From dridi.boukelmoune at gmail.com Tue May 2 08:06:05 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 02 May 2017 10:06:05 +0200 Subject: [master] 714774d Make the VSL_TAG macro responsible for casting Message-ID: commit 714774d3ef58b4032f28ee7d60986eb9c9052ff1 Author: Dridi Boukelmoune Date: Tue May 2 10:03:50 2017 +0200 Make the VSL_TAG macro responsible for casting diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h index c28604d..bfc24e5 100644 --- a/include/vapi/vsl_int.h +++ b/include/vapi/vsl_int.h @@ -70,7 +70,7 @@ #define VSL_END(ptr, len) ((ptr) + 2 + VSL_WORDS(len)) #define VSL_NEXT(ptr) VSL_END(ptr, VSL_LEN(ptr)) #define VSL_LEN(ptr) ((ptr)[0] & VSL_LENMASK) -#define VSL_TAG(ptr) ((ptr)[0] >> 24) +#define VSL_TAG(ptr) ((enum VSL_tag_e)((ptr)[0] >> 24)) #define VSL_ID(ptr) (((ptr)[1]) & VSL_IDENTMASK) #define VSL_CLIENT(ptr) (((ptr)[1]) & VSL_CLIENTMARKER) #define VSL_BACKEND(ptr) (((ptr)[1]) & VSL_BACKENDMARKER) diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index b418c64..2566561 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -172,7 +172,7 @@ vsl_match_IX(struct VSL_data *vsl, const vslf_list *list, const struct vslf *vslf; (void)vsl; - tag = (enum VSL_tag_e)VSL_TAG(c->rec.ptr); + tag = VSL_TAG(c->rec.ptr); cdata = VSL_CDATA(c->rec.ptr); len = VSL_LEN(c->rec.ptr); @@ -194,7 +194,7 @@ VSL_Match(struct VSL_data *vsl, const struct VSL_cursor *c) CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC); if (c == NULL || c->rec.ptr == NULL) return (0); - tag = (enum VSL_tag_e)VSL_TAG(c->rec.ptr); + tag = VSL_TAG(c->rec.ptr); if (tag <= SLT__Bogus || tag >= SLT__Reserved) return (0); if (vsl->c_opt && !VSL_CLIENT(c->rec.ptr)) @@ -245,7 +245,7 @@ VSL_Print(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo) return (0); if (fo == NULL) fo = stdout; - tag = (enum VSL_tag_e)VSL_TAG(c->rec.ptr); + tag = VSL_TAG(c->rec.ptr); vxid = VSL_ID(c->rec.ptr); len = VSL_LEN(c->rec.ptr); type = VSL_CLIENT(c->rec.ptr) ? 'c' : VSL_BACKEND(c->rec.ptr) ? @@ -283,7 +283,7 @@ VSL_PrintTerse(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo) return (0); if (fo == NULL) fo = stdout; - tag = (enum VSL_tag_e)VSL_TAG(c->rec.ptr); + tag = VSL_TAG(c->rec.ptr); len = VSL_LEN(c->rec.ptr); data = VSL_CDATA(c->rec.ptr); diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index e1d633c..a19ce5d 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -861,7 +861,7 @@ vtx_scan(struct VSLQ *vslq, struct vtx *vtx) continue; } - tag = (enum VSL_tag_e)VSL_TAG(ptr); + tag = VSL_TAG(ptr); assert(tag != SLT__Batch); switch (tag) { From dridi.boukelmoune at gmail.com Tue May 2 08:20:06 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 02 May 2017 10:20:06 +0200 Subject: [master] d320b77 Include to define mode_t in vpf.h Message-ID: commit d320b774e71875818504658e629a6b18c44d51cc Author: Dridi Boukelmoune Date: Tue May 2 10:16:03 2017 +0200 Include to define mode_t in vpf.h Inspired by: https://git.alpinelinux.org/cgit/aports/plain/main/varnish/musl-mode_t.patch?h=3.5-stable diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c index 6f8fdb6..b4739e5 100644 --- a/bin/varnishlog/varnishlog.c +++ b/bin/varnishlog/varnishlog.c @@ -39,6 +39,7 @@ #include #include #include +#include #include "vapi/vsm.h" #include "vapi/vsl.h" diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index 5c5b90e..e3bc741 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -41,6 +41,7 @@ #include #include #include +#include #include "compat/daemon.h" #include "vdef.h" From dridi.boukelmoune at gmail.com Tue May 2 22:11:05 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 03 May 2017 00:11:05 +0200 Subject: [master] 9867631 Prevent storage backends name collisions Message-ID: commit 98676319878b7a6aca77be7e207e140202800ae1 Author: Dridi Boukelmoune Date: Tue May 2 23:57:15 2017 +0200 Prevent storage backends name collisions Because of Transient storage being a special-case stevedore, the STV_Foreach logic would only work after the child's startup once all storage backends are defined. So during the setup, collisions would only be detected for the Transient storage. The underlying STV__iter function is now pretty much like a plain VTAILQ_FOREACH loop with an additional CHECK_OBJ_ORNULL step on all iterations. As a result, the transient storage is appended to the list at the end of the manager's setup and is now reported last in the CLI's storage.list command. Error messages related to the -s option are slightly more helpful. Fixes #2321 diff --git a/bin/varnishd/storage/mgt_stevedore.c b/bin/varnishd/storage/mgt_stevedore.c index 304b3bd..1ddf8a5 100644 --- a/bin/varnishd/storage/mgt_stevedore.c +++ b/bin/varnishd/storage/mgt_stevedore.c @@ -57,17 +57,11 @@ STV__iter(struct stevedore ** const pp) AN(pp); CHECK_OBJ_ORNULL(*pp, STEVEDORE_MAGIC); - if (*pp == stv_transient) { - *pp = NULL; - return (0); - } if (*pp != NULL) *pp = VTAILQ_NEXT(*pp, list); else *pp = VTAILQ_FIRST(&stevedores); - if (*pp == NULL) - *pp = stv_transient; - return (1); + return (*pp != NULL); } /*--------------------------------------------------------------------*/ @@ -128,6 +122,26 @@ static const struct choice STV_choice[] = { { NULL, NULL } }; +static void +stv_check_ident(const char *spec, const char *ident) +{ + struct stevedore *stv; + unsigned found = 0; + + if (!strcmp(ident, TRANSIENT_STORAGE)) + found = (stv_transient != NULL); + else { + STV_Foreach(stv) + if (!strcmp(stv->ident, ident)) { + found = 1; + break; + } + } + + if (found) + ARGV_ERR("(-s %s) '%s' is already defined\n", spec, ident); +} + void STV_Config(const char *spec) { @@ -135,7 +149,6 @@ STV_Config(const char *spec) const char *p, *q; struct stevedore *stv; const struct stevedore *stv2; - struct stevedore *stv3; int ac, l; static unsigned seq = 0; @@ -183,15 +196,12 @@ STV_Config(const char *spec) bprintf(stv->ident, "%.*s", l, spec); } - STV_Foreach(stv3) - if (!strcmp(stv3->ident, stv->ident)) - ARGV_ERR("(-s%s=%s) already defined once\n", - stv->ident, stv->name); + stv_check_ident(spec, stv->ident); if (stv->init != NULL) stv->init(stv, ac, av); else if (ac != 0) - ARGV_ERR("(-s%s) too many arguments\n", stv->name); + ARGV_ERR("(-s %s) too many arguments\n", stv->name); AN(stv->allocobj); AN(stv->methods); @@ -199,9 +209,8 @@ STV_Config(const char *spec) if (!strcmp(stv->ident, TRANSIENT_STORAGE)) { AZ(stv_transient); stv_transient = stv; - } else { + } else VTAILQ_INSERT_TAIL(&stevedores, stv, list); - } /* NB: Do not free av, stevedore gets to keep it */ } @@ -216,4 +225,5 @@ STV_Config_Transient(void) VCLS_AddFunc(mgt_cls, MCF_AUTH, cli_stv); if (stv_transient == NULL) STV_Config(TRANSIENT_STORAGE "=malloc"); + VTAILQ_INSERT_TAIL(&stevedores, stv_transient, list); } diff --git a/bin/varnishtest/tests/r02321.vtc b/bin/varnishtest/tests/r02321.vtc new file mode 100644 index 0000000..33c7658 --- /dev/null +++ b/bin/varnishtest/tests/r02321.vtc @@ -0,0 +1,19 @@ +varnishtest "Storage name collisions" + +# intentional collision +shell -err -expect "Error: (-s main=malloc,100m) 'main' is already defined" { + exec varnishd -a :0 -f '' -n ${tmpdir} \ + -s main=malloc,10m -s main=malloc,100m +} + +# pseudo-accidental collision +shell -err -expect "Error: (-s s0=malloc,100m) 's0' is already defined" { + exec varnishd -a :0 -f '' -n ${tmpdir} \ + -s malloc,10m -s s0=malloc,100m +} + +# Transient collision +shell -err -expect "Error: (-s Transient=malloc,100m) 'Transient' is already defined" { + exec varnishd -a :0 -f '' -n ${tmpdir} \ + -s Transient=malloc,10m -s Transient=malloc,100m +} From varnish-commit at varnish-cache.org Wed May 3 20:35:11 2017 From: varnish-commit at varnish-cache.org (mplijnz) Date: Thu, 4 May 2017 04:35:11 +0800 Subject: =?utf-8?B?dmFybmlzaC1jb21taXQ65Lu75q2j6Z2e5aaC5L2V?= =?utf-8?B?546p6L2s5Y2O5Li65LqL5Lia5ZCI5LyZ5Lq677yfMjc0NTQ=?= Message-ID: <20170504043519622376@slxjepyz.net> ????????---????????????????? ??????2017?5?14-15?? 6?09-10?? ????????????????????????? ????????????????????????????????????????? ??????5980?/2?/1???????????????? ? ??????021-31006787?0755-61280006?13381601000 ??? ??????320588808 at qq.com ?QQ/???320588808 ?????? ???????????????????????????????????????? ?????????????????????????? ?????????????????????????????? ???????????????????????????????? ?????? 1????????????????????? 2???5?????????????? 3???????????4???? 4??????????????? 5???????????????????????????? 6??????????????????????????????????? ????? ??1????????? ??2????????? ??3????????? ??1???????? ??2????????? ??3?????????? ??4???????? ?????? ???? ????????----?????????????? ???????????? ??????????????? ?????????? ???? ????????????????????????? ?????????????? ??1???????????????????????????????????????? ??1???????? ??????? ?????????????????? ???????? ????????????????2015?? ???? ?????????????????? ????????????????????? ?????? ???????????????????? ??2????????? ????? ?????????????? ??3?????????? ???? ?????????????????? ????????? ??2????????? ?????????? ???????--????????????? ?????????????? ?????????? 1??????---??3???????PB/PS/PE 2??????????? ?????????????????? ?????????? 1????? 2????? ?????????? 1?????? 2?????????? 3?IPO???? ???????LP?????? ???? ???????????? 1?????????????? 2???????????????????????????????????? ???????????????????? ???? ?????????????????? ????????? ???????????????? ?????????? ??1??????-----????????????????????????? ??2???????-----???????????? ??3?AB???----- Google???AB???????????? ???? ???? 1?????????????? 2????????????????????????????????????? ???? ???????????????? ?????????????? ??????? ???????????????????? ??4???????? ??????? ????????????????????? ??????? 1??????????????? ???VIE????????? 2??????????? ???????? ???????????????? ?????? ??? ????????????? ????????????????? ???????????????? ????????????? ?????????????? ????Bel Fuse Inc.?????????????? ????500???????????? ??????????????? ????? ???20??HR??????????????????????????500????????500?????????????????????????????????????????????????????? ??????????????????????????????????????IPO???????????????????????? ?????????????????????HR??????????????????HR????????????????????????????????????????????????????????????????????????????? ????? ???????????????????HR?????????????????????????????????????????????????????????????smart?????????? ????? ?????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? ?????????????????????????????????????????TCL?????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? ? 2017/5/4 ???4:35:19 -------------- next part -------------- An HTML attachment was scrubbed... URL: From dridi.boukelmoune at gmail.com Thu May 4 09:03:05 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 04 May 2017 11:03:05 +0200 Subject: [master] 995c8b2 Fix typo in lib/libvarnish/vtim.c Message-ID: commit 995c8b2cfcbb440f1d8574bb30b9ca047ea360d5 Author: Enerdhil Date: Thu May 4 10:51:43 2017 +0200 Fix typo in lib/libvarnish/vtim.c diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index 5e84e65..5e71a20 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -490,7 +490,7 @@ tst_delta() err += tst_delta_check("VTIM_real", r_begin, r_end, ref); if (err) { - printf("%d time delta test errrors\n", err); + printf("%d time delta test errors\n", err); exit(4); } } From phk at FreeBSD.org Fri May 5 19:19:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 05 May 2017 21:19:05 +0200 Subject: [master] fcfc338 Instead of our own home-rolled collision check, reuse VPF and always put a pid-file in the workdir. Message-ID: commit fcfc338d762c1e74e093defdb5d2c7a7ce90102e Author: Poul-Henning Kamp Date: Fri May 5 19:17:56 2017 +0000 Instead of our own home-rolled collision check, reuse VPF and always put a pid-file in the workdir. Improve the collision error message for -P by including the other pid. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 2d3a9dc..712c106 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -499,6 +499,7 @@ main(int argc, char * const *argv) struct f_arg *fa; struct vsb *vsb; VTAILQ_HEAD(,f_arg) f_args = VTAILQ_HEAD_INITIALIZER(f_args); + pid_t pid; setbuf(stdout, NULL); setbuf(stderr, NULL); @@ -787,9 +788,15 @@ main(int argc, char * const *argv) } VJ_master(JAIL_MASTER_FILE); - if (P_arg && (pfh = VPF_Open(P_arg, 0644, NULL)) == NULL) - ARGV_ERR("Could not open pid/lock (-P) file (%s): %s\n", - P_arg, strerror(errno)); + if (P_arg) { + pfh = VPF_Open(P_arg, 0644, &pid); + if (pfh == NULL && errno == EEXIST) + ARGV_ERR("Varnishd is already running (pid=%jd)\n", + (intmax_t)pid); + if (pfh == NULL) + ARGV_ERR("Could not open pid-file (%s): %s\n", + P_arg, strerror(errno)); + } VJ_master(JAIL_MASTER_LOW); /* If no -s argument specified, process default -s argument */ diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c index e41018d..45bb39e 100644 --- a/bin/varnishd/mgt/mgt_shmem.c +++ b/bin/varnishd/mgt/mgt_shmem.c @@ -43,6 +43,7 @@ #include "common/heritage.h" #include "vfl.h" +#include "vpf.h" #include "vsm_priv.h" #include "vfil.h" @@ -59,6 +60,8 @@ static void *mgt_vsm_p; static ssize_t mgt_vsm_l; +static struct vpf_fh *priv_vpf; + /*-------------------------------------------------------------------- * Use a bogo-VSM to hold master-copies of the VSM chunks the master * publishes, such as -S & -T arguments. @@ -84,60 +87,6 @@ mgt_SHM_static_alloc(const void *ptr, ssize_t size, } /*-------------------------------------------------------------------- - * Check that we are not started with the same -n argument as an already - * running varnishd. - * - * Non-zero return means we should exit and not trample the file. - * - */ - -static int -vsm_n_check(void) -{ - int fd, i; - struct stat st; - pid_t pid; - struct VSM_head vsmh; - int retval = 1; - - fd = open(VSM_FILENAME, O_RDWR); - if (fd < 0) - return (0); - - AZ(fstat(fd, &st)); - if (!S_ISREG(st.st_mode)) { - fprintf(stderr, - "VSM (%s) not a regular file.\n", VSM_FILENAME); - } else { - i = VFL_Test(fd, &pid); - if (i < 0) { - fprintf(stderr, - "Cannot determine locking status of VSM (%s)\n.", - VSM_FILENAME); - } else if (i == 0) { - /* - * File is unlocked, mark it as dead, to help any - * consumers still stuck on it. - */ - if (pread(fd, &vsmh, sizeof vsmh, 0) == sizeof vsmh) { - vsmh.alloc_seq = 0; - assert(sizeof vsmh == - pwrite(fd, &vsmh, sizeof vsmh, 0)); - } - retval = 0; - } else { - /* The VSM is locked, we won't touch it. */ - fprintf(stderr, - "VSM locked by running varnishd master (pid=%jd)\n" - "(Use unique -n arguments if you want" - " multiple instances)\n", (intmax_t)pid); - } - } - (void)close(fd); - return (retval); -} - -/*-------------------------------------------------------------------- * Build a zeroed file */ @@ -327,12 +276,15 @@ mgt_shm_atexit(void) void mgt_SHM_Init(void) { - int i; - - /* Collision check with already running varnishd */ - i = vsm_n_check(); - if (i) - exit(2); + pid_t pid = 0; + + priv_vpf = VPF_Open("_.pid", 0644, &pid); + if (priv_vpf == NULL && errno == EEXIST) + ARGV_ERR("Varnishd is already running (pid=%jd)\n", + (intmax_t)pid); + if (priv_vpf == NULL) + ARGV_ERR("Failure on _.pid: %s\n", strerror(errno)); + AZ(VPF_Write(priv_vpf)); /* Create our static VSM instance */ static_vsm = VSM_common_new(static_vsm_buf, sizeof static_vsm_buf); diff --git a/bin/varnishtest/tests/b00045.vtc b/bin/varnishtest/tests/b00045.vtc index 8863bad..47a676f 100644 --- a/bin/varnishtest/tests/b00045.vtc +++ b/bin/varnishtest/tests/b00045.vtc @@ -14,5 +14,14 @@ client c1 { delay .2 -shell -err -expect {Could not open pid/lock} \ - "varnishd -P ${v1_name}/varnishd.pid -b 127.0.0.1:80 -a :0 -n ${tmpdir}" +shell -err -expect {Error: Could not open pid-file} { + varnishd -P /dev/tty -b 127.0.0.1:80 -a :0 -n ${tmpdir} +} + +shell -err -expect {Error: Varnishd is already running} { + varnishd -P ${v1_name}/varnishd.pid -b 127.0.0.1:80 -a :0 -n ${tmpdir} +} + +shell -err -expect {Error: Varnishd is already running} { + varnishd -b 127.0.0.1:80 -a:0 -n ${tmpdir}/v1 -F +} From phk at FreeBSD.org Fri May 5 19:53:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 05 May 2017 21:53:05 +0200 Subject: [master] bd3b2dc Assert that we always have a transient storage. Message-ID: commit bd3b2dc29fdda6657d9a63c1d5781e61f58e41ad Author: Poul-Henning Kamp Date: Fri May 5 19:26:02 2017 +0000 Assert that we always have a transient storage. diff --git a/bin/varnishd/storage/mgt_stevedore.c b/bin/varnishd/storage/mgt_stevedore.c index 1ddf8a5..da3f4a1 100644 --- a/bin/varnishd/storage/mgt_stevedore.c +++ b/bin/varnishd/storage/mgt_stevedore.c @@ -225,5 +225,6 @@ STV_Config_Transient(void) VCLS_AddFunc(mgt_cls, MCF_AUTH, cli_stv); if (stv_transient == NULL) STV_Config(TRANSIENT_STORAGE "=malloc"); + AN(stv_transient); VTAILQ_INSERT_TAIL(&stevedores, stv_transient, list); } From phk at FreeBSD.org Fri May 5 21:01:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 05 May 2017 23:01:05 +0200 Subject: [master] d9c2408 Forward declare struct stevedore before we use it to unconfuse FlexeLint. Message-ID: commit d9c2408ec670886f64ef2614f3164accb9e462d6 Author: Poul-Henning Kamp Date: Fri May 5 20:59:08 2017 +0000 Forward declare struct stevedore before we use it to unconfuse FlexeLint. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index ee33de0..72140ee 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -113,6 +113,7 @@ struct poolparam; struct transport; struct req; struct sess; +struct stevedore; struct suckaddr; struct vrt_priv; struct vsb; From phk at FreeBSD.org Fri May 5 21:07:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 05 May 2017 23:07:06 +0200 Subject: [master] 7620b76 Always forward declare structs, and this FlexeLint hack goes away. Message-ID: commit 7620b76053210b140669eebb9e87fd7178977ed2 Author: Poul-Henning Kamp Date: Fri May 5 21:05:51 2017 +0000 Always forward declare structs, and this FlexeLint hack goes away. diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h index 5d6a8af..055b240 100644 --- a/bin/varnishd/common/heritage.h +++ b/bin/varnishd/common/heritage.h @@ -31,6 +31,7 @@ struct vsm_sc; struct suckaddr; +struct listen_sock; struct transport; struct listen_arg { diff --git a/include/vqueue.h b/include/vqueue.h index f5d39ea..3bbb7a1 100644 --- a/include/vqueue.h +++ b/include/vqueue.h @@ -435,7 +435,7 @@ struct { \ struct type *vtqe_next; /* next element */ \ struct type **vtqe_prev; /* address of previous next element */\ TRACEBUF \ -} /*lint -e(18) */ +} /* * Tail queue functions. From phk at FreeBSD.org Fri May 5 21:35:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 05 May 2017 23:35:05 +0200 Subject: [master] 7687c20 Forward declaration of initialized static variables is not possible in C. (One of the thing you'd think ISO-C had solved long time ago...) Message-ID: commit 7687c2029716d17c9984d1c182b9e4d43a947fb8 Author: Poul-Henning Kamp Date: Fri May 5 21:19:50 2017 +0000 Forward declaration of initialized static variables is not possible in C. (One of the thing you'd think ISO-C had solved long time ago...) diff --git a/bin/varnishtest/flint.lnt b/bin/varnishtest/flint.lnt index 65c57c0..b6b36df 100644 --- a/bin/varnishtest/flint.lnt +++ b/bin/varnishtest/flint.lnt @@ -9,6 +9,8 @@ -esym(534, snprintf) // Only for varnishtest, and not really nice +-esym(765, http_cmds) // No, cannot be made static + -e712 // 14 Info 712 Loss of precision (___) (___ to ___) -e747 // 16 Info 747 Significant prototype coercion (___) ___ to ___ diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 044581e..22c00bd 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -49,7 +49,7 @@ #include "vtcp.h" #include "hpack.h" -static const struct cmds http_cmds[]; +extern const struct cmds http_cmds[]; /* SECTION: client-server client/server * @@ -1780,7 +1780,7 @@ cmd_http_write_body(CMD_ARGS) * Execute HTTP specifications */ -static const struct cmds http_cmds[] = { +const struct cmds http_cmds[] = { #define CMD(n) { #n, cmd_##n }, #define CMD_HTTP(n) { #n, cmd_http_##n }, /* session */ From phk at FreeBSD.org Fri May 5 21:35:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 05 May 2017 23:35:05 +0200 Subject: [master] b498b6d Add some asserts to ensure we catch NULL pointers. Message-ID: commit b498b6d02862262463790bfbc1ba990839c2f3b0 Author: Poul-Henning Kamp Date: Fri May 5 21:29:35 2017 +0000 Add some asserts to ensure we catch NULL pointers. diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c index 0ab5679..e811ca0 100644 --- a/lib/libvarnishapi/vsl_arg.c +++ b/lib/libvarnishapi/vsl_arg.c @@ -199,6 +199,7 @@ VSLQ_Name2Grouping(const char *name, int l) { int i, n; + AN(name); if (l == -1) l = strlen(name); n = -1; @@ -264,6 +265,7 @@ vsl_IX_arg(struct VSL_data *vsl, int opt, const char *arg) struct vbitmap *tags = NULL; CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC); + AN(arg); vsl->flags |= F_SEEN_ixIX; b = arg; @@ -337,6 +339,7 @@ VSL_Arg(struct VSL_data *vsl, int opt, const char *arg) case 'i': case 'x': return (vsl_ix_arg(vsl, opt, arg)); case 'I': case 'X': return (vsl_IX_arg(vsl, opt, arg)); case 'L': + AN(arg); l = strtol(arg, &p, 0); while (isspace(*p)) p++; @@ -347,6 +350,7 @@ VSL_Arg(struct VSL_data *vsl, int opt, const char *arg) vsl->L_opt = (int)l; return (1); case 'T': + AN(arg); d = VNUM(arg); if (isnan(d)) return (vsl_diag(vsl, "-T: Syntax error")); From phk at FreeBSD.org Fri May 5 21:50:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 05 May 2017 23:50:06 +0200 Subject: [master] f1d356a Handle our private pid-file parallel to the -P one for simplicity. Message-ID: commit f1d356a0031c4e82b3c65528ded8582b6aa81ee6 Author: Poul-Henning Kamp Date: Fri May 5 21:48:55 2017 +0000 Handle our private pid-file parallel to the -P one for simplicity. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 712c106..c8f783f 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -70,7 +70,8 @@ struct VSC_C_mgt *VSC_C_mgt; static int I_fd = -1; static char Cn_arg[] = "/tmp/varnishd_C_XXXXXXX"; -static struct vpf_fh *pfh = NULL; +static struct vpf_fh *pfh1 = NULL; +static struct vpf_fh *pfh2 = NULL; int optreset; // Some has it, some doesn't. Cheaper than auto* @@ -179,8 +180,9 @@ mgt_stdin_close(void *priv) if (d_flag) { MCH_Stop_Child(); mgt_cli_close_all(); - if (pfh != NULL) - (void)VPF_Remove(pfh); + (void)VPF_Remove(pfh1); + if (pfh2 != NULL) + (void)VPF_Remove(pfh2); exit(0); } else { VFIL_null_fd(STDIN_FILENO); @@ -787,13 +789,25 @@ main(int argc, char * const *argv) dirname, strerror(errno)); } + vsb = VSB_new_auto(); + AN(vsb); + VSB_printf(vsb, "%s/_.pid", dirname); + AZ(VSB_finish(vsb)); VJ_master(JAIL_MASTER_FILE); + pfh1 = VPF_Open(VSB_data(vsb), 0644, &pid); + if (pfh1 == NULL && errno == EEXIST) + ARGV_ERR("Varnishd is already running (pid=%jd)\n", + (intmax_t)pid); + if (pfh1 == NULL) + ARGV_ERR("Could not open pid-file (%s): %s\n", + VSB_data(vsb), strerror(errno)); + VSB_destroy(&vsb); if (P_arg) { - pfh = VPF_Open(P_arg, 0644, &pid); - if (pfh == NULL && errno == EEXIST) + pfh2 = VPF_Open(P_arg, 0644, &pid); + if (pfh2 == NULL && errno == EEXIST) ARGV_ERR("Varnishd is already running (pid=%jd)\n", (intmax_t)pid); - if (pfh == NULL) + if (pfh2 == NULL) ARGV_ERR("Could not open pid-file (%s): %s\n", P_arg, strerror(errno)); } @@ -854,7 +868,8 @@ main(int argc, char * const *argv) S_arg = make_secret(dirname); AN(S_arg); - assert(pfh == NULL || !VPF_Write(pfh)); + assert(!VPF_Write(pfh1)); + assert(pfh2 == NULL || !VPF_Write(pfh2)); MGT_Complain(C_DEBUG, "Platform: %s", VSB_data(vident) + 1); @@ -931,7 +946,8 @@ main(int argc, char * const *argv) MGT_Complain(C_ERR, "vev_schedule() = %d", o); MGT_Complain(C_INFO, "manager dies"); - if (pfh != NULL) - (void)VPF_Remove(pfh); + (void)VPF_Remove(pfh1); + if (pfh2 != NULL) + (void)VPF_Remove(pfh2); exit(exit_status); } diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c index 45bb39e..858d871 100644 --- a/bin/varnishd/mgt/mgt_shmem.c +++ b/bin/varnishd/mgt/mgt_shmem.c @@ -43,7 +43,6 @@ #include "common/heritage.h" #include "vfl.h" -#include "vpf.h" #include "vsm_priv.h" #include "vfil.h" @@ -60,8 +59,6 @@ static void *mgt_vsm_p; static ssize_t mgt_vsm_l; -static struct vpf_fh *priv_vpf; - /*-------------------------------------------------------------------- * Use a bogo-VSM to hold master-copies of the VSM chunks the master * publishes, such as -S & -T arguments. @@ -276,16 +273,6 @@ mgt_shm_atexit(void) void mgt_SHM_Init(void) { - pid_t pid = 0; - - priv_vpf = VPF_Open("_.pid", 0644, &pid); - if (priv_vpf == NULL && errno == EEXIST) - ARGV_ERR("Varnishd is already running (pid=%jd)\n", - (intmax_t)pid); - if (priv_vpf == NULL) - ARGV_ERR("Failure on _.pid: %s\n", strerror(errno)); - AZ(VPF_Write(priv_vpf)); - /* Create our static VSM instance */ static_vsm = VSM_common_new(static_vsm_buf, sizeof static_vsm_buf); From fgsch at lodoss.net Fri May 5 22:12:06 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 06 May 2017 00:12:06 +0200 Subject: [master] 489dcd6 Bump the stack size when --enable-*san is used Message-ID: commit 489dcd6264578c734ac4b41b44e3ae55eaa0c7e1 Author: Federico G. Schwindt Date: Thu May 4 19:35:58 2017 +0100 Bump the stack size when --enable-*san is used The generated code needs a larger stack when optimizations are disabled, i.e. with --enable-debugging-symbols. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index c8f783f..6e24a05 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -276,7 +276,11 @@ init_params(struct cli *cli) low = sysconf(_SC_THREAD_STACK_MIN); MCF_ParamConf(MCF_MINIMUM, "thread_pool_stack", "%jdb", (intmax_t)low); +#if defined(WITH_SANITIZERS) + def = 92 * 1024; +#else def = 48 * 1024; +#endif if (def < low) def = low; MCF_ParamConf(MCF_DEFAULT, "thread_pool_stack", "%jdb", (intmax_t)def); diff --git a/configure.ac b/configure.ac index d556c23..0c95ea0 100644 --- a/configure.ac +++ b/configure.ac @@ -258,7 +258,7 @@ AC_ARG_ENABLE(msan, MSAN_FLAGS="-fsanitize=memory") if test "x$UBSAN_FLAGS$TSAN_FLAGS$ASAN_FLAGS$MSAN_FLAGS" != "x"; then - SAN_CFLAGS="${UBSAN_FLAGS} ${TSAN_FLAGS} ${ASAN_FLAGS} ${MSAN_FLAGS} -fPIC -fPIE -fno-omit-frame-pointer" + SAN_CFLAGS="-DWITH_SANITIZERS=1 ${UBSAN_FLAGS} ${TSAN_FLAGS} ${ASAN_FLAGS} ${MSAN_FLAGS} -fPIC -fPIE -fno-omit-frame-pointer" SAN_LDFLAGS="${UBSAN_FLAGS} ${TSAN_FLAGS} ${ASAN_FLAGS} ${MSAN_FLAGS} -pie" fi AC_SUBST(SAN_CFLAGS) From fgsch at lodoss.net Fri May 5 22:12:06 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 06 May 2017 00:12:06 +0200 Subject: [master] b264be7 More coverage Message-ID: commit b264be7b44b13d06ccf414e4ede1ee5028485396 Author: Federico G. Schwindt Date: Fri May 5 10:15:12 2017 +0100 More coverage diff --git a/bin/varnishtest/tests/u00005.vtc b/bin/varnishtest/tests/u00005.vtc index d80f197..cd87c93 100644 --- a/bin/varnishtest/tests/u00005.vtc +++ b/bin/varnishtest/tests/u00005.vtc @@ -22,10 +22,6 @@ shell -err "grep -q LCK.vbe.destroy ${p1_out}" process p2 {varnishstat -1 -n ${v1_name} -f ^*vbe.destroy \ -f *vbe* -f *mempool* | tr '[1-9]' '0'} -run -shell "grep -q vbe ${p2_out}" -shell "grep -q mempool ${p2_out}" -shell -err "grep -q LCK.vbe.destroy ${p2_out}" - shell "cmp -s ${p1_out} ${p2_out}" shell -expect "cache_hit" \ @@ -39,8 +35,12 @@ shell -expect "Copyright (c) 2006 Verdens Gang AS" \ "varnishstat -V" shell -err -expect "Usage: varnishstat " \ "varnishstat extra" +shell -err -expect "-t: Range error" \ + "varnishstat -n /nonexistent -t -1" +shell -err -expect "-t: Syntax error" \ + "varnishstat -n /nonexistent -t foo" shell -err -expect "Cannot open /nonexistent/_.vsm" \ - "varnishstat -n /nonexistent" + "varnishstat -n /nonexistent -t 1" shell -expect "MAIN.uptime" \ "varnishstat -n ${v1_name} -1" shell -expect "MAIN" \ diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc index 221138e..196514e 100644 --- a/bin/varnishtest/tests/u00006.vtc +++ b/bin/varnishtest/tests/u00006.vtc @@ -41,6 +41,8 @@ shell -err -expect {-X: Syntax error in "**"} \ {varnishlog -X "**:bar"} shell -err -expect "Cannot open output file (No such file or directory)" \ {varnishlog -w /nonexistent/file} +shell -err -expect "Only one of -n, -N and -r options may be used" \ + {varnishlog -n ${v1_name} -N ${v1_name}/_.vsm} process p1 -wait shell {grep -q "0 CLI" ${tmpdir}/vlog} From phk at FreeBSD.org Fri May 5 22:52:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 06 May 2017 00:52:05 +0200 Subject: [master] 79d906b Track FreeBSD's pidfile.c as close as possible. Message-ID: commit 79d906b7b26e4dd6e878b9aea4e321daa712daa7 Author: Poul-Henning Kamp Date: Fri May 5 22:07:19 2017 +0000 Track FreeBSD's pidfile.c as close as possible. diff --git a/lib/libvarnish/vpf.c b/lib/libvarnish/vpf.c index a5fd683..3da5748 100644 --- a/lib/libvarnish/vpf.c +++ b/lib/libvarnish/vpf.c @@ -76,7 +76,7 @@ vpf_read(const char *path, pid_t *pidptr) char buf[16], *endptr; int error, fd, i; - fd = open(path, O_RDONLY); + fd = open(path, O_RDONLY | O_CLOEXEC); if (fd == -1) return (errno); @@ -85,6 +85,8 @@ vpf_read(const char *path, pid_t *pidptr) (void)close(fd); if (i == -1) return (error); + else if (i == 0) + return (EAGAIN); buf[i] = '\0'; *pidptr = strtol(buf, &endptr, 10); @@ -105,17 +107,10 @@ VPF_Open(const char *path, mode_t mode, pid_t *pidptr) if (pfh == NULL) return (NULL); -#if 0 - if (path == NULL) - len = snprintf(pfh->pf_path, sizeof(pfh->pf_path), - "/var/run/%s.pid", getprogname()); - else -#endif - { - assert(path != NULL); - len = snprintf(pfh->pf_path, sizeof(pfh->pf_path), - "%s", path); - } + assert(path != NULL); + len = snprintf(pfh->pf_path, sizeof(pfh->pf_path), + "%s", path); + if (len >= (int)sizeof(pfh->pf_path)) { free(pfh); errno = ENAMETOOLONG; @@ -124,12 +119,12 @@ VPF_Open(const char *path, mode_t mode, pid_t *pidptr) /* * Open the PID file and obtain exclusive lock. - * We truncate PID file here only to remove old PID immediatelly, + * We truncate PID file here only to remove old PID immediately, * PID file will be truncated again in VPF_Write(), so * VPF_Write() can be called multiple times. */ fd = VFL_Open(pfh->pf_path, - O_WRONLY | O_CREAT | O_TRUNC | O_NONBLOCK, mode); + O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NONBLOCK, mode); if (fd == -1) { if (errno == EWOULDBLOCK && pidptr != NULL) { errno = vpf_read(pfh->pf_path, pidptr); @@ -139,6 +134,7 @@ VPF_Open(const char *path, mode_t mode, pid_t *pidptr) free(pfh); return (NULL); } + /* * Remember file information, so in VPF_Write() we are sure we write * to the proper descriptor. diff --git a/tools/audit_vpf.sh b/tools/audit_vpf.sh new file mode 100644 index 0000000..2115e6c --- /dev/null +++ b/tools/audit_vpf.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# +# Script to compare vpf.c with FreeBSD's pidfile.c +# +# Run this on a up-to-date FreeBSD source tree + +sed ' +s/vpf_/pidfile_/g +s/VPF_/pidfile_/g +s/pidfile_fh/pidfh/g +s/pidfile_Write/pidfile_write/g +s/pidfile_Close/pidfile_close/g +s/pidfile_Remove/pidfile_remove/g +s/pidfile_Open/pidfile_open/g +s/ (void)/ /g +' lib/libvarnish/vpf.c | + diff -ub /usr/src/lib/libutil/pidfile.c - + From phk at FreeBSD.org Fri May 5 22:52:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 06 May 2017 00:52:05 +0200 Subject: [master] 1f631aa Update our vsha256 with the latest improvements from Colin Percival: Message-ID: commit 1f631aa61dd170aa3bdd6485fbdb801c7cc36eb0 Author: Poul-Henning Kamp Date: Fri May 5 22:50:03 2017 +0000 Update our vsha256 with the latest improvements from Colin Percival: Retune SHA2 code for improved performance on CPUs with more ILP and a preference for memory load instructions over large code footprints with embedded immediate variables. On amd64 CPUs from 2007-2008 there is not a significant change, but amd64 CPUs from 2009-2010 get roughly 10% more throughput with this code; amd64 CPUs from 2011-2012 get roughly 15% more throughput; and AMD64 CPUs from 2013-2015 get 20-25% more throughput. The Raspberry Pi 2 increases its throughput by 6-8%. Sponsored by: Tarsnap Backup Inc. diff --git a/include/vsha256.h b/include/vsha256.h index bf42c11..e3852e9 100644 --- a/include/vsha256.h +++ b/include/vsha256.h @@ -30,6 +30,7 @@ #define _SHA256_H_ #define SHA256_LEN 32 +#define SHA256_DIGEST_LENGTH 32 typedef struct SHA256Context { uint32_t state[8]; diff --git a/lib/libvarnish/flint.lnt b/lib/libvarnish/flint.lnt index 0a014ba..636828a 100644 --- a/lib/libvarnish/flint.lnt +++ b/lib/libvarnish/flint.lnt @@ -1,5 +1,10 @@ -efunc(835, SHA256_Transform) // A zero has been given as ___ argument to operator '___' -efunc(778, SHA256_Transform) // Constant expression evaluates to 0 in operation '___' +-efunc(661, SHA256_Transform) // Possible access of out-of-bounds pointer +-efunc(662, SHA256_Transform) // Possible creation of out-of-bounds pointer +-efunc(662, SHA256_Update) // Possible creation of out-of-bounds pointer +-efunc(662, be32dec_vect) // Possible creation of out-of-bounds pointer +-efunc(670, SHA256_Update) // Possible access beyond array for function '___', +fan // Allow anon unions -e713 // Loss of precision (assignment) (unsigned long long to long long) @@ -7,3 +12,4 @@ -dVARNISH_STATE_DIR="foo" --emacro((835),BINHEAP_NOIDX) +--emacro((835),O_CLOEXEC) diff --git a/lib/libvarnish/vsha256.c b/lib/libvarnish/vsha256.c index 7e73710..c668474 100644 --- a/lib/libvarnish/vsha256.c +++ b/lib/libvarnish/vsha256.c @@ -23,7 +23,8 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * From: $FreeBSD: head/lib/libmd/sha256c.c 154479 2006-01-17 15:35:57Z phk $ + * From: + * $FreeBSD: sys/crypto/sha2/sha256c.c 300966 2016-05-29 17:26:40Z cperciva $ */ #include "config.h" @@ -84,7 +85,27 @@ be32dec_vect(uint32_t *dst, const unsigned char *src, size_t len) dst[i] = vbe32dec(src + i * 4); } -#endif +#endif /* BYTE_ORDER != BIG_ENDIAN */ + +/* SHA256 round constants. */ +static const uint32_t K[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +}; /* Elementary functions used by SHA256 */ #define Ch(x, y, z) ((x & (y ^ z)) ^ z) @@ -97,19 +118,28 @@ be32dec_vect(uint32_t *dst, const unsigned char *src, size_t len) #define s1(x) (ROTR(x, 17) ^ ROTR(x, 19) ^ SHR(x, 10)) /* SHA256 round function */ -#define RND(a, b, c, d, e, f, g, h, k) \ - t0 = h + S1(e) + Ch(e, f, g) + k; \ - t1 = S0(a) + Maj(a, b, c); \ - d += t0; \ - h = t0 + t1; +#define RND(a, b, c, d, e, f, g, h, k) \ + do { \ + h += S1(e) + Ch(e, f, g) + (k); \ + d += h; \ + h += S0(a) + Maj(a, b, c); \ + } while (0) /* Adjusted round function for rotating state */ -#define RNDr(S, W, i, k) \ +#define RNDr(S, W, i, ii) \ RND(S[(64 - i) % 8], S[(65 - i) % 8], \ S[(66 - i) % 8], S[(67 - i) % 8], \ S[(68 - i) % 8], S[(69 - i) % 8], \ S[(70 - i) % 8], S[(71 - i) % 8], \ - (W[i] + k)) + W[i + ii] + K[i + ii]) + +/* Message schedule computation */ +#define MSCH(W, ii, i) \ + do { \ + W[i + ii + 16] = \ + s1(W[i + ii + 14]) + W[i + ii + 9] + \ + s0(W[i + ii + 1]) + W[i + ii]; \ + } while (0) /* * SHA256 block compression function. The 256-bit state is transformed via @@ -120,82 +150,52 @@ SHA256_Transform(uint32_t * state, const unsigned char block[64]) { uint32_t W[64]; uint32_t S[8]; - uint32_t t0, t1; int i; - /* 1. Prepare message schedule W. */ + /* 1. Prepare the first part of the message schedule W. */ be32dec_vect(W, block, 64); - for (i = 16; i < 64; i++) - W[i] = s1(W[i - 2]) + W[i - 7] + s0(W[i - 15]) + W[i - 16]; /* 2. Initialize working variables. */ memcpy(S, state, 32); /* 3. Mix. */ - RNDr(S, W, 0, 0x428a2f98); - RNDr(S, W, 1, 0x71374491); - RNDr(S, W, 2, 0xb5c0fbcf); - RNDr(S, W, 3, 0xe9b5dba5); - RNDr(S, W, 4, 0x3956c25b); - RNDr(S, W, 5, 0x59f111f1); - RNDr(S, W, 6, 0x923f82a4); - RNDr(S, W, 7, 0xab1c5ed5); - RNDr(S, W, 8, 0xd807aa98); - RNDr(S, W, 9, 0x12835b01); - RNDr(S, W, 10, 0x243185be); - RNDr(S, W, 11, 0x550c7dc3); - RNDr(S, W, 12, 0x72be5d74); - RNDr(S, W, 13, 0x80deb1fe); - RNDr(S, W, 14, 0x9bdc06a7); - RNDr(S, W, 15, 0xc19bf174); - RNDr(S, W, 16, 0xe49b69c1); - RNDr(S, W, 17, 0xefbe4786); - RNDr(S, W, 18, 0x0fc19dc6); - RNDr(S, W, 19, 0x240ca1cc); - RNDr(S, W, 20, 0x2de92c6f); - RNDr(S, W, 21, 0x4a7484aa); - RNDr(S, W, 22, 0x5cb0a9dc); - RNDr(S, W, 23, 0x76f988da); - RNDr(S, W, 24, 0x983e5152); - RNDr(S, W, 25, 0xa831c66d); - RNDr(S, W, 26, 0xb00327c8); - RNDr(S, W, 27, 0xbf597fc7); - RNDr(S, W, 28, 0xc6e00bf3); - RNDr(S, W, 29, 0xd5a79147); - RNDr(S, W, 30, 0x06ca6351); - RNDr(S, W, 31, 0x14292967); - RNDr(S, W, 32, 0x27b70a85); - RNDr(S, W, 33, 0x2e1b2138); - RNDr(S, W, 34, 0x4d2c6dfc); - RNDr(S, W, 35, 0x53380d13); - RNDr(S, W, 36, 0x650a7354); - RNDr(S, W, 37, 0x766a0abb); - RNDr(S, W, 38, 0x81c2c92e); - RNDr(S, W, 39, 0x92722c85); - RNDr(S, W, 40, 0xa2bfe8a1); - RNDr(S, W, 41, 0xa81a664b); - RNDr(S, W, 42, 0xc24b8b70); - RNDr(S, W, 43, 0xc76c51a3); - RNDr(S, W, 44, 0xd192e819); - RNDr(S, W, 45, 0xd6990624); - RNDr(S, W, 46, 0xf40e3585); - RNDr(S, W, 47, 0x106aa070); - RNDr(S, W, 48, 0x19a4c116); - RNDr(S, W, 49, 0x1e376c08); - RNDr(S, W, 50, 0x2748774c); - RNDr(S, W, 51, 0x34b0bcb5); - RNDr(S, W, 52, 0x391c0cb3); - RNDr(S, W, 53, 0x4ed8aa4a); - RNDr(S, W, 54, 0x5b9cca4f); - RNDr(S, W, 55, 0x682e6ff3); - RNDr(S, W, 56, 0x748f82ee); - RNDr(S, W, 57, 0x78a5636f); - RNDr(S, W, 58, 0x84c87814); - RNDr(S, W, 59, 0x8cc70208); - RNDr(S, W, 60, 0x90befffa); - RNDr(S, W, 61, 0xa4506ceb); - RNDr(S, W, 62, 0xbef9a3f7); - RNDr(S, W, 63, 0xc67178f2); + for (i = 0; i < 64; i += 16) { + RNDr(S, W, 0, i); + RNDr(S, W, 1, i); + RNDr(S, W, 2, i); + RNDr(S, W, 3, i); + RNDr(S, W, 4, i); + RNDr(S, W, 5, i); + RNDr(S, W, 6, i); + RNDr(S, W, 7, i); + RNDr(S, W, 8, i); + RNDr(S, W, 9, i); + RNDr(S, W, 10, i); + RNDr(S, W, 11, i); + RNDr(S, W, 12, i); + RNDr(S, W, 13, i); + RNDr(S, W, 14, i); + RNDr(S, W, 15, i); + + if (i == 48) + break; + MSCH(W, 0, i); + MSCH(W, 1, i); + MSCH(W, 2, i); + MSCH(W, 3, i); + MSCH(W, 4, i); + MSCH(W, 5, i); + MSCH(W, 6, i); + MSCH(W, 7, i); + MSCH(W, 8, i); + MSCH(W, 9, i); + MSCH(W, 10, i); + MSCH(W, 11, i); + MSCH(W, 12, i); + MSCH(W, 13, i); + MSCH(W, 14, i); + MSCH(W, 15, i); + } /* 4. Mix local working variables into global state */ for (i = 0; i < 8; i++) @@ -213,23 +213,29 @@ static const unsigned char PAD[64] = { static void SHA256_Pad(SHA256_CTX * ctx) { - unsigned char len[8]; - uint32_t r, plen; - - /* - * Convert length to bits and encode as a vector of bytes - * -- we do this now rather than later because the length - * will change after we pad. - */ - vbe64enc(len, ctx->count << 3); - - /* Add 1--64 bytes so that the resulting length is 56 mod 64 */ - r = ctx->count & 0x3f; - plen = (r < 56) ? (56 - r) : (120 - r); - SHA256_Update(ctx, PAD, (size_t)plen); - - /* Add the terminating bit-count */ - SHA256_Update(ctx, len, 8); + size_t r; + + /* Figure out how many bytes we have buffered. */ + r = (ctx->count >> 3) & 0x3f; + + /* Pad to 56 mod 64, transforming if we finish a block en route. */ + if (r < 56) { + /* Pad to 56 mod 64. */ + memcpy(&ctx->buf[r], PAD, 56 - r); + } else { + /* Finish the current block and mix. */ + memcpy(&ctx->buf[r], PAD, 64 - r); + SHA256_Transform(ctx->state, ctx->buf); + + /* The start of the final block is all zeroes. */ + memset(&ctx->buf[0], 0, 56); + } + + /* Add the terminating bit-count. */ + be64enc(&ctx->buf[56], ctx->count); + + /* Mix in the final block. */ + SHA256_Transform(ctx->state, ctx->buf); } /* SHA-256 initialization. Begins a SHA-256 operation. */ @@ -255,23 +261,40 @@ SHA256_Init(SHA256_CTX * ctx) void SHA256_Update(SHA256_CTX * ctx, const void *in, size_t len) { - uint32_t r, l; + uint64_t bitlen; + uint32_t r; const unsigned char *src = in; /* Number of bytes left in the buffer from previous updates */ - r = ctx->count & 0x3f; - while (len > 0) { - l = 64 - r; - if (l > len) - l = len; - memcpy(&ctx->buf[r], src, l); - len -= l; - src += l; - ctx->count += l; - r = ctx->count & 0x3f; - if (r == 0) - SHA256_Transform(ctx->state, ctx->buf); + r = (ctx->count >> 3) & 0x3f; + + /* Convert the length into a number of bits */ + bitlen = len << 3; + + /* Update number of bits */ + ctx->count += bitlen; + + /* Handle the case where we don't need to perform any transforms */ + if (len < 64 - r) { + memcpy(&ctx->buf[r], src, len); + return; } + + /* Finish the current block */ + memcpy(&ctx->buf[r], src, 64 - r); + SHA256_Transform(ctx->state, ctx->buf); + src += 64 - r; + len -= 64 - r; + + /* Perform complete blocks */ + while (len >= 64) { + SHA256_Transform(ctx->state, src); + src += 64; + len -= 64; + } + + /* Copy left over data into buffer */ + memcpy(ctx->buf, src, len); } /* @@ -279,14 +302,14 @@ SHA256_Update(SHA256_CTX * ctx, const void *in, size_t len) * and clears the context state. */ void -SHA256_Final(unsigned char digest[32], SHA256_CTX * ctx) +SHA256_Final(unsigned char digest[static SHA256_DIGEST_LENGTH], SHA256_CTX *ctx) { /* Add padding */ SHA256_Pad(ctx); /* Write the hash */ - be32enc_vect(digest, ctx->state, 32); + be32enc_vect(digest, ctx->state, SHA256_DIGEST_LENGTH); /* Clear the context state */ memset((void *)ctx, 0, sizeof(*ctx)); diff --git a/tools/audit_vsha256.sh b/tools/audit_vsha256.sh new file mode 100644 index 0000000..a81c8c0 --- /dev/null +++ b/tools/audit_vsha256.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# +# Script to compare vsha256.c with FreeBSD's sha256c.c +# +# Run this on a up-to-date FreeBSD source tree + +sed ' +s/vbe32/be32/g +' lib/libvarnish/vsha256.c | + diff -ub /usr/src/sys/crypto/sha2/sha256c.c - + From phk at FreeBSD.org Fri May 5 22:56:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 06 May 2017 00:56:05 +0200 Subject: [master] 4d648b8 Missed one freebsd->varnish change: [v]be64enc() Message-ID: commit 4d648b8bfdaf795ee6497d2b29c06cda43cf981a Author: Poul-Henning Kamp Date: Fri May 5 22:54:34 2017 +0000 Missed one freebsd->varnish change: [v]be64enc() diff --git a/lib/libvarnish/vsha256.c b/lib/libvarnish/vsha256.c index c668474..5b898a9 100644 --- a/lib/libvarnish/vsha256.c +++ b/lib/libvarnish/vsha256.c @@ -232,7 +232,7 @@ SHA256_Pad(SHA256_CTX * ctx) } /* Add the terminating bit-count. */ - be64enc(&ctx->buf[56], ctx->count); + vbe64enc(&ctx->buf[56], ctx->count); /* Mix in the final block. */ SHA256_Transform(ctx->state, ctx->buf); From fgsch at lodoss.net Fri May 5 23:53:05 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 06 May 2017 01:53:05 +0200 Subject: [master] 8309f55 Display usage if we don't have any tests to run Message-ID: commit 8309f5541127967b81fbe7dca3c0c4056d958091 Author: Federico G. Schwindt Date: Sat May 6 00:51:06 2017 +0100 Display usage if we don't have any tests to run diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index e7e778f..e872807 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -145,7 +145,6 @@ usage(void) fprintf(stderr, FMT, "-t duration", "Time tests out after this long"); fprintf(stderr, FMT, "-v", "Verbose mode: always report test log"); fprintf(stderr, FMT, "-W", "Enable the witness facility for locking"); - fprintf(stderr, "\n"); exit(1); } @@ -653,7 +652,10 @@ main(int argc, char * const *argv) argc -= optind; argv += optind; - for (;argc > 0; argc--, argv++) { + if (argc < 1) + usage(); + + for (; argc > 0; argc--, argv++) { if (!read_file(*argv, ntest)) continue; if (!vtc_continue) From phk at FreeBSD.org Sun May 7 08:45:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 07 May 2017 10:45:05 +0200 Subject: [master] 5fae561 Update our slightly enhanced version zlib from 1.2.8 to 1.2.11 Message-ID: commit 5fae56145cc83007df6dfcc08f9b91134dafd757 Author: Poul-Henning Kamp Date: Sun May 7 08:43:45 2017 +0000 Update our slightly enhanced version zlib from 1.2.8 to 1.2.11 diff --git a/lib/libvgz/adler32.c b/lib/libvgz/adler32.c index a868f07..d0be438 100644 --- a/lib/libvgz/adler32.c +++ b/lib/libvgz/adler32.c @@ -1,5 +1,5 @@ /* adler32.c -- compute the Adler-32 checksum of a data stream - * Copyright (C) 1995-2011 Mark Adler + * Copyright (C) 1995-2011, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -7,11 +7,9 @@ #include "zutil.h" -#define local static - local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); -#define BASE 65521 /* largest prime smaller than 65536 */ +#define BASE 65521U /* largest prime smaller than 65536 */ #define NMAX 5552 /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ @@ -62,10 +60,10 @@ local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); #endif /* ========================================================================= */ -uLong ZEXPORT adler32(adler, buf, len) +uLong ZEXPORT adler32_z(adler, buf, len) uLong adler; const Bytef *buf; - uInt len; + z_size_t len; { unsigned long sum2; unsigned n; @@ -133,6 +131,15 @@ uLong ZEXPORT adler32(adler, buf, len) } /* ========================================================================= */ +uLong ZEXPORT adler32(adler, buf, len) + uLong adler; + const Bytef *buf; + uInt len; +{ + return adler32_z(adler, buf, len); +} + +/* ========================================================================= */ local uLong adler32_combine_(adler1, adler2, len2) uLong adler1; uLong adler2; @@ -156,7 +163,7 @@ local uLong adler32_combine_(adler1, adler2, len2) sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem; if (sum1 >= BASE) sum1 -= BASE; if (sum1 >= BASE) sum1 -= BASE; - if (sum2 >= (BASE << 1)) sum2 -= (BASE << 1); + if (sum2 >= ((unsigned long)BASE << 1)) sum2 -= ((unsigned long)BASE << 1); if (sum2 >= BASE) sum2 -= BASE; return sum1 | (sum2 << 16); } diff --git a/lib/libvgz/compress.c b/lib/libvgz/compress.c index 114fdb1..6134fad 100644 --- a/lib/libvgz/compress.c +++ b/lib/libvgz/compress.c @@ -1,5 +1,5 @@ /* compress.c -- compress a memory buffer - * Copyright (C) 1995-2005 Jean-loup Gailly. + * Copyright (C) 1995-2005, 2014, 2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -28,16 +28,11 @@ int ZEXPORT compress2 (dest, destLen, source, sourceLen, level) { z_stream stream; int err; + const uInt max = (uInt)-1; + uLong left; - stream.next_in = (z_const Bytef *)source; - stream.avail_in = (uInt)sourceLen; -#ifdef MAXSEG_64K - /* Check for source > 64K on 16-bit machine: */ - if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR; -#endif - stream.next_out = dest; - stream.avail_out = (uInt)*destLen; - if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR; + left = *destLen; + *destLen = 0; stream.zalloc = (alloc_func)0; stream.zfree = (free_func)0; @@ -46,15 +41,26 @@ int ZEXPORT compress2 (dest, destLen, source, sourceLen, level) err = deflateInit(&stream, level); if (err != Z_OK) return err; - err = deflate(&stream, Z_FINISH); - if (err != Z_STREAM_END) { - deflateEnd(&stream); - return err == Z_OK ? Z_BUF_ERROR : err; - } - *destLen = stream.total_out; + stream.next_out = dest; + stream.avail_out = 0; + stream.next_in = (z_const Bytef *)source; + stream.avail_in = 0; + + do { + if (stream.avail_out == 0) { + stream.avail_out = left > (uLong)max ? max : (uInt)left; + left -= stream.avail_out; + } + if (stream.avail_in == 0) { + stream.avail_in = sourceLen > (uLong)max ? max : (uInt)sourceLen; + sourceLen -= stream.avail_in; + } + err = deflate(&stream, sourceLen ? Z_NO_FLUSH : Z_FINISH); + } while (err == Z_OK); - err = deflateEnd(&stream); - return err; + *destLen = stream.total_out; + deflateEnd(&stream); + return err == Z_STREAM_END ? Z_OK : err; } /* =========================================================================== diff --git a/lib/libvgz/crc32.c b/lib/libvgz/crc32.c index 979a719..9580440 100644 --- a/lib/libvgz/crc32.c +++ b/lib/libvgz/crc32.c @@ -1,5 +1,5 @@ /* crc32.c -- compute the CRC-32 of a data stream - * Copyright (C) 1995-2006, 2010, 2011, 2012 Mark Adler + * Copyright (C) 1995-2006, 2010, 2011, 2012, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h * * Thanks to Rodney Brown for his contribution of faster @@ -30,17 +30,15 @@ #include "zutil.h" /* for STDC and FAR definitions */ -#define local static - /* Definitions for doing the crc four data bytes at a time. */ #if !defined(NOBYFOUR) && defined(Z_U4) # define BYFOUR #endif #ifdef BYFOUR local unsigned long crc32_little OF((unsigned long, - const unsigned char FAR *, unsigned)); + const unsigned char FAR *, z_size_t)); local unsigned long crc32_big OF((unsigned long, - const unsigned char FAR *, unsigned)); + const unsigned char FAR *, z_size_t)); # define TBLS 8 #else # define TBLS 1 @@ -201,10 +199,10 @@ const z_crc_t FAR * ZEXPORT get_crc_table() #define DO8 DO1; DO1; DO1; DO1; DO1; DO1; DO1; DO1 /* ========================================================================= */ -unsigned long ZEXPORT crc32(crc, buf, len) +unsigned long ZEXPORT crc32_z(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - uInt len; + z_size_t len; { if (buf == Z_NULL) return 0UL; @@ -235,8 +233,29 @@ unsigned long ZEXPORT crc32(crc, buf, len) return crc ^ 0xffffffffUL; } +/* ========================================================================= */ +unsigned long ZEXPORT crc32(crc, buf, len) + unsigned long crc; + const unsigned char FAR *buf; + uInt len; +{ + return crc32_z(crc, buf, len); +} + #ifdef BYFOUR +/* + This BYFOUR code accesses the passed unsigned char * buffer with a 32-bit + integer pointer type. This violates the strict aliasing rule, where a + compiler can assume, for optimization purposes, that two pointers to + fundamentally different types won't ever point to the same memory. This can + manifest as a problem only if one of the pointers is written to. This code + only reads from those pointers. So long as this code remains isolated in + this compilation unit, there won't be a problem. For this reason, this code + should not be copied and pasted into a compilation unit in which other code + writes to the buffer that is passed to these routines. + */ + /* ========================================================================= */ #define DOLIT4 c ^= *buf4++; \ c = crc_table[3][c & 0xff] ^ crc_table[2][(c >> 8) & 0xff] ^ \ @@ -247,7 +266,7 @@ unsigned long ZEXPORT crc32(crc, buf, len) local unsigned long crc32_little(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - unsigned len; + z_size_t len; { register z_crc_t c; register const z_crc_t FAR *buf4; @@ -278,7 +297,7 @@ local unsigned long crc32_little(crc, buf, len) } /* ========================================================================= */ -#define DOBIG4 c ^= *++buf4; \ +#define DOBIG4 c ^= *buf4++; \ c = crc_table[4][c & 0xff] ^ crc_table[5][(c >> 8) & 0xff] ^ \ crc_table[6][(c >> 16) & 0xff] ^ crc_table[7][c >> 24] #define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4 @@ -287,7 +306,7 @@ local unsigned long crc32_little(crc, buf, len) local unsigned long crc32_big(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - unsigned len; + z_size_t len; { register z_crc_t c; register const z_crc_t FAR *buf4; @@ -300,7 +319,6 @@ local unsigned long crc32_big(crc, buf, len) } buf4 = (const z_crc_t FAR *)(const void FAR *)buf; - buf4--; while (len >= 32) { DOBIG32; len -= 32; @@ -309,7 +327,6 @@ local unsigned long crc32_big(crc, buf, len) DOBIG4; len -= 4; } - buf4++; buf = (const unsigned char FAR *)buf4; if (len) do { diff --git a/lib/libvgz/deflate.c b/lib/libvgz/deflate.c index 4a7c6b3..b3adb63 100644 --- a/lib/libvgz/deflate.c +++ b/lib/libvgz/deflate.c @@ -1,5 +1,5 @@ /* deflate.c -- compress data using the deflation algorithm - * Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler + * Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -53,7 +53,7 @@ extern const char deflate_copyright[]; const char deflate_copyright[] = - " deflate 1.2.8 Copyright 1995-2013 Jean-loup Gailly and Mark Adler "; + " deflate 1.2.11 Copyright 1995-2017 Jean-loup Gailly and Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -74,6 +74,8 @@ typedef enum { typedef block_state (*compress_func) OF((deflate_state *s, int flush)); /* Compression function. Returns the block state after the call. */ +local int deflateStateCheck OF((z_streamp strm)); +local void slide_hash OF((deflate_state *s)); local void fill_window OF((deflate_state *s)); local block_state deflate_stored OF((deflate_state *s, int flush)); local block_state deflate_fast OF((deflate_state *s, int flush)); @@ -85,15 +87,16 @@ local block_state deflate_huff OF((deflate_state *s, int flush)); local void lm_init OF((deflate_state *s)); local void putShortMSB OF((deflate_state *s, uInt b)); local void flush_pending OF((z_streamp strm)); -local int read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); +local unsigned read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); #ifdef ASMV +# pragma message("Assembler code may have bugs -- use at your own risk") void match_init OF((void)); /* asm code initialization */ uInt longest_match OF((deflate_state *s, IPos cur_match)); #else local uInt longest_match OF((deflate_state *s, IPos cur_match)); #endif -#ifdef DEBUG +#ifdef ZLIB_DEBUG local void check_match OF((deflate_state *s, IPos start, IPos match, int length)); #endif @@ -149,21 +152,14 @@ local const config configuration_table[10] = { * meaning. */ -#define EQUAL 0 -/* result of memcmp for equal strings */ - -#ifndef NO_DUMMY_DECL -struct static_tree_desc_s {int dummy;}; /* for buggy compilers */ -#endif - /* rank Z_BLOCK between Z_NO_FLUSH and Z_PARTIAL_FLUSH */ #define RANK(f) (((f) * 2) - ((f) > 4 ? 9 : 0)) /* =========================================================================== * Update a hash value with the given input byte - * IN assertion: all calls to to UPDATE_HASH are made with consecutive - * input characters, so that a running hash key can be computed from the - * previous key instead of complete recalculation each time. + * IN assertion: all calls to UPDATE_HASH are made with consecutive input + * characters, so that a running hash key can be computed from the previous + * key instead of complete recalculation each time. */ #define UPDATE_HASH(s,h,c) (h = (((h)<hash_shift) ^ (c)) & s->hash_mask) @@ -174,9 +170,9 @@ struct static_tree_desc_s {int dummy;}; /* for buggy compilers */ * the previous length of the hash chain. * If this file is compiled with -DFASTEST, the compression level is forced * to 1, and no hash chains are maintained. - * IN assertion: all calls to to INSERT_STRING are made with consecutive - * input characters and the first MIN_MATCH bytes of str are valid - * (except for the last MIN_MATCH-1 bytes of the input file). + * IN assertion: all calls to INSERT_STRING are made with consecutive input + * characters and the first MIN_MATCH bytes of str are valid (except for + * the last MIN_MATCH-1 bytes of the input file). */ #ifdef FASTEST #define INSERT_STRING(s, str, match_head) \ @@ -198,6 +194,37 @@ struct static_tree_desc_s {int dummy;}; /* for buggy compilers */ s->head[s->hash_size-1] = NIL; \ zmemzero((Bytef *)s->head, (unsigned)(s->hash_size-1)*sizeof(*s->head)); +/* =========================================================================== + * Slide the hash table when sliding the window down (could be avoided with 32 + * bit values at the expense of memory usage). We slide even when level == 0 to + * keep the hash table consistent if we switch back to level > 0 later. + */ +local void slide_hash(s) + deflate_state *s; +{ + unsigned n, m; + Posf *p; + uInt wsize = s->w_size; + + n = s->hash_size; + p = &s->head[n]; + do { + m = *--p; + *p = (Pos)(m >= wsize ? m - wsize : NIL); + } while (--n); + n = wsize; +#ifndef FASTEST + p = &s->prev[n]; + do { + m = *--p; + *p = (Pos)(m >= wsize ? m - wsize : NIL); + /* If n is not on any hash chain, prev[n] is garbage but + * its value will never be used. + */ + } while (--n); +#endif +} + /* ========================================================================= */ int ZEXPORT deflateInit_(strm, level, version, stream_size) z_streamp strm; @@ -271,7 +298,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, #endif if (memLevel < 1 || memLevel > MAX_MEM_LEVEL || method != Z_DEFLATED || windowBits < 8 || windowBits > 15 || level < 0 || level > 9 || - strategy < 0 || strategy > Z_FIXED) { + strategy < 0 || strategy > Z_FIXED || (windowBits == 8 && wrap != 1)) { return Z_STREAM_ERROR; } if (windowBits == 8) windowBits = 9; /* until 256-byte window bug fixed */ @@ -279,14 +306,15 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, if (s == Z_NULL) return Z_MEM_ERROR; strm->state = (struct internal_state FAR *)s; s->strm = strm; + s->status = INIT_STATE; /* to pass state test in deflateReset() */ s->wrap = wrap; s->gzhead = Z_NULL; - s->w_bits = windowBits; + s->w_bits = (uInt)windowBits; s->w_size = 1 << s->w_bits; s->w_mask = s->w_size - 1; - s->hash_bits = memLevel + 7; + s->hash_bits = (uInt)memLevel + 7; s->hash_size = 1 << s->hash_bits; s->hash_mask = s->hash_size - 1; s->hash_shift = ((s->hash_bits+MIN_MATCH-1)/MIN_MATCH); @@ -320,6 +348,31 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, return deflateReset(strm); } +/* ========================================================================= + * Check for a valid deflate stream state. Return 0 if ok, 1 if not. + */ +local int deflateStateCheck (strm) + z_streamp strm; +{ + deflate_state *s; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + s = strm->state; + if (s == Z_NULL || s->strm != strm || (s->status != INIT_STATE && +#ifdef GZIP + s->status != GZIP_STATE && +#endif + s->status != EXTRA_STATE && + s->status != NAME_STATE && + s->status != COMMENT_STATE && + s->status != HCRC_STATE && + s->status != BUSY_STATE && + s->status != FINISH_STATE)) + return 1; + return 0; +} + /* ========================================================================= */ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) z_streamp strm; @@ -332,7 +385,7 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) unsigned avail; z_const unsigned char *next; - if (strm == Z_NULL || strm->state == Z_NULL || dictionary == Z_NULL) + if (deflateStateCheck(strm) || dictionary == Z_NULL) return Z_STREAM_ERROR; s = strm->state; wrap = s->wrap; @@ -390,13 +443,34 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) } /* ========================================================================= */ +int ZEXPORT deflateGetDictionary (strm, dictionary, dictLength) + z_streamp strm; + Bytef *dictionary; + uInt *dictLength; +{ + deflate_state *s; + uInt len; + + if (deflateStateCheck(strm)) + return Z_STREAM_ERROR; + s = strm->state; + len = s->strstart + s->lookahead; + if (len > s->w_size) + len = s->w_size; + if (dictionary != Z_NULL && len) + zmemcpy(dictionary, s->window + s->strstart + s->lookahead - len, len); + if (dictLength != Z_NULL) + *dictLength = len; + return Z_OK; +} + +/* ========================================================================= */ int ZEXPORT deflateResetKeep (strm) z_streamp strm; { deflate_state *s; - if (strm == Z_NULL || strm->state == Z_NULL || - strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) { + if (deflateStateCheck(strm)) { return Z_STREAM_ERROR; } @@ -412,7 +486,11 @@ int ZEXPORT deflateResetKeep (strm) if (s->wrap < 0) { s->wrap = -s->wrap; /* was made negative by deflate(..., Z_FINISH); */ } - s->status = s->wrap ? INIT_STATE : BUSY_STATE; + s->status = +#ifdef GZIP + s->wrap == 2 ? GZIP_STATE : +#endif + s->wrap ? INIT_STATE : BUSY_STATE; strm->adler = #ifdef GZIP s->wrap == 2 ? crc32(0L, Z_NULL, 0) : @@ -442,8 +520,8 @@ int ZEXPORT deflateSetHeader (strm, head) z_streamp strm; gz_headerp head; { - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; - if (strm->state->wrap != 2) return Z_STREAM_ERROR; + if (deflateStateCheck(strm) || strm->state->wrap != 2) + return Z_STREAM_ERROR; strm->state->gzhead = head; return Z_OK; } @@ -454,7 +532,7 @@ int ZEXPORT deflatePending (strm, pending, bits) int *bits; z_streamp strm; { - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; if (pending != Z_NULL) *pending = strm->state->pending; if (bits != Z_NULL) @@ -471,7 +549,7 @@ int ZEXPORT deflatePrime (strm, bits, value) deflate_state *s; int put; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; if ((Bytef *)(s->d_buf) < s->pending_out + ((Buf_size + 7) >> 3)) return Z_BUF_ERROR; @@ -496,9 +574,8 @@ int ZEXPORT deflateParams(strm, level, strategy) { deflate_state *s; compress_func func; - int err = Z_OK; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; #ifdef FASTEST @@ -512,13 +589,22 @@ int ZEXPORT deflateParams(strm, level, strategy) func = configuration_table[s->level].func; if ((strategy != s->strategy || func != configuration_table[level].func) && - strm->total_in != 0) { + s->high_water) { /* Flush the last buffer: */ - err = deflate(strm, Z_BLOCK); - if (err == Z_BUF_ERROR && s->pending == 0) - err = Z_OK; + int err = deflate(strm, Z_BLOCK); + if (err == Z_STREAM_ERROR) + return err; + if (strm->avail_out == 0) + return Z_BUF_ERROR; } if (s->level != level) { + if (s->level == 0 && s->matches != 0) { + if (s->matches == 1) + slide_hash(s); + else + CLEAR_HASH(s); + s->matches = 0; + } s->level = level; s->max_lazy_match = configuration_table[level].max_lazy; s->good_match = configuration_table[level].good_length; @@ -526,7 +612,7 @@ int ZEXPORT deflateParams(strm, level, strategy) s->max_chain_length = configuration_table[level].max_chain; } s->strategy = strategy; - return err; + return Z_OK; } /* ========================================================================= */ @@ -539,12 +625,12 @@ int ZEXPORT deflateTune(strm, good_length, max_lazy, nice_length, max_chain) { deflate_state *s; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; - s->good_match = good_length; - s->max_lazy_match = max_lazy; + s->good_match = (uInt)good_length; + s->max_lazy_match = (uInt)max_lazy; s->nice_match = nice_length; - s->max_chain_length = max_chain; + s->max_chain_length = (uInt)max_chain; return Z_OK; } @@ -571,14 +657,13 @@ uLong ZEXPORT deflateBound(strm, sourceLen) { deflate_state *s; uLong complen, wraplen; - Bytef *str; /* conservative upper bound for compressed data */ complen = sourceLen + ((sourceLen + 7) >> 3) + ((sourceLen + 63) >> 6) + 5; /* if can't get parameters, return conservative bound plus zlib wrapper */ - if (strm == Z_NULL || strm->state == Z_NULL) + if (deflateStateCheck(strm)) return complen + 6; /* compute wrapper length */ @@ -590,9 +675,11 @@ uLong ZEXPORT deflateBound(strm, sourceLen) case 1: /* zlib wrapper */ wraplen = 6 + (s->strstart ? 4 : 0); break; +#ifdef GZIP case 2: /* gzip wrapper */ wraplen = 18; if (s->gzhead != Z_NULL) { /* user-supplied gzip header */ + Bytef *str; if (s->gzhead->extra != Z_NULL) wraplen += 2 + s->gzhead->extra_len; str = s->gzhead->name; @@ -609,6 +696,7 @@ uLong ZEXPORT deflateBound(strm, sourceLen) wraplen += 2; } break; +#endif default: /* for compiler happiness */ wraplen = 6; } @@ -636,10 +724,10 @@ local void putShortMSB (s, b) } /* ========================================================================= - * Flush as much pending output as possible. All deflate() output goes - * through this function so some applications may wish to modify it - * to avoid allocating a large strm->next_out buffer and copying into it. - * (See also read_buf()). + * Flush as much pending output as possible. All deflate() output, except for + * some deflate_stored() output, goes through this function so some + * applications may wish to modify it to avoid allocating a large + * strm->next_out buffer and copying into it. (See also read_buf()). */ local void flush_pending(strm) z_streamp strm; @@ -656,13 +744,23 @@ local void flush_pending(strm) strm->next_out += len; s->pending_out += len; strm->total_out += len; - strm->avail_out -= len; - s->pending -= len; + strm->avail_out -= len; + s->pending -= len; if (s->pending == 0) { s->pending_out = s->pending_buf; } } +/* =========================================================================== + * Update the header CRC with the bytes s->pending_buf[beg..s->pending - 1]. + */ +#define HCRC_UPDATE(beg) \ + do { \ + if (s->gzhead->hcrc && s->pending > (beg)) \ + strm->adler = crc32(strm->adler, s->pending_buf + (beg), \ + s->pending - (beg)); \ + } while (0) + /* ========================================================================= */ int ZEXPORT deflate (strm, flush) z_streamp strm; @@ -671,230 +769,229 @@ int ZEXPORT deflate (strm, flush) int old_flush; /* value of flush param for previous deflate call */ deflate_state *s; - if (strm == Z_NULL || strm->state == Z_NULL || - flush > Z_BLOCK || flush < 0) { + if (deflateStateCheck(strm) || flush > Z_BLOCK || flush < 0) { return Z_STREAM_ERROR; } s = strm->state; if (strm->next_out == Z_NULL || - (strm->next_in == Z_NULL && strm->avail_in != 0) || + (strm->avail_in != 0 && strm->next_in == Z_NULL) || (s->status == FINISH_STATE && flush != Z_FINISH)) { ERR_RETURN(strm, Z_STREAM_ERROR); } if (strm->avail_out == 0) ERR_RETURN(strm, Z_BUF_ERROR); - s->strm = strm; /* just in case */ old_flush = s->last_flush; s->last_flush = flush; + /* Flush as much pending output as possible */ + if (s->pending != 0) { + flush_pending(strm); + if (strm->avail_out == 0) { + /* Since avail_out is 0, deflate will be called again with + * more output space, but possibly with both pending and + * avail_in equal to zero. There won't be anything to do, + * but this is not an error situation so make sure we + * return OK instead of BUF_ERROR at next call of deflate: + */ + s->last_flush = -1; + return Z_OK; + } + + /* Make sure there is something to do and avoid duplicate consecutive + * flushes. For repeated and useless calls with Z_FINISH, we keep + * returning Z_STREAM_END instead of Z_BUF_ERROR. + */ + } else if (strm->avail_in == 0 && RANK(flush) <= RANK(old_flush) && + flush != Z_FINISH) { + ERR_RETURN(strm, Z_BUF_ERROR); + } + + /* User must not provide more input after the first FINISH: */ + if (s->status == FINISH_STATE && strm->avail_in != 0) { + ERR_RETURN(strm, Z_BUF_ERROR); + } + /* Write the header */ if (s->status == INIT_STATE) { -#ifdef GZIP - if (s->wrap == 2) { - strm->adler = crc32(0L, Z_NULL, 0); - put_byte(s, 31); - put_byte(s, 139); - put_byte(s, 8); - if (s->gzhead == Z_NULL) { - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, s->level == 9 ? 2 : - (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? - 4 : 0)); - put_byte(s, OS_CODE); - s->status = BUSY_STATE; - } - else { - put_byte(s, (s->gzhead->text ? 1 : 0) + - (s->gzhead->hcrc ? 2 : 0) + - (s->gzhead->extra == Z_NULL ? 0 : 4) + - (s->gzhead->name == Z_NULL ? 0 : 8) + - (s->gzhead->comment == Z_NULL ? 0 : 16) - ); - put_byte(s, (Byte)(s->gzhead->time & 0xff)); - put_byte(s, (Byte)((s->gzhead->time >> 8) & 0xff)); - put_byte(s, (Byte)((s->gzhead->time >> 16) & 0xff)); - put_byte(s, (Byte)((s->gzhead->time >> 24) & 0xff)); - put_byte(s, s->level == 9 ? 2 : - (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? - 4 : 0)); - put_byte(s, s->gzhead->os & 0xff); - if (s->gzhead->extra != Z_NULL) { - put_byte(s, s->gzhead->extra_len & 0xff); - put_byte(s, (s->gzhead->extra_len >> 8) & 0xff); - } - if (s->gzhead->hcrc) - strm->adler = crc32(strm->adler, s->pending_buf, - s->pending); - s->gzindex = 0; - s->status = EXTRA_STATE; - } - } + /* zlib header */ + uInt header = (Z_DEFLATED + ((s->w_bits-8)<<4)) << 8; + uInt level_flags; + + if (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2) + level_flags = 0; + else if (s->level < 6) + level_flags = 1; + else if (s->level == 6) + level_flags = 2; else -#endif - { - uInt header = (Z_DEFLATED + ((s->w_bits-8)<<4)) << 8; - uInt level_flags; - - if (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2) - level_flags = 0; - else if (s->level < 6) - level_flags = 1; - else if (s->level == 6) - level_flags = 2; - else - level_flags = 3; - header |= (level_flags << 6); - if (s->strstart != 0) header |= PRESET_DICT; - header += 31 - (header % 31); + level_flags = 3; + header |= (level_flags << 6); + if (s->strstart != 0) header |= PRESET_DICT; + header += 31 - (header % 31); + + putShortMSB(s, header); + /* Save the adler32 of the preset dictionary: */ + if (s->strstart != 0) { + putShortMSB(s, (uInt)(strm->adler >> 16)); + putShortMSB(s, (uInt)(strm->adler & 0xffff)); + } + strm->adler = adler32(0L, Z_NULL, 0); + s->status = BUSY_STATE; + + /* Compression must start with an empty pending buffer */ + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } +#ifdef GZIP + if (s->status == GZIP_STATE) { + /* gzip header */ + strm->adler = crc32(0L, Z_NULL, 0); + put_byte(s, 31); + put_byte(s, 139); + put_byte(s, 8); + if (s->gzhead == Z_NULL) { + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, s->level == 9 ? 2 : + (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? + 4 : 0)); + put_byte(s, OS_CODE); s->status = BUSY_STATE; - putShortMSB(s, header); - /* Save the adler32 of the preset dictionary: */ - if (s->strstart != 0) { - putShortMSB(s, (uInt)(strm->adler >> 16)); - putShortMSB(s, (uInt)(strm->adler & 0xffff)); + /* Compression must start with an empty pending buffer */ + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } + else { + put_byte(s, (s->gzhead->text ? 1 : 0) + + (s->gzhead->hcrc ? 2 : 0) + + (s->gzhead->extra == Z_NULL ? 0 : 4) + + (s->gzhead->name == Z_NULL ? 0 : 8) + + (s->gzhead->comment == Z_NULL ? 0 : 16) + ); + put_byte(s, (Byte)(s->gzhead->time & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 8) & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 16) & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 24) & 0xff)); + put_byte(s, s->level == 9 ? 2 : + (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? + 4 : 0)); + put_byte(s, s->gzhead->os & 0xff); + if (s->gzhead->extra != Z_NULL) { + put_byte(s, s->gzhead->extra_len & 0xff); + put_byte(s, (s->gzhead->extra_len >> 8) & 0xff); } - strm->adler = adler32(0L, Z_NULL, 0); + if (s->gzhead->hcrc) + strm->adler = crc32(strm->adler, s->pending_buf, + s->pending); + s->gzindex = 0; + s->status = EXTRA_STATE; } } -#ifdef GZIP if (s->status == EXTRA_STATE) { if (s->gzhead->extra != Z_NULL) { - uInt beg = s->pending; /* start of bytes to update crc */ - - while (s->gzindex < (s->gzhead->extra_len & 0xffff)) { - if (s->pending == s->pending_buf_size) { - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - flush_pending(strm); - beg = s->pending; - if (s->pending == s->pending_buf_size) - break; + ulg beg = s->pending; /* start of bytes to update crc */ + uInt left = (s->gzhead->extra_len & 0xffff) - s->gzindex; + while (s->pending + left > s->pending_buf_size) { + uInt copy = s->pending_buf_size - s->pending; + zmemcpy(s->pending_buf + s->pending, + s->gzhead->extra + s->gzindex, copy); + s->pending = s->pending_buf_size; + HCRC_UPDATE(beg); + s->gzindex += copy; + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; } - put_byte(s, s->gzhead->extra[s->gzindex]); - s->gzindex++; - } - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - if (s->gzindex == s->gzhead->extra_len) { - s->gzindex = 0; - s->status = NAME_STATE; + beg = 0; + left -= copy; } + zmemcpy(s->pending_buf + s->pending, + s->gzhead->extra + s->gzindex, left); + s->pending += left; + HCRC_UPDATE(beg); + s->gzindex = 0; } - else - s->status = NAME_STATE; + s->status = NAME_STATE; } if (s->status == NAME_STATE) { if (s->gzhead->name != Z_NULL) { - uInt beg = s->pending; /* start of bytes to update crc */ + ulg beg = s->pending; /* start of bytes to update crc */ int val; - do { if (s->pending == s->pending_buf_size) { - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); + HCRC_UPDATE(beg); flush_pending(strm); - beg = s->pending; - if (s->pending == s->pending_buf_size) { - val = 1; - break; + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; } + beg = 0; } val = s->gzhead->name[s->gzindex++]; put_byte(s, val); } while (val != 0); - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - if (val == 0) { - s->gzindex = 0; - s->status = COMMENT_STATE; - } + HCRC_UPDATE(beg); + s->gzindex = 0; } - else - s->status = COMMENT_STATE; + s->status = COMMENT_STATE; } if (s->status == COMMENT_STATE) { if (s->gzhead->comment != Z_NULL) { - uInt beg = s->pending; /* start of bytes to update crc */ + ulg beg = s->pending; /* start of bytes to update crc */ int val; - do { if (s->pending == s->pending_buf_size) { - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); + HCRC_UPDATE(beg); flush_pending(strm); - beg = s->pending; - if (s->pending == s->pending_buf_size) { - val = 1; - break; + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; } + beg = 0; } val = s->gzhead->comment[s->gzindex++]; put_byte(s, val); } while (val != 0); - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - if (val == 0) - s->status = HCRC_STATE; + HCRC_UPDATE(beg); } - else - s->status = HCRC_STATE; + s->status = HCRC_STATE; } if (s->status == HCRC_STATE) { if (s->gzhead->hcrc) { - if (s->pending + 2 > s->pending_buf_size) + if (s->pending + 2 > s->pending_buf_size) { flush_pending(strm); - if (s->pending + 2 <= s->pending_buf_size) { - put_byte(s, (Byte)(strm->adler & 0xff)); - put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); - strm->adler = crc32(0L, Z_NULL, 0); - s->status = BUSY_STATE; + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } } + put_byte(s, (Byte)(strm->adler & 0xff)); + put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); + strm->adler = crc32(0L, Z_NULL, 0); } - else - s->status = BUSY_STATE; - } -#endif + s->status = BUSY_STATE; - /* Flush as much pending output as possible */ - if (s->pending != 0) { + /* Compression must start with an empty pending buffer */ flush_pending(strm); - if (strm->avail_out == 0) { - /* Since avail_out is 0, deflate will be called again with - * more output space, but possibly with both pending and - * avail_in equal to zero. There won't be anything to do, - * but this is not an error situation so make sure we - * return OK instead of BUF_ERROR at next call of deflate: - */ + if (s->pending != 0) { s->last_flush = -1; return Z_OK; } - - /* Make sure there is something to do and avoid duplicate consecutive - * flushes. For repeated and useless calls with Z_FINISH, we keep - * returning Z_STREAM_END instead of Z_BUF_ERROR. - */ - } else if (strm->avail_in == 0 && RANK(flush) <= RANK(old_flush) && - flush != Z_FINISH) { - ERR_RETURN(strm, Z_BUF_ERROR); - } - - /* User must not provide more input after the first FINISH: */ - if (s->status == FINISH_STATE && strm->avail_in != 0) { - ERR_RETURN(strm, Z_BUF_ERROR); } +#endif if (strm->start_bit == 0) strm->start_bit = (strm->total_out + s->pending) * 8 + s->bi_valid; @@ -905,9 +1002,10 @@ int ZEXPORT deflate (strm, flush) (flush != Z_NO_FLUSH && s->status != FINISH_STATE)) { block_state bstate; - bstate = s->strategy == Z_HUFFMAN_ONLY ? deflate_huff(s, flush) : - (s->strategy == Z_RLE ? deflate_rle(s, flush) : - (*(configuration_table[s->level].func))(s, flush)); + bstate = s->level == 0 ? deflate_stored(s, flush) : + s->strategy == Z_HUFFMAN_ONLY ? deflate_huff(s, flush) : + s->strategy == Z_RLE ? deflate_rle(s, flush) : + (*(configuration_table[s->level].func))(s, flush); if (bstate == finish_started || bstate == finish_done) { s->status = FINISH_STATE; @@ -949,7 +1047,6 @@ int ZEXPORT deflate (strm, flush) } } } - Assert(strm->avail_out > 0, "bug2"); if (flush != Z_FINISH) return Z_OK; if (s->wrap <= 0) return Z_STREAM_END; @@ -986,18 +1083,9 @@ int ZEXPORT deflateEnd (strm) { int status; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; status = strm->state->status; - if (status != INIT_STATE && - status != EXTRA_STATE && - status != NAME_STATE && - status != COMMENT_STATE && - status != HCRC_STATE && - status != BUSY_STATE && - status != FINISH_STATE) { - return Z_STREAM_ERROR; - } /* Deallocate in reverse order of allocations: */ TRY_FREE(strm, strm->state->pending_buf); @@ -1028,7 +1116,7 @@ int ZEXPORT deflateCopy (dest, source) ushf *overlay; - if (source == Z_NULL || dest == Z_NULL || source->state == Z_NULL) { + if (deflateStateCheck(source) || dest == Z_NULL) { return Z_STREAM_ERROR; } @@ -1078,7 +1166,7 @@ int ZEXPORT deflateCopy (dest, source) * allocating a large strm->next_in buffer and copying from it. * (See also flush_pending()). */ -local int read_buf(strm, buf, size) +local unsigned read_buf(strm, buf, size) z_streamp strm; Bytef *buf; unsigned size; @@ -1102,7 +1190,7 @@ local int read_buf(strm, buf, size) strm->next_in += len; strm->total_in += len; - return (int)len; + return len; } /* =========================================================================== @@ -1156,9 +1244,9 @@ local uInt longest_match(s, cur_match) { unsigned chain_length = s->max_chain_length;/* max hash chain length */ register Bytef *scan = s->window + s->strstart; /* current string */ - register Bytef *match; /* matched string */ + register Bytef *match; /* matched string */ register int len; /* length of current match */ - int best_len = s->prev_length; /* best match length so far */ + int best_len = (int)s->prev_length; /* best match length so far */ int nice_match = s->nice_match; /* stop if match long enough */ IPos limit = s->strstart > (IPos)MAX_DIST(s) ? s->strstart - (IPos)MAX_DIST(s) : NIL; @@ -1193,7 +1281,7 @@ local uInt longest_match(s, cur_match) /* Do not look for matches beyond the end of the input. This is necessary * to make deflate deterministic. */ - if ((uInt)nice_match > s->lookahead) nice_match = s->lookahead; + if ((uInt)nice_match > s->lookahead) nice_match = (int)s->lookahead; Assert((ulg)s->strstart <= s->window_size-MIN_LOOKAHEAD, "need lookahead"); @@ -1354,7 +1442,11 @@ local uInt longest_match(s, cur_match) #endif /* FASTEST */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG + +#define EQUAL 0 +/* result of memcmp for equal strings */ + /* =========================================================================== * Check that the match at match_start is indeed a match. */ @@ -1380,7 +1472,7 @@ local void check_match(s, start, match, length) } #else # define check_match(s, start, match, length) -#endif /* DEBUG */ +#endif /* ZLIB_DEBUG */ /* =========================================================================== * Fill the window when the lookahead becomes insufficient. @@ -1395,8 +1487,7 @@ local void check_match(s, start, match, length) local void fill_window(s) deflate_state *s; { - register unsigned n, m; - register Posf *p; + unsigned n; unsigned more; /* Amount of free space at the end of the window. */ uInt wsize = s->w_size; @@ -1423,35 +1514,11 @@ local void fill_window(s) */ if (s->strstart >= wsize+MAX_DIST(s)) { - zmemcpy(s->window, s->window+wsize, (unsigned)wsize); + zmemcpy(s->window, s->window+wsize, (unsigned)wsize - more); s->match_start -= wsize; s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ s->block_start -= (long) wsize; - - /* Slide the hash table (could be avoided with 32 bit values - at the expense of memory usage). We slide even when level == 0 - to keep the hash table consistent if we switch back to level > 0 - later. (Using level 0 permanently is not an optimal usage of - zlib, so we don't care about this pathological case.) - */ - n = s->hash_size; - p = &s->head[n]; - do { - m = *--p; - *p = (Pos)(m >= wsize ? m-wsize : NIL); - } while (--n); - - n = wsize; -#ifndef FASTEST - p = &s->prev[n]; - do { - m = *--p; - *p = (Pos)(m >= wsize ? m-wsize : NIL); - /* If n is not on any hash chain, prev[n] is garbage but - * its value will never be used. - */ - } while (--n); -#endif + slide_hash(s); more += wsize; } if (s->strm->avail_in == 0) break; @@ -1557,70 +1624,199 @@ local void fill_window(s) if (s->strm->avail_out == 0) return (last) ? finish_started : need_more; \ } +/* Maximum stored block length in deflate format (not including header). */ +#define MAX_STORED 65535 + +/* Minimum of a and b. */ +#define MIN(a, b) ((a) > (b) ? (b) : (a)) + /* =========================================================================== * Copy without compression as much as possible from the input stream, return * the current block state. - * This function does not insert new strings in the dictionary since - * uncompressible data is probably not useful. This function is used - * only for the level=0 compression option. - * NOTE: this function should be optimized to avoid extra copying from - * window to pending_buf. + * + * In case deflateParams() is used to later switch to a non-zero compression + * level, s->matches (otherwise unused when storing) keeps track of the number + * of hash table slides to perform. If s->matches is 1, then one hash table + * slide will be done when switching. If s->matches is 2, the maximum value + * allowed here, then the hash table will be cleared, since two or more slides + * is the same as a clear. + * + * deflate_stored() is written to minimize the number of times an input byte is + * copied. It is most efficient with large input and output buffers, which + * maximizes the opportunites to have a single copy from next_in to next_out. */ local block_state deflate_stored(s, flush) deflate_state *s; int flush; { - /* Stored blocks are limited to 0xffff bytes, pending_buf is limited - * to pending_buf_size, and each stored block has a 5 byte header: + /* Smallest worthy block size when not flushing or finishing. By default + * this is 32K. This can be as small as 507 bytes for memLevel == 1. For + * large input and output buffers, the stored block size will be larger. */ - ulg max_block_size = 0xffff; - ulg max_start; - - if (max_block_size > s->pending_buf_size - 5) { - max_block_size = s->pending_buf_size - 5; - } + unsigned min_block = MIN(s->pending_buf_size - 5, s->w_size); - /* Copy as much as possible from input to output: */ - for (;;) { - /* Fill the window as much as possible: */ - if (s->lookahead <= 1) { - - Assert(s->strstart < s->w_size+MAX_DIST(s) || - s->block_start >= (long)s->w_size, "slide too late"); + /* Copy as many min_block or larger stored blocks directly to next_out as + * possible. If flushing, copy the remaining available input to next_out as + * stored blocks, if there is enough space. + */ + unsigned len, left, have, last = 0; + unsigned used = s->strm->avail_in; + do { + /* Set len to the maximum size block that we can copy directly with the + * available input data and output space. Set left to how much of that + * would be copied from what's left in the window. + */ + len = MAX_STORED; /* maximum deflate stored block length */ + have = (s->bi_valid + 42) >> 3; /* number of header bytes */ + if (s->strm->avail_out < have) /* need room for header */ + break; + /* maximum stored block length that will fit in avail_out: */ + have = s->strm->avail_out - have; + left = s->strstart - s->block_start; /* bytes left in window */ + if (len > (ulg)left + s->strm->avail_in) + len = left + s->strm->avail_in; /* limit len to the input */ + if (len > have) + len = have; /* limit len to the output */ + + /* If the stored block would be less than min_block in length, or if + * unable to copy all of the available input when flushing, then try + * copying to the window and the pending buffer instead. Also don't + * write an empty block when flushing -- deflate() does that. + */ + if (len < min_block && ((len == 0 && flush != Z_FINISH) || + flush == Z_NO_FLUSH || + len != left + s->strm->avail_in)) + break; - fill_window(s); - if (s->lookahead == 0 && flush == Z_NO_FLUSH) return need_more; + /* Make a dummy stored block in pending to get the header bytes, + * including any pending bits. This also updates the debugging counts. + */ + last = flush == Z_FINISH && len == left + s->strm->avail_in ? 1 : 0; + _tr_stored_block(s, (char *)0, 0L, last); + + /* Replace the lengths in the dummy stored block with len. */ + s->pending_buf[s->pending - 4] = len; + s->pending_buf[s->pending - 3] = len >> 8; + s->pending_buf[s->pending - 2] = ~len; + s->pending_buf[s->pending - 1] = ~len >> 8; + + /* Write the stored block header bytes. */ + flush_pending(s->strm); + +#ifdef ZLIB_DEBUG + /* Update debugging counts for the data about to be copied. */ + s->compressed_len += len << 3; + s->bits_sent += len << 3; +#endif - if (s->lookahead == 0) break; /* flush the current block */ + /* Copy uncompressed bytes from the window to next_out. */ + if (left) { + if (left > len) + left = len; + zmemcpy(s->strm->next_out, s->window + s->block_start, left); + s->strm->next_out += left; + s->strm->avail_out -= left; + s->strm->total_out += left; + s->block_start += left; + len -= left; } - Assert(s->block_start >= 0L, "block gone"); - - s->strstart += s->lookahead; - s->lookahead = 0; - - /* Emit a stored block if pending_buf will be full: */ - max_start = s->block_start + max_block_size; - if (s->strstart == 0 || (ulg)s->strstart >= max_start) { - /* strstart == 0 is possible when wraparound on 16-bit machine */ - s->lookahead = (uInt)(s->strstart - max_start); - s->strstart = (uInt)max_start; - FLUSH_BLOCK(s, 0); + + /* Copy uncompressed bytes directly from next_in to next_out, updating + * the check value. + */ + if (len) { + read_buf(s->strm, s->strm->next_out, len); + s->strm->next_out += len; + s->strm->avail_out -= len; + s->strm->total_out += len; } - /* Flush if we may have to slide, otherwise block_start may become - * negative and the data will be gone: + } while (last == 0); + + /* Update the sliding window with the last s->w_size bytes of the copied + * data, or append all of the copied data to the existing window if less + * than s->w_size bytes were copied. Also update the number of bytes to + * insert in the hash tables, in the event that deflateParams() switches to + * a non-zero compression level. + */ + used -= s->strm->avail_in; /* number of input bytes directly copied */ + if (used) { + /* If any input was used, then no unused input remains in the window, + * therefore s->block_start == s->strstart. */ - if (s->strstart - (uInt)s->block_start >= MAX_DIST(s)) { - FLUSH_BLOCK(s, 0); + if (used >= s->w_size) { /* supplant the previous history */ + s->matches = 2; /* clear hash */ + zmemcpy(s->window, s->strm->next_in - s->w_size, s->w_size); + s->strstart = s->w_size; } + else { + if (s->window_size - s->strstart <= used) { + /* Slide the window down. */ + s->strstart -= s->w_size; + zmemcpy(s->window, s->window + s->w_size, s->strstart); + if (s->matches < 2) + s->matches++; /* add a pending slide_hash() */ + } + zmemcpy(s->window + s->strstart, s->strm->next_in - used, used); + s->strstart += used; + } + s->block_start = s->strstart; + s->insert += MIN(used, s->w_size - s->insert); } - s->insert = 0; - if (flush == Z_FINISH) { - FLUSH_BLOCK(s, 1); + if (s->high_water < s->strstart) + s->high_water = s->strstart; + + /* If the last block was written to next_out, then done. */ + if (last) return finish_done; + + /* If flushing and all input has been consumed, then done. */ + if (flush != Z_NO_FLUSH && flush != Z_FINISH && + s->strm->avail_in == 0 && (long)s->strstart == s->block_start) + return block_done; + + /* Fill the window with any remaining input. */ + have = s->window_size - s->strstart - 1; + if (s->strm->avail_in > have && s->block_start >= (long)s->w_size) { + /* Slide the window down. */ + s->block_start -= s->w_size; + s->strstart -= s->w_size; + zmemcpy(s->window, s->window + s->w_size, s->strstart); + if (s->matches < 2) + s->matches++; /* add a pending slide_hash() */ + have += s->w_size; /* more space now */ } - if ((long)s->strstart > s->block_start) - FLUSH_BLOCK(s, 0); - return block_done; + if (have > s->strm->avail_in) + have = s->strm->avail_in; + if (have) { + read_buf(s->strm, s->window + s->strstart, have); + s->strstart += have; + } + if (s->high_water < s->strstart) + s->high_water = s->strstart; + + /* There was not enough avail_out to write a complete worthy or flushed + * stored block to next_out. Write a stored block to pending instead, if we + * have enough input for a worthy block, or if flushing and there is enough + * room for the remaining input as a stored block in the pending buffer. + */ + have = (s->bi_valid + 42) >> 3; /* number of header bytes */ + /* maximum stored block length that will fit in pending: */ + have = MIN(s->pending_buf_size - have, MAX_STORED); + min_block = MIN(have, s->w_size); + left = s->strstart - s->block_start; + if (left >= min_block || + ((left || flush == Z_FINISH) && flush != Z_NO_FLUSH && + s->strm->avail_in == 0 && left <= have)) { + len = MIN(left, have); + last = flush == Z_FINISH && s->strm->avail_in == 0 && + len == left ? 1 : 0; + _tr_stored_block(s, (charf *)s->window + s->block_start, len, last); + s->block_start += len; + flush_pending(s->strm); + } + + /* We've done all we can with the available input and output. */ + return last ? finish_started : need_more; } /* =========================================================================== @@ -1897,7 +2093,7 @@ local block_state deflate_rle(s, flush) prev == *++scan && prev == *++scan && prev == *++scan && prev == *++scan && scan < strend); - s->match_length = MAX_MATCH - (int)(strend - scan); + s->match_length = MAX_MATCH - (uInt)(strend - scan); if (s->match_length > s->lookahead) s->match_length = s->lookahead; } diff --git a/lib/libvgz/deflate.h b/lib/libvgz/deflate.h index ce0299e..23ecdd3 100644 --- a/lib/libvgz/deflate.h +++ b/lib/libvgz/deflate.h @@ -1,5 +1,5 @@ /* deflate.h -- internal compression state - * Copyright (C) 1995-2012 Jean-loup Gailly + * Copyright (C) 1995-2016 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -51,13 +51,16 @@ #define Buf_size 16 /* size of bit buffer in bi_buf */ -#define INIT_STATE 42 -#define EXTRA_STATE 69 -#define NAME_STATE 73 -#define COMMENT_STATE 91 -#define HCRC_STATE 103 -#define BUSY_STATE 113 -#define FINISH_STATE 666 +#define INIT_STATE 42 /* zlib header -> BUSY_STATE */ +#ifdef GZIP +# define GZIP_STATE 57 /* gzip header -> BUSY_STATE | EXTRA_STATE */ +#endif +#define EXTRA_STATE 69 /* gzip extra block -> NAME_STATE */ +#define NAME_STATE 73 /* gzip file name -> COMMENT_STATE */ +#define COMMENT_STATE 91 /* gzip comment -> HCRC_STATE */ +#define HCRC_STATE 103 /* gzip header CRC -> BUSY_STATE */ +#define BUSY_STATE 113 /* deflate -> FINISH_STATE */ +#define FINISH_STATE 666 /* stream complete */ /* Stream status */ @@ -83,7 +86,7 @@ typedef struct static_tree_desc_s static_tree_desc; typedef struct tree_desc_s { ct_data *dyn_tree; /* the dynamic tree */ int max_code; /* largest code with non zero frequency */ - static_tree_desc *stat_desc; /* the corresponding static tree */ + const static_tree_desc *stat_desc; /* the corresponding static tree */ } FAR tree_desc; typedef ush Pos; @@ -100,10 +103,10 @@ typedef struct internal_state { Bytef *pending_buf; /* output still pending */ ulg pending_buf_size; /* size of pending_buf */ Bytef *pending_out; /* next pending byte to output to the stream */ - uInt pending; /* nb of bytes in the pending buffer */ + ulg pending; /* nb of bytes in the pending buffer */ int wrap; /* bit 0 true for zlib, bit 1 true for gzip */ gz_headerp gzhead; /* gzip header information to write */ - uInt gzindex; /* where in extra, name, or comment */ + ulg gzindex; /* where in extra, name, or comment */ Byte method; /* can only be DEFLATED */ int last_flush; /* value of flush param for previous deflate call */ @@ -249,7 +252,7 @@ typedef struct internal_state { uInt matches; /* number of string matches in current block */ uInt insert; /* bytes at end of window left to insert */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG ulg compressed_len; /* total bit length of compressed file mod 2^32 */ ulg bits_sent; /* bit length of compressed data sent mod 2^32 */ #endif @@ -275,7 +278,7 @@ typedef struct internal_state { /* Output a byte on the stream. * IN assertion: there is enough room in pending_buf. */ -#define put_byte(s, c) {s->pending_buf[s->pending++] = (c);} +#define put_byte(s, c) {s->pending_buf[s->pending++] = (Bytef)(c);} #define MIN_LOOKAHEAD (MAX_MATCH+MIN_MATCH+1) @@ -309,7 +312,7 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, * used. */ -#ifndef DEBUG +#ifndef ZLIB_DEBUG /* Inline versions of _tr_tally for speed: */ #if defined(GEN_TREES_H) || !defined(STDC) @@ -328,8 +331,8 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, flush = (s->last_lit == s->lit_bufsize-1); \ } # define _tr_tally_dist(s, distance, length, flush) \ - { uch len = (length); \ - ush dist = (distance); \ + { uch len = (uch)(length); \ + ush dist = (ush)(distance); \ s->d_buf[s->last_lit] = dist; \ s->l_buf[s->last_lit++] = len; \ dist--; \ diff --git a/lib/libvgz/gzguts.h b/lib/libvgz/gzguts.h index 53e5aa0..0c6f7b3 100644 --- a/lib/libvgz/gzguts.h +++ b/lib/libvgz/gzguts.h @@ -1,5 +1,5 @@ /* gzguts.h -- zlib internal header definitions for gz* operations - * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -25,6 +25,10 @@ # include # include #endif + +#ifndef _POSIX_SOURCE +# define _POSIX_SOURCE +#endif #include #ifdef _WIN32 @@ -35,6 +39,10 @@ # include #endif +#if defined(_WIN32) || defined(__CYGWIN__) +# define WIDECHAR +#endif + #ifdef WINAPI_FAMILY # define open _open # define read _read @@ -95,18 +103,19 @@ # endif #endif -/* unlike snprintf (which is required in C99, yet still not supported by - Microsoft more than a decade later!), _snprintf does not guarantee null - termination of the result -- however this is only used in gzlib.c where +/* unlike snprintf (which is required in C99), _snprintf does not guarantee + null termination of the result -- however this is only used in gzlib.c where the result is assured to fit in the space provided */ -#ifdef _MSC_VER +#if defined(_MSC_VER) && _MSC_VER < 1900 # define snprintf _snprintf #endif #ifndef local # define local static #endif -/* compile with -Dlocal if your debugger can't find static symbols */ +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ /* gz* functions always use library allocation functions */ #ifndef STDC @@ -170,7 +179,7 @@ typedef struct { char *path; /* path or fd for error messages */ unsigned size; /* buffer size, zero if not allocated yet */ unsigned want; /* requested buffer size, default is GZBUFSIZE */ - unsigned char *in; /* input buffer */ + unsigned char *in; /* input buffer (double-sized when writing) */ unsigned char *out; /* output buffer (double-sized when reading) */ int direct; /* 0 if processing gzip, 1 if transparent */ /* just for reading */ diff --git a/lib/libvgz/infback.c b/lib/libvgz/infback.c index 4cffcc7..d632415 100644 --- a/lib/libvgz/infback.c +++ b/lib/libvgz/infback.c @@ -1,5 +1,5 @@ /* infback.c -- inflate using a call-back interface - * Copyright (C) 1995-2011 Mark Adler + * Copyright (C) 1995-2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -61,7 +61,7 @@ int stream_size; Tracev((stderr, "inflate: allocated\n")); strm->state = (struct internal_state FAR *)state; state->dmax = 32768U; - state->wbits = windowBits; + state->wbits = (uInt)windowBits; state->wsize = 1U << windowBits; state->window = window; state->wnext = 0; diff --git a/lib/libvgz/inffast.c b/lib/libvgz/inffast.c index 73aecb9..c869998 100644 --- a/lib/libvgz/inffast.c +++ b/lib/libvgz/inffast.c @@ -1,5 +1,5 @@ /* inffast.c -- fast decoding - * Copyright (C) 1995-2008, 2010, 2013 Mark Adler + * Copyright (C) 1995-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -8,26 +8,9 @@ #include "inflate.h" #include "inffast.h" -#ifndef ASMINF - -/* Allow machine dependent optimization for post-increment or pre-increment. - Based on testing to date, - Pre-increment preferred for: - - PowerPC G3 (Adler) - - MIPS R5000 (Randers-Pehrson) - Post-increment preferred for: - - none - No measurable difference: - - Pentium III (Anderson) - - M68060 (Nikl) - */ -#ifdef POSTINC -# define OFF 0 -# define PUP(a) *(a)++ +#ifdef ASMINF +# pragma message("Assembler code may have bugs -- use at your own risk") #else -# define OFF 1 -# define PUP(a) *++(a) -#endif /* Decode literal, length, and distance codes and write out the resulting @@ -96,9 +79,9 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ /* copy state to local variables */ state = (struct inflate_state FAR *)strm->state; - in = strm->next_in - OFF; + in = strm->next_in; last = in + (strm->avail_in - 5); - out = strm->next_out - OFF; + out = strm->next_out; beg = out - (start - strm->avail_out); end = out + (strm->avail_out - 257); #ifdef INFLATE_STRICT @@ -119,9 +102,9 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ input data or output space */ do { if (bits < 15) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } here = lcode[hold & lmask]; @@ -134,14 +117,14 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? "inflate: literal '%c'\n" : "inflate: literal 0x%02x\n", here.val)); - PUP(out) = (unsigned char)(here.val); + *out++ = (unsigned char)(here.val); } else if (op & 16) { /* length base */ len = (unsigned)(here.val); op &= 15; /* number of extra bits */ if (op) { if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } len += (unsigned)hold & ((1U << op) - 1); @@ -150,9 +133,9 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ } Tracevv((stderr, "inflate: length %u\n", len)); if (bits < 15) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } here = dcode[hold & dmask]; @@ -165,10 +148,10 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ dist = (unsigned)(here.val); op &= 15; /* number of extra bits */ if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } } @@ -196,30 +179,30 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ #ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR if (len <= op - whave) { do { - PUP(out) = 0; + *out++ = 0; } while (--len); continue; } len -= op - whave; do { - PUP(out) = 0; + *out++ = 0; } while (--op > whave); if (op == 0) { from = out - dist; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--len); continue; } #endif } - from = window - OFF; + from = window; if (wnext == 0) { /* very common case */ from += wsize - op; if (op < len) { /* some from window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } @@ -230,14 +213,14 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ if (op < len) { /* some from end of window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); - from = window - OFF; + from = window; if (wnext < len) { /* some from start of window */ op = wnext; len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } @@ -248,35 +231,35 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ if (op < len) { /* some from window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } } while (len > 2) { - PUP(out) = PUP(from); - PUP(out) = PUP(from); - PUP(out) = PUP(from); + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; len -= 3; } if (len) { - PUP(out) = PUP(from); + *out++ = *from++; if (len > 1) - PUP(out) = PUP(from); + *out++ = *from++; } } else { from = out - dist; /* copy direct from output */ do { /* minimum length is three */ - PUP(out) = PUP(from); - PUP(out) = PUP(from); - PUP(out) = PUP(from); + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; len -= 3; } while (len > 2); if (len) { - PUP(out) = PUP(from); + *out++ = *from++; if (len > 1) - PUP(out) = PUP(from); + *out++ = *from++; } } } @@ -313,8 +296,8 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ hold &= (1U << bits) - 1; /* update state and return */ - strm->next_in = in + OFF; - strm->next_out = out + OFF; + strm->next_in = in; + strm->next_out = out; strm->avail_in = (unsigned)(in < last ? 5 + (last - in) : 5 - (in - last)); strm->avail_out = (unsigned)(out < end ? 257 + (end - out) : 257 - (out - end)); diff --git a/lib/libvgz/inflate.c b/lib/libvgz/inflate.c index 5c1a641..cf0d2b0 100644 --- a/lib/libvgz/inflate.c +++ b/lib/libvgz/inflate.c @@ -1,5 +1,5 @@ /* inflate.c -- zlib decompression - * Copyright (C) 1995-2012 Mark Adler + * Copyright (C) 1995-2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -92,6 +92,7 @@ #endif /* function prototypes */ +local int inflateStateCheck OF((z_streamp strm)); local void fixedtables OF((struct inflate_state FAR *state)); local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, unsigned copy)); @@ -101,12 +102,26 @@ local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf, unsigned len)); +local int inflateStateCheck(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + state = (struct inflate_state FAR *)strm->state; + if (state == Z_NULL || state->strm != strm || + state->mode < HEAD || state->mode > SYNC) + return 1; + return 0; +} + int ZEXPORT inflateResetKeep(strm) z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; strm->total_in = strm->total_out = state->total = 0; strm->start_bit = strm->stop_bit = strm->last_bit = 0; @@ -132,7 +147,7 @@ z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; state->wsize = 0; state->whave = 0; @@ -148,7 +163,7 @@ int windowBits; struct inflate_state FAR *state; /* get the state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; /* extract wrap request from windowBits parameter */ @@ -157,7 +172,7 @@ int windowBits; windowBits = -windowBits; } else { - wrap = (windowBits >> 4) + 1; + wrap = (windowBits >> 4) + 5; #ifdef GUNZIP if (windowBits < 48) windowBits &= 15; @@ -211,7 +226,9 @@ int stream_size; if (state == Z_NULL) return Z_MEM_ERROR; Tracev((stderr, "inflate: allocated\n")); strm->state = (struct internal_state FAR *)state; + state->strm = strm; state->window = Z_NULL; + state->mode = HEAD; /* to pass state test in inflateReset2() */ ret = inflateReset2(strm, windowBits); if (ret != Z_OK) { ZFREE(strm, state); @@ -235,17 +252,17 @@ int value; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (bits < 0) { state->hold = 0; state->bits = 0; return Z_OK; } - if (bits > 16 || state->bits + bits > 32) return Z_STREAM_ERROR; + if (bits > 16 || state->bits + (uInt)bits > 32) return Z_STREAM_ERROR; value &= (1L << bits) - 1; - state->hold += value << state->bits; - state->bits += bits; + state->hold += (unsigned)value << state->bits; + state->bits += (uInt)bits; return Z_OK; } @@ -626,7 +643,7 @@ int flush; static const unsigned short order[19] = /* permutation of code lengths */ {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; - if (strm == Z_NULL || strm->state == Z_NULL || strm->next_out == Z_NULL || + if (inflateStateCheck(strm) || strm->next_out == Z_NULL || (strm->next_in == Z_NULL && strm->avail_in != 0)) return Z_STREAM_ERROR; @@ -646,6 +663,8 @@ int flush; NEEDBITS(16); #ifdef GUNZIP if ((state->wrap & 2) && hold == 0x8b1f) { /* gzip header */ + if (state->wbits == 0) + state->wbits = 15; state->check = crc32(0L, Z_NULL, 0); CRC2(state->check, hold); INITBITS(); @@ -673,7 +692,7 @@ int flush; len = BITS(4) + 8; if (state->wbits == 0) state->wbits = len; - else if (len > state->wbits) { + if (len > 15 || len > state->wbits) { strm->msg = "invalid window size"; state->mode = BAD; break; @@ -700,14 +719,16 @@ int flush; } if (state->head != Z_NULL) state->head->text = (int)((hold >> 8) & 1); - if (state->flags & 0x0200) CRC2(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); INITBITS(); state->mode = TIME; case TIME: NEEDBITS(32); if (state->head != Z_NULL) state->head->time = hold; - if (state->flags & 0x0200) CRC4(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC4(state->check, hold); INITBITS(); state->mode = OS; case OS: @@ -716,7 +737,8 @@ int flush; state->head->xflags = (int)(hold & 0xff); state->head->os = (int)(hold >> 8); } - if (state->flags & 0x0200) CRC2(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); INITBITS(); state->mode = EXLEN; case EXLEN: @@ -725,7 +747,8 @@ int flush; state->length = (unsigned)(hold); if (state->head != Z_NULL) state->head->extra_len = (unsigned)hold; - if (state->flags & 0x0200) CRC2(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); INITBITS(); } else if (state->head != Z_NULL) @@ -743,7 +766,7 @@ int flush; len + copy > state->head->extra_max ? state->head->extra_max - len : copy); } - if (state->flags & 0x0200) + if ((state->flags & 0x0200) && (state->wrap & 4)) state->check = crc32(state->check, next, copy); have -= copy; next += copy; @@ -762,9 +785,9 @@ int flush; if (state->head != Z_NULL && state->head->name != Z_NULL && state->length < state->head->name_max) - state->head->name[state->length++] = len; + state->head->name[state->length++] = (Bytef)len; } while (len && copy < have); - if (state->flags & 0x0200) + if ((state->flags & 0x0200) && (state->wrap & 4)) state->check = crc32(state->check, next, copy); have -= copy; next += copy; @@ -783,9 +806,9 @@ int flush; if (state->head != Z_NULL && state->head->comment != Z_NULL && state->length < state->head->comm_max) - state->head->comment[state->length++] = len; + state->head->comment[state->length++] = (Bytef)len; } while (len && copy < have); - if (state->flags & 0x0200) + if ((state->flags & 0x0200) && (state->wrap & 4)) state->check = crc32(state->check, next, copy); have -= copy; next += copy; @@ -797,7 +820,7 @@ int flush; case HCRC: if (state->flags & 0x0200) { NEEDBITS(16); - if (hold != (state->check & 0xffff)) { + if ((state->wrap & 4) && hold != (state->check & 0xffff)) { strm->msg = "header crc mismatch"; state->mode = BAD; break; @@ -837,7 +860,7 @@ int flush; } NEEDBITS(3); state->last = BITS(1); - if (state->last) + if (state->last) strm->last_bit = 8 * (strm->total_in + in - have) - bits; DROPBITS(1); switch (BITS(2)) { @@ -1183,11 +1206,11 @@ int flush; out -= left; strm->total_out += out; state->total += out; - if (out) + if ((state->wrap & 4) && out) strm->adler = state->check = UPDATE(state->check, put - out, out); out = left; - if (( + if ((state->wrap & 4) && ( #ifdef GUNZIP state->flags ? hold : #endif @@ -1246,10 +1269,10 @@ int flush; strm->total_in += in; strm->total_out += out; state->total += out; - if (state->wrap && out) + if ((state->wrap & 4) && out) strm->adler = state->check = UPDATE(state->check, strm->next_out - out, out); - strm->data_type = state->bits + (state->last ? 64 : 0) + + strm->data_type = (int)state->bits + (state->last ? 64 : 0) + (state->mode == TYPE ? 128 : 0) + (state->mode == LEN_ || state->mode == COPY_ ? 256 : 0); if (((in == 0 && out == 0) || flush == Z_FINISH) && ret == Z_OK) @@ -1261,7 +1284,7 @@ int ZEXPORT inflateEnd(strm) z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL || strm->zfree == (free_func)0) + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (state->window != Z_NULL) ZFREE(strm, state->window); @@ -1279,7 +1302,7 @@ uInt *dictLength; struct inflate_state FAR *state; /* check state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; /* copy dictionary */ @@ -1304,7 +1327,7 @@ uInt dictLength; int ret; /* check state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (state->wrap != 0 && state->mode != DICT) return Z_STREAM_ERROR; @@ -1336,7 +1359,7 @@ gz_headerp head; struct inflate_state FAR *state; /* check state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if ((state->wrap & 2) == 0) return Z_STREAM_ERROR; @@ -1389,7 +1412,7 @@ z_streamp strm; struct inflate_state FAR *state; /* check parameters */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (strm->avail_in == 0 && state->bits < 8) return Z_BUF_ERROR; @@ -1436,7 +1459,7 @@ z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; return state->mode == STORED && state->bits == 0; } @@ -1451,8 +1474,7 @@ z_streamp source; unsigned wsize; /* check input */ - if (dest == Z_NULL || source == Z_NULL || source->state == Z_NULL || - source->zalloc == (alloc_func)0 || source->zfree == (free_func)0) + if (inflateStateCheck(source) || dest == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)source->state; @@ -1473,6 +1495,7 @@ z_streamp source; /* copy state */ zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); zmemcpy((voidpf)copy, (voidpf)state, sizeof(struct inflate_state)); + copy->strm = dest; if (state->lencode >= state->codes && state->lencode <= state->codes + ENOUGH - 1) { copy->lencode = copy->codes + (state->lencode - state->codes); @@ -1494,25 +1517,51 @@ int subvert; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; - state->sane = !subvert; #ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + state->sane = !subvert; return Z_OK; #else + (void)subvert; state->sane = 1; return Z_DATA_ERROR; #endif } +int ZEXPORT inflateValidate(strm, check) +z_streamp strm; +int check; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (check) + state->wrap |= 4; + else + state->wrap &= ~4; + return Z_OK; +} + long ZEXPORT inflateMark(strm) z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return -(1L << 16); + if (inflateStateCheck(strm)) + return -(1L << 16); state = (struct inflate_state FAR *)strm->state; - return ((long)(state->back) << 16) + + return (long)(((unsigned long)((long)state->back)) << 16) + (state->mode == COPY ? state->length : (state->mode == MATCH ? state->was - state->length : 0)); } + +unsigned long ZEXPORT inflateCodesUsed(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) return (unsigned long)-1; + state = (struct inflate_state FAR *)strm->state; + return (unsigned long)(state->next - state->codes); +} diff --git a/lib/libvgz/inflate.h b/lib/libvgz/inflate.h index 95f4986..a46cce6 100644 --- a/lib/libvgz/inflate.h +++ b/lib/libvgz/inflate.h @@ -1,5 +1,5 @@ /* inflate.h -- internal inflate state definition - * Copyright (C) 1995-2009 Mark Adler + * Copyright (C) 1995-2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -18,7 +18,7 @@ /* Possible inflate modes between inflate() calls */ typedef enum { - HEAD, /* i: waiting for magic header */ + HEAD = 16180, /* i: waiting for magic header */ FLAGS, /* i: waiting for method and flags (gzip) */ TIME, /* i: waiting for modification time (gzip) */ OS, /* i: waiting for extra flags and operating system (gzip) */ @@ -77,11 +77,14 @@ typedef enum { CHECK -> LENGTH -> DONE */ -/* state maintained between inflate() calls. Approximately 10K bytes. */ +/* State maintained between inflate() calls -- approximately 7K bytes, not + including the allocated sliding window, which is up to 32K bytes. */ struct inflate_state { + z_streamp strm; /* pointer back to this zlib stream */ inflate_mode mode; /* current inflate mode */ int last; /* true if processing last block */ - int wrap; /* bit 0 true for zlib, bit 1 true for gzip */ + int wrap; /* bit 0 true for zlib, bit 1 true for gzip, + bit 2 true to validate check value */ int havedict; /* true if dictionary provided */ int flags; /* gzip header method and flags (0 if zlib) */ unsigned dmax; /* zlib header max distance (INFLATE_STRICT) */ diff --git a/lib/libvgz/inftrees.c b/lib/libvgz/inftrees.c index 9d75fd2..d89a533 100644 --- a/lib/libvgz/inftrees.c +++ b/lib/libvgz/inftrees.c @@ -1,5 +1,5 @@ /* inftrees.c -- generate Huffman trees for efficient decoding - * Copyright (C) 1995-2013 Mark Adler + * Copyright (C) 1995-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -10,7 +10,7 @@ extern const char inflate_copyright[]; const char inflate_copyright[] = - " inflate 1.2.8 Copyright 1995-2013 Mark Adler "; + " inflate 1.2.11 Copyright 1995-2017 Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -55,7 +55,7 @@ unsigned short FAR *work; code FAR *next; /* next available space in table */ const unsigned short FAR *base; /* base value table to use */ const unsigned short FAR *extra; /* extra bits table to use */ - int end; /* use base and extra for symbol > end */ + unsigned match; /* use base and extra for symbol >= match */ unsigned short count[MAXBITS+1]; /* number of codes of each length */ unsigned short offs[MAXBITS+1]; /* offsets in table for each length */ static const unsigned short lbase[31] = { /* Length codes 257..285 base */ @@ -63,7 +63,7 @@ unsigned short FAR *work; 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0}; static const unsigned short lext[31] = { /* Length codes 257..285 extra */ 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, - 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 72, 78}; + 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 77, 202}; static const unsigned short dbase[32] = { /* Distance codes 0..29 base */ 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, @@ -182,19 +182,17 @@ unsigned short FAR *work; switch (type) { case CODES: base = extra = work; /* dummy value--not used */ - end = 19; + match = 20; break; case LENS: base = lbase; - base -= 257; extra = lext; - extra -= 257; - end = 256; + match = 257; break; - default: /* DISTS */ + default: /* DISTS */ base = dbase; extra = dext; - end = -1; + match = 0; } /* initialize state for loop */ @@ -217,13 +215,13 @@ unsigned short FAR *work; for (;;) { /* create table entry */ here.bits = (unsigned char)(len - drop); - if ((int)(work[sym]) < end) { + if (work[sym] + 1U < match) { here.op = (unsigned char)0; here.val = work[sym]; } - else if ((int)(work[sym]) > end) { - here.op = (unsigned char)(extra[work[sym]]); - here.val = base[work[sym]]; + else if (work[sym] >= match) { + here.op = (unsigned char)(extra[work[sym] - match]); + here.val = base[work[sym] - match]; } else { here.op = (unsigned char)(32 + 64); /* end of block */ diff --git a/lib/libvgz/trees.c b/lib/libvgz/trees.c index 708649c..10464f6 100644 --- a/lib/libvgz/trees.c +++ b/lib/libvgz/trees.c @@ -1,5 +1,5 @@ /* trees.c -- output deflated data using Huffman coding - * Copyright (C) 1995-2012 Jean-loup Gailly + * Copyright (C) 1995-2017 Jean-loup Gailly * detect_data_type() function provided freely by Cosmin Truta, 2006 * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -36,7 +36,7 @@ #include "deflate.h" -#ifdef DEBUG +#ifdef ZLIB_DEBUG # include #endif @@ -152,18 +152,16 @@ local int detect_data_type OF((deflate_state *s)); local unsigned bi_reverse OF((unsigned value, int length)); local void bi_windup OF((deflate_state *s)); local void bi_flush OF((deflate_state *s)); -local void copy_block OF((deflate_state *s, charf *buf, unsigned len, - int header)); #ifdef GEN_TREES_H local void gen_trees_header OF((void)); #endif -#ifndef DEBUG +#ifndef ZLIB_DEBUG # define send_code(s, c, tree) send_bits(s, tree[c].Code, tree[c].Len) /* Send a code of the given tree. c and tree must not have side effects */ -#else /* DEBUG */ +#else /* !ZLIB_DEBUG */ # define send_code(s, c, tree) \ { if (z_verbose>2) fprintf(stderr,"\ncd %3d ",(c)); \ send_bits(s, tree[c].Code, tree[c].Len); } @@ -182,7 +180,7 @@ local void gen_trees_header OF((void)); * Send a value on a given number of bits. * IN assertion: length <= 16 and value fits in length bits. */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG local void send_bits OF((deflate_state *s, int value, int length)); local void send_bits(s, value, length) @@ -208,12 +206,12 @@ local void send_bits(s, value, length) s->bi_valid += length; } } -#else /* !DEBUG */ +#else /* !ZLIB_DEBUG */ #define send_bits(s, value, length) \ { int len = length;\ if (s->bi_valid > (int)Buf_size - len) {\ - int val = value;\ + int val = (int)value;\ s->bi_buf |= (ush)val << s->bi_valid;\ put_short(s, s->bi_buf);\ s->bi_buf = (ush)val >> (Buf_size - s->bi_valid);\ @@ -223,7 +221,7 @@ local void send_bits(s, value, length) s->bi_valid += len;\ }\ } -#endif /* DEBUG */ +#endif /* ZLIB_DEBUG */ /* the arguments must not have side effects */ @@ -317,7 +315,7 @@ local void tr_static_init() * Genererate the file trees.h describing the static trees. */ #ifdef GEN_TREES_H -# ifndef DEBUG +# ifndef ZLIB_DEBUG # include # endif @@ -394,7 +392,7 @@ void ZLIB_INTERNAL _tr_init(s) s->bi_buf = 0; s->bi_valid = 0; -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len = 0L; s->bits_sent = 0L; #endif @@ -522,12 +520,12 @@ local void gen_bitlen(s, desc) xbits = 0; if (n >= base) xbits = extra[n-base]; f = tree[n].Freq; - s->opt_len += (ulg)f * (bits + xbits); - if (stree) s->static_len += (ulg)f * (stree[n].Len + xbits); + s->opt_len += (ulg)f * (unsigned)(bits + xbits); + if (stree) s->static_len += (ulg)f * (unsigned)(stree[n].Len + xbits); } if (overflow == 0) return; - Trace((stderr,"\nbit length overflow\n")); + Tracev((stderr,"\nbit length overflow\n")); /* This happens for example on obj2 and pic of the Calgary corpus */ /* Find the first bit length which could increase: */ @@ -554,9 +552,8 @@ local void gen_bitlen(s, desc) m = s->heap[--h]; if (m > max_code) continue; if ((unsigned) tree[m].Len != (unsigned) bits) { - Trace((stderr,"code %d bits %d->%d\n", m, tree[m].Len, bits)); - s->opt_len += ((long)bits - (long)tree[m].Len) - *(long)tree[m].Freq; + Tracev((stderr,"code %d bits %d->%d\n", m, tree[m].Len, bits)); + s->opt_len += ((ulg)bits - tree[m].Len) * tree[m].Freq; tree[m].Len = (ush)bits; } n--; @@ -578,7 +575,7 @@ local void gen_codes (tree, max_code, bl_count) ushf *bl_count; /* number of codes at each bit length */ { ush next_code[MAX_BITS+1]; /* next code value for each bit length */ - ush code = 0; /* running code value */ + unsigned code = 0; /* running code value */ int bits; /* bit index */ int n; /* code index */ @@ -586,7 +583,8 @@ local void gen_codes (tree, max_code, bl_count) * without bit reversal. */ for (bits = 1; bits <= MAX_BITS; bits++) { - next_code[bits] = code = (code + bl_count[bits-1]) << 1; + code = (code + bl_count[bits-1]) << 1; + next_code[bits] = (ush)code; } /* Check that the bit counts in bl_count are consistent. The last code * must be all ones. @@ -599,7 +597,7 @@ local void gen_codes (tree, max_code, bl_count) int len = tree[n].Len; if (len == 0) continue; /* Now reverse the bits */ - tree[n].Code = bi_reverse(next_code[len]++, len); + tree[n].Code = (ush)bi_reverse(next_code[len]++, len); Tracecv(tree != static_ltree, (stderr,"\nn %3d %c l %2d c %4x (%x) ", n, (isgraph(n) ? n : ' '), len, tree[n].Code, next_code[len]-1)); @@ -821,7 +819,7 @@ local int build_bl_tree(s) if (s->bl_tree[bl_order[max_blindex]].Len != 0) break; } /* Update opt_len to include the bit length tree and counts */ - s->opt_len += 3*(max_blindex+1) + 5+5+4; + s->opt_len += 3*((ulg)max_blindex+1) + 5+5+4; Tracev((stderr, "\ndyn trees: dyn %ld, stat %ld", s->opt_len, s->static_len)); @@ -868,12 +866,25 @@ void ZLIB_INTERNAL _tr_stored_block(s, buf, stored_len, last) ulg stored_len; /* length of input block */ int last; /* one if this is the last block for a file */ { + if (last) + s->strm->last_bit = + (s->strm->total_out + s->pending) * 8 + s->bi_valid; + send_bits(s, (STORED_BLOCK<<1)+last, 3); /* send block type */ -#ifdef DEBUG + bi_windup(s); /* align on byte boundary */ + put_short(s, (ush)stored_len); + put_short(s, (ush)~stored_len); + zmemcpy(s->pending_buf + s->pending, (Bytef *)buf, stored_len); + s->pending += stored_len; +#ifdef ZLIB_DEBUG s->compressed_len = (s->compressed_len + 3 + 7) & (ulg)~7L; s->compressed_len += (stored_len + 4) << 3; + s->bits_sent += 2*16; + s->bits_sent += stored_len<<3; #endif - copy_block(s, buf, (unsigned)stored_len, 1); /* with header */ + if (last) + s->strm->stop_bit = + (s->strm->total_out + s->pending) * 8 + s->bi_valid; } /* =========================================================================== @@ -894,7 +905,7 @@ void ZLIB_INTERNAL _tr_align(s) { send_bits(s, STATIC_TREES<<1, 3); send_code(s, END_BLOCK, static_ltree); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 10L; /* 3 for block type, 7 for EOB */ #endif bi_flush(s); @@ -902,7 +913,7 @@ void ZLIB_INTERNAL _tr_align(s) /* =========================================================================== * Determine the best encoding for the current block: dynamic trees, static - * trees or store, and output the encoded block to the zip file. + * trees or store, and write out the encoded block. */ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) deflate_state *s; @@ -978,7 +989,7 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) send_bits(s, (STATIC_TREES<<1)+last, 3); compress_block(s, (const ct_data *)static_ltree, (const ct_data *)static_dtree); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 3 + s->static_len; #endif } else { @@ -987,7 +998,7 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) max_blindex+1); compress_block(s, (const ct_data *)s->dyn_ltree, (const ct_data *)s->dyn_dtree); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 3 + s->opt_len; #endif } @@ -1001,7 +1012,7 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) s->strm->stop_bit = (s->strm->total_out + s->pending) * 8 + s->bi_valid; bi_windup(s); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 7; /* align on byte boundary */ #endif } @@ -1096,7 +1107,7 @@ local void compress_block(s, ltree, dtree) send_code(s, code, dtree); /* send the distance code */ extra = extra_dbits[code]; if (extra != 0) { - dist -= base_dist[code]; + dist -= (unsigned)base_dist[code]; send_bits(s, dist, extra); /* send the extra distance bits */ } } /* literal or match pair ? */ @@ -1199,34 +1210,7 @@ local void bi_windup(s) } s->bi_buf = 0; s->bi_valid = 0; -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->bits_sent = (s->bits_sent+7) & ~7; #endif } - -/* =========================================================================== - * Copy a stored block, storing first the length and its - * one's complement if requested. - */ -local void copy_block(s, buf, len, header) - deflate_state *s; - charf *buf; /* the input data */ - unsigned len; /* its length */ - int header; /* true if block header must be written */ -{ - bi_windup(s); /* align on byte boundary */ - - if (header) { - put_short(s, (ush)len); - put_short(s, (ush)~len); -#ifdef DEBUG - s->bits_sent += 2*16; -#endif - } -#ifdef DEBUG - s->bits_sent += (ulg)len<<3; -#endif - while (len--) { - put_byte(s, *buf++); - } -} diff --git a/lib/libvgz/uncompr.c b/lib/libvgz/uncompr.c index 0f6a951..a1212de 100644 --- a/lib/libvgz/uncompr.c +++ b/lib/libvgz/uncompr.c @@ -1,5 +1,5 @@ /* uncompr.c -- decompress a memory buffer - * Copyright (C) 1995-2003, 2010 Jean-loup Gailly. + * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -9,51 +9,85 @@ #include "vgz.h" /* =========================================================================== - Decompresses the source buffer into the destination buffer. sourceLen is - the byte length of the source buffer. Upon entry, destLen is the total - size of the destination buffer, which must be large enough to hold the - entire uncompressed data. (The size of the uncompressed data must have - been saved previously by the compressor and transmitted to the decompressor - by some mechanism outside the scope of this compression library.) - Upon exit, destLen is the actual size of the compressed buffer. - - uncompress returns Z_OK if success, Z_MEM_ERROR if there was not - enough memory, Z_BUF_ERROR if there was not enough room in the output - buffer, or Z_DATA_ERROR if the input data was corrupted. + Decompresses the source buffer into the destination buffer. *sourceLen is + the byte length of the source buffer. Upon entry, *destLen is the total size + of the destination buffer, which must be large enough to hold the entire + uncompressed data. (The size of the uncompressed data must have been saved + previously by the compressor and transmitted to the decompressor by some + mechanism outside the scope of this compression library.) Upon exit, + *destLen is the size of the decompressed data and *sourceLen is the number + of source bytes consumed. Upon return, source + *sourceLen points to the + first unused input byte. + + uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_BUF_ERROR if there was not enough room in the output buffer, or + Z_DATA_ERROR if the input data was corrupted, including if the input data is + an incomplete zlib stream. */ -int ZEXPORT uncompress (dest, destLen, source, sourceLen) +int ZEXPORT uncompress2 (dest, destLen, source, sourceLen) Bytef *dest; uLongf *destLen; const Bytef *source; - uLong sourceLen; + uLong *sourceLen; { z_stream stream; int err; + const uInt max = (uInt)-1; + uLong len, left; + Byte buf[1]; /* for detection of incomplete stream when *destLen == 0 */ - stream.next_in = (z_const Bytef *)source; - stream.avail_in = (uInt)sourceLen; - /* Check for source > 64K on 16-bit machine: */ - if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR; - - stream.next_out = dest; - stream.avail_out = (uInt)*destLen; - if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR; + len = *sourceLen; + if (*destLen) { + left = *destLen; + *destLen = 0; + } + else { + left = 1; + dest = buf; + } + stream.next_in = (z_const Bytef *)source; + stream.avail_in = 0; stream.zalloc = (alloc_func)0; stream.zfree = (free_func)0; + stream.opaque = (voidpf)0; err = inflateInit(&stream); if (err != Z_OK) return err; - err = inflate(&stream, Z_FINISH); - if (err != Z_STREAM_END) { - inflateEnd(&stream); - if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0)) - return Z_DATA_ERROR; - return err; - } - *destLen = stream.total_out; + stream.next_out = dest; + stream.avail_out = 0; - err = inflateEnd(&stream); - return err; + do { + if (stream.avail_out == 0) { + stream.avail_out = left > (uLong)max ? max : (uInt)left; + left -= stream.avail_out; + } + if (stream.avail_in == 0) { + stream.avail_in = len > (uLong)max ? max : (uInt)len; + len -= stream.avail_in; + } + err = inflate(&stream, Z_NO_FLUSH); + } while (err == Z_OK); + + *sourceLen -= len + stream.avail_in; + if (dest != buf) + *destLen = stream.total_out; + else if (stream.total_out && err == Z_BUF_ERROR) + left = 1; + + inflateEnd(&stream); + return err == Z_STREAM_END ? Z_OK : + err == Z_NEED_DICT ? Z_DATA_ERROR : + err == Z_BUF_ERROR && left + stream.avail_out ? Z_DATA_ERROR : + err; +} + +int ZEXPORT uncompress (dest, destLen, source, sourceLen) + Bytef *dest; + uLongf *destLen; + const Bytef *source; + uLong sourceLen; +{ + return uncompress2(dest, destLen, source, &sourceLen); } diff --git a/lib/libvgz/vgz.h b/lib/libvgz/vgz.h index 6846fa0..dc3e126 100644 --- a/lib/libvgz/vgz.h +++ b/lib/libvgz/vgz.h @@ -1,7 +1,7 @@ /* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.8, April 28th, 2013 + version 1.2.11, January 15th, 2017 - Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages @@ -37,11 +37,11 @@ extern "C" { #endif -#define ZLIB_VERSION "1.2.8" -#define ZLIB_VERNUM 0x1280 +#define ZLIB_VERSION "1.2.11" +#define ZLIB_VERNUM 0x12b0 #define ZLIB_VER_MAJOR 1 #define ZLIB_VER_MINOR 2 -#define ZLIB_VER_REVISION 8 +#define ZLIB_VER_REVISION 11 #define ZLIB_VER_SUBREVISION 0 /* @@ -65,7 +65,8 @@ extern "C" { with "gz". The gzip format is different from the zlib format. gzip is a gzip wrapper, documented in RFC 1952, wrapped around a deflate stream. - This library can optionally read and write gzip streams in memory as well. + This library can optionally read and write gzip and raw deflate streams in + memory as well. The zlib format was designed to be compact and fast for use in memory and on communications channels. The gzip format was designed for single- @@ -74,7 +75,7 @@ extern "C" { The library does not install any signal handler. The decoder checks the consistency of the compressed data, so the library should never crash - even in case of corrupted input. + even in the case of corrupted input. */ typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size)); @@ -87,7 +88,7 @@ typedef struct z_stream_s { uInt avail_in; /* number of bytes available at next_in */ uLong total_in; /* total number of input bytes read so far */ - Bytef *next_out; /* next output byte should be put there */ + Bytef *next_out; /* next output byte will go here */ uInt avail_out; /* remaining free space at next_out */ uLong total_out; /* total number of bytes output so far */ @@ -98,8 +99,9 @@ typedef struct z_stream_s { free_func zfree; /* used to free the internal state */ voidpf opaque; /* private data object passed to zalloc and zfree */ - int data_type; /* best guess about the data type: binary or text */ - uLong adler; /* adler32 value of the uncompressed data */ + int data_type; /* best guess about the data type: binary or text + for deflate, or the decoding state for inflate */ + uLong adler; /* Adler-32 or CRC-32 value of the uncompressed data */ uLong reserved; /* reserved for future use */ uLong start_bit; /* Bit pos of first deflate block */ @@ -146,7 +148,9 @@ typedef gz_header FAR *gz_headerp; zalloc must return Z_NULL if there is not enough memory for the object. If zlib is used in a multi-threaded application, zalloc and zfree must be - thread safe. + thread safe. In that case, zlib is thread-safe. When zalloc and zfree are + Z_NULL on entry to the initialization function, they are set to internal + routines that use the standard library functions malloc() and free(). On 16-bit systems, the functions zalloc and zfree must be able to allocate exactly 65536 bytes, but will not be required to allocate more than this if @@ -159,7 +163,7 @@ typedef gz_header FAR *gz_headerp; The fields total_in and total_out can be used for statistics or progress reports. After compression, total_in holds the total size of the - uncompressed data and may be saved for use in the decompressor (particularly + uncompressed data and may be saved for use by the decompressor (particularly if the decompressor wants to decompress everything in a single step). */ @@ -204,7 +208,7 @@ typedef gz_header FAR *gz_headerp; #define Z_TEXT 1 #define Z_ASCII Z_TEXT /* for compatibility with 1.2.2 and earlier */ #define Z_UNKNOWN 2 -/* Possible values of the data_type field (though see inflate()) */ +/* Possible values of the data_type field for deflate() */ #define Z_DEFLATED 8 /* The deflate compression method (the only one supported in this version) */ @@ -262,11 +266,11 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); enough room in the output buffer), next_in and avail_in are updated and processing will resume at this point for the next call of deflate(). - - Provide more output starting at next_out and update next_out and avail_out + - Generate more output starting at next_out and update next_out and avail_out accordingly. This action is forced if the parameter flush is non zero. Forcing flush frequently degrades the compression ratio, so this parameter - should be set only when necessary (in interactive applications). Some - output may be provided even if flush is not set. + should be set only when necessary. Some output may be provided even if + flush is zero. Before the call of deflate(), the application should ensure that at least one of the actions is possible, by providing more input and/or consuming more @@ -275,7 +279,9 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); output when it wants, for example when the output buffer is full (avail_out == 0), or after each call of deflate(). If deflate returns Z_OK and with zero avail_out, it must be called again after making room in the output - buffer because there might be more output pending. + buffer because there might be more output pending. See deflatePending(), + which can be used if desired to determine whether or not there is more ouput + in that case. Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to decide how much data to accumulate before producing output, in order to @@ -296,8 +302,8 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); input data so far will be available to the decompressor, as for Z_SYNC_FLUSH. This completes the current deflate block and follows it with an empty fixed codes block that is 10 bits long. This assures that enough bytes are output - in order for the decompressor to finish the block before the empty fixed code - block. + in order for the decompressor to finish the block before the empty fixed + codes block. If flush is set to Z_BLOCK, a deflate block is completed and emitted, as for Z_SYNC_FLUSH, but the output is not aligned on a byte boundary, and up to @@ -323,34 +329,38 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); If the parameter flush is set to Z_FINISH, pending input is processed, pending output is flushed and deflate returns with Z_STREAM_END if there was - enough output space; if deflate returns with Z_OK, this function must be - called again with Z_FINISH and more output space (updated avail_out) but no - more input data, until it returns with Z_STREAM_END or an error. After - deflate has returned Z_STREAM_END, the only possible operations on the stream - are deflateReset or deflateEnd. - - Z_FINISH can be used immediately after deflateInit if all the compression - is to be done in a single step. In this case, avail_out must be at least the - value returned by deflateBound (see below). Then deflate is guaranteed to - return Z_STREAM_END. If not enough output space is provided, deflate will - not return Z_STREAM_END, and it must be called again as described above. - - deflate() sets strm->adler to the adler32 checksum of all input read - so far (that is, total_in bytes). + enough output space. If deflate returns with Z_OK or Z_BUF_ERROR, this + function must be called again with Z_FINISH and more output space (updated + avail_out) but no more input data, until it returns with Z_STREAM_END or an + error. After deflate has returned Z_STREAM_END, the only possible operations + on the stream are deflateReset or deflateEnd. + + Z_FINISH can be used in the first deflate call after deflateInit if all the + compression is to be done in a single step. In order to complete in one + call, avail_out must be at least the value returned by deflateBound (see + below). Then deflate is guaranteed to return Z_STREAM_END. If not enough + output space is provided, deflate will not return Z_STREAM_END, and it must + be called again as described above. + + deflate() sets strm->adler to the Adler-32 checksum of all input read + so far (that is, total_in bytes). If a gzip stream is being generated, then + strm->adler will be the CRC-32 checksum of the input read so far. (See + deflateInit2 below.) deflate() may update strm->data_type if it can make a good guess about - the input data type (Z_BINARY or Z_TEXT). In doubt, the data is considered - binary. This field is only for information purposes and does not affect the - compression algorithm in any manner. + the input data type (Z_BINARY or Z_TEXT). If in doubt, the data is + considered binary. This field is only for information purposes and does not + affect the compression algorithm in any manner. deflate() returns Z_OK if some progress has been made (more input processed or more output produced), Z_STREAM_END if all input has been consumed and all output has been produced (only when flush is set to Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example - if next_in or next_out was Z_NULL), Z_BUF_ERROR if no progress is possible - (for example avail_in or avail_out was zero). Note that Z_BUF_ERROR is not - fatal, and deflate() can be called again with more input and more output - space to continue compressing. + if next_in or next_out was Z_NULL or the state was inadvertently written over + by the application), or Z_BUF_ERROR if no progress is possible (for example + avail_in or avail_out was zero). Note that Z_BUF_ERROR is not fatal, and + deflate() can be called again with more input and more output space to + continue compressing. */ @@ -373,23 +383,21 @@ ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm)); Initializes the internal stream state for decompression. The fields next_in, avail_in, zalloc, zfree and opaque must be initialized before by - the caller. If next_in is not Z_NULL and avail_in is large enough (the - exact value depends on the compression method), inflateInit determines the - compression method from the zlib header and allocates all data structures - accordingly; otherwise the allocation will be deferred to the first call of - inflate. If zalloc and zfree are set to Z_NULL, inflateInit updates them to - use default allocation functions. + the caller. In the current version of inflate, the provided input is not + read or consumed. The allocation of a sliding window will be deferred to + the first call of inflate (if the decompression does not complete on the + first call). If zalloc and zfree are set to Z_NULL, inflateInit updates + them to use default allocation functions. inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_VERSION_ERROR if the zlib library version is incompatible with the version assumed by the caller, or Z_STREAM_ERROR if the parameters are invalid, such as a null pointer to the structure. msg is set to null if - there is no error message. inflateInit does not perform any decompression - apart from possibly reading the zlib header if present: actual decompression - will be done by inflate(). (So next_in and avail_in may be modified, but - next_out and avail_out are unused and unchanged.) The current implementation - of inflateInit() does not process any header information -- that is deferred - until inflate() is called. + there is no error message. inflateInit does not perform any decompression. + Actual decompression will be done by inflate(). So next_in, and avail_in, + next_out, and avail_out are unused and unchanged. The current + implementation of inflateInit() does not process any header information -- + that is deferred until inflate() is called. */ @@ -405,17 +413,20 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); - Decompress more input starting at next_in and update next_in and avail_in accordingly. If not all input can be processed (because there is not - enough room in the output buffer), next_in is updated and processing will - resume at this point for the next call of inflate(). + enough room in the output buffer), then next_in and avail_in are updated + accordingly, and processing will resume at this point for the next call of + inflate(). - - Provide more output starting at next_out and update next_out and avail_out + - Generate more output starting at next_out and update next_out and avail_out accordingly. inflate() provides as much output as possible, until there is no more input data or no more space in the output buffer (see below about the flush parameter). Before the call of inflate(), the application should ensure that at least one of the actions is possible, by providing more input and/or consuming more - output, and updating the next_* and avail_* values accordingly. The + output, and updating the next_* and avail_* values accordingly. If the + caller of inflate() does not provide both available input and available + output space, it is possible that there will be no progress made. The application can consume the uncompressed output when it wants, for example when the output buffer is full (avail_out == 0), or after each call of inflate(). If inflate returns Z_OK and with zero avail_out, it must be @@ -432,7 +443,7 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); gets to the end of that block, or when it runs out of data. The Z_BLOCK option assists in appending to or combining deflate streams. - Also to assist in this, on return inflate() will set strm->data_type to the + To assist in this, on return inflate() always sets strm->data_type to the number of unused bits in the last byte taken from strm->next_in, plus 64 if inflate() is currently decoding the last block in the deflate stream, plus 128 if inflate() returned immediately after decoding an end-of-block code or @@ -458,7 +469,7 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); this case all pending input is processed and all pending output is flushed; avail_out must be large enough to hold all of the uncompressed data for the operation to complete. (The size of the uncompressed data may have been - saved by the compressor for this purpose.) The use of Z_FINISH is not + saved by the compressor for this purpose.) The use of Z_FINISH is not required to perform an inflation in one step. However it may be used to inform inflate that a faster approach can be used for the single inflate() call. Z_FINISH also informs inflate to not maintain a sliding window if the @@ -480,32 +491,33 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); chosen by the compressor and returns Z_NEED_DICT; otherwise it sets strm->adler to the Adler-32 checksum of all output produced so far (that is, total_out bytes) and returns Z_OK, Z_STREAM_END or an error code as described - below. At the end of the stream, inflate() checks that its computed adler32 + below. At the end of the stream, inflate() checks that its computed Adler-32 checksum is equal to that saved by the compressor and returns Z_STREAM_END only if the checksum is correct. inflate() can decompress and check either zlib-wrapped or gzip-wrapped deflate data. The header type is detected automatically, if requested when initializing with inflateInit2(). Any information contained in the gzip - header is not retained, so applications that need that information should - instead use raw inflate, see inflateInit2() below, or inflateBack() and - perform their own processing of the gzip header and trailer. When processing + header is not retained unless inflateGetHeader() is used. When processing gzip-wrapped deflate data, strm->adler32 is set to the CRC-32 of the output - producted so far. The CRC-32 is checked against the gzip trailer. + produced so far. The CRC-32 is checked against the gzip trailer, as is the + uncompressed length, modulo 2^32. inflate() returns Z_OK if some progress has been made (more input processed or more output produced), Z_STREAM_END if the end of the compressed data has been reached and all uncompressed output has been produced, Z_NEED_DICT if a preset dictionary is needed at this point, Z_DATA_ERROR if the input data was corrupted (input stream not conforming to the zlib format or incorrect check - value), Z_STREAM_ERROR if the stream structure was inconsistent (for example - next_in or next_out was Z_NULL), Z_MEM_ERROR if there was not enough memory, - Z_BUF_ERROR if no progress is possible or if there was not enough room in the - output buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and + value, in which case strm->msg points to a string with a more specific + error), Z_STREAM_ERROR if the stream structure was inconsistent (for example + next_in or next_out was Z_NULL, or the state was inadvertently written over + by the application), Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR + if no progress was possible or if there was not enough room in the output + buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and inflate() can be called again with more input and more output space to continue decompressing. If Z_DATA_ERROR is returned, the application may then call inflateSync() to look for a good compression block if a partial - recovery of the data is desired. + recovery of the data is to be attempted. */ @@ -515,9 +527,8 @@ ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm)); This function discards any unprocessed input and does not flush any pending output. - inflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state - was inconsistent. In the error case, msg may be set but then points to a - static string (which must not be deallocated). + inflateEnd returns Z_OK if success, or Z_STREAM_ERROR if the stream state + was inconsistent. */ @@ -548,16 +559,29 @@ ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm, compression at the expense of memory usage. The default value is 15 if deflateInit is used instead. + For the current implementation of deflate(), a windowBits value of 8 (a + window size of 256 bytes) is not supported. As a result, a request for 8 + will result in 9 (a 512-byte window). In that case, providing 8 to + inflateInit2() will result in an error when the zlib header with 9 is + checked against the initialization of inflate(). The remedy is to not use 8 + with deflateInit2() with this initialization, or at least in that case use 9 + with inflateInit2(). + windowBits can also be -8..-15 for raw deflate. In this case, -windowBits determines the window size. deflate() will then generate raw deflate data - with no zlib header or trailer, and will not compute an adler32 check value. + with no zlib header or trailer, and will not compute a check value. windowBits can also be greater than 15 for optional gzip encoding. Add 16 to windowBits to write a simple gzip header and trailer around the compressed data instead of a zlib wrapper. The gzip header will have no file name, no extra data, no comment, no modification time (set to zero), no - header crc, and the operating system will be set to 255 (unknown). If a - gzip stream is being written, strm->adler is a crc32 instead of an adler32. + header crc, and the operating system will be set to the appropriate value, + if the operating system was determined at compile time. If a gzip stream is + being written, strm->adler is a CRC-32 instead of an Adler-32. + + For raw deflate or gzip encoding, a request for a 256-byte window is + rejected as invalid, since only the zlib header provides a means of + transmitting the window size to the decompressor. The memLevel parameter specifies how much memory should be allocated for the internal compression state. memLevel=1 uses minimum memory but is @@ -618,12 +642,12 @@ ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, addition, the current implementation of deflate will use at most the window size minus 262 bytes of the provided dictionary. - Upon return of this function, strm->adler is set to the adler32 value + Upon return of this function, strm->adler is set to the Adler-32 value of the dictionary; the decompressor may later use this value to determine - which dictionary has been used by the compressor. (The adler32 value + which dictionary has been used by the compressor. (The Adler-32 value applies to the whole dictionary even if only a subset of the dictionary is actually used by the compressor.) If a raw deflate was requested, then the - adler32 value is not computed and strm->adler is not set. + Adler-32 value is not computed and strm->adler is not set. deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is @@ -632,6 +656,28 @@ ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, not perform any compression: this will be done by deflate(). */ +ZEXTERN int ZEXPORT deflateGetDictionary OF((z_streamp strm, + Bytef *dictionary, + uInt *dictLength)); +/* + Returns the sliding dictionary being maintained by deflate. dictLength is + set to the number of bytes in the dictionary, and that many bytes are copied + to dictionary. dictionary must have enough space, where 32768 bytes is + always enough. If deflateGetDictionary() is called with dictionary equal to + Z_NULL, then only the dictionary length is returned, and nothing is copied. + Similary, if dictLength is Z_NULL, then it is not set. + + deflateGetDictionary() may return a length less than the window size, even + when more than the window size in input has been provided. It may return up + to 258 bytes less in that case, due to how zlib's implementation of deflate + manages the sliding window and lookahead for matches, where matches can be + up to 258 bytes long. If the application needs the last window-size bytes of + input, then that would need to be saved by the application outside of zlib. + + deflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the + stream state is inconsistent. +*/ + ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, z_streamp source)); /* @@ -652,10 +698,10 @@ ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm)); /* - This function is equivalent to deflateEnd followed by deflateInit, - but does not free and reallocate all the internal compression state. The - stream will keep the same compression level and any other attributes that - may have been set by deflateInit2. + This function is equivalent to deflateEnd followed by deflateInit, but + does not free and reallocate the internal compression state. The stream + will leave the compression level and any other attributes that may have been + set unchanged. deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc or state being Z_NULL). @@ -666,20 +712,36 @@ ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm, int strategy)); /* Dynamically update the compression level and compression strategy. The - interpretation of level and strategy is as in deflateInit2. This can be + interpretation of level and strategy is as in deflateInit2(). This can be used to switch between compression and straight copy of the input data, or to switch to a different kind of input data requiring a different strategy. - If the compression level is changed, the input available so far is - compressed with the old level (and may be flushed); the new level will take - effect only at the next call of deflate(). - - Before the call of deflateParams, the stream state must be set as for - a call of deflate(), since the currently available input may have to be - compressed and flushed. In particular, strm->avail_out must be non-zero. - - deflateParams returns Z_OK if success, Z_STREAM_ERROR if the source - stream state was inconsistent or if a parameter was invalid, Z_BUF_ERROR if - strm->avail_out was zero. + If the compression approach (which is a function of the level) or the + strategy is changed, and if any input has been consumed in a previous + deflate() call, then the input available so far is compressed with the old + level and strategy using deflate(strm, Z_BLOCK). There are three approaches + for the compression levels 0, 1..3, and 4..9 respectively. The new level + and strategy will take effect at the next call of deflate(). + + If a deflate(strm, Z_BLOCK) is performed by deflateParams(), and it does + not have enough output space to complete, then the parameter change will not + take effect. In this case, deflateParams() can be called again with the + same parameters and more output space to try again. + + In order to assure a change in the parameters on the first try, the + deflate stream should be flushed using deflate() with Z_BLOCK or other flush + request until strm.avail_out is not zero, before calling deflateParams(). + Then no more input data should be provided before the deflateParams() call. + If this is done, the old level and strategy will be applied to the data + compressed before deflateParams(), and the new level and strategy will be + applied to the the data compressed after deflateParams(). + + deflateParams returns Z_OK on success, Z_STREAM_ERROR if the source stream + state was inconsistent or if a parameter was invalid, or Z_BUF_ERROR if + there was not enough output space to complete the compression of the + available input data before a change in the strategy or approach. Note that + in the case of a Z_BUF_ERROR, the parameters are not changed. A return + value of Z_BUF_ERROR is not fatal, in which case deflateParams() can be + retried with more output space. */ ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm, @@ -797,7 +859,7 @@ ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, is for use with other formats that use the deflate compressed data format such as zip. Those formats provide their own check values. If a custom format is developed using the raw deflate format for compressed data, it is - recommended that a check value such as an adler32 or a crc32 be applied to + recommended that a check value such as an Adler-32 or a CRC-32 be applied to the uncompressed data as is done in the zlib, gzip, and zip formats. For most applications, the zlib format should be used as is. Note that comments above on the use in deflateInit2() applies to the magnitude of windowBits. @@ -806,7 +868,10 @@ ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, 32 to windowBits to enable zlib and gzip decoding with automatic header detection, or add 16 to decode only the gzip format (the zlib format will return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a - crc32 instead of an adler32. + CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see + below), inflate() will not automatically decode concatenated gzip streams. + inflate() will return Z_STREAM_END at the end of the gzip stream. The state + would need to be reset to continue decoding a subsequent gzip stream. inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_VERSION_ERROR if the zlib library version is incompatible with the @@ -827,7 +892,7 @@ ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, Initializes the decompression dictionary from the given uncompressed byte sequence. This function must be called immediately after a call of inflate, if that call returned Z_NEED_DICT. The dictionary chosen by the compressor - can be determined from the adler32 value returned by that call of inflate. + can be determined from the Adler-32 value returned by that call of inflate. The compressor and decompressor must use exactly the same dictionary (see deflateSetDictionary). For raw inflate, this function can be called at any time to set the dictionary. If the provided dictionary is smaller than the @@ -838,7 +903,7 @@ ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the - expected one (incorrect adler32 value). inflateSetDictionary does not + expected one (incorrect Adler-32 value). inflateSetDictionary does not perform any decompression: this will be done by subsequent calls of inflate(). */ @@ -896,7 +961,7 @@ ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest, ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm)); /* This function is equivalent to inflateEnd followed by inflateInit, - but does not free and reallocate all the internal decompression state. The + but does not free and reallocate the internal decompression state. The stream will keep attributes that may have been set by inflateInit2. inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source @@ -908,7 +973,9 @@ ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm, /* This function is the same as inflateReset, but it also permits changing the wrap and window size requests. The windowBits parameter is interpreted - the same as it is for inflateInit2. + the same as it is for inflateInit2. If the window size is changed, then the + memory allocated for the window is freed, and the window will be reallocated + by inflate() if needed. inflateReset2 returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc or state being Z_NULL), or if @@ -960,7 +1027,7 @@ ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm)); location in the input stream can be determined from avail_in and data_type as noted in the description for the Z_BLOCK flush parameter for inflate. - inflateMark returns the value noted above or -1 << 16 if the provided + inflateMark returns the value noted above, or -65536 if the provided source stream state was inconsistent. */ @@ -1052,9 +1119,9 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, This routine would normally be used in a utility that reads zip or gzip files and writes out uncompressed files. The utility would decode the header and process the trailer on its own, hence this routine expects only - the raw deflate stream to decompress. This is different from the normal - behavior of inflate(), which expects either a zlib or gzip header and - trailer around the deflate stream. + the raw deflate stream to decompress. This is different from the default + behavior of inflate(), which expects a zlib header and trailer around the + deflate stream. inflateBack() uses two subroutines supplied by the caller that are then called by inflateBack() for input and output. inflateBack() calls those @@ -1063,12 +1130,12 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, parameters and return types are defined above in the in_func and out_func typedefs. inflateBack() will call in(in_desc, &buf) which should return the number of bytes of provided input, and a pointer to that input in buf. If - there is no input available, in() must return zero--buf is ignored in that - case--and inflateBack() will return a buffer error. inflateBack() will call - out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. out() - should return zero on success, or non-zero on failure. If out() returns - non-zero, inflateBack() will return with an error. Neither in() nor out() - are permitted to change the contents of the window provided to + there is no input available, in() must return zero -- buf is ignored in that + case -- and inflateBack() will return a buffer error. inflateBack() will + call out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. + out() should return zero on success, or non-zero on failure. If out() + returns non-zero, inflateBack() will return with an error. Neither in() nor + out() are permitted to change the contents of the window provided to inflateBackInit(), which is also the buffer that out() uses to write from. The length written by out() will be at most the window size. Any non-zero amount of input may be provided by in(). @@ -1096,7 +1163,7 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, using strm->next_in which will be Z_NULL only if in() returned an error. If strm->next_in is not Z_NULL, then the Z_BUF_ERROR was due to out() returning non-zero. (in() will always be called before out(), so strm->next_in is - assured to be defined if out() returns non-zero.) Note that inflateBack() + assured to be defined if out() returns non-zero.) Note that inflateBack() cannot return Z_OK. */ @@ -1118,7 +1185,7 @@ ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void)); 7.6: size of z_off_t Compiler, assembler, and debug options: - 8: DEBUG + 8: ZLIB_DEBUG 9: ASMV or ASMINF -- use ASM code 10: ZLIB_WINAPI -- exported functions use the WINAPI calling convention 11: 0 (reserved) @@ -1168,7 +1235,8 @@ ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, the byte length of the source buffer. Upon entry, destLen is the total size of the destination buffer, which must be at least the value returned by compressBound(sourceLen). Upon exit, destLen is the actual size of the - compressed buffer. + compressed data. compress() is equivalent to compress2() with a level + parameter of Z_DEFAULT_COMPRESSION. compress returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output @@ -1184,7 +1252,7 @@ ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen, length of the source buffer. Upon entry, destLen is the total size of the destination buffer, which must be at least the value returned by compressBound(sourceLen). Upon exit, destLen is the actual size of the - compressed buffer. + compressed data. compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output buffer, @@ -1207,7 +1275,7 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, uncompressed data. (The size of the uncompressed data must have been saved previously by the compressor and transmitted to the decompressor by some mechanism outside the scope of this compression library.) Upon exit, destLen - is the actual size of the uncompressed buffer. + is the actual size of the uncompressed data. uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output @@ -1216,6 +1284,14 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, buffer with the uncompressed data up to that point. */ +ZEXTERN int ZEXPORT uncompress2 OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen)); +/* + Same as uncompress, except that sourceLen is a pointer, where the + length of the source is *sourceLen. On return, *sourceLen is the number of + source bytes consumed. +*/ + /* gzip file access functions */ /* @@ -1294,10 +1370,9 @@ ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); default buffer size is 8192 bytes. This function must be called after gzopen() or gzdopen(), and before any other calls that read or write the file. The buffer memory allocation is always deferred to the first read or - write. Two buffers are allocated, either both of the specified size when - writing, or one of the specified size and the other twice that size when - reading. A larger buffer size of, for example, 64K or 128K bytes will - noticeably increase the speed of decompression (reading). + write. Three times that size in buffer space is allocated. A larger buffer + size of, for example, 64K or 128K bytes will noticeably increase the speed + of decompression (reading). The new buffer size also affects the maximum length for gzprintf(). @@ -1308,10 +1383,12 @@ ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy)); /* Dynamically update the compression level or strategy. See the description - of deflateInit2 for the meaning of these parameters. + of deflateInit2 for the meaning of these parameters. Previously provided + data is flushed before the parameter change. - gzsetparams returns Z_OK if success, or Z_STREAM_ERROR if the file was not - opened for writing. + gzsetparams returns Z_OK if success, Z_STREAM_ERROR if the file was not + opened for writing, Z_ERRNO if there is an error writing the flushed data, + or Z_MEM_ERROR if there is a memory allocation error. */ ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); @@ -1339,7 +1416,35 @@ ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); case. gzread returns the number of uncompressed bytes actually read, less than - len for end of file, or -1 for error. + len for end of file, or -1 for error. If len is too large to fit in an int, + then nothing is read, -1 is returned, and the error state is set to + Z_STREAM_ERROR. +*/ + +ZEXTERN z_size_t ZEXPORT gzfread OF((voidp buf, z_size_t size, z_size_t nitems, + gzFile file)); +/* + Read up to nitems items of size size from file to buf, otherwise operating + as gzread() does. This duplicates the interface of stdio's fread(), with + size_t request and return types. If the library defines size_t, then + z_size_t is identical to size_t. If not, then z_size_t is an unsigned + integer type that can contain a pointer. + + gzfread() returns the number of full items read of size size, or zero if + the end of the file was reached and a full item could not be read, or if + there was an error. gzerror() must be consulted if zero is returned in + order to determine if there was an error. If the multiplication of size and + nitems overflows, i.e. the product does not fit in a z_size_t, then nothing + is read, zero is returned, and the error state is set to Z_STREAM_ERROR. + + In the event that the end of file is reached and only a partial item is + available at the end, i.e. the remaining uncompressed data length is not a + multiple of size, then the final partial item is nevetheless read into buf + and the end-of-file flag is set. The length of the partial item read is not + provided, but could be inferred from the result of gztell(). This behavior + is the same as the behavior of fread() implementations in common libraries, + but it prevents the direct use of gzfread() to read a concurrently written + file, reseting and retrying on end-of-file, when size is not 1. */ ZEXTERN int ZEXPORT gzwrite OF((gzFile file, @@ -1350,19 +1455,33 @@ ZEXTERN int ZEXPORT gzwrite OF((gzFile file, error. */ +ZEXTERN z_size_t ZEXPORT gzfwrite OF((voidpc buf, z_size_t size, + z_size_t nitems, gzFile file)); +/* + gzfwrite() writes nitems items of size size from buf to file, duplicating + the interface of stdio's fwrite(), with size_t request and return types. If + the library defines size_t, then z_size_t is identical to size_t. If not, + then z_size_t is an unsigned integer type that can contain a pointer. + + gzfwrite() returns the number of full items written of size size, or zero + if there was an error. If the multiplication of size and nitems overflows, + i.e. the product does not fit in a z_size_t, then nothing is written, zero + is returned, and the error state is set to Z_STREAM_ERROR. +*/ + ZEXTERN int ZEXPORTVA gzprintf Z_ARG((gzFile file, const char *format, ...)); /* Converts, formats, and writes the arguments to the compressed file under control of the format string, as in fprintf. gzprintf returns the number of - uncompressed bytes actually written, or 0 in case of error. The number of - uncompressed bytes written is limited to 8191, or one less than the buffer - size given to gzbuffer(). The caller should assure that this limit is not - exceeded. If it is exceeded, then gzprintf() will return an error (0) with - nothing written. In this case, there may also be a buffer overflow with - unpredictable consequences, which is possible only if zlib was compiled with - the insecure functions sprintf() or vsprintf() because the secure snprintf() - or vsnprintf() functions were not available. This can be determined using - zlibCompileFlags(). + uncompressed bytes actually written, or a negative zlib error code in case + of error. The number of uncompressed bytes written is limited to 8191, or + one less than the buffer size given to gzbuffer(). The caller should assure + that this limit is not exceeded. If it is exceeded, then gzprintf() will + return an error (0) with nothing written. In this case, there may also be a + buffer overflow with unpredictable consequences, which is possible only if + zlib was compiled with the insecure functions sprintf() or vsprintf() + because the secure snprintf() or vsnprintf() functions were not available. + This can be determined using zlibCompileFlags(). */ ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s)); @@ -1422,7 +1541,7 @@ ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush)); If the flush parameter is Z_FINISH, the remaining data is written and the gzip stream is completed in the output. If gzwrite() is called again, a new gzip stream will be started in the output. gzread() is able to read such - concatented gzip streams. + concatenated gzip streams. gzflush should be called only when strictly necessary because it will degrade compression if called too often. @@ -1576,7 +1695,7 @@ ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); return the updated checksum. If buf is Z_NULL, this function returns the required initial value for the checksum. - An Adler-32 checksum is almost as reliable as a CRC32 but can be computed + An Adler-32 checksum is almost as reliable as a CRC-32 but can be computed much faster. Usage example: @@ -1589,6 +1708,12 @@ ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); if (adler != original_adler) error(); */ +ZEXTERN uLong ZEXPORT adler32_z OF((uLong adler, const Bytef *buf, + z_size_t len)); +/* + Same as adler32(), but with a size_t length. +*/ + /* ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2, z_off_t len2)); @@ -1618,6 +1743,12 @@ ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len)); if (crc != original_crc) error(); */ +ZEXTERN uLong ZEXPORT crc32_z OF((uLong adler, const Bytef *buf, + z_size_t len)); +/* + Same as crc32(), but with a size_t length. +*/ + /* ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2)); @@ -1648,19 +1779,35 @@ ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits, unsigned char FAR *window, const char *version, int stream_size)); -#define deflateInit(strm, level) \ - deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) -#define inflateInit(strm) \ - inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) -#define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ - deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ - (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) -#define inflateInit2(strm, windowBits) \ - inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ - (int)sizeof(z_stream)) -#define inflateBackInit(strm, windowBits, window) \ - inflateBackInit_((strm), (windowBits), (window), \ - ZLIB_VERSION, (int)sizeof(z_stream)) +#ifdef Z_PREFIX_SET +# define z_deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define z_inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) +#else +# define deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) +#endif #ifndef Z_SOLO @@ -1680,10 +1827,10 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ #ifdef Z_PREFIX_SET # undef z_gzgetc # define z_gzgetc(g) \ - ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : gzgetc(g)) + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) #else # define gzgetc(g) \ - ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : gzgetc(g)) + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) #endif /* provide 64-bit offset functions if _LARGEFILE64_SOURCE defined, and/or @@ -1741,19 +1888,16 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ #endif /* !Z_SOLO */ -/* hack for buggy compilers */ -#if !defined(ZUTIL_H) && !defined(NO_DUMMY_DECL) - struct internal_state {int dummy;}; -#endif - /* undocumented functions */ ZEXTERN const char * ZEXPORT zError OF((int)); ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp)); ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table OF((void)); ZEXTERN int ZEXPORT inflateUndermine OF((z_streamp, int)); +ZEXTERN int ZEXPORT inflateValidate OF((z_streamp, int)); +ZEXTERN unsigned long ZEXPORT inflateCodesUsed OF ((z_streamp)); ZEXTERN int ZEXPORT inflateResetKeep OF((z_streamp)); ZEXTERN int ZEXPORT deflateResetKeep OF((z_streamp)); -#if defined(_WIN32) && !defined(Z_SOLO) +#if (defined(_WIN32) || defined(__CYGWIN__)) && !defined(Z_SOLO) ZEXTERN gzFile ZEXPORT gzopen_w OF((const wchar_t *path, const char *mode)); #endif diff --git a/lib/libvgz/zconf.h b/lib/libvgz/zconf.h index 63eda6a..1afeed6 100644 --- a/lib/libvgz/zconf.h +++ b/lib/libvgz/zconf.h @@ -1,9 +1,9 @@ /* zconf.h -- configuration of the zlib compression library - * Copyright (C) 1995-2013 Jean-loup Gailly. + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ -/* @(#) $FreeBSD: head/lib/libz/zconf.h 250261 2013-05-05 06:20:49Z delphij $ */ +/* @(#) $FreeBSD: head/contrib/zlib/zconf.h 311285 2017-01-04 16:09:08Z delphij $ */ #ifndef ZCONF_H #define ZCONF_H @@ -17,7 +17,7 @@ #ifdef Z_PREFIX /* may be set to #if 1 by ./configure */ # define Z_PREFIX_SET -/* all linked symbols */ +/* all linked symbols and init macros */ # define _dist_code z__dist_code # define _length_code z__length_code # define _tr_align z__tr_align @@ -29,6 +29,7 @@ # define adler32 z_adler32 # define adler32_combine z_adler32_combine # define adler32_combine64 z_adler32_combine64 +# define adler32_z z_adler32_z # ifndef Z_SOLO # define compress z_compress # define compress2 z_compress2 @@ -37,10 +38,14 @@ # define crc32 z_crc32 # define crc32_combine z_crc32_combine # define crc32_combine64 z_crc32_combine64 +# define crc32_z z_crc32_z # define deflate z_deflate # define deflateBound z_deflateBound # define deflateCopy z_deflateCopy # define deflateEnd z_deflateEnd +# define deflateGetDictionary z_deflateGetDictionary +# define deflateInit z_deflateInit +# define deflateInit2 z_deflateInit2 # define deflateInit2_ z_deflateInit2_ # define deflateInit_ z_deflateInit_ # define deflateParams z_deflateParams @@ -67,6 +72,8 @@ # define gzeof z_gzeof # define gzerror z_gzerror # define gzflush z_gzflush +# define gzfread z_gzfread +# define gzfwrite z_gzfwrite # define gzgetc z_gzgetc # define gzgetc_ z_gzgetc_ # define gzgets z_gzgets @@ -78,7 +85,6 @@ # define gzopen_w z_gzopen_w # endif # define gzprintf z_gzprintf -# define gzvprintf z_gzvprintf # define gzputc z_gzputc # define gzputs z_gzputs # define gzread z_gzread @@ -89,32 +95,39 @@ # define gztell z_gztell # define gztell64 z_gztell64 # define gzungetc z_gzungetc +# define gzvprintf z_gzvprintf # define gzwrite z_gzwrite # endif # define inflate z_inflate # define inflateBack z_inflateBack # define inflateBackEnd z_inflateBackEnd +# define inflateBackInit z_inflateBackInit # define inflateBackInit_ z_inflateBackInit_ +# define inflateCodesUsed z_inflateCodesUsed # define inflateCopy z_inflateCopy # define inflateEnd z_inflateEnd +# define inflateGetDictionary z_inflateGetDictionary # define inflateGetHeader z_inflateGetHeader +# define inflateInit z_inflateInit +# define inflateInit2 z_inflateInit2 # define inflateInit2_ z_inflateInit2_ # define inflateInit_ z_inflateInit_ # define inflateMark z_inflateMark # define inflatePrime z_inflatePrime # define inflateReset z_inflateReset # define inflateReset2 z_inflateReset2 +# define inflateResetKeep z_inflateResetKeep # define inflateSetDictionary z_inflateSetDictionary -# define inflateGetDictionary z_inflateGetDictionary # define inflateSync z_inflateSync # define inflateSyncPoint z_inflateSyncPoint # define inflateUndermine z_inflateUndermine -# define inflateResetKeep z_inflateResetKeep +# define inflateValidate z_inflateValidate # define inflate_copyright z_inflate_copyright # define inflate_fast z_inflate_fast # define inflate_table z_inflate_table # ifndef Z_SOLO # define uncompress z_uncompress +# define uncompress2 z_uncompress2 # endif # define zError z_zError # ifndef Z_SOLO @@ -224,9 +237,19 @@ # define z_const #endif -/* Some Mac compilers merge all .h files incorrectly: */ -#if defined(__MWERKS__)||defined(applec)||defined(THINK_C)||defined(__SC__) -# define NO_DUMMY_DECL +#ifdef Z_SOLO + typedef unsigned long z_size_t; +#else +# define z_longlong long long +# if defined(NO_SIZE_T) + typedef unsigned NO_SIZE_T z_size_t; +# elif defined(STDC) +# include + typedef size_t z_size_t; +# else + typedef unsigned long z_size_t; +# endif +# undef z_longlong #endif /* Maximum value for memLevel in deflateInit2 */ @@ -256,7 +279,7 @@ Of course this will generally degrade compression (there's no free lunch). The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus a few kilobytes + that is, 32K for windowBits=15 (default value) plus about 7 kilobytes for small objects. */ diff --git a/lib/libvgz/zutil.c b/lib/libvgz/zutil.c index 23d2ebe..a76c6b0 100644 --- a/lib/libvgz/zutil.c +++ b/lib/libvgz/zutil.c @@ -1,5 +1,5 @@ /* zutil.c -- target dependent utility functions for the compression library - * Copyright (C) 1995-2005, 2010, 2011, 2012 Jean-loup Gailly. + * Copyright (C) 1995-2017 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -10,21 +10,18 @@ # include "gzguts.h" #endif -#ifndef NO_DUMMY_DECL -struct internal_state {int dummy;}; /* for buggy compilers */ -#endif - z_const char * const z_errmsg[10] = { -"need dictionary", /* Z_NEED_DICT 2 */ -"stream end", /* Z_STREAM_END 1 */ -"", /* Z_OK 0 */ -"file error", /* Z_ERRNO (-1) */ -"stream error", /* Z_STREAM_ERROR (-2) */ -"data error", /* Z_DATA_ERROR (-3) */ -"insufficient memory", /* Z_MEM_ERROR (-4) */ -"buffer error", /* Z_BUF_ERROR (-5) */ -"incompatible version",/* Z_VERSION_ERROR (-6) */ -""}; + (z_const char *)"need dictionary", /* Z_NEED_DICT 2 */ + (z_const char *)"stream end", /* Z_STREAM_END 1 */ + (z_const char *)"", /* Z_OK 0 */ + (z_const char *)"file error", /* Z_ERRNO (-1) */ + (z_const char *)"stream error", /* Z_STREAM_ERROR (-2) */ + (z_const char *)"data error", /* Z_DATA_ERROR (-3) */ + (z_const char *)"insufficient memory", /* Z_MEM_ERROR (-4) */ + (z_const char *)"buffer error", /* Z_BUF_ERROR (-5) */ + (z_const char *)"incompatible version",/* Z_VERSION_ERROR (-6) */ + (z_const char *)"" +}; const char * ZEXPORT zlibVersion() @@ -61,7 +58,7 @@ uLong ZEXPORT zlibCompileFlags() case 8: flags += 2 << 6; break; default: flags += 3 << 6; } -#ifdef DEBUG +#ifdef ZLIB_DEBUG flags += 1 << 8; #endif #if defined(ASMV) || defined(ASMINF) @@ -115,8 +112,8 @@ uLong ZEXPORT zlibCompileFlags() return flags; } -#ifdef DEBUG - +#ifdef ZLIB_DEBUG +#include # ifndef verbose # define verbose 0 # endif @@ -219,9 +216,11 @@ local ptr_table table[MAX_PTR]; voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, unsigned items, unsigned size) { - voidpf buf = opaque; /* just to make some compilers happy */ + voidpf buf; ulg bsize = (ulg)items*size; + (void)opaque; + /* If we allocate less than 65520 bytes, we assume that farmalloc * will return a usable pointer which doesn't have to be normalized. */ @@ -244,6 +243,9 @@ voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, unsigned items, unsigned size) void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) { int n; + + (void)opaque; + if (*(ush*)&ptr != 0) { /* object < 64K */ farfree(ptr); return; @@ -259,7 +261,6 @@ void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) next_ptr--; return; } - ptr = opaque; /* just to make some compilers happy */ Assert(0, "zcfree: ptr not found"); } @@ -278,13 +279,13 @@ void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, uInt items, uInt size) { - if (opaque) opaque = 0; /* to make compiler happy */ + (void)opaque; return _halloc((long)items, size); } void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) { - if (opaque) opaque = 0; /* to make compiler happy */ + (void)opaque; _hfree(ptr); } @@ -306,7 +307,7 @@ voidpf ZLIB_INTERNAL zcalloc (opaque, items, size) unsigned items; unsigned size; { - if (opaque) items += size - size; /* make compiler happy */ + (void)opaque; return sizeof(uInt) > 2 ? (voidpf)malloc(items * size) : (voidpf)calloc(items, size); } @@ -315,8 +316,8 @@ void ZLIB_INTERNAL zcfree (opaque, ptr) voidpf opaque; voidpf ptr; { + (void)opaque; free(ptr); - if (opaque) return; /* make compiler happy */ } #endif /* MY_ZCALLOC */ diff --git a/lib/libvgz/zutil.h b/lib/libvgz/zutil.h index f6dd6d5..1a66997 100644 --- a/lib/libvgz/zutil.h +++ b/lib/libvgz/zutil.h @@ -1,5 +1,5 @@ /* zutil.h -- internal interface and configuration of the compression library - * Copyright (C) 1995-2013 Jean-loup Gailly. + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -36,7 +36,9 @@ #ifndef local # define local static #endif -/* compile with -Dlocal if your debugger can't find static symbols */ +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ typedef unsigned char uch; typedef uch FAR uchf; @@ -98,28 +100,38 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #endif #ifdef AMIGA -# define OS_CODE 0x01 +# define OS_CODE 1 #endif #if defined(VAXC) || defined(VMS) -# define OS_CODE 0x02 +# define OS_CODE 2 # define F_OPEN(name, mode) \ fopen((name), (mode), "mbc=60", "ctx=stm", "rfm=fix", "mrs=512") #endif +#ifdef __370__ +# if __TARGET_LIB__ < 0x20000000 +# define OS_CODE 4 +# elif __TARGET_LIB__ < 0x40000000 +# define OS_CODE 11 +# else +# define OS_CODE 8 +# endif +#endif + #if defined(ATARI) || defined(atarist) -# define OS_CODE 0x05 +# define OS_CODE 5 #endif #ifdef OS2 -# define OS_CODE 0x06 +# define OS_CODE 6 # if defined(M_I86) && !defined(Z_SOLO) # include # endif #endif #if defined(MACOS) || defined(TARGET_OS_MAC) -# define OS_CODE 0x07 +# define OS_CODE 7 # ifndef Z_SOLO # if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os # include /* for fdopen */ @@ -131,18 +143,24 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # endif #endif -#ifdef TOPS20 -# define OS_CODE 0x0a +#ifdef __acorn +# define OS_CODE 13 #endif -#ifdef WIN32 -# ifndef __CYGWIN__ /* Cygwin is Unix, not Win32 */ -# define OS_CODE 0x0b -# endif +#if defined(WIN32) && !defined(__CYGWIN__) +# define OS_CODE 10 +#endif + +#ifdef _BEOS_ +# define OS_CODE 16 +#endif + +#ifdef __TOS_OS400__ +# define OS_CODE 18 #endif -#ifdef __50SERIES /* Prime/PRIMOS */ -# define OS_CODE 0x0f +#ifdef __APPLE__ +# define OS_CODE 19 #endif #if defined(_BEOS_) || defined(RISCOS) @@ -177,7 +195,7 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* common defaults */ #ifndef OS_CODE -# define OS_CODE 0x03 /* assume Unix */ +# define OS_CODE 3 /* assume Unix */ #endif #ifndef F_OPEN @@ -216,7 +234,7 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #endif /* Diagnostic functions */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG # include extern int ZLIB_INTERNAL z_verbose; extern void ZLIB_INTERNAL z_error OF((char *m)); diff --git a/tools/audit_vgz.sh b/tools/audit_vgz.sh new file mode 100644 index 0000000..16ebf6b --- /dev/null +++ b/tools/audit_vgz.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# +# Script to compare vgz with FreeBSD's copy of zlib +# +# Run this on a up-to-date FreeBSD source tree + +for i in lib/libvgz/*.[ch] +do + sed ' + s/"vgz.h"/"zlib.h"/ + s/msg = "/msg = (char *)"/ + ' $i | + diff -u /usr/src/contrib/zlib/`basename $i` - +done +diff -u /usr/src/contrib/zlib/zlib.h lib/libvgz/vgz.h + From phk at FreeBSD.org Sun May 7 21:03:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 07 May 2017 23:03:06 +0200 Subject: [master] 496be87 Update vfl.c to track FreeBSD's flopen.c Message-ID: commit 496be87b3d3b8e9bfe17556c886444047ca53d98 Author: Poul-Henning Kamp Date: Sun May 7 12:59:13 2017 +0000 Update vfl.c to track FreeBSD's flopen.c diff --git a/lib/libvarnish/vfl.c b/lib/libvarnish/vfl.c index f7302ec..690128d 100644 --- a/lib/libvarnish/vfl.c +++ b/lib/libvarnish/vfl.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2007 Dag-Erling Co?dan Sm?rgrav + * Copyright (c) 2007-2009 Dag-Erling Co?dan Sm?rgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -24,26 +24,33 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * Derived from: - * $FreeBSD: head/lib/libutil/flopen.c 184094 2008-10-20 18:11:30Z des $ + * $FreeBSD: head/lib/libutil/flopen.c 309344 2016-12-01 02:21:36Z cem $ */ #include "config.h" +#include #include #include -#include #include #include #include #include "vfl.h" +/* + * Reliably open and lock a file. + * + * Please do not modify this code without first reading the revision history + * and discussing your changes with . Don't be fooled by the + * code's apparent simplicity; there would be no need for this function if it + * was easy to get right. + */ int VFL_Open(const char *path, int flags, ...) { int fd, operation, serrno, trunc; - struct flock lock; struct stat sb, fsb; mode_t mode; @@ -60,10 +67,9 @@ VFL_Open(const char *path, int flags, ...) va_end(ap); } - memset(&lock, 0, sizeof lock); - lock.l_type = ((flags & O_ACCMODE) == O_RDONLY) ? F_RDLCK : F_WRLCK; - lock.l_whence = SEEK_SET; - operation = (flags & O_NONBLOCK) ? F_SETLK : F_SETLKW; + operation = LOCK_EX; + if (flags & O_NONBLOCK) + operation |= LOCK_NB; trunc = (flags & O_TRUNC); flags &= ~O_TRUNC; @@ -72,7 +78,7 @@ VFL_Open(const char *path, int flags, ...) if ((fd = open(path, flags, mode)) == -1) /* non-existent or no access */ return (-1); - if (fcntl(fd, operation, &lock) == -1) { + if (flock(fd, operation) == -1) { /* unsupported or interrupted */ serrno = errno; (void)close(fd); @@ -104,6 +110,18 @@ VFL_Open(const char *path, int flags, ...) errno = serrno; return (-1); } + /* + * The following change is provided as a specific example to + * avoid. + */ +#if 0 + if (fcntl(fd, F_SETFD, FD_CLOEXEC) != 0) { + serrno = errno; + (void)close(fd); + errno = serrno; + return (-1); + } +#endif return (fd); } } diff --git a/tools/audit_vfl.sh b/tools/audit_vfl.sh new file mode 100644 index 0000000..4be60a9 --- /dev/null +++ b/tools/audit_vfl.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# +# Script to compare vfl.c with FreeBSD's flopen.c +# +# Run this on a up-to-date FreeBSD source tree + +sed ' +s/VFL_Open/flopen/ +' lib/libvarnish/vfl.c | + diff -u /usr/src/lib/libutil/flopen.c - + From phk at FreeBSD.org Sun May 7 21:03:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 07 May 2017 23:03:06 +0200 Subject: [master] bdf37b2 Remove unnecessary #include of vsm_api.h Message-ID: commit bdf37b207ae50eefc84b85c2b71f53e71b4fed1a Author: Poul-Henning Kamp Date: Sun May 7 21:02:07 2017 +0000 Remove unnecessary #include of vsm_api.h diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index 2566561..127a4bb 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -55,7 +55,6 @@ #include "vapi/vsl.h" #include "vsl_api.h" -#include "vsm_api.h" /*--------------------------------------------------------------------*/ diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c index e811ca0..9c8032c 100644 --- a/lib/libvarnishapi/vsl_arg.c +++ b/lib/libvarnishapi/vsl_arg.c @@ -55,7 +55,6 @@ #include "vapi/vsl.h" #include "vsl_api.h" -#include "vsm_api.h" /*-------------------------------------------------------------------- * Look up a tag From phk at FreeBSD.org Sun May 7 21:45:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 07 May 2017 23:45:05 +0200 Subject: [master] 77ef704 Remove old debug relic Message-ID: commit 77ef70483741187d6ed488fa768edacec497195c Author: Poul-Henning Kamp Date: Sun May 7 21:44:05 2017 +0000 Remove old debug relic diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 5165620..3431dbf 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -307,8 +307,6 @@ vsc_add_pt(struct vsc *vsc, const volatile void *ptr, /*-------------------------------------------------------------------- */ -#include - static void vsc_build_vf_list(struct VSM_data *vd) { From phk at FreeBSD.org Sun May 7 21:47:04 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 07 May 2017 23:47:04 +0200 Subject: [master] 28ad2f7 Put fcntl.h include back Message-ID: commit 28ad2f72ee3b9e2354e64d2e6b9253670832ac46 Author: Poul-Henning Kamp Date: Sun May 7 21:45:53 2017 +0000 Put fcntl.h include back diff --git a/lib/libvarnish/vfl.c b/lib/libvarnish/vfl.c index 690128d..9af496a 100644 --- a/lib/libvarnish/vfl.c +++ b/lib/libvarnish/vfl.c @@ -33,6 +33,7 @@ #include #include +#include #include #include #include From phk at FreeBSD.org Sun May 7 22:44:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 08 May 2017 00:44:05 +0200 Subject: [master] d24c996 Start wrecking VSM/VSC apis: Remove unnecessary VSM_fantom in VSC_section{} Message-ID: commit d24c9962aba76a675b893740b9377fd9a6fbba74 Author: Poul-Henning Kamp Date: Sun May 7 22:25:08 2017 +0000 Start wrecking VSM/VSC apis: Remove unnecessary VSM_fantom in VSC_section{} diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c index c1f152e..5065d84 100644 --- a/bin/varnishstat/varnishstat.c +++ b/bin/varnishstat/varnishstat.c @@ -68,10 +68,10 @@ do_xml_cb(void *priv, const struct VSC_point * const pt) sec = pt->section; printf("\t\n"); - if (strcmp(sec->fantom->type, "")) - printf("\t\t%s\n", sec->fantom->type); - if (strcmp(sec->fantom->ident, "")) - printf("\t\t%s\n", sec->fantom->ident); + if (strcmp(sec->type, "")) + printf("\t\t%s\n", sec->type); + if (strcmp(sec->ident, "")) + printf("\t\t%s\n", sec->ident); printf("\t\t%s\n", pt->desc->name); printf("\t\t%ju\n", (uintmax_t)val); printf("\t\t%c\n", pt->desc->semantics); @@ -120,17 +120,17 @@ do_json_cb(void *priv, const struct VSC_point * const pt) printf(" \""); /* build the JSON key name. */ - if (sec->fantom->type[0]) - printf("%s.", sec->fantom->type); - if (sec->fantom->ident[0]) - printf("%s.", sec->fantom->ident); + if (sec->type[0]) + printf("%s.", sec->type); + if (sec->ident[0]) + printf("%s.", sec->ident); printf("%s\": {\n", pt->desc->name); printf(" \"description\": \"%s\",\n", pt->desc->sdesc); - if (strcmp(sec->fantom->type, "")) - printf(" \"type\": \"%s\", ", sec->fantom->type); - if (strcmp(sec->fantom->ident, "")) - printf("\"ident\": \"%s\", ", sec->fantom->ident); + if (strcmp(sec->type, "")) + printf(" \"type\": \"%s\", ", sec->type); + if (strcmp(sec->ident, "")) + printf("\"ident\": \"%s\", ", sec->ident); printf("\"flag\": \"%c\", ", pt->desc->semantics); printf("\"format\": \"%c\",\n", pt->desc->format); printf(" \"value\": %ju", (uintmax_t)val); @@ -182,10 +182,10 @@ do_once_cb(void *priv, const struct VSC_point * const pt) val = *(const volatile uint64_t*)pt->ptr; sec = pt->section; i = 0; - if (strcmp(sec->fantom->type, "")) - i += printf("%s.", sec->fantom->type); - if (strcmp(sec->fantom->ident, "")) - i += printf("%s.", sec->fantom->ident); + if (strcmp(sec->type, "")) + i += printf("%s.", sec->type); + if (strcmp(sec->ident, "")) + i += printf("%s.", sec->ident); i += printf("%s", pt->desc->name); if (i >= op->pad) op->pad = i + 1; @@ -227,10 +227,10 @@ do_list_cb(void *priv, const struct VSC_point * const pt) sec = pt->section; i = 0; - if (strcmp(sec->fantom->type, "")) - i += printf("%s.", sec->fantom->type); - if (strcmp(sec->fantom->ident, "")) - i += printf("%s.", sec->fantom->ident); + if (strcmp(sec->type, "")) + i += printf("%s.", sec->type); + if (strcmp(sec->ident, "")) + i += printf("%s.", sec->ident); i += printf("%s", pt->desc->name); if (i < 30) printf("%*s", i - 30, ""); diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h index bf1b1f5..ebd003e 100644 --- a/include/vapi/vsc.h +++ b/include/vapi/vsc.h @@ -117,7 +117,6 @@ struct VSC_section { const char *type; const char *ident; const struct VSC_type_desc *desc; - struct VSM_fantom *fantom; }; /* See include/tbl/vsc_fields.h for descriptions */ diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 3431dbf..9f0a4ff 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -256,7 +256,6 @@ vsc_add_vf(struct vsc *vsc, const struct VSM_fantom *fantom, vf->section.type = vf->fantom.type; vf->section.ident = vf->fantom.ident; vf->section.desc = desc; - vf->section.fantom = &vf->fantom; vf->order = order; VTAILQ_FOREACH(vf2, &vsc->vf_list, list) { From phk at FreeBSD.org Sun May 7 22:44:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 08 May 2017 00:44:05 +0200 Subject: [master] eacdb4f Decouple VSC api from VSM internal format Message-ID: commit eacdb4ff5bf962bbeeda5f2e9d27184e0e7915e0 Author: Poul-Henning Kamp Date: Sun May 7 22:43:32 2017 +0000 Decouple VSC api from VSM internal format diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 72140ee..b772822 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -39,7 +39,6 @@ #include "common/common.h" #include "vapi/vsl_int.h" -#include "vapi/vsm_int.h" #include "waiter/waiter.h" diff --git a/bin/varnishd/common/common_vsm.c b/bin/varnishd/common/common_vsm.c index 7b38c4a..22ae5c3 100644 --- a/bin/varnishd/common/common_vsm.c +++ b/bin/varnishd/common/common_vsm.c @@ -27,7 +27,7 @@ * * VSM stuff common to manager and child. * - * Please see comments in for details of protocols and + * Please see comments in for details of protocols and * data consistency. * */ diff --git a/include/Makefile.am b/include/Makefile.am index f6a08eb..9adaebe 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -46,7 +46,6 @@ nobase_pkginclude_HEADERS = \ tbl/vsl_tags_http.h \ tbl/waiters.h \ vapi/vsm.h \ - vapi/vsm_int.h \ vapi/vsc.h \ vapi/vsc_int.h \ vapi/vsl.h \ diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h index ebd003e..900a795 100644 --- a/include/vapi/vsc.h +++ b/include/vapi/vsc.h @@ -114,8 +114,8 @@ struct VSC_type_desc { }; struct VSC_section { - const char *type; - const char *ident; + char *type; + char *ident; const struct VSC_type_desc *desc; }; diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h index 18d809b..79cba08 100644 --- a/include/vapi/vsm.h +++ b/include/vapi/vsm.h @@ -36,8 +36,6 @@ #ifndef VAPI_VSM_H_INCLUDED #define VAPI_VSM_H_INCLUDED -#include "vsm_int.h" - struct VSM_chunk; struct VSM_data; @@ -50,12 +48,12 @@ struct VSM_fantom { void *b; /* first byte of payload */ void *e; /* first byte past payload */ uintptr_t priv; /* VSM private */ - char class[VSM_MARKER_LEN]; - char type[VSM_MARKER_LEN]; - char ident[VSM_IDENT_LEN]; + char *class; + char *type; + char *ident; }; -#define VSM_FANTOM_NULL { 0, 0, 0, 0, {0}, {0}, {0} } +#define VSM_FANTOM_NULL { 0, 0, 0, 0, 0, 0, 0 } /*--------------------------------------------------------------------- * VSM level access functions diff --git a/include/vapi/vsm_int.h b/include/vapi/vsm_int.h deleted file mode 100644 index deed9f7..0000000 --- a/include/vapi/vsm_int.h +++ /dev/null @@ -1,44 +0,0 @@ -/*- - * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2015 Varnish Software AS - * All rights reserved. - * - * Author: Poul-Henning Kamp - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * Define the internal details which must be kept in sync between - * vsm_priv.h and vapi/vsm.h, and this file SHALL not be included - * from anywhere but those two files. - * - * NB: THIS IS NOT A PUBLIC API TO VARNISH! - * - */ - -#ifndef VSM_INT_H_INCLUDED -#define VSM_INT_H_INCLUDED - -#define VSM_FILENAME "_.vsm" -#define VSM_MARKER_LEN 8 -#define VSM_IDENT_LEN 128 - -#endif /* VSM_INT_H_INCLUDED */ diff --git a/include/vsl_priv.h b/include/vsl_priv.h index 6296a5b..8ac6c66 100644 --- a/include/vsl_priv.h +++ b/include/vsl_priv.h @@ -36,7 +36,7 @@ #define VSL_PRIV_H_INCLUDED #include "vapi/vsl_int.h" -#include "vapi/vsm_int.h" +#include "vsm_priv.h" #define VSL_CLASS "Log" #define VSL_SEGMENTS 8U // power of two diff --git a/include/vsm_priv.h b/include/vsm_priv.h index 4d0d401..20f4f90 100644 --- a/include/vsm_priv.h +++ b/include/vsm_priv.h @@ -96,7 +96,9 @@ #ifndef VSM_PRIV_H_INCLUDED #define VSM_PRIV_H_INCLUDED -#include +#define VSM_FILENAME "_.vsm" +#define VSM_MARKER_LEN 8 +#define VSM_IDENT_LEN 128 struct VSM_chunk { #define VSM_CHUNK_MARKER "VSMCHUNK" diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 9f0a4ff..5fd55fd 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -253,8 +253,8 @@ vsc_add_vf(struct vsc *vsc, const struct VSM_fantom *fantom, ALLOC_OBJ(vf, VSC_VF_MAGIC); AN(vf); vf->fantom = *fantom; - vf->section.type = vf->fantom.type; - vf->section.ident = vf->fantom.ident; + REPLACE(vf->section.type, vf->fantom.type); + REPLACE(vf->section.ident, vf->fantom.ident); vf->section.desc = desc; vf->order = order; @@ -336,7 +336,7 @@ vsc_build_pt_list(struct VSM_data *vd) VTAILQ_FOREACH(vf, &vsc->vf_list, list) { #define VSC_DO(U,l,t,h) \ CHECK_OBJ_NOTNULL(vf, VSC_VF_MAGIC); \ - if (!strcmp(vf->fantom.type, t)) \ + if (!strcmp(vf->section.type, t)) \ iter_##l(vsc, VSC_desc_##l, vf); #define VSC_F(n,t,l,s,f,v,d,e) #define VSC_DONE(a,b,c) diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c index ae0ace8..3805237 100644 --- a/lib/libvarnishapi/vsm.c +++ b/lib/libvarnishapi/vsm.c @@ -397,12 +397,15 @@ VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf) vf->priv = vd->head->alloc_seq; vf->b = (void*)(vf->chunk + 1); vf->e = (char*)vf->b + vf->chunk->len; - strncpy(vf->class, vf->chunk->class, sizeof vf->class); - vf->class[sizeof vf->class - 1] = '\0'; - strncpy(vf->type, vf->chunk->type, sizeof vf->type); - vf->type[sizeof vf->type - 1] = '\0'; - strncpy(vf->ident, vf->chunk->ident, sizeof vf->ident); - vf->ident[sizeof vf->ident - 1] = '\0'; + + AZ(vf->chunk->class[sizeof vf->chunk->class - 1]); + REPLACE(vf->class, vf->chunk->class); + + AZ(vf->chunk->type[sizeof vf->chunk->type - 1]); + REPLACE(vf->type, vf->chunk->type); + + AZ(vf->chunk->ident[sizeof vf->chunk->ident - 1]); + REPLACE(vf->ident, vf->chunk->ident); return (1); } From fgsch at lodoss.net Sun May 7 23:39:05 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 08 May 2017 01:39:05 +0200 Subject: [master] 4518b0b A few more tests Message-ID: commit 4518b0b74da68ddc32367422970e581078d5ab72 Author: Federico G. Schwindt Date: Mon May 8 00:26:11 2017 +0100 A few more tests diff --git a/bin/varnishtest/tests/u00005.vtc b/bin/varnishtest/tests/u00005.vtc index cd87c93..45a533e 100644 --- a/bin/varnishtest/tests/u00005.vtc +++ b/bin/varnishtest/tests/u00005.vtc @@ -36,9 +36,9 @@ shell -expect "Copyright (c) 2006 Verdens Gang AS" \ shell -err -expect "Usage: varnishstat " \ "varnishstat extra" shell -err -expect "-t: Range error" \ - "varnishstat -n /nonexistent -t -1" + "varnishstat -t -1" shell -err -expect "-t: Syntax error" \ - "varnishstat -n /nonexistent -t foo" + "varnishstat -t foo" shell -err -expect "Cannot open /nonexistent/_.vsm" \ "varnishstat -n /nonexistent -t 1" shell -expect "MAIN.uptime" \ diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc index 196514e..35c8e01 100644 --- a/bin/varnishtest/tests/u00006.vtc +++ b/bin/varnishtest/tests/u00006.vtc @@ -23,6 +23,12 @@ shell -err -expect "Usage: varnishlog " \ "varnishlog extra" shell -err -expect "Missing -w option" \ "varnishlog -D" +shell -err -expect "Ambiguous grouping type: r" \ + "varnishlog -g r" +shell -err -expect "Unknown grouping type: foo" \ + "varnishlog -g foo" +shell -err -expect "-k: Invalid number 'foo'" \ + "varnishlog -k foo" shell -err -expect "-L: Range error" \ "varnishlog -L 0" shell -err -expect {-i: "foo" matches zero tags} \ @@ -35,6 +41,10 @@ shell -err -expect {-I: "Resp" is ambiguous} \ "varnishlog -I Resp:bar" shell -err -expect {-I: Regex error at position 4 (missing ))} \ {varnishlog -I "(foo"} +shell -err -expect "-t: Range error" \ + "varnishlog -t -1" +shell -err -expect "-t: Syntax error" \ + "varnishlog -t foo" shell -err -expect {-x: Syntax error in "**"} \ {varnishlog -x "**"} shell -err -expect {-X: Syntax error in "**"} \ From phk at FreeBSD.org Mon May 8 08:11:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 08 May 2017 10:11:05 +0200 Subject: [master] e9c70c7 Retire the -N option, it won't make any sense with multi-file VSM layout. Message-ID: commit e9c70c7306eb94c14768648dca03fc5b6c3252f9 Author: Poul-Henning Kamp Date: Mon May 8 08:10:06 2017 +0000 Retire the -N option, it won't make any sense with multi-file VSM layout. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 6e24a05..4f9b324 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -774,7 +774,7 @@ main(int argc, char * const *argv) VJ_master(JAIL_MASTER_LOW); } - if (VIN_N_Arg(n_arg, &heritage.name, &dirname, NULL) != 0) + if (VIN_n_Arg(n_arg, &heritage.name, &dirname) != 0) ARGV_ERR("Invalid instance (-n) name: %s\n", strerror(errno)); #ifdef HAVE_SETPROCTITLE diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index a9bedea..a4f70cd 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -73,7 +73,6 @@ HIS_OPT_g VUT_OPT_h VSL_OPT_L VUT_OPT_n -VUT_OPT_N HIS_OPT_p #define HIS_CLIENT "client" #define HIS_BACKEND "backend" diff --git a/bin/varnishlog/varnishlog_options.h b/bin/varnishlog/varnishlog_options.h index 197b8c2..4847268 100644 --- a/bin/varnishlog/varnishlog_options.h +++ b/bin/varnishlog/varnishlog_options.h @@ -67,7 +67,6 @@ VSL_OPT_I VUT_OPT_k VSL_OPT_L VUT_OPT_n -VUT_OPT_N VUT_OPT_P VUT_OPT_q VUT_OPT_r diff --git a/bin/varnishncsa/varnishncsa_options.h b/bin/varnishncsa/varnishncsa_options.h index 98ab675..6fe13de 100644 --- a/bin/varnishncsa/varnishncsa_options.h +++ b/bin/varnishncsa/varnishncsa_options.h @@ -85,7 +85,6 @@ NCSA_OPT_g VUT_OPT_h VSL_OPT_L VUT_OPT_n -VUT_OPT_N VUT_OPT_P VUT_OPT_q VUT_OPT_r diff --git a/bin/varnishstat/varnishstat_options.h b/bin/varnishstat/varnishstat_options.h index 65a961c..47dd18b 100644 --- a/bin/varnishstat/varnishstat_options.h +++ b/bin/varnishstat/varnishstat_options.h @@ -64,7 +64,6 @@ VUT_OPT_h STAT_OPT_j STAT_OPT_l VUT_OPT_n -VUT_OPT_N VUT_OPT_t VUT_OPT_V STAT_OPT_x diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc index 35c8e01..a36eff2 100644 --- a/bin/varnishtest/tests/u00006.vtc +++ b/bin/varnishtest/tests/u00006.vtc @@ -51,8 +51,8 @@ shell -err -expect {-X: Syntax error in "**"} \ {varnishlog -X "**:bar"} shell -err -expect "Cannot open output file (No such file or directory)" \ {varnishlog -w /nonexistent/file} -shell -err -expect "Only one of -n, -N and -r options may be used" \ - {varnishlog -n ${v1_name} -N ${v1_name}/_.vsm} +shell -err -expect "Only one of -n and -r options may be used" \ + {varnishlog -n ${v1_name} -r ${v1_name}/_.vsm} process p1 -wait shell {grep -q "0 CLI" ${tmpdir}/vlog} diff --git a/bin/varnishtop/varnishtop_options.h b/bin/varnishtop/varnishtop_options.h index b65a961..1cf0e20 100644 --- a/bin/varnishtop/varnishtop_options.h +++ b/bin/varnishtop/varnishtop_options.h @@ -65,7 +65,6 @@ VSL_OPT_i VSL_OPT_I VSL_OPT_L VUT_OPT_n -VUT_OPT_N TOP_OPT_p VUT_OPT_q VUT_OPT_r diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h index 79cba08..52d8167 100644 --- a/include/vapi/vsm.h +++ b/include/vapi/vsm.h @@ -98,18 +98,6 @@ int VSM_n_Arg(struct VSM_data *vd, const char *n_arg); * <0 on failure, VSM_Error() returns diagnostic string */ -#define VSM_N_USAGE "[-N filename]" - -int VSM_N_Arg(struct VSM_data *vd, const char *N_arg); - /* - * Configure the library to use the specified VSM file name. This - * bypasses abandonment checks and allows looking at stale VSM - * files without a running Varnish instance. - * - * Returns: - * 1 on success - */ - const char *VSM_Name(const struct VSM_data *vd); /* * Return the instance name. diff --git a/include/vin.h b/include/vin.h index f627ab5..e0fe073 100644 --- a/include/vin.h +++ b/include/vin.h @@ -32,5 +32,5 @@ #define VIN_H_INCLUDED /* This function lives in both libvarnish and libvarnishapi */ -int VIN_N_Arg(const char *n_arg, char **name, char **dir, char **vsl); +int VIN_n_Arg(const char *n_arg, char **name, char **dir); #endif diff --git a/include/vut.h b/include/vut.h index 5825994..1e1942c 100644 --- a/include/vut.h +++ b/include/vut.h @@ -43,7 +43,6 @@ struct VUT { int g_arg; int k_arg; char *n_arg; - char *N_arg; char *P_arg; char *q_arg; char *r_arg; diff --git a/include/vut_options.h b/include/vut_options.h index 7afc723..5e6d094 100644 --- a/include/vut_options.h +++ b/include/vut_options.h @@ -63,12 +63,6 @@ " the host name is used." \ ) -#define VUT_OPT_N \ - VOPT("N:", "[-N ]", "VSM filename", \ - "Specify the filename of a stale VSM instance. When using" \ - " this option the abandonment checking is disabled." \ - ) - #define VUT_OPT_P \ VOPT("P:", "[-P ]", "PID file", \ "Write the process' PID to the specified file." \ diff --git a/lib/libvarnish/vin.c b/lib/libvarnish/vin.c index e06a2c0..1cb064c 100644 --- a/lib/libvarnish/vin.c +++ b/lib/libvarnish/vin.c @@ -43,7 +43,7 @@ #include "vin.h" int -VIN_N_Arg(const char *n_arg, char **name, char **dir, char **vsl) +VIN_n_Arg(const char *n_arg, char **name, char **dir) { char nm[PATH_MAX]; char dn[PATH_MAX]; @@ -92,11 +92,5 @@ VIN_N_Arg(const char *n_arg, char **name, char **dir, char **vsl) if (*dir == NULL) return (-1); } - if (vsl != NULL) { - bprintf(nm, "%s%s", dn, VSM_FILENAME); - *vsl = strdup(nm); - if (*vsl == NULL) - return (-1); - } return (0); } diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 5fd55fd..a17d3d0 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -201,7 +201,6 @@ VSC_Arg(struct VSM_data *vd, int arg, const char *opt) switch (arg) { case 'f': return (vsc_f_arg(vd, opt)); case 'n': return (VSM_n_Arg(vd, opt)); - case 'N': return (VSM_N_Arg(vd, opt)); default: return (0); } diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c index 3805237..18c7b6c 100644 --- a/lib/libvarnishapi/vsm.c +++ b/lib/libvarnishapi/vsm.c @@ -132,48 +132,33 @@ int VSM_n_Arg(struct VSM_data *vd, const char *arg) { char *name = NULL; - char *fname = NULL; + char *dname = NULL; + struct vsb *vsb; CHECK_OBJ_NOTNULL(vd, VSM_MAGIC); if (vd->head) return (vsm_diag(vd, "VSM_n_Arg: Already open")); - if (VIN_N_Arg(arg, &name, NULL, &fname)) + if (VIN_n_Arg(arg, &name, &dname)) return (vsm_diag(vd, "Invalid instance name: %s", strerror(errno))); AN(name); - AN(fname); + AN(dname); + vsb = VSB_new_auto(); + AN(vsb); + VSB_printf(vsb, "%s%s", dname, VSM_FILENAME); + AZ(VSB_finish(vsb)); - if (vd->name) - free(vd->name); - vd->name = name; - if (vd->fname) - free(vd->fname); - vd->fname = fname; - vd->N_opt = 0; + REPLACE(vd->name, name); + REPLACE(vd->dname, dname); + REPLACE(vd->iname, VSB_data(vsb)); + VSB_destroy(&vsb); return (1); } /*--------------------------------------------------------------------*/ -int -VSM_N_Arg(struct VSM_data *vd, const char *arg) -{ - - CHECK_OBJ_NOTNULL(vd, VSM_MAGIC); - AN(arg); - - if (vd->head) - return (vsm_diag(vd, "VSM_N_Arg: Already open")); - REPLACE(vd->name, arg); - REPLACE(vd->fname, arg); - vd->N_opt = 1; - return (1); -} - -/*--------------------------------------------------------------------*/ - const char * VSM_Name(const struct VSM_data *vd) { @@ -196,7 +181,7 @@ VSM_Delete(struct VSM_data *vd) VSC_Delete(vd); VSM_ResetError(vd); free(vd->name); - free(vd->fname); + free(vd->dname); FREE_OBJ(vd); } @@ -224,43 +209,42 @@ VSM_Open(struct VSM_data *vd) /* Already open */ return (0); - if (vd->fname == NULL) { + if (vd->dname == NULL) { /* Use default (hostname) */ i = VSM_n_Arg(vd, ""); if (i < 0) return (i); - AN(vd->fname); + AN(vd->dname); } - vd->vsm_fd = open(vd->fname, O_RDONLY); + vd->vsm_fd = open(vd->iname, O_RDONLY); if (vd->vsm_fd < 0) return (vsm_diag(vd, "Cannot open %s: %s", - vd->fname, strerror(errno))); + vd->iname, strerror(errno))); AZ(fstat(vd->vsm_fd, &vd->fstat)); if (!S_ISREG(vd->fstat.st_mode)) { closefd(&vd->vsm_fd); - return (vsm_diag(vd, "%s is not a regular file", - vd->fname)); + return (vsm_diag(vd, "%s is not a regular file", vd->iname)); } i = read(vd->vsm_fd, &slh, sizeof slh); if (i != sizeof slh) { closefd(&vd->vsm_fd); return (vsm_diag(vd, "Cannot read %s: %s", - vd->fname, strerror(errno))); + vd->iname, strerror(errno))); } if (memcmp(slh.marker, VSM_HEAD_MARKER, sizeof slh.marker)) { closefd(&vd->vsm_fd); - return (vsm_diag(vd, "Not a VSM file %s", vd->fname)); + return (vsm_diag(vd, "Not a VSM file %s", vd->iname)); } - if (!vd->N_opt && slh.alloc_seq == 0) { + if (slh.alloc_seq == 0) { closefd(&vd->vsm_fd); return (vsm_diag(vd, "Abandoned VSM file (Varnish not running?) %s", - vd->fname)); + vd->iname)); } v = mmap(NULL, slh.shm_size, @@ -268,7 +252,7 @@ VSM_Open(struct VSM_data *vd) if (v == MAP_FAILED) { closefd(&vd->vsm_fd); return (vsm_diag(vd, "Cannot mmap %s: %s", - vd->fname, strerror(errno))); + vd->iname, strerror(errno))); } vd->head = v; vd->b = v; @@ -321,9 +305,6 @@ VSM_Abandoned(struct VSM_data *vd) if (vd->head == NULL) /* Not open */ return (1); - if (vd->N_opt) - /* No abandonment check should be done */ - return (0); if (!vd->head->alloc_seq) /* Flag of abandonment set by mgt */ return (1); @@ -333,7 +314,7 @@ VSM_Abandoned(struct VSM_data *vd) now = VTIM_mono(); if (vd->head->age == vd->age_ok && now - vd->t_ok > 2.) { /* No age change for 2 seconds, stat the file */ - if (stat(vd->fname, &st)) + if (stat(vd->iname, &st)) return (1); if (st.st_dev != vd->fstat.st_dev) return (1); @@ -370,11 +351,11 @@ VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf) if (!vd->head) return (0); /* Not open */ - if (!vd->N_opt && vd->head->alloc_seq == 0) + if (vd->head->alloc_seq == 0) return (0); /* abandoned VSM */ else if (vf->chunk != NULL) { /* get next chunk */ - if (!vd->N_opt && vf->priv != vd->head->alloc_seq) + if (vf->priv != vd->head->alloc_seq) return (0); /* changes during iteration */ if (vf->chunk->len == 0) return (0); /* free'd during iteration */ @@ -421,7 +402,7 @@ VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf) AN(vf); if (!vd->head) return (VSM_invalid); - if (!vd->N_opt && !vd->head->alloc_seq) + if (!vd->head->alloc_seq) return (VSM_invalid); if (vf->chunk == NULL) return (VSM_invalid); diff --git a/lib/libvarnishapi/vsm_api.h b/lib/libvarnishapi/vsm_api.h index 8ef537b..e357a8f 100644 --- a/lib/libvarnishapi/vsm_api.h +++ b/lib/libvarnishapi/vsm_api.h @@ -38,8 +38,8 @@ struct VSM_data { struct vsb *diag; char *name; - char *fname; - int N_opt; + char *dname; + char *iname; struct stat fstat; diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index e3bc741..9079b22 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -167,10 +167,6 @@ VUT_Arg(int opt, const char *arg) /* Varnish instance name */ REPLACE(VUT.n_arg, arg); return (1); - case 'N': - /* Varnish stale VSM file */ - REPLACE(VUT.N_arg, arg); - return (1); case 'P': /* PID file */ REPLACE(VUT.P_arg, arg); @@ -242,9 +238,8 @@ VUT_Setup(void) /* Check input arguments (2 used for bug in FlexeLint) */ if ((VUT.n_arg == NULL ? 0 : 2) + - (VUT.N_arg == NULL ? 0 : 2) + (VUT.r_arg == NULL ? 0 : 2) > 2) - VUT_Error(1, "Only one of -n, -N and -r options may be used"); + VUT_Error(1, "Only one of -n and -r options may be used"); /* Create and validate the query expression */ VUT.vslq = VSLQ_New(VUT.vsl, NULL, VUT.g_arg, VUT.q_arg); @@ -263,8 +258,6 @@ VUT_Setup(void) AN(VUT.vsm); if (VUT.n_arg && VSM_n_Arg(VUT.vsm, VUT.n_arg) <= 0) VUT_Error(1, "%s", VSM_Error(VUT.vsm)); - if (VUT.N_arg && VSM_N_Arg(VUT.vsm, VUT.N_arg) <= 0) - VUT_Error(1, "%s", VSM_Error(VUT.vsm)); REPLACE(VUT.name, VSM_Name(VUT.vsm)); t_start = NAN; c = NULL; @@ -336,7 +329,6 @@ void VUT_Fini(void) { free(VUT.n_arg); - free(VUT.N_arg); free(VUT.r_arg); free(VUT.P_arg); free(VUT.name); From martin at varnish-software.com Mon May 8 12:56:06 2017 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 08 May 2017 14:56:06 +0200 Subject: [master] d246d52 Remember to actually record the length of variable length attributes Message-ID: commit d246d525978363ae778dc5b793e1b201714b4080 Author: Martin Blix Grydeland Date: Mon May 8 14:28:55 2017 +0200 Remember to actually record the length of variable length attributes A variable length attribute (OA_HEADERS and OA_VARY) are allowed to be overwritten if the new data has the exact same length as the previous data. To check this, its length is recorded. But the length wasn't recorded, and the code would treat it as a new attribute. The reason this hasn't caused issues is that the code looking at OA_HEADERS and OA_VARY ignores the stored attribute length, and that these aren't ever overwritten. diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c index c3bffe9..aaf2c48 100644 --- a/bin/varnishd/storage/storage_simple.c +++ b/bin/varnishd/storage/storage_simple.c @@ -585,6 +585,7 @@ sml_setattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr, assert(st->len + len <= st->space); \ o->va_##l = st->ptr + st->len; \ st->len += len; \ + o->va_##l##_len = len; \ retval = o->va_##l; \ } \ break; From martin at varnish-software.com Mon May 8 12:56:06 2017 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 08 May 2017 14:56:06 +0200 Subject: [master] b950644 Add an assert that we get the vary attribute Message-ID: commit b950644283d4e6bfd7a5663aa756446b2f1ffedd Author: Martin Blix Grydeland Date: Mon May 8 14:41:17 2017 +0200 Add an assert that we get the vary attribute If an object has OA_VARY, we should succeed in getting it. This might help to narrow down #2319 diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 8019591..6dd6e61 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -428,6 +428,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, if (ObjHasAttr(wrk, oc, OA_VARY)) { vary = ObjGetAttr(wrk, oc, OA_VARY, NULL); + AN(vary); if (!VRY_Match(req, vary)) continue; } From fgsch at lodoss.net Fri May 12 09:34:05 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 12 May 2017 11:34:05 +0200 Subject: [master] 1401f91 Division by zero is undefined Message-ID: commit 1401f910331bc6a7d51f82a29e3e7cb3562afa7a Author: Federico G. Schwindt Date: Thu May 11 02:38:15 2017 +0100 Division by zero is undefined diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c index 5065d84..7650bf1 100644 --- a/bin/varnishstat/varnishstat.c +++ b/bin/varnishstat/varnishstat.c @@ -192,7 +192,8 @@ do_once_cb(void *priv, const struct VSC_point * const pt) printf("%*.*s", op->pad - i, op->pad - i, ""); if (pt->desc->semantics == 'c') printf("%12ju %12.2f %s\n", - (uintmax_t)val, val / op->up, pt->desc->sdesc); + (uintmax_t)val, op->up ? val / op->up : 0, + pt->desc->sdesc); else printf("%12ju %12s %s\n", (uintmax_t)val, ". ", pt->desc->sdesc); From fgsch at lodoss.net Fri May 12 09:34:05 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 12 May 2017 11:34:05 +0200 Subject: [master] d0b28d5 Rework -f option Message-ID: commit d0b28d539e4a7cc9d52fcd22539390149c57dbfe Author: Federico G. Schwindt Date: Thu May 11 02:32:03 2017 +0100 Rework -f option We now process the inclusive globs first, and then the exclusive ones. The former are accumulative, while the latter will filter the output from the first. Discussed in detail with and final diff from @mbgrydeland. Tests tweaking by yours truly. Fixes #2320. diff --git a/bin/varnishstat/varnishstat_options.h b/bin/varnishstat/varnishstat_options.h index 47dd18b..5307a1d 100644 --- a/bin/varnishstat/varnishstat_options.h +++ b/bin/varnishstat/varnishstat_options.h @@ -40,9 +40,9 @@ VOPT("f:", "[-f ]", "Field inclusion glob", \ "Field inclusion glob." \ " Use backslash to escape characters. If the argument" \ - " starts with '^' it is used as an exclusion glob." \ - " Multiple -f arguments may be given, and they will be" \ - " applied in order." \ + " starts with '^' it is used as an exclusive glob." \ + " Multiple -f arguments may be given. Inclusive globs" \ + " are accumulative and are run before exclusive ones." \ ) #define STAT_OPT_j \ VOPT("j", "[-j]", "Print statistics to stdout as JSON", \ diff --git a/bin/varnishtest/tests/u00005.vtc b/bin/varnishtest/tests/u00005.vtc index 45a533e..798a6cb 100644 --- a/bin/varnishtest/tests/u00005.vtc +++ b/bin/varnishtest/tests/u00005.vtc @@ -13,19 +13,21 @@ varnish v1 -vcl+backend {} -start delay 1 process p1 {varnishstat -1 -n ${v1_name} -f ^LCK.vbe.destroy \ - -f LCK.vbe.* -f LCK.mempool.* | tr '[1-9]' '0'} -run + -f LCK.vbe.* -f LCK.mempool.creat | tr '[1-9]' '0'} -run shell "grep -q vbe ${p1_out}" -shell "grep -q mempool ${p1_out}" +shell "grep -q LCK.mempool.creat ${p1_out}" shell -err "grep -q LCK.vbe.destroy ${p1_out}" -process p2 {varnishstat -1 -n ${v1_name} -f ^*vbe.destroy \ - -f *vbe* -f *mempool* | tr '[1-9]' '0'} -run +process p2 {varnishstat -1 -n ${v1_name} -f *vbe* \ + -f ^*vbe.destroy -f LCK.mempool.c* | tr '[1-9]' '0'} -run shell "cmp -s ${p1_out} ${p2_out}" -shell -expect "cache_hit" \ - "varnishstat -1 -n ${v1_name} -f MAIN.cache_hit" +shell -expect "MGT.uptime" \ + "varnishstat -1 -n ${v1_name} -f ^MAIN*" +shell -match "^MAIN" \ + "varnishstat -1 -n ${v1_name} -f ^foo" shell -expect "Usage: varnishstat " \ "varnishstat -h" diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index a17d3d0..c3b8a24 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -71,22 +71,24 @@ struct vsc_pt { VTAILQ_ENTRY(vsc_pt) list; struct VSC_point point; }; +VTAILQ_HEAD(vsc_pt_head, vsc_pt); struct vsc_sf { unsigned magic; #define VSC_SF_MAGIC 0x558478dd VTAILQ_ENTRY(vsc_sf) list; char *pattern; - unsigned exclude; }; +VTAILQ_HEAD(vsc_sf_head, vsc_sf); struct vsc { unsigned magic; #define VSC_MAGIC 0x3373554a VTAILQ_HEAD(, vsc_vf) vf_list; - VTAILQ_HEAD(, vsc_pt) pt_list; - VTAILQ_HEAD(, vsc_sf) sf_list; + struct vsc_pt_head pt_list; + struct vsc_sf_head sf_list_include; + struct vsc_sf_head sf_list_exclude; struct VSM_fantom iter_fantom; }; @@ -103,7 +105,8 @@ vsc_setup(struct VSM_data *vd) AN(vd->vsc); VTAILQ_INIT(&vd->vsc->vf_list); VTAILQ_INIT(&vd->vsc->pt_list); - VTAILQ_INIT(&vd->vsc->sf_list); + VTAILQ_INIT(&vd->vsc->sf_list_include); + VTAILQ_INIT(&vd->vsc->sf_list_exclude); } CHECK_OBJ_NOTNULL(vd->vsc, VSC_MAGIC); return (vd->vsc); @@ -125,27 +128,27 @@ vsc_delete_vf_list(struct vsc *vsc) } static void -vsc_delete_pt_list(struct vsc *vsc) +vsc_delete_pt_list(struct vsc_pt_head *head) { struct vsc_pt *pt; - while (!VTAILQ_EMPTY(&vsc->pt_list)) { - pt = VTAILQ_FIRST(&vsc->pt_list); + while (!VTAILQ_EMPTY(head)) { + pt = VTAILQ_FIRST(head); CHECK_OBJ_NOTNULL(pt, VSC_PT_MAGIC); - VTAILQ_REMOVE(&vsc->pt_list, pt, list); + VTAILQ_REMOVE(head, pt, list); FREE_OBJ(pt); } } static void -vsc_delete_sf_list(struct vsc *vsc) +vsc_delete_sf_list(struct vsc_sf_head *head) { struct vsc_sf *sf; - while (!VTAILQ_EMPTY(&vsc->sf_list)) { - sf = VTAILQ_FIRST(&vsc->sf_list); + while (!VTAILQ_EMPTY(head)) { + sf = VTAILQ_FIRST(head); CHECK_OBJ_NOTNULL(sf, VSC_SF_MAGIC); - VTAILQ_REMOVE(&vsc->sf_list, sf, list); + VTAILQ_REMOVE(head, sf, list); free(sf->pattern); FREE_OBJ(sf); } @@ -160,8 +163,9 @@ VSC_Delete(struct VSM_data *vd) vsc = vd->vsc; vd->vsc = NULL; CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC); - vsc_delete_sf_list(vsc); - vsc_delete_pt_list(vsc); + vsc_delete_sf_list(&vsc->sf_list_include); + vsc_delete_sf_list(&vsc->sf_list_exclude); + vsc_delete_pt_list(&vsc->pt_list); vsc_delete_vf_list(vsc); FREE_OBJ(vsc); } @@ -173,6 +177,7 @@ vsc_f_arg(struct VSM_data *vd, const char *opt) { struct vsc *vsc = vsc_setup(vd); struct vsc_sf *sf; + unsigned exclude = 0; AN(vd); AN(opt); @@ -181,14 +186,18 @@ vsc_f_arg(struct VSM_data *vd, const char *opt) AN(sf); if (opt[0] == '^') { - sf->exclude = 1; + exclude = 1; opt++; } sf->pattern = strdup(opt); AN(sf->pattern); - VTAILQ_INSERT_TAIL(&vsc->sf_list, sf, list); + if (exclude) + VTAILQ_INSERT_TAIL(&vsc->sf_list_exclude, sf, list); + else + VTAILQ_INSERT_TAIL(&vsc->sf_list_include, sf, list); + return (1); } @@ -310,7 +319,7 @@ vsc_build_vf_list(struct VSM_data *vd) { struct vsc *vsc = vsc_setup(vd); - vsc_delete_pt_list(vsc); + vsc_delete_pt_list(&vsc->pt_list); vsc_delete_vf_list(vsc); VSM_FOREACH(&vsc->iter_fantom, vd) { @@ -330,7 +339,7 @@ vsc_build_pt_list(struct VSM_data *vd) struct vsc *vsc = vsc_setup(vd); struct vsc_vf *vf; - vsc_delete_pt_list(vsc); + vsc_delete_pt_list(&vsc->pt_list); VTAILQ_FOREACH(vf, &vsc->vf_list, list) { #define VSC_DO(U,l,t,h) \ @@ -346,48 +355,71 @@ vsc_build_pt_list(struct VSM_data *vd) /*-------------------------------------------------------------------- */ +static int +vsc_filter_match_pt(struct vsb *vsb, const struct vsc_sf *sf, const + struct vsc_pt *pt) +{ + VSB_clear(vsb); + if (strcmp(pt->point.section->type, "")) + VSB_printf(vsb, "%s.", pt->point.section->type); + if (strcmp(pt->point.section->ident, "")) + VSB_printf(vsb, "%s.", pt->point.section->ident); + VSB_printf(vsb, "%s", pt->point.desc->name); + AZ(VSB_finish(vsb)); + return (!fnmatch(sf->pattern, VSB_data(vsb), 0)); +} + static void vsc_filter_pt_list(struct VSM_data *vd) { struct vsc *vsc = vsc_setup(vd); + struct vsc_pt_head tmplist; struct vsc_sf *sf; struct vsc_pt *pt, *pt2; - VTAILQ_HEAD(, vsc_pt) pt_list; struct vsb *vsb; - if (VTAILQ_EMPTY(&vsc->sf_list)) + if (VTAILQ_EMPTY(&vsc->sf_list_include) && + VTAILQ_EMPTY(&vsc->sf_list_exclude)) return; vsb = VSB_new_auto(); AN(vsb); + VTAILQ_INIT(&tmplist); + + /* Include filters. Empty include filter list implies one that + * matches everything. Points are sorted by the order of include + * filter they match. */ + if (!VTAILQ_EMPTY(&vsc->sf_list_include)) { + VTAILQ_FOREACH(sf, &vsc->sf_list_include, list) { + CHECK_OBJ_NOTNULL(sf, VSC_SF_MAGIC); + VTAILQ_FOREACH_SAFE(pt, &vsc->pt_list, list, pt2) { + CHECK_OBJ_NOTNULL(pt, VSC_PT_MAGIC); + if (vsc_filter_match_pt(vsb, sf, pt)) { + VTAILQ_REMOVE(&vsc->pt_list, + pt, list); + VTAILQ_INSERT_TAIL(&tmplist, + pt, list); + } + } + } + vsc_delete_pt_list(&vsc->pt_list); + VTAILQ_CONCAT(&vsc->pt_list, &tmplist, list); + } - VTAILQ_INIT(&pt_list); - VTAILQ_FOREACH(sf, &vsc->sf_list, list) { + /* Exclude filters */ + VTAILQ_FOREACH(sf, &vsc->sf_list_exclude, list) { CHECK_OBJ_NOTNULL(sf, VSC_SF_MAGIC); VTAILQ_FOREACH_SAFE(pt, &vsc->pt_list, list, pt2) { CHECK_OBJ_NOTNULL(pt, VSC_PT_MAGIC); - VSB_clear(vsb); - if (strcmp(pt->point.section->type, "")) - VSB_printf(vsb, "%s.", - pt->point.section->type); - if (strcmp(pt->point.section->ident, "")) - VSB_printf(vsb, "%s.", - pt->point.section->ident); - VSB_printf(vsb, "%s", pt->point.desc->name); - AZ(VSB_finish(vsb)); - if (fnmatch(sf->pattern, VSB_data(vsb), 0)) - continue; - VTAILQ_REMOVE(&vsc->pt_list, pt, list); - if (sf->exclude) - FREE_OBJ(pt); - else - VTAILQ_INSERT_TAIL(&pt_list, pt, list); + if (vsc_filter_match_pt(vsb, sf, pt)) { + VTAILQ_REMOVE(&vsc->pt_list, pt, list); + VTAILQ_INSERT_TAIL(&tmplist, pt, list); + } } } + vsc_delete_pt_list(&tmplist); VSB_destroy(&vsb); - vsc_delete_pt_list(vsc); - VTAILQ_CONCAT(&vsc->pt_list, &pt_list, list); } /*-------------------------------------------------------------------- From dridi.boukelmoune at gmail.com Fri May 12 11:14:06 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 12 May 2017 13:14:06 +0200 Subject: [master] ea11115 Protect the CLEAR_HASH macro from zlib Message-ID: commit ea111153098fe636f7c42ab8da4efdc5954e894e Author: Dridi Boukelmoune Date: Fri May 12 13:09:56 2017 +0200 Protect the CLEAR_HASH macro from zlib Spotted by Coverity: /lib/libvgz/deflate.c: 605 in deflateParams() 599 } 600 if (s->level != level) { 601 if (s->level == 0 && s->matches != 0) { 602 if (s->matches == 1) 603 slide_hash(s); 604 else >>> CID 1401053: Control flow issues (NESTING_INDENT_MISMATCH) >>> The macro on this line expands into multiple statements, only >>> the first of which is nested within the preceding parent while >>> the rest are not. 605 CLEAR_HASH(s); 606 s->matches = 0; 607 } 608 s->level = level; 609 s->max_lazy_match = configuration_table[level].max_lazy; 610 s->good_match = configuration_table[level].good_length; diff --git a/lib/libvgz/deflate.c b/lib/libvgz/deflate.c index b3adb63..c796cb8 100644 --- a/lib/libvgz/deflate.c +++ b/lib/libvgz/deflate.c @@ -191,8 +191,10 @@ local const config configuration_table[10] = { * prev[] will be initialized on the fly. */ #define CLEAR_HASH(s) \ +do { \ s->head[s->hash_size-1] = NIL; \ - zmemzero((Bytef *)s->head, (unsigned)(s->hash_size-1)*sizeof(*s->head)); + zmemzero((Bytef *)s->head, (unsigned)(s->hash_size-1)*sizeof(*s->head)); \ +} while (0) /* =========================================================================== * Slide the hash table when sliding the window down (could be avoided with 32 From nils.goroll at uplex.de Fri May 12 12:53:05 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 12 May 2017 14:53:05 +0200 Subject: [master] e216e02 document that send_timeout/idle_send_timeout have no effect unless SO_SNDTIMEO works Message-ID: commit e216e02cae5f9af66c36851bc7877a17d4f83a93 Author: Nils Goroll Date: Fri May 12 14:12:28 2017 +0200 document that send_timeout/idle_send_timeout have no effect unless SO_SNDTIMEO works diff --git a/include/tbl/params.h b/include/tbl/params.h index 5658f4b..d0f689c 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -697,6 +697,15 @@ PARAM( /* func */ NULL ) +#if defined(XYZZY) + #error "Temporary macro XYZZY already defined" +#endif + +#if defined(SO_SNDTIMEO_WORKS) + #define XYZZY DELAYED_EFFECT +#else + #define XYZZY NOT_IMPLEMENTED +#endif PARAM( /* name */ idle_send_timeout, /* typ */ timeout, @@ -704,7 +713,7 @@ PARAM( /* max */ NULL, /* default */ "60.000", /* units */ "seconds", - /* flags */ DELAYED_EFFECT, + /* flags */ XYZZY, /* s-text */ "Time to wait with no data sent. If no data has been transmitted " "in this many\n" @@ -713,6 +722,7 @@ PARAM( /* l-text */ "", /* func */ NULL ) +#undef XYZZY PARAM( /* name */ listen_depth, @@ -952,6 +962,15 @@ PARAM( /* func */ NULL ) +#if defined(XYZZY) + #error "Temporary macro XYZZY already defined" +#endif + +#if defined(SO_SNDTIMEO_WORKS) + #define XYZZY DELAYED_EFFECT +#else + #define XYZZY NOT_IMPLEMENTED +#endif PARAM( /* name */ send_timeout, /* typ */ timeout, @@ -959,7 +978,7 @@ PARAM( /* max */ NULL, /* default */ "600.000", /* units */ "seconds", - /* flags */ DELAYED_EFFECT, + /* flags */ XYZZY, /* s-text */ "Send timeout for client connections. If the HTTP response hasn't " "been transmitted in this many\n" @@ -968,6 +987,7 @@ PARAM( /* l-text */ "", /* func */ NULL ) +#undef XYZZY #if 0 /* actual location mgt_param_tbl.c */ From nils.goroll at uplex.de Fri May 12 12:53:05 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 12 May 2017 14:53:05 +0200 Subject: [master] 637bdca document that timeout_idle has no effect unless SO_RCVTIMEO works Message-ID: commit 637bdca0d30637115ffbf00fc326357f0eb61717 Author: Nils Goroll Date: Fri May 12 12:07:40 2017 +0200 document that timeout_idle has no effect unless SO_RCVTIMEO works diff --git a/include/tbl/params.h b/include/tbl/params.h index d0f689c..2dee510 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1355,6 +1355,15 @@ PARAM( ) #endif +#if defined(XYZZY) + #error "Temporary macro XYZZY already defined" +#endif + +#if defined(SO_RCVTIMEO_WORKS) + #define XYZZY 0 +#else + #define XYZZY NOT_IMPLEMENTED +#endif PARAM( /* name */ timeout_idle, /* typ */ timeout, @@ -1362,7 +1371,7 @@ PARAM( /* max */ NULL, /* default */ "5.000", /* units */ "seconds", - /* flags */ 0, + /* flags */ XYZZY, /* s-text */ "Idle timeout for client connections.\n" "A connection is considered idle, until we have received the full " @@ -1370,6 +1379,7 @@ PARAM( /* l-text */ "", /* func */ NULL ) +#undef XYZZY PARAM( /* name */ timeout_linger, From nils.goroll at uplex.de Fri May 12 12:53:05 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 12 May 2017 14:53:05 +0200 Subject: [master] 1469881 Improve general documentation on parameters and link to the list of them Message-ID: commit 146988123a466f6d06c64e4894354886fa715342 Author: Nils Goroll Date: Fri May 12 12:55:08 2017 +0200 Improve general documentation on parameters and link to the list of them diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 3b34004..931fbb9 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -421,6 +421,8 @@ relative to the values listed below, in order to conserve VM space: * gzip_stack_buffer: 4k * thread_pool_stack: 64k +.. _List of Parameters: + List of Parameters ------------------ diff --git a/doc/sphinx/users-guide/params.rst b/doc/sphinx/users-guide/params.rst index 5422c98..3fc5cd2 100644 --- a/doc/sphinx/users-guide/params.rst +++ b/doc/sphinx/users-guide/params.rst @@ -4,26 +4,13 @@ Parameters ---------- Varnish Cache comes with a set of parameters that affects behaviour and -performance. Most of these parameters can be set on the Varnish -command line (through `varnishadm`) using the ``param.set`` keyword. - -Some parameters can, for security purposes be read only using the '-r' -command line switch to `varnishd`. +performance. Parameters are set either though command line +arguments to ``varnishd`` or at runtime through ``varnishadm`` using +the ``param.set`` CLI command. We don't recommend that you tweak parameters unless you're sure of what you're doing. We've worked hard to make the defaults sane and Varnish should be able to handle most workloads with the default settings. -For a complete listing of all the parameters and a short descriptions -type ``param.show`` in the CLI. To inspect a certain parameter and get -a somewhat longer description on what it does and what the default is -type ``param.show`` and the name of the parameter, like this:: - - varnish> param.show shortlived - 200 - shortlived 10.000000 [s] - Default is 10.0 - Objects created with TTL shorter than this are - always put in transient storage. - - +For a complete listing of all parameters and their specifics see +:ref:`List of Parameters`. From nils.goroll at uplex.de Fri May 12 12:53:05 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 12 May 2017 14:53:05 +0200 Subject: [master] 97d2c57 sync idle_send_timeout and send_timeout documentation with reality Message-ID: commit 97d2c575521fc75a7e63c05545658131b300aece Author: Nils Goroll Date: Fri May 12 13:03:39 2017 +0200 sync idle_send_timeout and send_timeout documentation with reality The name idle_send_timeout does not seem to describe well where the timeout applies (individual write()/writev() calls for arbitraty amounts of data). The fact that the explanation is complicated might be an indication that a different timeout implementation be more appropriate. HTTP2 should probably implement send_timeout somewhen. diff --git a/include/tbl/params.h b/include/tbl/params.h index 2dee510..870ba41 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -715,10 +715,11 @@ PARAM( /* units */ "seconds", /* flags */ XYZZY, /* s-text */ - "Time to wait with no data sent. If no data has been transmitted " - "in this many\n" - "seconds the session is closed.\n" - "See setsockopt(2) under SO_SNDTIMEO for more information.", + "Send timeout for individual pieces of data on client connections." + " May get extended if 'send_timeout' applies.\n\n" + "When this timeout is hit, the session is closed.\n\n" + "See the man page for `setsockopt(2)` under ``SO_SNDTIMEO`` for more" + " information.", /* l-text */ "", /* func */ NULL ) @@ -980,10 +981,12 @@ PARAM( /* units */ "seconds", /* flags */ XYZZY, /* s-text */ - "Send timeout for client connections. If the HTTP response hasn't " - "been transmitted in this many\n" - "seconds the session is closed.\n" - "See setsockopt(2) under SO_SNDTIMEO for more information.", + "Total timeout for ordinary HTTP1 responses. Does not apply to some" + " internally generated errors and pipe mode.\n\n" + "When 'send_idle_timeout' is hit while sending an HTTP1 response, the" + " timeout is extended unless the total time already taken for sending" + " the response in its entirety exceeds this many seconds.\n\n" + "When this timeout is hit, the session is closed", /* l-text */ "", /* func */ NULL ) From nils.goroll at uplex.de Fri May 12 12:53:05 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 12 May 2017 14:53:05 +0200 Subject: [master] 39e20ba improve timeout_idle documentation and formatting Message-ID: commit 39e20baf91970e727a056072918691f1f82690b0 Author: Nils Goroll Date: Fri May 12 14:50:30 2017 +0200 improve timeout_idle documentation and formatting diff --git a/include/tbl/params.h b/include/tbl/params.h index 870ba41..7511f5a 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1376,9 +1376,12 @@ PARAM( /* units */ "seconds", /* flags */ XYZZY, /* s-text */ - "Idle timeout for client connections.\n" - "A connection is considered idle, until we have received the full " - "request headers.", + "Idle timeout for client connections.\n\n" + "A connection is considered idle until we have received the full" + " request headers.\n\n" + "This parameter is particularly relevant for HTTP1 keepalive " + " connections which are closed unless the next request is received" + " before this timeout is reached.", /* l-text */ "", /* func */ NULL ) From nils.goroll at uplex.de Fri May 12 13:01:05 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 12 May 2017 15:01:05 +0200 Subject: [master] f211c5c yes, the parameter names are confusing Message-ID: commit f211c5cc37aebf37e4b99fbad301b83d9a81d305 Author: Nils Goroll Date: Fri May 12 15:00:11 2017 +0200 yes, the parameter names are confusing diff --git a/include/tbl/params.h b/include/tbl/params.h index 7511f5a..fc25054 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -983,7 +983,7 @@ PARAM( /* s-text */ "Total timeout for ordinary HTTP1 responses. Does not apply to some" " internally generated errors and pipe mode.\n\n" - "When 'send_idle_timeout' is hit while sending an HTTP1 response, the" + "When 'idle_send_timeout' is hit while sending an HTTP1 response, the" " timeout is extended unless the total time already taken for sending" " the response in its entirety exceeds this many seconds.\n\n" "When this timeout is hit, the session is closed", From dridi.boukelmoune at gmail.com Fri May 12 18:53:05 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 12 May 2017 20:53:05 +0200 Subject: [master] 94b7087 Don't assert on vtc_error Message-ID: commit 94b70875d1bd90194ec13e6adc29d329d14a02d4 Author: Dridi Boukelmoune Date: Fri May 12 20:43:41 2017 +0200 Don't assert on vtc_error Reported by Coverity: ID 1406102: Incorrect expression (ASSERT_SIDE_EFFECT) /bin/varnishtest/vtc_http2.c: 852 in receive_frame() 846 s->wf = 0; 847 AZ(pthread_cond_signal(&s->cond)); 848 } 849 continue; 850 } 851 AZ(pthread_mutex_unlock(&hp->mtx)); >>> CID 1406102: Incorrect expression (ASSERT_SIDE_EFFECT) >>> Argument "vtc_error" of assert() has a side effect because the >>> variable is volatile. The containing function might work >>> differently in a non-debug build. 852 assert(vtc_error || vsb == NULL); 853 return (NULL); 854 } 855 856 #define STRTOU32(n, ss, p, v, c) \ 857 do { \ Although I'm not sure how relevant it is since assert() from vas.h doesn't remove side effects when no-op'd. diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 31591ae..4fcd243 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -849,7 +849,8 @@ receive_frame(void *priv) continue; } AZ(pthread_mutex_unlock(&hp->mtx)); - assert(vtc_error || vsb == NULL); + if (!vtc_error) + AZ(vsb); return (NULL); } From fgsch at lodoss.net Sat May 13 13:01:06 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 13 May 2017 15:01:06 +0200 Subject: [master] 10dafbc Plug harmless leak Message-ID: commit 10dafbc6c2154128eb426419cb39c4f73e7d4014 Author: Federico G. Schwindt Date: Sat May 13 13:43:16 2017 +0100 Plug harmless leak Spotted by coverity. diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c index 18c7b6c..da5f90f 100644 --- a/lib/libvarnishapi/vsm.c +++ b/lib/libvarnishapi/vsm.c @@ -152,7 +152,10 @@ VSM_n_Arg(struct VSM_data *vd, const char *arg) REPLACE(vd->name, name); REPLACE(vd->dname, dname); REPLACE(vd->iname, VSB_data(vsb)); + VSB_destroy(&vsb); + free(name); + free(dname); return (1); } From fgsch at lodoss.net Sat May 13 13:01:06 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 13 May 2017 15:01:06 +0200 Subject: [master] b4fc49a On error free any allocated resources Message-ID: commit b4fc49a92a67480b763da21fd2b0087357b0fac4 Author: Federico G. Schwindt Date: Sat May 13 13:33:41 2017 +0100 On error free any allocated resources Spotted by coverity. diff --git a/lib/libvarnish/vin.c b/lib/libvarnish/vin.c index 1cb064c..9e47867 100644 --- a/lib/libvarnish/vin.c +++ b/lib/libvarnish/vin.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -89,8 +90,10 @@ VIN_n_Arg(const char *n_arg, char **name, char **dir) } if (dir != NULL) { *dir = strdup(dn); - if (*dir == NULL) + if (*dir == NULL) { + free(*name); return (-1); + } } return (0); } From fgsch at lodoss.net Sat May 13 13:01:06 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 13 May 2017 15:01:06 +0200 Subject: [master] 3bec8f4 Follow existing code for consistency Message-ID: commit 3bec8f4700327dc95e9958fdfb62c29fdb659b9c Author: Federico G. Schwindt Date: Sat May 13 13:43:47 2017 +0100 Follow existing code for consistency diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index c3b8a24..99add7b 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -64,6 +64,7 @@ struct vsc_vf { struct VSC_section section; int order; }; +VTAILQ_HEAD(vsc_vf_head, vsc_vf); struct vsc_pt { unsigned magic; @@ -85,7 +86,7 @@ struct vsc { unsigned magic; #define VSC_MAGIC 0x3373554a - VTAILQ_HEAD(, vsc_vf) vf_list; + struct vsc_vf_head vf_list; struct vsc_pt_head pt_list; struct vsc_sf_head sf_list_include; struct vsc_sf_head sf_list_exclude; @@ -115,14 +116,14 @@ vsc_setup(struct VSM_data *vd) /*--------------------------------------------------------------------*/ static void -vsc_delete_vf_list(struct vsc *vsc) +vsc_delete_vf_list(struct vsc_vf_head *head) { struct vsc_vf *vf; - while (!VTAILQ_EMPTY(&vsc->vf_list)) { - vf = VTAILQ_FIRST(&vsc->vf_list); + while (!VTAILQ_EMPTY(head)) { + vf = VTAILQ_FIRST(head); CHECK_OBJ_NOTNULL(vf, VSC_VF_MAGIC); - VTAILQ_REMOVE(&vsc->vf_list, vf, list); + VTAILQ_REMOVE(head, vf, list); FREE_OBJ(vf); } } @@ -166,7 +167,7 @@ VSC_Delete(struct VSM_data *vd) vsc_delete_sf_list(&vsc->sf_list_include); vsc_delete_sf_list(&vsc->sf_list_exclude); vsc_delete_pt_list(&vsc->pt_list); - vsc_delete_vf_list(vsc); + vsc_delete_vf_list(&vsc->vf_list); FREE_OBJ(vsc); } @@ -320,7 +321,7 @@ vsc_build_vf_list(struct VSM_data *vd) struct vsc *vsc = vsc_setup(vd); vsc_delete_pt_list(&vsc->pt_list); - vsc_delete_vf_list(vsc); + vsc_delete_vf_list(&vsc->vf_list); VSM_FOREACH(&vsc->iter_fantom, vd) { if (strcmp(vsc->iter_fantom.class, VSC_CLASS)) From fgsch at lodoss.net Sat May 13 16:13:05 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 13 May 2017 18:13:05 +0200 Subject: [master] 8293fb5 Bump xcode version Message-ID: commit 8293fb534379e90a685cfb42eddceb236dc086d6 Author: Federico G. Schwindt Date: Sat May 13 17:10:00 2017 +0100 Bump xcode version diff --git a/.travis.yml b/.travis.yml index 732d895..9fd348d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ matrix: sudo: false compiler: clang - os: osx - osx_image: xcode8.2 + osx_image: xcode8.3 compiler: clang allow_failures: - os: osx From phk at FreeBSD.org Mon May 15 09:45:07 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 May 2017 11:45:07 +0200 Subject: [master] f3244fd GC a lot of #includes Message-ID: commit f3244fda73750b6702b771af50192c081b16472a Author: Poul-Henning Kamp Date: Mon May 15 09:44:00 2017 +0000 GC a lot of #includes diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index 1c046b2..efca881 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -58,7 +58,6 @@ #include #include -#include "vapi/vsl.h" #include "vapi/vsm.h" #include "vas.h" #include "vcli.h" diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 52de48f..c3cfd8c 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -43,9 +43,6 @@ #include #include -#include -#include - #include "binary_heap.h" #include "vcli_serve.h" #include "vrt.h" diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index b2d013c..46a5d7d 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -32,7 +32,6 @@ #include "cache.h" -#include #include #include "cache_transport.h" diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 4dcedd5..cf7b708 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -87,7 +87,6 @@ #include "cache_obj.h" #include "vend.h" #include "storage/storage.h" -#include "hash/hash_slinger.h" static const struct obj_methods * obj_getmethods(const struct objcore *oc) diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index 57b7e8d..671ceca 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -35,7 +35,6 @@ #include "config.h" -#include #include #include "cache.h" diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index a3499ca..601574a 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -34,7 +34,6 @@ #include "cache.h" -#include #include #include diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index d0380a8..5c7a7b1 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -32,7 +32,6 @@ #include "cache.h" #include "common/heritage.h" -#include "hash/hash_slinger.h" #include "cache_director.h" #include "vrt.h" diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c index 97a18b2..65e7074 100644 --- a/bin/varnishd/cache/cache_vrt_vmod.c +++ b/bin/varnishd/cache/cache_vrt_vmod.c @@ -34,11 +34,8 @@ #include "cache.h" #include -#include -#include #include #include -#include #include "vcli_serve.h" #include "vrt.h" diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c index c064f73..9cd6a5c 100644 --- a/bin/varnishd/common/common_vsc.c +++ b/bin/varnishd/common/common_vsc.c @@ -30,8 +30,6 @@ #include "config.h" -#include "common.h" - /* Build the static vsc type names */ #define VSC_TYPE_F(n,t,l,e,d) const char *VSC_type_##n = t; #include "tbl/vsc_types.h" diff --git a/bin/varnishd/common/common_vsm.c b/bin/varnishd/common/common_vsm.c index 22ae5c3..7994d95 100644 --- a/bin/varnishd/common/common_vsm.c +++ b/bin/varnishd/common/common_vsm.c @@ -34,7 +34,6 @@ #include "config.h" -#include #include #include #include diff --git a/bin/varnishd/hash/mgt_hash.c b/bin/varnishd/hash/mgt_hash.c index 6e772b2..fc715c0 100644 --- a/bin/varnishd/hash/mgt_hash.c +++ b/bin/varnishd/hash/mgt_hash.c @@ -30,7 +30,6 @@ #include "config.h" -#include #include #include #include diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index 5d3933a..e5032eb 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -35,12 +35,9 @@ #include #include -#include "vrt.h" #include "vtcp.h" #include "vtim.h" -#include "hash/hash_slinger.h" - #include "cache_http1.h" /*-------------------------------------------------------------------- diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c index bd68ae8..ceaf4c8 100644 --- a/bin/varnishd/http1/cache_http1_fsm.c +++ b/bin/varnishd/http1/cache_http1_fsm.c @@ -39,7 +39,6 @@ #include #include -#include "cache/cache_filter.h" #include "cache/cache_transport.h" #include "cache_http1.h" #include "hash/hash_slinger.h" diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c index 678a2b1..b6a2e30 100644 --- a/bin/varnishd/http1/cache_http1_pipe.c +++ b/bin/varnishd/http1/cache_http1_pipe.c @@ -36,8 +36,6 @@ #include #include -#include "vrt.h" - #include "cache_http1.h" static struct lock pipestat_mtx; diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index dc3ac30..0e04a84 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -34,10 +34,8 @@ #include #include "cache/cache_transport.h" -#include "cache/cache_filter.h" #include "http2/cache_http2.h" -#include "vend.h" #include "vtim.h" static const char h2_resp_101[] = diff --git a/bin/varnishd/mgt/mgt_acceptor.c b/bin/varnishd/mgt/mgt_acceptor.c index f9926f4..38be706 100644 --- a/bin/varnishd/mgt/mgt_acceptor.c +++ b/bin/varnishd/mgt/mgt_acceptor.c @@ -34,7 +34,6 @@ #include #include #include -#include #include #include #include diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 338ca53..ed398b7 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -31,13 +31,13 @@ #include "config.h" -#include #include #include #include #include #include +#include #include #include #include diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 4f9b324..1a50c51 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -31,7 +31,6 @@ #include "config.h" -#include #include #include #include @@ -53,7 +52,6 @@ #include "vin.h" #include "vpf.h" #include "vrnd.h" -#include "vsb.h" #include "vsha256.h" #include "vsub.h" #include "vtim.h" diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c index 89b6005..b2edd05 100644 --- a/bin/varnishd/mgt/mgt_param_bits.c +++ b/bin/varnishd/mgt/mgt_param_bits.c @@ -29,7 +29,6 @@ #include "config.h" -#include #include #include "mgt/mgt.h" diff --git a/bin/varnishd/mgt/mgt_param_tcp.c b/bin/varnishd/mgt/mgt_param_tcp.c index 9567d6f..1519d9d 100644 --- a/bin/varnishd/mgt/mgt_param_tcp.c +++ b/bin/varnishd/mgt/mgt_param_tcp.c @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c index b26581e..0f95237 100644 --- a/bin/varnishd/mgt/mgt_param_tweak.c +++ b/bin/varnishd/mgt/mgt_param_tweak.c @@ -36,7 +36,6 @@ #include #include #include -#include #include "mgt/mgt.h" diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c index 858d871..771080f 100644 --- a/bin/varnishd/mgt/mgt_shmem.c +++ b/bin/varnishd/mgt/mgt_shmem.c @@ -31,7 +31,6 @@ #include "config.h" #include -#include #include #include diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index d2fcdb1..00f1e6f 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -31,7 +31,6 @@ #include "config.h" -#include #include #include #include diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index d4231bc..0f6b691 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -34,7 +34,6 @@ #include #include -#include #include "cache/cache_transport.h" diff --git a/bin/varnishd/storage/mgt_stevedore.c b/bin/varnishd/storage/mgt_stevedore.c index da3f4a1..b05ecf8 100644 --- a/bin/varnishd/storage/mgt_stevedore.c +++ b/bin/varnishd/storage/mgt_stevedore.c @@ -33,7 +33,6 @@ #include "config.h" -#include #include #include #include diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 48a3c7b..f8253cc 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -38,8 +38,6 @@ #include #include -#include "hash/hash_slinger.h" - #include "storage/storage.h" #include "vrt.h" #include "vrt_obj.h" diff --git a/bin/varnishd/storage/stevedore_utils.c b/bin/varnishd/storage/stevedore_utils.c index 6225112..d6ea9a4 100644 --- a/bin/varnishd/storage/stevedore_utils.c +++ b/bin/varnishd/storage/stevedore_utils.c @@ -31,12 +31,10 @@ #include "config.h" -#include #include #include #include -#include #include #include #include diff --git a/bin/varnishd/storage/storage_lru.c b/bin/varnishd/storage/storage_lru.c index f3588b1..e7f0bff 100644 --- a/bin/varnishd/storage/storage_lru.c +++ b/bin/varnishd/storage/storage_lru.c @@ -33,7 +33,6 @@ #include #include "cache/cache.h" -#include "cache/cache_obj.h" #include "hash/hash_slinger.h" #include "storage/storage.h" diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index 9bdf536..9d2cc92 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -37,7 +37,6 @@ #include "cache/cache.h" -#include #include #include diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c index 5061188..780df0d 100644 --- a/bin/varnishd/storage/storage_persistent_silo.c +++ b/bin/varnishd/storage/storage_persistent_silo.c @@ -39,7 +39,6 @@ #include #include -#include "cache/cache_obj.h" #include "storage/storage.h" #include "storage/storage_simple.h" diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c index aaf2c48..ad33b7e 100644 --- a/bin/varnishd/storage/storage_simple.c +++ b/bin/varnishd/storage/storage_simple.c @@ -31,9 +31,6 @@ #include "cache/cache.h" -#include -#include - #include "cache/cache_obj.h" #include "hash/hash_slinger.h" diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c index 28f50b1..3d4b17d 100644 --- a/bin/varnishd/waiter/cache_waiter.c +++ b/bin/varnishd/waiter/cache_waiter.c @@ -32,7 +32,6 @@ #include "cache/cache.h" -#include #include #include "binary_heap.h" diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c index 23684b6..ffbf947 100644 --- a/bin/varnishd/waiter/cache_waiter_epoll.c +++ b/bin/varnishd/waiter/cache_waiter_epoll.c @@ -37,13 +37,11 @@ #include -#include #include #include "cache/cache.h" #include "waiter/waiter_priv.h" -#include "waiter/mgt_waiter.h" #include "vtim.h" #ifndef EPOLLRDHUP diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c index 19fde5d..7d84c03 100644 --- a/bin/varnishd/waiter/cache_waiter_poll.c +++ b/bin/varnishd/waiter/cache_waiter_poll.c @@ -37,7 +37,6 @@ #include "cache/cache.h" #include "waiter/waiter_priv.h" -#include "waiter/mgt_waiter.h" #include "vtim.h" struct vwp { diff --git a/bin/varnishd/waiter/mgt_waiter.c b/bin/varnishd/waiter/mgt_waiter.c index 3010d86..01780d3 100644 --- a/bin/varnishd/waiter/mgt_waiter.c +++ b/bin/varnishd/waiter/mgt_waiter.c @@ -29,9 +29,7 @@ */ #include "config.h" -#include #include -#include #include "mgt/mgt.h" #include "waiter/mgt_waiter.h" diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index 51c5038..d82b1ce 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -33,15 +33,11 @@ #include "config.h" -#include - #include #include #include #include -#include #include -#include #include #include #include diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c index b4739e5..87f98a3 100644 --- a/bin/varnishlog/varnishlog.c +++ b/bin/varnishlog/varnishlog.c @@ -37,16 +37,12 @@ #include #include #include -#include #include -#include -#include "vapi/vsm.h" #include "vapi/vsl.h" #include "vapi/voptget.h" #include "vas.h" #include "vdef.h" -#include "vpf.h" #include "vut.h" #include "miniobj.h" diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 1629aa1..dc64f4b 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -54,7 +54,6 @@ #include #include "base64.h" -#include "vapi/vsm.h" #include "vapi/vsl.h" #include "vapi/voptget.h" #include "vas.h" diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c index 7650bf1..5dfabb9 100644 --- a/bin/varnishstat/varnishstat.c +++ b/bin/varnishstat/varnishstat.c @@ -32,14 +32,12 @@ #include "config.h" -#include #include #include #include #include #include #include -#include #include "vapi/voptget.h" #include "vapi/vsl.h" diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index 586ee18..59bdb32 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -35,12 +35,9 @@ #include "config.h" #include -#include #include #include #include -#include -#include #include #include #include @@ -49,8 +46,6 @@ #include "vas.h" #include "miniobj.h" #include "vqueue.h" -#include "vapi/vsm.h" -#include "vapi/vsc.h" #include "vtim.h" #include "varnishstat.h" diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 0934661..feadd9f 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -28,7 +28,6 @@ #include "config.h" -#include #include #include @@ -36,9 +35,7 @@ #include #include #include -#include #include -#include #include #include #include diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index 8cd80d9..0280211 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -29,14 +29,11 @@ #include "config.h" #include -#include #include #include #include #include -#include -#include #include #include "vtc.h" diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index 7f317ac..4515282 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -28,7 +28,6 @@ #include "config.h" -#include #include #include diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 22c00bd..7645faa 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -28,7 +28,6 @@ #include "config.h" -#include #include #include diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 4fcd243..c819cdd 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -38,14 +38,12 @@ #include #include #include -#include #include #include "vtc.h" #include "vtc_http.h" #include "vfil.h" -#include "vgz.h" #include "hpack.h" #include "vend.h" diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index e872807..975f0b8 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -30,13 +30,10 @@ #include #include -#include -#include #include #include #include -#include #include #include #include @@ -49,7 +46,6 @@ #include "vev.h" #include "vfil.h" #include "vnum.h" -#include "vqueue.h" #include "vrnd.h" #include "vss.h" #include "vsub.h" diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index eaf8d02..f3b9943 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -29,13 +29,10 @@ #include "config.h" #include -#include -#include #include #include #include -#include #include #include #include diff --git a/bin/varnishtest/vtc_proxy.c b/bin/varnishtest/vtc_proxy.c index b1a6b05..03e186b 100644 --- a/bin/varnishtest/vtc_proxy.c +++ b/bin/varnishtest/vtc_proxy.c @@ -28,7 +28,6 @@ #include "config.h" -#include #include #include diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 13dd336..fb811df 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -28,7 +28,6 @@ #include "config.h" -#include #include #include diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c index c7d48ff..1911dd3 100644 --- a/bin/varnishtop/varnishtop.c +++ b/bin/varnishtop/varnishtop.c @@ -35,17 +35,14 @@ #include #include -#include #include #include #include -#include #include #include #include #include "vcurses.h" -#include "vapi/vsm.h" #include "vapi/vsl.h" #include "vapi/voptget.h" #include "vas.h" diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 251b420..1f8af2a 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -30,16 +30,11 @@ #include "config.h" -#include #include -#include - #include -#include #include #include -#include #include "vas.h" #include "vsa.h" diff --git a/lib/libvarnish/vsub.c b/lib/libvarnish/vsub.c index ea86699..9abf3f7 100644 --- a/lib/libvarnish/vsub.c +++ b/lib/libvarnish/vsub.c @@ -34,10 +34,7 @@ #include #include -#include -#include #include -#include #include #include diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 9122f3e..941a90b 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -32,7 +32,6 @@ #include #include #include -#include #ifdef HAVE_SYS_FILIO_H # include #endif @@ -45,7 +44,6 @@ #include #include #include -#include #include #include diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index 5e71a20..b2d47dd 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -61,7 +61,6 @@ #include #include #include -#include #ifdef __MACH__ #include #endif diff --git a/lib/libvarnishapi/daemon.c b/lib/libvarnishapi/daemon.c index feb7924..1d253c7 100644 --- a/lib/libvarnishapi/daemon.c +++ b/lib/libvarnishapi/daemon.c @@ -32,11 +32,7 @@ //lint -e{766} #include "config.h" -#include -#include #include -#include -#include #include "compat/daemon.h" #include "vdef.h" diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 99add7b..725a5f7 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -30,12 +30,10 @@ #include "config.h" -#include #include #include #include -#include #include #include diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index 127a4bb..3a879d3 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -30,16 +30,10 @@ #include "config.h" -#include -#include - #include #include #include #include -#include -#include -#include #include "vdef.h" #include "vas.h" @@ -51,7 +45,6 @@ #include "vre.h" #include "vsb.h" -#include "vapi/vsm.h" #include "vapi/vsl.h" #include "vsl_api.h" diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c index 9c8032c..829dbc3 100644 --- a/lib/libvarnishapi/vsl_arg.c +++ b/lib/libvarnishapi/vsl_arg.c @@ -30,18 +30,11 @@ #include "config.h" -#include -#include - #include -#include #include #include -#include #include #include -#include -#include #include "vdef.h" #include "vas.h" diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index a19ce5d..67d3cb0 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -35,7 +35,6 @@ #include #include #include -#include #include "vdef.h" #include "vas.h" diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c index b2d43a8..42596af 100644 --- a/lib/libvarnishapi/vsl_query.c +++ b/lib/libvarnishapi/vsl_query.c @@ -32,12 +32,7 @@ #include #include -#include #include -#include -#include -#include -#include #include "vdef.h" #include "vas.h" diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c index da5f90f..83bab97 100644 --- a/lib/libvarnishapi/vsm.c +++ b/lib/libvarnishapi/vsm.c @@ -32,13 +32,11 @@ #include #include -#include #include #include #include #include -#include #include #include #include diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index 9079b22..83a08da 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -41,7 +41,6 @@ #include #include #include -#include #include "compat/daemon.h" #include "vdef.h" diff --git a/lib/libvarnishapi/vxp.c b/lib/libvarnishapi/vxp.c index 558a99d..ae86282 100644 --- a/lib/libvarnishapi/vxp.c +++ b/lib/libvarnishapi/vxp.c @@ -31,7 +31,6 @@ #include "config.h" #include -#include #include #include diff --git a/lib/libvarnishapi/vxp_lexer.c b/lib/libvarnishapi/vxp_lexer.c index d858d40..0848ee3 100644 --- a/lib/libvarnishapi/vxp_lexer.c +++ b/lib/libvarnishapi/vxp_lexer.c @@ -31,7 +31,6 @@ #include "config.h" #include -#include #include #include diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c index 201d03a..a37faec 100644 --- a/lib/libvarnishapi/vxp_parse.c +++ b/lib/libvarnishapi/vxp_parse.c @@ -32,10 +32,7 @@ #include #include -#include #include -#include -#include #include "vdef.h" #include "vas.h" diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index b54f5ad..c98dfea 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -29,15 +29,11 @@ #include "config.h" -#include - -#include #include #include #include -#include #include #include diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 16a29d1..1f9b0eb 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -52,10 +52,8 @@ #include "config.h" -#include #include #include -#include #include #include diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index fd7daf1..5de8a2a 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -28,8 +28,6 @@ * */ -#include - #include #include diff --git a/lib/libvcc/vcc_types.c b/lib/libvcc/vcc_types.c index 4f0a5c4..af41455 100644 --- a/lib/libvcc/vcc_types.c +++ b/lib/libvcc/vcc_types.c @@ -30,10 +30,6 @@ #include "config.h" -#include -#include -#include -#include #include #include "vcc_compile.h" diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index 89ccccc..7d464c5 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -32,13 +32,11 @@ #include #include #include -#include #include #include "vcc_compile.h" #include "vre.h" -#include "vrt.h" #include "vsa.h" #include "vss.h" #include "vtcp.h" diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c index 00fc022..9e4c8cb 100644 --- a/lib/libvcc/vcc_var.c +++ b/lib/libvcc/vcc_var.c @@ -29,9 +29,6 @@ #include "config.h" -#include -#include - #include "vcc_compile.h" #include "vct.h" diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 717982e..28cfe2f 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -29,14 +29,12 @@ #include "config.h" #include -#include #include #include #include #include -#include #include "cache/cache.h" From phk at FreeBSD.org Mon May 15 10:02:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 May 2017 12:02:06 +0200 Subject: [master] c2ecb4d Add back #include for solaris Message-ID: commit c2ecb4deee2ed2c075b8ce965077ed13079c9c37 Author: Poul-Henning Kamp Date: Mon May 15 10:01:45 2017 +0000 Add back #include for solaris diff --git a/lib/libvarnish/vsub.c b/lib/libvarnish/vsub.c index 9abf3f7..e3f7453 100644 --- a/lib/libvarnish/vsub.c +++ b/lib/libvarnish/vsub.c @@ -35,6 +35,7 @@ #include #include +#include // Solaris closefrom(3c) #include #include From dridi.boukelmoune at gmail.com Mon May 15 15:51:05 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 15 May 2017 17:51:05 +0200 Subject: [master] 4c9b8d8 Only root can jail Message-ID: commit 4c9b8d8913eaf48e60efa42ad0eb80f5de0fbf0b Author: Dridi Boukelmoune Date: Mon May 15 17:46:46 2017 +0200 Only root can jail Fixes #2329 diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 931fbb9..56a90c4 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -309,8 +309,8 @@ specific options. Available jails are: -j - Default on all other platforms when `varnishd` is either started with - an effective uid of 0 ("as root") or as user ``varnish``. + Default on all other platforms when `varnishd` is started with an + effective uid of 0 ("as root"). With the ``unix`` jail mechanism activated, varnish will switch to an alternative user for subprocesses and change the effective uid of From phk at FreeBSD.org Mon May 15 20:00:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 May 2017 22:00:06 +0200 Subject: [master] ec5eba3 A little too aggressive on the #include removal here. Message-ID: commit ec5eba3fafe2fb9bb9c3c49adc44cddb12af5f51 Author: Poul-Henning Kamp Date: Mon May 15 19:56:34 2017 +0000 A little too aggressive on the #include removal here. diff --git a/lib/libvarnishapi/daemon.c b/lib/libvarnishapi/daemon.c index 1d253c7..bd72351 100644 --- a/lib/libvarnishapi/daemon.c +++ b/lib/libvarnishapi/daemon.c @@ -32,7 +32,11 @@ //lint -e{766} #include "config.h" +#include +#include #include +#include +#include #include "compat/daemon.h" #include "vdef.h" From phk at FreeBSD.org Mon May 15 20:00:07 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 May 2017 22:00:07 +0200 Subject: [master] 37eb85b Don't derefrence NULL pointers Message-ID: commit 37eb85ba98cc8ae6d10f2f2218cfd639fff38a42 Author: Poul-Henning Kamp Date: Mon May 15 19:56:47 2017 +0000 Don't derefrence NULL pointers diff --git a/lib/libvarnish/vin.c b/lib/libvarnish/vin.c index 9e47867..fecc554 100644 --- a/lib/libvarnish/vin.c +++ b/lib/libvarnish/vin.c @@ -91,7 +91,8 @@ VIN_n_Arg(const char *n_arg, char **name, char **dir) if (dir != NULL) { *dir = strdup(dn); if (*dir == NULL) { - free(*name); + if (name != NULL) + free(*name); return (-1); } } From phk at FreeBSD.org Mon May 15 20:36:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 May 2017 22:36:05 +0200 Subject: [master] 6c0c10f Polish the VSC RST typography a bit. Message-ID: commit 6c0c10ffae90e7599e0db16542c4cb5816f67663 Author: Poul-Henning Kamp Date: Mon May 15 20:35:24 2017 +0000 Polish the VSC RST typography a bit. diff --git a/bin/varnishstat/vsc2rst.c b/bin/varnishstat/vsc2rst.c index 6446ccb..760f130 100644 --- a/bin/varnishstat/vsc2rst.c +++ b/bin/varnishstat/vsc2rst.c @@ -34,7 +34,7 @@ #include #define VSC_LEVEL_F(v,l,e,d) \ - static const char VSC_level_##v[] = l; + static const char VSC_level_##v[] = #v; #include "tbl/vsc_levels.h" #undef VSC_LEVEL_F @@ -61,8 +61,19 @@ int main(int argc, char **argv) #define VSC_DO(U,l,t,h) L(h); -#define VSC_F(n, t, l, s, f, v, d, e) \ - printf("%s ? %s (%s)\n\t%s\n\n", #n, d, VSC_level_##v, e); +#define VSC_F(n, t, l, s, f, v, d, e) \ + do { \ + printf("``%s`` ? ", #n); \ + if (s == 'c') \ + printf("`counter` - "); \ + else if (s == 'g') \ + printf("`gauge` - "); \ + else if (s == 'b') \ + printf("`bitmap` - "); \ + printf("%s\n\n", VSC_level_##v); \ + printf("\t%s\n\n", d); \ + printf("\t%s\n\n", e); \ + } while (0); #define VSC_DONE(U,l,t) From phk at FreeBSD.org Mon May 15 21:04:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 May 2017 23:04:05 +0200 Subject: [master] f5892a5 Un-neuter flexelint Message-ID: commit f5892a5fb7052ed71838f5d540a89a806bb51c33 Author: Poul-Henning Kamp Date: Mon May 15 21:03:46 2017 +0000 Un-neuter flexelint diff --git a/bin/varnishncsa/flint.lnt b/bin/varnishncsa/flint.lnt index ac808d4..5270ff2 100644 --- a/bin/varnishncsa/flint.lnt +++ b/bin/varnishncsa/flint.lnt @@ -1,25 +1,2 @@ -+d__attribute__()= - -efile(451, "varnishncsa_options.h") - --e763 // Redundant declaration for symbol '...' previously declared - -// cache_vcl.c --e641 // Converting enum 'cli_status_e' to int - -// Review all below this line /////////////////////////////////////////////// - --e767 // Macro redef (system queue.h vs ours ) - --e712 // Loss of precision (assignment) (long long to --e747 // Significant prototype coercion (arg. no. 2) long --e713 // Loss of precision (assignment) (unsigned long long to long long) --e506 // Constant value boolean --e818 // Pointer parameter '...' could be declared as pointing to const --e774 // Boolean within 'if' always evaluates to False --e534 // Ignoring return value of function --e557 // unrecog format --e732 // Loss of sign (arg. no. 2) (int to unsigned --e788 // tag not used in defaulted switch - From dridi.boukelmoune at gmail.com Mon May 15 21:22:05 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 15 May 2017 23:22:05 +0200 Subject: [master] 74dd224 No need to strdup the default varnishncsa format Message-ID: commit 74dd2249f4181869d74b0260e3b7c8fb231accbe Author: Dridi Boukelmoune Date: Mon May 15 23:20:43 2017 +0200 No need to strdup the default varnishncsa format diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index dc64f4b..04efbda 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -678,6 +678,9 @@ parse_format(const char *format) struct vsb *vsb; char buf[256]; + if (format == NULL) + format = FORMAT; + vsb = VSB_new_auto(); AN(vsb); @@ -1212,8 +1215,6 @@ main(int argc, char * const *argv) VSLQ_grouping[VUT.g_arg]); /* Prepare output format */ - if (format == NULL) - format = strdup(FORMAT); parse_format(format); free(format); format = NULL; From dridi.boukelmoune at gmail.com Tue May 16 09:19:05 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 16 May 2017 11:19:05 +0200 Subject: [master] 138fecb A quick word on thread_pool bounds Message-ID: commit 138fecbd777c76e5b5ee0eb78cb8dd84bfd9fee7 Author: Dridi Boukelmoune Date: Tue May 16 11:16:56 2017 +0200 A quick word on thread_pool bounds It's sometimes confusing new users, and it managed to confuse me too a couple months ago. diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index 95e667c..49b9b5e 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -99,7 +99,8 @@ struct parspec WRK_parspec[] = { "2", "pools" }, { "thread_pool_max", tweak_thread_pool_max, &mgt_param.wthread_max, NULL, NULL, - "The maximum number of worker threads in each pool.\n" + "The maximum number of worker threads in each pool. The " + "minimum value depends on thread_pool_min.\n" "\n" "Do not set this higher than you have to, since excess " "worker threads soak up RAM and CPU and generally just get " @@ -108,7 +109,8 @@ struct parspec WRK_parspec[] = { "5000", "threads" }, { "thread_pool_min", tweak_thread_pool_min, &mgt_param.wthread_min, NULL, NULL, - "The minimum number of worker threads in each pool.\n" + "The minimum number of worker threads in each pool. The " + "maximum value depends on thread_pool_max.\n" "\n" "Increasing this may help ramp up faster from low load " "situations or when threads have expired.\n" From Daphne at varnish-cache.org Tue May 16 10:17:33 2017 From: Daphne at varnish-cache.org (Daphne) Date: Tue, 16 May 2017 12:17:33 +0200 Subject: Emailing: 5243494.pdf Message-ID: <81799285844357ACE7C2A3DA87B04BAA@varnish-cache.org> The message is ready to be sent with the following file or link attachments: 5243494.pdf Note: To protect against computer viruses, e-mail programs may prevent sending or receiving certain types of file attachments. Check your e-mail security settings to determine how attachments are handled. -------------- next part -------------- An HTML attachment was scrubbed... URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: 5243494.pdf.pdf Type: application/pdf Size: 51796 bytes Desc: not available URL: From geoff at uplex.de Tue May 16 12:23:06 2017 From: geoff at uplex.de (Geoff Simmons) Date: Tue, 16 May 2017 14:23:06 +0200 Subject: [master] a339f63 Fix an off-by-one error in WS_Assert_Allocated(). Message-ID: commit a339f63b02b5373baea43f72e8acd577cd31627d Author: Geoff Simmons Date: Tue May 16 14:22:14 2017 +0200 Fix an off-by-one error in WS_Assert_Allocated(). diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index 13ad4f0..270aa16 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -81,7 +81,7 @@ WS_Assert_Allocated(const struct ws *ws, const void *ptr, ssize_t len) WS_Assert(ws); if (len < 0) len = strlen(p) + 1; - assert(p >= ws->s && (p + len) < ws->f); + assert(p >= ws->s && (p + len) <= ws->f); } /* From nils.goroll at uplex.de Tue May 16 17:34:12 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 16 May 2017 19:34:12 +0200 Subject: [master] 636585d TLC for the smartos install instructions Message-ID: commit 636585d401f7396ef473c64cd50e6d81ceb36e31 Author: Nils Goroll Date: Tue May 16 18:18:00 2017 +0200 TLC for the smartos install instructions Thank you to @RantyDave Closes #2326 diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst index cd1a65f..640cd3a 100644 --- a/doc/sphinx/installation/install.rst +++ b/doc/sphinx/installation/install.rst @@ -119,14 +119,14 @@ Optionally, to rebuild the svg files: Build dependencies on a SmartOS Zone ------------------------------------ -As of SmartOS pkgsrc 2015Q4, install the following packages:: +As of SmartOS pkgsrc 2017Q1, install the following packages:: pkgin in autoconf automake libedit libtool ncurses \ pcre py27-sphinx python27 gmake gcc49 pkg-config -Optionally, to rebuild the svg files: +Optionally, to rebuild the svg files:: - pkgin in graphviz + pkgin in graphviz Optionally, to pull from a repository:: @@ -138,6 +138,12 @@ Building on Solaris and other Solaris-ish OSes Building with gcc should be straight forward, as long as the above requirements are installed. +By convention, consider installing Varnish under `/opt/local` using:: + + ./configure \ + --prefix=/opt/local \ + --mandir=/opt/local/man + Alternatively, building with Solaris Studio 12.4 should work considering the following recommendations: @@ -146,6 +152,8 @@ considering the following recommendations: dependencies are installed. Example for `/opt/local`:: ./configure \ + --prefix=/opt/local \ + --mandir=/opt/local/man CPPFLAGS="-I/opt/local/include" \ CFLAGS="-m64" \ LDFLAGS="-L/opt/local/lib -R/opt/local/lib" From fgsch at lodoss.net Fri May 19 08:26:05 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 19 May 2017 10:26:05 +0200 Subject: [master] d7e81eb GC leftover from the past Message-ID: commit d7e81eb4f029c2eeabbe3e83b70b53075edf1d38 Author: Federico G. Schwindt Date: Thu May 18 22:10:09 2017 +0200 GC leftover from the past diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index 7d464c5..f0a62fe 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -55,7 +55,6 @@ vcc_regexp(struct vcc *tl) Expect(tl, CSTR); if (tl->err) return (NULL); - memset(&t, 0, sizeof t); t = VRE_compile(tl->t->dec, 0, &error, &erroroffset); if (t == NULL) { VSB_printf(tl->sb, From fgsch at lodoss.net Sat May 20 10:28:05 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 20 May 2017 12:28:05 +0200 Subject: [master] 77f4d84 Plug harmless memleak Message-ID: commit 77f4d84c39d810fdb2d66aac8faaf3d956711b4b Author: Federico G. Schwindt Date: Sat May 20 11:09:09 2017 +0100 Plug harmless memleak diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index c819cdd..d734a79 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -754,6 +754,7 @@ receive_frame(void *priv) AZ(pthread_mutex_lock(&hp->mtx)); VTAILQ_FOREACH(s, &hp->streams, list) AZ(pthread_cond_signal(&s->cond)); + clean_frame(&f); AZ(pthread_mutex_unlock(&hp->mtx)); vtc_log(hp->vl, hp->fatal, "could not get frame body"); From fgsch at lodoss.net Sat May 20 10:28:05 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 20 May 2017 12:28:05 +0200 Subject: [master] 17f8593 Remove dead assignments and whitespace Message-ID: commit 17f85938c0f283bee423a51058cb0a4334c85668 Author: Federico G. Schwindt Date: Sat May 20 11:09:19 2017 +0100 Remove dead assignments and whitespace diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 670dc8f..e82fe5c 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -518,7 +518,6 @@ vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo) AN(vfc->oc->flags & OC_F_PASS); VSLb(wrk->vsl, SLT_FetchError, "Pass delivery abandoned"); - vfps = VFP_END; bo->htc->doclose = SC_RX_BODY; break; } diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index d734a79..c79d886 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -359,30 +359,18 @@ explain_flags(uint8_t flags, uint8_t type, struct vtclog *vl) { if (flags & ACK && (type == TYPE_PING || type == TYPE_SETTINGS)) { vtc_log(vl, 3, "flag: ACK"); - flags &= ~ACK; - } else if (flags & END_STREAM && - (type == TYPE_HEADERS || - type == TYPE_PUSH_PROMISE || - type == TYPE_DATA)) { + } else if (flags & END_STREAM && (type == TYPE_HEADERS || + type == TYPE_PUSH_PROMISE || type == TYPE_DATA)) { vtc_log(vl, 3, "flag: END_STREAM"); - flags &= ~END_STREAM; - } else if (flags & END_HEADERS && - (type == TYPE_HEADERS || - type == TYPE_PUSH_PROMISE || - type == TYPE_CONTINUATION)) { + } else if (flags & END_HEADERS && (type == TYPE_HEADERS || + type == TYPE_PUSH_PROMISE || type == TYPE_CONTINUATION)) { vtc_log(vl, 3, "flag: END_TYPE_HEADERS"); - flags &= ~END_HEADERS; - } else if (flags & PRIORITY && - (type == TYPE_HEADERS || - type == TYPE_PUSH_PROMISE)) { + } else if (flags & PRIORITY && (type == TYPE_HEADERS || + type == TYPE_PUSH_PROMISE)) { vtc_log(vl, 3, "flag: END_PRIORITY"); - flags &= ~PRIORITY; - } else if (flags & PADDED && - (type == TYPE_DATA || - type == TYPE_HEADERS || - type == TYPE_PUSH_PROMISE)) { + } else if (flags & PADDED && (type == TYPE_DATA || type == + TYPE_HEADERS || type == TYPE_PUSH_PROMISE)) { vtc_log(vl, 3, "flag: PADDED"); - flags &= ~PADDED; } else if (flags) vtc_log(vl, 3, "UNKNOWN FLAG(S): 0x%02x", flags); } From fgsch at lodoss.net Sat May 20 10:28:05 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 20 May 2017 12:28:05 +0200 Subject: [master] d141e3a This only requires a single expression Message-ID: commit d141e3a8a3a7a3970ce9f34af5f27de7e4301432 Author: Federico G. Schwindt Date: Sat May 20 11:17:05 2017 +0100 This only requires a single expression diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 344b831..abd210e 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -433,7 +433,7 @@ vcc_Eval_Regsub(struct vcc *tl, struct expr **e, const struct symbol *sym, vcc_NextToken(tl); bprintf(buf, "VRT_regsub(ctx, %d,\v+\n\v1,\n%s", all, p); - *e = vcc_expr_edit(STRING, buf, e2, *e); + *e = vcc_expr_edit(STRING, buf, e2, NULL); SkipToken(tl, ','); vcc_expr0(tl, &e2, STRING); From phk at FreeBSD.org Sun May 21 09:46:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 21 May 2017 11:46:05 +0200 Subject: [master] 96b1a54 Neuter parts of libvgz which we will never use. Message-ID: commit 96b1a5471c6a681de6271c8ee94b7c4e30343fa3 Author: Poul-Henning Kamp Date: Sun May 21 09:45:26 2017 +0000 Neuter parts of libvgz which we will never use. diff --git a/lib/libvgz/deflate.c b/lib/libvgz/deflate.c index c796cb8..17c2016 100644 --- a/lib/libvgz/deflate.c +++ b/lib/libvgz/deflate.c @@ -82,8 +82,10 @@ local block_state deflate_fast OF((deflate_state *s, int flush)); #ifndef FASTEST local block_state deflate_slow OF((deflate_state *s, int flush)); #endif +#ifdef NOVGZ local block_state deflate_rle OF((deflate_state *s, int flush)); local block_state deflate_huff OF((deflate_state *s, int flush)); +#endif local void lm_init OF((deflate_state *s)); local void putShortMSB OF((deflate_state *s, uInt b)); local void flush_pending OF((z_streamp strm)); @@ -260,6 +262,9 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, * output size for (length,distance) codes is <= 24 bits. */ + if (strategy != Z_DEFAULT_STRATEGY) + return Z_STREAM_ERROR; + if (version == Z_NULL || version[0] != my_version[0] || stream_size != sizeof(z_stream)) { return Z_VERSION_ERROR; @@ -375,6 +380,8 @@ local int deflateStateCheck (strm) return 0; } +#ifdef NOVGZ + /* ========================================================================= */ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) z_streamp strm; @@ -466,6 +473,8 @@ int ZEXPORT deflateGetDictionary (strm, dictionary, dictLength) return Z_OK; } +#endif + /* ========================================================================= */ int ZEXPORT deflateResetKeep (strm) z_streamp strm; @@ -517,6 +526,8 @@ int ZEXPORT deflateReset (strm) return ret; } +#ifdef NOVGZ + /* ========================================================================= */ int ZEXPORT deflateSetHeader (strm, head) z_streamp strm; @@ -712,6 +723,8 @@ uLong ZEXPORT deflateBound(strm, sourceLen) (sourceLen >> 25) + 13 - 6 + wraplen; } +#endif + /* ========================================================================= * Put a short in the pending buffer. The 16-bit value is put in MSB order. * IN assertion: the stream state is correct and there is enough room in @@ -1005,8 +1018,8 @@ int ZEXPORT deflate (strm, flush) block_state bstate; bstate = s->level == 0 ? deflate_stored(s, flush) : - s->strategy == Z_HUFFMAN_ONLY ? deflate_huff(s, flush) : - s->strategy == Z_RLE ? deflate_rle(s, flush) : + // s->strategy == Z_HUFFMAN_ONLY ? deflate_huff(s, flush) : + // s->strategy == Z_RLE ? deflate_rle(s, flush) : (*(configuration_table[s->level].func))(s, flush); if (bstate == finish_started || bstate == finish_done) { @@ -2056,6 +2069,8 @@ local block_state deflate_slow(s, flush) } #endif /* FASTEST */ +#ifdef NOVGZ + /* =========================================================================== * For Z_RLE, simply look for runs of bytes, generate matches only of distance * one. Do not maintain a hash table. (It will be regenerated if this run of @@ -2168,3 +2183,5 @@ local block_state deflate_huff(s, flush) FLUSH_BLOCK(s, 0); return block_done; } + +#endif diff --git a/lib/libvgz/inflate.c b/lib/libvgz/inflate.c index cf0d2b0..fdefc73 100644 --- a/lib/libvgz/inflate.c +++ b/lib/libvgz/inflate.c @@ -99,8 +99,10 @@ local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, #ifdef BUILDFIXED void makefixed OF((void)); #endif +#ifdef NOVGZ local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf, unsigned len)); +#endif local int inflateStateCheck(strm) z_streamp strm; @@ -1294,6 +1296,8 @@ z_streamp strm; return Z_OK; } +#ifdef NOVGZ + int ZEXPORT inflateGetDictionary(strm, dictionary, dictLength) z_streamp strm; Bytef *dictionary; @@ -1565,3 +1569,5 @@ z_streamp strm; state = (struct inflate_state FAR *)strm->state; return (unsigned long)(state->next - state->codes); } + +#endif From phk at FreeBSD.org Sun May 21 14:15:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 21 May 2017 16:15:05 +0200 Subject: [master] 324d6e4 Wait up to one third of total test duration for varnishd to phone home Message-ID: commit 324d6e4ed1677550e14482b7932e1cd57259e9ba Author: Poul-Henning Kamp Date: Sun May 21 14:13:39 2017 +0000 Wait up to one third of total test duration for varnishd to phone home diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index f098ef8..c385ffc 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -476,7 +476,7 @@ varnish_launch(struct varnish *v) fd[0].events = POLLIN; fd[1].fd = v->fds[1]; fd[1].events = POLLIN; - i = poll(fd, 2, 10000); + i = poll(fd, 2, vtc_maxdur * 1000 / 3); vtc_log(v->vl, 4, "CLIPOLL %d 0x%x 0x%x", i, fd[0].revents, fd[1].revents); if (i == 0) From hermunn at varnish-software.com Sun May 21 15:05:06 2017 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Sun, 21 May 2017 17:05:06 +0200 Subject: [4.1] 9367a63 Ignore missing reason in probe responses Message-ID: commit 9367a6329e67b626c4606820ece3edd2326b5a9e Author: Dridi Boukelmoune Date: Mon Sep 12 18:53:05 2016 +0200 Ignore missing reason in probe responses This is a cherry-pick of 3271c51e303b1f187d851cdaf056a940ccaa8365 that fixed the issue for Varnish 3.0 since it wasn't a problem in the 4.1 series then. This is only a back-port of the regression test since the referenced commits already fixed the regression. Refs 5e4d6c843376acc0bf14872d89a4e8bb2f77186b Refs 9f4226200ef7784b597056fbb788b57f64ca6ef3 Refs #2069 diff --git a/bin/varnishtest/tests/r02069.vtc b/bin/varnishtest/tests/r02069.vtc new file mode 100644 index 0000000..9e59644 --- /dev/null +++ b/bin/varnishtest/tests/r02069.vtc @@ -0,0 +1,38 @@ +varnishtest "Probe response without a reason" + +server s1 -repeat 20 { + rxreq + send "HTTP/1.1 200\r\n\r\n" +} -start + +varnish v1 -vcl { + import std; + + backend s1 { + .host = "${s1_addr}"; + .port = "${s1_port}"; + .probe = { + .initial = 0; + .window = 5; + .threshold = 5; + .interval = 100ms; + } + } + + sub vcl_recv { + if (std.healthy(req.backend_hint)) { + return (synth(200)); + } + else { + return (synth(500)); + } + } +} -start + +delay 1 + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run From hermunn at varnish-software.com Sun May 21 15:15:05 2017 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Sun, 21 May 2017 17:15:05 +0200 Subject: [4.1] a9ad661 Fix typo in lib/libvarnish/vtim.c Message-ID: commit a9ad661b3b94e12b2d1c23ae9ccd0a6ae7f54363 Author: Enerdhil Date: Thu May 4 10:51:43 2017 +0200 Fix typo in lib/libvarnish/vtim.c diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index 509ba9a..c892dbf 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -461,7 +461,7 @@ tst_delta() err += tst_delta_check("VTIM_real", r_begin, r_end, ref); if (err) { - printf("%d time delta test errrors\n", err); + printf("%d time delta test errors\n", err); exit(4); } } From dridi.boukelmoune at gmail.com Sun May 21 17:09:05 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Sun, 21 May 2017 19:09:05 +0200 Subject: [4.1] 1201a1f Adapt r02069.vtc to 4.1 Message-ID: commit 1201a1fa815383677abf1b12842a434e30bcaccd Author: Dridi Boukelmoune Date: Sun May 21 19:07:42 2017 +0200 Adapt r02069.vtc to 4.1 diff --git a/bin/varnishtest/tests/r02069.vtc b/bin/varnishtest/tests/r02069.vtc index 9e59644..959f492 100644 --- a/bin/varnishtest/tests/r02069.vtc +++ b/bin/varnishtest/tests/r02069.vtc @@ -6,7 +6,7 @@ server s1 -repeat 20 { } -start varnish v1 -vcl { - import std; + import ${vmod_std}; backend s1 { .host = "${s1_addr}"; From phk at FreeBSD.org Sun May 21 20:59:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 21 May 2017 22:59:05 +0200 Subject: [master] 1478588 More libvgz neutering Message-ID: commit 1478588525251ad0afb602781d2801e1e2d25d98 Author: Poul-Henning Kamp Date: Sun May 21 20:57:49 2017 +0000 More libvgz neutering diff --git a/lib/libvgz/deflate.c b/lib/libvgz/deflate.c index 17c2016..ba50571 100644 --- a/lib/libvgz/deflate.c +++ b/lib/libvgz/deflate.c @@ -229,6 +229,8 @@ local void slide_hash(s) #endif } +#ifdef NOVGZ + /* ========================================================================= */ int ZEXPORT deflateInit_(strm, level, version, stream_size) z_streamp strm; @@ -241,6 +243,8 @@ int ZEXPORT deflateInit_(strm, level, version, stream_size) /* To do: ignore strm->next_in if we use it as window */ } +#endif + /* ========================================================================= */ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, version, stream_size) @@ -1114,6 +1118,8 @@ int ZEXPORT deflateEnd (strm) return status == BUSY_STATE ? Z_DATA_ERROR : Z_OK; } +#ifdef NOVGZ + /* ========================================================================= * Copy the source state to the destination state. * To simplify the source, this is not supported for 16-bit MSDOS (which @@ -1174,6 +1180,8 @@ int ZEXPORT deflateCopy (dest, source) #endif /* MAXSEG_64K */ } +#endif + /* =========================================================================== * Read a new buffer from the current input stream, update the adler32 * and total number of bytes read. All deflate() input goes through diff --git a/lib/libvgz/inflate.c b/lib/libvgz/inflate.c index fdefc73..ca7b608 100644 --- a/lib/libvgz/inflate.c +++ b/lib/libvgz/inflate.c @@ -239,6 +239,8 @@ int stream_size; return ret; } +#ifdef NOVGZ + int ZEXPORT inflateInit_(strm, version, stream_size) z_streamp strm; const char *version; @@ -268,6 +270,8 @@ int value; return Z_OK; } +#endif + /* Return state with length and distance decoding tables and index sizes set to fixed code decoding. Normally this returns fixed tables from inffixed.h. From phk at FreeBSD.org Sun May 21 21:23:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 21 May 2017 23:23:05 +0200 Subject: [master] cc295cf A little bit more libvgz coverage Message-ID: commit cc295cf0bd0dff9e5e4eb8f193fef90dd9205199 Author: Poul-Henning Kamp Date: Sun May 21 21:22:27 2017 +0000 A little bit more libvgz coverage diff --git a/bin/varnishtest/tests/g00001.vtc b/bin/varnishtest/tests/g00001.vtc index 0f84f54..4d0cfe1 100644 --- a/bin/varnishtest/tests/g00001.vtc +++ b/bin/varnishtest/tests/g00001.vtc @@ -3,7 +3,13 @@ varnishtest "test basic gunzip for client" server s1 { rxreq expect req.http.accept-encoding == "gzip" - txresp -gzipbody FOO + txresp -nolen -hdr "Content-Length: 49" -hdr "Content-Encoding: gzip" + + # "date > _ ; gzip -9 _" - contains filename + timestamp + sendhex "1f 8b 08 08 ef 00 22 59 02 03 5f 00 0b 2e cd 53" + sendhex "f0 4d ac 54 30 32 04 22 2b 03 13 2b 13 73 85 d0" + sendhex "10 67 05 23 03 43 73 2e 00 cf 9b db c0 1d 00 00" + sendhex "00" } -start varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend { @@ -12,26 +18,26 @@ varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend { client c1 { txreq rxresp - expect resp.bodylen == "3" + expect resp.bodylen == "29" expect resp.http.content-encoding == txreq -hdr "Accept-encoding: gzip;q=0.1" rxresp expect resp.http.content-encoding == "gzip" gunzip - expect resp.bodylen == "3" + expect resp.bodylen == "29" } -run -delay .1 +varnish v1 -vsl_catchup client c1 { txreq -proto HTTP/1.0 rxresp - expect resp.bodylen == "3" + expect resp.bodylen == "29" expect resp.http.content-encoding == } -run -delay .1 +varnish v1 -vsl_catchup client c1 { txreq -req HEAD @@ -40,7 +46,7 @@ client c1 { txreq -req HEAD -hdr "Accept-encoding: gzip;q=0.1" rxresp -no_obj - expect resp.http.content-length == "26" + expect resp.http.content-length == "49" expect resp.http.content-encoding == "gzip" } -run diff --git a/lib/libvgz/deflate.c b/lib/libvgz/deflate.c index ba50571..4fce32a 100644 --- a/lib/libvgz/deflate.c +++ b/lib/libvgz/deflate.c @@ -1118,7 +1118,7 @@ int ZEXPORT deflateEnd (strm) return status == BUSY_STATE ? Z_DATA_ERROR : Z_OK; } -#ifdef NOVGZ +#ifdef NOVGZ /* ========================================================================= * Copy the source state to the destination state. From phk at FreeBSD.org Mon May 22 23:44:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 May 2017 01:44:05 +0200 Subject: [master] 3a504de Simplify Message-ID: commit 3a504de047c252f33841efe842f555f0667f31f2 Author: Poul-Henning Kamp Date: Mon May 22 22:12:05 2017 +0000 Simplify diff --git a/include/Makefile.am b/include/Makefile.am index 9adaebe..64fa642 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -119,11 +119,7 @@ BUILT_SOURCES = $(GENERATED_H) MAINTAINERCLEANFILES = vcs_version.h CLEANFILES = \ - tbl/vcl_returns.h \ - tbl/vrt_stv_var.h \ - vcl.h \ - vrt_obj.h \ - vmod_abi.h \ + $(GENERATED_H) \ vrt_test \ _vrt_test \ vrt.c From phk at FreeBSD.org Mon May 22 23:44:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 May 2017 01:44:05 +0200 Subject: [master] c875e90 Generate builtin_vcl.c instead of builtin_vcl.h to avoid a not fully understood circular-ish make distcheck dependency problem. Message-ID: commit c875e9089fa2f3ed19df4870ae0f5d3db516f217 Author: Poul-Henning Kamp Date: Mon May 22 23:42:39 2017 +0000 Generate builtin_vcl.c instead of builtin_vcl.h to avoid a not fully understood circular-ish make distcheck dependency problem. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 844b158..f548ab6 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -109,8 +109,10 @@ varnishd_SOURCES = \ waiter/cache_waiter_poll.c \ waiter/cache_waiter_ports.c +nodist_varnishd_SOURCES = \ + builtin_vcl.c + noinst_HEADERS = \ - builtin_vcl.h \ cache/cache_ban.h \ cache/cache_esi.h \ cache/cache_obj.h \ @@ -187,17 +189,20 @@ TESTS = vhp_table_test vhp_decode_test # # Turn the builtin.vcl file into a C-string we can include in the program. # -builtin_vcl.h: builtin.vcl +builtin_vcl.c: builtin.vcl echo '/*' > $@ echo ' * NB: This file is machine generated, DO NOT EDIT!' >> $@ echo ' *' >> $@ echo ' * Edit builtin.vcl instead and run make' >> $@ echo ' *' >> $@ echo ' */' >> $@ + echo '#include "mgt/mgt.h"' >> $@ echo '' >> $@ + echo 'const char * const builtin_vcl =' >> $@ sed -e 's/"/\\"/g' \ -e 's/$$/\\n"/' \ -e 's/^/ "/' $(srcdir)/builtin.vcl >> $@ + echo ';' >> $@ EXTRA_DIST = builtin.vcl @@ -205,10 +210,8 @@ vhp_hufdec.h: vhp_gen_hufdec $(AM_V_GEN) ./vhp_gen_hufdec > vhp_hufdec.h_ mv vhp_hufdec.h_ vhp_hufdec.h -DISTCLEANFILES = builtin_vcl.h +DISTCLEANFILES = builtin_vcl.c BUILT_SOURCES = vhp_hufdec.h DISTCLEANFILES += vhp_hufdec.h -# Explicitly record dependency -mgt/mgt_vcc.c: builtin_vcl.h diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index f38ddd6..3f8ff11 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -47,6 +47,10 @@ extern struct vev_base *mgt_evb; extern unsigned d_flag; extern int exit_status; +/* builtin_vcl.c */ + +extern const char * const builtin_vcl; + /* mgt_acceptor.c */ void MAC_Arg(const char *); diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 00f1e6f..83887c2 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -71,10 +71,6 @@ unsigned mgt_vcc_unsafe_path; /*--------------------------------------------------------------------*/ -static const char * const builtin_vcl = -#include "builtin_vcl.h" - "" ; - void mgt_DumpBuiltin(void) { From phk at FreeBSD.org Tue May 23 07:32:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 May 2017 09:32:05 +0200 Subject: [master] 883fddf Use sphinx source/build facility directly Message-ID: commit 883fddfea4e185ff417762fb64207b81d4cea971 Author: Poul-Henning Kamp Date: Tue May 23 07:30:43 2017 +0000 Use sphinx source/build facility directly diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index 255ce4e..a98eb31 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -5,9 +5,9 @@ SPHINXOPTS = SPHINXBUILD = sphinx-build -W -q -N PAPER = a4 -BUILDDIR = build +BUILDDIR = $(builddir)/build -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees -D latex_elements.papersize=$(PAPER) $(SPHINXOPTS) $(builddir) +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees -D latex_elements.papersize=$(PAPER) $(SPHINXOPTS) $(srcdir) .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest @@ -27,22 +27,12 @@ help: clean: -rm -rf $(BUILDDIR)/* -# use index.rst as an indicator if we have copied already -.PHONY: link_srcdir -link_srcdir: - if test "x$(srcdir)" != "x$(builddir)" && test ! -f index.rst ; then \ - d=`pwd`/$(builddir) ; \ - cd $(srcdir) && find . -type f | cpio -dmp $${d} || true ; \ - fi - # work around for make html called within doc/sphinx .PHONY: graphviz graphviz: cd ../graphviz && $(MAKE) html -sphinx_prereq: link_srcdir graphviz conf.py - -all: link_srcdir +sphinx_prereq: graphviz conf.py html: sphinx_prereq $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html diff --git a/man/Makefile.am b/man/Makefile.am index 2b61fbb..d5332f6 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -23,65 +23,65 @@ CLEANFILES = $(dist_man_MANS) RST2ANY_FLAGS = --halt=2 varnish-cli.7: $(top_builddir)/doc/sphinx/reference/varnish-cli.rst - ${RST2MAN} $(RST2ANY_FLAGS) $(top_builddir)/doc/sphinx/reference/varnish-cli.rst $@ + ${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst $@ varnish-counters.7: $(top_builddir)/doc/sphinx/reference/varnish-counters.rst - ${RST2MAN} $(RST2ANY_FLAGS) $(top_builddir)/doc/sphinx/reference/varnish-counters.rst $@ + ${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnish-counters.rst $@ vcl.7: $(top_builddir)/doc/sphinx/reference/vcl.rst \ $(top_builddir)/bin/varnishd/builtin.vcl - ${RST2MAN} $(RST2ANY_FLAGS) $(top_builddir)/doc/sphinx/reference/vcl.rst $@ + ${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/vcl.rst $@ vsl.7: $(top_builddir)/doc/sphinx/reference/vsl.rst \ $(top_builddir)/doc/sphinx/include/vsl-tags.rst - ${RST2MAN} $(RST2ANY_FLAGS) $(top_builddir)/doc/sphinx/reference/vsl.rst $@ + ${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/vsl.rst $@ vsl-query.7: $(top_builddir)/doc/sphinx/reference/vsl-query.rst - ${RST2MAN} $(RST2ANY_FLAGS) $(top_builddir)/doc/sphinx/reference/vsl-query.rst $@ + ${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/vsl-query.rst $@ varnishadm.1: $(top_builddir)/doc/sphinx/reference/varnishadm.rst - ${RST2MAN} $(RST2ANY_FLAGS) $(top_builddir)/doc/sphinx/reference/varnishadm.rst $@ + ${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishadm.rst $@ varnishd.1: \ $(top_builddir)/doc/sphinx/reference/varnishd.rst \ $(top_builddir)/doc/sphinx/include/params.rst - ${RST2MAN} $(RST2ANY_FLAGS) $(top_builddir)/doc/sphinx/reference/varnishd.rst $@ + ${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishd.rst $@ varnishncsa.1: \ $(top_builddir)/doc/sphinx/reference/varnishncsa.rst \ $(top_builddir)/doc/sphinx/include/varnishncsa_options.rst \ $(top_builddir)/doc/sphinx/include/varnishncsa_synopsis.rst - ${RST2MAN} $(RST2ANY_FLAGS) $(top_builddir)/doc/sphinx/reference/varnishncsa.rst $@ + ${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishncsa.rst $@ varnishlog.1: \ $(top_builddir)/doc/sphinx/reference/varnishlog.rst \ $(top_builddir)/doc/sphinx/include/varnishlog_options.rst \ $(top_builddir)/doc/sphinx/include/varnishlog_synopsis.rst - ${RST2MAN} $(RST2ANY_FLAGS) $(top_builddir)/doc/sphinx/reference/varnishlog.rst $@ + ${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishlog.rst $@ varnishstat.1: $(top_builddir)/doc/sphinx/reference/varnishstat.rst \ $(top_builddir)/doc/sphinx/include/varnishstat_options.rst \ $(top_builddir)/doc/sphinx/include/varnishstat_synopsis.rst - ${RST2MAN} $(RST2ANY_FLAGS) $(top_builddir)/doc/sphinx/reference/varnishstat.rst $@ + ${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishstat.rst $@ varnishtest.1: $(top_builddir)/doc/sphinx/reference/varnishtest.rst - ${RST2MAN} $(RST2ANY_FLAGS) $(top_builddir)/doc/sphinx/reference/varnishtest.rst $@ + ${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishtest.rst $@ vtc.7: $(top_builddir)/doc/sphinx/reference/vtc.rst \ $(top_builddir)/doc/sphinx/include/vtc-syntax.rst - ${RST2MAN} $(RST2ANY_FLAGS) $(top_builddir)/doc/sphinx/reference/vtc.rst $@ + ${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/vtc.rst $@ varnishtop.1: \ $(top_builddir)/doc/sphinx/reference/varnishtop.rst \ $(top_builddir)/doc/sphinx/include/varnishtop_options.rst \ $(top_builddir)/doc/sphinx/include/varnishtop_synopsis.rst - ${RST2MAN} $(RST2ANY_FLAGS) $(top_builddir)/doc/sphinx/reference/varnishtop.rst $@ + ${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishtop.rst $@ varnishhist.1: \ $(top_builddir)/doc/sphinx/reference/varnishhist.rst \ $(top_builddir)/doc/sphinx/include/varnishhist_options.rst \ $(top_builddir)/doc/sphinx/include/varnishhist_synopsis.rst - ${RST2MAN} $(RST2ANY_FLAGS) $(top_builddir)/doc/sphinx/reference/varnishhist.rst $@ + ${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishhist.rst $@ vmod_std.3: $(top_builddir)/lib/libvmod_std/vmod_std.man.rst ${RST2MAN} $(RST2ANY_FLAGS) $? $@ From phk at FreeBSD.org Tue May 23 08:34:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 May 2017 10:34:05 +0200 Subject: [master] a32fb96 First babystep of the new selfdocumenting VSC counters. Message-ID: commit a32fb96c1475b342b96de7083e35419582ef5d57 Author: Poul-Henning Kamp Date: Tue May 23 08:30:36 2017 +0000 First babystep of the new selfdocumenting VSC counters. The .VSC file steals .RST's directive syntax, but it is not a RST file, and it is not processed as such. Maybe later it will become a .RST file, but that is TBD. A C-struct for use in varnishd is built, (along with a JSON structure which describes the counters.) For now that structure is identical with what comes out of include/tbl/vsc_f_main, so nothing special happens in the VSM yet. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index f548ab6..c03ff70 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -107,12 +107,14 @@ varnishd_SOURCES = \ waiter/cache_waiter_epoll.c \ waiter/cache_waiter_kqueue.c \ waiter/cache_waiter_poll.c \ - waiter/cache_waiter_ports.c + waiter/cache_waiter_ports.c \ + VSC_main.c nodist_varnishd_SOURCES = \ builtin_vcl.c noinst_HEADERS = \ + VSC_main.h \ cache/cache_ban.h \ cache/cache_esi.h \ cache/cache_obj.h \ @@ -215,3 +217,12 @@ DISTCLEANFILES = builtin_vcl.c BUILT_SOURCES = vhp_hufdec.h DISTCLEANFILES += vhp_hufdec.h +####################################################################### +VSC_main.c VSC_main.h: $(srcdir)/main.vsc $(top_builddir)/lib/libvcc/vsctool.py + $(PYTHON) $(top_builddir)/lib/libvcc/vsctool.py $(srcdir)/main.vsc + +$(varnishd_OBJECTS): VSC_main.h + +EXTRA_DIST += main.vsc +DISTCLEANFILES += VSC_main.c VSC_main.h + diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index b772822..a5fcef8 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -977,7 +977,6 @@ void SES_Set_String_Attr(struct sess *sp, enum sess_attr a, const char *src); const char *SES_Get_String_Attr(const struct sess *sp, enum sess_attr a); /* cache_shmlog.c */ -extern struct VSC_C_main *VSC_C_main; void *VSM_Alloc(unsigned size, const char *class, const char *type, const char *ident); void VSM_Free(void *ptr); diff --git a/bin/varnishd/cache/cache_priv.h b/bin/varnishd/cache/cache_priv.h index 0ff51eb..215ebc9 100644 --- a/bin/varnishd/cache/cache_priv.h +++ b/bin/varnishd/cache/cache_priv.h @@ -29,6 +29,8 @@ * Stuff that should *never* be exposed to a VMOD */ +#include "VSC_main.h" + /* Prototypes etc ----------------------------------------------------*/ /* cache_acceptor.c */ @@ -103,6 +105,7 @@ void SES_NewPool(struct pool *, unsigned pool_no); void SES_DestroyPool(struct pool *); /* cache_shmlog.c */ +extern struct VSC_main *VSC_C_main; void VSM_Init(void); void VSL_Setup(struct vsl_log *vsl, void *ptr, size_t len); void VSL_ChgId(struct vsl_log *vsl, const char *typ, const char *why, diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 66fac59..113d0fc 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -51,8 +51,7 @@ static uint32_t *vsl_ptr; static unsigned vsl_segment_n; static ssize_t vsl_segsize; -struct VSC_C_main *VSC_C_main; - +struct VSC_main *VSC_C_main; static void vsl_sanity(const struct vsl_log *vsl) @@ -471,6 +470,28 @@ VSL_End(struct vsl_log *vsl) /*--------------------------------------------------------------------*/ +void * +VSC_Alloc(const char *nm, size_t sj, const unsigned char *zj, size_t szj, + const char *fmt, va_list va) +{ + (void)nm; + (void)zj; + (void)szj; + (void)sj; + (void)fmt; + (void)va; + return (0); +} + +void +VSC_Destroy(const char *nm, void *p) +{ + (void)nm; + (void)p; +} + +/*--------------------------------------------------------------------*/ + static void * vsm_cleaner(void *priv) { @@ -478,7 +499,7 @@ vsm_cleaner(void *priv) THR_SetName("vsm_cleaner"); while (1) { AZ(pthread_mutex_lock(&vsm_mtx)); - VSM_common_cleaner(heritage.vsm, VSC_C_main); + VSM_common_cleaner(heritage.vsm, (void*)VSC_C_main); AZ(pthread_mutex_unlock(&vsm_mtx)); VTIM_sleep(1.1); } diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h index 0c302c2..107f5b8 100644 --- a/bin/varnishd/common/common.h +++ b/bin/varnishd/common/common.h @@ -33,6 +33,7 @@ #endif #define COMMON_COMMON_H +#include #include #include @@ -90,5 +91,9 @@ void VSM_common_copy(struct vsm_sc *to, const struct vsm_sc *from); void VSM_common_cleaner(struct vsm_sc *sc, struct VSC_C_main *stats); void VSM_common_ageupdate(const struct vsm_sc *sc); +void *VSC_Alloc(const char *, size_t, const unsigned char *, size_t, + const char *, va_list); +void VSC_Destroy(const char *, void *); + /* mgt_cli.c */ extern struct VCLS *mgt_cls; diff --git a/bin/varnishd/main.vsc b/bin/varnishd/main.vsc new file mode 100644 index 0000000..f901fca --- /dev/null +++ b/bin/varnishd/main.vsc @@ -0,0 +1,756 @@ +.. + This is *NOT* a RST file but the syntax has been chosen so + that it may become an RST file at some later date. + + XXX: For now this file must be kept in 100% agreement with + XXX: include/tbl/vsc_f_main.h + +.. varnish_vsc_begin:: main + :oneliner: Main counters + +.. varnish_vsc:: uptime + :oneliner: Child process uptime + +How long the child process has been running. + +.. varnish_vsc:: sess_conn + :oneliner: Sessions accepted + +Count of sessions successfully accepted + +.. varnish_vsc:: sess_drop + :oneliner: Sessions dropped + +Count of sessions silently dropped due to lack of worker thread. + +.. varnish_vsc:: sess_fail + :oneliner: Session accept failures + +Count of failures to accept TCP connection. Either the client +changed its mind, or the kernel ran out of some resource like file +descriptors. + +.. varnish_vsc:: client_req_400 + :oneliner: Client requests received, subject to 400 errors + +400 means we couldn't make sense of the request, it was malformed +in some drastic way. + +.. varnish_vsc:: client_req_417 + :oneliner: Client requests received, subject to 417 errors + +417 means that something went wrong with an Expect: header. + +.. varnish_vsc:: client_req + :oneliner: Good client requests received + +The count of parseable client requests seen. + +.. varnish_vsc:: cache_hit + :oneliner: Cache hits + +Count of cache hits. A cache hit indicates that an object has been +delivered to a client without fetching it from a backend server. + +.. varnish_vsc:: cache_hitpass + :oneliner: Cache hits for pass. + +Count of hits for pass. A cache hit for pass indicates that Varnish +is going to pass the request to the backend and this decision has +been cached in it self. This counts how many times the cached +decision is being used. + +.. varnish_vsc:: cache_hitmiss + :oneliner: Cache hits for miss. + +Count of hits for miss. A cache hit for miss indicates that Varnish +is going to proceed as for a cache miss without request coalescing, +and this decision has been cached. This counts how many times the +cached decision is being used. + +.. varnish_vsc:: cache_miss + :oneliner: Cache misses + +Count of misses. A cache miss indicates the object was fetched from +the backend before delivering it to the client. + +.. varnish_vsc:: backend_conn + :oneliner: Backend conn. success + +How many backend connections have successfully been established. + +.. varnish_vsc:: backend_unhealthy + :oneliner: Backend conn. not attempted + + +.. varnish_vsc:: backend_busy + :oneliner: Backend conn. too many + + +.. varnish_vsc:: backend_fail + :oneliner: Backend conn. failures + + +.. varnish_vsc:: backend_reuse + :oneliner: Backend conn. reuses + +Count of backend connection reuses. This counter is increased +whenever we reuse a recycled connection. + +.. varnish_vsc:: backend_recycle + :oneliner: Backend conn. recycles + +Count of backend connection recycles. This counter is increased +whenever we have a keep-alive connection that is put back into the +pool of connections. It has not yet been used, but it might be, +unless the backend closes it. + +.. varnish_vsc:: backend_retry + :oneliner: Backend conn. retry + + +.. varnish_vsc:: fetch_head + :oneliner: Fetch no body (HEAD) + +beresp with no body because the request is HEAD. + +.. varnish_vsc:: fetch_length + :oneliner: Fetch with Length + +beresp.body with Content-Length. + +.. varnish_vsc:: fetch_chunked + :oneliner: Fetch chunked + +beresp.body with Chunked. + +.. varnish_vsc:: fetch_eof + :oneliner: Fetch EOF + +beresp.body with EOF. + +.. varnish_vsc:: fetch_bad + :oneliner: Fetch bad T-E + +beresp.body length/fetch could not be determined. + +.. varnish_vsc:: fetch_none + :oneliner: Fetch no body + +beresp.body empty + +.. varnish_vsc:: fetch_1xx + :oneliner: Fetch no body (1xx) + +beresp with no body because of 1XX response. + +.. varnish_vsc:: fetch_204 + :oneliner: Fetch no body (204) + +beresp with no body because of 204 response. + +.. varnish_vsc:: fetch_304 + :oneliner: Fetch no body (304) + +beresp with no body because of 304 response. + +.. varnish_vsc:: fetch_failed + :oneliner: Fetch failed (all causes) + +beresp fetch failed. + +.. varnish_vsc:: fetch_no_thread + :oneliner: Fetch failed (no thread) + +beresp fetch failed, no thread available. + +.. varnish_vsc:: pools + :type: gauge + :oneliner: Number of thread pools + +Number of thread pools. See also parameter thread_pools. NB: Presently +pools cannot be removed once created. + +.. varnish_vsc:: threads + :type: gauge + :oneliner: Total number of threads + +Number of threads in all pools. See also parameters thread_pools, +thread_pool_min and thread_pool_max. + +.. varnish_vsc:: threads_limited + :oneliner: Threads hit max + +Number of times more threads were needed, but limit was reached in +a thread pool. See also parameter thread_pool_max. + +.. varnish_vsc:: threads_created + :oneliner: Threads created + +Total number of threads created in all pools. + +.. varnish_vsc:: threads_destroyed + :oneliner: Threads destroyed + +Total number of threads destroyed in all pools. + +.. varnish_vsc:: threads_failed + :oneliner: Thread creation failed + +Number of times creating a thread failed. See VSL::Debug for +diagnostics. See also parameter thread_fail_delay. + +.. varnish_vsc:: thread_queue_len + :type: gauge + :oneliner: Length of session queue + +Length of session queue waiting for threads. NB: Only updates once +per second. See also parameter thread_queue_limit. + +.. varnish_vsc:: busy_sleep + :oneliner: Number of requests sent to sleep on busy objhdr + +Number of requests sent to sleep without a worker thread because +they found a busy object. + +.. varnish_vsc:: busy_wakeup + :oneliner: Number of requests woken after sleep on busy objhdr + +Number of requests taken off the busy object sleep list and rescheduled. + +.. varnish_vsc:: busy_killed + :oneliner: Number of requests killed after sleep on busy objhdr + +Number of requests killed from the busy object sleep list due to +lack of resources. + +.. varnish_vsc:: sess_queued + :oneliner: Sessions queued for thread + +Number of times session was queued waiting for a thread. See also +parameter thread_queue_limit. + +.. varnish_vsc:: sess_dropped + :oneliner: Sessions dropped for thread + +Number of times session was dropped because the queue were too long +already. See also parameter thread_queue_limit. + +.. varnish_vsc:: n_object + :type: gauge + :oneliner: object structs made + +Approximate number of HTTP objects (headers + body, if present) in +the cache. + +.. varnish_vsc:: n_vampireobject + :type: gauge + :level: diag + :oneliner: unresurrected objects + +Number of unresurrected objects + +.. varnish_vsc:: n_objectcore + :type: gauge + :oneliner: objectcore structs made + +Approximate number of object metadata elements in the cache. Each +object needs an objectcore, extra objectcores are for hit-for-miss, +hit-for-pass and busy objects. + +.. varnish_vsc:: n_objecthead + :type: gauge + :oneliner: objecthead structs made + +Approximate number of different hash entries in the cache. + +.. varnish_vsc:: n_backend + :type: gauge + :oneliner: Number of backends + +Number of backends known to us. + +.. varnish_vsc:: n_expired + :type: gauge + :oneliner: Number of expired objects + +Number of objects that expired from cache because of old age. + +.. varnish_vsc:: n_lru_nuked + :type: gauge + :oneliner: Number of LRU nuked objects + +How many objects have been forcefully evicted from storage to make +room for a new object. + +.. varnish_vsc:: n_lru_moved + :type: gauge + :level: diag + :oneliner: Number of LRU moved objects + +Number of move operations done on the LRU list. + +.. varnish_vsc:: losthdr + :oneliner: HTTP header overflows + + +.. varnish_vsc:: s_sess + :oneliner: Total sessions seen + + +.. varnish_vsc:: s_req + :oneliner: Total requests seen + + +.. varnish_vsc:: s_pipe + :oneliner: Total pipe sessions seen + + +.. varnish_vsc:: s_pass + :oneliner: Total pass-ed requests seen + + +.. varnish_vsc:: s_fetch + :oneliner: Total backend fetches initiated + + +.. varnish_vsc:: s_synth + :oneliner: Total synthethic responses made + + +.. varnish_vsc:: s_req_hdrbytes + :oneliner: Request header bytes + +Total request header bytes received + +.. varnish_vsc:: s_req_bodybytes + :oneliner: Request body bytes + +Total request body bytes received + +.. varnish_vsc:: s_resp_hdrbytes + :oneliner: Response header bytes + +Total response header bytes transmitted + +.. varnish_vsc:: s_resp_bodybytes + :oneliner: Response body bytes + +Total response body bytes transmitted + +.. varnish_vsc:: s_pipe_hdrbytes + :oneliner: Pipe request header bytes + +Total request bytes received for piped sessions + +.. varnish_vsc:: s_pipe_in + :oneliner: Piped bytes from client + +Total number of bytes forwarded from clients in pipe sessions + +.. varnish_vsc:: s_pipe_out + :oneliner: Piped bytes to client + +Total number of bytes forwarded to clients in pipe sessions + +.. varnish_vsc:: sess_closed + :oneliner: Session Closed + + +.. varnish_vsc:: sess_closed_err + :oneliner: Session Closed with error + +Total number of sessions closed with errors. See sc_* diag counters +for detailed breakdown + +.. varnish_vsc:: sess_readahead + :oneliner: Session Read Ahead + + +.. varnish_vsc:: sess_herd + :level: diag + :oneliner: Session herd + +Number of times the timeout_linger triggered + +.. varnish_vsc:: sc_rem_close + :level: diag + :oneliner: Session OK REM_CLOSE + +Number of session closes with REM_CLOSE (Client Closed) + +.. varnish_vsc:: sc_req_close + :level: diag + :oneliner: Session OK REQ_CLOSE + +Number of session closes with REQ_CLOSE (Client requested close) + +.. varnish_vsc:: sc_req_http10 + :level: diag + :oneliner: Session Err REQ_HTTP10 + +Number of session closes with Error REQ_HTTP10 (Proto < HTTP/1.1) + +.. varnish_vsc:: sc_rx_bad + :level: diag + :oneliner: Session Err RX_BAD + +Number of session closes with Error RX_BAD (Received bad req/resp) + +.. varnish_vsc:: sc_rx_body + :level: diag + :oneliner: Session Err RX_BODY + +Number of session closes with Error RX_BODY (Failure receiving req.body) + +.. varnish_vsc:: sc_rx_junk + :level: diag + :oneliner: Session Err RX_JUNK + +Number of session closes with Error RX_JUNK (Received junk data) + +.. varnish_vsc:: sc_rx_overflow + :level: diag + :oneliner: Session Err RX_OVERFLOW + +Number of session closes with Error RX_OVERFLOW (Received buffer overflow) + +.. varnish_vsc:: sc_rx_timeout + :level: diag + :oneliner: Session Err RX_TIMEOUT + +Number of session closes with Error RX_TIMEOUT (Receive timeout) + +.. varnish_vsc:: sc_tx_pipe + :level: diag + :oneliner: Session OK TX_PIPE + +Number of session closes with TX_PIPE (Piped transaction) + +.. varnish_vsc:: sc_tx_error + :level: diag + :oneliner: Session Err TX_ERROR + +Number of session closes with Error TX_ERROR (Error transaction) + +.. varnish_vsc:: sc_tx_eof + :level: diag + :oneliner: Session OK TX_EOF + +Number of session closes with TX_EOF (EOF transmission) + +.. varnish_vsc:: sc_resp_close + :level: diag + :oneliner: Session OK RESP_CLOSE + +Number of session closes with RESP_CLOSE (Backend/VCL requested close) + +.. varnish_vsc:: sc_overload + :level: diag + :oneliner: Session Err OVERLOAD + +Number of session closes with Error OVERLOAD (Out of some resource) + +.. varnish_vsc:: sc_pipe_overflow + :level: diag + :oneliner: Session Err PIPE_OVERFLOW + +Number of session closes with Error PIPE_OVERFLOW (Session pipe overflow) + +.. varnish_vsc:: sc_range_short + :level: diag + :oneliner: Session Err RANGE_SHORT + +Number of session closes with Error RANGE_SHORT (Insufficient data for range) + +.. varnish_vsc:: sc_req_http20 + :level: diag + :oneliner: Session Err REQ_HTTP20 + +Number of session closes with Error REQ_HTTP20 (HTTP2 not accepted) + +.. varnish_vsc:: sc_vcl_failure + :level: diag + :oneliner: Session Err VCL_FAILURE + +Number of session closes with Error VCL_FAILURE (VCL failure) + +.. varnish_vsc:: shm_records + :level: diag + :oneliner: SHM records + + +.. varnish_vsc:: shm_writes + :level: diag + :oneliner: SHM writes + + +.. varnish_vsc:: shm_flushes + :level: diag + :oneliner: SHM flushes due to overflow + + +.. varnish_vsc:: shm_cont + :level: diag + :oneliner: SHM MTX contention + + +.. varnish_vsc:: shm_cycles + :level: diag + :oneliner: SHM cycles through buffer + + +.. varnish_vsc:: backend_req + :oneliner: Backend requests made + + +.. varnish_vsc:: n_vcl + :oneliner: Number of loaded VCLs in total + + +.. varnish_vsc:: n_vcl_avail + :level: diag + :oneliner: Number of VCLs available + + +.. varnish_vsc:: n_vcl_discard + :level: diag + :oneliner: Number of discarded VCLs + + +.. varnish_vsc:: vcl_fail + :oneliner: VCL failures + +Count of failures which prevented VCL from completing. + +.. varnish_vsc:: bans + :type: gauge + :oneliner: Count of bans + +Number of all bans in system, including bans superseded by newer +bans and bans already checked by the ban-lurker. + +.. varnish_vsc:: bans_completed + :type: gauge + :level: diag + :oneliner: Number of bans marked 'completed' + +Number of bans which are no longer active, either because they got +checked by the ban-lurker or superseded by newer identical bans. + +.. varnish_vsc:: bans_obj + :type: gauge + :level: diag + :oneliner: Number of bans using obj.* + +Number of bans which use obj.* variables. These bans can possibly +be washed by the ban-lurker. + +.. varnish_vsc:: bans_req + :type: gauge + :level: diag + :oneliner: Number of bans using req.* + +Number of bans which use req.* variables. These bans can not be +washed by the ban-lurker. + +.. varnish_vsc:: bans_added + :level: diag + :oneliner: Bans added + +Counter of bans added to ban list. + +.. varnish_vsc:: bans_deleted + :level: diag + :oneliner: Bans deleted + +Counter of bans deleted from ban list. + +.. varnish_vsc:: bans_tested + :level: diag + :oneliner: Bans tested against objects (lookup) + +Count of how many bans and objects have been tested against each +other during hash lookup. + +.. varnish_vsc:: bans_obj_killed + :level: diag + :oneliner: Objects killed by bans (lookup) + +Number of objects killed by bans during object lookup. + +.. varnish_vsc:: bans_lurker_tested + :level: diag + :oneliner: Bans tested against objects (lurker) + +Count of how many bans and objects have been tested against each +other by the ban-lurker. + +.. varnish_vsc:: bans_tests_tested + :level: diag + :oneliner: Ban tests tested against objects (lookup) + +Count of how many tests and objects have been tested against each +other during lookup. 'ban req.url == foo && req.http.host == bar' +counts as one in 'bans_tested' and as two in 'bans_tests_tested' + +.. varnish_vsc:: bans_lurker_tests_tested + :level: diag + :oneliner: Ban tests tested against objects (lurker) + +Count of how many tests and objects have been tested against each +other by the ban-lurker. 'ban req.url == foo && req.http.host == +bar' counts as one in 'bans_tested' and as two in 'bans_tests_tested' + +.. varnish_vsc:: bans_lurker_obj_killed + :level: diag + :oneliner: Objects killed by bans (lurker) + +Number of objects killed by the ban-lurker. + +.. varnish_vsc:: bans_lurker_obj_killed_cutoff + :level: diag + :oneliner: Objects killed by bans for cutoff (lurker) + +Number of objects killed by the ban-lurker to keep the number of +bans below ban_cutoff. + +.. varnish_vsc:: bans_dups + :level: diag + :oneliner: Bans superseded by other bans + +Count of bans replaced by later identical bans. + +.. varnish_vsc:: bans_lurker_contention + :level: diag + :oneliner: Lurker gave way for lookup + +Number of times the ban-lurker had to wait for lookups. + +.. varnish_vsc:: bans_persisted_bytes + :type: gauge + :level: diag + :oneliner: Bytes used by the persisted ban lists + +Number of bytes used by the persisted ban lists. + +.. varnish_vsc:: bans_persisted_fragmentation + :type: gauge + :level: diag + :oneliner: Extra bytes in persisted ban lists due to fragmentation + +Number of extra bytes accumulated through dropped and completed +bans in the persistent ban lists. + +.. varnish_vsc:: n_purges + :type: gauge + :oneliner: Number of purge operations executed + + +.. varnish_vsc:: n_obj_purged + :type: gauge + :oneliner: Number of purged objects + + +.. varnish_vsc:: exp_mailed + :level: diag + :oneliner: Number of objects mailed to expiry thread + +Number of objects mailed to expiry thread for handling. + +.. varnish_vsc:: exp_received + :level: diag + :oneliner: Number of objects received by expiry thread + +Number of objects received by expiry thread for handling. + +.. varnish_vsc:: hcb_nolock + :level: debug + :oneliner: HCB Lookups without lock + + +.. varnish_vsc:: hcb_lock + :level: debug + :oneliner: HCB Lookups with lock + + +.. varnish_vsc:: hcb_insert + :level: debug + :oneliner: HCB Inserts + + +.. varnish_vsc:: esi_errors + :level: diag + :oneliner: ESI parse errors (unlock) + + +.. varnish_vsc:: esi_warnings + :level: diag + :oneliner: ESI parse warnings (unlock) + + +.. varnish_vsc:: vmods + :type: gauge + :oneliner: Loaded VMODs + + +.. varnish_vsc:: n_gzip + :oneliner: Gzip operations + + +.. varnish_vsc:: n_gunzip + :oneliner: Gunzip operations + + +.. varnish_vsc:: n_test_gunzip + :oneliner: Test gunzip operations + +Those operations occur when Varnish receives a compressed object +from a backend. They are done to verify the gzip stream while it's +inserted in storage. + +.. varnish_vsc:: vsm_free + :type: gauge + :level: diag + :oneliner: Free VSM space + +Number of bytes free in the shared memory used to communicate with +tools like varnishstat, varnishlog etc. + +.. varnish_vsc:: vsm_used + :type: gauge + :level: diag + :oneliner: Used VSM space + +Number of bytes used in the shared memory used to communicate with +tools like varnishstat, varnishlog etc. + +.. varnish_vsc:: vsm_cooling + :type: gauge + :level: debug + :oneliner: Cooling VSM space + +Number of bytes which will soon (max 1 minute) be freed in the +shared memory used to communicate with tools like varnishstat, +varnishlog etc. + +.. varnish_vsc:: vsm_overflow + :type: gauge + :level: diag + :oneliner: Overflow VSM space + +Number of bytes which does not fit in the shared memory used to +communicate with tools like varnishstat, varnishlog etc. If this +counter is not zero, consider increasing the runtime variable +vsm_space. + +.. varnish_vsc:: vsm_overflowed + :level: diag + :oneliner: Overflowed VSM space + +Total number of bytes which did not fit in the shared memory used +to communicate with tools like varnishstat, varnishlog etc. If this +counter is not zero, consider increasing the runtime variable +vsm_space. + +.. varnish_vsc_end:: MAIN diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am index 795b173..133dd1b 100644 --- a/lib/libvcc/Makefile.am +++ b/lib/libvcc/Makefile.am @@ -36,7 +36,8 @@ EXTRA_DIST = \ generate.py dist_pkgdata_SCRIPTS = \ - vmodtool.py + vmodtool.py \ + vsctool.py vcc_obj.c vcc_fixed_token.c vcc_token_defs.h: \ $(top_builddir)/include/vcl.h diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py new file mode 100644 index 0000000..afc2ffb --- /dev/null +++ b/lib/libvcc/vsctool.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python +# +# Copyright (c) 2017 Varnish Software AS +# All rights reserved. +# +# Author: Poul-Henning Kamp +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +""" +This program compiles a .vsc file to C language constructs. +""" + +from __future__ import print_function + +import getopt +import json +import sys +import gzip +import StringIO +import collections + +def gzip_str(s): + out = StringIO.StringIO() + gzip.GzipFile(fileobj=out, mode="w").write(s) + return out.getvalue() + +def genhdr(fo, name): + fo.write('/*\n') + fo.write(' * NB: This file is machine generated, DO NOT EDIT!\n') + fo.write(' *\n') + fo.write(' * Edit %s.vsc run lib/libvcc/vsctool.py instead.\n' % name) + fo.write(' */\n') + fo.write('\n') + +####################################################################### + +class vscset(object): + def __init__(self, name, m): + self.name = name + self.struct = "struct VSC_" + name + self.mbrs = [] + self.head = m + self.completed = False + + def addmbr(self, m): + assert not self.completed + self.mbrs.append(m) + + def complete(self): + self.completed = True + + def emit_json(self, fo): + dd = collections.OrderedDict() + dd["version"] = "1" + dd["name"] = self.name + dd["1line"] = self.head.param["oneliner"].strip() + dd["docs"] = self.head.getdoc() + dd["elements"] = len(self.mbrs) + el = collections.OrderedDict() + dd["elem"] = el + en = 0 + for i in self.mbrs: + en += 1 + ed = collections.OrderedDict() + el[i.arg] = ed + ed["index"] = en + ed["name"] = i.arg + ed["type"] = i.param["type"] + ed["level"] = i.param["level"] + ed["1line"] = i.param["oneliner"].strip() + ed["docs"] = i.getdoc() + s=json.dumps(dd, separators=(",",":")) + fo.write("\nstatic const size_t %s_jsonlen = %dL;\n" % + (self.name, len(s))) + z = gzip_str(s) + fo.write("\nstatic const unsigned char"); + fo.write(" %s_zjson[%d] = {\n" % (self.name, len(z))) + bz = bytearray(z) + t = "\t" + for i in bz: + t += "%d," % i + if len(t) >= 70: + fo.write(t + "\n") + t = "\t" + if len(t) > 1: + fo.write(t[:-1]) + fo.write("\n};\n") + s = json.dumps(dd, indent=2, separators=(',', ': ')) + fo.write("\n// ") + fo.write("\n// ".join(s.split("\n"))) + fo.write("\n") + + + def emit_h(self): + fon="VSC_" + self.name + ".h" + fo = open(fon, "w") + genhdr(fo, self.name) + fo.write(self.struct + " {\n") + for i in self.mbrs: + fo.write("\tuint64_t\t%s;\n" % i.arg) + fo.write("};\n") + fo.write("\n"); + fo.write(self.struct + " *VSC_" + self.name + "_New") + fo.write("(const char *fmt, ...);\n"); + fo.write("void VSCL_" + self.name + "_Destroy") + fo.write("(" + self.struct + "**);\n") + + def emit_c(self): + fon="VSC_" + self.name + ".c" + fo = open(fon, "w") + genhdr(fo, self.name) + fo.write('#include "config.h"\n') + fo.write('#include \n') + fo.write('#include \n') + fo.write('#include \n') + fo.write('#include "common/common.h"\n') + fo.write('#include "VSC_%s.h"\n' % self.name) + self.emit_json(fo) + fo.write("\n") + fo.write(self.struct + "*\n"); + fo.write("VSC_" + self.name + "_New") + fo.write("(const char *fmt, ...)\n"); + fo.write("{\n") + fo.write("\tva_list ap;\n") + fo.write("\t" + self.struct + " *retval;\n") + fo.write("\n") + fo.write("\tva_start(ap, fmt);\n") + fo.write("\tretval = VSC_Alloc") + fo.write('("' + self.name + '", ') + fo.write(self.name + "_jsonlen, ") + fo.write(self.name + "_zjson, ") + fo.write("sizeof " + self.name + "_zjson,\n") + fo.write("\t fmt, ap);\n") + fo.write("\tva_end(ap);\n") + fo.write("\treturn(retval);\n") + fo.write("}\n") + fo.write("\n") + fo.write("void\n") + fo.write("VSCL_" + self.name + "_Destroy") + fo.write("(" + self.struct + "**pp)\n") + fo.write("{\n") + fo.write("\n") + fo.write("\tAN(pp);\n") + fo.write('\tVSC_Destroy("%s", *pp);\n' % self.name) + fo.write("\t*pp = NULL;\n") + fo.write("}\n") + +####################################################################### + +class directive(object): + def __init__(self, s): + ll = s.split("\n") + i = ll.pop(0).split("::", 2) + self.cmd = i[0] + self.arg = i[1].strip() + assert len(self.arg.split()) == 1 + + self.param = {} + while len(ll): + j = ll[0].split(":",2) + if len(j) != 3 or not j[0].isspace(): + break + self.param[j[1]] = j[2].strip() + ll.pop(0) + self.ldoc = ll + + def getdoc(self): + while len(self.ldoc) and self.ldoc[0].strip() == "": + self.ldoc.pop(0) + while len(self.ldoc) and self.ldoc[-1].strip() == "": + self.ldoc.pop(-1) + return self.ldoc + + def moredoc(self, s): + self.getdoc() + self.ldoc += s.split("\n") + + def emit_rst(self, fo): + fo.write("\n.. " + self.cmd + ":: " + self.arg + "\n") + self.emit_rst_doc(fo) + + def emit_rst_doc(self, fo): + fo.write("\n".join(self.ldoc)) + + def emit_h(self, fo): + return + +class rst_vsc_begin(directive): + def __init__(self, s): + super(rst_vsc_begin, self).__init__(s) + + def vscset(self, ss): + ss.append(vscset(self.arg, self)) + +class rst_vsc(directive): + def __init__(self, s): + super(rst_vsc, self).__init__(s) + if "type" not in self.param: + self.param["type"] = "counter" + if "level" not in self.param: + self.param["level"] = "info" + + def emit_rst(self, fo): + fo.write("\n``%s`` - " % self.arg) + fo.write("`%s` - " % self.param["type"]) + fo.write("%s\n" % self.param["level"]) + self.emit_rst_doc(fo) + + def vscset(self, ss): + ss[-1].addmbr(self) + + +class rst_vsc_end(directive): + def __init__(self, s): + super(rst_vsc_end, self).__init__(s) + + def vscset(self, ss): + ss[-1].complete() + +class other(object): + def __init__(self, s): + self.s = s + + def emit_rst(self, fo): + fo.write(self.s) + + def emit_h(self, fo): + return + + def vscset(self, ss): + return + +####################################################################### + +class vsc_file(object): + def __init__(self, fin): + self.c = [] + scs = open(fin).read().split("\n.. ") + self.c.append(other(scs[0])) + ld = None + for i in scs[1:]: + j = i.split(None, 1) + f = { + "varnish_vsc_begin::": rst_vsc_begin, + "varnish_vsc::": rst_vsc, + "varnish_vsc_end::": rst_vsc_end, + }.get(j[0]) + if f is None: + s = "\n.. " + i + o = other(s) + if ld is not None: + ld.moredoc(s) + else: + o = f(i) + ld = o + self.c.append(o) + + self.vscset = [] + for i in self.c: + i.vscset(self.vscset) + + print(self.vscset) + + def emit_h(self): + for i in self.vscset: + i.emit_h() + + def emit_c(self): + for i in self.vscset: + i.emit_c() + + def emit_rst(self, fon): + fo = open(fon, "w") + for i in self.c: + i.emit_rst(fo) + +####################################################################### + +if __name__ == "__main__": + + optlist, args = getopt.getopt(sys.argv[1:], "") + + fo = sys.stdout + + for f, v in optlist: + assert False + if len(args) != 1: + print("Need exactly one filename argument") + exit(2) + + vf = vsc_file(args[0]) + vf.emit_rst("_.rst") + vf.emit_h() + vf.emit_c() From phk at FreeBSD.org Tue May 23 12:11:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 May 2017 14:11:06 +0200 Subject: [master] c122a39 Also move VSC::LCK to The New World Order Message-ID: commit c122a396936906bf583d986fe08d069ac74afb85 Author: Poul-Henning Kamp Date: Tue May 23 12:00:02 2017 +0000 Also move VSC::LCK to The New World Order diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index c03ff70..5586cf4 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -107,8 +107,7 @@ varnishd_SOURCES = \ waiter/cache_waiter_epoll.c \ waiter/cache_waiter_kqueue.c \ waiter/cache_waiter_poll.c \ - waiter/cache_waiter_ports.c \ - VSC_main.c + waiter/cache_waiter_ports.c nodist_varnishd_SOURCES = \ builtin_vcl.c @@ -219,10 +218,21 @@ DISTCLEANFILES += vhp_hufdec.h ####################################################################### VSC_main.c VSC_main.h: $(srcdir)/main.vsc $(top_builddir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_builddir)/lib/libvcc/vsctool.py $(srcdir)/main.vsc + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py $(srcdir)/main.vsc $(varnishd_OBJECTS): VSC_main.h EXTRA_DIST += main.vsc DISTCLEANFILES += VSC_main.c VSC_main.h +nodist_varnishd_SOURCES += VSC_main.c +####################################################################### + +VSC_lck.c VSC_lck.h: $(srcdir)/lck.vsc $(top_builddir)/lib/libvcc/vsctool.py + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py $(srcdir)/lck.vsc + +$(varnishd_OBJECTS): VSC_lck.h + +EXTRA_DIST += lck.vsc +DISTCLEANFILES += VSC_lck.c VSC_lck.h +nodist_varnishd_SOURCES += VSC_lck.c diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index a5fcef8..5256c3f 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -95,7 +95,7 @@ enum { /*--------------------------------------------------------------------*/ -struct VSC_C_lck; +struct VSC_lck; struct ban; struct ban_proto; struct backend; @@ -828,7 +828,7 @@ extern pthread_key_t witness_key; void Lck__Lock(struct lock *lck, const char *p, int l); void Lck__Unlock(struct lock *lck, const char *p, int l); int Lck__Trylock(struct lock *lck, const char *p, int l); -void Lck__New(struct lock *lck, struct VSC_C_lck *, const char *); +void Lck__New(struct lock *lck, struct VSC_lck *, const char *); int Lck__Held(const struct lock *lck); int Lck__Owned(const struct lock *lck); @@ -846,9 +846,9 @@ int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, double); assert(Lck__Owned(a)); \ } while (0) -struct VSC_C_lck *Lck_CreateClass(const char *name); +struct VSC_lck *Lck_CreateClass(const char *name); -#define LOCK(nam) extern struct VSC_C_lck *lck_##nam; +#define LOCK(nam) extern struct VSC_lck *lck_##nam; #include "tbl/locks.h" /* cache_mempool.c */ diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c index bf8151f..eb39cbd 100644 --- a/bin/varnishd/cache/cache_lck.c +++ b/bin/varnishd/cache/cache_lck.c @@ -41,6 +41,8 @@ #include #include +#include "VSC_lck.h" + struct ilck { unsigned magic; #define ILCK_MAGIC 0x7b86c8a5 @@ -48,7 +50,7 @@ struct ilck { pthread_mutex_t mtx; pthread_t owner; const char *w; - struct VSC_C_lck *stat; + struct VSC_lck *stat; }; static pthread_mutexattr_t attr; @@ -214,7 +216,7 @@ Lck_CondWait(pthread_cond_t *cond, struct lock *lck, double when) } void -Lck__New(struct lock *lck, struct VSC_C_lck *st, const char *w) +Lck__New(struct lock *lck, struct VSC_lck *st, const char *w) { struct ilck *ilck; @@ -242,14 +244,13 @@ Lck_Delete(struct lock *lck) FREE_OBJ(ilck); } -struct VSC_C_lck * +struct VSC_lck * Lck_CreateClass(const char *name) { - return(VSM_Alloc(sizeof(struct VSC_C_lck), - VSC_CLASS, VSC_type_lck, name)); + return(VSC_lck_New(name)); } -#define LOCK(nam) struct VSC_C_lck *lck_##nam; +#define LOCK(nam) struct VSC_lck *lck_##nam; #include "tbl/locks.h" void diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 113d0fc..5429a14 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -37,6 +37,8 @@ #include "common/heritage.h" +#include "vend.h" +#include "vgz.h" #include "vsl_priv.h" #include "vmb.h" #include "vtim.h" @@ -471,16 +473,32 @@ VSL_End(struct vsl_log *vsl) /*--------------------------------------------------------------------*/ void * -VSC_Alloc(const char *nm, size_t sj, const unsigned char *zj, size_t szj, +VSC_Alloc(const char *nm, size_t sd, + size_t sj, const unsigned char *zj, size_t szj, const char *fmt, va_list va) { + char *p; + z_stream vz; + (void)nm; - (void)zj; - (void)szj; - (void)sj; (void)fmt; (void)va; - return (0); + + p = VSM_Alloc(sd + sj, VSC_CLASS, nm, fmt); + AN(p); + + memset(p, 0, sd); + vbe64enc(p, sd); // fills .zero + + memset(&vz, 0, sizeof vz); + assert(Z_OK == inflateInit2(&vz, 31)); + vz.next_in = TRUST_ME(zj); + vz.avail_in = szj; + vz.next_out = (void*)(p + sd); + vz.avail_out = sj; + assert(Z_STREAM_END == inflate(&vz, Z_FINISH)); + assert(Z_OK == inflateEnd(&vz)); + return (p); } void @@ -540,10 +558,8 @@ VSM_Init(void) VWMB(); memcpy(vsl_head->marker, VSL_HEAD_MARKER, sizeof vsl_head->marker); - VSC_C_main = VSM_Alloc(sizeof *VSC_C_main, - VSC_CLASS, VSC_type_main, ""); + VSC_C_main = VSC_main_New(""); AN(VSC_C_main); - memset(VSC_C_main, 0, sizeof *VSC_C_main); AZ(pthread_create(&tp, NULL, vsm_cleaner, NULL)); } diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h index 107f5b8..2fbe5d2 100644 --- a/bin/varnishd/common/common.h +++ b/bin/varnishd/common/common.h @@ -91,7 +91,7 @@ void VSM_common_copy(struct vsm_sc *to, const struct vsm_sc *from); void VSM_common_cleaner(struct vsm_sc *sc, struct VSC_C_main *stats); void VSM_common_ageupdate(const struct vsm_sc *sc); -void *VSC_Alloc(const char *, size_t, const unsigned char *, size_t, +void *VSC_Alloc(const char *, size_t, size_t, const unsigned char *, size_t, const char *, va_list); void VSC_Destroy(const char *, void *); diff --git a/bin/varnishd/hash/hash_classic.c b/bin/varnishd/hash/hash_classic.c index 6ad5a0d..e89e411 100644 --- a/bin/varnishd/hash/hash_classic.c +++ b/bin/varnishd/hash/hash_classic.c @@ -38,7 +38,7 @@ #include "hash/hash_slinger.h" -static struct VSC_C_lck *lck_hcl; +static struct VSC_lck *lck_hcl; /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/hash/hash_simple_list.c b/bin/varnishd/hash/hash_simple_list.c index a946d16..ffaa2d5 100644 --- a/bin/varnishd/hash/hash_simple_list.c +++ b/bin/varnishd/hash/hash_simple_list.c @@ -35,7 +35,7 @@ #include "hash/hash_slinger.h" -static struct VSC_C_lck *lck_hsl; +static struct VSC_lck *lck_hsl; /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/lck.vsc b/bin/varnishd/lck.vsc new file mode 100644 index 0000000..936df89 --- /dev/null +++ b/bin/varnishd/lck.vsc @@ -0,0 +1,30 @@ +.. + This is *NOT* a RST file but the syntax has been chosen so + that it may become an RST file at some later date. + + XXX: For now this file must be kept in 100% agreement with + XXX: include/tbl/vsc_fields.h + +.. varnish_vsc_begin:: lck + :oneliner: LOCK COUNTERS (LCK.*) + +.. varnish_vsc:: creat + :type: counter + :level: debug + :oneliner: Created locks + + +.. varnish_vsc:: destroy + :type: counter + :level: debug + :oneliner: Destroyed locks + + +.. varnish_vsc:: locks + :type: counter + :level: debug + :oneliner: Lock Operations + + +.. varnish_vsc_end:: lck + diff --git a/bin/varnishd/main.vsc b/bin/varnishd/main.vsc index f901fca..242681d 100644 --- a/bin/varnishd/main.vsc +++ b/bin/varnishd/main.vsc @@ -753,4 +753,4 @@ to communicate with tools like varnishstat, varnishlog etc. If this counter is not zero, consider increasing the runtime variable vsm_space. -.. varnish_vsc_end:: MAIN +.. varnish_vsc_end:: main diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c index 143ab9d..7de1a34 100644 --- a/bin/varnishd/storage/storage_file.c +++ b/bin/varnishd/storage/storage_file.c @@ -64,7 +64,7 @@ */ #define NBUCKET (128 / 4 + 1) -static struct VSC_C_lck *lck_smf; +static struct VSC_lck *lck_smf; /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/storage/storage_malloc.c b/bin/varnishd/storage/storage_malloc.c index 0361e52..19a37ba 100644 --- a/bin/varnishd/storage/storage_malloc.c +++ b/bin/varnishd/storage/storage_malloc.c @@ -59,7 +59,7 @@ struct sma { struct sma_sc *sc; }; -static struct VSC_C_lck *lck_sma; +static struct VSC_lck *lck_sma; static struct storage * __match_proto__(sml_alloc_f) sma_alloc(const struct stevedore *st, size_t size) diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index 9d2cc92..b4283f3 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -55,7 +55,7 @@ static struct obj_methods smp_oc_realmethods; -static struct VSC_C_lck *lck_smp; +static struct VSC_lck *lck_smp; static void smp_init(void); diff --git a/include/vapi/vsc_int.h b/include/vapi/vsc_int.h index 352c8ba..9d20665 100644 --- a/include/vapi/vsc_int.h +++ b/include/vapi/vsc_int.h @@ -43,7 +43,7 @@ enum VSC_level_e { #include "tbl/vsc_types.h" /* Define the vsc type structs */ -#define VSC_DO(u,l,t,h) struct VSC_C_##l { +#define VSC_DO(u,l,t,h) struct VSC_C_##l { uint64_t zero; #define VSC_F(n,t,l,s,f,v,d,e) t n; #define VSC_DONE(u,l,t) }; #include "tbl/vsc_all.h" diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index afc2ffb..ee70033 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -49,8 +49,8 @@ def genhdr(fo, name): fo.write(' * NB: This file is machine generated, DO NOT EDIT!\n') fo.write(' *\n') fo.write(' * Edit %s.vsc run lib/libvcc/vsctool.py instead.\n' % name) - fo.write(' */\n') - fo.write('\n') + fo.write(' */\n') + fo.write('\n') ####################################################################### @@ -90,11 +90,11 @@ class vscset(object): ed["1line"] = i.param["oneliner"].strip() ed["docs"] = i.getdoc() s=json.dumps(dd, separators=(",",":")) - fo.write("\nstatic const size_t %s_jsonlen = %dL;\n" % + fo.write("\nstatic const size_t vsc_%s_jsonlen = %dL;\n" % (self.name, len(s))) z = gzip_str(s) fo.write("\nstatic const unsigned char"); - fo.write(" %s_zjson[%d] = {\n" % (self.name, len(z))) + fo.write(" vsc_%s_zjson[%d] = {\n" % (self.name, len(z))) bz = bytearray(z) t = "\t" for i in bz: @@ -116,6 +116,7 @@ class vscset(object): fo = open(fon, "w") genhdr(fo, self.name) fo.write(self.struct + " {\n") + fo.write("\tuint64_t\tzero;\n") for i in self.mbrs: fo.write("\tuint64_t\t%s;\n" % i.arg) fo.write("};\n") @@ -135,6 +136,10 @@ class vscset(object): fo.write('#include \n') fo.write('#include "common/common.h"\n') fo.write('#include "VSC_%s.h"\n' % self.name) + fo.write("\n") + fo.write('static const char vsc_%s_name[] = "%s";\n' % + (self.name, self.name.upper())) + self.emit_json(fo) fo.write("\n") fo.write(self.struct + "*\n"); @@ -146,10 +151,11 @@ class vscset(object): fo.write("\n") fo.write("\tva_start(ap, fmt);\n") fo.write("\tretval = VSC_Alloc") - fo.write('("' + self.name + '", ') - fo.write(self.name + "_jsonlen, ") - fo.write(self.name + "_zjson, ") - fo.write("sizeof " + self.name + "_zjson,\n") + fo.write("(vsc_" + self.name + "_name, ") + fo.write("sizeof(" + self.struct + "),\n\t ") + fo.write("vsc_" + self.name + "_jsonlen, ") + fo.write("vsc_" + self.name + "_zjson, ") + fo.write("sizeof vsc_" + self.name + "_zjson,\n") fo.write("\t fmt, ap);\n") fo.write("\tva_end(ap);\n") fo.write("\treturn(retval);\n") @@ -309,6 +315,6 @@ if __name__ == "__main__": exit(2) vf = vsc_file(args[0]) - vf.emit_rst("_.rst") + # vf.emit_rst("_.rst") vf.emit_h() vf.emit_c() From phk at FreeBSD.org Tue May 23 12:11:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 May 2017 14:11:06 +0200 Subject: [master] 3dbbcaf Move VBE stats to New World Order Message-ID: commit 3dbbcaf09dcbb532ca90bcb60dd4e2f93f1d2af7 Author: Poul-Henning Kamp Date: Tue May 23 12:09:02 2017 +0000 Move VBE stats to New World Order diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 5586cf4..9afd301 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -236,3 +236,14 @@ $(varnishd_OBJECTS): VSC_lck.h EXTRA_DIST += lck.vsc DISTCLEANFILES += VSC_lck.c VSC_lck.h nodist_varnishd_SOURCES += VSC_lck.c + +####################################################################### + +VSC_vbe.c VSC_vbe.h: $(srcdir)/vbe.vsc $(top_builddir)/lib/libvcc/vsctool.py + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py $(srcdir)/vbe.vsc + +$(varnishd_OBJECTS): VSC_vbe.h + +EXTRA_DIST += vbe.vsc +DISTCLEANFILES += VSC_vbe.c VSC_vbe.h +nodist_varnishd_SOURCES += VSC_vbe.c diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 3e7ecdb..d9759d8 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -45,6 +45,8 @@ #include "cache_transport.h" #include "http1/cache_http1.h" +#include "VSC_vbe.h" + #define FIND_TMO(tmx, dst, bo, be) \ do { \ CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); \ diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index 0bc73ed..da5ffdc 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -67,7 +67,7 @@ struct backend { const char *admin_health; double health_changed; - struct VSC_C_vbe *vsc; + struct VSC_vbe *vsc; struct tcp_pool *tcp_pool; diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c index e4acf8a..0b31cef 100644 --- a/bin/varnishd/cache/cache_backend_cfg.c +++ b/bin/varnishd/cache/cache_backend_cfg.c @@ -46,6 +46,8 @@ #include "cache_director.h" #include "cache_backend.h" +#include "VSC_vbe.h" + static VTAILQ_HEAD(, backend) backends = VTAILQ_HEAD_INITIALIZER(backends); static VTAILQ_HEAD(, backend) cool_backends = VTAILQ_HEAD_INITIALIZER(cool_backends); @@ -174,8 +176,7 @@ VBE_Event(struct backend *be, enum vcl_event_e ev) CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); if (ev == VCL_EVENT_WARM) { - be->vsc = VSM_Alloc(sizeof *be->vsc, - VSC_CLASS, VSC_type_vbe, be->display_name); + be->vsc = VSC_vbe_New(be->display_name); AN(be->vsc); } diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index c3cfd8c..af3336d 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -53,6 +53,8 @@ #include "cache_director.h" #include "cache_backend.h" +#include "VSC_vbe.h" + /* Default averaging rate, we want something pretty responsive */ #define AVG_RATE 4 diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h index f3591127..8b4c592 100644 --- a/bin/varnishd/http1/cache_http1.h +++ b/bin/varnishd/http1/cache_http1.h @@ -27,6 +27,8 @@ * */ +struct VSC_vbe; + /* cache_http1_fetch.c [V1F] */ int V1F_SendReq(struct worker *, struct busyobj *, uint64_t *ctr, int onlycached); @@ -49,7 +51,7 @@ struct v1p_acct { }; void V1P_Process(struct req *, int fd, struct v1p_acct *); -void V1P_Charge(struct req *, const struct v1p_acct *, struct VSC_C_vbe *); +void V1P_Charge(struct req *, const struct v1p_acct *, struct VSC_vbe *); /* cache_http1_line.c */ void V1L_Chunked(const struct worker *w); diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c index b6a2e30..49a10e5 100644 --- a/bin/varnishd/http1/cache_http1_pipe.c +++ b/bin/varnishd/http1/cache_http1_pipe.c @@ -38,6 +38,8 @@ #include "cache_http1.h" +#include "VSC_vbe.h" + static struct lock pipestat_mtx; static int @@ -61,7 +63,7 @@ rdf(int fd0, int fd1, uint64_t *pcnt) } void -V1P_Charge(struct req *req, const struct v1p_acct *a, struct VSC_C_vbe *b) +V1P_Charge(struct req *req, const struct v1p_acct *a, struct VSC_vbe *b) { AN(b); diff --git a/bin/varnishd/vbe.vsc b/bin/varnishd/vbe.vsc new file mode 100644 index 0000000..f959ca9 --- /dev/null +++ b/bin/varnishd/vbe.vsc @@ -0,0 +1,76 @@ +.. + This is *NOT* a RST file but the syntax has been chosen so + that it may become an RST file at some later date. + + XXX: For now this file must be kept in 100% agreement with + XXX: include/tbl/vsc_fields.h + +.. varnish_vsc_begin:: vbe + :oneliner: BACKEND COUNTERS (VBE.*) + +.. varnish_vsc:: happy + :type: bitmap + :level: info + :oneliner: Happy health probes + +.. varnish_vsc:: bereq_hdrbytes + :type: counter + :level: info + :oneliner: Request header bytes + +Total backend request header bytes sent + +.. varnish_vsc:: bereq_bodybytes + :type: counter + :level: info + :oneliner: Request body bytes + +Total backend request body bytes sent + +.. varnish_vsc:: beresp_hdrbytes + :type: counter + :level: info + :oneliner: Response header bytes + +Total backend response header bytes received + +.. varnish_vsc:: beresp_bodybytes + :type: counter + :level: info + :oneliner: Response body bytes + +Total backend response body bytes received + +.. varnish_vsc:: pipe_hdrbytes + :type: counter + :level: info + :oneliner: Pipe request header bytes + +Total request bytes sent for piped sessions + +.. varnish_vsc:: pipe_out + :type: counter + :level: info + :oneliner: Piped bytes to backend + +Total number of bytes forwarded to backend in pipe sessions + +.. varnish_vsc:: pipe_in + :type: counter + :level: info + :oneliner: Piped bytes from backend + +Total number of bytes forwarded from backend in pipe sessions + +.. varnish_vsc:: conn + :type: gauge + :level: info + :oneliner: Concurrent connections to backend + +.. varnish_vsc:: req + :type: counter + :level: info + :oneliner: Backend requests sent + +.. varnish_vsc_end:: vbe + From phk at FreeBSD.org Tue May 23 12:56:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 May 2017 14:56:05 +0200 Subject: [master] 15e2caa Move the remaining VSC's to The New World Order Message-ID: commit 15e2caa6095d3563fb8d59761c813d607c84bba6 Author: Poul-Henning Kamp Date: Tue May 23 12:34:36 2017 +0000 Move the remaining VSC's to The New World Order diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 9afd301..2b3220f 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -247,3 +247,48 @@ $(varnishd_OBJECTS): VSC_vbe.h EXTRA_DIST += vbe.vsc DISTCLEANFILES += VSC_vbe.c VSC_vbe.h nodist_varnishd_SOURCES += VSC_vbe.c + +####################################################################### + +VSC_sma.c VSC_sma.h: $(srcdir)/sma.vsc $(top_builddir)/lib/libvcc/vsctool.py + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py $(srcdir)/sma.vsc + +$(varnishd_OBJECTS): VSC_sma.h + +EXTRA_DIST += sma.vsc +DISTCLEANFILES += VSC_sma.c VSC_sma.h +nodist_varnishd_SOURCES += VSC_sma.c + +####################################################################### + +VSC_smf.c VSC_smf.h: $(srcdir)/smf.vsc $(top_builddir)/lib/libvcc/vsctool.py + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py $(srcdir)/smf.vsc + +$(varnishd_OBJECTS): VSC_smf.h + +EXTRA_DIST += smf.vsc +DISTCLEANFILES += VSC_smf.c VSC_smf.h +nodist_varnishd_SOURCES += VSC_smf.c + +####################################################################### + +VSC_mempool.c VSC_mempool.h: $(srcdir)/mempool.vsc $(top_builddir)/lib/libvcc/vsctool.py + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py $(srcdir)/mempool.vsc + +$(varnishd_OBJECTS): VSC_mempool.h + +EXTRA_DIST += mempool.vsc +DISTCLEANFILES += VSC_mempool.c VSC_mempool.h +nodist_varnishd_SOURCES += VSC_mempool.c + +####################################################################### + +VSC_mgt.c VSC_mgt.h: $(srcdir)/mgt.vsc $(top_builddir)/lib/libvcc/vsctool.py + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py $(srcdir)/mgt.vsc + +$(varnishd_OBJECTS): VSC_mgt.h + +EXTRA_DIST += mgt.vsc +DISTCLEANFILES += VSC_mgt.c VSC_mgt.h +nodist_varnishd_SOURCES += VSC_mgt.c + diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 5256c3f..6711c89 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -977,8 +977,6 @@ void SES_Set_String_Attr(struct sess *sp, enum sess_attr a, const char *src); const char *SES_Get_String_Attr(const struct sess *sp, enum sess_attr a); /* cache_shmlog.c */ -void *VSM_Alloc(unsigned size, const char *class, const char *type, - const char *ident); void VSM_Free(void *ptr); #ifdef VSL_ENDMARKER void VSLv(enum VSL_tag_e tag, uint32_t vxid, const char *fmt, va_list va); diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c index 56edb20..b021150 100644 --- a/bin/varnishd/cache/cache_mempool.c +++ b/bin/varnishd/cache/cache_mempool.c @@ -38,6 +38,8 @@ #include "vtim.h" +#include "VSC_mempool.h" + struct memitem { unsigned magic; #define MEMITEM_MAGIC 0x42e55401 @@ -58,7 +60,7 @@ struct mempool { volatile struct poolparam *param; volatile unsigned *cur_size; uint64_t live; - struct VSC_C_mempool *vsc; + struct VSC_mempool *vsc; unsigned n_pool; pthread_t thread; double t_now; @@ -236,8 +238,7 @@ MPL_New(const char *name, VTAILQ_INIT(&mpl->surplus); Lck_New(&mpl->mtx, lck_mempool); /* XXX: prealloc min_pool */ - mpl->vsc = VSM_Alloc(sizeof *mpl->vsc, - VSC_CLASS, VSC_type_mempool, mpl->name + 4); + mpl->vsc = VSC_mempool_New(mpl->name + 4); AN(mpl->vsc); AZ(pthread_create(&mpl->thread, NULL, mpl_guard, mpl)); AZ(pthread_detach(mpl->thread)); diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 5429a14..8ff321d 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -55,6 +55,9 @@ static ssize_t vsl_segsize; struct VSC_main *VSC_C_main; +static void *VSM_Alloc(unsigned size, const char *class, const char *type, + const char *ident); + static void vsl_sanity(const struct vsl_log *vsl) { @@ -566,7 +569,7 @@ VSM_Init(void) /*--------------------------------------------------------------------*/ -void * +static void * VSM_Alloc(unsigned size, const char *class, const char *type, const char *ident) { diff --git a/bin/varnishd/mempool.vsc b/bin/varnishd/mempool.vsc new file mode 100644 index 0000000..b947031 --- /dev/null +++ b/bin/varnishd/mempool.vsc @@ -0,0 +1,77 @@ +.. + This is *NOT* a RST file but the syntax has been chosen so + that it may become an RST file at some later date. + + XXX: For now this file must be kept in 100% agreement with + XXX: include/tbl/vsc_fields.h + +.. varnish_vsc_begin:: mempool + :oneliner: MEMORY POOL COUNTERS (MEMPOOL.*) + +.. varnish_vsc:: live + :type: gauge + :level: debug + :oneliner: In use + + +.. varnish_vsc:: pool + :type: gauge + :level: debug + :oneliner: In Pool + + +.. varnish_vsc:: sz_wanted + :type: gauge + :level: debug + :oneliner: Size requested + + +.. varnish_vsc:: sz_actual + :type: gauge + :level: debug + :oneliner: Size allocated + + +.. varnish_vsc:: allocs + :type: counter + :level: debug + :oneliner: Allocations + + +.. varnish_vsc:: frees + :type: counter + :level: debug + :oneliner: Frees + + +.. varnish_vsc:: recycle + :type: counter + :level: debug + :oneliner: Recycled from pool + + +.. varnish_vsc:: timeout + :type: counter + :level: debug + :oneliner: Timed out from pool + + +.. varnish_vsc:: toosmall + :type: counter + :level: debug + :oneliner: Too small to recycle + + +.. varnish_vsc:: surplus + :type: counter + :level: debug + :oneliner: Too many for pool + + +.. varnish_vsc:: randry + :type: counter + :level: debug + :oneliner: Pool ran dry + + +.. varnish_vsc_end:: mempool diff --git a/bin/varnishd/mgt.vsc b/bin/varnishd/mgt.vsc new file mode 100644 index 0000000..d1d7984 --- /dev/null +++ b/bin/varnishd/mgt.vsc @@ -0,0 +1,61 @@ +.. + This is *NOT* a RST file but the syntax has been chosen so + that it may become an RST file at some later date. + + XXX: For now this file must be kept in 100% agreement with + XXX: include/tbl/vsc_fields.h + +.. varnish_vsc_begin:: mgt + :class: MGT + :oneliner: MANAGEMENT PROCESS COUNTERS (MGT.*) + +.. varnish_vsc:: uptime + :type: counter + :level: info + :oneliner: Management process uptime + +Uptime in seconds of the management process + +.. varnish_vsc:: child_start + :type: counter + :level: diag + :oneliner: Child process started + +Number of times the child process has been started + +.. varnish_vsc:: child_exit + :type: counter + :level: diag + :oneliner: Child process normal exit + +Number of times the child process has been cleanly stopped + +.. varnish_vsc:: child_stop + :type: counter + :level: diag + :oneliner: Child process unexpected exit + +Number of times the child process has exited with an unexpected return code + +.. varnish_vsc:: child_died + :type: counter + :level: diag + :oneliner: Child process died (signal) + +Number of times the child process has died due to signals + +.. varnish_vsc:: child_dump + :type: counter + :level: diag + :oneliner: Child process core dumped + +Number of times the child process has produced core dumps + +.. varnish_vsc:: child_panic + :type: counter + :level: diag + :oneliner: Child process panic + +Number of times the management process has caught a child panic + +.. varnish_vsc_end:: mgt diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index 3f8ff11..5d81642 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -38,6 +38,8 @@ #include "common/common.h" #include "common/com_params.h" +#include "VSC_mgt.h" + struct cli; struct parspec; struct vcc; @@ -130,8 +132,8 @@ extern const struct jail_tech jail_tech_unix; extern const struct jail_tech jail_tech_solaris; /* mgt_main.c */ -extern struct VSC_C_mgt *VSC_C_mgt; -extern struct VSC_C_mgt static_VSC_C_mgt; +extern struct VSC_mgt *VSC_C_mgt; +extern struct VSC_mgt static_VSC_C_mgt; struct choice { const char *name; const void *ptr; diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 1a50c51..f0797a7 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -63,8 +63,8 @@ pid_t mgt_pid; struct vev_base *mgt_evb; int exit_status = 0; struct vsb *vident; -struct VSC_C_mgt static_VSC_C_mgt; -struct VSC_C_mgt *VSC_C_mgt; +struct VSC_mgt static_VSC_C_mgt; +struct VSC_mgt *VSC_C_mgt; static int I_fd = -1; static char Cn_arg[] = "/tmp/varnishd_C_XXXXXXX"; diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c index 771080f..914a5ef 100644 --- a/bin/varnishd/mgt/mgt_shmem.c +++ b/bin/varnishd/mgt/mgt_shmem.c @@ -193,8 +193,7 @@ mgt_SHM_Create(void) AN(heritage.panic_str); /* Copy management counters to shm and update pointer */ - VSC_C_mgt = VSM_common_alloc(heritage.vsm, - sizeof *VSC_C_mgt, VSC_CLASS, VSC_type_mgt, ""); + VSC_C_mgt = VSC_mgt_New(""); AN(VSC_C_mgt); *VSC_C_mgt = static_VSC_C_mgt; diff --git a/bin/varnishd/sma.vsc b/bin/varnishd/sma.vsc new file mode 100644 index 0000000..da4e18a --- /dev/null +++ b/bin/varnishd/sma.vsc @@ -0,0 +1,60 @@ +.. + This is *NOT* a RST file but the syntax has been chosen so + that it may become an RST file at some later date. + + XXX: For now this file must be kept in 100% agreement with + XXX: include/tbl/vsc_fields.h + +.. varnish_vsc_begin:: sma + :oneliner: MALLOC STORAGE COUNTERS (SMA.*) + +.. varnish_vsc:: c_req + :type: counter + :level: info + :oneliner: Allocator requests + +Number of times the storage has been asked to provide a storage segment. + +.. varnish_vsc:: c_fail + :type: counter + :level: info + :oneliner: Allocator failures + +Number of times the storage has failed to provide a storage segment. + +.. varnish_vsc:: c_bytes + :type: counter + :level: info + :oneliner: Bytes allocated + +Number of total bytes allocated by this storage. + +.. varnish_vsc:: c_freed + :type: counter + :level: info + :oneliner: Bytes freed + +Number of total bytes returned to this storage. + +.. varnish_vsc:: g_alloc + :type: gauge + :level: info + :oneliner: Allocations outstanding + +Number of storage allocations outstanding. + +.. varnish_vsc:: g_bytes + :type: gauge + :level: info + :oneliner: Bytes outstanding + +Number of bytes allocated from the storage. + +.. varnish_vsc:: g_space + :type: gauge + :level: info + :oneliner: Bytes available + +Number of bytes left in the storage. + +.. varnish_vsc_end:: sma diff --git a/bin/varnishd/smf.vsc b/bin/varnishd/smf.vsc new file mode 100644 index 0000000..29f38d9 --- /dev/null +++ b/bin/varnishd/smf.vsc @@ -0,0 +1,78 @@ +.. + This is *NOT* a RST file but the syntax has been chosen so + that it may become an RST file at some later date. + + XXX: For now this file must be kept in 100% agreement with + XXX: include/tbl/vsc_fields.h + +.. varnish_vsc_begin:: smf + :oneliner: FILE STORAGE COUNTERS (SMF.*) + +.. varnish_vsc:: c_req + :type: counter + :level: info + :oneliner: Allocator requests + +Number of times the storage has been asked to provide a storage segment. + +.. varnish_vsc:: c_fail + :type: counter + :level: info + :oneliner: Allocator failures + +Number of times the storage has failed to provide a storage segment. + +.. varnish_vsc:: c_bytes + :type: counter + :level: info + :oneliner: Bytes allocated + +Number of total bytes allocated by this storage. + +.. varnish_vsc:: c_freed + :type: counter + :level: info + :oneliner: Bytes freed + +Number of total bytes returned to this storage. + +.. varnish_vsc:: g_alloc + :type: gauge + :level: info + :oneliner: Allocations outstanding + +Number of storage allocations outstanding. + +.. varnish_vsc:: g_bytes + :type: gauge + :level: info + :oneliner: Bytes outstanding + +Number of bytes allocated from the storage. + +.. varnish_vsc:: g_space + :type: gauge + :level: info + :oneliner: Bytes available + +Number of bytes left in the storage. + +.. varnish_vsc:: g_smf + :type: gauge + :level: info + :oneliner: N struct smf + + +.. varnish_vsc:: g_smf_frag + :type: gauge + :level: info + :oneliner: N small free smf + + +.. varnish_vsc:: g_smf_large + :type: gauge + :level: info + :oneliner: N large free smf + + +.. varnish_vsc_end:: smf diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c index 7de1a34..675bf62 100644 --- a/bin/varnishd/storage/storage_file.c +++ b/bin/varnishd/storage/storage_file.c @@ -45,6 +45,8 @@ #include "vnum.h" #include "vfil.h" +#include "VSC_smf.h" + #ifndef MAP_NOCORE #define MAP_NOCORE 0 /* XXX Linux */ #endif @@ -91,7 +93,7 @@ struct smf_sc { unsigned magic; #define SMF_SC_MAGIC 0x52962ee7 struct lock mtx; - struct VSC_C_smf *stats; + struct VSC_smf *stats; const char *filename; int fd; @@ -410,8 +412,7 @@ smf_open(struct stevedore *st) if (lck_smf == NULL) lck_smf = Lck_CreateClass("smf"); CAST_OBJ_NOTNULL(sc, st->priv, SMF_SC_MAGIC); - sc->stats = VSM_Alloc(sizeof *sc->stats, - VSC_CLASS, VSC_type_smf, st->ident); + sc->stats = VSC_smf_New(st->ident); Lck_New(&sc->mtx, lck_smf); Lck_Lock(&sc->mtx); smf_open_chunk(sc, sc->filesize, 0, &fail, &sum); diff --git a/bin/varnishd/storage/storage_malloc.c b/bin/varnishd/storage/storage_malloc.c index 19a37ba..420b925 100644 --- a/bin/varnishd/storage/storage_malloc.c +++ b/bin/varnishd/storage/storage_malloc.c @@ -42,13 +42,15 @@ #include "vrt.h" #include "vnum.h" +#include "VSC_sma.h" + struct sma_sc { unsigned magic; #define SMA_SC_MAGIC 0x1ac8a345 struct lock sma_mtx; size_t sma_max; size_t sma_alloc; - struct VSC_C_sma *stats; + struct VSC_sma *stats; }; struct sma { @@ -211,8 +213,7 @@ sma_open(struct stevedore *st) lck_sma = Lck_CreateClass("sma"); CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC); Lck_New(&sma_sc->sma_mtx, lck_sma); - sma_sc->stats = VSM_Alloc(sizeof *sma_sc->stats, - VSC_CLASS, VSC_type_sma, st->ident); + sma_sc->stats = VSC_sma_New(st->ident); memset(sma_sc->stats, 0, sizeof *sma_sc->stats); if (sma_sc->sma_max != SIZE_MAX) sma_sc->stats->g_space = sma_sc->sma_max; From phk at FreeBSD.org Tue May 23 12:56:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 May 2017 14:56:05 +0200 Subject: [master] cbda52a We need VSM_Alloc() for test c00083 for now. Message-ID: commit cbda52aea7bc40a642d03e1fd75f68aaf57fb6cc Author: Poul-Henning Kamp Date: Tue May 23 12:47:07 2017 +0000 We need VSM_Alloc() for test c00083 for now. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 6711c89..5256c3f 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -977,6 +977,8 @@ void SES_Set_String_Attr(struct sess *sp, enum sess_attr a, const char *src); const char *SES_Get_String_Attr(const struct sess *sp, enum sess_attr a); /* cache_shmlog.c */ +void *VSM_Alloc(unsigned size, const char *class, const char *type, + const char *ident); void VSM_Free(void *ptr); #ifdef VSL_ENDMARKER void VSLv(enum VSL_tag_e tag, uint32_t vxid, const char *fmt, va_list va); diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 8ff321d..5429a14 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -55,9 +55,6 @@ static ssize_t vsl_segsize; struct VSC_main *VSC_C_main; -static void *VSM_Alloc(unsigned size, const char *class, const char *type, - const char *ident); - static void vsl_sanity(const struct vsl_log *vsl) { @@ -569,7 +566,7 @@ VSM_Init(void) /*--------------------------------------------------------------------*/ -static void * +void * VSM_Alloc(unsigned size, const char *class, const char *type, const char *ident) { From phk at FreeBSD.org Tue May 23 17:05:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 May 2017 19:05:06 +0200 Subject: [master] 5739314 Add a tiny JSON parser for private use by libvarnishapi. Message-ID: commit 5739314c680ee7fb58ff394b4cd35acdb24411e7 Author: Poul-Henning Kamp Date: Tue May 23 17:03:33 2017 +0000 Add a tiny JSON parser for private use by libvarnishapi. diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am index 769e658..60a7661 100644 --- a/lib/libvarnishapi/Makefile.am +++ b/lib/libvarnishapi/Makefile.am @@ -12,6 +12,8 @@ lib_LTLIBRARIES = libvarnishapi.la libvarnishapi_la_LDFLAGS = $(AM_LDFLAGS) -version-info 1:6:0 libvarnishapi_la_SOURCES = \ + vjsn.c \ + vjsn.h \ vsm_api.h \ vsl_api.h \ vxp.h \ diff --git a/lib/libvarnishapi/flint.lnt b/lib/libvarnishapi/flint.lnt index 59ad7b6..2d2538e 100644 --- a/lib/libvarnishapi/flint.lnt +++ b/lib/libvarnishapi/flint.lnt @@ -1,5 +1,7 @@ +fan +-esym(759, VJSN_*) // could be moved hdr->mod +-esym(765, VJSN_*) // could be made static -esym(788, vex_rhs_e::*) -esym(788, VSL_transaction_e::*) -esym(769, vex_rhs_e::VEX__UNSET) diff --git a/lib/libvarnishapi/vjsn.c b/lib/libvarnishapi/vjsn.c new file mode 100644 index 0000000..d3d5f30 --- /dev/null +++ b/lib/libvarnishapi/vjsn.c @@ -0,0 +1,561 @@ +/*- + * Copyright (c) 2017 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "vas.h" +#include "miniobj.h" +#include "vqueue.h" +#include "vjsn.h" + +const char VJSN_OBJECT[] = "object"; +const char VJSN_ARRAY[] = "array"; +const char VJSN_NUMBER[] = "number"; +const char VJSN_STRING[] = "string"; +const char VJSN_TRUE[] = "true"; +const char VJSN_FALSE[] = "false"; +const char VJSN_NULL[] = "null"; + +#define VJSN_EXPECT(js, xxx, ret) \ + do { \ + AZ(js->err); \ + if (*((js)->ptr) != xxx) { \ + js->err = "Expected " #xxx " not found."; \ + return (ret); \ + } else { \ + *js->ptr++ = '\0'; \ + } \ + } while (0) + +static struct vjsn_val *vjsn_value(struct vjsn *); + +static struct vjsn_val * +vjsn_val_new(const char *type) +{ + struct vjsn_val *jsv; + + ALLOC_OBJ(jsv, VJSN_VAL_MAGIC); + AN(jsv); + VTAILQ_INIT(&jsv->children); + jsv->type = type; + return (jsv); +} + +static void +vjsn_val_delete(struct vjsn_val *jsv) +{ + struct vjsn_val *jsve; + + CHECK_OBJ_NOTNULL(jsv, VJSN_VAL_MAGIC); + do { + jsve = VTAILQ_FIRST(&jsv->children); + if (jsve != NULL) { + VTAILQ_REMOVE(&jsv->children, jsve, list); + vjsn_val_delete(jsve); + } + } while (jsve != NULL); + FREE_OBJ(jsv); +} + +static void +vjsn_delete(struct vjsn **jp) +{ + struct vjsn *js; + + AN(jp); + js = *jp; + *jp = NULL; + CHECK_OBJ_NOTNULL(js, VJSN_MAGIC); + if (js->value != NULL) + vjsn_val_delete(js->value); + free(js->raw); + FREE_OBJ(js); +} + +static void +vjsn_skip_ws(struct vjsn *js) +{ + char c; + + while (1) { + c = js->ptr[0]; + if (c == 0x09 || c == 0x0a || c == 0x0d || c == 0x20) { + *js->ptr++ = '\0'; + continue; + } +#ifdef VJSN_COMMENTS + if (c == '/' && js->ptr[1] == '*') { + js->ptr += 2; + while (js->ptr[0] != '*' || js->ptr[1] != '/') + js->ptr++; + js->ptr += 2; + continue; + } +#endif + return; + } +} + +static unsigned +vjsn_unumber(struct vjsn *js) +{ + unsigned u = 0; + char c; + int i; + + VJSN_EXPECT(js, '\\', 0); + VJSN_EXPECT(js, 'u', 0); + for(i = 0; i < 4; i++) { + u <<= 4; + c = *js->ptr; + if (c >= '0' && c <= '9') + u |= c - '0'; /*lint !e737 */ + else if (c >= 'A' && c <= 'F') + u |= c - '7'; /*lint !e737 */ + else if (c >= 'a' && c <= 'f') + u |= c - 'W'; /*lint !e737 */ + else { + js->err = "Illegal \\uXXXX sequence"; + return (0); + } + js->ptr++; + } + return (u); +} + +static void +vjsn_unicode(struct vjsn *js, char **d) +{ + unsigned u1, u2; + + u1 = vjsn_unumber(js); + + if (u1 >= 0xd800 && u1 <= 0xdc00) { + u2 = vjsn_unumber(js); + if (u2 < 0xdc00 || u2 > 0xdfff) { + js->err = "Bad UTF-16 Surrogate pair"; + return; + } + u1 -= 0xd800; + u2 -= 0xdc00; + u1 <<= 10; + u1 |= u2; + } + /*lint -save -e734 -e713 */ + if (u1 < 0x80) + *(*d)++ = u1; + else if (u1 < 0x800) { + *(*d)++ = 0xc0 + u1 / 64; + *(*d)++ = 0x80 + u1 % 64; + } else if (u1 - 0xd800u < 0x800) { + js->err = "Bad UNICODE point"; + } else if (u1 < 0x10000) { + *(*d)++ = 0xe0 + u1 / 4096; + *(*d)++ = 0x80 + u1 / 64 % 64; + *(*d)++ = 0x80 + u1 % 64; + } else if (u1 < 0x110000) { + *(*d)++ = 0xf0 + u1 / 262144; + *(*d)++ = 0x80 + u1 / 4096 % 64; + *(*d)++ = 0x80 + u1 / 64 % 64; + *(*d)++ = 0x80 + u1 % 64; + } else { + js->err = "Bad UNICODE point"; + } + /*lint -restore */ +} + +static char * +vjsn_string(struct vjsn *js) +{ + char *p, *b; + + vjsn_skip_ws(js); + VJSN_EXPECT(js, '"', NULL); + b = p = js->ptr; + while (*js->ptr != '"') { + if (*js->ptr == '\0') { + js->err = "Unterminate string"; + return (NULL); + } + if (*js->ptr >= 0x00 && *js->ptr <= 0x1f) { + js->err = "unescaped control char in string"; + return (NULL); + } + if (*js->ptr != '\\') { + *p++ = *js->ptr++; + continue; + } + switch(js->ptr[1]) { + case '\\': + case '/': + case '"': *p++ = js->ptr[1]; js->ptr += 2; break; + case 'b': *p++ = 0x08; js->ptr += 2; break; + case 'f': *p++ = 0x0c; js->ptr += 2; break; + case 't': *p++ = 0x09; js->ptr += 2; break; + case 'n': *p++ = 0x0a; js->ptr += 2; break; + case 'r': *p++ = 0x0d; js->ptr += 2; break; + case 'u': + vjsn_unicode(js, &p); + if(js->err != NULL) + return(NULL); + break; + default: + js->err = "Bad string escape"; + return (NULL); + } + } + VJSN_EXPECT(js, '"', NULL); + *p = '\0'; + return (b); +} + +static struct vjsn_val * +vjsn_object(struct vjsn *js) +{ + struct vjsn_val *jsv, *jsve; + char *s; + + VJSN_EXPECT(js, '{', NULL); + + jsv = vjsn_val_new(VJSN_OBJECT); + AN(jsv); + + vjsn_skip_ws(js); + if (*js->ptr != '}') { + while (1) { + s = vjsn_string(js); + if (js->err != NULL) + return (jsv); + vjsn_skip_ws(js); + VJSN_EXPECT(js, ':', jsv); + jsve = vjsn_value(js); + if (js->err != NULL) + return (jsv); + CHECK_OBJ_NOTNULL(jsve, VJSN_VAL_MAGIC); + jsve->name = s; + VTAILQ_INSERT_TAIL(&jsv->children, jsve, list); + vjsn_skip_ws(js); + if(*js->ptr == '}') + break; + VJSN_EXPECT(js, ',', jsv); + } + } + VJSN_EXPECT(js, '}', jsv); + return (jsv); +} + +static struct vjsn_val * +vjsn_array(struct vjsn *js) +{ + struct vjsn_val *jsv, *jsve; + + VJSN_EXPECT(js, '[', NULL); + + jsv = vjsn_val_new(VJSN_ARRAY); + AN(jsv); + + vjsn_skip_ws(js); + if (*js->ptr != ']') { + while (1) { + jsve = vjsn_value(js); + if (js->err != NULL) + return (jsv); + CHECK_OBJ_NOTNULL(jsve, VJSN_VAL_MAGIC); + VTAILQ_INSERT_TAIL(&jsv->children, jsve, list); + vjsn_skip_ws(js); + if(*js->ptr == ']') + break; + VJSN_EXPECT(js, ',', jsv); + } + } + VJSN_EXPECT(js, ']', jsv); + return (jsv); +} + +static struct vjsn_val * +vjsn_number(struct vjsn *js) +{ + struct vjsn_val *jsv; + + jsv = vjsn_val_new(VJSN_NUMBER); + AN(jsv); + + jsv->value = js->ptr; + + if (*js->ptr == '-') + js->ptr++; + if (*js->ptr < '0' || *js->ptr > '9') { + js->err = "Bad number"; + return (jsv); + } + if (*js->ptr == '0' && js->ptr[1] >= '0' && js->ptr[1] <= '9') { + js->err = "Bad number"; + return (jsv); + } + while (*js->ptr >= '0' && *js->ptr <= '9') + js->ptr++; + if (*js->ptr == '.') { + js->ptr++; + if (*js->ptr < '0' || *js->ptr > '9') { + js->err = "Bad number"; + return (jsv); + } + while (*js->ptr >= '0' && *js->ptr <= '9') + js->ptr++; + } + if (*js->ptr == 'e' || *js->ptr == 'E') { + js->ptr++; + if (*js->ptr == '-' || *js->ptr == '+') + js->ptr++; + if (*js->ptr < '0' || *js->ptr > '9') { + js->err = "Bad number"; + return (jsv); + } + while (*js->ptr >= '0' && *js->ptr <= '9') + js->ptr++; + } + return (jsv); +} + +static struct vjsn_val * +vjsn_value(struct vjsn *js) +{ + struct vjsn_val *jsv; + + AZ(js->err); + vjsn_skip_ws(js); + if (*js->ptr == '{') + return (vjsn_object(js)); + if (*js->ptr== '[') + return (vjsn_array(js)); + if (*js->ptr== '"') { + jsv = vjsn_val_new(VJSN_STRING); + jsv->value = vjsn_string(js); + if (js->err != NULL) + return (jsv); + AN(jsv->value); + return (jsv); + } + if (!memcmp(js->ptr, "true", 4)) { + js->ptr += 4; + return (vjsn_val_new(VJSN_TRUE)); + } + if (!memcmp(js->ptr, "false", 4)) { + js->ptr += 5; + return (vjsn_val_new(VJSN_FALSE)); + } + if (!memcmp(js->ptr, "null", 4)) { + js->ptr += 4; + return (vjsn_val_new(VJSN_NULL)); + } + if (*js->ptr == '-' || (*js->ptr >= '0' && *js->ptr <= '9')) + return (vjsn_number(js)); + js->err = "Unrecognized value"; + return (NULL); +} + +struct vjsn * +vjsn_parse(const char *src, const char **err) +{ + struct vjsn *js; + char *p, *e; + + AN(src); + + AN(err); + *err = NULL; + + p = strdup(src); + AN(p); + e = strchr(p, '\0'); + AN(e); + + ALLOC_OBJ(js, VJSN_MAGIC); + AN(js); + js->raw = p; + js->ptr = p; + + js->value = vjsn_value(js); + if (js->err != NULL) { + *err = js->err; + vjsn_delete(&js); + return (NULL); + } + + vjsn_skip_ws(js); + if (js->ptr != e) { + *err = "Garbage after value"; + vjsn_delete(&js); + return (NULL); + } + return (js); +} + +static void +vjsn_dump_i(const struct vjsn_val *jsv, FILE *fo, int indent) +{ + struct vjsn_val *jsve; + + printf("%*s", indent, ""); + if (jsv->name != NULL) + printf("[\"%s\"]: ", jsv->name); + printf("{%s}", jsv->type); + if (jsv->value != NULL) { + if (strlen(jsv->value) < 20) + printf(" <%s", jsv->value); + else + printf(" <%.10s[...#%zu]", + jsv->value, strlen(jsv->value) - 10L); + printf(">"); + } + printf("\n"); + VTAILQ_FOREACH(jsve, &jsv->children, list) + vjsn_dump_i(jsve, fo, indent + 2); +} + +void +vjsn_dump(const struct vjsn *js, FILE *fo) +{ + + CHECK_OBJ_NOTNULL(js, VJSN_MAGIC); + AN(fo); + vjsn_dump_i(js->value, fo, 0); +} + +#ifdef VJSN_TEST + +static struct vjsn * +vjsn_file(const char *fn, const char **err) +{ + int fd; + struct stat st; + char *p, *e; + ssize_t sz; + struct vjsn *js; + + AN(fn); + AN(err); + *err = NULL; + fd = open(fn, O_RDONLY); + if (fd < 0) { + *err = strerror(errno); + return (NULL); + } + AZ(fstat(fd, &st)); + if (!S_ISREG(st.st_mode)) { + AZ(close(fd)); + *err = "Not a regular file"; + return (NULL); + } + + if (st.st_size == 0) { + AZ(close(fd)); + *err = "Empty file"; + return (NULL); + } + + p = malloc(st.st_size + 1); + AN(p); + + sz = read(fd, p, st.st_size + 1); + if (sz < 0) { + *err = strerror(errno); + AZ(close(fd)); + free(p); + return (NULL); + } + AZ(close(fd)); + + if (sz < st.st_size) { + free(p); + *err = "Short read"; + return (NULL); + } + + if (sz > st.st_size) { + free(p); + *err = "Long read"; + return (NULL); + } + + p[sz] = '\0'; + e = p + sz; + + ALLOC_OBJ(js, VJSN_MAGIC); + AN(js); + js->raw = p; + js->ptr = p; + + js->value = vjsn_value(js); + if (js->err != NULL) { + *err = js->err; + vjsn_delete(&js); + return (NULL); + } + + vjsn_skip_ws(js); + if (js->ptr != e) { + *err = "Garbage after value"; + vjsn_delete(&js); + return (NULL); + } + return (js); +} + + +static const char * const fnx = "tst.vjsn"; + +int +main(int argc, char **argv) +{ + struct vjsn *js; + const char *err; + + if (argc == 1) + js = vjsn_file(fnx, &err); + else + js = vjsn_file(argv[1], &err); + if (err != NULL) { + fprintf(stderr, "ERROR: %s\n", err); + AZ(js); + return (1); + } else { + if (0) + vjsn_dump(js, stdout); + vjsn_delete(&js); + } + return (0); +} + +#endif diff --git a/lib/libvarnishapi/vjsn.h b/lib/libvarnishapi/vjsn.h new file mode 100644 index 0000000..91f0e4f --- /dev/null +++ b/lib/libvarnishapi/vjsn.h @@ -0,0 +1,58 @@ +/*- + * Copyright (c) 2017 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +extern const char VJSN_OBJECT[]; +extern const char VJSN_ARRAY[]; +extern const char VJSN_NUMBER[]; +extern const char VJSN_STRING[]; +extern const char VJSN_TRUE[]; +extern const char VJSN_FALSE[]; +extern const char VJSN_NULL[]; + +struct vjsn_val { + unsigned magic; +#define VJSN_VAL_MAGIC 0x08a06b80 + const char *type; + const char *name; + VTAILQ_ENTRY(vjsn_val) list; + VTAILQ_HEAD(,vjsn_val) children; + char *value; +}; + +struct vjsn { + unsigned magic; +#define VJSN_MAGIC 0x86a7f02b + + char *raw; + char *ptr; + struct vjsn_val *value; + const char *err; +}; + +struct vjsn *vjsn_parse(const char *, const char **); +void vjsn_dump(const struct vjsn *js, FILE *fo); From phk at FreeBSD.org Tue May 23 18:20:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 May 2017 20:20:05 +0200 Subject: [master] e1cde63 Try to fix solaris compile nit Message-ID: commit e1cde638076b386c0e03dd5a7e93fabc87db296a Author: Poul-Henning Kamp Date: Tue May 23 18:19:02 2017 +0000 Try to fix solaris compile nit diff --git a/lib/libvarnishapi/vjsn.c b/lib/libvarnishapi/vjsn.c index d3d5f30..4a8b001 100644 --- a/lib/libvarnishapi/vjsn.c +++ b/lib/libvarnishapi/vjsn.c @@ -436,7 +436,7 @@ vjsn_dump_i(const struct vjsn_val *jsv, FILE *fo, int indent) printf(" <%s", jsv->value); else printf(" <%.10s[...#%zu]", - jsv->value, strlen(jsv->value) - 10L); + jsv->value, strlen(jsv->value + 10)); printf(">"); } printf("\n"); diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 725a5f7..08948d6 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -34,6 +34,7 @@ #include #include +#include #include #include @@ -41,7 +42,9 @@ #include "vas.h" #include "miniobj.h" #include "vqueue.h" +#include "vjsn.h" #include "vsb.h" +#include "vend.h" #include "vapi/vsc.h" #include "vapi/vsm.h" @@ -314,9 +317,25 @@ vsc_add_pt(struct vsc *vsc, const volatile void *ptr, */ static void +vsc_build_old_vf_list(struct vsc *vsc) +{ +#define VSC_TYPE_F(n,t,l,e,d) \ + if (!strcmp(vsc->iter_fantom.type, t)) \ + vsc_add_vf(vsc, &vsc->iter_fantom, \ + &VSC_type_desc_##n, VSC_type_order_##n); +#include "tbl/vsc_types.h" +} + +#include + +static void vsc_build_vf_list(struct VSM_data *vd) { + uint64_t u; struct vsc *vsc = vsc_setup(vd); + struct vjsn *vj; + const char *p; + const char *e; vsc_delete_pt_list(&vsc->pt_list); vsc_delete_vf_list(&vsc->vf_list); @@ -324,11 +343,20 @@ vsc_build_vf_list(struct VSM_data *vd) VSM_FOREACH(&vsc->iter_fantom, vd) { if (strcmp(vsc->iter_fantom.class, VSC_CLASS)) continue; -#define VSC_TYPE_F(n,t,l,e,d) \ - if (!strcmp(vsc->iter_fantom.type, t)) \ - vsc_add_vf(vsc, &vsc->iter_fantom, \ - &VSC_type_desc_##n, VSC_type_order_##n); -#include "tbl/vsc_types.h" + u = vbe64dec(vsc->iter_fantom.b); + vsc_build_old_vf_list(vsc); + if (u == 0) { + fprintf(stderr, "%s has no JSON\n", vsc->iter_fantom.type); + exit(2); + } + p = (char*)vsc->iter_fantom.b + u; + vj = vjsn_parse(p, &e); + if (e != NULL) { + fprintf(stderr, "%s\n", p); + fprintf(stderr, "JSON ERROR %s\n", e); + } + AZ(e); + //vjsn_dump(vj, stdout); } } From phk at FreeBSD.org Tue May 23 18:21:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 May 2017 20:21:06 +0200 Subject: [master] 5cc0e27 Revert debugging which snug into previous commit Message-ID: commit 5cc0e2705938fd1f2b1ef2f3954db65991cdee02 Author: Poul-Henning Kamp Date: Tue May 23 18:20:41 2017 +0000 Revert debugging which snug into previous commit diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 08948d6..725a5f7 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -34,7 +34,6 @@ #include #include -#include #include #include @@ -42,9 +41,7 @@ #include "vas.h" #include "miniobj.h" #include "vqueue.h" -#include "vjsn.h" #include "vsb.h" -#include "vend.h" #include "vapi/vsc.h" #include "vapi/vsm.h" @@ -317,25 +314,9 @@ vsc_add_pt(struct vsc *vsc, const volatile void *ptr, */ static void -vsc_build_old_vf_list(struct vsc *vsc) -{ -#define VSC_TYPE_F(n,t,l,e,d) \ - if (!strcmp(vsc->iter_fantom.type, t)) \ - vsc_add_vf(vsc, &vsc->iter_fantom, \ - &VSC_type_desc_##n, VSC_type_order_##n); -#include "tbl/vsc_types.h" -} - -#include - -static void vsc_build_vf_list(struct VSM_data *vd) { - uint64_t u; struct vsc *vsc = vsc_setup(vd); - struct vjsn *vj; - const char *p; - const char *e; vsc_delete_pt_list(&vsc->pt_list); vsc_delete_vf_list(&vsc->vf_list); @@ -343,20 +324,11 @@ vsc_build_vf_list(struct VSM_data *vd) VSM_FOREACH(&vsc->iter_fantom, vd) { if (strcmp(vsc->iter_fantom.class, VSC_CLASS)) continue; - u = vbe64dec(vsc->iter_fantom.b); - vsc_build_old_vf_list(vsc); - if (u == 0) { - fprintf(stderr, "%s has no JSON\n", vsc->iter_fantom.type); - exit(2); - } - p = (char*)vsc->iter_fantom.b + u; - vj = vjsn_parse(p, &e); - if (e != NULL) { - fprintf(stderr, "%s\n", p); - fprintf(stderr, "JSON ERROR %s\n", e); - } - AZ(e); - //vjsn_dump(vj, stdout); +#define VSC_TYPE_F(n,t,l,e,d) \ + if (!strcmp(vsc->iter_fantom.type, t)) \ + vsc_add_vf(vsc, &vsc->iter_fantom, \ + &VSC_type_desc_##n, VSC_type_order_##n); +#include "tbl/vsc_types.h" } } From phk at FreeBSD.org Tue May 23 19:41:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 May 2017 21:41:06 +0200 Subject: [master] f066db7 Take the 'zero' element out of the VSC structures again, and syntax check the JSON payload. Message-ID: commit f066db70d86b1a1701c98cbdb0eb7fdd0c236697 Author: Poul-Henning Kamp Date: Tue May 23 19:32:01 2017 +0000 Take the 'zero' element out of the VSC structures again, and syntax check the JSON payload. diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c index 0b31cef..976b647 100644 --- a/bin/varnishd/cache/cache_backend_cfg.c +++ b/bin/varnishd/cache/cache_backend_cfg.c @@ -186,10 +186,8 @@ VBE_Event(struct backend *be, enum vcl_event_e ev) if (be->probe != NULL && ev == VCL_EVENT_COLD) VBP_Control(be, 0); - if (ev == VCL_EVENT_COLD) { - VSM_Free(be->vsc); - be->vsc = NULL; - } + if (ev == VCL_EVENT_COLD) + VSC_vbe_Destroy(&be->vsc); } void diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c index b021150..45197e8 100644 --- a/bin/varnishd/cache/cache_mempool.c +++ b/bin/varnishd/cache/cache_mempool.c @@ -160,7 +160,7 @@ mpl_guard(void *priv) FREE_OBJ(mi); mi = NULL; } - VSM_Free(mpl->vsc); + VSC_mempool_Destroy(&mpl->vsc); Lck_Unlock(&mpl->mtx); Lck_Delete(&mpl->mtx); FREE_OBJ(mpl); diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 5429a14..84a6b47 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -472,6 +472,18 @@ VSL_End(struct vsl_log *vsl) /*--------------------------------------------------------------------*/ +struct vsc_segs { + unsigned magic; +#define VSC_SEGS_MAGIC 0x9b355991 + + VTAILQ_ENTRY(vsc_segs) list; + void *seg; + void *ptr; +}; + +static VTAILQ_HEAD(,vsc_segs) vsc_seglist = + VTAILQ_HEAD_INITIALIZER(vsc_seglist); + void * VSC_Alloc(const char *nm, size_t sd, size_t sj, const unsigned char *zj, size_t szj, @@ -479,33 +491,50 @@ VSC_Alloc(const char *nm, size_t sd, { char *p; z_stream vz; + struct vsc_segs *vsg; (void)nm; (void)fmt; (void)va; - p = VSM_Alloc(sd + sj, VSC_CLASS, nm, fmt); + + p = VSM_Alloc(8 + sd + sj, VSC_CLASS, nm, fmt); AN(p); memset(p, 0, sd); - vbe64enc(p, sd); // fills .zero + vbe64enc(p, sd); memset(&vz, 0, sizeof vz); assert(Z_OK == inflateInit2(&vz, 31)); vz.next_in = TRUST_ME(zj); vz.avail_in = szj; - vz.next_out = (void*)(p + sd); + vz.next_out = (void*)(p + 8 + sd); vz.avail_out = sj; assert(Z_STREAM_END == inflate(&vz, Z_FINISH)); assert(Z_OK == inflateEnd(&vz)); - return (p); + ALLOC_OBJ(vsg, VSC_SEGS_MAGIC); + vsg->seg = p; + vsg->ptr = p + 8; + VTAILQ_INSERT_TAIL(&vsc_seglist, vsg, list); + return (p + 8); } void VSC_Destroy(const char *nm, void *p) { + struct vsc_segs *vsg; + (void)nm; - (void)p; + VTAILQ_FOREACH(vsg, &vsc_seglist, list) { + if (vsg->ptr != p) + continue; + VSM_Free(vsg->seg); + VTAILQ_REMOVE(&vsc_seglist, vsg, list); + FREE_OBJ(vsg); + return; + + } + WRONG("Freeing unknown VSC"); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index f0797a7..809c006 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -47,6 +47,7 @@ #include "hash/hash_slinger.h" #include "vav.h" #include "vcli_serve.h" +#include "vend.h" #include "vev.h" #include "vfil.h" #include "vin.h" diff --git a/bin/varnishd/storage/storage_malloc.c b/bin/varnishd/storage/storage_malloc.c index 420b925..083b741 100644 --- a/bin/varnishd/storage/storage_malloc.c +++ b/bin/varnishd/storage/storage_malloc.c @@ -214,7 +214,6 @@ sma_open(struct stevedore *st) CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC); Lck_New(&sma_sc->sma_mtx, lck_sma); sma_sc->stats = VSC_sma_New(st->ident); - memset(sma_sc->stats, 0, sizeof *sma_sc->stats); if (sma_sc->sma_max != SIZE_MAX) sma_sc->stats->g_space = sma_sc->sma_max; } diff --git a/include/vapi/vsc_int.h b/include/vapi/vsc_int.h index 9d20665..352c8ba 100644 --- a/include/vapi/vsc_int.h +++ b/include/vapi/vsc_int.h @@ -43,7 +43,7 @@ enum VSC_level_e { #include "tbl/vsc_types.h" /* Define the vsc type structs */ -#define VSC_DO(u,l,t,h) struct VSC_C_##l { uint64_t zero; +#define VSC_DO(u,l,t,h) struct VSC_C_##l { #define VSC_F(n,t,l,s,f,v,d,e) t n; #define VSC_DONE(u,l,t) }; #include "tbl/vsc_all.h" diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 725a5f7..e58f64f 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -34,6 +34,7 @@ #include #include +#include #include #include @@ -41,7 +42,9 @@ #include "vas.h" #include "miniobj.h" #include "vqueue.h" +#include "vjsn.h" #include "vsb.h" +#include "vend.h" #include "vapi/vsc.h" #include "vapi/vsm.h" @@ -246,7 +249,7 @@ VSC_Get(const struct VSM_data *vd, struct VSM_fantom *fantom, const char *type, if (VSM_invalid == VSM_StillValid(vd, fantom) && !VSM_Get(vd, fantom, VSC_CLASS, type, ident)) return (NULL); - return ((void*)fantom->b); + return ((void*)((char*)fantom->b + 8)); } /*--------------------------------------------------------------------*/ @@ -300,7 +303,7 @@ vsc_add_pt(struct vsc *vsc, const volatile void *ptr, struct VSC_C_##l *st; \ \ CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC); \ - st = vf->fantom.b; + st = (void*)((char*)vf->fantom.b + 8); #define VSC_F(nn,tt,ll,ss,ff,vv,dd,ee) \ vsc_add_pt(vsc, &st->nn, descs++, vf); @@ -314,9 +317,25 @@ vsc_add_pt(struct vsc *vsc, const volatile void *ptr, */ static void +vsc_build_old_vf_list(struct vsc *vsc) +{ +#define VSC_TYPE_F(n,t,l,e,d) \ + if (!strcmp(vsc->iter_fantom.type, t)) \ + vsc_add_vf(vsc, &vsc->iter_fantom, \ + &VSC_type_desc_##n, VSC_type_order_##n); +#include "tbl/vsc_types.h" +} + +#include + +static void vsc_build_vf_list(struct VSM_data *vd) { + uint64_t u; struct vsc *vsc = vsc_setup(vd); + struct vjsn *vj; + const char *p; + const char *e; vsc_delete_pt_list(&vsc->pt_list); vsc_delete_vf_list(&vsc->vf_list); @@ -324,11 +343,20 @@ vsc_build_vf_list(struct VSM_data *vd) VSM_FOREACH(&vsc->iter_fantom, vd) { if (strcmp(vsc->iter_fantom.class, VSC_CLASS)) continue; -#define VSC_TYPE_F(n,t,l,e,d) \ - if (!strcmp(vsc->iter_fantom.type, t)) \ - vsc_add_vf(vsc, &vsc->iter_fantom, \ - &VSC_type_desc_##n, VSC_type_order_##n); -#include "tbl/vsc_types.h" + u = vbe64dec(vsc->iter_fantom.b); + vsc_build_old_vf_list(vsc); + if (u == 0) { + fprintf(stderr, "%s has no JSON\n", vsc->iter_fantom.type); + exit(2); + } + p = (char*)vsc->iter_fantom.b + 8 + u; + vj = vjsn_parse(p, &e); + if (e != NULL) { + fprintf(stderr, "%s\n", p); + fprintf(stderr, "JSON ERROR %s\n", e); + } + AZ(e); + //vjsn_dump(vj, stdout); } } diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index ee70033..29aeb5c 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -89,7 +89,7 @@ class vscset(object): ed["level"] = i.param["level"] ed["1line"] = i.param["oneliner"].strip() ed["docs"] = i.getdoc() - s=json.dumps(dd, separators=(",",":")) + s=json.dumps(dd, separators=(",",":")) + "\0" fo.write("\nstatic const size_t vsc_%s_jsonlen = %dL;\n" % (self.name, len(s))) z = gzip_str(s) @@ -116,14 +116,13 @@ class vscset(object): fo = open(fon, "w") genhdr(fo, self.name) fo.write(self.struct + " {\n") - fo.write("\tuint64_t\tzero;\n") for i in self.mbrs: fo.write("\tuint64_t\t%s;\n" % i.arg) fo.write("};\n") fo.write("\n"); fo.write(self.struct + " *VSC_" + self.name + "_New") fo.write("(const char *fmt, ...);\n"); - fo.write("void VSCL_" + self.name + "_Destroy") + fo.write("void VSC_" + self.name + "_Destroy") fo.write("(" + self.struct + "**);\n") def emit_c(self): @@ -162,7 +161,7 @@ class vscset(object): fo.write("}\n") fo.write("\n") fo.write("void\n") - fo.write("VSCL_" + self.name + "_Destroy") + fo.write("VSC_" + self.name + "_Destroy") fo.write("(" + self.struct + "**pp)\n") fo.write("{\n") fo.write("\n") @@ -285,8 +284,6 @@ class vsc_file(object): for i in self.c: i.vscset(self.vscset) - print(self.vscset) - def emit_h(self): for i in self.vscset: i.emit_h() From phk at FreeBSD.org Tue May 23 19:41:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 May 2017 21:41:06 +0200 Subject: [master] 7a8dc37 polish Message-ID: commit 7a8dc379a711bd7178c292ba0cd4e390c01b9779 Author: Poul-Henning Kamp Date: Tue May 23 19:40:48 2017 +0000 polish diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 84a6b47..6bf507d 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -513,6 +513,7 @@ VSC_Alloc(const char *nm, size_t sd, assert(Z_STREAM_END == inflate(&vz, Z_FINISH)); assert(Z_OK == inflateEnd(&vz)); ALLOC_OBJ(vsg, VSC_SEGS_MAGIC); + AN(vsg); vsg->seg = p; vsg->ptr = p + 8; VTAILQ_INSERT_TAIL(&vsc_seglist, vsg, list); @@ -520,7 +521,7 @@ VSC_Alloc(const char *nm, size_t sd, } void -VSC_Destroy(const char *nm, void *p) +VSC_Destroy(const char *nm, const void *p) { struct vsc_segs *vsg; diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h index 2fbe5d2..a6bbbbb 100644 --- a/bin/varnishd/common/common.h +++ b/bin/varnishd/common/common.h @@ -93,7 +93,7 @@ void VSM_common_ageupdate(const struct vsm_sc *sc); void *VSC_Alloc(const char *, size_t, size_t, const unsigned char *, size_t, const char *, va_list); -void VSC_Destroy(const char *, void *); +void VSC_Destroy(const char *, const void *); /* mgt_cli.c */ extern struct VCLS *mgt_cls; From phk at FreeBSD.org Tue May 23 19:48:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 May 2017 21:48:06 +0200 Subject: [master] 7e7d72a Polish for sunos compiler Message-ID: commit 7e7d72aac25ca1bcb00ff56f63620568e24d0fb5 Author: Poul-Henning Kamp Date: Tue May 23 19:47:11 2017 +0000 Polish for sunos compiler diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index e58f64f..70053ec 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -333,7 +333,6 @@ vsc_build_vf_list(struct VSM_data *vd) { uint64_t u; struct vsc *vsc = vsc_setup(vd); - struct vjsn *vj; const char *p; const char *e; @@ -343,20 +342,16 @@ vsc_build_vf_list(struct VSM_data *vd) VSM_FOREACH(&vsc->iter_fantom, vd) { if (strcmp(vsc->iter_fantom.class, VSC_CLASS)) continue; - u = vbe64dec(vsc->iter_fantom.b); vsc_build_old_vf_list(vsc); - if (u == 0) { - fprintf(stderr, "%s has no JSON\n", vsc->iter_fantom.type); - exit(2); - } + u = vbe64dec(vsc->iter_fantom.b); + assert(u > 0); p = (char*)vsc->iter_fantom.b + 8 + u; - vj = vjsn_parse(p, &e); + (void)vjsn_parse(p, &e); if (e != NULL) { fprintf(stderr, "%s\n", p); fprintf(stderr, "JSON ERROR %s\n", e); } AZ(e); - //vjsn_dump(vj, stdout); } } From fgsch at lodoss.net Wed May 24 00:58:05 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 24 May 2017 02:58:05 +0200 Subject: [master] 4b78318 Fix make check under osx while VSC work is ongoing Message-ID: commit 4b78318decc308b83139d3411146bdcdd0b673a0 Author: Federico G. Schwindt Date: Wed May 24 01:54:58 2017 +0100 Fix make check under osx while VSC work is ongoing diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 6bf507d..5dcf603 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -45,7 +45,7 @@ /* These cannot be struct lock, which depends on vsm/vsl working */ static pthread_mutex_t vsl_mtx; -static pthread_mutex_t vsm_mtx; +static pthread_mutex_t vsm_mtx = PTHREAD_MUTEX_INITIALIZER; static struct VSL_head *vsl_head; static const uint32_t *vsl_end; From phk at FreeBSD.org Wed May 24 06:11:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 24 May 2017 08:11:05 +0200 Subject: [master] 98be9d5 Run Nicolas Seriot's JSON test-cases. Message-ID: commit 98be9d5e6652e65b48a3dcd81e57639dc03adff2 Author: Poul-Henning Kamp Date: Wed May 24 06:08:59 2017 +0000 Run Nicolas Seriot's JSON test-cases. diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am index 60a7661..ab85776 100644 --- a/lib/libvarnishapi/Makefile.am +++ b/lib/libvarnishapi/Makefile.am @@ -104,3 +104,11 @@ vsl_glob_test_SOURCES = \ vsl_glob_test_LDADD = @PCRE_LIBS@ ${RT_LIBS} ${LIBM} libvarnishapi.la vsl_glob_test_CFLAGS = -I$(top_srcdir)/include + +TESTS = vjsn_test + +noinst_PROGRAMS += ${TESTS} + +vjsn_test_SOURCES = vjsn.c +vjsn_test_CFLAGS = -DVJSN_TEST +vjsn_test_LDADD = libvarnishapi.la diff --git a/lib/libvarnishapi/flint.lnt b/lib/libvarnishapi/flint.lnt index 2d2538e..40439a5 100644 --- a/lib/libvarnishapi/flint.lnt +++ b/lib/libvarnishapi/flint.lnt @@ -2,6 +2,9 @@ -esym(759, VJSN_*) // could be moved hdr->mod -esym(765, VJSN_*) // could be made static +-esym(759, vjsn_dump) // could be moved hdr->mod +-esym(765, vjsn_dump) // could be made static +-esym(714, vjsn_dump) // not ref -esym(788, vex_rhs_e::*) -esym(788, VSL_transaction_e::*) -esym(769, vex_rhs_e::VEX__UNSET) diff --git a/lib/libvarnishapi/vjsn.c b/lib/libvarnishapi/vjsn.c index 4a8b001..6c0b1b7 100644 --- a/lib/libvarnishapi/vjsn.c +++ b/lib/libvarnishapi/vjsn.c @@ -26,6 +26,8 @@ * SUCH DAMAGE. */ +#include "config.h" + #include #include #include @@ -455,106 +457,446 @@ vjsn_dump(const struct vjsn *js, FILE *fo) #ifdef VJSN_TEST -static struct vjsn * -vjsn_file(const char *fn, const char **err) +/* + * Test-cases by Nicolas Seriot + * + * See: http://seriot.ch/parsing_json.php + * + * MIT License + * + * Copyright (c) 2016 Nicolas Seriot + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * We skip tests containing NUL, because we're lazy (The code will actually + * pass these tests if you provide for a way to pass the true length of the + * input to it, and we skip really huge tests, because we are only limited + * by available memory. + * + * To produce the C-data-structures: + * + * Clone https://github.com/nst/JSONTestSuite.git + * + * And run this python in test_parsing: + + import glob + + def emit(fin): + if fin in skip: + return + x = bytearray(open(fin).read()) + if 0 in x: + return + if len(x) > 1000: + return + t = '\t"' + for i in x: + t += "\\x%02x" % i + if len(t) > 64: + print(t + '"') + t = '\t"' + print(t + '",') + + print("const char *good[] = {") + for f in glob.glob("y_*"): + emit(f) + print("\tNULL") + print("};") + + print("const char *bad[] = {") + for f in glob.glob("n_*"): + emit(f) + print("\tNULL") + print("};") + +*/ + +const char *good[] = { + "\x5b\x31\x32\x33\x65\x36\x35\x5d", + "\x5b\x5b\x5d\x20\x20\x20\x5d", + "\x5b\x22\x22\x5d", + "\x5b\x5d", + "\x5b\x22\x61\x22\x5d", + "\x5b\x66\x61\x6c\x73\x65\x5d", + "\x5b\x6e\x75\x6c\x6c\x2c\x20\x31\x2c\x20\x22\x31\x22\x2c\x20\x7b" + "\x7d\x5d", + "\x5b\x6e\x75\x6c\x6c\x5d", + "\x5b\x31\x0a\x5d", + "\x20\x5b\x31\x5d", + "\x5b\x31\x2c\x6e\x75\x6c\x6c\x2c\x6e\x75\x6c\x6c\x2c\x6e\x75\x6c" + "\x6c\x2c\x32\x5d", + "\x5b\x32\x5d\x20", + "\x5b\x30\x65\x2b\x31\x5d", + "\x5b\x30\x65\x31\x5d", + "\x5b\x20\x34\x5d", + "\x5b\x2d\x30\x2e\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30" + "\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30" + "\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30" + "\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30" + "\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30" + "\x30\x31\x5d\x0a", + "\x5b\x32\x30\x65\x31\x5d", + "\x5b\x2d\x30\x5d", + "\x5b\x2d\x31\x32\x33\x5d", + "\x5b\x2d\x31\x5d", + "\x5b\x2d\x30\x5d", + "\x5b\x31\x45\x32\x32\x5d", + "\x5b\x31\x45\x2d\x32\x5d", + "\x5b\x31\x45\x2b\x32\x5d", + "\x5b\x31\x32\x33\x65\x34\x35\x5d", + "\x5b\x31\x32\x33\x2e\x34\x35\x36\x65\x37\x38\x5d", + "\x5b\x31\x65\x2d\x32\x5d", + "\x5b\x31\x65\x2b\x32\x5d", + "\x5b\x31\x32\x33\x5d", + "\x5b\x31\x32\x33\x2e\x34\x35\x36\x37\x38\x39\x5d", + "\x7b\x22\x61\x73\x64\x22\x3a\x22\x73\x64\x66\x22\x2c\x20\x22\x64" + "\x66\x67\x22\x3a\x22\x66\x67\x68\x22\x7d", + "\x7b\x22\x61\x73\x64\x22\x3a\x22\x73\x64\x66\x22\x7d", + "\x7b\x22\x61\x22\x3a\x22\x62\x22\x2c\x22\x61\x22\x3a\x22\x63\x22" + "\x7d", + "\x7b\x22\x61\x22\x3a\x22\x62\x22\x2c\x22\x61\x22\x3a\x22\x62\x22" + "\x7d", + "\x7b\x7d", + "\x7b\x22\x22\x3a\x30\x7d", + "\x7b\x22\x66\x6f\x6f\x5c\x75\x30\x30\x30\x30\x62\x61\x72\x22\x3a" + "\x20\x34\x32\x7d", + "\x7b\x20\x22\x6d\x69\x6e\x22\x3a\x20\x2d\x31\x2e\x30\x65\x2b\x32" + "\x38\x2c\x20\x22\x6d\x61\x78\x22\x3a\x20\x31\x2e\x30\x65\x2b\x32" + "\x38\x20\x7d", + "\x7b\x22\x78\x22\x3a\x5b\x7b\x22\x69\x64\x22\x3a\x20\x22\x78\x78" + "\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78" + "\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78" + "\x78\x78\x78\x78\x78\x78\x22\x7d\x5d\x2c\x20\x22\x69\x64\x22\x3a" + "\x20\x22\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78" + "\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78" + "\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x22\x7d", + "\x7b\x22\x61\x22\x3a\x5b\x5d\x7d", + "\x7b\x22\x74\x69\x74\x6c\x65\x22\x3a\x22\x5c\x75\x30\x34\x31\x66" + "\x5c\x75\x30\x34\x33\x65\x5c\x75\x30\x34\x33\x62\x5c\x75\x30\x34" + "\x34\x32\x5c\x75\x30\x34\x33\x65\x5c\x75\x30\x34\x34\x30\x5c\x75" + "\x30\x34\x33\x30\x20\x5c\x75\x30\x34\x31\x37\x5c\x75\x30\x34\x33" + "\x35\x5c\x75\x30\x34\x33\x63\x5c\x75\x30\x34\x33\x62\x5c\x75\x30" + "\x34\x33\x35\x5c\x75\x30\x34\x33\x61\x5c\x75\x30\x34\x33\x65\x5c" + "\x75\x30\x34\x33\x66\x5c\x75\x30\x34\x33\x30\x22\x20\x7d", + "\x7b\x0a\x22\x61\x22\x3a\x20\x22\x62\x22\x0a\x7d", + "\x5b\x22\x5c\x75\x44\x38\x30\x31\x5c\x75\x64\x63\x33\x37\x22\x5d" + "", + "\x5b\x22\x5c\x75\x30\x30\x36\x30\x5c\x75\x30\x31\x32\x61\x5c\x75" + "\x31\x32\x41\x42\x22\x5d", + "\x5b\x22\x5c\x75\x64\x38\x33\x64\x5c\x75\x64\x65\x33\x39\x5c\x75" + "\x64\x38\x33\x64\x5c\x75\x64\x63\x38\x64\x22\x5d", + "\x5b\x22\x5c\x22\x5c\x5c\x5c\x2f\x5c\x62\x5c\x66\x5c\x6e\x5c\x72" + "\x5c\x74\x22\x5d", + "\x5b\x22\x5c\x5c\x75\x30\x30\x30\x30\x22\x5d", + "\x5b\x22\x5c\x22\x22\x5d", + "\x5b\x22\x61\x2f\x2a\x62\x2a\x2f\x63\x2f\x2a\x64\x2f\x2f\x65\x22" + "\x5d", + "\x5b\x22\x5c\x5c\x61\x22\x5d", + "\x5b\x22\x5c\x5c\x6e\x22\x5d", + "\x5b\x22\x5c\x75\x30\x30\x31\x32\x22\x5d", + "\x5b\x22\x5c\x75\x46\x46\x46\x46\x22\x5d", + "\x5b\x22\x61\x73\x64\x22\x5d", + "\x5b\x22\x6e\x65\x77\x5c\x75\x30\x30\x41\x30\x6c\x69\x6e\x65\x22" + "\x5d", + "\x5b\x20\x22\x61\x73\x64\x22\x5d", + "\x5b\x22\x5c\x75\x44\x42\x46\x46\x5c\x75\x44\x46\x46\x46\x22\x5d" + "", + "\x5b\x22\xf4\x8f\xbf\xbf\x22\x5d", + "\x5b\x22\xf0\x9b\xbf\xbf\x22\x5d", + "\x5b\x22\xef\xbf\xbf\x22\x5d", + "\x5b\x22\x5c\x75\x30\x30\x30\x30\x22\x5d", + "\x5b\x22\x5c\x75\x30\x30\x32\x63\x22\x5d", + "\x5b\x22\xcf\x80\x22\x5d", + "\x5b\x22\x61\x73\x64\x20\x22\x5d", + "\x22\x20\x22", + "\x5b\x22\x5c\x75\x44\x38\x33\x34\x5c\x75\x44\x64\x31\x65\x22\x5d" + "", + "\x5b\x22\x5c\x75\x30\x38\x32\x31\x22\x5d", + "\x5b\x22\x5c\x75\x30\x31\x32\x33\x22\x5d", + "\x5b\x22\xe2\x80\xa8\x22\x5d", + "\x5b\x22\xe2\x80\xa9\x22\x5d", + "\x5b\x22\x5c\x75\x30\x30\x36\x31\x5c\x75\x33\x30\x61\x66\x5c\x75" + "\x33\x30\x45\x41\x5c\x75\x33\x30\x62\x39\x22\x5d", + "\x5b\x22\x6e\x65\x77\x5c\x75\x30\x30\x30\x41\x6c\x69\x6e\x65\x22" + "\x5d", + "\x5b\x22\x7f\x22\x5d", + "\x5b\x22\x5c\x75\x41\x36\x36\x44\x22\x5d", + "\x5b\x22\x5c\x75\x30\x30\x35\x43\x22\x5d", + "\x5b\x22\xe2\x8d\x82\xe3\x88\xb4\xe2\x8d\x82\x22\x5d", + "\x5b\x22\x5c\x75\x44\x42\x46\x46\x5c\x75\x44\x46\x46\x45\x22\x5d" + "", + "\x5b\x22\x5c\x75\x44\x38\x33\x46\x5c\x75\x44\x46\x46\x45\x22\x5d" + "", + "\x5b\x22\xe2\x82\xac\xf0\x9d\x84\x9e\x22\x5d", + "\x5b\x22\x5c\x75\x32\x30\x30\x42\x22\x5d", + "\x5b\x22\x5c\x75\x32\x30\x36\x34\x22\x5d", + "\x5b\x22\x5c\x75\x46\x44\x44\x30\x22\x5d", + "\x5b\x22\x5c\x75\x46\x46\x46\x45\x22\x5d", + "\x5b\x22\x5c\x75\x30\x30\x32\x32\x22\x5d", + "\x5b\x22\x61\x7f\x61\x22\x5d", + "\x66\x61\x6c\x73\x65", + "\x34\x32", + "\x2d\x30\x2e\x31", + "\x6e\x75\x6c\x6c", + "\x22\x61\x73\x64\x22", + "\x74\x72\x75\x65", + "\x22\x22", + "\x5b\x22\x61\x22\x5d\x0a", + "\x5b\x74\x72\x75\x65\x5d", + "\x20\x5b\x5d\x20", + NULL +}; +const char *bad[] = { + "\x5b\x31\x20\x74\x72\x75\x65\x5d", + "\x5b\x61\xe5\x5d", + "\x5b\x22\x22\x3a\x20\x31\x5d", + "\x5b\x22\x22\x5d\x2c", + "\x5b\x2c\x31\x5d", + "\x5b\x31\x2c\x2c\x32\x5d", + "\x5b\x22\x78\x22\x2c\x2c\x5d", + "\x5b\x22\x78\x22\x5d\x5d", + "\x5b\x22\x22\x2c\x5d", + "\x5b\x22\x78\x22", + "\x5b\xff\x5d", + "\x5b\x78", + "\x5b\x33\x5b\x34\x5d\x5d", + "\x5b\x31\x3a\x32\x5d", + "\x5b\x2c\x5d", + "\x5b\x2d\x5d", + "\x5b\x20\x20\x20\x2c\x20\x22\x22\x5d", + "\x5b\x22\x61\x22\x2c\x0a\x34\x0a\x2c\x31\x2c", + "\x5b\x31\x2c\x5d", + "\x5b\x31\x2c\x2c\x5d", + "\x5b\x22\x0b\x61\x22\x5c\x66\x5d", + "\x5b\x2a\x5d", + "\x5b\x22\x22", + "\x5b\x31\x2c", + "\x5b\x31\x2c\x0a\x31\x0a\x2c\x31", + "\x5b\x7b\x7d", + "\x5b\x66\x61\x6c\x73\x5d", + "\x5b\x6e\x75\x6c\x5d", + "\x5b\x74\x72\x75\x5d", + "\x5b\x2b\x2b\x31\x32\x33\x34\x5d", + "\x5b\x2b\x31\x5d", + "\x5b\x2b\x49\x6e\x66\x5d", + "\x5b\x2d\x30\x31\x5d", + "\x5b\x2d\x31\x2e\x30\x2e\x5d", + "\x5b\x2d\x32\x2e\x5d", + "\x5b\x2d\x4e\x61\x4e\x5d", + "\x5b\x2e\x2d\x31\x5d", + "\x5b\x2e\x32\x65\x2d\x33\x5d", + "\x5b\x30\x2e\x31\x2e\x32\x5d", + "\x5b\x30\x2e\x33\x65\x2b\x5d", + "\x5b\x30\x2e\x33\x65\x5d", + "\x5b\x30\x2e\x65\x31\x5d", + "\x5b\x30\x45\x2b\x5d", + "\x5b\x30\x45\x5d", + "\x5b\x30\x65\x2b\x5d", + "\x5b\x30\x65\x5d", + "\x5b\x31\x2e\x30\x65\x2b\x5d", + "\x5b\x31\x2e\x30\x65\x2d\x5d", + "\x5b\x31\x2e\x30\x65\x5d", + "\x5b\x31\x20\x30\x30\x30\x2e\x30\x5d", + "\x5b\x31\x65\x45\x32\x5d", + "\x5b\x32\x2e\x65\x2b\x33\x5d", + "\x5b\x32\x2e\x65\x2d\x33\x5d", + "\x5b\x32\x2e\x65\x33\x5d", + "\x5b\x39\x2e\x65\x2b\x5d", + "\x5b\x49\x6e\x66\x5d", + "\x5b\x4e\x61\x4e\x5d", + "\x5b\xef\xbc\x91\x5d", + "\x5b\x31\x2b\x32\x5d", + "\x5b\x30\x78\x31\x5d", + "\x5b\x30\x78\x34\x32\x5d", + "\x5b\x49\x6e\x66\x69\x6e\x69\x74\x79\x5d", + "\x5b\x30\x65\x2b\x2d\x31\x5d", + "\x5b\x2d\x31\x32\x33\x2e\x31\x32\x33\x66\x6f\x6f\x5d", + "\x5b\x31\x32\x33\xe5\x5d", + "\x5b\x31\x65\x31\xe5\x5d", + "\x5b\x30\xe5\x5d\x0a", + "\x5b\x2d\x49\x6e\x66\x69\x6e\x69\x74\x79\x5d", + "\x5b\x2d\x66\x6f\x6f\x5d", + "\x5b\x2d\x20\x31\x5d", + "\x5b\x2d\x30\x31\x32\x5d", + "\x5b\x2d\x2e\x31\x32\x33\x5d", + "\x5b\x2d\x31\x78\x5d", + "\x5b\x31\x65\x61\x5d", + "\x5b\x31\x65\xe5\x5d", + "\x5b\x31\x2e\x5d", + "\x5b\x2e\x31\x32\x33\x5d", + "\x5b\x31\x2e\x32\x61\x2d\x33\x5d", + "\x5b\x31\x2e\x38\x30\x31\x31\x36\x37\x30\x30\x33\x33\x33\x37\x36" + "\x35\x31\x34\x48\x2d\x33\x30\x38\x5d", + "\x5b\x30\x31\x32\x5d", + "\x5b\x22\x78\x22\x2c\x20\x74\x72\x75\x74\x68\x5d", + "\x7b\x5b\x3a\x20\x22\x78\x22\x7d\x0a", + "\x7b\x22\x78\x22\x2c\x20\x6e\x75\x6c\x6c\x7d", + "\x7b\x22\x78\x22\x3a\x3a\x22\x62\x22\x7d", + "\x7b\xf0\x9f\x87\xa8\xf0\x9f\x87\xad\x7d", + "\x7b\x22\x61\x22\x3a\x22\x61\x22\x20\x31\x32\x33\x7d", + "\x7b\x6b\x65\x79\x3a\x20\x27\x76\x61\x6c\x75\x65\x27\x7d", + "\x7b\x22\x61\x22\x20\x62\x7d", + "\x7b\x3a\x22\x62\x22\x7d", + "\x7b\x22\x61\x22\x20\x22\x62\x22\x7d", + "\x7b\x22\x61\x22\x3a", + "\x7b\x22\x61\x22", + "\x7b\x31\x3a\x31\x7d", + "\x7b\x39\x39\x39\x39\x45\x39\x39\x39\x39\x3a\x31\x7d", + "\x7b\x22\xb9\x22\x3a\x22\x30\x22\x2c\x7d", + "\x7b\x6e\x75\x6c\x6c\x3a\x6e\x75\x6c\x6c\x2c\x6e\x75\x6c\x6c\x3a" + "\x6e\x75\x6c\x6c\x7d", + "\x7b\x22\x69\x64\x22\x3a\x30\x2c\x2c\x2c\x2c\x2c\x7d", + "\x7b\x27\x61\x27\x3a\x30\x7d", + "\x7b\x22\x69\x64\x22\x3a\x30\x2c\x7d", + "\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x2f\x2a\x2a\x2f", + "\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x2f\x2a\x2a\x2f\x2f", + "\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x2f\x2f", + "\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x2f", + "\x7b\x22\x61\x22\x3a\x22\x62\x22\x2c\x2c\x22\x63\x22\x3a\x22\x64" + "\x22\x7d", + "\x7b\x61\x3a\x20\x22\x62\x22\x7d", + "\x7b\x22\x61\x22\x3a\x22\x61", + "\x7b\x20\x22\x66\x6f\x6f\x22\x20\x3a\x20\x22\x62\x61\x72\x22\x2c" + "\x20\x22\x61\x22\x20\x7d", + "\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x23", + "\x20", + "\x5b\x22\x5c\x75\x44\x38\x30\x30\x5c\x22\x5d", + "\x5b\x22\x5c\x75\x44\x38\x30\x30\x5c\x75\x22\x5d", + "\x5b\x22\x5c\x75\x44\x38\x30\x30\x5c\x75\x31\x22\x5d", + "\x5b\x22\x5c\x75\x44\x38\x30\x30\x5c\x75\x31\x78\x22\x5d", + "\x5b\xc3\xa9\x5d", + "\x5b\x22\x5c\x78\x30\x30\x22\x5d", + "\x5b\x22\x5c\x5c\x5c\x22\x5d", + "\x5b\x22\x5c\x09\x22\x5d", + "\x5b\x22\x5c\xf0\x9f\x8c\x80\x22\x5d", + "\x5b\x22\x5c\x22\x5d", + "\x5b\x22\x5c\x75\x30\x30\x41\x22\x5d", + "\x5b\x22\x5c\x75\x44\x38\x33\x34\x5c\x75\x44\x64\x22\x5d", + "\x5b\x22\x5c\x75\x44\x38\x30\x30\x5c\x75\x44\x38\x30\x30\x5c\x78" + "\x22\x5d", + "\x5b\x22\x5c\x75\xe5\x22\x5d", + "\x5b\x22\x5c\x61\x22\x5d", + "\x5b\x22\x5c\x75\x71\x71\x71\x71\x22\x5d", + "\x5b\x22\x5c\xe5\x22\x5d", + "\x5b\x5c\x75\x30\x30\x32\x30\x22\x61\x73\x64\x22\x5d", + "\x5b\x5c\x6e\x5d", + "\x22", + "\x5b\x27\x73\x69\x6e\x67\x6c\x65\x20\x71\x75\x6f\x74\x65\x27\x5d" + "", + "\x61\x62\x63", + "\x5b\x22\x5c", + "\x5b\x22\x6e\x65\x77\x0a\x6c\x69\x6e\x65\x22\x5d", + "\x5b\x22\x09\x22\x5d", + "\x22\x5c\x55\x41\x36\x36\x44\x22", + "\x22\x22\x78", + "\x5b\xe2\x81\xa0\x5d", + "\xef\xbb\xbf", + "\x3c\x2e\x3e", + "\x5b\x3c\x6e\x75\x6c\x6c\x3e\x5d", + "\x5b\x31\x5d\x78", + "\x5b\x31\x5d\x5d", + "\x5b\x22\x61\x73\x64\x5d", + "\x61\xc3\xa5", + "\x5b\x54\x72\x75\x65\x5d", + "\x31\x5d", + "\x7b\x22\x78\x22\x3a\x20\x74\x72\x75\x65\x2c", + "\x5b\x5d\x5b\x5d", + "\x5d", + "\xef\xbb\x7b\x7d", + "\xe5", + "\x5b", + "", + "\x32\x40", + "\x7b\x7d\x7d", + "\x7b\x22\x22\x3a", + "\x7b\x22\x61\x22\x3a\x2f\x2a\x63\x6f\x6d\x6d\x65\x6e\x74\x2a\x2f" + "\x22\x62\x22\x7d", + "\x7b\x22\x61\x22\x3a\x20\x74\x72\x75\x65\x7d\x20\x22\x78\x22", + "\x5b\x27", + "\x5b\x2c", + "\x5b\x7b", + "\x5b\x22\x61", + "\x5b\x22\x61\x22", + "\x7b", + "\x7b\x5d", + "\x7b\x2c", + "\x7b\x5b", + "\x7b\x22\x61", + "\x7b\x27\x61\x27", + "\x5b\x22\x5c\x7b\x5b\x22\x5c\x7b\x5b\x22\x5c\x7b\x5b\x22\x5c\x7b" + "", + "\xe9", + "\x2a", + "\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x23\x7b\x7d", + "\x5b\x5c\x75\x30\x30\x30\x41\x22\x22\x5d", + "\x5b\x31", + "\x5b\x20\x66\x61\x6c\x73\x65\x2c\x20\x6e\x75\x6c", + "\x5b\x20\x74\x72\x75\x65\x2c\x20\x66\x61\x6c\x73", + "\x5b\x20\x66\x61\x6c\x73\x65\x2c\x20\x74\x72\x75", + "\x7b\x22\x61\x73\x64\x22\x3a\x22\x61\x73\x64\x22", + "\xc3\xa5", + "\x5b\xe2\x81\xa0\x5d", + "\x5b\x0c\x5d", + NULL +}; + +static void +test_good(const char *j) { - int fd; - struct stat st; - char *p, *e; - ssize_t sz; struct vjsn *js; + const char *err; - AN(fn); - AN(err); - *err = NULL; - fd = open(fn, O_RDONLY); - if (fd < 0) { - *err = strerror(errno); - return (NULL); - } - AZ(fstat(fd, &st)); - if (!S_ISREG(st.st_mode)) { - AZ(close(fd)); - *err = "Not a regular file"; - return (NULL); - } - - if (st.st_size == 0) { - AZ(close(fd)); - *err = "Empty file"; - return (NULL); - } - - p = malloc(st.st_size + 1); - AN(p); - - sz = read(fd, p, st.st_size + 1); - if (sz < 0) { - *err = strerror(errno); - AZ(close(fd)); - free(p); - return (NULL); - } - AZ(close(fd)); - - if (sz < st.st_size) { - free(p); - *err = "Short read"; - return (NULL); + js = vjsn_parse(j, &err); + if (js == NULL || err != NULL) { + fprintf(stderr, "Parse error: %s\n%s\n", err, j); + exit(1); } + printf("GOOD: %s\n", j); + vjsn_dump(js, stdout); + vjsn_delete(&js); +} - if (sz > st.st_size) { - free(p); - *err = "Long read"; - return (NULL); - } - - p[sz] = '\0'; - e = p + sz; - - ALLOC_OBJ(js, VJSN_MAGIC); - AN(js); - js->raw = p; - js->ptr = p; - - js->value = vjsn_value(js); - if (js->err != NULL) { - *err = js->err; - vjsn_delete(&js); - return (NULL); - } +static void +test_bad(const char *j) +{ + struct vjsn *js; + const char *err; - vjsn_skip_ws(js); - if (js->ptr != e) { - *err = "Garbage after value"; - vjsn_delete(&js); - return (NULL); + js = vjsn_parse(j, &err); + if (js != NULL || err == NULL) { + fprintf(stderr, "Parse succeeded %s\n", j); + exit(1); } - return (js); + printf("BAD: %s %s\n", err, j); } - -static const char * const fnx = "tst.vjsn"; - int main(int argc, char **argv) { - struct vjsn *js; - const char *err; - - if (argc == 1) - js = vjsn_file(fnx, &err); - else - js = vjsn_file(argv[1], &err); - if (err != NULL) { - fprintf(stderr, "ERROR: %s\n", err); - AZ(js); - return (1); - } else { - if (0) - vjsn_dump(js, stdout); - vjsn_delete(&js); - } + const char **s; + + (void)argc; + (void)argv; + for(s = good; *s != NULL; s++) + test_good(*s); + for(s = bad; *s != NULL; s++) + test_bad(*s); return (0); } From phk at FreeBSD.org Wed May 24 08:17:04 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 24 May 2017 10:17:04 +0200 Subject: [master] 47106ec Retire the VSC_Mgt() and VSC_Main() functions. Message-ID: commit 47106ec3dc02c7879b699a66f3e9131a9af6875c Author: Poul-Henning Kamp Date: Wed May 24 08:16:18 2017 +0000 Retire the VSC_Mgt() and VSC_Main() functions. diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 5dcf603..9e6ec9e 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -533,7 +533,6 @@ VSC_Destroy(const char *nm, const void *p) VTAILQ_REMOVE(&vsc_seglist, vsg, list); FREE_OBJ(vsg); return; - } WRONG("Freeing unknown VSC"); } diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c index 5dfabb9..2323b2b 100644 --- a/bin/varnishstat/varnishstat.c +++ b/bin/varnishstat/varnishstat.c @@ -166,6 +166,25 @@ struct once_priv { }; static int +do_once_cb_first(void *priv, const struct VSC_point * const pt) +{ + struct once_priv *op; + uint64_t val; + const struct VSC_section *sec; + + if (pt == NULL) + return (0); + op = priv; + AZ(strcmp(pt->desc->ctype, "uint64_t")); + sec = pt->section; + if (strcmp(sec->type, "MAIN") || strcmp(pt->desc->name, "uptime")) + return (0); + val = *(const volatile uint64_t*)pt->ptr; + op->up = (double)val; + return (1); +} + +static int do_once_cb(void *priv, const struct VSC_point * const pt) { struct once_priv *op; @@ -199,15 +218,14 @@ do_once_cb(void *priv, const struct VSC_point * const pt) } static void -do_once(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main) +do_once(struct VSM_data *vd) { struct once_priv op; memset(&op, 0, sizeof op); - if (VSC_C_main != NULL) - op.up = VSC_C_main->uptime; op.pad = 18; + (void)VSC_Iter(vd, NULL, do_once_cb_first, &op); (void)VSC_Iter(vd, NULL, do_once_cb, &op); } @@ -352,7 +370,7 @@ main(int argc, char * const *argv) else if (json) do_json(vd); else if (once) - do_once(vd, VSC_Main(vd, NULL)); + do_once(vd); else if (f_list) list_fields(vd); else diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index 59bdb32..36cac9b 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -102,8 +102,12 @@ static VTAILQ_HEAD(, pt) ptlist = VTAILQ_HEAD_INITIALIZER(ptlist); static int n_ptlist = 0; static int n_ptarray = 0; static struct pt **ptarray = NULL; -static const struct VSC_C_mgt *VSC_C_mgt = NULL; -static const struct VSC_C_main *VSC_C_main = NULL; +static const volatile uint64_t *mgt_uptime; +static const volatile uint64_t *main_uptime; +static const volatile uint64_t *main_hit; +static const volatile uint64_t *main_miss; +static const volatile uint64_t *main_cache_hit; +static const volatile uint64_t *main_cache_miss; static int l_status, l_bar_t, l_points, l_bar_b, l_info; static int colw_name = COLW_NAME_MIN; @@ -129,9 +133,9 @@ static void init_hitrate(void) { memset(&hitrate, 0, sizeof (struct hitrate)); - if (VSC_C_main != NULL) { - hitrate.lhit = VSC_C_main->cache_hit; - hitrate.lmiss = VSC_C_main->cache_miss; + if (main_cache_hit != NULL) { + hitrate.lhit = *main_cache_hit; + hitrate.lmiss = *main_cache_miss; } hitrate.hr_10.nmax = 10; hitrate.hr_100.nmax = 100; @@ -275,6 +279,19 @@ build_pt_list_cb(void *priv, const struct VSC_point *vpt) bprintf(buf, "%s.%s.%s", vpt->section->type, vpt->section->ident, vpt->desc->name); + if (!strcmp(buf, "MGT..uptime")) + mgt_uptime = vpt->ptr; + if (!strcmp(buf, "MAIN..uptime")) + main_uptime = vpt->ptr; + if (!strcmp(buf, "MAIN..hit")) + main_hit = vpt->ptr; + if (!strcmp(buf, "MAIN..miss")) + main_miss = vpt->ptr; + if (!strcmp(buf, "MAIN..cache_hit")) + main_cache_hit = vpt->ptr; + if (!strcmp(buf, "MAIN..cache_miss")) + main_cache_miss = vpt->ptr; + VTAILQ_FOREACH(pt, &ptlist, list) { CHECK_OBJ_NOTNULL(pt, PT_MAGIC); AN(pt->key); @@ -339,6 +356,13 @@ build_pt_list(struct VSM_data *vd, struct VSM_fantom *fantom) VTAILQ_INIT(&pt_priv.ptlist); pt_priv.n_ptlist = 0; + mgt_uptime = NULL; + main_uptime = NULL; + main_hit = NULL; + main_miss = NULL; + main_cache_hit = NULL; + main_cache_miss = NULL; + (void)VSC_Iter(vd, fantom, build_pt_list_cb, &pt_priv); delete_pt_list(); AN(VTAILQ_EMPTY(&ptlist)); @@ -384,8 +408,8 @@ sample_points(void) update_ma(&pt->ma_100, (int64_t)pt->cur); update_ma(&pt->ma_1000, (int64_t)pt->cur); } else if (pt->semantics == 'c') { - if (VSC_C_main != NULL && VSC_C_main->uptime) - pt->avg = pt->cur / VSC_C_main->uptime; + if (main_uptime != NULL && *main_uptime) + pt->avg = pt->cur / *main_uptime; else pt->avg = 0.; if (pt->t_last) { @@ -403,11 +427,11 @@ sample_hitrate(void) double hr, mr, ratio; uint64_t hit, miss; - if (VSC_C_main == NULL) + if (main_hit == NULL) return; - hit = VSC_C_main->cache_hit; - miss = VSC_C_main->cache_miss; + hit = *main_hit; + miss = *main_miss; hr = hit - hitrate.lhit; mr = miss - hitrate.lmiss; hitrate.lhit = hit; @@ -542,17 +566,17 @@ draw_status(void) werase(w_status); - if (VSC_C_mgt != NULL) - up_mgt = VSC_C_mgt->uptime; - if (VSC_C_main != NULL) - up_chld = VSC_C_main->uptime; + if (mgt_uptime != NULL) + up_mgt = *mgt_uptime; + if (main_uptime != NULL) + up_chld = *main_uptime; mvwprintw(w_status, 0, 0, "Uptime mgt: "); print_duration(w_status, up_mgt); mvwprintw(w_status, 1, 0, "Uptime child:"); print_duration(w_status, up_chld); - if (VSC_C_mgt == NULL) + if (mgt_uptime == NULL) mvwprintw(w_status, 0, COLS - strlen(discon), discon); else if (COLS > 70) { mvwprintw(w_status, 0, getmaxx(w_status) - 37, @@ -1053,8 +1077,6 @@ do_curses(struct VSM_data *vd, double delay) long t; int ch; double now; - struct VSM_fantom f_main = VSM_FANTOM_NULL; - struct VSM_fantom f_mgt = VSM_FANTOM_NULL; struct VSM_fantom f_iter = VSM_FANTOM_NULL; interval = delay; @@ -1071,8 +1093,6 @@ do_curses(struct VSM_data *vd, double delay) make_windows(); doupdate(); - VSC_C_mgt = VSC_Mgt(vd, &f_mgt); - VSC_C_main = VSC_Main(vd, &f_main); init_hitrate(); while (keep_running) { if (VSM_Abandoned(vd)) { @@ -1081,8 +1101,6 @@ do_curses(struct VSM_data *vd, double delay) VSM_Close(vd); VSM_Open(vd); } - VSC_C_mgt = VSC_Mgt(vd, &f_mgt); - VSC_C_main = VSC_Main(vd, &f_main); if (VSM_valid != VSM_StillValid(vd, &f_iter)) build_pt_list(vd, &f_iter); diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h index 900a795..5e36f3d 100644 --- a/include/vapi/vsc.h +++ b/include/vapi/vsc.h @@ -53,29 +53,6 @@ int VSC_Arg(struct VSM_data *vd, int arg, const char *opt); * 1 Handled. */ -struct VSC_C_mgt *VSC_Mgt(const struct VSM_data *vd, - struct VSM_fantom *fantom); -struct VSC_C_main *VSC_Main(const struct VSM_data *vd, - struct VSM_fantom *fantom); - /* - * Looks up and returns the management stats and the child main - * stats structure. If fantom is non-NULL, it can later be used - * with VSM_StillValid. Returns NULL until the management process - * has finished initialization or the child has been started. - * - * The returned structure is valid for at most 60 seconds after - * VSM_StillValid(,fantom) starts returning VSM_invalid. Using the - * pointer after this event gives undefined behavior. - * - * Arguments: - * vd: The VSM_data context - * fantom: Pointer to a fantom. Can be NULL. - * - * Return values: - * NULL: Failure - * non-NULL: Success - */ - void *VSC_Get(const struct VSM_data *vd, struct VSM_fantom *fantom, const char *type, const char *ident); /* diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 70053ec..744b35b 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -217,24 +217,6 @@ VSC_Arg(struct VSM_data *vd, int arg, const char *opt) } } -/*--------------------------------------------------------------------*/ - -struct VSC_C_mgt * -VSC_Mgt(const struct VSM_data *vd, struct VSM_fantom *fantom) -{ - - return (VSC_Get(vd, fantom, VSC_type_mgt, "")); -} - -/*--------------------------------------------------------------------*/ - -struct VSC_C_main * -VSC_Main(const struct VSM_data *vd, struct VSM_fantom *fantom) -{ - - return (VSC_Get(vd, fantom, VSC_type_main, "")); -} - /*-------------------------------------------------------------------- */ From phk at FreeBSD.org Wed May 24 10:58:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 24 May 2017 12:58:06 +0200 Subject: [master] 42e764e Polish a little bit. Message-ID: commit 42e764e24c305cfd7bc804a1b026f2e362e9da43 Author: Poul-Henning Kamp Date: Wed May 24 10:57:33 2017 +0000 Polish a little bit. diff --git a/lib/libvarnishapi/vjsn.c b/lib/libvarnishapi/vjsn.c index 6c0b1b7..9994d23 100644 --- a/lib/libvarnishapi/vjsn.c +++ b/lib/libvarnishapi/vjsn.c @@ -162,7 +162,13 @@ vjsn_unicode(struct vjsn *js, char **d) unsigned u1, u2; u1 = vjsn_unumber(js); + if (js->err) + return; + if (u1 >= 0xdc00 && u1 <= 0xdfff) { + js->err = "Lone second UTF-16 Surrogate"; + return; + } if (u1 >= 0xd800 && u1 <= 0xdc00) { u2 = vjsn_unumber(js); if (u2 < 0xdc00 || u2 > 0xdfff) { @@ -173,26 +179,24 @@ vjsn_unicode(struct vjsn *js, char **d) u2 -= 0xdc00; u1 <<= 10; u1 |= u2; + u1 |= 0x10000; } + assert(u1 < 0x110000); /*lint -save -e734 -e713 */ if (u1 < 0x80) *(*d)++ = u1; else if (u1 < 0x800) { *(*d)++ = 0xc0 + u1 / 64; *(*d)++ = 0x80 + u1 % 64; - } else if (u1 - 0xd800u < 0x800) { - js->err = "Bad UNICODE point"; } else if (u1 < 0x10000) { *(*d)++ = 0xe0 + u1 / 4096; *(*d)++ = 0x80 + u1 / 64 % 64; *(*d)++ = 0x80 + u1 % 64; - } else if (u1 < 0x110000) { + } else { *(*d)++ = 0xf0 + u1 / 262144; *(*d)++ = 0x80 + u1 / 4096 % 64; *(*d)++ = 0x80 + u1 / 64 % 64; *(*d)++ = 0x80 + u1 % 64; - } else { - js->err = "Bad UNICODE point"; } /*lint -restore */ } @@ -897,6 +901,13 @@ main(int argc, char **argv) test_good(*s); for(s = bad; *s != NULL; s++) test_bad(*s); + + /* + * This is part of Nicolas i(ndeterminate) test set, for reasons I + * do not fully grasp, but we want it to test bad. + */ + test_bad("\"\\uDFAA\""); + printf("Tests done\n"); return (0); } From fgsch at lodoss.net Thu May 25 21:15:05 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 25 May 2017 23:15:05 +0200 Subject: [master] 4a494bc Fix building with sanitizers Message-ID: commit 4a494bca4ed9dd1995d5fa04dd78d4fa76bade92 Author: Federico G. Schwindt Date: Thu May 25 22:13:11 2017 +0100 Fix building with sanitizers diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am index ab85776..5b9722a 100644 --- a/lib/libvarnishapi/Makefile.am +++ b/lib/libvarnishapi/Makefile.am @@ -110,5 +110,5 @@ TESTS = vjsn_test noinst_PROGRAMS += ${TESTS} vjsn_test_SOURCES = vjsn.c -vjsn_test_CFLAGS = -DVJSN_TEST -vjsn_test_LDADD = libvarnishapi.la +vjsn_test_CFLAGS = -DVJSN_TEST @SAN_LDFLAGS@ +vjsn_test_LDADD = libvarnishapi.la @SAN_LDFLAGS@ From fgsch at lodoss.net Thu May 25 21:15:05 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 25 May 2017 23:15:05 +0200 Subject: [master] 452872d Update and some housekeeping Message-ID: commit 452872ddf0e04c1587ebf4308a0207b6b3f29c55 Author: Federico G. Schwindt Date: Thu May 25 22:13:38 2017 +0100 Update and some housekeeping diff --git a/.gitignore b/.gitignore index 41b82cf..1c55b88 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ Makefile.in # Various auto-tools artifacts /aclocal.m4 /autom4te.cache/ +/build-aux /compile /config.guess /config.h @@ -38,17 +39,18 @@ Makefile.in /missing /stamp-h1 /varnishapi.pc -/build-aux +/varnishapi-uninstalled.pc TAGS tags cscope.*out -/varnishapi-uninstalled.pc # Default vcl made from bin/varnishd/default.vcl -/bin/varnishd/builtin_vcl.h +/bin/varnishd/builtin_vcl.c /etc/builtin.vcl # Various auto-generated code snippets +/bin/varnishd/vhp_hufdec.h +/bin/varnishd/vhp_gen_hufdec /include/vcl.h /include/vrt_obj.h /include/vmod_abi.h @@ -56,21 +58,18 @@ cscope.*out /include/tbl/vrt_stv_var.h /include/tbl/vcc_types.h /include/vcs_version.h -/include/vbm_test -/include/vbm_test.log -/include/test-suite.log /lib/libvcc/vcc_fixed_token.c /lib/libvcc/vcc_obj.c /lib/libvcc/vcc_token_defs.h -/lib/libvarnish/vnum_c_test -/lib/libvarnish/vnum_c_test.log -/lib/libvarnish/test-suite.log +/lib/libvarnishapi/vsl2rst /lib/libvarnishapi/vxp_fixed_token.c /lib/libvarnishapi/vxp_tokens.h -/lib/libvarnishapi/vsl2rst + +# Stats +/bin/varnishd/VSC_*.c +/bin/varnishd/VSC_*.h # Misc. generated files for included vmods. -/bin/varnishd/vhp_hufdec.h /lib/libvmod_*/vcc_if.c /lib/libvmod_*/vcc_if.h /lib/libvmod_*/vmod_*.rst @@ -103,22 +102,28 @@ cscope.*out /nbproject/private/ # Test droppings +/bin/varnishd/*.log /bin/varnishd/vhp_decode_test -/bin/varnishd/vhp_gen_hufdec /bin/varnishd/vhp_table_test -/bin/varnishd/*.log +/bin/varnishtest/*.log /bin/varnishtest/tests/*.log /bin/varnishtest/tests/*.log-t -/bin/varnishtest/test-suite.log +/include/*.log /include/vrt.c /include/_vrt_test /include/vrt_test -/include/vrt_test.* +/include/vbm_test +/lib/libvarnish/*.log +/lib/libvarnish/vnum_c_test +/lib/libvarnishapi/*.log +/lib/libvarnishapi/vjsn_test # vtest.sh droppings -tmp/ -vt_key -vt_key.pub +/tools/tmp/ +/tools/_vtest_tmp/ +/tools/varnish-cache/ +/tools/vt_key +/tools/vt_key.pub # Coverity output /cov-int From fgsch at lodoss.net Thu May 25 21:15:05 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 25 May 2017 23:15:05 +0200 Subject: [master] 2e0339c Correct check Message-ID: commit 2e0339cd73c7b6b93059b6a7b2dd59301d741b07 Author: Federico G. Schwindt Date: Thu May 25 22:14:30 2017 +0100 Correct check diff --git a/lib/libvarnishapi/vjsn.c b/lib/libvarnishapi/vjsn.c index 9994d23..444a471 100644 --- a/lib/libvarnishapi/vjsn.c +++ b/lib/libvarnishapi/vjsn.c @@ -377,7 +377,7 @@ vjsn_value(struct vjsn *js) js->ptr += 4; return (vjsn_val_new(VJSN_TRUE)); } - if (!memcmp(js->ptr, "false", 4)) { + if (!memcmp(js->ptr, "false", 5)) { js->ptr += 5; return (vjsn_val_new(VJSN_FALSE)); } From fgsch at lodoss.net Thu May 25 22:40:06 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 26 May 2017 00:40:06 +0200 Subject: [master] 5aa6fac And now using the correct variable, tsk tsk Message-ID: commit 5aa6fac40ca3afe90bb676bc60b9ee5d3d95244f Author: Federico G. Schwindt Date: Thu May 25 23:26:23 2017 +0100 And now using the correct variable, tsk tsk diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am index 5b9722a..0e5f34c 100644 --- a/lib/libvarnishapi/Makefile.am +++ b/lib/libvarnishapi/Makefile.am @@ -110,5 +110,5 @@ TESTS = vjsn_test noinst_PROGRAMS += ${TESTS} vjsn_test_SOURCES = vjsn.c -vjsn_test_CFLAGS = -DVJSN_TEST @SAN_LDFLAGS@ +vjsn_test_CFLAGS = -DVJSN_TEST @SAN_CFLAGS@ vjsn_test_LDADD = libvarnishapi.la @SAN_LDFLAGS@ From copier at varnish-cache.org Fri May 26 11:56:13 2017 From: copier at varnish-cache.org (copier at varnish-cache.org) Date: Fri, 26 May 2017 17:26:13 +0530 Subject: Scanned Image from a Xerox WorkCentre Message-ID: <976A10288AD3C8AD61843BCA4DC0DFCBF6EAB5@varnish-cache.org> You have a received a new image from Xerox WorkCentre. Sent by: copier at varnish-cache.org Number of Images: 2 Attachment File Type: PDF WorkCentre Pro Location: Machine location not set Device Name: copier at varnish-cache.org Attached file is scanned image in PDF format. -------------- next part -------------- A non-text attachment was scrubbed... Name: Scan_0064_0181650123.zip Type: application/zip Size: 7042 bytes Desc: not available URL: From phk at FreeBSD.org Fri May 26 21:23:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 26 May 2017 23:23:05 +0200 Subject: [master] fd55b85 Polish for flexelint Message-ID: commit fd55b85cdf8d0aa96c4640bb76ad1b00f0c51304 Author: Poul-Henning Kamp Date: Fri May 26 20:30:28 2017 +0000 Polish for flexelint diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 65bea1f..d22c1a1 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -552,7 +552,7 @@ vmod_test_probe(VRT_CTX, VCL_PROBE probe, VCL_PROBE same) VCL_INT vmod_typesize(VRT_CTX, VCL_STRING s) { - int i = 0; + size_t i = 0; const char *p; (void)ctx; @@ -570,5 +570,5 @@ vmod_typesize(VRT_CTX, VCL_STRING s) default: return(-1); } } - return (i); + return ((VCL_INT)i); } From phk at FreeBSD.org Fri May 26 21:23:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 26 May 2017 23:23:05 +0200 Subject: [master] f4ee973 Join the long-doc strings with "\n" Message-ID: commit f4ee973bd20a2abf4f56bc7c6c551b1461a4eca8 Author: Poul-Henning Kamp Date: Fri May 26 20:58:49 2017 +0000 Join the long-doc strings with "\n" diff --git a/bin/varnishd/lck.vsc b/bin/varnishd/lck.vsc index 936df89..91da95a 100644 --- a/bin/varnishd/lck.vsc +++ b/bin/varnishd/lck.vsc @@ -6,7 +6,7 @@ XXX: include/tbl/vsc_fields.h .. varnish_vsc_begin:: lck - :oneliner: LOCK COUNTERS (LCK.*) + :oneliner: Lock Counters .. varnish_vsc:: creat :type: counter diff --git a/bin/varnishd/mempool.vsc b/bin/varnishd/mempool.vsc index b947031..082c3b3 100644 --- a/bin/varnishd/mempool.vsc +++ b/bin/varnishd/mempool.vsc @@ -6,7 +6,7 @@ XXX: include/tbl/vsc_fields.h .. varnish_vsc_begin:: mempool - :oneliner: MEMORY POOL COUNTERS (MEMPOOL.*) + :oneliner: Memory Pool Counters .. varnish_vsc:: live :type: gauge diff --git a/bin/varnishd/mgt.vsc b/bin/varnishd/mgt.vsc index d1d7984..3525177 100644 --- a/bin/varnishd/mgt.vsc +++ b/bin/varnishd/mgt.vsc @@ -6,8 +6,7 @@ XXX: include/tbl/vsc_fields.h .. varnish_vsc_begin:: mgt - :class: MGT - :oneliner: MANAGEMENT PROCESS COUNTERS (MGT.*) + :oneliner: Management Process Counters .. varnish_vsc:: uptime :type: counter diff --git a/bin/varnishd/sma.vsc b/bin/varnishd/sma.vsc index da4e18a..e38b148 100644 --- a/bin/varnishd/sma.vsc +++ b/bin/varnishd/sma.vsc @@ -6,7 +6,7 @@ XXX: include/tbl/vsc_fields.h .. varnish_vsc_begin:: sma - :oneliner: MALLOC STORAGE COUNTERS (SMA.*) + :oneliner: Malloc Stevedore Counters .. varnish_vsc:: c_req :type: counter diff --git a/bin/varnishd/smf.vsc b/bin/varnishd/smf.vsc index 29f38d9..fb4d46d 100644 --- a/bin/varnishd/smf.vsc +++ b/bin/varnishd/smf.vsc @@ -6,7 +6,7 @@ XXX: include/tbl/vsc_fields.h .. varnish_vsc_begin:: smf - :oneliner: FILE STORAGE COUNTERS (SMF.*) + :oneliner: File Stevedore Counters .. varnish_vsc:: c_req :type: counter diff --git a/bin/varnishd/vbe.vsc b/bin/varnishd/vbe.vsc index f959ca9..23ac573 100644 --- a/bin/varnishd/vbe.vsc +++ b/bin/varnishd/vbe.vsc @@ -6,7 +6,7 @@ XXX: include/tbl/vsc_fields.h .. varnish_vsc_begin:: vbe - :oneliner: BACKEND COUNTERS (VBE.*) + :oneliner: Backend Counters .. varnish_vsc:: happy :type: bitmap diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index 29aeb5c..0a4a17b 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -74,7 +74,7 @@ class vscset(object): dd["version"] = "1" dd["name"] = self.name dd["1line"] = self.head.param["oneliner"].strip() - dd["docs"] = self.head.getdoc() + dd["docs"] = "\n".join(self.head.getdoc()) dd["elements"] = len(self.mbrs) el = collections.OrderedDict() dd["elem"] = el @@ -88,7 +88,7 @@ class vscset(object): ed["type"] = i.param["type"] ed["level"] = i.param["level"] ed["1line"] = i.param["oneliner"].strip() - ed["docs"] = i.getdoc() + ed["docs"] = "\n".join(i.getdoc()) s=json.dumps(dd, separators=(",",":")) + "\0" fo.write("\nstatic const size_t vsc_%s_jsonlen = %dL;\n" % (self.name, len(s))) From phk at FreeBSD.org Fri May 26 21:23:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 26 May 2017 23:23:05 +0200 Subject: [master] 4944555 Confine the VSC_C structs to vsc.c Message-ID: commit 49445552fcb4558e814c366d8f458862e7a9cc31 Author: Poul-Henning Kamp Date: Fri May 26 21:10:13 2017 +0000 Confine the VSC_C structs to vsc.c diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h index a6bbbbb..f0b4cf8 100644 --- a/bin/varnishd/common/common.h +++ b/bin/varnishd/common/common.h @@ -81,14 +81,14 @@ int VCL_TestLoad(const char *); /* common_vsm.c */ struct vsm_sc; -struct VSC_C_main; +struct VSC_main; struct vsm_sc *VSM_common_new(void *ptr, ssize_t len); void *VSM_common_alloc(struct vsm_sc *sc, ssize_t size, const char *class, const char *type, const char *ident); void VSM_common_free(struct vsm_sc *sc, void *ptr); void VSM_common_delete(struct vsm_sc **sc); void VSM_common_copy(struct vsm_sc *to, const struct vsm_sc *from); -void VSM_common_cleaner(struct vsm_sc *sc, struct VSC_C_main *stats); +void VSM_common_cleaner(struct vsm_sc *sc, struct VSC_main *stats); void VSM_common_ageupdate(const struct vsm_sc *sc); void *VSC_Alloc(const char *, size_t, size_t, const unsigned char *, size_t, diff --git a/bin/varnishd/common/common_vsm.c b/bin/varnishd/common/common_vsm.c index 7994d95..fb30c70 100644 --- a/bin/varnishd/common/common_vsm.c +++ b/bin/varnishd/common/common_vsm.c @@ -45,6 +45,8 @@ #include "vmb.h" #include "vtim.h" +#include "VSC_main.h" + extern volatile struct params * cache_param; /*--------------------------------------------------------------------*/ @@ -165,7 +167,7 @@ VSM_common_new(void *p, ssize_t l) */ void -VSM_common_cleaner(struct vsm_sc *sc, struct VSC_C_main *stats) +VSM_common_cleaner(struct vsm_sc *sc, struct VSC_main *stats) { double now = VTIM_real(); struct vsm_range *vr, *vr2; diff --git a/include/vapi/vsc_int.h b/include/vapi/vsc_int.h index 352c8ba..cc48d9c 100644 --- a/include/vapi/vsc_int.h +++ b/include/vapi/vsc_int.h @@ -41,9 +41,3 @@ enum VSC_level_e { /* Forward declare the static vsc type names */ #define VSC_TYPE_F(n,t,l,e,d) extern const char *VSC_type_##n; #include "tbl/vsc_types.h" - -/* Define the vsc type structs */ -#define VSC_DO(u,l,t,h) struct VSC_C_##l { -#define VSC_F(n,t,l,s,f,v,d,e) t n; -#define VSC_DONE(u,l,t) }; -#include "tbl/vsc_all.h" diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 744b35b..748fb27 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -51,6 +51,12 @@ #include "vsm_api.h" +/* Define the vsc type structs */ +#define VSC_DO(u,l,t,h) struct VSC_C_##l { +#define VSC_F(n,t,l,s,f,v,d,e) t n; +#define VSC_DONE(u,l,t) }; +#include "tbl/vsc_all.h" + enum { #define VSC_TYPE_F(n,t,l,e,d) \ VSC_type_order_##n, From phk at FreeBSD.org Fri May 26 21:23:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 26 May 2017 23:23:05 +0200 Subject: [master] d982e81 Privatize VSC_level_desc_* VSC_type_desc_* and VSC_desc_* to vsc.c Message-ID: commit d982e81a9f3828dc1043b3ffa707a9f44c1f93ad Author: Poul-Henning Kamp Date: Fri May 26 21:22:01 2017 +0000 Privatize VSC_level_desc_* VSC_type_desc_* and VSC_desc_* to vsc.c diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h index 5e36f3d..a3e1f42 100644 --- a/include/vapi/vsc.h +++ b/include/vapi/vsc.h @@ -147,23 +147,4 @@ int VSC_Iter(struct VSM_data *vd, struct VSM_fantom *fantom, VSC_iter_f *func, const struct VSC_level_desc *VSC_LevelDesc(unsigned level); -/********************************************************************** - * Precompiled VSC_type_desc's and VSC_desc's for all know VSCs. - */ - -#define VSC_LEVEL_F(v,l,e,d) \ - extern const struct VSC_level_desc VSC_level_desc_##v; -#include "tbl/vsc_levels.h" -#undef VSC_LEVEL_F - -#define VSC_TYPE_F(n,t,l,e,d) \ - extern const struct VSC_type_desc VSC_type_desc_##n; -#include "tbl/vsc_types.h" -#undef VSC_TYPE_F - -#define VSC_DO(U,l,t,h) extern const struct VSC_desc VSC_desc_##l[]; -#define VSC_F(n,t,l,s,f,v,d,e) -#define VSC_DONE(U,l,t) -#include "tbl/vsc_all.h" - #endif /* VAPI_VSC_H_INCLUDED */ diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 748fb27..d9caf37 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -100,6 +100,25 @@ struct vsc { struct VSM_fantom iter_fantom; }; +/********************************************************************** + * Precompiled VSC_type_desc's and VSC_desc's for all know VSCs. + */ + +#define VSC_LEVEL_F(v,l,e,d) \ + extern const struct VSC_level_desc VSC_level_desc_##v; +#include "tbl/vsc_levels.h" +#undef VSC_LEVEL_F + +#define VSC_TYPE_F(n,t,l,e,d) \ + extern const struct VSC_type_desc VSC_type_desc_##n; +#include "tbl/vsc_types.h" +#undef VSC_TYPE_F + +#define VSC_DO(U,l,t,h) extern const struct VSC_desc VSC_desc_##l[]; +#define VSC_F(n,t,l,s,f,v,d,e) +#define VSC_DONE(U,l,t) +#include "tbl/vsc_all.h" + /*--------------------------------------------------------------------*/ From phk at FreeBSD.org Fri May 26 21:43:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 26 May 2017 23:43:05 +0200 Subject: [master] 40a253b Simplify Message-ID: commit 40a253bd2f7275d17e0b014c2f9cc3c310b7a2d3 Author: Poul-Henning Kamp Date: Fri May 26 21:28:39 2017 +0000 Simplify diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index d9caf37..3528633 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -100,25 +100,27 @@ struct vsc { struct VSM_fantom iter_fantom; }; -/********************************************************************** - * Precompiled VSC_type_desc's and VSC_desc's for all know VSCs. +/*-------------------------------------------------------------------- + * Build the static level, type and point descriptions */ -#define VSC_LEVEL_F(v,l,e,d) \ - extern const struct VSC_level_desc VSC_level_desc_##v; +#define VSC_LEVEL_F(v,l,e,d) \ + const struct VSC_level_desc VSC_level_desc_##v = \ + {VSC_level_##v, l, e, d}; #include "tbl/vsc_levels.h" #undef VSC_LEVEL_F -#define VSC_TYPE_F(n,t,l,e,d) \ - extern const struct VSC_type_desc VSC_type_desc_##n; +#define VSC_TYPE_F(n,t,l,e,d) const char *VSC_type_##n = t; #include "tbl/vsc_types.h" -#undef VSC_TYPE_F -#define VSC_DO(U,l,t,h) extern const struct VSC_desc VSC_desc_##l[]; -#define VSC_F(n,t,l,s,f,v,d,e) -#define VSC_DONE(U,l,t) -#include "tbl/vsc_all.h" +#define VSC_TYPE_F(n,t,l,e,d) \ + const struct VSC_type_desc VSC_type_desc_##n = {l,e,d}; +#include "tbl/vsc_types.h" +#define VSC_DO(U,l,t,h) const struct VSC_desc VSC_desc_##l[] = { +#define VSC_F(n,t,l,s,f,v,d,e) {#n,#t,s,f,&VSC_level_desc_##v,d,e}, +#define VSC_DONE(U,l,t) }; +#include "tbl/vsc_all.h" /*--------------------------------------------------------------------*/ @@ -496,24 +498,3 @@ VSC_LevelDesc(unsigned level) } } -/*-------------------------------------------------------------------- - * Build the static level, type and point descriptions - */ - -#define VSC_LEVEL_F(v,l,e,d) \ - const struct VSC_level_desc VSC_level_desc_##v = \ - {VSC_level_##v, l, e, d}; -#include "tbl/vsc_levels.h" -#undef VSC_LEVEL_F - -#define VSC_TYPE_F(n,t,l,e,d) const char *VSC_type_##n = t; -#include "tbl/vsc_types.h" - -#define VSC_TYPE_F(n,t,l,e,d) \ - const struct VSC_type_desc VSC_type_desc_##n = {l,e,d}; -#include "tbl/vsc_types.h" - -#define VSC_DO(U,l,t,h) const struct VSC_desc VSC_desc_##l[] = { -#define VSC_F(n,t,l,s,f,v,d,e) {#n,#t,s,f,&VSC_level_desc_##v,d,e}, -#define VSC_DONE(U,l,t) }; -#include "tbl/vsc_all.h" From phk at FreeBSD.org Fri May 26 21:43:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 26 May 2017 23:43:05 +0200 Subject: [master] cb72dc4 Remove the unused type_desc Message-ID: commit cb72dc4fe61bfdb3c698033d54a269f10da75e55 Author: Poul-Henning Kamp Date: Fri May 26 21:42:39 2017 +0000 Remove the unused type_desc diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h index a3e1f42..8d0e993 100644 --- a/include/vapi/vsc.h +++ b/include/vapi/vsc.h @@ -84,16 +84,9 @@ struct VSC_level_desc { const char *ldesc; /* long description */ }; -struct VSC_type_desc { - const char *label; /* label */ - const char *sdesc; /* short description */ - const char *ldesc; /* long description */ -}; - struct VSC_section { char *type; char *ident; - const struct VSC_type_desc *desc; }; /* See include/tbl/vsc_fields.h for descriptions */ diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 3528633..179e2ef 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -113,10 +113,6 @@ struct vsc { #define VSC_TYPE_F(n,t,l,e,d) const char *VSC_type_##n = t; #include "tbl/vsc_types.h" -#define VSC_TYPE_F(n,t,l,e,d) \ - const struct VSC_type_desc VSC_type_desc_##n = {l,e,d}; -#include "tbl/vsc_types.h" - #define VSC_DO(U,l,t,h) const struct VSC_desc VSC_desc_##l[] = { #define VSC_F(n,t,l,s,f,v,d,e) {#n,#t,s,f,&VSC_level_desc_##v,d,e}, #define VSC_DONE(U,l,t) }; @@ -264,8 +260,7 @@ VSC_Get(const struct VSM_data *vd, struct VSM_fantom *fantom, const char *type, /*--------------------------------------------------------------------*/ static void -vsc_add_vf(struct vsc *vsc, const struct VSM_fantom *fantom, - const struct VSC_type_desc *desc, int order) +vsc_add_vf(struct vsc *vsc, const struct VSM_fantom *fantom, int order) { struct vsc_vf *vf, *vf2; @@ -274,7 +269,7 @@ vsc_add_vf(struct vsc *vsc, const struct VSM_fantom *fantom, vf->fantom = *fantom; REPLACE(vf->section.type, vf->fantom.type); REPLACE(vf->section.ident, vf->fantom.ident); - vf->section.desc = desc; + // vf->section.desc = desc; vf->order = order; VTAILQ_FOREACH(vf2, &vsc->vf_list, list) { @@ -331,7 +326,7 @@ vsc_build_old_vf_list(struct vsc *vsc) #define VSC_TYPE_F(n,t,l,e,d) \ if (!strcmp(vsc->iter_fantom.type, t)) \ vsc_add_vf(vsc, &vsc->iter_fantom, \ - &VSC_type_desc_##n, VSC_type_order_##n); + VSC_type_order_##n); #include "tbl/vsc_types.h" } From phk at FreeBSD.org Fri May 26 22:06:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 27 May 2017 00:06:05 +0200 Subject: [master] dfe2104 File more bits off the VSC api Message-ID: commit dfe21045bd76881909a75a2cb6767ebc681421d1 Author: Poul-Henning Kamp Date: Fri May 26 22:05:22 2017 +0000 File more bits off the VSC api diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 9e6ec9e..310df37 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -40,6 +40,7 @@ #include "vend.h" #include "vgz.h" #include "vsl_priv.h" +#include "vapi/vsc_int.h" #include "vmb.h" #include "vtim.h" diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h index f0b4cf8..00a8dc6 100644 --- a/bin/varnishd/common/common.h +++ b/bin/varnishd/common/common.h @@ -44,7 +44,6 @@ #include "vdef.h" #include "vqueue.h" #include "vsb.h" -#include "vapi/vsc_int.h" /**********************************************************************/ diff --git a/include/vapi/vsc_int.h b/include/vapi/vsc_int.h index cc48d9c..a767369 100644 --- a/include/vapi/vsc_int.h +++ b/include/vapi/vsc_int.h @@ -37,7 +37,3 @@ enum VSC_level_e { #include "tbl/vsc_levels.h" VSC_level_MAX, }; - -/* Forward declare the static vsc type names */ -#define VSC_TYPE_F(n,t,l,e,d) extern const char *VSC_type_##n; -#include "tbl/vsc_types.h" diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 179e2ef..76f62b5 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -269,7 +269,6 @@ vsc_add_vf(struct vsc *vsc, const struct VSM_fantom *fantom, int order) vf->fantom = *fantom; REPLACE(vf->section.type, vf->fantom.type); REPLACE(vf->section.ident, vf->fantom.ident); - // vf->section.desc = desc; vf->order = order; VTAILQ_FOREACH(vf2, &vsc->vf_list, list) { From phk at FreeBSD.org Fri May 26 22:39:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 27 May 2017 00:39:05 +0200 Subject: [master] 78cb659 Internalize the VSC levels, and expose a function to increase/decrease. Message-ID: commit 78cb659978c27f7dab456a47a7a56b761614d440 Author: Poul-Henning Kamp Date: Fri May 26 22:38:17 2017 +0000 Internalize the VSC levels, and expose a function to increase/decrease. diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index 36cac9b..6d3d142 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -117,7 +117,7 @@ static WINDOW *w_points = NULL; static WINDOW *w_bar_b = NULL; static WINDOW *w_info = NULL; -static int verbosity = VSC_level_info; +static const struct VSC_level_desc *verbosity; static int keep_running = 1; static int hide_unseen = 1; static int page_start = 0; @@ -216,7 +216,7 @@ build_pt_array(void) CHECK_OBJ_NOTNULL(pt, PT_MAGIC); if (!pt->seen && hide_unseen) continue; - if (pt->vpt->desc->level->verbosity > verbosity) + if (pt->vpt->desc->level > verbosity) continue; assert(n_ptarray < n_ptlist); ptarray[n_ptarray++] = pt; @@ -938,7 +938,6 @@ static void draw_bar_b(void) { int x, X; - const struct VSC_level_desc *level; char buf[64]; AN(w_bar_b); @@ -959,11 +958,10 @@ draw_bar_b(void) mvwprintw(w_bar_b, 0, X - strlen(buf), "%s", buf); X -= strlen(buf) + 2; - level = VSC_LevelDesc(verbosity); - if (level != NULL) { - mvwprintw(w_bar_b, 0, X - strlen(level->label), "%s", - level->label); - X -= strlen(level->label) + 2; + if (verbosity != NULL) { + mvwprintw(w_bar_b, 0, X - strlen(verbosity->label), "%s", + verbosity->label); + X -= strlen(verbosity->label) + 2; } if (!hide_unseen) mvwprintw(w_bar_b, 0, X - 6, "%s", "UNSEEN"); @@ -1045,9 +1043,11 @@ handle_keypress(int ch) page_start = (current - l_points) + 1; break; case 'v': - verbosity++; - if (VSC_LevelDesc(verbosity) == NULL) - verbosity = 0; + verbosity = VSC_ChangeLevel(verbosity, 1); + rebuild = 1; + break; + case 'V': + verbosity = VSC_ChangeLevel(verbosity, -1); rebuild = 1; break; case 'q': @@ -1081,6 +1081,8 @@ do_curses(struct VSM_data *vd, double delay) interval = delay; + verbosity = VSC_ChangeLevel(NULL, 0); + initscr(); raw(); noecho(); diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h index 8d0e993..3f96f09 100644 --- a/include/vapi/vsc.h +++ b/include/vapi/vsc.h @@ -78,7 +78,7 @@ struct VSC_desc; struct VSC_point; struct VSC_level_desc { - unsigned verbosity; + const char *name; /* name */ const char *label; /* label */ const char *sdesc; /* short description */ const char *ldesc; /* long description */ @@ -138,6 +138,6 @@ int VSC_Iter(struct VSM_data *vd, struct VSM_fantom *fantom, VSC_iter_f *func, * 0: Done */ -const struct VSC_level_desc *VSC_LevelDesc(unsigned level); +const struct VSC_level_desc *VSC_ChangeLevel(const struct VSC_level_desc*, int); #endif /* VAPI_VSC_H_INCLUDED */ diff --git a/include/vapi/vsc_int.h b/include/vapi/vsc_int.h index a767369..6859ac6 100644 --- a/include/vapi/vsc_int.h +++ b/include/vapi/vsc_int.h @@ -29,11 +29,3 @@ */ #define VSC_CLASS "Stat" - -/* vsc levels */ -enum VSC_level_e { -#define VSC_LEVEL_F(v,l,e,d) \ - VSC_level_##v, -#include "tbl/vsc_levels.h" - VSC_level_MAX, -}; diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map index a07bd30..14c4418 100644 --- a/lib/libvarnishapi/libvarnishapi.map +++ b/lib/libvarnishapi/libvarnishapi.map @@ -175,3 +175,8 @@ LIBVARNISHAPI_1.6 { VTIM_timespec; VTIM_timeval; } LIBVARNISHAPI_1.0; + +LIBVARNISHAPI_1.7 { + global: + VSC_ChangeLevel; +} LIBVARNISHAPI_1.0; diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 76f62b5..f600c00 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -104,17 +104,24 @@ struct vsc { * Build the static level, type and point descriptions */ -#define VSC_LEVEL_F(v,l,e,d) \ - const struct VSC_level_desc VSC_level_desc_##v = \ - {VSC_level_##v, l, e, d}; +#define VSC_LEVEL_F(v,l,e,d) \ + static const struct VSC_level_desc level_##v = {#v, l, e, d}; +#include "tbl/vsc_levels.h" + +static const struct VSC_level_desc * const levels[] = { +#define VSC_LEVEL_F(v,l,e,d) &level_##v, #include "tbl/vsc_levels.h" #undef VSC_LEVEL_F +}; + +static const size_t nlevels = + sizeof(levels)/sizeof(*levels); #define VSC_TYPE_F(n,t,l,e,d) const char *VSC_type_##n = t; #include "tbl/vsc_types.h" #define VSC_DO(U,l,t,h) const struct VSC_desc VSC_desc_##l[] = { -#define VSC_F(n,t,l,s,f,v,d,e) {#n,#t,s,f,&VSC_level_desc_##v,d,e}, +#define VSC_F(n,t,l,s,f,v,d,e) {#n,#t,s,f,&level_##v,d,e}, #define VSC_DONE(U,l,t) }; #include "tbl/vsc_all.h" @@ -479,16 +486,23 @@ VSC_Iter(struct VSM_data *vd, struct VSM_fantom *fantom, VSC_iter_f *func, */ const struct VSC_level_desc * -VSC_LevelDesc(unsigned level) +VSC_ChangeLevel(const struct VSC_level_desc *old, int chg) { - switch (level) { -#define VSC_LEVEL_F(v,l,e,d) \ - case VSC_level_##v: \ - return (&VSC_level_desc_##v); -#include "tbl/vsc_levels.h" -#undef VSC_LEVEL_F - default: - return (NULL); - } + int i; + + if (old == NULL) + old = levels[0]; + for (i = 0; i < nlevels; i++) + if (old == levels[i]) + break; + if (i == nlevels) + i = 0; + else + i += chg; + if (i >= nlevels) + i = nlevels - 1; + if (i < 0) + i = 0; + return (levels[i]); } From phk at FreeBSD.org Sat May 27 08:58:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 27 May 2017 10:58:06 +0200 Subject: [master] 18ad341 Support for optional Summing function. Message-ID: commit 18ad341a78c8efa7e190b8f6f6d4af7b8af67990 Author: Poul-Henning Kamp Date: Sat May 27 07:05:57 2017 +0000 Support for optional Summing function. We don't optimize but emit straight C-code summing each element in turn, so that it will work with any arithmetic C type down the road. diff --git a/bin/varnishd/main.vsc b/bin/varnishd/main.vsc index 242681d..16106f7 100644 --- a/bin/varnishd/main.vsc +++ b/bin/varnishd/main.vsc @@ -7,6 +7,7 @@ .. varnish_vsc_begin:: main :oneliner: Main counters + :sumfunction: .. varnish_vsc:: uptime :oneliner: Child process uptime diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index 0a4a17b..432f67f 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -120,11 +120,18 @@ class vscset(object): fo.write("\tuint64_t\t%s;\n" % i.arg) fo.write("};\n") fo.write("\n"); + fo.write(self.struct + " *VSC_" + self.name + "_New") fo.write("(const char *fmt, ...);\n"); + fo.write("void VSC_" + self.name + "_Destroy") fo.write("(" + self.struct + "**);\n") + if 'sumfunction' in self.head.param: + fo.write("void VSC_" + self.name + "_Summ") + fo.write("(" + self.struct + " *, ") + fo.write("const " + self.struct + " *);\n") + def emit_c(self): fon="VSC_" + self.name + ".c" fo = open(fon, "w") @@ -140,6 +147,7 @@ class vscset(object): (self.name, self.name.upper())) self.emit_json(fo) + fo.write("\n") fo.write(self.struct + "*\n"); fo.write("VSC_" + self.name + "_New") @@ -159,6 +167,7 @@ class vscset(object): fo.write("\tva_end(ap);\n") fo.write("\treturn(retval);\n") fo.write("}\n") + fo.write("\n") fo.write("void\n") fo.write("VSC_" + self.name + "_Destroy") @@ -170,6 +179,21 @@ class vscset(object): fo.write("\t*pp = NULL;\n") fo.write("}\n") + if 'sumfunction' in self.head.param: + fo.write("\n") + fo.write("void\n") + fo.write("VSC_" + self.name + "_Summ") + fo.write("(" + self.struct + " *dst, ") + fo.write("const " + self.struct + " *src)\n") + fo.write("{\n") + fo.write("\n") + fo.write("\tAN(dst);\n") + fo.write("\tAN(src);\n") + for i in self.mbrs: + fo.write("\tdst->" + i.arg) + fo.write(" += src->" + i.arg + ";\n") + fo.write("}\n") + ####################################################################### class directive(object): From phk at FreeBSD.org Sat May 27 08:58:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 27 May 2017 10:58:06 +0200 Subject: [master] ee58cb2 This file is now surplus to requirements. Message-ID: commit ee58cb2317d6d690c2b75c39f0812501603db3c9 Author: Poul-Henning Kamp Date: Sat May 27 07:26:51 2017 +0000 This file is now surplus to requirements. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 2b3220f..cd3e421 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -53,7 +53,6 @@ varnishd_SOURCES = \ cache/cache_wrk.c \ cache/cache_ws.c \ common/common_vsm.c \ - common/common_vsc.c \ hash/hash_classic.c \ hash/hash_critbit.c \ hash/mgt_hash.c \ diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c deleted file mode 100644 index 9cd6a5c..0000000 --- a/bin/varnishd/common/common_vsc.c +++ /dev/null @@ -1,36 +0,0 @@ -/*- - * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2014 Varnish Software AS - * All rights reserved. - * - * Author: Martin Blix Grydeland - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - */ - -#include "config.h" - -/* Build the static vsc type names */ -#define VSC_TYPE_F(n,t,l,e,d) const char *VSC_type_##n = t; -#include "tbl/vsc_types.h" -#undef VSC_TYPE_F From phk at FreeBSD.org Sat May 27 08:58:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 27 May 2017 10:58:06 +0200 Subject: [master] 44a3513 Hide struct dstat from vmods. Message-ID: commit 44a35137598a342f9967f844891fa8151906574c Author: Poul-Henning Kamp Date: Sat May 27 07:41:24 2017 +0000 Hide struct dstat from vmods. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 5256c3f..70ee5f1 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -250,18 +250,6 @@ struct acct_bereq { /*--------------------------------------------------------------------*/ -#define L0(t, n) -#define L1(t, n) t n; -#define VSC_FF(n,t,l,s,f,v,d,e) L##l(t, n) -struct dstat { - unsigned summs; -#include "tbl/vsc_f_main.h" -}; -#undef L0 -#undef L1 - -/*--------------------------------------------------------------------*/ - struct vsl_log { uint32_t *wlb, *wlp, *wle; unsigned wlr; @@ -315,7 +303,7 @@ struct worker { struct objhead *nobjhead; struct objcore *nobjcore; void *nhashpriv; - struct dstat stats[1]; + struct dstat *stats; struct vsl_log *vsl; // borrowed from req/bo struct pool_task task; @@ -860,8 +848,8 @@ void *MPL_Get(struct mempool *mpl, unsigned *size); void MPL_Free(struct mempool *mpl, void *item); /* cache_obj.c */ -struct objcore * ObjNew(struct worker *); -void ObjDestroy(struct worker *, struct objcore **); +struct objcore * ObjNew(const struct worker *); +void ObjDestroy(const struct worker *, struct objcore **); typedef int objiterate_f(void *priv, int flush, const void *ptr, ssize_t len); int ObjIterate(struct worker *, struct objcore *, void *priv, objiterate_f *func, int final); @@ -920,8 +908,8 @@ const char *sess_close_2str(enum sess_close sc, int want_desc); int Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio prio); int Pool_Task_Arg(struct worker *, enum task_prio, task_func_t *, const void *arg, size_t arg_len); -void Pool_Sumstat(struct worker *w); -int Pool_TrySumstat(struct worker *wrk); +void Pool_Sumstat(const struct worker *w); +int Pool_TrySumstat(const struct worker *wrk); void Pool_PurgeStat(unsigned nobj); int Pool_Task_Any(struct pool_task *task, enum task_prio prio); diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 6dd6e61..986af6f 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -72,7 +72,8 @@ struct rush { static const struct hash_slinger *hash; static struct objhead *private_oh; -static void hsh_rush1(struct worker *, struct objhead *, struct rush *, int); +static void hsh_rush1(const struct worker *, struct objhead *, + struct rush *, int); static void hsh_rush2(struct worker *, struct rush *); /*---------------------------------------------------------------------*/ @@ -116,7 +117,7 @@ hsh_prealloc(struct worker *wrk) /*---------------------------------------------------------------------*/ struct objcore * -HSH_Private(struct worker *wrk) +HSH_Private(const struct worker *wrk) { struct objcore *oc; @@ -156,7 +157,7 @@ HSH_Cleanup(struct worker *wrk) } void -HSH_DeleteObjHead(struct worker *wrk, struct objhead *oh) +HSH_DeleteObjHead(const struct worker *wrk, struct objhead *oh) { AZ(oh->refcnt); @@ -529,7 +530,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, */ static void -hsh_rush1(struct worker *wrk, struct objhead *oh, struct rush *r, int max) +hsh_rush1(const struct worker *wrk, struct objhead *oh, struct rush *r, int max) { unsigned u; struct req *req; diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index cf7b708..a139bcf 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -132,7 +132,7 @@ obj_deleteboc(struct boc **p) */ struct objcore * -ObjNew(struct worker *wrk) +ObjNew(const struct worker *wrk) { struct objcore *oc; @@ -155,7 +155,7 @@ ObjNew(struct worker *wrk) */ void -ObjDestroy(struct worker *wrk, struct objcore **p) +ObjDestroy(const struct worker *wrk, struct objcore **p) { struct objcore *oc; diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index 671ceca..6f8ba29 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -64,7 +64,7 @@ pool_sumstat(const struct dstat *src) } void -Pool_Sumstat(struct worker *wrk) +Pool_Sumstat(const struct worker *wrk) { Lck_Lock(&wstat_mtx); @@ -74,7 +74,7 @@ Pool_Sumstat(struct worker *wrk) } int -Pool_TrySumstat(struct worker *wrk) +Pool_TrySumstat(const struct worker *wrk) { if (Lck_Trylock(&wstat_mtx)) return (0); diff --git a/bin/varnishd/cache/cache_priv.h b/bin/varnishd/cache/cache_priv.h index 215ebc9..d83629a 100644 --- a/bin/varnishd/cache/cache_priv.h +++ b/bin/varnishd/cache/cache_priv.h @@ -31,6 +31,18 @@ #include "VSC_main.h" +/*--------------------------------------------------------------------*/ + +#define L0(t, n) +#define L1(t, n) t n; +#define VSC_FF(n,t,l,s,f,v,d,e) L##l(t, n) +struct dstat { + unsigned summs; +#include "tbl/vsc_f_main.h" +}; +#undef L0 +#undef L1 + /* Prototypes etc ----------------------------------------------------*/ /* cache_acceptor.c */ diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index f3dfc3b..be38c8d 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -61,10 +61,13 @@ wrk_bgthread(void *arg) { struct bgthread *bt; struct worker wrk; + struct dstat ds; CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC); THR_SetName(bt->name); INIT_OBJ(&wrk, WORKER_MAGIC); + memset(&ds, 0, sizeof ds); + wrk.stats = &ds; (void)bt->func(&wrk, bt->priv); @@ -93,6 +96,7 @@ static void WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace) { struct worker *w, ww; + struct dstat ds; unsigned char ws[thread_workspace]; uintptr_t u; @@ -104,6 +108,8 @@ WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace) w = &ww; INIT_OBJ(w, WORKER_MAGIC); w->lastused = NAN; + memset(&ds, 0, sizeof ds); + w->stats = &ds; AZ(pthread_cond_init(&w->cond, NULL)); WS_Init(w->aws, "wrk", ws, thread_workspace); diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index 006cab5..92f9a72 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -74,7 +74,7 @@ void HSH_Purge(struct worker *, struct objhead *, double ttl, double grace, void HSH_config(const char *h_arg); struct boc *HSH_RefBoc(const struct objcore *); void HSH_DerefBoc(struct worker *wrk, struct objcore *); -struct objcore *HSH_Private(struct worker *wrk); +struct objcore *HSH_Private(const struct worker *wrk); void HSH_Abandon(struct objcore *oc); int HSH_Snipe(const struct worker *, struct objcore *); void HSH_Kill(struct objcore *); @@ -107,7 +107,7 @@ struct objhead { void HSH_Fail(struct objcore *); void HSH_Unbusy(struct worker *, struct objcore *); -void HSH_DeleteObjHead(struct worker *, struct objhead *); +void HSH_DeleteObjHead(const struct worker *, struct objhead *); int HSH_DerefObjHead(struct worker *, struct objhead **); int HSH_DerefObjCore(struct worker *, struct objcore **, int); #define HSH_RUSH_POLICY -1 diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c index 780df0d..36aa348 100644 --- a/bin/varnishd/storage/storage_persistent_silo.c +++ b/bin/varnishd/storage/storage_persistent_silo.c @@ -389,7 +389,7 @@ smp_loaded_st(const struct smp_sc *sc, const struct smp_seg *sg, * objcore methods for persistent objects */ -struct object * +struct object * __match_proto__(sml_getobj_f) smp_sml_getobj(struct worker *wrk, struct objcore *oc) { struct object *o; From phk at FreeBSD.org Sat May 27 08:58:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 27 May 2017 10:58:06 +0200 Subject: [master] e49e3cb For now substitute VSC_main for dstat. We will circle back to this laster to get the memory/summing cost back down. Message-ID: commit e49e3cb3083605f59ef19e106fcbe08184ccda04 Author: Poul-Henning Kamp Date: Sat May 27 07:55:17 2017 +0000 For now substitute VSC_main for dstat. We will circle back to this laster to get the memory/summing cost back down. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 70ee5f1..86410bd 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -119,6 +119,10 @@ struct vsb; struct worker; struct v1l; +struct VSC_main; + +#define dstat VSC_main + #define DIGEST_LEN 32 /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index 6f8ba29..03b3911 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -55,12 +55,7 @@ pool_sumstat(const struct dstat *src) { Lck_AssertHeld(&wstat_mtx); -#define L0(n) -#define L1(n) (VSC_C_main->n += src->n) -#define VSC_FF(n,t,l,s,f,v,d,e) L##l(n); -#include "tbl/vsc_f_main.h" -#undef L0 -#undef L1 + VSC_main_Summ(VSC_C_main, src); } void diff --git a/bin/varnishd/cache/cache_priv.h b/bin/varnishd/cache/cache_priv.h index d83629a..215ebc9 100644 --- a/bin/varnishd/cache/cache_priv.h +++ b/bin/varnishd/cache/cache_priv.h @@ -31,18 +31,6 @@ #include "VSC_main.h" -/*--------------------------------------------------------------------*/ - -#define L0(t, n) -#define L1(t, n) t n; -#define VSC_FF(n,t,l,s,f,v,d,e) L##l(t, n) -struct dstat { - unsigned summs; -#include "tbl/vsc_f_main.h" -}; -#undef L0 -#undef L1 - /* Prototypes etc ----------------------------------------------------*/ /* cache_acceptor.c */ diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index be38c8d..0da3ce7 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -144,12 +144,7 @@ pool_addstat(struct dstat *dst, struct dstat *src) { dst->summs++; -#define L0(n) -#define L1(n) (dst->n += src->n) -#define VSC_FF(n,t,l,s,f,v,d,e) L##l(n); -#include "tbl/vsc_f_main.h" -#undef L0 -#undef L1 + VSC_main_Summ(dst, src); memset(src, 0, sizeof *src); } diff --git a/bin/varnishd/main.vsc b/bin/varnishd/main.vsc index 16106f7..d3c939a 100644 --- a/bin/varnishd/main.vsc +++ b/bin/varnishd/main.vsc @@ -9,6 +9,9 @@ :oneliner: Main counters :sumfunction: +.. varnish_vsc:: summs + :oneliner: stat summ operations + .. varnish_vsc:: uptime :oneliner: Child process uptime diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h index 206cd03..2a109c8 100644 --- a/include/tbl/vsc_f_main.h +++ b/include/tbl/vsc_f_main.h @@ -33,6 +33,8 @@ /*lint -save -e525 -e539 */ +VSC_FF(summs, uint64_t, 0, 'c', 'd', debug, "", "") + /*-------------------------------------------------------------------- * Globals, not related to traffic */ From phk at FreeBSD.org Sat May 27 08:58:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 27 May 2017 10:58:06 +0200 Subject: [master] 2643ce3 Make options control what is emitted. Message-ID: commit 2643ce32f3861ec80d9d69566c3369f3d1e8e998 Author: Poul-Henning Kamp Date: Sat May 27 08:03:16 2017 +0000 Make options control what is emitted. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index cd3e421..e35a2f2 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -217,7 +217,7 @@ DISTCLEANFILES += vhp_hufdec.h ####################################################################### VSC_main.c VSC_main.h: $(srcdir)/main.vsc $(top_builddir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py $(srcdir)/main.vsc + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/main.vsc $(varnishd_OBJECTS): VSC_main.h @@ -228,7 +228,7 @@ nodist_varnishd_SOURCES += VSC_main.c ####################################################################### VSC_lck.c VSC_lck.h: $(srcdir)/lck.vsc $(top_builddir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py $(srcdir)/lck.vsc + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/lck.vsc $(varnishd_OBJECTS): VSC_lck.h @@ -239,7 +239,7 @@ nodist_varnishd_SOURCES += VSC_lck.c ####################################################################### VSC_vbe.c VSC_vbe.h: $(srcdir)/vbe.vsc $(top_builddir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py $(srcdir)/vbe.vsc + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/vbe.vsc $(varnishd_OBJECTS): VSC_vbe.h @@ -250,7 +250,7 @@ nodist_varnishd_SOURCES += VSC_vbe.c ####################################################################### VSC_sma.c VSC_sma.h: $(srcdir)/sma.vsc $(top_builddir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py $(srcdir)/sma.vsc + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/sma.vsc $(varnishd_OBJECTS): VSC_sma.h @@ -261,7 +261,7 @@ nodist_varnishd_SOURCES += VSC_sma.c ####################################################################### VSC_smf.c VSC_smf.h: $(srcdir)/smf.vsc $(top_builddir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py $(srcdir)/smf.vsc + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/smf.vsc $(varnishd_OBJECTS): VSC_smf.h @@ -272,7 +272,7 @@ nodist_varnishd_SOURCES += VSC_smf.c ####################################################################### VSC_mempool.c VSC_mempool.h: $(srcdir)/mempool.vsc $(top_builddir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py $(srcdir)/mempool.vsc + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/mempool.vsc $(varnishd_OBJECTS): VSC_mempool.h @@ -283,7 +283,7 @@ nodist_varnishd_SOURCES += VSC_mempool.c ####################################################################### VSC_mgt.c VSC_mgt.h: $(srcdir)/mgt.vsc $(top_builddir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py $(srcdir)/mgt.vsc + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/mgt.vsc $(varnishd_OBJECTS): VSC_mgt.h diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index 432f67f..26d22d9 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -325,17 +325,19 @@ class vsc_file(object): if __name__ == "__main__": - optlist, args = getopt.getopt(sys.argv[1:], "") + optlist, args = getopt.getopt(sys.argv[1:], "chr") fo = sys.stdout - for f, v in optlist: - assert False if len(args) != 1: print("Need exactly one filename argument") exit(2) vf = vsc_file(args[0]) - # vf.emit_rst("_.rst") - vf.emit_h() - vf.emit_c() + for f,v in optlist: + if f == '-r': + vf.emit_rst("_.rst") + if f == '-h': + vf.emit_h() + if f == '-c': + vf.emit_c() From phk at FreeBSD.org Sat May 27 08:58:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 27 May 2017 10:58:06 +0200 Subject: [master] b4f4e20 Make vsctool.py produce the VSC .rst doc. Message-ID: commit b4f4e20efadfa43f7a2e0fc1d3e7ee07596d7f29 Author: Poul-Henning Kamp Date: Sat May 27 08:56:31 2017 +0000 Make vsctool.py produce the VSC .rst doc. Retire vsc2rst diff --git a/bin/varnishd/lck.vsc b/bin/varnishd/lck.vsc index 91da95a..104de02 100644 --- a/bin/varnishd/lck.vsc +++ b/bin/varnishd/lck.vsc @@ -8,6 +8,9 @@ .. varnish_vsc_begin:: lck :oneliner: Lock Counters + Counters which track the activity in the different classes + of mutex-locks. + .. varnish_vsc:: creat :type: counter :level: debug diff --git a/bin/varnishd/main.vsc b/bin/varnishd/main.vsc index d3c939a..8b30981 100644 --- a/bin/varnishd/main.vsc +++ b/bin/varnishd/main.vsc @@ -10,78 +10,79 @@ :sumfunction: .. varnish_vsc:: summs + :level: debug :oneliner: stat summ operations .. varnish_vsc:: uptime :oneliner: Child process uptime -How long the child process has been running. + How long the child process has been running. .. varnish_vsc:: sess_conn :oneliner: Sessions accepted -Count of sessions successfully accepted + Count of sessions successfully accepted .. varnish_vsc:: sess_drop :oneliner: Sessions dropped -Count of sessions silently dropped due to lack of worker thread. + Count of sessions silently dropped due to lack of worker thread. .. varnish_vsc:: sess_fail :oneliner: Session accept failures -Count of failures to accept TCP connection. Either the client -changed its mind, or the kernel ran out of some resource like file -descriptors. + Count of failures to accept TCP connection. Either the client + changed its mind, or the kernel ran out of some resource like file + descriptors. .. varnish_vsc:: client_req_400 :oneliner: Client requests received, subject to 400 errors -400 means we couldn't make sense of the request, it was malformed -in some drastic way. + 400 means we couldn't make sense of the request, it was malformed + in some drastic way. .. varnish_vsc:: client_req_417 :oneliner: Client requests received, subject to 417 errors -417 means that something went wrong with an Expect: header. + 417 means that something went wrong with an Expect: header. .. varnish_vsc:: client_req :oneliner: Good client requests received -The count of parseable client requests seen. + The count of parseable client requests seen. .. varnish_vsc:: cache_hit :oneliner: Cache hits -Count of cache hits. A cache hit indicates that an object has been -delivered to a client without fetching it from a backend server. + Count of cache hits. A cache hit indicates that an object has been + delivered to a client without fetching it from a backend server. .. varnish_vsc:: cache_hitpass :oneliner: Cache hits for pass. -Count of hits for pass. A cache hit for pass indicates that Varnish -is going to pass the request to the backend and this decision has -been cached in it self. This counts how many times the cached -decision is being used. + Count of hits for pass. A cache hit for pass indicates that Varnish + is going to pass the request to the backend and this decision has + been cached in it self. This counts how many times the cached + decision is being used. .. varnish_vsc:: cache_hitmiss :oneliner: Cache hits for miss. -Count of hits for miss. A cache hit for miss indicates that Varnish -is going to proceed as for a cache miss without request coalescing, -and this decision has been cached. This counts how many times the -cached decision is being used. + Count of hits for miss. A cache hit for miss indicates that Varnish + is going to proceed as for a cache miss without request coalescing, + and this decision has been cached. This counts how many times the + cached decision is being used. .. varnish_vsc:: cache_miss :oneliner: Cache misses -Count of misses. A cache miss indicates the object was fetched from -the backend before delivering it to the client. + Count of misses. A cache miss indicates the object was fetched from + the backend before delivering it to the client. .. varnish_vsc:: backend_conn :oneliner: Backend conn. success -How many backend connections have successfully been established. + How many backend connections have successfully been established. .. varnish_vsc:: backend_unhealthy :oneliner: Backend conn. not attempted @@ -98,16 +99,16 @@ How many backend connections have successfully been established. .. varnish_vsc:: backend_reuse :oneliner: Backend conn. reuses -Count of backend connection reuses. This counter is increased -whenever we reuse a recycled connection. + Count of backend connection reuses. This counter is increased + whenever we reuse a recycled connection. .. varnish_vsc:: backend_recycle :oneliner: Backend conn. recycles -Count of backend connection recycles. This counter is increased -whenever we have a keep-alive connection that is put back into the -pool of connections. It has not yet been used, but it might be, -unless the backend closes it. + Count of backend connection recycles. This counter is increased + whenever we have a keep-alive connection that is put back into the + pool of connections. It has not yet been used, but it might be, + unless the backend closes it. .. varnish_vsc:: backend_retry :oneliner: Backend conn. retry @@ -116,183 +117,183 @@ unless the backend closes it. .. varnish_vsc:: fetch_head :oneliner: Fetch no body (HEAD) -beresp with no body because the request is HEAD. + beresp with no body because the request is HEAD. .. varnish_vsc:: fetch_length :oneliner: Fetch with Length -beresp.body with Content-Length. + beresp.body with Content-Length. .. varnish_vsc:: fetch_chunked :oneliner: Fetch chunked -beresp.body with Chunked. + beresp.body with Chunked. .. varnish_vsc:: fetch_eof :oneliner: Fetch EOF -beresp.body with EOF. + beresp.body with EOF. .. varnish_vsc:: fetch_bad :oneliner: Fetch bad T-E -beresp.body length/fetch could not be determined. + beresp.body length/fetch could not be determined. .. varnish_vsc:: fetch_none :oneliner: Fetch no body -beresp.body empty + beresp.body empty .. varnish_vsc:: fetch_1xx :oneliner: Fetch no body (1xx) -beresp with no body because of 1XX response. + beresp with no body because of 1XX response. .. varnish_vsc:: fetch_204 :oneliner: Fetch no body (204) -beresp with no body because of 204 response. + beresp with no body because of 204 response. .. varnish_vsc:: fetch_304 :oneliner: Fetch no body (304) -beresp with no body because of 304 response. + beresp with no body because of 304 response. .. varnish_vsc:: fetch_failed :oneliner: Fetch failed (all causes) -beresp fetch failed. + beresp fetch failed. .. varnish_vsc:: fetch_no_thread :oneliner: Fetch failed (no thread) -beresp fetch failed, no thread available. + beresp fetch failed, no thread available. .. varnish_vsc:: pools :type: gauge :oneliner: Number of thread pools -Number of thread pools. See also parameter thread_pools. NB: Presently -pools cannot be removed once created. + Number of thread pools. See also parameter thread_pools. NB: Presently + pools cannot be removed once created. .. varnish_vsc:: threads :type: gauge :oneliner: Total number of threads -Number of threads in all pools. See also parameters thread_pools, -thread_pool_min and thread_pool_max. + Number of threads in all pools. See also parameters thread_pools, + thread_pool_min and thread_pool_max. .. varnish_vsc:: threads_limited :oneliner: Threads hit max -Number of times more threads were needed, but limit was reached in -a thread pool. See also parameter thread_pool_max. + Number of times more threads were needed, but limit was reached in + a thread pool. See also parameter thread_pool_max. .. varnish_vsc:: threads_created :oneliner: Threads created -Total number of threads created in all pools. + Total number of threads created in all pools. .. varnish_vsc:: threads_destroyed :oneliner: Threads destroyed -Total number of threads destroyed in all pools. + Total number of threads destroyed in all pools. .. varnish_vsc:: threads_failed :oneliner: Thread creation failed -Number of times creating a thread failed. See VSL::Debug for -diagnostics. See also parameter thread_fail_delay. + Number of times creating a thread failed. See VSL::Debug for + diagnostics. See also parameter thread_fail_delay. .. varnish_vsc:: thread_queue_len :type: gauge :oneliner: Length of session queue -Length of session queue waiting for threads. NB: Only updates once -per second. See also parameter thread_queue_limit. + Length of session queue waiting for threads. NB: Only updates once + per second. See also parameter thread_queue_limit. .. varnish_vsc:: busy_sleep :oneliner: Number of requests sent to sleep on busy objhdr -Number of requests sent to sleep without a worker thread because -they found a busy object. + Number of requests sent to sleep without a worker thread because + they found a busy object. .. varnish_vsc:: busy_wakeup :oneliner: Number of requests woken after sleep on busy objhdr -Number of requests taken off the busy object sleep list and rescheduled. + Number of requests taken off the busy object sleep list and rescheduled. .. varnish_vsc:: busy_killed :oneliner: Number of requests killed after sleep on busy objhdr -Number of requests killed from the busy object sleep list due to -lack of resources. + Number of requests killed from the busy object sleep list due to + lack of resources. .. varnish_vsc:: sess_queued :oneliner: Sessions queued for thread -Number of times session was queued waiting for a thread. See also -parameter thread_queue_limit. + Number of times session was queued waiting for a thread. See also + parameter thread_queue_limit. .. varnish_vsc:: sess_dropped :oneliner: Sessions dropped for thread -Number of times session was dropped because the queue were too long -already. See also parameter thread_queue_limit. + Number of times session was dropped because the queue were too long + already. See also parameter thread_queue_limit. .. varnish_vsc:: n_object :type: gauge :oneliner: object structs made -Approximate number of HTTP objects (headers + body, if present) in -the cache. + Approximate number of HTTP objects (headers + body, if present) in + the cache. .. varnish_vsc:: n_vampireobject :type: gauge :level: diag :oneliner: unresurrected objects -Number of unresurrected objects + Number of unresurrected objects .. varnish_vsc:: n_objectcore :type: gauge :oneliner: objectcore structs made -Approximate number of object metadata elements in the cache. Each -object needs an objectcore, extra objectcores are for hit-for-miss, -hit-for-pass and busy objects. + Approximate number of object metadata elements in the cache. Each + object needs an objectcore, extra objectcores are for hit-for-miss, + hit-for-pass and busy objects. .. varnish_vsc:: n_objecthead :type: gauge :oneliner: objecthead structs made -Approximate number of different hash entries in the cache. + Approximate number of different hash entries in the cache. .. varnish_vsc:: n_backend :type: gauge :oneliner: Number of backends -Number of backends known to us. + Number of backends known to us. .. varnish_vsc:: n_expired :type: gauge :oneliner: Number of expired objects -Number of objects that expired from cache because of old age. + Number of objects that expired from cache because of old age. .. varnish_vsc:: n_lru_nuked :type: gauge :oneliner: Number of LRU nuked objects -How many objects have been forcefully evicted from storage to make -room for a new object. + How many objects have been forcefully evicted from storage to make + room for a new object. .. varnish_vsc:: n_lru_moved :type: gauge :level: diag :oneliner: Number of LRU moved objects -Number of move operations done on the LRU list. + Number of move operations done on the LRU list. .. varnish_vsc:: losthdr :oneliner: HTTP header overflows @@ -325,37 +326,37 @@ Number of move operations done on the LRU list. .. varnish_vsc:: s_req_hdrbytes :oneliner: Request header bytes -Total request header bytes received + Total request header bytes received .. varnish_vsc:: s_req_bodybytes :oneliner: Request body bytes -Total request body bytes received + Total request body bytes received .. varnish_vsc:: s_resp_hdrbytes :oneliner: Response header bytes -Total response header bytes transmitted + Total response header bytes transmitted .. varnish_vsc:: s_resp_bodybytes :oneliner: Response body bytes -Total response body bytes transmitted + Total response body bytes transmitted .. varnish_vsc:: s_pipe_hdrbytes :oneliner: Pipe request header bytes -Total request bytes received for piped sessions + Total request bytes received for piped sessions .. varnish_vsc:: s_pipe_in :oneliner: Piped bytes from client -Total number of bytes forwarded from clients in pipe sessions + Total number of bytes forwarded from clients in pipe sessions .. varnish_vsc:: s_pipe_out :oneliner: Piped bytes to client -Total number of bytes forwarded to clients in pipe sessions + Total number of bytes forwarded to clients in pipe sessions .. varnish_vsc:: sess_closed :oneliner: Session Closed @@ -364,8 +365,8 @@ Total number of bytes forwarded to clients in pipe sessions .. varnish_vsc:: sess_closed_err :oneliner: Session Closed with error -Total number of sessions closed with errors. See sc_* diag counters -for detailed breakdown + Total number of sessions closed with errors. See sc_* diag counters + for detailed breakdown .. varnish_vsc:: sess_readahead :oneliner: Session Read Ahead @@ -375,109 +376,109 @@ for detailed breakdown :level: diag :oneliner: Session herd -Number of times the timeout_linger triggered + Number of times the timeout_linger triggered .. varnish_vsc:: sc_rem_close :level: diag :oneliner: Session OK REM_CLOSE -Number of session closes with REM_CLOSE (Client Closed) + Number of session closes with REM_CLOSE (Client Closed) .. varnish_vsc:: sc_req_close :level: diag :oneliner: Session OK REQ_CLOSE -Number of session closes with REQ_CLOSE (Client requested close) + Number of session closes with REQ_CLOSE (Client requested close) .. varnish_vsc:: sc_req_http10 :level: diag :oneliner: Session Err REQ_HTTP10 -Number of session closes with Error REQ_HTTP10 (Proto < HTTP/1.1) + Number of session closes with Error REQ_HTTP10 (Proto < HTTP/1.1) .. varnish_vsc:: sc_rx_bad :level: diag :oneliner: Session Err RX_BAD -Number of session closes with Error RX_BAD (Received bad req/resp) + Number of session closes with Error RX_BAD (Received bad req/resp) .. varnish_vsc:: sc_rx_body :level: diag :oneliner: Session Err RX_BODY -Number of session closes with Error RX_BODY (Failure receiving req.body) + Number of session closes with Error RX_BODY (Failure receiving req.body) .. varnish_vsc:: sc_rx_junk :level: diag :oneliner: Session Err RX_JUNK -Number of session closes with Error RX_JUNK (Received junk data) + Number of session closes with Error RX_JUNK (Received junk data) .. varnish_vsc:: sc_rx_overflow :level: diag :oneliner: Session Err RX_OVERFLOW -Number of session closes with Error RX_OVERFLOW (Received buffer overflow) + Number of session closes with Error RX_OVERFLOW (Received buffer overflow) .. varnish_vsc:: sc_rx_timeout :level: diag :oneliner: Session Err RX_TIMEOUT -Number of session closes with Error RX_TIMEOUT (Receive timeout) + Number of session closes with Error RX_TIMEOUT (Receive timeout) .. varnish_vsc:: sc_tx_pipe :level: diag :oneliner: Session OK TX_PIPE -Number of session closes with TX_PIPE (Piped transaction) + Number of session closes with TX_PIPE (Piped transaction) .. varnish_vsc:: sc_tx_error :level: diag :oneliner: Session Err TX_ERROR -Number of session closes with Error TX_ERROR (Error transaction) + Number of session closes with Error TX_ERROR (Error transaction) .. varnish_vsc:: sc_tx_eof :level: diag :oneliner: Session OK TX_EOF -Number of session closes with TX_EOF (EOF transmission) + Number of session closes with TX_EOF (EOF transmission) .. varnish_vsc:: sc_resp_close :level: diag :oneliner: Session OK RESP_CLOSE -Number of session closes with RESP_CLOSE (Backend/VCL requested close) + Number of session closes with RESP_CLOSE (Backend/VCL requested close) .. varnish_vsc:: sc_overload :level: diag :oneliner: Session Err OVERLOAD -Number of session closes with Error OVERLOAD (Out of some resource) + Number of session closes with Error OVERLOAD (Out of some resource) .. varnish_vsc:: sc_pipe_overflow :level: diag :oneliner: Session Err PIPE_OVERFLOW -Number of session closes with Error PIPE_OVERFLOW (Session pipe overflow) + Number of session closes with Error PIPE_OVERFLOW (Session pipe overflow) .. varnish_vsc:: sc_range_short :level: diag :oneliner: Session Err RANGE_SHORT -Number of session closes with Error RANGE_SHORT (Insufficient data for range) + Number of session closes with Error RANGE_SHORT (Insufficient data for range) .. varnish_vsc:: sc_req_http20 :level: diag :oneliner: Session Err REQ_HTTP20 -Number of session closes with Error REQ_HTTP20 (HTTP2 not accepted) + Number of session closes with Error REQ_HTTP20 (HTTP2 not accepted) .. varnish_vsc:: sc_vcl_failure :level: diag :oneliner: Session Err VCL_FAILURE -Number of session closes with Error VCL_FAILURE (VCL failure) + Number of session closes with Error VCL_FAILURE (VCL failure) .. varnish_vsc:: shm_records :level: diag @@ -525,126 +526,126 @@ Number of session closes with Error VCL_FAILURE (VCL failure) .. varnish_vsc:: vcl_fail :oneliner: VCL failures -Count of failures which prevented VCL from completing. + Count of failures which prevented VCL from completing. .. varnish_vsc:: bans :type: gauge :oneliner: Count of bans -Number of all bans in system, including bans superseded by newer -bans and bans already checked by the ban-lurker. + Number of all bans in system, including bans superseded by newer + bans and bans already checked by the ban-lurker. .. varnish_vsc:: bans_completed :type: gauge :level: diag :oneliner: Number of bans marked 'completed' -Number of bans which are no longer active, either because they got -checked by the ban-lurker or superseded by newer identical bans. + Number of bans which are no longer active, either because they got + checked by the ban-lurker or superseded by newer identical bans. .. varnish_vsc:: bans_obj :type: gauge :level: diag :oneliner: Number of bans using obj.* -Number of bans which use obj.* variables. These bans can possibly -be washed by the ban-lurker. + Number of bans which use obj.* variables. These bans can possibly + be washed by the ban-lurker. .. varnish_vsc:: bans_req :type: gauge :level: diag :oneliner: Number of bans using req.* -Number of bans which use req.* variables. These bans can not be -washed by the ban-lurker. + Number of bans which use req.* variables. These bans can not be + washed by the ban-lurker. .. varnish_vsc:: bans_added :level: diag :oneliner: Bans added -Counter of bans added to ban list. + Counter of bans added to ban list. .. varnish_vsc:: bans_deleted :level: diag :oneliner: Bans deleted -Counter of bans deleted from ban list. + Counter of bans deleted from ban list. .. varnish_vsc:: bans_tested :level: diag :oneliner: Bans tested against objects (lookup) -Count of how many bans and objects have been tested against each -other during hash lookup. + Count of how many bans and objects have been tested against each + other during hash lookup. .. varnish_vsc:: bans_obj_killed :level: diag :oneliner: Objects killed by bans (lookup) -Number of objects killed by bans during object lookup. + Number of objects killed by bans during object lookup. .. varnish_vsc:: bans_lurker_tested :level: diag :oneliner: Bans tested against objects (lurker) -Count of how many bans and objects have been tested against each -other by the ban-lurker. + Count of how many bans and objects have been tested against each + other by the ban-lurker. .. varnish_vsc:: bans_tests_tested :level: diag :oneliner: Ban tests tested against objects (lookup) -Count of how many tests and objects have been tested against each -other during lookup. 'ban req.url == foo && req.http.host == bar' -counts as one in 'bans_tested' and as two in 'bans_tests_tested' + Count of how many tests and objects have been tested against each + other during lookup. 'ban req.url == foo && req.http.host == bar' + counts as one in 'bans_tested' and as two in 'bans_tests_tested' .. varnish_vsc:: bans_lurker_tests_tested :level: diag :oneliner: Ban tests tested against objects (lurker) -Count of how many tests and objects have been tested against each -other by the ban-lurker. 'ban req.url == foo && req.http.host == -bar' counts as one in 'bans_tested' and as two in 'bans_tests_tested' + Count of how many tests and objects have been tested against each + other by the ban-lurker. 'ban req.url == foo && req.http.host == + bar' counts as one in 'bans_tested' and as two in 'bans_tests_tested' .. varnish_vsc:: bans_lurker_obj_killed :level: diag :oneliner: Objects killed by bans (lurker) -Number of objects killed by the ban-lurker. + Number of objects killed by the ban-lurker. .. varnish_vsc:: bans_lurker_obj_killed_cutoff :level: diag :oneliner: Objects killed by bans for cutoff (lurker) -Number of objects killed by the ban-lurker to keep the number of -bans below ban_cutoff. + Number of objects killed by the ban-lurker to keep the number of + bans below ban_cutoff. .. varnish_vsc:: bans_dups :level: diag :oneliner: Bans superseded by other bans -Count of bans replaced by later identical bans. + Count of bans replaced by later identical bans. .. varnish_vsc:: bans_lurker_contention :level: diag :oneliner: Lurker gave way for lookup -Number of times the ban-lurker had to wait for lookups. + Number of times the ban-lurker had to wait for lookups. .. varnish_vsc:: bans_persisted_bytes :type: gauge :level: diag :oneliner: Bytes used by the persisted ban lists -Number of bytes used by the persisted ban lists. + Number of bytes used by the persisted ban lists. .. varnish_vsc:: bans_persisted_fragmentation :type: gauge :level: diag :oneliner: Extra bytes in persisted ban lists due to fragmentation -Number of extra bytes accumulated through dropped and completed -bans in the persistent ban lists. + Number of extra bytes accumulated through dropped and completed + bans in the persistent ban lists. .. varnish_vsc:: n_purges :type: gauge @@ -660,13 +661,13 @@ bans in the persistent ban lists. :level: diag :oneliner: Number of objects mailed to expiry thread -Number of objects mailed to expiry thread for handling. + Number of objects mailed to expiry thread for handling. .. varnish_vsc:: exp_received :level: diag :oneliner: Number of objects received by expiry thread -Number of objects received by expiry thread for handling. + Number of objects received by expiry thread for handling. .. varnish_vsc:: hcb_nolock :level: debug @@ -709,52 +710,52 @@ Number of objects received by expiry thread for handling. .. varnish_vsc:: n_test_gunzip :oneliner: Test gunzip operations -Those operations occur when Varnish receives a compressed object -from a backend. They are done to verify the gzip stream while it's -inserted in storage. + Those operations occur when Varnish receives a compressed object + from a backend. They are done to verify the gzip stream while it's + inserted in storage. .. varnish_vsc:: vsm_free :type: gauge :level: diag :oneliner: Free VSM space -Number of bytes free in the shared memory used to communicate with -tools like varnishstat, varnishlog etc. + Number of bytes free in the shared memory used to communicate with + tools like varnishstat, varnishlog etc. .. varnish_vsc:: vsm_used :type: gauge :level: diag :oneliner: Used VSM space -Number of bytes used in the shared memory used to communicate with -tools like varnishstat, varnishlog etc. + Number of bytes used in the shared memory used to communicate with + tools like varnishstat, varnishlog etc. .. varnish_vsc:: vsm_cooling :type: gauge :level: debug :oneliner: Cooling VSM space -Number of bytes which will soon (max 1 minute) be freed in the -shared memory used to communicate with tools like varnishstat, -varnishlog etc. + Number of bytes which will soon (max 1 minute) be freed in the + shared memory used to communicate with tools like varnishstat, + varnishlog etc. .. varnish_vsc:: vsm_overflow :type: gauge :level: diag :oneliner: Overflow VSM space -Number of bytes which does not fit in the shared memory used to -communicate with tools like varnishstat, varnishlog etc. If this -counter is not zero, consider increasing the runtime variable -vsm_space. + Number of bytes which does not fit in the shared memory used to + communicate with tools like varnishstat, varnishlog etc. If this + counter is not zero, consider increasing the runtime variable + vsm_space. .. varnish_vsc:: vsm_overflowed :level: diag :oneliner: Overflowed VSM space -Total number of bytes which did not fit in the shared memory used -to communicate with tools like varnishstat, varnishlog etc. If this -counter is not zero, consider increasing the runtime variable -vsm_space. + Total number of bytes which did not fit in the shared memory used + to communicate with tools like varnishstat, varnishlog etc. If this + counter is not zero, consider increasing the runtime variable + vsm_space. .. varnish_vsc_end:: main diff --git a/bin/varnishd/mempool.vsc b/bin/varnishd/mempool.vsc index 082c3b3..d649023 100644 --- a/bin/varnishd/mempool.vsc +++ b/bin/varnishd/mempool.vsc @@ -37,13 +37,11 @@ :level: debug :oneliner: Allocations - .. varnish_vsc:: frees :type: counter :level: debug :oneliner: Frees - .. varnish_vsc:: recycle :type: counter :level: debug diff --git a/bin/varnishd/mgt.vsc b/bin/varnishd/mgt.vsc index 3525177..2f8ee8b 100644 --- a/bin/varnishd/mgt.vsc +++ b/bin/varnishd/mgt.vsc @@ -13,48 +13,49 @@ :level: info :oneliner: Management process uptime -Uptime in seconds of the management process + Uptime in seconds of the management process .. varnish_vsc:: child_start :type: counter :level: diag :oneliner: Child process started -Number of times the child process has been started + Number of times the child process has been started .. varnish_vsc:: child_exit :type: counter :level: diag :oneliner: Child process normal exit -Number of times the child process has been cleanly stopped + Number of times the child process has been cleanly stopped .. varnish_vsc:: child_stop :type: counter :level: diag :oneliner: Child process unexpected exit -Number of times the child process has exited with an unexpected return code + Number of times the child process has exited with an + unexpected return code .. varnish_vsc:: child_died :type: counter :level: diag :oneliner: Child process died (signal) -Number of times the child process has died due to signals + Number of times the child process has died due to signals .. varnish_vsc:: child_dump :type: counter :level: diag :oneliner: Child process core dumped -Number of times the child process has produced core dumps + Number of times the child process has produced core dumps .. varnish_vsc:: child_panic :type: counter :level: diag :oneliner: Child process panic -Number of times the management process has caught a child panic + Number of times the management process has caught a child panic .. varnish_vsc_end:: mgt diff --git a/bin/varnishd/sma.vsc b/bin/varnishd/sma.vsc index e38b148..0221c78 100644 --- a/bin/varnishd/sma.vsc +++ b/bin/varnishd/sma.vsc @@ -13,48 +13,48 @@ :level: info :oneliner: Allocator requests -Number of times the storage has been asked to provide a storage segment. + Number of times the storage has been asked to provide a storage segment. .. varnish_vsc:: c_fail :type: counter :level: info :oneliner: Allocator failures -Number of times the storage has failed to provide a storage segment. + Number of times the storage has failed to provide a storage segment. .. varnish_vsc:: c_bytes :type: counter :level: info :oneliner: Bytes allocated -Number of total bytes allocated by this storage. + Number of total bytes allocated by this storage. .. varnish_vsc:: c_freed :type: counter :level: info :oneliner: Bytes freed -Number of total bytes returned to this storage. + Number of total bytes returned to this storage. .. varnish_vsc:: g_alloc :type: gauge :level: info :oneliner: Allocations outstanding -Number of storage allocations outstanding. + Number of storage allocations outstanding. .. varnish_vsc:: g_bytes :type: gauge :level: info :oneliner: Bytes outstanding -Number of bytes allocated from the storage. + Number of bytes allocated from the storage. .. varnish_vsc:: g_space :type: gauge :level: info :oneliner: Bytes available -Number of bytes left in the storage. + Number of bytes left in the storage. .. varnish_vsc_end:: sma diff --git a/bin/varnishd/smf.vsc b/bin/varnishd/smf.vsc index fb4d46d..f781227 100644 --- a/bin/varnishd/smf.vsc +++ b/bin/varnishd/smf.vsc @@ -13,49 +13,49 @@ :level: info :oneliner: Allocator requests -Number of times the storage has been asked to provide a storage segment. + Number of times the storage has been asked to provide a storage segment. .. varnish_vsc:: c_fail :type: counter :level: info :oneliner: Allocator failures -Number of times the storage has failed to provide a storage segment. + Number of times the storage has failed to provide a storage segment. .. varnish_vsc:: c_bytes :type: counter :level: info :oneliner: Bytes allocated -Number of total bytes allocated by this storage. + Number of total bytes allocated by this storage. .. varnish_vsc:: c_freed :type: counter :level: info :oneliner: Bytes freed -Number of total bytes returned to this storage. + Number of total bytes returned to this storage. .. varnish_vsc:: g_alloc :type: gauge :level: info :oneliner: Allocations outstanding -Number of storage allocations outstanding. + Number of storage allocations outstanding. .. varnish_vsc:: g_bytes :type: gauge :level: info :oneliner: Bytes outstanding -Number of bytes allocated from the storage. + Number of bytes allocated from the storage. .. varnish_vsc:: g_space :type: gauge :level: info :oneliner: Bytes available -Number of bytes left in the storage. + Number of bytes left in the storage. .. varnish_vsc:: g_smf :type: gauge diff --git a/bin/varnishd/vbe.vsc b/bin/varnishd/vbe.vsc index 23ac573..751e1ab 100644 --- a/bin/varnishd/vbe.vsc +++ b/bin/varnishd/vbe.vsc @@ -18,49 +18,49 @@ :level: info :oneliner: Request header bytes -Total backend request header bytes sent + Total backend request header bytes sent .. varnish_vsc:: bereq_bodybytes :type: counter :level: info :oneliner: Request body bytes -Total backend request body bytes sent + Total backend request body bytes sent .. varnish_vsc:: beresp_hdrbytes :type: counter :level: info :oneliner: Response header bytes -Total backend response header bytes received + Total backend response header bytes received .. varnish_vsc:: beresp_bodybytes :type: counter :level: info :oneliner: Response body bytes -Total backend response body bytes received + Total backend response body bytes received .. varnish_vsc:: pipe_hdrbytes :type: counter :level: info :oneliner: Pipe request header bytes -Total request bytes sent for piped sessions + Total request bytes sent for piped sessions .. varnish_vsc:: pipe_out :type: counter :level: info :oneliner: Piped bytes to backend -Total number of bytes forwarded to backend in pipe sessions + Total number of bytes forwarded to backend in pipe sessions .. varnish_vsc:: pipe_in :type: counter :level: info :oneliner: Piped bytes from backend -Total number of bytes forwarded from backend in pipe sessions + Total number of bytes forwarded from backend in pipe sessions .. varnish_vsc:: conn :type: gauge diff --git a/bin/varnishstat/Makefile.am b/bin/varnishstat/Makefile.am index 34c7d41..9e404bb 100644 --- a/bin/varnishstat/Makefile.am +++ b/bin/varnishstat/Makefile.am @@ -20,6 +20,3 @@ varnishstat_LDADD = \ $(top_builddir)/lib/libvarnishapi/libvarnishapi.la \ @SAN_LDFLAGS@ \ @CURSES_LIB@ ${RT_LIBS} ${LIBM} ${PTHREAD_LIBS} - -noinst_PROGRAMS = vsc2rst -vsc2rst_SOURCES = vsc2rst.c $(top_srcdir)/include/tbl/vsc_fields.h diff --git a/bin/varnishstat/vsc2rst.c b/bin/varnishstat/vsc2rst.c deleted file mode 100644 index 760f130..0000000 --- a/bin/varnishstat/vsc2rst.c +++ /dev/null @@ -1,84 +0,0 @@ -/*- - * Copyright (c) 2011-2015 Varnish Software AS - * All rights reserved. - * - * Author: Tollef Fog Heen - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - */ - -#ifndef __FLEXELINT__ - -#include "config.h" - -#include - -#define VSC_LEVEL_F(v,l,e,d) \ - static const char VSC_level_##v[] = #v; -#include "tbl/vsc_levels.h" -#undef VSC_LEVEL_F - -static void -L(const char *s) -{ - printf("\n%s\n", s); - for (;*s != '\0'; s++) - putchar('='); - putchar('\n'); - putchar('\n'); -} - -int main(int argc, char **argv) -{ - (void)argc; - (void)argv; - L("COUNTER LEVELS"); - -#define VSC_LEVEL_F(v,l,e,d) \ - printf("%s ? %s\n\t%s\n\n", l, e, d); - -#include "tbl/vsc_levels.h" - -#define VSC_DO(U,l,t,h) L(h); - -#define VSC_F(n, t, l, s, f, v, d, e) \ - do { \ - printf("``%s`` ? ", #n); \ - if (s == 'c') \ - printf("`counter` - "); \ - else if (s == 'g') \ - printf("`gauge` - "); \ - else if (s == 'b') \ - printf("`bitmap` - "); \ - printf("%s\n\n", VSC_level_##v); \ - printf("\t%s\n\n", d); \ - printf("\t%s\n\n", e); \ - } while (0); - -#define VSC_DONE(U,l,t) - -#include "tbl/vsc_all.h" - return (0); -} - -#endif // __FLEXELINT__ diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index a98eb31..dc14c84 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -123,8 +123,21 @@ include/params.rst: $(top_builddir)/bin/varnishd/varnishd $(top_builddir)/bin/varnishd/varnishd -x parameter > $@ BUILT_SOURCES += include/params.rst -include/counters.rst: $(top_builddir)/bin/varnishstat/vsc2rst - $(top_builddir)/bin/varnishstat/vsc2rst > $@ +COUNTERS = \ + $(top_srcdir)/bin/varnishd/main.vsc \ + $(top_srcdir)/bin/varnishd/mgt.vsc \ + $(top_srcdir)/bin/varnishd/mempool.vsc \ + $(top_srcdir)/bin/varnishd/sma.vsc \ + $(top_srcdir)/bin/varnishd/smf.vsc \ + $(top_srcdir)/bin/varnishd/vbe.vsc \ + $(top_srcdir)/bin/varnishd/lck.vsc + +include/counters.rst: $(top_srcdir)/lib/libvcc/vsctool.py $(COUNTERS) + echo -n '' > $@ + for i in $(COUNTERS); do \ + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -r $$i >> $@ ; \ + done + BUILT_SOURCES += include/counters.rst # XXX add varnishstat here when it's been _opt2rst'ed diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index 26d22d9..6140095 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- encoding: utf-8 -*- # # Copyright (c) 2017 Varnish Software AS # All rights reserved. @@ -241,6 +242,16 @@ class rst_vsc_begin(directive): def vscset(self, ss): ss.append(vscset(self.arg, self)) + def emit_rst(self, fo): + fo.write("\n..\n\t" + self.cmd + ":: " + self.arg + "\n") + + s = self.arg.upper() + " ? " + self.param["oneliner"] + fo.write("\n") + fo.write(s + "\n") + fo.write("=" * len(s) + "\n") + + self.emit_rst_doc(fo) + class rst_vsc(directive): def __init__(self, s): super(rst_vsc, self).__init__(s) @@ -250,9 +261,11 @@ class rst_vsc(directive): self.param["level"] = "info" def emit_rst(self, fo): - fo.write("\n``%s`` - " % self.arg) + fo.write("\n``%s`` ? " % self.arg) fo.write("`%s` - " % self.param["type"]) - fo.write("%s\n" % self.param["level"]) + fo.write("%s\n\n" % self.param["level"]) + + fo.write("\t" + self.param["oneliner"] + "\n") self.emit_rst_doc(fo) def vscset(self, ss): @@ -266,6 +279,10 @@ class rst_vsc_end(directive): def vscset(self, ss): ss[-1].complete() + def emit_rst(self, fo): + fo.write("\n..\n\t" + self.cmd + ":: " + self.arg + "\n") + self.emit_rst_doc(fo) + class other(object): def __init__(self, s): self.s = s @@ -316,8 +333,8 @@ class vsc_file(object): for i in self.vscset: i.emit_c() - def emit_rst(self, fon): - fo = open(fon, "w") + def emit_rst(self): + fo = sys.stdout for i in self.c: i.emit_rst(fo) @@ -330,13 +347,13 @@ if __name__ == "__main__": fo = sys.stdout if len(args) != 1: - print("Need exactly one filename argument") + sys.stderr.write("Need exactly one filename argument\n") exit(2) vf = vsc_file(args[0]) for f,v in optlist: if f == '-r': - vf.emit_rst("_.rst") + vf.emit_rst() if f == '-h': vf.emit_h() if f == '-c': From phk at FreeBSD.org Sat May 27 11:50:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 27 May 2017 13:50:06 +0200 Subject: [master] 6ae8119 Make C11 _Static_assert() available if we have it Message-ID: commit 6ae8119e0f96e652cc1ba7b1e298438a850f52b7 Author: Poul-Henning Kamp Date: Sat May 27 09:50:27 2017 +0000 Make C11 _Static_assert() available if we have it diff --git a/configure.ac b/configure.ac index 0c95ea0..a3adfb6 100644 --- a/configure.ac +++ b/configure.ac @@ -359,6 +359,27 @@ AC_CHECK_DECL([SO_ACCEPTFILTER], ] ) +# Check for _Static_assert() +# We dont' want to go all C11 on this, and we have vas.h instead +# of assert.h, so check the brutal way + +save_LIBS="${LIBS}" +LIBS="" +AC_CACHE_CHECK([for _Static_assert], + [ac_cv_static_assert], + [AC_RUN_IFELSE( + [AC_LANG_PROGRAM([[ +_Static_assert(1 == sizeof(char), "didn't work"); + ]],[[ + ]])], + [ac_cv_static_assert=yes], + [ac_cv_static_assert=no]) + ]) +if test "$ac_cv_static_assert" = yes; then + AC_DEFINE([STATIC_ASSERT], [1], [Define if _Static_assert is availabel]) +fi +LIBS="${save_LIBS}" + # Older Solaris versions define SO_{RCV,SND}TIMEO, but do not # implement them. # diff --git a/include/vdef.h b/include/vdef.h index f0afe09..aa9be07 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -125,4 +125,9 @@ #define NEEDLESS(s) s #endif +#ifndef STATIC_ASSERT +# define _Static_assert(a,b) +#endif + + #endif /* VDEF_H_INCLUDED */ From phk at FreeBSD.org Sat May 27 11:50:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 27 May 2017 13:50:06 +0200 Subject: [master] e57c486 Use _Static_assert to ensure we know how the struct is packed. Message-ID: commit e57c486a1b23b500261464396f15bdd925176709 Author: Poul-Henning Kamp Date: Sat May 27 09:50:43 2017 +0000 Use _Static_assert to ensure we know how the struct is packed. diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index 6140095..31d4088 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -62,10 +62,13 @@ class vscset(object): self.mbrs = [] self.head = m self.completed = False + self.off = 8 def addmbr(self, m): assert not self.completed self.mbrs.append(m) + m.param["index"] = self.off + self.off += 8 def complete(self): self.completed = True @@ -79,12 +82,10 @@ class vscset(object): dd["elements"] = len(self.mbrs) el = collections.OrderedDict() dd["elem"] = el - en = 0 for i in self.mbrs: - en += 1 ed = collections.OrderedDict() el[i.arg] = ed - ed["index"] = en + ed["index"] = i.param["index"] ed["name"] = i.arg ed["type"] = i.param["type"] ed["level"] = i.param["level"] @@ -138,15 +139,24 @@ class vscset(object): fo = open(fon, "w") genhdr(fo, self.name) fo.write('#include "config.h"\n') + fo.write('#include \n') fo.write('#include \n') fo.write('#include \n') fo.write('#include \n') fo.write('#include "common/common.h"\n') fo.write('#include "VSC_%s.h"\n' % self.name) + fo.write("\n") fo.write('static const char vsc_%s_name[] = "%s";\n' % (self.name, self.name.upper())) + fo.write("\n") + for i in self.mbrs: + fo.write("_Static_assert(offsetof(" + self.struct) + fo.write(", " + i.arg + ")") + fo.write(" == %d," % (i.param["index"] - 8)) + fo.write(' "VSC element offset is wrong");\n') + self.emit_json(fo) fo.write("\n") From phk at FreeBSD.org Sat May 27 11:50:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 27 May 2017 13:50:06 +0200 Subject: [master] 85949af Check VSC parameters harder. Message-ID: commit 85949afda2ee78f88486c4855a93596039a7cf92 Author: Poul-Henning Kamp Date: Sat May 27 10:04:44 2017 +0000 Check VSC parameters harder. diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index 31d4088..4273c22 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -40,6 +40,17 @@ import gzip import StringIO import collections +TYPES = [ "counter", "gauge", "bitmap" ] +CTYPES = [ "uint64_t" ] +LEVELS = [ "info", "diag", "debug" ] + +PARAMS = { + "type": ["counter", TYPES], + "ctype": ["uint64_t", CTYPES], + "level": ["info", LEVELS], + "oneliner": True, +} + def gzip_str(s): out = StringIO.StringIO() gzip.GzipFile(fileobj=out, mode="w").write(s) @@ -265,10 +276,27 @@ class rst_vsc_begin(directive): class rst_vsc(directive): def __init__(self, s): super(rst_vsc, self).__init__(s) - if "type" not in self.param: - self.param["type"] = "counter" - if "level" not in self.param: - self.param["level"] = "info" + + for i,v in PARAMS.iteritems(): + if v is not True: + self.do_default(i, v[0], v[1]) + + for p in self.param: + if p in PARAMS: + continue + sys.stderr.write("Unknown paramter ") + sys.stderr.write("'" + p + "'") + sys.stderr.write(" on field '" + self.arg + "'\n") + exit(2) + + def do_default(self, p, v, s): + if p not in self.param: + self.param[p] = v + if self.param[p] not in s: + sys.stderr.write("Wrong " + p + " '" + self.param[p]) + sys.stderr.write("' on field '" + self.arg + "'\n") + exit(2) + def emit_rst(self, fo): fo.write("\n``%s`` ? " % self.arg) From phk at FreeBSD.org Sat May 27 11:50:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 27 May 2017 13:50:06 +0200 Subject: [master] 1f22984 Use -match instead of -expect Message-ID: commit 1f2298481b8dcc87bae1dd655cbe6577af077485 Author: Poul-Henning Kamp Date: Sat May 27 11:45:52 2017 +0000 Use -match instead of -expect diff --git a/bin/varnishtest/tests/u00005.vtc b/bin/varnishtest/tests/u00005.vtc index 798a6cb..4b068d4 100644 --- a/bin/varnishtest/tests/u00005.vtc +++ b/bin/varnishtest/tests/u00005.vtc @@ -47,5 +47,5 @@ shell -expect "MAIN.uptime" \ "varnishstat -n ${v1_name} -1" shell -expect "MAIN" \ "varnishstat -n ${v1_name} -x" -shell -expect "MAIN.uptime\":" \ +shell -match {"MAIN.uptime":} \ "varnishstat -n ${v1_name} -j" From phk at FreeBSD.org Sat May 27 11:50:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 27 May 2017 13:50:06 +0200 Subject: [master] 48ecd7d Add :order: parameters to match vsc_types.h ordering Message-ID: commit 48ecd7df2236b322f36e3c9b503c7e18264d53c0 Author: Poul-Henning Kamp Date: Sat May 27 11:46:20 2017 +0000 Add :order: parameters to match vsc_types.h ordering diff --git a/bin/varnishd/lck.vsc b/bin/varnishd/lck.vsc index 104de02..99a7824 100644 --- a/bin/varnishd/lck.vsc +++ b/bin/varnishd/lck.vsc @@ -7,6 +7,7 @@ .. varnish_vsc_begin:: lck :oneliner: Lock Counters + :order: 70 Counters which track the activity in the different classes of mutex-locks. diff --git a/bin/varnishd/main.vsc b/bin/varnishd/main.vsc index 8b30981..c903c28 100644 --- a/bin/varnishd/main.vsc +++ b/bin/varnishd/main.vsc @@ -7,6 +7,7 @@ .. varnish_vsc_begin:: main :oneliner: Main counters + :order: 10 :sumfunction: .. varnish_vsc:: summs diff --git a/bin/varnishd/mempool.vsc b/bin/varnishd/mempool.vsc index d649023..2890863 100644 --- a/bin/varnishd/mempool.vsc +++ b/bin/varnishd/mempool.vsc @@ -7,6 +7,7 @@ .. varnish_vsc_begin:: mempool :oneliner: Memory Pool Counters + :order: 30 .. varnish_vsc:: live :type: gauge diff --git a/bin/varnishd/mgt.vsc b/bin/varnishd/mgt.vsc index 2f8ee8b..6aee001 100644 --- a/bin/varnishd/mgt.vsc +++ b/bin/varnishd/mgt.vsc @@ -7,6 +7,7 @@ .. varnish_vsc_begin:: mgt :oneliner: Management Process Counters + :order: 20 .. varnish_vsc:: uptime :type: counter diff --git a/bin/varnishd/sma.vsc b/bin/varnishd/sma.vsc index 0221c78..9b4aadd 100644 --- a/bin/varnishd/sma.vsc +++ b/bin/varnishd/sma.vsc @@ -7,6 +7,7 @@ .. varnish_vsc_begin:: sma :oneliner: Malloc Stevedore Counters + :order: 40 .. varnish_vsc:: c_req :type: counter diff --git a/bin/varnishd/smf.vsc b/bin/varnishd/smf.vsc index f781227..806b2a6 100644 --- a/bin/varnishd/smf.vsc +++ b/bin/varnishd/smf.vsc @@ -7,6 +7,7 @@ .. varnish_vsc_begin:: smf :oneliner: File Stevedore Counters + :order: 50 .. varnish_vsc:: c_req :type: counter diff --git a/bin/varnishd/vbe.vsc b/bin/varnishd/vbe.vsc index 751e1ab..a84acbb 100644 --- a/bin/varnishd/vbe.vsc +++ b/bin/varnishd/vbe.vsc @@ -7,6 +7,7 @@ .. varnish_vsc_begin:: vbe :oneliner: Backend Counters + :order: 60 .. varnish_vsc:: happy :type: bitmap From phk at FreeBSD.org Sat May 27 11:50:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 27 May 2017 13:50:06 +0200 Subject: [master] 4c6b271 Retire the old VSC field definitions entirely. Message-ID: commit 4c6b2717c61644db4bd6804372b591598cdd720f Author: Poul-Henning Kamp Date: Sat May 27 11:47:46 2017 +0000 Retire the old VSC field definitions entirely. diff --git a/include/Makefile.am b/include/Makefile.am index 64fa642..d256448 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -37,11 +37,7 @@ nobase_pkginclude_HEADERS = \ tbl/vhp_huffman.h \ tbl/vhp_static.h \ tbl/vrt_stv_var.h \ - tbl/vsc_all.h \ - tbl/vsc_f_main.h \ - tbl/vsc_fields.h \ tbl/vsc_levels.h \ - tbl/vsc_types.h \ tbl/vsl_tags.h \ tbl/vsl_tags_http.h \ tbl/waiters.h \ diff --git a/include/tbl/vsc_all.h b/include/tbl/vsc_all.h deleted file mode 100644 index c77c2b8..0000000 --- a/include/tbl/vsc_all.h +++ /dev/null @@ -1,90 +0,0 @@ -/*- - * Copyright (c) 2010-2015 Varnish Software AS - * All rights reserved. - * - * Author: Poul-Henning Kamp - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - */ - -/*lint -save -e525 -e539 */ - -VSC_DO(MAIN, main, VSC_type_main, "MAIN COUNTERS (MAIN.*)") - #define VSC_FF VSC_F - #include "tbl/vsc_f_main.h" - #undef VSC_FF -VSC_DONE(MAIN, main, VSC_type_main) - -VSC_DO(MGT, mgt, VSC_type_mgt, "MANAGEMENT PROCESS COUNTERS (MGT.*)") - #define VSC_DO_MGT - #define VSC_FF VSC_F - #include "tbl/vsc_fields.h" - #undef VSC_FF - #undef VSC_DO_MGT -VSC_DONE(MGT, mgt, VSC_type_mgt) - -VSC_DO(MEMPOOL, mempool, VSC_type_mempool, "MEMORY POOL COUNTERS (MEMPOOL.*)") - #define VSC_DO_MEMPOOL - #define VSC_FF VSC_F - #include "tbl/vsc_fields.h" - #undef VSC_FF - #undef VSC_DO_MEMPOOL -VSC_DONE(MEMPOOL, mempool, VSC_type_mempool) - -VSC_DO(SMA, sma, VSC_type_sma, "MALLOC STORAGE COUNTERS (SMA.*)") - #define VSC_DO_SMA - #define VSC_FF VSC_F - #include "tbl/vsc_fields.h" - #undef VSC_FF - #undef VSC_DO_SMA -VSC_DONE(SMA, sma, VSC_type_sma) - -VSC_DO(SMF, smf, VSC_type_smf, "FILE STORAGE COUNTERS (SMF.*)") - #define VSC_DO_SMF - #define VSC_FF VSC_F - #include "tbl/vsc_fields.h" - #undef VSC_FF - #undef VSC_DO_SMF -VSC_DONE(SMF, smf, VSC_type_smf) - -VSC_DO(VBE, vbe, VSC_type_vbe, "BACKEND COUNTERS (VBE.*)") - #define VSC_DO_VBE - #define VSC_FF VSC_F - #include "tbl/vsc_fields.h" - #undef VSC_FF - #undef VSC_DO_VBE -VSC_DONE(VBE, vbe, VSC_type_vbe) - -VSC_DO(LCK, lck, VSC_type_lck, "LOCK COUNTERS (LCK.*)") - #define VSC_DO_LCK - #define VSC_FF VSC_F - #include "tbl/vsc_fields.h" - #undef VSC_FF - #undef VSC_DO_LCK -VSC_DONE(LCK, lck, VSC_type_lck) - -#undef VSC_DO -#undef VSC_F -#undef VSC_DONE - -/*lint -restore */ diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h deleted file mode 100644 index 2a109c8..0000000 --- a/include/tbl/vsc_f_main.h +++ /dev/null @@ -1,735 +0,0 @@ -/*- - * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2015 Varnish Software AS - * All rights reserved. - * - * Author: Poul-Henning Kamp - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * Definition of the main shared memory statistics below. - * - * See include/tbl/vsc_fields.h for the table definition. - */ - -/*lint -save -e525 -e539 */ - -VSC_FF(summs, uint64_t, 0, 'c', 'd', debug, "", "") - -/*-------------------------------------------------------------------- - * Globals, not related to traffic - */ - -VSC_FF(uptime, uint64_t, 0, 'c', 'd', info, - "Child process uptime", - "How long the child process has been running." -) - -/*--------------------------------------------------------------------- - * Sessions - */ - -VSC_FF(sess_conn, uint64_t, 1, 'c', 'i', info, - "Sessions accepted", - "Count of sessions successfully accepted" -) - -VSC_FF(sess_drop, uint64_t, 1, 'c', 'i', info, - "Sessions dropped", - "Count of sessions silently dropped due to lack of worker thread." -) - -VSC_FF(sess_fail, uint64_t, 1, 'c', 'i', info, - "Session accept failures", - "Count of failures to accept TCP connection." - " Either the client changed its mind, or the kernel ran out of" - " some resource like file descriptors." -) - -/*---------------------------------------------------------------------*/ - -VSC_FF(client_req_400, uint64_t, 1, 'c', 'i', info, - "Client requests received, subject to 400 errors", - "400 means we couldn't make sense of the request, it was" - " malformed in some drastic way." -) - -VSC_FF(client_req_417, uint64_t, 1, 'c', 'i', info, - "Client requests received, subject to 417 errors", - "417 means that something went wrong with an Expect: header." -) - -VSC_FF(client_req, uint64_t, 1, 'c', 'i', info, - "Good client requests received", - "The count of parseable client requests seen." -) - -/*---------------------------------------------------------------------*/ - -VSC_FF(cache_hit, uint64_t, 1, 'c', 'i', info, - "Cache hits", - "Count of cache hits. " - " A cache hit indicates that an object has been delivered to a" - " client without fetching it from a backend server." -) - -VSC_FF(cache_hitpass, uint64_t, 1, 'c', 'i', info, - "Cache hits for pass.", - "Count of hits for pass." - " A cache hit for pass indicates that Varnish is going to" - " pass the request to the backend and this decision has been" - " cached in it self. This counts how many times the cached" - " decision is being used." -) - -VSC_FF(cache_hitmiss, uint64_t, 1, 'c', 'i', info, - "Cache hits for miss.", - "Count of hits for miss." - " A cache hit for miss indicates that Varnish is going to" - " proceed as for a cache miss without request coalescing, and" - " this decision has been cached. This counts how many times the" - " cached decision is being used." -) - -VSC_FF(cache_miss, uint64_t, 1, 'c', 'i', info, - "Cache misses", - "Count of misses." - " A cache miss indicates the object was fetched from the" - " backend before delivering it to the client." -) - -/*---------------------------------------------------------------------*/ - -VSC_FF(backend_conn, uint64_t, 0, 'c', 'i', info, - "Backend conn. success", - "How many backend connections have successfully been" - " established." -) - -VSC_FF(backend_unhealthy, uint64_t, 0, 'c', 'i', info, - "Backend conn. not attempted", - "" -) - -VSC_FF(backend_busy, uint64_t, 0, 'c', 'i', info, - "Backend conn. too many", - "" -) - -VSC_FF(backend_fail, uint64_t, 0, 'c', 'i', info, - "Backend conn. failures", - "" -) - -VSC_FF(backend_reuse, uint64_t, 0, 'c', 'i', info, - "Backend conn. reuses", - "Count of backend connection reuses." - " This counter is increased whenever we reuse a recycled connection." -) - -VSC_FF(backend_recycle, uint64_t, 0, 'c', 'i', info, - "Backend conn. recycles", - "Count of backend connection recycles." - " This counter is increased whenever we have a keep-alive" - " connection that is put back into the pool of connections." - " It has not yet been used, but it might be, unless the backend" - " closes it." -) - -VSC_FF(backend_retry, uint64_t, 0, 'c', 'i', info, - "Backend conn. retry", - "" -) - -/*--------------------------------------------------------------------- - * Backend fetch statistics - */ - -VSC_FF(fetch_head, uint64_t, 1, 'c', 'i', info, - "Fetch no body (HEAD)", - "beresp with no body because the request is HEAD." -) - -VSC_FF(fetch_length, uint64_t, 1, 'c', 'i', info, - "Fetch with Length", - "beresp.body with Content-Length." -) - -VSC_FF(fetch_chunked, uint64_t, 1, 'c', 'i', info, - "Fetch chunked", - "beresp.body with Chunked." -) - -VSC_FF(fetch_eof, uint64_t, 1, 'c', 'i', info, - "Fetch EOF", - "beresp.body with EOF." -) - -VSC_FF(fetch_bad, uint64_t, 1, 'c', 'i', info, - "Fetch bad T-E", - "beresp.body length/fetch could not be determined." -) - -VSC_FF(fetch_none, uint64_t, 1, 'c', 'i', info, - "Fetch no body", - "beresp.body empty" -) - -VSC_FF(fetch_1xx, uint64_t, 1, 'c', 'i', info, - "Fetch no body (1xx)", - "beresp with no body because of 1XX response." -) - -VSC_FF(fetch_204, uint64_t, 1, 'c', 'i', info, - "Fetch no body (204)", - "beresp with no body because of 204 response." -) - -VSC_FF(fetch_304, uint64_t, 1, 'c', 'i', info, - "Fetch no body (304)", - "beresp with no body because of 304 response." -) - -VSC_FF(fetch_failed, uint64_t, 1, 'c', 'i', info, - "Fetch failed (all causes)", - "beresp fetch failed." -) - -VSC_FF(fetch_no_thread, uint64_t, 1, 'c', 'i', info, - "Fetch failed (no thread)", - "beresp fetch failed, no thread available." -) - -/*--------------------------------------------------------------------- - * Pools, threads, and sessions - * see: cache_pool.c - * - */ - -VSC_FF(pools, uint64_t, 0, 'g', 'i', info, - "Number of thread pools", - "Number of thread pools. See also parameter thread_pools." - " NB: Presently pools cannot be removed once created." -) - -VSC_FF(threads, uint64_t, 0, 'g', 'i', info, - "Total number of threads", - "Number of threads in all pools." - " See also parameters thread_pools, thread_pool_min and" - " thread_pool_max." -) - -VSC_FF(threads_limited, uint64_t, 0, 'c', 'i', info, - "Threads hit max", - "Number of times more threads were needed, but limit was reached" - " in a thread pool. See also parameter thread_pool_max." -) - -VSC_FF(threads_created, uint64_t, 0, 'c', 'i', info, - "Threads created", - "Total number of threads created in all pools." -) - -VSC_FF(threads_destroyed, uint64_t, 0, 'c', 'i', info, - "Threads destroyed", - "Total number of threads destroyed in all pools." -) - -VSC_FF(threads_failed, uint64_t, 0, 'c', 'i', info, - "Thread creation failed", - "Number of times creating a thread failed." - " See VSL::Debug for diagnostics." - " See also parameter thread_fail_delay." -) - -VSC_FF(thread_queue_len, uint64_t, 0, 'g', 'i', info, - "Length of session queue", - "Length of session queue waiting for threads." - " NB: Only updates once per second." - " See also parameter thread_queue_limit." -) - -VSC_FF(busy_sleep, uint64_t, 1, 'c', 'i', info, - "Number of requests sent to sleep on busy objhdr", - "Number of requests sent to sleep without a worker thread because" - " they found a busy object." -) - -VSC_FF(busy_wakeup, uint64_t, 1, 'c', 'i', info, - "Number of requests woken after sleep on busy objhdr", - "Number of requests taken off the busy object sleep list and" - " rescheduled." -) - -VSC_FF(busy_killed, uint64_t, 1, 'c', 'i', info, - "Number of requests killed after sleep on busy objhdr", - "Number of requests killed from the busy object sleep list" - " due to lack of resources." -) - -VSC_FF(sess_queued, uint64_t, 0, 'c', 'i', info, - "Sessions queued for thread", - "Number of times session was queued waiting for a thread." - " See also parameter thread_queue_limit." -) - -VSC_FF(sess_dropped, uint64_t, 0, 'c', 'i', info, - "Sessions dropped for thread", - "Number of times session was dropped because the queue were too" - " long already. See also parameter thread_queue_limit." -) - -/*---------------------------------------------------------------------*/ - -VSC_FF(n_object, uint64_t, 1, 'g', 'i', info, - "object structs made", - "Approximate number of HTTP objects (headers + body, if present)" - " in the cache." -) - -VSC_FF(n_vampireobject, uint64_t, 1, 'g', 'i', diag, - "unresurrected objects", - "Number of unresurrected objects" -) - -VSC_FF(n_objectcore, uint64_t, 1, 'g', 'i', info, - "objectcore structs made", - "Approximate number of object metadata elements in the cache." - " Each object needs an objectcore, extra objectcores are for" - " hit-for-miss, hit-for-pass and busy objects." -) - -VSC_FF(n_objecthead, uint64_t, 1, 'g', 'i', info, - "objecthead structs made", - "Approximate number of different hash entries in the cache." -) - -VSC_FF(n_backend, uint64_t, 0, 'g', 'i', info, - "Number of backends", - "Number of backends known to us." -) - -VSC_FF(n_expired, uint64_t, 0, 'g', 'i', info, - "Number of expired objects", - "Number of objects that expired from cache" - " because of old age." -) - -VSC_FF(n_lru_nuked, uint64_t, 0, 'g', 'i', info, - "Number of LRU nuked objects", - "How many objects have been forcefully evicted" - " from storage to make room for a new object." -) - -VSC_FF(n_lru_moved, uint64_t, 0, 'g', 'i', diag, - "Number of LRU moved objects", - "Number of move operations done on the LRU list." -) - -VSC_FF(losthdr, uint64_t, 0, 'c', 'i', info, - "HTTP header overflows", - "" -) - -VSC_FF(s_sess, uint64_t, 1, 'c', 'i', info, - "Total sessions seen", - "" -) - -VSC_FF(s_req, uint64_t, 1, 'c', 'i', info, - "Total requests seen", - "" -) - -VSC_FF(s_pipe, uint64_t, 1, 'c', 'i', info, - "Total pipe sessions seen", - "" -) - -VSC_FF(s_pass, uint64_t, 1, 'c', 'i', info, - "Total pass-ed requests seen", - "" -) - -VSC_FF(s_fetch, uint64_t, 1, 'c', 'i', info, - "Total backend fetches initiated", - "" -) - -VSC_FF(s_synth, uint64_t, 1, 'c', 'i', info, - "Total synthethic responses made", - "" -) - -VSC_FF(s_req_hdrbytes, uint64_t, 1, 'c', 'B', info, - "Request header bytes", - "Total request header bytes received" -) - -VSC_FF(s_req_bodybytes, uint64_t, 1, 'c', 'B', info, - "Request body bytes", - "Total request body bytes received" -) - -VSC_FF(s_resp_hdrbytes, uint64_t, 1, 'c', 'B', info, - "Response header bytes", - "Total response header bytes transmitted" -) - -VSC_FF(s_resp_bodybytes, uint64_t, 1, 'c', 'B', info, - "Response body bytes", - "Total response body bytes transmitted" -) - -VSC_FF(s_pipe_hdrbytes, uint64_t, 0, 'c', 'B', info, - "Pipe request header bytes", - "Total request bytes received for piped sessions" -) - -VSC_FF(s_pipe_in, uint64_t, 0, 'c', 'B', info, - "Piped bytes from client", - "Total number of bytes forwarded from clients in" - " pipe sessions" -) - -VSC_FF(s_pipe_out, uint64_t, 0, 'c', 'B', info, - "Piped bytes to client", - "Total number of bytes forwarded to clients in" - " pipe sessions" -) - -VSC_FF(sess_closed, uint64_t, 1, 'c', 'i', info, - "Session Closed", - "" -) - -VSC_FF(sess_closed_err, uint64_t, 0, 'c', 'i', info, - "Session Closed with error", - "Total number of sessions closed with errors." - " See sc_* diag counters for detailed breakdown" -) - -VSC_FF(sess_readahead, uint64_t, 1, 'c', 'i', info, - "Session Read Ahead", - "" -) - -VSC_FF(sess_herd, uint64_t, 1, 'c', 'i', diag, - "Session herd", - "Number of times the timeout_linger triggered" -) - -#define SESS_CLOSE_ERR0 "OK " -#define SESS_CLOSE_ERR1 "Err " -#define SESS_CLOSE_ERROR0 "" -#define SESS_CLOSE_ERROR1 "Error " -#define SESS_CLOSE(r, f, e, s) \ -VSC_FF(sc_ ## f, uint64_t, 0, 'c', 'i', diag, \ - "Session " SESS_CLOSE_ERR ## e #r, \ - "Number of session closes with " \ - SESS_CLOSE_ERROR ## e #r " (" s ")" \ -) -#include "tbl/sess_close.h" -#undef SESS_CLOSE_ERROR1 -#undef SESS_CLOSE_ERROR0 -#undef SESS_CLOSE_ERR1 -#undef SESS_CLOSE_ERR0 -#undef SESS_CLOSE - -/*--------------------------------------------------------------------*/ - -VSC_FF(shm_records, uint64_t, 0, 'c', 'i', diag, - "SHM records", - "" -) - -VSC_FF(shm_writes, uint64_t, 0, 'c', 'i', diag, - "SHM writes", - "" -) - -VSC_FF(shm_flushes, uint64_t, 0, 'c', 'i', diag, - "SHM flushes due to overflow", - "" -) - -VSC_FF(shm_cont, uint64_t, 0, 'c', 'i', diag, - "SHM MTX contention", - "" -) - -VSC_FF(shm_cycles, uint64_t, 0, 'c', 'i', diag, - "SHM cycles through buffer", - "" -) - -/*--------------------------------------------------------------------*/ - -VSC_FF(backend_req, uint64_t, 0, 'c', 'i', info, - "Backend requests made", - "" -) - -/*--------------------------------------------------------------------*/ - -VSC_FF(n_vcl, uint64_t, 0, 'c', 'i', info, - "Number of loaded VCLs in total", - "" -) - -VSC_FF(n_vcl_avail, uint64_t, 0, 'c', 'i', diag, - "Number of VCLs available", - "" -) - -VSC_FF(n_vcl_discard, uint64_t, 0, 'c', 'i', diag, - "Number of discarded VCLs", - "" -) - -VSC_FF(vcl_fail, uint64_t, 1, 'c', 'i', info, - "VCL failures", - "Count of failures which prevented VCL from completing." -) - -/*--------------------------------------------------------------------*/ - -VSC_FF(bans, uint64_t, 0, 'g', 'i', info, - "Count of bans", - "Number of all bans in system, including bans superseded" - " by newer bans and bans already checked by the ban-lurker." -) - -VSC_FF(bans_completed, uint64_t, 0, 'g', 'i', diag, - "Number of bans marked 'completed'", - "Number of bans which are no longer active, either because they" - " got checked by the ban-lurker or superseded by newer identical bans." -) - -VSC_FF(bans_obj, uint64_t, 0, 'g', 'i', diag, - "Number of bans using obj.*", - "Number of bans which use obj.* variables. These bans can possibly" - " be washed by the ban-lurker." -) - -VSC_FF(bans_req, uint64_t, 0, 'g', 'i', diag, - "Number of bans using req.*", - "Number of bans which use req.* variables. These bans can not" - " be washed by the ban-lurker." -) - -VSC_FF(bans_added, uint64_t, 0, 'c', 'i', diag, - "Bans added", - "Counter of bans added to ban list." -) - -VSC_FF(bans_deleted, uint64_t, 0, 'c', 'i', diag, - "Bans deleted", - "Counter of bans deleted from ban list." -) - -VSC_FF(bans_tested, uint64_t, 0, 'c', 'i', diag, - "Bans tested against objects (lookup)", - "Count of how many bans and objects have been tested against" - " each other during hash lookup." -) - -VSC_FF(bans_obj_killed, uint64_t, 0, 'c', 'i', diag, - "Objects killed by bans (lookup)", - "Number of objects killed by bans during object lookup." -) - -VSC_FF(bans_lurker_tested, uint64_t, 0, 'c', 'i', diag, - "Bans tested against objects (lurker)", - "Count of how many bans and objects have been tested against" - " each other by the ban-lurker." -) - -VSC_FF(bans_tests_tested, uint64_t, 0, 'c', 'i', diag, - "Ban tests tested against objects (lookup)", - "Count of how many tests and objects have been tested against" - " each other during lookup." - " 'ban req.url == foo && req.http.host == bar'" - " counts as one in 'bans_tested' and as two in 'bans_tests_tested'" -) - -VSC_FF(bans_lurker_tests_tested, uint64_t, 0, 'c', 'i', diag, - "Ban tests tested against objects (lurker)", - "Count of how many tests and objects have been tested against" - " each other by the ban-lurker." - " 'ban req.url == foo && req.http.host == bar'" - " counts as one in 'bans_tested' and as two in 'bans_tests_tested'" -) - -VSC_FF(bans_lurker_obj_killed, uint64_t, 0, 'c', 'i', diag, - "Objects killed by bans (lurker)", - "Number of objects killed by the ban-lurker." -) - -VSC_FF(bans_lurker_obj_killed_cutoff, uint64_t, 0, 'c', 'i', diag, - "Objects killed by bans for cutoff (lurker)", - "Number of objects killed by the ban-lurker to keep the number of" - " bans below ban_cutoff." -) - -VSC_FF(bans_dups, uint64_t, 0, 'c', 'i', diag, - "Bans superseded by other bans", - "Count of bans replaced by later identical bans." -) - -VSC_FF(bans_lurker_contention, uint64_t, 0, 'c', 'i', diag, - "Lurker gave way for lookup", - "Number of times the ban-lurker had to wait for lookups." -) - -VSC_FF(bans_persisted_bytes, uint64_t, 0, 'g', 'B', diag, - "Bytes used by the persisted ban lists", - "Number of bytes used by the persisted ban lists." -) - -VSC_FF(bans_persisted_fragmentation, uint64_t, 0, 'g', 'B', diag, - "Extra bytes in persisted ban lists due to fragmentation", - "Number of extra bytes accumulated through dropped and" - " completed bans in the persistent ban lists." -) - -/*--------------------------------------------------------------------*/ - -VSC_FF(n_purges, uint64_t, 0, 'g', 'i', info, - "Number of purge operations executed", - "" -) - -VSC_FF(n_obj_purged, uint64_t, 0, 'g', 'i', info, - "Number of purged objects", - "" -) - -/*--------------------------------------------------------------------*/ - -VSC_FF(exp_mailed, uint64_t, 0, 'c', 'i', diag, - "Number of objects mailed to expiry thread", - "Number of objects mailed to expiry thread for handling." -) - -VSC_FF(exp_received, uint64_t, 0, 'c', 'i', diag, - "Number of objects received by expiry thread", - "Number of objects received by expiry thread for handling." -) - -/*--------------------------------------------------------------------*/ - -VSC_FF(hcb_nolock, uint64_t, 1, 'c', 'i', debug, - "HCB Lookups without lock", - "" -) - -VSC_FF(hcb_lock, uint64_t, 0, 'c', 'i', debug, - "HCB Lookups with lock", - "" -) - -VSC_FF(hcb_insert, uint64_t, 0, 'c', 'i', debug, - "HCB Inserts", - "" -) - -/*--------------------------------------------------------------------*/ - -VSC_FF(esi_errors, uint64_t, 0, 'c', 'i', diag, - "ESI parse errors (unlock)", - "" -) - -VSC_FF(esi_warnings, uint64_t, 0, 'c', 'i', diag, - "ESI parse warnings (unlock)", - "" -) - -/*--------------------------------------------------------------------*/ - -VSC_FF(vmods, uint64_t, 0, 'g', 'i', info, - "Loaded VMODs", - "" -) - -/*--------------------------------------------------------------------*/ - -VSC_FF(n_gzip, uint64_t, 0, 'c', 'i', info, - "Gzip operations", - "" -) - -VSC_FF(n_gunzip, uint64_t, 0, 'c', 'i', info, - "Gunzip operations", - "" -) - -VSC_FF(n_test_gunzip, uint64_t, 0, 'c', 'i', info, - "Test gunzip operations", - "Those operations occur when Varnish receives a compressed" - " object from a backend. They are done to verify the gzip" - " stream while it's inserted in storage." -) - -/*--------------------------------------------------------------------*/ - -VSC_FF(vsm_free, uint64_t, 0, 'g', 'B', diag, - "Free VSM space", - "Number of bytes free in the shared memory used to communicate" - " with tools like varnishstat, varnishlog etc." -) - -VSC_FF(vsm_used, uint64_t, 0, 'g', 'B', diag, - "Used VSM space", - "Number of bytes used in the shared memory used to communicate" - " with tools like varnishstat, varnishlog etc." -) - -VSC_FF(vsm_cooling, uint64_t, 0, 'g', 'B', debug, - "Cooling VSM space", - "Number of bytes which will soon (max 1 minute) be freed" - " in the shared memory used to communicate" - " with tools like varnishstat, varnishlog etc." -) - -VSC_FF(vsm_overflow, uint64_t, 0, 'g', 'B', diag, - "Overflow VSM space", - "Number of bytes which does not fit" - " in the shared memory used to communicate" - " with tools like varnishstat, varnishlog etc." - " If this counter is not zero, consider" - " increasing the runtime variable vsm_space." -) - -VSC_FF(vsm_overflowed, uint64_t, 0, 'c', 'B', diag, - "Overflowed VSM space", - "Total number of bytes which did not fit" - " in the shared memory used to communicate" - " with tools like varnishstat, varnishlog etc." - " If this counter is not zero, consider" - " increasing the runtime variable vsm_space." -) - -#undef VSC_FF - -/*lint -restore */ diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h deleted file mode 100644 index 30c16bd..0000000 --- a/include/tbl/vsc_fields.h +++ /dev/null @@ -1,316 +0,0 @@ -/*- - * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2015 Varnish Software AS - * All rights reserved. - * - * Author: Poul-Henning Kamp - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * Definition of all shared memory statistics below (except main - see - * include/tbl/vsc_f_main.h). - * - * Fields (n, t, l, s, f, v, d, e): - * n - Name: Field name, in C-source and stats programs - * t - C-type: uint64_t, unless marked in 's' - * l - Local: Local counter in worker thread. - * s - Semantics: Semantics of the value in this field - * 'b' - Bitmap - * 'c' - Counter, never decreases. - * 'g' - Gauge, goes up and down - * f - Format: Display format for the field - * 'b' - Bitmap - * 'i' - Integer - * 'd' - Duration - * v - Verbosity: Counter verbosity level (see vsc_levels.h) - * d - Description: Short description of field (for screen use) - * e - Explanation: Long explanation of field (for doc use) - * - * Please describe Gauge variables as "Number of..." to indicate that - * this is a snapshot, and Counter variables as "Count of" to indicate - * accumulative count. - * - * ----------------------- - * NB: Cleanup in progress - * ----------------------- - * - * Insufficient attention has caused this to become a swamp of conflicting - * conventions, shorthands and general mumbo-jumbo. I'm trying to clean - * it up as I go over the code in other business. - * - * Please see the sessmem section for how it should look. - * - */ - -/*lint -save -e525 -e539 */ - -/**********************************************************************/ - -#ifdef VSC_DO_MGT - -VSC_FF(uptime, uint64_t, 0, 'c', 'd', info, - "Management process uptime", - "Uptime in seconds of the management process" -) - -VSC_FF(child_start, uint64_t, 0, 'c', 'i', diag, - "Child process started", - "Number of times the child process has been started" -) - -VSC_FF(child_exit, uint64_t, 0, 'c', 'i', diag, - "Child process normal exit", - "Number of times the child process has been cleanly stopped" -) - -VSC_FF(child_stop, uint64_t, 0, 'c', 'i', diag, - "Child process unexpected exit", - "Number of times the child process has exited with an unexpected" - " return code" -) - -VSC_FF(child_died, uint64_t, 0, 'c', 'i', diag, - "Child process died (signal)", - "Number of times the child process has died due to signals" -) - -VSC_FF(child_dump, uint64_t, 0, 'c', 'i', diag, - "Child process core dumped", - "Number of times the child process has produced core dumps" -) - -VSC_FF(child_panic, uint64_t, 0, 'c', 'i', diag, - "Child process panic", - "Number of times the management process has caught a child panic" -) - -#endif - -/**********************************************************************/ - -#ifdef VSC_DO_LCK - -VSC_FF(creat, uint64_t, 0, 'c', 'i', debug, - "Created locks", - "" -) - -VSC_FF(destroy, uint64_t, 0, 'c', 'i', debug, - "Destroyed locks", - "" -) - -VSC_FF(locks, uint64_t, 0, 'c', 'i', debug, - "Lock Operations", - "" -) - -#endif - -/********************************************************************** - * All Stevedores support these counters - */ - -#if defined(VSC_DO_SMA) || defined (VSC_DO_SMF) -VSC_FF(c_req, uint64_t, 0, 'c', 'i', info, - "Allocator requests", - "Number of times the storage has been asked to provide a storage segment." -) - -VSC_FF(c_fail, uint64_t, 0, 'c', 'i', info, - "Allocator failures", - "Number of times the storage has failed to provide a storage segment." -) - -VSC_FF(c_bytes, uint64_t, 0, 'c', 'B', info, - "Bytes allocated", - "Number of total bytes allocated by this storage." -) - -VSC_FF(c_freed, uint64_t, 0, 'c', 'B', info, - "Bytes freed", - "Number of total bytes returned to this storage." -) - -VSC_FF(g_alloc, uint64_t, 0, 'g', 'i', info, - "Allocations outstanding", - "Number of storage allocations outstanding." -) - -VSC_FF(g_bytes, uint64_t, 0, 'g', 'B', info, - "Bytes outstanding", - "Number of bytes allocated from the storage." -) - -VSC_FF(g_space, uint64_t, 0, 'g', 'B', info, - "Bytes available", - "Number of bytes left in the storage." -) -#endif - -/**********************************************************************/ - -#ifdef VSC_DO_SMA -/* No SMA specific counters */ -#endif - -/**********************************************************************/ - -#ifdef VSC_DO_SMF -VSC_FF(g_smf, uint64_t, 0, 'g', 'i', info, - "N struct smf", - "" -) - -VSC_FF(g_smf_frag, uint64_t, 0, 'g', 'i', info, - "N small free smf", - "" -) - -VSC_FF(g_smf_large, uint64_t, 0, 'g', 'i', info, - "N large free smf", - "" -) - -#endif - -/**********************************************************************/ - -#ifdef VSC_DO_VBE - -VSC_FF(happy, uint64_t, 0, 'b', 'b', info, - "Happy health probes", - "" -) - -VSC_FF(bereq_hdrbytes, uint64_t, 0, 'c', 'B', info, - "Request header bytes", - "Total backend request header bytes sent" -) - -VSC_FF(bereq_bodybytes, uint64_t, 0, 'c', 'B', info, - "Request body bytes", - "Total backend request body bytes sent" -) - -VSC_FF(beresp_hdrbytes, uint64_t, 0, 'c', 'B', info, - "Response header bytes", - "Total backend response header bytes received" -) - -VSC_FF(beresp_bodybytes, uint64_t, 0, 'c', 'B', info, - "Response body bytes", - "Total backend response body bytes received" -) - -VSC_FF(pipe_hdrbytes, uint64_t, 0, 'c', 'B', info, - "Pipe request header bytes", - "Total request bytes sent for piped sessions" -) - -VSC_FF(pipe_out, uint64_t, 0, 'c', 'B', info, - "Piped bytes to backend", - "Total number of bytes forwarded to backend in" - " pipe sessions" -) - -VSC_FF(pipe_in, uint64_t, 0, 'c', 'B', info, - "Piped bytes from backend", - "Total number of bytes forwarded from backend in" - " pipe sessions" -) - -VSC_FF(conn, uint64_t, 0, 'g', 'i', info, - "Concurrent connections to backend", - "" -) - -VSC_FF(req, uint64_t, 0, 'c', 'i', info, - "Backend requests sent", - "" -) - -#endif - -/**********************************************************************/ -#ifdef VSC_DO_MEMPOOL - -VSC_FF(live, uint64_t, 0, 'g', 'i', debug, - "In use", - "" -) - -VSC_FF(pool, uint64_t, 0, 'g', 'i', debug, - "In Pool", - "" -) - -VSC_FF(sz_wanted, uint64_t, 0, 'g', 'B', debug, - "Size requested", - "" -) - -VSC_FF(sz_actual, uint64_t, 0, 'g', 'B', debug, - "Size allocated", - "" -) - -VSC_FF(allocs, uint64_t, 0, 'c', 'i', debug, - "Allocations", - "" -) - -VSC_FF(frees, uint64_t, 0, 'c', 'i', debug, - "Frees", - "" -) - -VSC_FF(recycle, uint64_t, 0, 'c', 'i', debug, - "Recycled from pool", - "" -) - -VSC_FF(timeout, uint64_t, 0, 'c', 'i', debug, - "Timed out from pool", - "" -) - -VSC_FF(toosmall, uint64_t, 0, 'c', 'i', debug, - "Too small to recycle", - "" -) - -VSC_FF(surplus, uint64_t, 0, 'c', 'i', debug, - "Too many for pool", - "" -) - -VSC_FF(randry, uint64_t, 0, 'c', 'i', debug, - "Pool ran dry", - "" -) - -#endif - -#undef VSC_FF - -/*lint -restore */ diff --git a/include/tbl/vsc_types.h b/include/tbl/vsc_types.h deleted file mode 100644 index 20adc1c..0000000 --- a/include/tbl/vsc_types.h +++ /dev/null @@ -1,73 +0,0 @@ -/*- - * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2014 Varnish Software AS - * All rights reserved. - * - * Author: Martin Blix Grydeland - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * Fields (n, l, e, d): - * n - Name: Field name, in C-source - * t - Type: Type name, in shm chunk - * l - Label: Display name, in stats programs - * e - Explanation: Short description of this counter type - * d - Description: Long description of this counter type - * - * The order in which the types are defined in this file determines the - * order in which counters are reported in the API, and then also the - * display order in varnishstat. - */ - -/*lint -save -e525 -e539 */ - -VSC_TYPE_F(main, "MAIN", "", "Child", - "Child process main counters" -) - -VSC_TYPE_F(mgt, "MGT", "MGT", "Master", - "Management process counters" -) - -VSC_TYPE_F(mempool, "MEMPOOL", "MEMPOOL", "Memory pool", - "Memory pool counters" -) - -VSC_TYPE_F(sma, "SMA", "SMA", "Storage malloc", - "Malloc storage counters" -) - -VSC_TYPE_F(smf, "SMF", "SMF", "Storage file", - "File storage counters" -) - -VSC_TYPE_F(vbe, "VBE", "VBE", "Backend", - "Backend counters" -) - -VSC_TYPE_F(lck, "LCK", "LCK", "Lock", - "Mutex lock counters" -) - -#undef VSC_TYPE_F - -/*lint -restore */ diff --git a/lib/libvarnishapi/vjsn.c b/lib/libvarnishapi/vjsn.c index 444a471..01f10c2 100644 --- a/lib/libvarnishapi/vjsn.c +++ b/lib/libvarnishapi/vjsn.c @@ -428,6 +428,20 @@ vjsn_parse(const char *src, const char **err) return (js); } +struct vjsn_val * +vjsn_child(const struct vjsn_val *vv, const char *key) +{ + struct vjsn_val *vc; + + CHECK_OBJ_NOTNULL(vv, VJSN_VAL_MAGIC); + AN(key); + VTAILQ_FOREACH(vc, &vv->children, list) { + if (vc->name != NULL && !strcmp(vc->name, key)) + return (vc); + } + return (NULL); +} + static void vjsn_dump_i(const struct vjsn_val *jsv, FILE *fo, int indent) { diff --git a/lib/libvarnishapi/vjsn.h b/lib/libvarnishapi/vjsn.h index 91f0e4f..ee17a3d 100644 --- a/lib/libvarnishapi/vjsn.h +++ b/lib/libvarnishapi/vjsn.h @@ -56,3 +56,4 @@ struct vjsn { struct vjsn *vjsn_parse(const char *, const char **); void vjsn_dump(const struct vjsn *js, FILE *fo); +struct vjsn_val *vjsn_child(const struct vjsn_val *, const char *); diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index f600c00..c42f501 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -51,24 +51,13 @@ #include "vsm_api.h" -/* Define the vsc type structs */ -#define VSC_DO(u,l,t,h) struct VSC_C_##l { -#define VSC_F(n,t,l,s,f,v,d,e) t n; -#define VSC_DONE(u,l,t) }; -#include "tbl/vsc_all.h" - -enum { -#define VSC_TYPE_F(n,t,l,e,d) \ - VSC_type_order_##n, -#include "tbl/vsc_types.h" -}; - struct vsc_vf { unsigned magic; #define VSC_VF_MAGIC 0x516519f8 VTAILQ_ENTRY(vsc_vf) list; struct VSM_fantom fantom; struct VSC_section section; + struct vjsn *vjsn; int order; }; VTAILQ_HEAD(vsc_vf_head, vsc_vf); @@ -117,14 +106,6 @@ static const struct VSC_level_desc * const levels[] = { static const size_t nlevels = sizeof(levels)/sizeof(*levels); -#define VSC_TYPE_F(n,t,l,e,d) const char *VSC_type_##n = t; -#include "tbl/vsc_types.h" - -#define VSC_DO(U,l,t,h) const struct VSC_desc VSC_desc_##l[] = { -#define VSC_F(n,t,l,s,f,v,d,e) {#n,#t,s,f,&level_##v,d,e}, -#define VSC_DONE(U,l,t) }; -#include "tbl/vsc_all.h" - /*--------------------------------------------------------------------*/ static struct vsc * @@ -266,7 +247,7 @@ VSC_Get(const struct VSM_data *vd, struct VSM_fantom *fantom, const char *type, /*--------------------------------------------------------------------*/ -static void +static struct vsc_vf * vsc_add_vf(struct vsc *vsc, const struct VSM_fantom *fantom, int order) { struct vsc_vf *vf, *vf2; @@ -286,9 +267,11 @@ vsc_add_vf(struct vsc *vsc, const struct VSM_fantom *fantom, int order) VTAILQ_INSERT_BEFORE(vf2, vf, list); else VTAILQ_INSERT_TAIL(&vsc->vf_list, vf, list); + return (vf); } /*lint -esym(528, vsc_add_pt) */ +/*lint -sem(vsc_add_pt, custodial(3)) */ static void vsc_add_pt(struct vsc *vsc, const volatile void *ptr, const struct VSC_desc *desc, const struct vsc_vf *vf) @@ -305,46 +288,19 @@ vsc_add_pt(struct vsc *vsc, const volatile void *ptr, VTAILQ_INSERT_TAIL(&vsc->pt_list, pt, list); } -#define VSC_DO(U,l,t,h) \ - static void \ - iter_##l(struct vsc *vsc, const struct VSC_desc *descs, \ - struct vsc_vf *vf) \ - { \ - struct VSC_C_##l *st; \ - \ - CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC); \ - st = (void*)((char*)vf->fantom.b + 8); - -#define VSC_F(nn,tt,ll,ss,ff,vv,dd,ee) \ - vsc_add_pt(vsc, &st->nn, descs++, vf); - -#define VSC_DONE(U,l,t) \ - } - -#include "tbl/vsc_all.h" - /*-------------------------------------------------------------------- */ static void -vsc_build_old_vf_list(struct vsc *vsc) -{ -#define VSC_TYPE_F(n,t,l,e,d) \ - if (!strcmp(vsc->iter_fantom.type, t)) \ - vsc_add_vf(vsc, &vsc->iter_fantom, \ - VSC_type_order_##n); -#include "tbl/vsc_types.h" -} - -#include - -static void vsc_build_vf_list(struct VSM_data *vd) { uint64_t u; struct vsc *vsc = vsc_setup(vd); const char *p; const char *e; + struct vjsn *vj; + struct vjsn_val *vv; + struct vsc_vf *vf; vsc_delete_pt_list(&vsc->pt_list); vsc_delete_vf_list(&vsc->vf_list); @@ -352,15 +308,23 @@ vsc_build_vf_list(struct VSM_data *vd) VSM_FOREACH(&vsc->iter_fantom, vd) { if (strcmp(vsc->iter_fantom.class, VSC_CLASS)) continue; - vsc_build_old_vf_list(vsc); u = vbe64dec(vsc->iter_fantom.b); assert(u > 0); p = (char*)vsc->iter_fantom.b + 8 + u; - (void)vjsn_parse(p, &e); + vj = vjsn_parse(p, &e); if (e != NULL) { fprintf(stderr, "%s\n", p); fprintf(stderr, "JSON ERROR %s\n", e); + exit(2); } + AN(vj); + vv = vjsn_child(vj->value, "order"); + AN(vv); + assert(vv->type == VJSN_NUMBER); + vf = vsc_add_vf(vsc, &vsc->iter_fantom, atoi(vv->value)); + AN(vf); + vf->vjsn = vj; + // vjsn_dump(vf->vjsn, stderr); AZ(e); } } @@ -370,17 +334,36 @@ vsc_build_pt_list(struct VSM_data *vd) { struct vsc *vsc = vsc_setup(vd); struct vsc_vf *vf; + struct vjsn_val *vve, *vv, *vt; + struct VSC_desc *vdsc = NULL; vsc_delete_pt_list(&vsc->pt_list); VTAILQ_FOREACH(vf, &vsc->vf_list, list) { -#define VSC_DO(U,l,t,h) \ - CHECK_OBJ_NOTNULL(vf, VSC_VF_MAGIC); \ - if (!strcmp(vf->section.type, t)) \ - iter_##l(vsc, VSC_desc_##l, vf); -#define VSC_F(n,t,l,s,f,v,d,e) -#define VSC_DONE(a,b,c) -#include "tbl/vsc_all.h" + vve = vjsn_child(vf->vjsn->value, "elem"); + AN(vve); + VTAILQ_FOREACH(vv, &vve->children, list) { + vdsc = calloc(sizeof *vd, 1); + AN(vdsc); + +#define DOF(n, k) \ + vt = vjsn_child(vv, k); \ + AN(vt); \ + assert(vt->type == VJSN_STRING); \ + vdsc->n = vt->value; + + DOF(name, "name"); + DOF(ctype, "ctype"); + DOF(sdesc, "oneliner"); + DOF(ldesc, "docs"); +#undef DOF + vdsc->level = &level_info; + vt = vjsn_child(vv, "index"); + AN(vt); + vsc_add_pt(vsc, + (char*)vf->fantom.b + atoi(vt->value), + vdsc, vf); + } } } diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index 4273c22..ee5432b 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -88,7 +88,8 @@ class vscset(object): dd = collections.OrderedDict() dd["version"] = "1" dd["name"] = self.name - dd["1line"] = self.head.param["oneliner"].strip() + dd["oneliner"] = self.head.param["oneliner"].strip() + dd["order"] = int(self.head.param["order"]) dd["docs"] = "\n".join(self.head.getdoc()) dd["elements"] = len(self.mbrs) el = collections.OrderedDict() @@ -96,11 +97,11 @@ class vscset(object): for i in self.mbrs: ed = collections.OrderedDict() el[i.arg] = ed + for j in PARAMS: + if j in i.param: + ed[j] = i.param[j] ed["index"] = i.param["index"] ed["name"] = i.arg - ed["type"] = i.param["type"] - ed["level"] = i.param["level"] - ed["1line"] = i.param["oneliner"].strip() ed["docs"] = "\n".join(i.getdoc()) s=json.dumps(dd, separators=(",",":")) + "\0" fo.write("\nstatic const size_t vsc_%s_jsonlen = %dL;\n" % From phk at FreeBSD.org Sat May 27 20:00:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 27 May 2017 22:00:06 +0200 Subject: [master] 8763a45 Never trust the values of VSC-counters: They are mmap'ed in other processes and therefore they are not trustworthy. Message-ID: commit 8763a45177b8c3b2f61a2416ea4b53466e0eb1e2 Author: Poul-Henning Kamp Date: Sat May 27 19:57:45 2017 +0000 Never trust the values of VSC-counters: They are mmap'ed in other processes and therefore they are not trustworthy. diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index 2de444b..55932e8 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -47,6 +47,8 @@ struct ban * volatile ban_start; static pthread_t ban_thread; static int ban_holds; +uint64_t bans_persisted_bytes; +uint64_t bans_persisted_fragmentation; struct ban_test { uint8_t oper; @@ -165,8 +167,9 @@ ban_mark_completed(struct ban *b) VWMB(); vbe32enc(b->spec + BANS_LENGTH, BANS_HEAD_LEN); VSC_C_main->bans_completed++; - VSC_C_main->bans_persisted_fragmentation += - ln - ban_len(b->spec); + bans_persisted_fragmentation += ln - ban_len(b->spec); + VSC_C_main->bans_persisted_fragmentation = + bans_persisted_fragmentation; } } @@ -303,19 +306,19 @@ ban_export(void) unsigned ln; Lck_AssertHeld(&ban_mtx); - ln = VSC_C_main->bans_persisted_bytes - - VSC_C_main->bans_persisted_fragmentation; + ln = bans_persisted_bytes - bans_persisted_fragmentation; vsb = VSB_new_auto(); AN(vsb); - VTAILQ_FOREACH_REVERSE(b, &ban_head, banhead_s, list) { + VTAILQ_FOREACH_REVERSE(b, &ban_head, banhead_s, list) AZ(VSB_bcat(vsb, b->spec, ban_len(b->spec))); - } AZ(VSB_finish(vsb)); assert(VSB_len(vsb) == ln); STV_BanExport((const uint8_t *)VSB_data(vsb), VSB_len(vsb)); VSB_destroy(&vsb); - VSC_C_main->bans_persisted_bytes = ln; - VSC_C_main->bans_persisted_fragmentation = 0; + VSC_C_main->bans_persisted_bytes = + bans_persisted_bytes = ln; + VSC_C_main->bans_persisted_fragmentation = + bans_persisted_fragmentation = 0; } /* @@ -395,7 +398,8 @@ ban_reload(const uint8_t *ban, unsigned len) VTAILQ_INSERT_TAIL(&ban_head, b2, list); else VTAILQ_INSERT_BEFORE(b, b2, list); - VSC_C_main->bans_persisted_bytes += len; + bans_persisted_bytes += len; + VSC_C_main->bans_persisted_bytes = bans_persisted_bytes; /* Hunt down older duplicates */ for (b = VTAILQ_NEXT(b2, list); b != NULL; b = VTAILQ_NEXT(b, list)) { diff --git a/bin/varnishd/cache/cache_ban.h b/bin/varnishd/cache/cache_ban.h index f3992bb..6267c4e 100644 --- a/bin/varnishd/cache/cache_ban.h +++ b/bin/varnishd/cache/cache_ban.h @@ -106,6 +106,8 @@ extern int ban_shutdown; extern struct banhead_s ban_head; extern struct ban * volatile ban_start; extern pthread_cond_t ban_lurker_cond; +extern uint64_t bans_persisted_bytes; +extern uint64_t bans_persisted_fragmentation; void ban_mark_completed(struct ban *b); unsigned ban_len(const uint8_t *banspec); diff --git a/bin/varnishd/cache/cache_ban_build.c b/bin/varnishd/cache/cache_ban_build.c index 0a3378a..c98baa6 100644 --- a/bin/varnishd/cache/cache_ban_build.c +++ b/bin/varnishd/cache/cache_ban_build.c @@ -296,7 +296,8 @@ BAN_Commit(struct ban_proto *bp) VSC_C_main->bans++; VSC_C_main->bans_added++; - VSC_C_main->bans_persisted_bytes += ln; + bans_persisted_bytes += ln; + VSC_C_main->bans_persisted_bytes = bans_persisted_bytes; if (b->flags & BANS_FLAG_OBJ) VSC_C_main->bans_obj++; diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index 9a5b43b..ad1bd12 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -86,8 +86,10 @@ ban_cleantail(const struct ban *victim) VSC_C_main->bans_deleted++; VTAILQ_REMOVE(&ban_head, b, list); VTAILQ_INSERT_TAIL(&freelist, b, list); - VSC_C_main->bans_persisted_fragmentation += + bans_persisted_fragmentation += ban_len(b->spec); + VSC_C_main->bans_persisted_fragmentation = + bans_persisted_fragmentation; ban_info_drop(b->spec, ban_len(b->spec)); } else { b = NULL; From phk at FreeBSD.org Sun May 28 07:18:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 28 May 2017 09:18:06 +0200 Subject: [master] 6411bab Fold generated lines Message-ID: commit 6411bab3cdcfa075e1e76ad2bba5d85f8f3770ca Author: Poul-Henning Kamp Date: Sat May 27 20:52:33 2017 +0000 Fold generated lines diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index ee5432b..13f7c01 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -166,8 +166,8 @@ class vscset(object): for i in self.mbrs: fo.write("_Static_assert(offsetof(" + self.struct) fo.write(", " + i.arg + ")") - fo.write(" == %d," % (i.param["index"] - 8)) - fo.write(' "VSC element offset is wrong");\n') + fo.write(" == %d,\n" % (i.param["index"] - 8)) + fo.write(' "VSC element offset is wrong");\n') self.emit_json(fo) From phk at FreeBSD.org Sun May 28 07:18:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 28 May 2017 09:18:06 +0200 Subject: [master] 8a7253e Rename VSM_common to CVSM and hide it from vmods Message-ID: commit 8a7253e25aa54e46d004e060bddc04d331e56bbf Author: Poul-Henning Kamp Date: Sun May 28 04:44:47 2017 +0000 Rename VSM_common to CVSM and hide it from vmods diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index e35a2f2..4d3262d 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -120,6 +120,7 @@ noinst_HEADERS = \ cache/cache_priv.h \ cache/cache_transport.h \ common/heritage.h \ + common/common_vsm.h \ hash/hash_slinger.h \ hpack/vhp.h \ http1/cache_http1.h \ diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 310df37..969ff97 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -36,6 +36,7 @@ #include #include "common/heritage.h" +#include "common/common_vsm.h" #include "vend.h" #include "vgz.h" @@ -547,7 +548,7 @@ vsm_cleaner(void *priv) THR_SetName("vsm_cleaner"); while (1) { AZ(pthread_mutex_lock(&vsm_mtx)); - VSM_common_cleaner(heritage.vsm, (void*)VSC_C_main); + CVSM_cleaner(heritage.vsm, (void*)VSC_C_main); AZ(pthread_mutex_unlock(&vsm_mtx)); VTIM_sleep(1.1); } @@ -603,7 +604,7 @@ VSM_Alloc(unsigned size, const char *class, const char *type, volatile void *p; AZ(pthread_mutex_lock(&vsm_mtx)); - p = VSM_common_alloc(heritage.vsm, size, class, type, ident); + p = CVSM_alloc(heritage.vsm, size, class, type, ident); AZ(pthread_mutex_unlock(&vsm_mtx)); return (TRUST_ME(p)); } @@ -613,6 +614,6 @@ VSM_Free(void *ptr) { AZ(pthread_mutex_lock(&vsm_mtx)); - VSM_common_free(heritage.vsm, ptr); + CVSM_free(heritage.vsm, ptr); AZ(pthread_mutex_unlock(&vsm_mtx)); } diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h index 00a8dc6..cbbeb47 100644 --- a/bin/varnishd/common/common.h +++ b/bin/varnishd/common/common.h @@ -78,21 +78,5 @@ const struct transport *XPORT_Find(const char *name); /* cache/cache_vcl.c */ int VCL_TestLoad(const char *); -/* common_vsm.c */ -struct vsm_sc; -struct VSC_main; -struct vsm_sc *VSM_common_new(void *ptr, ssize_t len); -void *VSM_common_alloc(struct vsm_sc *sc, ssize_t size, - const char *class, const char *type, const char *ident); -void VSM_common_free(struct vsm_sc *sc, void *ptr); -void VSM_common_delete(struct vsm_sc **sc); -void VSM_common_copy(struct vsm_sc *to, const struct vsm_sc *from); -void VSM_common_cleaner(struct vsm_sc *sc, struct VSC_main *stats); -void VSM_common_ageupdate(const struct vsm_sc *sc); - -void *VSC_Alloc(const char *, size_t, size_t, const unsigned char *, size_t, - const char *, va_list); -void VSC_Destroy(const char *, const void *); - /* mgt_cli.c */ extern struct VCLS *mgt_cls; diff --git a/bin/varnishd/common/common_vsm.c b/bin/varnishd/common/common_vsm.c index fb30c70..f900b26 100644 --- a/bin/varnishd/common/common_vsm.c +++ b/bin/varnishd/common/common_vsm.c @@ -40,6 +40,7 @@ #include "common.h" #include "common/com_params.h" +#include "common/common_vsm.h" #include "vsm_priv.h" #include "vmb.h" @@ -127,7 +128,7 @@ vsm_common_insert_free(struct vsm_sc *sc, struct vsm_range *vr) */ struct vsm_sc * -VSM_common_new(void *p, ssize_t l) +CVSM_new(void *p, ssize_t l) { struct vsm_sc *sc; struct vsm_range *vr; @@ -167,7 +168,7 @@ VSM_common_new(void *p, ssize_t l) */ void -VSM_common_cleaner(struct vsm_sc *sc, struct VSC_main *stats) +CVSM_cleaner(struct vsm_sc *sc, struct VSC_main *stats) { double now = VTIM_real(); struct vsm_range *vr, *vr2; @@ -195,7 +196,7 @@ VSM_common_cleaner(struct vsm_sc *sc, struct VSC_main *stats) */ void * -VSM_common_alloc(struct vsm_sc *sc, ssize_t size, +CVSM_alloc(struct vsm_sc *sc, ssize_t size, const char *class, const char *type, const char *ident) { struct vsm_range *vr, *vr2, *vr3; @@ -286,7 +287,7 @@ VSM_common_alloc(struct vsm_sc *sc, ssize_t size, */ void -VSM_common_free(struct vsm_sc *sc, void *ptr) +CVSM_free(struct vsm_sc *sc, void *ptr) { struct vsm_range *vr, *vr2; @@ -337,7 +338,7 @@ VSM_common_free(struct vsm_sc *sc, void *ptr) */ void -VSM_common_delete(struct vsm_sc **scp) +CVSM_delete(struct vsm_sc **scp) { struct vsm_range *vr, *vr2; struct vsm_sc *sc; @@ -366,7 +367,7 @@ VSM_common_delete(struct vsm_sc **scp) */ void -VSM_common_copy(struct vsm_sc *to, const struct vsm_sc *from) +CVSM_copy(struct vsm_sc *to, const struct vsm_sc *from) { struct vsm_range *vr; void *p; @@ -374,7 +375,7 @@ VSM_common_copy(struct vsm_sc *to, const struct vsm_sc *from) CHECK_OBJ_NOTNULL(to, VSM_SC_MAGIC); CHECK_OBJ_NOTNULL(from, VSM_SC_MAGIC); VTAILQ_FOREACH(vr, &from->r_used, list) { - p = VSM_common_alloc(to, vr->chunk->len, + p = CVSM_alloc(to, vr->chunk->len, vr->chunk->class, vr->chunk->type, vr->chunk->ident); AN(p); memcpy(p, vr->chunk + 1, vr->chunk->len); @@ -386,7 +387,7 @@ VSM_common_copy(struct vsm_sc *to, const struct vsm_sc *from) */ void -VSM_common_ageupdate(const struct vsm_sc *sc) +CVSM_ageupdate(const struct vsm_sc *sc) { CHECK_OBJ_NOTNULL(sc, VSM_SC_MAGIC); diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 809c006..85113f8 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -43,6 +43,7 @@ #include "mgt/mgt.h" #include "common/heritage.h" +#include "common/common_vsm.h" #include "hash/hash_slinger.h" #include "vav.h" @@ -448,7 +449,7 @@ mgt_uptime(const struct vev *e, int what) VSC_C_mgt->uptime = static_VSC_C_mgt.uptime = (uint64_t)(VTIM_real() - mgt_uptime_t0); if (heritage.vsm != NULL) - VSM_common_ageupdate(heritage.vsm); + CVSM_ageupdate(heritage.vsm); return (0); } diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c index 914a5ef..7c05ff7 100644 --- a/bin/varnishd/mgt/mgt_shmem.c +++ b/bin/varnishd/mgt/mgt_shmem.c @@ -40,6 +40,7 @@ #include "mgt/mgt.h" #include "common/heritage.h" +#include "common/common_vsm.h" #include "vfl.h" #include "vsm_priv.h" @@ -72,11 +73,11 @@ mgt_SHM_static_alloc(const void *ptr, ssize_t size, { void *p; - p = VSM_common_alloc(static_vsm, size, class, type, ident); + p = CVSM_alloc(static_vsm, size, class, type, ident); AN(p); memcpy(p, ptr, size); if (heritage.vsm != NULL) { - p = VSM_common_alloc(heritage.vsm, size, class, type, ident); + p = CVSM_alloc(heritage.vsm, size, class, type, ident); AN(p); memcpy(p, ptr, size); } @@ -178,17 +179,17 @@ mgt_SHM_Create(void) /* This may or may not work */ (void)mlock(p, size); - heritage.vsm = VSM_common_new(p, size); + heritage.vsm = CVSM_new(p, size); - VSM_common_copy(heritage.vsm, static_vsm); + CVSM_copy(heritage.vsm, static_vsm); - heritage.param = VSM_common_alloc(heritage.vsm, + heritage.param = CVSM_alloc(heritage.vsm, sizeof *heritage.param, VSM_CLASS_PARAM, "", ""); AN(heritage.param); *heritage.param = mgt_param; heritage.panic_str_len = 64 * 1024; - heritage.panic_str = VSM_common_alloc(heritage.vsm, + heritage.panic_str = CVSM_alloc(heritage.vsm, heritage.panic_str_len, PAN_CLASS, "", ""); AN(heritage.panic_str); @@ -228,7 +229,7 @@ mgt_SHM_Destroy(int keep) heritage.panic_str = NULL; heritage.panic_str_len = 0; heritage.param = NULL; - VSM_common_delete(&heritage.vsm); + CVSM_delete(&heritage.vsm); AZ(munmap(mgt_vsm_p, mgt_vsm_l)); mgt_vsm_p = NULL; mgt_vsm_l = 0; @@ -261,7 +262,7 @@ mgt_shm_atexit(void) if (getpid() != mgt_pid) return; if (heritage.vsm != NULL) - VSM_common_delete(&heritage.vsm); + CVSM_delete(&heritage.vsm); } /*-------------------------------------------------------------------- @@ -272,7 +273,7 @@ void mgt_SHM_Init(void) { /* Create our static VSM instance */ - static_vsm = VSM_common_new(static_vsm_buf, sizeof static_vsm_buf); + static_vsm = CVSM_new(static_vsm_buf, sizeof static_vsm_buf); /* Setup atexit handler */ AZ(atexit(mgt_shm_atexit)); diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index 13f7c01..bc7abfb 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -155,7 +155,9 @@ class vscset(object): fo.write('#include \n') fo.write('#include \n') fo.write('#include \n') - fo.write('#include "common/common.h"\n') + fo.write('#include "vdef.h"\n') + fo.write('#include "vas.h"\n') + fo.write('#include "common/common_vsm.h"\n') fo.write('#include "VSC_%s.h"\n' % self.name) fo.write("\n") From phk at FreeBSD.org Sun May 28 07:18:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 28 May 2017 09:18:06 +0200 Subject: [master] b82c6d2 Forgot to add this one. Message-ID: commit b82c6d2d3b309ba39d7ba4cc7de0755dd675d857 Author: Poul-Henning Kamp Date: Sun May 28 07:12:27 2017 +0000 Forgot to add this one. diff --git a/bin/varnishd/common/common_vsm.h b/bin/varnishd/common/common_vsm.h new file mode 100644 index 0000000..45e6dd8 --- /dev/null +++ b/bin/varnishd/common/common_vsm.h @@ -0,0 +1,50 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2017 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#ifdef COMMON_COMMON_VSM_H +#error "Multiple includes of common/common_vsm.h" +#endif +#define COMMON_COMMON_VSM_H + +/* common_vsm.c */ +struct vsm_sc; +struct VSC_main; +struct vsm_sc *CVSM_new(void *ptr, ssize_t len); +void *CVSM_alloc(struct vsm_sc *sc, ssize_t size, + const char *class, const char *type, const char *ident); +void CVSM_free(struct vsm_sc *sc, void *ptr); +void CVSM_delete(struct vsm_sc **sc); +void CVSM_copy(struct vsm_sc *to, const struct vsm_sc *from); +void CVSM_cleaner(struct vsm_sc *sc, struct VSC_main *stats); +void CVSM_ageupdate(const struct vsm_sc *sc); + +void *VSC_Alloc(const char *, size_t, size_t, const unsigned char *, size_t, + const char *, va_list); +void VSC_Destroy(const char *, const void *); From phk at FreeBSD.org Sun May 28 11:23:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 28 May 2017 13:23:05 +0200 Subject: [master] 59e5695 Move VSC_{Alloc|Destroy}() into common Message-ID: commit 59e5695d8ca96ef7f6169607f1c89019302e6ebe Author: Poul-Henning Kamp Date: Sun May 28 08:48:49 2017 +0000 Move VSC_{Alloc|Destroy}() into common diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 4d3262d..8248966 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -52,6 +52,7 @@ varnishd_SOURCES = \ cache/cache_vrt_vmod.c \ cache/cache_wrk.c \ cache/cache_ws.c \ + common/common_vsc.c \ common/common_vsm.c \ hash/hash_classic.c \ hash/hash_critbit.c \ diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 969ff97..986f685 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -41,7 +41,6 @@ #include "vend.h" #include "vgz.h" #include "vsl_priv.h" -#include "vapi/vsc_int.h" #include "vmb.h" #include "vtim.h" @@ -474,73 +473,6 @@ VSL_End(struct vsl_log *vsl) /*--------------------------------------------------------------------*/ -struct vsc_segs { - unsigned magic; -#define VSC_SEGS_MAGIC 0x9b355991 - - VTAILQ_ENTRY(vsc_segs) list; - void *seg; - void *ptr; -}; - -static VTAILQ_HEAD(,vsc_segs) vsc_seglist = - VTAILQ_HEAD_INITIALIZER(vsc_seglist); - -void * -VSC_Alloc(const char *nm, size_t sd, - size_t sj, const unsigned char *zj, size_t szj, - const char *fmt, va_list va) -{ - char *p; - z_stream vz; - struct vsc_segs *vsg; - - (void)nm; - (void)fmt; - (void)va; - - - p = VSM_Alloc(8 + sd + sj, VSC_CLASS, nm, fmt); - AN(p); - - memset(p, 0, sd); - vbe64enc(p, sd); - - memset(&vz, 0, sizeof vz); - assert(Z_OK == inflateInit2(&vz, 31)); - vz.next_in = TRUST_ME(zj); - vz.avail_in = szj; - vz.next_out = (void*)(p + 8 + sd); - vz.avail_out = sj; - assert(Z_STREAM_END == inflate(&vz, Z_FINISH)); - assert(Z_OK == inflateEnd(&vz)); - ALLOC_OBJ(vsg, VSC_SEGS_MAGIC); - AN(vsg); - vsg->seg = p; - vsg->ptr = p + 8; - VTAILQ_INSERT_TAIL(&vsc_seglist, vsg, list); - return (p + 8); -} - -void -VSC_Destroy(const char *nm, const void *p) -{ - struct vsc_segs *vsg; - - (void)nm; - VTAILQ_FOREACH(vsg, &vsc_seglist, list) { - if (vsg->ptr != p) - continue; - VSM_Free(vsg->seg); - VTAILQ_REMOVE(&vsc_seglist, vsg, list); - FREE_OBJ(vsg); - return; - } - WRONG("Freeing unknown VSC"); -} - -/*--------------------------------------------------------------------*/ - static void * vsm_cleaner(void *priv) { diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c new file mode 100644 index 0000000..6157c8c --- /dev/null +++ b/bin/varnishd/common/common_vsc.c @@ -0,0 +1,115 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include +#include +#include + +#include "common.h" +#include "common/common_vsm.h" + +#include "vend.h" +#include "vgz.h" +#include "vapi/vsc_int.h" + +/*--------------------------------------------------------------------*/ + +void *VSM_Alloc(unsigned size, const char *class, const char *type, + const char *ident); +void VSM_Free(void *ptr); + +/*--------------------------------------------------------------------*/ + +struct vsc_segs { + unsigned magic; +#define VSC_SEGS_MAGIC 0x9b355991 + + VTAILQ_ENTRY(vsc_segs) list; + void *seg; + void *ptr; +}; + +static VTAILQ_HEAD(,vsc_segs) vsc_seglist = + VTAILQ_HEAD_INITIALIZER(vsc_seglist); + +void * +VSC_Alloc(const char *nm, size_t sd, + size_t sj, const unsigned char *zj, size_t szj, + const char *fmt, va_list va) +{ + char *p; + z_stream vz; + struct vsc_segs *vsg; + + (void)nm; + (void)fmt; + (void)va; + + + p = VSM_Alloc(8 + sd + sj, VSC_CLASS, nm, fmt); + AN(p); + + memset(p, 0, sd); + vbe64enc(p, sd); + + memset(&vz, 0, sizeof vz); + assert(Z_OK == inflateInit2(&vz, 31)); + vz.next_in = TRUST_ME(zj); + vz.avail_in = szj; + vz.next_out = (void*)(p + 8 + sd); + vz.avail_out = sj; + assert(Z_STREAM_END == inflate(&vz, Z_FINISH)); + assert(Z_OK == inflateEnd(&vz)); + ALLOC_OBJ(vsg, VSC_SEGS_MAGIC); + AN(vsg); + vsg->seg = p; + vsg->ptr = p + 8; + VTAILQ_INSERT_TAIL(&vsc_seglist, vsg, list); + return (p + 8); +} + +void +VSC_Destroy(const char *nm, const void *p) +{ + struct vsc_segs *vsg; + + (void)nm; + VTAILQ_FOREACH(vsg, &vsc_seglist, list) { + if (vsg->ptr != p) + continue; + VSM_Free(vsg->seg); + VTAILQ_REMOVE(&vsc_seglist, vsg, list); + FREE_OBJ(vsg); + return; + } + WRONG("Freeing unknown VSC"); +} From phk at FreeBSD.org Sun May 28 11:23:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 28 May 2017 13:23:05 +0200 Subject: [master] a283499 Fix destroy arguments Message-ID: commit a283499221093d6e21a7b5ae52eb2ee3e6796a61 Author: Poul-Henning Kamp Date: Sun May 28 11:00:12 2017 +0000 Fix destroy arguments diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index bc7abfb..08e248e 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -196,11 +196,11 @@ class vscset(object): fo.write("\n") fo.write("void\n") fo.write("VSC_" + self.name + "_Destroy") - fo.write("(" + self.struct + "**pp)\n") + fo.write("(" + self.struct + " **pp)\n") fo.write("{\n") fo.write("\n") fo.write("\tAN(pp);\n") - fo.write('\tVSC_Destroy("%s", *pp);\n' % self.name) + fo.write('\tVSC_Destroy(vsc_%s_name, *pp);\n' % self.name) fo.write("\t*pp = NULL;\n") fo.write("}\n") From phk at FreeBSD.org Sun May 28 13:49:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 28 May 2017 15:49:05 +0200 Subject: [master] 7906f09 Add the new VSM writer bits. Not used yet. Message-ID: commit 7906f096d1db35f746e5a7f2ef68b582dcac5e3c Author: Poul-Henning Kamp Date: Sun May 28 13:40:19 2017 +0000 Add the new VSM writer bits. Not used yet. diff --git a/include/Makefile.am b/include/Makefile.am index d256448..2feb37f 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -90,6 +90,7 @@ nobase_noinst_HEADERS = \ vnum.h \ vpf.h \ vsl_priv.h \ + vsmw.h \ vsm_priv.h \ vsub.h \ vss.h \ diff --git a/include/vsmw.h b/include/vsmw.h new file mode 100644 index 0000000..1ba2975 --- /dev/null +++ b/include/vsmw.h @@ -0,0 +1,41 @@ +/*- + * Copyright (c) 2017 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifdef VSMW_H_INCLUDED +#error "Multiple includes of vsmw.h" +#endif +#define VSMW_H_INCLUDED + +struct vsmw; + +void *VSMW_Allocv(struct vsmw *, const char *, size_t, const char *, va_list); +void *VSMW_Allocf(struct vsmw *, const char *, size_t, const char *, ...); +void VSMW_Free(struct vsmw *, void **); + +struct vsmw *VSMW_New(int, int, const char *); +void VSMW_Destroy(struct vsmw **); diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am index 6e008d9..bbef1a2 100644 --- a/lib/libvarnish/Makefile.am +++ b/lib/libvarnish/Makefile.am @@ -34,6 +34,7 @@ libvarnish_a_SOURCES = \ vsa.c \ vsb.c \ vsha256.c \ + vsmw.c \ vss.c \ vsub.c \ vtcp.c \ diff --git a/lib/libvarnish/vsmw.c b/lib/libvarnish/vsmw.c new file mode 100644 index 0000000..0f6cb08 --- /dev/null +++ b/lib/libvarnish/vsmw.c @@ -0,0 +1,294 @@ +/*- + * Copyright (c) 2010-2011 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * VSM stuff common to manager and child. + * + * Please see comments in for details of protocols and + * data consistency. + * + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vdef.h" +#include "vas.h" +#include "vsb.h" +#include "miniobj.h" +#include "vsmw.h" +#include "vqueue.h" + +#include "vfil.h" +#include "vrnd.h" + +#ifndef MAP_HASSEMAPHORE +# define MAP_HASSEMAPHORE 0 /* XXX Linux */ +#endif + +#ifndef MAP_NOSYNC +# define MAP_NOSYNC 0 /* XXX Linux */ +#endif + +/*--------------------------------------------------------------------*/ + +struct vsmwseg { + unsigned magic; +#define VSMWSEG_MAGIC 0x7e4ccaea + VTAILQ_ENTRY(vsmwseg) list; + char *fn; + + char *class; + size_t len; + char *id; + void *ptr; +}; + +struct vsmw { + unsigned magic; +#define VSMW_MAGIC 0xc2ca2cd9 + int vdirfd; + int mode; + char *idx; + VTAILQ_HEAD(, vsmwseg) segs; + struct vsb *vsb; +}; + +/*--------------------------------------------------------------------*/ + +static void +vsmw_write_index(const struct vsmw *vsmw, int fd, const struct vsmwseg *seg) +{ + ssize_t s; + + CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC); + CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC); + VSB_clear(vsmw->vsb); + VSB_printf(vsmw->vsb, "%s %zu %s %s\n", + seg->fn, + seg->len, + seg->class, + seg->id); + AZ(VSB_finish(vsmw->vsb)); + s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb)); + assert(s == VSB_len(vsmw->vsb)); +} + +/*--------------------------------------------------------------------*/ + +static void +vsmw_mkent(const struct vsmw *vsmw, const char *pfx) +{ + int fd; + + while (1) { + VSB_clear(vsmw->vsb); + VSB_printf(vsmw->vsb, "%s.%lx", pfx, VRND_RandomTestable()); + AZ(VSB_finish(vsmw->vsb)); + fd = openat(vsmw->vdirfd, VSB_data(vsmw->vsb), O_RDONLY); + if (fd < 0 && errno == ENOENT) + return; + if (fd >= 0) + AZ(close(fd)); + } +} + +/*--------------------------------------------------------------------*/ + +void * +VSMW_Allocv(struct vsmw *vsmw, const char *class, size_t len, + const char *fmt, va_list va) +{ + struct vsmwseg *seg; + int fd; + size_t ps; + + CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC); + + ALLOC_OBJ(seg, VSMWSEG_MAGIC); + AN(seg); + REPLACE(seg->class, class); + seg->len = len; + + VSB_clear(vsmw->vsb); + VSB_vprintf(vsmw->vsb, fmt, va); + AZ(VSB_finish(vsmw->vsb)); + REPLACE(seg->id, VSB_data(vsmw->vsb)); + + vsmw_mkent(vsmw, class); + REPLACE(seg->fn, VSB_data(vsmw->vsb)); + + ps = getpagesize(); + len = RUP2(len, ps); + + fd = openat(vsmw->vdirfd, seg->fn, + O_RDWR | O_CREAT | O_EXCL, vsmw->mode); + assert(fd >= 0); + + AZ(VFIL_allocate(fd, (off_t)len, 1)); + + seg->ptr = (void *)mmap(NULL, len, + PROT_READ|PROT_WRITE, + MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED, + fd, 0); + + AZ(close(fd)); + assert(seg->ptr != MAP_FAILED); + + VTAILQ_INSERT_TAIL(&vsmw->segs, seg, list); + fd = openat(vsmw->vdirfd, vsmw->idx, O_APPEND | O_WRONLY); + assert(fd >= 0); + vsmw_write_index(vsmw, fd, seg); + AZ(close(fd)); + + return (seg->ptr); +} + +void * +VSMW_Allocf(struct vsmw *vsmw, const char *class, size_t len, + const char *fmt, ...) +{ + va_list ap; + void *p; + + va_start(ap, fmt); + p = VSMW_Allocv(vsmw, class, len, fmt, ap); + va_end(ap); + return (p); +} + +/*--------------------------------------------------------------------*/ +static void +vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx) +{ + char *t = NULL; + int fd; + size_t len; + + CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC); + CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC); + + len = getpagesize(); + len = RUP2(seg->len, len); + AZ(munmap(seg->ptr, len)); + + VTAILQ_REMOVE(&vsmw->segs, seg, list); + AZ(unlinkat(vsmw->vdirfd, seg->fn, 0)); + REPLACE(seg->fn, NULL); + REPLACE(seg->id, NULL); + FREE_OBJ(seg); + + if (fixidx) { + vsmw_mkent(vsmw, vsmw->idx); + REPLACE(t, VSB_data(vsmw->vsb)); + AN(t); + fd = open(t, O_WRONLY|O_CREAT|O_EXCL, vsmw->mode); + assert(fd >= 0); + VTAILQ_FOREACH(seg, &vsmw->segs, list) + vsmw_write_index(vsmw, fd, seg); + AZ(close(fd)); + AZ(rename(t, vsmw->idx)); + REPLACE(t, NULL); + } +} + +/*--------------------------------------------------------------------*/ + +void +VSMW_Free(struct vsmw *vsmw, void **pp) +{ + struct vsmwseg *seg; + void *p; + + CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC); + AN(pp); + p = *pp; + AN(p); + *pp = NULL; + VTAILQ_FOREACH(seg, &vsmw->segs, list) + if (seg->ptr == p) + break; + AN(seg); + vsmw_delseg(vsmw, seg, 1); +} + +/*--------------------------------------------------------------------*/ + +struct vsmw * +VSMW_New(int vdirfd, int mode, const char *idxname) +{ + struct vsmw *vsmw; + int fd; + + assert(vdirfd > 0); + assert(mode > 0); + AN(idxname); + ALLOC_OBJ(vsmw, VSMW_MAGIC); + AN(vsmw); + VTAILQ_INIT(&vsmw->segs); + vsmw->vsb = VSB_new_auto(); + AN(vsmw->vsb); + REPLACE(vsmw->idx, idxname); + vsmw->mode = mode; + vsmw->vdirfd = vdirfd; + + (void)unlinkat(vdirfd, vsmw->idx, 0); + fd = openat(vdirfd, + vsmw->idx, O_APPEND | O_WRONLY | O_CREAT, vsmw->mode); + assert(fd >= 0); + AZ(close(fd)); + + return (vsmw); +} + +void +VSMW_Destroy(struct vsmw **pp) +{ + struct vsmw *vsmw; + struct vsmwseg *seg, *s2; + + AN(pp); + AN(*pp); + vsmw = *pp; + *pp = NULL; + CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC); + AZ(unlinkat(vsmw->vdirfd, vsmw->idx, 0)); + REPLACE(vsmw->idx, NULL); + VTAILQ_FOREACH_SAFE(seg, &vsmw->segs, list, s2) + vsmw_delseg(vsmw, seg, 0); + VSB_destroy(&vsmw->vsb); + AZ(close(vsmw->vdirfd)); + FREE_OBJ(vsmw); +} From phk at FreeBSD.org Sun May 28 13:54:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 28 May 2017 15:54:06 +0200 Subject: [master] d87b58e Include stdint Message-ID: commit d87b58e260836723da597700e33b75b7ff55871d Author: Poul-Henning Kamp Date: Sun May 28 13:53:10 2017 +0000 Include stdint diff --git a/lib/libvarnish/vsmw.c b/lib/libvarnish/vsmw.c index 0f6cb08..d57557e 100644 --- a/lib/libvarnish/vsmw.c +++ b/lib/libvarnish/vsmw.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include From martin at varnish-software.com Mon May 29 11:09:05 2017 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 29 May 2017 13:09:05 +0200 Subject: [master] 7163c34 Corrected grammar for install.rst Message-ID: commit 7163c3469e39879f82ebadec1d1caa826e4b20cc Author: Akshay Ranganath Date: Fri May 26 13:33:37 2017 -0700 Corrected grammar for install.rst In the section related to make test, the sentences were a bit hard incorrect. I've corrected it. diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst index 640cd3a..0596db5 100644 --- a/doc/sphinx/installation/install.rst +++ b/doc/sphinx/installation/install.rst @@ -178,11 +178,11 @@ tea while it runs, it usually takes a couple of minutes:: make check -Don't worry if one or two tests fail, some of the tests are a -bit too timing sensitive (Please tell us which so we can fix them.) but -if a lot of them fails, and in particular if the `b00000.vtc` test -fails, something is horribly wrong, and you will get nowhere without -figuring out what. +Don't worry if one or two tests fail. Some of the tests are a +bit too timing sensitive (Please tell us which so we can fix them). +However, if a lot of them fail, and in particular if the `b00000.vtc` test +fails, something is horribly wrong. You will get nowhere without +figuring this one out. Installing ---------- From dridi.boukelmoune at gmail.com Mon May 29 12:27:05 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 29 May 2017 14:27:05 +0200 Subject: [master] 5c2ce94 Typo Message-ID: commit 5c2ce94a6aa01cd3c09199b85e89c0d6720ccc87 Author: Dridi Boukelmoune Date: Mon May 29 14:26:35 2017 +0200 Typo diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index 08e248e..00a1714 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -287,7 +287,7 @@ class rst_vsc(directive): for p in self.param: if p in PARAMS: continue - sys.stderr.write("Unknown paramter ") + sys.stderr.write("Unknown parameter ") sys.stderr.write("'" + p + "'") sys.stderr.write(" on field '" + self.arg + "'\n") exit(2) From fgsch at lodoss.net Tue May 30 20:37:05 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 30 May 2017 22:37:05 +0200 Subject: [master] ab5c3ad Remove duplicated test Message-ID: commit ab5c3adf6de8d94893cee87723dcd4663ffc74ff Author: Federico G. Schwindt Date: Tue May 30 21:36:28 2017 +0100 Remove duplicated test c00037.vtc already tests hash_always_miss. diff --git a/bin/varnishtest/tests/c00042.vtc b/bin/varnishtest/tests/c00042.vtc deleted file mode 100644 index 390f574..0000000 --- a/bin/varnishtest/tests/c00042.vtc +++ /dev/null @@ -1,42 +0,0 @@ -varnishtest "using req.hash_always_miss to force fetch" - -server s1 { - rxreq - expect req.url == "/1" - txresp -bodylen 1 - rxreq - expect req.url == "/1" - txresp -bodylen 2 - -} -start - -varnish v1 -vcl+backend { - sub vcl_recv { - if (req.http.short) { - set req.hash_always_miss = true; - } - } -} -start - -client c1 { - txreq -url "/1" - rxresp - expect resp.status == 200 - expect resp.bodylen == 1 - - txreq -url "/1" - rxresp - expect resp.status == 200 - expect resp.bodylen == 1 - - txreq -url "/1" -hdr "short: yes" - rxresp - expect resp.status == 200 - expect resp.bodylen == 2 - - txreq -url "/1" - rxresp - expect resp.status == 200 - expect resp.bodylen == 2 - -} -run From fgsch at lodoss.net Tue May 30 21:00:07 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 30 May 2017 23:00:07 +0200 Subject: [master] e1c4d9a Undefined behaviour is undefined Message-ID: commit e1c4d9a6db62560360cc0c509f3244e43ca0047f Author: Federico G. Schwindt Date: Tue May 30 21:46:14 2017 +0100 Undefined behaviour is undefined Spotted by ubsan. diff --git a/bin/varnishd/hpack/vhp_table.c b/bin/varnishd/hpack/vhp_table.c index 3357844..3ac2996 100644 --- a/bin/varnishd/hpack/vhp_table.c +++ b/bin/varnishd/hpack/vhp_table.c @@ -373,10 +373,12 @@ VHT_SetProtoMax(struct vht_table *tbl, size_t protomax) if (buf == NULL) return (-1); - memcpy(buf, tbl->buf, tbl->size); - memcpy(ENTRIES(buf, bufsize, tbl->n), TBLENTRIES(tbl), - sizeof (struct vht_entry) * tbl->n); - free(tbl->buf); + if (tbl->buf != NULL) { + memcpy(buf, tbl->buf, tbl->size); + memcpy(ENTRIES(buf, bufsize, tbl->n), TBLENTRIES(tbl), + sizeof (struct vht_entry) * tbl->n); + free(tbl->buf); + } tbl->buf = buf; tbl->bufsize = bufsize; tbl->protomax = protomax; diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index c79d886..7b5f3e2 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -400,13 +400,17 @@ parse_data(struct stream *s, struct frame *f) vtc_log(hp->vl, 4, "padding: %3d", f->md.padded); } - if (!size) - vtc_log(hp->vl, 4, "s%u - no data", s->id); - if (s->id) s->ws -= size; + s->hp->ws -= size; + if (!size) { + AZ(data); + vtc_log(hp->vl, 4, "s%u - no data", s->id); + return; + } + if (s->body) { s->body = realloc(s->body, s->bodylen + size + 1L); } else { diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c index e4ad629..7c8011f 100644 --- a/lib/libvarnish/vev.c +++ b/lib/libvarnish/vev.c @@ -160,9 +160,10 @@ vev_get_sig(int sig) if (os == NULL) return (ENOMEM); - memcpy(os, vev_sigs, vev_nsig * sizeof *os); - - free(vev_sigs); + if (vev_sigs != NULL) { + memcpy(os, vev_sigs, vev_nsig * sizeof *os); + free(vev_sigs); + } vev_sigs = os; vev_nsig = sig + 1; From fgsch at lodoss.net Tue May 30 21:00:07 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 30 May 2017 23:00:07 +0200 Subject: [master] 92f372d Always initialise err Message-ID: commit 92f372d50c2f1683b3dbd06db5734a7ea145d173 Author: Federico G. Schwindt Date: Tue May 30 21:48:29 2017 +0100 Always initialise err If txrst is used without -err we want a valid error. Spotted by msan. diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 7b5f3e2..a24dd1e 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -1706,7 +1706,7 @@ cmd_txrst(CMD_ARGS) { struct stream *s; char *p; - uint32_t err; + uint32_t err = 0; struct frame f; (void)cmd; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC);