From nils.goroll at uplex.de Tue Nov 1 09:14:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 01 Nov 2016 10:14:05 +0100 Subject: [master] 5e5b4d5 wake up herder from sleeping after destroying, sleep for longer Message-ID: commit 5e5b4d5943ed8f5d3e8f18cdb1f678ab2ff6e098 Author: Nils Goroll Date: Thu Oct 27 07:04:00 2016 +0200 wake up herder from sleeping after destroying, sleep for longer Previously, after sending a thread to varnish heaven, the herder slept for wthread_destroy_delay unconditionally. Instead, we now wait on the cv so we get woken up in case we run dry during the delay. This change is relevant proportionally to the value of wthread_destroy_delay if the spread between thread_pool_min and thread_pool_max is big and varnish is exposed to sudden traffic peaks. IOW, it will probably be only relevant for high performance setups. Also, we now sleep for thread_pool_timeout unless a shorter delay is warranted. This will delay the effect of thread parameter changes for up to thread_pool_timeout seconds unless the pool runs dry, in which case they will become effective immediately. diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 4e8af44..6ba528e 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -435,6 +435,7 @@ pool_herder(void *priv) struct pool_task *pt; double t_idle; struct worker *wrk; + int delay; CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC); @@ -447,6 +448,8 @@ pool_herder(void *priv) pool_breed(pp); continue; } + + delay = cache_param->wthread_timeout; assert(pp->nthr >= cache_param->wthread_min); if (pp->nthr > cache_param->wthread_min) { @@ -472,8 +475,10 @@ pool_herder(void *priv) &wrk->task, list); wrk->task.func = pool_kiss_of_death; AZ(pthread_cond_signal(&wrk->cond)); - } else + } else { + delay = wrk->lastused - t_idle; wrk = NULL; + } } Lck_Unlock(&pp->mtx); @@ -483,15 +488,15 @@ pool_herder(void *priv) VSC_C_main->threads--; VSC_C_main->threads_destroyed++; Lck_Unlock(&pool_mtx); - VTIM_sleep(cache_param->wthread_destroy_delay); - continue; - } + delay = cache_param->wthread_destroy_delay; + } else if (delay < cache_param->wthread_destroy_delay) + delay = cache_param->wthread_destroy_delay; } Lck_Lock(&pp->mtx); if (!pp->dry) { (void)Lck_CondWait(&pp->herder_cond, &pp->mtx, - VTIM_real() + 5); + VTIM_real() + delay); } else { /* XXX: unsafe counters */ VSC_C_main->threads_limited++; diff --git a/bin/varnishtest/tests/r01490.vtc b/bin/varnishtest/tests/r01490.vtc index 82ffdb4..76b32be 100644 --- a/bin/varnishtest/tests/r01490.vtc +++ b/bin/varnishtest/tests/r01490.vtc @@ -9,6 +9,7 @@ varnish v1 \ -arg "-p thread_pool_min=2" \ -arg "-p thread_pool_max=3" \ -arg "-p thread_pools=1" \ + -arg "-p thread_pool_timeout=10" \ -vcl+backend {} varnish v1 -start @@ -21,16 +22,16 @@ logexpect l1 -v v1 -g raw { varnish v1 -cliok "param.set thread_pool_min 3" -# Herder thread sleeps 5 seconds. Have to wait longer than that. -delay 6 +# Have to wait longer than thread_pool_timeout +delay 11 varnish v1 -expect threads == 3 varnish v1 -cliok "param.set thread_pool_min 2" varnish v1 -cliok "param.set thread_pool_max 2" -# Herder thread sleeps 5 seconds. Have to wait longer than that. -delay 6 +# Have to wait longer than thread_pool_timeout +delay 11 varnish v1 -expect threads == 2 From nils.goroll at uplex.de Tue Nov 1 09:14:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 01 Nov 2016 10:14:05 +0100 Subject: [master] ee4ae94 fix a potential race Message-ID: commit ee4ae94cb53a481afdc0222b7a17feec9155df37 Author: Nils Goroll Date: Thu Oct 27 13:00:20 2016 +0200 fix a potential race The nthr => wthread_min assertion could trigger if wthread_min was changed after the first comparison. diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 6ba528e..c497451 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -435,24 +435,26 @@ pool_herder(void *priv) struct pool_task *pt; double t_idle; struct worker *wrk; - int delay; + int delay, wthread_min; CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC); THR_SetName("pool_herder"); while (1) { + wthread_min = cache_param->wthread_min; + /* Make more threads if needed and allowed */ - if (pp->nthr < cache_param->wthread_min || + if (pp->nthr < wthread_min || (pp->dry && pp->nthr < cache_param->wthread_max)) { pool_breed(pp); continue; } delay = cache_param->wthread_timeout; - assert(pp->nthr >= cache_param->wthread_min); + assert(pp->nthr >= wthread_min); - if (pp->nthr > cache_param->wthread_min) { + if (pp->nthr > wthread_min) { t_idle = VTIM_real() - cache_param->wthread_timeout; From nils.goroll at uplex.de Tue Nov 1 09:14:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 01 Nov 2016 10:14:05 +0100 Subject: [master] f88ca87 Apply queue limits only to requests Message-ID: commit f88ca87d8e6106ec2d7a3813d750a35814aa60bf Author: Nils Goroll Date: Thu Oct 27 07:52:46 2016 +0200 Apply queue limits only to requests We should queue bo tasks to avoid failing client requests unnecessarily. The detailed parameter description of the queue_limit was already correct, improve the short one. diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index c497451..e1cdde1 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -243,8 +243,11 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_how how) return (0); } - /* Acceptors are not subject to queue limits */ - if (how == TASK_QUEUE_VCA || + /* + * queue limits only apply to client threads - all other + * work is vital and needs do be done at the earliest + */ + if (how != TASK_QUEUE_REQ || pp->lqueue < cache_param->wthread_max + cache_param->wthread_queue_limit + pp->nthr) { pp->nqueued++; diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index 1e75953..8791a44 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -178,7 +178,7 @@ struct parspec WRK_parspec[] = { "10", "requests" }, { "thread_queue_limit", tweak_uint, &mgt_param.wthread_queue_limit, "0", NULL, - "Permitted queue length per thread-pool.\n" + "Permitted request queue length per thread-pool.\n" "\n" "This sets the number of requests we will queue, waiting " "for an available thread. Above this limit sessions will " diff --git a/include/tbl/params.h b/include/tbl/params.h index 71e2949..1e01384 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1215,7 +1215,7 @@ PARAM( /* units */ NULL, /* flags */ EXPERIMENTAL, /* s-text */ - "Permitted queue length per thread-pool.\n" + "Permitted request queue length per thread-pool.\n" "\n" "This sets the number of requests we will queue, waiting for an " "available thread. Above this limit sessions will be dropped " From nils.goroll at uplex.de Tue Nov 1 09:14:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 01 Nov 2016 10:14:05 +0100 Subject: [master] 2e2bfad clarify that the enum implies a priority Message-ID: commit 2e2bfad363ba5b2c8c6a5a03ff935060a5bd9499 Author: Nils Goroll Date: Thu Oct 27 08:56:56 2016 +0200 clarify that the enum implies a priority diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index ba28151..4b3604e 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -303,7 +303,8 @@ struct pool_task { void *priv; }; -enum task_how { +/* tasks are taken off the queues in this order */ +enum task_prio { TASK_QUEUE_BO, TASK_QUEUE_REQ, TASK_QUEUE_VCA, @@ -918,13 +919,13 @@ const char *body_status_2str(enum body_status e); const char *sess_close_2str(enum sess_close sc, int want_desc); /* cache_pool.c */ -int Pool_Task(struct pool *pp, struct pool_task *task, enum task_how how); +int Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio how); int Pool_Task_Arg(struct worker *, task_func_t *, const void *arg, size_t arg_len); void Pool_Sumstat(struct worker *w); int Pool_TrySumstat(struct worker *wrk); void Pool_PurgeStat(unsigned nobj); -int Pool_Task_Any(struct pool_task *task, enum task_how how); +int Pool_Task_Any(struct pool_task *task, enum task_prio how); /* cache_range.c [VRG] */ void VRG_dorange(struct req *req, const char *r); diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index f337714..9dfa2ed 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -91,7 +91,7 @@ Pool_TrySumstat(struct worker *wrk) */ int -Pool_Task_Any(struct pool_task *task, enum task_how how) +Pool_Task_Any(struct pool_task *task, enum task_prio how) { struct pool *pp; diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index e1cdde1..1db93a8 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -218,7 +218,7 @@ Pool_Task_Arg(struct worker *wrk, task_func_t *func, */ int -Pool_Task(struct pool *pp, struct pool_task *task, enum task_how how) +Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio how) { struct worker *wrk; int retval = 0; From nils.goroll at uplex.de Tue Nov 1 09:14:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 01 Nov 2016 10:14:05 +0100 Subject: [master] 42959df keep a reserve of idle theads for vital tasks Message-ID: commit 42959df8c23914e7124ad888e5de7d061bbc72e5 Author: Nils Goroll Date: Thu Oct 27 09:33:02 2016 +0200 keep a reserve of idle theads for vital tasks (backend tasks for the time being) Add parameter thread_pool_reserve to tweak diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 4b3604e..0d3543a 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -303,9 +303,14 @@ struct pool_task { void *priv; }; -/* tasks are taken off the queues in this order */ +/* + * tasks are taken off the queues in this order + * + * prios up to TASK_QUEUE_RESERVE are run from the reserve + */ enum task_prio { TASK_QUEUE_BO, +#define TASK_QUEUE_RESERVE TASK_QUEUE_BO TASK_QUEUE_REQ, TASK_QUEUE_VCA, TASK_QUEUE_END @@ -920,7 +925,7 @@ const char *sess_close_2str(enum sess_close sc, int want_desc); /* cache_pool.c */ int Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio how); -int Pool_Task_Arg(struct worker *, task_func_t *, +int Pool_Task_Arg(struct worker *, enum task_prio, task_func_t *, const void *arg, size_t arg_len); void Pool_Sumstat(struct worker *w); int Pool_TrySumstat(struct worker *wrk); diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 4b58fd7..9d69daa 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -447,7 +447,8 @@ vca_accept_task(struct worker *wrk, void *arg) wa.acceptsock = i; - if (!Pool_Task_Arg(wrk, vca_make_session, &wa, sizeof wa)) { + if (!Pool_Task_Arg(wrk, TASK_QUEUE_VCA, + vca_make_session, &wa, sizeof wa)) { /* * We couldn't get another thread, so we will handle * the request in this worker thread, but first we diff --git a/bin/varnishd/cache/cache_pool.h b/bin/varnishd/cache/cache_pool.h index c400a31..fa3663a 100644 --- a/bin/varnishd/cache/cache_pool.h +++ b/bin/varnishd/cache/cache_pool.h @@ -40,6 +40,7 @@ struct pool { pthread_t herder_thr; struct lock mtx; + unsigned nidle; struct taskhead idle_queue; struct taskhead queues[TASK_QUEUE_END]; unsigned nthr; diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 1db93a8..1fca8da 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -148,17 +148,35 @@ pool_addstat(struct dstat *dst, struct dstat *src) memset(src, 0, sizeof *src); } +static inline int +pool_reserve(void) +{ + unsigned lim; + + if (cache_param->wthread_reserve == 0) + return (cache_param->wthread_min / 20 + 1); + lim = cache_param->wthread_min * 950 / 1000; + if (cache_param->wthread_reserve > lim) + return (lim); + return (cache_param->wthread_reserve); +} + /*--------------------------------------------------------------------*/ static struct worker * -pool_getidleworker(struct pool *pp) +pool_getidleworker(struct pool *pp, enum task_prio how) { - struct pool_task *pt; + struct pool_task *pt = NULL; struct worker *wrk; CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); Lck_AssertHeld(&pp->mtx); - pt = VTAILQ_FIRST(&pp->idle_queue); + if (how <= TASK_QUEUE_RESERVE || pp->nidle > pool_reserve()) { + pt = VTAILQ_FIRST(&pp->idle_queue); + if (pt == NULL) + AZ(pp->nidle); + } + if (pt == NULL) { if (pp->nthr < cache_param->wthread_max) { pp->dry++; @@ -179,7 +197,7 @@ pool_getidleworker(struct pool *pp) */ int -Pool_Task_Arg(struct worker *wrk, task_func_t *func, +Pool_Task_Arg(struct worker *wrk, enum task_prio how, task_func_t *func, const void *arg, size_t arg_len) { struct pool *pp; @@ -193,9 +211,11 @@ Pool_Task_Arg(struct worker *wrk, task_func_t *func, CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); Lck_Lock(&pp->mtx); - wrk2 = pool_getidleworker(pp); + wrk2 = pool_getidleworker(pp, how); if (wrk2 != NULL) { + AN(pp->nidle); VTAILQ_REMOVE(&pp->idle_queue, &wrk2->task, list); + pp->nidle--; retval = 1; } else { wrk2 = wrk; @@ -222,7 +242,6 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio how) { struct worker *wrk; int retval = 0; - CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); AN(task); AN(task->func); @@ -232,9 +251,11 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio how) /* The common case first: Take an idle thread, do it. */ - wrk = pool_getidleworker(pp); + wrk = pool_getidleworker(pp, how); if (wrk != NULL) { + AN(pp->nidle); VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list); + pp->nidle--; AZ(wrk->task.func); wrk->task.func = task->func; wrk->task.priv = task->priv; @@ -282,7 +303,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk) { struct pool_task *tp; struct pool_task tpx, tps; - int i; + int i, prio_lim; CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); wrk->pool = pp; @@ -294,7 +315,12 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk) WS_Reset(wrk->aws, NULL); AZ(wrk->vsl); - for (i = 0; i < TASK_QUEUE_END; i++) { + if (pp->nidle < pool_reserve()) + prio_lim = TASK_QUEUE_RESERVE + 1; + else + prio_lim = TASK_QUEUE_END; + + for (i = 0; i < prio_lim; i++) { tp = VTAILQ_FIRST(&pp->queues[i]); if (tp != NULL) { pp->lqueue--; @@ -323,6 +349,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk) wrk->task.func = NULL; wrk->task.priv = wrk; VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list); + pp->nidle++; do { i = Lck_CondWait(&wrk->cond, &pp->mtx, wrk->vcl == NULL ? 0 : wrk->lastused+60.); @@ -470,6 +497,7 @@ pool_herder(void *priv) wrk = NULL; pt = VTAILQ_LAST(&pp->idle_queue, taskhead); if (pt != NULL) { + AN(pp->nidle); AZ(pt->func); CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC); @@ -478,6 +506,7 @@ pool_herder(void *priv) /* Give it a kiss on the cheek... */ VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list); + pp->nidle--; wrk->task.func = pool_kiss_of_death; AZ(pthread_cond_signal(&wrk->cond)); } else { diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h index 678ed26..9eabc2c 100644 --- a/bin/varnishd/common/params.h +++ b/bin/varnishd/common/params.h @@ -91,6 +91,7 @@ struct params { /* Worker threads and pool */ unsigned wthread_min; unsigned wthread_max; + unsigned wthread_reserve; double wthread_timeout; unsigned wthread_pools; double wthread_add_delay; diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index 8791a44..03e98fd 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -62,6 +62,8 @@ tweak_thread_pool_min(struct vsb *vsb, const struct parspec *par, return (-1); MCF_ParamConf(MCF_MINIMUM, "thread_pool_max", "%u", mgt_param.wthread_min); + MCF_ParamConf(MCF_MAXIMUM, "thread_pool_reserve", + "%u", mgt_param.wthread_min * 950 / 1000); return (0); } @@ -116,6 +118,25 @@ struct parspec WRK_parspec[] = { "Minimum is 10 threads.", DELAYED_EFFECT, "100", "threads" }, + { "thread_pool_reserve", tweak_uint, &mgt_param.wthread_reserve, + 0, NULL, + "The number of worker threads reserved for vital tasks " + "in each pool.\n" + "\n" + "Tasks may require other tasks to complete (for example, " + "client requests may require backend requests). This reserve " + "is to ensure that such tasks still get to run even under high " + "load.\n" + "\n" + "Increasing the reserve may help setups with a high number of " + "backend requests at the expense of client performance. " + "Setting it too high will waste resources by keeping threads " + "unused.\n" + "\n" + "Default is 0 to auto-tune (currently 5% of thread_pool_min).\n" + "Minimum is 1 otherwise, maximum is 95% of thread_pool_min.", + DELAYED_EFFECT, + "0", "threads" }, { "thread_pool_timeout", tweak_timeout, &mgt_param.wthread_timeout, "10", NULL, diff --git a/bin/varnishtest/tests/c00079.vtc b/bin/varnishtest/tests/c00079.vtc new file mode 100644 index 0000000..a0d4c2e --- /dev/null +++ b/bin/varnishtest/tests/c00079.vtc @@ -0,0 +1,23 @@ +varnishtest "thread_pool_reserve max adjustment" + +server s1 { +} -start + +varnish v1 \ + -arg "-p thread_pool_min=10" \ + -arg "-p thread_pool_max=100" \ + -arg "-p thread_pools=1" \ + -arg "-p thread_pool_timeout=10" \ + -vcl+backend {} +varnish v1 -start + +varnish v1 -cliok "param.set thread_pool_reserve 0" +varnish v1 -cliok "param.set thread_pool_reserve 1" +varnish v1 -cliok "param.set thread_pool_reserve 9" +varnish v1 -clierr 106 "param.set thread_pool_reserve 10" + +varnish v1 -cliok "param.set thread_pool_min 100" +varnish v1 -cliok "param.set thread_pool_reserve 0" +varnish v1 -cliok "param.set thread_pool_reserve 1" +varnish v1 -cliok "param.set thread_pool_reserve 95" +varnish v1 -clierr 106 "param.set thread_pool_reserve 96" diff --git a/include/tbl/params.h b/include/tbl/params.h index 1e01384..0250dc2 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1142,6 +1142,34 @@ PARAM( /* l-text */ "", /* func */ NULL ) +/* actual location mgt_pool.c */ +PARAM( + /* name */ thread_pool_reserve, + /* typ */ thread_pool_reserve, + /* min */ NULL, + /* max */ NULL, + /* default */ "0", + /* units */ "threads", + /* flags */ DELAYED_EFFECT| EXPERIMENTAL, + /* s-text */ + "The number of worker threads reserved for vital tasks " + "in each pool.\n" + "\n" + "Tasks may require other tasks to complete (for example, " + "client requests may require backend requests). This reserve " + "is to ensure that such tasks still get to run even under high " + "load.\n" + "\n" + "Increasing the reserve may help setups with a high number of " + "backend requests at the expense of client performance. " + "Setting it too high will waste resources by keeping threads " + "unused.\n" + "\n" + "Default is 0 to auto-tune (currently 5% of thread_pool_min).\n" + "Minimum is 1 otherwise, maximum is 95% of thread_pool_min.", + /* l-text */ "", + /* func */ NULL +) /* actual location mgt_pool.c */ PARAM( From nils.goroll at uplex.de Tue Nov 1 09:14:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 01 Nov 2016 10:14:05 +0100 Subject: [master] 549fdfc re-sync comment Message-ID: commit 549fdfc565bf6d4220b878f346412b48fa3df177 Author: Nils Goroll Date: Thu Oct 27 19:55:11 2016 +0200 re-sync comment diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 1fca8da..33a556f 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -446,7 +446,8 @@ pool_breed(struct pool *qp) /*-------------------------------------------------------------------- * Herd a single pool * - * This thread wakes every 5 seconds and whenever a pool queues. + * This thread wakes up every thread_pool_timeout seconds, whenever a pool + * queues and when threads need to be destroyed * * The trick here is to not be too aggressive about creating threads. In * pool_breed(), we sleep whenever we create a thread and a little while longer From phk at FreeBSD.org Tue Nov 1 11:46:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 01 Nov 2016 12:46:04 +0100 Subject: [master] 9509373 Move the TCP parameters into the tbl #include Message-ID: commit 95093730385d355825428ea46fb30ad30111749e Author: Poul-Henning Kamp Date: Tue Nov 1 10:25:23 2016 +0000 Move the TCP parameters into the tbl #include diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h index 9eabc2c..aadb0ea 100644 --- a/bin/varnishd/common/params.h +++ b/bin/varnishd/common/params.h @@ -101,12 +101,6 @@ struct params { ssize_t wthread_stacksize; unsigned wthread_queue_limit; -#ifdef HAVE_TCP_KEEP - double tcp_keepalive_time; - unsigned tcp_keepalive_probes; - double tcp_keepalive_intvl; -#endif - struct vre_limits vre_limits; struct poolparam req_pool; diff --git a/bin/varnishd/mgt/mgt_param_tcp.c b/bin/varnishd/mgt/mgt_param_tcp.c index 6d40097..342a527 100644 --- a/bin/varnishd/mgt/mgt_param_tcp.c +++ b/bin/varnishd/mgt/mgt_param_tcp.c @@ -51,28 +51,6 @@ #ifdef HAVE_TCP_KEEP -static struct parspec mgt_parspec_tcp_keep[] = { - { "tcp_keepalive_time", tweak_timeout, &mgt_param.tcp_keepalive_time, - "1", "7200", - "The number of seconds a connection needs to be idle before " - "TCP begins sending out keep-alive probes.", - EXPERIMENTAL, - "", "seconds" }, - { "tcp_keepalive_probes", tweak_uint, &mgt_param.tcp_keepalive_probes, - "1", "100", - "The maximum number of TCP keep-alive probes to send before " - "giving up and killing the connection if no response is " - "obtained from the other end.", - EXPERIMENTAL, - "", "probes" }, - { "tcp_keepalive_intvl", tweak_timeout, &mgt_param.tcp_keepalive_intvl, - "1", "100", - "The number of seconds between TCP keep-alive probes.", - EXPERIMENTAL, - "", "seconds" }, - { NULL, NULL, NULL } -}; - static void tcp_probe(int sock, int nam, const char *param, unsigned def) { @@ -108,7 +86,6 @@ void MCF_TcpParams(void) { #ifdef HAVE_TCP_KEEP - MCF_AddParams(mgt_parspec_tcp_keep); tcp_keep_probes(); #endif } diff --git a/include/tbl/params.h b/include/tbl/params.h index 0250dc2..7bbf54a 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -992,14 +992,13 @@ PARAM( /* func */ NULL ) -#if 0 -/* actual location mgt_param_tcp.c */ +#if defined(HAVE_TCP_KEEP) PARAM( /* name */ tcp_keepalive_intvl, /* typ */ timeout, - /* min */ "1.000", - /* max */ "100.000", - /* default */ "5.000", + /* min */ "1", + /* max */ "100", + /* default */ "", /* units */ "seconds", /* flags */ EXPERIMENTAL, /* s-text */ @@ -1008,13 +1007,12 @@ PARAM( /* func */ NULL ) -/* actual location mgt_param_tcp.c */ PARAM( /* name */ tcp_keepalive_probes, /* typ */ uint, /* min */ "1", /* max */ "100", - /* default */ "5", + /* default */ "", /* units */ "probes", /* flags */ EXPERIMENTAL, /* s-text */ @@ -1025,13 +1023,12 @@ PARAM( /* func */ NULL ) -/* actual location mgt_param_tcp.c */ PARAM( /* name */ tcp_keepalive_time, /* typ */ timeout, - /* min */ "1.000", - /* max */ "7200.000", - /* default */ "600.000", + /* min */ "1", + /* max */ "7200", + /* default */ "", /* units */ "seconds", /* flags */ EXPERIMENTAL, /* s-text */ @@ -1041,6 +1038,9 @@ PARAM( /* func */ NULL ) +#endif /* HAVE_TCP_KEEP */ + +#if 0 /* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_add_delay, From phk at FreeBSD.org Tue Nov 1 11:46:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 01 Nov 2016 12:46:04 +0100 Subject: [master] 5eb7124 Add some asserts for Flexelint and general paranoia Message-ID: commit 5eb7124955b048a55372e7f335cc00ce03fb7ec2 Author: Poul-Henning Kamp Date: Tue Nov 1 10:35:37 2016 +0000 Add some asserts for Flexelint and general paranoia diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index af9e46b..730f2d6 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -531,6 +531,10 @@ smp_allocobj(struct worker *wrk, const struct stevedore *stv, ltot = sizeof(struct object) + PRNDUP(wsl); ltot = IRNUP(sc, ltot); + st = NULL; + sg = NULL; + so = NULL; + objidx = 0; for (; nuke_limit >= 0; nuke_limit--) { st = smp_allocx(stv, ltot, ltot, &so, &objidx, &sg); if (st != NULL && st->space < ltot) { @@ -544,6 +548,8 @@ smp_allocobj(struct worker *wrk, const struct stevedore *stv, } AN(st); + AN(sg); + AN(so); assert(st->space >= ltot); o = SML_MkObject(stv, oc, st->ptr); From phk at FreeBSD.org Tue Nov 1 11:46:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 01 Nov 2016 12:46:04 +0100 Subject: [master] b2143c8 Unused include Message-ID: commit b2143c8d9420d4659524dec344028bc8245f856a Author: Poul-Henning Kamp Date: Tue Nov 1 10:36:02 2016 +0000 Unused include diff --git a/bin/varnishd/mgt/mgt_param_tcp.c b/bin/varnishd/mgt/mgt_param_tcp.c index 342a527..27dd249 100644 --- a/bin/varnishd/mgt/mgt_param_tcp.c +++ b/bin/varnishd/mgt/mgt_param_tcp.c @@ -47,8 +47,6 @@ #include "vtcp.h" -#include "mgt/mgt_param.h" - #ifdef HAVE_TCP_KEEP static void From phk at FreeBSD.org Tue Nov 1 11:46:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 01 Nov 2016 12:46:04 +0100 Subject: [master] a83b79b Make sure we have no random pointer values. Message-ID: commit a83b79b92c43aa954915877a93d8729ac55aaf51 Author: Poul-Henning Kamp Date: Tue Nov 1 10:36:14 2016 +0000 Make sure we have no random pointer values. diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 33a556f..6c66d67 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -301,7 +301,7 @@ pool_kiss_of_death(struct worker *wrk, void *priv) static void Pool_Work_Thread(struct pool *pp, struct worker *wrk) { - struct pool_task *tp; + struct pool_task *tp = NULL; struct pool_task tpx, tps; int i, prio_lim; From phk at FreeBSD.org Tue Nov 1 12:23:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 01 Nov 2016 13:23:05 +0100 Subject: [master] 722f654 Add a STV_Foreach() iterator, to isolate the clue to Transients special treatment. Message-ID: commit 722f654feede371ab9c98dd240d458b8e6954844 Author: Poul-Henning Kamp Date: Tue Nov 1 12:22:28 2016 +0000 Add a STV_Foreach() iterator, to isolate the clue to Transients special treatment. diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index f8d6224..2cbaaeb 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -104,10 +104,9 @@ run_vcc(void *priv) VCC_Err_Unref(vcc, mgt_vcc_err_unref); VCC_Allow_InlineC(vcc, mgt_vcc_allow_inline_c); VCC_Unsafe_Path(vcc, mgt_vcc_unsafe_path); - VTAILQ_FOREACH(stv, &stv_stevedores, list) + STV_Foreach(stv) VCC_Predef(vcc, "VCL_STEVEDORE", stv->ident); mgt_vcl_export_labels(vcc); - VCC_Predef(vcc, "VCL_STEVEDORE", stv_transient->ident); csrc = VCC_Compile(vcc, &sb, vp->vclsrc, vp->vclsrcfile); AZ(VSB_finish(sb)); if (VSB_len(sb)) diff --git a/bin/varnishd/storage/mgt_stevedore.c b/bin/varnishd/storage/mgt_stevedore.c index dff69fd..89399f1 100644 --- a/bin/varnishd/storage/mgt_stevedore.c +++ b/bin/varnishd/storage/mgt_stevedore.c @@ -51,6 +51,25 @@ struct stevedore *stv_transient; /*--------------------------------------------------------------------*/ +int +STV__iter(struct stevedore **pp) +{ + + if (*pp == stv_transient) { + *pp = NULL; + return (0); + } + if (*pp != NULL) + *pp = VTAILQ_NEXT(*pp, list); + else + *pp = VTAILQ_FIRST(&stv_stevedores); + if (*pp == NULL) + *pp = stv_transient; + return (1); +} + +/*--------------------------------------------------------------------*/ + static void __match_proto__(cli_func_t) stv_cli_list(struct cli *cli, const char * const *av, void *priv) { diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 072ba26..300faca 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -99,19 +99,13 @@ STV_open(void) char buf[1024]; ASSERT_CLI(); - VTAILQ_FOREACH(stv, &stv_stevedores, list) { + STV_Foreach(stv) { bprintf(buf, "storage.%s", stv->ident); stv->vclname = strdup(buf); AN(stv->vclname); if (stv->open != NULL) stv->open(stv); } - stv = stv_transient; - bprintf(buf, "storage.%s", stv->ident); - stv->vclname = strdup(buf); - AN(stv->vclname); - if (stv->open != NULL) - stv->open(stv); stv_next = VTAILQ_FIRST(&stv_stevedores); } @@ -124,11 +118,7 @@ STV_close(void) ASSERT_CLI(); for (i = 1; i >= 0; i--) { /* First send close warning */ - VTAILQ_FOREACH(stv, &stv_stevedores, list) - if (stv->close != NULL) - stv->close(stv, i); - stv = stv_transient; - if (stv->close != NULL) + STV_Foreach(stv) if (stv->close != NULL) stv->close(stv, i); } diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index db8856b..c87c5e9 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -131,6 +131,10 @@ extern struct stevedore_head stv_stevedores; extern struct stevedore *stv_transient; /*--------------------------------------------------------------------*/ +#define STV_Foreach(arg) for(arg = NULL; STV__iter(&arg);) +int STV__iter(struct stevedore **); + +/*--------------------------------------------------------------------*/ int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx); uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity, const char *ctx); From info at varnish-cache.org Tue Nov 1 13:06:09 2016 From: info at varnish-cache.org (varnish-cache.org) Date: Tue, 01 Nov 2016 11:06:09 -0200 Subject: Invoice No. 49654597 for VARNISH-COMMIT Message-ID: <88DB940C0C8BCD48390A2CDAD1A73037@varnish-cache.org> REF: 10652342 INVOICE NUMBER: 49654597 FROM: Vincent & Gorbing (175-187 Linthorpe Road,Buxton, Derbyshire, SK17 9QF, UNITED KINGDOM) DATE: 01/11/2016 --------------------------------- This is an automated message generated by the Banana ERP?Accounting. Any questions? Reply to this e-mail address. Do not hesitate to contact us. -------------- next part -------------- An HTML attachment was scrubbed... URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: INV_NO_49654597.zip Type: application/zip Size: 8230 bytes Desc: not available URL: From phk at FreeBSD.org Tue Nov 1 19:45:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 01 Nov 2016 20:45:05 +0100 Subject: [master] 561adad Also use STV_Foreach() in the worker process Message-ID: commit 561adad261715c7326e62036173dcef729dd7e5e Author: Poul-Henning Kamp Date: Tue Nov 1 12:27:07 2016 +0000 Also use STV_Foreach() in the worker process diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 300faca..c32105a 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -136,7 +136,7 @@ STV_BanInfoDrop(const uint8_t *ban, unsigned len) struct stevedore *stv; int r = 0; - VTAILQ_FOREACH(stv, &stv_stevedores, list) + STV_Foreach(stv) if (stv->baninfo != NULL) r |= stv->baninfo(stv, BI_DROP, ban, len); @@ -149,7 +149,7 @@ STV_BanInfoNew(const uint8_t *ban, unsigned len) struct stevedore *stv; int r = 0; - VTAILQ_FOREACH(stv, &stv_stevedores, list) + STV_Foreach(stv) if (stv->baninfo != NULL) r |= stv->baninfo(stv, BI_NEW, ban, len); @@ -167,7 +167,7 @@ STV_BanExport(const uint8_t *bans, unsigned len) { struct stevedore *stv; - VTAILQ_FOREACH(stv, &stv_stevedores, list) + STV_Foreach(stv) if (stv->banexport != NULL) stv->banexport(stv, bans, len); } @@ -179,13 +179,11 @@ STV_BanExport(const uint8_t *bans, unsigned len) const struct stevedore * STV_find(const char *nm) { - const struct stevedore *stv; + struct stevedore *stv; - VTAILQ_FOREACH(stv, &stv_stevedores, list) + STV_Foreach(stv) if (!strcmp(stv->ident, nm)) return (stv); - if (!strcmp(TRANSIENT_STORAGE, nm)) - return (stv_transient); return (NULL); } From phk at FreeBSD.org Tue Nov 1 19:45:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 01 Nov 2016 20:45:05 +0100 Subject: [master] cc663d7 Don't spit Compiled VCL on stderr Message-ID: commit cc663d7ce9241aff09c0d90cafee69a783658fb5 Author: Poul-Henning Kamp Date: Tue Nov 1 18:39:58 2016 +0000 Don't spit Compiled VCL on stderr diff --git a/bin/varnishtest/tests/a00009.vtc b/bin/varnishtest/tests/a00009.vtc index a1bf19e..3ae0c2d 100644 --- a/bin/varnishtest/tests/a00009.vtc +++ b/bin/varnishtest/tests/a00009.vtc @@ -1,6 +1,6 @@ varnishtest "Code coverage of VCL compiler and RSTdump etc" -shell "varnishd -b 127.0.0.1:80 -C -n ${tmpdir} > /${tmpdir}/_.c" +shell "varnishd -b 127.0.0.1:80 -C -n ${tmpdir} 2> /${tmpdir}/_.c" shell "varnishd -x dumprstparam > /${tmpdir}/_.param" shell "varnishd -x dumprstvsl > /${tmpdir}/_.vsl" shell "varnishd -x dumprstcli > /${tmpdir}/_.cli" From phk at FreeBSD.org Tue Nov 1 19:45:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 01 Nov 2016 20:45:05 +0100 Subject: [master] 0020803 More polishing of the stevedore code, to try to find out what FlexeLint is getting its knickers in a twist about. Message-ID: commit 0020803ef0685089d3371c33ace9423d28a290f1 Author: Poul-Henning Kamp Date: Tue Nov 1 19:43:58 2016 +0000 More polishing of the stevedore code, to try to find out what FlexeLint is getting its knickers in a twist about. diff --git a/bin/varnishd/storage/mgt_stevedore.c b/bin/varnishd/storage/mgt_stevedore.c index 89399f1..b199ce6 100644 --- a/bin/varnishd/storage/mgt_stevedore.c +++ b/bin/varnishd/storage/mgt_stevedore.c @@ -44,17 +44,19 @@ #include "storage/storage.h" #include "vav.h" -struct stevedore_head stv_stevedores = - VTAILQ_HEAD_INITIALIZER(stv_stevedores); +static VTAILQ_HEAD(, stevedore) stevedores = + VTAILQ_HEAD_INITIALIZER(stevedores); struct stevedore *stv_transient; /*--------------------------------------------------------------------*/ int -STV__iter(struct stevedore **pp) +STV__iter(struct stevedore ** const pp) { + AN(pp); + CHECK_OBJ_ORNULL(*pp, STEVEDORE_MAGIC); if (*pp == stv_transient) { *pp = NULL; return (0); @@ -62,7 +64,7 @@ STV__iter(struct stevedore **pp) if (*pp != NULL) *pp = VTAILQ_NEXT(*pp, list); else - *pp = VTAILQ_FIRST(&stv_stevedores); + *pp = VTAILQ_FIRST(&stevedores); if (*pp == NULL) *pp = stv_transient; return (1); @@ -79,9 +81,7 @@ stv_cli_list(struct cli *cli, const char * const *av, void *priv) (void)av; (void)priv; VCLI_Out(cli, "Storage devices:\n"); - stv = stv_transient; - VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name); - VTAILQ_FOREACH(stv, &stv_stevedores, list) + STV_Foreach(stv) VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name); } @@ -138,6 +138,7 @@ STV_Config(const char *spec) const char *p, *q; struct stevedore *stv; const struct stevedore *stv2; + struct stevedore *stv3; int ac, l; static unsigned seq = 0; @@ -185,12 +186,10 @@ STV_Config(const char *spec) bprintf(stv->ident, "%.*s", l, spec); } - VTAILQ_FOREACH(stv2, &stv_stevedores, list) { - if (strcmp(stv2->ident, stv->ident)) - continue; - ARGV_ERR("(-s%s=%s) already defined once\n", - stv->ident, stv->name); - } + STV_Foreach(stv3) + if (!strcmp(stv3->ident, stv->ident)) + ARGV_ERR("(-s%s=%s) already defined once\n", + stv->ident, stv->name); if (stv->init != NULL) stv->init(stv, ac, av); @@ -204,7 +203,7 @@ STV_Config(const char *spec) AZ(stv_transient); stv_transient = stv; } else { - VTAILQ_INSERT_TAIL(&stv_stevedores, stv, list); + VTAILQ_INSERT_TAIL(&stevedores, stv, list); } /* NB: Do not free av, stevedore gets to keep it */ } diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index c32105a..ef90263 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -43,7 +43,6 @@ #include "vrt.h" #include "vrt_obj.h" -static const struct stevedore * volatile stv_next; /*-------------------------------------------------------------------- * XXX: trust pointer writes to be atomic @@ -52,16 +51,15 @@ static const struct stevedore * volatile stv_next; const struct stevedore * STV_next() { - struct stevedore *stv; - if (stv_next == NULL) - return (stv_transient); - /* pick a stevedore and bump the head along */ - stv = VTAILQ_NEXT(stv_next, list); - if (stv == NULL) - stv = VTAILQ_FIRST(&stv_stevedores); + static struct stevedore *stv; + + if (!STV__iter(&stv)) + AN(STV__iter(&stv)); + if (stv == stv_transient) { + stv = NULL; + AN(STV__iter(&stv)); + } AN(stv); - AN(stv->name); - stv_next = stv; return (stv); } @@ -106,7 +104,6 @@ STV_open(void) if (stv->open != NULL) stv->open(stv); } - stv_next = VTAILQ_FIRST(&stv_stevedores); } void diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index c87c5e9..849e442 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -125,14 +125,13 @@ struct stevedore { char *vclname; }; -VTAILQ_HEAD(stevedore_head, stevedore); - -extern struct stevedore_head stv_stevedores; extern struct stevedore *stv_transient; /*--------------------------------------------------------------------*/ + #define STV_Foreach(arg) for(arg = NULL; STV__iter(&arg);) -int STV__iter(struct stevedore **); + +int STV__iter(struct stevedore ** const ); /*--------------------------------------------------------------------*/ int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx); diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c index bf0834c..957c8f4 100644 --- a/bin/varnishd/storage/storage_simple.c +++ b/bin/varnishd/storage/storage_simple.c @@ -134,7 +134,7 @@ SML_allocobj(struct worker *wrk, const struct stevedore *stv, struct objcore *oc, unsigned wsl, int nuke_limit) { struct object *o; - struct storage *st; + struct storage *st = NULL; unsigned ltot; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); diff --git a/bin/varnishtest/tests/c00044.vtc b/bin/varnishtest/tests/c00044.vtc index 46bd2b3..2c0cf66 100644 --- a/bin/varnishtest/tests/c00044.vtc +++ b/bin/varnishtest/tests/c00044.vtc @@ -18,9 +18,9 @@ server s1 { } -start varnish v1 \ - -arg "-smalloc,1m" \ - -arg "-smalloc,1m" \ - -arg "-smalloc,1m" \ + -arg "-ss1=malloc,1m" \ + -arg "-ss2=malloc,1m" \ + -arg "-ss0=malloc,1m" \ -vcl+backend { sub vcl_backend_response { set beresp.do_stream = false; diff --git a/bin/varnishtest/tests/c00078.vtc b/bin/varnishtest/tests/c00078.vtc index e7047ff..3a33888 100644 --- a/bin/varnishtest/tests/c00078.vtc +++ b/bin/varnishtest/tests/c00078.vtc @@ -5,8 +5,11 @@ server s1 -repeat 6 { txresp } -start -varnish v1 -arg "-smalloc,1m" -arg "-smalloc,1m" \ - -arg "-smalloc,1m" -vcl+backend { +varnish v1 \ + -arg "-ss1=malloc,1m" \ + -arg "-ss2=malloc,1m" \ + -arg "-ss0=malloc,1m" \ + -vcl+backend { import debug; sub vcl_backend_response { if (bereq.url == "/1") { From varnish-commit at varnish-cache.org Wed Nov 2 02:25:13 2016 From: varnish-commit at varnish-cache.org (rcnui) Date: Wed, 2 Nov 2016 10:25:13 +0800 Subject: =?utf-8?B?cmNudWktLeS4uuS7gOS5iOS7k+W6k+eahOWIqeeUqOeOh+S9jj8=?= Message-ID: <20161102102521020077@exhv.com> varnish-commit:??? ??????????????????? ????????????? ??????????? ????????? ?????????????? ????????? ????????? ????????? ???????? ??????? ????????????????????????????????????????????????????????????????????????????????????????????????? ?????????????? -------------- next part -------------- An HTML attachment was scrubbed... URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: ?????????????.docx Type: application/vnd.openxmlformats-officedocument.wordprocessingml.document Size: 23568 bytes Desc: not available URL: From varnish-commit at varnish-cache.org Wed Nov 2 16:12:40 2016 From: varnish-commit at varnish-cache.org (puiwnwjwl) Date: Thu, 3 Nov 2016 00:12:40 +0800 Subject: =?utf-8?B?dmFybmlzaC1jb21taXQ65aaC5L2V5aSE55CG5ZGY?= =?utf-8?B?5bel6L+d57qq6Zeu6aKYPyBzcmE=?= Message-ID: <20161103001245740232@qubfwiomz.net> varnish-commit: ?? 1.???????????????????????????? 2.?????????????????? 3.???????????????????????? 4.?????????????????????????? 5.?????????????? 6.?????????????????????????? 7.?????????????????????????? 8.??????????????????? 9.??????????????????????? ??????????????????? ???????????????????????????????????????????????? 2016-11-30:12:41 ??? -------------- next part -------------- An HTML attachment was scrubbed... URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: ??????????????????????.docx Type: application/vnd.openxmlformats-officedocument.wordprocessingml.document Size: 27044 bytes Desc: not available URL: From nils.goroll at uplex.de Wed Nov 2 18:12:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 02 Nov 2016 19:12:05 +0100 Subject: [master] 32090ef document the legal PROXY protocol versions Message-ID: commit 32090ef3da34b1c8f589fb8731e18a7ccadbc319 Author: Nils Goroll Date: Wed Nov 2 18:34:05 2016 +0100 document the legal PROXY protocol versions diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index d9f10fc..cc2b5bb 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -238,7 +238,7 @@ are available: proxy_header The PROXY protocol version Varnish should use when connecting to - this backend. + this backend. Allowed values are ``1`` and ``2``. max_connections Maximum number of open connections towards this backend. If From nils.goroll at uplex.de Wed Nov 2 18:12:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 02 Nov 2016 19:12:05 +0100 Subject: [master] 0a4d618 more consistent and RST-ish inline-markup -- starting with vcl.rst Message-ID: commit 0a4d6183cfa22e7d889bc24ff0b43630a4f614d3 Author: Nils Goroll Date: Wed Nov 2 19:03:01 2016 +0100 more consistent and RST-ish inline-markup -- starting with vcl.rst in particular * VCL language and other literals as ``literal`` * placeholders and emphasis as *emphasis* * no `interpreted text` except where it actually *is* that Ref http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#character-level-inline-markup diff --git a/doc/README.WRITING_RST.rst b/doc/README.WRITING_RST.rst index 19e7fe6..7b8078e 100644 --- a/doc/README.WRITING_RST.rst +++ b/doc/README.WRITING_RST.rst @@ -1,6 +1,20 @@ THINGS TO CONSIDER WHEN WRITING VARNISH RST DOCUMENTATION ========================================================= +Inline Markup +------------- + +Please try to be consistent with inline markup and fix places which do +not follow the style: + +* VCL language and other literals as ``literal`` + +* placeholders and emphasis as *emphasis* + +* no `interpreted text` except where it actually *is* that + +.. _Reference: http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#character-level-inline-markup + References are tricky --------------------- diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index cc2b5bb..075c345 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -27,9 +27,9 @@ This document focuses on the syntax of the VCL language. For a full description of syntax and semantics, with ample examples, please see the online documentation at https://www.varnish-cache.org/docs/ . -Starting with Varnish 4.0, each VCL file must start by declaring its version -with "vcl X.Y;" marker at the top of the file. See more about this -under Versioning below. +Starting with Varnish 4.0, each VCL file must start by declaring its +version with ``vcl`` *.*\ ``;`` marker at the top of +the file. See more about this under Versioning below. Operators @@ -37,30 +37,31 @@ Operators The following operators are available in VCL: - = + ``=`` Assignment operator. - == + ``==`` Comparison. - ~ + ``~`` Match. Can either be used with regular expressions or ACLs. - ! + ``!`` Negation. - && + ``&&`` Logical and. - || + ``||`` Logical or. Conditionals ------------ -VCL has *if* and *else* statements. Nested logic can be implemented -with the *elseif* statement (*elsif*/*elif*/*else if* are equivalent). +VCL has ``if`` and ``else`` statements. Nested logic can be +implemented with the ``elseif`` statement (``elsif``\ /\ ``elif``\ /\ +``else if`` are equivalent). Note that there are no loops or iterators of any kind in VCL. @@ -68,7 +69,7 @@ Note that there are no loops or iterators of any kind in VCL. Strings, booleans, time, duration, integers and real numbers ------------------------------------------------------------ -These are the data types in Varnish. You can *set* or *unset* these. +These are the data types in Varnish. You can ``set`` or ``unset`` these. Example:: @@ -79,58 +80,59 @@ Example:: Strings ~~~~~~~ -Basic strings are enclosed in double quotes (" ... "), and may not contain -newlines. Long strings are enclosed in {" ... "}. They may contain any -character including single double quotes ("), newline and other control -characters except for the NUL (0x00) character. +Basic strings are enclosed in double quotes ``"``\ *...*\ ``"``, and +may not contain newlines. Long strings are enclosed in +``{"``\ *...*\ ``"}``. They may contain any character including single +double quotes ``"``, newline and other control characters except for the +*NUL* (0x00) character. Booleans ~~~~~~~~ -Booleans can be either *true* or *false*. In addition, in a boolean -context some data types will evaluate to *true* or *false* depending on +Booleans can be either ``true`` or ``false``. In addition, in a boolean +context some data types will evaluate to ``true`` or ``false`` depending on their value. -String types will evaluate to *false* if they are empty; backend types -will evalute to *false* if they don't have a backend assigned; integer -types will evaluate to *false* if their value is zero; duration types -will evaluate to *false* if their value is equal or less than zero. +String types will evaluate to ``false`` if they are empty; backend types +will evalute to ``false`` if they don't have a backend assigned; integer +types will evaluate to ``false`` if their value is zero; duration types +will evaluate to ``false`` if their value is equal or less than zero. Time ~~~~ VCL has time. A duration can be added to a time to make another time. In string context they return a formatted string in RFC1123 format, -e.g. Sun, 06 Nov 1994 08:49:37 GMT. +e.g. ``Sun, 06 Nov 1994 08:49:37 GMT``. -The keyword *now* returns a time representing the current time in seconds +The keyword ``now`` returns a time representing the current time in seconds since the Epoch. Durations ~~~~~~~~~ Durations are defined by a number followed by a unit. The number can -include a fractional part, e.g. 1.5s. The supported units are: +include a fractional part, e.g. ``1.5s``. The supported units are: - ms + ``ms`` milliseconds - s + ``s`` seconds - m + ``m`` minutes - h + ``h`` hours - d + ``d`` days - w + ``w`` weeks - y + ``y`` years Integers @@ -172,7 +174,7 @@ To include a VCL file in another file use the include keyword:: Import statement ---------------- -The *import* statement is used to load Varnish Modules (VMODs.) +The ``import`` statement is used to load Varnish Modules (VMODs.) Example:: @@ -184,8 +186,9 @@ Example:: Comments -------- -Single lines of VCL can be commented out using // or #. Multi-line blocks can -be commented out with \/\* block \/\*. +Single lines of VCL can be commented out using ``//`` or +``#``. Multi-line blocks can be commented out with +``/*``\ *block*\ ``*/``. Example:: @@ -202,45 +205,45 @@ Backend definition ------------------ A backend declaration creates and initialises a named backend object. A -declaration start with the keyword *backend* followed by the name of the +declaration start with the keyword ``backend`` followed by the name of the backend. The actual declaration is in curly brackets, in a key/value fashion.:: backend name { .attribute = "value"; } -The only mandatory attribute is *host*. The attributes will inherit +The only mandatory attribute is ``.host``. The attributes will inherit their defaults from the global parameters. The following attributes are available: - host (mandatory) + ``.host`` *(mandatory)* The host to be used. IP address or a hostname that resolves to a single IP address. - port + ``.port`` The port on the backend that Varnish should connect to. - host_header + ``.host_header`` A host header to add to probes and regular backend requests if they have no such header. - connect_timeout + ``.connect_timeout`` Timeout for connections. - first_byte_timeout + ``.first_byte_timeout`` Timeout for first byte. - between_bytes_timeout + ``.between_bytes_timeout`` Timeout between bytes. - probe + ``.probe`` Attach a probe to the backend. See `Probes`_ - proxy_header + ``.proxy_header`` The PROXY protocol version Varnish should use when connecting to this backend. Allowed values are ``1`` and ``2``. - max_connections + ``.max_connections`` Maximum number of open connections towards this backend. If Varnish reaches the maximum Varnish it will start failing connections. @@ -260,41 +263,41 @@ the backend as down it they fail. A probe is defined as this:: .attribute = "value"; } -The probe named `default` is special and will be used for all backends +The probe named ``default`` is special and will be used for all backends which do not explicitly reference a probe. There are no mandatory options. These are the options you can set: - url - The URL to query. Defaults to "/". + ``.url`` + The URL to query. Defaults to ``/``. - request - Specify a full HTTP request using multiple strings. .request will - have \\r\\n automatically inserted after every string. If - specified, .request will take precedence over .url. + ``.request`` + Specify a full HTTP request using multiple strings. ``.request`` will + have ``\r\n`` automatically inserted after every string. If + specified, ``.request`` will take precedence over ``.url``. - expected_response - The expected HTTP response code. Defaults to 200. + ``.expected_response`` + The expected HTTP response code. Defaults to ``200``. - timeout - The timeout for the probe. Default is 2s. + ``.timeout`` + The timeout for the probe. Default is ``2s``. - interval - How often the probe is run. Default is 5s. + ``.interval`` + How often the probe is run. Default is ``5s``. - initial - How many of the polls in .window are considered good when Varnish - starts. Defaults to the value of threshold - 1. In this case, the + ``.initial`` + How many of the polls in ``.window`` are considered good when Varnish + starts. Defaults to the value of ``.threshold`` - 1. In this case, the backend starts as sick and requires one single poll to be considered healthy. - window + ``.window`` How many of the latest polls we examine to determine backend health. - Defaults to 8. + Defaults to ``8``. - threshold + ``.threshold`` How many of the polls in .window must have succeeded for us to - consider the backend healthy. Defaults to 3. + consider the backend healthy. Defaults to ``3``. Access Control List (ACL) @@ -325,14 +328,14 @@ To match an IP address against an ACL, simply use the match operator:: VCL objects ----------- -A VCL object can be instantiated with the *new* keyword:: +A VCL object can be instantiated with the ``new`` keyword:: sub vcl_init { new b = directors.round_robin() b.add_backend(node1); } -This is only available in vcl_init. +This is only available in ``vcl_init``. Subroutines ----------- @@ -346,10 +349,11 @@ A subroutine is used to group code for legibility or reusability:: } Subroutines in VCL do not take arguments, nor do they return -values. The built in subroutines all have names beginning with vcl\_, +values. The built in subroutines all have names beginning with ``vcl_``, which is reserved. -To call a subroutine, use the call keyword followed by the subroutine's name:: +To call a subroutine, use the ``call`` keyword followed by the +subroutine's name:: sub vcl_recv { call pipe_if_local; @@ -358,11 +362,11 @@ To call a subroutine, use the call keyword followed by the subroutine's name:: Return statements ~~~~~~~~~~~~~~~~~ -The ongoing vcl\_* subroutine execution ends when a return(*action*) statement -is made. +The ongoing ``vcl_*`` subroutine execution ends when a +``return(``\ **\ ``)`` statement is made. -The *action* specifies how execution should proceed. The context defines -which actions are available. +The ** specifies how execution should proceed. The context +defines which actions are available. Multiple subroutines ~~~~~~~~~~~~~~~~~~~~ @@ -397,59 +401,59 @@ ban(STRING) Invalidates all objects in cache that match the given expression with the ban mechanism. - The format of `STRING` is:: + The format of *STRING* is:: [&& ...] - * ``: + * **: * ``req.url``: The request url * ``req.http.*``: Any request header * ``obj.status``: The cache object status * ``obj.http.*``: Any cache object header - * ``: + * **: - * ``==``: `` and `` are equal strings (case sensitive) - * ``!=``: `` and `` are unequal strings (case sensitive) - * ``~``: `` matches the regular expression `` - * ``!~``:`` does not match the regular expression `` + * ``==``: ** and ** are equal strings (case sensitive) + * ``!=``: ** and ** are unequal strings (case sensitive) + * ``~``: ** matches the regular expression ** + * ``!~``:** does not match the regular expression ** - * ``: Either a literal string or a regular expression. Note - that `` does not use any of the string delimiters like ``"`` - or ``{"..."}`` used elsewhere in varnish. To match against strings - containing whitespace, regular expressions containing ``\s`` can - be used. + * **: Either a literal string or a regular expression. Note + that ** does not use any of the string delimiters like ``"`` + or ``{"``\ *...*\ ``"}`` used elsewhere in varnish. To match + against strings containing whitespace, regular expressions + containing ``\s`` can be used. - Expressions can be chained using the `and` operator ``&&``. For `or` + Expressions can be chained using the *and* operator ``&&``. For *or* semantics, use several bans. hash_data(input) ~~~~~~~~~~~~~~~~ - Adds an input to the hash input. In the built-in VCL hash_data() - is called on the host and URL of the *request*. Available in vcl_hash. + Adds an input to the hash input. In the built-in VCL ``hash_data()`` + is called on the host and URL of the request. Available in ``vcl_hash``. synthetic(STRING) ~~~~~~~~~~~~~~~~~ - Prepare a synthetic response body containing the STRING. Available in - vcl_synth and vcl_backend_error. + Prepare a synthetic response body containing the *STRING*. Available in + ``vcl_synth`` and ``vcl_backend_error``. .. list above comes from struct action_table[] in vcc_action.c. regsub(str, regex, sub) ~~~~~~~~~~~~~~~~~~~~~~~ - Returns a copy of str with the first occurrence of the regular - expression regex replaced with sub. Within sub, \\0 (which can - also be spelled \\&) is replaced with the entire matched string, - and \\n is replaced with the contents of subgroup n in the - matched string. + Returns a copy of *str* with the first occurrence of the regular + expression *regex* replaced with *sub*. Within *sub*, ``\0`` (which + can also be spelled ``\&``) is replaced with the entire matched + string, and ``\``\ *n* is replaced with the contents of subgroup *n* + in the matched string. regsuball(str, regex, sub) ~~~~~~~~~~~~~~~~~~~~~~~~~~ - As regsub() but this replaces all occurrences. + As ``regsub()``, but this replaces all occurrences. .. regsub* is in vcc_expr.c @@ -462,18 +466,19 @@ Versioning Multiple versions of the VCL syntax can coexist within certain constraints. -The VCL syntax version at the start of VCL file specified with ''-f'' +The VCL syntax version at the start of VCL file specified with ``-f`` sets the hard limit that cannot be exceeded anywhere, and it selects the appropriate version of the builtin VCL. -That means that you can never include "vcl 9.1;" from "vcl 8.7;", but -the opposite *may* be possible, to the extent the compiler supports it. +That means that you can never include ``vcl 9.1;`` from ``vcl 8.7;``, +but the opposite *may* be possible, to the extent the compiler +supports it. -Files pulled in via ``include`` do not need to have a "vcl X.Y;" but -it may be a good idea to do it anyway, to not have surprises in the -future. The syntax version set in an included file only applies to -that file and any files it includes - unless these set their own VCL -syntax version. +Files pulled in via ``include`` do not need to have a +``vcl`` *X.Y*\ ``;`` but it may be a good idea to do it anyway, to +not have surprises in the future. The syntax version set in an +included file only applies to that file and any files it includes - +unless these set their own VCL syntax version. The version of Varnish this file belongs to supports syntax 4.0 only. From nils.goroll at uplex.de Thu Nov 3 10:16:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 03 Nov 2016 11:16:05 +0100 Subject: [master] e651f6e remove now obsolete comment Message-ID: commit e651f6e9b257e0b709204cd00675e7981eb091c6 Author: Nils Goroll Date: Thu Nov 3 11:07:56 2016 +0100 remove now obsolete comment see vwe_enter diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c index 3c2b42c..0997979 100644 --- a/bin/varnishd/waiter/cache_waiter_epoll.c +++ b/bin/varnishd/waiter/cache_waiter_epoll.c @@ -26,9 +26,6 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * XXX: We need to pass sessions back into the event engine when they are - * reused. Not sure what the most efficient way is for that. For now - * write the session pointer to a pipe which the event engine monitors. */ //lint -e{766} From nils.goroll at uplex.de Thu Nov 3 13:45:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 03 Nov 2016 14:45:05 +0100 Subject: [master] 1ae8c00 improve the long descriptions of n_object, n_objectcore and n_objecthead Message-ID: commit 1ae8c00840ae503b1b871b6550af6cc0e4fa40fc Author: Nils Goroll Date: Thu Nov 3 14:44:35 2016 +0100 improve the long descriptions of n_object, n_objectcore and n_objecthead diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h index 6b69d07..34f2000 100644 --- a/include/tbl/vsc_f_main.h +++ b/include/tbl/vsc_f_main.h @@ -275,7 +275,8 @@ VSC_F(sess_dropped, uint64_t, 0, 'c', 'i', info, VSC_F(n_object, uint64_t, 1, 'g', 'i', info, "object structs made", - "Number of object structs made" + "Approximate number of HTTP objects (headers + body, if present)" + " in the cache." ) VSC_F(n_vampireobject, uint64_t, 1, 'g', 'i', diag, "unresurrected objects", @@ -283,11 +284,13 @@ VSC_F(n_vampireobject, uint64_t, 1, 'g', 'i', diag, ) VSC_F(n_objectcore, uint64_t, 1, 'g', 'i', info, "objectcore structs made", - "Number of objectcore structs made" + "Approximate number of object metadata elements in the cache." + " Each object needs an objectcore, extra objectcores are for" + " hit-for-pass and busy objects." ) VSC_F(n_objecthead, uint64_t, 1, 'g', 'i', info, "objecthead structs made", - "Number of objecthead structs made" + "Approximate number of different hash entries in the cache." ) VSC_F(n_backend, uint64_t, 0, 'g', 'i', info, From dridi.boukelmoune at gmail.com Thu Nov 3 17:04:04 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 03 Nov 2016 18:04:04 +0100 Subject: [master] fc71348 s/how/prio/ for consistency (followup to 2e2bfad) Message-ID: commit fc71348b5ae6b1f4a1ea72dd4d57be2a93186b66 Author: Dridi Boukelmoune Date: Thu Nov 3 18:02:50 2016 +0100 s/how/prio/ for consistency (followup to 2e2bfad) diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 0d3543a..d71504e 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -924,13 +924,13 @@ const char *body_status_2str(enum body_status e); const char *sess_close_2str(enum sess_close sc, int want_desc); /* cache_pool.c */ -int Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio how); +int Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio prio); int Pool_Task_Arg(struct worker *, enum task_prio, task_func_t *, const void *arg, size_t arg_len); void Pool_Sumstat(struct worker *w); int Pool_TrySumstat(struct worker *wrk); void Pool_PurgeStat(unsigned nobj); -int Pool_Task_Any(struct pool_task *task, enum task_prio how); +int Pool_Task_Any(struct pool_task *task, enum task_prio prio); /* cache_range.c [VRG] */ void VRG_dorange(struct req *req, const char *r); diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index 9dfa2ed..2a64d4f 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -91,7 +91,7 @@ Pool_TrySumstat(struct worker *wrk) */ int -Pool_Task_Any(struct pool_task *task, enum task_prio how) +Pool_Task_Any(struct pool_task *task, enum task_prio prio) { struct pool *pp; @@ -105,7 +105,7 @@ Pool_Task_Any(struct pool_task *task, enum task_prio how) if (pp == NULL) return (-1); // NB: When we remove pools, is there a race here ? - return (Pool_Task(pp, task, how)); + return (Pool_Task(pp, task, prio)); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 6c66d67..55d149c 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -164,14 +164,14 @@ pool_reserve(void) /*--------------------------------------------------------------------*/ static struct worker * -pool_getidleworker(struct pool *pp, enum task_prio how) +pool_getidleworker(struct pool *pp, enum task_prio prio) { struct pool_task *pt = NULL; struct worker *wrk; CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); Lck_AssertHeld(&pp->mtx); - if (how <= TASK_QUEUE_RESERVE || pp->nidle > pool_reserve()) { + if (prio <= TASK_QUEUE_RESERVE || pp->nidle > pool_reserve()) { pt = VTAILQ_FIRST(&pp->idle_queue); if (pt == NULL) AZ(pp->nidle); @@ -197,7 +197,7 @@ pool_getidleworker(struct pool *pp, enum task_prio how) */ int -Pool_Task_Arg(struct worker *wrk, enum task_prio how, task_func_t *func, +Pool_Task_Arg(struct worker *wrk, enum task_prio prio, task_func_t *func, const void *arg, size_t arg_len) { struct pool *pp; @@ -211,7 +211,7 @@ Pool_Task_Arg(struct worker *wrk, enum task_prio how, task_func_t *func, CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); Lck_Lock(&pp->mtx); - wrk2 = pool_getidleworker(pp, how); + wrk2 = pool_getidleworker(pp, prio); if (wrk2 != NULL) { AN(pp->nidle); VTAILQ_REMOVE(&pp->idle_queue, &wrk2->task, list); @@ -238,20 +238,20 @@ Pool_Task_Arg(struct worker *wrk, enum task_prio how, task_func_t *func, */ int -Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio how) +Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio prio) { struct worker *wrk; int retval = 0; CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); AN(task); AN(task->func); - assert(how < TASK_QUEUE_END); + assert(prio < TASK_QUEUE_END); Lck_Lock(&pp->mtx); /* The common case first: Take an idle thread, do it. */ - wrk = pool_getidleworker(pp, how); + wrk = pool_getidleworker(pp, prio); if (wrk != NULL) { AN(pp->nidle); VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list); @@ -268,12 +268,12 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio how) * queue limits only apply to client threads - all other * work is vital and needs do be done at the earliest */ - if (how != TASK_QUEUE_REQ || + if (prio != TASK_QUEUE_REQ || pp->lqueue < cache_param->wthread_max + cache_param->wthread_queue_limit + pp->nthr) { pp->nqueued++; pp->lqueue++; - VTAILQ_INSERT_TAIL(&pp->queues[how], task, list); + VTAILQ_INSERT_TAIL(&pp->queues[prio], task, list); } else { pp->ndropped++; retval = -1; From nils.goroll at uplex.de Fri Nov 4 07:12:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 04 Nov 2016 08:12:05 +0100 Subject: [master] 4ec789a Add a trivial benchmark to the vtim test Message-ID: commit 4ec789af6a6e8cadf3c81af1939834379949e835 Author: Nils Goroll Date: Fri Nov 4 08:09:07 2016 +0100 Add a trivial benchmark to the vtim test diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index 509ba9a..cd8c2ae 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -466,6 +466,29 @@ tst_delta() } } +static void +bench() +{ + double s, e, t; + int i; + + t = 0; + s = VTIM_real(); + for (i=0; i<100000; i++) + t += VTIM_real(); + e = VTIM_real(); + printf("real: %fs / %d = %fns - tst val %f\n", + e - s, i, 1e9 * (e - s) / i, t); + + t = 0; + s = VTIM_real(); + for (i=0; i<100000; i++) + t += VTIM_mono(); + e = VTIM_real(); + printf("mono: %fs / %d = %fns - tst val %f\n", + e - s, i, 1e9 * (e - s) / i, t); +} + int main(int argc, char **argv) { @@ -478,6 +501,8 @@ main(int argc, char **argv) AZ(setenv("TZ", "UTC", 1)); assert(sizeof t >= 8); + bench(); + /* Brute force test against libc version */ for (t = -2209852800; t < 20000000000; t += 3599) { gmtime_r(&t, &tm); From martin at varnish-software.com Fri Nov 4 12:23:05 2016 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Fri, 04 Nov 2016 13:23:05 +0100 Subject: [master] 9efd7fc Include the current time of the panic in the panic output Message-ID: commit 9efd7fca94a65a8d4774e61ac4a4f923b69d47ec Author: Martin Blix Grydeland Date: Thu Oct 27 14:02:43 2016 +0200 Include the current time of the panic in the panic output Show both the real and the monotonic time in the panic output when the panic is created. This is useful to have something to compare other panic timestamps against. The existing timestamp in the panic output is recorded by the management process at the time of receiving the panic from the child process. Due to processing and possible core dump creation, this time isn't very useful for comparison purposes. diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index ffe6fd1..fb57e86 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -39,6 +39,8 @@ #include #include +#include "vtim.h" + #include "cache.h" #include "cache_transport.h" @@ -629,6 +631,8 @@ pan_ic(const char *func, const char *file, int line, const char *cond, VSB_printf(pan_vsb, "version = %s\n", VCS_version); VSB_printf(pan_vsb, "ident = %s,%s\n", VSB_data(vident) + 1, Waiter_GetName()); + VSB_printf(pan_vsb, "now = %f (mono), %f (real)\n", + VTIM_mono(), VTIM_real()); pan_backtrace(pan_vsb); From martin at varnish-software.com Fri Nov 4 12:23:05 2016 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Fri, 04 Nov 2016 13:23:05 +0100 Subject: [master] cdf7f5b Introduce a parameter value for allowed clock step Message-ID: commit cdf7f5b8b99ce97b21910b8d082725052944e253 Author: Martin Blix Grydeland Date: Thu Oct 27 14:44:21 2016 +0200 Introduce a parameter value for allowed clock step This value is used in calculations that could reveal clock step, specifying the amount of error we allow before panicking. It defaults to a conservative 1 second. diff --git a/include/tbl/params.h b/include/tbl/params.h index 7bbf54a..d40aedd 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -307,6 +307,21 @@ PARAM( ) PARAM( + /* name */ clock_step, + /* typ */ timeout, + /* min */ "0.000", + /* max */ NULL, + /* default */ "1.000", + /* units */ "seconds", + /* flags */ 0, + /* s-text */ + "How much observed clock step we are willing to accept before " + "we panic.", + /* l-text */ "", + /* func */ NULL +) + +PARAM( /* name */ connect_timeout, /* typ */ timeout, /* min */ "0.000", From martin at varnish-software.com Fri Nov 4 12:23:05 2016 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Fri, 04 Nov 2016 13:23:05 +0100 Subject: [master] c2a62c3 Allow up to clock_step error when checking the session timestamps Message-ID: commit c2a62c39c5e3addd82003bdc24bbe293f30b9fdb Author: Martin Blix Grydeland Date: Thu Oct 27 14:59:17 2016 +0200 Allow up to clock_step error when checking the session timestamps When checking the timestamps on deleting a session, allow up to clock_step seconds error before bailing. Fixes: #1874 diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 2eeec0a..5e263e2 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -518,11 +518,15 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now) if (isnan(now)) now = VTIM_real(); AZ(isnan(sp->t_open)); + if (now < sp->t_open) { + if (now + cache_param->clock_step < sp->t_open) + WRONG("Clock step detected"); + now = sp->t_open; /* Do not log negatives */ + } if (reason == SC_NULL) reason = (enum sess_close)-sp->fd; - assert(now >= sp->t_open); assert(VTAILQ_EMPTY(&sp->privs->privs)); VSL(SLT_SessClose, sp->vxid, "%s %.3f", sess_close_2str(reason, 0), now - sp->t_open); From martin at varnish-software.com Fri Nov 4 12:23:05 2016 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Fri, 04 Nov 2016 13:23:05 +0100 Subject: [master] 4703d92 Log a message on clock step Message-ID: commit 4703d9207adbbddef5babec5bce6b6e918ffddba Author: Martin Blix Grydeland Date: Fri Nov 4 13:20:47 2016 +0100 Log a message on clock step diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 5e263e2..9bcb824 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -519,6 +519,9 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now) now = VTIM_real(); AZ(isnan(sp->t_open)); if (now < sp->t_open) { + VSL(SLT_Debug, sp->vxid, + "Clock step (now=%f < t_open=%f)", + now, sp->t_open); if (now + cache_param->clock_step < sp->t_open) WRONG("Clock step detected"); now = sp->t_open; /* Do not log negatives */ From document at varnish-cache.org Fri Nov 4 15:57:45 2016 From: document at varnish-cache.org (document at varnish-cache.org) Date: Fri, 04 Nov 2016 21:27:45 +0530 Subject: Please find attached invoice no: 7020538 Message-ID: <71759-28F1a2771-37eE85f20-832e94FA-290bcCa0@varnish-cache.org> Attached is a Print Manager form. Format = Portable Document Format File (PDF) ________________________________ Disclaimer This email/fax transmission is confidential and intended solely for the person or organisation to whom it is addressed. If you are not the intended recipient, you must not copy, distribute or disseminate the information, or take any action in reliance of it. Any views expressed in this message are those of the individual sender, except where the sender specifically states them to be the views of any organisation or employer. If you have received this message in error, do not open any attachment but please notify the sender (above) deleting this message from your system. For email transmissions please rely on your own virus check no responsibility is taken by the sender for any damage rising out of any bug or virus infection. -------------- next part -------------- A non-text attachment was scrubbed... Name: A3bd16C.zip Type: application/zip Size: 6832 bytes Desc: not available URL: From fgsch at lodoss.net Sun Nov 6 12:47:05 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 06 Nov 2016 13:47:05 +0100 Subject: [master] 438c9b4 Polish Message-ID: commit 438c9b43b1a0f28b27dcc23489c1162dd6453054 Author: Federico G. Schwindt Date: Sat Nov 5 18:02:27 2016 +0000 Polish diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c index 66752be..ce1dc7e 100644 --- a/bin/varnishd/http1/cache_http1_fsm.c +++ b/bin/varnishd/http1/cache_http1_fsm.c @@ -263,16 +263,21 @@ http1_dissect(struct worker *wrk, struct req *req) assert (req->req_body_status == REQ_BODY_INIT); - if (req->htc->body_status == BS_CHUNKED) { + switch (req->htc->body_status) { + case BS_CHUNKED: req->req_body_status = REQ_BODY_WITHOUT_LEN; - } else if (req->htc->body_status == BS_LENGTH) { + break; + case BS_LENGTH: req->req_body_status = REQ_BODY_WITH_LEN; - } else if (req->htc->body_status == BS_NONE) { + break; + case BS_NONE: req->req_body_status = REQ_BODY_NONE; - } else if (req->htc->body_status == BS_EOF) { + break; + case BS_EOF: req->req_body_status = REQ_BODY_WITHOUT_LEN; - } else { - WRONG("Unknown req.body_length situation"); + break; + default: + WRONG("Unknown req_body_status situation"); } if (http_GetHdr(req->http, H_Expect, &p)) { @@ -404,7 +409,8 @@ HTTP1_Session(struct worker *wrk, struct req *req) "H2 Prior Knowledge Upgrade"); http1_setstate(sp, NULL); req->err_code = 1; - SES_SetTransport(wrk, sp, req, &H2_transport); + SES_SetTransport(wrk, sp, req, + &H2_transport); return; } @@ -424,7 +430,8 @@ HTTP1_Session(struct worker *wrk, struct req *req) VSLb(req->vsl, SLT_Debug, "H2 upgrade attempt has body"); } else { - VSLb(req->vsl, SLT_Debug, "H2 Upgrade"); + VSLb(req->vsl, SLT_Debug, + "H2 Upgrade"); http1_setstate(sp, NULL); req->err_code = 2; SES_SetTransport(wrk, sp, req, @@ -441,7 +448,8 @@ HTTP1_Session(struct worker *wrk, struct req *req) */ if (VTCP_check_hup(sp->fd)) { AN(req->hash_objhead); - (void)HSH_DerefObjHead(wrk, &req->hash_objhead); + (void)HSH_DerefObjHead(wrk, + &req->hash_objhead); AZ(req->hash_objhead); SES_Close(sp, SC_REM_CLOSE); AN(Req_Cleanup(sp, wrk, req)); From fgsch at lodoss.net Sun Nov 6 13:45:04 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 06 Nov 2016 14:45:04 +0100 Subject: [master] 351f58c Skip this test if gradm2 is around Message-ID: commit 351f58c7162e1d03fb287eb27a6d5af275850ff7 Author: Federico G. Schwindt Date: Sun Nov 6 13:26:28 2016 +0000 Skip this test if gradm2 is around Some hardened Linux' will map the segment using random addresses as well. diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc index 9b280ec..7d7a6b7 100644 --- a/bin/varnishtest/tests/r00962.vtc +++ b/bin/varnishtest/tests/r00962.vtc @@ -1,7 +1,9 @@ varnishtest "Test address remapping" -# VM-remapping is to random on OSX +# VM-remapping is too random on OSX feature !OSX +# Same on some hardened Linux +feature cmd "gradm2 >& >/dev/null; [ $? -ne 0 ]" server s1 { rxreq From fgsch at lodoss.net Sun Nov 6 16:48:05 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 06 Nov 2016 17:48:05 +0100 Subject: [master] 5cfa334 Better and simpler check Message-ID: commit 5cfa3340b6f16d9c3afe28dee1555455bf99b28b Author: Federico G. Schwindt Date: Sun Nov 6 16:44:46 2016 +0000 Better and simpler check diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc index 7d7a6b7..dccf0af 100644 --- a/bin/varnishtest/tests/r00962.vtc +++ b/bin/varnishtest/tests/r00962.vtc @@ -3,7 +3,7 @@ varnishtest "Test address remapping" # VM-remapping is too random on OSX feature !OSX # Same on some hardened Linux -feature cmd "gradm2 >& >/dev/null; [ $? -ne 0 ]" +feature cmd "test ! -c /dev/grsec" server s1 { rxreq From fgsch at lodoss.net Mon Nov 7 14:18:04 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 07 Nov 2016 15:18:04 +0100 Subject: [master] 0ce5b8b Typo Message-ID: commit 0ce5b8b71f138ceb16e23104bc1f4eba0d76f363 Author: Federico G. Schwindt Date: Mon Nov 7 13:45:57 2016 +0000 Typo diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h index 34f2000..3952302 100644 --- a/include/tbl/vsc_f_main.h +++ b/include/tbl/vsc_f_main.h @@ -249,8 +249,8 @@ VSC_F(busy_sleep, uint64_t, 1, 'c', 'i', info, VSC_F(busy_wakeup, uint64_t, 1, 'c', 'i', info, "Number of requests woken after sleep on busy objhdr", - "Number of requests taken of the busy object sleep list and" - " and rescheduled." + "Number of requests taken off the busy object sleep list and" + " rescheduled." ) VSC_F(busy_killed, uint64_t, 1, 'c', 'i', info, From office at varnish-cache.org Mon Nov 7 15:40:29 2016 From: office at varnish-cache.org (office at varnish-cache.org) Date: Mon, 07 Nov 2016 21:10:29 +0530 Subject: Scanned image from MX2310U@varnish-cache.org Message-ID: <20161107211029.8FB5.OFFICE@varnish-cache.org> Reply to: office at varnish-cache.org Device Name: MX2310U at varnish-cache.org Device Model: MX-2310U Location: Reception File Format: PDF MMR(G4) Resolution: 200dpi x 200dpi Attached file is scanned image in PDF format(RAR archive). Use Acrobat(R)Reader(R) or Adobe(R)Reader(R) of Adobe Systems Incorporated to view the document. Adobe(R)Reader(R) can be downloaded from the following URL: Adobe, the Adobe logo, Acrobat, the Adobe PDF logo, and Reader are registered trademarks or trademarks of Adobe Systems Incorporated in the United States and other countries. http://www.adobe.com/ -------------- next part -------------- A non-text attachment was scrubbed... Name: office at varnish-cache.org_20161107_211029.zip Type: application/octet-stream Size: 7933 bytes Desc: not available URL: From nils.goroll at uplex.de Mon Nov 7 17:19:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 07 Nov 2016 18:19:05 +0100 Subject: [master] afbc4a2 test PRIV_TASK (and std.log) in vcl_fini Message-ID: commit afbc4a2a77d009eb4b33cb5bde559ec761f7f9d3 Author: Nils Goroll Date: Mon Nov 7 18:17:54 2016 +0100 test PRIV_TASK (and std.log) in vcl_fini diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index 7678798..e3636e5 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -5,9 +5,12 @@ server s1 { txresp rxreq txresp + + rxreq + txresp } -start -varnish v1 -vcl+backend { +varnish v1 -arg "-p debug=+vclrel" -vcl+backend { import debug; import std; @@ -46,6 +49,14 @@ varnish v1 -vcl+backend { set beresp.http.bx1 = debug.test_priv_task(""); set beresp.http.bo1 = obj.test_priv_task(""); } + + sub vcl_fini { + debug.test_priv_task("cleaning"); + debug.test_priv_task("up"); + std.log("func " + debug.test_priv_task()); + std.log("obj " + obj.test_priv_task()); + } + } -start logexpect l1 -v v1 -g raw -d 1 { @@ -64,6 +75,10 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 = VCL_Log ^foo expect 0 = BereqHeader {^bx0: b /snafu} expect 0 = VCL_Log ^bar + + expect * 0 Debug {^vcl1: VCL_EVENT_COLD} + expect * = VCL_Log {^func cleaning up} + expect 0 = VCL_Log {^obj cleaning up} } -start client c1 { @@ -86,4 +101,19 @@ client c1 { expect resp.http.bo1 == "b /snafu" } -run +shell "echo 'vcl 4.0; backend foo { .host = \"${s1_addr}\"; .port = \"${s1_port}\"; }' > ${tmpdir}/_b00014.vcl" + +varnish v1 -cliok "vcl.load foo ${tmpdir}/_b00014.vcl" \ + -cliok "vcl.use foo" \ + -cliok "vcl.list" \ + -cliok "vcl.discard vcl1" \ + -cliok "vcl.list" + +client c1 { + txreq -url /foo + rxresp +} -run + +varnish v1 -cliok "vcl.list" + logexpect l1 -wait From dridi.boukelmoune at gmail.com Tue Nov 8 15:30:06 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 08 Nov 2016 16:30:06 +0100 Subject: [master] 4af8328 Reorganize the syntax description in varnish-cli(7) Message-ID: commit 4af8328933cf63c54ba5c1766a049ff9a6b0d40d Author: Dridi Boukelmoune Date: Tue Nov 8 15:21:57 2016 +0100 Reorganize the syntax description in varnish-cli(7) Ramp up from simple tokens to here documents. diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index fda59c9..c0c911a 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -51,30 +51,49 @@ that service. Please see :ref:`varnishd(1)` for details. Syntax ------ -Commands are usually terminated with a newline. Long command can be -entered using shell-style *here document* (here-document or heredoc). -The format of here document is:: +The Varnish CLI is similar to another command line interface, the Bourne +Shell. Commands are usually terminated with a newline, and they may take +arguments. The command and its arguments are *tokenized* before parsing, +and as such arguments containing must must be enclosed in double quotes. + +It means that command parsing of + +:: + + help banner + +is equivalent to + +:: + + "help" banner + +because the double quotes only indicate the boundaries of the ``help`` +token. + +Within double quotes you can escape characters with \\ (backslash). The \\n, +\\r, and \\t get translated to newlines, carriage returns, an tabs. Double +quotes and backslashes themselves can be escaped with \\" and \\\\ +respectively. + +To enter characters in octals use the \\nnn syntax. Hexadecimals can +be entered with the \\xnn syntax. + +Commands may not end with a newline when a shell-style *here document* +(here-document or heredoc) is used. The format of a here document is:: << word here document word *word* can be any continuous string chosen to make sure it doesn't appear -naturally in the following *here document*. Often EOF or END is used. +naturally in the following *here document*. Traditionally EOF or END is +used. When using the here document style of input there are no restrictions on length. When using newline-terminated commands maximum length is limited by the varnishd parameter *cli_buffer*. -When commands are newline terminated they get *tokenized* before -parsing so if you have significant spaces enclose your strings in -double quotes. Within the quotes you can escape characters with -\\. The \n, \r and \t get translated to newlines, carriage returns and -tabs. Double quotes themselves can be escaped with a backslash. - -To enter characters in octals use the \\nnn syntax. Hexadecimals can -be entered with the \\xnn syntax. - Commands -------- From dridi.boukelmoune at gmail.com Tue Nov 8 15:30:06 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 08 Nov 2016 16:30:06 +0100 Subject: [master] 10ac7e1 Explain what may go wrong with quoting and the CLI Message-ID: commit 10ac7e1f480a7382c8fd12eb30193b7aff349847 Author: Dridi Boukelmoune Date: Tue Nov 8 15:43:06 2016 +0100 Explain what may go wrong with quoting and the CLI Fixes #1899 Closes #2110 diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index c0c911a..58c80ef 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -94,6 +94,86 @@ When using the here document style of input there are no restrictions on length. When using newline-terminated commands maximum length is limited by the varnishd parameter *cli_buffer*. +Quoting pitfalls +---------------- + +Integrating with the Varnish CLI can be sometimes surprising when quoting +is involved. For instance in Bourne Shell the delimiter used with here +documents may or may not be separated by spaces from the ``<<`` token:: + + cat <' Line 1 Pos 1) + < commit e3363acc3a293904ad35f5110e0367380e93c27d Author: Nils Goroll Date: Tue Nov 8 20:17:56 2016 +0100 monotonic clock for macos The otherwise recommended clock_get_time(SYSTEM_TIME) is about 40x slower than mach_absolute_time() even after minimizing overhead by retrieving the port right only once per process - see https://github.com/nigoroll/varnish-cache/tree/macos_SYSTEM_CLOCK So we revert to mach_absolute_time() which needs some scaling but otherwise does the job. It is reported to halt on iOS devices when sleeping, but even if anyone came up with the idea to sell varnish over the app store, sleeping would probably need to be avoided in the first place and otherwise the right way would probably be to adjust the offset using pre/post sleep callbacks. As MacOS should have been the last platform using the real clock fallback for VTIM_mono, we now bail out at compile-time for any platform we have overlooked. diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index cd8c2ae..8b43f50 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -62,6 +62,9 @@ #include #include #include +#ifdef __MACH__ +#include +#endif #include "vas.h" #include "vtim.h" @@ -87,6 +90,29 @@ static const int days_before_month[] = { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 }; +#ifdef __MACH__ +// http://stackoverflow.com/a/21352348 +static uint64_t mt_base; +static double mt_scale; + +void +mach_time_init(void) +{ + mach_timebase_info_data_t timebase; + + mt_base = mach_absolute_time(); + + AZ(mach_timebase_info(&timebase)); + mt_scale = (double)timebase.numer / (double)timebase.denom * 1e-9; +} + +__attribute__((constructor)) void +init(void) +{ + mach_time_init(); +} +#endif + /* * Note on Solaris: for some reason, clock_gettime(CLOCK_MONOTONIC, &ts) is not * implemented in assembly, but falls into a syscall, while gethrtime() doesn't, @@ -103,11 +129,12 @@ VTIM_mono(void) AZ(clock_gettime(CLOCK_MONOTONIC, &ts)); return (ts.tv_sec + 1e-9 * ts.tv_nsec); -#else - struct timeval tv; +#elif defined(__MACH__) + uint64_t mt = mach_absolute_time() - mt_base; - AZ(gettimeofday(&tv, NULL)); - return (tv.tv_sec + 1e-6 * tv.tv_usec); + return (mt * mt_scale); +#else +#error Varnish needs some monotonic time source #endif } From nils.goroll at uplex.de Tue Nov 8 19:34:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 08 Nov 2016 20:34:05 +0100 Subject: [master] 0b2d05c my macos test was missing the strict compiler flags Message-ID: commit 0b2d05cf1b59612228f30ca3f5eb25495f31e6ac Author: Nils Goroll Date: Tue Nov 8 20:28:14 2016 +0100 my macos test was missing the strict compiler flags diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index 8b43f50..ae19c6f 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -95,7 +95,7 @@ static const int days_before_month[] = { static uint64_t mt_base; static double mt_scale; -void +static void mach_time_init(void) { mach_timebase_info_data_t timebase; @@ -106,7 +106,7 @@ mach_time_init(void) mt_scale = (double)timebase.numer / (double)timebase.denom * 1e-9; } -__attribute__((constructor)) void +static __attribute__((constructor)) void init(void) { mach_time_init(); From nils.goroll at uplex.de Wed Nov 9 13:46:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 09 Nov 2016 14:46:05 +0100 Subject: [master] ca6f951 plaster some additional assertions for vary processing Message-ID: commit ca6f951317f2afaa991fc36475c0ff35c1df60d4 Author: Nils Goroll Date: Wed Nov 9 14:02:57 2016 +0100 plaster some additional assertions for vary processing Related to #2130 diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c index b983b54..d5d3e4b 100644 --- a/bin/varnishd/cache/cache_vary.c +++ b/bin/varnishd/cache/cache_vary.c @@ -117,8 +117,8 @@ VRY_Create(struct busyobj *bo, struct vsb **psb) /* Build a header-matching string out of it */ VSB_clear(sbh); - VSB_printf(sbh, "%c%.*s:%c", - (char)(1 + (q - p)), (int)(q - p), p, 0); + AZ(VSB_printf(sbh, "%c%.*s:%c", + (char)(1 + (q - p)), (int)(q - p), p, 0)); AZ(VSB_finish(sbh)); if (http_GetHdr(bo->bereq, VSB_data(sbh), &h)) { @@ -139,11 +139,11 @@ VRY_Create(struct busyobj *bo, struct vsb **psb) e = h; l = 0xffff; } - VSB_printf(sb, "%c%c", (int)(l >> 8), (int)(l & 0xff)); + AZ(VSB_printf(sb, "%c%c", (int)(l >> 8), (int)(l & 0xff))); /* Append to vary matching string */ - VSB_bcat(sb, VSB_data(sbh), VSB_len(sbh)); + AZ(VSB_bcat(sb, VSB_data(sbh), VSB_len(sbh))); if (e != h) - VSB_bcat(sb, h, e - h); + AZ(VSB_bcat(sb, h, e - h)); while (vct_issp(*q)) q++; @@ -291,6 +291,7 @@ VRY_Match(struct req *req, const uint8_t *vary) int i, oflo = 0; AN(vsp); + AN(vary); while (vary[2]) { if (vsp + 2 >= req->vary_e) { /* From nils.goroll at uplex.de Wed Nov 9 13:46:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 09 Nov 2016 14:46:05 +0100 Subject: [master] 37f565d in workspace panic info, use +0 offset output for {f, r, e} == s also Message-ID: commit 37f565ddb28d9ba04fa705030540fcb636911892 Author: Nils Goroll Date: Wed Nov 9 14:31:20 2016 +0100 in workspace panic info, use +0 offset output for {f, r, e} == s also diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 69edef8..1685d61 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -163,15 +163,15 @@ pan_ws(struct vsb *vsb, const struct ws *ws) VSB_printf(vsb, "OVERFLOWED "); VSB_printf(vsb, "id = \"%s\",\n", ws->id); VSB_printf(vsb, "{s, f, r, e} = {%p", ws->s); - if (ws->f > ws->s) + if (ws->f >= ws->s) VSB_printf(vsb, ", +%ld", (long) (ws->f - ws->s)); else VSB_printf(vsb, ", %p", ws->f); - if (ws->r > ws->s) + if (ws->r >= ws->s) VSB_printf(vsb, ", +%ld", (long) (ws->r - ws->s)); else VSB_printf(vsb, ", %p", ws->r); - if (ws->e > ws->s) + if (ws->e >= ws->s) VSB_printf(vsb, ", +%ld", (long) (ws->e - ws->s)); else VSB_printf(vsb, ", %p", ws->e); From nils.goroll at uplex.de Wed Nov 9 13:46:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 09 Nov 2016 14:46:05 +0100 Subject: [master] 31e0eaf panic: use pan_ws for all the workspaces Message-ID: commit 31e0eafe1420ac3b65d64b4e68dec3d554567be8 Author: Nils Goroll Date: Wed Nov 9 14:27:58 2016 +0100 panic: use pan_ws for all the workspaces diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index fb57e86..69edef8 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -193,7 +193,7 @@ pan_htc(struct vsb *vsb, const struct http_conn *htc) PAN_CheckMagic(vsb, htc, HTTP_CONN_MAGIC); VSB_printf(vsb, "fd = %d,\n", htc->fd); VSB_printf(vsb, "doclose = %s,\n", sess_close_2str(htc->doclose, 0)); - VSB_printf(vsb, "ws = %p,\n", htc->ws); + pan_ws(vsb, htc->ws); VSB_printf(vsb, "{rxbuf_b, rxbuf_e} = {%p, %p},\n", htc->rxbuf_b, htc->rxbuf_e); VSB_printf(vsb, "{pipeline_b, pipeline_e} = {%p, %p},\n", @@ -222,7 +222,7 @@ pan_http(struct vsb *vsb, const char *id, const struct http *h) return; VSB_indent(vsb, 2); PAN_CheckMagic(vsb, h, HTTP_MAGIC); - VSB_printf(vsb, "ws[%s] = %p,\n", h->ws ? h->ws->id : "", h->ws); + pan_ws(vsb, h->ws); VSB_printf(vsb, "hdrs {\n"); VSB_indent(vsb, 2); for (i = 0; i < h->nhd; ++i) { From nils.goroll at uplex.de Wed Nov 9 13:46:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 09 Nov 2016 14:46:05 +0100 Subject: [master] 04c1c92 panic: fix indent Message-ID: commit 04c1c92d8b9a3bb87375d5f66708516443a5e2e9 Author: Nils Goroll Date: Wed Nov 9 14:45:34 2016 +0100 panic: fix indent diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 1685d61..68253ff 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -518,9 +518,10 @@ pan_sess(struct vsb *vsb, const struct sess *sp) VSB_indent(vsb, -2); VSB_printf(vsb, "}"); } + VSB_printf(vsb, "\n"); ci = SES_Get_String_Attr(sp, SA_CLIENT_IP); cp = SES_Get_String_Attr(sp, SA_CLIENT_PORT); - VSB_printf(vsb, "\nclient = %s %s,\n", ci, cp); + VSB_printf(vsb, "client = %s %s,\n", ci, cp); VSB_indent(vsb, -2); VSB_printf(vsb, "},\n"); From nils.goroll at uplex.de Wed Nov 9 13:57:04 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 09 Nov 2016 14:57:04 +0100 Subject: [master] c457b96 serialize STV_next Message-ID: commit c457b965383087714e3afcce60a763fbec295bb1 Author: Nils Goroll Date: Wed Nov 9 14:55:13 2016 +0100 serialize STV_next May be a preliminary solution should phk decide to do this differently. Fixes #2128 diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index ef90263..6ecf8ee 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -52,13 +52,16 @@ const struct stevedore * STV_next() { static struct stevedore *stv; + static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; + AZ(pthread_mutex_lock(&mtx)); if (!STV__iter(&stv)) AN(STV__iter(&stv)); if (stv == stv_transient) { stv = NULL; AN(STV__iter(&stv)); } + AZ(pthread_mutex_unlock(&mtx)); AN(stv); return (stv); } From scanner at varnish-cache.org Wed Nov 9 13:30:05 2016 From: scanner at varnish-cache.org (scanner at varnish-cache.org) Date: Wed, 09 Nov 2016 06:30:05 -0700 Subject: Message from KMBT_C220 Message-ID: <6929EF40.016.DE68B32DB432.scanner@varnish-cache.org> A non-text attachment was scrubbed... Name: SKMBT_C21641175352915.zip Type: application/zip Size: 6637 bytes Desc: not available URL: From nils.goroll at uplex.de Wed Nov 9 15:39:04 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 09 Nov 2016 16:39:04 +0100 Subject: [master] 3e9fcfc thread_pool_stack: better documentation, not experimental but delayed Message-ID: commit 3e9fcfc885edbfcea00e18dab23f7bad04569933 Author: Nils Goroll Date: Wed Nov 9 16:27:44 2016 +0100 thread_pool_stack: better documentation, not experimental but delayed doc-fixes #2129 diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index 03e98fd..7cb807d 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -211,8 +211,39 @@ struct parspec WRK_parspec[] = { NULL, NULL, "Worker thread stack size.\n" "This will likely be rounded up to a multiple of 4k" - " (or whatever the page_size might be) by the kernel.", - EXPERIMENTAL, + " (or whatever the page_size might be) by the kernel.\n" + "\n" + "The required stack size is primarily driven by the" + " depth of the call-tree. The most common relevant" + " determining factors in varnish core code are GZIP" + " (un)compression, ESI processing and regular" + " expression matches. VMODs may also require" + " significant amounts of additional stack. The" + " nesting depth of VCL subs is another factor," + " although typically not predominant.\n" + "\n" + "The stack size is per thread, so the maximum total" + " memory required for worker thread stacks is in the" + " order of size = thread_pools x thread_pool_max x" + " thread_pool_stack.\n" + "\n" + "Thus, in particular for setups with many threads," + " keeping the stack size at a minimum helps reduce" + " the amount of memory required by Varnish.\n" + "\n" + "On the other hand, thread_pool_stack must be large" + " enough under all circumstances, otherwise varnish" + " will crash due to a stack overflow. Usually, a" + " stack overflow manifests itself as a segmentation" + " fault (aka segfault / SIGSEGV) with the faulting" + " address being near the stack pointer (sp).\n" + "\n" + "Unless stack usage can be reduced," + " thread_pool_stack must be increased when a stack" + " overflow occurs. Setting it in 150%-200%" + " increments is recommended until stack overflows" + " cease to occur.", + DELAYED_EFFECT, NULL, "bytes" }, // default set in mgt_main.c { NULL, NULL, NULL } }; From varnish-commit at varnish-cache.org Thu Nov 10 11:20:07 2016 From: varnish-commit at varnish-cache.org (etfgmwo) Date: Thu, 10 Nov 2016 19:20:07 +0800 Subject: =?utf-8?B?dmFybmlzaC1jb21taXQ65YWz6ZSu5Lq65omN566h?= =?utf-8?B?55CG5L2T57O76K6+6K6hIHBnNmQ=?= Message-ID: <20161110192021715317@eqkggnlw.com> varnish-commit: ? ?? ????????????????????? lt3d4 2016-11-1019:20:20 -------------- next part -------------- An HTML attachment was scrubbed... URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: ???????????????.docx Type: application/vnd.openxmlformats-officedocument.wordprocessingml.document Size: 24106 bytes Desc: not available URL: From phk at FreeBSD.org Fri Nov 11 08:16:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 11 Nov 2016 09:16:05 +0100 Subject: [master] bd9269a Make this test jail-compatible Message-ID: commit bd9269aabecee77e39961032f63bb0e639af535a Author: Poul-Henning Kamp Date: Fri Nov 11 08:14:53 2016 +0000 Make this test jail-compatible diff --git a/bin/varnishtest/tests/v00025.vtc b/bin/varnishtest/tests/v00025.vtc index 3e1acd3..8e96582 100644 --- a/bin/varnishtest/tests/v00025.vtc +++ b/bin/varnishtest/tests/v00025.vtc @@ -85,7 +85,7 @@ client c1 { expect resp.status == 200 expect resp.http.bereq_backend == "rr" expect resp.http.beresp_backend == "s1" - expect resp.http.be_ip == "127.0.0.1" + expect resp.http.be_ip == "${localhost}" expect resp.http.be_nm == "s1" expect resp.http.be == "rr" From nils.goroll at uplex.de Fri Nov 11 11:26:04 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 11 Nov 2016 12:26:04 +0100 Subject: [master] 45c79d0 Save the stv value to return within the protected reagion Message-ID: commit 45c79d06fd020bc60899b3fd6624fd7036d25535 Author: Nils Goroll Date: Fri Nov 11 12:21:17 2016 +0100 Save the stv value to return within the protected reagion commit c457b965383087714e3afcce60a763fbec295bb1 was incomplete This should fix #2128 for good diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 6ecf8ee..c646bdd 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -52,6 +52,7 @@ const struct stevedore * STV_next() { static struct stevedore *stv; + struct stevedore *r; static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; AZ(pthread_mutex_lock(&mtx)); @@ -61,9 +62,10 @@ STV_next() stv = NULL; AN(STV__iter(&stv)); } + r = stv; AZ(pthread_mutex_unlock(&mtx)); - AN(stv); - return (stv); + AN(r); + return (r); } /*------------------------------------------------------------------- From phk at FreeBSD.org Mon Nov 14 07:58:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 14 Nov 2016 08:58:05 +0100 Subject: [master] 718531c Make the size check work on arm/32bit Message-ID: commit 718531c591ae3d6b95fc7f19183fa5045f948f23 Author: Poul-Henning Kamp Date: Mon Nov 14 07:57:02 2016 +0000 Make the size check work on arm/32bit diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 086a779..994a255 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -600,7 +600,7 @@ parse_x_format(char *buf) { char *e, *r, *s; int slt; - long i; + intmax_t i; if (!strcmp(buf, "Varnish:time_firstbyte")) { addf_fragment(&CTX.frag[F_ttfb], ""); @@ -640,7 +640,7 @@ parse_x_format(char *buf) if (r == buf || r[1] == ']') VUT_Error(1, "Syntax error: VSL:%s", buf); e[-1] = '\0'; - i = strtol(r + 1, &s, 10); + i = strtoimax(r + 1, &s, 10); if (s != e - 1) VUT_Error(1, "Syntax error: VSL:%s]", buf); if (i <= 0) @@ -650,7 +650,7 @@ parse_x_format(char *buf) buf); if (i > INT_MAX) { VUT_Error(1, - "Field specifier %ld for the tag VSL:%s]" + "Field specifier %jd for the tag VSL:%s]" " is probably too high", i, buf); } From nils.goroll at uplex.de Mon Nov 14 09:20:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 14 Nov 2016 10:20:05 +0100 Subject: [master] b6f67fb Format 0 bytes as 0b, be consistent about internal byte arguments Message-ID: commit b6f67fb7e2bd6c8d23ac7e858e7f5c75fe46c8ed Author: Nils Goroll Date: Mon Oct 31 18:55:23 2016 +0100 Format 0 bytes as 0b, be consistent about internal byte arguments 0 bytes are not a bogus number diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index a31d1c7..61f8a90 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -443,12 +443,12 @@ init_params(struct cli *cli) #endif low = sysconf(_SC_THREAD_STACK_MIN); - MCF_ParamConf(MCF_MINIMUM, "thread_pool_stack", "%jd", (intmax_t)low); + MCF_ParamConf(MCF_MINIMUM, "thread_pool_stack", "%jdb", (intmax_t)low); def = 48 * 1024; if (def < low) def = low; - MCF_ParamConf(MCF_DEFAULT, "thread_pool_stack", "%jd", (intmax_t)def); + MCF_ParamConf(MCF_DEFAULT, "thread_pool_stack", "%jdb", (intmax_t)def); MCF_ParamConf(MCF_MAXIMUM, "thread_pools", "%d", MAX_THREAD_POOLS); diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c index d58e032..70e1c9a 100644 --- a/bin/varnishd/mgt/mgt_param_tweak.c +++ b/bin/varnishd/mgt/mgt_param_tweak.c @@ -223,7 +223,7 @@ fmt_bytes(struct vsb *vsb, uintmax_t t) { const char *p; - if (t & 0xff) { + if (t == 0 || t & 0xff) { VSB_printf(vsb, "%jub", t); return; } From nils.goroll at uplex.de Mon Nov 14 09:20:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 14 Nov 2016 10:20:05 +0100 Subject: [master] 1a989e2 make strings be strings even inside the #if 0 region Message-ID: commit 1a989e21eede0df420ffa6162fd15979a68ba82a Author: Nils Goroll Date: Tue Nov 1 07:54:09 2016 +0100 make strings be strings even inside the #if 0 region diff --git a/include/tbl/params.h b/include/tbl/params.h index d40aedd..7854f46 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1409,7 +1409,7 @@ PARAM( /* typ */ string, /* min */ NULL, /* max */ NULL, - /* default */ /opt/varnish/etc/varnish, + /* default */ "/opt/varnish/etc/varnish", /* units */ NULL, /* flags */ 0, /* s-text */ @@ -1425,7 +1425,7 @@ PARAM( /* typ */ string, /* min */ NULL, /* max */ NULL, - /* default */ /opt/varnish/lib/varnish/vmods, + /* default */ "/opt/varnish/lib/varnish/vmods", /* units */ NULL, /* flags */ 0, /* s-text */ @@ -1530,7 +1530,7 @@ PARAM( /* typ */ waiter, /* min */ NULL, /* max */ NULL, - /* default */ kqueue (possible values: kqueue, poll), + /* default */ "kqueue (possible values: kqueue, poll)", /* units */ NULL, /* flags */ MUST_RESTART| WIZARD, /* s-text */ From nils.goroll at uplex.de Mon Nov 14 12:14:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 14 Nov 2016 13:14:05 +0100 Subject: [master] 03d3f82 Check if events we received are actually being waited for Message-ID: commit 03d3f82a452846920863b53d4c67797a410385b2 Author: Nils Goroll Date: Thu Nov 3 12:11:54 2016 +0100 Check if events we received are actually being waited for For epoll, we tolerate spurious reports, for all other waiters we assert. fixes #2117 diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c index 80def71..f05a865 100644 --- a/bin/varnishd/waiter/cache_waiter.c +++ b/bin/varnishd/waiter/cache_waiter.c @@ -97,13 +97,15 @@ Wait_HeapInsert(const struct waiter *w, struct waited *wp) * XXX: any harm to come from it. Caveat Emptor. */ -void +int Wait_HeapDelete(const struct waiter *w, const struct waited *wp) { CHECK_OBJ_NOTNULL(w, WAITER_MAGIC); CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC); - if (wp->idx != BINHEAP_NOIDX) - binheap_delete(w->heap, wp->idx); + if (wp->idx == BINHEAP_NOIDX) + return (0); + binheap_delete(w->heap, wp->idx); + return (1); } double diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c index 0997979..6b6d39b 100644 --- a/bin/varnishd/waiter/cache_waiter_epoll.c +++ b/bin/varnishd/waiter/cache_waiter_epoll.c @@ -26,6 +26,8 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * + * Recommended reading: libev(3) "EVBACKEND_EPOLL" section + * - thank you, Marc Alexander Lehmann */ //lint -e{766} @@ -73,7 +75,7 @@ vwe_thread(void *priv) struct waited *wp; struct waiter *w; double now, then; - int i, n; + int i, n, active; struct vwe *vwe; char c; @@ -101,7 +103,7 @@ vwe_thread(void *priv) CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC); AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_DEL, wp->fd, NULL)); vwe->nwaited--; - Wait_HeapDelete(w, wp); + AN(Wait_HeapDelete(w, wp)); Lck_Unlock(&vwe->mtx); Wait_Call(w, wp, WAITER_TIMEOUT, now); } @@ -125,8 +127,12 @@ vwe_thread(void *priv) } CAST_OBJ_NOTNULL(wp, ep->data.ptr, WAITED_MAGIC); Lck_Lock(&vwe->mtx); - Wait_HeapDelete(w, wp); + active = Wait_HeapDelete(w, wp); Lck_Unlock(&vwe->mtx); + if (! active) { + VSL(SLT_Debug, wp->fd, "epoll: spurious event"); + continue; + } AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_DEL, wp->fd, NULL)); vwe->nwaited--; if (ep->events & EPOLLIN) diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c index e1c7642..42d5245 100644 --- a/bin/varnishd/waiter/cache_waiter_kqueue.c +++ b/bin/varnishd/waiter/cache_waiter_kqueue.c @@ -98,7 +98,7 @@ vwk_thread(void *priv) CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC); EV_SET(ke, wp->fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); AZ(kevent(vwk->kq, ke, 1, NULL, 0, NULL)); - Wait_HeapDelete(w, wp); + AN(Wait_HeapDelete(w, wp)); Lck_Unlock(&vwk->mtx); Wait_Call(w, wp, WAITER_TIMEOUT, now); } @@ -118,7 +118,7 @@ vwk_thread(void *priv) } CAST_OBJ_NOTNULL(wp, ke[j].udata, WAITED_MAGIC); Lck_Lock(&vwk->mtx); - Wait_HeapDelete(w, wp); + AN(Wait_HeapDelete(w, wp)); Lck_Unlock(&vwk->mtx); vwk->nwaited--; if (kp->flags & EV_EOF) diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c index 5e2edb8..ed3a39c 100644 --- a/bin/varnishd/waiter/cache_waiter_poll.c +++ b/bin/varnishd/waiter/cache_waiter_poll.c @@ -190,13 +190,13 @@ VSL(SLT_Debug, vwp->pollfd[i].fd, "POLL loop i=%d revents=0x%x", i, vwp->pollfd[ v--; then = Wait_When(wp); if (then <= now) { - Wait_HeapDelete(w, wp); + AN(Wait_HeapDelete(w, wp)); Wait_Call(w, wp, WAITER_TIMEOUT, now); vwp_del(vwp, i); } else if (vwp->pollfd[i].revents & POLLIN) { assert(wp->fd > 0); assert(wp->fd == vwp->pollfd[i].fd); - Wait_HeapDelete(w, wp); + AN(Wait_HeapDelete(w, wp)); Wait_Call(w, wp, WAITER_ACTION, now); vwp_del(vwp, i); } else { diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c index f208f8a..5b11be3 100644 --- a/bin/varnishd/waiter/cache_waiter_ports.c +++ b/bin/varnishd/waiter/cache_waiter_ports.c @@ -128,7 +128,7 @@ vws_port_ev(struct vws *vws, struct waiter *w, port_event_t *ev, double now) * threadID=129476&tstart=0 */ vws_del(vws, wp->fd); - Wait_HeapDelete(w, wp); + AN(Wait_HeapDelete(w, wp)); Wait_Call(w, wp, ev->portev_events & POLLERR ? WAITER_REMCLOSE : WAITER_ACTION, now); @@ -167,7 +167,7 @@ vws_thread(void *priv) } CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC); vws_del(vws, wp->fd); - Wait_HeapDelete(w, wp); + AN(Wait_HeapDelete(w, wp)); Wait_Call(w, wp, WAITER_TIMEOUT, now); } then = vws->next - now; diff --git a/bin/varnishd/waiter/waiter_priv.h b/bin/varnishd/waiter/waiter_priv.h index c7d1d25..6db06ca 100644 --- a/bin/varnishd/waiter/waiter_priv.h +++ b/bin/varnishd/waiter/waiter_priv.h @@ -78,5 +78,5 @@ Wait_When(const struct waited *wp) void Wait_Call(const struct waiter *, struct waited *, enum wait_event ev, double now); void Wait_HeapInsert(const struct waiter *, struct waited *); -void Wait_HeapDelete(const struct waiter *, const struct waited *); +int Wait_HeapDelete(const struct waiter *, const struct waited *); double Wait_HeapDue(const struct waiter *, struct waited **); From nils.goroll at uplex.de Mon Nov 14 12:58:04 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 14 Nov 2016 13:58:04 +0100 Subject: [master] 841d306 document that a 200 in vcl_deliver may turn into a 304 downstream Message-ID: commit 841d306903a67e7d57912ebb8fb44763913702f4 Author: Nils Goroll Date: Mon Nov 14 13:55:44 2016 +0100 document that a 200 in vcl_deliver may turn into a 304 downstream diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 0843ee5..37aa744 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -743,6 +743,10 @@ sp_variables = [ Assigning a HTTP standardized code to resp.status will also set resp.reason to the corresponding status message. + + resp.status 200 will get changed into 304 by core code after + a return(deliver) from vcl_deliver for conditional requests + to cached content if validation succeeds. """ ), ('resp.reason', From varnish-commit at varnish-cache.org Mon Nov 14 14:21:53 2016 From: varnish-commit at varnish-cache.org (xtjsrjsqn) Date: Mon, 14 Nov 2016 22:21:53 +0800 Subject: =?utf-8?B?dmFybmlzaC1jb21taXTvvJrkuIrmtbfkuqTlpKdF?= =?utf-8?B?TUJB5oC76KOB54+t77yM5qyi6L+O5oKo55qE5Y+C5Yqg77yBNTUzNQ==?= Message-ID: <20161114222159082806@jhzcpaazr.com> varnish-commit: ? ?? ???????????????????????????????????????????????EMBA??????????????????????????????????????????????????????????????????????????????????????? ??????????????????????????????????????????EMBA??????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? ?????EMBA???????????????????????????????????????????????????????????????????????????????????????????????????????????????????? ??????????????? 2016-11-1422:21:57 -------------- next part -------------- An HTML attachment was scrubbed... URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: ?????????EMBA????31??.docx Type: application/vnd.openxmlformats-officedocument.wordprocessingml.document Size: 75099 bytes Desc: not available URL: From phk at FreeBSD.org Tue Nov 15 15:25:06 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 15 Nov 2016 16:25:06 +0100 Subject: [master] e6e697c Don't assert perfect consistency during reset. Message-ID: commit e6e697cab3078af9ad1a93945982327d35f3103a Author: Poul-Henning Kamp Date: Tue Nov 15 15:23:32 2016 +0000 Don't assert perfect consistency during reset. diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index a5e5b47..4052770 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -417,10 +417,6 @@ cmd_barrier(CMD_ARGS) AZ(pthread_mutex_lock(&b->mtx)); switch (b->type) { case BARRIER_COND: - if (b->cyclic) - AZ(b->waiters); - else - assert(b->waiters == b->expected); break; case BARRIER_SOCK: b->active = 0; From phk at FreeBSD.org Tue Nov 15 15:25:06 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 15 Nov 2016 16:25:06 +0100 Subject: [master] 2487092 Make this test more robust. Message-ID: commit 24870927e94c2da9503e830d30d966382c7542f0 Author: Poul-Henning Kamp Date: Tue Nov 15 15:23:56 2016 +0000 Make this test more robust. diff --git a/bin/varnishtest/tests/r01927.vtc b/bin/varnishtest/tests/r01927.vtc index d1478c6..9286188 100644 --- a/bin/varnishtest/tests/r01927.vtc +++ b/bin/varnishtest/tests/r01927.vtc @@ -1,5 +1,7 @@ varnishtest "Test requests other than GET are cacheable" +barrier b1 cond 2 + server s1 { rxreq expect req.method == "POST" @@ -9,14 +11,22 @@ server s1 { expect req.method == "POST" expect req.body == "foo" txresp -body baz + barrier b1 sync } -start varnish v1 -vcl+backend { sub vcl_recv { # We ignore the actual body for this test. set req.http.method = req.method; + set req.http.hit = "No"; return (hash); } + sub vcl_hit { + set req.http.hit = "Yes"; + } + sub vcl_deliver { + set resp.http.hit = req.http.hit; + } sub vcl_backend_fetch { set bereq.method = bereq.http.method; } @@ -29,14 +39,27 @@ client c1 { txreq -req "POST" -body "foo" rxresp expect resp.body == "bar" + expect resp.http.hit == "No" + txreq -req "POST" -body "foo" rxresp expect resp.body == "bar" - delay 0.5 + expect resp.http.hit == "Yes" + + # Wait until between ttl&grace + delay 1.0 + + # Trigger bg fetch txreq -req "POST" -body "foo" rxresp expect resp.body == "bar" + expect resp.http.hit == "Yes" + + barrier b1 sync + + # Get new object, from cache txreq -req "POST" -body "foo" rxresp expect resp.body == "baz" + expect resp.http.hit == "Yes" } -run From phk at FreeBSD.org Wed Nov 16 12:32:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 16 Nov 2016 13:32:05 +0100 Subject: [master] 918cd81 Don't start a subprocess just to kill our own child process. Message-ID: commit 918cd814e343dfe99aa949524f2a4b3647133024 Author: Poul-Henning Kamp Date: Wed Nov 16 12:31:01 2016 +0000 Don't start a subprocess just to kill our own child process. diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index 3dba551..dd553c9 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -247,8 +247,7 @@ process_run(struct process *p) static void process_kill(const struct process *p, const char *sig) { - int s; - char buf[64]; + int j; CHECK_OBJ_NOTNULL(p, PROCESS_MAGIC); AN(sig); @@ -256,12 +255,19 @@ process_kill(const struct process *p, const char *sig) if (!p->running || !p->pid) vtc_log(p->vl, 0, "Cannot signal a non-running process"); - bprintf(buf, "kill -%s -%d", sig, p->pid); - vtc_log(p->vl, 4, "CMD: %s", buf); - - s = system(buf); - if (s != 0) - vtc_log(p->vl, 0, "Failed to send signal (exit status: %d)", s); + if (!strcmp(sig, "TERM")) + j = SIGTERM; + else if (!strcmp(sig, "HUP")) + j = SIGHUP; + else if (!strcmp(sig, "KILL")) + j = SIGKILL; + else + j = strtoul(sig, NULL, 10); + + if (kill(p->pid, j) < 0) + vtc_log(p->vl, 0, "Failed to send signal %d (%s)", j, strerror(errno)); + else + vtc_log(p->vl, 4, "Sent signal %d", j); } static inline void From dridi at varni.sh Wed Nov 16 12:41:05 2016 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 16 Nov 2016 13:41:05 +0100 Subject: [master] 918cd81 Don't start a subprocess just to kill our own child process. In-Reply-To: References: Message-ID: On Wed, Nov 16, 2016 at 1:32 PM, Poul-Henning Kamp wrote: > > commit 918cd814e343dfe99aa949524f2a4b3647133024 > Author: Poul-Henning Kamp > Date: Wed Nov 16 12:31:01 2016 +0000 > > Don't start a subprocess just to kill our own child process. > > diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c > index 3dba551..dd553c9 100644 > --- a/bin/varnishtest/vtc_process.c > +++ b/bin/varnishtest/vtc_process.c > @@ -247,8 +247,7 @@ process_run(struct process *p) > static void > process_kill(const struct process *p, const char *sig) > { > - int s; > - char buf[64]; > + int j; > > CHECK_OBJ_NOTNULL(p, PROCESS_MAGIC); > AN(sig); > @@ -256,12 +255,19 @@ process_kill(const struct process *p, const char *sig) > if (!p->running || !p->pid) > vtc_log(p->vl, 0, "Cannot signal a non-running process"); > > - bprintf(buf, "kill -%s -%d", sig, p->pid); > - vtc_log(p->vl, 4, "CMD: %s", buf); > - > - s = system(buf); > - if (s != 0) > - vtc_log(p->vl, 0, "Failed to send signal (exit status: %d)", s); > + if (!strcmp(sig, "TERM")) > + j = SIGTERM; > + else if (!strcmp(sig, "HUP")) > + j = SIGHUP; > + else if (!strcmp(sig, "KILL")) > + j = SIGKILL; > + else > + j = strtoul(sig, NULL, 10); Hi, This had been discussed during the review of background processes in varnishtest. The consensus was to let the kill(1) command do the signal parsing for better portability. Since this patch is restricting the signal spec to 3 strings or a number, either revert this change or update the documentation too. Cheers, Dridi > + > + if (kill(p->pid, j) < 0) > + vtc_log(p->vl, 0, "Failed to send signal %d (%s)", j, strerror(errno)); > + else > + vtc_log(p->vl, 4, "Sent signal %d", j); > } > > static inline void > > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From hermunn at varnish-software.com Wed Nov 16 12:55:05 2016 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Wed, 16 Nov 2016 13:55:05 +0100 Subject: [4.1] c855794 Include the current time of the panic in the panic output Message-ID: commit c855794616fd2f3bd74a250cefcb88a8ac3415f9 Author: Martin Blix Grydeland Date: Thu Oct 27 14:02:43 2016 +0200 Include the current time of the panic in the panic output Show both the real and the monotonic time in the panic output when the panic is created. This is useful to have something to compare other panic timestamps against. The existing timestamp in the panic output is recorded by the management process at the time of receiving the panic from the child process. Due to processing and possible core dump creation, this time isn't very useful for comparison purposes. diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index c475605..589f9f5 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -39,6 +39,8 @@ #include #include +#include "vtim.h" + #include "cache.h" #include "cache_filter.h" @@ -526,6 +528,8 @@ pan_ic(const char *func, const char *file, int line, const char *cond, VSB_printf(pan_vsb, "version = %s\n", VCS_version); VSB_printf(pan_vsb, "ident = %s,%s\n", VSB_data(vident) + 1, Waiter_GetName()); + VSB_printf(pan_vsb, "now = %f (mono), %f (real)\n", + VTIM_mono(), VTIM_real()); pan_backtrace(pan_vsb); From hermunn at varnish-software.com Wed Nov 16 12:55:05 2016 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Wed, 16 Nov 2016 13:55:05 +0100 Subject: [4.1] 5ed4b4b Introduce a parameter value for allowed clock step Message-ID: commit 5ed4b4bee3e2f76d3e7d8e27cb06bce2eb3713d8 Author: Martin Blix Grydeland Date: Thu Oct 27 14:44:21 2016 +0200 Introduce a parameter value for allowed clock step This value is used in calculations that could reveal clock step, specifying the amount of error we allow before panicking. It defaults to a conservative 1 second. diff --git a/include/tbl/params.h b/include/tbl/params.h index d1da313..a5157ee 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -290,6 +290,21 @@ PARAM( ) PARAM( + /* name */ clock_step, + /* typ */ timeout, + /* min */ "0.000", + /* max */ NULL, + /* default */ "1.000", + /* units */ "seconds", + /* flags */ 0, + /* s-text */ + "How much observed clock step we are willing to accept before " + "we panic.", + /* l-text */ "", + /* func */ NULL +) + +PARAM( /* name */ connect_timeout, /* typ */ timeout, /* min */ "0.000", From hermunn at varnish-software.com Wed Nov 16 12:55:05 2016 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Wed, 16 Nov 2016 13:55:05 +0100 Subject: [4.1] 4f58bb8 Allow up to clock_step error when checking the session timestamps Message-ID: commit 4f58bb832a65dce60cedbeccd07cae917b58361f Author: Martin Blix Grydeland Date: Thu Oct 27 14:59:17 2016 +0200 Allow up to clock_step error when checking the session timestamps When checking the timestamps on deleting a session, allow up to clock_step seconds error before bailing. Fixes: #1874 diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index b5be7ea..0e2a746 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -537,11 +537,15 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now) if (isnan(now)) now = VTIM_real(); AZ(isnan(sp->t_open)); + if (now < sp->t_open) { + if (now + cache_param->clock_step < sp->t_open) + WRONG("Clock step detected"); + now = sp->t_open; /* Do not log negatives */ + } if (reason == SC_NULL) reason = (enum sess_close)-sp->fd; - assert(now >= sp->t_open); assert(VTAILQ_EMPTY(&sp->privs->privs)); VSL(SLT_SessClose, sp->vxid, "%s %.3f", sess_close_2str(reason, 0), now - sp->t_open); From phk at phk.freebsd.dk Wed Nov 16 21:01:07 2016 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Wed, 16 Nov 2016 21:01:07 +0000 Subject: [master] 918cd81 Don't start a subprocess just to kill our own child process. In-Reply-To: References: Message-ID: <81248.1479330067@critter.freebsd.dk> -------- In message , Dridi Boukelmoune writes: >Since this patch is restricting the signal spec to 3 strings or a >number, either revert this change or update the documentation too. On my TODO list. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi.boukelmoune at gmail.com Thu Nov 17 09:46:04 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 17 Nov 2016 10:46:04 +0100 Subject: [4.1] 055551d Reorganize the syntax description in varnish-cli(7) Message-ID: commit 055551d091f83a9ecc65f18dfc3cea40497304e6 Author: Dridi Boukelmoune Date: Tue Nov 8 15:21:57 2016 +0100 Reorganize the syntax description in varnish-cli(7) Ramp up from simple tokens to here documents. Conflicts: doc/sphinx/reference/varnish-cli.rst diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index 7dcc9a4..3e785d4 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -50,30 +50,49 @@ that service. Please see :ref:`varnishd(1)` for details. Syntax ------ -Commands are usually terminated with a newline. Long command can be -entered using sh style *here documents*. The format of here-documents -is:: +The Varnish CLI is similar to another command line interface, the Bourne +Shell. Commands are usually terminated with a newline, and they may take +arguments. The command and its arguments are *tokenized* before parsing, +and as such arguments containing must must be enclosed in double quotes. + +It means that command parsing of + +:: + + help banner + +is equivalent to + +:: + + "help" banner + +because the double quotes only indicate the boundaries of the ``help`` +token. + +Within double quotes you can escape characters with \\ (backslash). The \\n, +\\r, and \\t get translated to newlines, carriage returns, an tabs. Double +quotes and backslashes themselves can be escaped with \\" and \\\\ +respectively. + +To enter characters in octals use the \\nnn syntax. Hexadecimals can +be entered with the \\xnn syntax. + +Commands may not end with a newline when a shell-style *here document* +(here-document or heredoc) is used. The format of a here document is:: << word here document word -*word* can be any continuous string chosen to make sure it doesn't -appear naturally in the following *here document*. +*word* can be any continuous string chosen to make sure it doesn't appear +naturally in the following *here document*. Traditionally EOF or END is +used. When using the here document style of input there are no restrictions on length. When using newline-terminated commands maximum length is limited by the varnishd parameter *cli_buffer*. -When commands are newline terminated they get *tokenized* before -parsing so if you have significant spaces enclose your strings in -double quotes. Within the quotes you can escape characters with -\\. The \n, \r and \t get translated to newlines, carriage returns and -tabs. Double quotes themselves can be escaped with a backslash. - -To enter characters in octals use the \\nnn syntax. Hexadecimals can -be entered with the \\xnn syntax. - Commands -------- From dridi.boukelmoune at gmail.com Thu Nov 17 09:46:04 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 17 Nov 2016 10:46:04 +0100 Subject: [4.1] f1bc4e8 Explain what may go wrong with quoting and the CLI Message-ID: commit f1bc4e88751070fc60033535c2e1ab1cb936fd99 Author: Dridi Boukelmoune Date: Tue Nov 8 15:43:06 2016 +0100 Explain what may go wrong with quoting and the CLI Fixes #1899 Closes #2110 diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index 3e785d4..02c86a4 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -93,6 +93,86 @@ When using the here document style of input there are no restrictions on length. When using newline-terminated commands maximum length is limited by the varnishd parameter *cli_buffer*. +Quoting pitfalls +---------------- + +Integrating with the Varnish CLI can be sometimes surprising when quoting +is involved. For instance in Bourne Shell the delimiter used with here +documents may or may not be separated by spaces from the ``<<`` token:: + + cat <' Line 1 Pos 1) + < commit feac030170411d2fb33c4c4a33c4c6cbf52099b9 Author: Poul-Henning Kamp Date: Thu Nov 17 22:07:36 2016 +0000 Don't use SIGHUP, it fails if varnishtest is run under nohup(1) We recognize only the most famous signals by name SIGTERM, SIGINT and SIGKILL, if you want a special signal you will have to use "-15" or whatever. The primary reason for this is that there is still no portable API for translating "SIGXXX" to an integer. It used to be that bin/kill.c contained an array: const char *const sys_signame[NSIG] = { [0] = "Signal 0", [SIGHUP] = "HUP", [SIGINT] = "INT", [SIGQUIT] = "QUIT", [SIGILL] = "ILL", [...] BSD unix sensibly moved this into libc, to avoid duplicating this all over the place, but Linux has not done that. The right way to do this, would have been to have contain a table: #ifdef SIGNAL_DOC(n,s,l) SIGNAL_DOC(1, "HUP", "Hangup") SIGNAL_DOC(2, "INT", "Interrupt") ... #endif That way nobody would ever need another #ifdef SIGFOO. Anyway... Rather than pointlessly add a semi-complete list of signals no sensible person should ever use in a varnishtest (SIGWINCH anybody ?) we support the famous three by name, and the rest by number. diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc index abe5f0e..1f3bb59 100644 --- a/bin/varnishtest/tests/u00000.vtc +++ b/bin/varnishtest/tests/u00000.vtc @@ -16,7 +16,7 @@ delay 0.5 # stop process p1 -stop process p2 -close -process p3 -kill "HUP" +process p3 -kill "INT" # wait process p1 -wait diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index dd553c9..f2fb1c8 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -247,7 +247,7 @@ process_run(struct process *p) static void process_kill(const struct process *p, const char *sig) { - int j; + int j = 0; CHECK_OBJ_NOTNULL(p, PROCESS_MAGIC); AN(sig); @@ -257,12 +257,14 @@ process_kill(const struct process *p, const char *sig) if (!strcmp(sig, "TERM")) j = SIGTERM; - else if (!strcmp(sig, "HUP")) - j = SIGHUP; + else if (!strcmp(sig, "INT")) + j = SIGINT; else if (!strcmp(sig, "KILL")) j = SIGKILL; + else if (*sig == '-') + j = strtoul(sig + 1, NULL, 10); else - j = strtoul(sig, NULL, 10); + vtc_log(p->vl, 0, "Could not grok signal (%s)", sig); if (kill(p->pid, j) < 0) vtc_log(p->vl, 0, "Failed to send signal %d (%s)", j, strerror(errno)); From fgsch at lodoss.net Fri Nov 18 05:46:05 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 18 Nov 2016 06:46:05 +0100 Subject: [master] cb80bb2 Fix make check Message-ID: commit cb80bb27eb796e1cd1ddfafb540157ac155b06a0 Author: Federico G. Schwindt Date: Fri Nov 18 05:17:09 2016 +0000 Fix make check diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc index 1f3bb59..e42afd8 100644 --- a/bin/varnishtest/tests/u00000.vtc +++ b/bin/varnishtest/tests/u00000.vtc @@ -16,7 +16,7 @@ delay 0.5 # stop process p1 -stop process p2 -close -process p3 -kill "INT" +process p3 -kill -1 # wait process p1 -wait From fgsch at lodoss.net Fri Nov 18 05:46:05 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 18 Nov 2016 06:46:05 +0100 Subject: [master] 3b0b3b7 Document workuser and minor improvements Message-ID: commit 3b0b3b729d3b510b3eea434fab5ebe9ce03b54a8 Author: Federico G. Schwindt Date: Fri Nov 18 05:10:35 2016 +0000 Document workuser and minor improvements diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 0b31a48..aff1f72 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -72,8 +72,8 @@ OPTIONS -h - Specifies the hash algorithm. See `Hash Algorithm Options`_ for a - list of supported algorithms. + Specifies the hash algorithm. See `Hash Algorithm`_ section for a list + of supported algorithms. -i identity @@ -82,7 +82,7 @@ OPTIONS -j - Specify the jailing technology to use. + Specify the jailing mechanism to use. See `Jail`_ section. -l @@ -131,7 +131,7 @@ OPTIONS -s <[name=]type[,options]> - Use the specified storage backend, see `Storage Backend Options`_. + Use the specified storage backend. See `Storage Backend`_ section. This option can be used multiple times to specify multiple storage files. Names are referenced in logs, VCL, statistics, etc. @@ -157,8 +157,8 @@ OPTIONS .. _opt_h: -Hash Algorithm Options ----------------------- +Hash Algorithm +-------------- The following hash algorithms are available: @@ -184,8 +184,8 @@ The following hash algorithms are available: .. _ref-varnishd-opt_s: -Storage Backend Options ------------------------ +Storage Backend +--------------- The following storage types are available: @@ -219,8 +219,8 @@ The following storage types are available: .. _ref-varnishd-opt_j: -Jail Options ------------- +Jail +---- Varnish jails are a generalization over various platform specific methods to reduce the privileges of varnish processes. They may have @@ -232,24 +232,27 @@ specific options. Available jails are: required set. Only available on platforms which have the setppriv(2) call. --j +-j - Default on all other platforms if `varnishd` is either started with + Default on all other platforms when `varnishd` is either started with an effective uid of 0 ("as root") or as user ``varnish``. - With the ``unix`` jail technology activated, varnish will switch to + With the ``unix`` jail mechanism activated, varnish will switch to an alternative user for subprocesses and change the effective uid of the master process whenever possible. The optional `user` argument specifies which alternative user to - use. It defaults to ``varnish`` + use. It defaults to ``varnish``. The optional `ccgroup` argument specifies a group to add to varnish subprocesses requiring access to a c-compiler. There is no default. + The optional `workuser` argument specifies an alternative user to use + for the worker process. It defaults to ``vcache``. + -j none - last resort jail choice: With jail technology ``none``, varnish will + last resort jail choice: With jail mechanism ``none``, varnish will run all processes with the privileges it was started with. From hermunn at varnish-software.com Fri Nov 18 09:08:05 2016 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Fri, 18 Nov 2016 10:08:05 +0100 Subject: [4.1] 49ce7f8 Check if events we received are actually being waited for Message-ID: commit 49ce7f87d310f00186a7c44215cc90102970c106 Author: Nils Goroll Date: Thu Nov 3 12:11:54 2016 +0100 Check if events we received are actually being waited for For epoll, we tolerate spurious reports, for all other waiters we assert. fixes #2117 Conflicts: bin/varnishd/waiter/cache_waiter_epoll.c diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c index b1d0801..3e6e632 100644 --- a/bin/varnishd/waiter/cache_waiter.c +++ b/bin/varnishd/waiter/cache_waiter.c @@ -97,13 +97,15 @@ Wait_HeapInsert(const struct waiter *w, struct waited *wp) * XXX: any harm to come from it. Caveat Emptor. */ -void +int Wait_HeapDelete(const struct waiter *w, const struct waited *wp) { CHECK_OBJ_NOTNULL(w, WAITER_MAGIC); CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC); - if (wp->idx != BINHEAP_NOIDX) - binheap_delete(w->heap, wp->idx); + if (wp->idx == BINHEAP_NOIDX) + return (0); + binheap_delete(w->heap, wp->idx); + return (1); } double diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c index f50ae46..68afcb2 100644 --- a/bin/varnishd/waiter/cache_waiter_epoll.c +++ b/bin/varnishd/waiter/cache_waiter_epoll.c @@ -29,6 +29,9 @@ * XXX: We need to pass sessions back into the event engine when they are * reused. Not sure what the most efficient way is for that. For now * write the session pointer to a pipe which the event engine monitors. + * + * Recommended reading: libev(3) "EVBACKEND_EPOLL" section + * - thank you, Marc Alexander Lehmann */ #include "config.h" @@ -75,7 +78,7 @@ vwe_thread(void *priv) struct waited *wp; struct waiter *w; double now, then; - int i, n; + int i, n, active; struct vwe *vwe; char c; @@ -103,7 +106,7 @@ vwe_thread(void *priv) CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC); AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_DEL, wp->fd, NULL)); vwe->nwaited--; - Wait_HeapDelete(w, wp); + AN(Wait_HeapDelete(w, wp)); Lck_Unlock(&vwe->mtx); Wait_Call(w, wp, WAITER_TIMEOUT, now); } @@ -127,8 +130,12 @@ vwe_thread(void *priv) } CAST_OBJ_NOTNULL(wp, ep->data.ptr, WAITED_MAGIC); Lck_Lock(&vwe->mtx); - Wait_HeapDelete(w, wp); + active = Wait_HeapDelete(w, wp); Lck_Unlock(&vwe->mtx); + if (! active) { + VSL(SLT_Debug, wp->fd, "epoll: spurious event"); + continue; + } AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_DEL, wp->fd, NULL)); vwe->nwaited--; if (ep->events & EPOLLIN) diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c index 02d7adc..5518875 100644 --- a/bin/varnishd/waiter/cache_waiter_kqueue.c +++ b/bin/varnishd/waiter/cache_waiter_kqueue.c @@ -97,7 +97,7 @@ vwk_thread(void *priv) CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC); EV_SET(ke, wp->fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); AZ(kevent(vwk->kq, ke, 1, NULL, 0, NULL)); - Wait_HeapDelete(w, wp); + AN(Wait_HeapDelete(w, wp)); Lck_Unlock(&vwk->mtx); Wait_Call(w, wp, WAITER_TIMEOUT, now); } @@ -117,7 +117,7 @@ vwk_thread(void *priv) } CAST_OBJ_NOTNULL(wp, ke[j].udata, WAITED_MAGIC); Lck_Lock(&vwk->mtx); - Wait_HeapDelete(w, wp); + AN(Wait_HeapDelete(w, wp)); Lck_Unlock(&vwk->mtx); vwk->nwaited--; if (kp->flags & EV_EOF) diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c index 5e2edb8..ed3a39c 100644 --- a/bin/varnishd/waiter/cache_waiter_poll.c +++ b/bin/varnishd/waiter/cache_waiter_poll.c @@ -190,13 +190,13 @@ VSL(SLT_Debug, vwp->pollfd[i].fd, "POLL loop i=%d revents=0x%x", i, vwp->pollfd[ v--; then = Wait_When(wp); if (then <= now) { - Wait_HeapDelete(w, wp); + AN(Wait_HeapDelete(w, wp)); Wait_Call(w, wp, WAITER_TIMEOUT, now); vwp_del(vwp, i); } else if (vwp->pollfd[i].revents & POLLIN) { assert(wp->fd > 0); assert(wp->fd == vwp->pollfd[i].fd); - Wait_HeapDelete(w, wp); + AN(Wait_HeapDelete(w, wp)); Wait_Call(w, wp, WAITER_ACTION, now); vwp_del(vwp, i); } else { diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c index 0850441..705e442 100644 --- a/bin/varnishd/waiter/cache_waiter_ports.c +++ b/bin/varnishd/waiter/cache_waiter_ports.c @@ -126,7 +126,7 @@ vws_port_ev(struct vws *vws, struct waiter *w, port_event_t *ev, double now) { * threadID=129476&tstart=0 */ vws_del(vws, wp->fd); - Wait_HeapDelete(w, wp); + AN(Wait_HeapDelete(w, wp)); Wait_Call(w, wp, ev->portev_events & POLLERR ? WAITER_REMCLOSE : WAITER_ACTION, now); @@ -165,7 +165,7 @@ vws_thread(void *priv) } CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC); vws_del(vws, wp->fd); - Wait_HeapDelete(w, wp); + AN(Wait_HeapDelete(w, wp)); Wait_Call(w, wp, WAITER_TIMEOUT, now); } then = vws->next - now; diff --git a/bin/varnishd/waiter/waiter_priv.h b/bin/varnishd/waiter/waiter_priv.h index c7d1d25..6db06ca 100644 --- a/bin/varnishd/waiter/waiter_priv.h +++ b/bin/varnishd/waiter/waiter_priv.h @@ -78,5 +78,5 @@ Wait_When(const struct waited *wp) void Wait_Call(const struct waiter *, struct waited *, enum wait_event ev, double now); void Wait_HeapInsert(const struct waiter *, struct waited *); -void Wait_HeapDelete(const struct waiter *, const struct waited *); +int Wait_HeapDelete(const struct waiter *, const struct waited *); double Wait_HeapDue(const struct waiter *, struct waited **); From phk at FreeBSD.org Fri Nov 18 09:22:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 18 Nov 2016 10:22:05 +0100 Subject: [master] a3d1721 Revert to SIGINT, our commits must have crossed each other. Message-ID: commit a3d1721c879c9df54e8bb7e60ae8c233a9e3f8c6 Author: Poul-Henning Kamp Date: Fri Nov 18 09:20:46 2016 +0000 Revert to SIGINT, our commits must have crossed each other. diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc index e42afd8..57308c7 100644 --- a/bin/varnishtest/tests/u00000.vtc +++ b/bin/varnishtest/tests/u00000.vtc @@ -16,7 +16,7 @@ delay 0.5 # stop process p1 -stop process p2 -close -process p3 -kill -1 +process p3 -kill INT # wait process p1 -wait From phk at FreeBSD.org Fri Nov 18 09:57:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 18 Nov 2016 10:57:05 +0100 Subject: [master] 3efd809 OK, for reasons I don't understand, SIGINT does not work here. Message-ID: commit 3efd809dc844a64d9d609d678cf6973667aa88f7 Author: Poul-Henning Kamp Date: Fri Nov 18 09:56:10 2016 +0000 OK, for reasons I don't understand, SIGINT does not work here. Use SIGKILL which is sure to work. diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc index 57308c7..cba95c0 100644 --- a/bin/varnishtest/tests/u00000.vtc +++ b/bin/varnishtest/tests/u00000.vtc @@ -16,7 +16,7 @@ delay 0.5 # stop process p1 -stop process p2 -close -process p3 -kill INT +process p3 -kill KILL # wait process p1 -wait From phk at FreeBSD.org Fri Nov 18 11:10:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 18 Nov 2016 12:10:05 +0100 Subject: [master] 520de7f Re-add a detail I missed: Kill the process group instead of just the direct child process. Message-ID: commit 520de7f7cd4ed52af930314a21f7c2730afd6a18 Author: Poul-Henning Kamp Date: Fri Nov 18 11:08:44 2016 +0000 Re-add a detail I missed: Kill the process group instead of just the direct child process. diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index f2fb1c8..effb25c 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -266,7 +266,7 @@ process_kill(const struct process *p, const char *sig) else vtc_log(p->vl, 0, "Could not grok signal (%s)", sig); - if (kill(p->pid, j) < 0) + if (kill(-p->pid, j) < 0) vtc_log(p->vl, 0, "Failed to send signal %d (%s)", j, strerror(errno)); else vtc_log(p->vl, 4, "Sent signal %d", j); From phk at FreeBSD.org Fri Nov 18 11:12:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 18 Nov 2016 12:12:04 +0100 Subject: [master] db4a524 Add the vtest.sh script which is the client side of our new homebuilt CI setup. Message-ID: commit db4a5243235a5d45738ccc0adf40fe0772b77d0c Author: Poul-Henning Kamp Date: Fri Nov 18 11:10:45 2016 +0000 Add the vtest.sh script which is the client side of our new homebuilt CI setup. diff --git a/tools/vtest.sh b/tools/vtest.sh new file mode 100644 index 0000000..d61ec33 --- /dev/null +++ b/tools/vtest.sh @@ -0,0 +1,169 @@ +#!/bin/sh +# +# Copyright (c) 2006-2016 Varnish Software AS +# All rights reserved. +# +# Author: Poul-Henning Kamp +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +set -e + +####################################################################### +# Parameters + +export MAKEFLAGS=-j2 + +export TMPDIR=`pwd`/tmp +mkdir -p tmp + +# Contact email of responsible person +CONTACT=phk at varnish-cache.org + +WAITPERIOD=60 # unit: Seconds + +WAITGOOD=60 # unit: WAITPERIOD +WAITBAD=1 # unit: WAITPERIOD + +####################################################################### + +if [ ! -d varnish-cache ] ; then + git clone \ + https://github.com/varnishcache/varnish-cache.git \ + varnish-cache +fi + +if [ ! -f vt_key.pub ] ; then + ssh-keygen -t ed25519 -N "" -f vt_key +fi + +autogen () ( + set -e + cd varnish-cache + nice make distclean > /dev/null 2>&1 || true + nice /usr/bin/time sh autogen.des +) + +makedistcheck () ( + set -e + cd varnish-cache + nice /usr/bin/time make distcheck +) + +failedtests () ( + for t in `grep '^FAIL: tests/' ${1} | sort -u | sed 's/.* //'` + do + echo -n "VTCGITREV ${t} " + ( + cd varnish-cache/bin/varnishtest/ + git log -n 1 ${t} | head -1 + ) + b=`basename ${t} .vtc` + for i in `find varnish-cache -name ${b}.log -print` + do + if [ -f ${i} ] ; then + mv ${i} "_report/_${b}.log" + echo "MANIFEST _${b}.log" >> ${LOG} + fi + done + done +) + +pack () ( + cd _report + tar czf - _log \ + `grep '^MANIFEST ' _log | sort -u | sed 's/^MANIFEST *//'` \ +) + +submit () ( + ssh \ + -T \ + -o StrictHostKeyChecking=no \ + -o PasswordAuthentication=no \ + -o NumberOfPasswordPrompts=0 \ + -i vt_key \ + -p 203 \ + vtest at varnish-cache.org < ${1} +) + +rm -f _report.tgz +touch _report.tgz +if ! submit _report.tgz; then + echo "Test submit failed" + echo + echo "You probably need to email this VTEST specific ssh-key" + echo "to phk at varnish-cache.org" + echo + sed 's/^/ /' vt_key.pub + echo + exit 2 +fi + +orev=000 +waitnext=${WAITBAD} + +while true +do + (cd varnish-cache && git pull > /dev/null 2>&1 || true) + rev=`cd varnish-cache && git show -s --pretty=format:%H` + if [ "${waitnext}" -gt 0 -a "x${rev}" = "x${orev}" ] ; then + sleep ${WAITPERIOD} + waitnext=`expr ${waitnext} - 1 || true` + continue + fi + waitnext=${WAITBAD} + orev=${rev} + + rm -rf _report + mkdir _report + export LOG=_report/_log + + echo "VTEST 1.01" > ${LOG} + echo "DATE `date +%s`" >> ${LOG} + echo "BRANCH trunk" >> ${LOG} + echo "HOST `hostname`" >> ${LOG} + echo "UNAME `uname -a`" >> ${LOG} + if [ -x /usr/bin/lsb_release ] ; then + echo "LSB `lsb_release -d`" >> ${LOG} + else + echo "LSB none" >> ${LOG} + fi + echo "GITREV $rev" >> ${LOG} + if ! autogen >> _report/_autogen 2>&1 ; then + echo "AUTOGEN BAD" >> ${LOG} + echo "MANIFEST _autogen" >> ${LOG} + else + echo "AUTOGEN GOOD" >> ${LOG} + if ! makedistcheck >> _report/_makedistcheck 2>&1 ; then + echo "MAKEDISTCHECK BAD" >> ${LOG} + echo "MANIFEST _autogen" >> ${LOG} + echo "MANIFEST _makedistcheck" >> ${LOG} + failedtests _report/_makedistcheck >> ${LOG} + else + echo "MAKEDISTCHECK GOOD" >> ${LOG} + waitnext=${WAITGOOD} + fi + fi + echo "VTEST END" >> ${LOG} + pack > _report.tgz + submit _report.tgz +done From dridi.boukelmoune at gmail.com Fri Nov 18 11:19:05 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 18 Nov 2016 12:19:05 +0100 Subject: [master] a535210 Let's avoid embarrassing commits Message-ID: commit a5352109a0ea09dc4444bd141e555def56c47d61 Author: Dridi Boukelmoune Date: Fri Nov 18 12:17:49 2016 +0100 Let's avoid embarrassing commits diff --git a/.gitignore b/.gitignore index 27b5ec7..5d9e107 100644 --- a/.gitignore +++ b/.gitignore @@ -110,6 +110,11 @@ cscope.*out /bin/varnishtest/tests/*.log-t /bin/varnishtest/test-suite.log +# vtest.sh droppings +tmp/ +vt_key +vt_key.pub + # Coverity output /cov-int /myproject.tgz From hermunn at varnish-software.com Fri Nov 18 12:44:04 2016 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Fri, 18 Nov 2016 13:44:04 +0100 Subject: [4.1] d5cf934 thread_pool_stack: better documentation, not experimental but delayed Message-ID: commit d5cf934f3bdeffce6cd38cb927fabf333b20abb0 Author: Nils Goroll Date: Wed Nov 9 16:27:44 2016 +0100 thread_pool_stack: better documentation, not experimental but delayed doc-fixes #2129 diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index 54407d0..f5a9cf4 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -199,8 +199,39 @@ struct parspec WRK_parspec[] = { NULL, NULL, "Worker thread stack size.\n" "This will likely be rounded up to a multiple of 4k" - " (or whatever the page_size might be) by the kernel.", - EXPERIMENTAL, + " (or whatever the page_size might be) by the kernel.\n" + "\n" + "The required stack size is primarily driven by the" + " depth of the call-tree. The most common relevant" + " determining factors in varnish core code are GZIP" + " (un)compression, ESI processing and regular" + " expression matches. VMODs may also require" + " significant amounts of additional stack. The" + " nesting depth of VCL subs is another factor," + " although typically not predominant.\n" + "\n" + "The stack size is per thread, so the maximum total" + " memory required for worker thread stacks is in the" + " order of size = thread_pools x thread_pool_max x" + " thread_pool_stack.\n" + "\n" + "Thus, in particular for setups with many threads," + " keeping the stack size at a minimum helps reduce" + " the amount of memory required by Varnish.\n" + "\n" + "On the other hand, thread_pool_stack must be large" + " enough under all circumstances, otherwise varnish" + " will crash due to a stack overflow. Usually, a" + " stack overflow manifests itself as a segmentation" + " fault (aka segfault / SIGSEGV) with the faulting" + " address being near the stack pointer (sp).\n" + "\n" + "Unless stack usage can be reduced," + " thread_pool_stack must be increased when a stack" + " overflow occurs. Setting it in 150%-200%" + " increments is recommended until stack overflows" + " cease to occur.", + DELAYED_EFFECT, NULL, "bytes" }, // default set in mgt_main.c { NULL, NULL, NULL } }; From phk at FreeBSD.org Fri Nov 18 13:35:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 18 Nov 2016 14:35:05 +0100 Subject: [master] 9bc12a0 /usr/bin/time is not portable. Message-ID: commit 9bc12a0bd2d749a829a8baaa89bd7fff1bd74887 Author: Poul-Henning Kamp Date: Fri Nov 18 13:33:54 2016 +0000 /usr/bin/time is not portable. diff --git a/tools/vtest.sh b/tools/vtest.sh index d61ec33..d5f43f7 100644 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -60,13 +60,13 @@ autogen () ( set -e cd varnish-cache nice make distclean > /dev/null 2>&1 || true - nice /usr/bin/time sh autogen.des + nice sh autogen.des ) makedistcheck () ( set -e cd varnish-cache - nice /usr/bin/time make distcheck + nice make distcheck ) failedtests () ( From dridi.boukelmoune at gmail.com Fri Nov 18 15:31:05 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 18 Nov 2016 16:31:05 +0100 Subject: [master] 346d543 Whitespace OCD Message-ID: commit 346d543be36ad1b6ec382857b6b06367b3eaf2d5 Author: Dridi Boukelmoune Date: Fri Nov 18 15:56:02 2016 +0100 Whitespace OCD diff --git a/tools/vtest.sh b/tools/vtest.sh index d5f43f7..cb1b5b2 100644 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -4,7 +4,7 @@ # All rights reserved. # # Author: Poul-Henning Kamp -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: @@ -13,7 +13,7 @@ # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. -# +# # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -53,7 +53,7 @@ if [ ! -d varnish-cache ] ; then fi if [ ! -f vt_key.pub ] ; then - ssh-keygen -t ed25519 -N "" -f vt_key + ssh-keygen -t ed25519 -N "" -f vt_key fi autogen () ( @@ -136,7 +136,7 @@ do rm -rf _report mkdir _report export LOG=_report/_log - + echo "VTEST 1.01" > ${LOG} echo "DATE `date +%s`" >> ${LOG} echo "BRANCH trunk" >> ${LOG} From dridi.boukelmoune at gmail.com Fri Nov 18 15:31:05 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 18 Nov 2016 16:31:05 +0100 Subject: [master] 2ebf30f Make MAKEFLAGS override-able Message-ID: commit 2ebf30f4030c3e77c4838080f5184bb9cb137308 Author: Dridi Boukelmoune Date: Fri Nov 18 15:56:29 2016 +0100 Make MAKEFLAGS override-able diff --git a/tools/vtest.sh b/tools/vtest.sh index cb1b5b2..33152e4 100644 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -31,7 +31,7 @@ set -e ####################################################################### # Parameters -export MAKEFLAGS=-j2 +export MAKEFLAGS="${MAKEFLAGS:--j2}" export TMPDIR=`pwd`/tmp mkdir -p tmp From dridi.boukelmoune at gmail.com Fri Nov 18 15:31:05 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 18 Nov 2016 16:31:05 +0100 Subject: [master] 4dcff1d Tolerate symbolic links Message-ID: commit 4dcff1d3a1d773054452fc4117551a7cbff2f0ce Author: Dridi Boukelmoune Date: Fri Nov 18 16:28:29 2016 +0100 Tolerate symbolic links diff --git a/tools/vtest.sh b/tools/vtest.sh index 33152e4..0aff2f4 100644 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -46,7 +46,7 @@ WAITBAD=1 # unit: WAITPERIOD ####################################################################### -if [ ! -d varnish-cache ] ; then +if ! (cd varnish-cache 2>/dev/null) ; then git clone \ https://github.com/varnishcache/varnish-cache.git \ varnish-cache From dridi.boukelmoune at gmail.com Fri Nov 18 15:31:06 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 18 Nov 2016 16:31:06 +0100 Subject: [master] 9e5e617 chmod +x tools/vtest.sh Message-ID: commit 9e5e617b45ee0989c6ffc658e2ca771ff46cdd32 Author: Dridi Boukelmoune Date: Fri Nov 18 16:29:32 2016 +0100 chmod +x tools/vtest.sh diff --git a/tools/vtest.sh b/tools/vtest.sh old mode 100644 new mode 100755 From nils.goroll at uplex.de Fri Nov 18 16:01:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 18 Nov 2016 17:01:05 +0100 Subject: [master] 15bfb54 Allow a custom DST Message-ID: commit 15bfb54aa349122f5785e8444d96173c86c25acd Author: Nils Goroll Date: Fri Nov 18 16:58:23 2016 +0100 Allow a custom DST The hardcoded paths are the one thing which has kept me from sing autogen.des regularly diff --git a/autogen.des b/autogen.des index 42b1508..d98980d 100755 --- a/autogen.des +++ b/autogen.des @@ -12,7 +12,9 @@ if [ -f /usr/bin/clang -a "x${CC}" = "x" ] ; then export CC fi -if [ "x`uname -o`" = "xFreeBSD" ] ; then +if [ "x$DST" != "x" ] ; then + : +elif [ "x`uname -o`" = "xFreeBSD" ] ; then DST="--prefix=/usr/local --mandir=/usr/local/man" else DST="--prefix=/opt/varnish --mandir=/opt/varnish/man" From phk at FreeBSD.org Fri Nov 18 21:45:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 18 Nov 2016 22:45:04 +0100 Subject: [master] f921816 Make SSH destination a parameter (so we can run on tools) Don't request a TTY Message-ID: commit f921816d89b6a0e20d5cf63da1d539558a1e7a5a Author: Poul-Henning Kamp Date: Fri Nov 18 21:43:58 2016 +0000 Make SSH destination a parameter (so we can run on tools) Don't request a TTY diff --git a/tools/vtest.sh b/tools/vtest.sh index 0aff2f4..cf4b10a 100755 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -44,6 +44,8 @@ WAITPERIOD=60 # unit: Seconds WAITGOOD=60 # unit: WAITPERIOD WAITBAD=1 # unit: WAITPERIOD +SSH_DST="-p 203 vtest at varnish-cache.org" + ####################################################################### if ! (cd varnish-cache 2>/dev/null) ; then @@ -100,9 +102,10 @@ submit () ( -o StrictHostKeyChecking=no \ -o PasswordAuthentication=no \ -o NumberOfPasswordPrompts=0 \ + -o RequestTTY=no \ -i vt_key \ - -p 203 \ - vtest at varnish-cache.org < ${1} + ${SSH_DST} \ + < ${1} ) rm -f _report.tgz From phk at FreeBSD.org Fri Nov 18 21:55:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 18 Nov 2016 22:55:04 +0100 Subject: [master] d433353 Use true(1) as dummy command Message-ID: commit d433353fe60bfd57555ee4237aa9f03d2f0e802b Author: Poul-Henning Kamp Date: Fri Nov 18 21:54:46 2016 +0000 Use true(1) as dummy command diff --git a/tools/vtest.sh b/tools/vtest.sh index cf4b10a..8424475 100755 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -105,6 +105,7 @@ submit () ( -o RequestTTY=no \ -i vt_key \ ${SSH_DST} \ + true \ < ${1} ) From phk at FreeBSD.org Sun Nov 20 14:02:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 20 Nov 2016 15:02:04 +0100 Subject: [master] 9897a93 Disable these very flakey check until we understand why they are flakey Message-ID: commit 9897a939dc55ce63075a98a93d2c2818ce170a36 Author: Poul-Henning Kamp Date: Sun Nov 20 14:01:14 2016 +0000 Disable these very flakey check until we understand why they are flakey diff --git a/bin/varnishtest/tests/t02001.vtc b/bin/varnishtest/tests/t02001.vtc index 43601a7..06ab867 100644 --- a/bin/varnishtest/tests/t02001.vtc +++ b/bin/varnishtest/tests/t02001.vtc @@ -61,7 +61,8 @@ client c1 { } -run } -run -varnish v1 -expect MEMPOOL.req0.live == 0 -varnish v1 -expect MEMPOOL.req1.live == 0 -varnish v1 -expect MEMPOOL.sess0.live == 0 -varnish v1 -expect MEMPOOL.sess1.live == 0 +# Tests temporarily neutered, they are too flakey +#varnish v1 -expect MEMPOOL.req0.live == 0 +#varnish v1 -expect MEMPOOL.req1.live == 0 +#varnish v1 -expect MEMPOOL.sess0.live == 0 +#varnish v1 -expect MEMPOOL.sess1.live == 0 From fgsch at lodoss.net Mon Nov 21 10:09:05 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 21 Nov 2016 11:09:05 +0100 Subject: [master] 3318bd3 This is trunk now Message-ID: commit 3318bd3217d14a00c7f2a9266494613eac482a6f Author: Federico G. Schwindt Date: Mon Nov 21 10:08:15 2016 +0000 This is trunk now diff --git a/configure.ac b/configure.ac index eac833e..bfc0099 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS Copyright (c) 2006-2016 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [5.0.0], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [trunk], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) From scanner at varnish-cache.org Mon Nov 21 10:47:30 2016 From: scanner at varnish-cache.org (scanner at varnish-cache.org) Date: Mon, 21 Nov 2016 16:17:30 +0530 Subject: Message from KMBT_C220 Message-ID: <7CF264A6.016.5C562FAC2E52.scanner@varnish-cache.org> A non-text attachment was scrubbed... Name: SKMBT_C5030380428.zip Type: application/zip Size: 7640 bytes Desc: not available URL: From phk at FreeBSD.org Mon Nov 21 11:08:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 21 Nov 2016 12:08:04 +0100 Subject: [master] ab404cc Why I wrote VTEST Message-ID: commit ab404cc830697532418b830cf037dca804436994 Author: Poul-Henning Kamp Date: Mon Nov 21 11:07:34 2016 +0000 Why I wrote VTEST diff --git a/doc/sphinx/phk/bjarne.jpeg b/doc/sphinx/phk/bjarne.jpeg new file mode 100644 index 0000000..83aa66c Binary files /dev/null and b/doc/sphinx/phk/bjarne.jpeg differ diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index 7dc5978..2dfcdb7 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -8,6 +8,7 @@ You may or may not want to know what Poul-Henning thinks. .. toctree:: :maxdepth: 1 + trialerror.rst farfaraway.rst thatslow.rst firstdesign.rst diff --git a/doc/sphinx/phk/trialerror.rst b/doc/sphinx/phk/trialerror.rst new file mode 100644 index 0000000..86009b3 --- /dev/null +++ b/doc/sphinx/phk/trialerror.rst @@ -0,0 +1,112 @@ +.. _phk_trialerror: + +================================================= +Trial&Error - Prototyping - Continous Integration +================================================= + +The other day I chatted to a friend who wrote his phd thesis with +`David Wheeler `_ as his advisor, that made me feel young, because Wheeler +was the guy who invented the subroutine. + +No, not 'a subroutine' but 'the subroutine'. + +In the 1980'ies, right when I had started in IT, there was the new +fashion: "Prototyping". + +It was all over the place, in 1983 Datamation you could about *"Data +driven prototyping"*, and by 1990 it had bubbled up to management +and Information and Software Technology could publish *"Organization +and Management of Systems Prototyping"* etc. etc. + +The grey-beard at my workplace laconically said *"We already do that, +only we call it Trial&Error."* + +Programming as always been Trial&Error, and probably always will. + +All the early pioneers, like Wheeler, complained about how +batch-scheduling of machine resources removed the "intimate" contact +with the running program and argued that it prolonged the debugging +process. + +Practically all the oral histories from back then are about people +sneaking in to university or work at night, to get the computer for +themselves. + +But we could call it "Prototyping" if that sounded better, and now +that the history-deficient dot-com generation has "invented" it, +we can call it "Continous Integration". + +I don't care - it's still Trial&Error to me. + +They guy I chatted with told how after his phd thesis he +*"swore to never again attempt to solve a problem with inadequate +tools"*. + +That is sound advice, and we all tend to forget it all the time, +so reminded, I did a mental inventory on the train home: Which tools +do I use even though I find them inadequate. + +And then I decided to do something about them. + +First thing was my lab which has - erhh... evolved? - over the last 15 years. + +Some of the original assumptions were plain wrong, and a lot of "As +a temporary solution I can ..." hacks became permanent, and so on. + +I spent two days cleaning, moving, shuffling and generally tidying +my lab, (Amongst other discoveries: The original two SCSI disks +from the first "freefall.freebsd.org" machine built by Rod Grimes.) +and it is now a lot more pleasant for the work I do these days. + +Second thing was the Jenkins and Travis we use for Tria^H^H^H^Continuous +Integration in the Varnish Project. + +Jenkins and Travis are both general purpose +program-test-framework-cloud-thingies, and they're fine up to a +point, but they are inadequate tools for me in too many ways. + +Jenkins is written in Java, which is not something I want to inflict +on computers volutarily, in particular not on computers people lend +us to test varnish on. + +Travis is Linux only, which is fine if you run Linux only, but I don't. + +But worst of all: Neither of them fully understand of our varnishtest +tool, and therefore their failure reports are tedious and cumbersome +to use. + +So, taking my friends advice, I sat down and wrote VTEST, which +consists of two small pieces of code: Tester and Reporter. + +The tester is a small, 173 lines, `portable and simple shell script +`_ +which runs on the computer, physical or virtual, where we want +to test Varnish. + +It obviously needs the compilers and tools we require to compile +Varnish, (autocrap, sphinx, graphviz) but it doesn't anything +beyond that, in particular it does not need a java runtime, a +GUI or a hole in your firewall. + +The tester sends a report to the project server with ssh(1), and +the reporter, which is just 750 lines of python code, ingests and +digests the report and spits out some `pidgin HTML +`_ with the information I actually +want to see. + +And just like with the varnishtest program before it, once I +had done it, my first thought was *"Why didn't I do that long time ago?"* + +So it is only fair to dedicate VTEST to the friend I chatted with: + +.. image:: bjarne.jpeg + +`Bjarne `_ tried to model how to best +distribute operating system kernels across a network, wrote a +adequate programming language tool for the job, which wass also +an adeqaute tool for a lot of other programming jobs. + +Thanks Bjarne! + +Poul-Henning, 2016-11-21 + From dridi.boukelmoune at gmail.com Mon Nov 21 11:21:04 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 21 Nov 2016 12:21:04 +0100 Subject: [master] 4567031 Give this test a chance not to be forgotten Message-ID: commit 45670312a6143c45b34468f20348f7f989eb3169 Author: Dridi Boukelmoune Date: Mon Nov 21 12:18:51 2016 +0100 Give this test a chance not to be forgotten diff --git a/bin/varnishtest/tests/t02001.vtc b/bin/varnishtest/tests/t02001.vtc index 06ab867..dafcded 100644 --- a/bin/varnishtest/tests/t02001.vtc +++ b/bin/varnishtest/tests/t02001.vtc @@ -61,7 +61,7 @@ client c1 { } -run } -run -# Tests temporarily neutered, they are too flakey +# XXX: Tests temporarily neutered, they are too flakey #varnish v1 -expect MEMPOOL.req0.live == 0 #varnish v1 -expect MEMPOOL.req1.live == 0 #varnish v1 -expect MEMPOOL.sess0.live == 0 From nils.goroll at uplex.de Mon Nov 21 16:56:04 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 21 Nov 2016 17:56:04 +0100 Subject: [master] b499bec ignore cpio errors for our srcdir to builddir fixup Message-ID: commit b499bec9bd7bca1c3aff9bf23a133295458c4adc Author: Nils Goroll Date: Mon Nov 21 17:52:41 2016 +0100 ignore cpio errors for our srcdir to builddir fixup This should make distcheck happy on SmartOS diff --git a/doc/graphviz/Makefile.am b/doc/graphviz/Makefile.am index dfbf7ab..6de1705 100644 --- a/doc/graphviz/Makefile.am +++ b/doc/graphviz/Makefile.am @@ -12,7 +12,7 @@ link_srcdir: test ! -f $(builddir)/cache_http1_fsm.svg ; then \ d=`pwd`/$(builddir) ; \ cd $(srcdir) && find . -name \*.svg -type f | \ - cpio -ldmp $${d} ; \ + cpio -ldmp $${d} || true ; \ fi dist-hook: diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index c88b46b..424ebb7 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -32,7 +32,7 @@ clean: link_srcdir: if test "x$(srcdir)" != "x$(builddir)" && test ! -f index.rst ; then \ d=`pwd`/$(builddir) ; \ - cd $(srcdir) && find . -type f | cpio -ldmp $${d} ; \ + cd $(srcdir) && find . -type f | cpio -ldmp $${d} || true ; \ fi # work around for make html called within doc/sphinx From phk at FreeBSD.org Mon Nov 21 20:48:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 21 Nov 2016 21:48:05 +0100 Subject: [master] 5d3effc Add a owner-settable "MESSAGE" Message-ID: commit 5d3effc1534fb77228aff9f80c7857b91bfb4518 Author: Poul-Henning Kamp Date: Mon Nov 21 20:45:45 2016 +0000 Add a owner-settable "MESSAGE" This goes into the 'id' on the server side, and is necessary because certain distingushing facts, like compiler/target architecture, cannot be determined until after autogen and the configure it creates have been run. diff --git a/tools/vtest.sh b/tools/vtest.sh index 8424475..dd7a36b 100755 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -36,8 +36,9 @@ export MAKEFLAGS="${MAKEFLAGS:--j2}" export TMPDIR=`pwd`/tmp mkdir -p tmp -# Contact email of responsible person -CONTACT=phk at varnish-cache.org +# Message to be shown in result pages +# Max 10 char of [A-Za-z0-9/. _-] +MESSAGE= WAITPERIOD=60 # unit: Seconds @@ -141,7 +142,7 @@ do mkdir _report export LOG=_report/_log - echo "VTEST 1.01" > ${LOG} + echo "VTEST 1.02" > ${LOG} echo "DATE `date +%s`" >> ${LOG} echo "BRANCH trunk" >> ${LOG} echo "HOST `hostname`" >> ${LOG} @@ -151,6 +152,7 @@ do else echo "LSB none" >> ${LOG} fi + echo "MESSAGE ${MESSAGE}" >> ${LOG} echo "GITREV $rev" >> ${LOG} if ! autogen >> _report/_autogen 2>&1 ; then echo "AUTOGEN BAD" >> ${LOG} From dridi.boukelmoune at gmail.com Tue Nov 22 10:17:05 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 22 Nov 2016 11:17:05 +0100 Subject: [master] ad63142 workspace_client is always above 4k Message-ID: commit ad631423922892742b4eff80e4e5d0408f6154d4 Author: Dridi Boukelmoune Date: Tue Nov 22 11:13:16 2016 +0100 workspace_client is always above 4k Should the minimum be bumped from 9k to 12k? diff --git a/include/tbl/params.h b/include/tbl/params.h index 7854f46..bf1080d 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1564,8 +1564,8 @@ PARAM( /* units */ "bytes", /* flags */ DELAYED_EFFECT, /* s-text */ - "Bytes of HTTP protocol workspace for clients HTTP req/resp. If " - "larger than 4k, use a multiple of 4k for VM efficiency.", + "Bytes of HTTP protocol workspace for clients HTTP req/resp. Use a " + "multiple of 4k for VM efficiency.", /* l-text */ "", /* func */ NULL ) From varnish-commit at varnish-cache.org Tue Nov 22 12:52:14 2016 From: varnish-commit at varnish-cache.org (ydbyyura) Date: Tue, 22 Nov 2016 20:52:14 +0800 Subject: =?utf-8?B?dmFybmlzaC1jb21taXQ6SFLmnIDmo5jmiYs=?= =?utf-8?B?55qE5Y2B57G7NjDkuKrmoLjlv4Ppl67popjlj4rop6PlhrPmlrnmoYjvvJ80MjY2?= =?utf-8?B?NA==?= Message-ID: <20161122205219255857@wyzrrv.net> HR??????60?????????? 2016? 11?29? ?? ?????3500 ????????????????? ???????????????????????????? ?????021-31006787, 18917870808 ??? ???QQ?320588808 ?Training Background|????? ????????????HR????????????????????????????????????????????????????????????????????????????????????????? ????????????????????????????????????????????????????????????????????? ICH?????????????????????????????????????60????????????????????????????????????????????????????????????????????????????????? ?Outline Content|????? ?????????? 1????????????????????????????? 2???offer,????????????? 3?????????????????offer? 4????????????????????????? 5??????????????????? ??????????? 1???????????????????????????? 2??????????????????????? 3???????????????? 4?????????????????????????????????? 5??????????????????????????????????? ???????????? 1???????????????????????? 2????????????????????????????? 3??????????????????????????????????? 4???????1:1???????????????1.5????? 5???????6??????????????? 6????????????????? 7???????????????????????????????????????????????????? ?????????? 1?????????????????? 2???????1??????????????? 3???????????????????????????????? 4???????????????????????? 5?????????????????????????? ?????????? 1????????????????? 2????????????????????? 3??????????????????? ?????????????? 1????????????????? 2???????????????? 3??????1?????????????????? 4??????????????????????????????????? ?????????????????? 1?????????????????????????????? 2??????????????????????????????????????? 3??????????????????????????????????????????????????????? 4???????????????????????????????? 5?????????????????????????? 6??????????????????????????? ???????????? 1?????????????1??????? 2????????????????????????????? 3?????????????????????????? 4????10??????????????????? 5??????????????????????????????????? 6???????????????????????????? ?????????? 1????????????????????????? 2????????30???????????????????????????? 3?????????????????????????????? 4??????????????????? 5????????????????????????????? 6??????????????????????????24?????? 7??????????????????????????????????????? 8?????????????????????????????? 9?????????????????????????? 10??????????????? 11?HR??????????? 12????????????????????????????????? 13?????????????????????? 14???????????????????????? ??????????? 1????????????????????? 2???????????????????????? 3?????????????? 4?????????????????? 5?????????????????? ????????????????????tuiding02 at 163.com,??????????????? -------------- next part -------------- An HTML attachment was scrubbed... URL: From hermunn at varnish-software.com Tue Nov 22 13:31:04 2016 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Tue, 22 Nov 2016 14:31:04 +0100 Subject: [4.1] f5d5587 wake up herder from sleeping after destroying, sleep for longer Message-ID: commit f5d5587e09677a94f45f2c73137585b228d84ea0 Author: Nils Goroll Date: Thu Oct 27 07:04:00 2016 +0200 wake up herder from sleeping after destroying, sleep for longer Previously, after sending a thread to varnish heaven, the herder slept for wthread_destroy_delay unconditionally. Instead, we now wait on the cv so we get woken up in case we run dry during the delay. This change is relevant proportionally to the value of wthread_destroy_delay if the spread between thread_pool_min and thread_pool_max is big and varnish is exposed to sudden traffic peaks. IOW, it will probably be only relevant for high performance setups. Also, we now sleep for thread_pool_timeout unless a shorter delay is warranted. This will delay the effect of thread parameter changes for up to thread_pool_timeout seconds unless the pool runs dry, in which case they will become effective immediately. diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 7176f09..43c0384 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -435,6 +435,7 @@ pool_herder(void *priv) struct pool_task *pt; double t_idle; struct worker *wrk; + int delay; CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC); @@ -445,6 +446,8 @@ pool_herder(void *priv) pool_breed(pp); continue; } + + delay = cache_param->wthread_timeout; assert(pp->nthr >= cache_param->wthread_min); if (pp->nthr > cache_param->wthread_min) { @@ -470,8 +473,10 @@ pool_herder(void *priv) &wrk->task, list); wrk->task.func = pool_kiss_of_death; AZ(pthread_cond_signal(&wrk->cond)); - } else + } else { + delay = wrk->lastused - t_idle; wrk = NULL; + } } Lck_Unlock(&pp->mtx); @@ -481,15 +486,15 @@ pool_herder(void *priv) VSC_C_main->threads--; VSC_C_main->threads_destroyed++; Lck_Unlock(&pool_mtx); - VTIM_sleep(cache_param->wthread_destroy_delay); - continue; - } + delay = cache_param->wthread_destroy_delay; + } else if (delay < cache_param->wthread_destroy_delay) + delay = cache_param->wthread_destroy_delay; } Lck_Lock(&pp->mtx); if (!pp->dry) { (void)Lck_CondWait(&pp->herder_cond, &pp->mtx, - VTIM_real() + 5); + VTIM_real() + delay); } else { /* XXX: unsafe counters */ VSC_C_main->threads_limited++; diff --git a/bin/varnishtest/tests/r01490.vtc b/bin/varnishtest/tests/r01490.vtc index 82ffdb4..76b32be 100644 --- a/bin/varnishtest/tests/r01490.vtc +++ b/bin/varnishtest/tests/r01490.vtc @@ -9,6 +9,7 @@ varnish v1 \ -arg "-p thread_pool_min=2" \ -arg "-p thread_pool_max=3" \ -arg "-p thread_pools=1" \ + -arg "-p thread_pool_timeout=10" \ -vcl+backend {} varnish v1 -start @@ -21,16 +22,16 @@ logexpect l1 -v v1 -g raw { varnish v1 -cliok "param.set thread_pool_min 3" -# Herder thread sleeps 5 seconds. Have to wait longer than that. -delay 6 +# Have to wait longer than thread_pool_timeout +delay 11 varnish v1 -expect threads == 3 varnish v1 -cliok "param.set thread_pool_min 2" varnish v1 -cliok "param.set thread_pool_max 2" -# Herder thread sleeps 5 seconds. Have to wait longer than that. -delay 6 +# Have to wait longer than thread_pool_timeout +delay 11 varnish v1 -expect threads == 2 From hermunn at varnish-software.com Tue Nov 22 13:31:04 2016 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Tue, 22 Nov 2016 14:31:04 +0100 Subject: [4.1] 674cd52 fix a potential race Message-ID: commit 674cd52130457657db62b7c5c810477a74ca3885 Author: Nils Goroll Date: Thu Oct 27 13:00:20 2016 +0200 fix a potential race The nthr => wthread_min assertion could trigger if wthread_min was changed after the first comparison. diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 43c0384..3625365 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -435,22 +435,24 @@ pool_herder(void *priv) struct pool_task *pt; double t_idle; struct worker *wrk; - int delay; + int delay, wthread_min; CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC); while (1) { + wthread_min = cache_param->wthread_min; + /* Make more threads if needed and allowed */ - if (pp->nthr < cache_param->wthread_min || + if (pp->nthr < wthread_min || (pp->dry && pp->nthr < cache_param->wthread_max)) { pool_breed(pp); continue; } delay = cache_param->wthread_timeout; - assert(pp->nthr >= cache_param->wthread_min); + assert(pp->nthr >= wthread_min); - if (pp->nthr > cache_param->wthread_min) { + if (pp->nthr > wthread_min) { t_idle = VTIM_real() - cache_param->wthread_timeout; From hermunn at varnish-software.com Tue Nov 22 13:31:05 2016 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Tue, 22 Nov 2016 14:31:05 +0100 Subject: [4.1] 6a672d4 Apply queue limits only to requests Message-ID: commit 6a672d40982cb590a3a60e427209b91d4d016f77 Author: Nils Goroll Date: Thu Oct 27 07:52:46 2016 +0200 Apply queue limits only to requests We should queue bo tasks to avoid failing client requests unnecessarily. The detailed parameter description of the queue_limit was already correct, improve the short one. diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 3625365..4f7460f 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -243,8 +243,11 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_how how) return (0); } - /* Acceptors are not subject to queue limits */ - if (how == TASK_QUEUE_VCA || + /* + * queue limits only apply to client threads - all other + * work is vital and needs do be done at the earliest + */ + if (how != TASK_QUEUE_REQ || pp->lqueue < cache_param->wthread_max + cache_param->wthread_queue_limit + pp->nthr) { pp->nqueued++; diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index f5a9cf4..724087a 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -187,7 +187,7 @@ struct parspec WRK_parspec[] = { "10", "requests" }, { "thread_queue_limit", tweak_uint, &mgt_param.wthread_queue_limit, "0", NULL, - "Permitted queue length per thread-pool.\n" + "Permitted request queue length per thread-pool.\n" "\n" "This sets the number of requests we will queue, waiting " "for an available thread. Above this limit sessions will " diff --git a/include/tbl/params.h b/include/tbl/params.h index a5157ee..c68500b 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1230,7 +1230,7 @@ PARAM( /* units */ NULL, /* flags */ EXPERIMENTAL, /* s-text */ - "Permitted queue length per thread-pool.\n" + "Permitted request queue length per thread-pool.\n" "\n" "This sets the number of requests we will queue, waiting for an " "available thread. Above this limit sessions will be dropped " From hermunn at varnish-software.com Tue Nov 22 13:31:05 2016 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Tue, 22 Nov 2016 14:31:05 +0100 Subject: [4.1] d564695 clarify that the enum implies a priority Message-ID: commit d564695c0fd6c3e68d3848944d8e9e0e8ca7469c Author: Nils Goroll Date: Thu Oct 27 08:56:56 2016 +0200 clarify that the enum implies a priority diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index d64d007..6ac1972 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -314,7 +314,8 @@ struct pool_task { void *priv; }; -enum task_how { +/* tasks are taken off the queues in this order */ +enum task_prio { TASK_QUEUE_BO, TASK_QUEUE_REQ, TASK_QUEUE_VCA, @@ -887,13 +888,13 @@ const char *body_status_2str(enum body_status e); const char *sess_close_2str(enum sess_close sc, int want_desc); /* cache_pool.c */ -int Pool_Task(struct pool *pp, struct pool_task *task, enum task_how how); +int Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio how); int Pool_Task_Arg(struct worker *, task_func_t *, const void *arg, size_t arg_len); void Pool_Sumstat(struct worker *w); int Pool_TrySumstat(struct worker *wrk); void Pool_PurgeStat(unsigned nobj); -int Pool_Task_Any(struct pool_task *task, enum task_how how); +int Pool_Task_Any(struct pool_task *task, enum task_prio how); /* cache_range.c [VRG] */ void VRG_dorange(struct req *req, const char *r); diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index d3deed2..409d2c3 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -91,7 +91,7 @@ Pool_TrySumstat(struct worker *wrk) */ int -Pool_Task_Any(struct pool_task *task, enum task_how how) +Pool_Task_Any(struct pool_task *task, enum task_prio how) { struct pool *pp; diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 4f7460f..01f5902 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -218,7 +218,7 @@ Pool_Task_Arg(struct worker *wrk, task_func_t *func, */ int -Pool_Task(struct pool *pp, struct pool_task *task, enum task_how how) +Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio how) { struct worker *wrk; int retval = 0; From hermunn at varnish-software.com Tue Nov 22 13:31:05 2016 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Tue, 22 Nov 2016 14:31:05 +0100 Subject: [4.1] 199d1ca keep a reserve of idle theads for vital tasks Message-ID: commit 199d1ca4be308ddd9333fac1878a8d3e4707ef2a Author: Nils Goroll Date: Thu Oct 27 09:33:02 2016 +0200 keep a reserve of idle theads for vital tasks (backend tasks for the time being) Add parameter thread_pool_reserve to tweak Conflicts: bin/varnishd/mgt/mgt_pool.c diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 6ac1972..1ea38e2 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -314,9 +314,14 @@ struct pool_task { void *priv; }; -/* tasks are taken off the queues in this order */ +/* + * tasks are taken off the queues in this order + * + * prios up to TASK_QUEUE_RESERVE are run from the reserve + */ enum task_prio { TASK_QUEUE_BO, +#define TASK_QUEUE_RESERVE TASK_QUEUE_BO TASK_QUEUE_REQ, TASK_QUEUE_VCA, TASK_QUEUE_END @@ -889,7 +894,7 @@ const char *sess_close_2str(enum sess_close sc, int want_desc); /* cache_pool.c */ int Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio how); -int Pool_Task_Arg(struct worker *, task_func_t *, +int Pool_Task_Arg(struct worker *, enum task_prio, task_func_t *, const void *arg, size_t arg_len); void Pool_Sumstat(struct worker *w); int Pool_TrySumstat(struct worker *wrk); diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index e718374..5b2827a 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -438,7 +438,8 @@ vca_accept_task(struct worker *wrk, void *arg) wa.acceptsock = i; - if (!Pool_Task_Arg(wrk, vca_make_session, &wa, sizeof wa)) { + if (!Pool_Task_Arg(wrk, TASK_QUEUE_VCA, + vca_make_session, &wa, sizeof wa)) { /* * We couldn't get another thread, so we will handle * the request in this worker thread, but first we diff --git a/bin/varnishd/cache/cache_pool.h b/bin/varnishd/cache/cache_pool.h index d3b9e70..239b9e3 100644 --- a/bin/varnishd/cache/cache_pool.h +++ b/bin/varnishd/cache/cache_pool.h @@ -42,6 +42,7 @@ struct pool { pthread_t herder_thr; struct lock mtx; + unsigned nidle; struct taskhead idle_queue; struct taskhead queues[TASK_QUEUE_END]; unsigned nthr; diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 01f5902..ede1329 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -148,17 +148,35 @@ pool_addstat(struct dstat *dst, struct dstat *src) memset(src, 0, sizeof *src); } +static inline int +pool_reserve(void) +{ + unsigned lim; + + if (cache_param->wthread_reserve == 0) + return (cache_param->wthread_min / 20 + 1); + lim = cache_param->wthread_min * 950 / 1000; + if (cache_param->wthread_reserve > lim) + return (lim); + return (cache_param->wthread_reserve); +} + /*--------------------------------------------------------------------*/ static struct worker * -pool_getidleworker(struct pool *pp) +pool_getidleworker(struct pool *pp, enum task_prio how) { - struct pool_task *pt; + struct pool_task *pt = NULL; struct worker *wrk; CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); Lck_AssertHeld(&pp->mtx); - pt = VTAILQ_FIRST(&pp->idle_queue); + if (how <= TASK_QUEUE_RESERVE || pp->nidle > pool_reserve()) { + pt = VTAILQ_FIRST(&pp->idle_queue); + if (pt == NULL) + AZ(pp->nidle); + } + if (pt == NULL) { if (pp->nthr < cache_param->wthread_max) { pp->dry++; @@ -179,7 +197,7 @@ pool_getidleworker(struct pool *pp) */ int -Pool_Task_Arg(struct worker *wrk, task_func_t *func, +Pool_Task_Arg(struct worker *wrk, enum task_prio how, task_func_t *func, const void *arg, size_t arg_len) { struct pool *pp; @@ -193,9 +211,11 @@ Pool_Task_Arg(struct worker *wrk, task_func_t *func, CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); Lck_Lock(&pp->mtx); - wrk2 = pool_getidleworker(pp); + wrk2 = pool_getidleworker(pp, how); if (wrk2 != NULL) { + AN(pp->nidle); VTAILQ_REMOVE(&pp->idle_queue, &wrk2->task, list); + pp->nidle--; retval = 1; } else { wrk2 = wrk; @@ -222,7 +242,6 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio how) { struct worker *wrk; int retval = 0; - CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); AN(task); AN(task->func); @@ -232,9 +251,11 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio how) /* The common case first: Take an idle thread, do it. */ - wrk = pool_getidleworker(pp); + wrk = pool_getidleworker(pp, how); if (wrk != NULL) { + AN(pp->nidle); VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list); + pp->nidle--; AZ(wrk->task.func); wrk->task.func = task->func; wrk->task.priv = task->priv; @@ -282,7 +303,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk) { struct pool_task *tp; struct pool_task tpx, tps; - int i; + int i, prio_lim; CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); wrk->pool = pp; @@ -294,7 +315,12 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk) WS_Reset(wrk->aws, NULL); AZ(wrk->vsl); - for (i = 0; i < TASK_QUEUE_END; i++) { + if (pp->nidle < pool_reserve()) + prio_lim = TASK_QUEUE_RESERVE + 1; + else + prio_lim = TASK_QUEUE_END; + + for (i = 0; i < prio_lim; i++) { tp = VTAILQ_FIRST(&pp->queues[i]); if (tp != NULL) { pp->lqueue--; @@ -323,6 +349,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk) wrk->task.func = NULL; wrk->task.priv = wrk; VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list); + pp->nidle++; do { i = Lck_CondWait(&wrk->cond, &pp->mtx, wrk->vcl == NULL ? 0 : wrk->lastused+60.); @@ -468,6 +495,7 @@ pool_herder(void *priv) wrk = NULL; pt = VTAILQ_LAST(&pp->idle_queue, taskhead); if (pt != NULL) { + AN(pp->nidle); AZ(pt->func); CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC); @@ -476,6 +504,7 @@ pool_herder(void *priv) /* Give it a kiss on the cheek... */ VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list); + pp->nidle--; wrk->task.func = pool_kiss_of_death; AZ(pthread_cond_signal(&wrk->cond)); } else { diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h index 678ed26..9eabc2c 100644 --- a/bin/varnishd/common/params.h +++ b/bin/varnishd/common/params.h @@ -91,6 +91,7 @@ struct params { /* Worker threads and pool */ unsigned wthread_min; unsigned wthread_max; + unsigned wthread_reserve; double wthread_timeout; unsigned wthread_pools; double wthread_add_delay; diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index 724087a..beb5f56 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -55,6 +55,7 @@ static char min_val[20]; static char max_val[20]; +static char reserve_max_val[20]; static int tweak_thread_pool_min(struct vsb *vsb, const struct parspec *par, @@ -64,10 +65,18 @@ tweak_thread_pool_min(struct vsb *vsb, const struct parspec *par, if (tweak_generic_uint(vsb, par->priv, arg, par->min, par->max)) return (-1); + AN(VSB_new(&v2, min_val, sizeof min_val, 0)); AZ(tweak_generic_uint(&v2, &mgt_param.wthread_min, NULL, NULL, NULL)); AZ(VSB_finish(&v2)); MCF_SetMinimum("thread_pool_max", min_val); + + unsigned t = mgt_param.wthread_min * 19 / 20; + AN(VSB_new(&v2, reserve_max_val, sizeof reserve_max_val, 0)); + AZ(tweak_generic_uint(&v2, &t, NULL, NULL, NULL)); + AZ(VSB_finish(&v2)); + MCF_SetMaximum("thread_pool_reserve", reserve_max_val); + return (0); } @@ -125,6 +134,25 @@ struct parspec WRK_parspec[] = { "Minimum is 10 threads.", DELAYED_EFFECT, "100", "threads" }, + { "thread_pool_reserve", tweak_uint, &mgt_param.wthread_reserve, + 0, NULL, + "The number of worker threads reserved for vital tasks " + "in each pool.\n" + "\n" + "Tasks may require other tasks to complete (for example, " + "client requests may require backend requests). This reserve " + "is to ensure that such tasks still get to run even under high " + "load.\n" + "\n" + "Increasing the reserve may help setups with a high number of " + "backend requests at the expense of client performance. " + "Setting it too high will waste resources by keeping threads " + "unused.\n" + "\n" + "Default is 0 to auto-tune (currently 5% of thread_pool_min).\n" + "Minimum is 1 otherwise, maximum is 95% of thread_pool_min.", + DELAYED_EFFECT, + "0", "threads" }, { "thread_pool_timeout", tweak_timeout, &mgt_param.wthread_timeout, "10", NULL, diff --git a/bin/varnishtest/tests/c00079.vtc b/bin/varnishtest/tests/c00079.vtc new file mode 100644 index 0000000..a0d4c2e --- /dev/null +++ b/bin/varnishtest/tests/c00079.vtc @@ -0,0 +1,23 @@ +varnishtest "thread_pool_reserve max adjustment" + +server s1 { +} -start + +varnish v1 \ + -arg "-p thread_pool_min=10" \ + -arg "-p thread_pool_max=100" \ + -arg "-p thread_pools=1" \ + -arg "-p thread_pool_timeout=10" \ + -vcl+backend {} +varnish v1 -start + +varnish v1 -cliok "param.set thread_pool_reserve 0" +varnish v1 -cliok "param.set thread_pool_reserve 1" +varnish v1 -cliok "param.set thread_pool_reserve 9" +varnish v1 -clierr 106 "param.set thread_pool_reserve 10" + +varnish v1 -cliok "param.set thread_pool_min 100" +varnish v1 -cliok "param.set thread_pool_reserve 0" +varnish v1 -cliok "param.set thread_pool_reserve 1" +varnish v1 -cliok "param.set thread_pool_reserve 95" +varnish v1 -clierr 106 "param.set thread_pool_reserve 96" diff --git a/include/tbl/params.h b/include/tbl/params.h index c68500b..9a272a3 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1157,6 +1157,34 @@ PARAM( /* l-text */ "", /* func */ NULL ) +/* actual location mgt_pool.c */ +PARAM( + /* name */ thread_pool_reserve, + /* typ */ thread_pool_reserve, + /* min */ NULL, + /* max */ NULL, + /* default */ "0", + /* units */ "threads", + /* flags */ DELAYED_EFFECT| EXPERIMENTAL, + /* s-text */ + "The number of worker threads reserved for vital tasks " + "in each pool.\n" + "\n" + "Tasks may require other tasks to complete (for example, " + "client requests may require backend requests). This reserve " + "is to ensure that such tasks still get to run even under high " + "load.\n" + "\n" + "Increasing the reserve may help setups with a high number of " + "backend requests at the expense of client performance. " + "Setting it too high will waste resources by keeping threads " + "unused.\n" + "\n" + "Default is 0 to auto-tune (currently 5% of thread_pool_min).\n" + "Minimum is 1 otherwise, maximum is 95% of thread_pool_min.", + /* l-text */ "", + /* func */ NULL +) /* actual location mgt_pool.c */ PARAM( From donotreply at varnish-cache.org Tue Nov 22 15:38:16 2016 From: donotreply at varnish-cache.org (donotreply at varnish-cache.org) Date: Tue, 22 Nov 2016 22:38:16 +0700 Subject: Message from "RNP002EDF52F413" Message-ID: <20161122223816C.DCSML-S026800000.8C9D78A06F64@varnish-cache.org> This E-mail was sent from "RNP002EDF52F413" (Aficio MP 2352). Scan Date: Tue, 22 Nov 2016 22:38:16 +0700) Queries to: donotreply at varnish-cache.org -------------- next part -------------- A non-text attachment was scrubbed... Name: 20161122223816106_003.zip Type: application/zip Size: 6451 bytes Desc: not available URL: From donotreply at varnish-cache.org Tue Nov 22 16:33:24 2016 From: donotreply at varnish-cache.org (donotreply at varnish-cache.org) Date: Tue, 22 Nov 2016 16:33:24 -0000 Subject: Message from "RNP002E9867B362" Message-ID: <20161122163324C.DCSML-S026800000.B7FA8E6FF581@varnish-cache.org> This E-mail was sent from "RNP002E9867B362" (Aficio MP 2352). Scan Date: Tue, 22 Nov 2016 16:33:24 -0000) Queries to: donotreply at varnish-cache.org -------------- next part -------------- A non-text attachment was scrubbed... Name: 20161122163324455_009.zip Type: application/zip Size: 6443 bytes Desc: not available URL: From varnish-commit at varnish-cache.org Wed Nov 23 04:14:26 2016 From: varnish-commit at varnish-cache.org (sc) Date: Wed, 23 Nov 2016 12:14:26 +0800 Subject: =?utf-8?B?dmFybmlzaC1jb21taXTvvJrkvIHkuJrlpoLkvZXlgZo=?= =?utf-8?B?5aW95YWz6ZSu5Lq65omN55qE566h55CG77yf?= Message-ID: <20161123121440108787@tokfqt.com> ??????????????? ?????2016?11?25-26? ?? 2016?12?2-3? ?? ??????????????????????? ?????3800?/????????????????????????? ?????021-31006787?0755-61280006, 18917870808 ??? ?????320588808 at vip.qq.com QQ/???320588808 ????? ???????????????????? ---????????????? ---??????????????? ---??????????????? ---????????????? ...... ??????????????????????????????????????????????????????????????????????????????????????????????????? ????????????+???4.0?????????????????????????????????????????????????????????????????????????????????????????????????HR???????????????????????????????????????????????????????????????????????????????????????????????????8090????????????????????????????????????????????????????????????????????????????????????????????????????HR????????????????????????????????????????????????????????????????????????????????????????? ?????????????????????????????????????????????????????????????????HR????????????????? ??????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? ????? ---????????????????????????? ---????????????? ---???????????? ---???????????????????????? ---??????????????????? ...... ????? ???????????????????????????? 1??????????????????????????? 2?????????????????? 3????????????????????? 4???????????5??????????????????????? 5??????????????????HR????? ???????????? 1??????? 2????????4?? 3???1???????????? 4???2???????? 5???3????????? 6???4?????????????????????? 7????????????????????????? ???????????? 1??????? 2????????3??3?? 3???1??????????????????????????????? 4???2??????????????????????????????? 5???3???????????????????????????????????? 6????????????????????????????????? 7??????1??????????? 2?X??????????? 3???????????????? 8??????????????? ???????????? 1???????4???2?? 2???1??????????????????????? 3???2??????????????????????????? 4???3???????????????????????????? 5???4????????????????????????????? 6?????????????????????? 7???????????????????X???????????? ?????????????? 1??????????5???2?? 2???1??????????????????????? 3???2??????? 4???3??????????????????????????? 5???4?????????????????????????????? 6???5?????????????????????????????? 7??????????????????????? 8???????????????X????????????X????????? ???????????? 1??????????????????????????? 2????????????? 3???????? 4??????????? 5?????????????????????????X?????????? 6????????????????/???? ???????????????? 1??????????????????????????????????? 2??????????? 3???????? 4? ???? 5??????????????????X?????????X?????? 6?????????????????? -------------- next part -------------- An HTML attachment was scrubbed... URL: From antonia.carnon at varnish-cache.org Wed Nov 23 09:40:06 2016 From: antonia.carnon at varnish-cache.org (antonia carnon) Date: Wed, 23 Nov 2016 11:40:06 +0200 Subject: Bill-1151714 Message-ID: -------------- next part -------------- A non-text attachment was scrubbed... Name: 2ecbce9b2ce3f9a40b955f7a1a083bb5.zip Type: application/zip Size: 7672 bytes Desc: not available URL: From garrett.dubrick at varnish-cache.org Wed Nov 23 09:55:44 2016 From: garrett.dubrick at varnish-cache.org (garrett dubrick) Date: Wed, 23 Nov 2016 11:55:44 +0200 Subject: Bill-85628 Message-ID: -------------- next part -------------- A non-text attachment was scrubbed... Name: b50de2908d54a830acc57726b6c92423.zip Type: application/zip Size: 7672 bytes Desc: not available URL: From gale.selbye at varnish-cache.org Wed Nov 23 11:42:27 2016 From: gale.selbye at varnish-cache.org (gale selbye) Date: Wed, 23 Nov 2016 18:42:27 +0700 Subject: Bill-4797 Message-ID: <2F123913-3521-F054-987C-F5E8EF49C6B4@varnish-cache.org> -------------- next part -------------- A non-text attachment was scrubbed... Name: cf5f58a841dca0197b0d7d2c0b3ef436.zip Type: application/zip Size: 7678 bytes Desc: not available URL: From hermunn at varnish-software.com Wed Nov 23 14:58:05 2016 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Wed, 23 Nov 2016 15:58:05 +0100 Subject: [4.1] c9c1eff Update changelog Message-ID: commit c9c1effac3a79e37e92849f35149e304d941dfab Author: P?l Hermunn Johansen Date: Wed Nov 23 14:22:06 2016 +0100 Update changelog diff --git a/doc/changes.rst b/doc/changes.rst index 6ca2418..0136676 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,4 +1,26 @@ ====================================== +Varnish Cache 4.1.4-beta3 (unreleased) +====================================== + +Changes since 4.1.4-beta2: + +* Include the current time of the panic in the panic output +* Keep a reserve of idle threads for vital tasks + +Bugs fixed +---------- + +* 1874_ - clock-step related crash +* 1889_ - (docfix) What does -p flag for backend.list command means +* 2115_ - VSM temporary files are not always deleted +* 2129_ - (docfix) stack overflow with >4 level esi + +.. _1874: https://github.com/varnishcache/varnish-cache/issues/1874 +.. _1889: https://github.com/varnishcache/varnish-cache/issues/1889 +.. _2115: https://github.com/varnishcache/varnish-cache/issues/2115 +.. _2129: https://github.com/varnishcache/varnish-cache/issues/2129 + +====================================== Varnish Cache 4.1.4-beta2 (2016-10-13) ====================================== From varnish-commit at varnish-cache.org Wed Nov 23 18:29:04 2016 From: varnish-commit at varnish-cache.org (xh) Date: Thu, 24 Nov 2016 02:29:04 +0800 Subject: =?utf-8?B?dmFybmlzaC1jb21taXTlpoLkvZXkvJjljJbph4fotK0=?= =?utf-8?B?6L+Q5L2c5rWB56iL77yf?= Message-ID: <20161124022912482682@ruja.org> ??????????????? ?????? 2016?11?26-27?? ?????? ???????????????????????????????????????? ??? ?????? 3200/2?/1??????????????? ????0755-6128-0006 18917870808 ??? QQ????320588808 ????? ????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? ????? ?????????????? ??????????? ????????????? ???????????? ???????????? ????????? ?????????? ????????????? ?????????? ????? ?????????????????? ???????????????? ?????????????? ??????????? ??????????????? ??????????? ????????????? ??????????????? ????????????? ????????????? ??????????? ?????????? ????????????? ????????????? ????? ??????????? ??????????????? ????????????? ??????????? ????????????? ????????????? ???????????? ?????? XX???????????? XX??????????? ??????????????????????? ?????????? ?????-??? ?????-??? ??????? ??????????????? ?????????? ????????????? ?????????????? ?????????????? ??????????? ???????????????? ??????????????? ?????????????? ????????? ????????????????? ?????????? ???????? ??????????? ???????????? ?????????? ????????????? ??????????????? ???????????? ??????????? ??????????? ???????????? ???????????? ????????? ???????????? ??????? ??????????????? ???????????? ?????????? ????????????????????? ????????????????? ???????????????? ??????????? ?????????? ????80:20?? ????????????? ????????????? ???????????? ????????? ??????????? ???????????? ???????? ????????? ????????? ?????????? ??????????? ??????? ?????????? ???????????????? ???????? ???????????? ??????????????? ?????????? ?????????????? ?????????? ???????????? ?????????? ?????????? ???? ????? ???????????? ???????? ??????????? ??????? ???CPI??? ???PPI? ???PMI? ???????????? ???????????????? ???????? ?????????? ?????????????? ??????? ???????? ???????????? ???????????? ?????????? ??????????????? ????????? ????????? ????????????? ??????????? ?????????????? ????????????? ????? ???????? ????????????? ??????? ???????? ???? ??????????? ??????? ??????? ?????????? ??????????? ??????? ??????????? ???????????? ???:????? ???????? ??????????? ??????? ????????? ???????????????? ??????????? ????????? ????????? ????????????? ???????????????? ????????????? ?????????? ?????????????? ??????????? ???????? ????????? ?????????? ??????? ?????????? ????????????? ???????????? ????????????????? ???????????? ????????? ????? ??????????? ???????????????????? ??????????????? ???????????????????? ????????????? ??????????? ???????????? ??????????? ???? ??????????? ????????? ??????????? ??????? ????? ?????????? ????? ????????? ??????????? ?????????? ????????? ?????????? ???????????? ???????????? ????????????? ?????????????????? ??????? ?????????????? ???????????? ?????????????? ???????????????? ?????????????????? ?????????????????? ????????????? ?????????? ???? ???????????? ???? ???? ??????????? ?????????? ?????????? ????????????? ??????????????? ??????????????? ????????????? ????????SWOT???? ??????? ????????? ????????????????? ??????????????? ???????????? ????????????? ?????????? ????? ???????????????????1986 ???? Gerber???????????????(MichiganState University )???????????,?????Heinz(??)????? ????: ???????????????????????????????????????????????????2000???,?????????,??????????????????????500? ????????????,??????????? ???????????????????????????????????????????????????????????????????? ????: ?????????????? ??????????????? ????????????? ?????????????? PMC??-?????????????? ????????? ????: * ???????, ?????, ?????? * ????????????????????????????????????????????????????? * ?????????????????????,???????????????,????????????????????? ?????????????????????tuiding02 at 163.com,??????????????? -------------- next part -------------- An HTML attachment was scrubbed... URL: From hermunn at varnish-software.com Thu Nov 24 11:51:05 2016 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Thu, 24 Nov 2016 12:51:05 +0100 Subject: [4.1] 9dc2ec5 Prepare for 4.1.4-beta3 release Message-ID: commit 9dc2ec53e232cb230dcd3673c80bb8f3420313ee Author: P?l Hermunn Johansen Date: Thu Nov 24 12:48:52 2016 +0100 Prepare for 4.1.4-beta3 release diff --git a/configure.ac b/configure.ac index 1653b25..fe9739d 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS Copyright (c) 2006-2016 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [4.1.4-beta2], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [4.1.4-beta3], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/changes.rst b/doc/changes.rst index 0136676..0207b59 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,5 +1,5 @@ ====================================== -Varnish Cache 4.1.4-beta3 (unreleased) +Varnish Cache 4.1.4-beta3 (2016-11-24) ====================================== Changes since 4.1.4-beta2: From dridi.boukelmoune at gmail.com Thu Nov 24 16:01:05 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 24 Nov 2016 17:01:05 +0100 Subject: [master] 7cdb853 Use both Message-ID: commit 7cdb8531b38353b0570920bb798c59d567c99d47 Author: Dridi Boukelmoune Date: Thu Nov 24 16:59:12 2016 +0100 Use both diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 37aa744..e006b89 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -160,7 +160,7 @@ returns = ( sp_variables = [ ('remote.ip', 'IP', - ('client', 'backend'), + ('both'), (), """ The IP address of the other end of the TCP connection. This can either be the clients IP, or the outgoing IP @@ -169,7 +169,7 @@ sp_variables = [ ), ('client.ip', 'IP', - ('client', 'backend'), + ('both'), (), """ The client's IP address. """ @@ -185,14 +185,14 @@ sp_variables = [ ), ('local.ip', 'IP', - ('client', 'backend'), + ('both'), (), """ The IP address of the local end of the TCP connection. """ ), ('server.ip', 'IP', - ('client', 'backend'), + ('both'), (), """ The IP address of the socket on which the client connection was received. From dridi.boukelmoune at gmail.com Thu Nov 24 16:08:04 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 24 Nov 2016 17:08:04 +0100 Subject: [master] 3f0b634 Missing comma Message-ID: commit 3f0b634bb491cee0ec8ca306f7ba8f0f8513a0db Author: Dridi Boukelmoune Date: Thu Nov 24 17:07:21 2016 +0100 Missing comma diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index e006b89..d002541 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -160,7 +160,7 @@ returns = ( sp_variables = [ ('remote.ip', 'IP', - ('both'), + ('both',), (), """ The IP address of the other end of the TCP connection. This can either be the clients IP, or the outgoing IP @@ -169,7 +169,7 @@ sp_variables = [ ), ('client.ip', 'IP', - ('both'), + ('both',), (), """ The client's IP address. """ @@ -185,14 +185,14 @@ sp_variables = [ ), ('local.ip', 'IP', - ('both'), + ('both',), (), """ The IP address of the local end of the TCP connection. """ ), ('server.ip', 'IP', - ('both'), + ('both',), (), """ The IP address of the socket on which the client connection was received. From voicemail at varnish-cache.org Fri Nov 25 12:50:24 2016 From: voicemail at varnish-cache.org (voicemail at varnish-cache.org) Date: Fri, 25 Nov 2016 12:50:24 -0000 Subject: [Vigor2820 Series] New voice mail message from 01451309204 on 2016/11/25 14:50:14 Message-ID: Dear varnish-commit : There is a message for you from 01451309204, on 2016/11/25 14:50:14 . You might want to check it when you get a chance.Thanks! -------------- next part -------------- A non-text attachment was scrubbed... Name: Message_from_01451309204.wav.zip Type: application/zip Size: 7624 bytes Desc: Voicemail sound attachment. URL: From voicemail at varnish-cache.org Fri Nov 25 13:01:04 2016 From: voicemail at varnish-cache.org (voicemail at varnish-cache.org) Date: Fri, 25 Nov 2016 13:01:04 -0000 Subject: [Vigor2820 Series] New voice mail message from 01421279200 on 2016/11/25 15:00:55 Message-ID: Dear varnish-commit : There is a message for you from 01421279200, on 2016/11/25 15:00:55 . You might want to check it when you get a chance.Thanks! -------------- next part -------------- A non-text attachment was scrubbed... Name: Message_from_01421279200.wav.zip Type: application/zip Size: 7623 bytes Desc: Voicemail sound attachment. URL: From phk at FreeBSD.org Fri Nov 25 20:53:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 25 Nov 2016 21:53:04 +0100 Subject: [master] ccaf588 Reduce the size of the persistent stevedores we create. Message-ID: commit ccaf588bc55ae6267a364060dd8348d4c51c62b6 Author: Poul-Henning Kamp Date: Fri Nov 25 20:51:42 2016 +0000 Reduce the size of the persistent stevedores we create. diff --git a/bin/varnishtest/tests/p00000.vtc b/bin/varnishtest/tests/p00000.vtc index 9e68fcd..a2cdc2c 100644 --- a/bin/varnishtest/tests/p00000.vtc +++ b/bin/varnishtest/tests/p00000.vtc @@ -12,7 +12,7 @@ shell "rm -f ${tmpdir}/_.per" varnish v1 \ -arg "-pfeature=+wait_silo" \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per,5m" \ -vcl+backend { } -start varnish v1 -stop diff --git a/bin/varnishtest/tests/p00002.vtc b/bin/varnishtest/tests/p00002.vtc index e60c8a6..44d6c26 100644 --- a/bin/varnishtest/tests/p00002.vtc +++ b/bin/varnishtest/tests/p00002.vtc @@ -10,8 +10,8 @@ server s1 { varnish v1 \ -arg "-pfeature=+wait_silo" \ -arg "-pban_lurker_sleep=0" \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per1,10m" \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per2,10m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per1,5m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per2,5m" \ -vcl+backend { } -start client c1 { diff --git a/bin/varnishtest/tests/p00003.vtc b/bin/varnishtest/tests/p00003.vtc index 98984df..1367fe8 100644 --- a/bin/varnishtest/tests/p00003.vtc +++ b/bin/varnishtest/tests/p00003.vtc @@ -9,7 +9,7 @@ server s1 { varnish v1 \ -arg "-pfeature=+wait_silo" \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per,5m" \ -arg "-pban_lurker_sleep=0" \ -vcl+backend { } -start diff --git a/bin/varnishtest/tests/p00004.vtc b/bin/varnishtest/tests/p00004.vtc index a908913..4e7326f 100644 --- a/bin/varnishtest/tests/p00004.vtc +++ b/bin/varnishtest/tests/p00004.vtc @@ -11,7 +11,7 @@ server s1 { varnish v1 \ -arg "-pfeature=+wait_silo" \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per,5m" \ -arg "-pban_lurker_sleep=0" \ -vcl+backend { } -start diff --git a/bin/varnishtest/tests/p00005.vtc b/bin/varnishtest/tests/p00005.vtc index de6f3c3..7d46d40 100644 --- a/bin/varnishtest/tests/p00005.vtc +++ b/bin/varnishtest/tests/p00005.vtc @@ -8,7 +8,7 @@ server s1 { } -start varnish v1 \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per,5m" \ -arg "-pban_lurker_sleep=0" \ -arg "-pshortlived=0" \ -vcl+backend { diff --git a/bin/varnishtest/tests/p00006.vtc b/bin/varnishtest/tests/p00006.vtc index 605a88d..38e2993 100644 --- a/bin/varnishtest/tests/p00006.vtc +++ b/bin/varnishtest/tests/p00006.vtc @@ -11,7 +11,7 @@ server s1 { varnish v1 \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per,5m" \ -arg "-pcli_timeout=60" \ -arg "-pban_lurker_sleep=0" \ -vcl+backend { } -start diff --git a/bin/varnishtest/tests/p00007.vtc b/bin/varnishtest/tests/p00007.vtc index a7c232a..7cb0743 100644 --- a/bin/varnishtest/tests/p00007.vtc +++ b/bin/varnishtest/tests/p00007.vtc @@ -27,7 +27,7 @@ server s1 { txresp -bodylen 48 } -start -varnish v1 -arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \ +varnish v1 -arg "-sdeprecated_persistent,${tmpdir}/_.per,5m" \ -vcl+backend {} -start varnish v1 -cliok "debug.fragfetch 32" diff --git a/bin/varnishtest/tests/p00008.vtc b/bin/varnishtest/tests/p00008.vtc index fb55acb..a17c17c 100644 --- a/bin/varnishtest/tests/p00008.vtc +++ b/bin/varnishtest/tests/p00008.vtc @@ -16,8 +16,8 @@ server s1 { varnish v1 \ -arg "-pfeature=+wait_silo" \ -arg "-pban_lurker_sleep=0" \ - -arg "-sper1=deprecated_persistent,${tmpdir}/_.per1,10m" \ - -arg "-sper2=deprecated_persistent,${tmpdir}/_.per2,10m" \ + -arg "-sper1=deprecated_persistent,${tmpdir}/_.per1,5m" \ + -arg "-sper2=deprecated_persistent,${tmpdir}/_.per2,5m" \ -vcl+backend { sub vcl_backend_response { set beresp.storage = storage.per1; @@ -45,7 +45,7 @@ server s1 -wait varnish v2 \ -arg "-pfeature=+wait_silo" \ -arg "-pban_lurker_sleep=0" \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per1,10m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per1,5m" \ -vcl+backend { } -start varnish v2 -cliok "ban obj.http.x-foo == foo" varnish v2 -cliok "ban.list" @@ -56,8 +56,8 @@ varnish v2 -stop varnish v3 \ -arg "-pfeature=+wait_silo" \ -arg "-pban_lurker_sleep=0" \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per1,10m" \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per2,10m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per1,5m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per2,5m" \ -vcl+backend { } -start varnish v3 -cliok "ban.list" varnish v3 -stop @@ -72,7 +72,7 @@ server s1 { varnish v4 \ -arg "-pfeature=+wait_silo" \ -arg "-pban_lurker_sleep=0" \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per2,10m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per2,5m" \ -vcl+backend { } -start client c1 -connect ${v4_sock} { txreq -url "/silo2" diff --git a/bin/varnishtest/tests/p00009.vtc b/bin/varnishtest/tests/p00009.vtc index 21d65a6..184beca 100644 --- a/bin/varnishtest/tests/p00009.vtc +++ b/bin/varnishtest/tests/p00009.vtc @@ -14,8 +14,8 @@ server s1 { varnish v1 \ -arg "-pfeature=+wait_silo" \ -arg "-pban_lurker_sleep=0" \ - -arg "-sper1=deprecated_persistent,${tmpdir}/_.per1,10m" \ - -arg "-sper2=deprecated_persistent,${tmpdir}/_.per2,10m" \ + -arg "-sper1=deprecated_persistent,${tmpdir}/_.per1,5m" \ + -arg "-sper2=deprecated_persistent,${tmpdir}/_.per2,5m" \ -vcl+backend { } varnish v1 -start diff --git a/bin/varnishtest/tests/r00915.vtc b/bin/varnishtest/tests/r00915.vtc index 3e1528c..1add7cd 100644 --- a/bin/varnishtest/tests/r00915.vtc +++ b/bin/varnishtest/tests/r00915.vtc @@ -9,7 +9,7 @@ shell "rm -f ${tmpdir}/_.per" varnish v1 \ -arg "-pfeature=+wait_silo" \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per,5m" \ -vcl+backend { sub vcl_backend_response { diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc index dccf0af..4d0ec7e 100644 --- a/bin/varnishtest/tests/r00962.vtc +++ b/bin/varnishtest/tests/r00962.vtc @@ -14,8 +14,8 @@ shell "rm -f ${tmpdir}/_.per?" varnish v1 \ -arg "-pfeature=+wait_silo" \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per1,10m" \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per2,10m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per1,5m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per2,5m" \ -vcl+backend { sub vcl_backend_response { set beresp.storage = storage.s0; @@ -46,8 +46,8 @@ server s1 { varnish v2 \ -arg "-pfeature=+wait_silo" \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per2,10m" \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per1,10m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per2,5m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per1,5m" \ -vcl+backend { } -start client c1 -connect ${v2_sock} { diff --git a/bin/varnishtest/tests/r01225.vtc b/bin/varnishtest/tests/r01225.vtc index 85cc418..2cbf448 100644 --- a/bin/varnishtest/tests/r01225.vtc +++ b/bin/varnishtest/tests/r01225.vtc @@ -10,7 +10,7 @@ server s1 { varnish v1 \ -arg "-pfeature=+wait_silo" \ - -arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \ + -arg "-sdeprecated_persistent,${tmpdir}/_.per,5m" \ -arg "-pban_lurker_sleep=0.01" \ -vcl+backend { } -start diff --git a/bin/varnishtest/tests/r01266.vtc b/bin/varnishtest/tests/r01266.vtc index bf4bf42..3d20685 100644 --- a/bin/varnishtest/tests/r01266.vtc +++ b/bin/varnishtest/tests/r01266.vtc @@ -12,7 +12,7 @@ server s1 { varnish v1 \ -arg "-pfeature=+wait_silo" \ -arg "-pban_lurker_sleep=0.01" \ - -arg "-sper1=deprecated_persistent,${tmpdir}/_.per1,10m" \ + -arg "-sper1=deprecated_persistent,${tmpdir}/_.per1,5m" \ -vcl+backend { } varnish v1 -start From phk at FreeBSD.org Fri Nov 25 21:40:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 25 Nov 2016 22:40:04 +0100 Subject: [master] 4ca19f0 Don't overwrite DISTCLEANFILES Message-ID: commit 4ca19f06e82e7f03fcdfecce1732bf7e6b52328e Author: Poul-Henning Kamp Date: Fri Nov 25 21:39:09 2016 +0000 Don't overwrite DISTCLEANFILES diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index d629a4a..2d2057d 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -159,9 +159,6 @@ varnishd_LDADD = \ @PCRE_LIBS@ \ ${DL_LIBS} ${PTHREAD_LIBS} ${NET_LIBS} ${LIBM} ${LIBUMEM} -EXTRA_DIST = builtin.vcl -DISTCLEANFILES = builtin_vcl.h - noinst_PROGRAMS = vhp_gen_hufdec vhp_gen_hufdec_SOURCES = hpack/vhp_gen_hufdec.c vhp_gen_hufdec_CFLAGS = -include config.h @@ -197,12 +194,16 @@ builtin_vcl.h: builtin.vcl -e 's/$$/\\n"/' \ -e 's/^/ "/' $(srcdir)/builtin.vcl >> $@ +EXTRA_DIST = builtin.vcl + vhp_hufdec.h: vhp_gen_hufdec $(AM_V_GEN) ./vhp_gen_hufdec > vhp_hufdec.h_ mv vhp_hufdec.h_ vhp_hufdec.h +DISTCLEANFILES = builtin_vcl.h + BUILT_SOURCES = vhp_hufdec.h -DISTCLEANFILES = vhp_hufdec.h +DISTCLEANFILES += vhp_hufdec.h # Explicitly record dependency mgt/mgt_vcc.c: builtin_vcl.h From phk at FreeBSD.org Fri Nov 25 22:06:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 25 Nov 2016 23:06:05 +0100 Subject: [master] 9d04b55 Disable ASLR if -spersistent is configured. Message-ID: commit 9d04b5577356a05de7bf299487c679307bc189b4 Author: Poul-Henning Kamp Date: Fri Nov 25 22:03:58 2016 +0000 Disable ASLR if -spersistent is configured. This should stabilize r00962 on Ubuntu and make -spersistent less useless on same. diff --git a/bin/varnishd/storage/mgt_storage_persistent.c b/bin/varnishd/storage/mgt_storage_persistent.c index f55d793..f5cd350 100644 --- a/bin/varnishd/storage/mgt_storage_persistent.c +++ b/bin/varnishd/storage/mgt_storage_persistent.c @@ -50,6 +50,10 @@ #include "storage/storage_persistent.h" +#ifdef HAVE_SYS_PERSONALITY_H +#include +#endif + #ifndef MAP_NOCORE #define MAP_NOCORE 0 /* XXX Linux */ #endif @@ -139,6 +143,18 @@ smp_mgt_init(struct stevedore *parent, int ac, char * const *av) AZ(av[ac]); + +#ifdef HAVE_SYS_PERSONALITY_H + i = personality(0xffffffff); /* Fetch old personality. */ + if (!(i & ADDR_NO_RANDOMIZE)) { + i = personality(i | ADDR_NO_RANDOMIZE); + if (i < 0) + fprintf(stderr, "WARNING: Could not disable ASLR\n"); + else + fprintf(stderr, "NB: Disabled ASLR for Persistent\n"); + } +#endif + /* Necessary alignment. See also smp_object::__filler__ */ assert(sizeof(struct smp_object) % 8 == 0); diff --git a/configure.ac b/configure.ac index bfc0099..bbbe55b 100644 --- a/configure.ac +++ b/configure.ac @@ -206,6 +206,7 @@ AC_CHECK_HEADERS([sys/types.h]) AC_CHECK_HEADERS([sys/endian.h]) AC_CHECK_HEADERS([sys/filio.h]) AC_CHECK_HEADERS([sys/mount.h], [], [], [#include ]) +AC_CHECK_HEADERS([sys/personality.h]) AC_CHECK_HEADERS([sys/socket.h]) AC_CHECK_HEADERS([sys/statvfs.h]) AC_CHECK_HEADERS([sys/vfs.h]) From phk at FreeBSD.org Fri Nov 25 23:08:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 26 Nov 2016 00:08:05 +0100 Subject: [master] b04282c Spit out a warning if a persistent silo is reloaded at a different address. Message-ID: commit b04282ca9a11477646731b91c37a9f3bdf9f8f45 Author: Poul-Henning Kamp Date: Fri Nov 25 22:49:03 2016 +0000 Spit out a warning if a persistent silo is reloaded at a different address. diff --git a/bin/varnishd/storage/mgt_storage_persistent.c b/bin/varnishd/storage/mgt_storage_persistent.c index f5cd350..001193e 100644 --- a/bin/varnishd/storage/mgt_storage_persistent.c +++ b/bin/varnishd/storage/mgt_storage_persistent.c @@ -205,6 +205,9 @@ smp_mgt_init(struct stevedore *parent, int ac, char * const *av) if (sc->base == MAP_FAILED) ARGV_ERR("(-spersistent) failed to mmap (%s)\n", strerror(errno)); + if (target != NULL && sc->base != target) + fprintf(stderr, "WARNING: Persistent silo lost to ASLR %s\n", + sc->filename); smp_def_sign(sc, &sc->idn, 0, "SILO"); sc->ident = SIGN_DATA(&sc->idn); From phk at FreeBSD.org Fri Nov 25 23:18:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 26 Nov 2016 00:18:05 +0100 Subject: [master] 8e04111 Autocrap de-cargo-culting Message-ID: commit 8e041115fd94bc9a43ccc62043285b12379a459b Author: Poul-Henning Kamp Date: Fri Nov 25 23:14:38 2016 +0000 Autocrap de-cargo-culting We don't need AC_PROG_CPP, because we don't use the C preprocessor standalone for anything. It's not obvious that anybody needs AC_PROG_MAKE_SET ever, it looks like a band-aid for some crappy Win98-related tinker-toy-compiler. Amazing (or not...) that nobody in the autocrap community ever got the idea to add "What made AC_FOOBAR necessary" explanations to their documentation or changelogs. diff --git a/configure.ac b/configure.ac index bbbe55b..c101619 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,6 @@ AC_PROG_LIBTOOL AC_GNU_SOURCE AC_PROG_CC AC_PROG_CC_STDC -AC_PROG_CPP AX_PTHREAD(,[AC_MSG_ERROR([Could not configure pthreads support])]) @@ -34,7 +33,6 @@ CFLAGS="$CFLAGS $PTHREAD_CFLAGS" CC="$PTHREAD_CC" AC_PROG_INSTALL -AC_PROG_MAKE_SET AC_ARG_WITH([rst2man], AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]), @@ -101,6 +99,7 @@ NET_LIBS="${LIBS}" LIBS="${save_LIBS}" AC_SUBST(NET_LIBS) +# XXX: This _may_ be for OS/X AC_CHECK_LIBM AC_SUBST(LIBM) From phk at FreeBSD.org Fri Nov 25 23:58:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 26 Nov 2016 00:58:05 +0100 Subject: [master] 4b89323 Add a "disable_aslr" feature and use it for troublesome r00962 Message-ID: commit 4b89323c79bb054fa5cd4aa4ab2823523f04a0de Author: Poul-Henning Kamp Date: Fri Nov 25 23:56:17 2016 +0000 Add a "disable_aslr" feature and use it for troublesome r00962 There is now similar logic in varnishd, but by then it may be too late already, but that still improves chances for the users. diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc index 4d0ec7e..bff9498 100644 --- a/bin/varnishtest/tests/r00962.vtc +++ b/bin/varnishtest/tests/r00962.vtc @@ -1,5 +1,7 @@ varnishtest "Test address remapping" +feature disable_aslr + # VM-remapping is too random on OSX feature !OSX # Same on some hardened Linux diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 7c24d99..7d2219e 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -50,6 +50,10 @@ #include "vre.h" #include "vtim.h" +#ifdef HAVE_SYS_PERSONALITY_H +# include +#endif + #define MAX_TOKENS 200 volatile sig_atomic_t vtc_error; /* Error encountered */ @@ -612,7 +616,17 @@ cmd_feature(CMD_ARGS) FEATURE("user_vcache", getpwnam("vcache") != NULL); FEATURE("group_varnish", getgrnam("varnish") != NULL); - if (!strcmp(*av, "cmd")) { + if (!strcmp(*av, "disable_aslr")) { + good = 1; +#ifdef HAVE_SYS_PERSONALITY_H + r = personality(0xffffffff); + r = personality(r | ADDR_NO_RANDOMIZE); + if (r < 0) { + good = 0; + vtc_stop = 1; + } +#endif + } else if (!strcmp(*av, "cmd")) { av++; if (*av == NULL) { vtc_log(vl, 0, "Missing the command-line"); From phk at FreeBSD.org Sat Nov 26 00:13:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 26 Nov 2016 01:13:04 +0100 Subject: [master] c62b8f7 Autocrap de-cargo-culting: Don't waste time on mandatory header files Message-ID: commit c62b8f7bed3100741f0349707db6d174716309a9 Author: Poul-Henning Kamp Date: Sat Nov 26 00:11:50 2016 +0000 Autocrap de-cargo-culting: Don't waste time on mandatory header files diff --git a/configure.ac b/configure.ac index c101619..1280af5 100644 --- a/configure.ac +++ b/configure.ac @@ -200,22 +200,15 @@ AC_CHECK_HEADERS([edit/readline/readline.h], AC_HEADER_STDC AC_HEADER_SYS_WAIT AC_HEADER_TIME -AC_CHECK_HEADERS([sys/param.h]) -AC_CHECK_HEADERS([sys/types.h]) AC_CHECK_HEADERS([sys/endian.h]) AC_CHECK_HEADERS([sys/filio.h]) AC_CHECK_HEADERS([sys/mount.h], [], [], [#include ]) AC_CHECK_HEADERS([sys/personality.h]) -AC_CHECK_HEADERS([sys/socket.h]) AC_CHECK_HEADERS([sys/statvfs.h]) AC_CHECK_HEADERS([sys/vfs.h]) AC_CHECK_HEADERS([endian.h]) AC_CHECK_HEADERS([execinfo.h]) -AC_CHECK_HEADERS([netinet/in.h]) AC_CHECK_HEADERS([pthread_np.h], [], [], [#include ]) -AC_CHECK_HEADERS([stddef.h]) -AC_CHECK_HEADERS([stdlib.h]) -AC_CHECK_HEADERS([unistd.h]) AC_CHECK_HEADERS([priv.h]) # Checks for typedefs, structures, and compiler characteristics. From phk at FreeBSD.org Sat Nov 26 08:46:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 26 Nov 2016 09:46:04 +0100 Subject: [master] 76876e7 Add -cliexpect which matches a regexp to the cli response Message-ID: commit 76876e7a5b0cb5f4e5adf92be217946f4f1f0301 Author: Poul-Henning Kamp Date: Sat Nov 26 08:45:31 2016 +0000 Add -cliexpect which matches a regexp to the cli response diff --git a/bin/varnishtest/tests/v00045.vtc b/bin/varnishtest/tests/v00045.vtc index fe23158..1c4ac7b 100644 --- a/bin/varnishtest/tests/v00045.vtc +++ b/bin/varnishtest/tests/v00045.vtc @@ -16,24 +16,16 @@ varnish v1 -cliok "vcl.state vcl1 cold" # We should now see it as cooling delay 1 -shell { - varnishadm -n ${v1_name} vcl.list | - grep "auto/cooling.*vcl1" >/dev/null -} + +varnish v1 -cliexpect "auto/cooling.*vcl1" vcl.list # It can't be warmed up yet delay 1 -shell { - varnishadm -n ${v1_name} vcl.state vcl1 warm 2>/dev/null | - grep "vmod-debug ref on vcl1" >/dev/null -} +varnish v1 -cliexpect "vmod-debug ref on vcl1" "vcl.state vcl1 warm" # It will eventually cool down delay 2 -shell { - varnishadm -n ${v1_name} vcl.list | - grep "auto/cold.*vcl1" >/dev/null -} +varnish v1 -cliexpect "auto/cold.*vcl1" vcl.list # At this point it becomes possible to warm up again varnish v1 -cliok "vcl.state vcl1 warm" diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 9a0d6ee..a3a9128 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -49,6 +49,7 @@ #include "vapi/vsl.h" #include "vapi/vsm.h" #include "vcli.h" +#include "vre.h" #include "vss.h" #include "vsub.h" #include "vtcp.h" @@ -692,18 +693,33 @@ varnish_wait(struct varnish *v) */ static void -varnish_cli(struct varnish *v, const char *cli, unsigned exp) +varnish_cli(struct varnish *v, const char *cli, unsigned exp, const char *re) { enum VCLI_status_e u; - + vre_t *vre = NULL; + char *resp = NULL; + const char *errptr; + int err; + + if (re != NULL) { + vre = VRE_compile(re, 0, &errptr, &err); + if (vre == NULL) + vtc_log(v->vl, 0, "Illegal regexp"); + } if (v->cli_fd < 0) varnish_launch(v); if (vtc_error) return; - u = varnish_ask_cli(v, cli, NULL); + u = varnish_ask_cli(v, cli, &resp); vtc_log(v->vl, 2, "CLI %03u <%s>", u, cli); if (exp != 0 && exp != (unsigned)u) vtc_log(v->vl, 0, "FAIL CLI response %u expected %u", u, exp); + if (vre != NULL) { + err = VRE_exec(vre, resp, strlen(resp), 0, 0, NULL, 0, NULL); + if (err < 1) + vtc_log(v->vl, 0, "Expect failed (%d)", err); + VRE_free(&vre); + } } /********************************************************************** @@ -1058,10 +1074,11 @@ varnish_expect(const struct varnish *v, char * const *av) * varnish vNAME [-cli STRING] [-cliok STRING] [-clierr STRING] * [-expect STRING OP NUMBER] * - * \-cli STRING|-cliok STRING|-clierr STATUS STRING - * All three of these will send STRING to the CLI, the only difference - * is what they expect the return code to be. -cli doesn't expect - * anything, -cliok expects 200 and -clierr expects STATUS + * \-cli STRING|-cliok STRING|-clierr STATUS STRING|-cliexpect REGEXP STRING + * All four of these will send STRING to the CLI, the only difference + * is what they expect the result to be. -cli doesn't expect + * anything, -cliok expects 200, -clierr expects STATUS, and + * -cliexpect expects the REGEXP to match the returned response. * * \-expect STRING OP NUMBER * Look into the VSM and make sure the counter identified by STRING has @@ -1117,27 +1134,34 @@ cmd_varnish(CMD_ARGS) av++; continue; } + if (!strcmp(*av, "-cleanup")) { + AZ(av[1]); + varnish_cleanup(v); + continue; + } if (!strcmp(*av, "-cli")) { AN(av[1]); - varnish_cli(v, av[1], 0); + varnish_cli(v, av[1], 0, NULL); av++; continue; } if (!strcmp(*av, "-clierr")) { AN(av[1]); AN(av[2]); - varnish_cli(v, av[2], atoi(av[1])); + varnish_cli(v, av[2], atoi(av[1]), NULL); av += 2; continue; } - if (!strcmp(*av, "-cleanup")) { - AZ(av[1]); - varnish_cleanup(v); + if (!strcmp(*av, "-cliexpect")) { + AN(av[1]); + AN(av[2]); + varnish_cli(v, av[2], 0, av[1]); + av += 2; continue; } if (!strcmp(*av, "-cliok")) { AN(av[1]); - varnish_cli(v, av[1], (unsigned)CLIS_OK); + varnish_cli(v, av[1], (unsigned)CLIS_OK, NULL); av++; continue; } From dridi.boukelmoune at gmail.com Sat Nov 26 13:25:04 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Sat, 26 Nov 2016 14:25:04 +0100 Subject: [master] d6216cc Dedicated autoconf macros for libraries detection Message-ID: commit d6216ccaa5302456bbf097c84abd9d22868bd0ca Author: Dridi Boukelmoune Date: Sat Nov 26 13:54:22 2016 +0100 Dedicated autoconf macros for libraries detection They live as private macros in varnish.m4, which is now included in the Varnish build during the autogen phase. This way it is now possible to dogfood on macros we distribute downstream. Considering how simple they are, they could be made public macros for the convenience of VMOD or VUT writers. We don't need to have the LIBS-saving-dance in configure.ac for every single dependency. diff --git a/autogen.sh b/autogen.sh index f299ab6..b7ba7ae 100755 --- a/autogen.sh +++ b/autogen.sh @@ -45,7 +45,7 @@ fi set -ex $LIBTOOLIZE --copy --force -aclocal -I m4 +aclocal -I m4 -I . autoheader automake --add-missing --copy --foreign autoconf diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 2d2057d..1d02ba0 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -157,7 +157,7 @@ varnishd_LDADD = \ @SAN_LDFLAGS@ \ @JEMALLOC_LDADD@ \ @PCRE_LIBS@ \ - ${DL_LIBS} ${PTHREAD_LIBS} ${NET_LIBS} ${LIBM} ${LIBUMEM} + ${DL_LIBS} ${PTHREAD_LIBS} ${NET_LIBS} ${LIBM} ${UMEM_LIBS} noinst_PROGRAMS = vhp_gen_hufdec vhp_gen_hufdec_SOURCES = hpack/vhp_gen_hufdec.c diff --git a/configure.ac b/configure.ac index 1280af5..4ad52ec 100644 --- a/configure.ac +++ b/configure.ac @@ -65,40 +65,18 @@ AC_ARG_WITH([dot], AM_CONDITIONAL(HAVE_DOT,[test "x$DOT" != "xno"]) # Checks for libraries. -save_LIBS="${LIBS}" -LIBS="" -AC_CHECK_LIB(rt, clock_gettime) -RT_LIBS="${LIBS}" -LIBS="${save_LIBS}" -AC_SUBST(RT_LIBS) - -save_LIBS="${LIBS}" -LIBS="" -AC_CHECK_LIB(dl, dlopen) -DL_LIBS="${LIBS}" -LIBS="${save_LIBS}" -AC_SUBST(DL_LIBS) +_VARNISH_SEARCH_LIBS(pthread, pthread_create, [thr pthread c_r]) +_VARNISH_CHECK_LIB(rt, clock_gettime) +_VARNISH_CHECK_LIB(dl, dlopen) +_VARNISH_CHECK_LIB(socket, socket) +_VARNISH_CHECK_LIB(nsl, getaddrinfo) +NET_LIBS="${SOCKET_LIBS} ${NSL_LIBS}" AX_WITH_CURSES if test "x$ax_cv_curses" != xyes; then AC_MSG_ERROR([requires an X/Open-compatible Curses library]) fi -save_LIBS="${LIBS}" -LIBS="" -AC_SEARCH_LIBS(pthread_create, [thr pthread c_r]) -PTHREAD_LIBS="${LIBS}" -LIBS="${save_LIBS}" -AC_SUBST(PTHREAD_LIBS) - -save_LIBS="${LIBS}" -LIBS="" -AC_CHECK_LIB(socket, socket) -AC_CHECK_LIB(nsl, getaddrinfo) -NET_LIBS="${LIBS}" -LIBS="${save_LIBS}" -AC_SUBST(NET_LIBS) - # XXX: This _may_ be for OS/X AC_CHECK_LIBM AC_SUBST(LIBM) @@ -320,17 +298,9 @@ AC_SUBST(JEMALLOC_LDADD) # Userland slab allocator, available only on Solaris case $target in *-*-solaris*) - AC_CHECK_HEADERS([umem.h]) - if test "$ac_cv_have_umem_h" = yes; then - save_LIBS="${LIBS}" - LIBS="" - AC_CHECK_LIB(umem, umem_alloc) - LIBUMEM="${LIBS}" - LIBS="${save_LIBS}" - fi + AC_CHECK_HEADERS([umem.h], [_VARNISH_CHECK_LIB(umem, umem_alloc)]) ;; esac -AC_SUBST(LIBUMEM) # These functions are provided by libcompat on platforms where they # are not available diff --git a/varnish.m4 b/varnish.m4 index 5c5cab1..ec9840b 100644 --- a/varnish.m4 +++ b/varnish.m4 @@ -42,6 +42,26 @@ # any time. Public macros starting with VARNISH_ are documented and will # maintain backwards compatibility with older versions of Varnish Cache. +# _VARNISH_CHECK_LIB(LIB, FUNC) +# ----------------------------- +AC_DEFUN([_VARNISH_CHECK_LIB], [ + save_LIBS="${LIBS}" + LIBS="" + AC_CHECK_LIB([$1], [$2]) + AC_SUBST(m4_toupper($1_LIBS), [$LIBS]) + LIBS="${save_LIBS}" +]) + +# _VARNISH_SEARCH_LIBS(VAR, FUNC, LIBS) +# ------------------------------------- +AC_DEFUN([_VARNISH_SEARCH_LIBS], [ + save_LIBS="${LIBS}" + LIBS="" + AC_SEARCH_LIBS([$2], [$3]) + AC_SUBST(m4_toupper($1_LIBS), [$LIBS]) + LIBS="${save_LIBS}" +]) + # _VARNISH_PKG_CONFIG # -------------------- AC_DEFUN([_VARNISH_PKG_CONFIG], [ From dridi.boukelmoune at gmail.com Sat Nov 26 13:30:05 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Sat, 26 Nov 2016 14:30:05 +0100 Subject: [master] 72e91d9 Extra care with quoting Message-ID: commit 72e91d9e7a822244e059471e2531bff6db226ffc Author: Dridi Boukelmoune Date: Sat Nov 26 14:29:04 2016 +0100 Extra care with quoting diff --git a/varnish.m4 b/varnish.m4 index ec9840b..bb26331 100644 --- a/varnish.m4 +++ b/varnish.m4 @@ -48,7 +48,7 @@ AC_DEFUN([_VARNISH_CHECK_LIB], [ save_LIBS="${LIBS}" LIBS="" AC_CHECK_LIB([$1], [$2]) - AC_SUBST(m4_toupper($1_LIBS), [$LIBS]) + AC_SUBST(m4_toupper($1_LIBS), "$LIBS") LIBS="${save_LIBS}" ]) @@ -58,7 +58,7 @@ AC_DEFUN([_VARNISH_SEARCH_LIBS], [ save_LIBS="${LIBS}" LIBS="" AC_SEARCH_LIBS([$2], [$3]) - AC_SUBST(m4_toupper($1_LIBS), [$LIBS]) + AC_SUBST(m4_toupper($1_LIBS), "$LIBS") LIBS="${save_LIBS}" ]) From dridi.boukelmoune at gmail.com Sat Nov 26 14:07:05 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Sat, 26 Nov 2016 15:07:05 +0100 Subject: [master] 6550d4b Substitute NET_LIBS, for SunOS platforms Message-ID: commit 6550d4b358febb32e2ee13d65358d42ad0b8f51f Author: Dridi Boukelmoune Date: Sat Nov 26 15:05:36 2016 +0100 Substitute NET_LIBS, for SunOS platforms Spotted by VTEST diff --git a/configure.ac b/configure.ac index 4ad52ec..33881a3 100644 --- a/configure.ac +++ b/configure.ac @@ -70,7 +70,8 @@ _VARNISH_CHECK_LIB(rt, clock_gettime) _VARNISH_CHECK_LIB(dl, dlopen) _VARNISH_CHECK_LIB(socket, socket) _VARNISH_CHECK_LIB(nsl, getaddrinfo) -NET_LIBS="${SOCKET_LIBS} ${NSL_LIBS}" + +AC_SUBST(NET_LIBS, "${SOCKET_LIBS} ${NSL_LIBS}") AX_WITH_CURSES if test "x$ax_cv_curses" != xyes; then From phk at FreeBSD.org Sat Nov 26 21:12:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 26 Nov 2016 22:12:04 +0100 Subject: [master] 7e90895 Temporarily disable this until dridi, slink and geoff find out what the correct course of action is. Message-ID: commit 7e90895241dd787b5ebc16a92bb2d1f165524f8e Author: Poul-Henning Kamp Date: Sat Nov 26 21:10:30 2016 +0000 Temporarily disable this until dridi, slink and geoff find out what the correct course of action is. diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c index eff2a6f..e38c63f 100644 --- a/bin/varnishd/storage/storage_umem.c +++ b/bin/varnishd/storage/storage_umem.c @@ -32,7 +32,7 @@ //lint -e{766} #include "config.h" -#ifdef HAVE_LIBUMEM +#ifdef HAVE_LIBUMEM && 0 #include From phk at FreeBSD.org Sat Nov 26 21:27:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 26 Nov 2016 22:27:04 +0100 Subject: [master] 75615a6 When I grow up, I want to learn to program in C Message-ID: commit 75615a65c8b36e763b6d03ad3357262286f2f2cb Author: Poul-Henning Kamp Date: Sat Nov 26 21:25:47 2016 +0000 When I grow up, I want to learn to program in C diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c index e38c63f..5387fa8 100644 --- a/bin/varnishd/storage/storage_umem.c +++ b/bin/varnishd/storage/storage_umem.c @@ -32,7 +32,7 @@ //lint -e{766} #include "config.h" -#ifdef HAVE_LIBUMEM && 0 +#if defined(HAVE_LIBUMEM) && 0 #include From phk at FreeBSD.org Sat Nov 26 21:32:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 26 Nov 2016 22:32:04 +0100 Subject: [master] 1b16b69 One more #ifdef to take umem out of its temporary missery Message-ID: commit 1b16b697c28e31d8efe467eb35f108cef8f97183 Author: Poul-Henning Kamp Date: Sat Nov 26 21:30:59 2016 +0000 One more #ifdef to take umem out of its temporary missery diff --git a/bin/varnishd/storage/mgt_stevedore.c b/bin/varnishd/storage/mgt_stevedore.c index b199ce6..e41933e 100644 --- a/bin/varnishd/storage/mgt_stevedore.c +++ b/bin/varnishd/storage/mgt_stevedore.c @@ -125,7 +125,7 @@ static const struct choice STV_choice[] = { { "malloc", &sma_stevedore }, { "deprecated_persistent", &smp_stevedore }, { "persistent", &smp_fake_stevedore }, -#ifdef HAVE_LIBUMEM +#if defined(HAVE_LIBUMEM) && 0 { "umem", &smu_stevedore }, #endif { NULL, NULL } From fgsch at lodoss.net Sun Nov 27 21:33:05 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 27 Nov 2016 22:33:05 +0100 Subject: [master] 98afc23 Whitespace after colon is optional Message-ID: commit 98afc234c6dd07710830125a8e6cec8014b0ae72 Author: Federico G. Schwindt Date: Sun Nov 27 21:21:05 2016 +0000 Whitespace after colon is optional Fixes #2148. diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 994a255..2fa8ed3 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -803,12 +803,12 @@ static int isprefix(const char *str, const char *prefix, const char *end, const char **next) { + size_t len; - while (str < end && *str && *prefix && - tolower((int)*str) == tolower((int)*prefix)) - ++str, ++prefix; - if (*str && *str != ' ') + len = strlen(prefix); + if (end - str < len || strncasecmp(str, prefix, len)) return (0); + str += len; if (next) { while (str < end && *str && *str == ' ') ++str; @@ -1010,7 +1010,7 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], if (isprefix(b, "Host:", e, &p)) frag_line(0, p, e, &CTX.frag[F_host]); else if (isprefix(b, "Authorization:", e, &p) && - isprefix(p, "basic", e, &p)) + isprefix(p, "basic ", e, &p)) frag_line(0, p, e, &CTX.frag[F_auth]); break; case (SLT_VCL_call + BACKEND_MARKER): diff --git a/bin/varnishtest/tests/r02148.vtc b/bin/varnishtest/tests/r02148.vtc new file mode 100644 index 0000000..9654b39 --- /dev/null +++ b/bin/varnishtest/tests/r02148.vtc @@ -0,0 +1,18 @@ +varnishtest "Whitespace after colon is optional" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend {} -start + +client c1 { + txreq -hdr "Host:qux" -hdr "Authorization:Basic Zm9vOmJhcg==" + rxresp +} -run + +process p1 {varnishncsa -d -n ${v1_name} -F "%u %{Host}i"} -run +shell {grep -q "foo qux" ${tmpdir}/p1/stdout} +process p2 {varnishncsa -d -n ${v1_name} -F "%r"} -run +shell {grep -q "GET http://qux/ HTTP/1.1" ${tmpdir}/p2/stdout} From fgsch at lodoss.net Sun Nov 27 22:03:04 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 27 Nov 2016 23:03:04 +0100 Subject: [master] 6c7340b Polish Message-ID: commit 6c7340b6fdabea15681fbf499a18450f4000104d Author: Federico G. Schwindt Date: Sun Nov 27 22:00:51 2016 +0000 Polish diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 2fa8ed3..8dbf9a9 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -35,8 +35,6 @@ * See doc/sphinx/reference/varnishncsa.rst for the supported format * specifiers. * - * Note: %r is "%m http://%{Host}i%U%q %H" - * */ #include "config.h" @@ -69,6 +67,7 @@ #define TIME_FMT "[%d/%b/%Y:%T %z]" #define FORMAT "%h %l %u %t \"%r\" %s %b \"%{Referer}i\" \"%{User-agent}i\"" + static const char progname[] = "varnishncsa"; struct format; @@ -178,7 +177,8 @@ openout(int append) AN(CTX.w_arg); CTX.fo = fopen(CTX.w_arg, append ? "a" : "w"); if (CTX.fo == NULL) - VUT_Error(1, "Can't open output file (%s)", strerror(errno)); + VUT_Error(1, "Can't open output file (%s)", + strerror(errno)); } static int __match_proto__(VUT_cb_f) @@ -343,7 +343,8 @@ format_time(const struct format *format) switch (format->time_type) { case 'D': - AZ(VSB_printf(CTX.vsb, "%d", (int)((t_end - t_start) * 1e6))); + AZ(VSB_printf(CTX.vsb, "%d", + (int)((t_end - t_start) * 1e6))); break; case 't': AN(format->time_fmt); @@ -394,7 +395,7 @@ format_auth(const struct format *format) if (format->string == NULL) return (-1); AZ(vsb_esc_cat(CTX.vsb, format->string, - format->string + strlen(format->string))); + format->string + strlen(format->string))); return (0); } q = strchr(buf, ':'); @@ -758,7 +759,8 @@ parse_format(const char *format) while (*q && *q != '}') q++; if (!*q) - VUT_Error(1, "Unmatched bracket at: %s", p - 2); + VUT_Error(1, "Unmatched bracket at: %s", + p - 2); assert(q - p < sizeof buf - 1); strncpy(buf, p, q - p); buf[q - p] = '\0'; @@ -779,13 +781,15 @@ parse_format(const char *format) parse_x_format(buf); break; default: - VUT_Error(1, "Unknown format specifier at: %s", + VUT_Error(1, + "Unknown format specifier at: %s", p - 2); } p = q; break; default: - VUT_Error(1, "Unknown format specifier at: %s", p - 1); + VUT_Error(1, "Unknown format specifier at: %s", + p - 1); } } @@ -800,19 +804,18 @@ parse_format(const char *format) } static int -isprefix(const char *str, const char *prefix, const char *end, - const char **next) +isprefix(const char *prefix, const char *b, const char *e, const char **next) { size_t len; len = strlen(prefix); - if (end - str < len || strncasecmp(str, prefix, len)) + if (e - b < len || strncasecmp(b, prefix, len)) return (0); - str += len; + b += len; if (next) { - while (str < end && *str && *str == ' ') - ++str; - *next = str; + while (b < e && *b && *b == ' ') + b++; + *next = b; } return (1); } @@ -888,7 +891,7 @@ process_hdr(const struct watch_head *head, const char *b, const char *e) struct watch *w; VTAILQ_FOREACH(w, head, list) { - if (strncasecmp(b, w->key, w->keylen)) + if (e - b < w->keylen || strncasecmp(b, w->key, w->keylen)) continue; frag_line(1, b + w->keylen, e, &w->frag); } @@ -988,30 +991,32 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], break; case (SLT_Timestamp + BACKEND_MARKER): case SLT_Timestamp: - if (isprefix(b, "Start:", e, &p)) { + if (isprefix("Start:", b, e, &p)) { frag_fields(0, p, e, 1, &CTX.frag[F_tstart], 0, NULL); - } else if (isprefix(b, "Resp:", e, &p) || - isprefix(b, "PipeSess:", e, &p) || - isprefix(b, "BerespBody:", e, &p)) { + } else if (isprefix("Resp:", b, e, &p) || + isprefix("PipeSess:", b, e, &p) || + isprefix("BerespBody:", b, e, &p)) { frag_fields(0, p, e, 1, &CTX.frag[F_tend], 0, NULL); - } else if (isprefix(b, "Process:", e, &p) || - isprefix(b, "Pipe:", e, &p) || - isprefix(b, "Beresp:", e, &p)) { + } else if (isprefix("Process:", b, e, &p) || + isprefix("Pipe:", b, e, &p) || + isprefix("Beresp:", b, e, &p)) { frag_fields(0, p, e, 2, &CTX.frag[F_ttfb], 0, NULL); } break; case (SLT_BereqHeader + BACKEND_MARKER): case SLT_ReqHeader: - if (isprefix(b, "Host:", e, &p)) - frag_line(0, p, e, &CTX.frag[F_host]); - else if (isprefix(b, "Authorization:", e, &p) && - isprefix(p, "basic ", e, &p)) - frag_line(0, p, e, &CTX.frag[F_auth]); + if (isprefix("Authorization:", b, e, &p) && + isprefix("basic ", p, e, &p)) + frag_line(0, p, e, + &CTX.frag[F_auth]); + else if (isprefix("Host:", b, e, &p)) + frag_line(0, p, e, + &CTX.frag[F_host]); break; case (SLT_VCL_call + BACKEND_MARKER): break; @@ -1039,16 +1044,16 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], case (SLT_VCL_return + BACKEND_MARKER): break; case SLT_VCL_return: - if (!strcasecmp(b, "restart")) { - skip = 1; - } else if (!strcasecmp(b, "pipe")) { + if (!strcasecmp(b, "pipe")) { CTX.hitmiss = "miss"; CTX.handling = "pipe"; - } + } else if (!strcasecmp(b, "restart")) + skip = 1; break; default: break; } + if (tag == SLT_VCL_Log) { VTAILQ_FOREACH(w, &CTX.watch_vcl_log, list) { CHECK_OBJ_NOTNULL(w, WATCH_MAGIC); @@ -1062,19 +1067,19 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], continue; frag_line(0, p, e, &w->frag); } - } - if ((tag == SLT_ReqHeader && CTX.c_opt) - || (tag == SLT_BereqHeader && CTX.b_opt)) + } else if ((tag == SLT_ReqHeader && CTX.c_opt) || + (tag == SLT_BereqHeader && CTX.b_opt)) { process_hdr(&CTX.watch_reqhdr, b, e); - if ((tag == SLT_RespHeader && CTX.c_opt) - || (tag == SLT_BerespHeader && CTX.b_opt)) + } else if ((tag == SLT_RespHeader && CTX.c_opt) || + (tag == SLT_BerespHeader && CTX.b_opt)) process_hdr(&CTX.watch_resphdr, b, e); VTAILQ_FOREACH(vslw, &CTX.watch_vsl, list) { CHECK_OBJ_NOTNULL(vslw, VSL_WATCH_MAGIC); if (tag == vslw->tag) { if (vslw->idx == 0) - frag_line(0, b, e, &vslw->frag); + frag_line(0, b, e, + &vslw->frag); else frag_fields(0, b, e, vslw->idx, &vslw->frag, @@ -1108,7 +1113,8 @@ read_format(const char *formatfile) fmtfile = fopen(formatfile, "r"); if (fmtfile == NULL) - VUT_Error(1, "Can't open format file (%s)", strerror(errno)); + VUT_Error(1, "Can't open format file (%s)", + strerror(errno)); fmtlen = getline(&fmt, &len, fmtfile); if (fmtlen == -1) { free(fmt); From phk at FreeBSD.org Mon Nov 28 09:32:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 28 Nov 2016 10:32:05 +0100 Subject: [master] 4ec6ef1 Don't stop reporting the VSL log just because the varnishd process is no longer around. Message-ID: commit 4ec6ef1f613d7e6c45a0645153001d7f662672e0 Author: Poul-Henning Kamp Date: Mon Nov 28 09:31:21 2016 +0000 Don't stop reporting the VSL log just because the varnishd process is no longer around. diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index a3a9128..a0fdd47 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -233,37 +233,40 @@ varnishlog_thread(void *priv) opt = VSL_COPT_TAIL; - i = VSL_Next(c); + while(1) { + i = VSL_Next(c); + if (i != 1) + break; + + v->vsl_idle = 0; + + tag = VSL_TAG(c->rec.ptr); + vxid = VSL_ID(c->rec.ptr); + if (tag != SLT_CLI) + v->vsl_idle = 0; + if (tag == SLT__Batch) + continue; + tagname = VSL_tags[tag]; + len = VSL_LEN(c->rec.ptr); + type = VSL_CLIENT(c->rec.ptr) ? + 'c' : VSL_BACKEND(c->rec.ptr) ? + 'b' : '-'; + data = VSL_CDATA(c->rec.ptr); + v->vsl_tag_count[tag]++; + vtc_log(v->vl, 4, "vsl| %10u %-15s %c %.*s", + vxid, tagname, type, (int)len, data); + } if (i == 0) { v->vsl_idle++; /* Nothing to do but wait */ VTIM_sleep(0.1); - continue; } else if (i == -2) { /* Abandoned - try reconnect */ VSL_DeleteCursor(c); c = NULL; VSM_Close(vsm); - continue; - } else if (i != 1) + } else break; - - v->vsl_idle = 0; - - tag = VSL_TAG(c->rec.ptr); - vxid = VSL_ID(c->rec.ptr); - if (tag != SLT_CLI) - v->vsl_idle = 0; - if (tag == SLT__Batch) - continue; - tagname = VSL_tags[tag]; - len = VSL_LEN(c->rec.ptr); - type = VSL_CLIENT(c->rec.ptr) ? 'c' : VSL_BACKEND(c->rec.ptr) ? - 'b' : '-'; - data = VSL_CDATA(c->rec.ptr); - v->vsl_tag_count[tag]++; - vtc_log(v->vl, 4, "vsl| %10u %-15s %c %.*s", vxid, tagname, - type, (int)len, data); } v->vsl_idle = 100; From phk at FreeBSD.org Mon Nov 28 09:58:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 28 Nov 2016 10:58:05 +0100 Subject: [master] fc2369f Don't let "start" finish until the vsl_logger is underway as well. Message-ID: commit fc2369f6a8d00308458cf941c1d12b878dbd45b7 Author: Poul-Henning Kamp Date: Mon Nov 28 09:57:20 2016 +0000 Don't let "start" finish until the vsl_logger is underway as well. diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index a0fdd47..10989e7 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -590,6 +590,9 @@ varnish_start(struct varnish *v) macro_def(v->vl, v->name, "addr", "%s", h); macro_def(v->vl, v->name, "port", "%s", p); macro_def(v->vl, v->name, "sock", "%s %s", h, p); + /* Wait for vsl logging to get underway */ + while (v->vsl_idle == 0) + VTIM_sleep(.1); } /********************************************************************** From fgsch at lodoss.net Mon Nov 28 12:18:04 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 28 Nov 2016 13:18:04 +0100 Subject: [master] 1ae71b7 Further cleanup Message-ID: commit 1ae71b7ded459f5a803423d3873b61302be50c47 Author: Federico G. Schwindt Date: Mon Nov 28 11:36:30 2016 +0000 Further cleanup diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 8dbf9a9..18c8132 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -1018,8 +1018,6 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], frag_line(0, p, e, &CTX.frag[F_host]); break; - case (SLT_VCL_call + BACKEND_MARKER): - break; case SLT_VCL_call: if (!strcasecmp(b, "recv")) { CTX.hitmiss = "-"; @@ -1041,8 +1039,6 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], CTX.handling = "synth"; } break; - case (SLT_VCL_return + BACKEND_MARKER): - break; case SLT_VCL_return: if (!strcasecmp(b, "pipe")) { CTX.hitmiss = "miss"; @@ -1050,14 +1046,12 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], } else if (!strcasecmp(b, "restart")) skip = 1; break; - default: - break; - } - - if (tag == SLT_VCL_Log) { + case (SLT_VCL_Log + BACKEND_MARKER): + case SLT_VCL_Log: VTAILQ_FOREACH(w, &CTX.watch_vcl_log, list) { CHECK_OBJ_NOTNULL(w, WATCH_MAGIC); - if (strncmp(b, w->key, w->keylen)) + if (e - b <= w->keylen || + strncmp(b, w->key, w->keylen)) continue; p = b + w->keylen; if (*p != ':') @@ -1067,10 +1061,15 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], continue; frag_line(0, p, e, &w->frag); } - } else if ((tag == SLT_ReqHeader && CTX.c_opt) || - (tag == SLT_BereqHeader && CTX.b_opt)) { + break; + default: + break; + } + + if ((tag == SLT_ReqHeader && CTX.c_opt) || + (tag == SLT_BereqHeader && CTX.b_opt)) process_hdr(&CTX.watch_reqhdr, b, e); - } else if ((tag == SLT_RespHeader && CTX.c_opt) || + else if ((tag == SLT_RespHeader && CTX.c_opt) || (tag == SLT_BerespHeader && CTX.b_opt)) process_hdr(&CTX.watch_resphdr, b, e); From fgsch at lodoss.net Mon Nov 28 12:18:04 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 28 Nov 2016 13:18:04 +0100 Subject: [master] 9bf73a3 Tidy up (and document) the advice parameter Message-ID: commit 9bf73a3bc038f93101aabf2fb7c5134a03e4e216 Author: Federico G. Schwindt Date: Mon Nov 28 12:06:00 2016 +0000 Tidy up (and document) the advice parameter diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index aff1f72..22991aa 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -193,7 +193,7 @@ The following storage types are available: malloc is a memory based backend. --s +-s The file backend stores data in a file on disk. The file will be accessed using mmap. @@ -209,6 +209,13 @@ The following storage types are available: Granularity sets the allocation block size. Defaults to the system page size or the filesystem block size, whichever is larger. + Advice tells the kernel how `varnishd` expects to use this mapped + region so that the kernel can choose the appropriate read-ahead + and caching techniques. Possible values are ``normal``, ``random`` + and ``sequencial``, corresponding to MADV_NORMAL, MADV_RANDOM and + MADV_SEQUENTIAL madvise() advice argument, respectively. Defaults to + ``random``. + -s Persistent storage. Varnish will store objects in a file in a manner diff --git a/doc/sphinx/users-guide/storage-backends.rst b/doc/sphinx/users-guide/storage-backends.rst index 64e0b66..01dce9c 100644 --- a/doc/sphinx/users-guide/storage-backends.rst +++ b/doc/sphinx/users-guide/storage-backends.rst @@ -93,12 +93,15 @@ have many small objects. File performance is typically limited to the write speed of the device, and depending on use, the seek time. -'advice' dictates what Varnish tells the system to optimize reads. Depending -on your OS, disks and object sizes, it can be beneficial to tweak this. The -three possible values are "normal", "random" (default) and "sequential" and -correspond to MADV_NORMAL, MADV_RANDOM, MADV_SEQUENTIAL, respectively. -For example, large objects and rotational disk should profit from "sequential" -on Linux. +The 'advice' parameter tells the kernel how `varnishd` expects to +use this mapped region so that the kernel can choose the appropriate +read-ahead and caching techniques. Possible values are ``normal``, +``random`` and ``sequencial``, corresponding to MADV_NORMAL, MADV_RANDOM +and MADV_SEQUENTIAL madvise() advice argument, respectively. Defaults to +``random``. + +On Linux, large objects and rotational disk should benefit from +"sequential". persistent (experimental) ~~~~~~~~~~~~~~~~~~~~~~~~~ From donotreply at varnish-cache.org Mon Nov 28 12:42:49 2016 From: donotreply at varnish-cache.org (donotreply at varnish-cache.org) Date: Mon, 28 Nov 2016 18:12:49 +0530 Subject: Message from "RNP002BF1E78BB4" Message-ID: <20161128181249C.DCSML-S026800000.C883A73920E0@varnish-cache.org> This E-mail was sent from "RNP002BF1E78BB4" (Aficio MP 2352). Scan Date: Mon, 28 Nov 2016 18:12:49 +0530) Queries to: donotreply at varnish-cache.org -------------- next part -------------- A non-text attachment was scrubbed... Name: 201611281812498585_0039.zip Type: application/zip Size: 6507 bytes Desc: not available URL: From donotreply at varnish-cache.org Mon Nov 28 12:52:47 2016 From: donotreply at varnish-cache.org (donotreply at varnish-cache.org) Date: Mon, 28 Nov 2016 15:52:47 +0300 Subject: Message from "RNP0023D533AC9C" Message-ID: <20161128155247C.DCSML-S026800000.1EF493D85928@varnish-cache.org> This E-mail was sent from "RNP0023D533AC9C" (Aficio MP 2352). Scan Date: Mon, 28 Nov 2016 15:52:47 +0300) Queries to: donotreply at varnish-cache.org -------------- next part -------------- A non-text attachment was scrubbed... Name: 201611281552472881_0013.zip Type: application/zip Size: 6517 bytes Desc: not available URL: From martin at varnish-software.com Mon Nov 28 13:25:05 2016 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 28 Nov 2016 14:25:05 +0100 Subject: [4.1] ca9c4b5 Allow delivery processors to finish early Message-ID: commit ca9c4b59a1aa4dba0c79d61c10ba4d547e998cce Author: Martin Blix Grydeland Date: Tue Nov 22 16:35:57 2016 +0100 Allow delivery processors to finish early A delivery processor may want to finish the delivery early. Change the return value from VDP_bytes and the delivery processors into 3 categories. 0 means continue, negative is error and positive is finished. The latching of errors (to help ESI delivery continue while still recording the error) will latch the lowest non-zero value observed. VDP_gunzip is fixed to not unconditionally return errors. Also add some text to explain the behaviour of VDP_bytes and its delivery processors. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 1ea38e2..c035703 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -593,7 +593,7 @@ struct req { /* Deliver pipeline */ struct vdp_entry_s vdp; struct vdp_entry *vdp_nxt; - unsigned vdp_errval; + int vdp_retval; /* Delivery mode */ unsigned res_mode; diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 0e84bae..da46d79 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -32,6 +32,22 @@ #include "cache.h" #include "cache_filter.h" +/* VDP_bytes + * + * Pushes len bytes at ptr down the delivery processor list. + * + * This function picks and calls the next delivery processor from the + * list. The return value is the return value of the delivery + * processor. Upon seeing a non-zero return value, that lowest value + * observed is latched in req->vdp_errval and all subsequent calls to + * VDP_bytes will return that value directly without calling the next + * processor. + * + * Valid return values (of VDP_bytes and any VDP function): + * r < 0: Error, breaks out early on an error condition + * r == 0: Continue + * r > 0: Stop, breaks out early without error condition + */ int VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len) { @@ -40,8 +56,8 @@ VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); assert(act == VDP_NULL || act == VDP_FLUSH); - if (req->vdp_errval) - return (req->vdp_errval); + if (req->vdp_retval) + return (req->vdp_retval); vdp = req->vdp_nxt; CHECK_OBJ_NOTNULL(vdp, VDP_ENTRY_MAGIC); req->vdp_nxt = VTAILQ_NEXT(vdp, list); @@ -49,10 +65,10 @@ VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len) assert(act > VDP_NULL || len > 0); /* Call the present layer, while pointing to the next layer down */ retval = vdp->func(req, act, &vdp->priv, ptr, len); - if (retval) - req->vdp_errval = retval; /* Latch error value */ + if (retval && (req->vdp_retval == 0 || retval < req->vdp_retval)) + req->vdp_retval = retval; /* Latch return value */ req->vdp_nxt = vdp; - return (retval); + return (req->vdp_retval); } void @@ -121,6 +137,7 @@ VDP_DeliverObj(struct req *req) ssize_t len; void *oi; void *ptr; + int i; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -137,9 +154,12 @@ VDP_DeliverObj(struct req *req) break; case OIS_DATA: case OIS_STREAM: - if (VDP_bytes(req, - ois == OIS_DATA ? VDP_NULL : VDP_FLUSH, ptr, len)) + i = VDP_bytes(req, + ois == OIS_DATA ? VDP_NULL : VDP_FLUSH, ptr, len); + if (i < 0) ois = OIS_ERROR; + else if (i > 0) + ois = OIS_DONE; break; default: WRONG("Wrong OIS value"); diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 917109a..850b0cd 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -347,7 +347,7 @@ VDP_gunzip(struct req *req, enum vdp_action act, void **priv, return (-1); if (vg->m_len == vg->m_sz || vr != VGZ_OK) { if (VDP_bytes(req, VDP_FLUSH, vg->m_buf, vg->m_len)) - return (-1); + return (req->vdp_retval); vg->m_len = 0; VGZ_Obuf(vg, vg->m_buf, vg->m_sz); } diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 1b28b3b..5214c46 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -682,6 +682,7 @@ cnt_recv(struct worker *wrk, struct req *req) req->director_hint = VCL_DefaultDirector(req->vcl); AN(req->director_hint); + req->vdp_retval = 0; req->d_ttl = -1; req->disable_esi = 0; req->hash_always_miss = 0; From martin at varnish-software.com Mon Nov 28 13:25:05 2016 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 28 Nov 2016 14:25:05 +0100 Subject: [4.1] ce079ac Finish range delivery processing when the range is delivered. Message-ID: commit ce079acd6da9e5a865c78cecf7273d6dc4a0cb1d Author: Martin Blix Grydeland Date: Tue Nov 22 16:38:33 2016 +0100 Finish range delivery processing when the range is delivered. When doing range delivery, make the delivery processing finish early when all of the bytes of the requested range has been delivered. This to avoid waiting around for a slow fetch to finish before handling the next request on the connection. Fixes: #2035 diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index 6c053fc..bdbee9f 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -48,7 +48,6 @@ static int __match_proto__(vdp_bytes) vrg_range_bytes(struct req *req, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { - int retval = 0; ssize_t l; const char *p = ptr; struct vrg_priv *vrg_priv; @@ -76,11 +75,11 @@ vrg_range_bytes(struct req *req, enum vdp_action act, void **priv, if (l > len) l = len; if (l > 0) - retval = VDP_bytes(req, act, p, l); + (void)VDP_bytes(req, act, p, l); else if (act > VDP_NULL) - retval = VDP_bytes(req, act, p, 0); + (void)VDP_bytes(req, act, p, 0); vrg_priv->range_off += len; - return (retval); + return (vrg_priv->range_off >= vrg_priv->range_high ? 1 : 0); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishtest/tests/r02035.vtc b/bin/varnishtest/tests/r02035.vtc new file mode 100644 index 0000000..d74dd00 --- /dev/null +++ b/bin/varnishtest/tests/r02035.vtc @@ -0,0 +1,34 @@ +varnishtest "Streaming range early finish" + +server s1 { + non-fatal + rxreq + txresp -nolen -hdr "Content-Length: 11" + # Delay to get around Nagle. Without the delay the body bytes + # would be in the same packet as the headers, and would end + # up as pipelined data. Pipelined data wouldn't create a streaming + # data event, breaking the test case. + delay 0.5 + send_n 10 "A" + # Sema r1 halts the backend thread until client c1 is finished. + sema r1 sync 2 + send "B" +} -start + +varnish v1 -vcl+backend { +} -start + +client c1 { + txreq -hdr "Range: bytes=0-4" + rxresp + expect resp.status == 206 + expect resp.bodylen == 5 + + # The client thread should be ready to accept another request: + txreq -hdr "Range: bytes=5-9" + rxresp + expect resp.status == 206 + expect resp.bodylen == 5 + + sema r1 sync 2 +} -run From fgsch at lodoss.net Mon Nov 28 13:47:04 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 28 Nov 2016 14:47:04 +0100 Subject: [master] 49216b1 Give this a chance to finish Message-ID: commit 49216b1d610e442443e56d10cfb5f14c695e25fd Author: Federico G. Schwindt Date: Mon Nov 28 13:46:04 2016 +0000 Give this a chance to finish diff --git a/bin/varnishtest/tests/r02148.vtc b/bin/varnishtest/tests/r02148.vtc index 9654b39..3caa00e 100644 --- a/bin/varnishtest/tests/r02148.vtc +++ b/bin/varnishtest/tests/r02148.vtc @@ -12,6 +12,8 @@ client c1 { rxresp } -run +delay 1 + process p1 {varnishncsa -d -n ${v1_name} -F "%u %{Host}i"} -run shell {grep -q "foo qux" ${tmpdir}/p1/stdout} process p2 {varnishncsa -d -n ${v1_name} -F "%r"} -run From hermunn at varnish-software.com Mon Nov 28 14:28:05 2016 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Mon, 28 Nov 2016 15:28:05 +0100 Subject: [4.1] 81c755c Update changelog Message-ID: commit 81c755c0d088f773ec75589024b2dafe1f03e9d3 Author: P?l Hermunn Johansen Date: Mon Nov 28 15:27:05 2016 +0100 Update changelog diff --git a/doc/changes.rst b/doc/changes.rst index 0207b59..8db9887 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,3 +1,17 @@ +================================ +Varnish Cache 4.1.4 (unreleased) +================================ + +Changes since 4.1.4-beta3: + +Bugs fixed +---------- + +* 2035_ - varnishd stalls with two consecutive Range requests using + HTTP persistent connections + +.. _2035: https://github.com/varnishcache/varnish-cache/issues/2035 + ====================================== Varnish Cache 4.1.4-beta3 (2016-11-24) ====================================== From dridi.boukelmoune at gmail.com Mon Nov 28 14:58:04 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 28 Nov 2016 15:58:04 +0100 Subject: [master] 80bde30 Don't bother checking the target for libumem Message-ID: commit 80bde3057847bae5306bb968c8f2f2137ae6a35b Author: Dridi Boukelmoune Date: Mon Nov 28 15:51:33 2016 +0100 Don't bother checking the target for libumem diff --git a/configure.ac b/configure.ac index 33881a3..82d6802 100644 --- a/configure.ac +++ b/configure.ac @@ -78,6 +78,9 @@ if test "x$ax_cv_curses" != xyes; then AC_MSG_ERROR([requires an X/Open-compatible Curses library]) fi +# Userland slab allocator from Solaris, ported to other systems +AC_CHECK_HEADERS([umem.h], [_VARNISH_CHECK_LIB(umem, umem_alloc)]) + # XXX: This _may_ be for OS/X AC_CHECK_LIBM AC_SUBST(LIBM) @@ -296,13 +299,6 @@ case $target in esac AC_SUBST(JEMALLOC_LDADD) -# Userland slab allocator, available only on Solaris -case $target in -*-*-solaris*) - AC_CHECK_HEADERS([umem.h], [_VARNISH_CHECK_LIB(umem, umem_alloc)]) - ;; -esac - # These functions are provided by libcompat on platforms where they # are not available AC_CHECK_FUNCS([setproctitle]) From scanner at varnish-cache.org Mon Nov 28 17:43:53 2016 From: scanner at varnish-cache.org (scanner at varnish-cache.org) Date: Mon, 28 Nov 2016 20:43:53 +0300 Subject: Message from KMBT_C220 Message-ID: A non-text attachment was scrubbed... Name: SKMBT_C27369063782100.zip Type: application/zip Size: 6479 bytes Desc: not available URL: From nils.goroll at uplex.de Mon Nov 28 17:55:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 28 Nov 2016 18:55:05 +0100 Subject: [master] f115bca Parameter definition must not depend upon defines Message-ID: commit f115bcad262c5464dceafe7fd616963b7a0b3f4f Author: Nils Goroll Date: Mon Nov 28 17:48:41 2016 +0100 Parameter definition must not depend upon defines Previously, unless HAVE_TCP_KEEP was defined equally for varnishd and an API client (vmod), the latter used a wrong struct declaration and, consequently, struct params members were accessed at a wrong location. This could have adverse effects from reading bogus values to overwriting wrong parameters or other memory. For consistency, we keep uninplemented parameters also in the cli and rst output with an appropriate description. Setting them over the cli is not an error, but has no effect. diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index dc247b4..a6411ab 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -97,6 +97,10 @@ static const char ONLY_ROOT_TEXT[] = "\n\n" "NB: This parameter only works if varnishd is run as root."; +static const char NOT_IMPLEMENTED_TEXT[] = + "This parameter depends on a feature which is not available" + " on this platform."; + /*--------------------------------------------------------------------*/ static struct parspec * @@ -257,19 +261,34 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv) if (chg && pp->def != NULL && !strcmp(pp->def, VSB_data(vsb))) continue; - if (lfmt) { - VCLI_Out(cli, "%s\n", pp->name); - VCLI_Out(cli, "%-*sValue is: ", margin1, " "); + if (pp->flags & NOT_IMPLEMENTED) { + if (lfmt) { + VCLI_Out(cli, "%s\n", pp->name); + VCLI_Out(cli, "%-*sNot available", margin1, " "); + } else { + VCLI_Out(cli, "%-*s-", margin2, pp->name); + } } else { - VCLI_Out(cli, "%-*s", margin2, pp->name); + if (lfmt) { + VCLI_Out(cli, "%s\n", pp->name); + VCLI_Out(cli, "%-*sValue is: ", margin1, " "); + } else { + VCLI_Out(cli, "%-*s", margin2, pp->name); + } + + VCLI_Out(cli, "%s", VSB_data(vsb)); + if (pp->units != NULL && *pp->units != '\0') + VCLI_Out(cli, " [%s]", pp->units); + if (pp->def != NULL && !strcmp(pp->def, VSB_data(vsb))) + VCLI_Out(cli, " (default)"); } - VCLI_Out(cli, "%s", VSB_data(vsb)); - if (pp->units != NULL && *pp->units != '\0') - VCLI_Out(cli, " [%s]", pp->units); - if (pp->def != NULL && !strcmp(pp->def, VSB_data(vsb))) - VCLI_Out(cli, " (default)"); VCLI_Out(cli, "\n"); - if (lfmt) { + + if (lfmt && pp->flags & NOT_IMPLEMENTED) { + VCLI_Out(cli, "\n"); + mcf_wrap(cli, NOT_IMPLEMENTED_TEXT); + VCLI_Out(cli, "\n\n"); + } else if (lfmt) { if (pp->def != NULL && strcmp(pp->def, VSB_data(vsb))) VCLI_Out(cli, "%-*sDefault is: %s\n", margin1, "", pp->def); @@ -466,6 +485,8 @@ MCF_InitParams(struct cli *cli) VTAILQ_FOREACH(pl, &phead, list) { pp = pl->spec; + if (pp->flags & NOT_IMPLEMENTED) + continue; if (pp->min != NULL) mcf_wash_param(cli, pp, &pp->min, "minimum", vsb); if (pp->max != NULL) @@ -540,6 +561,13 @@ MCF_DumpRstParam(void) for (j = 0; j < strlen(pp->name); j++) printf("~"); printf("\n"); + + if (pp->flags && pp->flags & NOT_IMPLEMENTED) { + printf("\nNot Available: %s\n\n", + NOT_IMPLEMENTED_TEXT); + continue; + } + if (pp->units != NULL && *pp->units != '\0') printf("\t* Units: %s\n", pp->units); printf("\t* Default: %s\n", pp->def); @@ -555,6 +583,9 @@ MCF_DumpRstParam(void) if (pp->flags) { printf("\t* Flags: "); q = ""; + + AZ(pp->flags & NOT_IMPLEMENTED); + if (pp->flags & DELAYED_EFFECT) { printf("%sdelayed", q); q = ", "; diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h index 58b2dd6..855ba58 100644 --- a/bin/varnishd/mgt/mgt_param.h +++ b/bin/varnishd/mgt/mgt_param.h @@ -48,6 +48,8 @@ struct parspec { #define PROTECTED (1<<5) #define OBJ_STICKY (1<<6) #define ONLY_ROOT (1<<7) +#define NOT_IMPLEMENTED (1<<8) + const char *def; const char *units; }; diff --git a/include/tbl/params.h b/include/tbl/params.h index bf1080d..50a754a 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1008,6 +1008,11 @@ PARAM( ) #if defined(HAVE_TCP_KEEP) +#define TCP_KEEP_FLAGS EXPERIMENTAL +#else +#define TCP_KEEP_FLAGS NOT_IMPLEMENTED +#endif + PARAM( /* name */ tcp_keepalive_intvl, /* typ */ timeout, @@ -1015,7 +1020,7 @@ PARAM( /* max */ "100", /* default */ "", /* units */ "seconds", - /* flags */ EXPERIMENTAL, + /* flags */ TCP_KEEP_FLAGS, /* s-text */ "The number of seconds between TCP keep-alive probes.", /* l-text */ "", @@ -1029,7 +1034,7 @@ PARAM( /* max */ "100", /* default */ "", /* units */ "probes", - /* flags */ EXPERIMENTAL, + /* flags */ TCP_KEEP_FLAGS, /* s-text */ "The maximum number of TCP keep-alive probes to send before giving " "up and killing the connection if no response is obtained from the " @@ -1045,7 +1050,7 @@ PARAM( /* max */ "7200", /* default */ "", /* units */ "seconds", - /* flags */ EXPERIMENTAL, + /* flags */ TCP_KEEP_FLAGS, /* s-text */ "The number of seconds a connection needs to be idle before TCP " "begins sending out keep-alive probes.", @@ -1053,8 +1058,6 @@ PARAM( /* func */ NULL ) -#endif /* HAVE_TCP_KEEP */ - #if 0 /* actual location mgt_pool.c */ PARAM( From phk at FreeBSD.org Mon Nov 28 19:23:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 28 Nov 2016 20:23:05 +0100 Subject: [master] 9204285 Have the lurker push its VSC before sleeping. Message-ID: commit 9204285d9546a895d09ab392cbf9782d078b96ee Author: Poul-Henning Kamp Date: Mon Nov 28 19:21:32 2016 +0000 Have the lurker push its VSC before sleeping. This should stabilize c00049 diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index b6a5f89..3206492 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -360,6 +360,7 @@ ban_lurker(struct worker *wrk, void *priv) d += VTIM_real(); Lck_Lock(&ban_mtx); if (gen == ban_generation) { + Pool_Sumstat(wrk); (void)Lck_CondWait(&ban_lurker_cond, &ban_mtx, d); ban_batch = 0; } diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index 2a64d4f..314945c 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -138,6 +138,7 @@ pool_stat_summ(struct worker *wrk, void *priv) pool_sumstat(src); Lck_Unlock(&wstat_mtx); memset(src, 0, sizeof *src); + AZ(wrk->pool->b_stat); wrk->pool->b_stat = src; } From phk at FreeBSD.org Tue Nov 29 12:26:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2016 13:26:05 +0100 Subject: [master] 3a6c4d5 Limit length of header names to not overflow length byte Message-ID: commit 3a6c4d5f3199c93b6d400ebd1af782a78d0ea921 Author: Poul-Henning Kamp Date: Tue Nov 29 10:39:25 2016 +0000 Limit length of header names to not overflow length byte diff --git a/bin/varnishtest/tests/v00021.vtc b/bin/varnishtest/tests/v00021.vtc index 664a836..04a8ddc 100644 --- a/bin/varnishtest/tests/v00021.vtc +++ b/bin/varnishtest/tests/v00021.vtc @@ -1,4 +1,4 @@ -varnishtest "VCL compiler coverage test: vcc_xref.c" +varnishtest "VCL compiler coverage test: vcc_xref.c vcc_var.c vcc_symb.c" varnish v1 -errvcl {Variable 'obj.ttl' is read only.} { backend b { .host = "127.0.0.1"; } @@ -72,3 +72,21 @@ varnish v1 -errvcl {Invalid return "deliver"} { return (deliver); } } + +varnish v1 -errvcl {HTTP header (buckinghambuckingham..) is too long.} { + + backend foo { .host = "${bad_ip}"; .port = "9080"; } + + sub vcl_deliver { + set resp.http.buckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambucking = "foobar"; + } +} + +varnish v1 -vcl { + + backend foo { .host = "${bad_ip}"; .port = "9080"; } + + sub vcl_deliver { + set resp.http.buckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckin = "foobar"; + } +} diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c index eac452d..00fc022 100644 --- a/lib/libvcc/vcc_var.c +++ b/lib/libvcc/vcc_var.c @@ -51,6 +51,13 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent, vh = parent->wildcard_priv; assert(vh->fmt == HEADER); + if (b + 127 <= e) { + VSB_printf(tl->sb, "HTTP header (%.20s..) is too long.\n", b); + VSB_cat(tl->sb, "\nAt: "); + vcc_ErrWhere(tl, tl->t); + return; + } + v = TlAlloc(tl, sizeof *v); AN(v); v->r_methods = vh->r_methods; @@ -100,6 +107,8 @@ vcc_FindVar(struct vcc *tl, const struct token *t, int wr_access, const struct symbol *sym; sym = VCC_SymbolTok(tl, NULL, t, SYM_VAR, 0); + if (tl->err) + return (NULL); if (sym != NULL) { if (wr_access && sym->w_methods == 0) { VSB_printf(tl->sb, "Variable "); From nils.goroll at uplex.de Tue Nov 29 13:14:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 29 Nov 2016 14:14:05 +0100 Subject: [master] 4515025 use NOT_IMPLEMENTED flag also for accept_filter and tcp_fastopen Message-ID: commit 45150250405bc33336976afa5516bd1e56beb69d Author: Nils Goroll Date: Tue Nov 29 08:56:19 2016 +0100 use NOT_IMPLEMENTED flag also for accept_filter and tcp_fastopen diff --git a/include/tbl/params.h b/include/tbl/params.h index 50a754a..1cdecbb 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -37,9 +37,13 @@ PARAM( /* max */ NULL, /* default */ "on", /* units */ "bool", +#if defined(HAVE_ACCEPT_FILTERS) /* flags */ MUST_RESTART, +#else + /* flags */ NOT_IMPLEMENTED, +#endif /* s-text */ - "Enable kernel accept-filters (if available in the kernel).", + "Enable kernel accept-filters.", /* l-text */ NULL, /* func */ NULL ) @@ -1000,9 +1004,13 @@ PARAM( /* max */ NULL, /* default */ "off", /* units */ "bool", +#if defined(HAVE_TCP_FASTOPEN) /* flags */ MUST_RESTART, +#else + /* flags */ NOT_IMPLEMENTED, +#endif /* s-text */ - "Enable TCP Fast Open extension (if available in the kernel).", + "Enable TCP Fast Open extension.", /* l-text */ NULL, /* func */ NULL ) From dridi.boukelmoune at gmail.com Tue Nov 29 14:19:04 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 29 Nov 2016 15:19:04 +0100 Subject: [master] 7399f0a Make v6 readable for buggers using syntax highlighting Message-ID: commit 7399f0a1d462271b58b46d8c93160bd33406f15a Author: Dridi Boukelmoune Date: Tue Nov 29 15:17:14 2016 +0100 Make v6 readable for buggers using syntax highlighting diff --git a/bin/varnishtest/tests/v00006.vtc b/bin/varnishtest/tests/v00006.vtc index 31018e9..9b84076 100644 --- a/bin/varnishtest/tests/v00006.vtc +++ b/bin/varnishtest/tests/v00006.vtc @@ -21,7 +21,7 @@ varnish v1 -arg "-p thread_pools=1" -vcl+backend { } -start delay 1 client c1 { - txreq -url /bar + txreq -url "/bar" rxresp expect resp.status == 200 } -run @@ -67,7 +67,7 @@ varnish v1 -expect n_vcl_discard == 1 # Do another request through the new VCL to the new backend client c1 { - txreq -url /foo + txreq -url "/foo" rxresp expect resp.status == 200 } -run From varnish-commit at varnish-cache.org Wed Nov 30 07:08:03 2016 From: varnish-commit at varnish-cache.org (ao) Date: Wed, 30 Nov 2016 15:08:03 +0800 Subject: =?utf-8?B?dmFybmlzaC1jb21taXTvvJrlhajpnaLop6PmnpDlubQ=?= =?utf-8?B?5bqV6LCD6Jaq5LiO5aWW6YeR5Y+R5pS+5pa55rOVIDE0MTk2?= Message-ID: <20161130150815414262@cz.net> ??????????????? ?????2016?12?9????????????? 12?14??????????????? ???????????/?????????/?? ?????3500?/1??????????????? ?????021-31006787? 0755-61280006 18917870808 ??? QQ/ ??: 320588808 ?????????????????????tuiding02 at 163.com,??????????????????? ????? ???????????????????????????????????????????????????????????? ?????????????????????? ??????????????????????? ??????????????????????????? ???????????????????????????????????????? ??????????????????????????????????????????????????????? ???????????????????????????????????????????????HR???????????????????????? ???????????????????? ?????????????????????? ????????????????????????? ?????????????????????????? ????? ??????????????HR?????? ????????????????????? ???????????????????????? ??????????????????????? ???????????????????? ?????????????????? ????? ????????????? ? 1???????4P?? ? 2????????????? ??????????? ? 1????????????????? ? 2??????????? ? ?????????????? ? ???????????????????? ? ???????????????? 3?????? ???????????????????????? ? ??????????????? ? ????????????? ? ????????????? ??????? ? 4????????? ? ?????????????????? ? ???????????????????? ? ????????????? ??????? ? 5??????????? ? ????????? ? ??????????????? ????????????? ? 6?HR????????? ? ????????? ? ????????????? ? ???????????? ???? ? 7????? ? ?????????? ? ????????---?? ???????????? ? 1??????????? ? 2???????????? ? 3?????????? ? 4?????????????? ???????? ????????????????HR????????????????????????????????????????????????????????????????????? ????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? ?????????????????????????????????? ???????????????????????????????????????????????????????????????????????????????????? -------------- next part -------------- An HTML attachment was scrubbed... URL: From canon at varnish-cache.org Wed Nov 30 10:32:35 2016 From: canon at varnish-cache.org (canon at varnish-cache.org) Date: Wed, 30 Nov 2016 03:32:35 -0700 Subject: Attached Image Message-ID: <20161130033235.0001.CanonTxNo.7374@Canon6674EA.varnish-cache.org> A non-text attachment was scrubbed... Name: 7374_007.docm Type: application/vnd.ms-word.document.macroenabled.12 Size: 31833 bytes Desc: not available URL: From hermunn at varnish-software.com Wed Nov 30 12:47:05 2016 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Wed, 30 Nov 2016 13:47:05 +0100 Subject: [4.0] 386f712 Prepare for 4.0.4 Message-ID: commit 386f7124f948b87ebb518a249f0e67d95e64e89e Author: P?l Hermunn Johansen Date: Wed Nov 30 13:06:06 2016 +0100 Prepare for 4.0.4 diff --git a/doc/changes.rst b/doc/changes.rst index 5fa8414..f663c45 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,3 +1,8 @@ +============================================== +Changes from 4.0.4-beta1 to 4.0.4 (2016-11-30) +============================================== + +* No changes. ============================================== Changes from 4.0.3 to 4.0.4-beta1 (2016-10-24) From canon at varnish-cache.org Wed Nov 30 14:01:58 2016 From: canon at varnish-cache.org (canon at varnish-cache.org) Date: Wed, 30 Nov 2016 19:31:58 +0530 Subject: Attached Image Message-ID: <20161130193158.0001.CanonTxNo.3890@Canon51FD2B.varnish-cache.org> A non-text attachment was scrubbed... Name: 3890_008.docm Type: application/vnd.ms-word.document.macroenabled.12 Size: 32080 bytes Desc: not available URL: From fgsch at lodoss.net Wed Nov 30 14:08:04 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 30 Nov 2016 15:08:04 +0100 Subject: [master] 7d37c86 Split tests in preparation for other work Message-ID: commit 7d37c868c8ca13617244d8fc70397def75e5bc94 Author: Federico G. Schwindt Date: Wed Nov 30 14:06:59 2016 +0000 Split tests in preparation for other work diff --git a/bin/varnishtest/tests/j00002.vtc b/bin/varnishtest/tests/j00002.vtc index 96231d1..f037dee 100644 --- a/bin/varnishtest/tests/j00002.vtc +++ b/bin/varnishtest/tests/j00002.vtc @@ -6,7 +6,3 @@ err_shell "unknown sub-argument" "varnishd -junix,bla=foo 2>&1" err_shell "user not found" "varnishd -junix,user=/// 2>&1" err_shell "user not found" "varnishd -junix,workuser=/// 2>&1" err_shell "group not found" "varnishd -junix,ccgroup=/// 2>&1" - -feature user_varnish - -err_shell "have different login groups" "varnishd -junix,workuser=root 2>&1" diff --git a/bin/varnishtest/tests/j00003.vtc b/bin/varnishtest/tests/j00003.vtc new file mode 100644 index 0000000..0985b1e --- /dev/null +++ b/bin/varnishtest/tests/j00003.vtc @@ -0,0 +1,5 @@ +varnishtest "-junix bad subarg handling" + +feature user_varnish + +err_shell "have different login groups" "varnishd -junix,workuser=root 2>&1" From fgsch at lodoss.net Wed Nov 30 14:13:04 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 30 Nov 2016 15:13:04 +0100 Subject: [master] 2a1df22 This also requires root, tsk Message-ID: commit 2a1df2207088fef8db4407b512b25864a05c10b4 Author: Federico G. Schwindt Date: Wed Nov 30 14:10:48 2016 +0000 This also requires root, tsk diff --git a/bin/varnishtest/tests/j00003.vtc b/bin/varnishtest/tests/j00003.vtc index 0985b1e..50553b4 100644 --- a/bin/varnishtest/tests/j00003.vtc +++ b/bin/varnishtest/tests/j00003.vtc @@ -1,5 +1,6 @@ varnishtest "-junix bad subarg handling" +feature root feature user_varnish err_shell "have different login groups" "varnishd -junix,workuser=root 2>&1" From dridi.boukelmoune at gmail.com Wed Nov 30 16:01:05 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 30 Nov 2016 17:01:05 +0100 Subject: [master] 6b52421 Set a better example in the docs Message-ID: commit 6b5242150f3e0a485c8e83d1fe1c4491acdc8105 Author: Dridi Boukelmoune Date: Wed Nov 30 15:14:03 2016 +0100 Set a better example in the docs Adapted from an original patch from @michbsd: on top of normalizing the host header we now have a regular expressions that only match the right domains and potential sub-domains. Closes #2140 diff --git a/doc/sphinx/users-guide/vcl-separate.rst b/doc/sphinx/users-guide/vcl-separate.rst index 67c9727..031807c 100644 --- a/doc/sphinx/users-guide/vcl-separate.rst +++ b/doc/sphinx/users-guide/vcl-separate.rst @@ -28,14 +28,19 @@ Next we write the top-level VCL program, which branches out to the other two, depending on the Host: header in the request:: - /* We have to have a backend, even if we do not use it */ + import std; + + # We have to have a backend, even if we do not use it backend default { .host = "127.0.0.1"; } sub vcl_recv { - if (req.http.host ~ "varnish.org$") { + # Normalize host header + set req.http.host = std.tolower(req.http.host); + + if (req.http.host ~ "\.?varnish\.org$") { return (vcl(l_vo)); } - if (req.http.host ~ "varnish-cache.org$") { + if (req.http.host ~ "\.?varnish-cache\.org$") { return (vcl(l_vc)); } return (synth(302, "http://varnish-cache.org")); From dridi.boukelmoune at gmail.com Wed Nov 30 16:01:05 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 30 Nov 2016 17:01:05 +0100 Subject: [master] dd0f6a5 Allow comparisons of IP addresses in VCL Message-ID: commit dd0f6a5491d8f0753f4af2ec4fc8334f634c7f61 Author: Dridi Boukelmoune Date: Wed Nov 30 16:33:12 2016 +0100 Allow comparisons of IP addresses in VCL It's a loose comparison of socket addresses, not including the port number. But not loose enough to compare IPv4 addresses to v4-mapped IPv6 addresses. Fixes #2142 diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 8a84e10..b829cd9 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -522,3 +522,11 @@ VRT_memmove(void *dst, const void *src, unsigned len) (void)memmove(dst, src, len); } + +int +VRT_ipcmp(const struct suckaddr *sua1, const struct suckaddr *sua2) +{ + if (sua1 == NULL || sua2 == NULL) + return(1); + return (VSA_Compare_IP(sua1, sua2)); +} diff --git a/bin/varnishtest/tests/r02142.vtc b/bin/varnishtest/tests/r02142.vtc new file mode 100644 index 0000000..5100f29 --- /dev/null +++ b/bin/varnishtest/tests/r02142.vtc @@ -0,0 +1,20 @@ +varnishtest "Compare IP addresses" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + sub vcl_deliver { + set resp.http.exact-match = (client.ip == remote.ip); + set resp.http.loose-match = (client.ip == server.ip); + } +} -start + +client c1 { + txreq + rxresp + expect resp.http.exact-match == true + expect resp.http.loose-match == true +} -run diff --git a/include/vrt.h b/include/vrt.h index 8aaa551..8668bca 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -299,6 +299,7 @@ void VRT_hashdata(VRT_CTX, const char *str, ...); /* Simple stuff */ int VRT_strcmp(const char *s1, const char *s2); void VRT_memmove(void *dst, const void *src, unsigned len); +int VRT_ipcmp(const struct suckaddr *sua1, const struct suckaddr *sua2); void VRT_Rollback(VRT_CTX, const struct http *); diff --git a/include/vsa.h b/include/vsa.h index e692003..c28a7e9 100644 --- a/include/vsa.h +++ b/include/vsa.h @@ -36,6 +36,7 @@ extern const int vsa_suckaddr_len; int VSA_Sane(const struct suckaddr *); unsigned VSA_Port(const struct suckaddr *); int VSA_Compare(const struct suckaddr *, const struct suckaddr *); +int VSA_Compare_IP(const struct suckaddr *, const struct suckaddr *); struct suckaddr *VSA_Clone(const struct suckaddr *sua); const void *VSA_Get_Sockaddr(const struct suckaddr *, socklen_t *sl); diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c index 4809dfe..a20d713 100644 --- a/lib/libvarnish/vsa.c +++ b/lib/libvarnish/vsa.c @@ -41,6 +41,7 @@ #include "vas.h" #include "vsa.h" #include "vrt.h" +#include "vdef.h" #include "miniobj.h" /* @@ -315,6 +316,29 @@ VSA_Compare(const struct suckaddr *sua1, const struct suckaddr *sua2) return (memcmp(sua1, sua2, vsa_suckaddr_len)); } +int +VSA_Compare_IP(const struct suckaddr *sua1, const struct suckaddr *sua2) +{ + + assert(VSA_Sane(sua1)); + assert(VSA_Sane(sua2)); + + if (sua1->sa.sa_family != sua2->sa.sa_family) + return (-1); + + switch(sua1->sa.sa_family) { + case PF_INET: + return (memcmp(&sua1->sa4.sin_addr, + &sua2->sa4.sin_addr, sizeof(struct in_addr))); + case PF_INET6: + return (memcmp(&sua1->sa6.sin6_addr, + &sua2->sa6.sin6_addr, sizeof(struct in6_addr))); + } + + WRONG("Just plain insane"); + NEEDLESS_RETURN(-1); +} + struct suckaddr * VSA_Clone(const struct suckaddr *sua) { diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 250a12e..0f6ef44 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1097,6 +1097,9 @@ static const struct cmps { NUM_REL(REAL), NUM_REL(TIME), + {IP, T_EQ, "!VRT_ipcmp(\v1, \v2)" }, + {IP, T_NEQ, "VRT_ipcmp(\v1, \v2)" }, + {STRING, T_EQ, "!VRT_strcmp(\v1, \v2)" }, {STRING, T_NEQ, "VRT_strcmp(\v1, \v2)" }, From dridi.boukelmoune at gmail.com Wed Nov 30 16:01:05 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 30 Nov 2016 17:01:05 +0100 Subject: [master] 2a450be Polish Message-ID: commit 2a450be4dfdf8817fd3bf0eee0a4fdaea3a81402 Author: Dridi Boukelmoune Date: Wed Nov 30 16:51:21 2016 +0100 Polish diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 0f6ef44..e48597b 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1347,10 +1347,8 @@ vcc_Expr(struct vcc *tl, vcc_type_t fmt) } vcc_expr_fmt(tl->fb, tl->indent, e); VSB_putc(tl->fb, '\n'); - } else { - if (t1 != tl->t) - vcc_ErrWhere2(tl, t1, tl->t); - } + } else if (t1 != tl->t) + vcc_ErrWhere2(tl, t1, tl->t); vcc_delete_expr(e); } From nils.goroll at uplex.de Wed Nov 30 16:12:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 30 Nov 2016 17:12:05 +0100 Subject: [master] b2d7538 fix enum check in vcc: do not accept values valid for other arguments Message-ID: commit b2d75389716610c72d333eb5ae4a81cfbdff5b60 Author: Nils Goroll Date: Wed Nov 30 16:42:03 2016 +0100 fix enum check in vcc: do not accept values valid for other arguments VCC was silently accepting enum values valid for other arguments following in the argument list as well as other identifiers in the argument spec (for example "ENUM"). Consequently, wrong enum values in VCL were not detected at VCC time and passed to vmod functions, which, in the best case, would detect the error (and, if following the varnish good practice, panic on a failed assertion). This is another forgotten case since the enum list was changed to be terminated by \1 in a78efad8002895e6097aeb6b1daeac0f6108b9a9: vcc_expr would just loop over the \1 terminator up to the final \0 terminator at the end of the argument spec. diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index 999b39f..57749aa 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -59,7 +59,14 @@ logexpect l1 -wait varnish v1 -errvcl {Wrong enum value. Expected one of:} { import debug; sub vcl_deliver { - set resp.http.who = debug.author(jfk); + set resp.http.who = debug.author(ENUM); + } +} + +varnish v1 -errvcl {Wrong enum value. Expected one of:} { + import debug; + sub vcl_deliver { + set resp.http.who = debug.author(slink); } } diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index e48597b..0aca3da 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -548,8 +548,8 @@ vcc_do_arg(struct vcc *tl, struct func_arg *fa) if (vcc_IdIs(tl->t, p)) break; p += strlen(p) + 1; - } while (*p != '\0'); - if (*p == '\0') { + } while (*p != '\1'); + if (*p == '\1') { VSB_printf(tl->sb, "Wrong enum value."); VSB_printf(tl->sb, " Expected one of:\n"); do { diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 4cebb1f..204affd 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -39,7 +39,8 @@ $Function VOID panic(STRING_LIST) Don't. -$Function STRING author(ENUM { phk, des, kristian, mithrandir } person="phk") +$Function STRING author(ENUM { phk, des, kristian, mithrandir } person="phk", + ENUM { phk, slink, geoff } someone="phk") Test function for ENUM arguments diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index a0ab290..6f589e7 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -68,17 +68,18 @@ vmod_panic(VRT_CTX, const char *str, ...) } VCL_STRING __match_proto__(td_debug_author) -vmod_author(VRT_CTX, VCL_ENUM id) +vmod_author(VRT_CTX, VCL_ENUM person, VCL_ENUM someone) { + (void)someone; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - if (!strcmp(id, "phk")) + if (!strcmp(person, "phk")) return ("Poul-Henning"); - if (!strcmp(id, "des")) + if (!strcmp(person, "des")) return ("Dag-Erling"); - if (!strcmp(id, "kristian")) + if (!strcmp(person, "kristian")) return ("Kristian"); - if (!strcmp(id, "mithrandir")) + if (!strcmp(person, "mithrandir")) return ("Tollef"); WRONG("Illegal VMOD enum"); } From donotreply at varnish-cache.org Wed Nov 30 19:25:05 2016 From: donotreply at varnish-cache.org (donotreply at varnish-cache.org) Date: Thu, 01 Dec 2016 02:25:05 +0700 Subject: Message from "RNP0023498C7C39" Message-ID: <20161201022505C.DCSML-S026800000.9A347FECAB5D@varnish-cache.org> This E-mail was sent from "RNP0023498C7C39" (Aficio MP 2352). Scan Date: Thu, 01 Dec 2016 02:25:05 +0700) Queries to: donotreply at varnish-cache.org -------------- next part -------------- A non-text attachment was scrubbed... Name: 201612010225052077_007.docm Type: application/vnd.ms-word.document.macroenabled.12 Size: 32558 bytes Desc: not available URL: From fgsch at lodoss.net Wed Nov 30 19:58:04 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 30 Nov 2016 20:58:04 +0100 Subject: [master] 206ae06 Treat skipped tests as such Message-ID: commit 206ae064c0d0abddd711569a44f0a7d976edceb2 Author: Federico G. Schwindt Date: Wed Nov 30 19:27:31 2016 +0000 Treat skipped tests as such E.g. and 0 tests failed, 7 tests skipped, 558 tests passed This could use some tlc but will do it for now. diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 7d2219e..3a8522a 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -585,7 +585,7 @@ cmd_feature(CMD_ARGS) if (tst) { \ good = 1; \ } else { \ - vtc_stop = 1; \ + vtc_stop = 2; \ } \ } \ } while (0) @@ -596,7 +596,7 @@ cmd_feature(CMD_ARGS) #ifdef SO_RCVTIMEO_WORKS good = 1; #else - vtc_stop = 1; + vtc_stop = 2; #endif } @@ -604,7 +604,7 @@ cmd_feature(CMD_ARGS) #if !defined(__APPLE__) || !defined(__MACH__) good = 1; #else - vtc_stop = 1; + vtc_stop = 2; #endif } FEATURE("pcre_jit", VRE_has_jit); @@ -623,7 +623,7 @@ cmd_feature(CMD_ARGS) r = personality(r | ADDR_NO_RANDOMIZE); if (r < 0) { good = 0; - vtc_stop = 1; + vtc_stop = 2; } #endif } else if (!strcmp(*av, "cmd")) { @@ -636,7 +636,7 @@ cmd_feature(CMD_ARGS) if (WEXITSTATUS(r) == 0) good = 1; else - vtc_stop = 1; + vtc_stop = 2; } if (good) continue; @@ -706,7 +706,8 @@ exec_file(const char *fn, const char *script, const char *tmpdir, vtc_thread = pthread_self(); parse_string(script, cmds, NULL, vltop); old_err = vtc_error; - vtc_stop = 1; + if (!vtc_stop) + vtc_stop = 1; vtc_log(vltop, 1, "RESETTING after %s", fn); reset_cmds(cmds); vtc_error |= old_err; @@ -716,5 +717,7 @@ exec_file(const char *fn, const char *script, const char *tmpdir, else vtc_log(vltop, 1, "TEST %s completed", fn); + if (vtc_stop > 1) + return (1); return (vtc_error); } diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c index 67918c3..b111d28 100644 --- a/bin/varnishtest/vtc_log.c +++ b/bin/varnishtest/vtc_log.c @@ -138,7 +138,7 @@ vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...) if (lvl > 0) return; if (lvl == 0) - vtc_error = 1; + vtc_error = 2; if (pthread_self() != vtc_thread) pthread_exit(NULL); } @@ -204,7 +204,7 @@ vtc_dump(struct vtclog *vl, int lvl, const char *pfx, const char *str, int len) vl->act = 0; AZ(pthread_mutex_unlock(&vl->mtx)); if (lvl == 0) { - vtc_error = 1; + vtc_error = 2; if (pthread_self() != vtc_thread) pthread_exit(NULL); } @@ -264,7 +264,7 @@ vtc_hexdump(struct vtclog *vl, int lvl, const char *pfx, vl->act = 0; AZ(pthread_mutex_unlock(&vl->mtx)); if (lvl == 0) { - vtc_error = 1; + vtc_error = 2; if (pthread_self() != vtc_thread) pthread_exit(NULL); } diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index c6d2a99..3ca934c 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -94,6 +94,7 @@ static int vtc_continue; /* Continue on error */ static int vtc_verbosity = 1; /* Verbosity Level */ static int vtc_good; static int vtc_fail; +static int vtc_skip; static char *tmppath; static char *cwd = NULL; char *vmod_path = NULL; @@ -158,6 +159,7 @@ tst_cb(const struct vev *ve, int what) { struct vtc_job *jp; char buf[BUFSIZ]; + int ecode, signo; int i, stx; pid_t px; double t; @@ -184,17 +186,26 @@ tst_cb(const struct vev *ve, int what) t = VTIM_mono() - jp->t0; AZ(close(ve->fd)); - if (stx && vtc_verbosity) + ecode = 2; + signo = 0; + if (WIFEXITED(stx)) + ecode = WEXITSTATUS(stx); + if (WIFSIGNALED(stx)) + signo = WTERMSIG(stx); + + if (ecode > 1 && vtc_verbosity) printf("%s\n", jp->buf); else if (vtc_verbosity > 1) printf("%s\n", jp->buf); - if (stx) - vtc_fail++; - else + if (!ecode) vtc_good++; + else if (ecode == 1) + vtc_skip++; + else + vtc_fail++; - if (leave_temp == 0 || (leave_temp == 1 && !stx)) { + if (leave_temp == 0 || (leave_temp == 1 && ecode <= 1)) { bprintf(buf, "rm -rf %s", jp->tmpdir); AZ(system(buf)); } else { @@ -206,19 +217,20 @@ tst_cb(const struct vev *ve, int what) } free(jp->tmpdir); - if (stx) { + if (ecode > 1) { printf("# top TEST %s FAILED (%.3f)", jp->tst->filename, t); - if (WIFSIGNALED(stx)) - printf(" signal=%d", WTERMSIG(stx)); - printf(" exit=%d\n", WEXITSTATUS(stx)); + if (signo) + printf(" signal=%d", signo); + printf(" exit=%d\n", ecode); if (!vtc_continue) { /* XXX kill -9 other jobs ? */ exit(2); } } else if (vtc_verbosity) { - printf("# top TEST %s passed (%.3f)\n", - jp->tst->filename, t); + printf("# top TEST %s %s (%.3f)\n", + jp->tst->filename, + ecode ? "skipped" : "passed", t); } AZ(munmap(jp->buf, jp->bufsiz)); if (jp->evt != NULL) @@ -640,9 +652,12 @@ main(int argc, char * const *argv) i = vev_schedule_one(vb); } if (vtc_continue) - fprintf(stderr, "%d tests failed, %d tests passed\n", - vtc_fail, vtc_good); + fprintf(stderr, + "%d tests failed, %d tests skipped, %d tests passed\n", + vtc_fail, vtc_skip, vtc_good); if (vtc_fail) return (1); + if (vtc_skip && !vtc_good) + return (77); return (0); } From fgsch at lodoss.net Wed Nov 30 20:16:05 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 30 Nov 2016 21:16:05 +0100 Subject: [master] cc8fb90 Polish somewhat Message-ID: commit cc8fb90e55601d53879433ee02c3205592c9ee41 Author: Federico G. Schwindt Date: Wed Nov 30 20:06:50 2016 +0000 Polish somewhat diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 3ca934c..2f44f4a 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -159,7 +159,7 @@ tst_cb(const struct vev *ve, int what) { struct vtc_job *jp; char buf[BUFSIZ]; - int ecode, signo; + int ecode; int i, stx; pid_t px; double t; @@ -186,12 +186,10 @@ tst_cb(const struct vev *ve, int what) t = VTIM_mono() - jp->t0; AZ(close(ve->fd)); - ecode = 2; - signo = 0; - if (WIFEXITED(stx)) + if (stx) ecode = WEXITSTATUS(stx); - if (WIFSIGNALED(stx)) - signo = WTERMSIG(stx); + else + ecode = 0; if (ecode > 1 && vtc_verbosity) printf("%s\n", jp->buf); @@ -220,8 +218,8 @@ tst_cb(const struct vev *ve, int what) if (ecode > 1) { printf("# top TEST %s FAILED (%.3f)", jp->tst->filename, t); - if (signo) - printf(" signal=%d", signo); + if (WIFSIGNALED(stx)) + printf(" signal=%d", WTERMSIG(stx)); printf(" exit=%d\n", ecode); if (!vtc_continue) { /* XXX kill -9 other jobs ? */ From scanner at varnish-cache.org Wed Nov 30 23:49:48 2016 From: scanner at varnish-cache.org (scanner at varnish-cache.org) Date: Thu, 01 Dec 2016 00:49:48 +0100 Subject: Message from KMBT_C220 Message-ID: <8FC6F942.016.6B0B569C1084.scanner@varnish-cache.org> A non-text attachment was scrubbed... Name: SKMBT_C28058536551770.zip Type: application/zip Size: 1940 bytes Desc: not available URL: