Code Coverage: vinyl-cache/bin/vinyld/cache/cache_wrk.c
 
82217aa4fc7094b3610340078afaf78cd95a8ec9
 
0
/*-
1
 * Copyright (c) 2006 Verdens Gang AS
2
 * Copyright (c) 2006-2011 Varnish Software AS
3
 * All rights reserved.
4
 *
5
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
6
 *
7
 * SPDX-License-Identifier: BSD-2-Clause
8
 *
9
 * Redistribution and use in source and binary forms, with or without
10
 * modification, are permitted provided that the following conditions
11
 * are met:
12
 * 1. Redistributions of source code must retain the above copyright
13
 *    notice, this list of conditions and the following disclaimer.
14
 * 2. Redistributions in binary form must reproduce the above copyright
15
 *    notice, this list of conditions and the following disclaimer in the
16
 *    documentation and/or other materials provided with the distribution.
17
 *
18
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
22
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28
 * SUCH DAMAGE.
29
 *
30
 * Worker thread stuff unrelated to the worker thread pools.
31
 *
32
 * --
33
 * signaling_note:
34
 *
35
 * note on worker wakeup signaling through the wrk condition variable (cv)
36
 *
37
 * In the general case, a cv needs to be signaled while holding the
38
 * corresponding mutex, otherwise the signal may be posted before the waiting
39
 * thread could register itself on the cv and, consequently, the signal may be
40
 * missed.
41
 *
42
 * In our case, any worker thread which we wake up comes from the idle queue,
43
 * where it put itself under the mutex, releasing that mutex implicitly via
44
 * Lck_CondWaitUntil() (which calls some variant of pthread_cond_wait). So we avoid
45
 * additional mutex contention knowing that any worker thread on the idle queue
46
 * is blocking on the cv.
47
 *
48
 * Except -- when it isn't, because it woke up for releasing its VCL
49
 * Reference. To account for this case, we check if the task function has been
50
 * set in the meantime, which in turn requires all of the task preparation to be
51
 * done holding the pool mutex. (see also #2719)
52
 */
53
54
#include "config.h"
55
56
#include <stdlib.h>
57
#include <sched.h>
58
59
#include "cache_int.h"
60
#include "cache_pool.h"
61
62
#include "vcli_serve.h"
63
#include "vtim.h"
64
65
#include "hash/hash_slinger.h"
66
67
static void Pool_Work_Thread(struct pool *pp, struct worker *wrk);
68
69
static uintmax_t reqpoolfail;
70
71
/*--------------------------------------------------------------------
72
 * Return all cached resources
73
 */
74
75
void
76
WRK_Cleanup(const struct worker *wrk)
77 23790
{
78
79 23790
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
80 23790
        CHECK_OBJ_NOTNULL(wrk->wpriv, WORKER_PRIV_MAGIC);
81 23790
        HSH_Cleanup(wrk);
82 23790
        if (wrk->wpriv->vcl != NULL)
83 1133
                VCL_Rel(&wrk->wpriv->vcl);
84 23790
}
85
86
/*--------------------------------------------------------------------
87
 * Create and start a back-ground thread which as its own worker and
88
 * session data structures;
89
 */
90
91
struct bgthread {
92
        unsigned        magic;
93
#define BGTHREAD_MAGIC  0x23b5152b
94
        const char      *name;
95
        bgthread_t      *func;
96
        void            *priv;
97
};
98
99
static void *
100
wrk_bgthread(void *arg)
101 4136
{
102 4136
        struct bgthread *bt;
103 4136
        struct worker wrk;
104 4136
        struct worker_priv wpriv[1];
105 4136
        struct VSC_main_wrk ds;
106 4136
        void *r;
107
108 4136
        CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC);
109 4136
        THR_SetName(bt->name);
110 4136
        THR_Init();
111 4136
        INIT_OBJ(&wrk, WORKER_MAGIC);
112 4136
        INIT_OBJ(wpriv, WORKER_PRIV_MAGIC);
113 4136
        wrk.wpriv = wpriv;
114
        // bgthreads do not have a vpi member
115 4136
        memset(&ds, 0, sizeof ds);
116 4136
        wrk.stats = &ds;
117
118 4136
        r = bt->func(&wrk, bt->priv);
119 4136
        WRK_Cleanup(&wrk);
120 4136
        Pool_Sumstat(&wrk);
121 4136
        return (r);
122 4136
}
123
124
void
125
WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv)
126 4136
{
127 4136
        struct bgthread *bt;
128
129 4136
        ALLOC_OBJ(bt, BGTHREAD_MAGIC);
130 4136
        AN(bt);
131
132 4136
        bt->name = name;
133 4136
        bt->func = func;
134 4136
        bt->priv = priv;
135 4136
        PTOK(pthread_create(thr, NULL, wrk_bgthread, bt));
136 4136
}
137
138
/*--------------------------------------------------------------------*/
139
140
static void
141
WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace)
142 21525
{
143
        // child_signal_handler stack overflow check uses struct worker addr
144 21525
        struct worker *w, ww;
145 21525
        struct VSC_main_wrk ds;
146 21525
        unsigned char ws[thread_workspace];
147 21525
        struct worker_priv wpriv[1];
148 21525
        unsigned char vpi[vpi_wrk_len];
149
150 21525
        AN(qp);
151 21525
        AN(stacksize);
152 21525
        AN(thread_workspace);
153
154 21525
        THR_SetName("cache-worker");
155 21525
        w = &ww;
156 21525
        INIT_OBJ(w, WORKER_MAGIC);
157 21525
        INIT_OBJ(wpriv, WORKER_PRIV_MAGIC);
158 21525
        w->wpriv = wpriv;
159 21525
        w->lastused = NAN;
160 21525
        memset(&ds, 0, sizeof ds);
161 21525
        w->stats = &ds;
162 21525
        THR_SetWorker(w);
163 21525
        PTOK(pthread_cond_init(&w->cond, NULL));
164
165 21532
        WS_Init(w->aws, "wrk", ws, thread_workspace);
166 21532
        VPI_wrk_init(w, vpi, sizeof vpi);
167 21532
        AN(w->vpi);
168
169 21532
        VSL(SLT_WorkThread, NO_VXID, "%p start", w);
170
171 21532
        Pool_Work_Thread(qp, w);
172 21532
        AZ(w->pool);
173
174 21532
        VSL(SLT_WorkThread, NO_VXID, "%p end", w);
175 21532
        PTOK(pthread_cond_destroy(&w->cond));
176 20708
        WRK_Cleanup(w);
177 20708
        Pool_Sumstat(w);
178 20708
}
179
180
/*--------------------------------------------------------------------
181
 * Summing of stats into pool counters
182
 */
183
184
static unsigned
185
wrk_addstat(const struct worker *wrk, const struct pool_task *tp, unsigned locked)
186 37914
{
187 37914
        struct pool *pp;
188
189 37914
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
190 37914
        pp = wrk->pool;
191 37914
        CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
192 37914
        if (locked)
193 37914
                Lck_AssertHeld(&pp->mtx);
194
195 37914
        if ((tp == NULL && wrk->stats->summs > 0) ||
196 37914
            (wrk->stats->summs >= cache_param->wthread_stats_rate)) {
197 8191
                if (!locked)
198 0
                        Lck_Lock(&pp->mtx);
199
200 8191
                pp->a_stat->summs++;
201 8191
                VSC_main_Summ_wrk_wrk(pp->a_stat, wrk->stats);
202 8191
                memset(wrk->stats, 0, sizeof *wrk->stats);
203
204 8191
                if (!locked)
205 0
                        Lck_Unlock(&pp->mtx);
206 8191
        }
207
208 37914
        return (tp != NULL);
209 37914
}
210
211
void
212
WRK_AddStat(const struct worker *wrk)
213 0
{
214
215 0
        (void)wrk_addstat(wrk, wrk->task, 0);
216 0
        wrk->stats->summs++;
217 0
}
218
219
/*--------------------------------------------------------------------
220
 * Pool reserve calculation
221
 */
222
223
static unsigned
224
pool_reserve(void)
225 46159
{
226 46159
        unsigned lim;
227
228 46159
        if (cache_param->wthread_reserve == 0) {
229 46067
                lim = cache_param->wthread_min / 20 + 1;
230 46067
        } else {
231 92
                lim = cache_param->wthread_min * 950 / 1000;
232 92
                if (cache_param->wthread_reserve < lim)
233 90
                        lim = cache_param->wthread_reserve;
234 92
        }
235 46159
        if (lim < TASK_QUEUE_RESERVE)
236 43391
                return (TASK_QUEUE_RESERVE);
237 2768
        return (lim);
238 46159
}
239
240
/*--------------------------------------------------------------------*/
241
242
static struct worker *
243
pool_getidleworker(struct pool *pp, enum task_prio prio)
244 8243
{
245 8243
        struct pool_task *pt = NULL;
246 8243
        struct worker *wrk;
247
248 8243
        CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
249 8243
        Lck_AssertHeld(&pp->mtx);
250 8243
        if (pp->nidle > (pool_reserve() * prio / TASK_QUEUE_RESERVE)) {
251 8225
                pt = VTAILQ_FIRST(&pp->idle_queue);
252 8225
                if (pt == NULL)
253 0
                        AZ(pp->nidle);
254 8225
        }
255
256 8243
        if (pt == NULL)
257 20
                return (NULL);
258
259 8223
        AZ(pt->func);
260 8223
        CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC);
261
262 8223
        AN(pp->nidle);
263 8223
        VTAILQ_REMOVE(&pp->idle_queue, wrk->task, list);
264 8223
        pp->nidle--;
265
266 8223
        return (wrk);
267 8223
}
268
269
/*--------------------------------------------------------------------
270
 * Special scheduling:  If no thread can be found, the current thread
271
 * will be prepared for rescheduling instead.
272
 * The selected threads workspace is reserved and the argument put there.
273
 * Return one if another thread was scheduled, otherwise zero.
274
 */
275
276
int
277
Pool_Task_Arg(struct worker *wrk, enum task_prio prio, task_func_t *func,
278
    const void *arg, size_t arg_len)
279 2395
{
280 2395
        struct pool *pp;
281 2395
        struct worker *wrk2;
282 2395
        int retval;
283
284 2395
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
285 2395
        AN(arg);
286 2395
        AN(arg_len);
287 2395
        pp = wrk->pool;
288 2395
        CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
289
290 2395
        Lck_Lock(&pp->mtx);
291 2395
        wrk2 = pool_getidleworker(pp, prio);
292 2395
        if (wrk2 != NULL)
293 2393
                retval = 1;
294 2
        else {
295 2
                wrk2 = wrk;
296 2
                retval = 0;
297 2
        }
298 2395
        AZ(wrk2->task->func);
299 2395
        assert(arg_len <= WS_ReserveSize(wrk2->aws, arg_len));
300 7185
        vmemcpy(WS_Reservation(wrk2->aws), arg, arg_len);
301 0
        wrk2->task->func = func;
302 7185
        wrk2->task->priv = WS_Reservation(wrk2->aws);
303 2395
        Lck_Unlock(&pp->mtx);
304
        // see signaling_note at the top for explanation
305 4790
        if (retval)
306 2395
                PTOK(pthread_cond_signal(&wrk2->cond));
307 2395
        return (retval);
308 4790
}
309
310
/*--------------------------------------------------------------------
311
 * Enter a new task to be done
312
 */
313
314
int
315
Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio prio)
316 5851
{
317 5851
        struct worker *wrk;
318 5851
        int retval = 0;
319 5851
        CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
320 5851
        AN(task);
321 5851
        AN(task->func);
322 5851
        assert(prio < TASK_QUEUE__END);
323
324 5851
        if (prio == TASK_QUEUE_REQ && reqpoolfail) {
325 1
                retval = reqpoolfail & 1;
326 1
                reqpoolfail >>= 1;
327 1
                if (retval) {
328 1
                        VSL(SLT_Debug, NO_VXID,
329 1
                            "Failing due to reqpoolfail (next= 0x%jx)",
330 1
                            reqpoolfail);
331 1
                        return (retval);
332 1
                }
333 1
        }
334
335 5850
        Lck_Lock(&pp->mtx);
336
337
        /* The common case first:  Take an idle thread, do it. */
338
339 5850
        wrk = pool_getidleworker(pp, prio);
340 5850
        if (wrk != NULL) {
341 5832
                AZ(wrk->task->func);
342 5832
                wrk->task->func = task->func;
343 5832
                wrk->task->priv = task->priv;
344 5832
                Lck_Unlock(&pp->mtx);
345
                // see signaling_note at the top for explanation
346 5832
                PTOK(pthread_cond_signal(&wrk->cond));
347 5832
                return (0);
348 5832
        }
349
350
        /* Vital work is always queued. Only priority classes that can
351
         * fit under the reserve capacity are eligible to queuing.
352
         */
353 18
        if (prio >= TASK_QUEUE_RESERVE || pp->die) {
354 10
                retval = -1;
355 10
        } else if (!TASK_QUEUE_LIMITED(prio) ||
356 8
            pp->lqueue + pp->nthr < cache_param->wthread_max +
357 7
            cache_param->wthread_queue_limit) {
358 7
                pp->stats->sess_queued++;
359 7
                pp->lqueue++;
360 7
                VTAILQ_INSERT_TAIL(&pp->queues[prio], task, list);
361 7
                PTOK(pthread_cond_signal(&pp->herder_cond));
362 7
        } else {
363
                /* NB: This is counter-intuitive but when we drop a REQ
364
                 * task, it is an HTTP/1 request and we effectively drop
365
                 * the whole session. It is otherwise an h2 stream with
366
                 * STR priority in which case we are dropping a request.
367
                 */
368 1
                if (prio == TASK_QUEUE_REQ)
369 0
                        pp->stats->sess_dropped++;
370 1
                else
371 1
                        pp->stats->req_dropped++;
372 1
                retval = -1;
373 1
        }
374 18
        Lck_Unlock(&pp->mtx);
375 18
        return (retval);
376 18
}
377
378
/*--------------------------------------------------------------------
379
 * Empty function used as a pointer value for the thread exit condition.
380
 */
381
382
static void v_matchproto_(task_func_t)
383
pool_kiss_of_death(struct worker *wrk, void *priv)
384 0
{
385 0
        (void)wrk;
386 0
        (void)priv;
387 0
}
388
389
390
/*--------------------------------------------------------------------
391
 * This is the work function for worker threads in the pool.
392
 */
393
394
static void
395
Pool_Work_Thread(struct pool *pp, struct worker *wrk)
396 21532
{
397 21532
        struct pool_task *tp;
398 21532
        struct pool_task tpx, tps;
399 21532
        vtim_real tmo, now;
400 21532
        unsigned i, reserve;
401
402 21532
        CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
403 21532
        wrk->pool = pp;
404 38724
        while (1) {
405 37911
                CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
406 37911
                tp = NULL;
407
408 37911
                WS_Rollback(wrk->aws, 0);
409 37911
                AZ(wrk->vsl);
410
411 37911
                Lck_Lock(&pp->mtx);
412 37911
                reserve = pool_reserve();
413
414 206619
                for (i = 0; i < TASK_QUEUE_RESERVE; i++) {
415 177256
                        if (pp->nidle < (reserve * i / TASK_QUEUE_RESERVE))
416 8541
                                break;
417 168715
                        tp = VTAILQ_FIRST(&pp->queues[i]);
418 168715
                        if (tp != NULL) {
419 7
                                pp->lqueue--;
420 7
                                pp->ndequeued--;
421 7
                                VTAILQ_REMOVE(&pp->queues[i], tp, list);
422 7
                                break;
423 7
                        }
424 168715
                }
425
426 37911
                if (wrk_addstat(wrk, tp, 1)) {
427 7
                        wrk->stats->summs++;
428 7
                        AN(tp);
429 37904
                } else if (pp->b_stat != NULL && pp->a_stat->summs) {
430
                        /* Nothing to do, push pool stats into global pool */
431 8189
                        tps.func = pool_stat_summ;
432 8189
                        tps.priv = pp->a_stat;
433 8189
                        pp->a_stat = pp->b_stat;
434 8189
                        pp->b_stat = NULL;
435 8189
                        tp = &tps;
436 29715
                } else {
437
                        /* Nothing to do: To sleep, perchance to dream ... */
438 29715
                        if (isnan(wrk->lastused))
439 21527
                                wrk->lastused = VTIM_real();
440 29715
                        wrk->task->func = NULL;
441 29715
                        wrk->task->priv = wrk;
442 29715
                        VTAILQ_INSERT_HEAD(&pp->idle_queue, wrk->task, list);
443 29715
                        pp->nidle++;
444 29715
                        now = wrk->lastused;
445 29952
                        do {
446
                                // see signaling_note at the top for explanation
447 29952
                                if (DO_DEBUG(DBG_VCLREL) &&
448 29952
                                    pp->b_stat == NULL && pp->a_stat->summs)
449
                                        /* We've released the VCL, but
450
                                         * there are pool stats not pushed
451
                                         * to the global stats and some
452
                                         * thread is busy pushing
453
                                         * stats. Set a 1 second timeout
454
                                         * so that we'll wake up and get a
455
                                         * chance to push stats. */
456 3
                                        tmo = now + 1.;
457 29949
                                else if (wrk->wpriv->vcl == NULL)
458 27388
                                        tmo = INFINITY;
459 2561
                                else if (DO_DEBUG(DBG_VTC_MODE))
460 2564
                                        tmo = now + 1.;
461 18446744073709551613
                                else
462 18446744073709551613
                                        tmo = now + 60.;
463 29952
                                (void)Lck_CondWaitUntil(
464 29952
                                    &wrk->cond, &pp->mtx, tmo);
465 29952
                                if (wrk->task->func != NULL) {
466
                                        /* We have been handed a new task */
467 28950
                                        tpx = *wrk->task;
468 28950
                                        tp = &tpx;
469 28950
                                        wrk->stats->summs++;
470 28950
                                } else if (pp->b_stat != NULL &&
471 1002
                                    pp->a_stat->summs) {
472
                                        /* Woken up to release the VCL,
473
                                         * and noticing that there are
474
                                         * pool stats not pushed to the
475
                                         * global stats and no active
476
                                         * thread currently doing
477
                                         * it. Remove ourself from the
478
                                         * idle queue and take on the
479
                                         * task. */
480 0
                                        assert(pp->nidle > 0);
481 0
                                        VTAILQ_REMOVE(&pp->idle_queue,
482 0
                                            wrk->task, list);
483 0
                                        pp->nidle--;
484 0
                                        tps.func = pool_stat_summ;
485 0
                                        tps.priv = pp->a_stat;
486 0
                                        pp->a_stat = pp->b_stat;
487 0
                                        pp->b_stat = NULL;
488 0
                                        tp = &tps;
489 1002
                                } else {
490
                                        // Presumably ETIMEDOUT but we do not
491
                                        // assert this because pthread condvars
492
                                        // are not airtight.
493 1002
                                        if (wrk->wpriv->vcl)
494 234
                                                VCL_Rel(&wrk->wpriv->vcl);
495 1002
                                        now = VTIM_real();
496 1002
                                }
497 29952
                        } while (tp == NULL);
498 29715
                }
499 37911
                Lck_Unlock(&pp->mtx);
500
501 37911
                if (tp->func == pool_kiss_of_death)
502 20719
                        break;
503
504 22457
                do {
505 22457
                        memset(wrk->task, 0, sizeof wrk->task);
506 22457
                        assert(wrk->pool == pp);
507 22457
                        AN(tp->func);
508 22457
                        tp->func(wrk, tp->priv);
509 22457
                        if (DO_DEBUG(DBG_VCLREL) && wrk->wpriv->vcl != NULL)
510 42
                                VCL_Rel(&wrk->wpriv->vcl);
511 22457
                        tpx = *wrk->task;
512 22457
                        tp = &tpx;
513 22457
                } while (tp->func != NULL);
514
515 17192
                if (WS_Overflowed(wrk->aws))
516 1
                        wrk->stats->ws_thread_overflow++;
517
                /* cleanup for next task */
518 17192
                wrk->seen_methods = 0;
519 17192
        }
520 21532
        wrk->pool = NULL;
521 21532
        Lck_Lock(&pp->mtx);
522 21532
        AN(pp->wrk_dying);
523 21532
        pp->wrk_dying--;
524 21532
        Lck_Unlock(&pp->mtx);
525 21532
}
526
527
/*--------------------------------------------------------------------
528
 * Create another worker thread.
529
 */
530
531
struct pool_info {
532
        unsigned                magic;
533
#define POOL_INFO_MAGIC         0x4e4442d3
534
        size_t                  stacksize;
535
        struct pool             *qp;
536
};
537
538
static void *
539
pool_thread(void *priv)
540 21529
{
541 21529
        struct pool_info *pi;
542
543 21529
        CAST_OBJ_NOTNULL(pi, priv, POOL_INFO_MAGIC);
544 21529
        THR_Init();
545 21529
        WRK_Thread(pi->qp, pi->stacksize, cache_param->workspace_thread);
546 21529
        FREE_OBJ(pi);
547 21529
        return (NULL);
548 21529
}
549
550
static void
551
pool_breed(struct pool *qp)
552 21532
{
553 21532
        pthread_t tp;
554 21532
        pthread_attr_t tp_attr;
555 21532
        struct pool_info *pi;
556
557 21532
        PTOK(pthread_attr_init(&tp_attr));
558 21532
        PTOK(pthread_attr_setdetachstate(&tp_attr, PTHREAD_CREATE_DETACHED));
559
560
        /* Set the stacksize for worker threads we create */
561 21532
        if (cache_param->wthread_stacksize != UINT_MAX)
562 21532
                PTOK(pthread_attr_setstacksize(&tp_attr, cache_param->wthread_stacksize));
563
564 21532
        ALLOC_OBJ(pi, POOL_INFO_MAGIC);
565 21532
        AN(pi);
566 21532
        PTOK(pthread_attr_getstacksize(&tp_attr, &pi->stacksize));
567 21532
        pi->qp = qp;
568
569 21532
        errno = pthread_create(&tp, &tp_attr, pool_thread, pi);
570 21532
        if (errno) {
571 0
                FREE_OBJ(pi);
572 0
                VSL(SLT_Debug, NO_VXID, "Create worker thread failed %d %s",
573 0
                    errno, VAS_errtxt(errno));
574 0
                Lck_Lock(&pool_mtx);
575 0
                VSC_C_main->threads_failed++;
576 0
                Lck_Unlock(&pool_mtx);
577 0
                VTIM_sleep(cache_param->wthread_fail_delay);
578 21532
        } else {
579 21532
                qp->nthr++;
580 21532
                Lck_Lock(&pool_mtx);
581 21532
                VSC_C_main->threads++;
582 21532
                VSC_C_main->threads_created++;
583 21532
                Lck_Unlock(&pool_mtx);
584 21532
                if (cache_param->wthread_add_delay > 0.0)
585 15
                        VTIM_sleep(cache_param->wthread_add_delay);
586 21517
                else
587 21517
                        (void)sched_yield();
588 21532
        }
589
590 21532
        PTOK(pthread_attr_destroy(&tp_attr));
591 21532
}
592
593
/*--------------------------------------------------------------------
594
 * Herd a single pool
595
 *
596
 * This thread wakes up every thread_pool_timeout seconds, whenever a pool
597
 * queues and when threads need to be destroyed
598
 *
599
 * The trick here is to not be too aggressive about creating threads.  In
600
 * pool_breed(), we sleep whenever we create a thread and a little while longer
601
 * whenever we fail to, hopefully missing a lot of cond_signals in the meantime.
602
 *
603
 * Idle threads are destroyed at a rate determined by wthread_destroy_delay
604
 *
605
 * XXX: probably need a lot more work.
606
 *
607
 */
608
609
void*
610
pool_herder(void *priv)
611 2038
{
612 2038
        struct pool *pp;
613 2038
        struct pool_task *pt;
614 2038
        double t_idle;
615 2038
        struct worker *wrk;
616 2038
        double delay;
617 2038
        unsigned wthread_min;
618 2038
        uintmax_t dq = (1ULL << 31);
619 2038
        vtim_mono dqt = 0;
620 2038
        int r = 0;
621
622 2038
        CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
623
624 2038
        THR_SetName("pool_herder");
625 2038
        THR_Init();
626
627 49966
        while (!pp->die || pp->nthr > 0) {
628
                /*
629
                 * If the worker pool is configured too small, we can
630
                 * end up deadlocking it (see #2418 for details).
631
                 *
632
                 * Recovering from this would require a lot of complicated
633
                 * code, and fundamentally, either people configured their
634
                 * pools wrong, in which case we want them to notice, or
635
                 * they are under DoS, in which case recovering gracefully
636
                 * is unlikely be a major improvement.
637
                 *
638
                 * Instead we implement a watchdog and kill the worker if
639
                 * nothing has been dequeued for that long.
640
                 */
641 47874
                if (VTAILQ_EMPTY(&pp->queues[TASK_QUEUE_HIGHEST_PRIORITY])) {
642
                        /* Watchdog only applies to no movement on the
643
                         * highest priority queue (TASK_QUEUE_BO) */
644 47874
                        dq = pp->ndequeued + 1;
645 18446744073709551615
                } else if (dq != pp->ndequeued) {
646 3
                        dq = pp->ndequeued;
647 3
                        dqt = VTIM_mono();
648 18446744073709551612
                } else if (VTIM_mono() - dqt > cache_param->wthread_watchdog) {
649 0
                        VSL(SLT_Error, NO_VXID,
650 0
                            "Pool Herder: Queue does not move ql=%u dt=%f",
651 0
                            pp->lqueue, VTIM_mono() - dqt);
652 0
                        WRONG("Worker Pool Queue does not move"
653 0
                              " - see thread_pool_watchdog parameter");
654 0
                }
655 47873
                wthread_min = cache_param->wthread_min;
656 47873
                if (pp->die)
657 20728
                        wthread_min = 0;
658
659
                /* Make more threads if needed and allowed */
660 47873
                if (pp->nthr < wthread_min ||
661 47873
                    (pp->lqueue > 0 && pp->nthr < cache_param->wthread_max)) {
662 21532
                        pool_breed(pp);
663 21532
                        continue;
664 21532
                }
665
666 26341
                delay = cache_param->wthread_timeout;
667 26341
                assert(pp->nthr >= wthread_min);
668
669 26341
                if (pp->nthr > wthread_min) {
670
671 20726
                        t_idle = VTIM_real() - cache_param->wthread_timeout;
672
673 20726
                        Lck_Lock(&pp->mtx);
674 20726
                        wrk = NULL;
675 20726
                        pt = VTAILQ_LAST(&pp->idle_queue, taskhead);
676 20726
                        if (pt != NULL) {
677 20679
                                AN(pp->nidle);
678 20679
                                AZ(pt->func);
679 20679
                                CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC);
680
681 20679
                                if (pp->die || wrk->lastused < t_idle ||
682 20679
                                    pp->nthr > cache_param->wthread_max) {
683
                                        /* Give it a kiss on the cheek... */
684 20675
                                        VTAILQ_REMOVE(&pp->idle_queue,
685 20675
                                            wrk->task, list);
686 20675
                                        pp->nidle--;
687 20675
                                        wrk->task->func = pool_kiss_of_death;
688 20675
                                        PTOK(pthread_cond_signal(&wrk->cond));
689 20730
                                        pp->nthr--;
690 20730
                                        pp->wrk_dying++;
691 20730
                                } else {
692 4
                                        delay = wrk->lastused - t_idle;
693 4
                                        wrk = NULL;
694 4
                                }
695 20679
                        }
696 20781
                        Lck_Unlock(&pp->mtx);
697
698 20781
                        if (wrk != NULL) {
699 20687
                                Lck_Lock(&pool_mtx);
700 20687
                                VSC_C_main->threads--;
701 20687
                                VSC_C_main->threads_destroyed++;
702 20687
                                Lck_Unlock(&pool_mtx);
703 20687
                                delay = cache_param->wthread_destroy_delay;
704 20687
                        } else
705 94
                                delay = vmax(delay,
706 20781
                                    cache_param->wthread_destroy_delay);
707 20781
                }
708
709 26396
                if (pp->die) {
710 20736
                        if (delay < 2)
711 20731
                                delay = .01;
712 5
                        else
713 5
                                delay = 1;
714 20736
                        VTIM_sleep(delay);
715 20736
                        continue;
716 20736
                }
717 5660
                Lck_Lock(&pp->mtx);
718 5660
                if (pp->lqueue == 0) {
719 5640
                        if (DO_DEBUG(DBG_VTC_MODE))
720 5634
                                delay = 0.5;
721 5640
                        r = Lck_CondWaitTimeout(
722 5640
                            &pp->herder_cond, &pp->mtx, delay);
723 5640
                } else if (pp->nthr >= cache_param->wthread_max) {
724
                        /* XXX: unsafe counters */
725 3
                        if (r != ETIMEDOUT)
726 1
                                VSC_C_main->threads_limited++;
727 3
                        r = Lck_CondWaitTimeout(
728 3
                            &pp->herder_cond, &pp->mtx, 1.0);
729 3
                }
730 5660
                Lck_Unlock(&pp->mtx);
731 5660
        }
732 2093
        return (NULL);
733 2038
}
734
735
/*--------------------------------------------------------------------
736
 * Debugging aids
737
 */
738
739
static void v_matchproto_(cli_func_t)
740
debug_reqpoolfail(struct cli *cli, const char * const *av, void *priv)
741 1
{
742 1
        uintmax_t u = 1;
743 1
        const char *p;
744
745 1
        (void)priv;
746 1
        (void)cli;
747 1
        reqpoolfail = 0;
748 2
        for (p = av[2]; *p != '\0'; p++) {
749 1
                if (*p == 'F' || *p == 'f')
750 1
                        reqpoolfail |= u;
751 1
                u <<= 1;
752 1
        }
753 1
}
754
755
static struct cli_proto debug_cmds[] = {
756
        { CLICMD_DEBUG_REQPOOLFAIL,             "d", debug_reqpoolfail },
757
        { NULL }
758
};
759
760
void
761
WRK_Log(enum VSL_tag_e tag, const char *fmt, ...)
762 6720
{
763 6720
        struct worker *wrk;
764 6720
        va_list ap;
765
766 6720
        AN(fmt);
767
768 6720
        wrk = THR_GetWorker();
769 6720
        CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC);
770
771 6720
        va_start(ap, fmt);
772 6720
        if (wrk != NULL && wrk->vsl != NULL)
773 4006
                VSLbv(wrk->vsl, tag, fmt, ap);
774 2714
        else
775 2714
                VSLv(tag, NO_VXID, fmt, ap);
776 6720
        va_end(ap);
777 6720
}
778
779
/*--------------------------------------------------------------------
780
 *
781
 */
782
783
void
784
WRK_Init(void)
785 1026
{
786 1026
        assert(cache_param->wthread_min >= TASK_QUEUE_RESERVE);
787 1026
        CLI_AddFuncs(debug_cmds);
788 1026
}