vinyl-cache/lib/libvinylapi/vut.c
0
/*-
1
 * Copyright (c) 2006 Verdens Gang AS
2
 * Copyright (c) 2006-2015 Varnish Software AS
3
 * All rights reserved.
4
 *
5
 * Author: Martin Blix Grydeland <martin@varnish-software.com>
6
 *
7
 * SPDX-License-Identifier: BSD-2-Clause
8
 *
9
 * Redistribution and use in source and binary forms, with or without
10
 * modification, are permitted provided that the following conditions
11
 * are met:
12
 * 1. Redistributions of source code must retain the above copyright
13
 *    notice, this list of conditions and the following disclaimer.
14
 * 2. Redistributions in binary form must reproduce the above copyright
15
 *    notice, this list of conditions and the following disclaimer in the
16
 *    documentation and/or other materials provided with the distribution.
17
 *
18
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
22
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28
 * SUCH DAMAGE.
29
 *
30
 * Common functions for the utilities
31
 */
32
33
#include "config.h"
34
35
#include <ctype.h>
36
#include <math.h>
37
#include <stdint.h>
38
#include <stdarg.h>
39
#include <stdlib.h>
40
#include <unistd.h>
41
#include <stdio.h>
42
#include <string.h>
43
#include <sys/stat.h> /* for MUSL */
44
45
#include "compat/daemon.h"
46
#include "vdef.h"
47
#include "vpf.h"
48
#include "vapi/vsm.h"
49
#include "vapi/vsl.h"
50
#include "vtim.h"
51
#include "vas.h"
52
#include "miniobj.h"
53
#include "vcs.h"
54
#include "vsb.h"
55
#include "vfil.h"
56
57
#include "vapi/voptget.h"
58
#include "vapi/vsig.h"
59
60
#include "vut.h"
61
62
63
static int vut_synopsis(const struct vopt_spec *);
64
static int vut_options(const struct vopt_spec *);
65
66
static struct vpf_fh    *pfh;
67
static unsigned         daemonized;
68
69
static struct VUT pfh_vut;
70
71
static int
72 6
vut_daemon(struct VUT *vut)
73
{
74 6
        if (daemonized)
75 0
                VUT_Error(vut, 1, "Already running as a daemon");
76 6
        daemonized = 1;
77 6
        return (vinyl_daemon(0, 0));
78
}
79
80
static void
81 12
vut_vpf_remove(void)
82
{
83
84 12
        CHECK_OBJ(&pfh_vut, VUT_MAGIC);
85 12
        AN(pfh);
86 12
        AN(pfh_vut.P_arg);
87
88 12
        VPF_Remove(pfh);
89 12
        free(pfh_vut.P_arg);
90 12
        ZERO_OBJ(&pfh_vut, sizeof pfh_vut);
91 12
        pfh = NULL;
92 12
}
93
94
static int v_matchproto_(VSLQ_dispatch_f)
95 3003
vut_dispatch(struct VSL_data *vsl, struct VSL_transaction * const trans[],
96
    void *priv)
97
{
98
        struct VUT *vut;
99
        int i;
100
101 3003
        CAST_OBJ_NOTNULL(vut, priv, VUT_MAGIC);
102
103 3003
        if (vut->k_arg == 0)
104 0
                return (-1);    /* End of file */
105 3003
        AN(vut->dispatch_f);
106 3003
        i = vut->dispatch_f(vsl, trans, vut->dispatch_priv);
107 3003
        if (vut->k_arg > 0)
108 9
                vut->k_arg--;
109 3003
        if (i >= 0 && vut->k_arg == 0)
110 9
                return (-1);    /* End of file */
111 2994
        return (i);
112 3003
}
113
114
void
115 291
VUT_Error(struct VUT *vut, int status, const char *fmt, ...)
116
{
117
        va_list ap;
118
119 291
        CHECK_OBJ_NOTNULL(vut, VUT_MAGIC);
120 291
        AN(status);
121
122 291
        va_start(ap, fmt);
123 291
        if (vut->error_f != NULL) {
124 0
                vut->error_f(vut, status, fmt, ap);
125 0
        } else {
126 291
                vfprintf(stderr, fmt, ap);
127 291
                fprintf(stderr, "\n");
128
        }
129 291
        va_end(ap);
130 291
        exit(status);
131
}
132
133
static void
134 177
vut_arg_q(struct VUT *vut, const char *arg)
135
{
136
        struct vsb *vsb;
137
138 177
        AN(arg);
139 177
        if (vut->q_arg == NULL) {
140 168
                REPLACE(vut->q_arg, arg);
141 168
                return;
142
        }
143
144 9
        vsb = VSB_new_auto();
145 9
        AN(vsb);
146 9
        AZ(VSB_printf(vsb, "%s\n%s", vut->q_arg, arg));
147 9
        AZ(VSB_finish(vsb));
148
149 9
        REPLACE(vut->q_arg, VSB_data(vsb));
150
151 9
        VSB_destroy(&vsb);
152 177
}
153
154
int
155 1287
VUT_Arg(struct VUT *vut, int opt, const char *arg)
156
{
157
        int i;
158
        char *p;
159
160 1287
        CHECK_OBJ_NOTNULL(vut, VUT_MAGIC);
161 1287
        AN(opt);
162
163 1287
        switch (opt) {
164
        case '0':
165 0
                vut->dryrun_opt = 1;
166 0
                return (1);
167
        case 'd':
168
                /* Head */
169 252
                vut->d_opt = 1;
170 252
                return (1);
171
        case 'D':
172
                /* Daemon mode */
173 18
                vut->D_opt = 1;
174 18
                return (1);
175
        case 'g':
176
                /* Grouping */
177 42
                AN(arg);
178 42
                vut->g_arg = VSLQ_Name2Grouping(arg, -1);
179 42
                if (vut->g_arg == -2)
180 3
                        VUT_Error(vut, 1, "Ambiguous grouping type: %s", arg);
181 39
                else if (vut->g_arg < 0)
182 3
                        VUT_Error(vut, 1, "Unknown grouping type: %s", arg);
183 36
                return (1);
184
        case 'k':
185
                /* Log transaction limit */
186 12
                AN(arg);
187 12
                vut->k_arg = (int)strtol(arg, &p, 10);
188 12
                if (*p != '\0' || vut->k_arg <= 0)
189 3
                        VUT_Error(vut, 1, "-k: Invalid number '%s'", arg);
190 9
                return (1);
191
        case 'n':
192
                /* Vinyl instance name */
193 288
                AN(arg);
194 288
                REPLACE(vut->n_arg, arg);
195 288
                return (1);
196
        case 'P':
197
                /* PID file */
198 12
                AN(arg);
199 12
                REPLACE(vut->P_arg, arg);
200 12
                return (1);
201
        case 'Q':
202 24
                AN(arg);
203 24
                p = VFIL_readfile(NULL, arg, NULL);
204 24
                if (p == NULL)
205 3
                        VUT_Error(vut, 1, "-Q %s: %s", arg, strerror(errno));
206 21
                vut_arg_q(vut, p);
207 21
                free(p);
208 21
                return (1);
209
        case 'q':
210
                /* Query to use */
211 156
                AN(arg);
212 156
                vut_arg_q(vut, arg);
213 156
                return (1);
214
        case 'r':
215
                /* Binary file input */
216 24
                AN(arg);
217 24
                REPLACE(vut->r_arg, arg);
218 24
                return (1);
219
        case 't':
220
                /* VSM connect timeout */
221 12
                REPLACE(vut->t_arg, arg);
222 12
                return (1);
223
        case 'V':
224
                /* Print version number and exit */
225 15
                VCS_Message(vut->progname);
226 15
                exit(0);
227
        default:
228 432
                AN(vut->vsl);
229 432
                i = VSL_Arg(vut->vsl, opt, arg);
230 432
                if (i < 0)
231 54
                        VUT_Error(vut, 1, "%s", VSL_Error(vut->vsl));
232 378
                return (i);
233
        }
234 1206
}
235
236
struct VUT *
237 723
VUT_Init(const char *progname, int argc, char * const *argv,
238
    const struct vopt_spec *voc)
239
{
240
        struct VUT *vut;
241
242 723
        AN(progname);
243 723
        AN(argv);
244 723
        AN(voc);
245
246 723
        VSIG_Arm_hup();
247 723
        VSIG_Arm_int();
248 723
        VSIG_Arm_term();
249 723
        VSIG_Arm_usr1();
250
251 723
        if (argc == 2 && !vstrcmp(argv[1], "--synopsis"))
252 15
                exit(vut_synopsis(voc));
253 708
        if (argc == 2 && !vstrcmp(argv[1], "--options"))
254 15
                exit(vut_options(voc));
255 693
        if (argc == 2 && !vstrcmp(argv[1], "--optstring")) {
256 0
                (void)printf("%s\n", voc->vopt_optstring);
257 0
                exit(0);
258
        }
259
260 693
        ALLOC_OBJ(vut, VUT_MAGIC);
261 693
        AN(vut);
262 693
        vut->progname = progname;
263 693
        vut->g_arg = VSL_g_vxid;
264 693
        vut->k_arg = -1;
265 693
        AZ(vut->vsl);
266 693
        vut->vsl = VSL_New();
267 693
        AN(vut->vsl);
268 693
        return (vut);
269
}
270
271
void
272 0
VUT_Signal(VUT_sighandler_f sig_cb)
273
{
274
275 0
        AN(sig_cb);
276 0
        (void)signal(SIGHUP, sig_cb);
277 0
        (void)signal(SIGINT, sig_cb);
278 0
        (void)signal(SIGTERM, sig_cb);
279 0
        (void)signal(SIGUSR1, sig_cb);
280 0
}
281
282
void
283 0
VUT_Signaled(struct VUT *vut, int sig)
284
{
285
286 0
        CHECK_OBJ_NOTNULL(vut, VUT_MAGIC);
287
#define VSIG_SIGNAL(UPPER, lower) \
288
        VSIG_##lower += (int)(sig == SIG##UPPER);
289
#include "tbl/vsig_list.h"
290
}
291
292
void
293 381
VUT_Setup(struct VUT *vut)
294
{
295
        struct VSL_cursor *c;
296
297 381
        CHECK_OBJ_NOTNULL(vut, VUT_MAGIC);
298 381
        AN(vut->vsl);
299 381
        AZ(vut->vsm);
300 381
        AZ(vut->vslq);
301
302
        /* Check input arguments (2 used for bug in FlexeLint) */
303 1143
        if ((vut->n_arg == NULL ? 0 : 2) +
304 762
            (vut->r_arg == NULL ? 0 : 2) > 2)
305 3
                VUT_Error(vut, 1, "Only one of -n and -r options may be used");
306
307 378
        if (vut->r_arg != NULL && !vstrcmp(vut->r_arg, "-") && vut->D_opt)
308 0
                VUT_Error(vut, 1, "Daemon cannot read from stdin");
309
310
        /* Create and validate the query expression */
311 756
        vut->vslq = VSLQ_New(vut->vsl, NULL,
312 378
            (enum VSL_grouping_e)vut->g_arg, vut->q_arg);
313 378
        if (vut->vslq == NULL)
314 192
                VUT_Error(vut, 1, "Query expression error:\n%s",
315 96
                    VSL_Error(vut->vsl));
316
317 282
        if (vut->dryrun_opt)
318 0
                return;
319
320
        /* Setup input */
321 282
        if (vut->r_arg) {
322 21
                c = VSL_CursorFile(vut->vsl, vut->r_arg, 0);
323 21
                if (c == NULL)
324 6
                        VUT_Error(vut, 1, "%s", VSL_Error(vut->vsl));
325 15
                VSLQ_SetCursor(vut->vslq, &c);
326 15
                AZ(c);
327 15
        } else {
328 261
                vut->vsm = VSM_New();
329 261
                AN(vut->vsm);
330 261
                if (vut->n_arg && VSM_Arg(vut->vsm, 'n', vut->n_arg) <= 0)
331 0
                        VUT_Error(vut, 1, "%s", VSM_Error(vut->vsm));
332 261
                if (vut->t_arg && VSM_Arg(vut->vsm, 't', vut->t_arg) <= 0)
333 9
                        VUT_Error(vut, 1, "%s", VSM_Error(vut->vsm));
334 252
                if (VSM_Attach(vut->vsm, STDERR_FILENO))
335 3
                        VUT_Error(vut, 1, "VSM: %s", VSM_Error(vut->vsm));
336
                // Cursor is handled in VUT_Main()
337
        }
338
339
        /* Open PID file */
340 264
        if (vut->P_arg) {
341 12
                if (pfh != NULL)
342 0
                        VUT_Error(vut, 1, "PID file already created");
343 12
                pfh = VPF_Open(vut->P_arg, 0644, NULL);
344 12
                if (pfh == NULL)
345 0
                        VUT_Error(vut, 1,
346 0
                            "%s: %s", vut->P_arg, strerror(errno));
347 12
        }
348
349
        /* Daemon mode */
350 264
        if (vut->D_opt && vut_daemon(vut) == -1)
351 0
                VUT_Error(vut, 1, "Daemon mode: %s", strerror(errno));
352
353
        /* Write PID and setup exit handler */
354 264
        if (vut->P_arg) {
355 12
                AN(pfh);
356 12
                VPF_Write(pfh);
357
358
                /* NB: move ownership to a global pseudo-VUT. */
359 12
                INIT_OBJ(&pfh_vut, VUT_MAGIC);
360 12
                pfh_vut.P_arg = vut->P_arg;
361 12
                pfh_vut.error_f = vut->error_f;
362 12
                vut->P_arg = NULL;
363
364 12
                AZ(atexit(vut_vpf_remove));
365 12
        }
366 264
}
367
368
void
369 264
VUT_Fini(struct VUT **vutp)
370
{
371
        struct VUT *vut;
372
373 264
        TAKE_OBJ_NOTNULL(vut, vutp, VUT_MAGIC);
374 264
        AN(vut->progname);
375
376 264
        free(vut->n_arg);
377 264
        free(vut->q_arg);
378 264
        free(vut->r_arg);
379 264
        free(vut->t_arg);
380 264
        AZ(vut->P_arg);
381
382 264
        if (vut->vslq)
383 264
                VSLQ_Delete(&vut->vslq);
384 264
        if (vut->vsl)
385 264
                VSL_Delete(vut->vsl);
386 264
        if (vut->vsm)
387 249
                VSM_Destroy(&vut->vsm);
388
389 264
        FREE_OBJ(vut);
390 264
}
391
392
static void
393 0
vut_CursorError(struct VUT *vut, vtim_mono *last)
394
{
395
        const char *diag;
396
        vtim_mono now;
397
398 0
        CHECK_OBJ_NOTNULL(vut, VUT_MAGIC);
399 0
        AN(vut->vsl);
400 0
        AN(last);
401
402 0
        diag = VSL_Error(vut->vsl);
403 0
        if (diag == NULL)
404 0
                diag = "Missing diagnostic";
405
406 0
        now = VTIM_mono();
407 0
        if (isnan(*last) || *last + 1 < now) {
408 0
                fprintf(stderr, "Failed to acquire log: %s\n", diag);
409 0
                *last = now;
410 0
        }
411 0
}
412
413
int
414 264
VUT_Main(struct VUT *vut)
415
{
416
        struct VSL_cursor *c;
417 264
        int i = -1;
418 264
        int hascursor = -1;
419 264
        vtim_mono t_failcursor = NAN;
420
421 264
        CHECK_OBJ_NOTNULL(vut, VUT_MAGIC);
422 264
        AN(vut->vslq);
423
424 20662
        while (!VSIG_int && !VSIG_term && !vut->dryrun_opt) {
425 20626
                if (VSIG_hup != vut->last_sighup) {
426
                        /* sighup callback */
427 12
                        vut->last_sighup = VSIG_hup;
428 12
                        if (vut->sighup_f != NULL)
429 6
                                i = vut->sighup_f(vut);
430
                        else
431 6
                                i = 1;
432 12
                        if (i)
433 6
                                break;
434 6
                }
435
436 20620
                if (VSIG_usr1 != vut->last_sigusr1) {
437
                        /* Flush and report any incomplete records */
438 0
                        vut->last_sigusr1 = VSIG_usr1;
439 0
                        (void)VSLQ_Flush(vut->vslq, vut_dispatch, vut);
440 0
                }
441
442
                /* We must repeatedly call VSM_Status() when !hascursor
443
                 * to make VSM discover our segment.
444
                 *
445
                 * XXX consider moving the error handling to VSLQ_Dispatch.
446
                 * or some other VSL utility function
447
                 * Reasons:
448
                 *
449
                 * - it does not seem to make much sense to call VSM_StillValid
450
                 *   in vsl if that can only detect invalid segments after
451
                 *   VSM_Status has run, so it appears both should be
452
                 *   consolidated
453
                 *
454
                 * - not all VSL Clients will use VUT, yet the log abandoned/
455
                 *   overrun situation will be occur for all of them.
456
                 */
457
458 20620
                if (vut->vsm != NULL &&
459 20605
                    (VSM_Status(vut->vsm) & VSM_WRK_RESTARTED)) {
460 0
                        if (hascursor < 1) {
461 0
                                fprintf(stderr, "Log abandoned (vsm)\n");
462 0
                                VSLQ_SetCursor(vut->vslq, NULL);
463 0
                                hascursor = 0;
464 0
                        }
465 0
                }
466 20620
                if (vut->vsm != NULL && hascursor < 1) {
467
                        /* Reconnect VSM */
468 249
                        AZ(vut->r_arg);
469 249
                        VTIM_sleep(0.1);
470 498
                        c = VSL_CursorVSM(vut->vsl, vut->vsm,
471 249
                            (vut->d_opt ? VSL_COPT_TAILSTOP : VSL_COPT_TAIL)
472 249
                            | VSL_COPT_BATCH);
473 249
                        if (c == NULL) {
474 0
                                vut_CursorError(vut, &t_failcursor);
475 0
                                VSL_ResetError(vut->vsl);
476 0
                                continue;
477
                        }
478 249
                        if (hascursor >= 0)
479 0
                                fprintf(stderr, "Log reacquired\n");
480 249
                        hascursor = 1;
481 249
                        VSLQ_SetCursor(vut->vslq, &c);
482 249
                        AZ(c);
483 249
                }
484
485 20620
                do
486 53734
                        i = VSLQ_Dispatch(vut->vslq, vut_dispatch, vut);
487 33114
                while (i == vsl_more &&
488 12494
                    VSIG_usr1 == vut->last_sigusr1 &&
489 12494
                    VSIG_hup == vut->last_sighup);
490
491 20620
                if (i == vsl_more)
492 0
                        continue;
493 20620
                else if (i == vsl_end) {
494 20398
                        if (vut->idle_f) {
495 9691
                                i = vut->idle_f(vut);
496 9691
                                if (i)
497 0
                                        break;
498 9691
                        }
499 20398
                        VTIM_sleep(0.01);
500 20398
                        continue;
501 222
                } else if (i == vsl_e_eof)
502 222
                        break;
503
504 0
                if (vut->vsm == NULL)
505 0
                        break;
506
507
                /* XXX: Make continuation optional */
508
509 0
                (void)VSLQ_Flush(vut->vslq, vut_dispatch, vut);
510
511 0
                if (i == vsl_e_abandon) {
512 0
                        fprintf(stderr, "Log abandoned (vsl)\n");
513 0
                        VSLQ_SetCursor(vut->vslq, NULL);
514 0
                        hascursor = 0;
515 0
                } else if (i == vsl_e_overrun) {
516 0
                        fprintf(stderr, "Log overrun\n");
517 0
                        VSLQ_SetCursor(vut->vslq, NULL);
518 0
                        hascursor = 0;
519 0
                } else
520 0
                        fprintf(stderr, "Error %d from VSLQ_Dispatch()", i);
521
        }
522
523 264
        return (i);
524
}
525
526
/**********************************************************************/
527
528
void v_noreturn_
529 30
VUT_Usage(const struct VUT *vut, const struct vopt_spec *voc, int status)
530
{
531
        const char **opt;
532
533 30
        fprintf(stderr, "Usage: %s <options>\n\n", vut->progname);
534 30
        fprintf(stderr, "Options:\n");
535 762
        for (opt = voc->vopt_usage; *opt != NULL; opt += 2)
536 732
                fprintf(stderr, " %-25s %s\n", *opt, *(opt + 1));
537 30
        exit(status);
538
}
539
540
/**********************************************************************/
541
542
543
static void
544 336
print_nobrackets(const char *s)
545
{
546
        const char *e;
547
548
        /* Remove whitespace */
549 336
        while (isspace(*s))
550 0
                s++;
551 336
        e = s + vstrlen(s);
552 336
        while (e > s && isspace(e[-1]))
553 0
                e--;
554
555
        /* Remove outer layer brackets if present */
556 336
        if (e > s && *s == '[' && e[-1] == ']') {
557 336
                s++;
558 336
                e--;
559 336
        }
560
561 336
        printf("%.*s", (int)(e - s), s);
562 336
}
563
564
static void
565 336
print_tabbed(const char *string, int tabs)
566
{
567
        int i;
568
        const char *c;
569
570 53991
        for (c = string; *c; c++) {
571 53655
                if (c == string || *(c - 1) == '\n')
572 720
                        for (i = 0; i < tabs; i++)
573 720
                                printf("\t");
574 53655
                printf("%c", *c);
575 53655
        }
576 336
}
577
578
static void
579 336
print_opt(const struct vopt_list *opt)
580
{
581 336
        print_nobrackets(opt->synopsis);
582 336
        printf("\n\n");
583 336
        print_tabbed(opt->ldesc, 1);
584 336
        printf("\n\n");
585 336
}
586
587
static int
588 15
vut_synopsis(const struct vopt_spec *voc)
589
{
590 15
        printf(".. |synopsis| replace:: %s\n", voc->vopt_synopsis);
591 15
        return (0);
592
}
593
594
static int
595 15
vut_options(const struct vopt_spec *voc)
596
{
597
        int i;
598
599 351
        for (i = 0; i < voc->vopt_list_n; i++)
600 336
                print_opt(&voc->vopt_list[i]);
601 15
        printf("--optstring\n"
602
            "\tPrint the optstring parameter to ``getopt(3)`` to help"
603
            " writing wrapper scripts.\n\n");
604 15
        return (0);
605
}