vinyl-cache/lib/libvinylapi/vsm.c
0
/*-
1
 * Copyright (c) 2006 Verdens Gang AS
2
 * Copyright (c) 2006-2015 Varnish Software AS
3
 * All rights reserved.
4
 *
5
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
6
 * Author: Martin Blix Grydeland <martin@varnish-software.com>
7
 *
8
 * SPDX-License-Identifier: BSD-2-Clause
9
 *
10
 * Redistribution and use in source and binary forms, with or without
11
 * modification, are permitted provided that the following conditions
12
 * are met:
13
 * 1. Redistributions of source code must retain the above copyright
14
 *    notice, this list of conditions and the following disclaimer.
15
 * 2. Redistributions in binary form must reproduce the above copyright
16
 *    notice, this list of conditions and the following disclaimer in the
17
 *    documentation and/or other materials provided with the distribution.
18
 *
19
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
23
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29
 * SUCH DAMAGE.
30
 */
31
32
#include "config.h"
33
34
#include <sys/mman.h>
35
#include <sys/stat.h>
36
37
#include <fcntl.h>
38
#include <float.h>
39
#include <math.h>
40
#include <stdarg.h>
41
#include <stdint.h>
42
#include <stdio.h>
43
#include <stdlib.h>
44
#include <string.h>
45
#include <unistd.h>
46
47
#include "vapi/vsig.h"
48
#include "vapi/vsm.h"
49
50
#include "vdef.h"
51
52
#include "miniobj.h"
53
#include "vas.h"
54
#include "vav.h"
55
#include "vin.h"
56
#include "vlu.h"
57
#include "vqueue.h"
58
#include "vsb.h"
59
#include "vsm_priv.h"
60
#include "vtim.h"
61
62
#ifndef MAP_HASSEMAPHORE
63
#  define MAP_HASSEMAPHORE 0 /* XXX Linux */
64
#endif
65
66
#ifndef MAP_NOSYNC
67
#  define MAP_NOSYNC 0 /* XXX Linux */
68
#endif
69
70
const struct vsm_valid VSM_invalid[1] = {{"invalid"}};
71
const struct vsm_valid VSM_valid[1] = {{"valid"}};
72
73
static vlu_f vsm_vlu_func;
74
75
#define VSM_PRIV_SHIFT                                                  \
76
        (sizeof (uint64_t) * 4)
77
#define VSM_PRIV_MASK                                                   \
78
        ((1ULL << VSM_PRIV_SHIFT) - 1)
79
#define VSM_PRIV_LOW(u)                                                 \
80
        ((uint64_t)(u) & VSM_PRIV_MASK)
81
#define VSM_PRIV_HIGH(u)                                                \
82
        (((uint64_t)(u) >> VSM_PRIV_SHIFT) & VSM_PRIV_MASK)
83
#define VSM_PRIV_MERGE(low, high)                                       \
84
        (VSM_PRIV_LOW(low) | (VSM_PRIV_LOW(high) << VSM_PRIV_SHIFT))
85
86
/*--------------------------------------------------------------------*/
87
88
struct vsm_set;
89
90
struct vsm_seg {
91
        unsigned                magic;
92
#define VSM_SEG_MAGIC           0xeb6c6dfd
93
        unsigned                flags;
94
#define VSM_FLAG_MARKSCAN       (1U<<1)
95
#define VSM_FLAG_STALE          (1U<<2)
96
#define VSM_FLAG_CLUSTER        (1U<<3)
97
        VTAILQ_ENTRY(vsm_seg)   list;
98
        VTAILQ_ENTRY(vsm_seg)   clist;
99
        struct vsm_set          *set;
100
        struct vsm_seg          *cluster;
101
        char                    **av;
102
        int                     refs;
103
        void                    *s;
104
        size_t                  sz;
105
        void                    *b;
106
        void                    *e;
107
        uint64_t                serial;
108
};
109
110
struct vsm_set {
111
        unsigned                magic;
112
#define VSM_SET_MAGIC           0xdee401b8
113
        const char              *dname;
114
        struct vsm              *vsm;
115
        VTAILQ_HEAD(,vsm_seg)   segs;
116
        VTAILQ_HEAD(,vsm_seg)   stale;
117
        VTAILQ_HEAD(,vsm_seg)   clusters;
118
119
        int                     dfd;
120
        struct stat             dst;
121
122
        int                     fd;
123
        struct stat             fst;
124
125
        uintmax_t               id1, id2;
126
127
        // _.index reading state
128
        struct vlu              *vlu;
129
        unsigned                lineno;
130
        unsigned                retval;
131
        struct vsm_seg          *vg;
132
133
        unsigned                flag_running;
134
        unsigned                flag_changed;
135
        unsigned                flag_restarted;
136
137
        int                     couldkill;
138
};
139
140
struct vsm {
141
        unsigned                magic;
142
#define VSM_MAGIC               0x6e3bd69b
143
144
        struct vsb              *diag;
145
        uint64_t                serial;
146
147
        int                     wdfd;
148
        struct stat             wdst;
149
        char                    *wdname;
150
151
        struct vsm_set          *mgt;
152
        struct vsm_set          *child;
153
154
        int                     attached;
155
        double                  patience;
156
};
157
158
/*--------------------------------------------------------------------*/
159
160
static int
161 1038
vsm_diag(struct vsm *vd, const char *fmt, ...)
162
{
163
        va_list ap;
164
165 1038
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
166 1038
        AN(fmt);
167
168 1038
        if (vd->diag == NULL)
169 1037
                vd->diag = VSB_new_auto();
170 1038
        AN(vd->diag);
171 1038
        VSB_clear(vd->diag);
172 1038
        va_start(ap, fmt);
173 1038
        VSB_vprintf(vd->diag, fmt, ap);
174 1038
        va_end(ap);
175 1038
        AZ(VSB_finish(vd->diag));
176 1038
        return (-1);
177
}
178
179
/*--------------------------------------------------------------------*/
180
181
static int
182 150753
vsm_mapseg(struct vsm *vd, struct vsm_seg *vg)
183
{
184
        size_t of, off, sz, ps, len;
185
        struct vsb *vsb;
186
        void *s;
187
        int fd;
188
189 150753
        CHECK_OBJ_NOTNULL(vg, VSM_SEG_MAGIC);
190
191 150753
        if (vg->s != NULL)
192 2121
                return (0);
193
194 148632
        ps = getpagesize();
195
196 148632
        of = strtoul(vg->av[2], NULL, 10);
197 148632
        off = RDN2(of, ps);
198
199 148632
        if (vg->flags & VSM_FLAG_CLUSTER)
200 876
                assert(of == 0);
201 148632
        assert(vg->cluster == NULL);
202
203 148632
        sz = strtoul(vg->av[3], NULL, 10);
204 148632
        assert(sz > 0);
205 148632
        assert(of >= off);
206 148632
        len = RUP2((of - off) + sz, ps);
207
208 148632
        vsb = VSB_new_auto();
209 148632
        AN(vsb);
210 148632
        VSB_printf(vsb, "%s/%s/%s", vd->wdname, vg->set->dname, vg->av[1]);
211 148632
        AZ(VSB_finish(vsb));
212
213 148632
        fd = open(VSB_data(vsb), O_RDONLY);     // XXX: openat
214 148632
        if (fd < 0) {
215 9
                VSB_destroy(&vsb);
216 9
                return (vsm_diag(vd, "Could not open segment"));
217
        }
218
219 297246
        s = (void*)mmap(NULL, len,
220
            PROT_READ,
221
            MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED,
222 148623
            fd, (off_t)off);
223
224 148623
        VSB_destroy(&vsb);
225
226 148623
        closefd(&fd);
227 148623
        if (s == MAP_FAILED)
228 0
                return (vsm_diag(vd, "Could not mmap segment"));
229
230 148623
        vg->s = s;
231 148623
        vg->b = (char*)(vg->s) + of - off;
232 148623
        vg->e = (char *)vg->b + sz;
233 148623
        vg->sz = len;
234
235 148623
        return (0);
236 150753
}
237
238
static void
239 132831
vsm_unmapseg(struct vsm_seg *vg)
240
{
241
242 132831
        CHECK_OBJ_NOTNULL(vg, VSM_SEG_MAGIC);
243
244 132831
        AN(vg->b);
245 132831
        AN(vg->e);
246 132831
        AZ(munmap(vg->s, vg->sz));
247 132831
        vg->s = vg->b = vg->e = NULL;
248 132831
        vg->sz = 0;
249 132831
}
250
251
/*--------------------------------------------------------------------*/
252
253
static void
254 1995669
vsm_delseg(struct vsm_seg *vg, int refsok)
255
{
256
257 1995669
        CHECK_OBJ_NOTNULL(vg, VSM_SEG_MAGIC);
258
259 1995669
        if (vg->set->vg == vg) {
260 111118
                AZ(vg->flags & VSM_FLAG_STALE);
261 111118
                vg->set->vg = VTAILQ_NEXT(vg, list);
262 111118
        }
263
264 1995669
        if (refsok && vg->refs) {
265 3287
                AZ(vg->flags & VSM_FLAG_STALE);
266 3287
                vg->flags |= VSM_FLAG_STALE;
267 3287
                VTAILQ_REMOVE(&vg->set->segs, vg, list);
268 3287
                VTAILQ_INSERT_TAIL(&vg->set->stale, vg, list);
269 3287
                return;
270
        }
271
272 1992382
        if (vg->s != NULL)
273 0
                vsm_unmapseg(vg);
274
275 1992382
        if (vg->flags & VSM_FLAG_CLUSTER) {
276 47082
                vg->flags &= ~VSM_FLAG_CLUSTER;
277 47082
                VTAILQ_REMOVE(&vg->set->clusters, vg, clist);
278 47082
        }
279
280 1992382
        if (vg->flags & VSM_FLAG_STALE)
281 3287
                VTAILQ_REMOVE(&vg->set->stale, vg, list);
282
        else
283 1989095
                VTAILQ_REMOVE(&vg->set->segs, vg, list);
284 1992382
        VAV_Free(vg->av);
285 1992382
        FREE_OBJ(vg);
286 1995669
}
287
288
/*--------------------------------------------------------------------*/
289
290
static struct vsm_set *
291 104534
vsm_newset(const char *dirname)
292
{
293
        struct vsm_set *vs;
294
295 104534
        ALLOC_OBJ(vs, VSM_SET_MAGIC);
296 104534
        AN(vs);
297 104534
        VTAILQ_INIT(&vs->segs);
298 104534
        VTAILQ_INIT(&vs->stale);
299 104534
        VTAILQ_INIT(&vs->clusters);
300 104534
        vs->dname = dirname;
301 104534
        vs->dfd = vs->fd = -1;
302 104534
        vs->vlu = VLU_New(vsm_vlu_func, vs, 0);
303 104534
        AN(vs->vlu);
304 104534
        if (getenv("VSM_NOPID") != NULL)
305 0
                vs->couldkill = -1;
306 104534
        return (vs);
307
}
308
309
static void
310 103190
vsm_delset(struct vsm_set **p)
311
{
312
        struct vsm_set *vs;
313
        struct vsm_seg *vg;
314
315 103190
        TAKE_OBJ_NOTNULL(vs, p, VSM_SET_MAGIC);
316
317 103190
        if (vs->fd >= 0)
318 64115
                closefd(&vs->fd);
319 103190
        if (vs->dfd >= 0)
320 85342
                closefd(&vs->dfd);
321 103190
        while ((vg = VTAILQ_FIRST(&vs->stale)) != NULL) {
322 0
                AN(vg->flags & VSM_FLAG_STALE);
323 0
                vsm_delseg(vg, 0);
324
        }
325 1945111
        while ((vg = VTAILQ_FIRST(&vs->segs)) != NULL) {
326 1841921
                AZ(vg->flags & VSM_FLAG_STALE);
327 1841921
                vsm_delseg(vg, 0);
328
        }
329 103190
        assert(VTAILQ_EMPTY(&vs->clusters));
330 103190
        VLU_Destroy(&vs->vlu);
331 103190
        FREE_OBJ(vs);
332 103190
}
333
334
static void
335 269762
vsm_wash_set(const struct vsm_set *vs, int all)
336
{
337
        struct vsm_seg *vg, *vg2;
338
339 2209265
        VTAILQ_FOREACH_SAFE(vg, &vs->segs, list, vg2) {
340 1939503
                if (all || (vg->flags & VSM_FLAG_MARKSCAN) == 0)
341 62866
                        vsm_delseg(vg, 1);
342 1939503
        }
343 269762
}
344
345
/*--------------------------------------------------------------------*/
346
347
struct vsm *
348 52267
VSM_New(void)
349
{
350
        struct vsm *vd;
351
352 52267
        ALLOC_OBJ(vd, VSM_MAGIC);
353 52267
        AN(vd);
354
355 52267
        vd->mgt = vsm_newset(VSM_MGT_DIRNAME);
356 52267
        vd->mgt->flag_running = VSM_MGT_RUNNING;
357 52267
        vd->mgt->flag_changed = VSM_MGT_CHANGED;
358 52267
        vd->mgt->flag_restarted = VSM_MGT_RESTARTED;
359
360 52267
        vd->child = vsm_newset(VSM_CHILD_DIRNAME);
361 52267
        vd->child->flag_running = VSM_WRK_RUNNING;
362 52267
        vd->child->flag_changed = VSM_WRK_CHANGED;
363 52267
        vd->child->flag_restarted = VSM_WRK_RESTARTED;
364
365 52267
        vd->mgt->vsm = vd;
366 52267
        vd->child->vsm = vd;
367 52267
        vd->wdfd = -1;
368 52267
        vd->patience = 5;
369 52267
        return (vd);
370
}
371
372
/*--------------------------------------------------------------------*/
373
374
int
375 53254
VSM_Arg(struct vsm *vd, char flag, const char *arg)
376
{
377 53254
        char *p = NULL;
378
379 53254
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
380
381 53254
        if (arg == NULL)
382 840
                return (1);
383 52414
        switch (flag) {
384
        case 't':
385 357
                if (!strcasecmp(arg, "off")) {
386 0
                        vd->patience = -1;
387 0
                } else {
388 357
                        vd->patience = strtod(arg, &p);
389 609
                        if ((p != NULL && *p != '\0') ||
390 252
                            !isfinite(vd->patience) || vd->patience < 0)
391 294
                                return (vsm_diag(vd,
392 147
                                    "-t: Invalid argument: %s", arg));
393
                }
394 210
                break;
395
        case 'n':
396 52057
                if (vd->wdname != NULL)
397 0
                        free(vd->wdname);
398 52057
                vd->wdname = VIN_n_Arg(arg);
399 52057
                if (vd->wdname == NULL)
400 0
                        return (vsm_diag(vd, "Invalid instance name: %s",
401 0
                            strerror(errno)));
402 52057
                break;
403
        default:
404 0
                return (vsm_diag(vd, "Unknown VSM_Arg('%c')", flag));
405
        }
406 52267
        return (1);
407 53254
}
408
409
/*--------------------------------------------------------------------*/
410
411
void
412 51595
VSM_Destroy(struct vsm **vdp)
413
{
414
        struct vsm *vd;
415
416 51595
        TAKE_OBJ_NOTNULL(vd, vdp, VSM_MAGIC);
417
418 51595
        VSM_ResetError(vd);
419 51595
        REPLACE(vd->wdname, NULL);
420 51595
        if (vd->diag != NULL)
421 0
                VSB_destroy(&vd->diag);
422 51595
        if (vd->wdfd >= 0)
423 51553
                closefd(&vd->wdfd);
424 51595
        vsm_delset(&vd->mgt);
425 51595
        vsm_delset(&vd->child);
426 51595
        FREE_OBJ(vd);
427 51595
}
428
429
/*--------------------------------------------------------------------*/
430
431
const char *
432 240
VSM_Error(const struct vsm *vd)
433
{
434
435 240
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
436
437 240
        if (vd->diag == NULL)
438 0
                return ("No VSM error");
439
        else
440 240
                return (VSB_data(vd->diag));
441 240
}
442
443
/*--------------------------------------------------------------------*/
444
445
void
446 104261
VSM_ResetError(struct vsm *vd)
447
{
448
449 104261
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
450
451 104261
        if (vd->diag == NULL)
452 103476
                return;
453 785
        VSB_destroy(&vd->diag);
454 104261
}
455
456
/*--------------------------------------------------------------------
457
 */
458
459
static int
460 2179752
vsm_cmp_av(char * const *a1, char * const *a2)
461
{
462
463 2579632
        while (1) {
464 2579632
                if (*a1 == NULL && *a2 == NULL)
465 88693
                        return (0);
466 2490939
                if (*a1 == NULL || *a2 == NULL)
467 0
                        return (1);
468 2490939
                if (vstrcmp(*a1, *a2))
469 2091059
                        return (1);
470 399880
                a1++;
471 399880
                a2++;
472
        }
473 2179752
}
474
475
static struct vsm_seg *
476 45536
vsm_findcluster(const struct vsm_set *vs, const char *cnam)
477
{
478
        struct vsm_seg *vg;
479 45536
        AN(vs);
480 45536
        AN(cnam);
481 63481
        VTAILQ_FOREACH(vg, &vs->clusters, clist) {
482 63481
                AN(vg->av[1]);
483 63481
                if (!vstrcmp(cnam, vg->av[1]))
484 45536
                        return (vg);
485 17945
        }
486 0
        return (NULL);
487 45536
}
488
489
static unsigned
490 1878283
vsm_running(struct vsm_set *vs, pid_t pid)
491
{
492
493 1878283
        AN(vs);
494
495 1878283
        if (pid == 0)
496 0
                return (0);
497
498 1878283
        if (kill(pid, 0) == 0) {
499 1827018
                vs->couldkill = 1;
500 1827018
                return (1);
501
        }
502 51265
        if (errno == EPERM)     /* a process exists, assume running */
503 0
                return (1);
504 51265
        assert(errno != EINVAL);
505 51265
        return (0);
506 1878283
}
507
508
static int
509 87457
vsm_vlu_hash(struct vsm_set *vs, const char *line)
510
{
511
        int i;
512
        uintmax_t id1, id2;
513
514 87457
        i = sscanf(line, "# %ju %ju", &id1, &id2);
515 87457
        if (i != 2) {
516 0
                vs->retval |= vs->flag_restarted;
517 0
                return (0);
518
        }
519 87457
        if (vs->couldkill >= 0 && vsm_running(vs, id1)) {
520
                /* nothing to do */
521 87457
        } else if (vs->couldkill > 0 && errno == ESRCH) {
522 0
                vs->retval |= vs->flag_restarted | VSM_MGT_CHANGED;
523 0
                return (0);
524
        }
525 87457
        vs->retval |= VSM_MGT_RUNNING;
526 87457
        if (id1 != vs->id1 || id2 != vs->id2) {
527 87421
                vs->retval |= vs->flag_restarted;
528 87421
                vs->id1 = id1;
529 87421
                vs->id2 = id2;
530 87421
        }
531 87457
        return (0);
532 87457
}
533
534
static char **
535 2101781
vsm_parse_plus_minus_line(struct vsm *vd, const char *line, int *acp)
536
{
537
        char **av;
538
        int ac;
539
540 2101781
        AN(acp);
541 2101781
        *acp = 0;
542
543 2101781
        av = VAV_Parse(line + 1, &ac, 0);
544 2101781
        AN(av);
545
546 2101781
        if (av[0] != NULL) {
547 21
                (void)vsm_diag(vd, "%s", av[0]);
548 21
                VAV_Free(av);
549 21
                return (NULL);
550
        }
551 2101760
        if (ac < 4 || ac > 6) {
552 21
                (void)vsm_diag(vd, "Wrong number of fields (%u)", ac-1);
553 21
                VAV_Free(av);
554 21
                return (NULL);
555
        }
556 2101739
        *acp = ac;
557 2101739
        return (av);
558 2101781
}
559
560
static int
561 2014186
vsm_vlu_plus(struct vsm *vd, struct vsm_set *vs, const char *line)
562
{
563
        char **av;
564
        int ac;
565
        struct vsm_seg *vg;
566
567 2014186
        av = vsm_parse_plus_minus_line(vd, line, &ac);
568 2014186
        if (av == NULL)
569 42
                return (-1);
570
571 2014144
        vg = vs->vg;
572 2014144
        CHECK_OBJ_ORNULL(vg, VSM_SEG_MAGIC);
573 2014144
        if (vg != NULL)
574 1098
                AZ(vg->flags & VSM_FLAG_STALE);
575 2014144
        while (vg != NULL && vsm_cmp_av(&vg->av[1], &av[1]))
576 0
                vg = VTAILQ_NEXT(vg, list);
577 2014144
        if (vg != NULL) {
578
                /* entry compared equal, so it survives */
579 1098
                CHECK_OBJ_NOTNULL(vg, VSM_SEG_MAGIC);
580 1098
                VAV_Free(av);
581 1098
                vg->flags |= VSM_FLAG_MARKSCAN;
582 1098
                vs->vg = VTAILQ_NEXT(vg, list);
583 1098
        } else {
584 2013046
                ALLOC_OBJ(vg, VSM_SEG_MAGIC);
585 2013046
                AN(vg);
586 2013046
                vg->av = av;
587 2013046
                vg->set = vs;
588 2013046
                vg->flags = VSM_FLAG_MARKSCAN;
589 2013046
                vg->serial = vd->serial;
590
591 2013046
                VTAILQ_INSERT_TAIL(&vs->segs, vg, list);
592 2013046
                if (ac == 4) {
593 47418
                        vg->flags |= VSM_FLAG_CLUSTER;
594 47418
                        VTAILQ_INSERT_TAIL(&vs->clusters, vg, clist);
595 2013046
                } else if (*vg->av[2] != '0') {
596 44436
                        vg->cluster = vsm_findcluster(vs, vg->av[1]);
597 44436
                        CHECK_OBJ_NOTNULL(vg->cluster, VSM_SEG_MAGIC);
598 44436
                }
599 2013046
                vs->retval |= vs->flag_changed;
600
        }
601 2014144
        return (0);
602 2014186
}
603
604
static int
605 87595
vsm_vlu_minus(struct vsm *vd, struct vsm_set *vs, const char *line)
606
{
607
        char **av;
608
        int ac;
609
        struct vsm_seg *vg;
610
611 87595
        av = vsm_parse_plus_minus_line(vd, line, &ac);
612 87595
        if (av == NULL)
613 0
                return (-1);
614
615
        /* Clustered segments cannot come before their cluster */
616 87595
        if (*av[2] != '0')
617 1100
                vg = vsm_findcluster(vs, av[1]);
618
        else
619 86495
                vg = VTAILQ_FIRST(&vs->segs);
620
621 2178654
        for (;vg != NULL; vg = VTAILQ_NEXT(vg, list)) {
622 2178654
                if (!vsm_cmp_av(&vg->av[1], &av[1])) {
623 87595
                        vs->retval |= vs->flag_changed;
624 87595
                        vsm_delseg(vg, 1);
625 87595
                        break;
626
                }
627 2091059
        }
628 87595
        AN(vg);
629 87595
        VAV_Free(av);
630 87595
        return (0);
631 87595
}
632
633
static int v_matchproto_(vlu_f)
634 2189259
vsm_vlu_func(void *priv, const char *line)
635
{
636
        struct vsm *vd;
637
        struct vsm_set *vs;
638 2189259
        int i = 0;
639
640 2189259
        CAST_OBJ_NOTNULL(vs, priv, VSM_SET_MAGIC);
641 2189259
        vd = vs->vsm;
642 2189259
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
643 2189259
        AN(line);
644
645 2189259
        vs->lineno++;
646
647
        /* Up the serial counter. This wraps at UINTPTR_MAX/2
648
         * because thats the highest value we can store in struct
649
         * vsm_fantom. */
650 2189259
        vd->serial = VSM_PRIV_LOW(vd->serial + 1);
651
652 2189259
        switch (line[0]) {
653
        case '#':
654 87457
                i = vsm_vlu_hash(vs, line);
655 151421
                VTAILQ_FOREACH(vs->vg, &vs->segs, list)
656 63964
                        vs->vg->flags &= ~VSM_FLAG_MARKSCAN;
657 87457
                if (!(vs->retval & vs->flag_restarted))
658 36
                        vs->vg = VTAILQ_FIRST(&vs->segs);
659 87457
                break;
660
        case '+':
661 2014186
                i = vsm_vlu_plus(vd, vs, line);
662 2014186
                break;
663
        case '-':
664 87595
                i = vsm_vlu_minus(vd, vs, line);
665 87595
                break;
666
        default:
667 21
                i = vsm_diag(vd, "Unknown ident");
668 21
                break;
669
        }
670 2189259
        return (i);
671
}
672
673
static void
674 1790912
vsm_readlines(struct vsm_set *vs)
675
{
676
        struct vsm *vd;
677
        int i;
678
679 1790912
        vd = vs->vsm;
680 1790912
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
681
682 1790912
        do {
683 2025010
                assert(vs->fd >= 0);
684 2025010
                i = VLU_Fd(vs->vlu, vs->fd);
685 2025010
        } while (!i);
686 1790912
        if (i != -2) {
687
                /* The line handler left the reason in the diag */
688 84
                if (vd->diag != NULL) {
689 63
                        fprintf(
690 63
                            stderr,
691
                            "Malformed _.index line (%u): %s\n",
692 63
                            vs->lineno,
693 63
                            VSB_data(vd->diag)
694
                        );
695 63
                } else {
696 21
                        fprintf(
697 21
                            stderr,
698
                            "Could not read _.index:\nPossibly relevant: %s",
699 21
                            strerror(errno)
700
                        );
701
                }
702 84
                exit(2);
703
        }
704 1790828
}
705
706
static unsigned
707 2013821
vsm_refresh_set(struct vsm *vd, struct vsm_set *vs)
708
{
709 2013821
        unsigned restarted = 0;
710
        struct stat st;
711
712 2013821
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
713 2013821
        CHECK_OBJ_NOTNULL(vs, VSM_SET_MAGIC);
714 2013821
        vs->retval = 0;
715 3757828
        if (vs->dfd >= 0 && (
716 1745349
            fstatat(vd->wdfd, vs->dname, &st, AT_SYMLINK_NOFOLLOW) ||
717 1745351
            st.st_ino != vs->dst.st_ino ||
718 1744007
            st.st_dev != vs->dst.st_dev ||
719 1744007
            st.st_mode != vs->dst.st_mode ||
720 1744007
            st.st_nlink == 0)) {
721 1352
                closefd(&vs->dfd);
722 1344
                restarted = vs->flag_restarted;
723 1344
        }
724
725 2013819
        if (vs->dfd < 0) {
726 269810
                if (vs->fd >= 0)
727 1337
                        closefd(&vs->fd);
728 269810
                vs->dfd = openat(vd->wdfd, vs->dname, O_RDONLY);
729 269810
        }
730
731 2013819
        if (vs->dfd < 0) {
732 182368
                vs->id1 = vs->id2 = 0;
733 182368
                vsm_wash_set(vs, 1);
734 182368
                return (vs->retval | restarted);
735
        }
736
737 1831451
        AZ(fstat(vs->dfd, &vs->dst));
738
739 3534886
        if (vs->fd >= 0 && (
740 1724703
            fstatat(vs->dfd, "_.index", &st, AT_SYMLINK_NOFOLLOW) ||
741 1703472
            st.st_ino != vs->fst.st_ino ||
742 1703436
            st.st_dev != vs->fst.st_dev ||
743 1703436
            st.st_mode != vs->fst.st_mode ||
744 1703435
            st.st_size < vs->fst.st_size ||
745 1703435
            st.st_nlink < 1)) {
746 21272
                closefd(&vs->fd);
747 21270
                vs->retval |= vs->flag_changed;
748 21270
        }
749
750 1831449
        if (vs->fd >= 0) {
751 1703435
                vs->vg = NULL;
752 1703435
                vsm_readlines(vs);
753 1703435
        } else {
754 394032
                VTAILQ_FOREACH(vs->vg, &vs->segs, list)
755 266018
                        vs->vg->flags &= ~VSM_FLAG_MARKSCAN;
756 128014
                vs->vg = VTAILQ_FIRST(&vs->segs);
757 128014
                vs->fd = openat(vs->dfd, "_.index", O_RDONLY);
758 128014
                if (vs->fd < 0)
759 40536
                        return (vs->retval | restarted);
760 87478
                VLU_Reset(vs->vlu);
761 87478
                vs->lineno = 0;
762 87478
                AZ(fstat(vs->fd, &vs->fst));
763 87478
                vsm_readlines(vs);
764 87478
                vsm_wash_set(vs, 0);
765
        }
766
767 1790913
        vs->fst.st_size = lseek(vs->fd, 0L, SEEK_CUR);
768
769 1790913
        if (vs->couldkill < 0 || vsm_running(vs, vs->id1))
770 1739651
                vs->retval |= vs->flag_running;
771 1790913
        return (vs->retval);
772 2013817
}
773
774
/*--------------------------------------------------------------------*/
775
776
unsigned
777 1027352
VSM_Status(struct vsm *vd)
778
{
779 1027352
        unsigned retval = 0;
780
        struct stat st;
781
782 1027352
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
783
784
        /* See if the -n workdir changed */
785 1027352
        if (vd->wdfd >= 0) {
786 974647
                AZ(fstat(vd->wdfd, &st));
787 1949293
                if (st.st_ino != vd->wdst.st_ino ||
788 974646
                    st.st_dev != vd->wdst.st_dev ||
789 974646
                    st.st_mode != vd->wdst.st_mode ||
790 974646
                    st.st_nlink == 0) {
791 2
                        closefd(&vd->wdfd);
792 0
                        vsm_wash_set(vd->mgt, 1);
793 0
                        vsm_wash_set(vd->child, 1);
794 0
                }
795 974645
        }
796
797
        /* Open workdir */
798 1027350
        if (vd->wdfd < 0) {
799 52708
                retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED;
800 52708
                retval |= VSM_WRK_RESTARTED | VSM_WRK_CHANGED;
801 52708
                vd->wdfd = open(vd->wdname, O_RDONLY);
802 52708
                if (vd->wdfd < 0)
803 735
                        (void)vsm_diag(vd,
804
                            "VSM_Status: Cannot open workdir");
805
                else
806 51973
                        AZ(fstat(vd->wdfd, &vd->wdst));
807 52708
        }
808
809 1027350
        if (vd->wdfd >= 0) {
810 1026533
                retval |= vsm_refresh_set(vd, vd->mgt);
811 1026533
                if (vd->mgt->couldkill > 0 && (retval & VSM_MGT_RESTARTED))
812 51889
                        vd->mgt->couldkill = 0;
813 1026533
                if (retval & VSM_MGT_RUNNING)
814 987198
                        retval |= vsm_refresh_set(vd, vd->child);
815 1026533
                if (vd->child->couldkill > 0 && (retval & VSM_WRK_RESTARTED))
816 35391
                        vd->child->couldkill = 0;
817 1026533
        }
818 1027350
        return (retval);
819
}
820
821
/*--------------------------------------------------------------------*/
822
823
int
824 52099
VSM_Attach(struct vsm *vd, int progress)
825
{
826
        const char *def;
827
        double t0;
828
        unsigned u;
829 52099
        int i, n = 0;
830
831 52099
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
832
833 52099
        if (vd->patience < 0)
834 42
                t0 = DBL_MAX;
835
        else
836 52057
                t0 = VTIM_mono() + vd->patience;
837
838 52099
        if (vd->wdname == NULL) {
839 147
                def = getenv("VINYL_DEFAULT_N");
840 147
                if (def == NULL)
841 0
                        def = ""; /* Use default (hostname) */
842 147
                i = VSM_Arg(vd, 'n', def);
843 147
                if (i < 0)
844 0
                        return (i);
845 147
                AN(vd->wdname);
846 147
        }
847
848 52099
        AZ(vd->attached);
849 52813
        while (!VSIG_int && !VSIG_term) {
850 52666
                u = VSM_Status(vd);
851 52666
                VSM_ResetError(vd);
852 52666
                if (u & VSM_MGT_RUNNING) {
853 51889
                        if (progress >= 0 && n > 4)
854 0
                                (void)write(progress, "\n", 1);
855 51889
                        vd->attached = 1;
856 51889
                        return (0);
857
                }
858 777
                if (t0 < VTIM_mono()) {
859 63
                        if (progress >= 0 && n > 4)
860 21
                                (void)write(progress, "\n", 1);
861 63
                        return (vsm_diag(vd,
862
                            "Could not get hold of vinyld, is it running?"));
863
                }
864 714
                if (progress >= 0 && !(++n % 4))
865 168
                        (void)write(progress, ".", 1);
866 714
                VTIM_sleep(.25);
867
        }
868 21
        return (vsm_diag(vd, "Attach interrupted"));
869 51973
}
870
871
/*--------------------------------------------------------------------*/
872
873
static struct vsm_seg *
874 196692
vsm_set_findseg(const struct vsm_set *vs, uintptr_t serial)
875
{
876
        struct vsm_seg *vg;
877
878 2402788
        VTAILQ_FOREACH(vg, &vs->segs, list) {
879 2289221
                if (vg->serial == serial)
880 83125
                        return (vg);
881 2206096
        }
882 196748
        VTAILQ_FOREACH(vg, &vs->stale, list) {
883 86426
                if (vg->serial == serial)
884 3245
                        return (vg);
885 83181
        }
886 110322
        return (NULL);
887 196692
}
888
889
static struct vsm_seg *
890 5351099
vsm_findseg(const struct vsm *vd, const struct vsm_fantom *vf)
891
{
892
        struct vsm_seg *vg;
893
        uint64_t x;
894
895 5351099
        x = VSM_PRIV_HIGH(vf->priv);
896 5351099
        if (x == vd->serial) {
897 5251748
                vg = (struct vsm_seg *)vf->priv2;
898 5251748
                if (!VALID_OBJ(vg, VSM_SEG_MAGIC) ||
899 5251748
                    vg->serial != VSM_PRIV_LOW(vf->priv))
900 0
                        WRONG("Corrupt fantom");
901 5251748
                return (vg);
902
        }
903
904 99351
        x = VSM_PRIV_LOW(vf->priv);
905 99351
        vg = vsm_set_findseg(vd->mgt, x);
906 99351
        if (vg == NULL)
907 97341
                vg = vsm_set_findseg(vd->child, x);
908 99351
        if (vg == NULL)
909 12981
                return (NULL);
910
911
        /* Update the fantom with the new priv so that lookups will be
912
         * fast on the next call. Note that this casts away the const. */
913 86370
        ((struct vsm_fantom *)TRUST_ME(vf))->priv =
914 86370
            VSM_PRIV_MERGE(vg->serial, vd->serial);
915 86370
        return (vg);
916 5351099
}
917
918
/*--------------------------------------------------------------------*/
919
920
void
921 238843
VSM__iter0(const struct vsm *vd, struct vsm_fantom *vf)
922
{
923
924 238843
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
925 238843
        AN(vf);
926
927 238843
        AN(vd->attached);
928 238843
        memset(vf, 0, sizeof *vf);
929 238843
}
930
931
int
932 3109280
VSM__itern(struct vsm *vd, struct vsm_fantom *vf)
933
{
934
        struct vsm_seg *vg;
935
936 3109280
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
937 3109280
        AN(vd->attached);
938 3109280
        AN(vf);
939
940 3109280
        if (vf->priv == 0) {
941 238844
                vg = VTAILQ_FIRST(&vd->mgt->segs);
942 238844
                if (vg == NULL)
943 1
                        return (0);
944 238843
        } else {
945 2870436
                vg = vsm_findseg(vd, vf);
946 2870436
                if (vg == NULL)
947 0
                        return (vsm_diag(vd, "VSM_FOREACH: inconsistency"));
948 2908544
                while (1) {
949 2908544
                        if (vg->set == vd->mgt && VTAILQ_NEXT(vg, list) == NULL)
950 236889
                                vg = VTAILQ_FIRST(&vd->child->segs);
951
                        else
952 2671655
                                vg = VTAILQ_NEXT(vg, list);
953 2908544
                        if (vg == NULL)
954 172490
                                return (0);
955 2736054
                        if (!(vg->flags & VSM_FLAG_CLUSTER))
956 2697946
                                break;
957
                }
958
        }
959 2936789
        memset(vf, 0, sizeof *vf);
960 2936789
        vf->priv = VSM_PRIV_MERGE(vg->serial, vd->serial);
961 2936789
        vf->priv2 = (uintptr_t)vg;
962 2936789
        vf->category = vg->av[4];
963 2936789
        vf->ident = vg->av[5];
964 2936789
        AN(vf->category);
965 2936789
        return (1);
966 3109280
}
967
968
/*--------------------------------------------------------------------*/
969
970
int
971 150753
VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
972
{
973
        struct vsm_seg *vg, *vgc;
974
        size_t of, sz;
975
        int r;
976
977 150753
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
978 150753
        AN(vd->attached);
979 150753
        AN(vf);
980 150753
        vg = vsm_findseg(vd, vf);
981 150753
        if (vg == NULL)
982 0
                return (vsm_diag(vd, "VSM_Map: bad fantom"));
983
984 150753
        assert(vg->serial == VSM_PRIV_LOW(vf->priv));
985 150753
        assert(vg->av[4] == vf->category);
986 150753
        assert(vg->av[5] == vf->ident);
987
988 150753
        if (vg->b != NULL) {
989 0
                assert(vg->refs > 0);
990 0
                AN(vg->e);
991 0
                vf->b = vg->b;
992 0
                vf->e = vg->e;
993 0
                vg->refs++;
994 0
                return (0);
995
        }
996
997 150753
        assert(vg->refs == 0);
998
999 150753
        vgc = vg->cluster;
1000
1001 150753
        if (vgc == NULL) {
1002 147756
                r = vsm_mapseg(vd, vg);
1003 147756
                if (r)
1004 9
                        return (r);
1005 147747
                vf->b = vg->b;
1006 147747
                vf->e = vg->e;
1007
1008 147747
                vg->refs++;
1009
1010 147747
                return (0);
1011
        }
1012
1013 2997
        CHECK_OBJ(vgc, VSM_SEG_MAGIC);
1014 2997
        assert(vgc->flags & VSM_FLAG_CLUSTER);
1015 2997
        assert(vg->s == NULL);
1016 2997
        assert(vg->sz == 0);
1017
1018 2997
        r = vsm_mapseg(vd, vgc);
1019 2997
        if (r)
1020 0
                return (r);
1021 2997
        vgc->refs++;
1022
1023 2997
        of = strtoul(vg->av[2], NULL, 10);
1024 2997
        sz = strtoul(vg->av[3], NULL, 10);
1025 2997
        assert(sz > 0);
1026
1027 2997
        assert(vgc->sz >= of + sz);
1028 2997
        assert(vgc->s == vgc->b);
1029 2997
        vg->b = (char *)vgc->b + of;
1030 2997
        vg->e = (char *)vg->b + sz;
1031
1032 2997
        vf->b = vg->b;
1033 2997
        vf->e = vg->e;
1034
1035 2997
        vg->refs++;
1036
1037 2997
        return (0);
1038 150753
}
1039
1040
/*--------------------------------------------------------------------*/
1041
1042
int
1043 132873
VSM_Unmap(struct vsm *vd, struct vsm_fantom *vf)
1044
{
1045
        struct vsm_seg *vg;
1046
1047 132873
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
1048 132873
        AN(vd->attached);
1049 132873
        AN(vf);
1050 132873
        AN(vf->b);
1051 132873
        vg = vsm_findseg(vd, vf);
1052 132873
        if (vg == NULL)
1053 0
                return (vsm_diag(vd, "VSM_Unmap: bad fantom"));
1054 132873
        CHECK_OBJ(vg, VSM_SEG_MAGIC);
1055 132873
        assert(vg->refs > 0);
1056 132873
        vg->refs--;
1057 132873
        vf->b = NULL;
1058 132873
        vf->e = NULL;
1059 132873
        if (vg->refs > 0)
1060 0
                return (0);
1061
1062 132873
        if (vg->cluster) {
1063 582
                CHECK_OBJ_NOTNULL(vg->cluster, VSM_SEG_MAGIC);
1064 582
                assert(vg->s == NULL);
1065 582
                assert(vg->sz == 0);
1066 582
                assert(vg->cluster->refs > 0);
1067 582
                if (--vg->cluster->refs == 0) {
1068 540
                        vsm_unmapseg(vg->cluster);
1069 540
                        if (vg->cluster->flags & VSM_FLAG_STALE) {
1070 42
                                AN(vg->flags & VSM_FLAG_STALE);
1071 42
                                vsm_delseg(vg->cluster, 0);
1072 42
                        }
1073 540
                }
1074 582
                vg->b = vg->e = NULL;
1075 582
        } else {
1076 132291
                vsm_unmapseg(vg);
1077
        }
1078 132873
        if (vg->flags & VSM_FLAG_STALE)
1079 3245
                vsm_delseg(vg, 0);
1080 132873
        return (0);
1081 132873
}
1082
1083
/*--------------------------------------------------------------------*/
1084
1085
const struct vsm_valid *
1086 2197108
VSM_StillValid(const struct vsm *vd, const struct vsm_fantom *vf)
1087
{
1088
        struct vsm_seg *vg;
1089
1090 2197108
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
1091 2197108
        AN(vf);
1092 2197108
        vg = vsm_findseg(vd, vf);
1093 2197108
        if (vg == NULL || vg->flags & VSM_FLAG_STALE)
1094 16495
                return (VSM_invalid);
1095 2180613
        return (VSM_valid);
1096 2197108
}
1097
1098
/*--------------------------------------------------------------------*/
1099
1100
int
1101 206057
VSM_Get(struct vsm *vd, struct vsm_fantom *vf,
1102
    const char *category, const char *ident)
1103
{
1104
1105 206057
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
1106 206057
        AN(vd->attached);
1107 1428952
        VSM_FOREACH(vf, vd) {
1108 1287294
                if (vstrcmp(vf->category, category))
1109 1222895
                        continue;
1110 64399
                if (ident != NULL && vstrcmp(vf->ident, ident))
1111 0
                        continue;
1112 64399
                return (1);
1113
        }
1114 141658
        memset(vf, 0, sizeof *vf);
1115 141658
        return (0);
1116 206057
}
1117
1118
/*--------------------------------------------------------------------*/
1119
1120
char *
1121 1953
VSM_Dup(struct vsm *vd, const char *category, const char *ident)
1122
{
1123
        struct vsm_fantom vf;
1124 1953
        char *p = NULL;
1125
1126 1953
        CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
1127 1953
        AN(vd->attached);
1128 7833
        VSM_FOREACH(&vf, vd) {
1129 7833
                if (vstrcmp(vf.category, category))
1130 3360
                        continue;
1131 4473
                if (ident != NULL && vstrcmp(vf.ident, ident))
1132 2520
                        continue;
1133 1953
                AZ(VSM_Map(vd, &vf));
1134 1953
                AN(vf.b);
1135 1953
                AN(vf.e);
1136 1953
                p = malloc((char*)vf.e - (char*)vf.b);
1137 1953
                AN(p);
1138 1953
                vmemcpy(p, vf.b, (char *)vf.e - (char *)vf.b);
1139 1953
                AZ(VSM_Unmap(vd, &vf));
1140 1953
                break;
1141
        }
1142 1953
        return (p);
1143
}