GCOV - Varnish Cache


/*-
 * Copyright (c) 2006 Verdens Gang AS
 * Copyright (c) 2006-2015 Varnish Software AS
 * All rights reserved.
 *
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include "config.h"

#include "cache_int.h"

#include <stdio.h>
#include <stdlib.h>

#include "vsl_priv.h"
#include "vmb.h"

#include "common/heritage.h"
#include "common/vsmw.h"

/* ------------------------------------------------------------
 * strands helpers - move elsewhere?
 */

static unsigned
strands_len(const struct strands *s)
{
        unsigned r = 0;
        int i;

        CHECK_OBJ_NOTNULL(s, STRANDS_MAGIC);
        for (i = 0; i < s->n; i++) {
                if (s->p[i] == NULL || *s->p[i] == '\0')
                        continue;
                r += vstrlen(s->p[i]);
        }

        return (r);
}

/*
 * like VRT_Strands(), but truncating instead of failing for end of buffer
 *
 * returns number of bytes including NUL
 */
static unsigned
strands_cat(char *buf, unsigned bufl, const struct strands *s)
{
        unsigned l = 0, ll;
        int i;

        /* NUL-terminated */
        assert(bufl > 0);
        bufl--;
        CHECK_OBJ_NOTNULL(s, STRANDS_MAGIC);

        for (i = 0; i < s->n && bufl > 0; i++) {
                if (s->p[i] == NULL || *s->p[i] == '\0')
                        continue;
                ll = vmin_t(unsigned, vstrlen(s->p[i]), bufl);
                vmemcpy(buf, s->p[i], ll);
                l += ll;
                buf += ll;
                bufl -= ll;
        }
        *buf = '\0';    /* NUL-terminated */
        return (l + 1);
}

/* These cannot be struct lock, which depends on vsm/vsl working */
static pthread_mutex_t vsl_mtx;
static pthread_mutex_t vsc_mtx;
static pthread_mutex_t vsm_mtx;

static struct VSL_head          *vsl_head;
static const uint32_t           *vsl_end;
static uint32_t                 *vsl_ptr;
static unsigned                 vsl_segment_n;
static ssize_t                  vsl_segsize;

struct VSC_main *VSC_C_main;

static void
vsl_sanity(const struct vsl_log *vsl)
{

        CHECK_OBJ_NOTNULL(vsl, VSL_LOG_MAGIC);
        AN(vsl);
        AN(vsl->wlp);
        AN(vsl->wlb);
        AN(vsl->wle);
        assert(vsl->wlb <= vsl->wlp);
        assert(vsl->wlp <= vsl->wle);
}

/*--------------------------------------------------------------------
 * Check if the VSL_tag is masked by parameter bitmap
 */

static inline int
vsl_tag_is_masked(enum VSL_tag_e tag)
{
        volatile uint8_t *bm = &cache_param->vsl_mask[0];
        uint8_t b;

        assert(tag > SLT__Bogus);
        assert(tag < SLT__Reserved);
        bm += ((unsigned)tag >> 3);
        b = (0x80 >> ((unsigned)tag & 7));
        return (*bm & b);
}

int
VSL_tag_is_masked(enum VSL_tag_e tag)
{
        return (vsl_tag_is_masked(tag));
}

/*--------------------------------------------------------------------
 * Lay down a header fields, and return pointer to the next record
 */

static inline uint32_t *
vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, vxid_t vxid)
{

        AZ((uintptr_t)p & 0x3);
        assert(tag > SLT__Bogus);
        assert(tag < SLT__Reserved);
        AZ(len & ~VSL_LENMASK);

        p[2] = vxid.vxid >> 32;
        p[1] = vxid.vxid;
        p[0] = (((unsigned)tag & VSL_IDMASK) << VSL_IDSHIFT) |
             (VSL_VERSION_3 << VSL_VERSHIFT) |
             len;
        return (VSL_END(p, len));
}

/*--------------------------------------------------------------------
 * Space available in a VSL buffer when accounting for overhead
 */

static unsigned
vsl_space(const struct vsl_log *vsl)
{
        ptrdiff_t mlen;

        mlen = vsl->wle - vsl->wlp;
        assert(mlen >= 0);
        if (mlen < VSL_OVERHEAD + 1)
                return (0);
        mlen -= VSL_OVERHEAD;
        mlen *= sizeof *vsl->wlp;
        if (mlen > cache_param->vsl_reclen)
                mlen = cache_param->vsl_reclen;
        return(mlen);
}

/*--------------------------------------------------------------------
 * Wrap the VSL buffer
 */

static void
vsl_wrap(void)
{

        assert(vsl_ptr >= vsl_head->log);
        assert(vsl_ptr < vsl_end);
        vsl_segment_n += VSL_SEGMENTS - (vsl_segment_n % VSL_SEGMENTS);
        assert(vsl_segment_n % VSL_SEGMENTS == 0);
        vsl_head->offset[0] = 0;
        vsl_head->log[0] = VSL_ENDMARKER;
        VWMB();
        if (vsl_ptr != vsl_head->log) {
                *vsl_ptr = VSL_WRAPMARKER;
                vsl_ptr = vsl_head->log;
        }
        vsl_head->segment_n = vsl_segment_n;
        VSC_C_main->shm_cycles++;
}

/*--------------------------------------------------------------------
 * Reserve bytes for a record, wrap if necessary
 */

static uint32_t *
vsl_get(unsigned len, unsigned records, unsigned flushes)
{
        uint32_t *p;
        int err;

        err = pthread_mutex_trylock(&vsl_mtx);
        if (err == EBUSY) {
                PTOK(pthread_mutex_lock(&vsl_mtx));
                VSC_C_main->shm_cont++;
        } else {
                AZ(err);
        }
        assert(vsl_ptr < vsl_end);
        AZ((uintptr_t)vsl_ptr & 0x3);

        VSC_C_main->shm_writes++;
        VSC_C_main->shm_flushes += flushes;
        VSC_C_main->shm_records += records;
        VSC_C_main->shm_bytes +=
            VSL_BYTES(VSL_OVERHEAD + VSL_WORDS((uint64_t)len));

        /* Wrap if necessary */
        if (VSL_END(vsl_ptr, len) >= vsl_end)
                vsl_wrap();

        p = vsl_ptr;
        vsl_ptr = VSL_END(vsl_ptr, len);
        assert(vsl_ptr < vsl_end);
        AZ((uintptr_t)vsl_ptr & 0x3);

        *vsl_ptr = VSL_ENDMARKER;

        while ((vsl_ptr - vsl_head->log) / vsl_segsize >
            vsl_segment_n % VSL_SEGMENTS) {
                vsl_segment_n++;
                vsl_head->offset[vsl_segment_n % VSL_SEGMENTS] =
                    vsl_ptr - vsl_head->log;
        }

        PTOK(pthread_mutex_unlock(&vsl_mtx));
        /* Implicit VWMB() in mutex op ensures ENDMARKER and new table
           values are seen before new segment number */
        vsl_head->segment_n = vsl_segment_n;

        return (p);
}

/*--------------------------------------------------------------------
 * Stick a finished record into VSL.
 */

static void
vslr(enum VSL_tag_e tag, vxid_t vxid, const char *b, unsigned len)
{
        uint32_t *p;
        unsigned mlen;

        mlen = cache_param->vsl_reclen;

        /* Truncate */
        if (len > mlen)
                len = mlen;

        p = vsl_get(len, 1, 0);

        vmemcpy(p + VSL_OVERHEAD, b, len);

        /*
         * the vxid needs to be written before the barrier to
         * ensure it is valid when vsl_hdr() marks the record
         * ready by writing p[0]
         */
        p[2] = vxid.vxid >> 32;
        p[1] = vxid.vxid;
        VWMB();
        (void)vsl_hdr(tag, p, len, vxid);
}

/*--------------------------------------------------------------------
 * Add a unbuffered record to VSL
 *
 * NB: This variant should be used sparingly and only for low volume
 * NB: since it significantly adds to the mutex load on the VSL.
 */

void
VSLv(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, va_list ap)
{
        unsigned n, mlen = cache_param->vsl_reclen;
        char buf[mlen];

        AN(fmt);
        if (vsl_tag_is_masked(tag))
                return;

        if (strchr(fmt, '%') == NULL) {
                vslr(tag, vxid, fmt, vstrlen(fmt) + 1);
        } else {
                n = vsnprintf(buf, mlen, fmt, ap);
                n = vmin(n, mlen - 1);
                buf[n++] = '\0'; /* NUL-terminated */
                vslr(tag, vxid, buf, n);
        }

}

void
VSLs(enum VSL_tag_e tag, vxid_t vxid, const struct strands *s)
{
        unsigned n, mlen = cache_param->vsl_reclen;
        char buf[mlen];

        if (vsl_tag_is_masked(tag))
                return;

        n = strands_cat(buf, mlen, s);

        vslr(tag, vxid, buf, n);
}

void
VSL(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, ...)
{
        va_list ap;

        va_start(ap, fmt);
        VSLv(tag, vxid, fmt, ap);
        va_end(ap);
}

/*--------------------------------------------------------------------*/

void
VSL_Flush(struct vsl_log *vsl, int overflow)
{
        uint32_t *p;
        unsigned l;

        vsl_sanity(vsl);
        l = pdiff(vsl->wlb, vsl->wlp);
        if (l == 0)
                return;

        assert(l >= 8);

        p = vsl_get(l, vsl->wlr, overflow);

        vmemcpy(p + VSL_OVERHEAD, vsl->wlb, l);
        p[1] = l;
        VWMB();
        p[0] = ((((unsigned)SLT__Batch & 0xff) << VSL_IDSHIFT));
        vsl->wlp = vsl->wlb;
        vsl->wlr = 0;
}

/*--------------------------------------------------------------------
 * Buffered VSLs
 */

static char *
vslb_get(struct vsl_log *vsl, enum VSL_tag_e tag, unsigned *length)
{
        unsigned mlen = cache_param->vsl_reclen;
        char *retval;

        vsl_sanity(vsl);
        if (*length < mlen)
                mlen = *length;

        if (VSL_END(vsl->wlp, mlen) > vsl->wle)
                VSL_Flush(vsl, 1);

        retval = VSL_DATA(vsl->wlp);

        /* If it still doesn't fit, truncate */
        if (VSL_END(vsl->wlp, mlen) > vsl->wle)
                mlen = vsl_space(vsl);

        vsl->wlp = vsl_hdr(tag, vsl->wlp, mlen, vsl->wid);
        vsl->wlr++;
        *length = mlen;
        return (retval);
}

static void
vslb_simple(struct vsl_log *vsl, enum VSL_tag_e tag,
    unsigned length, const char *str)
{
        char *p;

        if (length == 0)
                length = vstrlen(str);
        length += 1; // NUL
        p = vslb_get(vsl, tag, &length);
        vmemcpy(p, str, length - 1);
        p[length - 1] = '\0';

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

/*--------------------------------------------------------------------
 * VSL-buffered-txt
 */

void
VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t)
{

        Tcheck(t);
        if (vsl_tag_is_masked(tag))
                return;

        vslb_simple(vsl, tag, Tlen(t), t.b);
}

/*--------------------------------------------------------------------
 * VSL-buffered-strands
 */
void
VSLbs(struct vsl_log *vsl, enum VSL_tag_e tag, const struct strands *s)
{
        unsigned l;
        char *p;

        if (vsl_tag_is_masked(tag))
                return;

        l = strands_len(s) + 1;
        p = vslb_get(vsl, tag, &l);

        (void)strands_cat(p, l, s);

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

/*--------------------------------------------------------------------
 * VSL-buffered
 */

void
VSLbv(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, va_list ap)
{
        char *p, *p1;
        unsigned n = 0, mlen;
        va_list ap2;

        AN(fmt);
        if (vsl_tag_is_masked(tag))
                return;

        /*
         * If there are no printf-expansions, don't waste time expanding them
         */
        if (strchr(fmt, '%') == NULL) {
                vslb_simple(vsl, tag, 0, fmt);
                return;
        }

        /*
         * If the format is trivial, deal with it directly
         */
        if (!vstrcmp(fmt, "%s")) {
                p1 = va_arg(ap, char *);
                vslb_simple(vsl, tag, 0, p1);
                return;
        }

        vsl_sanity(vsl);

        mlen = vsl_space(vsl);

        // First attempt, only if any space at all
        if (mlen > 0) {
                p = VSL_DATA(vsl->wlp);
                va_copy(ap2, ap);
                n = vsnprintf(p, mlen, fmt, ap2);
                va_end(ap2);
        }

        // Second attempt, if a flush might help
        if (mlen == 0 || (n + 1 > mlen && n + 1 <= cache_param->vsl_reclen)) {
                VSL_Flush(vsl, 1);
                mlen = vsl_space(vsl);
                p = VSL_DATA(vsl->wlp);
                n = vsnprintf(p, mlen, fmt, ap);
        }
        if (n + 1 < mlen)
                mlen = n + 1;
        (void)vslb_get(vsl, tag, &mlen);

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

void
VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...)
{
        va_list ap;

        vsl_sanity(vsl);
        va_start(ap, fmt);
        VSLbv(vsl, tag, fmt, ap);
        va_end(ap);
}

#define Tf6 "%ju.%06ju"
#define Ta6(t) (uintmax_t)floor((t)), (uintmax_t)floor((t) * 1e6) % 1000000U

void
VSLb_ts(struct vsl_log *vsl, const char *event, vtim_real first,
    vtim_real *pprev, vtim_real now)
{

        /*
         * XXX: Make an option to turn off some unnecessary timestamp
         * logging. This must be done carefully because some functions
         * (e.g. V1L_Open) takes the last timestamp as its initial
         * value for timeout calculation.
         */
        vsl_sanity(vsl);
        AN(event);
        AN(pprev);
        assert(!isnan(now) && now != 0.);
        VSLb(vsl, SLT_Timestamp, "%s: " Tf6 " " Tf6 " " Tf6,
            event, Ta6(now), Ta6(now - first), Ta6(now - *pprev));
        *pprev = now;
}

void
VSLb_ts_req(struct req *req, const char *event, vtim_real now)
{

        if (isnan(req->t_first) || req->t_first == 0.)
                req->t_first = req->t_prev = now;
        VSLb_ts(req->vsl, event, req->t_first, &req->t_prev, now);
}

void
VSLb_ts_busyobj(struct busyobj *bo, const char *event, vtim_real now)
{

        if (isnan(bo->t_first) || bo->t_first == 0.)
                bo->t_first = bo->t_prev = now;
        VSLb_ts(bo->vsl, event, bo->t_first, &bo->t_prev, now);
}

void
VSLb_bin(struct vsl_log *vsl, enum VSL_tag_e tag, ssize_t len, const void *ptr)
{
        unsigned mlen;
        char *p;

        vsl_sanity(vsl);
        AN(ptr);
        if (vsl_tag_is_masked(tag))
                return;
        mlen = cache_param->vsl_reclen;

        /* Truncate */
        len = vmin_t(ssize_t, len, mlen);

        assert(vsl->wlp <= vsl->wle);

        /* Flush if necessary */
        if (VSL_END(vsl->wlp, len) > vsl->wle)
                VSL_Flush(vsl, 1);
        assert(VSL_END(vsl->wlp, len) <= vsl->wle);
        p = VSL_DATA(vsl->wlp);
        vmemcpy(p, ptr, len);
        vsl->wlp = vsl_hdr(tag, vsl->wlp, len, vsl->wid);
        assert(vsl->wlp <= vsl->wle);
        vsl->wlr++;

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

/*--------------------------------------------------------------------
 * Setup a VSL buffer
 */

void
VSL_Init(struct vsl_log *vsl, void *ptr, size_t len)
{

        AN(vsl);
        AN(ptr);
        AN(len);

        INIT_OBJ(vsl, VSL_LOG_MAGIC);
        vsl->wlp = ptr;
        vsl->wlb = ptr;
        vsl->wle = ptr;
        vsl->wle += len / sizeof(*vsl->wle);
        vsl->wlr = 0;
        vsl->wid = NO_VXID;
        vsl_sanity(vsl);
}

void
VSL_Alloc(struct vsl_log *vsl)
{
        size_t len = cache_param->vsl_buffer;
        void *ptr = malloc(len);
        AN(ptr);
        VSL_Init(vsl, ptr, len);
        vsl->flags |= VSL_LOG_F_MALLOCED;
}

void
VSL_Free(struct vsl_log *vsl)
{

        CHECK_OBJ_NOTNULL(vsl, VSL_LOG_MAGIC);
        assert(vsl->flags & VSL_LOG_F_MALLOCED);
        free(vsl->wlb);
        memset(vsl, 0, sizeof *vsl);
}

/*--------------------------------------------------------------------*/

void
VSL_ChgId(struct vsl_log *vsl, const char *typ, const char *why, vxid_t vxid)
{
        vxid_t ovxid;

        vsl_sanity(vsl);
        ovxid = vsl->wid;
        VSLb(vsl, SLT_Link, "%s %ju %s", typ, VXID(vxid), why);
        VSL_End(vsl);
        vsl->wid = vxid;
        VSLb(vsl, SLT_Begin, "%s %ju %s", typ, VXID(ovxid), why);
}

/*--------------------------------------------------------------------*/

void
VSL_End(struct vsl_log *vsl)
{
        txt t;
        char p[] = "";

        vsl_sanity(vsl);
        assert(!IS_NO_VXID(vsl->wid));
        t.b = p;
        t.e = p;
        VSLbt(vsl, SLT_End, t);
        VSL_Flush(vsl, 0);
        vsl->wid = NO_VXID;
}

static void v_matchproto_(vsm_lock_f)
vsm_vsc_lock(void)
{
        PTOK(pthread_mutex_lock(&vsc_mtx));
}

static void v_matchproto_(vsm_lock_f)
vsm_vsc_unlock(void)
{
        PTOK(pthread_mutex_unlock(&vsc_mtx));
}

static void v_matchproto_(vsm_lock_f)
vsm_vsmw_lock(void)
{
        PTOK(pthread_mutex_lock(&vsm_mtx));
}

static void v_matchproto_(vsm_lock_f)
vsm_vsmw_unlock(void)
{
        PTOK(pthread_mutex_unlock(&vsm_mtx));
}

/*--------------------------------------------------------------------*/

void
VSM_Init(void)
{
        unsigned u;

        assert(UINT_MAX % VSL_SEGMENTS == VSL_SEGMENTS - 1);

        PTOK(pthread_mutex_init(&vsl_mtx, &mtxattr_errorcheck));
        PTOK(pthread_mutex_init(&vsc_mtx, &mtxattr_errorcheck));
        PTOK(pthread_mutex_init(&vsm_mtx, &mtxattr_errorcheck));

        vsc_lock = vsm_vsc_lock;
        vsc_unlock = vsm_vsc_unlock;
        vsmw_lock = vsm_vsmw_lock;
        vsmw_unlock = vsm_vsmw_unlock;

        heritage.proc_vsmw = VSMW_New(heritage.vsm_fd, 0640, "_.index");
        AN(heritage.proc_vsmw);

        VSC_C_main = VSC_main_New(NULL, NULL, "");
        AN(VSC_C_main);

        AN(heritage.proc_vsmw);
        vsl_head = VSMW_Allocf(heritage.proc_vsmw, NULL, VSL_CLASS,
            cache_param->vsl_space, VSL_CLASS);
        AN(vsl_head);
        vsl_segsize = ((cache_param->vsl_space - sizeof *vsl_head) /
            sizeof *vsl_end) / VSL_SEGMENTS;
        vsl_end = vsl_head->log + vsl_segsize * VSL_SEGMENTS;
        /* Make segment_n always overflow on first log wrap to make any
           problems with regard to readers on that event visible */
        vsl_segment_n = UINT_MAX - (VSL_SEGMENTS - 1);
        AZ(vsl_segment_n % VSL_SEGMENTS);
        vsl_ptr = vsl_head->log;
        *vsl_ptr = VSL_ENDMARKER;

        memset(vsl_head, 0, sizeof *vsl_head);
        vsl_head->segsize = vsl_segsize;
        vsl_head->offset[0] = 0;
        vsl_head->segment_n = vsl_segment_n;
        for (u = 1; u < VSL_SEGMENTS; u++)
                vsl_head->offset[u] = -1;
        VWMB();
        vmemcpy(vsl_head->marker, VSL_HEAD_MARKER, sizeof vsl_head->marker);
}

		vinyl-cache/bin/vinyld/cache/cache_shmlog.c
0		/*-
1		* Copyright (c) 2006 Verdens Gang AS
2		* Copyright (c) 2006-2015 Varnish Software AS
3		* All rights reserved.
4		*
5		* Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
6		*
7		* SPDX-License-Identifier: BSD-2-Clause
8		*
9		* Redistribution and use in source and binary forms, with or without
10		* modification, are permitted provided that the following conditions
11		* are met:
12		* 1. Redistributions of source code must retain the above copyright
13		* notice, this list of conditions and the following disclaimer.
14		* 2. Redistributions in binary form must reproduce the above copyright
15		* notice, this list of conditions and the following disclaimer in the
16		* documentation and/or other materials provided with the distribution.
17		*
18		* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19		* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20		* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21		* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
22		* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23		* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24		* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25		* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26		* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27		* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28		* SUCH DAMAGE.
29		*/
30
31		#include "config.h"
32
33		#include "cache_int.h"
34
35		#include <stdio.h>
36		#include <stdlib.h>
37
38		#include "vsl_priv.h"
39		#include "vmb.h"
40
41		#include "common/heritage.h"
42		#include "common/vsmw.h"
43
44		/* ------------------------------------------------------------
45		* strands helpers - move elsewhere?
46		*/
47
48		static unsigned
49	1122744	strands_len(const struct strands *s)
50		{
51	1122744	unsigned r = 0;
52		int i;
53
54	1122744	CHECK_OBJ_NOTNULL(s, STRANDS_MAGIC);
55	2246780	for (i = 0; i < s->n; i++) {
56	1124036	if (s->p[i] == NULL \|\| *s->p[i] == '\0')
57	107587	continue;
58	1016449	r += vstrlen(s->p[i]);
59	1016449	}
60
61	1122744	return (r);
62		}
63
64		/*
65		* like VRT_Strands(), but truncating instead of failing for end of buffer
66		*
67		* returns number of bytes including NUL
68		*/
69		static unsigned
70	1126829	strands_cat(char buf, unsigned bufl, const struct strands s)
71		{
72	1126829	unsigned l = 0, ll;
73		int i;
74
75		/* NUL-terminated */
76	1126829	assert(bufl > 0);
77	1126829	bufl--;
78	1126829	CHECK_OBJ_NOTNULL(s, STRANDS_MAGIC);
79
80	2149498	for (i = 0; i < s->n && bufl > 0; i++) {
81	1022671	if (s->p[i] == NULL \|\| *s->p[i] == '\0')
82	56	continue;
83	1022627	ll = vmin_t(unsigned, vstrlen(s->p[i]), bufl);
84	1022627	vmemcpy(buf, s->p[i], ll);
85	1022627	l += ll;
86	1022627	buf += ll;
87	1022627	bufl -= ll;
88	1022627	}
89	1126827	buf = '\0'; / NUL-terminated */
90	1126827	return (l + 1);
91		}
92
93		/* These cannot be struct lock, which depends on vsm/vsl working */
94		static pthread_mutex_t vsl_mtx;
95		static pthread_mutex_t vsc_mtx;
96		static pthread_mutex_t vsm_mtx;
97
98		static struct VSL_head *vsl_head;
99		static const uint32_t *vsl_end;
100		static uint32_t *vsl_ptr;
101		static unsigned vsl_segment_n;
102		static ssize_t vsl_segsize;
103
104		struct VSC_main *VSC_C_main;
105
106		static void
107	11114432	vsl_sanity(const struct vsl_log *vsl)
108		{
109
110	11114432	CHECK_OBJ_NOTNULL(vsl, VSL_LOG_MAGIC);
111	11114432	AN(vsl);
112	11114432	AN(vsl->wlp);
113	11114432	AN(vsl->wlb);
114	11114432	AN(vsl->wle);
115	11114432	assert(vsl->wlb <= vsl->wlp);
116	11114432	assert(vsl->wlp <= vsl->wle);
117	11114432	}
118
119		/*--------------------------------------------------------------------
120		* Check if the VSL_tag is masked by parameter bitmap
121		*/
122
123		static inline int
124	8785632	vsl_tag_is_masked(enum VSL_tag_e tag)
125		{
126	8785632	volatile uint8_t *bm = &cache_param->vsl_mask[0];
127		uint8_t b;
128
129	8785632	assert(tag > SLT__Bogus);
130	8785632	assert(tag < SLT__Reserved);
131	8785632	bm += ((unsigned)tag >> 3);
132	8785632	b = (0x80 >> ((unsigned)tag & 7));
133	8785632	return (*bm & b);
134		}
135
136		int
137	27095	VSL_tag_is_masked(enum VSL_tag_e tag)
138		{
139	27095	return (vsl_tag_is_masked(tag));
140		}
141
142		/*--------------------------------------------------------------------
143		* Lay down a header fields, and return pointer to the next record
144		*/
145
146		static inline uint32_t *
147	6876380	vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, vxid_t vxid)
148		{
149
150	6876380	AZ((uintptr_t)p & 0x3);
151	6876380	assert(tag > SLT__Bogus);
152	6876380	assert(tag < SLT__Reserved);
153	6876380	AZ(len & ~VSL_LENMASK);
154
155	6876380	p[2] = vxid.vxid >> 32;
156	6876380	p[1] = vxid.vxid;
157	13752760	p[0] = (((unsigned)tag & VSL_IDMASK) << VSL_IDSHIFT) \|
158	6876380	(VSL_VERSION_3 << VSL_VERSHIFT) \|
159	6876380	len;
160	6876380	return (VSL_END(p, len));
161		}
162
163		/*--------------------------------------------------------------------
164		* Space available in a VSL buffer when accounting for overhead
165		*/
166
167		static unsigned
168	1672706	vsl_space(const struct vsl_log *vsl)
169		{
170		ptrdiff_t mlen;
171
172	1672706	mlen = vsl->wle - vsl->wlp;
173	1672706	assert(mlen >= 0);
174	1672706	if (mlen < VSL_OVERHEAD + 1)
175	42	return (0);
176	1672664	mlen -= VSL_OVERHEAD;
177	1672664	mlen = sizeof vsl->wlp;
178	1672664	if (mlen > cache_param->vsl_reclen)
179	1672233	mlen = cache_param->vsl_reclen;
180	1672664	return(mlen);
181	1672706	}
182
183		/*--------------------------------------------------------------------
184		* Wrap the VSL buffer
185		*/
186
187		static void
188	0	vsl_wrap(void)
189		{
190
191	0	assert(vsl_ptr >= vsl_head->log);
192	0	assert(vsl_ptr < vsl_end);
193	0	vsl_segment_n += VSL_SEGMENTS - (vsl_segment_n % VSL_SEGMENTS);
194	0	assert(vsl_segment_n % VSL_SEGMENTS == 0);
195	0	vsl_head->offset[0] = 0;
196	0	vsl_head->log[0] = VSL_ENDMARKER;
197	0	VWMB();
198	0	if (vsl_ptr != vsl_head->log) {
199	0	*vsl_ptr = VSL_WRAPMARKER;
200	0	vsl_ptr = vsl_head->log;
201	0	}
202	0	vsl_head->segment_n = vsl_segment_n;
203	0	VSC_C_main->shm_cycles++;
204	0	}
205
206		/*--------------------------------------------------------------------
207		* Reserve bytes for a record, wrap if necessary
208		*/
209
210		static uint32_t *
211	2053599	vsl_get(unsigned len, unsigned records, unsigned flushes)
212		{
213		uint32_t *p;
214		int err;
215
216	2053599	err = pthread_mutex_trylock(&vsl_mtx);
217	2053599	if (err == EBUSY) {
218	11687	PTOK(pthread_mutex_lock(&vsl_mtx));
219	11687	VSC_C_main->shm_cont++;
220	11687	} else {
221	2041912	AZ(err);
222		}
223	2053599	assert(vsl_ptr < vsl_end);
224	2053599	AZ((uintptr_t)vsl_ptr & 0x3);
225
226	2053599	VSC_C_main->shm_writes++;
227	2053599	VSC_C_main->shm_flushes += flushes;
228	2053599	VSC_C_main->shm_records += records;
229	2053599	VSC_C_main->shm_bytes +=
230	2053599	VSL_BYTES(VSL_OVERHEAD + VSL_WORDS((uint64_t)len));
231
232		/* Wrap if necessary */
233	2053599	if (VSL_END(vsl_ptr, len) >= vsl_end)
234	0	vsl_wrap();
235
236	2053599	p = vsl_ptr;
237	2053599	vsl_ptr = VSL_END(vsl_ptr, len);
238	2053599	assert(vsl_ptr < vsl_end);
239	2053599	AZ((uintptr_t)vsl_ptr & 0x3);
240
241	2053599	*vsl_ptr = VSL_ENDMARKER;
242
243	2053662	while ((vsl_ptr - vsl_head->log) / vsl_segsize >
244	2053662	vsl_segment_n % VSL_SEGMENTS) {
245	63	vsl_segment_n++;
246	63	vsl_head->offset[vsl_segment_n % VSL_SEGMENTS] =
247	63	vsl_ptr - vsl_head->log;
248		}
249
250	2053599	PTOK(pthread_mutex_unlock(&vsl_mtx));
251		/* Implicit VWMB() in mutex op ensures ENDMARKER and new table
252		values are seen before new segment number */
253	2053599	vsl_head->segment_n = vsl_segment_n;
254
255	2053599	return (p);
256		}
257
258		/*--------------------------------------------------------------------
259		* Stick a finished record into VSL.
260		*/
261
262		static void
263	1371977	vslr(enum VSL_tag_e tag, vxid_t vxid, const char *b, unsigned len)
264		{
265		uint32_t *p;
266		unsigned mlen;
267
268	1371977	mlen = cache_param->vsl_reclen;
269
270		/* Truncate */
271	1371977	if (len > mlen)
272	0	len = mlen;
273
274	1371977	p = vsl_get(len, 1, 0);
275
276	1371977	vmemcpy(p + VSL_OVERHEAD, b, len);
277
278		/*
279		* the vxid needs to be written before the barrier to
280		* ensure it is valid when vsl_hdr() marks the record
281		* ready by writing p[0]
282		*/
283	1371977	p[2] = vxid.vxid >> 32;
284	1371977	p[1] = vxid.vxid;
285	1371977	VWMB();
286	1371977	(void)vsl_hdr(tag, p, len, vxid);
287	1371977	}
288
289		/*--------------------------------------------------------------------
290		* Add a unbuffered record to VSL
291		*
292		* NB: This variant should be used sparingly and only for low volume
293		* NB: since it significantly adds to the mutex load on the VSL.
294		*/
295
296		void
297	2220560	VSLv(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, va_list ap)
298		{
299	2220560	unsigned n, mlen = cache_param->vsl_reclen;
300	2220560	char buf[mlen];
301
302	2220560	AN(fmt);
303	2220560	if (vsl_tag_is_masked(tag))
304	852827	return;
305
306	1367733	if (strchr(fmt, '%') == NULL) {
307	25059	vslr(tag, vxid, fmt, vstrlen(fmt) + 1);
308	25059	} else {
309	1342674	n = vsnprintf(buf, mlen, fmt, ap);
310	1342674	n = vmin(n, mlen - 1);
311	1342674	buf[n++] = '\0'; /* NUL-terminated */
312	1342674	vslr(tag, vxid, buf, n);
313		}
314
315	2220560	}
316
317		void
318	4242	VSLs(enum VSL_tag_e tag, vxid_t vxid, const struct strands *s)
319		{
320	4242	unsigned n, mlen = cache_param->vsl_reclen;
321	4242	char buf[mlen];
322
323	4242	if (vsl_tag_is_masked(tag))
324	0	return;
325
326	4242	n = strands_cat(buf, mlen, s);
327
328	4242	vslr(tag, vxid, buf, n);
329	4242	}
330
331		void
332	2175591	VSL(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, ...)
333		{
334		va_list ap;
335
336	2175591	va_start(ap, fmt);
337	2175591	VSLv(tag, vxid, fmt, ap);
338	2175591	va_end(ap);
339	2175591	}
340
341		/--------------------------------------------------------------------/
342
343		void
344	858048	VSL_Flush(struct vsl_log *vsl, int overflow)
345		{
346		uint32_t *p;
347		unsigned l;
348
349	858048	vsl_sanity(vsl);
350	858048	l = pdiff(vsl->wlb, vsl->wlp);
351	858048	if (l == 0)
352	178633	return;
353
354	679415	assert(l >= 8);
355
356	679415	p = vsl_get(l, vsl->wlr, overflow);
357
358	679415	vmemcpy(p + VSL_OVERHEAD, vsl->wlb, l);
359	679415	p[1] = l;
360	679415	VWMB();
361	679415	p[0] = ((((unsigned)SLT__Batch & 0xff) << VSL_IDSHIFT));
362	679415	vsl->wlp = vsl->wlb;
363	679415	vsl->wlr = 0;
364	858048	}
365
366		/*--------------------------------------------------------------------
367		* Buffered VSLs
368		*/
369
370		static char *
371	5478128	vslb_get(struct vsl_log vsl, enum VSL_tag_e tag, unsigned length)
372		{
373	5478128	unsigned mlen = cache_param->vsl_reclen;
374		char *retval;
375
376	5478128	vsl_sanity(vsl);
377	5478128	if (*length < mlen)
378	5463520	mlen = *length;
379
380	5478128	if (VSL_END(vsl->wlp, mlen) > vsl->wle)
381	210	VSL_Flush(vsl, 1);
382
383	5478128	retval = VSL_DATA(vsl->wlp);
384
385		/* If it still doesn't fit, truncate */
386	5478128	if (VSL_END(vsl->wlp, mlen) > vsl->wle)
387	42	mlen = vsl_space(vsl);
388
389	5478128	vsl->wlp = vsl_hdr(tag, vsl->wlp, mlen, vsl->wid);
390	5478128	vsl->wlr++;
391	5478128	*length = mlen;
392	5478128	return (retval);
393		}
394
395		static void
396	2685040	vslb_simple(struct vsl_log *vsl, enum VSL_tag_e tag,
397		unsigned length, const char *str)
398		{
399		char *p;
400
401	2685040	if (length == 0)
402	211852	length = vstrlen(str);
403	2685040	length += 1; // NUL
404	2685040	p = vslb_get(vsl, tag, &length);
405	2685040	vmemcpy(p, str, length - 1);
406	2685040	p[length - 1] = '\0';
407
408	2685040	if (DO_DEBUG(DBG_SYNCVSL))
409	247749	VSL_Flush(vsl, 0);
410	2685040	}
411
412		/*--------------------------------------------------------------------
413		* VSL-buffered-txt
414		*/
415
416		void
417	3082278	VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t)
418		{
419
420	3082278	Tcheck(t);
421	3082278	if (vsl_tag_is_masked(tag))
422	466955	return;
423
424	2615323	vslb_simple(vsl, tag, Tlen(t), t.b);
425	3082278	}
426
427		/*--------------------------------------------------------------------
428		* VSL-buffered-strands
429		*/
430		void
431	1327538	VSLbs(struct vsl_log vsl, enum VSL_tag_e tag, const struct strands s)
432		{
433		unsigned l;
434		char *p;
435
436	1327538	if (vsl_tag_is_masked(tag))
437	204762	return;
438
439	1122776	l = strands_len(s) + 1;
440	1122776	p = vslb_get(vsl, tag, &l);
441
442	1122776	(void)strands_cat(p, l, s);
443
444	1122776	if (DO_DEBUG(DBG_SYNCVSL))
445	98767	VSL_Flush(vsl, 0);
446	1327538	}
447
448		/*--------------------------------------------------------------------
449		* VSL-buffered
450		*/
451
452		void
453	2065955	VSLbv(struct vsl_log vsl, enum VSL_tag_e tag, const char fmt, va_list ap)
454		{
455		char p, p1;
456	2065955	unsigned n = 0, mlen;
457		va_list ap2;
458
459	2065955	AN(fmt);
460	2065955	if (vsl_tag_is_masked(tag))
461	323341	return;
462
463		/*
464		* If there are no printf-expansions, don't waste time expanding them
465		*/
466	1742614	if (strchr(fmt, '%') == NULL) {
467	69656	vslb_simple(vsl, tag, 0, fmt);
468	69656	return;
469		}
470
471		/*
472		* If the format is trivial, deal with it directly
473		*/
474	1672958	if (!vstrcmp(fmt, "%s")) {
475	168	p1 = va_arg(ap, char *);
476	168	vslb_simple(vsl, tag, 0, p1);
477	168	return;
478		}
479
480	1672790	vsl_sanity(vsl);
481
482	1672790	mlen = vsl_space(vsl);
483
484		// First attempt, only if any space at all
485	1672790	if (mlen > 0) {
486	1671939	p = VSL_DATA(vsl->wlp);
487	1671939	va_copy(ap2, ap);
488	1671939	n = vsnprintf(p, mlen, fmt, ap2);
489	1671939	va_end(ap2);
490	1671939	}
491
492		// Second attempt, if a flush might help
493	1672790	if (mlen == 0 \|\| (n + 1 > mlen && n + 1 <= cache_param->vsl_reclen)) {
494	1359	VSL_Flush(vsl, 1);
495	1359	mlen = vsl_space(vsl);
496	1359	p = VSL_DATA(vsl->wlp);
497	1359	n = vsnprintf(p, mlen, fmt, ap);
498	1359	}
499	1673000	if (n + 1 < mlen)