GCOV - Varnish Cache


/*-
 * Copyright (c) 2008-2015 Varnish Software AS
 * All rights reserved.
 *
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#ifdef VTEST_WITH_VTC_VINYL

#include "config.h"

#include <sys/socket.h>

#include <errno.h>
#include <fcntl.h>
#include <fnmatch.h>
#include <inttypes.h>
#include <limits.h>
#include <poll.h>
#include <pthread.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include "vapi/vsc.h"
#include "vapi/vsl.h"
#include "vapi/vsm.h"

#include "vdef.h"
#include "vqueue.h"

#include "miniobj.h"
#include "vas.h"
#include "vcli.h"
#include "vjsn.h"
#include "vre.h"
#include "vsb.h"
#include "vsub.h"
#include "vtcp.h"
#include "vtim.h"

#include <vtest_api.h>
#include "vtest_ext_vinyl.h"

struct vinyl {
        unsigned                magic;
#define VINYL_MAGIC             0x208cd8e4
        char                    *me;
        char                    *name;
        struct vtclog           *vl;
        VTAILQ_ENTRY(vinyl)     list;

        struct vsb              *args;
        int                     fds[4];
        pid_t                   pid;

        double                  syntax;

        pthread_t               tp;
        pthread_t               tp_vsl;
        int                     tp_started;

        int                     expect_exit;

        int                     cli_fd;
        int                     vcl_nbr;
        char                    *workdir;
        char                    *jail;
        char                    *proto;

        struct vsm              *vsm_vsl;
        struct vsm              *vsm_vsc;
        struct vsc              *vsc;
        int                     has_a_arg;

        unsigned                vsl_tag_count[256];

        volatile int            vsl_rec;
        volatile int            vsl_idle;
};

#define NONSENSE        "%XJEIFLH|)Xspa8P"

static VTAILQ_HEAD(, vinyl)     vinyles =
    VTAILQ_HEAD_INITIALIZER(vinyles);

/**********************************************************************
 * Fatal condition cleanup
 * Invalid to call in any code path not followed by vtc_fatal().
 */

static void
vinyl_fatal_cleanup(const struct vinyl *v)
{
        struct pollfd fd[1];
        int n;

        if (!pthread_equal(pthread_self(), vtc_thread))
                return;

        if (!v->tp_started)
                return;

        memset(fd, 0, sizeof(fd));
        fd[0].fd = v->fds[0];
        fd[0].events = POLLIN;

        do {
                n = poll(fd, sizeof(fd)/sizeof(fd[0]), 10);
                if (n == 1 && (fd[0].revents & (POLLHUP|POLLERR)) != 0) {
                        PTOK(pthread_join(v->tp, NULL));
                        break;
                }

                if (n == 1)
                        VTIM_sleep(0.01);
        } while (n > 0);
}

#define vinyl_fatal(v, ...)                                             \
    do {                                                                \
            vinyl_fatal_cleanup((v));                                   \
            vtc_fatal((v)->vl, __VA_ARGS__);                            \
    } while (0)

/**********************************************************************
 * Ask a question over CLI
 */

static enum VCLI_status_e
vinyl_ask_cli(const struct vinyl *v, const char *cmd, char **repl)
{
        int i;
        unsigned retval;
        char *r;

        if (cmd != NULL) {
                vtc_dump(v->vl, 4, "CLI TX", cmd, -1);
                i = write(v->cli_fd, cmd, vstrlen(cmd));
                if (i != vstrlen(cmd) && !vtc_stop)
                        vinyl_fatal(v, "CLI write failed (%s) = %u %s",
                            cmd, errno, strerror(errno));
                i = write(v->cli_fd, "\n", 1);
                if (i != 1 && !vtc_stop)
                        vinyl_fatal(v, "CLI write failed (%s) = %u %s",
                            cmd, errno, strerror(errno));
        }
        i = VCLI_ReadResult(v->cli_fd, &retval, &r, vtc_maxdur);
        if (i != 0 && !vtc_stop)
                vinyl_fatal(v, "CLI failed (%s) = %d %u %s",
                    cmd != NULL ? cmd : "NULL", i, retval, r);
        vtc_log(v->vl, 3, "CLI RX  %u", retval);
        vtc_dump(v->vl, 4, "CLI RX", r, -1);
        if (repl != NULL)
                *repl = r;
        else
                free(r);
        return ((enum VCLI_status_e)retval);
}

/**********************************************************************
 *
 */

static void
wait_stopped(const struct vinyl *v)
{
        char *r = NULL;
        enum VCLI_status_e st;

        vtc_log(v->vl, 3, "wait-stopped");
        while (1) {
                st = vinyl_ask_cli(v, "status", &r);
                if (st != CLIS_OK)
                        vinyl_fatal(v,
                            "CLI status command failed: %u %s", st, r);
                if (!vstrcmp(r, "Child in state stopped")) {
                        free(r);
                        break;
                }
                free(r);
                r = NULL;
                VTIM_sleep(0.2);
        }
}
/**********************************************************************
 *
 */

static void
wait_running(const struct vinyl *v)
{
        char *r = NULL;
        enum VCLI_status_e st;

        while (1) {
                vtc_log(v->vl, 3, "wait-running");
                st = vinyl_ask_cli(v, "status", &r);
                if (st != CLIS_OK)
                        vinyl_fatal(v,
                            "CLI status command failed: %u %s", st, r);
                if (!vstrcmp(r, "Child in state stopped"))
                        vinyl_fatal(v,
                            "Child stopped before running: %u %s", st, r);
                if (!vstrcmp(r, "Child in state running")) {
                        free(r);
                        r = NULL;
                        st = vinyl_ask_cli(v, "debug.listen_address", &r);
                        if (st != CLIS_OK)
                                vinyl_fatal(v,
                                    "CLI status command failed: %u %s", st, r);
                        free(r);
                        break;
                }
                free(r);
                r = NULL;
                VTIM_sleep(0.2);
        }
}

/**********************************************************************
 * Vinyllog gatherer thread
 */

static void *
vinyllog_thread(void *priv)
{
        struct vinyl *v;
        struct VSL_data *vsl;
        struct vsm *vsm;
        struct VSL_cursor *c;
        enum VSL_tag_e tag;
        uint64_t vxid;
        unsigned len, vs;
        const char *tagname, *data;
        int type, i, opt;
        struct vsb *vsb;

        CAST_OBJ_NOTNULL(v, priv, VINYL_MAGIC);

        vsl = VSL_New();
        AN(vsl);
        vsm = v->vsm_vsl;

        vsb = VSB_new_auto();
        AN(vsb);

        c = NULL;
        opt = 0;
        while (v->fds[1] > 0 || c != NULL) {    //lint !e845 bug in flint
                if (c == NULL) {
                        if (vtc_error)
                                break;
                        VTIM_sleep(0.1);
                        (void)VSM_Status(vsm);
                        c = VSL_CursorVSM(vsl, vsm, opt);
                        if (c == NULL) {
                                vtc_log(v->vl, 3, "vsl|%s", VSL_Error(vsl));
                                VSL_ResetError(vsl);
                                continue;
                        }
                }
                AN(c);

                opt = VSL_COPT_TAIL;

                while (1) {
                        i = VSL_Next(c);
                        if (i != 1)
                                break;

                        v->vsl_rec = 1;

                        tag = VSL_TAG(c->rec.ptr);
                        vxid = VSL_ID(c->rec.ptr);
                        if (tag == SLT__Batch)
                                continue;
                        tagname = VSL_tags[tag];
                        len = VSL_LEN(c->rec.ptr);
                        type = VSL_CLIENT(c->rec.ptr) ?
                            'c' : VSL_BACKEND(c->rec.ptr) ?
                            'b' : '-';
                        data = VSL_CDATA(c->rec.ptr);
                        v->vsl_tag_count[tag]++;
                        if (VSL_tagflags[tag] & SLT_F_BINARY) {
                                VSB_clear(vsb);
                                VSB_quote(vsb, data, len, VSB_QUOTE_HEX);
                                AZ(VSB_finish(vsb));
                                /* +2 to skip "0x" */
                                vtc_log(v->vl, 4, "vsl| %10ju %-15s %c [%s]",
                                    (uintmax_t)vxid, tagname, type,
                                    VSB_data(vsb) + 2);
                                continue;
                        }
                        if (VSL_tagflags[tag] & SLT_F_UNSAFE) {
                                // remove trailing NUL
                                if (len > 1 && data[len - 1] == '\0')
                                        len--;
                                VSB_clear(vsb);
                                VSB_quote(vsb, data, len, VSB_QUOTE_ESCHEX);
                                AZ(VSB_finish(vsb));
                                data = VSB_data(vsb);
                                len = VSB_len(vsb);
                        }
                        vtc_log(v->vl, 4, "vsl| %10ju %-15s %c %.*s",
                            (uintmax_t)vxid, tagname, type, (int)len,
                            data);
                }
                if (i == 0) {
                        /* Nothing to do but wait */
                        v->vsl_idle++;
                        vs = VSM_Status(vsm) & VSM_WRK_MASK;
                        if ((vs & ~VSM_WRK_CHANGED) != VSM_WRK_RUNNING) {
                                /* Abandoned - try reconnect */
                                VSL_DeleteCursor(c);
                                c = NULL;
                        } else {
                                VTIM_sleep(0.1);
                        }
                } else if (i == -2) {
                        /* Abandoned - try reconnect */
                        VSL_DeleteCursor(c);
                        c = NULL;
                } else
                        break;
        }

        if (c)
                VSL_DeleteCursor(c);
        VSL_Delete(vsl);
        VSB_destroy(&vsb);

        return (NULL);
}

/**********************************************************************
 * Allocate and initialize a vinyl
 */

static struct vinyl *
vinyl_new(const char *name)
{
        struct vinyl *v;
        struct vsb *vsb;
        char buf[1024];

        ALLOC_OBJ(v, VINYL_MAGIC);
        AN(v);
        REPLACE(v->name, name);

        REPLACE(v->jail, "");

        v->vl = vtc_logopen("%s", name);
        AN(v->vl);

        vsb = macro_expandf(v->vl, "${tmpdir}/%s", name);
        AN(vsb);
        v->workdir = strdup(VSB_data(vsb));
        AN(v->workdir);
        VSB_destroy(&vsb);

        bprintf(buf, "rm -rf %s ; mkdir -p %s", v->workdir, v->workdir);
        AZ(system(buf));

        v->args = VSB_new_auto();

        v->cli_fd = -1;
        VTAILQ_INSERT_TAIL(&vinyles, v, list);

        return (v);
}

/**********************************************************************
 * Delete a vinyl instance
 */

static void
vinyl_delete(struct vinyl *v)
{

        CHECK_OBJ_NOTNULL(v, VINYL_MAGIC);
        vtc_logclose(v->vl);
        free(v->name);
        free(v->jail);
        free(v->workdir);
        VSB_destroy(&v->args);
        if (v->vsc != NULL)
                VSC_Destroy(&v->vsc, v->vsm_vsc);
        if (v->vsm_vsc != NULL)
                VSM_Destroy(&v->vsm_vsc);
        if (v->vsm_vsl != NULL)
                VSM_Destroy(&v->vsm_vsl);

        /*
         * We do not delete the workdir, it may contain stuff people
         * want (coredumps, shmlog/stats etc), and trying to divine
         * "may want" is just too much trouble.  Leave it around and
         * nuke it at the start of the next test-run.
         */

        /* XXX: MEMLEAK (?) */
        FREE_OBJ(v);
}

/**********************************************************************
 * Vinyl listener
 */

static void *
vinyl_thread(void *priv)
{
        struct vinyl *v;

        CAST_OBJ_NOTNULL(v, priv, VINYL_MAGIC);
        return (vtc_record(v->vl, v->fds[0], NULL));
}

/**********************************************************************
 * Launch a Vinyl
 */

static void
vinyl_launch(struct vinyl *v)
{
        struct vsb *vsb, *vsb1;
        int i, nfd, asock;
        char abuf[VTCP_ADDRBUFSIZE];
        char pbuf[VTCP_PORTBUFSIZE];
        char lbuf[PATH_MAX];
        struct pollfd fd[3];
        enum VCLI_status_e u;
        const char *err;
        char *r = NULL;

        /* Create listener socket */
        asock = VTCP_listen_on(default_listen_addr, NULL, 1, &err);
        if (err != NULL)
                vinyl_fatal(v, "Create CLI listen socket failed: %s", err);
        assert(asock > 0);
        VTCP_myname(asock, abuf, sizeof abuf, pbuf, sizeof pbuf);

        AZ(VSB_finish(v->args));
        vtc_log(v->vl, 2, "Launch");
        vsb = VSB_new_auto();
        AN(vsb);
        VSB_cat(vsb, "cd ${pwd} &&");
        VSB_printf(vsb, " exec vinyld %s -d -n %s -i %s",
            v->jail, v->workdir, v->name);
        if (macro_isdef(NULL, "vinyld_args_prepend")) {
                VSB_putc(vsb, ' ');
                macro_cat(v->vl, vsb, "vinyld_args_prepend", NULL);
        }
        VSB_cat(vsb, VSB_data(params_vsb));
        if (leave_temp) {
                VSB_cat(vsb, " -p debug=+vcl_keep");
                VSB_cat(vsb, " -p debug=+vmod_so_keep");
                VSB_cat(vsb, " -p debug=+vsm_keep");
        }
        VSB_cat(vsb, " -l 2m");
        VSB_cat(vsb, " -p auto_restart=off");
        VSB_cat(vsb, " -p syslog_cli_traffic=off");
        VSB_cat(vsb, " -p thread_pool_min=10");
        VSB_cat(vsb, " -p debug=+vtc_mode");
        VSB_cat(vsb, " -p vsl_mask=+Debug,+H2RxHdr,+H2RxBody");
        VSB_cat(vsb, " -p h2_initial_window_size=1m");
        VSB_cat(vsb, " -p h2_rx_window_low_water=64k");
        if (!v->has_a_arg) {
                VSB_printf(vsb, " -a '%s'", default_listen_addr);
                if (v->proto != NULL)
                        VSB_printf(vsb, ",%s", v->proto);
        }
        VSB_printf(vsb, " -M '%s %s'", abuf, pbuf);
        VSB_printf(vsb, " -P %s/%sd.pid", v->workdir, v->me);
        if (vmod_path != NULL)
                VSB_printf(vsb, " -p vmod_path=%s", vmod_path);
        VSB_printf(vsb, " %s", VSB_data(v->args));
        if (macro_isdef(NULL, "vinyld_args_append")) {
                VSB_putc(vsb, ' ');
                macro_cat(v->vl, vsb, "vinyld_args_append", NULL);
        }
        AZ(VSB_finish(vsb));
        vtc_log(v->vl, 3, "CMD: %s", VSB_data(vsb));
        vsb1 = macro_expand(v->vl, VSB_data(vsb));
        AN(vsb1);
        VSB_destroy(&vsb);
        vsb = vsb1;
        vtc_log(v->vl, 3, "CMD: %s", VSB_data(vsb));
        AZ(pipe(&v->fds[0]));
        AZ(pipe(&v->fds[2]));
        v->pid = fork();
        assert(v->pid >= 0);
        if (v->pid == 0) {
                AZ(dup2(v->fds[0], 0));
                assert(dup2(v->fds[3], 1) == 1);
                assert(dup2(1, 2) == 2);
                closefd(&v->fds[0]);
                closefd(&v->fds[1]);
                closefd(&v->fds[2]);
                closefd(&v->fds[3]);
                VSUB_closefrom(STDERR_FILENO + 1);
                AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(vsb), (char*)0));
                exit(1);
        } else {
                vtc_log(v->vl, 3, "PID: %ld", (long)v->pid);
                macro_def(v->vl, v->name, "pid", "%ld", (long)v->pid);
                macro_def(v->vl, v->name, "name", "%s", v->workdir);
        }
        closefd(&v->fds[0]);
        closefd(&v->fds[3]);
        v->fds[0] = v->fds[2];
        v->fds[2] = v->fds[3] = -1;
        VSB_destroy(&vsb);
        AZ(v->tp_started);
        v->tp_started = 1;
        PTOK(pthread_create(&v->tp, NULL, vinyl_thread, v));

        /* Wait for the vinyl to call home */
        memset(fd, 0, sizeof fd);
        fd[0].fd = asock;
        fd[0].events = POLLIN;
        fd[1].fd = v->fds[1];
        fd[1].events = POLLIN;
        fd[2].fd = v->fds[2];
        fd[2].events = POLLIN;
        i = poll(fd, 2, (int)(vtc_maxdur * 1000 / 3));
        vtc_log(v->vl, 4, "CLIPOLL %d 0x%x 0x%x 0x%x",
            i, fd[0].revents, fd[1].revents, fd[2].revents);
        if (i == 0)
                vinyl_fatal(v, "FAIL timeout waiting for CLI connection");
        if (fd[1].revents & POLLHUP)
                vinyl_fatal(v, "FAIL debug pipe closed");
        if (!(fd[0].revents & POLLIN))
                vinyl_fatal(v, "FAIL CLI connection wait failure");
        nfd = accept(asock, NULL, NULL);
        closefd(&asock);
        if (nfd < 0)
                vinyl_fatal(v, "FAIL no CLI connection accepted");

        v->cli_fd = nfd;

        vtc_log(v->vl, 3, "CLI connection fd = %d", v->cli_fd);
        assert(v->cli_fd >= 0);

        /* Receive the banner or auth response */
        u = vinyl_ask_cli(v, NULL, &r);
        if (vtc_error)
                return;
        if (u != CLIS_AUTH)
                vinyl_fatal(v, "CLI auth demand expected: %u %s", u, r);

        bprintf(lbuf, "%s/_.secret", v->workdir);
        nfd = open(lbuf, O_RDONLY);
        assert(nfd >= 0);

        assert(sizeof lbuf >= CLI_AUTH_RESPONSE_LEN + 7);
        bstrcpy(lbuf, "auth ");
        VCLI_AuthResponse(nfd, r, lbuf + 5);
        closefd(&nfd);
        free(r);
        r = NULL;
        strcat(lbuf, "\n");

        u = vinyl_ask_cli(v, lbuf, &r);
        if (vtc_error)
                return;
        if (u != CLIS_OK)
                vinyl_fatal(v, "CLI auth command failed: %u %s", u, r);
        free(r);

        v->vsm_vsc = VSM_New();
        AN(v->vsm_vsc);
        v->vsc = VSC_New();
        AN(v->vsc);
        assert(VSM_Arg(v->vsm_vsc, 'n', v->workdir) > 0);
        AZ(VSM_Attach(v->vsm_vsc, -1));

        v->vsm_vsl = VSM_New();
        assert(VSM_Arg(v->vsm_vsl, 'n', v->workdir) > 0);
        AZ(VSM_Attach(v->vsm_vsl, -1));

        PTOK(pthread_create(&v->tp_vsl, NULL, vinyllog_thread, v));
}

#define VINYL_LAUNCH(v)                         \
        do {                                            \
                CHECK_OBJ_NOTNULL(v, VINYL_MAGIC);      \
                if (v->cli_fd < 0)                      \
                        vinyl_launch(v);                \
                if (vtc_error)                          \
                        return;                         \
        } while (0)

/**********************************************************************
 * Start a Vinyl
 */

static void
vinyl_listen(const struct vinyl *v, char *la)
{
        const char *a, *p, *n, *n2;
        char m[64], s[256];
        unsigned first;

        n2 = "";
        first = 1;

        while (*la != '\0') {
                n = la;
                la = strchr(la, ' ');
                AN(la);
                *la = '\0';
                a = ++la;
                la = strchr(la, ' ');
                AN(la);
                *la = '\0';
                p = ++la;
                la = strchr(la, '\n');
                AN(la);
                *la = '\0';
                la++;

                AN(*n);
                AN(*a);
                AN(*p);

                if (*p == '-') {
                        bprintf(s, "%s", a);
                        a = "0.0.0.0";
                        p = "0";
                } else if (strchr(a, ':')) {
                        bprintf(s, "[%s]:%s", a, p);
                } else {
                        bprintf(s, "%s:%s", a, p);
                }

                if (first) {
                        vtc_log(v->vl, 2, "Listen on %s %s", a, p);
                        macro_def(v->vl, v->name, "addr", "%s", a);
                        macro_def(v->vl, v->name, "port", "%s", p);
                        macro_def(v->vl, v->name, "sock", "%s", s);
                        first = 0;
                }

                if (!vstrcmp(n, n2))
                        continue;

                bprintf(m, "%s_addr", n);
                macro_def(v->vl, v->name, m, "%s", a);
                bprintf(m, "%s_port", n);
                macro_def(v->vl, v->name, m, "%s", p);
                bprintf(m, "%s_sock", n);
                macro_def(v->vl, v->name, m, "%s", s);
                n2 = n;
        }
}

static void
vinyl_start(struct vinyl *v)
{
        enum VCLI_status_e u;
        char *resp = NULL;

        VINYL_LAUNCH(v);
        vtc_log(v->vl, 2, "Start");
        u = vinyl_ask_cli(v, "start", &resp);
        if (vtc_error)
                return;
        if (u != CLIS_OK)
                vinyl_fatal(v, "CLI start command failed: %u %s", u, resp);
        wait_running(v);
        free(resp);
        resp = NULL;
        u = vinyl_ask_cli(v, "debug.xid 1000", &resp);
        if (vtc_error)
                return;
        if (u != CLIS_OK)
                vinyl_fatal(v, "CLI debug.xid command failed: %u %s",
                    u, resp);
        free(resp);
        resp = NULL;
        u = vinyl_ask_cli(v, "debug.listen_address", &resp);
        if (vtc_error)
                return;
        if (u != CLIS_OK)
                vinyl_fatal(v,
                    "CLI debug.listen_address command failed: %u %s", u, resp);
        vinyl_listen(v, resp);
        free(resp);
        /* Wait for vsl logging to get underway */
        while (v->vsl_rec == 0)
                VTIM_sleep(.1);
}

/**********************************************************************
 * Stop a Vinyl
 */

static void
vinyl_stop(struct vinyl *v)
{

        VINYL_LAUNCH(v);
        vtc_log(v->vl, 2, "Stop");
        (void)vinyl_ask_cli(v, "stop", NULL);
        wait_stopped(v);
}

/**********************************************************************
 * Cleanup
 */

static void
vinyl_cleanup(struct vinyl *v)
{
        void *p;

        /* Close the CLI connection */
        closefd(&v->cli_fd);

        /* Close the STDIN connection. */
        closefd(&v->fds[1]);

        /* Wait until STDOUT+STDERR closes */
        AN(v->tp_started);
        PTOK(pthread_join(v->tp, &p));
        closefd(&v->fds[0]);
        v->tp_started = 0;

        /* Pick up the VSL thread */
        PTOK(pthread_join(v->tp_vsl, &p));

        vtc_wait4(v->vl, v->pid, v->expect_exit, 0, 0);
        v->pid = 0;
}

/**********************************************************************
 * Wait for a Vinyl
 */

static void
vinyl_wait(struct vinyl *v)
{
        if (v->cli_fd < 0)
                return;

        vtc_log(v->vl, 2, "Wait");

        if (!vtc_error) {
                /* Do a backend.list to log if child is still running */
                (void)vinyl_ask_cli(v, "backend.list", NULL);
        }

        /* Then stop it */
        vinyl_stop(v);

        if (vinyl_ask_cli(v, "panic.show", NULL) != CLIS_CANT)
                vinyl_fatal(v, "Unexpected panic");

        vinyl_cleanup(v);
}


/**********************************************************************
 * Ask a CLI JSON question
 */

static void
vinyl_cli_json(struct vinyl *v, const char *cli)
{
        enum VCLI_status_e u;
        char *resp = NULL;
        const char *errptr;
        struct vjsn *vj;

        VINYL_LAUNCH(v);
        u = vinyl_ask_cli(v, cli, &resp);
        vtc_log(v->vl, 2, "CLI %03u <%s>", u, cli);
        if (u != CLIS_OK)
                vinyl_fatal(v,
                    "FAIL CLI response %u expected %u", u, CLIS_OK);
        vj = vjsn_parse(resp, &errptr);
        if (vj == NULL)
                vinyl_fatal(v, "FAIL CLI, not good JSON: %s", errptr);
        vjsn_delete(&vj);
        free(resp);
}

/**********************************************************************
 * Ask a CLI question
 */

static void
vinyl_cli(struct vinyl *v, const char *cli, unsigned exp, const char *re,
    int neg)
{
        enum VCLI_status_e u;
        struct vsb vsb[1];
        vre_t *vre = NULL;
        char *resp = NULL, errbuf[VRE_ERROR_LEN];
        int err, erroff;

        VINYL_LAUNCH(v);
        if (re != NULL) {
                vre = VRE_compile(re, 0, &err, &erroff, 1);
                if (vre == NULL) {
                        AN(VSB_init(vsb, errbuf, sizeof errbuf));
                        AZ(VRE_error(vsb, err));
                        AZ(VSB_finish(vsb));
                        VSB_fini(vsb);
                        vinyl_fatal(v, "Illegal regexp: %s (@%d)",
                            errbuf, erroff);
                }
        }
        u = vinyl_ask_cli(v, cli, &resp);
        vtc_log(v->vl, 2, "CLI %03u <%s>", u, cli);
        if (exp != 0 && exp != (unsigned)u)
                vinyl_fatal(v, "FAIL CLI response %u expected %u", u, exp);
        if (vre != NULL) {
                err = VRE_match(vre, resp, 0, 0, NULL);
                if (err < VRE_ERROR_NOMATCH) {
                        AN(VSB_init(vsb, errbuf, sizeof errbuf));
                        AZ(VRE_error(vsb, err));
                        AZ(VSB_finish(vsb));
                        VSB_fini(vsb);
                        vinyl_fatal(v, "Regexp failed: %s with %s (%d)",
                            re, errbuf, err);
                } else if ((err == VRE_ERROR_NOMATCH) == (neg == 0)) { /*lint !e731 */
                        vinyl_fatal(v, "Expect failed: regexp \"%s\" did%s "
                            "match response \"%s\"", re, neg ? "" : " not",
                            resp);
                }
                VRE_free(&vre);
        }
        free(resp);
}

static const char *
vcl_prepend(void)
{
        const char *vcl_pfx;

        vcl_pfx = getenv("VTEST_VINYL_VCL_PREPEND");
        return vcl_pfx ? vcl_pfx : "";
}

/**********************************************************************
 * Load a VCL program
 */

static void
vinyl_vcl(struct vinyl *v, const char *vcl, int fail, char **resp)
{
        struct vsb *vsb;
        enum VCLI_status_e u;

        VINYL_LAUNCH(v);
        vsb = VSB_new_auto();
        AN(vsb);

        VSB_printf(vsb, "vcl.inline vcl%d << %s\nvcl %.1f;\n%s%s\n%s\n",
            ++v->vcl_nbr, NONSENSE, v->syntax,
            vcl_prepend(), vcl, NONSENSE);
        AZ(VSB_finish(vsb));

        u = vinyl_ask_cli(v, VSB_data(vsb), resp);
        if (u == CLIS_OK) {
                VSB_clear(vsb);
                VSB_printf(vsb, "vcl.use vcl%d", v->vcl_nbr);
                AZ(VSB_finish(vsb));
                u = vinyl_ask_cli(v, VSB_data(vsb), NULL);
        }
        if (u == CLIS_OK && fail) {
                VSB_destroy(&vsb);
                vinyl_fatal(v, "VCL compilation succeeded expected failure");
        } else if (u != CLIS_OK && !fail) {
                VSB_destroy(&vsb);
                vinyl_fatal(v, "VCL compilation failed expected success");
        } else if (fail)
                vtc_log(v->vl, 2, "VCL compilation failed (as expected)");
        VSB_destroy(&vsb);
}

/**********************************************************************
 * Load a VCL program prefixed by backend decls for our servers
 */

static void
vinyl_vclbackend(struct vinyl *v, const char *vcl)
{
        struct vsb *vsb, *vsb2;
        enum VCLI_status_e u;

        VINYL_LAUNCH(v);
        vsb = VSB_new_auto();
        AN(vsb);

        vsb2 = VSB_new_auto();
        AN(vsb2);

        VSB_printf(vsb2, "vcl %.1f;\n", v->syntax);

        cmd_server_gen_vcl(vsb2);

        AZ(VSB_finish(vsb2));

        VSB_printf(vsb, "vcl.inline vcl%d << %s\n%s\n%s%s\n%s\n",
            ++v->vcl_nbr, NONSENSE, VSB_data(vsb2), vcl_prepend(),
            vcl, NONSENSE);
        AZ(VSB_finish(vsb));

        u = vinyl_ask_cli(v, VSB_data(vsb), NULL);
        if (u != CLIS_OK) {
                VSB_destroy(&vsb);
                VSB_destroy(&vsb2);
                vinyl_fatal(v, "FAIL VCL does not compile");
        }
        VSB_clear(vsb);
        VSB_printf(vsb, "vcl.use vcl%d", v->vcl_nbr);
        AZ(VSB_finish(vsb));
        u = vinyl_ask_cli(v, VSB_data(vsb), NULL);
        assert(u == CLIS_OK);
        VSB_destroy(&vsb);
        VSB_destroy(&vsb2);
}

/**********************************************************************
 */

struct dump_priv {
        const char *arg;
        const struct vinyl *v;
};

static int
do_stat_dump_cb(void *priv, const struct VSC_point * const pt)
{
        const struct vinyl *v;
        struct dump_priv *dp;
        uint64_t u;

        if (pt == NULL)
                return (0);
        dp = priv;
        v = dp->v;

        if (vstrcmp(pt->ctype, "uint64_t"))
                return (0);
        u = VSC_Value(pt);

        if (vstrcmp(dp->arg, "*")) {
                if (fnmatch(dp->arg, pt->name, 0))
                        return (0);
        }

        vtc_log(v->vl, 4, "VSC %s %ju",  pt->name, (uintmax_t)u);
        return (0);
}

static void
vinyl_vsc(struct vinyl *v, const char *arg)
{
        struct dump_priv dp;

        VINYL_LAUNCH(v);
        memset(&dp, 0, sizeof dp);
        dp.v = v;
        dp.arg = arg;
        (void)VSM_Status(v->vsm_vsc);
        (void)VSC_Iter(v->vsc, v->vsm_vsc, do_stat_dump_cb, &dp);
}

/**********************************************************************
 * Check statistics
 */

struct stat_arg {
        const char      *pattern;
        uintmax_t       val;
        unsigned        good;
};

struct stat_priv {
        struct stat_arg lhs;
        struct stat_arg rhs;
};

static int
stat_match(const char *pattern, const char *name)
{

        if (strchr(pattern, '.') == NULL) {
                if (fnmatch("MAIN.*", name, 0))
                        return (FNM_NOMATCH);
                name += 5;
        }
        return (fnmatch(pattern, name, 0));
}

static int
do_expect_cb(void *priv, const struct VSC_point * const pt)
{
        struct stat_priv *sp = priv;

        if (pt == NULL)
                return (0);

        if (!sp->lhs.good && stat_match(sp->lhs.pattern, pt->name) == 0) {
                AZ(vstrcmp(pt->ctype, "uint64_t"));
                AN(pt->ptr);
                sp->lhs.val = VSC_Value(pt);
                sp->lhs.good = 1;
        }

        if (sp->rhs.pattern == NULL) {
                sp->rhs.good = 1;
        } else if (!sp->rhs.good &&
            stat_match(sp->rhs.pattern, pt->name) == 0) {
                AZ(vstrcmp(pt->ctype, "uint64_t"));
                AN(pt->ptr);
                sp->rhs.val = VSC_Value(pt);
                sp->rhs.good = 1;
        }

        return (sp->lhs.good && sp->rhs.good);
}

/**********************************************************************
 */

static void
vinyl_expect(struct vinyl *v, char * const *av)
{
        struct stat_priv sp;
        int good, i, not;
        uintmax_t u;
        char *l, *p;

        VINYL_LAUNCH(v);
        ZERO_OBJ(&sp, sizeof sp);
        l = av[0];
        not = (*l == '!');
        if (not) {
                l++;
                AZ(av[1]);
        } else {
                AN(av[1]);
                AN(av[2]);
                errno = 0;
                u = strtoumax(av[2], &p, 0);
                if (errno != ERANGE && *p == '\0')
                        sp.rhs.val = u;
                else
                        sp.rhs.pattern = av[2];
        }

        sp.lhs.pattern = l;

        for (i = 0; i < 50; i++, (void)usleep(100000)) {
                (void)VSM_Status(v->vsm_vsc);
                sp.lhs.good = sp.rhs.good = 0;
                good = VSC_Iter(v->vsc, v->vsm_vsc, do_expect_cb, &sp);
                if (!good)
                        good = -2;
                if (good < 0)
                        continue;

                if (not)
                        vinyl_fatal(v, "Found (not expected): %s", l);

                good = -1;
                if (!strcmp(av[1], "==")) good = (sp.lhs.val == sp.rhs.val);
                if (!strcmp(av[1], "!=")) good = (sp.lhs.val != sp.rhs.val);
                if (!strcmp(av[1], ">" )) good = (sp.lhs.val >  sp.rhs.val);
                if (!strcmp(av[1], "<" )) good = (sp.lhs.val <  sp.rhs.val);
                if (!strcmp(av[1], ">=")) good = (sp.lhs.val >= sp.rhs.val);
                if (!strcmp(av[1], "<=")) good = (sp.lhs.val <= sp.rhs.val);
                if (good == -1)
                        vinyl_fatal(v, "comparison %s unknown", av[1]);
                if (good)
                        break;
        }
        if (good == -1) {
                vinyl_fatal(v, "VSM error: %s", VSM_Error(v->vsm_vsc));
        }
        if (good == -2) {
                if (not) {
                        vtc_log(v->vl, 2, "not found (as expected): %s", l);
                        return;
                }
                vinyl_fatal(v, "stats field %s unknown",
                    sp.lhs.good ? sp.rhs.pattern : sp.lhs.pattern);
        }

        if (good == 1) {
                vtc_log(v->vl, 2, "as expected: %s (%ju) %s %s (%ju)",
                    av[0], sp.lhs.val, av[1], av[2], sp.rhs.val);
        } else {
                vinyl_fatal(v, "Not true: %s (%ju) %s %s (%ju)",
                    av[0], sp.lhs.val, av[1], av[2], sp.rhs.val);
        }
}

static void
vsl_catchup(struct vinyl *v)
{
        int vsl_idle;

        VINYL_LAUNCH(v);
        vsl_idle = v->vsl_idle;
        while (!vtc_error && vsl_idle == v->vsl_idle)
                VTIM_sleep(0.1);
}

/* SECTION: vinyl vinyl
 *
 * Define and interact with vinyl instances.
 *
 * To define a ``vinyld``, you'll use this syntax::
 *
 *      vinyl vNAME [-arg STRING] [-vcl STRING] [-vcl+backend STRING]
 *              [-errvcl STRING STRING] [-jail STRING] [-proto PROXY]
 *
 * The first ``vinyl vNAME`` invocation will start the vinyld master
 * process in the background, waiting for the ``-start`` switch to actually
 * start the child.
 *
 * Types used in the description below:
 *
 * PATTERN
 *         is a 'glob' style pattern (ie: fnmatch(3)) as used in shell filename
 *         expansion.
 *
 * Arguments:
 *
 * vNAME
 *         Identify the ``vinyld`` server with a string, it must starts with 'v'.
 *
 * \-arg STRING
 *         Pass an argument to vinyld, for example "-h simple_list".
 *
 *         If the ${vinyld_args_prepend} or ${vinyld_args_append} macros are
 *         defined, they are expanded and inserted before / appended to the
 *         vinyld command line as constructed by vinyltest, before the
 *         command line itself is expanded. This enables tweaks to the vinyld
 *         command line without editing test cases. This macros can be defined
 *         using the ``-D`` option for vinyltest.
 *
 * \-vcl STRING
 *         Specify the VCL to load on this ``vinyld`` instance. You'll probably
 *         want to use multi-lines strings for this ({...}).
 *
 * \-vcl+backend STRING
 *         Do the exact same thing as -vcl, but adds the definition block of
 *         known backends (ie. already defined).
 *
 * \-errvcl STRING1 STRING2
 *         Load STRING2 as VCL, expecting it to fail, and ``vinyld`` to send an
 *         error string matching STRING1
 *
 * \-jail STRING
 *         Look at ``man vinyld`` (-j) for more information.
 *
 * \-proto PROXY
 *         Have ``vinyld`` use the proxy protocol. Note that PROXY here is the
 *         actual string.
 *
 * You can decide to start the ``vinyld`` instance and/or wait for several events::
 *
 *         vinyl vNAME [-start] [-wait] [-wait-running] [-wait-stopped]
 *
 * \-start
 *         Start the child process.
 *
 *         Once successfully started, the following macros are available for
 *         the default listen address: ``${vNAME_addr}``, ``${vNAME_port}``
 *         and ``${vNAME_sock}``. Additional macros are available, including
 *         the listen address name for each address vNAME listens to, like for
 *         example: ``${vNAME_a0_addr}``.
 *
 * \-stop
 *         Stop the child process.
 *
 * \-syntax
 *         Set the VCL syntax level for this command (default: 4.1)
 *
 * \-wait
 *         Wait for that instance to terminate.
 *
 * \-wait-running
 *         Wait for the ``vinyld`` child process to be started.
 *
 * \-wait-stopped
 *         Wait for the ``vinyld`` child process to stop.
 *
 * \-cleanup
 *         Once ``vinyld`` is stopped, clean everything after it. This is only used
 *         in very few tests and you should never need it.
 *
 * \-expectexit NUMBER
 *         Expect vinyld to exit(3) with this value
 *
 * Once ``vinyld`` is started, you can talk to it (as you would through
 * ``vinyladm``) with these additional switches::
 *
 *         vinyl vNAME [-cli STRING] [-cliok STRING] [-clierr STRING]
 *                       [-clijson STRING]
 *
 * \-cli STRING|-cliok STRING|-clierr STATUS STRING|-cliexpect [!] REGEXP STRING
 *         All four of these will send STRING to the CLI, the only difference
 *         is what they expect the result to be. -cli doesn't expect
 *         anything, -cliok expects 200, -clierr expects STATUS, and
 *         -cliexpect expects the REGEXP to match the returned response.
 *         -cliexpect ! ... negates the match
 *
 * \-clijson STRING
 *         Send STRING to the CLI, expect success (CLIS_OK/200) and check
 *         that the response is parsable JSON.
 *
 * It is also possible to interact with its shared memory (as you would
 * through tools like ``vinylstat``) with additional switches:
 *
 * \-expect \!PATTERN|PATTERN OP NUMBER|PATTERN OP PATTERN
 *         Look into the VSM and make sure the first VSC counter identified by
 *         PATTERN has a correct value. OP can be ==, >, >=, <, <=. For
 *         example::
 *
 *                 vinyl v1 -expect SM?.s1.g_space > 1000000
 *                 vinyl v1 -expect cache_hit >= cache_hit_grace
 *
 *         In the \! form the test fails if a counter matches PATTERN.
 *
 *         The ``MAIN.`` namespace can be omitted from PATTERN.
 *
 *         The test takes up to 5 seconds before timing out.
 *
 * \-vsc PATTERN
 *         Dump VSC counters matching PATTERN.
 *
 * \-vsl_catchup
 *         Wait until the logging thread has idled to make sure that all
 *         the generated log is flushed
 *
 * Environment variables:
 *
 * VTEST_VINYL_VCL_PREPEND can be used to inject VCL code into all VCL created
 * by the vinyl command before VCL provided by the caller using the -vcl or
 * -vcl+backend argument. That is, VTEST_VINYL_VCL_PREPEND is injected after the
 * implicit "vcl x.y;" statement and backend definitions generated by the vinyl
 * command.
 */

void
cmd_vinyl(CMD_ARGS)
{
        struct vinyl *v, *v2;
        const char *me;

        (void)priv;

        if (av == NULL) {
                /* Reset and free */
                VTAILQ_FOREACH_SAFE(v, &vinyles, list, v2) {
                        if (v->cli_fd >= 0)
                                vinyl_wait(v);
                        VTAILQ_REMOVE(&vinyles, v, list);
                        vinyl_delete(v);
                }
                return;
        }

        me = av[0];
        // AZ(strcmp(av[0], "vinyl"));
        av++;

        VTC_CHECK_NAME(vl, av[0], "Vinyl", 'v');
        VTAILQ_FOREACH(v, &vinyles, list)
                if (!vstrcmp(v->name, av[0]))
                        break;
        if (v == NULL) {
                v = vinyl_new(av[0]);
                v->me =strdup(me);
                AN(v->me);
        }
        av++;
        v->syntax = 4.1;

        for (; *av != NULL; av++) {
                if (vtc_error)
                        break;
                if (!vstrcmp(*av, "-arg")) {
                        AN(av[1]);
                        AZ(v->pid);
                        VSB_cat(v->args, " ");
                        VSB_cat(v->args, av[1]);
                        if (av[1][0] == '-' && av[1][1] == 'a')
                                v->has_a_arg = 1;
                        av++;
                        continue;
                }
                if (!vstrcmp(*av, "-cleanup")) {
                        AZ(av[1]);
                        vinyl_cleanup(v);
                        continue;
                }
                if (!vstrcmp(*av, "-cli")) {
                        AN(av[1]);
                        vinyl_cli(v, av[1], 0, NULL, 0);
                        av++;
                        continue;
                }
                if (!vstrcmp(*av, "-clierr")) {
                        AN(av[1]);
                        AN(av[2]);
                        vinyl_cli(v, av[2], atoi(av[1]), NULL, 0);
                        av += 2;
                        continue;
                }
                if (!vstrcmp(*av, "-cliexpect")) {
                        int neg = 0;

                        AN(av[1]);
                        AN(av[2]);
                        if (*av[1] == '!') {
                                neg = 1;
                                av++;
                                AN(av[2]);
                        }
                        vinyl_cli(v, av[2], 0, av[1], neg);
                        av += 2;
                        continue;
                }
                if (!vstrcmp(*av, "-clijson")) {
                        AN(av[1]);
                        vinyl_cli_json(v, av[1]);
                        av++;
                        continue;
                }
                if (!vstrcmp(*av, "-cliok")) {
                        AN(av[1]);
                        vinyl_cli(v, av[1], (unsigned)CLIS_OK, NULL, 0);
                        av++;
                        continue;
                }
                if (!vstrcmp(*av, "-errvcl")) {
                        char *r = NULL;
                        AN(av[1]);
                        AN(av[2]);
                        vinyl_vcl(v, av[2], 1, &r);
                        if (strstr(r, av[1]) == NULL)
                                vinyl_fatal(v,
                                    "Did not find expected string: (\"%s\")",
                                    av[1]);
                        else
                                vtc_log(v->vl, 3,
                                    "Found expected string: (\"%s\")",
                                    av[1]);
                        free(r);
                        av += 2;
                        continue;
                }
                if (!vstrcmp(*av, "-expect")) {
                        av++;
                        vinyl_expect(v, av);
                        av += 2;
                        continue;
                }
                if (!vstrcmp(*av, "-expectexit")) {
                        v->expect_exit = strtoul(av[1], NULL, 0);
                        av++;
                        continue;
                }
                if (!vstrcmp(*av, "-jail")) {
                        AN(av[1]);
                        AZ(v->pid);
                        REPLACE(v->jail, av[1]);
                        av++;
                        continue;
                }
                if (!vstrcmp(*av, "-proto")) {
                        AN(av[1]);
                        AZ(v->pid);
                        REPLACE(v->proto, av[1]);
                        av++;
                        continue;
                }
                if (!vstrcmp(*av, "-start")) {
                        vinyl_start(v);
                        continue;
                }
                if (!vstrcmp(*av, "-stop")) {
                        vinyl_stop(v);
                        continue;
                }
                if (!vstrcmp(*av, "-syntax")) {
                        AN(av[1]);
                        v->syntax = strtod(av[1], NULL);
                        av++;
                        continue;
                }
                if (!vstrcmp(*av, "-vcl")) {
                        AN(av[1]);
                        vinyl_vcl(v, av[1], 0, NULL);
                        av++;
                        continue;
                }
                if (!vstrcmp(*av, "-vcl+backend")) {
                        AN(av[1]);
                        vinyl_vclbackend(v, av[1]);
                        av++;
                        continue;
                }
                if (!vstrcmp(*av, "-vsc")) {
                        AN(av[1]);
                        vinyl_vsc(v, av[1]);
                        av++;
                        continue;
                }
                if (!vstrcmp(*av, "-wait-stopped")) {
                        wait_stopped(v);
                        continue;
                }
                if (!vstrcmp(*av, "-wait-running")) {
                        wait_running(v);
                        continue;
                }
                if (!vstrcmp(*av, "-wait")) {
                        vinyl_wait(v);
                        continue;
                }
                if (!vstrcmp(*av, "-vsl_catchup")) {
                        vsl_catchup(v);
                        continue;
                }
                vinyl_fatal(v, "Unknown %s argument: %s", me, *av);
        }
}

#endif /* VTEST_WITH_VTC_VINYL */

		vinyl-cache/bin/vinyltest/vtc_vinyl.c
0		/*-
1		* Copyright (c) 2008-2015 Varnish Software AS
2		* All rights reserved.
3		*
4		* Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
5		*
6		* SPDX-License-Identifier: BSD-2-Clause
7		*
8		* Redistribution and use in source and binary forms, with or without
9		* modification, are permitted provided that the following conditions
10		* are met:
11		* 1. Redistributions of source code must retain the above copyright
12		* notice, this list of conditions and the following disclaimer.
13		* 2. Redistributions in binary form must reproduce the above copyright
14		* notice, this list of conditions and the following disclaimer in the
15		* documentation and/or other materials provided with the distribution.
16		*
17		* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18		* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19		* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20		* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
21		* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22		* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23		* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24		* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25		* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26		* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27		* SUCH DAMAGE.
28		*/
29
30		#ifdef VTEST_WITH_VTC_VINYL
31
32		#include "config.h"
33
34		#include <sys/socket.h>
35
36		#include <errno.h>
37		#include <fcntl.h>
38		#include <fnmatch.h>
39		#include <inttypes.h>
40		#include <limits.h>
41		#include <poll.h>
42		#include <pthread.h>
43		#include <signal.h>
44		#include <stdio.h>
45		#include <stdlib.h>
46		#include <string.h>
47		#include <unistd.h>
48
49		#include "vapi/vsc.h"
50		#include "vapi/vsl.h"
51		#include "vapi/vsm.h"
52
53		#include "vdef.h"
54		#include "vqueue.h"
55
56		#include "miniobj.h"
57		#include "vas.h"
58		#include "vcli.h"
59		#include "vjsn.h"
60		#include "vre.h"
61		#include "vsb.h"
62		#include "vsub.h"
63		#include "vtcp.h"
64		#include "vtim.h"
65
66		#include <vtest_api.h>
67		#include "vtest_ext_vinyl.h"
68
69		struct vinyl {
70		unsigned magic;
71		#define VINYL_MAGIC 0x208cd8e4
72		char *me;
73		char *name;
74		struct vtclog *vl;
75		VTAILQ_ENTRY(vinyl) list;
76
77		struct vsb *args;
78		int fds[4];
79		pid_t pid;
80
81		double syntax;
82
83		pthread_t tp;
84		pthread_t tp_vsl;
85		int tp_started;
86
87		int expect_exit;
88
89		int cli_fd;
90		int vcl_nbr;
91		char *workdir;
92		char *jail;
93		char *proto;
94
95		struct vsm *vsm_vsl;
96		struct vsm *vsm_vsc;
97		struct vsc *vsc;
98		int has_a_arg;
99
100		unsigned vsl_tag_count[256];
101
102		volatile int vsl_rec;
103		volatile int vsl_idle;
104		};
105
106		#define NONSENSE "%XJEIFLH\|)Xspa8P"
107
108		static VTAILQ_HEAD(, vinyl) vinyles =
109		VTAILQ_HEAD_INITIALIZER(vinyles);
110
111		/**********************************************************************
112		* Fatal condition cleanup
113		* Invalid to call in any code path not followed by vtc_fatal().
114		*/
115
116		static void
117	21	vinyl_fatal_cleanup(const struct vinyl *v)
118		{
119		struct pollfd fd[1];
120		int n;
121
122	21	if (!pthread_equal(pthread_self(), vtc_thread))
123	0	return;
124
125	21	if (!v->tp_started)
126	21	return;
127
128	0	memset(fd, 0, sizeof(fd));
129	0	fd[0].fd = v->fds[0];
130	0	fd[0].events = POLLIN;
131
132	0	do {
133	0	n = poll(fd, sizeof(fd)/sizeof(fd[0]), 10);
134	0	if (n == 1 && (fd[0].revents & (POLLHUP\|POLLERR)) != 0) {
135	0	PTOK(pthread_join(v->tp, NULL));
136	0	break;
137		}
138
139	0	if (n == 1)
140	0	VTIM_sleep(0.01);
141	0	} while (n > 0);
142	21	}
143
144		#define vinyl_fatal(v, ...) \
145		do { \
146		vinyl_fatal_cleanup((v)); \
147		vtc_fatal((v)->vl, __VA_ARGS__); \
148		} while (0)
149
150		/**********************************************************************
151		* Ask a question over CLI
152		*/
153
154		static enum VCLI_status_e
155	322196	vinyl_ask_cli(const struct vinyl v, const char cmd, char **repl)
156		{
157		int i;
158		unsigned retval;
159		char *r;
160
161	322196	if (cmd != NULL) {
162	300924	vtc_dump(v->vl, 4, "CLI TX", cmd, -1);
163	300924	i = write(v->cli_fd, cmd, vstrlen(cmd));
164	300924	if (i != vstrlen(cmd) && !vtc_stop)
165	0	vinyl_fatal(v, "CLI write failed (%s) = %u %s",
166		cmd, errno, strerror(errno));
167	0	i = write(v->cli_fd, "\n", 1);
168	0	if (i != 1 && !vtc_stop)
169	0	vinyl_fatal(v, "CLI write failed (%s) = %u %s",
170		cmd, errno, strerror(errno));
171	300924	}
172	601848	i = VCLI_ReadResult(v->cli_fd, &retval, &r, vtc_maxdur);
173	601848	if (i != 0 && !vtc_stop)
174	0	vinyl_fatal(v, "CLI failed (%s) = %d %u %s",
175		cmd != NULL ? cmd : "NULL", i, retval, r);
176	322196	vtc_log(v->vl, 3, "CLI RX %u", retval);
177	322196	vtc_dump(v->vl, 4, "CLI RX", r, -1);
178	322196	if (repl != NULL)
179	205880	*repl = r;
180		else
181	116316	free(r);
182	322196	return ((enum VCLI_status_e)retval);
183		}
184
185		/**********************************************************************
186		*
187		*/
188
189		static void
190	22511	wait_stopped(const struct vinyl *v)
191		{
192	22511	char *r = NULL;
193		enum VCLI_status_e st;
194
195	22511	vtc_log(v->vl, 3, "wait-stopped");
196	22511	while (1) {
197	22511	st = vinyl_ask_cli(v, "status", &r);
198	22511	if (st != CLIS_OK)
199	0	vinyl_fatal(v,
200		"CLI status command failed: %u %s", st, r);
201	22511	if (!vstrcmp(r, "Child in state stopped")) {
202	22511	free(r);
203	22511	break;
204		}
205	0	free(r);
206	0	r = NULL;
207	0	VTIM_sleep(0.2);
208		}
209	22511	}
210		/**********************************************************************
211		*
212		*/
213
214		static void
215	21021	wait_running(const struct vinyl *v)
216		{
217	21021	char *r = NULL;
218		enum VCLI_status_e st;
219
220	21021	while (1) {
221	21021	vtc_log(v->vl, 3, "wait-running");
222	21021	st = vinyl_ask_cli(v, "status", &r);
223	21021	if (st != CLIS_OK)
224	0	vinyl_fatal(v,
225		"CLI status command failed: %u %s", st, r);
226	0	if (!vstrcmp(r, "Child in state stopped"))
227	0	vinyl_fatal(v,
228		"Child stopped before running: %u %s", st, r);
229	21021	if (!vstrcmp(r, "Child in state running")) {
230	21021	free(r);
231	21021	r = NULL;
232	21021	st = vinyl_ask_cli(v, "debug.listen_address", &r);
233	21021	if (st != CLIS_OK)
234	0	vinyl_fatal(v,
235		"CLI status command failed: %u %s", st, r);
236	21021	free(r);
237	21021	break;
238		}
239	0	free(r);
240	0	r = NULL;
241	0	VTIM_sleep(0.2);
242		}
243	21021	}
244
245		/**********************************************************************
246		* Vinyllog gatherer thread
247		*/
248
249		static void *
250	21272	vinyllog_thread(void *priv)
251		{
252		struct vinyl *v;
253		struct VSL_data *vsl;
254		struct vsm *vsm;
255		struct VSL_cursor *c;
256		enum VSL_tag_e tag;
257		uint64_t vxid;
258		unsigned len, vs;
259		const char tagname, data;
260		int type, i, opt;
261		struct vsb *vsb;
262
263	21272	CAST_OBJ_NOTNULL(v, priv, VINYL_MAGIC);
264
265	21272	vsl = VSL_New();
266	21272	AN(vsl);
267	21272	vsm = v->vsm_vsl;
268
269	21272	vsb = VSB_new_auto();
270	21272	AN(vsb);
271
272	21272	c = NULL;
273	21272	opt = 0;
274	760334	while (v->fds[1] > 0 \|\| c != NULL) { //lint !e845 bug in flint
275	739062	if (c == NULL) {
276	196797	if (vtc_error)
277	0	break;
278	196797	VTIM_sleep(0.1);
279	196797	(void)VSM_Status(vsm);
280	196797	c = VSL_CursorVSM(vsl, vsm, opt);
281	196797	if (c == NULL) {
282	141667	vtc_log(v->vl, 3, "vsl\|%s", VSL_Error(vsl));
283	141667	VSL_ResetError(vsl);
284	141667	continue;
285		}
286	55129	}
287	597394	AN(c);
288
289	597394	opt = VSL_COPT_TAIL;
290
291	7499214	while (1) {
292	7530735	i = VSL_Next(c);
293	7530735	if (i != 1)
294	597393	break;
295
296	6933341	v->vsl_rec = 1;
297
298	6933341	tag = VSL_TAG(c->rec.ptr);
299	6933341	vxid = VSL_ID(c->rec.ptr);
300	6933341	if (tag == SLT__Batch)
301	0	continue;
302	6933341	tagname = VSL_tags[tag];
303	6933341	len = VSL_LEN(c->rec.ptr);
304	6933341	type = VSL_CLIENT(c->rec.ptr) ?
305	2781838	'c' : VSL_BACKEND(c->rec.ptr) ?
306		'b' : '-';
307	6933342	data = VSL_CDATA(c->rec.ptr);
308	6933342	v->vsl_tag_count[tag]++;
309	6933342	if (VSL_tagflags[tag] & SLT_F_BINARY) {
310	31521	VSB_clear(vsb);
311	31521	VSB_quote(vsb, data, len, VSB_QUOTE_HEX);
312	31521	AZ(VSB_finish(vsb));
313		/* +2 to skip "0x" */
314	63042	vtc_log(v->vl, 4, "vsl\| %10ju %-15s %c [%s]",
315	31521	(uintmax_t)vxid, tagname, type,
316	31521	VSB_data(vsb) + 2);
317	31521	continue;
318		}
319	6901820	if (VSL_tagflags[tag] & SLT_F_UNSAFE) {
320		// remove trailing NUL
321	917965	if (len > 1 && data[len - 1] == '\0')
322	917944	len--;
323	917965	VSB_clear(vsb);
324	917965	VSB_quote(vsb, data, len, VSB_QUOTE_ESCHEX);
325	917965	AZ(VSB_finish(vsb));
326	917965	data = VSB_data(vsb);
327	917965	len = VSB_len(vsb);
328	917965	}
329	13803640	vtc_log(v->vl, 4, "vsl\| %10ju %-15s %c %.*s",
330	6901820	(uintmax_t)vxid, tagname, type, (int)len,
331	6901820	data);
332		}
333	597393	if (i == 0) {
334		/* Nothing to do but wait */
335	597393	v->vsl_idle++;
336	597393	vs = VSM_Status(vsm) & VSM_WRK_MASK;
337	597393	if ((vs & ~VSM_WRK_CHANGED) != VSM_WRK_RUNNING) {
338		/* Abandoned - try reconnect */
339	55129	VSL_DeleteCursor(c);
340	55129	c = NULL;
341	55129	} else {
342	542264	VTIM_sleep(0.1);
343		}
344	597393	} else if (i == -2) {
345		/* Abandoned - try reconnect */
346	0	VSL_DeleteCursor(c);
347	0	c = NULL;
348	0	} else
349	0	break;
350		}
351
352	0	if (c)
353	0	VSL_DeleteCursor(c);
354	21272	VSL_Delete(vsl);
355	21272	VSB_destroy(&vsb);
356
357	21272	return (NULL);
358		}
359
360		/**********************************************************************
361		* Allocate and initialize a vinyl
362		*/
363
364		static struct vinyl *
365	21293	vinyl_new(const char *name)
366		{
367		struct vinyl *v;
368		struct vsb *vsb;
369		char buf[1024];
370
371	21293	ALLOC_OBJ(v, VINYL_MAGIC);
372	21293	AN(v);
373	21293	REPLACE(v->name, name);
374
375	21293	REPLACE(v->jail, "");
376
377	21293	v->vl = vtc_logopen("%s", name);
378	21293	AN(v->vl);
379
380	21293	vsb = macro_expandf(v->vl, "${tmpdir}/%s", name);
381	21293	AN(vsb);
382	21293	v->workdir = strdup(VSB_data(vsb));
383	21293	AN(v->workdir);
384	21293	VSB_destroy(&vsb);
385
386	21293	bprintf(buf, "rm -rf %s ; mkdir -p %s", v->workdir, v->workdir);
387	21293	AZ(system(buf));
388
389	21293	v->args = VSB_new_auto();
390
391	21293	v->cli_fd = -1;
392	21293	VTAILQ_INSERT_TAIL(&vinyles, v, list);
393
394	21293	return (v);
395		}
396
397		/**********************************************************************
398		* Delete a vinyl instance
399		*/
400
401		static void
402	21293	vinyl_delete(struct vinyl *v)
403		{
404
405	21293	CHECK_OBJ_NOTNULL(v, VINYL_MAGIC);
406	21293	vtc_logclose(v->vl);
407	21293	free(v->name);
408	21293	free(v->jail);
409	21293	free(v->workdir);
410	21293	VSB_destroy(&v->args);
411	21293	if (v->vsc != NULL)
412	21272	VSC_Destroy(&v->vsc, v->vsm_vsc);
413	21272	if (v->vsm_vsc != NULL)
414	21272	VSM_Destroy(&v->vsm_vsc);
415	21272	if (v->vsm_vsl != NULL)
416	21272	VSM_Destroy(&v->vsm_vsl);
417
418		/*
419		* We do not delete the workdir, it may contain stuff people
420		* want (coredumps, shmlog/stats etc), and trying to divine
421		* "may want" is just too much trouble. Leave it around and
422		* nuke it at the start of the next test-run.
423		*/
424
425		/* XXX: MEMLEAK (?) */
426	21293	FREE_OBJ(v);
427	21293	}
428
429		/**********************************************************************
430		* Vinyl listener
431		*/
432
433		static void *
434	21272	vinyl_thread(void *priv)
435		{
436		struct vinyl *v;
437
438	21272	CAST_OBJ_NOTNULL(v, priv, VINYL_MAGIC);
439	21272	return (vtc_record(v->vl, v->fds[0], NULL));
440		}
441
442		/**********************************************************************
443		* Launch a Vinyl
444		*/
445
446		static void
447	21272	vinyl_launch(struct vinyl *v)
448		{
449		struct vsb vsb, vsb1;
450		int i, nfd, asock;
451		char abuf[VTCP_ADDRBUFSIZE];
452		char pbuf[VTCP_PORTBUFSIZE];
453		char lbuf[PATH_MAX];
454		struct pollfd fd[3];
455		enum VCLI_status_e u;
456		const char *err;
457	21272	char *r = NULL;
458
459		/* Create listener socket */
460	21272	asock = VTCP_listen_on(default_listen_addr, NULL, 1, &err);
461	21272	if (err != NULL)
462	0	vinyl_fatal(v, "Create CLI listen socket failed: %s", err);
463	21272	assert(asock > 0);
464	21272	VTCP_myname(asock, abuf, sizeof abuf, pbuf, sizeof pbuf);
465
466	21272	AZ(VSB_finish(v->args));
467	21272	vtc_log(v->vl, 2, "Launch");
468	21272	vsb = VSB_new_auto();
469	21272	AN(vsb);
470	21272	VSB_cat(vsb, "cd ${pwd} &&");
471	42544	VSB_printf(vsb, " exec vinyld %s -d -n %s -i %s",
472	21272	v->jail, v->workdir, v->name);
473	21272	if (macro_isdef(NULL, "vinyld_args_prepend")) {
474	0	VSB_putc(vsb, ' ');
475	0	macro_cat(v->vl, vsb, "vinyld_args_prepend", NULL);
476	0	}
477	0	VSB_cat(vsb, VSB_data(params_vsb));
478	0	if (leave_temp) {
479	0	VSB_cat(vsb, " -p debug=+vcl_keep");
480	0	VSB_cat(vsb, " -p debug=+vmod_so_keep");
481	0	VSB_cat(vsb, " -p debug=+vsm_keep");
482	0	}
483	20495	VSB_cat(vsb, " -l 2m");
484	20495	VSB_cat(vsb, " -p auto_restart=off");
485	20495	VSB_cat(vsb, " -p syslog_cli_traffic=off");
486	20495	VSB_cat(vsb, " -p thread_pool_min=10");
487	20495	VSB_cat(vsb, " -p debug=+vtc_mode");
488	20495	VSB_cat(vsb, " -p vsl_mask=+Debug,+H2RxHdr,+H2RxBody");
489	20495	VSB_cat(vsb, " -p h2_initial_window_size=1m");
490	20495	VSB_cat(vsb, " -p h2_rx_window_low_water=64k");
491	20495	if (!v->has_a_arg) {
492	20495	VSB_printf(vsb, " -a '%s'", default_listen_addr);
493	20495	if (v->proto != NULL)
494	336	VSB_printf(vsb, ",%s", v->proto);
495	20495	}
496	21251	VSB_printf(vsb, " -M '%s %s'", abuf, pbuf);
497	21251	VSB_printf(vsb, " -P %s/%sd.pid", v->workdir, v->me);
498	21251	if (vmod_path != NULL)
499	21251	VSB_printf(vsb, " -p vmod_path=%s", vmod_path);