Code Coverage: vinyl-cache/lib/libvcc/vcc_token.c
 
82217aa4fc7094b3610340078afaf78cd95a8ec9
 
0
/*-
1
 * Copyright (c) 2006 Verdens Gang AS
2
 * Copyright (c) 2006-2011 Varnish Software AS
3
 * All rights reserved.
4
 *
5
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
6
 *
7
 * SPDX-License-Identifier: BSD-2-Clause
8
 *
9
 * Redistribution and use in source and binary forms, with or without
10
 * modification, are permitted provided that the following conditions
11
 * are met:
12
 * 1. Redistributions of source code must retain the above copyright
13
 *    notice, this list of conditions and the following disclaimer.
14
 * 2. Redistributions in binary form must reproduce the above copyright
15
 *    notice, this list of conditions and the following disclaimer in the
16
 *    documentation and/or other materials provided with the distribution.
17
 *
18
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
22
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28
 * SUCH DAMAGE.
29
 */
30
31
#include "config.h"
32
33
#include <stdlib.h>
34
#include <string.h>
35
36
#include "vcc_compile.h"
37
38
#include "venc.h"
39
#include "vct.h"
40
41
/*--------------------------------------------------------------------*/
42
43
void
44
vcc_ErrToken(const struct vcc *tl, const struct token *t)
45 36
{
46
47 36
        if (t->tok == EOI)
48 0
                VSB_cat(tl->sb, "end of input");
49 36
        else if (t->tok == CSRC)
50 0
                VSB_cat(tl->sb, "C{ ... }C");
51 36
        else
52 36
                VSB_printf(tl->sb, "'%.*s'", PF(t));
53 36
}
54
55
void
56
vcc__ErrInternal(struct vcc *tl, const char *func, unsigned line)
57 0
{
58
59 0
        VSB_printf(tl->sb, "VCL compiler internal error at %s():%u\n",
60 0
            func, line);
61 0
        tl->err = 1;
62 0
}
63
64
/*--------------------------------------------------------------------
65
 * Find start of source-line of token
66
 */
67
68
static void
69
vcc_iline(const struct token *t, const char **ll, int tail)
70 715
{
71 715
        const char *p, *b, *x;
72
73 715
        b = t->src->b;
74 715
        if (ll != NULL)
75 715
                *ll = b;
76 715
        x = tail ? t->e - 1 : t->b;
77 277213
        for (p = b; p < x; p++) {
78 276498
                if (*p == '\n') {
79 8551
                        if (ll != NULL)
80 8551
                                *ll = p + 1;
81 8551
                }
82 276498
        }
83 715
}
84
85
/*--------------------------------------------------------------------
86
 * Find and print src+line+pos of this token
87
 */
88
89
static void
90
vcc_icoord(struct vsb *vsb, const struct token *t, int tail)
91 135870
{
92 135870
        unsigned lin, pos;
93 135870
        const char *p, *b, *x;
94
95 135870
        lin = 1;
96 135870
        pos = 0;
97 135870
        b = t->src->b;
98 135870
        x = tail ? t->e - 1 : t->b;
99 732666341
        for (p = b; p < x; p++) {
100 732530471
                if (*p == '\n') {
101 28422840
                        lin++;
102 28422840
                        pos = 0;
103 704107631
                } else if (*p == '\t') {
104 13512749
                        pos &= ~7;
105 13512749
                        pos += 8;
106 13512749
                } else
107 690594882
                        pos++;
108 732530471
        }
109 135870
        VSB_cat(vsb, "(");
110 135870
        if (tail < 2)
111 135743
                VSB_printf(vsb, "'%s' Line %u ", t->src->name, lin);
112 135870
        VSB_printf(vsb, "Pos %u)", pos + 1);
113 135870
}
114
115
/*--------------------------------------------------------------------*/
116
117
void
118
vcc_Coord(const struct vcc *tl, struct vsb *vsb, const struct token *t)
119 135155
{
120
121 135155
        if (t == NULL)
122 135155
                t = tl->t;
123 135155
        vcc_icoord(vsb, t, 0);
124 135155
}
125
126
/*--------------------------------------------------------------------
127
 * Output one line of source code, starting at 'l' and ending at the
128
 * first NL or 'le'.
129
 */
130
131
static void
132
vcc_quoteline(const struct vcc *tl, const char *l, const char *le)
133 588
{
134 588
        const char *p;
135 588
        unsigned x, y;
136
137 588
        x = y = 0;
138 18446
        for (p = l; p < le && *p != '\n'; p++) {
139 17858
                if (*p == '\t') {
140 785
                        y &= ~7;
141 785
                        y += 8;
142 7057
                        while (x < y) {
143 6272
                                VSB_putc(tl->sb, ' ');
144 6272
                                x++;
145 6272
                        }
146 17073
                } else {
147 17073
                        x++;
148 17073
                        y++;
149 17073
                        VSB_putc(tl->sb, *p);
150 17073
                }
151 17858
        }
152 588
        VSB_putc(tl->sb, '\n');
153 588
}
154
155
/*--------------------------------------------------------------------
156
 * Output a marker line for a sourceline starting at 'l' and ending at
157
 * the first NL or 'le'.  Characters between 'b' and 'e' are marked.
158
 */
159
160
static void
161
vcc_markline(const struct vcc *tl, const char *l, const char *le,
162
    const char *b, const char *e)
163 588
{
164 588
        const char *p;
165 588
        unsigned x, y;
166 588
        char c;
167
168 588
        x = y = 0;
169 18446
        for (p = l; p < le && *p != '\n'; p++) {
170 17858
                if (p >= b && p < e)
171 4506
                        c = '#';
172 13352
                else
173 13352
                        c = '-';
174
175 17858
                if (*p == '\t') {
176 785
                        y &= ~7;
177 785
                        y += 8;
178 785
                } else
179 17073
                        y++;
180 41203
                while (x < y) {
181 23345
                        VSB_putc(tl->sb, c);
182 23345
                        x++;
183 23345
                }
184 17858
        }
185 588
        VSB_putc(tl->sb, '\n');
186 588
}
187
188
void
189
vcc_Warn(struct vcc *tl)
190 159
{
191
192 159
        AN(tl);
193 159
        AN(tl->err);
194 159
        VSB_cat(tl->sb, "(That was just a warning)\n");
195 159
        tl->err = 0;
196 159
}
197
198
/*--------------------------------------------------------------------*/
199
/* XXX: should take first+last token */
200
201
void
202
vcc_ErrWhere2(struct vcc *tl, const struct token *t, const struct token *t2)
203 132
{
204 132
        const char  *l1, *l2, *l3;
205
206 132
        if (t == NULL) {
207 1
                vcc_ErrWhere(tl, t2);
208 1
                return;
209 1
        }
210 131
        if (t->src != t2->src) {
211 4
                vcc_ErrWhere(tl, t);
212 4
                return;
213 4
        }
214 127
        vcc_iline(t, &l1, 0);
215 127
        t2 = VTAILQ_PREV(t2, tokenhead, list);
216 127
        vcc_iline(t2, &l2, 1);
217
218
219 127
        if (l1 == l2) {
220 127
                vcc_icoord(tl->sb, t, 0);
221 127
                VSB_cat(tl->sb, " -- ");
222 127
                vcc_icoord(tl->sb, t2, 2);
223 127
                VSB_putc(tl->sb, '\n');
224
                /* Two tokens on same line */
225 127
                vcc_quoteline(tl, l1, t->src->e);
226 127
                vcc_markline(tl, l1, t->src->e, t->b, t2->e);
227 127
        } else {
228
                /* Two tokens different lines */
229 0
                l3 = strchr(l1, '\n');
230 0
                AN(l3);
231
                /* XXX: t had better be before t2 */
232 0
                vcc_icoord(tl->sb, t, 0);
233 0
                if (l3 + 1 == l2) {
234 0
                        VSB_cat(tl->sb, " -- ");
235 0
                        vcc_icoord(tl->sb, t2, 1);
236 0
                }
237 0
                VSB_putc(tl->sb, '\n');
238 0
                vcc_quoteline(tl, l1, t->src->e);
239 0
                vcc_markline(tl, l1, t->src->e, t->b, t2->e);
240 0
                if (l3 + 1 != l2) {
241 0
                        VSB_cat(tl->sb, "[...]\n");
242 0
                        vcc_icoord(tl->sb, t2, 1);
243 0
                        VSB_putc(tl->sb, '\n');
244 0
                }
245 0
                vcc_quoteline(tl, l2, t->src->e);
246 0
                vcc_markline(tl, l2, t->src->e, t->b, t2->e);
247 0
        }
248 127
        VSB_putc(tl->sb, '\n');
249 127
        tl->err = 1;
250 127
}
251
252
void
253
vcc_ErrWhere(struct vcc *tl, const struct token *t)
254 461
{
255 461
        const char  *l1;
256
257 461
        vcc_iline(t, &l1, 0);
258 461
        vcc_icoord(tl->sb, t, 0);
259 461
        VSB_putc(tl->sb, '\n');
260 461
        vcc_quoteline(tl, l1, t->src->e);
261 461
        vcc_markline(tl, l1, t->src->e, t->b, t->e);
262 461
        VSB_putc(tl->sb, '\n');
263 461
        tl->err = 1;
264 461
}
265
266
/*--------------------------------------------------------------------*/
267
268
struct token *
269
vcc_PeekTokenFrom(const struct vcc *tl, const struct token *t)
270 4579171
{
271 4579171
        struct token *t2;
272
273 4579171
        CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
274 4579171
        AN(t);
275 4579171
        assert(t->tok != EOI);
276 4579171
        t2 = VTAILQ_NEXT(t, list);
277 4579171
        AN(t2);
278 4579171
        return (t2);
279 4579171
}
280
281
struct token *
282
vcc_PeekToken(const struct vcc *tl)
283 330705
{
284
285 330705
        CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
286 330705
        return (vcc_PeekTokenFrom(tl, tl->t));
287 330705
}
288
289
void
290
vcc_NextToken(struct vcc *tl)
291 1741633
{
292
293 1741633
        CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
294 1741633
        tl->t = vcc_PeekTokenFrom(tl, tl->t);
295 1741633
}
296
297
void
298
vcc__Expect(struct vcc *tl, unsigned tok, unsigned line)
299 1211162
{
300 1211162
        if (tl->t->tok == tok)
301 1211146
                return;
302 16
        VSB_printf(tl->sb, "Expected %s got ", vcl_tnames[tok]);
303 16
        vcc_ErrToken(tl, tl->t);
304 16
        VSB_printf(tl->sb, "\n(program line %u), at\n", line);
305 16
        vcc_ErrWhere(tl, tl->t);
306 16
}
307
308
/*--------------------------------------------------------------------
309
 * Compare ID token to string, return true of match
310
 */
311
312
int
313
vcc_IdIs(const struct token *t, const char *p)
314 3230965
{
315 3230965
        const char *q;
316
317 3230965
        assert(t->tok == ID);
318 4294800
        for (q = t->b; q < t->e && *p != '\0'; p++, q++)
319 4052638
                if (*q != *p)
320 2988803
                        return (0);
321 242162
        if (q != t->e || *p != '\0')
322 211
                return (0);
323 241951
        return (1);
324 242162
}
325
326
/*--------------------------------------------------------------------
327
 * Check that we have a Vinyl identifier
328
 */
329
330
void
331
vcc_PrintTokens(const struct vcc *tl,
332
    const struct token *tb, const struct token *te)
333 35
{
334
335 35
        CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
336 35
        AN(tb);
337 35
        AN(te);
338 118
        while (tb != te) {
339 83
                VSB_printf(tl->sb, "%.*s", PF(tb));
340 83
                tb = vcc_PeekTokenFrom(tl, tb);
341 83
                AN(tb);
342 83
        }
343 35
}
344
345
void
346
vcc_ExpectVid(struct vcc *tl, const char *what)
347 137225
{
348 137225
        const char *bad = NULL;
349 137225
        struct token *t2;
350
351 137225
        ExpectErr(tl, ID);
352 137224
        ERRCHK(tl);
353
354 137224
        t2 = vcc_PeekToken(tl);
355 137224
        AN(t2);
356 137230
        while (t2->tok == '.') {
357 6
                bad = ".";
358 6
                t2 = vcc_PeekTokenFrom(tl, t2);
359 6
                AN(t2);
360 6
                if (t2->tok != ID)
361 0
                        break;
362 6
                t2 = vcc_PeekTokenFrom(tl, t2);
363 6
                AN(t2);
364 6
        }
365 137224
        if (bad == NULL)
366 137218
                bad = VCT_invalid_name(tl->t->b, tl->t->e);
367 137224
        if (bad != NULL) {
368 6
                VSB_printf(tl->sb, "Name of %s, '", what);
369 6
                vcc_PrintTokens(tl, tl->t, t2);
370 6
                VSB_printf(tl->sb,
371 6
                    "', contains illegal character '%c'\n", *bad);
372 6
                vcc_ErrWhere2(tl, tl->t, t2);
373 6
                return;
374 6
        }
375 137224
}
376
377
/*--------------------------------------------------------------------
378
 * Decode a string
379
 */
380
381
static void
382
vcc_decstr(struct vcc *tl, unsigned sep)
383 154709
{
384 154709
        char *q;
385 154709
        unsigned int l;
386
387 154709
        assert(tl->t->tok == CSTR);
388 154709
        l = pdiff(tl->t->b + sep, tl->t->e - sep);
389 154709
        tl->t->dec = TlAlloc(tl, l + 1);
390 154709
        AN(tl->t->dec);
391 154709
        q = tl->t->dec;
392 154709
        vmemcpy(q, tl->t->b + sep, l);
393 154709
        q[l] = '\0';
394 154709
}
395
396
/*--------------------------------------------------------------------
397
 * Add a token to the token list.
398
 */
399
400
static void
401
vcc_addtoken(struct vcc *tl, unsigned tok,
402
    struct source *sp, const char *b, const char *e)
403 3294361
{
404 3294361
        struct token *t;
405
406 3294361
        t = TlAlloc(tl, sizeof *t);
407 3294361
        assert(t != NULL);
408 3294361
        t->tok = tok;
409 3294361
        t->b = b;
410 3294361
        t->e = e;
411 3294361
        t->src = sp;
412 3294361
        VTAILQ_INSERT_TAIL(&sp->src_tokens, t, src_list);
413 3294361
        tl->t = t;
414 3294361
}
415
416
/*--------------------------------------------------------------------
417
 * Find a delimited token
418
 */
419
420
static const struct delim_def {
421
        const char      *name;
422
        const char      *b;
423
        const char      *e;
424
        unsigned        len;    /* NB: must be the same for both delimiters */
425
        unsigned        crlf;
426
        unsigned        tok;
427
} delim_defs[] = {
428
#define DELIM_DEF(nm, l, r, c, t)               \
429
        { nm, l, r, sizeof (l) - 1, c, t }
430
        DELIM_DEF("long-string", "\"\"\"", "\"\"\"", 1, CSTR),  /* """...""" */
431
        DELIM_DEF("long-string", "{\"", "\"}", 1, CSTR),        /*  {"..."}  */
432
        DELIM_DEF("string", "\"", "\"", 0, CSTR),               /*   "..."   */
433
        DELIM_DEF("inline C source", "C{", "}C", 1, CSRC),      /*  C{...}C  */
434
#undef DELIM_DEF
435
        { NULL }
436
};
437
438
static unsigned
439
vcc_delim_token(struct vcc *tl, struct source *sp, const char *p,
440
    const char **qp)
441 3289438
{
442 3289438
        const struct delim_def *dd;
443 3289438
        const char *q, *r;
444
445 16105234
        for (dd = delim_defs; dd->name != NULL; dd++)
446 12970533
                if (!vstrncmp(p, dd->b, dd->len))
447 154737
                        break;
448
449 3289438
        if (dd->name == NULL)
450 3134701
                return (0);
451
452 154737
        q = strstr(p + dd->len, dd->e);
453 154737
        if (q != NULL && !dd->crlf) {
454 122209
                r = strpbrk(p + dd->len, "\r\n");
455 122209
                if (r != NULL && r < q)
456 0
                        q = NULL;
457 122209
        }
458
459 154737
        if (q == NULL) {
460 4
                vcc_addtoken(tl, EOI, sp, p, p + dd->len);
461 4
                VSB_printf(tl->sb, "Unterminated %s, starting at\n", dd->name);
462 4
                vcc_ErrWhere(tl, tl->t);
463 4
                return (0);
464 4
        }
465
466 154733
        assert(q < sp->e);
467 154733
        vcc_addtoken(tl, dd->tok, sp, p, q + dd->len);
468 154733
        if (dd->tok == CSTR)
469 154709
                vcc_decstr(tl, dd->len);
470 154733
        *qp = q + dd->len;
471 154733
        return (1);
472 154733
}
473
474
/*--------------------------------------------------------------------
475
 * Lex a number, either CNUM or FNUM.
476
 * We enforce the RFC8941 restrictions on number of digits here.
477
 */
478
479
static int
480
vcc_lex_number_token_valid(struct vcc *tl)
481 45519
{
482 45519
        if (*tl->t->e == 'e' || *tl->t->e == 'E') {
483 2
                VSB_printf(tl->sb, "Unexpected character '%c'.\n", *tl->t->e);
484 2
                tl->t->e++;
485 2
                vcc_ErrWhere(tl, tl->t);
486 2
                return (0);
487 2
        }
488 45517
        return (1);
489 45519
}
490
491
static const char *
492
vcc_lex_number(struct vcc *tl, struct source *sp, const char *p)
493 45523
{
494 45523
        const char *q, *r;
495 45523
        char *s;
496
497 157687
        for (q = p; q < sp->e; q++)
498 157687
                if (!vct_isdigit(*q))
499 45523
                        break;
500 45523
        if (*q != '.') {
501 40477
                vcc_addtoken(tl, CNUM, sp, p, q);
502 40477
                if (q - p > 15) {
503 1
                        VSB_cat(tl->sb, "Too many digits for integer.\n");
504 1
                        vcc_ErrWhere(tl, tl->t);
505 1
                        return (NULL);
506 1
                }
507 40476
                if (! vcc_lex_number_token_valid(tl))
508 1
                        return (NULL);
509 40475
                tl->t->num = strtod(p, &s);
510 40475
                assert(s == tl->t->e);
511 40475
                return (q);
512 40475
        }
513 5046
        r = ++q;
514 10146
        for (; r < sp->e; r++)
515 10146
                if (!vct_isdigit(*r))
516 5046
                        break;
517 5046
        vcc_addtoken(tl, FNUM, sp, p, r);
518 5046
        if (q - p > 13 || r - q > 3) {
519 3
                VSB_cat(tl->sb, "Too many digits for real.\n");
520 3
                vcc_ErrWhere(tl, tl->t);
521 3
                return (NULL);
522 3
        }
523 5043
        if (! vcc_lex_number_token_valid(tl))
524 1
                return (NULL);
525 5042
        tl->t->num = strtod(p, &s);
526 5042
        assert(s == tl->t->e);
527 5042
        return (r);
528 5042
}
529
530
/*--------------------------------------------------------------------
531
 * Lexical analysis and token generation
532
 */
533
534
void
535
vcc_Lexer(struct vcc *tl, struct source *sp)
536 4927
{
537 4927
        const char *p, *q, *r;
538 4927
        unsigned u;
539 4927
        struct vsb *vsb;
540 4927
        char namebuf[40];
541
542 6714779
        for (p = sp->b; p < sp->e; ) {
543
544
                /* Skip any whitespace */
545 6709876
                if (vct_isspace(*p)) {
546 3160516
                        p++;
547 3160516
                        continue;
548 3160516
                }
549
550
                /* Skip '#.*\n' comments */
551 3549360
                if (*p == '#') {
552 12198187
                        while (p < sp->e && *p != '\n')
553 11938303
                                p++;
554 259884
                        continue;
555 259884
                }
556
557
                /* Skip C-style comments */
558 3289476
                if (*p == '/' && p[1] == '*') {
559 66
                        for (q = p + 2; q < sp->e; q++) {
560 65
                                if (*q == '/' && q[1] == '*') {
561 1
                                        VSB_cat(tl->sb,
562 1
                                            "/* ... */ comment contains /*\n");
563 1
                                        vcc_addtoken(tl, EOI, sp, p, p + 2);
564 1
                                        vcc_ErrWhere(tl, tl->t);
565 1
                                        vcc_addtoken(tl, EOI, sp, q, q + 2);
566 1
                                        vcc_ErrWhere(tl, tl->t);
567 1
                                        return;
568 1
                                }
569 64
                                if (*q == '*' && q[1] == '/') {
570 2
                                        p = q + 2;
571 2
                                        break;
572 2
                                }
573 64
                        }
574 3
                        if (q < sp->e)
575 2
                                continue;
576 1
                        vcc_addtoken(tl, EOI, sp, p, p + 2);
577 1
                        VSB_cat(tl->sb,
578 1
                            "Unterminated /* ... */ comment, starting at\n");
579 1
                        vcc_ErrWhere(tl, tl->t);
580 1
                        return;
581 3
                }
582
583
                /* Skip C++-style comments */
584 3289472
                if (*p == '/' && p[1] == '/') {
585 503
                        while (p < sp->e && *p != '\n')
586 486
                                p++;
587 17
                        continue;
588 17
                }
589
590
                /* Recognize BLOB (= SF-binary) */
591 3289455
                if (*p == ':') {
592 17
                        vsb = VSB_new_auto();
593 17
                        AN(vsb);
594 17
                        q = sp->e;
595 17
                        q -= (q - (p + 1)) % 4;
596 17
                        assert(q > p);
597 17
                        r = VENC_Decode_Base64(vsb, p + 1, q);
598 17
                        if (r == NULL && *q == ':')
599 2
                                r = q;
600 17
                        if (r == NULL) {
601 2
                                vcc_addtoken(tl, CBLOB, sp, p, q + 1);
602 2
                                VSB_cat(tl->sb,
603 2
                                    "Missing colon at end of BLOB:\n");
604 2
                                vcc_ErrWhere(tl, tl->t);
605 2
                                VSB_destroy(&vsb);
606 2
                                return;
607 2
                        }
608 15
                        vcc_addtoken(tl, CBLOB, sp, p, r + 1);
609 15
                        if (*r == ':' && ((r - p) % 4) != 1) {
610 3
                                VSB_cat(tl->sb,
611 3
                                    "BLOB must have n*4 base64 characters\n");
612 3
                                vcc_ErrWhere(tl, tl->t);
613 3
                                VSB_destroy(&vsb);
614 3
                                return;
615 3
                        }
616 12
                        if (*r == '=') {
617 3
                                VSB_cat(tl->sb,
618 3
                                    "Wrong padding ('=') in BLOB:\n");
619 3
                                vcc_ErrWhere(tl, tl->t);
620 3
                                VSB_destroy(&vsb);
621 3
                                return;
622 3
                        }
623 9
                        if (*r != ':') {
624 2
                                VSB_cat(tl->sb, "Illegal BLOB character:\n");
625 2
                                vcc_ErrWhere(tl, tl->t);
626 2
                                VSB_destroy(&vsb);
627 2
                                return;
628 2
                        }
629 7
                        r++;
630 7
                        AZ(VSB_finish(vsb));
631
632 7
                        bprintf(namebuf, "blob_%u", tl->unique++);
633 7
                        Fh(tl, 0,
634 7
                            "\nstatic const unsigned char %s_data[%zd] = {\n",
635 7
                            namebuf, VSB_len(vsb));
636 230
                        for (u = 0; u < VSB_len(vsb); u++) {
637 223
                                Fh(tl, 0, "\t0x%02x,", VSB_data(vsb)[u] & 0xff);
638 223
                                if ((u & 7) == 7)
639 26
                                        Fh(tl, 0, "\n");
640 223
                        }
641 7
                        if ((u & 7) != 7)
642 7
                                Fh(tl, 0, "\n");
643 7
                        Fh(tl, 0, "};\n");
644 7
                        Fh(tl, 0,
645 7
                            "\nstatic const struct vrt_blob %s[1] = {{\n",
646 7
                            namebuf);
647 7
                        Fh(tl, 0, "\t.magic =\tVRT_BLOB_MAGIC,\n");
648 7
                        Fh(tl, 0, "\t.len =\t%zd,\n", VSB_len(vsb));
649 7
                        Fh(tl, 0, "\t.blob =\t%s_data,\n", namebuf);
650 7
                        Fh(tl, 0, "}};\n");
651 7
                        REPLACE(tl->t->dec, namebuf);
652 7
                        VSB_destroy(&vsb);
653 7
                        p = r;
654 7
                        continue;
655 7
                }
656
657
                /* Match delimited tokens */
658 3289438
                if (vcc_delim_token(tl, sp, p, &q) != 0) {
659 154733
                        p = q;
660 154733
                        continue;
661 154733
                }
662 3134705
                ERRCHK(tl);
663
664
                /* Match for the fixed tokens (see generate.py) */
665 3134701
                u = vcl_fixed_token(p, &q);
666 3134701
                if (u != 0) {
667 1714366
                        vcc_addtoken(tl, u, sp, p, q);
668 1714366
                        p = q;
669 1714366
                        continue;
670 1714366
                }
671
672
                /* Match Identifiers */
673 1420335
                if (vct_isident1(*p)) {
674 10788606
                        for (q = p; q < sp->e; q++)
675 10788606
                                if (!vct_isident(*q))
676 1374810
                                        break;
677 1374810
                        vcc_addtoken(tl, ID, sp, p, q);
678 1374810
                        p = q;
679 1374810
                        continue;
680 1374810
                }
681
682
                /* Match numbers { [0-9]+ } */
683 45525
                if (vct_isdigit(*p)) {
684 45523
                        p = vcc_lex_number(tl, sp, p);
685 45523
                        if (p == NULL)
686 6
                                return;
687 45517
                        continue;
688 45523
                }
689 2
                vcc_addtoken(tl, EOI, sp, p, p + 1);
690 2
                VSB_cat(tl->sb, "Syntax error at\n");
691 2
                vcc_ErrWhere(tl, tl->t);
692 2
                return;
693 45525
        }
694 4903
        vcc_addtoken(tl, EOI, sp, sp->e, sp->e);
695 4903
}