From 0dc042eefa5b767a294a1abe6b8b7dd2542dd3ca Mon Sep 17 00:00:00 2001 From: Jo-Philipp Wich Date: Fri, 16 Oct 2026 09:35:43 +0200 Subject: [PATCH 01/14] source: rework the text access API The lexer, the error context formatter, the callee recovery, the debugger and the loader of precompiled programs all reached into the source object, using its file pointer and read offset directly. Give the source an interface for reading its text instead: - uc_source_read_block() lends a block of text and its offset to the caller, which lets the lexer drop its own read buffer and keep its own byte counter. - uc_source_read_line() returns the line holding a given offset and uc_source_callee_expr() reads the bytes in front of a call into a window and scans that in memory. Both leave the read position as they found it. - uc_source_read() with uc_source_offset() and uc_source_remaining() serve the loader of precompiled programs, which no longer needs a FILE pointer. - uc_source_read_content() returns the whole text for the debugger. - uc_source_type_test() is renamed to uc_source_begin(), as it positions the source on its first byte of code. The callee recovery no longer keeps its state in static variables. It reads a window of the source in front of a call into a heap buffer, which starts at 256 bytes and doubles up to 4096 bytes while the scan runs into its start, finds the start of the invoked expression, then collapses and copies the expression in place, and returns the buffer to the caller, which releases it using free(). The scan now also follows an expression which is broken across lines in front of a subscript or nested call. Signed-off-by: Jo-Philipp Wich --- compiler.c | 4 +- include/ucode/internal/lexer.h | 4 +- include/ucode/internal/source.h | 19 +- include/ucode/internal/types.h | 4 +- lexer.c | 42 +- lib.c | 62 +-- lib/debug.c | 21 +- program.c | 192 ++++--- source.c | 524 +++++++++++++++----- tests/custom/00_syntax/30_call_error_callee | 88 ++++ vm.c | 9 +- 11 files changed, 643 insertions(+), 326 deletions(-) create mode 100644 tests/custom/00_syntax/30_call_error_callee diff --git a/compiler.c b/compiler.c index 8688806c9..4b51368f9 100644 --- a/compiler.c +++ b/compiler.c @@ -3757,7 +3757,7 @@ uc_compiler_compile_module_source(uc_compiler_t *compiler, const char *modname, if (!loaded) { /* We do not yet support linking precompiled modules at compile time, turn static import operation into dynamic load one. */ - if (uc_source_type_test(source) == UC_SOURCE_TYPE_PRECOMPILED) + if (uc_source_begin(source) == UC_SOURCE_TYPE_PRECOMPILED) return uc_compiler_compile_dynload(compiler, modname, imports); load_idx = uc_program_function_id(compiler->program, @@ -4317,7 +4317,7 @@ uc_compile(uc_parse_config_t *config, uc_source_t *source, char **errp) if (!config) config = &uc_default_parse_config; - switch (uc_source_type_test(source)) { + switch (uc_source_begin(source)) { case UC_SOURCE_TYPE_PLAIN: prog = uc_compile_from_source(config, source, NULL, errp); break; diff --git a/include/ucode/internal/lexer.h b/include/ucode/internal/lexer.h index d8ffa3a4a..287e69adf 100644 --- a/include/ucode/internal/lexer.h +++ b/include/ucode/internal/lexer.h @@ -174,8 +174,8 @@ typedef struct { size_t count; char *entries; } buffer; - unsigned char *rbuf; - size_t rlen, rpos; + const uint8_t *rbuf; + size_t rlen, rpos, off; } uc_lexer_t; diff --git a/include/ucode/internal/source.h b/include/ucode/internal/source.h index b9a040ca3..8c4fe8a25 100644 --- a/include/ucode/internal/source.h +++ b/include/ucode/internal/source.h @@ -14,8 +14,6 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -/* Internal source API. */ - #ifndef UCODE_INTERNAL_SOURCE_H #define UCODE_INTERNAL_SOURCE_H @@ -25,7 +23,20 @@ __hidden void uc_source_free(uc_source_t *source); -__hidden uc_source_type_t uc_source_type_test(uc_source_t *source); +__hidden const uint8_t *uc_source_read_block(uc_source_t *source, size_t *len, + size_t *off); + +__hidden size_t uc_source_read_line(uc_source_t *source, size_t off, char *line, + size_t size, size_t *lpos, bool *truncated); + +char *uc_source_read_content(uc_source_t *source, size_t *len); + +__hidden size_t uc_source_read(uc_source_t *source, void *buf, size_t len); + +__hidden size_t uc_source_offset(uc_source_t *source); +__hidden size_t uc_source_remaining(uc_source_t *source); + +__hidden uc_source_type_t uc_source_begin(uc_source_t *source); __hidden void uc_source_line_next(uc_source_t *source); __hidden void uc_source_line_update(uc_source_t *source, size_t off); @@ -35,6 +46,6 @@ __hidden void uc_source_runpath_set(uc_source_t *source, const char *runpath); __hidden bool uc_source_export_add(uc_source_t *source, uc_value_t *name); __hidden ssize_t uc_source_export_lookup(uc_source_t *source, uc_value_t *name); -__hidden const char *uc_source_callee_expr(uc_source_t *source, size_t end); +__hidden char *uc_source_callee_expr(uc_source_t *source, size_t end); #endif /* UCODE_INTERNAL_SOURCE_H */ diff --git a/include/ucode/internal/types.h b/include/ucode/internal/types.h index 43ce6781e..cc78f703d 100644 --- a/include/ucode/internal/types.h +++ b/include/ucode/internal/types.h @@ -45,7 +45,9 @@ struct uc_source { uc_value_t header; char *filename, *runpath, *buffer; FILE *fp; - size_t off; + uint8_t *rbuf; + size_t off, size; + bool sized; uc_lineinfo_t lineinfo; struct { size_t count, offset; diff --git a/lexer.c b/lexer.c index 6613144ec..f1ce429fa 100644 --- a/lexer.c +++ b/lexer.c @@ -68,13 +68,18 @@ static const struct keyword reserved_words[] = { static int fill_buf(uc_lexer_t *lex) { - lex->rbuf = xrealloc(lex->rbuf, 128); - lex->rlen = fread(lex->rbuf, 1, 128, lex->source->fp); + size_t off; + + lex->rbuf = uc_source_read_block(lex->source, &lex->rlen, &off); lex->rpos = 0; - if (!lex->rlen) + if (!lex->rbuf) return EOF; + /* everything in front of the block has been consumed, so this merely + * establishes the offset of the first byte of code */ + lex->off = off; + lex->rpos++; return (int)lex->rbuf[0]; @@ -87,7 +92,7 @@ update_line(uc_lexer_t *lex, int ch) { else if (ch != EOF) uc_source_line_update(lex->source, 1); - lex->source->off++; + lex->off++; return ch; } @@ -131,11 +136,11 @@ emit_op(uc_lexer_t *lex, ssize_t pos, int type, uc_value_t *uv) lex->curr.uv = uv; if (pos < 0) - lex->curr.pos = lex->source->off + pos; + lex->curr.pos = lex->off + pos; else lex->curr.pos = (size_t)pos; - lex->curr.end = lex->source->off; + lex->curr.end = lex->off; return &lex->curr; } @@ -166,7 +171,7 @@ emit_buffer(uc_lexer_t *lex, ssize_t pos, int type, const char *strip_trailing_c static uc_token_t * parse_comment(uc_lexer_t *lex, int kind) { - size_t off = lex->source->off - 1; + size_t off = lex->off - 1; int ch; uc_vector_push(&lex->buffer, '/'); @@ -355,7 +360,7 @@ parse_string(uc_lexer_t *lex, int kind) else type = TK_STRING; - off = lex->source->off - 1; + off = lex->off - 1; for (ch = next_char(lex); ch != EOF; ch = next_char(lex)) { switch (ch) { @@ -786,7 +791,7 @@ lex_find_token(uc_lexer_t *lex) case '/': ch = lookahead_char(lex); - lex->lastoff = lex->source->off - 1; + lex->lastoff = lex->off - 1; if (ch == '/' || ch == '*') return parse_comment(lex, ch); @@ -876,7 +881,7 @@ lex_find_token(uc_lexer_t *lex) case '\'': case '"': case '`': - lex->lastoff = lex->source->off - 1; + lex->lastoff = lex->off - 1; return parse_string(lex, ch); @@ -1011,7 +1016,7 @@ lex_step(uc_lexer_t *lex) /* push out leading text */ tok = emit_buffer(lex, lex->lastoff, TK_TEXT, strip); - lex->lastoff = lex->source->off - 2; + lex->lastoff = lex->off - 2; if (!tok) continue; @@ -1045,7 +1050,7 @@ lex_step(uc_lexer_t *lex) tok = emit_op(lex, lex->lastoff, TK_COMMENT, NULL); - lex->lastoff = lex->source->off; + lex->lastoff = lex->off; lex->state = UC_LEX_IDENTIFY_BLOCK; return tok; @@ -1054,13 +1059,13 @@ lex_step(uc_lexer_t *lex) lex->state = UC_LEX_IDENTIFY_TOKEN; lex->block = EXPRESSION; - return emit_op(lex, lex->source->off - 2, TK_LEXP, NULL); + return emit_op(lex, lex->off - 2, TK_LEXP, NULL); case UC_LEX_BLOCK_STATEMENT_EMIT_TAG: lex->state = UC_LEX_IDENTIFY_TOKEN; lex->block = STATEMENTS; - return emit_op(lex, lex->source->off - 2, TK_LSTM, NULL); + return emit_op(lex, lex->off - 2, TK_LSTM, NULL); case UC_LEX_IDENTIFY_TOKEN: do { tok = lex_find_token(lex); } while (tok == NULL); @@ -1075,7 +1080,7 @@ lex_step(uc_lexer_t *lex) if (lex->modifier == UNSPEC && lex->config && lex->config->trim_blocks) lex->modifier = NEWLINE; - lex->lastoff = lex->source->off; + lex->lastoff = lex->off; lex->state = UC_LEX_IDENTIFY_BLOCK; lex->block = NONE; @@ -1084,7 +1089,7 @@ lex_step(uc_lexer_t *lex) /* found end of expression block */ else if (lex->block == EXPRESSION && tok->type == TK_REXP) { - lex->lastoff = lex->source->off; + lex->lastoff = lex->off; lex->state = UC_LEX_IDENTIFY_BLOCK; lex->block = NONE; } @@ -1140,7 +1145,7 @@ lex_step(uc_lexer_t *lex) } } - return emit_op(lex, lex->source->off, TK_EOF, NULL); + return emit_op(lex, lex->off, TK_EOF, NULL); } void @@ -1156,6 +1161,7 @@ uc_lexer_init(uc_lexer_t *lex, uc_parse_config_t *config, uc_source_t *source) lex->rlen = 0; lex->rpos = 0; + lex->off = 0; lex->rbuf = NULL; lex->buffer.count = 0; @@ -1181,8 +1187,6 @@ uc_lexer_free(uc_lexer_t *lex) uc_vector_clear(&lex->templates); uc_source_put(lex->source); - - free(lex->rbuf); } uc_token_t * diff --git a/lib.c b/lib.c index 45d642193..0484ca5f0 100644 --- a/lib.c +++ b/lib.c @@ -162,60 +162,40 @@ static bool format_context_range(uc_stringbuf_t *buf, uc_source_t *src, size_t off, size_t startoff, size_t endoff, bool compact) { - size_t len, rlen, lpos, eoff, sbegin, send, shift; + size_t len, lpos, eoff, sbegin, send, shift; bool truncated; char line[256]; - long srcpos; int eline; - srcpos = ftell(src->fp); + eline = uc_source_read_line(src, off, line, sizeof(line), &lpos, &truncated); - if (srcpos == -1) + if (!eline) return false; - fseek(src->fp, 0, SEEK_SET); - - truncated = false; - eline = 1; - rlen = 0; + if (compact) + ucv_stringbuf_printf(buf, "\033[2;40;97m%17s %s", + source_filename(src, eline), + truncated ? "..." : ""); + else + ucv_stringbuf_printf(buf, "\n `%s", + truncated ? "..." : ""); - while (fgets(line, sizeof(line), src->fp)) { - len = strlen(line); - rlen += len; + /* the index of the failing byte within the line and the columns gained by + * prefixing an overlong line with "..." */ + eoff = off - lpos; + shift = truncated ? 3 : 0; - if (rlen >= off) { - if (compact) - ucv_stringbuf_printf(buf, "\033[2;40;97m%17s %s", - source_filename(src, eline), - truncated ? "..." : ""); - else - ucv_stringbuf_printf(buf, "\n `%s", - truncated ? "..." : ""); - - /* the source offset this line starts at, the index of the failing - * byte within it and the columns gained by prefixing an overlong - * line with "..." */ - lpos = rlen - len; - eoff = off - lpos; - shift = truncated ? 3 : 0; - - sbegin = send = eoff; - - /* clip the statement's extent to the part of it on this line */ - if (endoff > startoff && endoff > lpos && startoff < lpos + len) { - sbegin = startoff > lpos ? startoff - lpos : 0; - send = endoff < lpos + len ? endoff - lpos : len; - } + sbegin = send = eoff; - format_context_line(buf, line, eoff + shift, sbegin + shift, send + shift, compact); - break; - } + /* clip the statement's extent to the part of it on this line */ + len = strlen(line); - truncated = (len > 0 && line[len-1] != '\n'); - eline += !truncated; + if (endoff > startoff && endoff > lpos && startoff < lpos + len) { + sbegin = startoff > lpos ? startoff - lpos : 0; + send = endoff < lpos + len ? endoff - lpos : len; } - fseek(src->fp, srcpos, SEEK_SET); + format_context_line(buf, line, eoff + shift, sbegin + shift, send + shift, compact); return true; } diff --git a/lib/debug.c b/lib/debug.c index 74e2c58a7..d7d1a7272 100644 --- a/lib/debug.c +++ b/lib/debug.c @@ -83,6 +83,7 @@ extern bool debug_remote_has_active_connection(void); #include "ucode/module.h" #include "ucode/internal/lib.h" #include "ucode/internal/program.h" +#include "ucode/internal/source.h" #include "ucode/internal/chunk.h" #include "ucode/internal/vm.h" #include "ucode/internal/platform.h" @@ -4955,6 +4956,8 @@ proto_cmd_source(uc_vm_t *vm, debug_breakpoint_t *dbk, uc_value_t *payload, int uc_value_t *filev = ucv_object_get(payload, "file", NULL); location_t loc = { 0 }; uc_value_t *obj; + char *text = NULL; + size_t len = 0; if (ucv_type(filev) != UC_STRING) { send_error(fd, vm, "Usage: SOURCE {\"file\":\"...\"}"); @@ -4968,22 +4971,16 @@ proto_cmd_source(uc_vm_t *vm, debug_breakpoint_t *dbk, uc_value_t *payload, int obj = ucv_object_new(vm); ucv_object_add(obj, "file", ucv_get(filev)); - if (!lookup_source(vm, &loc)) { + if (lookup_source(vm, &loc)) + text = uc_source_read_content(loc.source, &len); + + if (!text) { ucv_object_add(obj, "text", NULL); ucv_object_add(obj, "error", ucv_string_new("source not available on server")); } else { - uc_stringbuf_t text = { 0 }; - char buf[4096]; - size_t n; - - fseeko(loc.source->fp, 0, SEEK_SET); - - while ((n = fread(buf, 1, sizeof(buf), loc.source->fp)) > 0) - printbuf_memappend_fast((&text), buf, (int)n); - - ucv_object_add(obj, "text", ucv_string_new_length(text.buf, text.bpos)); - free(text.buf); + ucv_object_add(obj, "text", ucv_string_new_length(text, len)); + free(text); } debug_proto_write(fd, vm, "SOURCE", obj); diff --git a/program.c b/program.c index b30fc21f7..c762e48b7 100644 --- a/program.c +++ b/program.c @@ -445,48 +445,48 @@ uc_program_write(uc_program_t *prog, FILE *file, bool debug) } static bool -read_error(FILE *file, char **errp, const char *subject, size_t rlen, size_t len) +read_error(uc_source_t *in, char **errp, const char *subject, size_t rlen, size_t len) { const char *reason; - if (feof(file)) + if (!errno) reason = "Premature EOF"; else reason = strerror(errno); if (errp) xasprintf(errp, - "%s while reading %s at offset %ld, got %zu of %zu bytes\n", - reason, subject, ftell(file) - rlen, rlen, len); + "%s while reading %s at offset %zu, got %zu of %zu bytes\n", + reason, subject, uc_source_offset(in) - rlen, rlen, len); return false; } static bool -skip_padding(FILE *file, size_t len, char **errp) +skip_padding(uc_source_t *in, size_t len, char **errp) { size_t pad = (~len + 1) & (sizeof(uint32_t) - 1), rlen; char buf[sizeof(uint32_t) - 1]; if (pad != 0) { - rlen = fread(buf, 1, pad, file); + rlen = uc_source_read(in, buf, pad); if (rlen != pad) - return read_error(file, errp, "padding", rlen, pad); + return read_error(in, errp, "padding", rlen, pad); } return true; } static bool -read_u32(FILE *file, uint32_t *n, const char *subj, char **errp) +read_u32(uc_source_t *in, uint32_t *n, const char *subj, char **errp) { - size_t rlen = fread(n, 1, sizeof(*n), file); + size_t rlen = uc_source_read(in, n, sizeof(*n)); if (rlen != sizeof(*n)) { *n = 0; - return read_error(file, errp, subj ? subj : "uint32_t", rlen, sizeof(*n)); + return read_error(in, errp, subj ? subj : "uint32_t", rlen, sizeof(*n)); } *n = be32toh(*n); @@ -495,14 +495,14 @@ read_u32(FILE *file, uint32_t *n, const char *subj, char **errp) } static bool -read_u64(FILE *file, uint64_t *n, const char *subj, char **errp) +read_u64(uc_source_t *in, uint64_t *n, const char *subj, char **errp) { - size_t rlen = fread(n, 1, sizeof(*n), file); + size_t rlen = uc_source_read(in, n, sizeof(*n)); if (rlen != sizeof(*n)) { *n = 0; - return read_error(file, errp, subj ? subj : "uint64_t", rlen, sizeof(*n)); + return read_error(in, errp, subj ? subj : "uint64_t", rlen, sizeof(*n)); } *n = be64toh(*n); @@ -511,7 +511,7 @@ read_u64(FILE *file, uint64_t *n, const char *subj, char **errp) } static bool -read_size_t(FILE *file, size_t *n, size_t size, const char *subj, char **errp); +read_size_t(uc_source_t *in, size_t *n, size_t size, const char *subj, char **errp); /* A count or length taken from the file sizes an allocation, and every item * it counts is then read from that same file, so a value larger than the @@ -519,24 +519,12 @@ read_size_t(FILE *file, size_t *n, size_t size, const char *subj, char **errp); * field from reaching xalloc(), which aborts the process rather than * returning. */ static bool -length_fits(FILE *file, size_t n, const char *subj, char **errp) +length_fits(uc_source_t *in, size_t n, const char *subj, char **errp) { - long pos, end; - - pos = ftell(file); - - if (pos < 0 || fseek(file, 0, SEEK_END) != 0) - return true; - - end = ftell(file); - - if (fseek(file, pos, SEEK_SET) != 0) - return false; - - if (end >= pos && n > (size_t)(end - pos)) { + if (n > uc_source_remaining(in)) { if (errp) - xasprintf(errp, "Invalid length %zu for %s at offset %ld\n", - n, subj, pos); + xasprintf(errp, "Invalid length %zu for %s at offset %zu\n", + n, subj, uc_source_offset(in)); return false; } @@ -545,27 +533,15 @@ length_fits(FILE *file, size_t n, const char *subj, char **errp) } static bool -read_count(FILE *file, size_t *n, size_t itemsize, const char *subj, char **errp) +read_count(uc_source_t *in, size_t *n, size_t itemsize, const char *subj, char **errp) { - long pos, end; - - if (!read_size_t(file, n, sizeof(uint32_t), subj, errp)) - return false; - - pos = ftell(file); - - if (pos < 0 || fseek(file, 0, SEEK_END) != 0) - return true; - - end = ftell(file); - - if (fseek(file, pos, SEEK_SET) != 0) + if (!read_size_t(in, n, sizeof(uint32_t), subj, errp)) return false; - if (end >= pos && *n > (size_t)(end - pos) / (itemsize ? itemsize : 1)) { + if (*n > uc_source_remaining(in) / (itemsize ? itemsize : 1)) { if (errp) - xasprintf(errp, "Invalid count %zu for %s at offset %ld\n", - *n, subj, pos); + xasprintf(errp, "Invalid count %zu for %s at offset %zu\n", + *n, subj, uc_source_offset(in)); return false; } @@ -573,15 +549,15 @@ read_count(FILE *file, size_t *n, size_t itemsize, const char *subj, char **errp return true; } -#define read_length(file, n, subj, errp) read_count(file, n, 1, subj, errp) +#define read_length(in, n, subj, errp) read_count(in, n, 1, subj, errp) static bool -read_size_t(FILE *file, size_t *n, size_t size, const char *subj, char **errp) +read_size_t(uc_source_t *in, size_t *n, size_t size, const char *subj, char **errp) { union { uint8_t u8; uint16_t u16; uint32_t u32; uint64_t u64; } nval; size_t rlen; - rlen = fread(&nval.u64, 1, size, file); + rlen = uc_source_read(in, &nval.u64, size); if (rlen != size) { *n = 0; @@ -595,7 +571,7 @@ read_size_t(FILE *file, size_t *n, size_t size, const char *subj, char **errp) } } - return read_error(file, errp, subj, rlen, sizeof(nval)); + return read_error(in, errp, subj, rlen, sizeof(nval)); } switch (size) { @@ -609,7 +585,7 @@ read_size_t(FILE *file, size_t *n, size_t size, const char *subj, char **errp) } static bool -_read_vector(FILE *file, void *ptr, size_t itemsize, const char *subj, char **errp) +_read_vector(uc_source_t *in, void *ptr, size_t itemsize, const char *subj, char **errp) { struct { size_t count; void *data; } *vec = ptr; size_t rlen, len; @@ -617,15 +593,15 @@ _read_vector(FILE *file, void *ptr, size_t itemsize, const char *subj, char **er snprintf(subjbuf, sizeof(subjbuf), "%s vector size", subj); - if (!read_count(file, &vec->count, itemsize, subjbuf, errp)) + if (!read_count(in, &vec->count, itemsize, subjbuf, errp)) return false; vec->data = xcalloc(vec->count, itemsize); len = vec->count; - rlen = fread(vec->data, itemsize, len, file); + rlen = uc_source_read(in, vec->data, len * itemsize); - if (rlen != len) { + if (rlen != len * itemsize) { free(vec->data); vec->count = 0; @@ -633,30 +609,30 @@ _read_vector(FILE *file, void *ptr, size_t itemsize, const char *subj, char **er snprintf(subjbuf, sizeof(subjbuf), "%s vector data", subj); - return read_error(file, errp, subjbuf, rlen * itemsize, len * itemsize); + return read_error(in, errp, subjbuf, rlen, len * itemsize); } - return skip_padding(file, vec->count * itemsize, errp); + return skip_padding(in, vec->count * itemsize, errp); } -#define read_vector(file, vec, subj, errp) \ - _read_vector(file, vec, sizeof((vec)->entries[0]), subj, errp) +#define read_vector(in, vec, subj, errp) \ + _read_vector(in, vec, sizeof((vec)->entries[0]), subj, errp) static bool -read_string(FILE *file, char *dst, size_t len, const char *subj, char **errp) +read_string(uc_source_t *in, char *dst, size_t len, const char *subj, char **errp) { size_t rlen; - rlen = fread(dst, 1, len, file); + rlen = uc_source_read(in, dst, len); if (rlen != len) - return read_error(file, errp, subj, rlen, len); + return read_error(in, errp, subj, rlen, len); - return skip_padding(file, len, errp); + return skip_padding(in, len, errp); } static bool -read_vallist(FILE *file, uc_value_list_t *vallist, const char *subj, char **errp) +read_vallist(uc_source_t *in, uc_value_list_t *vallist, const char *subj, char **errp) { char subjbuf[64]; size_t i; @@ -664,7 +640,7 @@ read_vallist(FILE *file, uc_value_list_t *vallist, const char *subj, char **errp /* read index */ snprintf(subjbuf, sizeof(subjbuf), "%s index size", subj); - if (!read_count(file, &vallist->isize, sizeof(uint64_t), subjbuf, errp)) + if (!read_count(in, &vallist->isize, sizeof(uint64_t), subjbuf, errp)) goto out; vallist->index = xcalloc(sizeof(vallist->index[0]), vallist->isize); @@ -672,21 +648,21 @@ read_vallist(FILE *file, uc_value_list_t *vallist, const char *subj, char **errp for (i = 0; i < vallist->isize; i++) { snprintf(subjbuf, sizeof(subjbuf), "%s index entry %zu of %zu", subj, i, vallist->isize); - if (!read_u64(file, &vallist->index[i], subjbuf, errp)) + if (!read_u64(in, &vallist->index[i], subjbuf, errp)) goto out; } /* read data */ snprintf(subjbuf, sizeof(subjbuf), "%s data size", subj); - if (!read_length(file, &vallist->dsize, subjbuf, errp)) + if (!read_length(in, &vallist->dsize, subjbuf, errp)) goto out; vallist->data = xalloc(vallist->dsize); snprintf(subjbuf, sizeof(subjbuf), "%s data", subj); - if (!read_string(file, vallist->data, vallist->dsize, subj, errp)) + if (!read_string(in, vallist->data, vallist->dsize, subj, errp)) goto out; return true; @@ -705,7 +681,7 @@ read_vallist(FILE *file, uc_value_list_t *vallist, const char *subj, char **errp } static bool -read_exports(FILE *file, uc_source_t *source, uint32_t flags, const char *subj, char **errp) +read_exports(uc_source_t *in, uc_source_t *source, uint32_t flags, const char *subj, char **errp) { size_t i, num_exports, len; uc_stringbuf_t *buf; @@ -717,7 +693,7 @@ read_exports(FILE *file, uc_source_t *source, uint32_t flags, const char *subj, /* read export count */ snprintf(subjbuf, sizeof(subjbuf), "%s count", subj); - if (!read_count(file, &num_exports, sizeof(uint32_t), subjbuf, errp)) + if (!read_count(in, &num_exports, sizeof(uint32_t), subjbuf, errp)) return false; /* read export symbol names */ @@ -726,14 +702,14 @@ read_exports(FILE *file, uc_source_t *source, uint32_t flags, const char *subj, snprintf(subjbuf, sizeof(subjbuf), "%s entry %zu of %zu size", subj, i, num_exports); - if (!read_size_t(file, &len, sizeof(uint32_t), subjbuf, errp)) + if (!read_size_t(in, &len, sizeof(uint32_t), subjbuf, errp)) return false; if (len < 0xffffffff) { snprintf(subjbuf, sizeof(subjbuf), "%s entry %zu of %zu name", subj, i, num_exports); - if (!length_fits(file, len, subjbuf, errp)) + if (!length_fits(in, len, subjbuf, errp)) return false; buf = ucv_stringbuf_new(); @@ -747,7 +723,7 @@ read_exports(FILE *file, uc_source_t *source, uint32_t flags, const char *subj, return false; } - if (!read_string(file, buf->buf + buf->bpos - len - 1, len, subjbuf, errp)) + if (!read_string(in, buf->buf + buf->bpos - len - 1, len, subjbuf, errp)) { printbuf_free(buf); return false; @@ -775,23 +751,23 @@ read_sourceinfo(uc_source_t *input, uint32_t flags, char **errp, uc_program_t *p size_t len, count; if (flags & UC_PROGRAM_F_SOURCEINFO) { - if (!read_count(input->fp, &count, sizeof(uint32_t) * 2, + if (!read_count(input, &count, sizeof(uint32_t) * 2, "amount of source entries", errp)) return NULL; while (count > 0) { - if (!read_length(input->fp, &len, "sourceinfo filename length", errp)) + if (!read_length(input, &len, "sourceinfo filename length", errp)) return NULL; path = xalloc(len + 1); - if (!read_string(input->fp, path, len, "sourceinfo filename", errp)) { + if (!read_string(input, path, len, "sourceinfo filename", errp)) { free(path); return NULL; } - if (!read_length(input->fp, &len, "sourceinfo code buffer length", errp)) { + if (!read_length(input, &len, "sourceinfo code buffer length", errp)) { free(path); return NULL; @@ -800,7 +776,7 @@ read_sourceinfo(uc_source_t *input, uint32_t flags, char **errp, uc_program_t *p if (len > 0) { code = xalloc(len); - if (!read_string(input->fp, code, len, "sourceinfo code buffer data", errp)) { + if (!read_string(input, code, len, "sourceinfo code buffer data", errp)) { free(code); free(path); @@ -818,7 +794,7 @@ read_sourceinfo(uc_source_t *input, uint32_t flags, char **errp, uc_program_t *p } } - if (!read_vector(input->fp, &source->lineinfo, "sourceinfo lineinfo", errp)) { + if (!read_vector(input, &source->lineinfo, "sourceinfo lineinfo", errp)) { uc_source_put(source); free(path); @@ -844,7 +820,7 @@ read_sourceinfo(uc_source_t *input, uint32_t flags, char **errp, uc_program_t *p } static bool -read_chunk(FILE *file, uc_chunk_t *chunk, uint32_t flags, const char *subj, char **errp) +read_chunk(uc_source_t *in, uc_chunk_t *chunk, uint32_t flags, const char *subj, char **errp) { uc_varrange_t *varrange; uc_ehrange_t *ehrange; @@ -854,14 +830,14 @@ read_chunk(FILE *file, uc_chunk_t *chunk, uint32_t flags, const char *subj, char /* read bytecode data */ snprintf(subjbuf, sizeof(subjbuf), "%s byte code", subj); - if (!read_vector(file, chunk, subjbuf, errp)) + if (!read_vector(in, chunk, subjbuf, errp)) goto out; /* read exception ranges */ if (flags & UC_FUNCTION_F_HAS_EXCEPTIONS) { snprintf(subjbuf, sizeof(subjbuf), "%s exception ranges count", subj); - if (!read_count(file, &chunk->ehranges.count, + if (!read_count(in, &chunk->ehranges.count, sizeof(uint32_t) * 4, subjbuf, errp)) goto out; @@ -875,10 +851,10 @@ read_chunk(FILE *file, uc_chunk_t *chunk, uint32_t flags, const char *subj, char ehrange = &chunk->ehranges.entries[i]; - if (!read_size_t(file, &ehrange->from, sizeof(uint32_t), subjbuf, errp) || - !read_size_t(file, &ehrange->to, sizeof(uint32_t), subjbuf, errp) || - !read_size_t(file, &ehrange->target, sizeof(uint32_t), subjbuf, errp) || - !read_size_t(file, &ehrange->slot, sizeof(uint32_t), subjbuf, errp)) + if (!read_size_t(in, &ehrange->from, sizeof(uint32_t), subjbuf, errp) || + !read_size_t(in, &ehrange->to, sizeof(uint32_t), subjbuf, errp) || + !read_size_t(in, &ehrange->target, sizeof(uint32_t), subjbuf, errp) || + !read_size_t(in, &ehrange->slot, sizeof(uint32_t), subjbuf, errp)) goto out; } } @@ -887,7 +863,7 @@ read_chunk(FILE *file, uc_chunk_t *chunk, uint32_t flags, const char *subj, char if (flags & UC_FUNCTION_F_HAS_VARDBG) { snprintf(subjbuf, sizeof(subjbuf), "%s variable scopes count", subj); - if (!read_count(file, &chunk->debuginfo.variables.count, + if (!read_count(in, &chunk->debuginfo.variables.count, sizeof(uint32_t) * 4, subjbuf, errp)) goto out; @@ -901,10 +877,10 @@ read_chunk(FILE *file, uc_chunk_t *chunk, uint32_t flags, const char *subj, char varrange = &chunk->debuginfo.variables.entries[i]; - if (!read_size_t(file, &varrange->from, sizeof(uint32_t), subjbuf, errp) || - !read_size_t(file, &varrange->to, sizeof(uint32_t), subjbuf, errp) || - !read_size_t(file, &varrange->slot, sizeof(uint32_t), subjbuf, errp) || - !read_size_t(file, &varrange->nameidx, sizeof(uint32_t), subjbuf, errp)) + if (!read_size_t(in, &varrange->from, sizeof(uint32_t), subjbuf, errp) || + !read_size_t(in, &varrange->to, sizeof(uint32_t), subjbuf, errp) || + !read_size_t(in, &varrange->slot, sizeof(uint32_t), subjbuf, errp) || + !read_size_t(in, &varrange->nameidx, sizeof(uint32_t), subjbuf, errp)) goto out; if (varrange->slot >= ((uint32_t)-1 / 2)) @@ -913,7 +889,7 @@ read_chunk(FILE *file, uc_chunk_t *chunk, uint32_t flags, const char *subj, char snprintf(subjbuf, sizeof(subjbuf), "%s variable names", subj); - if (!read_vallist(file, &chunk->debuginfo.varnames, subjbuf, errp)) + if (!read_vallist(in, &chunk->debuginfo.varnames, subjbuf, errp)) goto out; } @@ -921,7 +897,7 @@ read_chunk(FILE *file, uc_chunk_t *chunk, uint32_t flags, const char *subj, char if (flags & UC_FUNCTION_F_HAS_OFFSETDBG) { snprintf(subjbuf, sizeof(subjbuf), "%s source offsets", subj); - if (!read_vector(file, &chunk->debuginfo.offsets, subjbuf, errp)) + if (!read_vector(in, &chunk->debuginfo.offsets, subjbuf, errp)) goto out; } @@ -947,7 +923,7 @@ read_chunk(FILE *file, uc_chunk_t *chunk, uint32_t flags, const char *subj, char } static bool -read_function(FILE *file, uc_program_t *program, size_t idx, char **errp) +read_function(uc_source_t *in, uc_program_t *program, size_t idx, char **errp) { size_t nargs, nupvals, srcidx, srcpos, namelen = 0; char subjbuf[64], *name = NULL; @@ -957,29 +933,29 @@ read_function(FILE *file, uc_program_t *program, size_t idx, char **errp) snprintf(subjbuf, sizeof(subjbuf), "function #%zu flags", idx); - if (!read_u32(file, &flags, subjbuf, errp)) + if (!read_u32(in, &flags, subjbuf, errp)) goto out; if (flags & UC_FUNCTION_F_HAS_NAME) { snprintf(subjbuf, sizeof(subjbuf), "function #%zu name length", idx); - if (!read_length(file, &namelen, subjbuf, errp)) + if (!read_length(in, &namelen, subjbuf, errp)) goto out; name = xalloc(namelen + 1); snprintf(subjbuf, sizeof(subjbuf), "function #%zu name", idx); - if (!read_string(file, name, namelen, subjbuf, errp)) + if (!read_string(in, name, namelen, subjbuf, errp)) goto out; } snprintf(subjbuf, sizeof(subjbuf), "function #%zu (%s) arg count and offset", idx, name ? name : "-"); - if (!read_size_t(file, &nargs, sizeof(uint16_t), subjbuf, errp) || - !read_size_t(file, &nupvals, sizeof(uint16_t), subjbuf, errp) || - !read_size_t(file, &srcidx, sizeof(uint32_t), subjbuf, errp) || - !read_size_t(file, &srcpos, sizeof(uint32_t), subjbuf, errp)) { + if (!read_size_t(in, &nargs, sizeof(uint16_t), subjbuf, errp) || + !read_size_t(in, &nupvals, sizeof(uint16_t), subjbuf, errp) || + !read_size_t(in, &srcidx, sizeof(uint32_t), subjbuf, errp) || + !read_size_t(in, &srcpos, sizeof(uint32_t), subjbuf, errp)) { goto out; } @@ -999,7 +975,7 @@ read_function(FILE *file, uc_program_t *program, size_t idx, char **errp) snprintf(subjbuf, sizeof(subjbuf), "function #%zu (%s) body", idx, name ? name : "-"); - if (!read_chunk(file, &func->chunk, flags, subjbuf, errp)) + if (!read_chunk(in, &func->chunk, flags, subjbuf, errp)) goto out; free(name); @@ -1019,7 +995,7 @@ uc_program_load(uc_source_t *input, char **errp) uint32_t flags, i; size_t nfuncs; - if (!read_u32(input->fp, &i, "file magic", errp)) + if (!read_u32(input, &i, "file magic", errp)) goto out; if (i != UC_PRECOMPILED_BYTECODE_MAGIC) { @@ -1027,7 +1003,7 @@ uc_program_load(uc_source_t *input, char **errp) goto out; } - if (!read_u32(input->fp, &flags, "program flags", errp)) + if (!read_u32(input, &flags, "program flags", errp)) goto out; if ((flags >> 24) != UCODE_BYTECODE_VERSION) { @@ -1042,18 +1018,18 @@ uc_program_load(uc_source_t *input, char **errp) if (!read_sourceinfo(input, flags, errp, program)) goto out; - if (!read_vallist(input->fp, &program->constants, "constants", errp)) + if (!read_vallist(input, &program->constants, "constants", errp)) goto out; - if (!read_exports(input->fp, program->sources.entries[0], flags, + if (!read_exports(input, program->sources.entries[0], flags, "exports", errp)) goto out; - if (!read_count(input->fp, &nfuncs, sizeof(uint32_t), "function count", errp)) + if (!read_count(input, &nfuncs, sizeof(uint32_t), "function count", errp)) goto out; for (i = 0; i < nfuncs; i++) - if (!read_function(input->fp, program, i, errp)) + if (!read_function(input, program, i, errp)) goto out; return program; diff --git a/source.c b/source.c index fed56447b..f31826593 100644 --- a/source.c +++ b/source.c @@ -66,6 +66,157 @@ uc_source_new_buffer(const char *name, char *buf, size_t len) return src; } +/* Number of source bytes lent to the lexer per block read. */ +#define UC_SOURCE_BLOCKSIZE 128 + +const uint8_t * +uc_source_read_block(uc_source_t *source, size_t *len, size_t *off) +{ + size_t rlen; + + if (!source->rbuf) + source->rbuf = xalloc(UC_SOURCE_BLOCKSIZE); + + rlen = fread(source->rbuf, 1, UC_SOURCE_BLOCKSIZE, source->fp); + + if (off) + *off = source->off; + + source->off += rlen; + + *len = rlen; + + return rlen ? source->rbuf : NULL; +} + +size_t +uc_source_read(uc_source_t *source, void *buf, size_t len) +{ + size_t rlen; + + errno = 0; + + rlen = fread(buf, 1, len, source->fp); + source->off += rlen; + + return rlen; +} + +size_t +uc_source_offset(uc_source_t *source) +{ + return source->off; +} + +size_t +uc_source_remaining(uc_source_t *source) +{ + long pos, end; + + if (!source->sized) { + /* the size of text that cannot seek is not known, treat it as + * unbounded */ + source->size = SIZE_MAX; + source->sized = true; + + pos = ftell(source->fp); + + if (pos >= 0 && fseek(source->fp, 0, SEEK_END) == 0) { + end = ftell(source->fp); + + if (end >= 0) + source->size = end; + + fseek(source->fp, pos, SEEK_SET); + } + } + + return (source->size > source->off) ? source->size - source->off : 0; +} + +/* Random access to the text of a source for error messages and the debugger, + * leaving the read position as found. */ +static FILE * +uc_source_text_open(uc_source_t *source, long *pos) +{ + *pos = ftell(source->fp); + + return (*pos != -1) ? source->fp : NULL; +} + +static void +uc_source_text_close(uc_source_t *source, FILE *fp, long pos) +{ + fseek(fp, pos, SEEK_SET); +} + +size_t +uc_source_read_line(uc_source_t *source, size_t off, char *line, size_t size, + size_t *lpos, bool *truncated) +{ + size_t len, rlen = 0, eline = 1; + bool cont = false; + long pos; + FILE *fp; + + *truncated = false; + + if (size < 2 || !(fp = uc_source_text_open(source, &pos))) + return 0; + + fseek(fp, 0, SEEK_SET); + + while (fgets(line, size, fp)) { + len = strlen(line); + rlen += len; + + if (rlen >= off) { + *truncated = cont; + *lpos = rlen - len; + + uc_source_text_close(source, fp, pos); + + return eline; + } + + /* a chunk without a newline is part of a line longer than the buffer, + * its remainder is delivered by the next one and does not advance the + * line count */ + cont = (len > 0 && line[len - 1] != '\n'); + eline += !cont; + } + + uc_source_text_close(source, fp, pos); + + return 0; +} + +char * +uc_source_read_content(uc_source_t *source, size_t *len) +{ + size_t rlen, tlen = 0; + char *text = NULL; + long pos; + FILE *fp; + + if (!(fp = uc_source_text_open(source, &pos))) + return NULL; + + fseek(fp, 0, SEEK_SET); + + do { + text = xrealloc(text, tlen + 4096); + rlen = fread(text + tlen, 1, 4096, fp); + tlen += rlen; + } while (rlen == 4096); + + uc_source_text_close(source, fp, pos); + + *len = tlen; + + return text; +} + uc_source_t * uc_source_get(uc_source_t *source) { @@ -125,11 +276,12 @@ uc_source_free(uc_source_t *source) uc_source_state_clear(source); fclose(source->fp); + free(source->rbuf); free(source->buffer); } uc_source_type_t -uc_source_type_test(uc_source_t *source) +uc_source_begin(uc_source_t *source) { union { char s[sizeof(uint32_t)]; uint32_t n; } buf = { 0 }; uc_source_type_t type = UC_SOURCE_TYPE_PLAIN; @@ -257,20 +409,29 @@ uc_source_export_lookup(uc_source_t *source, uc_value_t *name) * contain the name actually being invoked, are shown. */ #define UC_EXPR_MAXLEN 64 -/* The number of characters which may still be read while recovering an - * expression. Since only the source of the arguments passed to the failing call - * can be arbitrarily wide, this bounds the effort spent on an error message. */ -#define UC_EXPR_MAXREAD 4096 +/* The number of source bytes, ending with the closing parenthesis of the call, + * which are looked at while recovering an expression. Only the source of the + * arguments passed to the failing call can be arbitrarily wide, so the window + * starts out small and is doubled while the scan runs into its start, up to a + * limit which bounds the effort spent on an error message. */ +#define UC_EXPR_WINDOW_MIN 256 +#define UC_EXPR_WINDOW_MAX 4096 -/* The text of an invoked expression is recovered into a buffer which is filled - * from its end towards its beginning, so that the characters end up in the - * order they have in the source without a second pass. */ -static char uc_expr_text[UC_EXPR_MAXLEN + 1]; -static size_t uc_expr_pos, uc_expr_read; +/* The state of one recovery, which lives on the stack of expr_recover(). + * + * The window of source text being scanned is `len` bytes of the allocated + * buffer `win` starting at the source offset `base`. `cut` notes that the scan + * wanted to look in front of a window which does not start at the beginning of + * the file. */ +typedef struct { + char *win; + size_t base, len; + bool cut; +} expr_t; /* Returns true if `ch` closes a string or template literal. */ static bool -uc_expr_quote(int ch) +expr_quote(int ch) { return ch == '\'' || ch == '"' || ch == '`'; } @@ -278,7 +439,7 @@ uc_expr_quote(int ch) /* Returns true if `ch` is a member access operator, all of which may be * surrounded by whitespace without ceasing to belong to the same expression. */ static bool -uc_expr_member_char(int ch) +expr_member_char(int ch) { return ch == '.' || ch == '[' || ch == '?'; } @@ -289,66 +450,54 @@ uc_expr_member_char(int ch) * lexer, plus the member access operators. Notably `:` is absent, so that * `foo ? bar : baz()` identifies `baz`. */ static bool -uc_expr_spine_char(int ch) +expr_spine_char(int ch) { return isalnum(ch) || ch == '_' || ch == '.' || ch == '?'; } -/* Reads the character at `off`, returning EOF once the read budget is used up. - * The caller keeps track of the position, so that it stays in a register. */ +/* Returns the character at source offset `off`, or EOF if it is not within the + * window. */ static int -uc_expr_at(FILE *fp, size_t off) +expr_at(expr_t *x, size_t off) { - if (uc_expr_read == 0) - return EOF; - - uc_expr_read--; + if (off < x->base) { + x->cut |= (x->base > 0); - fseek(fp, (long)off, SEEK_SET); + return EOF; + } - return fgetc(fp); -} + if (off - x->base >= x->len) + return EOF; -/* Puts `ch` in front of the characters collected so far. Characters which no - * longer fit are dropped, since only the tail of an over-long expression is of - * interest. */ -static inline void -uc_expr_collect(int ch) -{ - if (uc_expr_pos > 0) - uc_expr_text[--uc_expr_pos] = ch; + return (unsigned char)x->win[off - x->base]; } /* Returns true if the literal quote at `off` is the opening one, which it is * when it is preceded by an even number of backslashes, since each of those * escapes the next character. */ static bool -uc_expr_quote_open(FILE *fp, size_t off) +expr_quote_open(expr_t *x, size_t off) { size_t n = 0; - while (off-- > 0 && uc_expr_at(fp, off) == '\\') + while (off-- > 0 && expr_at(x, off) == '\\') n++; return n % 2 == 0; } /* Returns the offset of the quote opening the literal which closes at `off`, or - * SIZE_MAX if there is none. The characters passed over are collected along the - * way; they are of no interest to the caller only in the latter case, in which - * the scan is abandoned altogether. */ + * SIZE_MAX if there is none. */ static size_t -uc_expr_back_literal(FILE *fp, int quote, size_t off) +expr_back_literal(expr_t *x, int quote, size_t off) { int ch; while (off-- > 0) { - if ((ch = uc_expr_at(fp, off)) == EOF) + if ((ch = expr_at(x, off)) == EOF) return SIZE_MAX; - uc_expr_collect(ch); - - if (ch == quote && uc_expr_quote_open(fp, off)) + if (ch == quote && expr_quote_open(x, off)) return off; } @@ -360,21 +509,19 @@ uc_expr_back_literal(FILE *fp, int quote, size_t off) * source. Literal values are skipped over as a whole, so that a bracket inside * one of them cannot be mistaken for the opening delimiter. */ static size_t -uc_expr_back_pair(FILE *fp, size_t off, int open) +expr_back_pair(expr_t *x, size_t off, int open) { size_t depth = 0, beg; int ch; while (off-- > 0) { - ch = uc_expr_at(fp, off); + ch = expr_at(x, off); if (ch == EOF) return SIZE_MAX; - uc_expr_collect(ch); - - if (uc_expr_quote(ch)) { - if ((beg = uc_expr_back_literal(fp, ch, off)) == SIZE_MAX) + if (expr_quote(ch)) { + if ((beg = expr_back_literal(x, ch, off)) == SIZE_MAX) return SIZE_MAX; off = beg; @@ -399,131 +546,242 @@ uc_expr_back_pair(FILE *fp, size_t off, int open) return SIZE_MAX; } -/* Extracts the expression invoked by the call whose closing parenthesis is at - * source offset `end`, for use in error messages about a failed call. +/* Returns the offset of the first byte of the expression invoked by the call + * whose closing parenthesis is at `end`, or SIZE_MAX if it cannot be + * determined; `*open` is set to the offset of the parenthesis opening the + * arguments, which is where the expression ends. * * The source in front of `end` is scanned backwards over the balanced pair - * ending there, which delimits the arguments passed to the failing call and - * whose opening parenthesis therefore marks the end of the invoked expression. - * The scan then continues over the spine of that expression, taking subscripts, - * nested calls and literal values as a whole so that their contents cannot be - * mistaken for part of the surrounding expression, and dropping whitespace - * which does not surround a member access operator, since such whitespace - * separates the expression from whatever precedes it. - * - * Returns the recovered expression, which stays valid until the next call, or - * NULL if it could not be determined. */ -__hidden const char * -uc_source_callee_expr(uc_source_t *source, size_t end) + * ending there, which delimits the arguments passed to the failing call. The + * scan then continues over the spine of the invoked expression, taking + * subscripts, nested calls and literal values as a whole so that their contents + * cannot be mistaken for part of the surrounding expression, and passing over + * whitespace which surrounds a member access operator, while any other + * whitespace separates the expression from whatever precedes it. */ +static size_t +expr_start(expr_t *x, size_t end, size_t *open) { - const char *res = NULL; - size_t i, beg, pos; + size_t i, beg, start; int last = 0; - long save; - FILE *fp; - - if (!source || !source->fp || end == 0) - return NULL; - - fp = source->fp; - save = ftell(fp); - - uc_expr_pos = UC_EXPR_MAXLEN; - uc_expr_read = UC_EXPR_MAXREAD; /* the instruction of a failing call refers to the closing parenthesis of * the arguments passed to it */ - if (uc_expr_at(fp, end) != ')') - goto out; - - /* the arguments themselves are of no interest, so whatever the scan - * collected while looking for the parenthesis opening them is dropped */ - pos = uc_expr_pos; + if (expr_at(x, end) != ')') + return SIZE_MAX; - if ((beg = uc_expr_back_pair(fp, end, '(')) == SIZE_MAX || beg == 0) - goto out; + if ((beg = expr_back_pair(x, end, '(')) == SIZE_MAX || beg == 0) + return SIZE_MAX; - uc_expr_pos = pos; + *open = start = i = beg; - i = beg; - - /* once the buffer is full, the expression is known to be incomplete, and - * whatever comes next cannot make a difference anymore */ - while (i-- > 0 && uc_expr_pos > 0) { - int ch = uc_expr_at(fp, i); + while (i-- > 0) { + int ch = expr_at(x, i); if (ch == EOF) break; - /* literal values, subscripts and nested calls are taken as a whole, so - * that their contents cannot be mistaken for part of the expression */ - if (uc_expr_quote(ch) || ch == ')' || ch == ']' || ch == '}') { - bool quote = uc_expr_quote(ch); + if (isspace((unsigned char)ch)) { + while (i > 0 && (ch = expr_at(x, --i)) != EOF + && isspace((unsigned char)ch)) + ; + + if (ch == EOF) + break; + + /* whitespace surrounding a member access operator is merely + * formatting and thus belongs to the expression, as is the + * whitespace in front of the arguments, which is skipped before any + * of the expression has been seen; everywhere else it ends the + * expression */ + if (!expr_member_char(last) && start != *open && + !expr_member_char(ch)) + break; + } + + if (expr_quote(ch) || ch == ')' || ch == ']' || ch == '}') { + bool quote = expr_quote(ch); /* the opener matching a closer sits one below it, except for the * square and curly braces, which sit two below */ - int open = quote ? ch : ch - 1 - (ch > ')'); - - uc_expr_collect(ch); + int opener = quote ? ch : ch - 1 - (ch > ')'); beg = quote - ? uc_expr_back_literal(fp, ch, i) - : uc_expr_back_pair(fp, i, open); + ? expr_back_literal(x, ch, i) + : expr_back_pair(x, i, opener); if (beg == SIZE_MAX) - goto out; + return SIZE_MAX; - i = beg; - last = open; + i = start = beg; + last = opener; continue; } - if (isspace((unsigned char)ch)) { - while (i > 0 && (ch = uc_expr_at(fp, --i)) != EOF - && isspace((unsigned char)ch)) - ; + if (!expr_spine_char(ch)) + break; - if (ch == EOF) - break; + start = i; + last = ch; + } - /* whitespace surrounding a member access operator is merely - * formatting and thus belongs to the expression, as is the - * whitespace in front of the arguments, which is skipped before any - * of the expression has been collected; everywhere else it ends - * the expression */ - if (!uc_expr_member_char(last) - && uc_expr_pos != UC_EXPR_MAXLEN - && !uc_expr_member_char(ch)) - break; + /* nothing but whitespace in front of the arguments, or the scan ran into + * the start of the window while the expression was still being followed */ + if (start == *open || x->cut) + return SIZE_MAX; + + return start; +} + +/* Rewrites the source text in `s` up to `e` into `d`, which does not lie behind + * `s`, leaving out the whitespace of the expression spine. Whitespace within + * subscripts, arguments and literal values is kept as it is. Returns the end of + * the output. */ +static char * +expr_format(char *d, const char *s, const char *e) +{ + size_t depth = 0; + int quote; + + while (s < e) { + if (expr_quote(*s)) { + quote = *s; + + for (*d++ = *s++; s < e && *s != quote; ) { + if (*s == '\\' && s + 1 < e) + *d++ = *s++; + + *d++ = *s++; + } + + if (s < e) + *d++ = *s++; + + continue; } - if (!uc_expr_spine_char((unsigned char)ch)) + switch (*s) { + case ' ': case '\t': case '\n': case '\v': case '\f': case '\r': + if (depth == 0) { + s++; + + continue; + } + + break; + + case '(': case '[': case '{': + depth++; + break; + + case ')': case ']': case '}': + depth -= (depth > 0); break; + } + + *d++ = *s++; + } - uc_expr_collect(ch); + return d; +} - last = ch; +/* Recovers the expression invoked by the call whose closing parenthesis is at + * source offset `end` from the `window` bytes of source ending there, see + * uc_source_callee_expr(). `*cut` is set if the scan wanted to look in front of + * the window, which makes a larger window worth a try if nothing was found. */ +static char * +expr_recover(uc_source_t *source, size_t end, size_t window, bool *cut) +{ + size_t start, open = 0, len; + expr_t expr, *x = &expr; + char *tail; + long save; + FILE *fp; + + *cut = false; + + /* read the bytes in front of the closing parenthesis, and the parenthesis + * itself, in one go and scan them in memory */ + x->base = (end >= window) ? end + 1 - window : 0; + x->len = end + 1 - x->base; + x->cut = false; + x->win = xrealloc(NULL, x->len); + + fp = uc_source_text_open(source, &save); + + if (!fp) { + free(x->win); + + return NULL; + } + + if (fseek(fp, x->base, SEEK_SET) != 0 || + fread(x->win, 1, x->len, fp) != x->len) { + uc_source_text_close(source, fp, save); + free(x->win); + + return NULL; + } + + uc_source_text_close(source, fp, save); + + start = expr_start(x, end, &open); + *cut = x->cut; + + if (start == SIZE_MAX) { + free(x->win); + + return NULL; } - if (uc_expr_pos == UC_EXPR_MAXLEN) - goto out; + /* the text is rewritten in place, and ends up at the start of the window */ + tail = expr_format(x->win, x->win + (start - x->base), x->win + (open - x->base)); + len = tail - x->win; + + if (len > UC_EXPR_MAXLEN) { + /* the expression is longer than the limit, so only its tail is known, + * and the dots replace some of its characters */ + memmove(x->win, tail - UC_EXPR_MAXLEN, UC_EXPR_MAXLEN); + memcpy(x->win, "...", 3); - /* the expression is longer than the buffer, so only its tail is known, and - * the dots overwrite some of the characters which were collected */ - if (uc_expr_pos == 0) - uc_expr_text[0] = uc_expr_text[1] = uc_expr_text[2] = '.'; + len = UC_EXPR_MAXLEN; + } /* an expression starting with a member access operator is missing its * left-hand side, so the scan could not be completed */ - else if (uc_expr_member_char(uc_expr_text[uc_expr_pos])) - goto out; + else if (len == 0 || expr_member_char((unsigned char)x->win[0])) { + free(x->win); - uc_expr_text[UC_EXPR_MAXLEN] = 0; + return NULL; + } - res = uc_expr_text + uc_expr_pos; + x->win[len] = 0; -out: - if (save >= 0) - fseek(fp, save, SEEK_SET); + return xrealloc(x->win, len + 1); +} - return res; +/* Extracts the expression invoked by the call whose closing parenthesis is at + * source offset `end`, for use in error messages about a failed call. + * + * The window of source text in front of `end` starts out small, and is doubled + * while the scan of the expression runs into its start. + * + * Returns the recovered expression, which the caller releases using free(), or + * NULL if it could not be determined. */ +__hidden char * +uc_source_callee_expr(uc_source_t *source, size_t end) +{ + size_t window = UC_EXPR_WINDOW_MIN; + bool cut; + char *res; + + if (!source || end == 0) + return NULL; + + while (true) { + res = expr_recover(source, end, window, &cut); + + if (res || !cut || window >= UC_EXPR_WINDOW_MAX) + return res; + + window *= 2; + } } diff --git a/tests/custom/00_syntax/30_call_error_callee b/tests/custom/00_syntax/30_call_error_callee new file mode 100644 index 000000000..9c03b11a5 --- /dev/null +++ b/tests/custom/00_syntax/30_call_error_callee @@ -0,0 +1,88 @@ +The expression invoked by a failing call is recovered from the source and +named in the error message, regardless of how it is formatted. + + +1. Whitespace and line breaks within the invoked expression are dropped, while + whitespace within subscripts is kept. + +-- Testcase -- +{% + let o = { a: { "x y": [ { f: 1 } ] } }; + + try { + o + .a + [ "x y" ] [0] + .f ( 1, + 2 ); + } + catch (e) { + print(e.message, "\n"); + } + + try { + o . a [ "x y" ][0] . f(); + } + catch (e) { + print(e.message, "\n"); + } +%} +-- End -- + +-- Expect stdout -- +`o.a[ "x y" ][0].f` is not a function, got 1 instead +`o.a[ "x y" ][0].f` is not a function, got 1 instead +-- End -- + + +2. An invoked expression longer than 64 characters shows its tail. + +-- Testcase -- +{% + let o = { aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa: { b: 1 } }; + + try { + o.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.b(); + } + catch (e) { + print(e.message, "\n"); + } +%} +-- End -- + +-- Expect stdout -- +`...aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.b` is not a function, got 1 instead +-- End -- + + +3. Arguments of any width up to a few kilobytes do not prevent the expression + from being recovered, while wider ones do. + +-- Testcase -- +{% + function args(n) { + let rv = []; + + for (let i = 0; i < n; i++) + push(rv, '"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"'); + + return join(",\n\t", rv); + } + + for (let n in [ 2, 20, 60, 120 ]) { + try { + loadstring(`{% let o = { f: { g: 1 } };\no.f.g(${args(n)}); %}`)(); + } + catch (e) { + print(n, ": ", e.message, "\n"); + } + } +%} +-- End -- + +-- Expect stdout -- +2: `o.f.g` is not a function, got 1 instead +20: `o.f.g` is not a function, got 1 instead +60: `o.f.g` is not a function, got 1 instead +120: left-hand side is not a function, got 1 instead +-- End -- diff --git a/vm.c b/vm.c index f30fa6fe1..d25ba9850 100644 --- a/vm.c +++ b/vm.c @@ -731,9 +731,9 @@ uc_vm_frame_reinit(uc_vm_t *vm, uc_closure_t *closure, uc_value_t *ctx, /* Returns the expression invoked by the call that the instruction the VM is * about to execute belongs to, or NULL if it cannot be determined. The result - * is meant for error messages describing a failed call, and stays valid until - * the next call. */ -static const char * + * is meant for error messages describing a failed call, and must be + * released using free(). */ +static char * uc_vm_callee_expr(uc_vm_t *vm) { uc_function_t *function; @@ -871,7 +871,7 @@ uc_vm_call_function(uc_vm_t *vm, uc_value_t *ctx, uc_value_t *fno, bool mcall, * both the offending expression and the value that was not a function; * a NULL vm skips __tostring__ dispatch, so describing the value has no * side effects */ - const char *callee = uc_vm_callee_expr(vm); + char *callee = uc_vm_callee_expr(vm); char *hint = ucv_to_jsonstring(NULL, fno); size_t hlen = strlen(hint); @@ -883,6 +883,7 @@ uc_vm_call_function(uc_vm_t *vm, uc_value_t *ctx, uc_value_t *fno, bool mcall, else uc_vm_raise_exception(vm, EXCEPTION_TYPE, "left-hand side is not a function, got %s instead", hint); + free(callee); free(hint); ucv_put(ctx); ucv_put(fno); From 1dd1f88c81bb0dcdd194615217085c60d441d988 Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Fri, 16 Oct 2026 09:37:20 +0200 Subject: [PATCH 02/14] source: close the file of a source after its compilation A source keeps its file open until the source is released, and a program holds its sources for as long as it lives. Each live program thus holds one open file per source file, which is what runs an application with many loaded files into the limit on open files. After the compilation, the file only serves the text for error messages. Open the file when its text is first read, and release it with uc_source_reset() when the compilation or the load of a precompiled program ends, or when the reading of the text reaches its end. Read the text for the error context, for the name of a failed call and for the debugger from the path of the source. Keep the absolute path for that, since a script can change its working directory. That file can be gone or changed by then; the message then has no source context or shows the text of the current file. A buffer source keeps its text in memory as before. Do not open the source files that a precompiled program names when it is loaded. The warning about a file that does not exist goes away with that. Signed-off-by: Felix Fietkau [adapted to the reworked source API: implicit open, uc_source_reset()] Signed-off-by: Jo-Philipp Wich --- compiler.c | 33 +++- include/ucode/internal/source.h | 4 + include/ucode/internal/types.h | 4 +- include/ucode/source.h | 1 + program.c | 7 +- source.c | 168 +++++++++++++++--- tests/custom/04_modules/19_source_file_closed | 83 +++++++++ 7 files changed, 266 insertions(+), 34 deletions(-) create mode 100644 tests/custom/04_modules/19_source_file_closed diff --git a/compiler.c b/compiler.c index 4b51368f9..9b4f1679a 100644 --- a/compiler.c +++ b/compiler.c @@ -3728,7 +3728,7 @@ uc_compiler_compile_module_source(uc_compiler_t *compiler, const char *modname, }; size_t i, load_idx = 0, n_imports = 0, n_exports = 0; - bool loaded = false; + bool loaded = false, res; uc_value_t *import; ssize_t slot; @@ -3757,16 +3757,34 @@ uc_compiler_compile_module_source(uc_compiler_t *compiler, const char *modname, if (!loaded) { /* We do not yet support linking precompiled modules at compile time, turn static import operation into dynamic load one. */ - if (uc_source_begin(source) == UC_SOURCE_TYPE_PRECOMPILED) + switch (uc_source_begin(source)) { + case UC_SOURCE_TYPE_PRECOMPILED: + uc_source_reset(source); + return uc_compiler_compile_dynload(compiler, modname, imports); + case UC_SOURCE_TYPE_UNAVAILABLE: + if (errp) + xasprintf(errp, "Unable to open source file %s: %s\n", + source->filename, strerror(errno)); + + return false; + + default: + break; + } + load_idx = uc_program_function_id(compiler->program, uc_program_function_last(compiler->program)) + 1; source->exports.offset = (size_t)-1; n_exports = uc_compiler_get_exportnum(compiler); - if (!uc_compile_from_source(&config, source, compiler->program, errp)) + res = uc_compile_from_source(&config, source, compiler->program, errp); + + uc_source_reset(source); + + if (!res) return false; /* The linked module's own exports are the last ones allocated while @@ -4326,6 +4344,13 @@ uc_compile(uc_parse_config_t *config, uc_source_t *source, char **errp) prog = uc_compile_from_bytecode(config, source, errp); break; + case UC_SOURCE_TYPE_UNAVAILABLE: + if (errp) + xasprintf(errp, "Unable to open source file %s: %s\n", + source->filename, strerror(errno)); + + break; + default: if (errp) xasprintf(errp, "Unrecognized source type\n"); @@ -4333,5 +4358,7 @@ uc_compile(uc_parse_config_t *config, uc_source_t *source, char **errp) break; } + uc_source_reset(source); + return prog; } diff --git a/include/ucode/internal/source.h b/include/ucode/internal/source.h index 8c4fe8a25..759c0ba8b 100644 --- a/include/ucode/internal/source.h +++ b/include/ucode/internal/source.h @@ -21,6 +21,8 @@ #include "ucode/internal/types.h" #include "ucode/internal/util.h" +__hidden uc_source_t *uc_source_new_path(const char *path); + __hidden void uc_source_free(uc_source_t *source); __hidden const uint8_t *uc_source_read_block(uc_source_t *source, size_t *len, @@ -36,6 +38,8 @@ __hidden size_t uc_source_read(uc_source_t *source, void *buf, size_t len); __hidden size_t uc_source_offset(uc_source_t *source); __hidden size_t uc_source_remaining(uc_source_t *source); +__hidden void uc_source_reset(uc_source_t *source); + __hidden uc_source_type_t uc_source_begin(uc_source_t *source); __hidden void uc_source_line_next(uc_source_t *source); diff --git a/include/ucode/internal/types.h b/include/ucode/internal/types.h index cc78f703d..63e5b10f3 100644 --- a/include/ucode/internal/types.h +++ b/include/ucode/internal/types.h @@ -43,11 +43,11 @@ uc_declare_vector(uc_lineinfo_t, uint8_t); struct uc_source { uc_value_t header; - char *filename, *runpath, *buffer; + char *filename, *runpath, *buffer, *path; FILE *fp; uint8_t *rbuf; size_t off, size; - bool sized; + bool done, sized; uc_lineinfo_t lineinfo; struct { size_t count, offset; diff --git a/include/ucode/source.h b/include/ucode/source.h index 0a426a64d..2b614de21 100644 --- a/include/ucode/source.h +++ b/include/ucode/source.h @@ -30,6 +30,7 @@ typedef enum { UC_SOURCE_TYPE_PLAIN = 0, UC_SOURCE_TYPE_PRECOMPILED = 1, + UC_SOURCE_TYPE_UNAVAILABLE = 2, } uc_source_type_t; /* Create a source from a file or an in-memory buffer. The buffer variant diff --git a/program.c b/program.c index c762e48b7..1920c09a3 100644 --- a/program.c +++ b/program.c @@ -786,12 +786,7 @@ read_sourceinfo(uc_source_t *input, uint32_t flags, char **errp, uc_program_t *p source = uc_source_new_buffer(path, code, len); } else { - source = uc_source_new_file(path); - - if (!source) { - fprintf(stderr, "Unable to open source file %s: %s\n", path, strerror(errno)); - source = uc_source_new_buffer(path, xstrdup(""), 0); - } + source = uc_source_new_path(path); } if (!read_vector(input, &source->lineinfo, "sourceinfo lineinfo", errp)) { diff --git a/source.c b/source.c index f31826593..34a5b2b7f 100644 --- a/source.c +++ b/source.c @@ -46,6 +46,7 @@ uc_source_new_file(const char *path) src = uc_source_alloc(path); src->fp = fp; + src->path = realpath(path, NULL); return src; } @@ -66,14 +67,81 @@ uc_source_new_buffer(const char *name, char *buf, size_t len) return src; } +uc_source_t * +uc_source_new_path(const char *path) +{ + uc_source_t *src = uc_source_alloc(path); + + src->path = realpath(path, NULL); + + return src; +} + /* Number of source bytes lent to the lexer per block read. */ #define UC_SOURCE_BLOCKSIZE 128 +/* The file is opened again after the working directory can have changed, so + * a source keeps the absolute path that was valid when it was created. */ +static FILE * +uc_source_file_open(uc_source_t *source) +{ + return fopen(source->path ? source->path : source->filename, "rb"); +} + +/* Make sure the stream is open. A source holds its file open only while it is + * being read, so that the number of live programs does not count against the + * limit on open files. */ +static bool +uc_source_stream_ensure(uc_source_t *source) +{ + if (!source->fp) + source->fp = uc_source_file_open(source); + + return source->fp != NULL; +} + +/* The end of the text was reached, or reading it failed. Buffer backed + * sources have nothing to close, but they are done all the same. */ +static void +uc_source_stream_finish(uc_source_t *source) +{ + source->done = true; + + if (source->buffer || !source->fp) + return; + + fclose(source->fp); + source->fp = NULL; +} + +void +uc_source_reset(uc_source_t *source) +{ + free(source->rbuf); + + source->rbuf = NULL; + source->off = 0; + source->done = false; + + if (source->buffer) { + fseek(source->fp, 0, SEEK_SET); + } + else if (source->fp) { + fclose(source->fp); + source->fp = NULL; + } +} + const uint8_t * uc_source_read_block(uc_source_t *source, size_t *len, size_t *off) { size_t rlen; + *len = 0; + + if (source->done || !uc_source_stream_ensure(source)) + return NULL; + if (!source->rbuf) source->rbuf = xalloc(UC_SOURCE_BLOCKSIZE); @@ -84,6 +152,9 @@ uc_source_read_block(uc_source_t *source, size_t *len, size_t *off) source->off += rlen; + if (rlen < UC_SOURCE_BLOCKSIZE || feof(source->fp) || ferror(source->fp)) + uc_source_stream_finish(source); + *len = rlen; return rlen ? source->rbuf : NULL; @@ -96,9 +167,17 @@ uc_source_read(uc_source_t *source, void *buf, size_t len) errno = 0; + if (!len || source->done || !uc_source_stream_ensure(source)) + return 0; + rlen = fread(buf, 1, len, source->fp); source->off += rlen; + /* a short read leaves errno set if the read failed, and clear if the text + * ended prematurely */ + if (rlen < len || uc_source_remaining(source) == 0) + uc_source_stream_finish(source); + return rlen; } @@ -111,43 +190,71 @@ uc_source_offset(uc_source_t *source) size_t uc_source_remaining(uc_source_t *source) { + FILE *fp; long pos, end; if (!source->sized) { + fp = source->fp ? source->fp : uc_source_file_open(source); + + if (!fp) + return 0; + /* the size of text that cannot seek is not known, treat it as * unbounded */ source->size = SIZE_MAX; source->sized = true; - pos = ftell(source->fp); + pos = ftell(fp); - if (pos >= 0 && fseek(source->fp, 0, SEEK_END) == 0) { - end = ftell(source->fp); + if (pos >= 0 && fseek(fp, 0, SEEK_END) == 0) { + end = ftell(fp); if (end >= 0) source->size = end; - fseek(source->fp, pos, SEEK_SET); + fseek(fp, pos, SEEK_SET); } + + if (fp != source->fp) + fclose(fp); } return (source->size > source->off) ? source->size - source->off : 0; } -/* Random access to the text of a source for error messages and the debugger, - * leaving the read position as found. */ +/* Random access to the text of a source for error messages. The text of a + * closed source is read from its path, which can be gone or differ from the + * file that was compiled. A file that was open is left positioned as found, + * one that was not is closed again. A path that names a pipe or a terminal + * by then has no text to show, and reading it would block. */ static FILE * -uc_source_text_open(uc_source_t *source, long *pos) +uc_source_text_open(uc_source_t *source, bool *temp, long *pos) { - *pos = ftell(source->fp); + FILE *fp = source->fp ? source->fp : uc_source_file_open(source); + + *temp = (fp != source->fp); - return (*pos != -1) ? source->fp : NULL; + if (!fp) + return NULL; + + *pos = ftell(fp); + + if (*pos != -1) + return fp; + + if (*temp) + fclose(fp); + + return NULL; } static void -uc_source_text_close(uc_source_t *source, FILE *fp, long pos) +uc_source_text_close(FILE *fp, bool temp, long pos) { - fseek(fp, pos, SEEK_SET); + if (temp) + fclose(fp); + else + fseek(fp, pos, SEEK_SET); } size_t @@ -155,13 +262,13 @@ uc_source_read_line(uc_source_t *source, size_t off, char *line, size_t size, size_t *lpos, bool *truncated) { size_t len, rlen = 0, eline = 1; - bool cont = false; + bool temp, cont = false; long pos; FILE *fp; *truncated = false; - if (size < 2 || !(fp = uc_source_text_open(source, &pos))) + if (size < 2 || !(fp = uc_source_text_open(source, &temp, &pos))) return 0; fseek(fp, 0, SEEK_SET); @@ -174,7 +281,7 @@ uc_source_read_line(uc_source_t *source, size_t off, char *line, size_t size, *truncated = cont; *lpos = rlen - len; - uc_source_text_close(source, fp, pos); + uc_source_text_close(fp, temp, pos); return eline; } @@ -186,7 +293,7 @@ uc_source_read_line(uc_source_t *source, size_t off, char *line, size_t size, eline += !cont; } - uc_source_text_close(source, fp, pos); + uc_source_text_close(fp, temp, pos); return 0; } @@ -196,10 +303,11 @@ uc_source_read_content(uc_source_t *source, size_t *len) { size_t rlen, tlen = 0; char *text = NULL; + bool temp; long pos; FILE *fp; - if (!(fp = uc_source_text_open(source, &pos))) + if (!(fp = uc_source_text_open(source, &temp, &pos))) return NULL; fseek(fp, 0, SEEK_SET); @@ -210,7 +318,7 @@ uc_source_read_content(uc_source_t *source, size_t *len) tlen += rlen; } while (rlen == 4096); - uc_source_text_close(source, fp, pos); + uc_source_text_close(fp, temp, pos); *len = tlen; @@ -275,9 +383,13 @@ uc_source_free(uc_source_t *source) uc_source_state_clear(source); - fclose(source->fp); - free(source->rbuf); + uc_source_reset(source); + + if (source->buffer) + fclose(source->fp); + free(source->buffer); + free(source->path); } uc_source_type_t @@ -285,10 +397,18 @@ uc_source_begin(uc_source_t *source) { union { char s[sizeof(uint32_t)]; uint32_t n; } buf = { 0 }; uc_source_type_t type = UC_SOURCE_TYPE_PLAIN; - FILE *fp = source->fp; size_t rlen; + FILE *fp; int c = 0; + /* start over from the beginning of the text */ + uc_source_reset(source); + + if (!uc_source_stream_ensure(source)) + return UC_SOURCE_TYPE_UNAVAILABLE; + + fp = source->fp; + if (fread(buf.s, 1, 2, fp) == 2 && !strncmp(buf.s, "#!", 2)) { source->off += 2; @@ -319,6 +439,7 @@ uc_source_begin(uc_source_t *source) } } + /* leave the stream on the first byte of code */ if (fseek(fp, -(long)rlen, SEEK_CUR) == -1) fprintf(stderr, "Failed to rewind source buffer: %s\n", strerror(errno)); @@ -694,6 +815,7 @@ expr_recover(uc_source_t *source, size_t end, size_t window, bool *cut) size_t start, open = 0, len; expr_t expr, *x = &expr; char *tail; + bool temp; long save; FILE *fp; @@ -706,7 +828,7 @@ expr_recover(uc_source_t *source, size_t end, size_t window, bool *cut) x->cut = false; x->win = xrealloc(NULL, x->len); - fp = uc_source_text_open(source, &save); + fp = uc_source_text_open(source, &temp, &save); if (!fp) { free(x->win); @@ -716,13 +838,13 @@ expr_recover(uc_source_t *source, size_t end, size_t window, bool *cut) if (fseek(fp, x->base, SEEK_SET) != 0 || fread(x->win, 1, x->len, fp) != x->len) { - uc_source_text_close(source, fp, save); + uc_source_text_close(fp, temp, save); free(x->win); return NULL; } - uc_source_text_close(source, fp, save); + uc_source_text_close(fp, temp, save); start = expr_start(x, end, &open); *cut = x->cut; diff --git a/tests/custom/04_modules/19_source_file_closed b/tests/custom/04_modules/19_source_file_closed new file mode 100644 index 000000000..809584e18 --- /dev/null +++ b/tests/custom/04_modules/19_source_file_closed @@ -0,0 +1,83 @@ +A source holds its file open only while it is compiled. A live program thus +holds no open file, also not for the modules that it links. The text for an +error message is read from the path of the source. If that file is gone, the +message has no source context. + +-- Testcase -- +import * as fs from 'fs'; + +function open_files() { + return length(fs.lsdir("/proc/self/fd")); +} + +function raise(label, fn, text) { + try { + fn(); + } + catch (e) { + print(label, ": ", e.message, ", line ", e.stacktrace[0].line, + index(e.stacktrace[0].context, text) < 0 ? ", no source text\n" : ", with source text\n"); + } +} + +fs.mkdir("files"); + +let fds = open_files(); +let held = []; + +for (let i = 0; i < 100; i++) { + fs.writefile(`files/mod${i}.uc`, "export const value = 1;\n"); + fs.writefile(`files/root${i}.uc`, `import { value } from './mod${i}.uc';\nreturn value;\n`); + push(held, loadfile(`files/root${i}.uc`)); +} + +print("open files for 100 live programs: ", open_files() - fds, "\n"); + +fs.writefile("files/fail.uc", "let o = {};\n\nreturn {\n\tstore: () => null.x = 1,\n\tcall: () => o.missing()\n};\n"); + +let fail = loadfile("files/fail.uc")(); + +raise("store", fail.store, "null.x = 1"); +raise("call", fail.call, "o.missing()"); + +fs.chdir("files"); +raise("store, other directory", fail.store, "null.x = 1"); +fs.chdir(".."); + +fs.unlink("files/fail.uc"); + +raise("store, file removed", fail.store, "null.x = 1"); +raise("call, file removed", fail.call, "o.missing()"); + +fs.writefile("files/broken.uc", "let x = 1;\nlet = 2;\n"); + +try { + loadfile("files/broken.uc"); +} +catch (e) { + print(e.message); +} + +print("open files at the end: ", open_files() - fds, "\n"); +-- End -- + +-- Args -- +-R +-- End -- + +-- Expect stdout -- +open files for 100 live programs: 0 +store: attempt to set property on null value, line 4, with source text +call: `o.missing` is not a function, got null instead, line 5, with source text +store, other directory: attempt to set property on null value, line 4, with source text +store, file removed: attempt to set property on null value, line 4, no source text +call, file removed: left-hand side is not a function, got null instead, line 5, no source text +Unable to compile source file 'files/broken.uc': + + | Syntax error: Expecting variable name + | In line 2, byte 5: + | + | `let = 2;` + | ^-- Near here +open files at the end: 0 +-- End -- From 69db946e7c3980bbbe44a921125f475b6aac0954 Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Tue, 13 Oct 2026 20:26:03 +0200 Subject: [PATCH 03/14] lib: remove the source cache of loadfile() Since commit f2d62f82a522 ("lib: deduplicate source files by resolved path in vm->sources cache"), uc_loadfile() and thus include() hand the same source object to uc_compile() on each call for one file. A compilation records its read position, its line information and its export table in the source, and the root source also holds the export count of the program. The next compilation of that source continues from this state, so the export slots of its imports lie behind the export table of the new program. A named import then yields null and a wildcard import leaves its name operands unread, which the VM decodes as instructions: Runtime error: unknown opcode 0 A file without imports is affected as well: the error locations of a later compilation are wrong, and the cache keeps each source until the VM is freed. The cache was added to limit the number of open files. A source no longer holds its file open after the compilation, so the cache has no purpose. Revert that commit: each call compiles a source of its own again, and each program keeps the line information and the export table of its own compilation. Keep the field of the VM as a reserved pointer, since nothing uses it. Fixes: f2d62f82a522 ("lib: deduplicate source files by resolved path in vm->sources cache") Signed-off-by: Felix Fietkau --- include/ucode/types.h | 2 +- lib.c | 27 +--- .../04_modules/20_repeated_source_compile | 147 ++++++++++++++++++ .../04_modules/21_changed_source_compile | 83 ++++++++++ vm.c | 19 --- 5 files changed, 237 insertions(+), 41 deletions(-) create mode 100644 tests/custom/04_modules/20_repeated_source_compile create mode 100644 tests/custom/04_modules/21_changed_source_compile diff --git a/include/ucode/types.h b/include/ucode/types.h index b096ae0ce..bb571e699 100644 --- a/include/ucode/types.h +++ b/include/ucode/types.h @@ -227,7 +227,7 @@ struct uc_vm { uc_parse_config_t *config; uc_value_t *globals; uc_value_t *registry; - struct lh_table *sources; + void *reserved; uc_weakref_t values; uc_resource_types_t restypes; uc_breakpoints_t breakpoints; diff --git a/lib.c b/lib.c index 0484ca5f0..0606d1479 100644 --- a/lib.c +++ b/lib.c @@ -5977,36 +5977,21 @@ static uc_value_t * uc_loadfile(uc_vm_t *vm, size_t nargs) { uc_value_t *path = uc_fn_arg(0); - uc_source_t *source = NULL; - char *resolved; + uc_source_t *source; if (ucv_type(path) != UC_STRING) return NULL; - resolved = realpath(ucv_string_get(path), NULL); - - if (resolved && lh_table_lookup_ex(vm->sources, (const void *)resolved, (void **)&source)) { - source = uc_source_get(source); - } + source = uc_source_new_file(ucv_string_get(path)); if (!source) { - source = uc_source_new_file(ucv_string_get(path)); - - if (!source) { - free(resolved); - uc_vm_raise_exception(vm, EXCEPTION_RUNTIME, - "Unable to open source file %s: %s", - ucv_string_get(path), strerror(errno)); - - return NULL; - } + uc_vm_raise_exception(vm, EXCEPTION_RUNTIME, + "Unable to open source file %s: %s", + ucv_string_get(path), strerror(errno)); - if (resolved) - lh_table_insert(vm->sources, xstrdup(resolved), uc_source_get(source)); + return NULL; } - free(resolved); - return uc_load_common(vm, nargs, source); } diff --git a/tests/custom/04_modules/20_repeated_source_compile b/tests/custom/04_modules/20_repeated_source_compile new file mode 100644 index 000000000..230260d59 --- /dev/null +++ b/tests/custom/04_modules/20_repeated_source_compile @@ -0,0 +1,147 @@ +Each loadfile() or include() call compiles a source of its own. A compilation +records its read position, its line information and its export table in the +source. With one source for several calls, the imports of a later compilation +resolve to wrong export slots, its error locations are wrong and a leading +interpreter line is parsed as code. + +-- Testcase -- +import * as fs from 'fs'; + +function run(label, fn) { + try { + fn(); + } + catch (e) { + let m = match(e.stacktrace[0].context, /`([^`]*)`/); + + printf("%s: %s: %s (line %d: %s)\n", label, e.type, e.message, + e.stacktrace[0].line, m ? m[1] : "no source line"); + } +} + +function repeat(label, count, fn) { + for (let i = 1; i <= count; i++) + run(`${label} #${i}`, fn); +} + +repeat("named loadfile", 3, () => loadfile("files/named.uc")()); +repeat("wildcard loadfile", 3, () => loadfile("files/wildcard.uc")()); +repeat("default loadfile", 2, () => loadfile("files/default.uc")()); +repeat("named include", 2, () => include("files/named.uc")); +repeat("wildcard include", 2, () => include("files/wildcard.uc")); + +fs.symlink("named.uc", `${TESTFILES_PATH}/link.uc`); +run("symlink #1", () => loadfile("files/link.uc")()); +run("symlink #2", () => loadfile("./files/../files/named.uc")()); + +repeat("no import", 2, () => loadfile("files/plain.uc")()); +repeat("native import", 2, () => loadfile("files/native.uc")()); +run("other file #1", () => loadfile("files/named.uc")()); +run("other file #2", () => loadfile("files/named2.uc")()); +repeat("loadstring", 2, () => loadstring(fs.readfile("files/plain.uc"))()); + +fs.symlink("dynamic.uc", `${TESTFILES_PATH}/alias.uc`); +run("dynamic import #1", () => print("dynamic: ", import("dynamic").own(), "\n")); +run("dynamic import #2", () => print("dynamic: ", import("alias").own(), "\n")); + +repeat("error location", 3, () => loadfile("files/error.uc")()); +repeat("interpreter line", 2, () => loadfile("files/shebang.uc")()); + +system([ fs.readlink("/proc/self/exe"), "-c", "-o", "files/precompiled.uc", "files/plain.uc" ]); +repeat("precompiled", 2, () => loadfile("files/precompiled.uc")()); +-- End -- + +-- File mod.uc -- +export function first() { return "first"; }; +export function second() { return "second"; }; +export default "default"; +-- End -- + +-- File named.uc -- +import { second, first } from "./mod.uc"; + +print("named: ", first(), " ", second(), "\n"); +-- End -- + +-- File named2.uc -- +import { first } from "./mod.uc"; + +print("named2: ", first(), "\n"); +-- End -- + +-- File wildcard.uc -- +import * as mod from "./mod.uc"; + +print("wildcard: ", mod.first(), " ", mod.second(), " ", mod.default, "\n"); +-- End -- + +-- File default.uc -- +import def, { second } from "./mod.uc"; + +print("default: ", def, " ", second(), "\n"); +-- End -- + +-- File plain.uc -- +print("plain\n"); +-- End -- + +-- File native.uc -- +import { basename } from "fs"; + +print("native: ", basename("/a/b"), "\n"); +-- End -- + +-- File dynamic.uc -- +import { first } from "./mod.uc"; + +export function own() { return "own " + first(); }; +-- End -- + +-- File error.uc -- +let a = 1; +let b = 2; +null.x = a + b; +-- End -- + +-- File shebang.uc -- +#!/usr/bin/env ucode +print("interpreter line\n"); +-- End -- + +-- Args -- +-R -L files/ +-- End -- + +-- Expect stdout -- +named: first second +named: first second +named: first second +wildcard: first second default +wildcard: first second default +wildcard: first second default +default: default second +default: default second +named: first second +named: first second +wildcard: first second default +wildcard: first second default +named: first second +named: first second +plain +plain +native: b +native: b +named: first second +named2: first +plain +plain +dynamic: own first +dynamic: own first +error location #1: Type error: attempt to set property on null value (line 3: null.x = a + b;) +error location #2: Type error: attempt to set property on null value (line 3: null.x = a + b;) +error location #3: Type error: attempt to set property on null value (line 3: null.x = a + b;) +interpreter line +interpreter line +plain +plain +-- End -- diff --git a/tests/custom/04_modules/21_changed_source_compile b/tests/custom/04_modules/21_changed_source_compile new file mode 100644 index 000000000..f622d59c7 --- /dev/null +++ b/tests/custom/04_modules/21_changed_source_compile @@ -0,0 +1,83 @@ +Each loadfile() or include() call reads the current content of the file, also +after an exception read the source line of an earlier version for its context. + +A program of an earlier call stays alive and keeps the line information of +its own compilation, also after a later compilation of that file failed. + +-- Testcase -- +import * as fs from 'fs'; + +function run(label, fn) { + try { + fn(); + } + catch (e) { + printf("%s: %s: %s\n", label, e.type, split(e.message, "\n")[0]); + } +} + +function load(label, path, code) { + fs.writefile(path, code); + run(label, () => loadfile(path)()); +} + +fs.mkdir("files"); + +load("longer #1", "files/longer.uc", "print('short\\n');\nnull.x = 1;\n"); +load("longer #2", "files/longer.uc", "print('a longer second version\\n');\n"); + +load("shorter #1", "files/shorter.uc", "print('a longer first version\\n');\nnull.x = 1;\n"); +load("shorter #2", "files/shorter.uc", "print('short\\n');\n"); + +load("broken #1", "files/broken.uc", "print('valid\\n');\nnull.x = 1;\n"); +load("broken #2", "files/broken.uc", "let = 1;\n"); +load("broken #3", "files/broken.uc", "print('valid again\\n');\n"); + +function line(label, fn) { + try { + fn(); + } + catch (e) { + printf("%s: line %d\n", label, e.stacktrace[0].line); + } +} + +let raise = "return function() {\n\tnull.x = 1;\n};\n"; + +fs.writefile("files/live.uc", raise); +let first = loadfile("files/live.uc")(); +line("first #1", first); + +fs.writefile("files/live.uc", "\n\n\n" + raise); +let second = loadfile("files/live.uc")(); +line("first #2", first); +line("second #1", second); + +fs.writefile("files/live.uc", "let = 1;\n"); +run("live", () => loadfile("files/live.uc")); +line("first #3", first); +line("second #2", second); +-- End -- + +-- Args -- +-R +-- End -- + +-- Expect stdout -- +short +longer #1: Type error: attempt to set property on null value +a longer second version +a longer first version +shorter #1: Type error: attempt to set property on null value +short +valid +broken #1: Type error: attempt to set property on null value +broken #2: Runtime error: Unable to compile source file 'files/broken.uc': +valid again +first #1: line 2 +first #2: line 2 +second #1: line 5 +live: Runtime error: Unable to compile source file 'files/live.uc': +first #3: line 2 +second #2: line 5 +-- End -- diff --git a/vm.c b/vm.c index d25ba9850..ba78a76e1 100644 --- a/vm.c +++ b/vm.c @@ -269,8 +269,6 @@ void uc_vm_init(uc_vm_t *vm, uc_parse_config_t *config) vm->open_upvals = NULL; - vm->sources = lh_kchar_table_new(16, NULL); - vm->values.prev = &vm->values; vm->values.next = &vm->values; @@ -327,23 +325,6 @@ void uc_vm_free(uc_vm_t *vm) ucv_freeall(vm); - /* Release source cache entries after freeall so that all program refs - * to sources are already gone and the cache refs are the last ones. */ - if (vm->sources) { - struct lh_entry *entry, *next; - - entry = vm->sources->head; - - while (entry) { - next = entry->next; - ucv_put((uc_value_t *)lh_entry_v(entry)); - free((char *)lh_entry_k(entry)); - entry = next; - } - - lh_table_free(vm->sources); - } - for (i = 0; i < vm->restypes.count; i++) free(vm->restypes.entries[i]); From 4436a931dc3842ee74a55d059af9f0a48bfee639 Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Fri, 16 Oct 2026 09:38:15 +0200 Subject: [PATCH 04/14] compiler: start each compilation from a clean source A compilation records its read position, its line information and its export table in the source. uc_compile() does not reset that state, so a second uc_compile() call for one source object continues from it. The error locations of the new program are wrong, and the export slots of its imports lie behind the export table of the new program. uc_loadfile() no longer passes a source twice, but uc_compile() and the source objects are public API and an embedder can do so. Drop the state of the previous compilation in uc_source_begin(), which every compilation of a source starts with. The sources of statically imported modules are created per program and need no reset. The source then describes its latest compilation only. A program of an earlier compilation that is still in use needs a source of its own. Signed-off-by: Felix Fietkau [adapted to the reworked source API: state is cleared in uc_source_begin()] Signed-off-by: Jo-Philipp Wich --- source.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source.c b/source.c index 34a5b2b7f..0eda8b1c4 100644 --- a/source.c +++ b/source.c @@ -122,6 +122,7 @@ uc_source_reset(uc_source_t *source) source->rbuf = NULL; source->off = 0; source->done = false; + source->sized = false; if (source->buffer) { fseek(source->fp, 0, SEEK_SET); @@ -401,8 +402,13 @@ uc_source_begin(uc_source_t *source) FILE *fp; int c = 0; - /* start over from the beginning of the text */ + /* A compilation records the line information and the export table in the + * source, and a source can outlive its compilation. Drop that state, and + * start over from the beginning of the text. */ uc_source_reset(source); + uc_source_state_clear(source); + + source->exports.offset = 0; if (!uc_source_stream_ensure(source)) return UC_SOURCE_TYPE_UNAVAILABLE; From 4503599e49f1fae4507667b349bb5938a465babd Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Tue, 13 Oct 2026 20:27:35 +0200 Subject: [PATCH 05/14] source: read a file that cannot seek into a buffer uc_source_begin() reads the first bytes of a source to look for an interpreter line and for the bytecode magic, and seeks back afterwards. The seek fails on a pipe, so the compiler never sees these bytes. A script that is passed as /dev/stdin from a pipe, or loaded from there with loadfile(), fails: Failed to rewind source buffer: Illegal seek Syntax error: Unexpected token The content of a pipe is also gone when an error message needs the source text afterwards. Read such a file into a buffer in uc_source_new_file() and create a buffer source from it, like main.c already does for the "-" argument. Move that code to the new uc_source_new_slurp(), which both use. While here, terminate the buffer of uc_source_new_slurp(), since uc_program_write() stores it as a string, and let uc_load_common() name the file in a compile error of a file that was read into a buffer. Signed-off-by: Felix Fietkau [adapted to the reworked source API: uc_source_new_stream() is uc_source_new_slurp()] Signed-off-by: Jo-Philipp Wich --- include/ucode/source.h | 8 +++- lib.c | 8 ++-- main.c | 26 ++--------- source.c | 39 ++++++++++++++++ .../04_modules/22_unseekable_source_compile | 44 +++++++++++++++++++ 5 files changed, 97 insertions(+), 28 deletions(-) create mode 100644 tests/custom/04_modules/22_unseekable_source_compile diff --git a/include/ucode/source.h b/include/ucode/source.h index 2b614de21..08c676689 100644 --- a/include/ucode/source.h +++ b/include/ucode/source.h @@ -34,10 +34,16 @@ typedef enum { } uc_source_type_t; /* Create a source from a file or an in-memory buffer. The buffer variant - * takes ownership of `buf` (it is freed when the source is released). */ + * takes ownership of `buf` (it is freed when the source is released). A + * file that cannot seek, such as a pipe, is read into a buffer. */ uc_source_t *uc_source_new_file(const char *path); uc_source_t *uc_source_new_buffer(const char *name, char *buf, size_t len); +/* Create a buffer source from the remaining content of `fp`, which is read to + * its end. The caller keeps `fp` and closes it. An empty input results in a + * buffer holding a single newline. */ +uc_source_t *uc_source_new_slurp(const char *name, FILE *fp); + /* Fetch the line containing the given byte offset; *offset is set to the * offset of the first character of that line. Returns the 1-based line number. */ size_t uc_source_get_line(uc_source_t *source, size_t *offset); diff --git a/lib.c b/lib.c index 0606d1479..41ab339ab 100644 --- a/lib.c +++ b/lib.c @@ -5843,7 +5843,7 @@ uc_compile_parse_config(uc_parse_config_t *config, uc_value_t *spec) } static uc_value_t * -uc_load_common(uc_vm_t *vm, size_t nargs, uc_source_t *source) +uc_load_common(uc_vm_t *vm, size_t nargs, uc_source_t *source, bool is_file) { uc_parse_config_t conf = *vm->config; uc_program_t *program; @@ -5866,7 +5866,7 @@ uc_load_common(uc_vm_t *vm, size_t nargs, uc_source_t *source) if (!closure) { uc_error_message_indent(&err); - if (source->buffer) + if (!is_file) uc_vm_raise_exception(vm, EXCEPTION_RUNTIME, "Unable to compile source string:\n\n%s", err); else @@ -5947,7 +5947,7 @@ uc_loadstring(uc_vm_t *vm, size_t nargs) return NULL; } - return uc_load_common(vm, nargs, source); + return uc_load_common(vm, nargs, source, false); } /** @@ -5992,7 +5992,7 @@ uc_loadfile(uc_vm_t *vm, size_t nargs) return NULL; } - return uc_load_common(vm, nargs, source); + return uc_load_common(vm, nargs, source, true); } /** diff --git a/main.c b/main.c index 873eb832d..18f64156f 100644 --- a/main.c +++ b/main.c @@ -378,38 +378,18 @@ compile(uc_vm_t *vm, uc_source_t *src, FILE *precompile, const compile_opts_t *o static uc_source_t * read_stdin(void) { - size_t rlen = 0, tlen = 0; - char buf[128], *p = NULL; + FILE *fp = stdin_unused; - if (!stdin_unused) { + if (!fp) { fprintf(stderr, "The stdin can only be read once\n"); errno = EINVAL; return NULL; } - while (true) { - rlen = fread(buf, 1, sizeof(buf), stdin_unused); - - if (rlen == 0) - break; - - p = xrealloc(p, tlen + rlen); - memcpy(p + tlen, buf, rlen); - tlen += rlen; - } - stdin_unused = NULL; - /* On empty stdin, provide a dummy buffer and ensure that it is - * at least one byte long, due to - * https://github.com/google/sanitizers/issues/627 */ - if (p == NULL) { - p = xstrdup("\n"); - tlen = 1; - } - - return uc_source_new_buffer("[stdin]", p, tlen); + return uc_source_new_slurp("[stdin]", fp); } static void diff --git a/source.c b/source.c index 0eda8b1c4..f75365de5 100644 --- a/source.c +++ b/source.c @@ -35,6 +35,36 @@ uc_source_alloc(const char *name) return src; } +uc_source_t * +uc_source_new_slurp(const char *name, FILE *fp) +{ + size_t rlen = 0, tlen = 0; + char buf[128], *p = NULL; + + while (true) { + rlen = fread(buf, 1, sizeof(buf), fp); + + if (rlen == 0) + break; + + /* uc_program_write() stores the buffer as a string */ + p = xrealloc(p, tlen + rlen + 1); + memcpy(p + tlen, buf, rlen); + tlen += rlen; + p[tlen] = 0; + } + + /* On empty input, provide a dummy buffer and ensure that it is + * at least one byte long, due to + * https://github.com/google/sanitizers/issues/627 */ + if (p == NULL) { + p = xstrdup("\n"); + tlen = 1; + } + + return uc_source_new_buffer(name, p, tlen); +} + uc_source_t * uc_source_new_file(const char *path) { @@ -44,6 +74,15 @@ uc_source_new_file(const char *path) if (!fp) return NULL; + /* the type test seeks in the file, and the content of a pipe cannot be + * read again for an error message */ + if (fseek(fp, 0L, SEEK_SET)) { + src = uc_source_new_slurp(path, fp); + fclose(fp); + + return src; + } + src = uc_source_alloc(path); src->fp = fp; src->path = realpath(path, NULL); diff --git a/tests/custom/04_modules/22_unseekable_source_compile b/tests/custom/04_modules/22_unseekable_source_compile new file mode 100644 index 000000000..4419f3e67 --- /dev/null +++ b/tests/custom/04_modules/22_unseekable_source_compile @@ -0,0 +1,44 @@ +A source file that cannot seek, such as a pipe, is read into a buffer when +it is opened: the type test, the interpreter line and the error context all +seek in the source. + +-- Testcase -- +import { readlink } from 'fs'; + +let ucode = readlink('/proc/self/exe'); + +system(`printf '#!/usr/bin/env ucode\\nprint("script\\\\n");\\nnull.x = 1;\\n' | ${ucode} /dev/stdin 2>&1`); +system(`printf 'print("loadfile\\\\n");\\n' | ${ucode} -e 'loadfile("/dev/stdin")();' 2>&1`); +system(`printf 'let = 1;\\n' | ${ucode} -e 'try { loadfile("/dev/stdin"); } catch (e) { print(substr(e.message, 0, index(e.message, "\\n")), "\\n"); }' 2>&1`); + +// A precompiled program names its source file for the error context only. +// The load of the program must not read a source that cannot seek. +let libs = join(" ", map(REQUIRE_SEARCH_PATH, (path) => `-L '${path}'`)); + +system(`${ucode} ${libs} -c -o files/reader.uc /dev/stdin < files/reader-source.uc 2>&1`); +system(`echo data | ${ucode} ${libs} files/reader.uc 2>&1`); +-- End -- + +-- File reader-source.uc -- +import { stdin } from "fs"; + +print("input: ", stdin.read("line")); +-- End -- + +-- Args -- +-R +-- End -- + +-- Expect stdout -- +Type error: attempt to set property on null value +In /dev/stdin, line 3, byte 10: + + `null.x = 1;` + ~~~~~~~~~^~ + + +script +loadfile +Unable to compile source file '/dev/stdin': +input: data +-- End -- From 330cdab782be6d486cc248c71ee11446b750a9ff Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Tue, 13 Oct 2026 11:59:04 +0200 Subject: [PATCH 06/14] compiler: release the name list of an import() statement uc_compiler_compile_import() allocates the list of imported names before it checks for the call form. A statement that consists of an import() call returns early and leaks that list on each compilation. Release the list on that path. Fixes: 926f20b55e13 ("compiler: add support for runtime module imports") Signed-off-by: Felix Fietkau --- compiler.c | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler.c b/compiler.c index 9b4f1679a..ee70961c8 100644 --- a/compiler.c +++ b/compiler.c @@ -4126,6 +4126,7 @@ uc_compiler_compile_import(uc_compiler_t *compiler) &compiler->parser->prev); uc_compiler_emit_insn(compiler, 0, I_POP); + ucv_put(namelist); return TK_SCOL; } From 153c627fd44566b65e9694f9155842f5475315ef Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Tue, 13 Oct 2026 11:59:12 +0200 Subject: [PATCH 07/14] lib: do not store a failed module in the module table uc_require_path() stores the result of a load in the module table as soon as the file exists, also if the module raised an exception. The next import() or require() of that name then returns the incomplete result of the failed load, for example an empty namespace, and raises no error. Store the module only if the load raised no exception, so that a later load runs the module again. Signed-off-by: Felix Fietkau --- lib.c | 2 +- .../04_modules/23_dynamic_import_failed | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 tests/custom/04_modules/23_dynamic_import_failed diff --git a/lib.c b/lib.c index 41ab339ab..cfb7e7a61 100644 --- a/lib.c +++ b/lib.c @@ -3094,7 +3094,7 @@ uc_require_path(uc_vm_t *vm, const char *path_template, const char *name, else if (!strcmp(p + 1, ".uc")) rv = uc_require_ucode(vm, buf->buf, NULL, res, true, module_mode); - if (rv) + if (rv && vm->exception.type == EXCEPTION_NONE) ucv_object_add(modtable, name, ucv_get(*res)); out: diff --git a/tests/custom/04_modules/23_dynamic_import_failed b/tests/custom/04_modules/23_dynamic_import_failed new file mode 100644 index 000000000..a31bc89de --- /dev/null +++ b/tests/custom/04_modules/23_dynamic_import_failed @@ -0,0 +1,45 @@ +A dynamically loaded module whose body raises an exception is not stored in +the module table. A later load runs the module again and does not return +the incomplete namespace of the failed load. + +-- Testcase -- +function run(label, fn) { + try { + print(label, ": ", fn(), "\n"); + } + catch (e) { + print(label, ": ", e.message, "\n"); + } +} + +global.fail = true; +run("failed load #1", () => keys(import("failing"))); +run("failed load #2", () => keys(import("failing"))); + +global.fail = false; +run("good load", () => keys(import("failing"))); +run("stored module", () => keys(import("failing"))); + +// A statement that only calls import() must not leak in the compiler. +import("failing"); +-- End -- + +-- File failing.uc -- +export const a = 1; + +if (global.fail) + die("module body failed"); + +export const b = 2; +-- End -- + +-- Args -- +-R -L files/ +-- End -- + +-- Expect stdout -- +failed load #1: module body failed +failed load #2: module body failed +good load: [ "a", "b" ] +stored module: [ "a", "b" ] +-- End -- From 3bf604919140025d63bacd7cf282986d1c0bd1af Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Tue, 13 Oct 2026 12:24:33 +0200 Subject: [PATCH 08/14] vm: keep the exception of a failed dynamic module load uc_require_library() returns the incomplete namespace of a module whose body raised an exception. uc_vm_insn_dynload() tests the return value only and goes on to bind the imported names. A name that the module did not export before it failed replaces the exception of the module: Reference error: Module does not export b Return if the load raised an exception. Signed-off-by: Felix Fietkau --- tests/custom/04_modules/23_dynamic_import_failed | 9 +++++++++ vm.c | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/custom/04_modules/23_dynamic_import_failed b/tests/custom/04_modules/23_dynamic_import_failed index a31bc89de..0ba9d6167 100644 --- a/tests/custom/04_modules/23_dynamic_import_failed +++ b/tests/custom/04_modules/23_dynamic_import_failed @@ -12,13 +12,20 @@ function run(label, fn) { } } +// A named import of a module that the compiler links dynamically reports the +// exception of the module, not a missing export. +let named = loadstring('import { b } from "failing";\nreturn b;\n', + { force_dynlink_list: [ "failing" ] }); + global.fail = true; run("failed load #1", () => keys(import("failing"))); run("failed load #2", () => keys(import("failing"))); +run("failed named import", named); global.fail = false; run("good load", () => keys(import("failing"))); run("stored module", () => keys(import("failing"))); +run("good named import", named); // A statement that only calls import() must not leak in the compiler. import("failing"); @@ -40,6 +47,8 @@ export const b = 2; -- Expect stdout -- failed load #1: module body failed failed load #2: module body failed +failed named import: module body failed good load: [ "a", "b" ] stored module: [ "a", "b" ] +good named import: 2 -- End -- diff --git a/vm.c b/vm.c index ba78a76e1..95f28b426 100644 --- a/vm.c +++ b/vm.c @@ -3082,8 +3082,11 @@ uc_vm_insn_dynload(uc_vm_t *vm, uc_vm_insn_t insn) modscope = uc_require_library(vm, name, true); ucv_put(name); - if (!modscope) + if (!modscope || vm->exception.type != EXCEPTION_NONE) { + ucv_put(modscope); + return; + } if (vm->exception.type != EXCEPTION_NONE) { ucv_put(modscope); From 71b4a144b438a5c00cfdea69986b28100fd491b7 Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Tue, 13 Oct 2026 12:24:33 +0200 Subject: [PATCH 09/14] lib: compile a file loaded from a module body as a program uc_require_ucode() points vm->config at a local configuration with compile_module set, and keeps it there while the body of the module runs. uc_load_common() copies that configuration, so a loadfile() or loadstring() call in the body of a dynamically loaded module compiles its argument as a module. A script that returns a value then fails: Syntax error: return must be inside function body Reset compile_module once the module itself is compiled. Signed-off-by: Felix Fietkau --- lib.c | 4 +++ .../custom/04_modules/24_load_in_module_body | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 tests/custom/04_modules/24_load_in_module_body diff --git a/lib.c b/lib.c index cfb7e7a61..c8043b17b 100644 --- a/lib.c +++ b/lib.c @@ -3020,6 +3020,10 @@ uc_require_ucode(uc_vm_t *vm, const char *path, uc_value_t *scope, uc_value_t ** closure = uc_loadfile(vm, 1); + /* the module body runs with this configuration; a file that it loads is + * not a module */ + config.compile_module = prev_config->compile_module; + ucv_put(uc_vm_stack_pop(vm)); if (closure) { diff --git a/tests/custom/04_modules/24_load_in_module_body b/tests/custom/04_modules/24_load_in_module_body new file mode 100644 index 000000000..9a7e01373 --- /dev/null +++ b/tests/custom/04_modules/24_load_in_module_body @@ -0,0 +1,27 @@ +A file or a string that the body of a dynamically loaded module loads is a +program of its own, not a module: it can return a value. + +-- Testcase -- +let loader = import("loader"); + +print("file: ", loader.file(), "\n"); +print("string: ", loader.string(), "\n"); +-- End -- + +-- File loader.uc -- +export const file = loadfile("files/script.uc"); +export const string = loadstring("return 2;\n"); +-- End -- + +-- File script.uc -- +return 1; +-- End -- + +-- Args -- +-R -L files/ +-- End -- + +-- Expect stdout -- +file: 1 +string: 2 +-- End -- From 080cbd79727fa6eaa37ea9591166fe7ecb6934b5 Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Tue, 13 Oct 2026 13:46:04 +0200 Subject: [PATCH 10/14] lib: release the main function of a required file uc_require_ucode() pushes the closure that uc_loadfile() returns on the stack, runs it, and pops it again without a release. Each include(), render(), require() and import() thus leaks one closure, which keeps the program of the file, until a collection or the end of the VM. Release the closure after the run. Signed-off-by: Felix Fietkau --- lib.c | 1 + .../04_modules/25_required_closure_release | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 tests/custom/04_modules/25_required_closure_release diff --git a/lib.c b/lib.c index c8043b17b..15110ea7d 100644 --- a/lib.c +++ b/lib.c @@ -3043,6 +3043,7 @@ uc_require_ucode(uc_vm_t *vm, const char *path, uc_value_t *scope, uc_value_t ** uc_vm_stack_pop(vm); uc_vm_stack_pop(vm); uc_vm_stack_pop(vm); + ucv_put(closure); } } } diff --git a/tests/custom/04_modules/25_required_closure_release b/tests/custom/04_modules/25_required_closure_release new file mode 100644 index 000000000..103a80acf --- /dev/null +++ b/tests/custom/04_modules/25_required_closure_release @@ -0,0 +1,29 @@ +include() and render() release the main function of the file that they ran. +Without the collector, a loop over them must not add live values. + +-- Testcase -- +function count(label, fn) { + let live = gc("count"); + + for (let i = 0; i < 100; i++) + fn(); + + print(label, ": ", gc("count") - live, "\n"); +} + +count("include", () => include("files/script.uc")); +count("render", () => render("files/script.uc")); +-- End -- + +-- File script.uc -- +let a = 1; +-- End -- + +-- Args -- +-R +-- End -- + +-- Expect stdout -- +include: 0 +render: 0 +-- End -- From 6c30ef7fd30c2458cfee983a114803ba39850863 Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Tue, 13 Oct 2026 16:09:43 +0200 Subject: [PATCH 11/14] vm: release the previous binding of an import uc_vm_insn_import() and uc_vm_insn_dynload() store the binding of an imported name in an upvalue slot of the running closure, and overwrite what the slot holds. A script can call the function that loadfile() returns more than once. Each call after the first one runs the import instructions of the same closure again and leaks the upvalue reference of the call before it, for a dynamically loaded module together with the reference to the imported value. Release the binding that a slot holds before it gets a new one. Signed-off-by: Felix Fietkau --- vm.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/vm.c b/vm.c index 95f28b426..8ef78518d 100644 --- a/vm.c +++ b/vm.c @@ -2997,6 +2997,16 @@ uc_vm_insn_delete(uc_vm_t *vm, uc_vm_insn_t insn) ucv_put(v); } +/* a closure that runs again holds the binding of its previous run */ +static void +uc_vm_import_bind(uc_closure_t *closure, size_t slot, uc_upvalref_t *ref) +{ + if (closure->upvals[slot]) + ucv_put(&closure->upvals[slot]->header); + + closure->upvals[slot] = ref; +} + static void uc_vm_insn_import(uc_vm_t *vm, uc_vm_insn_t insn) { @@ -3044,14 +3054,14 @@ uc_vm_insn_import(uc_vm_t *vm, uc_vm_insn_t insn) /* module export available, patch into upvalue */ else if (from < prog->exports.count && prog->exports.entries[from]) { - frame->closure->upvals[to] = prog->exports.entries[from]; ucv_get(&prog->exports.entries[from]->header); + uc_vm_import_bind(frame->closure, to, prog->exports.entries[from]); } /* module export missing, e.g. due to premature return in module, * patch up dummy upvalue ref with `null` value */ else { - frame->closure->upvals[to] = (uc_upvalref_t *)ucv_upvalref_new(0); + uc_vm_import_bind(frame->closure, to, (uc_upvalref_t *)ucv_upvalref_new(0)); frame->closure->upvals[to]->closed = true; } } @@ -3139,7 +3149,7 @@ uc_vm_insn_dynload(uc_vm_t *vm, uc_vm_insn_t insn) ucv_put(name); - frame->closure->upvals[to] = (uc_upvalref_t *)ucv_upvalref_new(0); + uc_vm_import_bind(frame->closure, to, (uc_upvalref_t *)ucv_upvalref_new(0)); frame->closure->upvals[to]->closed = true; frame->closure->upvals[to]->value = ucv_get(export); From 6e404de5c0b1b7645032248b2bae2ca736f6bdb1 Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Tue, 13 Oct 2026 16:10:34 +0200 Subject: [PATCH 12/14] vm: let the main closure of a program own its module exports Since commit d64d5d685d86 ("vm: maintain export symbol tables per program"), uc_vm_insn_export() stores the exports of statically linked modules in the program and keeps them until the program is freed. This has several defects: - An exported function is a closure that holds the program, so the table keeps the program alive. Without the collector, each loadfile() or include() of a file that imports such a module keeps its memory. - The collector does not treat the table as a root. A collection between two uc_vm_execute() calls of one program frees the exports, and the second call fails: Type error: `inc` is not a function, got null instead - A program that runs in a second VM binds the values of the first VM, also after that VM is freed. - Each run of a program appends its exports to the table again. - A call that starts while another call of the same main function is active shifts the entries, so an import that the outer call resolves afterwards binds an unrelated export. Keep the exports per closure of the entry function of a program, release them when that closure is freed, and mark them in the collector from that closure. A program that is loaded and run once is thus freed with its last closure. A function from loadfile() keeps its module instances for as long as the script holds it, so module state still carries over from one call to the next. A later or nested run of a closure runs the module bodies again, as before, but adds no entries: its imports resolve to the instances of the first run. For uc_vm_execute(), keep the closure that runs in the registry of the VM once it has module instances. A later execution of the program in that VM runs the same closure. Only the caller can tell that the program is still in use, and its reference is the one that no closure accounts for. Count the closures of a program and drop the registry entry in the collector once no other reference exists. Read the exports of a dynamically loaded module from its closure in uc_require_imports(). While here, read all name operands of a wildcard import if an export is missing. Signed-off-by: Felix Fietkau --- include/ucode/internal/program.h | 7 + include/ucode/internal/types.h | 20 ++- lib.c | 8 +- lib/debug.c | 5 +- program.c | 13 ++ tests/custom/04_modules/26_export_lifetime | 144 +++++++++++++++++++++ types.c | 72 ++++++++++- vm.c | 122 +++++++++++++++-- 8 files changed, 365 insertions(+), 26 deletions(-) create mode 100644 tests/custom/04_modules/26_export_lifetime diff --git a/include/ucode/internal/program.h b/include/ucode/internal/program.h index 0a08e4feb..81b14f42f 100644 --- a/include/ucode/internal/program.h +++ b/include/ucode/internal/program.h @@ -41,6 +41,13 @@ __hidden uc_function_t *uc_program_entry(uc_program_t *); +/* prefix of the registry keys under which a VM holds the closures that + * uc_vm_execute() runs; the address of the program follows */ +#define UC_PROGRAM_MAIN_KEY "vm.main." + +__hidden uc_export_instance_t *uc_program_instance_get(uc_program_t *, uc_closure_t *, bool); +__hidden void uc_program_instance_free(uc_program_t *, uc_closure_t *, bool); + __hidden uc_function_t *uc_program_function_new(uc_program_t *, const char *, uc_source_t *, size_t); __hidden size_t uc_program_function_id(uc_program_t *, uc_function_t *); __hidden uc_function_t *uc_program_function_load(uc_program_t *, size_t); diff --git a/include/ucode/internal/types.h b/include/ucode/internal/types.h index 63e5b10f3..f9a1c0127 100644 --- a/include/ucode/internal/types.h +++ b/include/ucode/internal/types.h @@ -121,12 +121,30 @@ typedef struct { uc_declare_vector(uc_sources_t, uc_source_t *); +/* The exports of the modules that a program links statically, as created by + * the runs of one closure of its entry function. */ +typedef struct { + uc_closure_t *closure; + uc_modexports_t exports; +} uc_export_instance_t; + +typedef struct { + uc_closure_t *closure; + size_t cursor; + bool keep; +} uc_program_run_t; + +uc_declare_vector(uc_export_instances_t, uc_export_instance_t); +uc_declare_vector(uc_program_runs_t, uc_program_run_t); + struct uc_program { uc_value_t header; uc_value_list_t constants; uc_weakref_t functions; uc_sources_t sources; - uc_modexports_t exports; + uc_export_instances_t instances; + uc_program_runs_t runs; + uint32_t closure_refs; }; diff --git a/lib.c b/lib.c index 15110ea7d..0537c26b5 100644 --- a/lib.c +++ b/lib.c @@ -2982,19 +2982,21 @@ uc_require_imports(uc_vm_t *vm, uc_value_t *closure) { uc_function_t *fn = ucv_as_closure(closure)->function; uc_source_t *src = uc_program_function_source(fn); + uc_export_instance_t *instance = + uc_program_instance_get(fn->program, ucv_as_closure(closure), false); uc_value_t *ns = ucv_object_new(vm); size_t i = src->exports.offset; uc_vector_foreach(&src->exports, sym) { - if (i >= fn->program->exports.count) + if (!instance || i >= instance->exports.count) break; if (ucv_type(*sym) == UC_STRING) ucv_object_add(ns, ucv_string_get(*sym), - ucv_get(&fn->program->exports.entries[i++]->header)); + ucv_get(&instance->exports.entries[i++]->header)); else if (ucv_type(*sym) == UC_NULL) ucv_object_add(ns, "default", - ucv_get(&fn->program->exports.entries[i++]->header)); + ucv_get(&instance->exports.entries[i++]->header)); } ucv_set_constant(ns, true); diff --git a/lib/debug.c b/lib/debug.c index d7d1a7272..374ccc623 100644 --- a/lib/debug.c +++ b/lib/debug.c @@ -204,8 +204,9 @@ uc_debug_discover_ucv(uc_value_t *uv, struct lh_table *seen) for (i = 0; i < program->sources.count; i++) uc_debug_discover_ucv(&program->sources.entries[i]->header, seen); - for (i = 0; i < program->exports.count; i++) - uc_debug_discover_ucv(&program->exports.entries[i]->header, seen); + uc_vector_foreach(&program->instances, instance) + for (i = 0; i < instance->exports.count; i++) + uc_debug_discover_ucv(&instance->exports.entries[i]->header, seen); break; diff --git a/program.c b/program.c index 1920c09a3..56ceb7886 100644 --- a/program.c +++ b/program.c @@ -1044,6 +1044,19 @@ uc_program_entry(uc_program_t *program) return (uc_function_t *)program->functions.prev; } +uc_export_instance_t * +uc_program_instance_get(uc_program_t *program, uc_closure_t *closure, bool create) +{ + uc_vector_foreach(&program->instances, instance) + if (instance->closure == closure) + return instance; + + if (!create) + return NULL; + + return uc_vector_push(&program->instances, { .closure = closure }); +} + uc_value_t * uc_program_main(uc_vm_t *vm, uc_program_t *prog) { diff --git a/tests/custom/04_modules/26_export_lifetime b/tests/custom/04_modules/26_export_lifetime new file mode 100644 index 000000000..82cb333f2 --- /dev/null +++ b/tests/custom/04_modules/26_export_lifetime @@ -0,0 +1,144 @@ +The exports of statically linked modules belong to the main function of a +program and live as long as that function. A program that is loaded and run +once does not wait for the collector, also if its module exports a function. +Each later call of the function binds the module instances of the first +call, so module state carries over. A call that starts while another call of +the function is active shares these instances, and the imports that the +outer call resolves afterwards bind the correct exports. Functions imported +during a call stay valid after it. + +-- Testcase -- +import * as fs from 'fs'; + +let live = gc("count"); + +for (let i = 0; i < 100; i++) + loadfile("files/counter.uc")(); + +print("live values after 100 loads: ", gc("count") - live, "\n"); + +let fn = loadfile("files/counter.uc"); +print("repeated call: ", fn(), " ", fn(), " ", fn(), "\n"); + +gc(); +print("call after a collection: ", fn(), "\n"); + +live = gc("count"); + +for (let i = 0; i < 100; i++) + fn(); + +print("live values after 100 calls: ", gc("count") - live, "\n"); + +let inc = loadfile("files/keep.uc")(); +print("kept function: ", inc(), " ", inc(), "\n"); + +global.depth = 0; +global.nested = loadfile("files/nested.uc"); +print("nested run: ", global.nested(), "\n"); + +print("tail call: ", loadfile("files/tail.uc")(), "\n"); + +let failing = loadfile("files/failing.uc"); + +for (let i = 0; i < 2; i++) { + try { + failing(); + } + catch (e) { + print("failing run: ", e.message, "\n"); + } +} + +let dynamic = import("dynamic"); +print("dynamic import: ", dynamic.inc(), " ", dynamic.inc(), " ", dynamic.name, "\n"); + +// A file that the body of a dynamically loaded module loads is a program of +// its own, not a module. +let loader = import("loader"); +print("loaded in a module body: ", loader.fn(), " ", loader.fn(), " ", loader.fn(), "\n"); + +system([ fs.readlink("/proc/self/exe"), "-c", "-o", "files/precompiled.uc", "files/counter.uc" ]); +fn = loadfile("files/precompiled.uc"); +print("precompiled: ", fn(), " ", fn(), "\n"); +-- End -- + +-- File mod.uc -- +let n = 0; + +export function inc() { return ++n; }; +export const name = "mod"; +-- End -- + +-- File counter.uc -- +import { inc } from "./mod.uc"; + +return inc(); +-- End -- + +-- File keep.uc -- +import { inc } from "./mod.uc"; + +return inc; +-- End -- + +-- File nested.uc -- +import { inc } from "./mod.uc"; + +let d = global.depth++; +let count = inc(); +let inner = (d < 2) ? global.nested() : "-"; + +import * as other from "./other.uc"; +import { value } from "./other.uc"; + +return `${d}:${count}:${inc()}:${value()}:${other.default} ${inner}`; +-- End -- + +-- File other.uc -- +export function value() { return "value"; }; +export default "default"; +-- End -- + +-- File tail.uc -- +import { inc } from "./mod.uc"; + +inc(); +return inc(); +-- End -- + +-- File failing.uc -- +import { inc } from "./mod.uc"; + +die(`count ${inc()}`); +-- End -- + +-- File loader.uc -- +export const fn = loadfile("files/counter.uc"); +-- End -- + +-- File dynamic.uc -- +import { inc as mod_inc, name as mod_name } from "./mod.uc"; + +export function inc() { return mod_inc(); }; +export const name = mod_name; +-- End -- + +-- Args -- +-R -L files/ +-- End -- + +-- Expect stdout -- +live values after 100 loads: 0 +repeated call: 1 2 3 +call after a collection: 4 +live values after 100 calls: 0 +kept function: 1 2 +nested run: 0:1:6:value:default 1:2:5:value:default 2:3:4:value:default - +tail call: 2 +failing run: count 1 +failing run: count 2 +dynamic import: 1 2 mod +loaded in a module body: 1 2 3 +precompiled: 1 2 +-- End -- diff --git a/types.c b/types.c index 6427d5033..effa531c2 100644 --- a/types.c +++ b/types.c @@ -157,6 +157,7 @@ ucv_gc_mark(uc_value_t *uv) uc_object_t *object; uc_array_t *array; uc_resource_type_t *restype; + uc_export_instance_t *instance; uc_program_t *program; uc_value_t **values; struct lh_entry *entry; @@ -202,6 +203,13 @@ ucv_gc_mark(uc_value_t *uv) for (i = 0; i < function->nupvals; i++) ucv_gc_mark(&closure->upvals[i]->header); + /* The exports are values of the VM of this closure. A program can + * hold instances of other VMs, so they are not marked from there. */ + instance = uc_program_instance_get(function->program, closure, false); + + for (i = 0; instance && i < instance->exports.count; i++) + ucv_gc_mark(&instance->exports.entries[i]->header); + ucv_gc_mark(&function->program->header); break; @@ -239,9 +247,6 @@ ucv_gc_mark(uc_value_t *uv) for (i = 0; i < program->sources.count; i++) ucv_gc_mark(&program->sources.entries[i]->header); - for (i = 0; i < program->exports.count; i++) - ucv_gc_mark(&program->exports.entries[i]->header); - break; default: @@ -249,6 +254,26 @@ ucv_gc_mark(uc_value_t *uv) } } +void +uc_program_instance_free(uc_program_t *program, uc_closure_t *closure, bool retain) +{ + uc_export_instance_t *instance = uc_program_instance_get(program, closure, false); + uc_modexports_t exports; + size_t i; + + if (!instance) + return; + + /* the release of an export can free another closure of the program */ + exports = instance->exports; + *instance = program->instances.entries[--program->instances.count]; + + for (i = 0; i < exports.count; i++) + ucv_put_value(&exports.entries[i]->header, retain); + + uc_vector_clear(&exports); +} + void ucv_free(uc_value_t *uv, bool retain) { @@ -301,10 +326,13 @@ ucv_free(uc_value_t *uv, bool retain) function = closure->function; ref = &closure->ref; + uc_program_instance_free(function->program, closure, retain); + for (i = 0; i < function->nupvals; i++) if (closure->upvals[i]) ucv_put_value(&closure->upvals[i]->header, retain); + function->program->closure_refs--; ucv_put_value(&function->program->header, retain); break; @@ -352,11 +380,10 @@ ucv_free(uc_value_t *uv, bool retain) for (i = 0; i < program->sources.count; i++) ucv_put_value(&program->sources.entries[i]->header, retain); - for (i = 0; i < program->exports.count; i++) - ucv_put_value(&program->exports.entries[i]->header, retain); - + /* each instance belongs to a closure, which holds the program */ uc_vector_clear(&program->sources); - uc_vector_clear(&program->exports); + uc_vector_clear(&program->instances); + uc_vector_clear(&program->runs); break; case UC_SOURCE: @@ -1231,6 +1258,7 @@ ucv_closure_new(uc_vm_t *vm, uc_function_t *function, bool arrow_fn) } uc_program_get(function->program); + function->program->closure_refs++; return &closure->header; } @@ -3028,6 +3056,33 @@ ucv_key_rawdelete(uc_vm_t *vm, uc_value_t *scope, uc_value_t *key) return ucv_object_del_key(vm, scope, key); } +/* The registry holds the closure that uc_vm_execute() runs, so that a later + * execution of the program finds the module instances of the earlier one. + * Only the caller of uc_vm_execute() can tell that the program is still in + * use, and it holds a program reference that is no closure. Without such a + * reference no later execution can follow. */ +static bool +ucv_gc_main_release(uc_vm_t *vm) +{ + uc_program_t *program; + + ucv_object_foreach(vm->registry, key, val) { + if (strncmp(key, UC_PROGRAM_MAIN_KEY, strlen(UC_PROGRAM_MAIN_KEY))) + continue; + + program = ucv_as_closure(val)->function->program; + + if (program->header.refcount > program->closure_refs) + continue; + + ucv_object_delete(vm->registry, key); + + return true; + } + + return false; +} + static void ucv_gc_common(uc_vm_t *vm, bool final) { @@ -3042,6 +3097,9 @@ ucv_gc_common(uc_vm_t *vm, bool final) return; if (!final) { + while (vm->registry && ucv_gc_main_release(vm)) + ; + /* mark reachable objects */ ucv_gc_mark(vm->globals); ucv_gc_mark(vm->registry); diff --git a/vm.c b/vm.c index 8ef78518d..335f69fe8 100644 --- a/vm.c +++ b/vm.c @@ -631,6 +631,70 @@ uc_vm_call_native(uc_vm_t *vm, uc_value_t *ctx, uc_cfunction_t *fptr, bool mcall ucv_put(res); } +/* The exports of the modules that a program links statically belong to the + * closure of its entry function that runs them, and live as long as that + * closure. An exported function holds the program, so exports that the program + * owned itself would keep the program alive. */ +static bool +uc_vm_closure_is_entry(uc_closure_t *closure) +{ + uc_function_t *function = closure->function; + + return function->program->functions.prev == &function->progref; +} + +static void +uc_vm_run_begin(uc_closure_t *closure, bool keep) +{ + if (uc_vm_closure_is_entry(closure)) + uc_vector_push(&closure->function->program->runs, { .closure = closure, .keep = keep }); +} + +static void +uc_vm_run_end(uc_closure_t *closure) +{ + if (uc_vm_closure_is_entry(closure)) + uc_vector_pop(&closure->function->program->runs); +} + +typedef char uc_vm_main_key_t[sizeof(UC_PROGRAM_MAIN_KEY) + sizeof("0x") + 2 * sizeof(void *)]; + +static const char * +uc_vm_main_key(uc_program_t *program, uc_vm_main_key_t key) +{ + snprintf(key, sizeof(uc_vm_main_key_t), UC_PROGRAM_MAIN_KEY "%p", (void *)program); + + return key; +} + +/* The caller of uc_vm_execute() holds the program only. Keep the closure that + * runs in the registry once it has module instances, so that a later execution + * in this VM runs the same closure and finds them. */ +static void +uc_vm_main_keep(uc_vm_t *vm, uc_program_t *program, uc_program_run_t *run) +{ + uc_vm_main_key_t key; + + if (!run->keep) + return; + + run->keep = false; + uc_vm_registry_set(vm, uc_vm_main_key(program, key), ucv_get(&run->closure->header)); +} + +static uc_modexports_t * +uc_vm_run_exports(uc_program_t *program, uc_program_run_t *run, bool create) +{ + uc_export_instance_t *instance; + + if (!run) + return NULL; + + instance = uc_program_instance_get(program, run->closure, create); + + return instance ? &instance->exports : NULL; +} + /* Replaces the current call frame by an invocation frame for the given closure, * reusing the stack slots formerly occupied by the current frame. * @@ -689,6 +753,11 @@ uc_vm_frame_reinit(uc_vm_t *vm, uc_closure_t *closure, uc_value_t *ctx, * occupies the slot below the invoked function value. the tco counter records * that this frame replaced the tail calling one, so stack traces can indicate * the collapsed frames */ + if (frame->closure) + uc_vm_run_end(frame->closure); + + uc_vm_run_begin(closure, false); + frame->stackframe = blockdst + (mcall ? 1 : 0); frame->cfunction = NULL; frame->closure = closure; @@ -926,6 +995,8 @@ uc_vm_call_function(uc_vm_t *vm, uc_value_t *ctx, uc_value_t *fno, bool mcall, .strict = function->strict }); + uc_vm_run_begin(closure, false); + if (vm->trace) uc_vm_frame_dump(vm, frame); @@ -3012,7 +3083,9 @@ uc_vm_insn_import(uc_vm_t *vm, uc_vm_insn_t insn) { uc_callframe_t *frame = uc_vm_current_frame(vm); uc_program_t *prog = uc_vm_current_program(vm); - uint16_t from = vm->arg.u32 & 0xffff; + uc_modexports_t *exports = uc_vm_run_exports(prog, uc_vector_last(&prog->runs), false); + size_t count = exports ? exports->count : 0; + size_t from = vm->arg.u32 & 0xffff; uint16_t to = vm->arg.u32 >> 16; uc_value_t *name, *modobj; uint32_t cidx; @@ -3025,10 +3098,10 @@ uc_vm_insn_import(uc_vm_t *vm, uc_vm_insn_t insn) /* instruction is followed by u16 containing the offset of the * first module export and `from` times u32 values containing * the constant indexes of the names */ - for (from = frame->ip[0] * 0x100 + frame->ip[1], frame->ip += 2; - from < prog->exports.count && to > 0; - from++, to--) { + from = frame->ip[0] * 0x100 + frame->ip[1]; + /* read all name operands, also those of a missing export */ + for (frame->ip += 2; to > 0; from++, to--) { cidx = ( frame->ip[0] * 0x1000000UL + frame->ip[1] * 0x10000UL + @@ -3038,11 +3111,14 @@ uc_vm_insn_import(uc_vm_t *vm, uc_vm_insn_t insn) frame->ip += 4; + if (from >= count || !exports->entries[from]) + continue; + name = uc_program_get_constant(uc_vm_current_program(vm), cidx); - if (ucv_type(name) == UC_STRING && prog->exports.entries[from]) + if (ucv_type(name) == UC_STRING) ucv_object_add(modobj, ucv_string_get(name), - ucv_get(&prog->exports.entries[from]->header)); + ucv_get(&exports->entries[from]->header)); ucv_put(name); } @@ -3053,9 +3129,9 @@ uc_vm_insn_import(uc_vm_t *vm, uc_vm_insn_t insn) } /* module export available, patch into upvalue */ - else if (from < prog->exports.count && prog->exports.entries[from]) { - ucv_get(&prog->exports.entries[from]->header); - uc_vm_import_bind(frame->closure, to, prog->exports.entries[from]); + else if (from < count && exports->entries[from]) { + ucv_get(&exports->entries[from]->header); + uc_vm_import_bind(frame->closure, to, exports->entries[from]); } /* module export missing, e.g. due to premature return in module, @@ -3071,9 +3147,23 @@ uc_vm_insn_export(uc_vm_t *vm, uc_vm_insn_t insn) { uc_callframe_t *frame = uc_vm_current_frame(vm); uc_program_t *prog = uc_vm_current_program(vm); - uc_upvalref_t *ref = uc_vm_capture_upval(vm, frame->stackframe + vm->arg.u32); + uc_program_run_t *run = uc_vector_last(&prog->runs); + uc_modexports_t *exports = uc_vm_run_exports(prog, run, true); + uc_upvalref_t *ref; + + if (!exports) + return; + + uc_vm_main_keep(vm, prog, run); - uc_vector_push(&prog->exports, ref); + /* A later run of the closure runs the module bodies again. Its imports + * resolve to the instances of the first run, which stay in place. */ + if (run->cursor++ < exports->count) + return; + + ref = uc_vm_capture_upval(vm, frame->stackframe + vm->arg.u32); + + uc_vector_push(exports, ref); ucv_get(&ref->header); } @@ -3211,8 +3301,10 @@ uc_vm_callframe_pop(uc_vm_t *vm) ucv_put(uc_vm_stack_pop(vm)); /* release function */ - if (frame->closure) + if (frame->closure) { + uc_vm_run_end(frame->closure); ucv_put(&frame->closure->header); + } if (frame->cfunction) ucv_put(&frame->cfunction->header); @@ -3635,7 +3727,9 @@ uc_vm_execute_chunk(uc_vm_t *vm) uc_vm_status_t uc_vm_execute(uc_vm_t *vm, uc_program_t *program, uc_value_t **retval) { - uc_closure_t *closure = ucv_as_closure(uc_program_main(vm, program)); + uc_vm_main_key_t key; + uc_value_t *kept = uc_vm_registry_get(vm, uc_vm_main_key(program, key)); + uc_closure_t *closure = ucv_as_closure(kept ? ucv_get(kept) : uc_program_main(vm, program)); uc_function_t *fn = closure->function; uc_vm_status_t status; uc_callframe_t *frame; @@ -3651,6 +3745,8 @@ uc_vm_execute(uc_vm_t *vm, uc_program_t *program, uc_value_t **retval) .strict = fn->strict }); + uc_vm_run_begin(closure, !kept); + if (vm->trace) { buf = xprintbuf_new(); From 0adb7ba6b4d16d2870a96d8be3faef2ed92cd714 Mon Sep 17 00:00:00 2001 From: Jo-Philipp Wich Date: Fri, 16 Oct 2026 12:19:45 +0200 Subject: [PATCH 13/14] vm: do not interrupt the exports of a module The VM runs the handlers of pending signals after each instruction, and uc_vm_insn_export() adds one export to the module instance of the running closure. A signal handler that raises an exception between two export instructions of one module leaves a part of that module instance for the next run of the closure. The compiler emits the exports of a module as one block. Do not run the signal handlers between two export instructions, so that no handler runs inside that block. Signed-off-by: Felix Fietkau [reworked: the main loop skips the signal dispatch between exports] Signed-off-by: Jo-Philipp Wich --- vm.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vm.c b/vm.c index 335f69fe8..0329fdf56 100644 --- a/vm.c +++ b/vm.c @@ -3710,6 +3710,12 @@ uc_vm_execute_chunk(uc_vm_t *vm) } } + /* the compiler emits the exports of a module as one block, do not run + * signal handlers inside of it, as an exception raised by one would + * leave a part of the module instance behind */ + if (insn == I_EXPORT && frame->ip[0] == I_EXPORT) + continue; + /* run handler for signal(s) delivered during previous instruction */ if (uc_vm_signal_dispatch(vm) != EXCEPTION_NONE) goto exception; From 8cfcffe5abcdc939cfd791724d7eafd3ad1713bc Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Tue, 13 Oct 2026 13:46:13 +0200 Subject: [PATCH 14/14] lib: find the exports of a precompiled module by their count uc_require_imports() takes the position of the own exports of a module from the export offset of its source. uc_program_write() does not store that offset, so it is zero for a precompiled module. If such a module links another module statically, its namespace object holds the exports of the linked module under the names of its own exports. A module exports its own symbols last, after those of the modules that it links. Take the position from the number of own exports, which the source and the bytecode both record. Signed-off-by: Felix Fietkau --- lib.c | 11 ++++- .../04_modules/27_precompiled_module_exports | 40 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 tests/custom/04_modules/27_precompiled_module_exports diff --git a/lib.c b/lib.c index 0537c26b5..828af1f3c 100644 --- a/lib.c +++ b/lib.c @@ -2985,7 +2985,16 @@ uc_require_imports(uc_vm_t *vm, uc_value_t *closure) uc_export_instance_t *instance = uc_program_instance_get(fn->program, ucv_as_closure(closure), false); uc_value_t *ns = ucv_object_new(vm); - size_t i = src->exports.offset; + size_t own = 0, i = 0; + + /* The module exports its own symbols last, after those of the modules + * that it links. A precompiled module does not record their offset. */ + uc_vector_foreach(&src->exports, sym) + if (ucv_type(*sym) == UC_STRING || ucv_type(*sym) == UC_NULL) + own++; + + if (instance && instance->exports.count > own) + i = instance->exports.count - own; uc_vector_foreach(&src->exports, sym) { if (!instance || i >= instance->exports.count) diff --git a/tests/custom/04_modules/27_precompiled_module_exports b/tests/custom/04_modules/27_precompiled_module_exports new file mode 100644 index 000000000..06af45946 --- /dev/null +++ b/tests/custom/04_modules/27_precompiled_module_exports @@ -0,0 +1,40 @@ +A precompiled module that links another module statically exports its own +symbols, like the module does when it is loaded from source. The exports of +the linked module come first in the export table and are not part of the +namespace. + +-- Testcase -- +import { readlink } from 'fs'; + +function show(label, mod) { + print(label, ": ", sort(keys(mod)), " ", mod.own(), " ", mod.name, " ", mod.default, "\n"); +} + +system([ readlink("/proc/self/exe"), "-cmodule", "-o", "files/precompiled.uc", "files/outer.uc" ]); + +show("source", import("outer")); +show("precompiled", import("precompiled")); +-- End -- + +-- File inner.uc -- +export function helper() { return "inner helper"; }; +export const name = "inner"; +export default "inner default"; +-- End -- + +-- File outer.uc -- +import { helper } from "./inner.uc"; + +export function own() { return `outer own, ${helper()}`; }; +export const name = "outer"; +export default "outer default"; +-- End -- + +-- Args -- +-R -L files/ +-- End -- + +-- Expect stdout -- +source: [ "default", "name", "own" ] outer own, inner helper outer outer default +precompiled: [ "default", "name", "own" ] outer own, inner helper outer outer default +-- End --