module load / compiler fixes

compiler.c

@@ -3728,7 +3728,7 @@ uc_compiler_compile_module_source(uc_compiler_t *compiler, const char *modname,
 	};
 
 	size_t i, load_idx = 0, n_imports = 0, n_exports = 0;
-	bool loaded = false;
+	bool loaded = false, res;
 	uc_value_t *import;
 	ssize_t slot;
 
@@ -3757,16 +3757,34 @@ uc_compiler_compile_module_source(uc_compiler_t *compiler, const char *modname,
 	if (!loaded) {
 		/* We do not yet support linking precompiled modules at compile time,
 		   turn static import operation into dynamic load one. */
-		if (uc_source_type_test(source) == UC_SOURCE_TYPE_PRECOMPILED)
+		switch (uc_source_begin(source)) {
+		case UC_SOURCE_TYPE_PRECOMPILED:
+			uc_source_reset(source);
+
 			return uc_compiler_compile_dynload(compiler, modname, imports);
 
+		case UC_SOURCE_TYPE_UNAVAILABLE:
+			if (errp)
+				xasprintf(errp, "Unable to open source file %s: %s\n",
+					source->filename, strerror(errno));
+
+			return false;
+
+		default:
+			break;
+		}
+
 		load_idx = uc_program_function_id(compiler->program,
 			uc_program_function_last(compiler->program)) + 1;
 
 		source->exports.offset = (size_t)-1;
 		n_exports = uc_compiler_get_exportnum(compiler);
 
-		if (!uc_compile_from_source(&config, source, compiler->program, errp))
+		res = uc_compile_from_source(&config, source, compiler->program, errp);
+
+		uc_source_reset(source);
+
+		if (!res)
 			return false;
 
 		/* The linked module's own exports are the last ones allocated while
@@ -4108,6 +4126,7 @@ uc_compiler_compile_import(uc_compiler_t *compiler)
 		                                       &compiler->parser->prev);
 
 		uc_compiler_emit_insn(compiler, 0, I_POP);
+		ucv_put(namelist);
 
 		return TK_SCOL;
 	}
@@ -4317,7 +4336,7 @@ uc_compile(uc_parse_config_t *config, uc_source_t *source, char **errp)
 	if (!config)
 		config = &uc_default_parse_config;
 
-	switch (uc_source_type_test(source)) {
+	switch (uc_source_begin(source)) {
 	case UC_SOURCE_TYPE_PLAIN:
 		prog = uc_compile_from_source(config, source, NULL, errp);
 		break;
@@ -4326,12 +4345,21 @@ uc_compile(uc_parse_config_t *config, uc_source_t *source, char **errp)
 		prog = uc_compile_from_bytecode(config, source, errp);
 		break;
 
+	case UC_SOURCE_TYPE_UNAVAILABLE:
+		if (errp)
+			xasprintf(errp, "Unable to open source file %s: %s\n",
+				source->filename, strerror(errno));
+
+		break;
+
 	default:
 		if (errp)
 			xasprintf(errp, "Unrecognized source type\n");
 
 		break;
 	}
 
+	uc_source_reset(source);
+
 	return prog;
 }

include/ucode/internal/lexer.h

@@ -174,8 +174,8 @@ typedef struct {
 		size_t count;
 		char *entries;
 	} buffer;
-	unsigned char *rbuf;
-	size_t rlen, rpos;
+	const uint8_t *rbuf;
+	size_t rlen, rpos, off;
 } uc_lexer_t;
 
 

include/ucode/internal/program.h

@@ -41,6 +41,13 @@
 
 __hidden uc_function_t *uc_program_entry(uc_program_t *);
 
+/* prefix of the registry keys under which a VM holds the closures that
+ * uc_vm_execute() runs; the address of the program follows */
+#define UC_PROGRAM_MAIN_KEY "vm.main."
+
+__hidden uc_export_instance_t *uc_program_instance_get(uc_program_t *, uc_closure_t *, bool);
+__hidden void uc_program_instance_free(uc_program_t *, uc_closure_t *, bool);
+
 __hidden uc_function_t *uc_program_function_new(uc_program_t *, const char *, uc_source_t *, size_t);
 __hidden size_t uc_program_function_id(uc_program_t *, uc_function_t *);
 __hidden uc_function_t *uc_program_function_load(uc_program_t *, size_t);

include/ucode/internal/source.h

@@ -14,18 +14,33 @@
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* Internal source API. */
-
 #ifndef UCODE_INTERNAL_SOURCE_H
 #define UCODE_INTERNAL_SOURCE_H
 
 #include "ucode/source.h"
 #include "ucode/internal/types.h"
 #include "ucode/internal/util.h"
 
+__hidden uc_source_t *uc_source_new_path(const char *path);
+
 __hidden void uc_source_free(uc_source_t *source);
 
-__hidden uc_source_type_t uc_source_type_test(uc_source_t *source);
+__hidden const uint8_t *uc_source_read_block(uc_source_t *source, size_t *len,
+                                             size_t *off);
+
+__hidden size_t uc_source_read_line(uc_source_t *source, size_t off, char *line,
+                                    size_t size, size_t *lpos, bool *truncated);
+
+char *uc_source_read_content(uc_source_t *source, size_t *len);
+
+__hidden size_t uc_source_read(uc_source_t *source, void *buf, size_t len);
+
+__hidden size_t uc_source_offset(uc_source_t *source);
+__hidden size_t uc_source_remaining(uc_source_t *source);
+
+__hidden void uc_source_reset(uc_source_t *source);
+
+__hidden uc_source_type_t uc_source_begin(uc_source_t *source);
 
 __hidden void uc_source_line_next(uc_source_t *source);
 __hidden void uc_source_line_update(uc_source_t *source, size_t off);
@@ -35,6 +50,6 @@ __hidden void uc_source_runpath_set(uc_source_t *source, const char *runpath);
 __hidden bool uc_source_export_add(uc_source_t *source, uc_value_t *name);
 __hidden ssize_t uc_source_export_lookup(uc_source_t *source, uc_value_t *name);
 
-__hidden const char *uc_source_callee_expr(uc_source_t *source, size_t end);
+__hidden char *uc_source_callee_expr(uc_source_t *source, size_t end);
 
 #endif /* UCODE_INTERNAL_SOURCE_H */

include/ucode/internal/types.h

@@ -43,9 +43,11 @@ uc_declare_vector(uc_lineinfo_t, uint8_t);
 
 struct uc_source {
 	uc_value_t header;
-	char *filename, *runpath, *buffer;
+	char *filename, *runpath, *buffer, *path;
 	FILE *fp;
-	size_t off;
+	uint8_t *rbuf;
+	size_t off, size;
+	bool done, sized;
 	uc_lineinfo_t lineinfo;
 	struct {
 		size_t count, offset;
@@ -119,12 +121,30 @@ typedef struct {
 
 uc_declare_vector(uc_sources_t, uc_source_t *);
 
+/* The exports of the modules that a program links statically, as created by
+ * the runs of one closure of its entry function. */
+typedef struct {
+	uc_closure_t *closure;
+	uc_modexports_t exports;
+} uc_export_instance_t;
+
+typedef struct {
+	uc_closure_t *closure;
+	size_t cursor;
+	bool keep;
+} uc_program_run_t;
+
+uc_declare_vector(uc_export_instances_t, uc_export_instance_t);
+uc_declare_vector(uc_program_runs_t, uc_program_run_t);
+
 struct uc_program {
 	uc_value_t header;
 	uc_value_list_t constants;
 	uc_weakref_t functions;
 	uc_sources_t sources;
-	uc_modexports_t exports;
+	uc_export_instances_t instances;
+	uc_program_runs_t runs;
+	uint32_t closure_refs;
 };
 
 

include/ucode/source.h

@@ -30,13 +30,20 @@
 typedef enum {
 	UC_SOURCE_TYPE_PLAIN = 0,
 	UC_SOURCE_TYPE_PRECOMPILED = 1,
+	UC_SOURCE_TYPE_UNAVAILABLE = 2,
 } uc_source_type_t;
 
 /* Create a source from a file or an in-memory buffer. The buffer variant
- * takes ownership of `buf` (it is freed when the source is released). */
+ * takes ownership of `buf` (it is freed when the source is released). A
+ * file that cannot seek, such as a pipe, is read into a buffer. */
 uc_source_t *uc_source_new_file(const char *path);
 uc_source_t *uc_source_new_buffer(const char *name, char *buf, size_t len);
 
+/* Create a buffer source from the remaining content of `fp`, which is read to
+ * its end. The caller keeps `fp` and closes it. An empty input results in a
+ * buffer holding a single newline. */
+uc_source_t *uc_source_new_slurp(const char *name, FILE *fp);
+
 /* Fetch the line containing the given byte offset; *offset is set to the
  * offset of the first character of that line. Returns the 1-based line number. */
 size_t uc_source_get_line(uc_source_t *source, size_t *offset);

include/ucode/types.h

@@ -227,7 +227,7 @@ struct uc_vm {
 	uc_parse_config_t *config;
 	uc_value_t *globals;
 	uc_value_t *registry;
-	struct lh_table *sources;
+	void *reserved;
 	uc_weakref_t values;
 	uc_resource_types_t restypes;
 	uc_breakpoints_t breakpoints;

lexer.c

@@ -68,13 +68,18 @@ static const struct keyword reserved_words[] = {
 
 static int
 fill_buf(uc_lexer_t *lex) {
-	lex->rbuf = xrealloc(lex->rbuf, 128);
-	lex->rlen = fread(lex->rbuf, 1, 128, lex->source->fp);
+	size_t off;
+
+	lex->rbuf = uc_source_read_block(lex->source, &lex->rlen, &off);
 	lex->rpos = 0;
 
-	if (!lex->rlen)
+	if (!lex->rbuf)
 		return EOF;
 
+	/* everything in front of the block has been consumed, so this merely
+	 * establishes the offset of the first byte of code */
+	lex->off = off;
+
 	lex->rpos++;
 
 	return (int)lex->rbuf[0];
@@ -87,7 +92,7 @@ update_line(uc_lexer_t *lex, int ch) {
 	else if (ch != EOF)
 		uc_source_line_update(lex->source, 1);
 
-	lex->source->off++;
+	lex->off++;
 
 	return ch;
 }
@@ -131,11 +136,11 @@ emit_op(uc_lexer_t *lex, ssize_t pos, int type, uc_value_t *uv)
 	lex->curr.uv = uv;
 
 	if (pos < 0)
-		lex->curr.pos = lex->source->off + pos;
+		lex->curr.pos = lex->off + pos;
 	else
 		lex->curr.pos = (size_t)pos;
 
-	lex->curr.end = lex->source->off;
+	lex->curr.end = lex->off;
 
 	return &lex->curr;
 }
@@ -166,7 +171,7 @@ emit_buffer(uc_lexer_t *lex, ssize_t pos, int type, const char *strip_trailing_c
 static uc_token_t *
 parse_comment(uc_lexer_t *lex, int kind)
 {
-	size_t off = lex->source->off - 1;
+	size_t off = lex->off - 1;
 	int ch;
 
 	uc_vector_push(&lex->buffer, '/');
@@ -355,7 +360,7 @@ parse_string(uc_lexer_t *lex, int kind)
 	else
 		type = TK_STRING;
 
-	off = lex->source->off - 1;
+	off = lex->off - 1;
 
 	for (ch = next_char(lex); ch != EOF; ch = next_char(lex)) {
 		switch (ch) {
@@ -786,7 +791,7 @@ lex_find_token(uc_lexer_t *lex)
 
 	case '/':
 		ch = lookahead_char(lex);
-		lex->lastoff = lex->source->off - 1;
+		lex->lastoff = lex->off - 1;
 
 		if (ch == '/' || ch == '*')
 			return parse_comment(lex, ch);
@@ -876,7 +881,7 @@ lex_find_token(uc_lexer_t *lex)
 	case '\'':
 	case '"':
 	case '`':
-		lex->lastoff = lex->source->off - 1;
+		lex->lastoff = lex->off - 1;
 
 		return parse_string(lex, ch);
 
@@ -1011,7 +1016,7 @@ lex_step(uc_lexer_t *lex)
 
 			/* push out leading text */
 			tok = emit_buffer(lex, lex->lastoff, TK_TEXT, strip);
-			lex->lastoff = lex->source->off - 2;
+			lex->lastoff = lex->off - 2;
 
 			if (!tok)
 				continue;
@@ -1045,7 +1050,7 @@ lex_step(uc_lexer_t *lex)
 
 			tok = emit_op(lex, lex->lastoff, TK_COMMENT, NULL);
 
-			lex->lastoff = lex->source->off;
+			lex->lastoff = lex->off;
 			lex->state = UC_LEX_IDENTIFY_BLOCK;
 
 			return tok;
@@ -1054,13 +1059,13 @@ lex_step(uc_lexer_t *lex)
 			lex->state = UC_LEX_IDENTIFY_TOKEN;
 			lex->block = EXPRESSION;
 
-			return emit_op(lex, lex->source->off - 2, TK_LEXP, NULL);
+			return emit_op(lex, lex->off - 2, TK_LEXP, NULL);
 
 		case UC_LEX_BLOCK_STATEMENT_EMIT_TAG:
 			lex->state = UC_LEX_IDENTIFY_TOKEN;
 			lex->block = STATEMENTS;
 
-			return emit_op(lex, lex->source->off - 2, TK_LSTM, NULL);
+			return emit_op(lex, lex->off - 2, TK_LSTM, NULL);
 
 		case UC_LEX_IDENTIFY_TOKEN:
 			do { tok = lex_find_token(lex); } while (tok == NULL);
@@ -1075,7 +1080,7 @@ lex_step(uc_lexer_t *lex)
 				if (lex->modifier == UNSPEC && lex->config && lex->config->trim_blocks)
 					lex->modifier = NEWLINE;
 
-				lex->lastoff = lex->source->off;
+				lex->lastoff = lex->off;
 				lex->state = UC_LEX_IDENTIFY_BLOCK;
 				lex->block = NONE;
 
@@ -1084,7 +1089,7 @@ lex_step(uc_lexer_t *lex)
 
 			/* found end of expression block */
 			else if (lex->block == EXPRESSION && tok->type == TK_REXP) {
-				lex->lastoff = lex->source->off;
+				lex->lastoff = lex->off;
 				lex->state = UC_LEX_IDENTIFY_BLOCK;
 				lex->block = NONE;
 			}
@@ -1140,7 +1145,7 @@ lex_step(uc_lexer_t *lex)
 		}
 	}
 
-	return emit_op(lex, lex->source->off, TK_EOF, NULL);
+	return emit_op(lex, lex->off, TK_EOF, NULL);
 }
 
 void
@@ -1156,6 +1161,7 @@ uc_lexer_init(uc_lexer_t *lex, uc_parse_config_t *config, uc_source_t *source)
 
 	lex->rlen = 0;
 	lex->rpos = 0;
+	lex->off = 0;
 	lex->rbuf = NULL;
 
 	lex->buffer.count = 0;
@@ -1181,8 +1187,6 @@ uc_lexer_free(uc_lexer_t *lex)
 	uc_vector_clear(&lex->templates);
 
 	uc_source_put(lex->source);
-
-	free(lex->rbuf);
 }
 
 uc_token_t *