diff --git a/daslib/jsonrpc.das b/daslib/jsonrpc.das new file mode 100644 index 000000000..bb25c910f --- /dev/null +++ b/daslib/jsonrpc.das @@ -0,0 +1,413 @@ +options gen2 +options no_unused_block_arguments = false +options no_unused_function_arguments = false +options indenting = 4 +options strict_smart_pointers = true + +module jsonrpc shared public + +//! JSON-RPC 2.0 envelope + parser, transport-agnostic. +//! +//! Implements the JSON-RPC 2.0 spec (https://www.jsonrpc.org/specification): +//! single-request and §6 batch parsing on the server side, request/notification +//! builders on the client side, response envelope/error envelope helpers, and +//! standard error code constants. No I/O and no dispatch — pair with a +//! transport (stdio loop, HTTP server, websocket) and a method dispatch +//! callback at the call site. +//! +//! The ``compact_json_whitespace`` helper is a transport-framing utility +//! (newline-delimited wires need single-line responses; ``write_json`` +//! pretty-prints). Not strictly JSON-RPC, but ships here because every +//! newline-framed transport hits the same problem. +//! +//! Permissive by default: the ``jsonrpc`` request member may be absent or +//! carry a value other than ``"2.0"`` — real-world clients are lazy about it. +//! Pass ``strict=true`` to ``parse_request`` / ``parse_batch`` / +//! ``dispatch_line`` for full §4 compliance (missing or non-"2.0" yields +//! INVALID_REQUEST). + +require daslib/json +require daslib/json_boost +require strings + + +// ===== Error code constants (JSON-RPC 2.0 §5.1) ===== + +let public PARSE_ERROR = -32700 //!< Invalid JSON received. +let public INVALID_REQUEST = -32600 //!< Not a valid request object. +let public METHOD_NOT_FOUND = -32601 //!< Method does not exist or is not available. +let public INVALID_PARAMS = -32602 //!< Invalid method params. +let public INTERNAL_ERROR = -32603 //!< Internal JSON-RPC error. + + +// ===== Parsed request shape (server side) ===== + +struct public ParsedRequest { + id_str : string //!< Serialized id ("null" / "7" / "\"abc\"") — embed verbatim into responses. + method : string //!< Method name. Empty when the request was malformed. + params_json : string //!< Raw serialized ``params`` value (e.g. ``"null"``, ``"[1,2]"``, ``"{\"x\":1}"``). + params : JsonValue? //!< Pre-parsed ``params`` JsonValue; null when absent. Saves callers a second ``read_json``. + is_notification : bool //!< True when the request omits ``id`` (spec §4.1 — MUST NOT send a response). + error_envelope : string //!< Pre-built error response for malformed input. Empty when the request was well-formed. +} + +struct public ParsedBatch { + is_batch : bool //!< True when the wire body was a JSON array. + requests : array //!< One entry per array element (or one for a single request). + framing_error : string //!< Top-level error envelope to send as-is (parse failure, empty array). Mutually exclusive with ``requests``. +} + + +// ===== Parsed response shape (client side) ===== + +struct public ParsedResponse { + id_str : string //!< Echoed id from the response. + is_success : bool //!< True when the response carries ``result``; false when it carries ``error``. + result_json : string //!< Raw ``result`` value; valid when ``is_success``. + error_code : int //!< Error code; 0 when ``is_success``. + error_msg : string //!< Error message; empty when ``is_success``. + error_data : string //!< Optional ``error.data`` field; empty when absent. + parse_error : string //!< Non-empty when the response wire itself was malformed. +} + +struct public ParsedResponseBatch { + is_batch : bool //!< True when the wire body was a JSON array. + responses : array //!< One per array element (or one for a single response). + parse_error : string //!< Top-level parse error; mutually exclusive with ``responses``. +} + + +// ===== Envelope builders (responses + errors) ===== + +def public response(id_str : string; result_json : string) : string { + //! Wrap ``result_json`` in a JSON-RPC 2.0 success response envelope. + //! ``id_str`` and ``result_json`` are embedded verbatim — both must be valid JSON. + return "\{\"jsonrpc\":\"2.0\",\"id\":{id_str},\"result\":{result_json}}" +} + +def public error(id_str : string; code : int; message : string) : string { + //! Wrap ``code`` + ``message`` in a JSON-RPC 2.0 error response envelope. + let msg_json = write_json(JV(message)) + return "\{\"jsonrpc\":\"2.0\",\"id\":{id_str},\"error\":\{\"code\":{code},\"message\":{msg_json}}}" +} + +def public error_with_data(id_str : string; code : int; message, data_json : string) : string { + //! Like ``error``, but also includes the optional ``data`` field (spec §5.1). + //! ``data_json`` is embedded verbatim — must be valid JSON. + let msg_json = write_json(JV(message)) + return "\{\"jsonrpc\":\"2.0\",\"id\":{id_str},\"error\":\{\"code\":{code},\"message\":{msg_json},\"data\":{data_json}}}" +} + +def public serialize_id(id_val : JsonValue?) : string { + //! Serialize a JSON-RPC id (string/number/null) to its wire form. + //! Returns ``"null"`` when ``id_val`` is null. + return id_val == null ? "null" : write_json(id_val) +} + + +// ===== Outgoing request builders (client side) ===== + +def public make_request(method : string; params_json : string; id : int) : string { + //! Build a JSON-RPC 2.0 request with an integer id. + //! ``params_json`` is the pre-serialized params value (``"null"``, ``"[1,2]"``, ``"{\"x\":1}"``). + let method_str = write_json(JV(method)) + return "\{\"jsonrpc\":\"2.0\",\"id\":{id},\"method\":{method_str},\"params\":{params_json}}" +} + +def public make_request(method : string; params_json : string; id : string) : string { + //! Build a JSON-RPC 2.0 request with a string id. + let method_str = write_json(JV(method)) + let id_str = write_json(JV(id)) + return "\{\"jsonrpc\":\"2.0\",\"id\":{id_str},\"method\":{method_str},\"params\":{params_json}}" +} + +def public make_notification(method : string; params_json : string) : string { + //! Build a JSON-RPC 2.0 notification (no id — server MUST NOT respond). + let method_str = write_json(JV(method)) + return "\{\"jsonrpc\":\"2.0\",\"method\":{method_str},\"params\":{params_json}}" +} + +def public make_batch(messages : array) : string { + //! Wrap pre-built request/notification strings as a JSON-RPC 2.0 §6 batch array. + //! Each element of ``messages`` must be a valid request or notification. + //! Empty input returns ``"[]"`` — note that ``[]`` is itself a -32600 invalid + //! request per the spec, so this is mostly useful for testing the rejection path. + return build_string() $(var w) { + write_char(w, int('[')) + var first = true + for (m in messages) { + if (!first) write_char(w, int(',')) + first = false + write(w, m) + } + write_char(w, int(']')) + } +} + + +// ===== Server-side parsers ===== + +def private parse_one_request(body_value : JsonValue?; strict : bool) : ParsedRequest { + var result = ParsedRequest(id_str = "null") + if (body_value == null || !(body_value.value is _object)) { + result.error_envelope = error("null", INVALID_REQUEST, "invalid request: not a JSON object") + return result + } + let body & = unsafe(body_value.value as _object) + let id_v = body?["id"] ?? null + let method_v = body?["method"] ?? null + // params_v needs the reinterpret: it's assigned to `result.params : JsonValue?`, + // and `?[]` on a const view returns `JsonValue? const` — assignment to a + // non-const field would fail. id_v / method_v above are read-only and don't need it. + var params_v = unsafe(reinterpret(body?["params"] ?? null)) + // §4: id MUST be string, number, or null. Reject objects/arrays/booleans — + // they'd also break framing since write_json would pretty-print across lines. + if (id_v != null && !(id_v.value is _string) && !(id_v.value is _number) && !(id_v.value is _longint) && !(id_v.value is _null)) { + result.error_envelope = error("null", INVALID_REQUEST, "invalid request: id must be string, number, or null") + return result + } + result.id_str = serialize_id(id_v) + if (method_v == null || !(method_v.value is _string)) { + // Don't flag as notification before method validation — `{}` or `{"method":42}` + // without id would otherwise swallow the envelope per spec §5.1. + result.error_envelope = error(result.id_str, INVALID_REQUEST, "invalid request: missing or non-string method") + return result + } + // Strict-mode jsonrpc field enforcement + if (strict) { + let jsonrpc_v = body?["jsonrpc"] ?? null + if (jsonrpc_v == null || !(jsonrpc_v.value is _string) || (jsonrpc_v.value as _string) != "2.0") { + result.error_envelope = error(result.id_str, INVALID_REQUEST, "invalid request: jsonrpc field must be \"2.0\"") + return result + } + } + // §4.2: params, if present, MUST be Array or Object. Reject scalars. + if (params_v != null && !(params_v.value is _object) && !(params_v.value is _array) && !(params_v.value is _null)) { + result.error_envelope = error(result.id_str, INVALID_PARAMS, "invalid params: must be array or object") + return result + } + result.is_notification = (id_v == null) + result.method = method_v.value as _string + result.params = params_v + result.params_json = params_v == null ? "null" : write_json(params_v) + return result +} + +def public parse_request(line : string; strict : bool = false) : ParsedRequest { + //! Parse a single JSON-RPC 2.0 request line. Pure function — testable in isolation. + //! + //! With ``strict=false`` (default), the ``jsonrpc`` field is optional and any + //! string value is accepted. With ``strict=true``, missing or non-``"2.0"`` + //! ``jsonrpc`` yields INVALID_REQUEST. All other §4 rules — id type, + //! notification semantics, required ``method``, ``params`` type — are + //! enforced in both modes. + var result = ParsedRequest(id_str = "null") + var parse_error_msg = "" + var parsed = read_json(line, parse_error_msg) + if (parsed == null) { + result.error_envelope = error("null", PARSE_ERROR, "parse error: {parse_error_msg}") + return result + } + return parse_one_request(parsed, strict) +} + +def public parse_batch(line : string; strict : bool = false) : ParsedBatch { + //! Parse a JSON-RPC 2.0 wire body that may be a single request or a §6 batch array. + //! + //! Semantics: + //! + //! * Single object → ``is_batch=false``, ``requests`` has one entry. + //! * Array of objects → ``is_batch=true``, ``requests`` has one entry per element. + //! * Empty array ``[]`` → ``is_batch=true``, ``requests`` empty, ``framing_error`` + //! carries a pre-built INVALID_REQUEST envelope (spec §6). + //! * Top-level parse failure → ``is_batch=false``, ``requests`` empty, + //! ``framing_error`` carries a PARSE_ERROR envelope. + var result : ParsedBatch + var parse_error_msg = "" + var parsed = read_json(line, parse_error_msg) + if (parsed == null) { + result.framing_error = error("null", PARSE_ERROR, "parse error: {parse_error_msg}") + return <- result + } + if (parsed.value is _array) { + result.is_batch = true + let arr & = unsafe(parsed.value as _array) + if (empty(arr)) { + result.framing_error = error("null", INVALID_REQUEST, "invalid request: empty batch") + return <- result + } + result.requests |> reserve(length(arr)) + for (entry in arr) { + let ev = unsafe(reinterpret(entry)) + result.requests |> push(parse_one_request(ev, strict)) + } + return <- result + } + // Single request — wrap in a one-element ParsedBatch + result.is_batch = false + result.requests |> push(parse_one_request(parsed, strict)) + return <- result +} + + +// ===== Client-side response parsers ===== + +def private parse_one_response(body_value : JsonValue?) : ParsedResponse { + var result = ParsedResponse(id_str = "null", is_success = false) + if (body_value == null || !(body_value.value is _object)) { + result.parse_error = "response is not a JSON object" + return result + } + let body & = unsafe(body_value.value as _object) + let id_v = body?["id"] ?? null + let result_v = body?["result"] ?? null + let error_v = body?["error"] ?? null + result.id_str = serialize_id(id_v) + if (result_v != null) { + result.is_success = true + result.result_json = write_json(result_v) + return result + } + if (error_v == null || !(error_v.value is _object)) { + result.parse_error = "response missing both `result` and a valid `error`" + return result + } + let err & = unsafe(error_v.value as _object) + let code_v = err?["code"] ?? null + let msg_v = err?["message"] ?? null + let data_v = err?["data"] ?? null + if (code_v != null && code_v.value is _longint) { + result.error_code = int(code_v.value as _longint) + } elif (code_v != null && code_v.value is _number) { + result.error_code = int(code_v.value as _number) + } + if (msg_v != null && msg_v.value is _string) { + result.error_msg = msg_v.value as _string + } + if (data_v != null) { + result.error_data = write_json(data_v) + } + return result +} + +def public parse_response(line : string) : ParsedResponse { + //! Parse a single JSON-RPC 2.0 response line. + //! + //! ``is_success`` distinguishes ``{"result":...}`` from ``{"error":...}``. + //! Optional ``error.data`` is surfaced as the raw JSON string in + //! ``error_data`` (empty when absent). Malformed wires set ``parse_error``. + var result = ParsedResponse(id_str = "null", is_success = false) + var parse_error_msg = "" + var parsed = read_json(line, parse_error_msg) + if (parsed == null) { + result.parse_error = "parse error: {parse_error_msg}" + return result + } + return parse_one_response(parsed) +} + +def public parse_response_batch(line : string) : ParsedResponseBatch { + //! Parse a JSON-RPC 2.0 response wire body that may be a single response or a §6 batch array. + //! + //! Semantics mirror ``parse_batch``: a top-level array sets ``is_batch=true``; + //! a single object sets ``is_batch=false`` with one entry. Top-level parse + //! failure populates ``parse_error``. + var result : ParsedResponseBatch + var parse_error_msg = "" + var parsed = read_json(line, parse_error_msg) + if (parsed == null) { + result.parse_error = "parse error: {parse_error_msg}" + return <- result + } + if (parsed.value is _array) { + result.is_batch = true + let arr & = unsafe(parsed.value as _array) + result.responses |> reserve(length(arr)) + for (entry in arr) { + let ev = unsafe(reinterpret(entry)) + result.responses |> push(parse_one_response(ev)) + } + return <- result + } + result.is_batch = false + result.responses |> push(parse_one_response(parsed)) + return <- result +} + + +// ===== Framing helper ===== + +def public compact_json_whitespace(s : string) : string { + //! Strip JSON pretty-printer whitespace (spaces, tabs, newlines, carriage returns) + //! that appears outside string literals. Whitespace inside ``"..."`` is preserved + //! verbatim, including backslash-escaped quotes. Used to flatten ``write_json`` + //! output for one-message-per-line wire framing. + return build_string() $(var w) { + peek_data(s) $(bytes) { + var in_string = false + var escape = false + for (b in bytes) { + if (in_string) { + write_char(w, int(b)) + if (escape) { + escape = false + } elif (b == uint8('\\')) { + escape = true + } elif (b == uint8('"')) { + in_string = false + } + } else { + if (b == uint8('"')) { + in_string = true + write_char(w, int(b)) + } elif (b != uint8(' ') && b != uint8('\n') && b != uint8('\t') && b != uint8('\r')) { + write_char(w, int(b)) + } + } + } + } + } +} + + +// ===== High-level dispatch convenience ===== + +def private dispatch_one(req : ParsedRequest; dispatcher : block<(method : string; params_json : string) : string>) : string { + if (!empty(req.error_envelope)) return req.is_notification ? "" : req.error_envelope + let result_json = invoke(dispatcher, req.method, req.params_json) + return req.is_notification ? "" : response(req.id_str, result_json) +} + +def public dispatch_line(line : string; strict : bool; dispatcher : block<(method : string; params_json : string) : string>) : string { + //! Dispatch a JSON-RPC 2.0 wire line (single or §6 batch) through ``dispatcher``, + //! returning the response string ready for the wire (or ``""`` for all-notifications). + //! + //! The dispatcher block receives ``(method, params_json)`` for each non-notification + //! request and returns a raw JSON result string. The library wraps the result in + //! a JSON-RPC envelope; notification semantics, batch wrapping, framing-error + //! suppression, and input-order preservation are all handled here. + //! + //! Caller-side error reporting: to signal ``METHOD_NOT_FOUND`` or other + //! per-method errors from the dispatcher, return a JSON object the wrapper + //! treats as result — or use ``parse_request`` / ``parse_batch`` directly and + //! emit ``error(...)`` envelopes yourself when the protocol requires it. + let pb = parse_batch(line, strict) + if (!empty(pb.framing_error)) return pb.framing_error + if (!pb.is_batch) return dispatch_one(pb.requests[0], dispatcher) + var parts : array + for (req in pb.requests) { + let r = dispatch_one(req, dispatcher) + if (!empty(r)) parts |> push(r) + } + if (empty(parts)) return "" + return build_string() $(var w) { + write_char(w, int('[')) + var first = true + for (p in parts) { + if (!first) write_char(w, int(',')) + first = false + write(w, p) + } + write_char(w, int(']')) + } +} diff --git a/doc/reflections/das2rst.das b/doc/reflections/das2rst.das index 567bb5657..8f8760843 100644 --- a/doc/reflections/das2rst.das +++ b/doc/reflections/das2rst.das @@ -16,6 +16,7 @@ require daslib/rst require daslib/functional require daslib/json require daslib/json_boost +require daslib/jsonrpc require daslib/regex require daslib/regex_boost require daslib/apply @@ -426,6 +427,19 @@ def document_module_json_boost(root : string) { document("Boost package for JSON", mod, "json_boost.rst", groups) } +def document_module_jsonrpc(root : string) { + var mod = find_module("jsonrpc") + var groups <- array( + group_by_regex("Envelope builders", mod, %regex~(response|error|error_with_data|serialize_id)$%%), + group_by_regex("Server-side parsers", mod, %regex~(parse_request|parse_batch)$%%), + group_by_regex("Outgoing request builders", mod, %regex~(make_request|make_notification|make_batch)$%%), + group_by_regex("Response parsers", mod, %regex~(parse_response|parse_response_batch)$%%), + group_by_regex("Framing helper", mod, %regex~(compact_json_whitespace)$%%), + group_by_regex("High-level dispatch", mod, %regex~(dispatch_line)$%%) + ) + document("JSON-RPC 2.0 envelope + parser, transport-agnostic", mod, "jsonrpc.rst", groups) +} + def document_module_regex(root : string) { var mod = find_module("regex") var groups <- array( @@ -1635,6 +1649,7 @@ def main { document_module_jobque_boost(root) document_module_json_boost(root) document_module_json(root) + document_module_jsonrpc(root) // document_module_linked_list(root) // spoof template, not documentable document_module_linq(root) document_module_linq_boost(root) diff --git a/doc/source/reference/language/options.rst b/doc/source/reference/language/options.rst index f3476685a..009f6b3a3 100644 --- a/doc/source/reference/language/options.rst +++ b/doc/source/reference/language/options.rst @@ -371,6 +371,10 @@ Debugging and Profiling - bool - false - Prints detailed compile time breakdown at the end of compilation. + * - ``log_module_compile_time`` + - bool + - false + - Prints per-module compile-time breakdown (parse / infer with pass count / optimize / macro (in infer) / macro mods) plus function count for each required module. Also enables per-context simulate timing logs and the top-level aggregate summary. CLI: ``-log-compile-time``. -------------------- RTTI diff --git a/doc/source/reference/tutorials.rst b/doc/source/reference/tutorials.rst index 059a2273e..1a42b52de 100644 --- a/doc/source/reference/tutorials.rst +++ b/doc/source/reference/tutorials.rst @@ -397,6 +397,27 @@ For the strudel-to-strudel.cc feature comparison, see tutorials/daStrudel_15_live_reloading.rst tutorials/daStrudel_16_hrtf_position.rst +.. _tutorials_jsonrpc: + +JSON-RPC 2.0 Tutorials +======================= + +These tutorials cover ``daslib/jsonrpc`` — the transport-agnostic +JSON-RPC 2.0 library: building requests, implementing servers with +``dispatch_line``, and the §6 batch semantics. The companion ``.das`` +files are in ``tutorials/jsonrpc/``. + +Run any tutorial from the project root:: + + daslang.exe tutorials/jsonrpc/01_request_response.das + +.. toctree:: + :maxdepth: 1 + + tutorials/jsonrpc_01_request_response.rst + tutorials/jsonrpc_02_dispatch_line.rst + tutorials/jsonrpc_03_batch.rst + .. _tutorials_daspeg: dasPEG (Parser Generator) Tutorials diff --git a/doc/source/reference/tutorials/jsonrpc_01_request_response.rst b/doc/source/reference/tutorials/jsonrpc_01_request_response.rst new file mode 100644 index 000000000..94cb132f9 --- /dev/null +++ b/doc/source/reference/tutorials/jsonrpc_01_request_response.rst @@ -0,0 +1,128 @@ +.. _tutorial_jsonrpc_request_response: + +================================================= +JSONRPC-01 — Building Requests, Parsing Responses +================================================= + +.. index:: + single: Tutorial; JSON-RPC + single: Tutorial; jsonrpc + single: Tutorial; daslib/jsonrpc + +This tutorial covers the client side of ``daslib/jsonrpc``: building +outgoing JSON-RPC 2.0 requests, notifications, and §6 batches, then +parsing the responses you get back. + +Setup +===== + +Import the module: + +.. code-block:: das + + require daslib/jsonrpc + +Building a request +================== + +``make_request(method, params_json, id)`` builds a JSON-RPC 2.0 request +as a single-line string. The ``params_json`` argument is a *pre-serialized* +JSON value — pass ``"null"`` for no params, or any valid JSON literal, +object, or array. The id may be ``int`` or ``string``: + +.. code-block:: das + + let req1 = make_request("echo", "[\"hello\"]", 1) + // → {"jsonrpc":"2.0","id":1,"method":"echo","params":["hello"]} + + let req2 = make_request("status", "null", "call-7") + // → {"jsonrpc":"2.0","id":"call-7","method":"status","params":null} + +Notifications +============= + +A notification omits the ``id`` field. Per JSON-RPC 2.0 §4.1, the server +MUST NOT send a response. Use these for fire-and-forget side effects: + +.. code-block:: das + + let notif = make_notification("log", "\{\"msg\":\"ready\"}") + // → {"jsonrpc":"2.0","method":"log","params":{"msg":"ready"}} + +Building a §6 batch +==================== + +``make_batch(messages)`` wraps an array of pre-built request / +notification strings as a JSON array. The server returns an array of +responses, with notification entries suppressed: + +.. code-block:: das + + let entries <- [ + make_request("status", "null", 10), + make_request("echo", "[1, 2, 3]", 11), + make_notification("log", "\{\"level\":\"info\"}") + ] + let batch = make_batch(entries) + +Parsing a success response +=========================== + +``parse_response(line)`` returns a ``ParsedResponse`` struct. Check +``is_success`` to distinguish ``{"result":...}`` from ``{"error":...}``: + +.. code-block:: das + + let wire = "\{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":[\"hello\"]}" + let r = parse_response(wire) + // r.is_success = true + // r.id_str = "1" + // r.result_json = ["hello"] + +Parsing an error response +========================== + +Error responses populate ``error_code``, ``error_msg``, and optionally +``error_data``. Standard codes are constants: ``PARSE_ERROR``, +``INVALID_REQUEST``, ``METHOD_NOT_FOUND``, ``INVALID_PARAMS``, +``INTERNAL_ERROR``: + +.. code-block:: das + + let wire = "\{\"jsonrpc\":\"2.0\",\"id\":1,\"error\":\{\"code\":-32601,\"message\":\"method not found\"}}" + let r = parse_response(wire) + // r.is_success = false + // r.error_code = -32601 (matches METHOD_NOT_FOUND) + // r.error_msg = "method not found" + +Parsing a batched response +=========================== + +``parse_response_batch`` detects whether the wire was a single response +or a JSON array. For batches, ``is_batch=true`` and ``responses[]`` +holds one entry per array element: + +.. code-block:: das + + let wire = "[\{\"id\":10,\"result\":\"ok\"},\{\"id\":11,\"error\":\{\"code\":-32602,\"message\":\"bad\"}}]" + let pb = parse_response_batch(wire) + for (entry in pb.responses) { + if (entry.is_success) print("[{entry.id_str}] result: {entry.result_json}\n") + else print("[{entry.id_str}] error {entry.error_code}: {entry.error_msg}\n") + } + +Running the tutorial +==================== + +:: + + daslang.exe tutorials/jsonrpc/01_request_response.das + +Full source: :download:`tutorials/jsonrpc/01_request_response.das <../../../../tutorials/jsonrpc/01_request_response.das>` + +See also +======== + +* **Next:** :ref:`tutorial_jsonrpc_dispatch_line` — implementing a server with ``dispatch_line`` +* :ref:`tutorial_jsonrpc_batch` — §6 batches end-to-end +* :ref:`stdlib_jsonrpc` — module reference diff --git a/doc/source/reference/tutorials/jsonrpc_02_dispatch_line.rst b/doc/source/reference/tutorials/jsonrpc_02_dispatch_line.rst new file mode 100644 index 000000000..7a6e2c5dd --- /dev/null +++ b/doc/source/reference/tutorials/jsonrpc_02_dispatch_line.rst @@ -0,0 +1,95 @@ +.. _tutorial_jsonrpc_dispatch_line: + +===================================================== +JSONRPC-02 — Implementing a Server with dispatch_line +===================================================== + +.. index:: + single: Tutorial; JSON-RPC; server + single: Tutorial; jsonrpc; dispatch_line + +This tutorial shows the server side of ``daslib/jsonrpc``: the +``dispatch_line`` convenience that wraps parsing, envelope building, +notification suppression, and batch fan-out in one call. The lower-level +``parse_request`` + envelope-builder path is also covered for cases +where ``dispatch_line`` doesn't fit. + +The dispatcher block +==================== + +``dispatch_line(line, strict, dispatcher_block)`` parses a wire line and +invokes the dispatcher block for each non-notification request. The +block receives ``(method, params_json)`` and returns the raw JSON result +string; the library wraps the result in a JSON-RPC envelope. + +.. code-block:: das + + require daslib/jsonrpc + + def dispatch(method, params_json : string) : string { + if (method == "ping") return "\"pong\"" + if (method == "echo") return jsonrpc::compact_json_whitespace(params_json) + return "\{\"error\":\"unknown: {method}\"}" + } + + [export] + def main() { + let resp = jsonrpc::dispatch_line("\{\"id\":1,\"method\":\"ping\"}", false) $(m, p) { + return dispatch(m, p) + } + print("{resp}\n") + // → {"jsonrpc":"2.0","id":1,"result":"pong"} + } + +Notification semantics +====================== + +For a notification (no ``id``), the dispatcher still runs (so side +effects happen) but the return value is discarded and ``dispatch_line`` +returns ``""``. Per JSON-RPC 2.0 §4.1, the wire MUST stay silent. + +Top-level parse failure +======================= + +If the wire line isn't valid JSON, ``dispatch_line`` returns a pre-built +``PARSE_ERROR`` envelope and the dispatcher is never called. Same for +top-level §6 violations such as the empty array ``[]``. + +Strict vs permissive +==================== + +``dispatch_line(line, strict=false)`` is permissive: the ``jsonrpc`` +field is optional and any string value is accepted. Pass ``strict=true`` +to enforce §4 — a missing or non-``"2.0"`` ``jsonrpc`` field yields +``INVALID_REQUEST``. + +When dispatch_line doesn't fit +============================== + +``dispatch_line`` wraps all dispatcher results in a *success* envelope. +When you need to emit specific error codes per method +(``METHOD_NOT_FOUND``, ``INVALID_PARAMS``), drop down to +``parse_request`` + envelope builders directly: + +.. code-block:: das + + let req = parse_request(line) + if (!empty(req.error_envelope)) return req.is_notification ? "" : req.error_envelope + if (req.method == "unknown") return error(req.id_str, METHOD_NOT_FOUND, "no such method") + return response(req.id_str, dispatch(req.method, req.params_json)) + +Running the tutorial +==================== + +:: + + daslang.exe tutorials/jsonrpc/02_dispatch_line.das + +Full source: :download:`tutorials/jsonrpc/02_dispatch_line.das <../../../../tutorials/jsonrpc/02_dispatch_line.das>` + +See also +======== + +* :ref:`tutorial_jsonrpc_request_response` — building requests, parsing responses (the client side) +* **Next:** :ref:`tutorial_jsonrpc_batch` — §6 batches end-to-end +* :ref:`stdlib_jsonrpc` — module reference diff --git a/doc/source/reference/tutorials/jsonrpc_03_batch.rst b/doc/source/reference/tutorials/jsonrpc_03_batch.rst new file mode 100644 index 000000000..1c6feb3b7 --- /dev/null +++ b/doc/source/reference/tutorials/jsonrpc_03_batch.rst @@ -0,0 +1,109 @@ +.. _tutorial_jsonrpc_batch: + +================================================== +JSONRPC-03 — §6 Batches: Many Messages in One Wire +================================================== + +.. index:: + single: Tutorial; JSON-RPC; batch + single: Tutorial; jsonrpc; §6 batch + +This tutorial covers JSON-RPC 2.0 §6 batch requests — sending and +receiving multiple messages in a single wire payload. The semantics +are subtle (empty array is an error, all-notifications batch yields +nothing on the wire, per-entry errors are individual envelopes), but +``daslib/jsonrpc`` handles all the edge cases. + +Mixed batch round-trip +====================== + +A client batches two requests plus one notification. The server +dispatches them. The response is an array of *two* entries: +the notification produces no response and input order is preserved. + +.. code-block:: das + + let entries <- [ + make_request("ping", "null", 1), + make_notification("log", "\{\"msg\":\"hello\"}"), + make_request("echo", "[42]", 2) + ] + let wire = make_batch(entries) + + let resp = jsonrpc::dispatch_line(wire, false) $(m, p) { + if (m == "ping") return "\"pong\"" + if (m == "echo") return jsonrpc::compact_json_whitespace(p) + return "\"unknown\"" + } + // resp = [{"id":1,"result":"pong"},{"id":2,"result":[42]}] + +Per-entry errors: continue-on-error +==================================== + +Malformed entries — missing method, bad id type, etc. — get individual +error envelopes in their array slot. Valid entries still dispatch: + +.. code-block:: das + + let wire = "[\{\"id\":1,\"method\":\"ping\"},\{\"id\":2},\{\"id\":3,\"method\":\"echo\",\"params\":[\"x\"]}]" + let resp = jsonrpc::dispatch_line(wire, false) $(m, p) { return "ok" } + // resp has three entries: + // id=1 result + // id=2 error -32600 (missing method) + // id=3 result + +Empty batch array +================= + +Per §6, an empty array ``[]`` is *itself* an invalid request. The server +responds with a SINGLE error envelope, not an array: + +.. code-block:: das + + let resp = jsonrpc::dispatch_line("[]", false) $(m, p) { return "" } + // resp = {"jsonrpc":"2.0","id":null,"error":{"code":-32600,"message":"invalid request: empty batch"}} + +All-notifications batch +======================= + +If every entry in a batch is a notification, the server processes them +(dispatcher runs for each) but emits nothing on the wire — the response +array would be empty, so per §6 the server returns nothing at all: + +.. code-block:: das + + let wire = "[\{\"method\":\"a\"},\{\"method\":\"b\"}]" + let resp = jsonrpc::dispatch_line(wire, false) $(m, p) { return "ignored" } + // resp == "" + +Manual batch handling +===================== + +For custom error semantics (per-method ``INVALID_PARAMS``, +``METHOD_NOT_FOUND``, logging per entry, etc.), use ``parse_batch`` +directly: + +.. code-block:: das + + let pb = parse_batch(wire) + if (!empty(pb.framing_error)) return pb.framing_error + for (req in pb.requests) { + if (!empty(req.error_envelope)) { /* per-entry error */ } + else { /* req.method, req.id_str, req.params, req.params_json available */ } + } + +Running the tutorial +==================== + +:: + + daslang.exe tutorials/jsonrpc/03_batch.das + +Full source: :download:`tutorials/jsonrpc/03_batch.das <../../../../tutorials/jsonrpc/03_batch.das>` + +See also +======== + +* :ref:`tutorial_jsonrpc_request_response` — building requests, parsing responses +* :ref:`tutorial_jsonrpc_dispatch_line` — server with ``dispatch_line`` +* :ref:`stdlib_jsonrpc` — module reference diff --git a/doc/source/reference/utils/daslang_live.rst b/doc/source/reference/utils/daslang_live.rst index acd574f63..030029970 100644 --- a/doc/source/reference/utils/daslang_live.rst +++ b/doc/source/reference/utils/daslang_live.rst @@ -533,8 +533,15 @@ Endpoints - Graceful shutdown. * - POST - ``/command`` - - Dispatch a ``[live_command]`` via JSON body: + - Dispatch a single ``[live_command]`` via JSON body: ``{"name":"cmd_name","args":{...}}``. + * - POST + - ``/commands`` + - Dispatch a batch of ``[live_command]`` in one round-trip. + Body is a JSON array of ``{name,args}`` objects; + response is a JSON array of per-entry results in input order. + Continue-on-error: malformed entries surface ``{"error":...}`` + in their slot. * - ANY - ``*`` - JSON help with all endpoints and curl examples. @@ -555,8 +562,14 @@ Call a live command:: curl -X POST http://localhost:9090/command \ -d '{"name":"set_color","args":{"r":1.0,"g":0.0,"b":0.0}}' +Call a batch of live commands:: + + curl -X POST http://localhost:9090/commands \ + -d '[{"name":"set_color","args":{"r":0.5}},{"name":"set_alpha","args":{"a":0.5}}]' + When using the daslang MCP server, prefer ``live_*`` MCP tools over -curl (see :ref:`utils_mcp`). +curl (see :ref:`utils_mcp`). The ``live_commands`` MCP tool maps to the +``POST /commands`` batch endpoint above. .. _stdio_api: @@ -589,37 +602,39 @@ an object when the command returned an object (``status``), a string when the command returned a string (``set_color`` in the demo returns a ``JV(string)`` so the result is a quoted JSON string), and so on. +**Implementation.** Parsing, envelope building, notification semantics, +and §6 batch handling all come from :ref:`stdlib_jsonrpc`. The transport +itself is just the stdin reader thread plus a thin +``dispatch_command`` bridge — see ``handle_jsonrpc_line`` in +``modules/dasLiveHost/live/live_api_stdio.das``. Clients that need the +parser / envelope helpers directly should ``require daslib/jsonrpc``. + **Framing guarantee.** The transport always writes exactly one response per line on stdout, with no embedded newlines in the envelope. ``write_json`` pretty-prints by default, so the dispatch result is -post-processed by ``compact_json_whitespace`` before being embedded in -the envelope — whitespace outside string literals is stripped; -whitespace inside ``"..."`` is preserved verbatim. +post-processed by ``jsonrpc::compact_json_whitespace`` before being +embedded in the envelope. **Notifications.** Per JSON-RPC 2.0 §4.1, a request that omits the ``id`` field is a *notification*: the server MUST NOT respond. ``live_api_stdio`` honors this — fire-and-forget commands like -``{"method":"shutdown"}`` produce no output, useful for clients that -don't want to bookkeep a response. An explicit ``"id":null`` is *not* -a notification; the server responds with ``"id":null``. Parse errors -and invalid requests are *not* treated as notifications either — they -emit ``-32700`` / ``-32600`` responses with ``"id":null`` so a buggy -client doesn't hang waiting for a reply. - -**Permissive JSON-RPC subset.** Two deviations from strict JSON-RPC 2.0 -to ease integration with real-world clients: - -* The ``jsonrpc`` member is **optional**. Strict compliance would - reject ``{"id":1,"method":"status"}`` because the ``"jsonrpc":"2.0"`` - field is missing — this transport accepts it. Requests with a - different version (``"jsonrpc":"1.0"``) are also accepted. -* All other JSON-RPC 2.0 rules are enforced: ``method`` is required and - must be a string, ``id`` must be string / number / null when present - (objects / arrays / booleans are rejected with ``-32600``), and - notifications produce no response. - -Error envelopes follow JSON-RPC 2.0 codes: ``-32700`` parse error, -``-32600`` invalid request (missing/non-string ``method``). +``{"method":"shutdown"}`` produce no output. An explicit ``"id":null`` +is *not* a notification; the server responds with ``"id":null``. Parse +errors and invalid requests still emit ``-32700`` / ``-32600`` +responses with ``"id":null``. + +**§6 batch support.** Multiple commands in one wire message — a JSON +array of requests at the top level — produce a JSON array of responses +(in input order, notification entries suppressed). Empty array and +top-level malformed JSON yield single error envelopes per spec. + +**Permissive by default.** The ``jsonrpc`` member is optional and any +value is accepted; pass ``strict=true`` through ``daslib/jsonrpc`` for +full §4 compliance. Other §4 rules (``method`` required and string, +``id`` string/number/null only) are always enforced. + +Error envelopes follow JSON-RPC 2.0 codes (``-32700`` / ``-32600`` / +``-32602``). Available methods (built-ins from ``live/live_api_builtins``): diff --git a/doc/source/stdlib/handmade/module-jsonrpc.rst b/doc/source/stdlib/handmade/module-jsonrpc.rst new file mode 100644 index 000000000..a15885203 --- /dev/null +++ b/doc/source/stdlib/handmade/module-jsonrpc.rst @@ -0,0 +1,85 @@ +The JSON-RPC module is a transport-agnostic JSON-RPC 2.0 implementation +(https://www.jsonrpc.org/specification). It provides envelope builders, +request/response parsers, optional §6 batch handling, and a high-level +``dispatch_line`` convenience for stdio-style servers. + +See :ref:`tutorial_jsonrpc_request_response` for a hands-on tutorial. + +All functions and symbols are in the ``jsonrpc`` module, use require to get access to it: + +.. code-block:: das + + require daslib/jsonrpc + +The library is permissive by default — the ``jsonrpc`` request member may +be absent or carry a value other than ``"2.0"`` (real-world clients are +lazy about it). Pass ``strict=true`` to ``parse_request`` / ``parse_batch`` +/ ``dispatch_line`` for full §4 compliance. + +Standard error codes are exposed as ``let public`` constants: + +============================ ========================================= +Constant Code +============================ ========================================= +``PARSE_ERROR`` ``-32700`` Invalid JSON received +``INVALID_REQUEST`` ``-32600`` Not a valid request object +``METHOD_NOT_FOUND`` ``-32601`` Method does not exist +``INVALID_PARAMS`` ``-32602`` Invalid method params +``INTERNAL_ERROR`` ``-32603`` Internal JSON-RPC error +============================ ========================================= + +Server-side example (one-shot): + +.. code-block:: das + + require daslib/jsonrpc + + [export] + def main() { + let wire = "\{\"id\":1,\"method\":\"ping\"}" + let response = jsonrpc::dispatch_line(wire, false) $(method, params_json) { + if (method == "ping") return "\"pong\"" + return "\"unknown\"" + } + print("{response}\n") + // → {"jsonrpc":"2.0","id":1,"result":"pong"} + } + +Client-side example (build + parse): + +.. code-block:: das + + require daslib/jsonrpc + + [export] + def main() { + let wire = make_request("echo", "[1,2,3]", 7) + // → {"jsonrpc":"2.0","id":7,"method":"echo","params":[1,2,3]} + let reply_wire = "\{\"jsonrpc\":\"2.0\",\"id\":7,\"result\":[1,2,3]}" + let r = parse_response(reply_wire) + if (r.is_success) print("got result for id={r.id_str}\n") + } + +§6 batch — multiple messages in one wire payload: + +.. code-block:: das + + let entries <- [ + make_request("a", "null", 1), + make_notification("log", "\{\"msg\":\"hi\"}"), + make_request("b", "null", 2) + ] + let batch = make_batch(entries) + let response = jsonrpc::dispatch_line(batch, false) $(method, params_json) { + return "\"ok-{method}\"" + } + // response is a JSON array with two entries (notification suppressed). + +The ``compact_json_whitespace`` helper flattens pretty-printed +``write_json`` output to a single line — useful when piping +``daslib/json`` output through a newline-framed JSON-RPC transport. + +In-tree consumers include +:doc:`/reference/utils/daslang_live` (stdio JSON-RPC transport for live +commands) and the daslang MCP server (``utils/mcp/protocol.das``). For a +complete pedagogical client+server pair, see ``examples/mcp/echo/``. diff --git a/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst b/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst index e39beb178..40d8a296e 100644 --- a/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst +++ b/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst @@ -16,6 +16,7 @@ Keep context alive after main function. Whether to use very safe context (delete of data is delayed, to avoid table[foo]=table[bar] lifetime bugs). Threshold for reporting candidates for function calls. If less than this number, we always report them. Maximum number of inference passes. +Maximum call expression nesting depth during inference. Stack size. Whether to intern strings. Whether to use persistent heap (or linear heap). @@ -65,6 +66,7 @@ Fails compilation if AOT is not available. Fails compilation if AOT export is not available. Log compile time. Log total compile time. +Log detailed per-module compile-time breakdown. Each required module emits its own block with parse / infer (with pass count) / optimize / macro (in infer) / macro mods times plus function count. Also enables a separate ``simulate (...) took X, modName (fileName)`` line for every ``Program::simulate`` call and the top-level aggregate summary at the end of compilation. Wired to the daslang CLI flag ``-log-compile-time``. Disables fast call optimization. Reuse stack memory after variables go out of scope. Force in-scope for POD-like types. diff --git a/doc/source/stdlib/sec_data_formats.rst b/doc/source/stdlib/sec_data_formats.rst index 0359a6fb1..8cd2b0a3f 100644 --- a/doc/source/stdlib/sec_data_formats.rst +++ b/doc/source/stdlib/sec_data_formats.rst @@ -10,6 +10,7 @@ JSON, XML, regular expressions, PEG parser generator, and reStructuredText proce generated/json.rst generated/json_boost.rst + generated/jsonrpc.rst generated/pugixml.rst generated/PUGIXML_boost.rst generated/regex.rst diff --git a/examples/mcp/echo/README.md b/examples/mcp/echo/README.md new file mode 100644 index 000000000..43b940ffc --- /dev/null +++ b/examples/mcp/echo/README.md @@ -0,0 +1,79 @@ +# examples/mcp/echo + +A minimal JSON-RPC 2.0 client + server pair built on +[daslib/jsonrpc](../../../daslib/jsonrpc.das) — the canonical bidirectional +JSON-RPC library in daslang. + +The server implements three toy methods (`echo`, `add`, `status`) over +newline-delimited stdio JSON-RPC. The client demonstrates the outgoing +side: request/notification/batch builders, response parsers. + +This is a teaching example — for the real production usage, see +[modules/dasLiveHost/live/live_api_stdio.das](../../../modules/dasLiveHost/live/live_api_stdio.das) +(live-command server) and [utils/mcp/protocol.das](../../../utils/mcp/protocol.das) +(MCP server). + +## Run + +The server reads newline-delimited JSON-RPC from stdin and writes responses +to stdout: + +```sh +echo '{"jsonrpc":"2.0","id":1,"method":"echo","params":["hello"]}' | daslang server.das +# → {"jsonrpc":"2.0","id":1,"result":["hello"]} + +echo '{"id":2,"method":"add","params":{"a":7,"b":35}}' | daslang server.das +# → {"jsonrpc":"2.0","id":2,"result":42} +``` + +A `§6` batch — multiple requests + a notification in one message — comes back +as a JSON array of responses, with the notification suppressed: + +```sh +echo '[{"id":1,"method":"add","params":{"a":1,"b":2}},{"method":"notif"},{"id":2,"method":"status"}]' \ + | daslang server.das +# → [{"jsonrpc":"2.0","id":1,"result":3},{"jsonrpc":"2.0","id":2,"result":"ok"}] +``` + +A pure notification (no `id`) produces no output: + +```sh +echo '{"method":"echo","params":["fire-and-forget"]}' | daslang server.das +# → (no output) +``` + +## Client modes + +The client uses [daslib/clargs](../../../daslib/clargs.das) for argument +parsing. Daslang's script host forwards args after `--`. + +**Demo mode** (default) — self-contained: builds sample wires, dispatches +them inline, parses sample responses: + +```sh +daslang client.das +``` + +**Emit mode** — print sample requests to stdout (pipe into the server for a +real round-trip): + +```sh +daslang client.das -- --mode=Emit | daslang server.das +``` + +## API surface used + +Server-side: +- `jsonrpc::dispatch_line(line, strict)` — high-level: handles parsing, + envelope wrapping, batch fan-out, notification suppression. Takes a + dispatcher block `(method, params_json) → result_json`. +- `jsonrpc::compact_json_whitespace(s)` — flattens pretty-printed + `write_json` output to a single line for newline-framed wires. + +Client-side: +- `make_request(method, params_json, id : int | string)` — outgoing request. +- `make_notification(method, params_json)` — fire-and-forget. +- `make_batch(messages)` — wrap N pre-built wires as a §6 batch. +- `parse_response(line)` — server response → `ParsedResponse {is_success, + result_json, error_code, error_msg, error_data}`. +- `parse_response_batch(line)` — same for §6 batched responses. diff --git a/examples/mcp/echo/client.das b/examples/mcp/echo/client.das new file mode 100644 index 000000000..a6f556a90 --- /dev/null +++ b/examples/mcp/echo/client.das @@ -0,0 +1,114 @@ +options gen2 +options no_unused_block_arguments = false +options no_unused_function_arguments = false + +// Companion to server.das — demonstrates the client side of daslib/jsonrpc. +// +// Two modes (daslang script-host uses `--` to forward flags): +// daslang client.das -- --mode=emit print sample requests to stdout +// daslang client.das -- --mode=demo self-contained build + parse demo +// daslang client.das defaults to demo +// +// Pipe through the echo server for a real round-trip: +// daslang client.das -- --mode=emit | daslang server.das + +require daslib/jsonrpc +require daslib/clargs + +enum Mode { + Demo + Emit +} + +[CommandLineArgs] +struct Config { + @clarg_short = "m" + @clarg_doc = "Output mode: Demo (build + parse inline) or Emit (print requests to stdout)" + mode : Mode + + @clarg_short = "?" + @clarg_name = "show-help" + @clarg_doc = "Show this help and exit" + help : bool +} + +def emit_requests() { + let echo_req = make_request("echo", "[\"hello\"]", 1) + print("{echo_req}\n") + let add_req = make_request("add", "\{\"a\":2,\"b\":3\}", 2) + print("{add_req}\n") + // Notification — no id; server runs the method but emits no response. + let status_notif = make_notification("status", "null") + print("{status_notif}\n") + // §6 batch: two requests + one notification. + let entries <- [ + make_request("status", "null", 10), + make_notification("log", "\{\"msg\":\"client says hi\"}"), + make_request("add", "\{\"a\":40,\"b\":2\}", 11) + ] + let batch_wire = make_batch(entries) + print("{batch_wire}\n") +} + +def demo() { + // === Client side: build outgoing wires === + print("=== Building requests ===\n") + let req_echo = make_request("echo", "[\"hello\"]", 1) + let req_add = make_request("add", "\{\"a\":2,\"b\":3\}", "call-2") + let notif = make_notification("status", "null") + let batch <- [ + make_request("status", "null", 10), + make_request("add", "\{\"a\":40,\"b\":2\}", 11) + ] + let req_batch = make_batch(batch) + print(" single (int id): {req_echo}\n") + print(" single (string id): {req_add}\n") + print(" notification: {notif}\n") + print(" batch: {req_batch}\n") + + // === Server side: parse + dispatch (inline, no subprocess) === + print("\n=== Server parses + dispatches ===\n") + let resp = jsonrpc::dispatch_line(req_echo, false) $(method, params_json) { + if (method == "echo") return jsonrpc::compact_json_whitespace(params_json) + return "\"unknown\"" + } + print(" echo response wire: {resp}\n") + + // === Client side: parse incoming responses === + print("\n=== Parsing responses ===\n") + let r = parse_response(resp) + print(" is_success={r.is_success}, id={r.id_str}, result={r.result_json |> compact_json_whitespace()}\n") + + // Error response parse + let err_wire = error("\"call-2\"", METHOD_NOT_FOUND, "method not found") + let er = parse_response(err_wire) + print(" is_success={er.is_success}, error_code={er.error_code}, error_msg={er.error_msg}\n") + + // Batch response parse + let batch_resp = "[\{\"jsonrpc\":\"2.0\",\"id\":10,\"result\":\"ok\"\},\{\"jsonrpc\":\"2.0\",\"id\":11,\"result\":42\}]" + let rb = parse_response_batch(batch_resp) + print(" batch is_batch={rb.is_batch}, count={length(rb.responses)}\n") + for (entry in rb.responses) { + print(" id={entry.id_str}, success={entry.is_success}, result={entry.result_json}\n") + } +} + +[export] +def main() { + var r <- parse_args(type) + if (r |> is_err) { + print("error: {r |> unwrap_err}\n\n") + print_help(get_command_info(type), "client") + return + } + let cfg <- r |> move_unwrap + if (cfg.help) { + print_help(get_command_info(type), "client") + return + } + if (cfg.mode == Mode.Emit) { + emit_requests() + } else { + demo() + } +} diff --git a/examples/mcp/echo/server.das b/examples/mcp/echo/server.das new file mode 100644 index 000000000..8e25a6800 --- /dev/null +++ b/examples/mcp/echo/server.das @@ -0,0 +1,67 @@ +options gen2 +options no_unused_block_arguments = false +options no_unused_function_arguments = false + +// Minimal JSON-RPC 2.0 stdio server built on daslib/jsonrpc. +// +// Three toy methods: +// echo(...) — returns params verbatim +// add({a,b}) — returns a + b +// status() — returns a greeting string +// +// Reads newline-delimited JSON-RPC from stdin, writes responses to stdout. +// Handles single requests, §6 batches, and notifications (no response). +// +// Usage: +// echo '{"jsonrpc":"2.0","id":1,"method":"echo","params":[42]}' | daslang server.das +// echo '[{"id":1,"method":"add","params":{"a":2,"b":3}},{"id":2,"method":"status"}]' | daslang server.das + +require daslib/jsonrpc +require daslib/json +require daslib/json_boost +require daslib/fio +require strings + +def dispatch_method(method, params_json : string) : string { + if (method == "echo") return jsonrpc::compact_json_whitespace(params_json) + if (method == "add") { + var err = "" + let p = read_json(params_json, err) + if (p == null || !(p.value is _object)) return "\{\"error\":\"add requires object params\"}" + let obj & = unsafe(p.value as _object) + let a_v = unsafe(reinterpret(obj?["a"] ?? null)) + let b_v = unsafe(reinterpret(obj?["b"] ?? null)) + let a = (a_v != null && a_v.value is _longint) ? int(a_v.value as _longint) : 0 + let b = (b_v != null && b_v.value is _longint) ? int(b_v.value as _longint) : 0 + return "{a + b}" + } + if (method == "status") return "\"ok\"" + // Unknown method — return an error JSON value as result. The transport + // wraps it in a result envelope; spec-strict servers would emit + // METHOD_NOT_FOUND via the lower-level parse_batch API instead. + return "\{\"error\":\"unknown method: {method}\"}" +} + +[export] +def main() { + let sin = fstdin() + let sout = fstdout() + while (!feof(sin)) { + let chunk = fgets(sin) // nolint:PERF003 + let len = length(chunk) + if (len == 0) break + var line = chunk + if (character_at(chunk, len - 1) == '\n') { // nolint:PERF003 + line = (len > 1 && character_at(chunk, len - 2) == '\r') ? slice(chunk, 0, len - 2) : slice(chunk, 0, len - 1) // nolint:PERF003 + } + if (empty(line)) continue + let response = jsonrpc::dispatch_line(line, false) $(method, params_json) { + return dispatch_method(method, params_json) + } + if (!empty(response)) { + fprint(sout, response) + fprint(sout, "\n") + fflush(sout) + } + } +} diff --git a/include/daScript/ast/ast.h b/include/daScript/ast/ast.h index 163987f95..0b00e222e 100644 --- a/include/daScript/ast/ast.h +++ b/include/daScript/ast/ast.h @@ -1568,6 +1568,7 @@ namespace das bool fail_on_lack_of_aot_export = false; // remove_unused_symbols = false is missing in the module, which is passed to AOT /*option*/ bool log_compile_time = false; // if true, then compile time will be printed at the end of the compilation /*option*/ bool log_total_compile_time = false; // if true, then detailed compile time will be printed at the end of the compilation + /*option*/ bool log_module_compile_time = false; // if true, every required module logs its own parse / infer (with pass count) / optimize / macro (in infer) / macro mods breakdown + function count; also enables per-context simulate timing and the top-level aggregate summary (CLI: -log-compile-time) /*option*/ bool no_fast_call = false; // disable fastcall /*option*/ bool scoped_stack_allocator = true; // reuse stack memory after variables out of scope /*option*/ bool force_inscope_pod = false; // force in-scope for POD-like types @@ -1757,6 +1758,7 @@ namespace das int totalFunctions = 0; int totalVariables = 0; int newLambdaIndex = 1; + int inferPassesUsed = 0; // sum of inferTypesDirty inner-loop pass counts across all inferTypes calls (incl. restartInfer legs) for this module; reset by parseDaScript once per module-compile; used by per-module compile-time log vector errors; vector aotErrors; uint32_t globalInitStackSize = 0; diff --git a/modules/dasLiveHost/live/live_api.das b/modules/dasLiveHost/live/live_api.das index 347886ac6..a7b284647 100644 --- a/modules/dasLiveHost/live/live_api.das +++ b/modules/dasLiveHost/live/live_api.das @@ -21,7 +21,8 @@ module live_api shared public //! POST /pause — pauses the host //! POST /unpause — unpauses the host //! POST /shutdown — graceful shutdown -//! POST /command — dispatches a [live_command] via host bridge +//! POST /command — dispatches a single [live_command] via host bridge +//! POST /commands — dispatches an array of [live_command]s (continue-on-error) //! ANY * — returns JSON help with all endpoints and curl examples //! //! When a compilation error is active, /command, /pause, and /unpause @@ -32,6 +33,7 @@ require daslib/json require daslib/json_boost require daslib/jobque_boost require daslib/debugger +require strings require live_host var live_api_port : int = 9090 @@ -84,6 +86,9 @@ def live_api_help_json() : string { let cmd_desc = "Dispatch a [live_command] (JSON body: \{\"name\":\"cmd\",\"args\":\{...\}\})" let cmd_curl = "curl -X POST -H \"Content-Type: application/json\" -d '\{\"name\":\"help\"\}' http://localhost:{live_api_port}/command" endpoints |> push(JV({ "method" => "POST", "path" => "/command", "description" => cmd_desc, "curl" => cmd_curl })) + let cmds_desc = "Dispatch a batch of [live_command]s in one round-trip (JSON array of \{name,args\}); continue-on-error, result array preserves input order" + let cmds_curl = "curl -X POST -H \"Content-Type: application/json\" -d '[\{\"name\":\"help\"\}]' http://localhost:{live_api_port}/commands" + endpoints |> push(JV({ "method" => "POST", "path" => "/commands", "description" => cmds_desc, "curl" => cmds_curl })) var result : table result |> insert("endpoints", JV(endpoints)) let err = get_last_error() @@ -156,6 +161,43 @@ class LiveApiServer : HvWebServer { return resp |> JSON(result, http_status.OK) } + // Batch: array of {name, args} objects. Continue-on-error — result array + // preserves input order; malformed entries get {"error":...} in their slot. + POST("/commands") <| @(var req : HttpRequest?; var resp : HttpResponse?) : http_status { + let guard = compilation_error_response(resp) + if (guard != http_status.OK) return guard + let body = string(req.body) + if (empty(body)) return resp |> JSON(write_json(JV("empty body")), http_status.BAD_REQUEST) + var parse_error = "" + var parsed = read_json(body, parse_error) + if (parsed == null || !(parsed.value is _array)) { + return resp |> JSON(write_json(JV("body must be a JSON array of \{name,args\} objects")), http_status.BAD_REQUEST) + } + let arr & = unsafe(parsed.value as _array) + let result = build_string() $(var w) { + write_char(w, int('[')) + var first = true + for (entry in arr) { + if (!first) write_char(w, int(',')) + first = false + let entry_v = unsafe(reinterpret(entry)) + if (entry_v == null || !(entry_v.value is _object)) { + write(w, "\{\"error\":\"invalid command object\"}") + continue + } + let entry_obj & = unsafe(entry_v.value as _object) + let name_v = unsafe(reinterpret(entry_obj?["name"] ?? null)) + if (name_v == null || !(name_v.value is _string)) { + write(w, "\{\"error\":\"missing or non-string name\"}") + continue + } + write(w, dispatch_command(write_json(entry_v))) + } + write_char(w, int(']')) + } + return resp |> JSON(result, http_status.OK) + } + // Catch-all: return help for any unmatched route ANY("*") <| @(var req : HttpRequest?; var resp : HttpResponse?) : http_status { return resp |> JSON(live_api_help_json(), http_status.OK) diff --git a/modules/dasLiveHost/live/live_api_stdio.das b/modules/dasLiveHost/live/live_api_stdio.das index da7f68dbb..0a823cd4e 100644 --- a/modules/dasLiveHost/live/live_api_stdio.das +++ b/modules/dasLiveHost/live/live_api_stdio.das @@ -27,24 +27,14 @@ module live_api_stdio shared public //! transport must redirect ``print()`` to stderr or a log file, or the //! application's output will interleave with JSON-RPC responses. //! -//! Spec compliance — this is a permissive JSON-RPC 2.0 subset, sized for -//! the live-command demo, not a general-purpose JSON-RPC server. If a -//! real-deal implementation is wanted under ``daslib/``, the gap list -//! against the spec (https://www.jsonrpc.org/specification) is: -//! -//! * Batch requests (§6) — array of requests / array of responses. Currently rejected as -32600. -//! * ``params`` type validation (§4.2) — spec says MUST be Array or Object; we forward whatever's there. -//! * ``-32601`` method not found and ``-32603`` internal error — ``dispatch_command`` returns its own error JSON inside ``result`` rather than a JSON-RPC ``error`` envelope. -//! * Optional ``data`` field on errors (genuinely optional, never emitted). -//! -//! Everything else (response envelope, id type validation, framing, -//! notification semantics, ``-32700`` / ``-32600``, permissive -//! ``jsonrpc`` member) is in place — the parser layer -//! (``parse_jsonrpc_request``, ``compact_json_whitespace``, -//! envelope helpers) is reusable as-is. +//! JSON-RPC 2.0 parsing/envelope/batch handling lives in ``daslib/jsonrpc``; +//! this module is the live_host bridge that wires ``dispatch_command`` to +//! that library plus the stdin/stdout transport. Permissive ``jsonrpc`` +//! field handling is the default — pass ``true`` to ``dispatch_line`` for +//! strict §4 compliance. -require daslib/json require daslib/json_boost +require daslib/jsonrpc require daslib/jobque_boost require daslib/debugger require daslib/fio @@ -58,127 +48,17 @@ struct StdioMsg { line : string } -// --- JSON-RPC 2.0 envelope --- - -def public json_rpc_response(id_json : string; result_json : string) : string { - return "\{\"jsonrpc\":\"2.0\",\"id\":{id_json},\"result\":{result_json}}" -} - -def public json_rpc_error(id_json : string; code : int; message : string) : string { - let msg_json = write_json(JV(message)) - return "\{\"jsonrpc\":\"2.0\",\"id\":{id_json},\"error\":\{\"code\":{code},\"message\":{msg_json}}}" -} - -def public serialize_id(id_val : JsonValue?) : string { - return id_val == null ? "null" : write_json(id_val) -} - -// --- Pure parsing layer (transport-agnostic, testable without dispatch) --- - -struct public ParsedJsonRpc { - id_str : string //!< Serialized JSON-RPC `id` field; "null" when missing or parse failed before id was reached. - method : string //!< Live command name; empty when the request was malformed. - inner_json : string //!< Synthesized `{"name":,"args":}` for live_host dispatch; empty when invalid. - error_envelope : string //!< Full JSON-RPC error response when the request was malformed; empty when valid. - is_notification : bool //!< True when the request omits `id` (JSON-RPC 2.0 notification — server MUST NOT respond). -} - -def public parse_jsonrpc_request(line : string) : ParsedJsonRpc { - //! Parse a single JSON-RPC 2.0 request line into its components without touching dispatch. - //! Pure function — testable in isolation from a live_host runtime. - //! - //! Permissive subset: the `jsonrpc` member is optional (a strict JSON-RPC - //! 2.0 server would reject requests omitting `"jsonrpc":"2.0"` with - //! `-32600`; this transport accepts them, since real-world clients are - //! lazy about the field). All other JSON-RPC 2.0 rules — id type - //! restrictions, notification semantics, required `method` — are - //! enforced. - var result = ParsedJsonRpc(id_str = "null") - var parse_error = "" - var parsed = read_json(line, parse_error) - if (parsed == null) { - result.error_envelope = json_rpc_error("null", -32700, "parse error: {parse_error}") - return result - } - if (!(parsed.value is _object)) { - result.error_envelope = json_rpc_error("null", -32600, "invalid request: not a JSON object") - return result - } - let body & = unsafe(parsed.value as _object) - let id_v = unsafe(reinterpret(body?["id"] ?? null)) - let method_v = unsafe(reinterpret(body?["method"] ?? null)) - let params_v = unsafe(reinterpret(body?["params"] ?? null)) - // JSON-RPC 2.0 §4: id MUST be string, number, or null. Reject objects / - // arrays / booleans — they'd also break framing since write_json would - // pretty-print them with embedded newlines. - if (id_v != null && !(id_v.value is _string) && !(id_v.value is _number) && !(id_v.value is _longint) && !(id_v.value is _null)) { - result.error_envelope = json_rpc_error("null", -32600, "invalid request: id must be string, number, or null") - return result - } - result.id_str = serialize_id(id_v) - if (method_v == null || !(method_v.value is _string)) { - // Don't flag as notification before method validation — `{}` or `{"method":42}` - // with no id would otherwise swallow the -32600 envelope per JSON-RPC §5.1. - result.error_envelope = json_rpc_error(result.id_str, -32600, "invalid request: missing or non-string method") - return result - } - result.is_notification = (id_v == null) - result.method = method_v.value as _string - let method_str = write_json(JV(result.method)) - let params_str = params_v == null ? "null" : write_json(params_v) - result.inner_json = "\{\"name\":{method_str},\"args\":{params_str}}" - return result -} - -// --- Compact serialization (write_json pretty-prints; the wire is newline-delimited) --- - -def public compact_json_whitespace(s : string) : string { - //! Strip JSON pretty-printer whitespace (spaces, tabs, newlines, carriage returns) - //! that appears outside string literals. JSON whitespace outside strings is - //! non-significant, so this is safe; whitespace inside `"..."` is preserved - //! verbatim. Used to flatten dispatch_command output for one-message-per-line - //! framing on the stdio wire. - return build_string() $(var w) { - peek_data(s) $(bytes) { - var in_string = false - var escape = false - for (b in bytes) { - if (in_string) { - write_char(w, int(b)) - if (escape) { - escape = false - } elif (b == uint8('\\')) { - escape = true - } elif (b == uint8('"')) { - in_string = false - } - } else { - if (b == uint8('"')) { - in_string = true - write_char(w, int(b)) - } elif (b != uint8(' ') && b != uint8('\n') && b != uint8('\t') && b != uint8('\r')) { - write_char(w, int(b)) - } - } - } - } - } -} - // --- Per-request dispatch --- def handle_jsonrpc_line(line : string) : string { - //! Dispatches a single JSON-RPC request line. Returns the response envelope - //! as a string, or an empty string for notifications (id-less requests) per - //! JSON-RPC 2.0 — callers must not write anything to the wire when the - //! return value is empty. - let req = parse_jsonrpc_request(line) - // Parse / validation errors: per spec, notifications still must not get a - // response even when malformed. - if (!empty(req.error_envelope)) return req.is_notification ? "" : req.error_envelope - let result_json = compact_json_whitespace(dispatch_command(req.inner_json)) - if (req.is_notification) return "" - return json_rpc_response(req.id_str, result_json) + //! Dispatch a JSON-RPC 2.0 wire line (single or §6 batch) through + //! ``dispatch_command``. Returns the response envelope ready for the wire, + //! or an empty string for notifications / all-notification batches. + return jsonrpc::dispatch_line(line, false) $(method, params_json) { + let method_str = write_json(JV(method)) + let inner = "\{\"name\":{method_str},\"args\":{params_json}}" + return jsonrpc::compact_json_whitespace(dispatch_command(inner)) + } } // --- Reader thread --- diff --git a/mouse-data/docs/add-cli-flag-to-daslang-executable.md b/mouse-data/docs/add-cli-flag-to-daslang-executable.md new file mode 100644 index 000000000..fe28c47f6 --- /dev/null +++ b/mouse-data/docs/add-cli-flag-to-daslang-executable.md @@ -0,0 +1,51 @@ +--- +slug: add-cli-flag-to-daslang-executable +title: How do I add a new command-line flag to the daslang executable that maps to a CodeOfPolicies field? +created: 2026-05-16 +last_verified: 2026-05-16 +links: [] +--- + +Touch four spots in `utils/daScript/main.cpp`: + +**1. File-scope static** (near the other flag statics around lines 34-57): + +```cpp +static bool logModuleCompileTime = false; +``` + +**2. Argv parser branch** in the main argv loop (`for ( int i=1; i < argc; ++i ) { if ( argv[i][0]=='-' ) { string cmd(argv[i]+1); ...`). Add an `else if` near related flags: + +```cpp +} else if ( cmd=="log-compile-time" ) { + logModuleCompileTime = true; +``` + +Single-dash style (`-log-compile-time`) for short or hyphenated flags. Double-dash (`--track-smart-ptr`) used for some longer ones. Existing convention is mostly single-dash — match the neighbors. + +**3. Wire to policies in BOTH population sites:** + +There are TWO CodeOfPolicies setup paths in `main.cpp`: +- `getPolicies()` near top of file — used by the AOT path (`das_aot_main`) +- `compile_and_run()` later — used by the normal run / JIT path + +Both need the new line: + +```cpp +policies.log_module_compile_time = logModuleCompileTime; +``` + +If you only update one, the flag silently does nothing in the other mode. (Easy to verify with `git grep -n "policies\.no_lint" utils/daScript/main.cpp` — every existing flag is set at both sites.) + +**4. Help text** in `print_help()`: + +```cpp +<< " -log-compile-time log detailed per-module compile-time breakdown (parse/infer/opt/macro/simulate)\n" +``` + +**Optional: also wire daslang-live** if the flag matters for the live-reload host. Its argv parser is `utils/daslang-live/main.cpp:648-680` and its `CodeOfPolicies` is constructed inside `compile_script()` ~line 171. Skip if the flag is daslang-only. + +**Rebuild target:** `cmake --build build --config Release --target daslang -j 64` (≈30s incremental). The static lives in `daslang.cpp`'s translation unit — no header changes, no library re-link of dependents. If you also touched `module_builtin_rtti.cpp` for the underlying policy registration, the full target list grows (rebuild test_aot if you care about AOT path). + +## Questions +- How do I add a new command-line flag to the daslang executable that maps to a CodeOfPolicies field? diff --git a/mouse-data/docs/add-field-to-codeofpolicies-full-checklist.md b/mouse-data/docs/add-field-to-codeofpolicies-full-checklist.md new file mode 100644 index 000000000..30a9a32f7 --- /dev/null +++ b/mouse-data/docs/add-field-to-codeofpolicies-full-checklist.md @@ -0,0 +1,46 @@ +--- +slug: add-field-to-codeofpolicies-full-checklist +title: What's the full checklist for adding a new field to CodeOfPolicies so it works end-to-end (compiler reads it, options.das_root maps it, RTTI sees it, das2rst documents it)? +created: 2026-05-16 +last_verified: 2026-05-16 +links: [] +--- + +Adding a `CodeOfPolicies` field looks like one edit but is actually FOUR coordinated changes. Miss any of them and you get silent failures (no compile error, just wrong rendered docs / option not visible to RTTI). + +**1. Declare the field** in `include/daScript/ast/ast.h` inside `struct CodeOfPolicies` (~line 1489). Annotate `/*option*/` if you want it overridable via per-file `options foo = true`: + +```cpp +/*option*/ bool log_module_compile_time = false; // short trailing comment is fine +``` + +**2. Register it for RTTI** in `src/builtin/module_builtin_rtti.cpp` (search for the existing `addField("...")` calls in the CodeOfPolicies registration block, ~line 940): + +```cpp +addField("log_module_compile_time"); +``` + +Without this, `for_each_field` (used by `daslib/rst.das`) doesn't see the field and `das2rst` silently skips it. + +**3. Add a description line** in `doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst`. This file is **strictly positional**: it maps line-by-line to RTTI-registered fields in *offset-sorted order* (which equals C++ declaration order for non-virtual structs). Place the new line at the position matching where you declared the field in step 1. + +**Silent shift trap:** if the handmade file ends up short by ONE line vs registered fields, EVERY field after the gap gets its predecessor's description — but no error, no warning. The only way to spot it is reading the rendered `doc/source/stdlib/generated/rtti.rst` and confirming field name + description agree. + +The reader code is `daslib/rst.das:817-880` (`document_topic`). It computes `headerLen = got - expected` and uses the first `headerLen` lines as preamble, then maps the remaining lines 1:1 to fields. So adding a line in the WRONG slot doesn't crash — it just silently mis-binds. + +**4. Add a row** to the language-reference options table at `doc/source/reference/language/options.rst` (search for `log_total_compile_time` — it's a worked example near the bottom). Only needed if the field is `/*option*/` (user-overridable per file). + +**Then regen + verify:** + +```bash +cmake --build build --config Release --target daslang -j 64 +bin/daslang doc/reflections/das2rst.das +grep -n "your_new_field" doc/source/stdlib/generated/rtti.rst # must show with correct description +``` + +If the description shown next to your field is for the WRONG field, your handmade file is short by one — find the missing description and add it. (I hit this in PR #2677: `max_call_depth` had been missing a description for who-knows-how-long, silently shifting ~30 later fields' descriptions up by one in the generated rtti.rst.) + +Bonus: if you also want a CLI flag for your option in `daslang`, the wiring is described separately under `add-cli-flag-to-daslang-executable`. + +## Questions +- What's the full checklist for adding a new field to CodeOfPolicies so it works end-to-end (compiler reads it, options.das_root maps it, RTTI sees it, das2rst documents it)? diff --git a/mouse-data/docs/das2rst-handmade-file-positional-mapping-and-silent-shift-trap.md b/mouse-data/docs/das2rst-handmade-file-positional-mapping-and-silent-shift-trap.md new file mode 100644 index 000000000..180d2078a --- /dev/null +++ b/mouse-data/docs/das2rst-handmade-file-positional-mapping-and-silent-shift-trap.md @@ -0,0 +1,37 @@ +--- +slug: das2rst-handmade-file-positional-mapping-and-silent-shift-trap +title: How does daslib/rst.das map handmade .rst description lines to struct fields, and what is the silent-misalignment trap when one line is missing? +created: 2026-05-16 +last_verified: 2026-05-16 +links: [] +--- + +`doc/source/stdlib/handmade/structure_annotation--.rst` files (and similar `class_annotation-*.rst`) are read **positionally** by `daslib/rst.das:817-880` (`document_topic`): + +1. Reader splits the file on `\n`, strips trailing blank lines → `got` lines. +2. Compares to `expected = length(tab) - 1` (one row per field, after header row). +3. Computes `headerLen = got - expected`. +4. First `headerLen` lines printed verbatim as preamble (typically the struct description). +5. **Last `expected` lines** mapped 1:1 to fields in **offset-sorted order** (= C++ declaration order for non-virtual structs). + +So for a struct with N fields, you want N+1 non-blank lines (1 struct description + N field descriptions). The mapping is by POSITION, not by field name — there's no field-name annotation in the .rst, just one description per line. + +**The silent shift trap:** if the file ends up with N lines instead of N+1 (one missing description anywhere in the middle), every field from the gap onwards inherits its successor's description. No compile error, no warning — `das2rst` happily produces garbage docs. The only way to detect: + +```bash +bin/daslang doc/reflections/das2rst.das +grep -A 1 "field_name_you_know" doc/source/stdlib/generated/rtti.rst # description text on next line must match +``` + +If the description belongs to a different field, your handmade file is short by one line somewhere between the file start and that field. Binary-search backwards to find where the shift begins — the first field with a wrong description marks the slot where one line is missing. + +**Why this is fragile:** the file uses no field-name markers, no JSON, no positional separators. A future contributor adds a new C++ field, forgets the handmade entry → the next field's description (and everything after) drifts. Has happened multiple times to `CodeOfPolicies`. + +**Fix recipe when you spot it:** read the offset of the misalignment in the rendered file, count back to find which C++ field has no description, add a line at the matching position in handmade. Repeat until rendered matches expected. + +If `bin/daslang doc/reflections/das2rst.das` panics with "has less documentation than values" — that's the FRIENDLY case (file has FEWER lines than fields). The silent case is when somebody added the right number of lines but in the wrong positions, or when an EARLIER field's description is missing. + +PR #2677 fixed a pre-existing silent shift in `structure_annotation-rtti-CodeOfPolicies.rst` (missing description for `max_call_depth`). About 30 fields downstream had their previous-field's description showing. + +## Questions +- How does daslib/rst.das map handmade .rst description lines to struct fields, and what is the silent-misalignment trap when one line is missing? diff --git a/mouse-data/docs/dasimgui-new-widget-module-needs-das-module-entry.md b/mouse-data/docs/dasimgui-new-widget-module-needs-das-module-entry.md new file mode 100644 index 000000000..102d4b76e --- /dev/null +++ b/mouse-data/docs/dasimgui-new-widget-module-needs-das-module-entry.md @@ -0,0 +1,24 @@ +--- +slug: dasimgui-new-widget-module-needs-das-module-entry +title: When adding a new .das sibling module under modules/dasImgui/widgets/, why does `require imgui/` fail with "missing prerequisite; file not found" (error 20605)? +created: 2026-05-15 +last_verified: 2026-05-15 +links: [] +--- + +**Cause:** dasImgui uses `.das_module` (at `modules/dasImgui/.das_module`) to map `imgui/` requires to widgets/*.das via `register_native_path("imgui", "name", "{project_path}/widgets/file.das")`. A new .das file under `widgets/` is **not** auto-discovered; the module path registration is what makes `require imgui/` resolve. + +**Fix:** add a `register_native_path` line in `modules/dasImgui/.das_module`'s `initialize()` function alongside the existing entries for `imgui_lint`, `imgui_live`, etc. + +```das +register_native_path("imgui", "imgui_harness", "{project_path}/widgets/imgui_harness.das") +``` + +No daslang rebuild needed — `.das_module` runs at module-load time on every invocation. + +**Counterpart for daslib/:** the `daslib/` path uses a different mechanism (auto-discovered from `modules/dasImgui/daslib/*.das`) and doesn't need `.das_module` entries. + +**Symptom this caused:** PR creating `widgets/imgui_harness.das` saw `require imgui/imgui_harness` fail with error 20605 until the `.das_module` line was added. + +## Questions +- When adding a new .das sibling module under modules/dasImgui/widgets/, why does `require imgui/` fail with "missing prerequisite; file not found" (error 20605)? diff --git a/mouse-data/docs/daslang-private-require-structural-symbol-gate.md b/mouse-data/docs/daslang-private-require-structural-symbol-gate.md new file mode 100644 index 000000000..600fd8a11 --- /dev/null +++ b/mouse-data/docs/daslang-private-require-structural-symbol-gate.md @@ -0,0 +1,29 @@ +--- +slug: daslang-private-require-structural-symbol-gate +title: How do I make a daslang wrapper module enforce "users cannot call symbols from the backend module" structurally — without relying on a lint or a runtime check? +created: 2026-05-15 +last_verified: 2026-05-15 +links: [] +--- + +**Pattern: private `require` for the backend.** + +```das +module my_wrapper shared public + +require imgui_backend_safe public // re-exported to consumers +require imgui_backend_glfw // PRIVATE — symbols stay inside this module +``` + +A `require X` (no `public`) is **private to the requiring module**. Symbols from X are visible inside `my_wrapper.das` but do NOT transitively re-export to anything that does `require my_wrapper`. Consumers literally cannot resolve `glfwInit` / `ImGui_ImplGlfw_*` etc. — those names aren't in their scope. + +**Why this matters:** lints catch use AFTER it happens; private-require prevents the symbol from existing in user scope at all. A user who tries to call a forbidden symbol gets the standard "function not found" diagnostic instead of a lint error. + +**A user who explicitly `require imgui_backend_glfw` themselves bypasses the structural guarantee** — and that's where a complementary lint catches the explicit-require attempt. Defense in depth: structure first, lint second. + +**Application:** dasImgui's `imgui_harness` (PR landing 2026-05-15) privately requires `imgui_app`, `glfw/glfw_boost`, `opengl/opengl_boost`, `live/glfw_live`, `live/opengl_live`, `imgui/imgui_live`, `imgui/imgui_visual_aids`. Consumers `require imgui/imgui_harness` and the GLFW/GL function names are not in their scope. + +**Source:** `modules/dasImgui/widgets/imgui_harness.das`. Same pattern works for any backend-abstraction wrapper. + +## Questions +- How do I make a daslang wrapper module enforce "users cannot call symbols from the backend module" structurally — without relying on a lint or a runtime check? diff --git a/mouse-data/docs/daslang-script-flags-need-dash-dash-separator.md b/mouse-data/docs/daslang-script-flags-need-dash-dash-separator.md new file mode 100644 index 000000000..ab78aaba7 --- /dev/null +++ b/mouse-data/docs/daslang-script-flags-need-dash-dash-separator.md @@ -0,0 +1,29 @@ +--- +slug: daslang-script-flags-need-dash-dash-separator +title: When daslang.exe runs a script with `daslang FILE.das --flag`, why doesn't the script's `get_user_args()` see `--flag`? +created: 2026-05-15 +last_verified: 2026-05-15 +links: [] +--- + +**Cause:** `get_user_args()` in `daslib/clargs` is **mode-aware**: +- Standalone exe (`daslang -exe`-built): returns `argv[1..]` +- Interpreter mode (regular `daslang.exe FILE.das`): returns the slice **after** the `--` separator. No `--` = empty args. + +So `daslang.exe FILE.das --headless` → `get_user_args()` returns `[]` because there's no `--` separator. The `--headless` token is consumed by daslang.exe itself (or treated as positional and ignored). + +**Fix — invoke with `--` separator:** +``` +daslang.exe FILE.das -- --headless +``` + +`get_user_args()` now returns `["--headless"]` and clargs `find_bool_flag_raw_value(args, "--headless")` resolves correctly. + +**For dastest:** `dastest --test FILE.das -- --headless` works the same way; dastest passes everything after its own `--` through to the script. + +**Source:** `daslib/clargs.das:74` `get_user_args()` / `:36` `get_cli_arguments()` — `find_index(all_args, "--")` is the gate. + +**Related memory:** `feedback_clargs_underscore_hyphen` (clargs field underscores → hyphens). Together they characterize the clargs invocation contract for daslang-script CLI flags. + +## Questions +- When daslang.exe runs a script with `daslang FILE.das --flag`, why doesn't the script's `get_user_args()` see `--flag`? diff --git a/mouse-data/docs/is-there-a-json-rpc-2-0-implementation-in-daslang-i-can-use-or-extend.md b/mouse-data/docs/is-there-a-json-rpc-2-0-implementation-in-daslang-i-can-use-or-extend.md index 9019c9f80..038389437 100644 --- a/mouse-data/docs/is-there-a-json-rpc-2-0-implementation-in-daslang-i-can-use-or-extend.md +++ b/mouse-data/docs/is-there-a-json-rpc-2-0-implementation-in-daslang-i-can-use-or-extend.md @@ -6,26 +6,28 @@ last_verified: 2026-05-16 links: [] --- -A partial one — `modules/dasLiveHost/live/live_api_stdio.das` ships an 80%-compliant JSON-RPC 2.0 transport for live commands over stdin/stdout (newline-delimited framing). Reusable building blocks are public: +Yes — **`daslib/jsonrpc`** is the canonical transport-agnostic JSON-RPC 2.0 implementation. `require daslib/jsonrpc` and you get: -- `parse_jsonrpc_request(line) → ParsedJsonRpc` — pure parser; returns id/method/inner_json/error_envelope/is_notification -- `compact_json_whitespace(s) → string` — escape-quote-aware whitespace stripper (one-message-per-line guarantee) -- `json_rpc_response(id_json, result_json) → string`, `json_rpc_error(id_json, code, message) → string` -- `serialize_id(id_val) → string` +- **Envelope builders:** `response(id, result_json)`, `error(id, code, message)`, `error_with_data(id, code, message, data_json)`, `serialize_id(JsonValue?)`. +- **Server-side parsers:** `parse_request(line, strict=false) → ParsedRequest`, `parse_batch(line, strict=false) → ParsedBatch` (handles §6 batch, empty array, top-level malformed JSON). +- **Client-side builders:** `make_request(method, params_json, id : int|string)`, `make_notification(method, params_json)`, `make_batch(messages)`. +- **Client-side response parsers:** `parse_response(line) → ParsedResponse`, `parse_response_batch(line) → ParsedResponseBatch`. +- **High-level convenience:** `dispatch_line(line, strict, dispatcher_block) → string` — handles parsing, envelope wrapping, batch fan-out, notification suppression in one call. +- **Standard error code constants:** `PARSE_ERROR (-32700)`, `INVALID_REQUEST (-32600)`, `METHOD_NOT_FOUND (-32601)`, `INVALID_PARAMS (-32602)`, `INTERNAL_ERROR (-32603)`. +- **Framing helper:** `compact_json_whitespace(s)` — escape-aware whitespace stripper for newline-framed wires. -What's solidly spec-compliant: response envelope, id type validation (string/number/null only — objects/arrays/bools get -32600), `id:null` on parse failure, method-must-be-string, -32700 / -32600, notification semantics for well-formed requests, framing. +Spec compliance: §4 (request shape, id type, params type), §5.1 (error codes), §6 (batch), notification semantics. Permissive default — `jsonrpc:"2.0"` field is optional; pass `strict=true` for full §4 enforcement. Tests live under `tests/jsonrpc/` (~120 [test] cases covering envelope, parser, batch, strict mode, sending API, round-trip). -What's permissive: `jsonrpc:"2.0"` member is optional; non-"2.0" values are accepted. Documented in the module docstring and `doc/source/reference/utils/daslang_live.rst`. +In-tree consumers: +- `modules/dasLiveHost/live/live_api_stdio.das` — JSON-RPC 2.0 stdio transport for live commands. +- `utils/mcp/protocol.das` — the daslang MCP server. -What's missing if you want a real-deal daslib JSON-RPC server (gap list also in the live_api_stdio.das module docstring as of PR #2674): -- **Batch requests (§6)** — array-of-requests / array-of-responses. Currently rejected as -32600. Biggest single gap vs spec. -- **`params` type validation (§4.2)** — spec says MUST be Array or Object; we forward whatever's there. No -32602. -- **-32601 method not found / -32603 internal error** — `dispatch_command` returns its own error JSON shape inside `result` instead of a JSON-RPC `error` envelope. Wrapping would belong at the transport layer. -- **Optional `data` field on errors** — never emitted (genuinely optional per spec). +For runnable examples, see `examples/mcp/echo/` (a minimal client+server pair) and the `tutorials/jsonrpc/` walkthrough. -Tests live at `tests/live_host/test_live_api_stdio.das` (~40 cases across 8 [test] functions) and cover the parser layer in isolation; the integration path is covered by `examples/daslive/hello_stdio/`. - -For a daslib promotion: lift `parse_jsonrpc_request` + `compact_json_whitespace` + envelope helpers into `daslib/jsonrpc.das`, add batch dispatch (top-level array → fan out → array response), validate `params`, and let callers wrap their own dispatch table with proper -32601 / -32603 envelope handling. +History: prior to PR introducing `daslib/jsonrpc` (May 2026), the same logic was hand-rolled twice — once in `live_api_stdio.das`, once in `utils/mcp/protocol.das`. The library extracted the shared bits; both transports now `require daslib/jsonrpc` and inherit batch support for free. ## Questions - Is there a JSON-RPC 2.0 implementation in daslang I can use or extend? +- How do I parse a JSON-RPC request in daslang? +- How do I send a JSON-RPC batch? +- Where are the JSON-RPC error code constants? diff --git a/mouse-data/docs/when-do-i-need-unsafe-reinterpret-lt-jsonvalue-gt-around-a-body-key-null-lookup-and-when-can-i-just-use-the-const-view-directly.md b/mouse-data/docs/when-do-i-need-unsafe-reinterpret-lt-jsonvalue-gt-around-a-body-key-null-lookup-and-when-can-i-just-use-the-const-view-directly.md new file mode 100644 index 000000000..fcf00294b --- /dev/null +++ b/mouse-data/docs/when-do-i-need-unsafe-reinterpret-lt-jsonvalue-gt-around-a-body-key-null-lookup-and-when-can-i-just-use-the-const-view-directly.md @@ -0,0 +1,35 @@ +--- +slug: when-do-i-need-unsafe-reinterpret-lt-jsonvalue-gt-around-a-body-key-null-lookup-and-when-can-i-just-use-the-const-view-directly +title: 'When do I need `unsafe(reinterpret(...))` around a `body?["key"] ?? null` lookup, and when can I just use the const view directly?' +created: 2026-05-16 +last_verified: 2026-05-16 +links: [] +--- + +**Only when you store the pointer somewhere that wants `JsonValue?` (non-const-data).** For read-only operations (`is _string` / `as _string` / `write_json(v)` / `v != null` / `v.value is _object`) the const view from `?[]` works directly — no `unsafe` cast needed. + +The `?[]` operator from `daslib/json_boost` has two overloads (json_boost.das:29-37): + +```das +def operator ?[] (a : JsonValue const? ==const; key : string) : JsonValue? const { ... } +def operator ?[] (var a : JsonValue? ==const; key : string) : JsonValue? { ... } +``` + +Indexing a `const`-bound view returns `JsonValue? const` (const-pointer wrapper, data is mutable). For reading the value out (the common case in any parser), this is fine. The `unsafe(reinterpret(...))` strips the outer const so you can: + +- **Assign to a non-const struct field** — the load-bearing case. Example: `result.params = params_v` in `daslib/jsonrpc.das:188` requires `var params_v = unsafe(reinterpret(body?["params"] ?? null))`. Drop the reinterpret and you get `error[30915]: can only copy compatible type; JsonValue?& = JsonValue? const&`. +- **Pass to a function that takes `var` JsonValue?** — same root cause. + +What does NOT need the cast (idiomatic): + +```das +let id_v = body?["id"] ?? null // const view +if (id_v != null && id_v.value is _string) // read OK +let s = id_v.value as _string // read OK +let j = write_json(id_v) // takes JsonValue? const, OK +``` + +Historical context: `live_api_stdio.das` (PR #2674, before the `daslib/jsonrpc` extraction in PR #2679) wrapped every `?[]` lookup in `unsafe(reinterpret(...))` defensively. When the same code moved into `daslib/jsonrpc.das`, four of the six call sites turned out to be read-only and the cast was dropped. Watch for this pattern next time you migrate JSON-traversal code into a library — the `unsafe` is usually noise. + +## Questions +- When do I need `unsafe(reinterpret<JsonValue?>(...))` around a `body?[\"key\"] ?? null` lookup, and when can I just use the const view directly? diff --git a/mouse-data/docs/why-does-my-dastest-integration-test-hang-at-readiness-gate-failed-when-external-curl-to-status-works-fine-is-it-a-require-order.md b/mouse-data/docs/why-does-my-dastest-integration-test-hang-at-readiness-gate-failed-when-external-curl-to-status-works-fine-is-it-a-require-order.md new file mode 100644 index 000000000..efa990fc8 --- /dev/null +++ b/mouse-data/docs/why-does-my-dastest-integration-test-hang-at-readiness-gate-failed-when-external-curl-to-status-works-fine-is-it-a-require-order.md @@ -0,0 +1,75 @@ +--- +slug: why-does-my-dastest-integration-test-hang-at-readiness-gate-failed-when-external-curl-to-status-works-fine-is-it-a-require-order +title: why does my dastest integration test hang at "readiness gate FAILED" when external curl to /status works fine — is it a require-order issue in daslang-live? +created: 2026-05-16 +last_verified: 2026-05-16 +links: [] +--- + +# Symptom + +`dastest` integration tests hang at the `imgui_playwright` readiness gate: + +``` +[imgui_playwright] subprocess up, polling /status... +[imgui_playwright] readiness gate FAILED +``` + +(30s `wait_until_ready` timeout, then 120s popen drain timeout. External `curl http://localhost:9090/status` from a sibling shell returns 200 with proper status JSON throughout — only the popen parent's request loop can't see it.) + +# Root cause + +`live/live_api` was required BEFORE `imgui_app + glfw/glfw_boost + opengl/* + glfw_live + opengl_live` somewhere in the requirer chain (usually a wrapper module like `imgui/imgui_harness`). The `[_macro] installing` in `live_api.das` calls `fork_debug_agent_context(@@debug_agent)` at compile time. If that fork happens before GLFW is initialized in the live runtime, the resulting LiveApiServer becomes unreachable from a popen parent on Windows. + +Filed: [#2677](https://github.com/GaijinEntertainment/daScript/issues/2677). Distinct from #2675 (`ANY("*")` route shadowing). + +# Fix (mechanical) + +In the requirer module (yours or a wrapper you control), reorder requires so the **windowed backend stack comes first**: + +```das +// Windowed backend FIRST (correctness, not aesthetics). +require imgui +require imgui_app +require glfw/glfw_boost +require opengl/opengl_boost +require live/glfw_live +require live/opengl_live + +// Live-host + boost-runtime stack AFTER. +require live/live_api +require live/live_commands +require live/live_vars +require live_host +require imgui/imgui_live +require imgui/imgui_boost_runtime +require imgui/imgui_boost_v2 +require imgui/imgui_widgets_builtin +require imgui/imgui_containers_builtin +require imgui/imgui_visual_aids +``` + +This mirrors the canonical order every pre-`imgui_harness` example/test used verbatim. Reordering is a no-op for visibility / re-export semantics — purely a workaround for the install-time ordering bug. + +# How to recognize this gotcha + +- Test hangs at `readiness gate FAILED` (not at `body did not converge` or similar). +- External `curl` to `localhost:9090/status` works while the test hangs (proves the server is up — the popen parent specifically can't reach it). +- Always reproduces — not a flaky timing issue. +- ONLY triggers when run via `popen` (via `with_imgui_app` in `imgui_playwright`, or any `dastest` integration test). Direct `bin/Release/daslang-live.exe