diff --git a/.github/workflows/c-cpp.yml b/.github/workflows/c-cpp.yml index 8d4c777..95a5cc0 100644 --- a/.github/workflows/c-cpp.yml +++ b/.github/workflows/c-cpp.yml @@ -2,9 +2,9 @@ name: C/C++ CI on: push: - branches: [ dev, stable, dev-0.9.9, dev-1.0.0] + branches: [ dev, stable, dev-0.9.9, dev-1.0.0, dev-rolling-context] pull_request: - branches: [ dev, stable, dev-0.9.9, dev-1.0.0] + branches: [ dev, stable, dev-0.9.9, dev-1.0.0, dev-rolling-context] env: SPECS_BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..9753040 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,58 @@ +# Agent Guidelines for specs2016 + +## General Project Guidelines + +- When adding or removing tests in `specs/src/test/ProcessingTest.cc`, also update `count_processing_tests` in `specs/tests/valgrind_unit_tests.py`. +- When adding or removing tests in `specs/src/test/ALUUnitTest.cc`, also update `count_ALU_tests` in `specs/tests/valgrind_unit_tests.py`. +- When adding or removing tests in `specs/src/test/TokenTest.cc`, also update `count_token_tests` in `specs/tests/valgrind_unit_tests.py`. +- Keep any new `ProcessingTest.cc` regression cases appended at the end when possible, to avoid unnecessary renumbering. +- `MYASSERT(cond)` and `MYASSERT_WITH_MSG(cond, msg)` (defined in `specs/src/utils/ErrorReporting.h`) are **always-on runtime checks** that throw `SpecsException` via `MYTHROW`. They are *not* compiled out by `NDEBUG`. Do not add redundant `if`/`MYTHROW` guards that duplicate what a `MYASSERT` already covers. +- When building the project, always use the command-line `make clean all`. Do not skip the `clean` target, and do not use the `-j` argument. +- When changing the structure of any of the classes that have dump_ macros in `specs_gdb.py` and `specs.gdb` update the relevant macros as well. + +## Rolling Context Feature (Issue #106) + +### Token Normalization in tokens.cc + +The CONTEXT keyword requires special handling during token normalization (`normalizeTokenList` in `tokens.cc`): + +1. **Initial State**: When a CONTEXT token is first parsed, its `Literal()` field is empty. + +2. **Normalization Process** (lines 920-951 in tokens.cc): + - Check if the next token exists: `i+1 < tokList->size()` + - If yes, extract the integer offset from the next token (which can be a RANGE or a literal) + - Store the offset string in the CONTEXT token's literal field + - Erase the next token from the list + - If no next token exists, the literal remains empty + +3. **Compile-Time Validation** (lines 473-489 in specItems.cc): + - When `itemGroup::Compile` processes a CONTEXT token, it must validate that the literal is not empty + - Use `tokenVec[index].argIndex()` (not the vector index) for error messages + - Call `std::stoi(tokenVec[index].Literal())` only after validation + +### Key Insight + +The `if` statement at line 923 in tokens.cc checks `tok.Literal()==""` because: +- If there is no next token (`i+1 >= tokList->size()`), the condition is false +- The literal is never set +- The error is caught at compile time in specItems.cc with a proper error message + +**Example that triggers the error:** +``` +specs '1-* 1 CONTEXT' +``` +This produces: "CONTEXT at index X must be followed by an integer offset" + +### Testing + +All rolling context tests are in `specs/src/test/ProcessingTest.cc` (tests #229-#241). +Run with: `../exe/ProcessingTest` + +### Verbose Output + +When rolling context is used and verbose mode is enabled (`-v` flag), the buffer sizes are printed: +``` +Rolling context buffer sizes: forward= backward= +``` + +This is implemented in `specs/src/test/specs.cc` after the compilation phase. diff --git a/README.md b/README.md index fb65923..91fa2b1 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ What's new: * Support Python in `MSBuild` builds * Added MSI and stand-alone Windows executable to release artifacts * Debugging aids for GDB + * Rolling context support *** 1-May-2026: Version 0.9.9 is here @@ -78,8 +79,10 @@ When starting a new version: Contributors ============ -* Yoav Nir ([yoavnir](https://github.com/yoavnir)) -* Jean-Baptiste Jouband ([Gawesomer](https://github.com/Gawesomer)) +- Yoav Nir ([yoavnir](https://github.com/yoavnir)) +- Jean-Baptiste Jouband ([Gawesomer](https://github.com/Gawesomer)) +- donglrd ([donglrd](https://github.com/donglrd)) +- Miriam-R-coder ([Miriam-R-coder](https://github.com/Miriam)) Documentation ============= diff --git a/agents.md b/agents.md deleted file mode 100644 index 2824abb..0000000 --- a/agents.md +++ /dev/null @@ -1,9 +0,0 @@ -# Agents Notes - -- When adding or removing tests in `specs/src/test/ProcessingTest.cc`, also update `count_processing_tests` in `specs/tests/valgrind_unit_tests.py`. -- When adding or removing tests in `specs/src/test/ALUUnitTest.cc`, also update `count_ALU_tests` in `specs/tests/valgrind_unit_tests.py`. -- When adding or removing tests in `specs/src/test/TokenTest.cc`, also update `count_token_tests` in `specs/tests/valgrind_unit_tests.py`. -- Keep any new `ProcessingTest.cc` regression cases appended at the end when possible, to avoid unnecessary renumbering. -- `MYASSERT(cond)` and `MYASSERT_WITH_MSG(cond, msg)` (defined in `specs/src/utils/ErrorReporting.h`) are **always-on runtime checks** that throw `SpecsException` via `MYTHROW`. They are *not* compiled out by `NDEBUG`. Do not add redundant `if`/`MYTHROW` guards that duplicate what a `MYASSERT` already covers. -- When building the project, always use the command-line `make clean all`. Do not skip the `clean` target, and do not use the `-j` argument. -- When changing the structure of any of the classes that have dump_ macros in `specs_gdb.py` and `specs.gdb` update the relevant macros as well. diff --git a/manpage b/manpage index 7183d75..65be818 100644 --- a/manpage +++ b/manpage @@ -357,6 +357,48 @@ Similarly: specs w1 a: SKIP-UNTIL "a=BEGIN" 1-* 1 .P skips records until the first record whose first word is BEGIN, then processes that record and all subsequent records normally. +.IP "CONTEXT" 3 +Temporarily changes the working string to a nearby record without consuming it. +.B CONTEXT +is followed by an integer offset: +.B CONTEXT 0 +resets the working string to the current record, +.B CONTEXT 1 +sets it to the next record, +.B CONTEXT -1 +sets it to the previous record, and so on. +The required buffer sizes are computed at parse time from the largest positive and negative offsets used. If the offset refers to a record beyond the beginning or end of the input, the working string is set to the empty string. +.P +.B CONTEXT +must not be abbreviated. It is not supported with threading or multiple input streams. +.P +Example: +.P + echo -e "A\\nB\\nC" | specs 1-* 1 CONTEXT 1 1-* NW +.P +produces: "A B", "B C", "C" -- each line shows the current record followed by the next. +.P +In expressions, the syntax +.B @+n +and +.B @-n +provides access to nearby records without changing the working string. For example, +.B @+1 +refers to the next record and +.B @-1 +refers to the previous record. +.B @+0 +and +.B @-0 +are equivalent to +.B @@ +(the current record). +.P +Example: +.P + echo -e "A\\nB\\nC" | specs print @+1 1 +.P +produces: "B", "C", "" -- each line shows the next record (empty for the last). .SS "MainOptions" These are optional spec units that appear at the beginning of the specification and modify the behavior of the entire specification. @@ -680,6 +722,21 @@ will output 4. The assignment is performed and the value stored in the counter. .IP "At-sign (@) Operator" 3 The at-sign allows the inclusion of user-defined and system-defined labels as strings in expressions. The double at-sign (@@) substitutes for the entire input record. +The syntax +.B @+n +and +.B @-n +(where n is a natural number) accesses nearby records from the rolling context buffer. +.B @+1 +is the next record, +.B @-1 +is the previous record. +.B @+0 +and +.B @-0 +are equivalent to +.B @@. +Out-of-bounds offsets return the empty string. The forward and backward buffer sizes are computed automatically at parse time. This feature requires non-threaded mode and a single input stream. .SS "Built-In Functions" .IP "abs(x)" 3 @@ -855,6 +912,15 @@ or spec units, then this number will be equal to the result of .B number(). Otherwise, it will be higher. +.IP "ctxrecno()" 3 +Returns the record number of the record that input parts work on. This is similar to +.B recno(), +but considers rolling context, which +.B recno() +does not. When no +.I CONTEXT +is active, returns the same value as +.B recno(). .IP "record()" 3 Returns the entire input record. Equivalent to .B range(1,-1) diff --git a/specs/docs/alu_adv.md b/specs/docs/alu_adv.md index b2dba3c..e81562e 100644 --- a/specs/docs/alu_adv.md +++ b/specs/docs/alu_adv.md @@ -164,6 +164,7 @@ All three regular expression functions have an argument called `matchFlags`. Thi | `number()` | Returns the number of processing cycles we have already gone through. Unless `READ` or `READSTOP` are used, this will be equal to the number of records read so far. | | `range(n,m)` | Returns the substring from the *n*-th character (default first) to the *m*-th character (default last) | | `recno()` | Returns the number of the currently read record. If the `READ` or `READSTOP` keywords are used this may be greater than `number()` | +| `ctxrecno()` | Returns the record number of the record that input parts work on. This is similar to `recno()`, but considers rolling context, which `recno()` does not. | | `record()` | Returns the entire input record | | `word(n)` | Returns the *n*-th word | | `wordrange(n,m)` | Returns the substring from the *n*-th word (default first) to the *m*-th word (default last) | diff --git a/specs/docs/debugging.md b/specs/docs/debugging.md index e2dcb1b..08b042c 100644 --- a/specs/docs/debugging.md +++ b/specs/docs/debugging.md @@ -21,7 +21,7 @@ To debug specs effectively, you must build with debug symbols enabled. ```bash cd specs/src -python setup.py -v DEBUG +python3 setup.py -v DEBUG make clean all ``` @@ -30,38 +30,30 @@ The `-v DEBUG` flag tells the setup script to enable debug symbols and disable o ### On Windows ```cmd -cd specs\src -python setup.py -v DEBUG -c VS msbuild specs\specs.sln /p:Configuration=Debug /p:Platform=x64 ``` +However, `gdb` is not normally the debugger that you use on Windows. + --- ## Loading the GDB Macros ### Automatic Loading (Recommended) -When you run GDB from the `specs/src/` directory, the `.gdbinit` file is automatically loaded: +When you run GDB from the `specs/src/` directory, the `.gdbinit` file is automatically loaded. So you can add the `specs.gdb` file to `.gdbinit`. Or you can specify it on the command line: ```bash cd specs/src -gdb ./specs +gdb ../exe/specs -x gdb/specs.gdb ``` -GDB will automatically source `gdb/specs.gdb`, which loads the Python extension and registers all dump commands. - ### Manual Loading -If you're running GDB from a different directory, you can manually load the macros: +If you're running GDB from a different directory, you can manually load the macros from within GDB: -```bash -gdb ./specs -x specs/src/gdb/specs.gdb ``` - -Or from within GDB: - -``` -(gdb) source specs/src/gdb/specs.gdb +(gdb) source gdb/specs.gdb ``` ### Verify Loading @@ -69,15 +61,41 @@ Or from within GDB: After loading, you should see a welcome message: ``` +specs GDB extension loaded successfully + ======================================== specs GDB debugging macros loaded ======================================== Available commands: - dump_pstate - Dump ProcessingState - dump_sb - Dump StringBuilder - dump_item - Dump Item (polymorphic) - ... +dump_pstate - Dump ProcessingState +dump_sb - Dump StringBuilder +dump_item - Dump Item (polymorphic) +dump_items - Dump itemGroup +dump_token - Dump Token +dump_alu_value - Dump ALUValue +dump_alu_counters - Dump ALUCounters +dump_alu_vec - Dump AluVec +dump_alu_function - Dump AluFunction +dump_external_func_rec - Dump ExternalFunctionRec +dump_python_func_collection - Dump PythonFunctionCollection +dump_python_func_by_name - Dump PythonFuncRec by name +dump_python_func_rec - Dump PythonFuncRec +dump_python_func_arg - Dump PythonFuncArg +dump_exception - Dump SpecsException + +Breakpoint helpers: +bp_apply - Break on all 9 Item subclass apply methods +bp_getstr - Break on InputPart::getStr +bp_compile - Break on itemGroup::Compile +bp_parseAluExpression - Break on parseAluExpression, where expressions are parsed +bp_pyfuncs - Break on the imoprtant functions related to Python functions: +. - PythonFunctionCollection::Initialize, where the Python Function Collection is initialized +. - PythonFunctionCollection::GetFunctionByName where the function record is retrieved based on name +. - PythonFuncRec::setArgValue, where an argument for an external function is set +. - PythonFuncRec::Call, where an external function is invoked + +For more help, type: help dump-processing-state ``` --- @@ -143,22 +161,25 @@ Available commands: ### Python Interface Commands -|| Command | Purpose | -||---------|----------| -|| `dump_alu_function ` | Dump an AluFunction (name, arg count, input dependency) | -|| `dump_external_func_rec ` | Dump an ExternalFunctionRec (polymorphic base class) | -|| `dump_external_func_collection ` | Dump an ExternalFunctionCollection (initialization state) | -|| `dump_python_func_collection ` | Dump a PythonFunctionCollection (internal Python function registry) | -|| `dump_python_func_rec ` | Dump a PythonFuncRec (Python function record with name and args) | -|| `dump_python_func_arg ` | Dump a PythonFuncArg (function argument with default value) | +| Command | Purpose | +|---------|----------| +| `dump_alu_function ` | Dump an AluFunction (name, arg count, input dependency) | +| `dump_external_func_rec ` | Dump an ExternalFunctionRec (polymorphic base class) | +| `dump_external_func_collection ` | Dump an ExternalFunctionCollection (initialization state) | +| `dump_python_func_collection ` | Dump a PythonFunctionCollection (internal Python function registry) | +| `dump_python_func_rec ` | Dump a PythonFuncRec (Python function record with name and args) | +| `dump_python_func_by_name ` | Dump a PythonFuncRec (Python function record with name and args) by collection and function name | +| `dump_python_func_arg ` | Dump a PythonFuncArg (function argument with default value) | ### Breakpoint Helpers -| Command | Purpose | -|---------|---------| -| `bp_apply` | Set breakpoint on Item::apply | -| `bp_getstr` | Set breakpoint on InputPart::getStr | -| `bp_compile` | Set breakpoint on itemGroup::Compile | +| Command | Description | +|---------|-------------| +| `bp_apply` | Set breakpoints on all 9 `::apply` methods of `Item` subclasses | +| `bp_getstr` | Set breakpoint on `InputPart::getStr` | +| `bp_compile` | Set breakpoint on `itemGroup::Compile` | +| `bp_parseAluExpression` | Set breakpoint on `parseAluExpression`, where expressions are parsed | +| `bp_pyfuncs` | Set breakpoints on all Python function-related methods. This includes `PythonFunctionCollection::Initialize`, where the Python Function Collection is initialized, `PythonFunctionCollection::GetFunctionByName` where the function record is retrieved based on name, `PythonFuncRec::setArgValue`, where an argument for an external function is set, and `PythonFuncRec::Call`, where an external function is invoked | --- @@ -169,30 +190,53 @@ Available commands: Suppose you're debugging a spec that processes records and you want to see the current state: ``` -(gdb) break Item::apply +(gdb) bp_apply Breakpoint 1 at 0x... (gdb) run < input.txt Starting program: ./specs ... -Breakpoint 1, Item::apply (this=0x..., pState=0x..., pSB=0x...) at specitems/specItems.cc:... +Breakpoint 2, DataField::apply (this=0x5fb8b0, pState=..., pSB=0x7fffffffcf50) at specitems/dataField.cc:444 (gdb) dump_pstate pState -ProcessingState @ 0x7fffffffde00 - Current Record: "hello world" - Previous Record: "goodbye world" +ProcessingState @ 0x7fffffffd070 + Current Record: "How am I doing?" + Previous Record: "well, hello there" + Input Record: "How am I doing?" Pad Char: ' ' (0x20) - Word Separator: " " - Field Separator: "\t" - Cycle Counter: 42 + Word Separator: "" (local) + Field Separator: " " + Cycle Counter: 2 Extra Reads: 0 - Record Count: 42 - Word Count: 2 - Field Count: 1 - Input Station: -1 + Record Count: 2 + Context Offset: 0 + Word Count: -1 + Field Count: -1 + Word Positions (3 cached): + [0] 1-5 + [1] 7-11 + [2] 13-17 + Field Positions (0 cached): + (none) + Field Identifiers (0 entries): + (none) + FI Statistics (0 entries): + (none) + Break Values (1 entries): + 'a' = "well," + Break Level: (none) + Frequency Maps (0 entries): + (none) + Conditions (0 deep): + (empty) + Loops (0 deep): + (empty) + Input Station: FIRST Input Stream: 1 + Stream Changed: False + Writers: 0x7fffffffcfe0 Output Index: 1 - No Write: false - EOF: false + No Write: False + EOF: False ``` This shows you exactly what the current record is, how many times we've processed records, and the current separators. @@ -203,13 +247,19 @@ When debugging a data field specification: ``` (gdb) dump_data_field pDataField -DataField @ 0x... - m_label: A - m_outStart: 10 - m_maxLength: 20 - m_strip: true - m_conversion: UCASE - m_alignment: Left +Item @ 0x5fb8b0 + Original Index: 1 + readsLines: False + producesOutput: False + forcesRunoutCycle: False + isBreak: False +----- end of 'Item' dump + Label: a + Output Start: 10 + Max Length: 20 + Strip: True + Conversion: UCASE + Alignment: Left ``` This tells you that the field is labeled 'A', outputs starting at column 10, has a max length of 20 characters, strips whitespace, converts to uppercase, and is left-aligned. @@ -236,13 +286,13 @@ Then you can inspect individual items: ``` (gdb) dump_item pItemGroup.m_items[0] -Item @ 0x... - m_originalIndex: 0 - Debug: {Source=Range[1:10];Dest=@10L20} - readsLines: true - producesOutput: true - forcesRunoutCycle: false - isBreak: false +Item @ 0x5fb410 + Original Index: 1 + readsLines: False + producesOutput: False + forcesRunoutCycle: False + isBreak: False +----- end of 'Item' dump ``` ### Example 4: Examining ALU Expressions @@ -251,15 +301,16 @@ When debugging expression evaluation: ``` (gdb) dump_alu_value myALUValue -ALUValue @ 0x... - m_type: Int - m_value: "42" - m_exact: true +ALUValue @ 0x5fdc90 + Type: Int + Value: "42" + Exact: True (gdb) dump_alu_counters g_counters -ALUCounters @ 0x... - Counters (map): - m_map @ 0x... +ALUCounters @ 0x5e2c60 + Counters (2 entries): + #1: (int) 117 (exact) + #2: (float) -0.019522002761880094 ``` ### Example 5: Conditional Breakpoints with Cycle Counter @@ -267,12 +318,12 @@ ALUCounters @ 0x... To break only on a specific record number: ``` -(gdb) break Item::apply if pState.m_CycleCounter == 100 +(gdb) break DataField::apply if pState.m_CycleCounter == 100 Breakpoint 1 at 0x... (gdb) run < input.txt ... -Breakpoint 1, Item::apply (this=0x..., pState=0x..., pSB=0x...) at specitems/specItems.cc:... +Breakpoint 1, DataField::apply (this=0x..., pState=0x..., pSB=0x...) at specitems/specItems.cc:... (gdb) dump_pstate pState ProcessingState @ 0x... @@ -284,35 +335,42 @@ This is useful for debugging issues that only occur on specific records. ### Example 6: Debugging Python Function Integration -When debugging Python function calls and integration: +When debugging Python function calls and integration, there is one macro that sets most of the important python-related breakpoints: ``` -(gdb) break PythonIntf.cc:167 -Breakpoint 1 at 0x... +(gdb) bp_pyfuncs +Breakpoint 2 at 0x50e8ce: file utils/PythonIntf.cc, line 351. +Breakpoint 3 at 0x51118c: file utils/PythonIntf.cc, line 629. +Breakpoint 4 at 0x50ce71: file utils/PythonIntf.cc, line 112. +Breakpoint 5 at 0x50d3c4: file utils/PythonIntf.cc, line 170. -(gdb) run -f myspec.txt < input.txt -... -Breakpoint 1, PyObject_CallObject (...) at PythonIntf.cc:167 +(gdb) run -f myspec < input.txt + +Breakpoint 2, PythonFunctionCollection::Initialize (this=0x5e6da0 , _path=0x5e5fd0 "/home/sio/specs") at utils/PythonIntf.cc:351 -(gdb) dump_python_func_collection g_PythonFunctions +(gdb) dump_python_func_collection gFunctionCollection PythonFunctionCollection @ 0x... - m_Initialized: true - m_Functions @ 0x... + Initialized: true + Functions (3 entries): + my_custom_function @ 0x... (2 args) + another_func @ 0x... (exact, 1 args) + third_func @ 0x... (0 args) -(gdb) dump_python_func_rec g_PythonFunctions.m_Functions[0] +(gdb) dump_python_func_by_name gFunctionCollection my_custom_function PythonFuncRec @ 0x... - m_name: my_custom_function - m_pFuncPtr: 0x... - m_doc: Computes the custom value based on input - m_pTuple: 0x... - m_args (2 items): - [0] input_value (default: counterType__Int) + Name: my_custom_function + Func Ptr: 0x... + doc: Computes the custom value based on input + Arg Type: no exactness information + Tuple: 0x... + Args (2 items): + [0] input_value (default: Int) = 0 - [1] multiplier (default: counterType__Float) + [1] multiplier (default: Float) = 1.5 ``` -This shows you the complete function signature, documentation, and argument defaults. The `m_pTuple` field shows whether arguments have been prepared for the function call. +This shows you the complete function signature, documentation, and argument defaults. The `Tuple` field shows whether arguments have been prepared for the function call. The collection dump now lists all functions with their key properties (exactness and argument count). --- diff --git a/specs/src/cli/tokens.cc b/specs/src/cli/tokens.cc index b82d9eb..c627e1a 100644 --- a/specs/src/cli/tokens.cc +++ b/specs/src/cli/tokens.cc @@ -367,6 +367,7 @@ void parseSingleToken(std::vector *pVec, std::string arg, int argidx) SIMPLETOKEN(skip-until, SKIPUNTIL); SIMPLETOKEN(splitw, SPLITW); SIMPLETOKEN(splitf, SPLITF); + SIMPLETOKEN(context, CONTEXT); /* question mark to replace PRINT */ if (arg[0]=='?') { @@ -916,6 +917,38 @@ void normalizeTokenList(std::vector *tokList) tok.setLiteral(separator); break; } + case TokenListType__CONTEXT: + { + if (i+1 < tokList->size()) { + std::string offsetStr; + if (TokenListType__RANGE == nextTok.Type() && nextTok.Range() && nextTok.Range()->isSingleNumber()) { + offsetStr = std::to_string(nextTok.Range()->getSingleNumber()); + nextTok.deallocDynamic(); + } else if (mayBeLiteral(nextTok)) { + offsetStr = getLiteral(nextTok); + } else { + std::string err = "CONTEXT at index " + std::to_string(tok.argIndex()) + + " must be followed by an integer offset, got <" + nextTok.Orig() + ">"; + MYTHROW(err); + } + // Validate that offsetStr is a valid integer + try { + std::stoi(offsetStr); + } catch (...) { + std::string err = "CONTEXT at index " + std::to_string(tok.argIndex()) + + " must be followed by an integer offset, got <" + offsetStr + ">"; + MYTHROW(err); + } + tok.setLiteral(offsetStr); + tokList->erase(tokList->begin()+(i+1)); + } + if (tok.Literal()=="") { + std::string err = "CONTEXT at index " + std::to_string(tok.argIndex()) + + " must be followed by an integer offset"; + MYTHROW(err); + } + break; + } default: break; } diff --git a/specs/src/cli/tokens.h b/specs/src/cli/tokens.h index 296b00f..369e42a 100644 --- a/specs/src/cli/tokens.h +++ b/specs/src/cli/tokens.h @@ -79,6 +79,7 @@ X(SKIPWHILE, false, true) \ X(SPLITW, false, false) \ X(SPLITF, false, false) \ + X(CONTEXT, false, true) \ X(DUMMY, false, false) #define X(t,r,l) TokenListType__##t, diff --git a/specs/src/gdb/COMMANDS.md b/specs/src/gdb/COMMANDS.md index 2c85084..d4276f3 100644 --- a/specs/src/gdb/COMMANDS.md +++ b/specs/src/gdb/COMMANDS.md @@ -63,33 +63,32 @@ ## Python Interface Commands -|| Command | Alias | Description | -||---------|-------|-------------| -|| `dump-alu-function` | `dump_alu_function` | Dump an AluFunction (name, arg count, input dependency) | -|| `dump-external-function-rec` | `dump_external_func_rec` | Dump an ExternalFunctionRec (calls virtual methods GetArgCount/GetFuncPtr) | -|| `dump-external-function-collection` | `dump_external_func_collection` | Dump an ExternalFunctionCollection (initialization state) | -|| `dump-python-function-collection` | `dump_python_func_collection` | Dump a PythonFunctionCollection (registry state and function count) | -|| `dump-python-func-rec` | `dump_python_func_rec` | Dump a PythonFuncRec (name, pointer, doc, and expanded argument list) | -|| `dump-python-func-arg` | `dump_python_func_arg` | Dump a PythonFuncArg (name, default type, and default value) | +| Command | Alias | Description | +|---------|-------|-------------| +| `dump-alu-function` | `dump_alu_function` | Dump an AluFunction (name, arg count, input dependency) | +| `dump-external-function-rec` | `dump_external_func_rec` | Dump an ExternalFunctionRec (calls virtual methods GetArgCount/GetFuncPtr) | +| `dump-external-function-collection` | `dump_external_func_collection` | Dump an ExternalFunctionCollection (initialization state) | +| `dump-python-function-collection` | `dump_python_func_collection` | Dump a PythonFunctionCollection (registry state, function count, and function list) | +| `dump-python-func-by-name` | `dump_python_func_by_name` | Dump a PythonFuncRec by looking it up in a collection by name | +| `dump-python-func-rec` | `dump_python_func_rec` | Dump a PythonFuncRec (name, pointer, doc, and expanded argument list) | +| `dump-python-func-arg` | `dump_python_func_arg` | Dump a PythonFuncArg (name, default type, and default value) | ## Utility Commands -|| Command | Alias | Description | -||---------|-------|-------------| -|| `dump-exception` | `dump_exception` | Dump a SpecsException | -|| `dump-all` | — | Dump all relevant debugging info | +| Command | Alias | Description | +|---------|-------|-------------| +| `dump-exception` | `dump_exception` | Dump a SpecsException | +| `dump-all` | — | Dump all relevant debugging info | ## Breakpoint Helpers -|| Command | Description | -||---------|-------------| -|| `bp_apply` | Set breakpoint on Item::apply | -|| `bp_getstr` | Set breakpoint on InputPart::getStr | -|| `bp_compile` | Set breakpoint on itemGroup::Compile | -|| `bp_parseAluExpression` | Set breakpoint on parseAluExpression, where expressions are parsed | -|| `bp_pfc_initialize` | Set breakpoint on PythonFunctionCollection::Initialize, where the Python Function Collection is initialized | -|| `bp_func_setargvalue` | Set breakpoint on PythonFuncRec::setArgValue, where an argument for an external function is set | -|| `bp_func_call` | Set breakpoint on PythonFuncRec::Call, where an external function is invoked | +| Command | Description | +|---------|-------------| +| `bp_apply` | Set breakpoints on all 9 `::apply` methods of `Item` subclasses | +| `bp_getstr` | Set breakpoint on `InputPart::getStr` | +| `bp_compile` | Set breakpoint on `itemGroup::Compile` | +| `bp_parseAluExpression` | Set breakpoint on `parseAluExpression`, where expressions are parsed | +| `bp_pyfuncs` | Set breakpoints on all Python function-related methods. This includes `PythonFunctionCollection::Initialize`, where the Python Function Collection is initialized, `PythonFunctionCollection::GetFunctionByName` where the function record is retrieved based on name, `PythonFuncRec::setArgValue`, where an argument for an external function is set, and `PythonFuncRec::Call`, where an external function is invoked | ## Usage Examples @@ -120,18 +119,26 @@ Item @ 0x... ### Dump ALUValue ```gdb (gdb) dump_alu_value myValue -ALUValue @ 0x... - m_type: Int - m_value: "42" - m_exact: true +ALUValue @ 0x5fdc30 + Type: Int + Value: "17" + Exact: True ``` -### Set Breakpoint on Item::apply +### Set Breakpoints on Item::apply ```gdb (gdb) bp_apply -Breakpoint 1 at 0x... +Breakpoint 1 at 0x...: DataField::apply +Breakpoint 2 at 0x...: TokenItem::apply +Breakpoint 3 at 0x...: SetItem::apply +Breakpoint 4 at 0x...: SkipItem::apply +Breakpoint 5 at 0x...: ConditionItem::apply +Breakpoint 6 at 0x...: BreakItem::apply +Breakpoint 7 at 0x...: SelectItem::apply +Breakpoint 8 at 0x...: SplitItem::apply +Breakpoint 9 at 0x...: ContextItem::apply (gdb) run -Breakpoint 1, Item::apply (this=0x..., pState=0x..., pSB=0x...) at specitems/specItems.cc:... +Breakpoint 1, DataField::apply (this=0x..., pState=..., pSB=0x...) at specitems/dataField.cc:... (gdb) dump_pstate pState ``` diff --git a/specs/src/gdb/specs.gdb b/specs/src/gdb/specs.gdb index 36fc163..b124feb 100644 --- a/specs/src/gdb/specs.gdb +++ b/specs/src/gdb/specs.gdb @@ -88,6 +88,10 @@ define dump_break_item dump-break-item $arg0 end +define dump_context_item + dump-context-item $arg0 +end + define dump_select_item dump-select-item $arg0 end @@ -169,6 +173,10 @@ define dump_python_func_collection dump-python-function-collection $arg0 end +define dump_python_func_by_name + dump-python-func-by-name $arg0 $arg1 +end + define dump_python_func_rec dump-python-func-rec $arg0 end @@ -187,10 +195,18 @@ end # ============================================================================ define bp_apply - break Item::apply + break DataField::apply + break TokenItem::apply + break SetItem::apply + break SkipItem::apply + break ConditionItem::apply + break BreakItem::apply + break SelectItem::apply + break SplitItem::apply + break ContextItem::apply end document bp_apply - Set a breakpoint on Item::apply to debug item application. + Set breakpoints on every Item subclass's apply method (9 breakpoints). end define bp_getstr @@ -214,25 +230,25 @@ document bp_parseAluExpression Set a breakpoint on parseAluExpression to debug parsing of mathematical expressions. end -define bp_pfc_initialize - break PythonFunctionCollection::Initialize +define bp_context_apply + break ContextItem::apply end -document bp_pfc_initialize - Set a breakpoint on PythonFunctionCollection::Initialize to debug the initialization of the Python Function Collection. +document bp_context_apply + Set a breakpoint on ContextItem::apply to debug rolling context operations. end -define bp_func_setargvalue +define bp_pyfuncs + break PythonFunctionCollection::Initialize + break PythonFunctionCollection::GetFunctionByName break PythonFuncRec::setArgValue -end -document bp_func_setargvalue - Set a breakpoint on PythonFuncRec::setArgValue to debug setting external function arguments. -end - -define bp_func_call break PythonFuncRec::Call end -document bp_func_call - Set a breakpoint on PythonFuncRec::Call to debug calling external functions. +document bp_pyfuncs + Set breakpoints on all Python function-related methods. This includes: + - PythonFunctionCollection::Initialize, where the Python Function Collection is initialized + - PythonFunctionCollection::GetFunctionByName where the function record is retrieved based on name + - PythonFuncRec::setArgValue, where an argument for an external function is set + - PythonFuncRec::Call, where an external function is invoked end # ============================================================================ @@ -265,18 +281,22 @@ echo dump_alu_counters - Dump ALUCounters\n echo dump_alu_vec - Dump AluVec\n echo dump_alu_function - Dump AluFunction\n echo dump_external_func_rec - Dump ExternalFunctionRec\n +echo dump_python_func_collection - Dump PythonFunctionCollection\n +echo dump_python_func_by_name - Dump PythonFuncRec by name\n echo dump_python_func_rec - Dump PythonFuncRec\n echo dump_python_func_arg - Dump PythonFuncArg\n echo dump_exception - Dump SpecsException\n echo \n echo Breakpoint helpers:\n -echo bp_apply - Break on Item::apply\n -echo bp_getstr - Break on InputPart::getStr\n -echo bp_compile - Break on itemGroup::Compile\n -echo bp_parseAluExpression - Break on parseAluExpression, where expressions are parsed\n -echo bp_pfc_initialize - Break on PythonFunctionCollection::Initialize, where the Python Function Collection is initialized\n -echo bp_func_setargvalue - Break on PythonFuncRec::setArgValue, where an argument for an external function is set\n -echo bp_func_call - Break on PythonFuncRec::Call, where an external function is invoked\n +echo bp_apply - Break on all 9 Item subclass apply methods\n +echo bp_getstr - Break on InputPart::getStr\n +echo bp_compile - Break on itemGroup::Compile\n +echo bp_parseAluExpression - Break on parseAluExpression, where expressions are parsed\n +echo bp_pyfuncs - Break on the imoprtant functions related to Python functions:\n +echo . - PythonFunctionCollection::Initialize, where the Python Function Collection is initialized\n +echo . - PythonFunctionCollection::GetFunctionByName where the function record is retrieved based on name\n +echo . - PythonFuncRec::setArgValue, where an argument for an external function is set\n +echo . - PythonFuncRec::Call, where an external function is invoked\n echo \n echo For more help, type: help dump-processing-state\n echo \n diff --git a/specs/src/gdb/specs_gdb.py b/specs/src/gdb/specs_gdb.py index af1ca04..708c0ba 100644 --- a/specs/src/gdb/specs_gdb.py +++ b/specs/src/gdb/specs_gdb.py @@ -158,6 +158,18 @@ 3: "NullStr", } +EXTREME_BOOL = { + 0: "False", + 1: "True", + 2: "DontCare", +} + +INPUT_STATION = { + -1: "FIRST", + -2: "SECOND", + 0: "STDERR", +} + # Token types (X-macro generated, simplified list) TOKEN_TYPES = { 0: "STOP", @@ -285,6 +297,65 @@ def std_vector_size(val): except: return 0 +def std_map_size(val): + """Get the size of a std::map.""" + try: + return int(val["_M_t"]["_M_impl"]["_M_node_count"]) + except: + return 0 + +def std_map_items(val): + """ + Iterate over key-value pairs in a std::map. + Uses GDB's default visualizer (libstdc++ StdMapPrinter). + Returns list of (key, value) tuples. + + Note: The libstdc++ StdMapPrinter yields *alternating* children: + [0] -> first key, [1] -> first value, + [2] -> second key, [3] -> second value, ... + This function pairs them up into (key, value) tuples. + """ + try: + items = [] + pp = gdb.default_visualizer(val) + if pp and hasattr(pp, 'children'): + children = list(pp.children()) + # Pair up alternating key/value entries + for i in range(0, len(children) - 1, 2): + key = children[i][1] # the gdb.Value for the key + value = children[i + 1][1] # the gdb.Value for the value + items.append((key, value)) + return items + except: + return [] + +def std_vector_int_items(val): + """ + Extract all elements from a std::vector as a Python list. + Returns an empty list on failure. + """ + try: + size = std_vector_size(val) + start = val["_M_impl"]["_M_start"] + return [int(start[i]) for i in range(size)] + except: + return [] + +def std_stack_items(val): + """ + Extract all elements from a std::stack as a Python list (bottom to top). + std::stack wraps a deque in its 'c' member. Uses GDB's pretty-printer + for the underlying deque to iterate. + """ + try: + deque = val["c"] + pp = gdb.default_visualizer(deque) + if pp and hasattr(pp, 'children'): + return [child[1] for child in pp.children()] + return [] + except: + return [] + def identify_dynamic_type(val): """ Identify the actual derived type of a polymorphic object by reading the vtable. @@ -313,6 +384,137 @@ def call_method_safe(val, method_name, *args): except: return None +def pyobj_repr(pyobj_ptr): + """ + Given a GDB value that is a PyObject*, return a human-readable string + representation of the Python object. Handles int, float, str, bool, + None, and tuple. Falls back to showing the type name for anything else. + """ + try: + if int(pyobj_ptr) == 0: + return "NULL" + + # Read ob_type->tp_name to discover the Python type + ob_type = pyobj_ptr["ob_type"] + tp_name = ob_type["tp_name"].string() + + if tp_name == "NoneType": + return "None" + + if tp_name == "bool": + # True/False are singletons; compare the pointer value to _Py_TrueStruct + try: + py_true = gdb.parse_and_eval("(PyObject*)&_Py_TrueStruct") + if int(pyobj_ptr) == int(py_true): + return "True" + return "False" + except: + return "" + + if tp_name == "float": + try: + float_type = gdb.lookup_type("PyFloatObject").pointer() + fval = pyobj_ptr.cast(float_type)["ob_fval"] + return str(float(fval)) + except: + return "" + + if tp_name == "int": + try: + long_type = gdb.lookup_type("PyLongObject").pointer() + long_obj = pyobj_ptr.cast(long_type) + lv_tag = int(long_obj["long_value"]["lv_tag"]) + # _PyLong_NON_SIZE_BITS = 3, _PyLong_SIGN_MASK = 3 + is_compact = lv_tag < (2 << 3) + if is_compact: + sign = 1 - (lv_tag & 3) + digit = int(long_obj["long_value"]["ob_digit"][0]) + return str(sign * digit) + else: + return "" + except: + return "" + + if tp_name == "str": + try: + ascii_type = gdb.lookup_type("PyASCIIObject").pointer() + ascii_obj = pyobj_ptr.cast(ascii_type) + length = int(ascii_obj["length"]) + kind = int(ascii_obj["state"]["kind"]) + is_ascii = int(ascii_obj["state"]["ascii"]) + is_compact = int(ascii_obj["state"]["compact"]) + if is_ascii and is_compact and kind == 1: + # Data follows immediately after the PyASCIIObject struct + data_ptr = (ascii_obj + 1).cast(gdb.lookup_type("char").pointer()) + s = data_ptr.string(length=min(length, 80)) + if length > 80: + return f'"{s}..."' + return f'"{s}"' + elif is_compact and kind == 1: + # Latin-1, data after PyCompactUnicodeObject + compact_type = gdb.lookup_type("PyCompactUnicodeObject").pointer() + compact_obj = pyobj_ptr.cast(compact_type) + data_ptr = (compact_obj + 1).cast(gdb.lookup_type("char").pointer()) + s = data_ptr.string(length=min(length, 80)) + if length > 80: + return f'"{s}..."' + return f'"{s}"' + else: + return f'' + except Exception as e: + return f"" + + if tp_name == "tuple": + try: + tuple_type = gdb.lookup_type("PyTupleObject").pointer() + tup = pyobj_ptr.cast(tuple_type) + # ob_size is in the PyVarObject header + var_type = gdb.lookup_type("PyVarObject").pointer() + size = int(pyobj_ptr.cast(var_type)["ob_size"]) + elems = [] + for i in range(min(size, 10)): + elem = tup["ob_item"][i] + elems.append(pyobj_repr(elem)) + inner = ", ".join(elems) + if size > 10: + inner += ", ..." + return f"({inner})" + except Exception as e: + return f"" + + # Fallback: just show the type name + return f"<{tp_name} object at {pyobj_ptr}>" + except: + return f"" + +def dump_pytuple(pyobj_ptr, arg_dict, indent=" "): + """ + Given a GDB value that is a PyObject* pointing to a Python tuple, print + its contents. The tuple holds the argument values for a Python function + call (populated by setArgValue, freed by ResetArgs). + """ + if not isinstance(arg_dict,dict): + arg_dict = dict() + try: + if int(pyobj_ptr) == 0: + print(f"{indent}Tuple: nullptr (no call arguments prepared)") + return + + var_type = gdb.lookup_type("PyVarObject").pointer() + size = int(pyobj_ptr.cast(var_type)["ob_size"]) + tuple_type = gdb.lookup_type("PyTupleObject").pointer() + tup = pyobj_ptr.cast(tuple_type) + + print(f"{indent}Tuple ({size} call args):") + for i in range(size): + elem = tup["ob_item"][i] + if i in arg_dict.keys(): + print(f"{indent} [{i}] {arg_dict[i]} = {pyobj_repr(elem)}") + else: + print(f"{indent} [{i}] {pyobj_repr(elem)}") + except Exception as e: + print(f"{indent}Tuple: {pyobj_ptr} (cannot inspect: {e})") + # ============================================================================ # PRETTY-PRINTERS # ============================================================================ @@ -447,7 +649,7 @@ def invoke(self, arg, from_tty): val = gdb.parse_and_eval(arg) m_str = std_string_to_str(val["m_Str"]) print(f"LiteralPart @ {val.address}") - print(f" m_Str: \"{m_str}\"") + print(f" String: \"{m_str}\"") except Exception as e: print(f"Error: {e}") @@ -463,8 +665,8 @@ def invoke(self, arg, from_tty): from_val = int(val["_from"]) to_val = int(val["_to"]) print(f"RangePart @ {val.address}") - print(f" _from: {from_val}") - print(f" _to: {to_val}") + print(f" From: {from_val}") + print(f" To: {to_val}") print(f" readsLines: true") except Exception as e: print(f"Error: {e}") @@ -482,9 +684,9 @@ def invoke(self, arg, from_tty): to_val = int(val["_to"]) sep = std_string_to_str(val["m_WordSep"]) print(f"WordRangePart @ {val.address}") - print(f" _from: {from_val}") - print(f" _to: {to_val}") - print(f" m_WordSep: \"{sep}\"") + print(f" From: {from_val}") + print(f" To: {to_val}") + print(f" Word Separator: \"{sep}\"") except Exception as e: print(f"Error: {e}") @@ -501,9 +703,9 @@ def invoke(self, arg, from_tty): to_val = int(val["_to"]) sep = std_string_to_str(val["m_FieldSep"]) print(f"FieldRangePart @ {val.address}") - print(f" _from: {from_val}") - print(f" _to: {to_val}") - print(f" m_FieldSep: \"{sep}\"") + print(f" From: {from_val}") + print(f" To: {to_val}") + print(f" Field Separator: \"{sep}\"") except Exception as e: print(f"Error: {e}") @@ -520,8 +722,8 @@ def invoke(self, arg, from_tty): type_str = CLOCK_TYPE.get(type_val, f"Unknown({type_val})") clock = int(val["m_StaticClock"]) print(f"ClockPart @ {val.address}") - print(f" m_Type: {type_str}") - print(f" m_StaticClock: {clock}") + print(f" Type: {type_str}") + print(f" Static Clock: {clock}") except Exception as e: print(f"Error: {e}") @@ -536,7 +738,7 @@ def invoke(self, arg, from_tty): val = gdb.parse_and_eval(arg) fid = std_string_to_str(val["m_fieldIdentifier"]) print(f"IDPart @ {val.address}") - print(f" m_fieldIdentifier: \"{fid}\"") + print(f" Field Identifier: \"{fid}\"") except Exception as e: print(f"Error: {e}") @@ -552,8 +754,8 @@ def invoke(self, arg, from_tty): raw_expr = std_string_to_str(val["m_rawExpression"]) is_assn = bool(val["m_isAssignment"]) print(f"ExpressionPart @ {val.address}") - print(f" m_rawExpression: \"{raw_expr}\"") - print(f" m_isAssignment: {is_assn}") + print(f" Expression: \"{raw_expr}\"") + print(f" Is Assignment: {is_assn}") except Exception as e: print(f"Error: {e}") @@ -572,7 +774,7 @@ def invoke(self, arg, from_tty): val = gdb.parse_and_eval(arg) orig_idx = int(val["m_originalIndex"]) print(f"Item @ {val.address}") - print(f" m_originalIndex: {orig_idx}") + print(f" Original Index: {orig_idx}") # Try to call virtual methods try: @@ -605,6 +807,8 @@ def invoke(self, arg, from_tty): print(f" isBreak: {bool(is_break)}") except: pass + + print("----- end of 'Item' dump") except Exception as e: print(f"Error: {e}") @@ -613,9 +817,17 @@ class DumpDataField(gdb.Command): def __init__(self): super(DumpDataField, self).__init__("dump-data-field", gdb.COMMAND_DATA) + self.dump_item = None def invoke(self, arg, from_tty): try: + # First, call DumpItem to print base class fields + if self.dump_item is None: + self.dump_item = DumpItem() + self.dump_item.invoke(arg, from_tty) + + # Then print derived class fields + print(f"DataField:") val = gdb.parse_and_eval(arg) label = chr(int(val["m_label"])) if int(val["m_label"]) > 0 else "none" out_start = int(val["m_outStart"]) @@ -627,13 +839,12 @@ def invoke(self, arg, from_tty): conv_str = STRING_CONVERSIONS.get(conv, f"Unknown({conv})") align_str = OUTPUT_ALIGNMENT.get(align, f"Unknown({align})") - print(f"DataField @ {val.address}") - print(f" m_label: {label}") - print(f" m_outStart: {out_start}") - print(f" m_maxLength: {max_len}") - print(f" m_strip: {strip}") - print(f" m_conversion: {conv_str}") - print(f" m_alignment: {align_str}") + print(f" Label: {label}") + print(f" Output Start: {out_start}") + print(f" Max Length: {max_len}") + print(f" Strip: {strip}") + print(f" Conversion: {conv_str}") + print(f" Alignment: {align_str}") except Exception as e: print(f"Error: {e}") @@ -642,18 +853,24 @@ class DumpTokenItem(gdb.Command): def __init__(self): super(DumpTokenItem, self).__init__("dump-token-item", gdb.COMMAND_DATA) + self.dump_item = None def invoke(self, arg, from_tty): try: + # First, call DumpItem to print base class fields + if self.dump_item is None: + self.dump_item = DumpItem() + self.dump_item.invoke(arg, from_tty) + + # Then print derived class fields + print(f"TokenItem:") val = gdb.parse_and_eval(arg) token = deref_shared_ptr(val["mp_Token"]) if token: type_val = int(token["m_type"]) type_str = TOKEN_TYPES.get(type_val, f"Unknown({type_val})") - print(f"TokenItem @ {val.address}") print(f" Token type: {type_str}") else: - print(f"TokenItem @ {val.address}") print(f" mp_Token: ") except Exception as e: print(f"Error: {e}") @@ -663,15 +880,22 @@ class DumpSetItem(gdb.Command): def __init__(self): super(DumpSetItem, self).__init__("dump-set-item", gdb.COMMAND_DATA) + self.dump_item = None def invoke(self, arg, from_tty): try: + # First, call DumpItem to print base class fields + if self.dump_item is None: + self.dump_item = DumpItem() + self.dump_item.invoke(arg, from_tty) + + # Then print derived class fields + print(f"SetItem:") val = gdb.parse_and_eval(arg) raw_expr = std_string_to_str(val["m_rawExpression"]) key = int(val["m_key"]) - print(f"SetItem @ {val.address}") - print(f" m_rawExpression: \"{raw_expr}\"") - print(f" m_key: {key}") + print(f" Expression: \"{raw_expr}\"") + print(f" Key: {key}") except Exception as e: print(f"Error: {e}") @@ -680,18 +904,25 @@ class DumpSkipItem(gdb.Command): def __init__(self): super(DumpSkipItem, self).__init__("dump-skip-item", gdb.COMMAND_DATA) + self.dump_item = None def invoke(self, arg, from_tty): try: + # First, call DumpItem to print base class fields + if self.dump_item is None: + self.dump_item = DumpItem() + self.dump_item.invoke(arg, from_tty) + + # Then print derived class fields + print(f"SkipItem:") val = gdb.parse_and_eval(arg) raw_expr = std_string_to_str(val["m_rawExpression"]) is_until = bool(val["m_bIsUntil"]) satisfied = bool(val["m_bSatisfied"]) skip_type = "SKIPUNTIL" if is_until else "SKIPWHILE" - print(f"SkipItem @ {val.address}") print(f" Type: {skip_type}") - print(f" m_rawExpression: \"{raw_expr}\"") - print(f" m_bSatisfied: {satisfied}") + print(f" Expression: \"{raw_expr}\"") + print(f" Satisfied: {satisfied}") except Exception as e: print(f"Error: {e}") @@ -700,18 +931,25 @@ class DumpConditionItem(gdb.Command): def __init__(self): super(DumpConditionItem, self).__init__("dump-condition-item", gdb.COMMAND_DATA) + self.dump_item = None def invoke(self, arg, from_tty): try: + # First, call DumpItem to print base class fields + if self.dump_item is None: + self.dump_item = DumpItem() + self.dump_item.invoke(arg, from_tty) + + # Then print derived class fields + print(f"ConditionItem:") val = gdb.parse_and_eval(arg) pred = int(val["m_pred"]) pred_str = CONDITION_PREDICATE.get(pred, f"Unknown({pred})") raw_expr = std_string_to_str(val["m_rawExpression"]) is_assn = bool(val["m_isAssignment"]) - print(f"ConditionItem @ {val.address}") - print(f" m_pred: {pred_str}") - print(f" m_rawExpression: \"{raw_expr}\"") - print(f" m_isAssignment: {is_assn}") + print(f" Predicate: {pred_str}") + print(f" Expression: \"{raw_expr}\"") + print(f" Is Assignment: {is_assn}") except Exception as e: print(f"Error: {e}") @@ -720,13 +958,42 @@ class DumpBreakItem(gdb.Command): def __init__(self): super(DumpBreakItem, self).__init__("dump-break-item", gdb.COMMAND_DATA) + self.dump_item = None def invoke(self, arg, from_tty): try: + # First, call DumpItem to print base class fields + if self.dump_item is None: + self.dump_item = DumpItem() + self.dump_item.invoke(arg, from_tty) + + # Then print derived class fields + print(f"BreakItem:") val = gdb.parse_and_eval(arg) ident = chr(int(val["m_identifier"])) - print(f"BreakItem @ {val.address}") - print(f" m_identifier: {ident}") + print(f" Identifier: {ident}") + except Exception as e: + print(f"Error: {e}") + +class DumpContextItem(gdb.Command): + """Dump a ContextItem.""" + + def __init__(self): + super(DumpContextItem, self).__init__("dump-context-item", gdb.COMMAND_DATA) + self.dump_item = None + + def invoke(self, arg, from_tty): + try: + # First, call DumpItem to print base class fields + if self.dump_item is None: + self.dump_item = DumpItem() + self.dump_item.invoke(arg, from_tty) + + # Then print derived class fields + print(f"ContextItem:") + val = gdb.parse_and_eval(arg) + offset = int(val["m_offset"]) + print(f" Offset: {offset}") except Exception as e: print(f"Error: {e}") @@ -735,15 +1002,22 @@ class DumpSelectItem(gdb.Command): def __init__(self): super(DumpSelectItem, self).__init__("dump-select-item", gdb.COMMAND_DATA) + self.dump_item = None def invoke(self, arg, from_tty): try: + # First, call DumpItem to print base class fields + if self.dump_item is None: + self.dump_item = DumpItem() + self.dump_item.invoke(arg, from_tty) + + # Then print derived class fields + print(f"SelectItem:") val = gdb.parse_and_eval(arg) stream = int(val["m_stream"]) b_output = bool(val["bOutput"]) - print(f"SelectItem @ {val.address}") - print(f" m_stream: {stream}") - print(f" bOutput: {b_output}") + print(f" Stream: {stream}") + print(f" Output: {b_output}") except Exception as e: print(f"Error: {e}") @@ -752,20 +1026,27 @@ class DumpSplitItem(gdb.Command): def __init__(self): super(DumpSplitItem, self).__init__("dump-split-item", gdb.COMMAND_DATA) + self.dump_item = None def invoke(self, arg, from_tty): try: + # First, call DumpItem to print base class fields + if self.dump_item is None: + self.dump_item = DumpItem() + self.dump_item.invoke(arg, from_tty) + + # Then print derived class fields + print(f"SplitItem:") val = gdb.parse_and_eval(arg) is_field = bool(val["m_isField"]) sep = std_string_to_str(val["m_separator"]) splitting = bool(val["m_splitting"]) current_piece = int(val["m_currentPiece"]) split_type = "SPLITF" if is_field else "SPLITW" - print(f"SplitItem @ {val.address}") print(f" Type: {split_type}") - print(f" m_separator: \"{sep}\"") - print(f" m_splitting: {splitting}") - print(f" m_currentPiece: {current_piece}") + print(f" Separator: \"{sep}\"") + print(f" Splitting: {splitting}") + print(f" Current Piece: {current_piece}") except Exception as e: print(f"Error: {e}") @@ -790,8 +1071,8 @@ def invoke(self, arg, from_tty): item_count = std_vector_size(items_vec) print(f"itemGroup @ {val.address}") - print(f" bNeedRunoutCycle: {need_runout}") - print(f" bFoundSelectSecond: {found_second}") + print(f" Need Runout Cycle: {need_runout}") + print(f" Found Select Second: {found_second}") print(f" Item count: {item_count}") print(f" Items:") @@ -828,10 +1109,10 @@ def invoke(self, arg, from_tty): orig = std_string_to_str(val["m_orig"]) print(f"Token @ {val.address}") - print(f" m_type: {type_str}") - print(f" m_literal: \"{literal}\"") - print(f" m_argc: {argc}") - print(f" m_orig: \"{orig}\"") + print(f" Type: {type_str}") + print(f" Literal: \"{literal}\"") + print(f" Arg Count: {argc}") + print(f" Original: \"{orig}\"") except Exception as e: print(f"Error: {e}") @@ -851,12 +1132,12 @@ def invoke(self, arg, from_tty): first = int(val["m_first"]) last = int(val["m_last"]) print(f"TokenFieldRangeSimple @ {val.address}") - print(f" m_first: {first}") - print(f" m_last: {last}") - print(f" bDone: {b_done}") + print(f" First: {first}") + print(f" Last: {last}") + print(f" Done: {b_done}") except: print(f"TokenFieldRange @ {val.address}") - print(f" bDone: {b_done}") + print(f" Done: {b_done}") except Exception as e: print(f"Error: {e}") @@ -870,53 +1151,279 @@ class DumpProcessingState(gdb.Command): def __init__(self): super(DumpProcessingState, self).__init__("dump-processing-state", gdb.COMMAND_DATA) + @staticmethod + def _fmt_record(shared_str): + """Format a PSpecString (shared_ptr) for display.""" + obj = deref_shared_ptr(shared_str) + if obj is None: + return "" + s = std_string_to_str(obj) + if len(s) > 60: + return f"\"{s[:60]}...\" (len={len(s)})" + return f"\"{s}\"" + def invoke(self, arg, from_tty): try: val = gdb.parse_and_eval(arg) + print(f"ProcessingState @ {val.address}") - # Current record - ps = deref_shared_ptr(val["m_ps"]) - if ps: - record_str = std_string_to_str(ps) - else: - record_str = "" + # --- Records --- + print(f" Current Record: {self._fmt_record(val['m_ps'])}") + print(f" Previous Record: {self._fmt_record(val['m_prevPs'])}") + print(f" Input Record: {self._fmt_record(val['m_inputRecord'])}") - # Previous record - prev_ps = deref_shared_ptr(val["m_prevPs"]) - if prev_ps: - prev_record_str = std_string_to_str(prev_ps) - else: - prev_record_str = "" + # --- Separators & Padding --- + try: + pad = chr(int(val["m_pad"])) + print(f" Pad Char: '{pad}' (0x{ord(pad):02x})") + except Exception as e: + print(f" Pad Char: (error: {e})") - pad = chr(int(val["m_pad"])) - word_sep = std_string_to_str(val["m_wordSeparator"]) - field_sep = std_string_to_str(val["m_fieldSeparator"]) - cycle = int(val["m_CycleCounter"]) - extra_reads = int(val["m_ExtraReads"]) - word_count = int(val["m_wordCount"]) - field_count = int(val["m_fieldCount"]) - input_station = int(val["m_inputStation"]) - input_stream = int(val["m_inputStream"]) - output_idx = int(val["m_outputIndex"]) - no_write = bool(val["m_bNoWrite"]) - eof = bool(val["m_bEOF"]) + try: + ws_local = bool(val["m_wordSeparatorLocal"]) + word_sep = std_string_to_str(val["m_wordSeparator"]) + local_tag = " (local)" if ws_local else "" + print(f" Word Separator: \"{word_sep}\"{local_tag}") + except Exception as e: + print(f" Word Separator: (error: {e})") - print(f"ProcessingState @ {val.address}") - print(f" Current Record: \"{record_str[:50]}{'...' if len(record_str) > 50 else ''}\"") - print(f" Previous Record: \"{prev_record_str[:50]}{'...' if len(prev_record_str) > 50 else ''}\"") - print(f" Pad Char: '{pad}' (0x{ord(pad):02x})") - print(f" Word Separator: \"{word_sep}\"") - print(f" Field Separator: \"{field_sep}\"") - print(f" Cycle Counter: {cycle}") - print(f" Extra Reads: {extra_reads}") - print(f" Record Count: {cycle + extra_reads}") - print(f" Word Count: {word_count}") - print(f" Field Count: {field_count}") - print(f" Input Station: {input_station}") - print(f" Input Stream: {input_stream}") - print(f" Output Index: {output_idx}") - print(f" No Write: {no_write}") - print(f" EOF: {eof}") + try: + field_sep = std_string_to_str(val["m_fieldSeparator"]) + print(f" Field Separator: \"{field_sep}\"") + except Exception as e: + print(f" Field Separator: (error: {e})") + + # --- Counters --- + try: + cycle = int(val["m_CycleCounter"]) + extra_reads = int(val["m_ExtraReads"]) + context_offset = int(val["m_contextOffset"]) + print(f" Cycle Counter: {cycle}") + print(f" Extra Reads: {extra_reads}") + print(f" Record Count: {cycle + extra_reads}") + print(f" Context Offset: {context_offset}") + except Exception as e: + print(f" Counters: (error: {e})") + + # --- Word / Field Caches --- + try: + word_count = int(val["m_wordCount"]) + field_count = int(val["m_fieldCount"]) + print(f" Word Count: {word_count}") + print(f" Field Count: {field_count}") + except Exception as e: + print(f" Word/Field Count: (error: {e})") + + try: + ws = std_vector_int_items(val["m_wordStart"]) + we = std_vector_int_items(val["m_wordEnd"]) + n = len(ws) + print(f" Word Positions ({n} cached):") + if n == 0: + print(f" (none)") + else: + limit = min(n, 20) + for i in range(limit): + end_val = we[i] if i < len(we) else "?" + print(f" [{i}] {ws[i]}-{end_val}") + if n > 20: + print(f" ... and {n - 20} more") + except Exception as e: + print(f" Word Positions: (error: {e})") + + try: + fs = std_vector_int_items(val["m_fieldStart"]) + fe = std_vector_int_items(val["m_fieldEnd"]) + n = len(fs) + print(f" Field Positions ({n} cached):") + if n == 0: + print(f" (none)") + else: + limit = min(n, 20) + for i in range(limit): + end_val = fe[i] if i < len(fe) else "?" + print(f" [{i}] {fs[i]}-{end_val}") + if n > 20: + print(f" ... and {n - 20} more") + except Exception as e: + print(f" Field Positions: (error: {e})") + + # --- Field Identifiers --- + try: + fi = val["m_fieldIdentifiers"] + fi_size = std_map_size(fi) + print(f" Field Identifiers ({fi_size} entries):") + if fi_size == 0: + print(f" (none)") + else: + items = std_map_items(fi) + for key, value in items: + try: + k = chr(int(key)) + s = deref_shared_ptr(value) + v = std_string_to_str(s) if s else "" + if len(v) > 40: + v = v[:40] + "..." + print(f" '{k}' = \"{v}\"") + except: + pass + except Exception as e: + print(f" Field Identifiers: (error: {e})") + + # --- FI Statistics --- + try: + fis = val["m_fiStatistics"] + fis_size = std_map_size(fis) + print(f" FI Statistics ({fis_size} entries):") + if fis_size == 0: + print(f" (none)") + else: + items = std_map_items(fis) + for key, value in items: + try: + k = chr(int(key)) + stats = deref_shared_ptr(value) + if stats: + total = int(stats["m_totalCount"]) + int_count = int(stats["m_intCount"]) + float_count = int(stats["m_floatCount"]) + print(f" '{k}': {total} values ({int_count} int, {float_count} float)") + else: + print(f" '{k}': ") + except: + pass + except Exception as e: + print(f" FI Statistics: (error: {e})") + + # --- Break Values --- + try: + bv = val["m_breakValues"] + bv_size = std_map_size(bv) + print(f" Break Values ({bv_size} entries):") + if bv_size == 0: + print(f" (none)") + else: + items = std_map_items(bv) + for key, value in items: + try: + k = chr(int(key)) + s = deref_shared_ptr(value) + v = std_string_to_str(s) if s else "" + if len(v) > 40: + v = v[:40] + "..." + print(f" '{k}' = \"{v}\"") + except: + pass + except Exception as e: + print(f" Break Values: (error: {e})") + + try: + bl = int(val["m_breakLevel"]) + if bl == 0: + print(f" Break Level: (none)") + else: + print(f" Break Level: '{chr(bl)}' (0x{bl:02x})") + except Exception as e: + print(f" Break Level: (error: {e})") + + # --- Frequency Maps --- + try: + fm = val["m_freqMaps"] + fm_size = std_map_size(fm) + print(f" Frequency Maps ({fm_size} entries):") + if fm_size == 0: + print(f" (none)") + else: + items = std_map_items(fm) + for key, value in items: + try: + k = chr(int(key)) + fmap = deref_shared_ptr(value) + if fmap: + nelem = int(fmap["map"]["_M_element_count"]) + counter = int(fmap["counter"]) + print(f" '{k}': {nelem} unique elements, {counter} total") + else: + print(f" '{k}': ") + except: + pass + except Exception as e: + print(f" Frequency Maps: (error: {e})") + + # --- Conditions Stack --- + try: + cond_items = std_stack_items(val["m_Conditions"]) + depth = len(cond_items) + print(f" Conditions ({depth} deep):") + if depth == 0: + print(f" (empty)") + else: + for i in range(depth - 1, -1, -1): + label = "top -> " if i == depth - 1 else " " + v = int(cond_items[i]) + name = EXTREME_BOOL.get(v, f"Unknown({v})") + print(f" {label}{name}") + except Exception as e: + print(f" Conditions: (error: {e})") + + # --- Loops Stack --- + try: + loop_items = std_stack_items(val["m_Loops"]) + depth = len(loop_items) + print(f" Loops ({depth} deep):") + if depth == 0: + print(f" (empty)") + else: + for i in range(depth - 1, -1, -1): + label = "top -> " if i == depth - 1 else " " + v = int(loop_items[i]) + print(f" {label}token #{v}") + except Exception as e: + print(f" Loops: (error: {e})") + + # --- I/O State --- + try: + input_station = int(val["m_inputStation"]) + station_name = INPUT_STATION.get(input_station, f"Stream({input_station})") + print(f" Input Station: {station_name}") + except Exception as e: + print(f" Input Station: (error: {e})") + + try: + input_stream = int(val["m_inputStream"]) + print(f" Input Stream: {input_stream}") + except Exception as e: + print(f" Input Stream: (error: {e})") + + try: + stream_changed = bool(val["m_inputStreamChanged"]) + print(f" Stream Changed: {stream_changed}") + except Exception as e: + print(f" Stream Changed: (error: {e})") + + try: + writers = val["m_Writers"] + print(f" Writers: {writers}") + except Exception as e: + print(f" Writers: (error: {e})") + + try: + output_idx = int(val["m_outputIndex"]) + print(f" Output Index: {output_idx}") + except Exception as e: + print(f" Output Index: (error: {e})") + + try: + no_write = bool(val["m_bNoWrite"]) + print(f" No Write: {no_write}") + except Exception as e: + print(f" No Write: (error: {e})") + + try: + eof = bool(val["m_bEOF"]) + print(f" EOF: {eof}") + except Exception as e: + print(f" EOF: (error: {e})") except Exception as e: print(f"Error: {e}") @@ -962,10 +1469,10 @@ def invoke(self, arg, from_tty): b_ran_dry = bool(val["m_bRanDry"]) print(f"Reader @ {val.address}") - print(f" m_countRead: {count_read}") - print(f" m_countUsed: {count_used}") - print(f" m_bAbort: {b_abort}") - print(f" m_bRanDry: {b_ran_dry}") + print(f" Count Read: {count_read}") + print(f" Count Used: {count_used}") + print(f" Abort: {b_abort}") + print(f" Ran Dry: {b_ran_dry}") except Exception as e: print(f"Error: {e}") @@ -983,9 +1490,9 @@ def invoke(self, arg, from_tty): ended = bool(val["m_ended"]) print(f"Writer @ {val.address}") - print(f" m_countGenerated: {count_gen}") - print(f" m_countWritten: {count_written}") - print(f" m_ended: {ended}") + print(f" Count Generated: {count_gen}") + print(f" Count Written: {count_written}") + print(f" Ended: {ended}") except Exception as e: print(f"Error: {e}") @@ -1008,9 +1515,9 @@ def invoke(self, arg, from_tty): exact = bool(val["m_exact"]) print(f"ALUValue @ {val.address}") - print(f" m_type: {type_str}") - print(f" m_value: \"{value_str}\"") - print(f" m_exact: {exact}") + print(f" Type: {type_str}") + print(f" Value: \"{value_str}\"") + print(f" Exact: {exact}") except Exception as e: print(f"Error: {e}") @@ -1023,11 +1530,28 @@ def __init__(self): def invoke(self, arg, from_tty): try: val = gdb.parse_and_eval(arg) - # m_map is a std::map + m_map = val["m_map"] + size = std_map_size(m_map) print(f"ALUCounters @ {val.address}") - print(f" Counters (map):") - # Simplified: just show the address - print(f" m_map @ {val['m_map'].address}") + print(f" Counters ({size} entries):") + if size == 0: + print(f" (empty)") + else: + items = std_map_items(m_map) + for key, value in items: + try: + k = int(key) + type_val = int(value["m_type"]) + type_str = ALU_COUNTER_TYPE.get(type_val, f"Unknown({type_val})").lower() + val_str = std_string_to_str(value["m_value"]) + exact = bool(value["m_exact"]) + exact_str = " (exact)" if exact else "" + if type_val == 0: # counterType__None + print(f" #{k}: (none)") + else: + print(f" #{k}: ({type_str}) {val_str}{exact_str}") + except Exception as item_e: + print(f" (error: {item_e})") except Exception as e: print(f"Error: {e}") @@ -1093,9 +1617,9 @@ def invoke(self, arg, from_tty): total_count = int(val["m_totalCount"]) print(f"AluValueStats @ {val.address}") - print(f" m_intCount: {int_count}") - print(f" m_floatCount: {float_count}") - print(f" m_totalCount: {total_count}") + print(f" Int Count: {int_count}") + print(f" Float Count: {float_count}") + print(f" Total Count: {total_count}") except Exception as e: print(f"Error: {e}") @@ -1160,9 +1684,9 @@ def invoke(self, arg, from_tty): relies_on_input = bool(val["m_reliesOnInput"]) print(f"AluFunction @ {val.address}") - print(f" m_FuncName: {func_name}") - print(f" m_ArgCount: {arg_count}") - print(f" m_reliesOnInput: {relies_on_input}") + print(f" Function Name: {func_name}") + print(f" Arg Count: {arg_count}") + print(f" Relies On Input: {relies_on_input}") except Exception as e: print(f"Error: {e}") @@ -1236,47 +1760,114 @@ def invoke(self, arg, from_tty): try: val = gdb.parse_and_eval(arg) + print(f"PythonFunctionCollection @ {val.address}") + # Access m_Initialized try: m_initialized = bool(val["m_Initialized"]) - print(f"PythonFunctionCollection @ {val.address}") - print(f" m_Initialized: {m_initialized}") + print(f" Initialized: {m_initialized}") + except Exception as e: + print(f" Initialized: (error: {e})") + + # Try to access m_Functions map and iterate + try: + m_functions = val["m_Functions"] + map_size = std_map_size(m_functions) + print(f" Functions ({map_size} entries):") - # Try to access m_Functions map (simplified) - try: - m_functions = val["m_Functions"] - print(f" m_Functions @ {m_functions.address}") - except: - pass - except: - print(f"PythonFunctionCollection @ {val.address}") + if map_size == 0: + print(f" (empty)") + else: + # Iterate the map using std_map_items + items = std_map_items(m_functions) + if items: + for key, value in items: + try: + # value is shared_ptr + func_rec = deref_shared_ptr(value) + if func_rec: + func_name = std_string_to_str(func_rec["m_name"]).ljust(20) + func_ptr = func_rec["m_pFuncPtr"] + arg_type_exact = bool(func_rec["m_argTypeExact"]) + arg_count = std_vector_size(func_rec["m_args"]) + + exact_str = ", exact arguments" if arg_type_exact else "" + print(f" {func_name} @ {func_ptr} ({arg_count} args{exact_str})") + except: + pass + else: + print(f" (iteration failed)") + except Exception as e: + print(f" Functions: (error: {e})") except Exception as e: print(f"Error: {e}") -class DumpPythonFuncRec(gdb.Command): - """Dump a PythonFuncRec (internal class from PythonIntf.cc).""" +class DumpPythonFuncByName(gdb.Command): + """Dump a PythonFuncRec by looking it up in a PythonFunctionCollection by name.""" def __init__(self): - super(DumpPythonFuncRec, self).__init__("dump-python-func-rec", gdb.COMMAND_DATA) + super(DumpPythonFuncByName, self).__init__("dump-python-func-by-name", gdb.COMMAND_DATA) def invoke(self, arg, from_tty): try: - val = gdb.parse_and_eval(arg) + # Parse arguments: collection_expr function_name + args = arg.split(None, 1) + if len(args) < 2: + print("Usage: dump-python-func-by-name ") + return + + collection_expr = args[0] + func_name_arg = args[1] + + # Remove quotes if present + if func_name_arg.startswith('"') and func_name_arg.endswith('"'): + func_name_arg = func_name_arg[1:-1] + elif func_name_arg.startswith("'") and func_name_arg.endswith("'"): + func_name_arg = func_name_arg[1:-1] + + # Evaluate the collection + collection = gdb.parse_and_eval(collection_expr) + # Access the m_Functions map and find the function by name + m_functions = collection["m_Functions"] + items = std_map_items(m_functions) + + found = False + if items: + for key, value in items: + try: + key_str = std_string_to_str(key) + if key_str == func_name_arg: + func_rec = deref_shared_ptr(value) + if func_rec: + self._dump_python_func_rec(func_rec) + found = True + break + except: + pass + + if not found: + print(f"Function '{func_name_arg}' not found in collection") + except Exception as e: + print(f"Error: {e}") + + def _dump_python_func_rec(self, val): + """Dump a PythonFuncRec (reuses logic from DumpPythonFuncRec).""" + try: print(f"PythonFuncRec @ {val.address}") # Access members try: m_name = std_string_to_str(val["m_name"]) - print(f" m_name: {m_name}") + print(f" Name: {m_name}") except Exception as e: - print(f" m_name: (error: {e})") + print(f" Name: (error: {e})") try: m_pFuncPtr = val["m_pFuncPtr"] - print(f" m_pFuncPtr: {m_pFuncPtr}") + print(f" Func Ptr: {m_pFuncPtr}") except Exception as e: - print(f" m_pFuncPtr: (error: {e})") + print(f" Func Ptr: (error: {e})") # Show m_doc try: @@ -1284,48 +1875,155 @@ def invoke(self, arg, from_tty): if m_doc: # Format multi-line docs nicely if "\n" in m_doc: - print(f" m_doc:") + print(f" doc:") for line in m_doc.split("\n"): print(f" {line}") else: - print(f" m_doc: {m_doc}") + print(f" doc: {m_doc}") else: - print(f" m_doc: (empty)") + print(f" doc: (empty)") except Exception as e: - print(f" m_doc: (error: {e})") + print(f" doc: (error: {e})") # Show m_argTypeExact try: m_argTypeExact = bool(val["m_argTypeExact"]) - print(f" m_argTypeExact: {m_argTypeExact}") + print(" Arg Type: {}".format("exact" if m_argTypeExact else "no exactness information")) except Exception as e: - print(f" m_argTypeExact: (error: {e})") + print(f" Arg Type: (error: {e})") - # Show m_pTuple + # Expand m_args vector + argDict = dict() + try: + m_args = val["m_args"] + arg_size = std_vector_size(m_args) + print(f" Args ({arg_size} items):") + + # Try to iterate and dump each argument + args_start = m_args["_M_impl"]["_M_start"] + for i in range(arg_size): + try: + arg_elem = args_start[i] + arg_name = std_string_to_str(arg_elem["m_name"]) + arg_default = int(arg_elem["m_default"]) + arg_default_str = ALU_COUNTER_TYPE.get(arg_default, f"Unknown({arg_default})") + + print(f" [{i}] {arg_name} (default: {arg_default_str})") + argDict[i] = arg_name + + # Show default value if present + if arg_default == 1: # counterType__Str + try: + defStr = std_string_to_str(arg_elem["m_defStr"]) + print(f" = \"{defStr}\"") + except: + pass + elif arg_default == 2: # counterType__Int + try: + defInt = int(arg_elem["m_defInt"]) + print(f" = {defInt}") + except: + pass + elif arg_default == 3: # counterType__Float + try: + defFloat = float(arg_elem["m_defFloat"]) + print(f" = {defFloat}") + except: + pass + except Exception as arg_e: + print(f" [{i}] (error: {arg_e})") + except Exception as e: + print(f" m_args: (error: {e})") + + # Show m_pTuple (argument values for the current Python function call) try: m_pTuple = val["m_pTuple"] - if m_pTuple == 0: - print(f" m_pTuple: nullptr") + dump_pytuple(m_pTuple, argDict) + except Exception as e: + print(f" Tuple: (error: {e})") + + + except Exception as e: + print(f"Error: {e}") + +class DumpPythonFuncRec(gdb.Command): + """Dump a PythonFuncRec (internal class from PythonIntf.cc).""" + + def __init__(self): + super(DumpPythonFuncRec, self).__init__("dump-python-func-rec", gdb.COMMAND_DATA) + + def invoke(self, arg, from_tty): + try: + val = gdb.parse_and_eval(arg) + + # Auto-cast: if val is a shared_ptr, extract + # _M_ptr and cast to PythonFuncRec* (the derived type). + val_type = val.type.strip_typedefs() + type_name = str(val_type) + if "shared_ptr" in type_name and "ExternalFunctionRec" in type_name: + ptr = val["_M_ptr"] + if ptr == 0: + print("PythonFuncRec: nullptr") + return + python_func_type = gdb.lookup_type("PythonFuncRec").pointer() + val = ptr.cast(python_func_type).dereference() + + print(f"PythonFuncRec @ {val.address}") + + # Access members + try: + m_name = std_string_to_str(val["m_name"]) + print(f" Name: {m_name}") + except Exception as e: + print(f" Name: (error: {e})") + + try: + m_pFuncPtr = val["m_pFuncPtr"] + print(f" Func Ptr: {m_pFuncPtr}") + except Exception as e: + print(f" Func Ptr: (error: {e})") + + # Show m_doc + try: + m_doc = std_string_to_str(val["m_doc"]) + if m_doc: + # Format multi-line docs nicely + if "\n" in m_doc: + print(f" doc:") + for line in m_doc.split("\n"): + print(f" {line}") + else: + print(f" doc: {m_doc}") else: - print(f" m_pTuple: {m_pTuple}") + print(f" doc: (empty)") + except Exception as e: + print(f" doc: (error: {e})") + + # Show m_argTypeExact + try: + m_argTypeExact = bool(val["m_argTypeExact"]) + print(" Arg Type: {}".format("exact" if m_argTypeExact else "no exactness information")) except Exception as e: - print(f" m_pTuple: (error: {e})") + print(f" Arg Type: (error: {e})") # Expand m_args vector + argDict = dict() try: m_args = val["m_args"] arg_size = std_vector_size(m_args) - print(f" m_args ({arg_size} items):") + print(f" Args ({arg_size} items):") # Try to iterate and dump each argument + args_start = m_args["_M_impl"]["_M_start"] for i in range(arg_size): try: - arg_elem = m_args[i] + arg_elem = args_start[i] arg_name = std_string_to_str(arg_elem["m_name"]) arg_default = int(arg_elem["m_default"]) arg_default_str = ALU_COUNTER_TYPE.get(arg_default, f"Unknown({arg_default})") print(f" [{i}] {arg_name} (default: {arg_default_str})") + argDict[i] = arg_name # Show default value if present if arg_default == 1: # counterType__Str @@ -1349,7 +2047,15 @@ def invoke(self, arg, from_tty): except Exception as arg_e: print(f" [{i}] (error: {arg_e})") except Exception as e: - print(f" m_args: (error: {e})") + print(f" Args: (error: {e})") + + # Show m_pTuple (argument values for the current Python function call) + try: + m_pTuple = val["m_pTuple"] + dump_pytuple(m_pTuple, argDict) + except Exception as e: + print(f" Tuple: (error: {e})") + except Exception as e: print(f"Error: {e}") @@ -1370,26 +2076,26 @@ def invoke(self, arg, from_tty): m_default_str = ALU_COUNTER_TYPE.get(m_default, f"Unknown({m_default})") print(f"PythonFuncArg @ {val.address}") - print(f" m_name: {m_name}") - print(f" m_default: {m_default_str}") + print(f" Name: {m_name}") + print(f" Default Type: {m_default_str}") # Try to get default value if m_default == 1: # counterType__Str try: m_defStr = std_string_to_str(val["m_defStr"]) - print(f" m_defStr: \"{m_defStr}\"") + print(f" Default String: \"{m_defStr}\"") except: pass elif m_default == 2: # counterType__Int try: m_defInt = int(val["m_defInt"]) - print(f" m_defInt: {m_defInt}") + print(f" Default Int: {m_defInt}") except: pass elif m_default == 3: # counterType__Float try: m_defFloat = float(val["m_defFloat"]) - print(f" m_defFloat: {m_defFloat}") + print(f" Default Float: {m_defFloat}") except: pass except Exception as inner_e: @@ -1437,6 +2143,7 @@ def register_commands(): DumpSkipItem() DumpConditionItem() DumpBreakItem() + DumpContextItem() DumpSelectItem() DumpSplitItem() @@ -1466,6 +2173,7 @@ def register_commands(): DumpExternalFunctionRec() DumpExternalFunctionCollection() DumpPythonFunctionCollection() + DumpPythonFuncByName() DumpPythonFuncRec() DumpPythonFuncArg() diff --git a/specs/src/processing/ProcessingState.cc b/specs/src/processing/ProcessingState.cc index 8940400..61c4e5d 100644 --- a/specs/src/processing/ProcessingState.cc +++ b/specs/src/processing/ProcessingState.cc @@ -49,6 +49,7 @@ void ProcessingState::Reset() m_wordCount = -1; m_CycleCounter = 0; m_ExtraReads = 0; + m_contextOffset = 0; m_inputStation = STATION_FIRST; m_breakLevel = 0; } @@ -57,6 +58,7 @@ ProcessingState::ProcessingState() Reset(); m_ps = nullptr; m_prevPs = nullptr; + m_inputRecord = nullptr; m_inputStream = DEFAULT_READER_IDX; m_inputStreamChanged = false; m_bNoWrite = false; @@ -75,8 +77,10 @@ ProcessingState::ProcessingState(ProcessingState& ps) m_wordCount = 0; m_CycleCounter = 0; m_ExtraReads = 0; + m_contextOffset = 0; m_ps = nullptr; m_prevPs = nullptr; + m_inputRecord = nullptr; m_inputStation = STATION_FIRST; m_breakLevel = 0; m_inputStream = DEFAULT_READER_IDX; @@ -97,8 +101,10 @@ ProcessingState::ProcessingState(ProcessingState* pPS) m_wordCount = 0; m_CycleCounter = 0; m_ExtraReads = 0; + m_contextOffset = 0; m_ps = nullptr; m_prevPs = nullptr; + m_inputRecord = nullptr; m_inputStation = STATION_FIRST; m_breakLevel = 0; m_inputStream = DEFAULT_READER_IDX; @@ -118,15 +124,16 @@ ProcessingState::~ProcessingState() void ProcessingState::setString(PSpecString ps, bool bResetState) { - if (m_ps && ps!=m_ps) { - m_prevPs = m_ps; + if (m_inputRecord) { + m_prevPs = m_inputRecord; } else { - MYASSERT(m_prevPs==nullptr); m_prevPs = std::make_shared(); } + m_inputRecord = ps; m_ps = ps; m_wordCount = -1; m_fieldCount = -1; + m_contextOffset = 0; if (bResetState) { fieldIdentifierClear(); resetBreaks(); @@ -138,6 +145,14 @@ void ProcessingState::setStringInPlace(PSpecString ps) m_ps = ps; } +void ProcessingState::setContextString(PSpecString ps, int offset) +{ + m_ps = ps; + m_wordCount = -1; + m_fieldCount = -1; + m_contextOffset = offset; +} + void ProcessingState::setFirst() { if (m_inputStation != STATION_FIRST) { diff --git a/specs/src/processing/ProcessingState.h b/specs/src/processing/ProcessingState.h index 2940c83..b25b48b 100644 --- a/specs/src/processing/ProcessingState.h +++ b/specs/src/processing/ProcessingState.h @@ -52,6 +52,7 @@ class ProcessingState : public stateQueryAgent { bool isRunIn() override { return (m_CycleCounter==1); } bool isRunOut() override { return (m_ps==nullptr); } // NOTE: will return true before first record ALUInt getRecordCount() override { return ALUInt(m_CycleCounter + m_ExtraReads); } + ALUInt getContextOffset() override { return ALUInt(m_contextOffset); } ALUInt getIterationCount() override { return ALUInt(m_CycleCounter); } bool breakEstablished(char id) override; PAluValueStats valueStatistics(char id) override; @@ -87,8 +88,10 @@ class ProcessingState : public stateQueryAgent { void setFirst(); void setSecond(); void setStream(int i); + void setContextString(PSpecString ps, int offset = 0); int getActiveInputStation() { return m_inputStation; } PSpecString currRecord() override { return (m_inputStation==STATION_FIRST) ? m_ps : m_prevPs; } + PSpecString inputRecord() override { return m_inputRecord; } bool recordNotAvailable() { return nullptr==currRecord(); } bool inputStreamHasChanged() { return m_inputStreamChanged; } void resetInputStreamFlag() { m_inputStreamChanged = false; } @@ -113,10 +116,12 @@ class ProcessingState : public stateQueryAgent { std::string m_fieldSeparator; PSpecString m_ps; // The current record PSpecString m_prevPs; // The previous record + PSpecString m_inputRecord; // The real input record (unaffected by CONTEXT) int m_wordCount; int m_fieldCount; unsigned int m_CycleCounter; unsigned int m_ExtraReads; + int m_contextOffset; std::vector m_wordStart; std::vector m_wordEnd; std::vector m_fieldStart; diff --git a/specs/src/processing/Reader.cc b/specs/src/processing/Reader.cc index 161e80c..4688165 100644 --- a/specs/src/processing/Reader.cc +++ b/specs/src/processing/Reader.cc @@ -5,6 +5,7 @@ #include "Reader.h" uint64_t g_readRecordCounter = 0; +Reader* g_pReader = nullptr; void ReadAllRecordsIntoReaderQueue(Reader* r) { @@ -108,6 +109,12 @@ void Reader::Begin() { mp_thread = std::unique_ptr(new std::thread(ReadAllRecordsIntoReaderQueue, this)); } +PSpecString Reader::peek(int offset) +{ + MYTHROW("Rolling context is not supported for this reader type"); + return nullptr; +} + StandardReader::StandardReader() { m_NeedToClose = false; @@ -115,6 +122,10 @@ StandardReader::StandardReader() { m_buffer = nullptr; m_recfm = RECFM_DELIMITED; m_lineDelimiter = 0; + m_forwardContextSize = 0; + m_backwardContextSize = 0; + m_currentRecord = nullptr; + m_contextInitialized = false; } StandardReader::StandardReader(std::istream* f) { @@ -128,6 +139,10 @@ StandardReader::StandardReader(std::istream* f) { m_buffer = nullptr; m_recfm = RECFM_DELIMITED; m_lineDelimiter = 0; + m_forwardContextSize = 0; + m_backwardContextSize = 0; + m_currentRecord = nullptr; + m_contextInitialized = false; } StandardReader::StandardReader(std::string& fn) { @@ -142,6 +157,10 @@ StandardReader::StandardReader(std::string& fn) { m_buffer = nullptr; m_recfm = RECFM_DELIMITED; m_lineDelimiter = 0; + m_forwardContextSize = 0; + m_backwardContextSize = 0; + m_currentRecord = nullptr; + m_contextInitialized = false; } StandardReader::StandardReader(pipeType pipe) { @@ -151,6 +170,10 @@ StandardReader::StandardReader(pipeType pipe) { m_buffer = nullptr; m_recfm = RECFM_DELIMITED; m_lineDelimiter = 0; + m_forwardContextSize = 0; + m_backwardContextSize = 0; + m_currentRecord = nullptr; + m_contextInitialized = false; } StandardReader::~StandardReader() { @@ -181,11 +204,68 @@ void StandardReader::setLineDelimiter(char c) m_lineDelimiter = c; } +void StandardReader::setContextSizes(unsigned int forward, unsigned int backward) +{ + m_forwardContextSize = forward; + m_backwardContextSize = backward; +} + +PSpecString StandardReader::peek(int offset) +{ + if (offset == 0) { + return m_currentRecord ? m_currentRecord : std::make_shared(); + } + if (offset < 0) { + unsigned int idx = (unsigned int)(-offset) - 1; + if (idx >= m_backwardBuffer.size()) return std::make_shared(); + return m_backwardBuffer[m_backwardBuffer.size() - 1 - idx]; + } + // offset > 0 + unsigned int idx = (unsigned int)offset - 1; + if (idx >= m_forwardBuffer.size()) return std::make_shared(); + return m_forwardBuffer[idx]; +} + bool StandardReader::endOfSource() { - return m_bAbort || m_EOF; + if (m_bAbort) return true; + if (m_contextInitialized && !m_forwardBuffer.empty()) return false; + return m_EOF; } PSpecString StandardReader::getNextRecord() { + if (m_forwardContextSize == 0 && m_backwardContextSize == 0) + return getNextRecordInternal(); + + if (!m_contextInitialized) { + m_currentRecord = getNextRecordInternal(); + if (!m_currentRecord) return nullptr; + for (unsigned int i = 0; i < m_forwardContextSize; i++) { + PSpecString rec = getNextRecordInternal(); + if (!rec) break; + m_forwardBuffer.push_back(rec); + } + m_contextInitialized = true; + return m_currentRecord; + } + + // Shift window forward + m_backwardBuffer.push_back(m_currentRecord); + if (m_backwardBuffer.size() > m_backwardContextSize) + m_backwardBuffer.pop_front(); + + if (!m_forwardBuffer.empty()) { + m_currentRecord = m_forwardBuffer.front(); + m_forwardBuffer.pop_front(); + PSpecString rec = getNextRecordInternal(); + if (rec) m_forwardBuffer.push_back(rec); + } else { + m_currentRecord = getNextRecordInternal(); + } + + return m_currentRecord; +} + +PSpecString StandardReader::getNextRecordInternal() { std::string line; bool ok; switch (m_recfm) { @@ -310,6 +390,16 @@ void TestReader::InsertString(PSpecString ps) mp_arr[m_count++] = ps; } +PSpecString TestReader::peek(int offset) +{ + // m_idx points to the *next* record to read, so current record is m_idx-1 + int target = int(m_idx) - 1 + offset; + if (target < 0 || target >= int(m_count)) { + return std::make_shared(); // empty string for out-of-bounds + } + return mp_arr[target]; +} + // #include // for memset // #include "utils/ErrorReporting.h" diff --git a/specs/src/processing/Reader.h b/specs/src/processing/Reader.h index a718f7d..e398b12 100644 --- a/specs/src/processing/Reader.h +++ b/specs/src/processing/Reader.h @@ -1,6 +1,7 @@ #ifndef SPECS2016__PROCESSING__READER__H #define SPECS2016__PROCESSING__READER__H +#include #include #include #include "utils/StringQueue.h" @@ -39,6 +40,8 @@ class Reader { virtual void setLineDelimiter(char c) { MYTHROW("Reader::setLineDelimiter: should not be called"); } + virtual PSpecString peek(int offset); + virtual void setContextSizes(unsigned int forward, unsigned int backward) {} protected: StringQueue m_queue; std::unique_ptr mp_thread; @@ -61,6 +64,7 @@ class TestReader : public Reader { bool endOfSource() override {return m_bAbort || (m_idx >= m_count); } PSpecString getNextRecord() override {return mp_arr[m_idx++];} PSpecString get(classifyingTimer& tmr, unsigned int& _readerCounter) override {return getNextRecord();} + PSpecString peek(int offset) override; private: PSpecString *mp_arr; size_t m_count; @@ -86,7 +90,10 @@ class StandardReader : public Reader { PSpecString getNextRecord() override; void setFormatFixed(unsigned int lrecl, bool blocked) override; void setLineDelimiter(char c) override; + PSpecString peek(int offset) override; + void setContextSizes(unsigned int forward, unsigned int backward) override; private: + PSpecString getNextRecordInternal(); std::shared_ptr m_File; pipeType m_pipe; char* m_buffer; @@ -95,6 +102,13 @@ class StandardReader : public Reader { recordFormat m_recfm; unsigned int m_lrecl; char m_lineDelimiter; + // Rolling context buffers + unsigned int m_forwardContextSize; + unsigned int m_backwardContextSize; + std::deque m_forwardBuffer; + std::deque m_backwardBuffer; + PSpecString m_currentRecord; + bool m_contextInitialized; }; typedef std::shared_ptr PStandardReader; @@ -129,5 +143,6 @@ class multiReader : public Reader { typedef std::shared_ptr PMultiReader; +extern Reader* g_pReader; #endif diff --git a/specs/src/specitems/item.h b/specs/src/specitems/item.h index fcb77c7..e90282e 100644 --- a/specs/src/specitems/item.h +++ b/specs/src/specitems/item.h @@ -390,4 +390,17 @@ class SplitItem : public Item { typedef std::shared_ptr PSplitItem; +class ContextItem : public Item { +public: + explicit ContextItem(int offset); + ~ContextItem() override {} + std::string Debug() override; + ApplyRet apply(ProcessingState& pState, StringBuilder* pSB) override; + bool readsLines() override { return true; } +private: + int m_offset; +}; + +typedef std::shared_ptr PContextItem; + #endif diff --git a/specs/src/specitems/specItems.cc b/specs/src/specitems/specItems.cc index 9ad0ac0..ae74bd8 100644 --- a/specs/src/specitems/specItems.cc +++ b/specs/src/specitems/specItems.cc @@ -12,6 +12,9 @@ bool g_keep_suppressed_record = false; extern uint64_t g_readRecordCounter; unsigned int g_WhileGuardLimit = 5000; +unsigned int g_forwardContext = 0; +unsigned int g_backwardContext = 0; + struct predicateStackItem { PConditionItem pred; unsigned int argIndex; @@ -467,6 +470,23 @@ void itemGroup::Compile(std::vector &tokenVec, unsigned int& index) addItem(pItem); break; } + case TokenListType__CONTEXT: + { + if (tokenVec[index].Literal().empty()) { + std::string err = "CONTEXT at index " + std::to_string(tokenVec[index].argIndex()) + + " must be followed by an integer offset"; + MYTHROW(err); + } + int offset = std::stoi(tokenVec[index].Literal()); + auto pItem = std::make_shared(offset); + addItem(pItem); + if (offset > 0 && (unsigned int)offset > g_forwardContext) + g_forwardContext = (unsigned int)offset; + if (offset < 0 && (unsigned int)(-offset) > g_backwardContext) + g_backwardContext = (unsigned int)(-offset); + index++; + break; + } case TokenListType__REQUIRES: { if (!configSpecLiteralExists(tokenVec[index].Literal())) { @@ -1264,6 +1284,24 @@ ApplyRet SelectItem::apply(ProcessingState& pState, StringBuilder* pSB) return ApplyRet__Continue; } +ContextItem::ContextItem(int offset) : m_offset(offset) {} + +std::string ContextItem::Debug() +{ + std::string ret = "CONTEXT "; + if (m_offset >= 0) ret += "+"; + ret += std::to_string(m_offset); + return ret; +} + +ApplyRet ContextItem::apply(ProcessingState& pState, StringBuilder* pSB) +{ + MYASSERT_WITH_MSG(g_pReader != nullptr, "Rolling context requires a reader"); + PSpecString ps = g_pReader->peek(m_offset); + pState.setContextString(ps, m_offset); + return ApplyRet__Continue; +} + #ifdef ndef std::ostream& operator<< (std::ostream& os, const SpecString &str) { diff --git a/specs/src/specitems/specItems.h b/specs/src/specitems/specItems.h index 683bb4b..89b016f 100644 --- a/specs/src/specitems/specItems.h +++ b/specs/src/specitems/specItems.h @@ -11,6 +11,9 @@ #define MAX_DEPTH_CONDITION_STATEMENTS 64 +extern unsigned int g_forwardContext; +extern unsigned int g_backwardContext; + class itemGroup { public: itemGroup(); diff --git a/specs/src/test/ProcessingTest.cc b/specs/src/test/ProcessingTest.cc index 57ac54e..cde37e8 100644 --- a/specs/src/test/ProcessingTest.cc +++ b/specs/src/test/ProcessingTest.cc @@ -8,6 +8,7 @@ #include "processing/Config.h" #include "processing/ProcessingState.h" #include "processing/StringBuilder.h" +#include "processing/Reader.h" extern ALUCounters g_counters; extern char g_printonly_rule; @@ -109,8 +110,11 @@ PSpecString runTestOnExample(const char* _specList, const char* _example) g_counters.clearAll(); g_keep_suppressed_record = false; g_printonly_rule = PRINTONLY_PRINTALL; + g_forwardContext = 0; + g_backwardContext = 0; TestReader tRead(100); + g_pReader = &tRead; unsigned int readerCounter = 1; char* example = strdup(_example); char* example_ctx = example; @@ -206,6 +210,7 @@ PSpecString runTestOnExample(const char* _specList, const char* _example) } end: + g_pReader = nullptr; free(example); while (!vec.empty()) { vec[0].deallocDynamic(); @@ -869,6 +874,86 @@ int main(int argc, char** argv) " PRINT 'exact(max(a))' NW"; VERIFY2(spec, "1.5\n2.5\n3.5", "0 0 0"); // TEST #228 + // === Rolling Context tests === + + // CONTEXT 0 resets to current record + spec = "CONTEXT 0 1-* 1"; + VERIFY2(spec, "alpha\nbeta\ngamma", "alpha\nbeta\ngamma"); // TEST #229 + + // CONTEXT +1 peeks at next record + spec = "1-* 1 CONTEXT 1 1-* NW"; + VERIFY2(spec, "alpha\nbeta\ngamma", "alpha beta\nbeta gamma\ngamma"); // TEST #230 + + // CONTEXT -1 peeks at previous record + spec = "1-* 1 CONTEXT -1 1-* NW"; + VERIFY2(spec, "alpha\nbeta\ngamma", "alpha\nbeta alpha\ngamma beta"); // TEST #231 + + // @+1 in expression peeks at next record + spec = "print @+1 1"; + VERIFY2(spec, "alpha\nbeta\ngamma", "beta\ngamma\n"); // TEST #232 + + // @-1 in expression peeks at previous record + spec = "print @-1 1"; + VERIFY2(spec, "alpha\nbeta\ngamma", "\nalpha\nbeta"); // TEST #233 + + // @+0 is the same as @@ + spec = "print @+0 1"; + VERIFY2(spec, "alpha\nbeta\ngamma", "alpha\nbeta\ngamma"); // TEST #234 + + // @-0 is the same as @@ + spec = "print @-0 1"; + VERIFY2(spec, "alpha\nbeta\ngamma", "alpha\nbeta\ngamma"); // TEST #235 + + // CONTEXT with larger forward offset + spec = "1-* 1 CONTEXT 2 1-* NW"; + VERIFY2(spec, "A\nB\nC\nD\nE", "A C\nB D\nC E\nD\nE"); // TEST #236 + + // CONTEXT with larger backward offset + spec = "1-* 1 CONTEXT -2 1-* NW"; + VERIFY2(spec, "A\nB\nC\nD\nE", "A\nB\nC A\nD B\nE C"); // TEST #237 + + // Combined forward and backward in one spec + spec = "CONTEXT -1 1-* 1 CONTEXT 0 1-* NW CONTEXT 1 1-* NW"; + VERIFY2(spec, "A\nB\nC", "A B\nA B C\nB C"); // TEST #238 + + // @+n in expression with function + spec = "PRINT 'length(@+1)' 1"; + VERIFY2(spec, "AB\nCDE\nF", "3\n1\n0"); // TEST #239 + + // Out-of-range forward context returns empty string + spec = "print @+5 1"; + VERIFY2(spec, "only", ""); // TEST #240 + + // Out-of-range backward context returns empty string + spec = "print @-5 1"; + VERIFY2(spec, "only", ""); // TEST #241 + + // @@ returns the real input record, not the CONTEXT-modified one + spec = "CONTEXT -1 PRINT '@@' 1 WRITE PRINT '@-1' 1 WRITE"; + VERIFY2(spec, "alpha\nbeta\ngamma", "alpha\n\nbeta\nalpha\ngamma\nbeta"); // TEST #242 + + // === ctxrecno tests === + + // ctxrecno without CONTEXT returns same as recno + spec = "PRINT 'ctxrecno()' 1"; + VERIFY2(spec, "a\nb\nc", "1\n2\n3"); // TEST #243 + + // ctxrecno with CONTEXT 1 returns recno + 1 + spec = "CONTEXT 1 PRINT 'ctxrecno()' 1"; + VERIFY2(spec, "a\nb\nc", "2\n3\n4"); // TEST #244 + + // ctxrecno with CONTEXT -1 returns recno - 1 + spec = "CONTEXT -1 PRINT 'ctxrecno()' 1"; + VERIFY2(spec, "a\nb\nc", "0\n1\n2"); // TEST #245 + + // ctxrecno with CONTEXT 0 returns same as recno + spec = "CONTEXT 0 PRINT 'ctxrecno()' 1"; + VERIFY2(spec, "a\nb\nc", "1\n2\n3"); // TEST #246 + + // ctxrecno resets after CONTEXT changes + spec = "PRINT 'ctxrecno()' 1 CONTEXT 1 PRINT 'ctxrecno()' NW"; + VERIFY2(spec, "a\nb\nc", "1 2\n2 3\n3 4"); // TEST #247 + if (errorCount) { std::cout << '\n' << errorCount << '/' << testCount << " tests failed.\n"; std::cout << "Failed tests: "; diff --git a/specs/src/test/specs.cc b/specs/src/test/specs.cc index b05d6e0..81fcfe4 100644 --- a/specs/src/test/specs.cc +++ b/specs/src/test/specs.cc @@ -270,6 +270,21 @@ int main (int argc, char** argv) exit (0); } + // Check for rolling context incompatibilities + if (g_forwardContext > 0 || g_backwardContext > 0) { + if (g_bThreaded) { + std::cerr << "Error: Rolling context (CONTEXT / @+n / @-n) is not supported with threading.\n"; + exit(0); + } + if (anyNonPrimaryInputStreamDefined()) { + std::cerr << "Error: Rolling context (CONTEXT / @+n / @-n) is not supported with multiple input streams.\n"; + exit(0); + } + if (g_bVerbose) { + std::cerr << "specs: Using a " << g_forwardContext + g_backwardContext + 1 << "-record rolling context: " << g_forwardContext << " records forward and " << g_backwardContext << " records backward.\n"; + } + } + // After the compilation, the token vector contents are no longer necessary for (size_t i=0; i 0 || g_backwardContext > 0) { + pRd->setContextSizes(g_forwardContext, g_backwardContext); + } + g_pReader = pRd.get(); + pRd->Begin(); timer.changeClass(timeClassProcessing); @@ -395,6 +415,7 @@ int main (int argc, char** argv) return -4; } + g_pReader = nullptr; pRd->End(); readLines = pRd->countRead(); usedLines = pRd->countUsed(); diff --git a/specs/src/utils/alu.cc b/specs/src/utils/alu.cc index 6f4325a..55b5596 100644 --- a/specs/src/utils/alu.cc +++ b/specs/src/utils/alu.cc @@ -16,8 +16,11 @@ #include "alu.h" #include "aluFunctions.h" #include "processing/Config.h" // for configured literals +#include "processing/Reader.h" // for g_pReader extern stateQueryAgent* g_pStateQueryAgent; +extern unsigned int g_forwardContext; +extern unsigned int g_backwardContext; void ALUValue::set(std::string& s) { @@ -861,12 +864,31 @@ PValue AluAssnOperator::computeAppnd(PValue operand, PValue prevOp) void AluInputRecord::_serialize(std::ostream& os) const { - os << "@@"; + if (m_offset == 0) { + os << "@@"; + } else if (m_offset > 0) { + os << "@+" << m_offset; + } else { + os << "@" << m_offset; + } +} + +std::string AluInputRecord::_identify() +{ + if (m_offset == 0) return "@@"; + if (m_offset > 0) return "@+" + std::to_string(m_offset); + return "@" + std::to_string(m_offset); } PValue AluInputRecord::evaluate() { - PSpecString ps = g_pStateQueryAgent->getFromTo(1,-1); + PSpecString ps; + if (m_offset == 0) { + ps = g_pStateQueryAgent->inputRecord(); + } else { + MYASSERT_WITH_MSG(g_pReader != nullptr, "Rolling context requires a reader"); + ps = g_pReader->peek(m_offset); + } PValue ret; if (ps) { ret = mkValue2(ps->data(), int(ps->length())); @@ -1233,6 +1255,28 @@ bool parseAluExpression(std::string& s, AluVec& vec) continue; } + // Rolling context: @+n or @-n + if (*c=='@' && (c[1]=='+' || c[1]=='-') && isDigit(c[2])) { + char sign = c[1]; + char* tokEnd = c + 2; + while (tokEnd(offset); + vec.push_back(pUnit); + prevUnitType = pUnit->type(); + c = tokEnd; + mayBeStart = false; + // Update rolling context size globals + if (offset > 0 && (unsigned int)offset > g_forwardContext) { + g_forwardContext = (unsigned int)offset; + } else if (offset < 0 && (unsigned int)(-offset) > g_backwardContext) { + g_backwardContext = (unsigned int)(-offset); + } + continue; + } + // Also a configured string if (*c=='@' && isFirstCharInIdentifier(c[1])) { char* tokEnd = ++c; diff --git a/specs/src/utils/alu.h b/specs/src/utils/alu.h index 0c21e25..3a180e2 100644 --- a/specs/src/utils/alu.h +++ b/specs/src/utils/alu.h @@ -285,13 +285,17 @@ class AluFunction : public AluUnit { class AluInputRecord : public AluUnit { public: - AluInputRecord() {} + AluInputRecord() : m_offset(0) {} + explicit AluInputRecord(int offset) : m_offset(offset) {} ~AluInputRecord() override {} void _serialize(std::ostream& os) const override; - std::string _identify() override {return "@@";} + std::string _identify() override; AluUnitType type() override {return UT_InputRecord;} PValue evaluate() override; bool requiresRead() override {return true;} + int offset() const {return m_offset;} +private: + int m_offset; }; class AluOtherToken : public AluUnit { diff --git a/specs/src/utils/aluFunctions.cc b/specs/src/utils/aluFunctions.cc index e649754..59c5fc2 100644 --- a/specs/src/utils/aluFunctions.cc +++ b/specs/src/utils/aluFunctions.cc @@ -414,6 +414,11 @@ PValue AluFunc_recno() return mkValue(g_pStateQueryAgent->getRecordCount()); } +PValue AluFunc_ctxrecno() +{ + return mkValue(g_pStateQueryAgent->getRecordCount() + g_pStateQueryAgent->getContextOffset()); +} + PValue AluFunc_eof() { bool isRunOut = g_pStateQueryAgent->isRunOut(); diff --git a/specs/src/utils/aluFunctions.h b/specs/src/utils/aluFunctions.h index ba4ab00..4021fc2 100644 --- a/specs/src/utils/aluFunctions.h +++ b/specs/src/utils/aluFunctions.h @@ -40,6 +40,8 @@ "() - Returns TRUE (1) if this is the first line.","") \ X(recno, 0, ALUFUNC_REGULAR, true, \ "() - Returns the record number of the current record.","Increments with every READ or READSTOP.") \ + X(ctxrecno, 0, ALUFUNC_REGULAR, true, \ + "() - Returns the record number of the record that input parts work on.","This is similar to recno, but considers rolling context, which recno does not.") \ X(number, 0, ALUFUNC_REGULAR, true, \ "() - Returns the number of times this specification has restarted","Does not increment with READ or READSTOP. Otherwise similar to recno().") \ X(eof, 0, ALUFUNC_REGULAR, false, \ @@ -427,9 +429,11 @@ class stateQueryAgent { return getFromTo(int(from), int(to)); } virtual PSpecString currRecord() = 0; + virtual PSpecString inputRecord() = 0; virtual bool isRunIn() = 0; virtual bool isRunOut() = 0; virtual ALUInt getRecordCount() = 0; + virtual ALUInt getContextOffset() = 0; virtual ALUInt getIterationCount() = 0; virtual bool breakEstablished(char id) = 0; virtual PAluValueStats valueStatistics(char id) = 0; diff --git a/specs/tests/pytest.py b/specs/tests/pytest.py index c313673..39b02a2 100644 --- a/specs/tests/pytest.py +++ b/specs/tests/pytest.py @@ -83,7 +83,7 @@ def set_localfuncs(lf): # But if we force it... sys.stdout.write("Test 04 (bad file; non-python function; force) -- ") ret = run_cmd('print "sqrt(81)" 1', True) -if ret=="Python Interface: Error loading local functions" or ret=="SyntaxError: invalid syntax": +if ret=="Python Interface: Error loading local functions" or ret.startswith("SyntaxError: invalid syntax"): sys.stdout.write("OK\n") else: sys.stdout.write("Not OK: <"+ret+">\n") @@ -92,7 +92,7 @@ def set_localfuncs(lf): # Or call a non-built-in function... sys.stdout.write("Test 05 (bad file; unknown function) -- ") ret = run_cmd('print "kuku(16)" 1') -if ret=="Python Interface: Error loading local functions" or ret=="SyntaxError: invalid syntax": +if ret=="Python Interface: Error loading local functions" or ret.startswith("SyntaxError: invalid syntax"): sys.stdout.write("OK\n") else: sys.stdout.write("Not OK: <"+ret+">\n") diff --git a/specs/tests/valgrind_unit_tests.py b/specs/tests/valgrind_unit_tests.py index 20dbd9a..c5329e2 100644 --- a/specs/tests/valgrind_unit_tests.py +++ b/specs/tests/valgrind_unit_tests.py @@ -1,7 +1,7 @@ import sys, memcheck, argparse -count_ALU_tests = 825 -count_processing_tests = 228 +count_ALU_tests = 832 +count_processing_tests = 247 count_token_tests = 17 # Parse the one command line options