feat: show source code lines / traceback from ReceiptAPI [APE-708] (ApeWorX#1337)

Author: antazoey

codeql-config.yml

@@ -5,4 +5,3 @@ queries:
 
 paths:
   - src
-

setup.py

@@ -19,7 +19,7 @@
     ],
     "lint": [
         "black>=23.3.0,<24",  # Auto-formatter and linter
-        "mypy>=0.991",  # Static type analyzer
+        "mypy>=0.991,<1",  # Static type analyzer
         "types-PyYAML",  # Needed due to mypy typeshed
         "types-requests",  # Needed due to mypy typeshed
         "types-setuptools",  # Needed due to mypy typeshed
@@ -121,8 +121,8 @@
         "web3[tester]>=6.0.0,<7",
         # ** Dependencies maintained by ApeWorX **
         "eip712>=0.2.1,<0.3",
-        "ethpm-types>=0.4.5,<0.5",
-        "evm-trace>=0.1.0a18",
+        "ethpm-types>=0.5.0,<0.6",
+        "evm-trace>=0.1.0a19",
     ],
     entry_points={
         "console_scripts": ["ape=ape._cli:cli"],

src/ape/api/compiler.py

@@ -1,10 +1,14 @@
 from pathlib import Path
-from typing import Dict, List, Optional, Set
+from typing import Dict, Iterator, List, Optional, Set, Tuple
 
-from ethpm_types import ContractType
+from ethpm_types import ContractType, HexBytes
+from ethpm_types.source import ContractSource
+from evm_trace.geth import TraceFrame as EvmTraceFrame
+from evm_trace.geth import create_call_node_data
 from semantic_version import Version  # type: ignore
 
 from ape.exceptions import ContractLogicError
+from ape.types.trace import SourceTraceback, TraceFrame
 from ape.utils import BaseInterfaceModel, abstractmethod, raises_not_implemented
 
 
@@ -125,3 +129,35 @@ def enrich_error(self, err: ContractLogicError) -> ContractLogicError:
         """
 
         return err
+
+    @raises_not_implemented
+    def trace_source(  # type: ignore[empty-body]
+        self, contract_type: ContractType, trace: Iterator[TraceFrame], calldata: HexBytes
+    ) -> SourceTraceback:
+        """
+        Get a source-traceback for the given contract type.
+        The source traceback object contains all the control paths taken in the transaction.
+        When available, source-code location information is accessible from the object.
+
+        Args:
+            contract_type (``ContractType``): A contract type that was created by this compiler.
+            trace (Iterator[:class:`~ape.types.trace.TraceFrame`]): The resulting frames from
+              executing a function defined in the given contract type.
+            calldata (``HexBytes``): Calldata passed to the top-level call.
+
+        Returns:
+            :class:`~ape.types.trace.SourceTraceback`
+        """
+
+    def _create_contract_from_call(
+        self, frame: TraceFrame
+    ) -> Tuple[Optional[ContractSource], HexBytes]:
+        evm_frame = EvmTraceFrame(**frame.raw)
+        data = create_call_node_data(evm_frame)
+        calldata = data["calldata"]
+        address = self.provider.network.ecosystem.decode_address(data["address"])
+        if address not in self.chain_manager.contracts:
+            return None, calldata
+
+        called_contract = self.chain_manager.contracts[address]
+        return self.project_manager._create_contract_source(called_contract), calldata

src/ape/api/projects.py

@@ -1,7 +1,7 @@
 import os.path
 import tempfile
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 import yaml
 from ethpm_types import Checksum, ContractType, PackageManifest, Source
@@ -171,17 +171,20 @@ def _create_manifest(
 
     @classmethod
     def _create_source_dict(
-        cls, contract_filepaths: List[Path], base_path: Path
+        cls, contract_filepaths: Union[Path, List[Path]], base_path: Path
     ) -> Dict[str, Source]:
+        filepaths = (
+            [contract_filepaths] if isinstance(contract_filepaths, Path) else contract_filepaths
+        )
         source_imports: Dict[str, List[str]] = cls.compiler_manager.get_imports(
-            contract_filepaths, base_path
+            filepaths, base_path
         )  # {source_id: [import_source_ids, ...], ...}
         source_references: Dict[str, List[str]] = cls.compiler_manager.get_references(
             imports_dict=source_imports
         )  # {source_id: [referring_source_ids, ...], ...}
 
         source_dict: Dict[str, Source] = {}
-        for source_path in contract_filepaths:
+        for source_path in filepaths:
             key = str(get_relative_path(source_path, base_path))
             source_dict[key] = Source(
                 checksum=Checksum(
@@ -354,7 +357,7 @@ def compile(self) -> PackageManifest:
                 # Create content, including sub-directories.
                 source_path.parent.mkdir(parents=True, exist_ok=True)
                 source_path.touch()
-                source_path.write_text(content)
+                source_path.write_text(str(content))
 
             # Handle import remapping entries indicated in the manifest file
             target_config_file = project.path / project.config_file_name

src/ape/api/transactions.py

@@ -19,7 +19,13 @@
     TransactionNotFoundError,
 )
 from ape.logging import logger
-from ape.types import AddressType, ContractLogContainer, TraceFrame, TransactionSignature
+from ape.types import (
+    AddressType,
+    ContractLogContainer,
+    SourceTraceback,
+    TraceFrame,
+    TransactionSignature,
+)
 from ape.utils import BaseInterfaceModel, abstractmethod, cached_property, raises_not_implemented
 
 if TYPE_CHECKING:
@@ -428,6 +434,15 @@ def return_value(self) -> Any:
 
         return output
 
+    @property
+    @raises_not_implemented
+    def source_traceback(self) -> SourceTraceback:  # type: ignore[empty-body]
+        """
+        A pythonic style traceback for both failing and non-failing receipts.
+        Requires a provider that implements
+        :meth:~ape.api.providers.ProviderAPI.get_transaction_trace`.
+        """
+
     @raises_not_implemented
     def show_trace(self, verbose: bool = False, file: IO[str] = sys.stdout):
         """
@@ -445,6 +460,14 @@ def show_gas_report(self, file: IO[str] = sys.stdout):
         Display a gas report for the calls made in this transaction.
         """
 
+    @raises_not_implemented
+    def show_source_traceback(self):
+        """
+        Show a receipt traceback mapping to lines in the source code.
+        Only works when the contract type and source code are both available,
+        like in local projects.
+        """
+
     def track_gas(self):
         """
         Track this receipt's gas in the on-going session gas-report.

src/ape/contracts/base.py

@@ -756,7 +756,7 @@ def receipt(self) -> Optional[ReceiptAPI]:
         if not self._cached_receipt and self.txn_hash:
             try:
                 receipt = self.chain_manager.get_receipt(self.txn_hash)
-            except TransactionNotFoundError:
+            except (TransactionNotFoundError, ValueError):
                 return None
 
             self._cached_receipt = receipt

src/ape/exceptions.py

@@ -1,10 +1,12 @@
 import sys
+import tempfile
 import time
 import traceback
 from collections import deque
 from functools import cached_property
 from inspect import getframeinfo, stack
 from pathlib import Path
+from types import CodeType, TracebackType
 from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional
 
 import click
@@ -19,7 +21,7 @@
     from ape.api.networks import NetworkAPI
     from ape.api.providers import SubprocessProvider
     from ape.api.transactions import TransactionAPI
-    from ape.types import AddressType, BlockID, SnapshotID, TraceFrame
+    from ape.types import AddressType, BlockID, SnapshotID, SourceTraceback, TraceFrame
 
 
 class ApeException(Exception):
@@ -112,9 +114,24 @@ def __init__(
         self.txn = txn
         self.trace = trace
         self.contract_address = contract_address
+        self.source_traceback: Optional["SourceTraceback"] = None
         ex_message = f"({code}) {message}" if code else message
+
+        # Finalizes expected revert message.
         super().__init__(ex_message)
 
+        if not txn:
+            return
+
+        ape_tb = _get_ape_traceback(self, txn)
+        if not ape_tb:
+            return
+
+        self.source_traceback = ape_tb
+        py_tb = _get_custom_python_traceback(self, txn, ape_tb)
+        if py_tb:
+            self.__traceback__ = py_tb
+
 
 class VirtualMachineError(TransactionError):
     """
@@ -532,9 +549,10 @@ def handle_ape_exception(err: ApeException, base_paths: List[Path]) -> bool:
     an exception on the exc-stack.
 
     Args:
-        err (:class:`~ape.exceptions.ApeException`): The transaction error
+        err (:class:`~ape.exceptions.TransactionError`): The transaction error
           being handled.
-        base_paths (List[Path]): Source base paths for allowed frames.
+        base_paths (Optional[List[Path]]): Optionally include additional
+          source-path prefixes to use when finding relevant frames.
 
     Returns:
         bool: ``True`` if outputted something.
@@ -621,3 +639,87 @@ def name(self) -> str:
         The name of the error.
         """
         return self.abi.name
+
+
+def _get_ape_traceback(err: TransactionError, txn: "TransactionAPI") -> Optional["SourceTraceback"]:
+    receipt = txn.receipt
+    if not receipt:
+        return None
+
+    try:
+        ape_traceback = receipt.source_traceback
+    except (ApeException, NotImplementedError):
+        return None
+
+    if ape_traceback is None or not len(ape_traceback):
+        return None
+
+    return ape_traceback
+
+
+def _get_custom_python_traceback(
+    err: TransactionError, txn: "TransactionAPI", ape_traceback: "SourceTraceback"
+) -> Optional[TracebackType]:
+    # Manipulate python traceback to show lines from contract.
+    # Help received from Jinja lib:
+    #  https://github.com/pallets/jinja/blob/main/src/jinja2/debug.py#L142
+
+    _, exc_value, tb = sys.exc_info()
+    depth = None
+    idx = len(ape_traceback) - 1
+    frames = []
+    project_path = txn.project_manager.path.as_posix()
+    while tb is not None:
+        if not tb.tb_frame.f_code.co_filename.startswith(project_path):
+            # Ignore frames outside the project.
+            # This allows both contract code an scripts to appear.
+            tb = tb.tb_next
+            continue
+
+        frames.append(tb)
+        tb = tb.tb_next
+
+    while (depth is None or depth > 1) and idx >= 0:
+        exec_item = ape_traceback[idx]
+        if depth is not None and exec_item.depth >= depth:
+            # Wait for decreasing depth.
+            continue
+
+        depth = exec_item.depth
+        lineno = exec_item.begin_lineno
+        if lineno is None:
+            continue
+
+        if exec_item.source_path is None:
+            # File is not local. Create a temporary file in its place.
+            # This is necessary for tracebacks to work in Python.
+            temp_file = tempfile.NamedTemporaryFile(prefix="unknown_contract_")
+            filename = temp_file.name
+        else:
+            filename = exec_item.source_path.as_posix()
+
+        # Raise an exception at the correct line number.
+        py_code: CodeType = compile(
+            "\n" * (lineno - 1) + "raise __ape_exception__", filename, "exec"
+        )
+        py_code = py_code.replace(co_name=exec_item.closure.name)
+
+        # Execute the new code to get a new (fake) tb with contract source info.
+        try:
+            exec(py_code, {"__ape_exception__": err}, {})
+        except BaseException:
+            fake_tb = sys.exc_info()[2].tb_next  # type: ignore
+            if isinstance(fake_tb, TracebackType):
+                frames.append(fake_tb)
+
+        idx -= 1
+
+    if not frames:
+        return None
+
+    tb_next = None
+    for tb in frames:
+        tb.tb_next = tb_next
+        tb_next = tb
+
+    return frames[-1]