diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a67ddebd..e1e78afa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,9 +28,6 @@ jobs: - name: Install dependencies run: | poetry install - - name: Ensure pure cbor2 is installed - run: | - make ensure-pure-cbor2 - name: Run unit tests run: | poetry run pytest --doctest-modules --ignore=examples --cov=pycardano --cov-config=.coveragerc --cov-report=xml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ef489fcd..c451ff40 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -24,9 +24,6 @@ jobs: - name: Install dependencies run: | poetry install - - name: Ensure pure cbor2 is installed - run: | - make ensure-pure-cbor2 - name: Lint with flake8 run: | poetry run flake8 pycardano diff --git a/Makefile b/Makefile index c72b327b..755e769e 100644 --- a/Makefile +++ b/Makefile @@ -23,25 +23,10 @@ export PRINT_HELP_PYSCRIPT BROWSER := poetry run python -c "$$BROWSER_PYSCRIPT" -ensure-pure-cbor2: ## ensures cbor2 is installed with pure Python implementation - @poetry run python -c "from importlib.metadata import version; \ - print(version('cbor2'))" > .cbor2_version - @poetry run python -c "import cbor2, inspect; \ - print('Checking cbor2 implementation...'); \ - decoder_path = inspect.getfile(cbor2.CBORDecoder); \ - using_c_ext = decoder_path.endswith('.so'); \ - print(f'Implementation path: {decoder_path}'); \ - print(f'Using C extension: {using_c_ext}'); \ - exit(1 if using_c_ext else 0)" || \ - (echo "Reinstalling cbor2 with pure Python implementation..." && \ - poetry run pip uninstall -y cbor2 && \ - CBOR2_BUILD_C_EXTENSION=0 poetry run pip install --no-binary cbor2 "cbor2==$$(cat .cbor2_version)" --force-reinstall && \ - rm .cbor2_version) - help: @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) -cov: ensure-pure-cbor2 ## check code coverage +cov: ## check code coverage poetry run pytest -n 4 --cov pycardano cov-html: cov ## check code coverage and generate an html report @@ -69,7 +54,7 @@ clean-test: ## remove test and coverage artifacts rm -fr cov_html/ rm -fr .pytest_cache -test: ensure-pure-cbor2 ## runs tests +test: ## runs tests poetry run pytest -vv -n 4 test-integration: ## runs integration tests @@ -78,7 +63,7 @@ test-integration: ## runs integration tests test-single: ## runs tests with "single" markers poetry run pytest -s -vv -m single -qa: ensure-pure-cbor2 ## runs static analyses +qa: ## runs static analyses poetry run flake8 pycardano poetry run mypy --install-types --non-interactive pycardano poetry run black --check . @@ -92,6 +77,6 @@ docs: ## build the documentation poetry run sphinx-build docs/source docs/build/html $(BROWSER) docs/build/html/index.html -release: clean qa test format ensure-pure-cbor2 ## build dist version and release to pypi +release: clean qa test format ## build dist version and release to pypi poetry build poetry publish \ No newline at end of file diff --git a/README.md b/README.md index 40f5a84a..b6281b5f 100644 --- a/README.md +++ b/README.md @@ -57,11 +57,13 @@ Install the library using [pip](https://pip.pypa.io/en/stable/): `pip install pycardano` -#### Install cbor2 pure python implementation (Optional) +#### cbor2 [cbor2](https://github.com/agronholm/cbor2/tree/master) is a dependency of pycardano. It is used to encode and decode CBOR data. It has two implementations: one is pure Python and the other is C, which is installed by default. The C implementation is faster, but it is less flexible than the pure Python implementation. -For some users, the C implementation may not work properly when deserializing cbor data. For example, the order of inputs of a transaction isn't guaranteed to be the same as the order of inputs in the original transaction (details could be found in [this issue](https://github.com/Python-Cardano/pycardano/issues/311)). This would result in a different transaction hash when the transaction is serialized again. For users who encounter this issue, we recommend to use the pure Python implementation of cbor2. You can do so by running [ensure_pure_cbor2.sh](./ensure_pure_cbor2.sh), which inspect the cbor2 installed in the running environment and force install pure python implementation if necessary. +For some users, the C implementation may not work properly when deserializing cbor data. For example, the order of inputs of a transaction isn't guaranteed to be the same as the order of inputs in the original transaction (details could be found in [this issue](https://github.com/Python-Cardano/pycardano/issues/311)). This would result in a different transaction hash when the transaction is serialized again. + +To solve this problem, a fork of cbor2 is created at [cbor2pure](https://github.com/cffls/cbor2pure). This fork removes C extension and only uses pure python for cbor decoding. By default, for correctness, pycardano uses cbor2pure in decoding. However, if speed is preferred over accuracy, users can set `CBOR_C_EXTENSION=1` in their environment, and the default C extension would be used instead. ```bash ensure_pure_cbor2.sh diff --git a/ensure_pure_cbor2.sh b/ensure_pure_cbor2.sh deleted file mode 100755 index 9c50a210..00000000 --- a/ensure_pure_cbor2.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Script to ensure cbor2 is installed with pure Python implementation - -set -x - -# Check if poetry is available, otherwise use python directly -if command -v poetry &> /dev/null; then - PYTHON="poetry run python" -else - PYTHON="python" -fi - -echo "Checking cbor2 version..." -$PYTHON -c "from importlib.metadata import version; print(version('cbor2'))" > .cbor2_version -CBOR2_VERSION=$(cat .cbor2_version) -echo "Found cbor2 version: $CBOR2_VERSION" - -echo "Checking cbor2 implementation..." -$PYTHON -c " -import cbor2, inspect, sys -decoder_path = inspect.getfile(cbor2.CBORDecoder) -using_c_ext = decoder_path.endswith('.so') -print(f'Implementation path: {decoder_path}') -print(f'Using C extension: {using_c_ext}') -sys.exit(1 if using_c_ext else 0) -" - -if [ $? -ne 0 ]; then - echo "Reinstalling cbor2 with pure Python implementation..." - $PYTHON -m pip uninstall -y cbor2 || uv pip uninstall cbor2 - CBOR2_BUILD_C_EXTENSION=0 $PYTHON -m pip install --no-binary cbor2 "cbor2==$CBOR2_VERSION" --force-reinstall || CBOR2_BUILD_C_EXTENSION=0 uv pip install --no-binary cbor2 "cbor2==$CBOR2_VERSION" --force-reinstall - echo "Successfully reinstalled cbor2 with pure Python implementation" -else - echo "Already using pure Python implementation of cbor2" -fi - -# Clean up -rm -f .cbor2_version diff --git a/integration-test/run_tests.sh b/integration-test/run_tests.sh index 11ea5242..8d32d0f9 100755 --- a/integration-test/run_tests.sh +++ b/integration-test/run_tests.sh @@ -6,7 +6,6 @@ set -o pipefail ROOT=$(pwd) poetry install -C .. -make ensure-pure-cbor2 -f ../Makefile #poetry run pip install ogmios ########## diff --git a/poetry.lock b/poetry.lock index c0dd4358..0eb28037 100644 --- a/poetry.lock +++ b/poetry.lock @@ -262,6 +262,21 @@ files = [ {file = "cbor2-5.7.0.tar.gz", hash = "sha256:3f6d843f4db4d0ec501c46453c22a4fbebb1abfb5b740e1bcab34c615cd7406b"}, ] +[[package]] +name = "cbor2pure" +version = "5.7.2" +description = "CBOR (de)serializer with extensive tag support" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "cbor2pure-5.7.2-py3-none-any.whl", hash = "sha256:cc3ea93afc3d92789343cdaa7c3f75802f56beecf6272ee6069e04a237641d4f"}, + {file = "cbor2pure-5.7.2.tar.gz", hash = "sha256:96bbd679260777c4e2075fe00a28a16a297207862d7e7ff80ab529c91c35925e"}, +] + +[package.dependencies] +cbor2 = "*" + [[package]] name = "certifi" version = "2025.8.3" @@ -2541,4 +2556,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = "^3.9.1" -content-hash = "ba802fdd52ec008c3af72f5c7b8765d18ba0875feab49b9a78c6b94cfffae680" +content-hash = "03156c658c4cb55fd0abc906c448aa1b0569a09e8321b00e32bd47a6b0bc7c1e" diff --git a/pycardano/address.py b/pycardano/address.py index 8d660995..0b4290ab 100644 --- a/pycardano/address.py +++ b/pycardano/address.py @@ -14,10 +14,10 @@ from typing import Optional, Type, Union import base58 -import cbor2 from cbor2 import CBORTag from typing_extensions import override +from pycardano.cbor import cbor2 from pycardano.crypto.bech32 import decode, encode from pycardano.exception import ( DecodingException, diff --git a/pycardano/backend/blockfrost.py b/pycardano/backend/blockfrost.py index 1d25a6f8..b7b11bad 100644 --- a/pycardano/backend/blockfrost.py +++ b/pycardano/backend/blockfrost.py @@ -5,7 +5,6 @@ from fractions import Fraction from typing import Dict, List, Optional, Union -import cbor2 from blockfrost import ApiError, ApiUrls, BlockFrostApi from blockfrost.utils import Namespace @@ -16,6 +15,7 @@ GenesisParameters, ProtocolParameters, ) +from pycardano.cbor import cbor2 from pycardano.exception import TransactionFailedException from pycardano.hash import SCRIPT_HASH_SIZE, DatumHash, ScriptHash from pycardano.nativescript import NativeScript diff --git a/pycardano/backend/cardano_cli.py b/pycardano/backend/cardano_cli.py index 4c4c0da7..61398c8f 100644 --- a/pycardano/backend/cardano_cli.py +++ b/pycardano/backend/cardano_cli.py @@ -12,7 +12,6 @@ from pathlib import Path from typing import Dict, List, Optional, Union -import cbor2 import docker from cachetools import Cache, LRUCache, TTLCache, func from docker.errors import APIError @@ -24,6 +23,7 @@ GenesisParameters, ProtocolParameters, ) +from pycardano.cbor import cbor2 from pycardano.exception import ( CardanoCliError, PyCardanoException, diff --git a/pycardano/cbor.py b/pycardano/cbor.py new file mode 100644 index 00000000..ddd6e2ab --- /dev/null +++ b/pycardano/cbor.py @@ -0,0 +1,14 @@ +""" +Conditional cbor2 import module. + +This module provides a centralized location for importing cbor2, +with support for both the C extension (cbor2) and pure Python (cbor2pure) versions. +Set the environment variable CBOR_C_EXTENSION=1 to use the C extension. +""" + +import os + +if os.getenv("CBOR_C_EXTENSION", "0") == "1": + import cbor2 # noqa: F401 +else: + import cbor2pure as cbor2 # type: ignore # noqa: F401 diff --git a/pycardano/plutus.py b/pycardano/plutus.py index 83c66705..98036de6 100644 --- a/pycardano/plutus.py +++ b/pycardano/plutus.py @@ -10,12 +10,12 @@ from hashlib import sha256 from typing import Any, List, Optional, Type, Union -import cbor2 from cbor2 import CBORTag from nacl.encoding import RawEncoder from nacl.hash import blake2b from typeguard import typechecked +from pycardano.cbor import cbor2 from pycardano.exception import DeserializeException, InvalidArgumentException from pycardano.hash import DATUM_HASH_SIZE, SCRIPT_HASH_SIZE, DatumHash, ScriptHash from pycardano.nativescript import NativeScript diff --git a/pycardano/serialization.py b/pycardano/serialization.py index 385fc120..bc6e1cec 100644 --- a/pycardano/serialization.py +++ b/pycardano/serialization.py @@ -31,8 +31,7 @@ get_type_hints, ) -import cbor2 - +from pycardano.cbor import cbor2 from pycardano.logging import logger # Remove the semantic decoder for 258 (CBOR tag for set) as we care about the order of elements diff --git a/pycardano/transaction.py b/pycardano/transaction.py index da3deb56..fa3f1666 100644 --- a/pycardano/transaction.py +++ b/pycardano/transaction.py @@ -6,13 +6,13 @@ from dataclasses import dataclass, field from typing import Any, Callable, List, Optional, Type, Union -import cbor2 from cbor2 import CBORTag from nacl.encoding import RawEncoder from nacl.hash import blake2b from pprintpp import pformat from pycardano.address import Address +from pycardano.cbor import cbor2 from pycardano.certificate import Certificate from pycardano.exception import InvalidDataException from pycardano.governance import ProposalProcedure, VotingProcedures @@ -105,7 +105,7 @@ def __add__(self, other: Asset) -> Asset: def __iadd__(self, other: Asset) -> Asset: new_item = self + other - self.data = new_item.data + self.data = new_item.data # type: ignore[has-type] return self.normalize() def __sub__(self, other: Asset) -> Asset: diff --git a/pycardano/utils.py b/pycardano/utils.py index 00c5f6a4..ddb50ff5 100644 --- a/pycardano/utils.py +++ b/pycardano/utils.py @@ -6,11 +6,11 @@ import sys from typing import Dict, List, Optional, Tuple, Union -import cbor2 from nacl.encoding import RawEncoder from nacl.hash import blake2b from pycardano.backend.base import ChainContext +from pycardano.cbor import cbor2 from pycardano.hash import SCRIPT_DATA_HASH_SIZE, SCRIPT_HASH_SIZE, ScriptDataHash from pycardano.plutus import COST_MODELS, CostModels, Datum, RedeemerMap, Redeemers from pycardano.serialization import NonEmptyOrderedSet, default_encoder diff --git a/pyproject.toml b/pyproject.toml index 83f0e1f9..bf45e5eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ ogmios = ">=1.4.2" requests = ">=2.32.3" websockets = ">=13.0" base58 = ">=2.1.0" +cbor2pure = ">=5.7.2" [tool.poetry.group.dev.dependencies] pytest = ">=8.2.0" diff --git a/test/pycardano/test_serialization.py b/test/pycardano/test_serialization.py index 607d3d18..256caf62 100644 --- a/test/pycardano/test_serialization.py +++ b/test/pycardano/test_serialization.py @@ -18,7 +18,6 @@ get_origin, ) -import cbor2 import pytest from cbor2 import CBORTag from frozenlist import FrozenList @@ -35,6 +34,7 @@ VerificationKey, VerificationKeyWitness, ) +from pycardano.cbor import cbor2 from pycardano.exception import ( DeserializeException, InvalidKeyTypeException,