From 427f36883b6bda6f0893ab00a131afc53b67badb Mon Sep 17 00:00:00 2001 From: pstlouis Date: Fri, 26 Jul 2024 10:32:00 -0400 Subject: [PATCH 1/3] bump ujson to 5.4.0 Signed-off-by: pstlouis --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ff846c2e3..86186a3ae 100644 --- a/setup.py +++ b/setup.py @@ -134,7 +134,7 @@ def run(self): ### Tests fail without version pin (GHA run: https://github.com/udosson/indy-plenum/actions/runs/1078741118) 'sortedcontainers==1.5.7', ### Tests fail without version pin (GHA run: https://github.com/udosson/indy-plenum/actions/runs/1078741118) - 'ujson==1.33', + 'ujson==5.4.0', ], setup_requires=['pytest-runner==5.3.0'], From f69ee5cf88bd21b4a6eccf72f8775dc62e29a5c4 Mon Sep 17 00:00:00 2001 From: pstlouis Date: Fri, 13 Sep 2024 14:11:59 -0400 Subject: [PATCH 2/3] remove ujson Signed-off-by: pstlouis --- .../ubuntu-2004/build-3rd-parties.sh | 1 - build-scripts/ubuntu-2004/prepare-package.sh | 1 - common/serializers/json_serializer.py | 46 ++++++------------- common/serializers/signing_serializer.py | 3 -- dev-setup/ubuntu/ubuntu-2004/SetupVMTest.txt | 1 - plenum/recorder/recorder.py | 6 +-- plenum/test/recorder/test_recorder.py | 6 +-- scripts/test_zmq/test_zmq/zstack.py | 7 +-- setup.py | 2 - stp_zmq/zstack.py | 5 +- 10 files changed, 17 insertions(+), 61 deletions(-) diff --git a/build-scripts/ubuntu-2004/build-3rd-parties.sh b/build-scripts/ubuntu-2004/build-3rd-parties.sh index 7770c5b28..da6c70802 100755 --- a/build-scripts/ubuntu-2004/build-3rd-parties.sh +++ b/build-scripts/ubuntu-2004/build-3rd-parties.sh @@ -126,4 +126,3 @@ build_from_pypi semver 2.13.0 build_from_pypi sha3 build_from_pypi six build_from_pypi sortedcontainers 1.5.7 -build_from_pypi ujson 1.33 diff --git a/build-scripts/ubuntu-2004/prepare-package.sh b/build-scripts/ubuntu-2004/prepare-package.sh index 7579ccf81..44eacc88a 100755 --- a/build-scripts/ubuntu-2004/prepare-package.sh +++ b/build-scripts/ubuntu-2004/prepare-package.sh @@ -29,7 +29,6 @@ if [ "$distro_packages" = "debian-packages" ]; then # Update the package names to match the versions that are pre-installed on the os. echo -e "\nAdapt the dependencies for the Canonical archive" #### ToDo adjust packages for the Cannonical archive for Ubuntu 20.04 (focal) - # sed -i "s~ujson==1.33~ujson==1.33-1build1~" setup.py # sed -i "s~prompt_toolkit==0.57~prompt_toolkit==0.57-1~" setup.py # sed -i "s~msgpack-python==0.4.6~msgpack==0.4.6-1build1~" setup.py elif [ "$distro_packages" = "python-packages" ]; then diff --git a/common/serializers/json_serializer.py b/common/serializers/json_serializer.py index a012dbe0c..40c7fc706 100644 --- a/common/serializers/json_serializer.py +++ b/common/serializers/json_serializer.py @@ -2,45 +2,25 @@ import base64 +import json from typing import Dict from common.serializers.mapping_serializer import MappingSerializer -try: - import ujson as json - from ujson import encode as uencode +class OrderedJsonEncoder(json.JSONEncoder): + def __init__(self, *args, **kwargs): + kwargs['ensure_ascii'] = False + kwargs['sort_keys'] = True + kwargs['separators'] = (',', ':') + super().__init__(*args, **kwargs) - # Older versions of ujson's encode do not support `sort_keys`, if that - # is the case default to using json - uencode({'xx': '123', 'aa': 90}, sort_keys=True) + def encode(self, o): + if isinstance(o, (bytes, bytearray)): + return '"{}"'.format(base64.b64encode(o).decode("utf-8")) + else: + return super().encode(o) - class UJsonEncoder: - @staticmethod - def encode(o): - if isinstance(o, (bytes, bytearray)): - return '"{}"'.format(base64.b64encode(o).decode("utf-8")) - else: - return uencode(o, sort_keys=True) - - JsonEncoder = UJsonEncoder() - -except (ImportError, TypeError): - import json - - class OrderedJsonEncoder(json.JSONEncoder): - def __init__(self, *args, **kwargs): - kwargs['ensure_ascii'] = False - kwargs['sort_keys'] = True - kwargs['separators'] = (',', ':') - super().__init__(*args, **kwargs) - - def encode(self, o): - if isinstance(o, (bytes, bytearray)): - return '"{}"'.format(base64.b64encode(o).decode("utf-8")) - else: - return super().encode(o) - - JsonEncoder = OrderedJsonEncoder() +JsonEncoder = OrderedJsonEncoder() class JsonSerializer(MappingSerializer): diff --git a/common/serializers/signing_serializer.py b/common/serializers/signing_serializer.py index 4888cb5f4..6ab526f8e 100644 --- a/common/serializers/signing_serializer.py +++ b/common/serializers/signing_serializer.py @@ -90,6 +90,3 @@ def serialize(self, obj, level=0, objname=None, topLevelKeysToIgnore=None, return res return res.encode('utf-8') - - # topLevelKeysToIgnore = topLevelKeysToIgnore or [] - # return ujson.dumps({k:obj[k] for k in obj.keys() if k not in topLevelKeysToIgnore}, sort_keys=True) diff --git a/dev-setup/ubuntu/ubuntu-2004/SetupVMTest.txt b/dev-setup/ubuntu/ubuntu-2004/SetupVMTest.txt index a67c63597..2e5bcf576 100644 --- a/dev-setup/ubuntu/ubuntu-2004/SetupVMTest.txt +++ b/dev-setup/ubuntu/ubuntu-2004/SetupVMTest.txt @@ -82,7 +82,6 @@ sortedcontainers==1.5.7 \ timeout-decorator==0.5.0 \ toml==0.10.2 \ - ujson==1.33 \ wcwidth==0.2.5 \ wheel==0.34.2 \ zipp==1.2.0 diff --git a/plenum/recorder/recorder.py b/plenum/recorder/recorder.py index ca897d9f3..3f3da8fd0 100644 --- a/plenum/recorder/recorder.py +++ b/plenum/recorder/recorder.py @@ -1,14 +1,10 @@ import os import time +import json from typing import Callable from storage.kv_store_rocksdb_int_keys import KeyValueStorageRocksdbIntKeys -try: - import ujson as json -except ImportError: - import json - class Recorder: INCOMING_FLAG = 0 diff --git a/plenum/test/recorder/test_recorder.py b/plenum/test/recorder/test_recorder.py index 29d85aa26..d9d78d156 100644 --- a/plenum/test/recorder/test_recorder.py +++ b/plenum/test/recorder/test_recorder.py @@ -1,14 +1,10 @@ import random import time +import json from collections import OrderedDict from plenum.common.util import randomString -try: - import ujson as json -except ImportError: - import json - import pytest from plenum.recorder.recorder import Recorder diff --git a/scripts/test_zmq/test_zmq/zstack.py b/scripts/test_zmq/test_zmq/zstack.py index f80b81823..a33e38a35 100644 --- a/scripts/test_zmq/test_zmq/zstack.py +++ b/scripts/test_zmq/test_zmq/zstack.py @@ -1,10 +1,5 @@ from test_zmq.authenticator import MultiZapAuthenticator - -try: - import ujson as json -except ImportError: - import json - +import json import os import shutil import sys diff --git a/setup.py b/setup.py index c9c7f0733..035f34544 100644 --- a/setup.py +++ b/setup.py @@ -131,8 +131,6 @@ def run(self): 'six', ### Tests fail without version pin (GHA run: https://github.com/udosson/indy-plenum/actions/runs/1078741118) 'sortedcontainers==1.5.7', - ### Tests fail without version pin (GHA run: https://github.com/udosson/indy-plenum/actions/runs/1078741118) - 'ujson==5.4.0', ], setup_requires=['pytest-runner==5.3.0'], diff --git a/stp_zmq/zstack.py b/stp_zmq/zstack.py index 32a2a07f6..f2e817dda 100644 --- a/stp_zmq/zstack.py +++ b/stp_zmq/zstack.py @@ -1,4 +1,5 @@ import inspect +import json from plenum.common.constants import OP_FIELD_NAME, BATCH from plenum.common.metrics_collector import NullMetricsCollector @@ -8,10 +9,6 @@ from stp_core.common.constants import CONNECTION_PREFIX, ZMQ_NETWORK_PROTOCOL from stp_zmq.client_message_provider import ClientMessageProvider -try: - import ujson as json -except ImportError: - import json import os import shutil From 8d299a74217ebdc9746f4b5a41ab4f00d8fe73fb Mon Sep 17 00:00:00 2001 From: pstlouis Date: Fri, 13 Sep 2024 14:19:23 -0400 Subject: [PATCH 3/3] linting errors Signed-off-by: pstlouis --- common/serializers/json_serializer.py | 2 ++ ledger/test/helper.py | 3 ++- plenum/common/channel.py | 2 +- .../plugin/stats_consumer/plugin_firebase_stats_consumer.py | 4 ++-- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/common/serializers/json_serializer.py b/common/serializers/json_serializer.py index 40c7fc706..71fa03da9 100644 --- a/common/serializers/json_serializer.py +++ b/common/serializers/json_serializer.py @@ -7,6 +7,7 @@ from common.serializers.mapping_serializer import MappingSerializer + class OrderedJsonEncoder(json.JSONEncoder): def __init__(self, *args, **kwargs): kwargs['ensure_ascii'] = False @@ -20,6 +21,7 @@ def encode(self, o): else: return super().encode(o) + JsonEncoder = OrderedJsonEncoder() diff --git a/ledger/test/helper.py b/ledger/test/helper.py index a1a2194e5..548a9a6ae 100644 --- a/ledger/test/helper.py +++ b/ledger/test/helper.py @@ -108,7 +108,6 @@ def create_ledger_rocksdb_storage(txn_serializer, hash_serializer, tempdir, init def create_ledger_chunked_file_storage(txn_serializer, hash_serializer, tempdir, init_genesis_txn_file=None): - chunk_creator = None db_name = 'transactions' if isinstance(txn_serializer, MsgPackSerializer): # TODO: fix chunk_creator support @@ -119,6 +118,8 @@ def chunk_creator(name): isLineNoKey=True, storeContentHash=False, ensureDurability=False) + else: + chunk_creator = None store = ChunkedFileStore(tempdir, db_name, isLineNoKey=True, diff --git a/plenum/common/channel.py b/plenum/common/channel.py index de782ca6a..323a3b864 100644 --- a/plenum/common/channel.py +++ b/plenum/common/channel.py @@ -156,7 +156,7 @@ def _process_sync(self, msg: Any): # This is done so that messages can include additional metadata # isinstance is not used here because it returns true for NamedTuple # as well. - if type(msg) != tuple: + if not isinstance(type(msg), tuple): msg = (msg,) handler = self._find_handler(msg[0]) if handler is None: diff --git a/plenum/server/plugin/stats_consumer/plugin_firebase_stats_consumer.py b/plenum/server/plugin/stats_consumer/plugin_firebase_stats_consumer.py index 1fab98b30..6bf2f4a45 100644 --- a/plenum/server/plugin/stats_consumer/plugin_firebase_stats_consumer.py +++ b/plenum/server/plugin/stats_consumer/plugin_firebase_stats_consumer.py @@ -6,11 +6,11 @@ from plenum.common.types import EVENT_PERIODIC_STATS_THROUGHPUT, \ EVENT_NODE_STARTED, EVENT_REQ_ORDERED, EVENT_PERIODIC_STATS_LATENCIES, \ PLUGIN_TYPE_STATS_CONSUMER, EVENT_VIEW_CHANGE, EVENT_PERIODIC_STATS_NODES, \ - EVENT_PERIODIC_STATS_TOTAL_REQUESTS, EVENT_PERIODIC_STATS_NODE_INFO,\ + EVENT_PERIODIC_STATS_TOTAL_REQUESTS, EVENT_PERIODIC_STATS_NODE_INFO, \ EVENT_PERIODIC_STATS_SYSTEM_PERFORMANCE_INFO from stp_core.common.log import getlogger from plenum.config import STATS_SERVER_IP, STATS_SERVER_PORT, STATS_SERVER_MESSAGE_BUFFER_MAX_SIZE -from plenum.server.plugin.stats_consumer.stats_publisher import StatsPublisher,\ +from plenum.server.plugin.stats_consumer.stats_publisher import StatsPublisher, \ Topic from plenum.server.plugin_loader import HasDynamicallyImportedModules from plenum.server.stats_consumer import StatsConsumer