diff --git a/quarkchain/cluster/cluster.py b/quarkchain/cluster/cluster.py index 1fb3cf13b..5b6271fda 100644 --- a/quarkchain/cluster/cluster.py +++ b/quarkchain/cluster/cluster.py @@ -106,7 +106,7 @@ async def run_master(self): extra_cmd += " --enable_profiler=true" master = await run_master(self.config.json_filepath, extra_cmd) prefix = "{}MASTER".format(self.cluster_id) - asyncio.ensure_future(print_output(prefix, master.stdout)) + asyncio.create_task(print_output(prefix, master.stdout)) self.procs.append((prefix, master)) async def run_slaves(self): @@ -117,7 +117,7 @@ async def run_slaves(self): slave.ID in self.args.profile.split(","), ) prefix = "{}SLAVE_{}".format(self.cluster_id, slave.ID) - asyncio.ensure_future(print_output(prefix, s.stdout)) + asyncio.create_task(print_output(prefix, s.stdout)) self.procs.append((prefix, s)) async def run_prom(self): @@ -149,10 +149,10 @@ async def shutdown(self): def start_and_loop(self): try: - asyncio.get_event_loop().run_until_complete(self.run()) + asyncio.run(self.run()) except KeyboardInterrupt: try: - asyncio.get_event_loop().run_until_complete(self.shutdown()) + asyncio.run(self.shutdown()) except Exception: pass diff --git a/quarkchain/cluster/jsonrpc.py b/quarkchain/cluster/jsonrpc.py index 8106fed1f..5ee107ad0 100644 --- a/quarkchain/cluster/jsonrpc.py +++ b/quarkchain/cluster/jsonrpc.py @@ -470,7 +470,7 @@ def _parse_log_request( # noinspection PyPep8Naming class JSONRPCHttpServer: @classmethod - def start_public_server(cls, env, master_server): + async def start_public_server(cls, env, master_server): server = cls( env, master_server, @@ -478,11 +478,11 @@ def start_public_server(cls, env, master_server): env.cluster_config.JSON_RPC_HOST, public_methods, ) - server.start() + await server.start() return server @classmethod - def start_private_server(cls, env, master_server): + async def start_private_server(cls, env, master_server): server = cls( env, master_server, @@ -490,11 +490,11 @@ def start_private_server(cls, env, master_server): env.cluster_config.PRIVATE_JSON_RPC_HOST, private_methods, ) - server.start() + await server.start() return server @classmethod - def start_test_server(cls, env, master_server): + async def start_test_server(cls, env, master_server): methods = AsyncMethods() for method in public_methods.values(): methods.add(method) @@ -507,7 +507,7 @@ def start_test_server(cls, env, master_server): env.cluster_config.JSON_RPC_HOST, methods, ) - server.start() + await server.start() return server def __init__( @@ -549,7 +549,7 @@ async def __handle(self, request): return web.Response() return web.json_response(response, status=response.http_status) - def start(self): + async def start(self): app = web.Application(client_max_size=JSON_RPC_CLIENT_REQUEST_MAX_SIZE) cors = aiohttp_cors.setup(app) route = app.router.add_post("/", self.__handle) @@ -565,12 +565,12 @@ def start(self): }, ) self.runner = web.AppRunner(app, access_log=None) - self.loop.run_until_complete(self.runner.setup()) + await self.runner.setup() site = web.TCPSite(self.runner, self.host, self.port) - self.loop.run_until_complete(site.start()) + await site.start() - def shutdown(self): - self.loop.run_until_complete(self.runner.cleanup()) + async def shutdown(self): + await self.runner.cleanup() # JSON RPC handlers @public_methods.add @@ -1452,7 +1452,7 @@ def get_data_default(key, decoder, default=None): class JSONRPCWebsocketServer: @classmethod - def start_websocket_server(cls, env, slave_server): + async def start_websocket_server(cls, env, slave_server): server = cls( env, slave_server, @@ -1460,7 +1460,7 @@ def start_websocket_server(cls, env, slave_server): env.slave_config.HOST, public_methods, ) - server.start() + await server.start() return server def __init__( @@ -1531,11 +1531,11 @@ async def __handle(self, websocket, path): except: pass - def start(self): + async def start(self): start_server = websockets.serve(self.__handle, self.host, self.port) - self.loop.run_until_complete(start_server) + await start_server - def shutdown(self): + async def shutdown(self): pass # TODO @staticmethod diff --git a/quarkchain/cluster/master.py b/quarkchain/cluster/master.py index e4ce6527b..bbc21bc22 100644 --- a/quarkchain/cluster/master.py +++ b/quarkchain/cluster/master.py @@ -456,7 +456,7 @@ def __init__( self.full_shard_id_list = full_shard_id_list check(len(full_shard_id_list) > 0) - asyncio.ensure_future(self.active_and_loop_forever()) + self._loop_task = asyncio.create_task(self.active_and_loop_forever()) def get_connection_to_forward(self, metadata): """Override ProxyConnection.get_connection_to_forward() @@ -763,7 +763,7 @@ class MasterServer: """ def __init__(self, env, root_state, name="master"): - self.loop = asyncio.get_event_loop() + self.loop = asyncio.get_running_loop() self.env = env self.root_state = root_state # type: RootState self.network = None # will be set by network constructor @@ -849,7 +849,7 @@ async def __connect(self, host, port): while True: try: reader, writer = await asyncio.open_connection( - host, port, loop=self.loop + host, port ) break except Exception as e: @@ -1068,29 +1068,31 @@ async def __init_cluster(self): self.cluster_active_future.set_result(None) def start(self): - self.loop.create_task(self.__init_cluster()) + self._init_task = self.loop.create_task(self.__init_cluster()) - def do_loop(self, callbacks: List[Callable]): + async def do_loop(self, callbacks: List[Callable]): if self.env.arguments.enable_profiler: profile = cProfile.Profile() profile.enable() try: - self.loop.run_until_complete(self.shutdown_future) + await self.shutdown_future except KeyboardInterrupt: pass finally: for callback in callbacks: if callable(callback): - callback() + result = callback() + if asyncio.iscoroutine(result): + await result if self.env.arguments.enable_profiler: profile.disable() profile.print_stats("time") - def wait_until_cluster_active(self): + async def wait_until_cluster_active(self): # Wait until cluster is ready - self.loop.run_until_complete(self.cluster_active_future) + await self.cluster_active_future def shutdown(self): # TODO: May set exception and disconnect all slaves @@ -1100,6 +1102,8 @@ def shutdown(self): self.cluster_active_future.set_exception( RuntimeError("failed to start the cluster") ) + if hasattr(self, '_init_task') and self._init_task and not self._init_task.done(): + self._init_task.cancel() def get_shutdown_future(self): return self.shutdown_future @@ -1848,21 +1852,17 @@ def parse_args(): return env -def main(): +async def _main_async(env): from quarkchain.cluster.jsonrpc import JSONRPCHttpServer - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - env = parse_args() - loop = asyncio.get_event_loop() root_state = RootState(env) master = MasterServer(env, root_state) if env.arguments.check_db: master.start() - master.wait_until_cluster_active() - asyncio.ensure_future(master.check_db()) - master.do_loop([]) + await master.wait_until_cluster_active() + asyncio.create_task(master.check_db()) + await master.do_loop([]) return # p2p discovery mode will disable master-slave communication and JSONRPC @@ -1875,31 +1875,38 @@ def main(): # only start the cluster if not in discovery-only mode if start_master: master.start() - master.wait_until_cluster_active() + await master.wait_until_cluster_active() # kick off simulated mining if enabled if env.cluster_config.START_SIMULATED_MINING: - asyncio.ensure_future(master.start_mining()) + asyncio.create_task(master.start_mining()) if env.cluster_config.use_p2p(): - network = P2PManager(env, master, loop) + network = P2PManager(env, master) else: - network = SimpleNetwork(env, master, loop) - network.start() + network = SimpleNetwork(env, master) + await network.start() callbacks = [network.shutdown] if env.cluster_config.ENABLE_PUBLIC_JSON_RPC: - public_json_rpc_server = JSONRPCHttpServer.start_public_server(env, master) + public_json_rpc_server = await JSONRPCHttpServer.start_public_server(env, master) callbacks.append(public_json_rpc_server.shutdown) if env.cluster_config.ENABLE_PRIVATE_JSON_RPC: - private_json_rpc_server = JSONRPCHttpServer.start_private_server(env, master) + private_json_rpc_server = await JSONRPCHttpServer.start_private_server(env, master) callbacks.append(private_json_rpc_server.shutdown) - master.do_loop(callbacks) + await master.do_loop(callbacks) Logger.info("Master server is shutdown") +def main(): + os.chdir(os.path.dirname(os.path.abspath(__file__))) + + env = parse_args() + asyncio.run(_main_async(env)) + + if __name__ == "__main__": main() diff --git a/quarkchain/cluster/miner.py b/quarkchain/cluster/miner.py index a7924846c..4a4c0d85a 100644 --- a/quarkchain/cluster/miner.py +++ b/quarkchain/cluster/miner.py @@ -187,10 +187,11 @@ def __init__( # key can be None, meaning default coinbase address from local config self.current_works = LRUCache(128) self.root_signer_private_key = root_signer_private_key + self._mining_task = None def start(self): self.enabled = True - self._mine_new_block_async() + self._mining_task = self._mine_new_block_async() def is_enabled(self): return self.enabled @@ -201,6 +202,9 @@ def disable(self): # end the mining process self.input_q.put((None, {})) self.enabled = False + if self._mining_task and not self._mining_task.done(): + self._mining_task.cancel() + self._mining_task = None def _mine_new_block_async(self): async def handle_mined_block(): @@ -266,7 +270,7 @@ async def mine_new_block(): # no-op if enabled or mining remotely if not self.enabled or self.remote: return None - return asyncio.ensure_future(mine_new_block()) + return asyncio.create_task(mine_new_block()) async def get_work(self, coinbase_addr: Address, now=None) -> (MiningWork, Block): if not self.remote: diff --git a/quarkchain/cluster/protocol.py b/quarkchain/cluster/protocol.py index f5670b868..7282ddc4a 100644 --- a/quarkchain/cluster/protocol.py +++ b/quarkchain/cluster/protocol.py @@ -22,7 +22,6 @@ def __init__( op_ser_map, op_non_rpc_map, op_rpc_map, - loop=None, metadata_class=None, name=None, command_size_limit=None, @@ -34,7 +33,6 @@ def __init__( op_ser_map, op_non_rpc_map, op_rpc_map, - loop=loop, metadata_class=metadata_class, name=name, command_size_limit=command_size_limit, @@ -110,12 +108,11 @@ def __init__( op_ser_map, op_non_rpc_map, op_rpc_map, - loop=None, metadata_class=Metadata, name=None, ): super().__init__( - op_ser_map, op_non_rpc_map, op_rpc_map, loop, metadata_class, name=name + op_ser_map, op_non_rpc_map, op_rpc_map, metadata_class, name=name ) self.read_deque = deque() self.read_event = asyncio.Event() @@ -147,7 +144,7 @@ def get_metadata_to_write(self, metadata): class NullConnection(AbstractConnection): def __init__(self): - super().__init__(dict(), dict(), dict(), None, Metadata, name="NULL_CONNECTION") + super().__init__(dict(), dict(), dict(), name="NULL_CONNECTION") def write_raw_data(self, metadata, raw_data): pass @@ -192,8 +189,7 @@ def __init__( op_ser_map, op_non_rpc_map, op_rpc_map, - loop=None, - metadata_class=None, + name=None, command_size_limit=None, ): super().__init__( @@ -203,9 +199,8 @@ def __init__( op_ser_map, op_non_rpc_map, op_rpc_map, - loop, - P2PMetadata, - name=metadata_class, + metadata_class=P2PMetadata, + name=name, command_size_limit=command_size_limit, ) @@ -233,7 +228,6 @@ def __init__( op_ser_map, op_non_rpc_map, op_rpc_map, - loop=None, name=None, ): super().__init__( @@ -243,8 +237,7 @@ def __init__( op_ser_map, op_non_rpc_map, op_rpc_map, - loop, - ClusterMetadata, + metadata_class=ClusterMetadata, name=name, ) self.peer_rpc_ids = dict() diff --git a/quarkchain/cluster/root_state.py b/quarkchain/cluster/root_state.py index 84f217aac..f1a3a3732 100644 --- a/quarkchain/cluster/root_state.py +++ b/quarkchain/cluster/root_state.py @@ -610,7 +610,7 @@ def add_block( "propagation_latency_ms": start_ms - tracking_data.get("mined", 0), "num_tx": len(block.minor_block_header_list), } - asyncio.ensure_future( + asyncio.create_task( self.env.cluster_config.kafka_logger.log_kafka_sample_async( self.env.cluster_config.MONITORING.PROPAGATION_TOPIC, sample ) diff --git a/quarkchain/cluster/shard.py b/quarkchain/cluster/shard.py index 429095156..731def59f 100644 --- a/quarkchain/cluster/shard.py +++ b/quarkchain/cluster/shard.py @@ -394,7 +394,7 @@ async def __run_sync(self, notify_sync: Callable): await self.shard.add_block(block) if counter % 100 == 0: sync_data = (block.header.height, block_header_chain[-1]) - asyncio.ensure_future(notify_sync(sync_data)) + asyncio.create_task(notify_sync(sync_data)) counter = 0 counter += 1 block_header_chain.pop(0) @@ -507,7 +507,7 @@ def __init__(self, env, full_shard_id, slave): self.state = ShardState(env, full_shard_id, self.__init_shard_db()) - self.loop = asyncio.get_event_loop() + self.loop = asyncio.get_running_loop() self.synchronizer = Synchronizer( self.state.subscription_manager.notify_sync, lambda: self.state.header_tip ) @@ -593,9 +593,9 @@ async def create_peer_shard_connections(self, cluster_peer_ids, master_conn): shard=self, name="{}_vconn_{}".format(master_conn.name, cluster_peer_id), ) - asyncio.ensure_future(peer_shard_conn.active_and_loop_forever()) + peer_shard_conn._loop_task = asyncio.create_task(peer_shard_conn.active_and_loop_forever()) conns.append(peer_shard_conn) - await asyncio.gather(*[conn.active_future for conn in conns]) + await asyncio.gather(*[conn.active_event.wait() for conn in conns]) for conn in conns: self.add_peer(conn) diff --git a/quarkchain/cluster/shard_state.py b/quarkchain/cluster/shard_state.py index f62fb5ce7..45bc9a8be 100644 --- a/quarkchain/cluster/shard_state.py +++ b/quarkchain/cluster/shard_state.py @@ -579,7 +579,7 @@ def add_tx(self, tx: TypedTransaction, xshard_gas_limit=None): return False self.tx_queue.add_transaction(tx) - asyncio.ensure_future( + asyncio.create_task( self.subscription_manager.notify_new_pending_tx( [tx_hash + evm_tx.from_full_shard_key.to_bytes(4, byteorder="big")] ) @@ -860,18 +860,18 @@ def __rewrite_block_index_to(self, minor_block, add_tx_back_to_queue=True): if add_tx_back_to_queue: self.__add_transactions_from_block(block) if len(old_chain) > 0: - asyncio.ensure_future(self.subscription_manager.notify_log(old_chain, True)) + asyncio.create_task(self.subscription_manager.notify_log(old_chain, True)) for block in new_chain: self.db.put_transaction_index_from_block(block) self.db.put_minor_block_index(block) self.__remove_transactions_from_block(block) # new_chain has at least one block, starting from minor_block with block height descending - asyncio.ensure_future( + asyncio.create_task( self.subscription_manager.notify_new_heads( sorted(new_chain, key=lambda x: x.header.height) ) ) - asyncio.ensure_future(self.subscription_manager.notify_log(new_chain)) + asyncio.create_task(self.subscription_manager.notify_log(new_chain)) # will be called for chain reorganization def __add_transactions_from_block(self, block): @@ -883,7 +883,7 @@ def __add_transactions_from_block(self, block): tx_hashes.append( tx_hash + evm_tx.from_full_shard_key.to_bytes(4, byteorder="big") ) - asyncio.ensure_future( + asyncio.create_task( self.subscription_manager.notify_new_pending_tx(tx_hashes) ) @@ -1050,7 +1050,7 @@ def add_block( "propagation_latency_ms": start_ms - tracking_data.get("mined", 0), "num_tx": len(block.tx_list), } - asyncio.ensure_future( + asyncio.create_task( self.env.cluster_config.kafka_logger.log_kafka_sample_async( self.env.cluster_config.MONITORING.PROPAGATION_TOPIC, sample ) @@ -1392,7 +1392,7 @@ def add_cross_shard_tx_list_by_minor_block_hash( tx.tx_hash + tx.from_address.full_shard_key.to_bytes(4, byteorder="big") for tx in tx_list.tx_list ] - asyncio.ensure_future( + asyncio.create_task( self.subscription_manager.notify_new_pending_tx(tx_hashes) ) diff --git a/quarkchain/cluster/simple_network.py b/quarkchain/cluster/simple_network.py index aba4bd406..48bbca30a 100644 --- a/quarkchain/cluster/simple_network.py +++ b/quarkchain/cluster/simple_network.py @@ -128,7 +128,7 @@ async def start(self, is_server=False): "Established virtual shard connections with peer {}".format(self.id.hex()) ) - asyncio.ensure_future(self.active_and_loop_forever()) + self._loop_task = asyncio.create_task(self.active_and_loop_forever()) await self.wait_until_active() # Only make the peer connection avaialbe after exchanging HELLO and creating virtual shard connections @@ -383,7 +383,7 @@ class AbstractNetwork: cluster_peer_pool = None # type: Dict[int, Peer] @abstractmethod - def start(self) -> None: + async def start(self) -> None: """ start the network server and discovery on the provided loop """ @@ -408,8 +408,7 @@ class SimpleNetwork(AbstractNetwork): """Fully connected P2P network for inter-cluster communication """ - def __init__(self, env, master_server, loop): - self.loop = loop + def __init__(self, env, master_server): self.env = env self.active_peer_pool = dict() # peer id => peer self.self_id = random_bytes(32) @@ -421,6 +420,7 @@ def __init__(self, env, master_server, loop): # 0 is reserved for master self.next_cluster_peer_id = 0 self.cluster_peer_pool = dict() # cluster peer id => peer + self._seed_task = None async def new_peer(self, client_reader, client_writer): peer = Peer( @@ -436,7 +436,7 @@ async def new_peer(self, client_reader, client_writer): async def connect(self, ip, port): Logger.info("connecting {} {}".format(ip, port)) try: - reader, writer = await asyncio.open_connection(ip, port, loop=self.loop) + reader, writer = await asyncio.open_connection(ip, port) except Exception as e: Logger.info("failed to connect {} {}: {}".format(ip, port, e)) return None @@ -472,7 +472,7 @@ async def connect_seed(self, ip, port): Logger.info("connecting {} peers ...".format(len(resp.peer_info_list))) for peer_info in resp.peer_info_list: - asyncio.ensure_future( + asyncio.create_task( self.connect(str(ipaddress.ip_address(peer_info.ip)), peer_info.port) ) @@ -487,23 +487,26 @@ def shutdown_peers(self): for peer_id, peer in active_peer_pool.items(): peer.close() - def start_server(self): - coro = asyncio.start_server(self.new_peer, "0.0.0.0", self.port, loop=self.loop) - self.server = self.loop.run_until_complete(coro) + async def start_server(self): + self.server = await asyncio.start_server( + self.new_peer, "0.0.0.0", self.port + ) Logger.info("Self id {}".format(self.self_id.hex())) Logger.info( "Listening on {} for p2p".format(self.server.sockets[0].getsockname()) ) - def shutdown(self): + async def shutdown(self): self.shutdown_peers() + if self._seed_task and not self._seed_task.done(): + self._seed_task.cancel() self.server.close() - self.loop.run_until_complete(self.server.wait_closed()) + await self.server.wait_closed() - def start(self): - self.start_server() + async def start(self): + await self.start_server() - self.loop.create_task( + self._seed_task = asyncio.create_task( self.connect_seed( self.env.cluster_config.SIMPLE_NETWORK.BOOTSTRAP_HOST, self.env.cluster_config.SIMPLE_NETWORK.BOOTSTRAP_PORT, diff --git a/quarkchain/cluster/slave.py b/quarkchain/cluster/slave.py index 81c570b41..ad597dd07 100644 --- a/quarkchain/cluster/slave.py +++ b/quarkchain/cluster/slave.py @@ -103,12 +103,12 @@ def __init__(self, env, reader, writer, slave_server, name=None): MASTER_OP_RPC_MAP, name=name, ) - self.loop = asyncio.get_event_loop() + self.loop = asyncio.get_running_loop() self.env = env self.slave_server = slave_server # type: SlaveServer self.shards = slave_server.shards # type: Dict[Branch, Shard] - asyncio.ensure_future(self.active_and_loop_forever()) + self._loop_task = asyncio.create_task(self.active_and_loop_forever()) # cluster_peer_id -> {branch_value -> shard_conn} self.v_conn_map = dict() @@ -346,8 +346,8 @@ async def handle_create_cluster_peer_connection_request(self, req): shard=shard, name="{}_vconn_{}".format(self.name, req.cluster_peer_id), ) - asyncio.ensure_future(peer_shard_conn.active_and_loop_forever()) - active_futures.append(peer_shard_conn.active_future) + peer_shard_conn._loop_task = asyncio.create_task(peer_shard_conn.active_and_loop_forever()) + active_futures.append(peer_shard_conn.active_event.wait()) shard_to_conn[shard] = peer_shard_conn # wait for all the connections to become active before return @@ -723,12 +723,12 @@ def __init__( self.full_shard_id_list = full_shard_id_list self.shards = self.slave_server.shards - self.ping_received_future = asyncio.get_event_loop().create_future() + self.ping_received_event = asyncio.Event() - asyncio.ensure_future(self.active_and_loop_forever()) + self._loop_task = asyncio.create_task(self.active_and_loop_forever()) async def wait_until_ping_received(self): - await self.ping_received_future + await self.ping_received_event.wait() def close_with_error(self, error): Logger.info("Closing connection with slave {}".format(self.id)) @@ -756,7 +756,7 @@ async def handle_ping(self, ping: Ping): "Empty shard mask list from slave {}".format(self.id) ) - self.ping_received_future.set_result(None) + self.ping_received_event.set() return Pong(self.slave_server.id, self.slave_server.full_shard_id_list) @@ -808,7 +808,7 @@ def __init__(self, env, slave_server): self.full_shard_id_to_slaves[full_shard_id] = [] self.slave_connections = set() self.slave_ids = set() # set(bytes) - self.loop = asyncio.get_event_loop() + self.loop = asyncio.get_running_loop() def close_all(self): for conn in self.slave_connections: @@ -850,7 +850,7 @@ async def connect_to_slave(self, slave_info: SlaveInfo) -> str: host = slave_info.host.decode("ascii") port = slave_info.port try: - reader, writer = await asyncio.open_connection(host, port, loop=self.loop) + reader, writer = await asyncio.open_connection(host, port) except Exception as e: err_msg = "Failed to connect {}:{} with exception {}".format(host, port, e) Logger.info(err_msg) @@ -887,7 +887,7 @@ class SlaveServer: """ Slave node in a cluster """ def __init__(self, env, name="slave"): - self.loop = asyncio.get_event_loop() + self.loop = asyncio.get_running_loop() self.env = env self.id = bytes(self.env.slave_config.ID, "ascii") self.full_shard_id_list = self.env.slave_config.FULL_SHARD_ID_LIST @@ -991,7 +991,6 @@ async def __start_server(self): self.__handle_new_connection, "0.0.0.0", self.env.slave_config.PORT, - loop=self.loop, ) Logger.info( "Listening on {} for intra-cluster RPC".format( @@ -1000,11 +999,11 @@ async def __start_server(self): ) def start(self): - self.loop.create_task(self.__start_server()) + self._server_task = self.loop.create_task(self.__start_server()) - def do_loop(self): + async def do_loop(self): try: - self.loop.run_until_complete(self.shutdown_future) + await self.shutdown_future except KeyboardInterrupt: pass @@ -1464,15 +1463,9 @@ def parse_args(): return env -def main(): +async def _main_async(env): from quarkchain.cluster.jsonrpc import JSONRPCWebsocketServer - os.chdir(os.path.dirname(os.path.abspath(__file__))) - env = parse_args() - - if env.arguments.enable_profiler: - profile = cProfile.Profile() - profile.enable() slave_server = SlaveServer(env) slave_server.start() @@ -1483,13 +1476,24 @@ def main(): ) callbacks.append(json_rpc_websocket_server.shutdown) - slave_server.do_loop() + await slave_server.do_loop() + Logger.info("Slave server is shutdown") + + +def main(): + os.chdir(os.path.dirname(os.path.abspath(__file__))) + env = parse_args() + + if env.arguments.enable_profiler: + profile = cProfile.Profile() + profile.enable() + + asyncio.run(_main_async(env)) + if env.arguments.enable_profiler: profile.disable() profile.print_stats("time") - Logger.info("Slave server is shutdown") - if __name__ == "__main__": main() diff --git a/quarkchain/cluster/subscription.py b/quarkchain/cluster/subscription.py index 11b6c84a7..545be6452 100644 --- a/quarkchain/cluster/subscription.py +++ b/quarkchain/cluster/subscription.py @@ -2,7 +2,7 @@ import json from typing import List, Dict, Tuple, Optional, Callable -from jsonrpcserver.exceptions import InvalidParams +from quarkchain.cluster.jsonrpcserver import InvalidParams from websockets import WebSocketServerProtocol from quarkchain.core import MinorBlock @@ -89,7 +89,7 @@ async def notify_sync(self, data: Optional[Tuple[int, ...]] = None): } for sub_id, websocket in self.subscribers[SUB_SYNC].items(): response = self.response_encoder(sub_id, result) - asyncio.ensure_future(websocket.send(json.dumps(response))) + asyncio.create_task(websocket.send(json.dumps(response))) @staticmethod def response_encoder(sub_id, result): diff --git a/quarkchain/cluster/tests/conftest.py b/quarkchain/cluster/tests/conftest.py new file mode 100644 index 000000000..82c806e19 --- /dev/null +++ b/quarkchain/cluster/tests/conftest.py @@ -0,0 +1,20 @@ +import asyncio + +import pytest + +from quarkchain.protocol import AbstractConnection + + +@pytest.fixture(autouse=True) +def ensure_event_loop(): + """Ensure an event loop exists after each test. + IsolatedAsyncioTestCase tears down its loop and sets the current loop to None, + which breaks subsequent sync tests that call asyncio.get_event_loop().""" + yield + AbstractConnection.aborted_rpc_count = 0 + try: + old_loop = asyncio.get_event_loop() + if old_loop.is_closed(): + asyncio.set_event_loop(asyncio.new_event_loop()) + except RuntimeError: + asyncio.set_event_loop(asyncio.new_event_loop()) diff --git a/quarkchain/cluster/tests/test_cluster.py b/quarkchain/cluster/tests/test_cluster.py index eb76458e0..8c747529a 100644 --- a/quarkchain/cluster/tests/test_cluster.py +++ b/quarkchain/cluster/tests/test_cluster.py @@ -1,3 +1,4 @@ +import asyncio import unittest from eth_keys.datatypes import PrivateKey @@ -25,8 +26,7 @@ ) from quarkchain.evm import opcodes from quarkchain.utils import ( - call_async, - assert_true_with_timeout, + async_assert_true_with_timeout, sha3_256, token_id_encode, ) @@ -49,23 +49,23 @@ def _tip_gen(shard_state): return b -class TestCluster(unittest.TestCase): - def test_single_cluster(self): +class TestCluster(unittest.IsolatedAsyncioTestCase): + async def test_single_cluster(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(1, acc1) as clusters: + async with ClusterContext(1, acc1) as clusters: self.assertEqual(len(clusters), 1) - def test_three_clusters(self): - with ClusterContext(3) as clusters: + async def test_three_clusters(self): + async with ClusterContext(3) as clusters: self.assertEqual(len(clusters), 3) - def test_create_shard_at_different_height(self): + async def test_create_shard_at_different_height(self): acc1 = Address.create_random_account(0) id1 = 0 << 16 | 1 | 0 id2 = 1 << 16 | 1 | 0 genesis_root_heights = {id1: 1, id2: 2} - with ClusterContext( + async with ClusterContext( 1, acc1, chain_size=2, @@ -78,7 +78,7 @@ def test_create_shard_at_different_height(self): self.assertIsNone(clusters[0].get_shard(id2)) # Add root block with height 1, which will automatically create genesis block for shard 0 - root0 = call_async(master.get_next_block_to_mine(acc1, branch_value=None)) + root0 = (await master.get_next_block_to_mine(acc1, branch_value=None)) self.assertEqual(root0.header.height, 1) self.assertEqual(len(root0.minor_block_header_list), 0) self.assertEqual( @@ -87,7 +87,7 @@ def test_create_shard_at_different_height(self): ], master.env.quark_chain_config.ROOT.COINBASE_AMOUNT, ) - call_async(master.add_root_block(root0)) + (await master.add_root_block(root0)) # shard 0 created at root height 1 self.assertIsNotNone(clusters[0].get_shard(id1)) @@ -110,7 +110,7 @@ def test_create_shard_at_different_height(self): ) # Add root block with height 2, which will automatically create genesis block for shard 1 - root1 = call_async(master.get_next_block_to_mine(acc1, branch_value=None)) + root1 = (await master.get_next_block_to_mine(acc1, branch_value=None)) self.assertEqual(len(root1.minor_block_header_list), 1) self.assertEqual( root1.header.coinbase_amount_map.balance_map[ @@ -122,7 +122,7 @@ def test_create_shard_at_different_height(self): ], ) self.assertEqual(root1.minor_block_header_list[0], shard_state.header_tip) - call_async(master.add_root_block(root1)) + (await master.add_root_block(root1)) self.assertIsNotNone(clusters[0].get_shard(id1)) # shard 1 created at root height 2 @@ -134,7 +134,7 @@ def test_create_shard_at_different_height(self): mblock.meta.xshard_tx_cursor_info, XshardTxCursorInfo(root1.header.height + 1, 0, 0), ) - call_async(clusters[0].get_shard(id1).add_block(mblock)) + (await clusters[0].get_shard(id1).add_block(mblock)) self.assertEqual( shard_state.get_token_balance( acc1.recipient, shard_state.env.quark_chain_config.genesis_token @@ -157,20 +157,20 @@ def test_create_shard_at_different_height(self): # Add root block with height 3, which will include # - the genesis block for shard 1; and # - the added block for shard 0. - root2 = call_async(master.get_next_block_to_mine(acc1, branch_value=None)) + root2 = (await master.get_next_block_to_mine(acc1, branch_value=None)) self.assertEqual(len(root2.minor_block_header_list), 2) - def test_get_primary_account_data(self): + async def test_get_primary_account_data(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_random_account(full_shard_key=1) - with ClusterContext(1, acc1) as clusters: + async with ClusterContext(1, acc1) as clusters: master = clusters[0].master slaves = clusters[0].slave_list self.assertEqual( - call_async(master.get_primary_account_data(acc1)).transaction_count, 0 + (await master.get_primary_account_data(acc1)).transaction_count, 0 ) tx = create_transfer_transaction( @@ -182,37 +182,37 @@ def test_get_primary_account_data(self): ) self.assertTrue(slaves[0].add_tx(tx)) - root = call_async( + root = (await master.get_next_block_to_mine(address=acc1, branch_value=None) ) - call_async(master.add_root_block(root)) + (await master.add_root_block(root)) - block1 = call_async( + block1 = (await master.get_next_block_to_mine(address=acc1, branch_value=0b10) ) self.assertTrue( - call_async( + (await master.add_raw_minor_block(block1.header.branch, block1.serialize()) ) ) self.assertEqual( - call_async(master.get_primary_account_data(acc1)).transaction_count, 1 + (await master.get_primary_account_data(acc1)).transaction_count, 1 ) self.assertEqual( - call_async(master.get_primary_account_data(acc2)).transaction_count, 0 + (await master.get_primary_account_data(acc2)).transaction_count, 0 ) - def test_add_transaction(self): + async def test_add_transaction(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id1, full_shard_key=1) - with ClusterContext(2, acc1, should_set_gas_price_limit=True) as clusters: + async with ClusterContext(2, acc1, should_set_gas_price_limit=True) as clusters: master = clusters[0].master - root = call_async(master.get_next_block_to_mine(acc1, branch_value=None)) - call_async(master.add_root_block(root)) + root = (await master.get_next_block_to_mine(acc1, branch_value=None)) + (await master.add_root_block(root)) # tx with gas price price lower than required (10 wei) should be rejected tx0 = create_transfer_transaction( @@ -223,7 +223,7 @@ def test_add_transaction(self): value=0, gas_price=9, ) - self.assertFalse(call_async(master.add_transaction(tx0))) + self.assertFalse((await master.add_transaction(tx0))) tx1 = create_transfer_transaction( shard_state=clusters[0].get_shard_state(0b10), @@ -233,7 +233,7 @@ def test_add_transaction(self): value=12345, gas_price=10, ) - self.assertTrue(call_async(master.add_transaction(tx1))) + self.assertTrue((await master.add_transaction(tx1))) self.assertEqual(len(clusters[0].get_shard_state(0b10).tx_queue), 1) tx2 = create_transfer_transaction( @@ -245,13 +245,13 @@ def test_add_transaction(self): gas=30000, gas_price=10, ) - self.assertTrue(call_async(master.add_transaction(tx2))) + self.assertTrue((await master.add_transaction(tx2))) self.assertEqual(len(clusters[0].get_shard_state(0b11).tx_queue), 1) # check the tx is received by the other cluster state0 = clusters[1].get_shard_state(0b10) tx_queue, expect_evm_tx1 = state0.tx_queue, tx1.tx.to_evm_tx() - assert_true_with_timeout(lambda: len(tx_queue) == 1) + await async_assert_true_with_timeout(lambda: len(tx_queue) == 1) actual_evm_tx = tx_queue.pop_transaction( state0.get_transaction_count ).tx.to_evm_tx() @@ -259,22 +259,22 @@ def test_add_transaction(self): state1 = clusters[1].get_shard_state(0b11) tx_queue, expect_evm_tx2 = state1.tx_queue, tx2.tx.to_evm_tx() - assert_true_with_timeout(lambda: len(tx_queue) == 1) + await async_assert_true_with_timeout(lambda: len(tx_queue) == 1) actual_evm_tx = tx_queue.pop_transaction( state1.get_transaction_count ).tx.to_evm_tx() self.assertEqual(actual_evm_tx, expect_evm_tx2) - def test_add_transaction_with_invalid_mnt(self): + async def test_add_transaction_with_invalid_mnt(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id1, full_shard_key=1) - with ClusterContext(2, acc1, should_set_gas_price_limit=True) as clusters: + async with ClusterContext(2, acc1, should_set_gas_price_limit=True) as clusters: master = clusters[0].master - root = call_async(master.get_next_block_to_mine(acc1, branch_value=None)) - call_async(master.add_root_block(root)) + root = (await master.get_next_block_to_mine(acc1, branch_value=None)) + (await master.add_root_block(root)) tx1 = create_transfer_transaction( shard_state=clusters[0].get_shard_state(0b10), @@ -285,7 +285,7 @@ def test_add_transaction_with_invalid_mnt(self): gas_price=10, gas_token_id=1, ) - self.assertFalse(call_async(master.add_transaction(tx1))) + self.assertFalse((await master.add_transaction(tx1))) tx2 = create_transfer_transaction( shard_state=clusters[0].get_shard_state(0b11), @@ -297,18 +297,18 @@ def test_add_transaction_with_invalid_mnt(self): gas_price=10, gas_token_id=1, ) - self.assertFalse(call_async(master.add_transaction(tx2))) + self.assertFalse((await master.add_transaction(tx2))) @mock_pay_native_token_as_gas(lambda *x: (50, x[-1] // 5)) - def test_add_transaction_with_valid_mnt(self): + async def test_add_transaction_with_valid_mnt(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(2, acc1, should_set_gas_price_limit=True) as clusters: + async with ClusterContext(2, acc1, should_set_gas_price_limit=True) as clusters: master = clusters[0].master - root = call_async(master.get_next_block_to_mine(acc1, branch_value=None)) - call_async(master.add_root_block(root)) + root = (await master.get_next_block_to_mine(acc1, branch_value=None)) + (await master.add_root_block(root)) # gasprice will be 9, which is smaller than 10 as required. tx0 = create_transfer_transaction( @@ -320,7 +320,7 @@ def test_add_transaction_with_valid_mnt(self): gas_price=49, gas_token_id=1, ) - self.assertFalse(call_async(master.add_transaction(tx0))) + self.assertFalse((await master.add_transaction(tx0))) # gasprice will be 10, but the balance will be insufficient. tx1 = create_transfer_transaction( @@ -332,7 +332,7 @@ def test_add_transaction_with_valid_mnt(self): gas_price=50, gas_token_id=1, ) - self.assertFalse(call_async(master.add_transaction(tx1))) + self.assertFalse((await master.add_transaction(tx1))) tx2 = create_transfer_transaction( shard_state=clusters[0].get_shard_state(0b10), @@ -344,23 +344,23 @@ def test_add_transaction_with_valid_mnt(self): gas_token_id=1, nonce=5, ) - self.assertTrue(call_async(master.add_transaction(tx2))) + self.assertTrue((await master.add_transaction(tx2))) # check the tx is received by the other cluster state1 = clusters[1].get_shard_state(0b10) tx_queue, expect_evm_tx2 = state1.tx_queue, tx2.tx.to_evm_tx() - assert_true_with_timeout(lambda: len(tx_queue) == 1) + await async_assert_true_with_timeout(lambda: len(tx_queue) == 1) actual_evm_tx = tx_queue.peek()[0].tx.tx.to_evm_tx() self.assertEqual(actual_evm_tx, expect_evm_tx2) - def test_add_minor_block_request_list(self): + async def test_add_minor_block_request_list(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(2, acc1) as clusters: + async with ClusterContext(2, acc1) as clusters: shard_state = clusters[0].get_shard_state(0b10) b1 = _tip_gen(shard_state) - add_result = call_async( + add_result = (await clusters[0].master.add_raw_minor_block(b1.header.branch, b1.serialize()) ) self.assertTrue(add_result) @@ -378,22 +378,22 @@ def test_add_minor_block_request_list(self): ) # Make sure another cluster received the new block - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: clusters[0] .get_shard_state(0b10) .contain_block_by_hash(b1.header.get_hash()) ) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: clusters[1].master.root_state.db.contain_minor_block_by_hash( b1.header.get_hash() ) ) - def test_add_root_block_request_list(self): + async def test_add_root_block_request_list(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(2, acc1) as clusters: + async with ClusterContext(2, acc1) as clusters: # shutdown cluster connection clusters[1].peer.close() @@ -402,7 +402,7 @@ def test_add_root_block_request_list(self): shard_state0 = clusters[0].get_shard_state(0b10) for i in range(7): b1 = _tip_gen(shard_state0) - add_result = call_async( + add_result = (await clusters[0].master.add_raw_minor_block( b1.header.branch, b1.serialize() ) @@ -413,7 +413,7 @@ def test_add_root_block_request_list(self): block_header_list.append(clusters[0].get_shard_state(2 | 1).header_tip) shard_state0 = clusters[0].get_shard_state(0b11) b2 = _tip_gen(shard_state0) - add_result = call_async( + add_result = (await clusters[0].master.add_raw_minor_block(b2.header.branch, b2.serialize()) ) self.assertTrue(add_result) @@ -422,7 +422,7 @@ def test_add_root_block_request_list(self): # add 1 block in cluster 1 shard_state1 = clusters[1].get_shard_state(0b11) b3 = _tip_gen(shard_state1) - add_result = call_async( + add_result = (await clusters[1].master.add_raw_minor_block(b3.header.branch, b3.serialize()) ) self.assertTrue(add_result) @@ -430,7 +430,7 @@ def test_add_root_block_request_list(self): self.assertEqual(clusters[1].get_shard_state(0b11).header_tip, b3.header) # reestablish cluster connection - call_async( + (await clusters[1].network.connect( "127.0.0.1", clusters[0].master.env.cluster_config.SIMPLE_NETWORK.BOOTSTRAP_PORT, @@ -440,32 +440,32 @@ def test_add_root_block_request_list(self): root_block1 = clusters[0].master.root_state.create_block_to_mine( block_header_list, acc1 ) - call_async(clusters[0].master.add_root_block(root_block1)) + (await clusters[0].master.add_root_block(root_block1)) # Make sure the root block tip of local cluster is changed self.assertEqual(clusters[0].master.root_state.tip, root_block1.header) # Make sure the root block tip of cluster 1 is changed - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: clusters[1].master.root_state.tip == root_block1.header, 2 ) # Minor block is downloaded self.assertEqual(b1.header.height, 7) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: clusters[1].get_shard_state(0b10).header_tip == b1.header ) # The tip is overwritten due to root chain first consensus - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: clusters[1].get_shard_state(0b11).header_tip == b2.header ) - def test_shard_synchronizer_with_fork(self): + async def test_shard_synchronizer_with_fork(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(2, acc1) as clusters: + async with ClusterContext(2, acc1) as clusters: # shutdown cluster connection clusters[1].peer.close() @@ -474,7 +474,7 @@ def test_shard_synchronizer_with_fork(self): shard_state0 = clusters[0].get_shard_state(0b10) for i in range(13): block = _tip_gen(shard_state0) - add_result = call_async( + add_result = (await clusters[0].master.add_raw_minor_block( block.header.branch, block.serialize() ) @@ -487,7 +487,7 @@ def test_shard_synchronizer_with_fork(self): shard_state0 = clusters[1].get_shard_state(0b10) for i in range(12): block = _tip_gen(shard_state0) - add_result = call_async( + add_result = (await clusters[1].master.add_raw_minor_block( block.header.branch, block.serialize() ) @@ -496,7 +496,7 @@ def test_shard_synchronizer_with_fork(self): self.assertEqual(clusters[1].get_shard_state(0b10).header_tip.height, 12) # reestablish cluster connection - call_async( + (await clusters[1].network.connect( "127.0.0.1", clusters[0].master.env.cluster_config.SIMPLE_NETWORK.BOOTSTRAP_PORT, @@ -506,7 +506,7 @@ def test_shard_synchronizer_with_fork(self): # a new block from cluster 0 will trigger sync in cluster 1 shard_state0 = clusters[0].get_shard_state(0b10) block = _tip_gen(shard_state0) - add_result = call_async( + add_result = (await clusters[0].master.add_raw_minor_block( block.header.branch, block.serialize() ) @@ -517,13 +517,13 @@ def test_shard_synchronizer_with_fork(self): # expect cluster 1 has all the blocks from cluster 0 and # has the same tip as cluster 0 for block in block_list: - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: clusters[1] .slave_list[0] .shards[Branch(0b10)] .state.contain_block_by_hash(block.header.get_hash()) ) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: clusters[ 1 ].master.root_state.db.contain_minor_block_by_hash( @@ -536,13 +536,13 @@ def test_shard_synchronizer_with_fork(self): clusters[0].get_shard_state(0b10).header_tip, ) - def test_shard_genesis_fork_fork(self): + async def test_shard_genesis_fork_fork(self): """ Test shard forks at genesis blocks due to root chain fork at GENESIS.ROOT_HEIGHT""" acc1 = Address.create_random_account(0) acc2 = Address.create_random_account(1) genesis_root_heights = {2: 0, 3: 1} - with ClusterContext( + async with ClusterContext( 2, acc1, chain_size=1, @@ -553,8 +553,8 @@ def test_shard_genesis_fork_fork(self): clusters[1].peer.close() master0 = clusters[0].master - root0 = call_async(master0.get_next_block_to_mine(acc1, branch_value=None)) - call_async(master0.add_root_block(root0)) + root0 = (await master0.get_next_block_to_mine(acc1, branch_value=None)) + (await master0.add_root_block(root0)) genesis0 = ( clusters[0].get_shard_state(2 | 1).db.get_minor_block_by_height(0) ) @@ -563,9 +563,9 @@ def test_shard_genesis_fork_fork(self): ) master1 = clusters[1].master - root1 = call_async(master1.get_next_block_to_mine(acc2, branch_value=None)) + root1 = (await master1.get_next_block_to_mine(acc2, branch_value=None)) self.assertNotEqual(root0.header.get_hash(), root1.header.get_hash()) - call_async(master1.add_root_block(root1)) + (await master1.add_root_block(root1)) genesis1 = ( clusters[1].get_shard_state(2 | 1).db.get_minor_block_by_height(0) ) @@ -576,19 +576,19 @@ def test_shard_genesis_fork_fork(self): self.assertNotEqual(genesis0.header.get_hash(), genesis1.header.get_hash()) # let's make cluster1's root chain longer than cluster0's - root2 = call_async(master1.get_next_block_to_mine(acc2, branch_value=None)) - call_async(master1.add_root_block(root2)) + root2 = (await master1.get_next_block_to_mine(acc2, branch_value=None)) + (await master1.add_root_block(root2)) self.assertEqual(master1.root_state.tip.height, 2) # reestablish cluster connection - call_async( + (await clusters[1].network.connect( "127.0.0.1", clusters[0].master.env.cluster_config.SIMPLE_NETWORK.BOOTSTRAP_PORT, ) ) # Expect cluster0's genesis change to genesis1 - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: clusters[0] .get_shard_state(2 | 1) .db.get_minor_block_by_height(0) @@ -597,13 +597,13 @@ def test_shard_genesis_fork_fork(self): ) self.assertTrue(clusters[0].get_shard_state(2 | 1).root_tip == root2.header) - def test_broadcast_cross_shard_transactions(self): + async def test_broadcast_cross_shard_transactions(self): """ Test the cross shard transactions are broadcasted to the destination shards """ id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc3 = Address.create_random_account(full_shard_key=1) - with ClusterContext(1, acc1) as clusters: + async with ClusterContext(1, acc1) as clusters: master = clusters[0].master slaves = clusters[0].slave_list genesis_token = ( @@ -612,12 +612,12 @@ def test_broadcast_cross_shard_transactions(self): # Add a root block first so that later minor blocks referring to this root # can be broadcasted to other shards - root_block = call_async( + root_block = (await master.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) tx1 = create_transfer_transaction( shard_state=clusters[0].get_shard_state(2 | 0), @@ -634,7 +634,7 @@ def test_broadcast_cross_shard_transactions(self): b2.header.create_time += 1 self.assertNotEqual(b1.header.get_hash(), b2.header.get_hash()) - call_async(clusters[0].get_shard(2 | 0).add_block(b1)) + (await clusters[0].get_shard(2 | 0).add_block(b1)) # expect shard 1 got the CrossShardTransactionList of b1 xshard_tx_list = ( @@ -648,7 +648,7 @@ def test_broadcast_cross_shard_transactions(self): self.assertEqual(xshard_tx_list.tx_list[0].to_address, acc3) self.assertEqual(xshard_tx_list.tx_list[0].value, 54321) - call_async(clusters[0].get_shard(2 | 0).add_block(b2)) + (await clusters[0].get_shard(2 | 0).add_block(b2)) # b2 doesn't update tip self.assertEqual(clusters[0].get_shard_state(2 | 0).header_tip, b1.header) @@ -669,12 +669,12 @@ def test_broadcast_cross_shard_transactions(self): .get_shard_state(2 | 1) .create_block_to_mine(address=acc1.address_in_shard(1)) ) - call_async(master.add_raw_minor_block(b3.header.branch, b3.serialize())) + (await master.add_raw_minor_block(b3.header.branch, b3.serialize())) - root_block = call_async( + root_block = (await master.get_next_block_to_mine(address=acc1, branch_value=None) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) # b4 should include the withdraw of tx1 b4 = ( @@ -685,25 +685,25 @@ def test_broadcast_cross_shard_transactions(self): # adding b1, b2, b3 again shouldn't affect b4 to be added later self.assertTrue( - call_async(master.add_raw_minor_block(b1.header.branch, b1.serialize())) + (await master.add_raw_minor_block(b1.header.branch, b1.serialize())) ) self.assertTrue( - call_async(master.add_raw_minor_block(b2.header.branch, b2.serialize())) + (await master.add_raw_minor_block(b2.header.branch, b2.serialize())) ) self.assertTrue( - call_async(master.add_raw_minor_block(b3.header.branch, b3.serialize())) + (await master.add_raw_minor_block(b3.header.branch, b3.serialize())) ) self.assertTrue( - call_async(master.add_raw_minor_block(b4.header.branch, b4.serialize())) + (await master.add_raw_minor_block(b4.header.branch, b4.serialize())) ) self.assertEqual( - call_async( + (await master.get_primary_account_data(acc3) ).token_balances.balance_map, {genesis_token: 54321}, ) - def test_broadcast_cross_shard_transactions_with_extra_gas(self): + async def test_broadcast_cross_shard_transactions_with_extra_gas(self): """ Test the cross shard transactions are broadcasted to the destination shards """ id1 = Identity.create_random_identity() id2 = Identity.create_random_identity() @@ -712,7 +712,7 @@ def test_broadcast_cross_shard_transactions_with_extra_gas(self): acc3 = Address.create_random_account(full_shard_key=1) acc4 = Address.create_random_account(full_shard_key=1) - with ClusterContext(1, acc1) as clusters: + async with ClusterContext(1, acc1) as clusters: master = clusters[0].master slaves = clusters[0].slave_list genesis_token = ( @@ -721,12 +721,12 @@ def test_broadcast_cross_shard_transactions_with_extra_gas(self): # Add a root block first so that later minor blocks referring to this root # can be broadcasted to other shards - root_block = call_async( + root_block = (await master.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) tx1 = create_transfer_transaction( shard_state=clusters[0].get_shard_state(2 | 0), @@ -740,10 +740,10 @@ def test_broadcast_cross_shard_transactions_with_extra_gas(self): self.assertTrue(slaves[0].add_tx(tx1)) b1 = clusters[0].get_shard_state(2 | 0).create_block_to_mine(address=acc2) - call_async(clusters[0].get_shard(2 | 0).add_block(b1)) + (await clusters[0].get_shard(2 | 0).add_block(b1)) self.assertEqual( - call_async( + (await master.get_primary_account_data(acc1) ).token_balances.balance_map, { @@ -753,13 +753,13 @@ def test_broadcast_cross_shard_transactions_with_extra_gas(self): }, ) - root_block = call_async( + root_block = (await master.get_next_block_to_mine(address=acc1, branch_value=None) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) self.assertEqual( - call_async( + (await master.get_primary_account_data(acc1.address_in_shard(1)) ).token_balances.balance_map, {genesis_token: 1000000}, @@ -767,13 +767,13 @@ def test_broadcast_cross_shard_transactions_with_extra_gas(self): # b2 should include the withdraw of tx1 b2 = clusters[0].get_shard_state(2 | 1).create_block_to_mine(address=acc4) - call_async(clusters[0].get_shard(2 | 1).add_block(b2)) + (await clusters[0].get_shard(2 | 1).add_block(b2)) - self.assert_balance( + await self.assert_balance( master, [acc3, acc1.address_in_shard(1)], [54321, 1012345] ) - def test_broadcast_cross_shard_transactions_with_extra_gas_old(self): + async def test_broadcast_cross_shard_transactions_with_extra_gas_old(self): """ Test the cross shard transactions are broadcasted to the destination shards """ id1 = Identity.create_random_identity() id2 = Identity.create_random_identity() @@ -782,7 +782,7 @@ def test_broadcast_cross_shard_transactions_with_extra_gas_old(self): acc3 = Address.create_random_account(full_shard_key=1) acc4 = Address.create_random_account(full_shard_key=1) - with ClusterContext(1, acc1) as clusters: + async with ClusterContext(1, acc1) as clusters: master = clusters[0].master slaves = clusters[0].slave_list genesis_token = ( @@ -799,12 +799,12 @@ def test_broadcast_cross_shard_transactions_with_extra_gas_old(self): # Add a root block first so that later minor blocks referring to this root # can be broadcasted to other shards - root_block = call_async( + root_block = (await master.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) tx1 = create_transfer_transaction( shard_state=clusters[0].get_shard_state(2 | 0), @@ -818,10 +818,10 @@ def test_broadcast_cross_shard_transactions_with_extra_gas_old(self): self.assertTrue(slaves[0].add_tx(tx1)) b1 = clusters[0].get_shard_state(2 | 0).create_block_to_mine(address=acc2) - call_async(clusters[0].get_shard(2 | 0).add_block(b1)) + (await clusters[0].get_shard(2 | 0).add_block(b1)) self.assertEqual( - call_async( + (await master.get_primary_account_data(acc1) ).token_balances.balance_map, { @@ -831,13 +831,13 @@ def test_broadcast_cross_shard_transactions_with_extra_gas_old(self): }, ) - root_block = call_async( + root_block = (await master.get_next_block_to_mine(address=acc1, branch_value=None) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) self.assertEqual( - call_async( + (await master.get_primary_account_data(acc1.address_in_shard(1)) ).token_balances.balance_map, {genesis_token: 1000000}, @@ -845,20 +845,20 @@ def test_broadcast_cross_shard_transactions_with_extra_gas_old(self): # b2 should include the withdraw of tx1 b2 = clusters[0].get_shard_state(2 | 1).create_block_to_mine(address=acc4) - call_async(clusters[0].get_shard(2 | 1).add_block(b2)) + (await clusters[0].get_shard(2 | 1).add_block(b2)) - self.assert_balance( + await self.assert_balance( master, [acc3, acc1.address_in_shard(1)], [54321, 1000000] ) - def test_broadcast_cross_shard_transactions_1x2(self): + async def test_broadcast_cross_shard_transactions_1x2(self): """ Test the cross shard transactions are broadcasted to the destination shards """ id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc3 = Address.create_random_account(full_shard_key=2 << 16) acc4 = Address.create_random_account(full_shard_key=3 << 16) - with ClusterContext(1, acc1, chain_size=8, shard_size=1) as clusters: + async with ClusterContext(1, acc1, chain_size=8, shard_size=1) as clusters: master = clusters[0].master slaves = clusters[0].slave_list genesis_token = ( @@ -867,12 +867,12 @@ def test_broadcast_cross_shard_transactions_1x2(self): # Add a root block first so that later minor blocks referring to this root # can be broadcasted to other shards - root_block = call_async( + root_block = (await master.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) tx1 = create_transfer_transaction( shard_state=clusters[0].get_shard_state(1), @@ -898,7 +898,7 @@ def test_broadcast_cross_shard_transactions_1x2(self): b2 = clusters[0].get_shard_state(1).create_block_to_mine(address=acc1) b2.header.create_time += 1 - call_async(clusters[0].get_shard(1).add_block(b1)) + (await clusters[0].get_shard(1).add_block(b1)) # expect chain 2 got the CrossShardTransactionList of b1 xshard_tx_list = ( @@ -924,7 +924,7 @@ def test_broadcast_cross_shard_transactions_1x2(self): self.assertEqual(xshard_tx_list.tx_list[0].to_address, acc4) self.assertEqual(xshard_tx_list.tx_list[0].value, 1234) - call_async(clusters[0].get_shard(1 | 0).add_block(b2)) + (await clusters[0].get_shard(1 | 0).add_block(b2)) # b2 doesn't update tip self.assertEqual(clusters[0].get_shard_state(1 | 0).header_tip, b1.header) @@ -957,12 +957,12 @@ def test_broadcast_cross_shard_transactions_1x2(self): .get_shard_state((2 << 16) | 1) .create_block_to_mine(address=acc1.address_in_shard(2 << 16)) ) - call_async(master.add_raw_minor_block(b3.header.branch, b3.serialize())) + (await master.add_raw_minor_block(b3.header.branch, b3.serialize())) - root_block = call_async( + root_block = (await master.get_next_block_to_mine(address=acc1, branch_value=None) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) # b4 should include the withdraw of tx1 b4 = ( @@ -971,10 +971,10 @@ def test_broadcast_cross_shard_transactions_1x2(self): .create_block_to_mine(address=acc1.address_in_shard(2 << 16)) ) self.assertTrue( - call_async(master.add_raw_minor_block(b4.header.branch, b4.serialize())) + (await master.add_raw_minor_block(b4.header.branch, b4.serialize())) ) self.assertEqual( - call_async( + (await master.get_primary_account_data(acc3) ).token_balances.balance_map, {genesis_token: 54321}, @@ -987,26 +987,26 @@ def test_broadcast_cross_shard_transactions_1x2(self): .create_block_to_mine(address=acc1.address_in_shard(3 << 16)) ) self.assertTrue( - call_async(master.add_raw_minor_block(b5.header.branch, b5.serialize())) + (await master.add_raw_minor_block(b5.header.branch, b5.serialize())) ) self.assertEqual( - call_async( + (await master.get_primary_account_data(acc4) ).token_balances.balance_map, {genesis_token: 1234}, ) - def assert_balance(self, master, account_list, balance_list): + async def assert_balance(self, master, account_list, balance_list): genesis_token = master.env.quark_chain_config.genesis_token for idx, account in enumerate(account_list): self.assertEqual( - call_async( + (await master.get_primary_account_data(account) ).token_balances.balance_map, {genesis_token: balance_list[idx]}, ) - def test_broadcast_cross_shard_transactions_2x1(self): + async def test_broadcast_cross_shard_transactions_2x1(self): """ Test the cross shard transactions are broadcasted to the destination shards """ id1 = Identity.create_random_identity() id2 = Identity.create_random_identity() @@ -1016,7 +1016,7 @@ def test_broadcast_cross_shard_transactions_2x1(self): acc4 = Address.create_random_account(full_shard_key=1 << 16) acc5 = Address.create_random_account(full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, chain_size=8, shard_size=1, mblock_coinbase_amount=1000000 ) as clusters: master = clusters[0].master @@ -1024,21 +1024,21 @@ def test_broadcast_cross_shard_transactions_2x1(self): # Add a root block first so that later minor blocks referring to this root # can be broadcasted to other shards - root_block = call_async( + root_block = (await master.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) b0 = ( clusters[0] .get_shard_state((1 << 16) + 1) .create_block_to_mine(address=acc2) ) - call_async(clusters[0].get_shard((1 << 16) + 1).add_block(b0)) + (await clusters[0].get_shard((1 << 16) + 1).add_block(b0)) - self.assert_balance(master, [acc1, acc2], [1000000, 500000]) + await self.assert_balance(master, [acc1, acc2], [1000000, 500000]) tx1 = create_transfer_transaction( shard_state=clusters[0].get_shard_state(1), @@ -1080,10 +1080,10 @@ def test_broadcast_cross_shard_transactions_2x1(self): .create_block_to_mine(address=acc4) ) - call_async(clusters[0].get_shard(1).add_block(b1)) - call_async(clusters[0].get_shard((1 << 16) + 1).add_block(b2)) + (await clusters[0].get_shard(1).add_block(b1)) + (await clusters[0].get_shard((1 << 16) + 1).add_block(b2)) - self.assert_balance( + await self.assert_balance( master, [acc1, acc1.address_in_shard(2 << 16), acc2, acc4, acc5], [ @@ -1098,7 +1098,7 @@ def test_broadcast_cross_shard_transactions_2x1(self): ], ) self.assertEqual( - call_async( + (await master.get_primary_account_data(acc3) ).token_balances.balance_map, {}, @@ -1122,10 +1122,10 @@ def test_broadcast_cross_shard_transactions_2x1(self): self.assertEqual(len(xshard_tx_list.tx_list), 1) self.assertEqual(xshard_tx_list.tx_list[0].tx_hash, tx3.get_hash()) - root_block = call_async( + root_block = (await master.get_next_block_to_mine(address=acc1, branch_value=None) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) # b3 should include the deposits of tx1, t2, t3 b3 = ( @@ -1134,9 +1134,9 @@ def test_broadcast_cross_shard_transactions_2x1(self): .create_block_to_mine(address=acc1.address_in_shard(2 << 16)) ) self.assertTrue( - call_async(master.add_raw_minor_block(b3.header.branch, b3.serialize())) + (await master.add_raw_minor_block(b3.header.branch, b3.serialize())) ) - self.assert_balance( + await self.assert_balance( master, [acc1, acc1.address_in_shard(2 << 16), acc2, acc3, acc4, acc5], [ @@ -1154,9 +1154,9 @@ def test_broadcast_cross_shard_transactions_2x1(self): b4 = clusters[0].get_shard_state(1).create_block_to_mine(address=acc1) self.assertTrue( - call_async(master.add_raw_minor_block(b4.header.branch, b4.serialize())) + (await master.add_raw_minor_block(b4.header.branch, b4.serialize())) ) - self.assert_balance( + await self.assert_balance( master, [acc1, acc1.address_in_shard(2 << 16), acc2, acc3, acc4, acc5], [ @@ -1175,11 +1175,11 @@ def test_broadcast_cross_shard_transactions_2x1(self): ], ) - root_block = call_async( + root_block = (await master.get_next_block_to_mine(address=acc3, branch_value=None) ) - call_async(master.add_root_block(root_block)) - self.assert_balance( + (await master.add_root_block(root_block)) + await self.assert_balance( master, [acc1, acc1.address_in_shard(2 << 16), acc2, acc3, acc4, acc5], [ @@ -1204,9 +1204,9 @@ def test_broadcast_cross_shard_transactions_2x1(self): .create_block_to_mine(address=acc3) ) self.assertTrue( - call_async(master.add_raw_minor_block(b5.header.branch, b5.serialize())) + (await master.add_raw_minor_block(b5.header.branch, b5.serialize())) ) - self.assert_balance( + await self.assert_balance( master, [acc1, acc1.address_in_shard(2 << 16), acc2, acc3, acc4, acc5], [ @@ -1231,11 +1231,11 @@ def test_broadcast_cross_shard_transactions_2x1(self): ], ) - root_block = call_async( + root_block = (await master.get_next_block_to_mine(address=acc4, branch_value=None) ) - call_async(master.add_root_block(root_block)) - self.assert_balance( + (await master.add_root_block(root_block)) + await self.assert_balance( master, [acc1, acc1.address_in_shard(2 << 16), acc2, acc3, acc4, acc5], [ @@ -1266,7 +1266,7 @@ def test_broadcast_cross_shard_transactions_2x1(self): .create_block_to_mine(address=acc4) ) self.assertTrue( - call_async(master.add_raw_minor_block(b6.header.branch, b6.serialize())) + (await master.add_raw_minor_block(b6.header.branch, b6.serialize())) ) balances = [ 120 * 10 ** 18 # root block coinbase reward @@ -1288,7 +1288,7 @@ def test_broadcast_cross_shard_transactions_2x1(self): 120 * 10 ** 18 + 500000 + 1000000 + opcodes.GTXCOST, 500000 + opcodes.GTXCOST * 2, ] - self.assert_balance( + await self.assert_balance( master, [acc1, acc1.address_in_shard(2 << 16), acc2, acc3, acc4, acc5], balances, @@ -1301,7 +1301,7 @@ def test_broadcast_cross_shard_transactions_2x1(self): + 500000, # post-tax mblock coinbase ) - def test_cross_shard_contract_call(self): + async def test_cross_shard_contract_call(self): """ Test the cross shard transactions are broadcasted to the destination shards """ id1 = Identity.create_random_identity() id2 = Identity.create_random_identity() @@ -1317,7 +1317,7 @@ def test_cross_shard_contract_call(self): 16, ) - with ClusterContext( + async with ClusterContext( 1, acc1, chain_size=8, shard_size=1, mblock_coinbase_amount=10000000 ) as clusters: master = clusters[0].master @@ -1328,12 +1328,12 @@ def test_cross_shard_contract_call(self): # Add a root block first so that later minor blocks referring to this root # can be broadcasted to other shards - root_block = call_async( + root_block = (await master.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) tx0 = create_contract_with_storage2_transaction( shard_state=clusters[0].get_shard_state((1 << 16) | 1), @@ -1343,13 +1343,13 @@ def test_cross_shard_contract_call(self): ) self.assertTrue(slaves[1].add_tx(tx0)) b0 = clusters[0].get_shard_state(1).create_block_to_mine(address=acc1) - call_async(clusters[0].get_shard(1).add_block(b0)) + (await clusters[0].get_shard(1).add_block(b0)) b1 = ( clusters[0] .get_shard_state((1 << 16) + 1) .create_block_to_mine(address=acc2) ) - call_async(clusters[0].get_shard((1 << 16) + 1).add_block(b1)) + (await clusters[0].get_shard((1 << 16) + 1).add_block(b1)) tx1 = create_transfer_transaction( shard_state=clusters[0].get_shard_state(1), @@ -1362,20 +1362,20 @@ def test_cross_shard_contract_call(self): self.assertTrue(slaves[0].add_tx(tx1)) b00 = clusters[0].get_shard_state(1).create_block_to_mine(address=acc1) - call_async(clusters[0].get_shard(1).add_block(b00)) + (await clusters[0].get_shard(1).add_block(b00)) self.assertEqual( - call_async( + (await master.get_primary_account_data(acc3) ).token_balances.balance_map, {genesis_token: 1500000}, ) - _, _, receipt = call_async( + _, _, receipt = (await master.get_transaction_receipt(tx0.get_hash(), b1.header.branch) ) self.assertEqual(receipt.success, b"\x01") contract_address = receipt.contract_address - result = call_async( + result = (await master.get_storage_at(contract_address, storage_key, b1.header.height) ) self.assertEqual( @@ -1386,12 +1386,12 @@ def test_cross_shard_contract_call(self): ) # should include b1 - root_block = call_async( + root_block = (await master.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) # call the contract with insufficient gas tx2 = create_transfer_transaction( @@ -1406,17 +1406,17 @@ def test_cross_shard_contract_call(self): ) self.assertTrue(slaves[0].add_tx(tx2)) b2 = clusters[0].get_shard_state(1).create_block_to_mine(address=acc1) - call_async(clusters[0].get_shard(1).add_block(b2)) + (await clusters[0].get_shard(1).add_block(b2)) # should include b2 - root_block = call_async( + root_block = (await master.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) self.assertEqual( - call_async( + (await master.get_primary_account_data(acc4) ).token_balances.balance_map, {}, @@ -1428,8 +1428,8 @@ def test_cross_shard_contract_call(self): .get_shard_state((1 << 16) + 1) .create_block_to_mine(address=acc2) ) - call_async(clusters[0].get_shard((1 << 16) + 1).add_block(b3)) - result = call_async( + (await clusters[0].get_shard((1 << 16) + 1).add_block(b3)) + result = (await master.get_storage_at(contract_address, storage_key, b3.header.height) ) self.assertEqual( @@ -1439,12 +1439,12 @@ def test_cross_shard_contract_call(self): ), ) self.assertEqual( - call_async( + (await master.get_primary_account_data(acc4) ).token_balances.balance_map, {}, ) - _, _, receipt = call_async( + _, _, receipt = (await master.get_transaction_receipt(tx2.get_hash(), b3.header.branch) ) self.assertEqual(receipt.success, b"") @@ -1463,15 +1463,15 @@ def test_cross_shard_contract_call(self): self.assertTrue(slaves[0].add_tx(tx3)) b4 = clusters[0].get_shard_state(1).create_block_to_mine(address=acc1) - call_async(clusters[0].get_shard(1).add_block(b4)) + (await clusters[0].get_shard(1).add_block(b4)) # should include b4 - root_block = call_async( + root_block = (await master.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) # The contract should be called b5 = ( @@ -1479,8 +1479,8 @@ def test_cross_shard_contract_call(self): .get_shard_state((1 << 16) + 1) .create_block_to_mine(address=acc2) ) - call_async(clusters[0].get_shard((1 << 16) + 1).add_block(b5)) - result = call_async( + (await clusters[0].get_shard((1 << 16) + 1).add_block(b5)) + result = (await master.get_storage_at(contract_address, storage_key, b5.header.height) ) self.assertEqual( @@ -1490,17 +1490,17 @@ def test_cross_shard_contract_call(self): ), ) self.assertEqual( - call_async( + (await master.get_primary_account_data(acc4) ).token_balances.balance_map, {genesis_token: 677758}, ) - _, _, receipt = call_async( + _, _, receipt = (await master.get_transaction_receipt(tx3.get_hash(), b3.header.branch) ) self.assertEqual(receipt.success, b"\x01") - def test_cross_shard_contract_create(self): + async def test_cross_shard_contract_create(self): """ Test the cross shard transactions are broadcasted to the destination shards """ id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -1513,7 +1513,7 @@ def test_cross_shard_contract_create(self): 16, ) - with ClusterContext( + async with ClusterContext( 1, acc1, chain_size=8, shard_size=1, mblock_coinbase_amount=1000000 ) as clusters: master = clusters[0].master @@ -1521,12 +1521,12 @@ def test_cross_shard_contract_create(self): # Add a root block first so that later minor blocks referring to this root # can be broadcasted to other shards - root_block = call_async( + root_block = (await master.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) tx1 = create_contract_with_storage2_transaction( shard_state=clusters[0].get_shard_state((1 << 16) | 1), @@ -1541,31 +1541,31 @@ def test_cross_shard_contract_create(self): .get_shard_state((1 << 16) + 1) .create_block_to_mine(address=acc2) ) - call_async(clusters[0].get_shard((1 << 16) + 1).add_block(b1)) + (await clusters[0].get_shard((1 << 16) + 1).add_block(b1)) - _, _, receipt = call_async( + _, _, receipt = (await master.get_transaction_receipt(tx1.get_hash(), b1.header.branch) ) self.assertEqual(receipt.success, b"\x01") # should include b1 - root_block = call_async( + root_block = (await master.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) b2 = clusters[0].get_shard_state(1).create_block_to_mine(address=acc1) - call_async(clusters[0].get_shard(1).add_block(b2)) + (await clusters[0].get_shard(1).add_block(b2)) # contract should be created - _, _, receipt = call_async( + _, _, receipt = (await master.get_transaction_receipt(tx1.get_hash(), b2.header.branch) ) self.assertEqual(receipt.success, b"\x01") contract_address = receipt.contract_address - result = call_async( + result = (await master.get_storage_at(contract_address, storage_key, b2.header.height) ) self.assertEqual( @@ -1589,13 +1589,13 @@ def test_cross_shard_contract_create(self): self.assertTrue(slaves[0].add_tx(tx2)) b3 = clusters[0].get_shard_state(1).create_block_to_mine(address=acc1) - call_async(clusters[0].get_shard(1).add_block(b3)) + (await clusters[0].get_shard(1).add_block(b3)) - _, _, receipt = call_async( + _, _, receipt = (await master.get_transaction_receipt(tx2.get_hash(), b3.header.branch) ) self.assertEqual(receipt.success, b"\x01") - result = call_async( + result = (await master.get_storage_at(contract_address, storage_key, b3.header.height) ) self.assertEqual( @@ -1605,28 +1605,28 @@ def test_cross_shard_contract_create(self): ), ) - def test_broadcast_cross_shard_transactions_to_neighbor_only(self): + async def test_broadcast_cross_shard_transactions_to_neighbor_only(self): """ Test the broadcast is only done to the neighbors """ id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) # create 64 shards so that the neighbor rule can kick in # explicitly set num_slaves to 4 so that it does not spin up 64 slaves - with ClusterContext(1, acc1, shard_size=64, num_slaves=4) as clusters: + async with ClusterContext(1, acc1, shard_size=64, num_slaves=4) as clusters: master = clusters[0].master # Add a root block first so that later minor blocks referring to this root # can be broadcasted to other shards - root_block = call_async( + root_block = (await master.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) b1 = clusters[0].get_shard_state(64).create_block_to_mine(address=acc1) self.assertTrue( - call_async(master.add_raw_minor_block(b1.header.branch, b1.serialize())) + (await master.add_raw_minor_block(b1.header.branch, b1.serialize())) ) neighbor_shards = [2 ** i for i in range(6)] @@ -1642,29 +1642,29 @@ def test_broadcast_cross_shard_transactions_to_neighbor_only(self): else: self.assertIsNone(xshard_tx_list) - def test_get_work_from_slave(self): + async def test_get_work_from_slave(self): genesis = Address.create_empty_account(full_shard_key=0) - with ClusterContext(1, genesis, remote_mining=True) as clusters: + async with ClusterContext(1, genesis, remote_mining=True) as clusters: slaves = clusters[0].slave_list # no posw state = clusters[0].get_shard_state(2 | 0) branch = state.create_block_to_mine().header.branch - work = call_async(slaves[0].get_work(branch)) + work = (await slaves[0].get_work(branch)) self.assertEqual(work.difficulty, 10) # enable posw, with total stakes cover all the window state.shard_config.POSW_CONFIG.ENABLED = True state.shard_config.POSW_CONFIG.TOTAL_STAKE_PER_BLOCK = 500000 - work = call_async(slaves[0].get_work(branch)) + work = (await slaves[0].get_work(branch)) self.assertEqual(work.difficulty, 0) - def test_handle_get_minor_block_list_request_with_total_diff(self): + async def test_handle_get_minor_block_list_request_with_total_diff(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(2, acc1) as clusters: + async with ClusterContext(2, acc1) as clusters: cluster0_root_state = clusters[0].master.root_state cluster1_root_state = clusters[1].master.root_state coinbase = cluster1_root_state._calculate_root_block_coinbase([], 0) @@ -1674,7 +1674,7 @@ def test_handle_get_minor_block_list_request_with_total_diff(self): rb1 = rb0.create_block_to_append(difficulty=int(1e6)).finalize(coinbase) # Establish cluster connection - call_async( + (await clusters[1].network.connect( "127.0.0.1", clusters[0].master.env.cluster_config.SIMPLE_NETWORK.BOOTSTRAP_PORT, @@ -1682,27 +1682,27 @@ def test_handle_get_minor_block_list_request_with_total_diff(self): ) # Cluster 0 broadcasts the root block to cluster 1 - call_async(clusters[0].master.add_root_block(rb1)) + (await clusters[0].master.add_root_block(rb1)) self.assertEqual(cluster0_root_state.tip.get_hash(), rb1.header.get_hash()) # Make sure the root block tip of cluster 1 is changed - assert_true_with_timeout(lambda: cluster1_root_state.tip == rb1.header, 2) + await async_assert_true_with_timeout(lambda: cluster1_root_state.tip == rb1.header, 2) # Cluster 1 generates a minor block and broadcasts to cluster 0 shard_state = clusters[1].get_shard_state(0b10) b1 = _tip_gen(shard_state) - add_result = call_async( + add_result = (await clusters[1].master.add_raw_minor_block(b1.header.branch, b1.serialize()) ) self.assertTrue(add_result) # Make sure another cluster received the new minor block - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: clusters[1] .get_shard_state(0b10) .contain_block_by_hash(b1.header.get_hash()) ) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: clusters[0].master.root_state.db.contain_minor_block_by_hash( b1.header.get_hash() ) @@ -1710,36 +1710,36 @@ def test_handle_get_minor_block_list_request_with_total_diff(self): # Cluster 1 generates a new root block with higher total difficulty rb2 = rb0.create_block_to_append(difficulty=int(3e6)).finalize(coinbase) - call_async(clusters[1].master.add_root_block(rb2)) + (await clusters[1].master.add_root_block(rb2)) self.assertEqual(cluster1_root_state.tip.get_hash(), rb2.header.get_hash()) # Generate a minor block b2 b2 = _tip_gen(shard_state) - add_result = call_async( + add_result = (await clusters[1].master.add_raw_minor_block(b2.header.branch, b2.serialize()) ) self.assertTrue(add_result) # Make sure another cluster received the new minor block - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: clusters[1] .get_shard_state(0b10) .contain_block_by_hash(b2.header.get_hash()) ) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: clusters[0].master.root_state.db.contain_minor_block_by_hash( b2.header.get_hash() ) ) - def test_new_block_header_pool(self): + async def test_new_block_header_pool(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(1, acc1) as clusters: + async with ClusterContext(1, acc1) as clusters: shard_state = clusters[0].get_shard_state(0b10) b1 = _tip_gen(shard_state) - add_result = call_async( + add_result = (await clusters[0].master.add_raw_minor_block(b1.header.branch, b1.serialize()) ) self.assertTrue(add_result) @@ -1751,41 +1751,41 @@ def test_new_block_header_pool(self): b2 = b1.create_block_to_append(difficulty=12345) shard = clusters[0].slave_list[0].shards[b2.header.branch] with self.assertRaises(ValueError): - call_async(shard.handle_new_block(b2)) + (await shard.handle_new_block(b2)) # Also the block should not exist in new block pool self.assertTrue( b2.header.get_hash() not in shard.state.new_block_header_pool ) - def test_get_root_block_headers_with_skip(self): + async def test_get_root_block_headers_with_skip(self): """ Test the broadcast is only done to the neighbors """ id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(2, acc1) as clusters: + async with ClusterContext(2, acc1) as clusters: master = clusters[0].master # Add a root block first so that later minor blocks referring to this root # can be broadcasted to other shards root_block_header_list = [master.root_state.tip] for i in range(10): - root_block = call_async( + root_block = (await master.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) root_block_header_list.append(root_block.header) self.assertEqual(root_block_header_list[-1].height, 10) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: clusters[1].master.root_state.tip.height == 10 ) peer = clusters[1].peer # Test Case 1 ################################################### - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_ROOT_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetRootBlockHeaderListWithSkipRequest.create_for_height( @@ -1798,7 +1798,7 @@ def test_get_root_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[1], root_block_header_list[3]) self.assertEqual(resp.block_header_list[2], root_block_header_list[5]) - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_ROOT_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetRootBlockHeaderListWithSkipRequest.create_for_hash( @@ -1815,7 +1815,7 @@ def test_get_root_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[2], root_block_header_list[5]) # Test Case 2 ################################################### - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_ROOT_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetRootBlockHeaderListWithSkipRequest.create_for_height( @@ -1828,7 +1828,7 @@ def test_get_root_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[1], root_block_header_list[5]) self.assertEqual(resp.block_header_list[2], root_block_header_list[8]) - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_ROOT_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetRootBlockHeaderListWithSkipRequest.create_for_hash( @@ -1845,7 +1845,7 @@ def test_get_root_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[2], root_block_header_list[8]) # Test Case 3 ################################################### - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_ROOT_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetRootBlockHeaderListWithSkipRequest.create_for_height( @@ -1860,7 +1860,7 @@ def test_get_root_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[3], root_block_header_list[9]) self.assertEqual(resp.block_header_list[4], root_block_header_list[10]) - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_ROOT_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetRootBlockHeaderListWithSkipRequest.create_for_hash( @@ -1879,7 +1879,7 @@ def test_get_root_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[4], root_block_header_list[10]) # Test Case 4 ################################################### - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_ROOT_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetRootBlockHeaderListWithSkipRequest.create_for_height( @@ -1889,7 +1889,7 @@ def test_get_root_block_headers_with_skip(self): ) self.assertEqual(len(resp.block_header_list), 1) self.assertEqual(resp.block_header_list[0], root_block_header_list[2]) - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_ROOT_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetRootBlockHeaderListWithSkipRequest.create_for_hash( @@ -1904,7 +1904,7 @@ def test_get_root_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[0], root_block_header_list[2]) # Test Case 5 ################################################### - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_ROOT_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetRootBlockHeaderListWithSkipRequest.create_for_height( @@ -1914,7 +1914,7 @@ def test_get_root_block_headers_with_skip(self): ) self.assertEqual(len(resp.block_header_list), 0) - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_ROOT_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetRootBlockHeaderListWithSkipRequest.create_for_hash( @@ -1925,7 +1925,7 @@ def test_get_root_block_headers_with_skip(self): self.assertEqual(len(resp.block_header_list), 0) # Test Case 6 ################################################### - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_ROOT_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetRootBlockHeaderListWithSkipRequest.create_for_height( @@ -1940,7 +1940,7 @@ def test_get_root_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[3], root_block_header_list[2]) self.assertEqual(resp.block_header_list[4], root_block_header_list[0]) - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_ROOT_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetRootBlockHeaderListWithSkipRequest.create_for_hash( @@ -1958,30 +1958,30 @@ def test_get_root_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[3], root_block_header_list[2]) self.assertEqual(resp.block_header_list[4], root_block_header_list[0]) - def test_get_root_block_header_sync_from_genesis(self): + async def test_get_root_block_header_sync_from_genesis(self): """ Test the broadcast is only done to the neighbors """ id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(2, acc1, connect=False) as clusters: + async with ClusterContext(2, acc1, connect=False) as clusters: master = clusters[0].master root_block_header_list = [master.root_state.tip] for i in range(10): - root_block = call_async( + root_block = (await master.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) root_block_header_list.append(root_block.header) # Connect and the synchronizer should automically download - call_async( + (await clusters[1].network.connect( "127.0.0.1", clusters[0].network.env.cluster_config.P2P_PORT ) ) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: clusters[1].master.root_state.tip == root_block_header_list[-1] ) self.assertEqual( @@ -1989,38 +1989,38 @@ def test_get_root_block_header_sync_from_genesis(self): len(root_block_header_list) - 1, ) - def test_get_root_block_header_sync_from_height_3(self): + async def test_get_root_block_header_sync_from_height_3(self): """ Test the broadcast is only done to the neighbors """ id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(2, acc1, connect=False) as clusters: + async with ClusterContext(2, acc1, connect=False) as clusters: master0 = clusters[0].master root_block_list = [] for i in range(10): - root_block = call_async( + root_block = (await master0.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master0.add_root_block(root_block)) + (await master0.add_root_block(root_block)) root_block_list.append(root_block) # Add 3 blocks to another cluster master1 = clusters[1].master for i in range(3): - call_async(master1.add_root_block(root_block_list[i])) - assert_true_with_timeout( + (await master1.add_root_block(root_block_list[i])) + await async_assert_true_with_timeout( lambda: master1.root_state.tip == root_block_list[2].header ) # Connect and the synchronizer should automically download - call_async( + (await clusters[1].network.connect( "127.0.0.1", clusters[0].network.env.cluster_config.P2P_PORT ) ) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: master1.root_state.tip == root_block_list[-1].header ) self.assertEqual( @@ -2028,40 +2028,40 @@ def test_get_root_block_header_sync_from_height_3(self): ) self.assertEqual(master1.synchronizer.stats.ancestor_lookup_requests, 1) - def test_get_root_block_header_sync_with_fork(self): + async def test_get_root_block_header_sync_with_fork(self): """ Test the broadcast is only done to the neighbors """ id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(2, acc1, connect=False) as clusters: + async with ClusterContext(2, acc1, connect=False) as clusters: master0 = clusters[0].master root_block_list = [] for i in range(10): - root_block = call_async( + root_block = (await master0.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master0.add_root_block(root_block)) + (await master0.add_root_block(root_block)) root_block_list.append(root_block) # Add 2+3 blocks to another cluster: 2 are the same as cluster 0, and 3 are the fork master1 = clusters[1].master for i in range(2): - call_async(master1.add_root_block(root_block_list[i])) + (await master1.add_root_block(root_block_list[i])) for i in range(3): - root_block = call_async( + root_block = (await master1.get_next_block_to_mine(acc1, branch_value=None) ) - call_async(master1.add_root_block(root_block)) + (await master1.add_root_block(root_block)) # Connect and the synchronizer should automically download - call_async( + (await clusters[1].network.connect( "127.0.0.1", clusters[0].network.env.cluster_config.P2P_PORT ) ) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: master1.root_state.tip == root_block_list[-1].header ) self.assertEqual( @@ -2069,188 +2069,188 @@ def test_get_root_block_header_sync_with_fork(self): ) self.assertEqual(master1.synchronizer.stats.ancestor_lookup_requests, 1) - def test_get_root_block_header_sync_with_staleness(self): + async def test_get_root_block_header_sync_with_staleness(self): """ Test the broadcast is only done to the neighbors """ id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(2, acc1, connect=False) as clusters: + async with ClusterContext(2, acc1, connect=False) as clusters: master0 = clusters[0].master root_block_list = [] for i in range(10): - root_block = call_async( + root_block = (await master0.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master0.add_root_block(root_block)) + (await master0.add_root_block(root_block)) root_block_list.append(root_block) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: master0.root_state.tip == root_block_list[-1].header ) # Add 3 blocks to another cluster master1 = clusters[1].master for i in range(8): - root_block = call_async( + root_block = (await master1.get_next_block_to_mine(acc1, branch_value=None) ) - call_async(master1.add_root_block(root_block)) + (await master1.add_root_block(root_block)) master1.env.quark_chain_config.ROOT.MAX_STALE_ROOT_BLOCK_HEIGHT_DIFF = 5 - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: master1.root_state.tip == root_block.header ) # Connect and the synchronizer should automically download - call_async( + (await clusters[1].network.connect( "127.0.0.1", clusters[0].network.env.cluster_config.P2P_PORT ) ) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: master1.synchronizer.stats.ancestor_not_found_count == 1 ) self.assertEqual(master1.synchronizer.stats.blocks_downloaded, 0) self.assertEqual(master1.synchronizer.stats.ancestor_lookup_requests, 1) - def test_get_root_block_header_sync_with_multiple_lookup(self): + async def test_get_root_block_header_sync_with_multiple_lookup(self): """ Test the broadcast is only done to the neighbors """ id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(2, acc1, connect=False) as clusters: + async with ClusterContext(2, acc1, connect=False) as clusters: master0 = clusters[0].master root_block_list = [] for i in range(12): - root_block = call_async( + root_block = (await master0.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master0.add_root_block(root_block)) + (await master0.add_root_block(root_block)) root_block_list.append(root_block) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: master0.root_state.tip == root_block_list[-1].header ) # Add 4+4 blocks to another cluster master1 = clusters[1].master for i in range(4): - call_async(master1.add_root_block(root_block_list[i])) + (await master1.add_root_block(root_block_list[i])) for i in range(4): - root_block = call_async( + root_block = (await master1.get_next_block_to_mine(acc1, branch_value=None) ) - call_async(master1.add_root_block(root_block)) + (await master1.add_root_block(root_block)) master1.synchronizer.root_block_header_list_limit = 4 # Connect and the synchronizer should automically download - call_async( + (await clusters[1].network.connect( "127.0.0.1", clusters[0].network.env.cluster_config.P2P_PORT ) ) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: master1.root_state.tip == root_block_list[-1].header ) self.assertEqual(master1.synchronizer.stats.blocks_downloaded, 8) self.assertEqual(master1.synchronizer.stats.headers_downloaded, 5 + 8) self.assertEqual(master1.synchronizer.stats.ancestor_lookup_requests, 2) - def test_get_root_block_header_sync_with_start_equal_end(self): + async def test_get_root_block_header_sync_with_start_equal_end(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(2, acc1, connect=False) as clusters: + async with ClusterContext(2, acc1, connect=False) as clusters: master0 = clusters[0].master root_block_list = [] for i in range(5): - root_block = call_async( + root_block = (await master0.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master0.add_root_block(root_block)) + (await master0.add_root_block(root_block)) root_block_list.append(root_block) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: master0.root_state.tip == root_block_list[-1].header ) # Add 3+1 blocks to another cluster master1 = clusters[1].master for i in range(3): - call_async(master1.add_root_block(root_block_list[i])) + (await master1.add_root_block(root_block_list[i])) for i in range(1): - root_block = call_async( + root_block = (await master1.get_next_block_to_mine(acc1, branch_value=None) ) - call_async(master1.add_root_block(root_block)) + (await master1.add_root_block(root_block)) master1.synchronizer.root_block_header_list_limit = 3 # Connect and the synchronizer should automically download - call_async( + (await clusters[1].network.connect( "127.0.0.1", clusters[0].network.env.cluster_config.P2P_PORT ) ) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: master1.root_state.tip == root_block_list[-1].header ) self.assertEqual(master1.synchronizer.stats.blocks_downloaded, 2) self.assertEqual(master1.synchronizer.stats.headers_downloaded, 6) self.assertEqual(master1.synchronizer.stats.ancestor_lookup_requests, 2) - def test_get_root_block_header_sync_with_best_ancestor(self): + async def test_get_root_block_header_sync_with_best_ancestor(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(2, acc1, connect=False) as clusters: + async with ClusterContext(2, acc1, connect=False) as clusters: master0 = clusters[0].master root_block_list = [] for i in range(5): - root_block = call_async( + root_block = (await master0.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master0.add_root_block(root_block)) + (await master0.add_root_block(root_block)) root_block_list.append(root_block) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: master0.root_state.tip == root_block_list[-1].header ) # Add 2+2 blocks to another cluster master1 = clusters[1].master for i in range(2): - call_async(master1.add_root_block(root_block_list[i])) + (await master1.add_root_block(root_block_list[i])) for i in range(2): - root_block = call_async( + root_block = (await master1.get_next_block_to_mine(acc1, branch_value=None) ) - call_async(master1.add_root_block(root_block)) + (await master1.add_root_block(root_block)) master1.synchronizer.root_block_header_list_limit = 3 # Lookup will be [0, 2, 4], and then [3], where 3 cannot be found and thus 2 is the best. # Connect and the synchronizer should automically download - call_async( + (await clusters[1].network.connect( "127.0.0.1", clusters[0].network.env.cluster_config.P2P_PORT ) ) - assert_true_with_timeout( + await async_assert_true_with_timeout( lambda: master1.root_state.tip == root_block_list[-1].header ) self.assertEqual(master1.synchronizer.stats.blocks_downloaded, 3) self.assertEqual(master1.synchronizer.stats.headers_downloaded, 4 + 3) self.assertEqual(master1.synchronizer.stats.ancestor_lookup_requests, 2) - def test_get_minor_block_headers_with_skip(self): + async def test_get_minor_block_headers_with_skip(self): """ Test the broadcast is only done to the neighbors """ id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext(2, acc1) as clusters: + async with ClusterContext(2, acc1) as clusters: master = clusters[0].master shard = next(iter(clusters[0].slave_list[0].shards.values())) @@ -2260,7 +2260,7 @@ def test_get_minor_block_headers_with_skip(self): branch = shard.state.header_tip.branch for i in range(10): b = shard.state.create_block_to_mine() - call_async(master.add_raw_minor_block(b.header.branch, b.serialize())) + (await master.add_raw_minor_block(b.header.branch, b.serialize())) minor_block_header_list.append(b.header) self.assertEqual(minor_block_header_list[-1].height, 10) @@ -2268,7 +2268,7 @@ def test_get_minor_block_headers_with_skip(self): peer = next(iter(clusters[1].slave_list[0].shards[branch].peers.values())) # Test Case 1 ################################################### - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_MINOR_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetMinorBlockHeaderListWithSkipRequest.create_for_height( @@ -2285,7 +2285,7 @@ def test_get_minor_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[1], minor_block_header_list[3]) self.assertEqual(resp.block_header_list[2], minor_block_header_list[5]) - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_MINOR_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetMinorBlockHeaderListWithSkipRequest.create_for_hash( @@ -2303,7 +2303,7 @@ def test_get_minor_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[2], minor_block_header_list[5]) # Test Case 2 ################################################### - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_MINOR_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetMinorBlockHeaderListWithSkipRequest.create_for_height( @@ -2320,7 +2320,7 @@ def test_get_minor_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[1], minor_block_header_list[5]) self.assertEqual(resp.block_header_list[2], minor_block_header_list[8]) - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_MINOR_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetMinorBlockHeaderListWithSkipRequest.create_for_hash( @@ -2338,7 +2338,7 @@ def test_get_minor_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[2], minor_block_header_list[8]) # Test Case 3 ################################################### - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_MINOR_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetMinorBlockHeaderListWithSkipRequest.create_for_height( @@ -2357,7 +2357,7 @@ def test_get_minor_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[3], minor_block_header_list[9]) self.assertEqual(resp.block_header_list[4], minor_block_header_list[10]) - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_MINOR_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetMinorBlockHeaderListWithSkipRequest.create_for_hash( @@ -2377,7 +2377,7 @@ def test_get_minor_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[4], minor_block_header_list[10]) # Test Case 4 ################################################### - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_MINOR_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetMinorBlockHeaderListWithSkipRequest.create_for_height( @@ -2391,7 +2391,7 @@ def test_get_minor_block_headers_with_skip(self): ) self.assertEqual(len(resp.block_header_list), 1) self.assertEqual(resp.block_header_list[0], minor_block_header_list[2]) - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_MINOR_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetMinorBlockHeaderListWithSkipRequest.create_for_hash( @@ -2407,7 +2407,7 @@ def test_get_minor_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[0], minor_block_header_list[2]) # Test Case 5 ################################################### - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_MINOR_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetMinorBlockHeaderListWithSkipRequest.create_for_height( @@ -2421,7 +2421,7 @@ def test_get_minor_block_headers_with_skip(self): ) self.assertEqual(len(resp.block_header_list), 0) - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_MINOR_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetMinorBlockHeaderListWithSkipRequest.create_for_hash( @@ -2436,7 +2436,7 @@ def test_get_minor_block_headers_with_skip(self): self.assertEqual(len(resp.block_header_list), 0) # Test Case 6 ################################################### - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_MINOR_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetMinorBlockHeaderListWithSkipRequest.create_for_height( @@ -2455,7 +2455,7 @@ def test_get_minor_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[3], minor_block_header_list[2]) self.assertEqual(resp.block_header_list[4], minor_block_header_list[0]) - op, resp, rpc_id = call_async( + op, resp, rpc_id = (await peer.write_rpc_request( op=CommandOp.GET_MINOR_BLOCK_HEADER_LIST_WITH_SKIP_REQUEST, cmd=GetMinorBlockHeaderListWithSkipRequest.create_for_hash( @@ -2474,26 +2474,26 @@ def test_get_minor_block_headers_with_skip(self): self.assertEqual(resp.block_header_list[3], minor_block_header_list[2]) self.assertEqual(resp.block_header_list[4], minor_block_header_list[0]) - def test_posw_on_root_chain(self): + async def test_posw_on_root_chain(self): """ Test the broadcast is only done to the neighbors """ staker_id = Identity.create_random_identity() staker_addr = Address.create_from_identity(staker_id, full_shard_key=0) signer_id = Identity.create_random_identity() signer_addr = Address.create_from_identity(signer_id, full_shard_key=0) - def add_root_block(addr, sign=False): - root_block = call_async( + async def add_root_block(addr, sign=False): + root_block = (await master.get_next_block_to_mine(addr, branch_value=None) ) # type: RootBlock if sign: root_block.header.sign_with_private_key(PrivateKey(signer_id.get_key())) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) - with ClusterContext(1, staker_addr, shard_size=1) as clusters: + async with ClusterContext(1, staker_addr, shard_size=1) as clusters: master = clusters[0].master # add a root block first to init shard chains - add_root_block(Address.create_empty_account()) + await add_root_block(Address.create_empty_account()) qkc_config = master.env.quark_chain_config qkc_config.ROOT.CONSENSUS_TYPE = ConsensusType.POW_DOUBLESHA256 @@ -2519,14 +2519,14 @@ def mock_get_root_chain_stakes(recipient, _): # fail, because signature mismatch with self.assertRaises(ValueError): - add_root_block(staker_addr) + await add_root_block(staker_addr) # succeed - add_root_block(staker_addr, sign=True) + await add_root_block(staker_addr, sign=True) # fail again, because quota used up with self.assertRaises(ValueError): - add_root_block(staker_addr, sign=True) + await add_root_block(staker_addr, sign=True) - def test_total_balance_handle_xshard_deposit(self): + async def test_total_balance_handle_xshard_deposit(self): """ Test the cross shard transactions are broadcasted to the destination shards """ id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -2534,7 +2534,7 @@ def test_total_balance_handle_xshard_deposit(self): qkc_token = token_id_encode("QKC") init_coinbase = 1000000 - with ClusterContext( + async with ClusterContext( 1, acc1, chain_size=2, @@ -2550,12 +2550,12 @@ def test_total_balance_handle_xshard_deposit(self): # add a root block first so that later minor blocks referring to this root # can be broadcasted to other shards - root_block = call_async( + root_block = (await master.get_next_block_to_mine( Address.create_empty_account(), branch_value=None ) ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) balance, _ = state2.get_total_balance( qkc_token, @@ -2578,19 +2578,19 @@ def test_total_balance_handle_xshard_deposit(self): self.assertTrue(slaves[0].add_tx(tx)) b1 = state1.create_block_to_mine(address=acc1) - call_async(clusters[0].get_shard(1).add_block(b1)) + (await clusters[0].get_shard(1).add_block(b1)) # add two blocks to shard 1, while only make the first included by root block b2s = [] for _ in range(2): b2 = state2.create_block_to_mine(address=acc2) - call_async(clusters[0].get_shard((1 << 16) + 1).add_block(b2)) + (await clusters[0].get_shard((1 << 16) + 1).add_block(b2)) b2s.append(b2) # add a root block so the xshard tx can be recorded root_block = master.root_state.create_block_to_mine( [b1.header, b2s[0].header], acc1 ) - call_async(master.add_root_block(root_block)) + (await master.add_root_block(root_block)) # check source shard balance, _ = state1.get_total_balance( @@ -2616,7 +2616,7 @@ def test_total_balance_handle_xshard_deposit(self): # query latest header, deposit should be executed, regardless of root block # once next block is available b2 = state2.create_block_to_mine(address=acc2) - call_async(clusters[0].get_shard((1 << 16) + 1).add_block(b2)) + (await clusters[0].get_shard((1 << 16) + 1).add_block(b2)) for rh in [None, root_block.header.get_hash()]: balance, _ = state2.get_total_balance( qkc_token, state2.header_tip.get_hash(), rh, 100, None diff --git a/quarkchain/cluster/tests/test_filter.py b/quarkchain/cluster/tests/test_filter.py index 454bd4b72..8b49c81ae 100644 --- a/quarkchain/cluster/tests/test_filter.py +++ b/quarkchain/cluster/tests/test_filter.py @@ -1,4 +1,5 @@ import unittest +import asyncio from copy import copy from quarkchain.cluster.log_filter import LogFilter @@ -14,9 +15,9 @@ import random -class TestFilter(unittest.TestCase): - def setUp(self): - super().setUp() +class TestFilter(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + await super().asyncSetUp() id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -88,14 +89,14 @@ def filter_gen_with_criteria(criteria, addresses=None, option="default"): self.filter_gen_with_criteria = filter_gen_with_criteria - def test_bloom_bits_in_cstor(self): + async def test_bloom_bits_in_cstor(self): criteria = [[tp] for tp in self.log.topics] f = self.filter_gen_with_criteria(criteria) # use sha3(b'Hi(address)') to test bits expected_indexes = bits_in_number(bloom(sha3_256(b"Hi(address)"))) self.assertEqual(expected_indexes, bits_in_number(f.bloom_bits[0][0])) - def test_get_block_candidates_hit(self): + async def test_get_block_candidates_hit(self): hit_criteria = [ [[tp] for tp in self.log.topics], # exact match [[self.log.topics[0]], []], # one wild card @@ -113,7 +114,7 @@ def test_get_block_candidates_hit(self): self.assertEqual(len(blocks), 1) self.assertEqual(blocks[0].header.height, self.start_height) - def test_get_block_candidates_miss(self): + async def test_get_block_candidates_miss(self): miss_criteria = [ [[self.log.topics[0]], [bytes.fromhex("1234")]] # one miss match ] @@ -122,7 +123,7 @@ def test_get_block_candidates_miss(self): blocks = f._get_block_candidates() self.assertEqual(len(blocks), 0) - def test_log_topics_match(self): + async def test_log_topics_match(self): criteria = [[tp] for tp in self.log.topics] f = self.filter_gen_with_criteria(criteria) log = copy(self.log) @@ -137,14 +138,14 @@ def test_log_topics_match(self): f = self.filter_gen_with_criteria(criteria) self.assertTrue(f._log_topics_match(log)) - def test_get_logs(self): + async def test_get_logs(self): criteria = [[tp] for tp in self.log.topics] addresses = [Address(self.log.recipient, 0)] f = self.filter_gen_with_criteria(criteria, addresses) logs = f._get_logs([self.hit_block]) self.assertListEqual([self.log], logs) - def test_get_block_candidates_height_ascending(self): + async def test_get_block_candidates_height_ascending(self): criteria = [] addresses = [] f = self.filter_gen_with_criteria(criteria, addresses) diff --git a/quarkchain/cluster/tests/test_jsonrpc.py b/quarkchain/cluster/tests/test_jsonrpc.py index 5a4050e98..1b76ea356 100644 --- a/quarkchain/cluster/tests/test_jsonrpc.py +++ b/quarkchain/cluster/tests/test_jsonrpc.py @@ -2,8 +2,7 @@ import json import logging import unittest -from contextlib import contextmanager - +from contextlib import asynccontextmanager import aiohttp from jsonrpcclient.aiohttp_client import aiohttpClient from jsonrpcclient.exceptions import ReceivedErrorResponse @@ -35,55 +34,51 @@ from quarkchain.env import DEFAULT_ENV from quarkchain.evm.messages import mk_contract_address from quarkchain.evm.transactions import Transaction as EvmTransaction -from quarkchain.utils import call_async, sha3_256, token_id_encode - +from quarkchain.utils import sha3_256, token_id_encode # disable jsonrpcclient verbose logging logging.getLogger("jsonrpcclient.client.request").setLevel(logging.WARNING) logging.getLogger("jsonrpcclient.client.response").setLevel(logging.WARNING) -@contextmanager -def jrpc_http_server_context(master): +@asynccontextmanager +async def jrpc_http_server_context(master): env = DEFAULT_ENV.copy() env.cluster_config = ClusterConfig() env.cluster_config.JSON_RPC_PORT = 38391 # to pass the circleCi env.cluster_config.JSON_RPC_HOST = "127.0.0.1" - server = JSONRPCHttpServer.start_test_server(env, master) + server = await JSONRPCHttpServer.start_test_server(env, master) try: yield server finally: - server.shutdown() + await server.shutdown() -def send_request(*args): - async def __send_request(*args): - async with aiohttp.ClientSession(loop=asyncio.get_event_loop()) as session: - client = aiohttpClient(session, "http://localhost:38391") - response = await client.request(*args) - return response +async def send_request(*args): + async with aiohttp.ClientSession() as session: + client = aiohttpClient(session, "http://localhost:38391") + response = await client.request(*args) + return response - return call_async(__send_request(*args)) - -class TestJSONRPCHttp(unittest.TestCase): - def test_getTransactionCount(self): +class TestJSONRPCHttp(unittest.IsolatedAsyncioTestCase): + async def test_getTransactionCount(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_random_account(full_shard_key=1) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master slaves = clusters[0].slave_list - stats = call_async(master.get_stats()) + stats = await master.get_stats() self.assertTrue("posw" in json.dumps(stats)) self.assertEqual( - call_async(master.get_primary_account_data(acc1)).transaction_count, 0 + (await master.get_primary_account_data(acc1)).transaction_count, 0 ) for i in range(3): tx = create_transfer_transaction( @@ -95,65 +90,65 @@ def test_getTransactionCount(self): ) self.assertTrue(slaves[0].add_tx(tx)) - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) self.assertEqual(i + 1, block.header.height) self.assertTrue( - call_async(clusters[0].get_shard(2 | 0).add_block(block)) + (await clusters[0].get_shard(2 | 0).add_block(block)) ) - response = send_request( + response = await send_request( "getTransactionCount", ["0x" + acc2.serialize().hex()] ) self.assertEqual(response, "0x0") - response = send_request( + response = await send_request( "getTransactionCount", ["0x" + acc1.serialize().hex()] ) self.assertEqual(response, "0x3") - response = send_request( + response = await send_request( "getTransactionCount", ["0x" + acc1.serialize().hex(), "latest"] ) self.assertEqual(response, "0x3") for i in range(3): - response = send_request( + response = await send_request( "getTransactionCount", ["0x" + acc1.serialize().hex(), hex(i + 1)] ) self.assertEqual(response, hex(i + 1)) - def test_getBalance(self): + async def test_getBalance(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): - response = send_request("getBalances", ["0x" + acc1.serialize().hex()]) + response = await send_request("getBalances", ["0x" + acc1.serialize().hex()]) self.assertListEqual( response["balances"], [{"tokenId": "0x8bb0", "tokenStr": "QKC", "balance": "0xf4240"}], ) - response = send_request("eth_getBalance", ["0x" + acc1.recipient.hex()]) + response = await send_request("eth_getBalance", ["0x" + acc1.recipient.hex()]) self.assertEqual(response, "0xf4240") - def test_sendTransaction(self): + async def test_sendTransaction(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_random_account(full_shard_key=1) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): slaves = clusters[0].slave_list master = clusters[0].master - block = call_async( - master.get_next_block_to_mine(address=acc2, branch_value=None) + block = await master.get_next_block_to_mine( + address=acc2, branch_value=None ) - call_async(master.add_root_block(block)) + await master.add_root_block(block) evm_tx = EvmTransaction( nonce=0, @@ -183,7 +178,7 @@ def test_sendTransaction(self): network_id=hex(slaves[0].env.quark_chain_config.NETWORK_ID), ) tx = TypedTransaction(SerializedEvmTransaction.from_evm_tx(evm_tx)) - response = send_request("sendTransaction", [request]) + response = await send_request("sendTransaction", [request]) self.assertEqual(response, "0x" + tx.get_hash().hex() + "00000000") state = clusters[0].get_shard_state(2 | 0) @@ -195,21 +190,21 @@ def test_sendTransaction(self): evm_tx, ) - def test_sendTransaction_with_bad_signature(self): + async def test_sendTransaction_with_bad_signature(self): """ sendTransaction validates signature """ id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_random_account(full_shard_key=1) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master - block = call_async( - master.get_next_block_to_mine(address=acc2, branch_value=None) + block = await master.get_next_block_to_mine( + address=acc2, branch_value=None ) - call_async(master.add_root_block(block)) + await master.add_root_block(block) request = dict( to="0x" + acc2.recipient.hex(), @@ -223,22 +218,22 @@ def test_sendTransaction_with_bad_signature(self): fromFullShardKey="0x00000000", toFullShardKey="0x00000001", ) - self.assertEqual(send_request("sendTransaction", [request]), EMPTY_TX_ID) + self.assertEqual(await send_request("sendTransaction", [request]), EMPTY_TX_ID) self.assertEqual(len(clusters[0].get_shard_state(2 | 0).tx_queue), 0) - def test_sendTransaction_missing_from_full_shard_key(self): + async def test_sendTransaction_missing_from_full_shard_key(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=None) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=None ) - call_async(master.add_root_block(block)) + await master.add_root_block(block) request = dict( to="0x" + acc1.recipient.hex(), @@ -252,20 +247,20 @@ def test_sendTransaction_missing_from_full_shard_key(self): ) with self.assertRaises(Exception): - send_request("sendTransaction", [request]) + await send_request("sendTransaction", [request]) - def test_getMinorBlock(self): + async def test_getMinorBlock(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master slaves = clusters[0].slave_list self.assertEqual( - call_async(master.get_primary_account_data(acc1)).transaction_count, 0 + (await master.get_primary_account_data(acc1)).transaction_count, 0 ) tx = create_transfer_transaction( shard_state=clusters[0].get_shard_state(2 | 0), @@ -276,14 +271,14 @@ def test_getMinorBlock(self): ) self.assertTrue(slaves[0].add_tx(tx)) - block1 = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + block1 = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(block1))) + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(block1))) # By id for need_extra_info in [True, False]: - resp = send_request( + resp = await send_request( "getMinorBlockById", [ "0x" + block1.header.get_hash().hex() + "0" * 8, @@ -295,7 +290,7 @@ def test_getMinorBlock(self): resp["transactions"][0], "0x" + tx.get_hash().hex() + "00000002" ) - resp = send_request( + resp = await send_request( "getMinorBlockById", ["0x" + block1.header.get_hash().hex() + "0" * 8, True], ) @@ -303,47 +298,47 @@ def test_getMinorBlock(self): resp["transactions"][0]["hash"], "0x" + tx.get_hash().hex() ) - resp = send_request("getMinorBlockById", ["0x" + "ff" * 36, True]) + resp = await send_request("getMinorBlockById", ["0x" + "ff" * 36, True]) self.assertIsNone(resp) # By height for need_extra_info in [True, False]: - resp = send_request( + resp = await send_request( "getMinorBlockByHeight", ["0x0", "0x1", False, need_extra_info] ) self.assertEqual( resp["transactions"][0], "0x" + tx.get_hash().hex() + "00000002" ) - resp = send_request("getMinorBlockByHeight", ["0x0", "0x1", True]) + resp = await send_request("getMinorBlockByHeight", ["0x0", "0x1", True]) self.assertEqual( resp["transactions"][0]["hash"], "0x" + tx.get_hash().hex() ) - resp = send_request("getMinorBlockByHeight", ["0x1", "0x2", False]) + resp = await send_request("getMinorBlockByHeight", ["0x1", "0x2", False]) self.assertIsNone(resp) - resp = send_request("getMinorBlockByHeight", ["0x0", "0x4", False]) + resp = await send_request("getMinorBlockByHeight", ["0x0", "0x4", False]) self.assertIsNone(resp) - def test_getRootblockConfirmationIdAndCount(self): + async def test_getRootblockConfirmationIdAndCount(self): # TODO test root chain forks id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master slaves = clusters[0].slave_list self.assertEqual( - call_async(master.get_primary_account_data(acc1)).transaction_count, 0 + (await master.get_primary_account_data(acc1)).transaction_count, 0 ) - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=None) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=None ) - call_async(master.add_root_block(block)) + await master.add_root_block(block) tx = create_transfer_transaction( shard_state=clusters[0].get_shard_state(2 | 0), @@ -354,17 +349,17 @@ def test_getRootblockConfirmationIdAndCount(self): ) self.assertTrue(slaves[0].add_tx(tx)) - block1 = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + block1 = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(block1))) + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(block1))) tx_id = ( "0x" + tx.get_hash().hex() + acc1.full_shard_key.to_bytes(4, "big").hex() ) - resp = send_request("getTransactionById", [tx_id]) + resp = await send_request("getTransactionById", [tx_id]) self.assertEqual(resp["hash"], "0x" + tx.get_hash().hex()) self.assertEqual( resp["blockId"], @@ -377,59 +372,59 @@ def test_getRootblockConfirmationIdAndCount(self): minor_hash = resp["blockId"] # zero root block confirmation - resp_hash = send_request( + resp_hash = await send_request( "getRootHashConfirmingMinorBlockById", [minor_hash] ) self.assertIsNone( resp_hash, "should return None for unconfirmed minor blocks" ) - resp_count = send_request( + resp_count = await send_request( "getTransactionConfirmedByNumberRootBlocks", [tx_id] ) self.assertEqual(resp_count, "0x0") # 1 root block confirmation - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=None) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=None ) - call_async(master.add_root_block(block)) - resp_hash = send_request( + await master.add_root_block(block) + resp_hash = await send_request( "getRootHashConfirmingMinorBlockById", [minor_hash] ) self.assertIsNotNone(resp_hash, "confirmed by root block") self.assertEqual(resp_hash, "0x" + block.header.get_hash().hex()) - resp_count = send_request( + resp_count = await send_request( "getTransactionConfirmedByNumberRootBlocks", [tx_id] ) self.assertEqual(resp_count, "0x1") # 2 root block confirmation - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=None) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=None ) - call_async(master.add_root_block(block)) - resp_hash = send_request( + await master.add_root_block(block) + resp_hash = await send_request( "getRootHashConfirmingMinorBlockById", [minor_hash] ) self.assertIsNotNone(resp_hash, "confirmed by root block") self.assertNotEqual(resp_hash, "0x" + block.header.get_hash().hex()) - resp_count = send_request( + resp_count = await send_request( "getTransactionConfirmedByNumberRootBlocks", [tx_id] ) self.assertEqual(resp_count, "0x2") - def test_getTransactionById(self): + async def test_getTransactionById(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master slaves = clusters[0].slave_list self.assertEqual( - call_async(master.get_primary_account_data(acc1)).transaction_count, 0 + (await master.get_primary_account_data(acc1)).transaction_count, 0 ) tx = create_transfer_transaction( shard_state=clusters[0].get_shard_state(2 | 0), @@ -440,12 +435,12 @@ def test_getTransactionById(self): ) self.assertTrue(slaves[0].add_tx(tx)) - block1 = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + block1 = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(block1))) + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(block1))) - resp = send_request( + resp = await send_request( "getTransactionById", [ "0x" @@ -455,16 +450,16 @@ def test_getTransactionById(self): ) self.assertEqual(resp["hash"], "0x" + tx.get_hash().hex()) - def test_call_success(self): + async def test_call_success(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): slaves = clusters[0].slave_list - response = send_request( + response = await send_request( "call", [{"to": "0x" + acc1.serialize().hex(), "gas": hex(21000)}] ) @@ -475,17 +470,17 @@ def test_call_success(self): "should not affect tx queue", ) - def test_call_success_default_gas(self): + async def test_call_success_default_gas(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): slaves = clusters[0].slave_list # gas is not specified in the request - response = send_request( + response = await send_request( "call", [{"to": "0x" + acc1.serialize().hex()}, "latest"] ) @@ -496,17 +491,17 @@ def test_call_success_default_gas(self): "should not affect tx queue", ) - def test_call_failure(self): + async def test_call_failure(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): slaves = clusters[0].slave_list # insufficient gas - response = send_request( + response = await send_request( "call", [{"to": "0x" + acc1.serialize().hex(), "gas": "0x1"}, None] ) @@ -517,22 +512,22 @@ def test_call_failure(self): "should not affect tx queue", ) - def test_getTransactionReceipt_not_exist(self): + async def test_getTransactionReceipt_not_exist(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): for endpoint in ("getTransactionReceipt", "eth_getTransactionReceipt"): - resp = send_request(endpoint, ["0x" + bytes(36).hex()]) + resp = await send_request(endpoint, ["0x" + bytes(36).hex()]) self.assertIsNone(resp) - def test_getTransactionReceipt_on_transfer(self): + async def test_getTransactionReceipt_on_transfer(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master @@ -547,13 +542,13 @@ def test_getTransactionReceipt_on_transfer(self): ) self.assertTrue(slaves[0].add_tx(tx)) - block1 = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + block1 = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(block1))) + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(block1))) for endpoint in ("getTransactionReceipt", "eth_getTransactionReceipt"): - resp = send_request( + resp = await send_request( endpoint, [ "0x" @@ -566,12 +561,12 @@ def test_getTransactionReceipt_on_transfer(self): self.assertEqual(resp["cumulativeGasUsed"], "0x5208") self.assertIsNone(resp["contractAddress"]) - def test_getTransactionReceipt_on_xshard_transfer_before_enabling_EVM(self): + async def test_getTransactionReceipt_on_xshard_transfer_before_enabling_EVM(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id1, full_shard_key=0x00010000) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master @@ -579,10 +574,10 @@ def test_getTransactionReceipt_on_xshard_transfer_before_enabling_EVM(self): # disable EVM to have fake xshard receipts master.env.quark_chain_config.ENABLE_EVM_TIMESTAMP = 2 ** 64 - 1 - block = call_async( - master.get_next_block_to_mine(address=acc2, branch_value=None) + block = await master.get_next_block_to_mine( + address=acc2, branch_value=None ) - call_async(master.add_root_block(block)) + await master.add_root_block(block) s1, s2 = ( clusters[0].get_shard_state(2 | 0), @@ -598,30 +593,30 @@ def test_getTransactionReceipt_on_xshard_transfer_before_enabling_EVM(self): ) tx1 = tx_gen(s1, acc1, acc2) self.assertTrue(slaves[0].add_tx(tx1)) - b1 = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + b1 = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(b1))) + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(b1))) - root_block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=None) + root_block = await master.get_next_block_to_mine( + address=acc1, branch_value=None ) - call_async(master.add_root_block(root_block)) + await master.add_root_block(root_block) tx2 = tx_gen(s2, acc2, acc2) self.assertTrue(slaves[0].add_tx(tx2)) - b3 = call_async( - master.get_next_block_to_mine(address=acc2, branch_value=0x00010002) + b3 = await master.get_next_block_to_mine( + address=acc2, branch_value=0x00010002 ) - self.assertTrue(call_async(clusters[0].get_shard(0x00010002).add_block(b3))) + self.assertTrue((await clusters[0].get_shard(0x00010002).add_block(b3))) # in-shard tx 21000 + receiving x-shard tx 9000 self.assertEqual(s2.evm_state.gas_used, 30000) self.assertEqual(s2.evm_state.xshard_receive_gas_used, 9000) for endpoint in ("getTransactionReceipt", "eth_getTransactionReceipt"): - resp = send_request( + resp = await send_request( endpoint, [ "0x" @@ -636,7 +631,7 @@ def test_getTransactionReceipt_on_xshard_transfer_before_enabling_EVM(self): self.assertIsNone(resp["contractAddress"]) # query xshard tx receipt on the target shard - resp = send_request( + resp = await send_request( endpoint, [ "0x" @@ -649,21 +644,21 @@ def test_getTransactionReceipt_on_xshard_transfer_before_enabling_EVM(self): self.assertEqual(resp["cumulativeGasUsed"], hex(0)) self.assertEqual(resp["gasUsed"], hex(0)) - def test_getTransactionReceipt_on_xshard_transfer_after_enabling_EVM(self): + async def test_getTransactionReceipt_on_xshard_transfer_after_enabling_EVM(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id1, full_shard_key=1) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master slaves = clusters[0].slave_list - block = call_async( - master.get_next_block_to_mine(address=acc2, branch_value=None) + block = await master.get_next_block_to_mine( + address=acc2, branch_value=None ) - call_async(master.add_root_block(block)) + await master.add_root_block(block) s1, s2 = ( clusters[0].get_shard_state(2 | 0), @@ -679,23 +674,23 @@ def test_getTransactionReceipt_on_xshard_transfer_after_enabling_EVM(self): ) self.assertTrue(slaves[0].add_tx(tx)) # source shard - b1 = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + b1 = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(b1))) + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(b1))) # root chain - root_block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=None) + root_block = await master.get_next_block_to_mine( + address=acc1, branch_value=None ) - call_async(master.add_root_block(root_block)) + await master.add_root_block(root_block) # target shard - b3 = call_async( - master.get_next_block_to_mine(address=acc2, branch_value=0b11) + b3 = await master.get_next_block_to_mine( + address=acc2, branch_value=0b11 ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 1).add_block(b3))) + self.assertTrue((await clusters[0].get_shard(2 | 1).add_block(b3))) # query xshard tx receipt on the target shard - resp = send_request( + resp = await send_request( "getTransactionReceipt", [ "0x" @@ -708,11 +703,11 @@ def test_getTransactionReceipt_on_xshard_transfer_after_enabling_EVM(self): self.assertEqual(resp["cumulativeGasUsed"], hex(9000)) self.assertEqual(resp["gasUsed"], hex(9000)) - def test_getTransactionReceipt_on_contract_creation(self): + async def test_getTransactionReceipt_on_contract_creation(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master @@ -727,13 +722,13 @@ def test_getTransactionReceipt_on_contract_creation(self): ) self.assertTrue(slaves[0].add_tx(tx)) - block1 = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + block1 = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(block1))) + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(block1))) for endpoint in ("getTransactionReceipt", "eth_getTransactionReceipt"): - resp = send_request(endpoint, ["0x" + tx.get_hash().hex() + "00000002"]) + resp = await send_request(endpoint, ["0x" + tx.get_hash().hex() + "00000002"]) self.assertEqual(resp["transactionHash"], "0x" + tx.get_hash().hex()) self.assertEqual(resp["status"], "0x1") self.assertEqual(resp["cumulativeGasUsed"], "0x213eb") @@ -748,11 +743,11 @@ def test_getTransactionReceipt_on_contract_creation(self): + to_full_shard_key.to_bytes(4, "big").hex(), ) - def test_getTransactionReceipt_on_xshard_contract_creation(self): + async def test_getTransactionReceipt_on_xshard_contract_creation(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master @@ -760,10 +755,10 @@ def test_getTransactionReceipt_on_xshard_contract_creation(self): # Add a root block to update block gas limit for xshard tx throttling # so that the following tx can be processed - root_block = call_async( - master.get_next_block_to_mine(acc1, branch_value=None) + root_block = await master.get_next_block_to_mine( + acc1, branch_value=None ) - call_async(master.add_root_block(root_block)) + await master.add_root_block(root_block) to_full_shard_key = acc1.full_shard_key + 1 tx = create_contract_creation_with_event_transaction( @@ -774,35 +769,35 @@ def test_getTransactionReceipt_on_xshard_contract_creation(self): ) self.assertTrue(slaves[0].add_tx(tx)) - block1 = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + block1 = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(block1))) + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(block1))) for endpoint in ("getTransactionReceipt", "eth_getTransactionReceipt"): - resp = send_request(endpoint, ["0x" + tx.get_hash().hex() + "00000002"]) + resp = await send_request(endpoint, ["0x" + tx.get_hash().hex() + "00000002"]) self.assertEqual(resp["transactionHash"], "0x" + tx.get_hash().hex()) self.assertEqual(resp["status"], "0x1") self.assertEqual(resp["cumulativeGasUsed"], "0x11374") self.assertIsNone(resp["contractAddress"]) # x-shard contract creation should succeed. check target shard - root_block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=None) + root_block = await master.get_next_block_to_mine( + address=acc1, branch_value=None ) # root chain - call_async(master.add_root_block(root_block)) - block2 = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b11) + await master.add_root_block(root_block) + block2 = await master.get_next_block_to_mine( + address=acc1, branch_value=0b11 ) # target shard - self.assertTrue(call_async(clusters[0].get_shard(2 | 1).add_block(block2))) + self.assertTrue((await clusters[0].get_shard(2 | 1).add_block(block2))) for endpoint in ("getTransactionReceipt", "eth_getTransactionReceipt"): - resp = send_request(endpoint, ["0x" + tx.get_hash().hex() + "00000003"]) + resp = await send_request(endpoint, ["0x" + tx.get_hash().hex() + "00000003"]) self.assertEqual(resp["transactionHash"], "0x" + tx.get_hash().hex()) self.assertEqual(resp["status"], "0x1") self.assertEqual(resp["cumulativeGasUsed"], "0xc515") self.assertIsNotNone(resp["contractAddress"]) - def test_getLogs(self): + async def test_getLogs(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -814,7 +809,7 @@ def test_getLogs(self): "data": "0x", } - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True, genesis_minor_quarkash=10000000 ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master @@ -822,10 +817,10 @@ def test_getLogs(self): # Add a root block to update block gas limit for xshard tx throttling # so that the following tx can be processed - root_block = call_async( - master.get_next_block_to_mine(acc1, branch_value=None) + root_block = await master.get_next_block_to_mine( + acc1, branch_value=None ) - call_async(master.add_root_block(root_block)) + await master.add_root_block(root_block) tx = create_contract_creation_with_event_transaction( shard_state=clusters[0].get_shard_state(2 | 0), @@ -836,10 +831,10 @@ def test_getLogs(self): expected_log_parts["transactionHash"] = "0x" + tx.get_hash().hex() self.assertTrue(slaves[0].add_tx(tx)) - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(block))) + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(block))) for using_eth_endpoint in (True, False): shard_id = hex(acc1.full_shard_key) @@ -850,15 +845,15 @@ def test_getLogs(self): req = lambda o: send_request("getLogs", [o, shard_id]) # no filter object as wild cards - resp = req({}) + resp = await req({}) self.assertEqual(1, len(resp)) self.assertDictContainsSubset(expected_log_parts, resp[0]) # filter with from/to blocks - resp = req({"fromBlock": "0x0", "toBlock": "0x1"}) + resp = await req({"fromBlock": "0x0", "toBlock": "0x1"}) self.assertEqual(1, len(resp)) self.assertDictContainsSubset(expected_log_parts, resp[0]) - resp = req({"fromBlock": "0x0", "toBlock": "0x0"}) + resp = await req({"fromBlock": "0x0", "toBlock": "0x0"}) self.assertEqual(0, len(resp)) # filter by contract address @@ -874,7 +869,7 @@ def test_getLogs(self): else hex(acc1.full_shard_key)[2:].zfill(8) ) } - resp = req(filter_obj) + resp = await req(filter_obj) self.assertEqual(1, len(resp)) # filter by topics @@ -891,7 +886,7 @@ def test_getLogs(self): ] } for f in (filter_obj, filter_obj_nested): - resp = req(f) + resp = await req(f) self.assertEqual(1, len(resp)) self.assertDictContainsSubset(expected_log_parts, resp[0]) self.assertEqual( @@ -907,22 +902,22 @@ def test_getLogs(self): to_full_shard_key=acc1.full_shard_key + 1, ) self.assertTrue(slaves[0].add_tx(tx)) - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) # source shard - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(block))) - root_block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=None) + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(block))) + root_block = await master.get_next_block_to_mine( + address=acc1, branch_value=None ) # root chain - call_async(master.add_root_block(root_block)) - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b11) + await master.add_root_block(root_block) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=0b11 ) # target shard - self.assertTrue(call_async(clusters[0].get_shard(2 | 1).add_block(block))) + self.assertTrue((await clusters[0].get_shard(2 | 1).add_block(block))) req = lambda o: send_request("getLogs", [o, hex(0b11)]) # no filter object as wild cards - resp = req({}) + resp = await req({}) self.assertEqual(1, len(resp)) expected_log_parts["transactionIndex"] = "0x3" # after root block coinbase expected_log_parts["transactionHash"] = "0x" + tx.get_hash().hex() @@ -932,27 +927,27 @@ def test_getLogs(self): # missing shard ID should fail for endpoint in ("getLogs", "eth_getLogs"): with self.assertRaises(ReceivedErrorResponse): - send_request(endpoint, [{}]) + await send_request(endpoint, [{}]) with self.assertRaises(ReceivedErrorResponse): - send_request(endpoint, [{}, None]) + await send_request(endpoint, [{}, None]) - def test_estimateGas(self): + async def test_estimateGas(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): payload = {"to": "0x" + acc1.serialize().hex()} - response = send_request("estimateGas", [payload]) + response = await send_request("estimateGas", [payload]) self.assertEqual(response, "0x5208") # 21000 # cross-shard from_addr = "0x" + acc1.address_in_shard(1).serialize().hex() payload["from"] = from_addr - response = send_request("estimateGas", [payload]) + response = await send_request("estimateGas", [payload]) self.assertEqual(response, "0x7530") # 30000 - def test_getStorageAt(self): + async def test_getStorageAt(self): key = bytes.fromhex( "c987d4506fb6824639f9a9e3b8834584f5165e94680501d1b0044071cd36c3b3" ) @@ -960,7 +955,7 @@ def test_getStorageAt(self): acc1 = Address.create_from_identity(id1, full_shard_key=0) created_addr = "0x8531eb33bba796115f56ffa1b7df1ea3acdd8cdd00000000" - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master @@ -974,10 +969,10 @@ def test_getStorageAt(self): ) self.assertTrue(slaves[0].add_tx(tx)) - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(block))) + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(block))) for using_eth_endpoint in (True, False): if using_eth_endpoint: @@ -988,7 +983,7 @@ def test_getStorageAt(self): req = lambda k: send_request("getStorageAt", [created_addr, k]) # first storage - response = req("0x0") + response = await req("0x0") # equals 1234 self.assertEqual( response, @@ -999,20 +994,20 @@ def test_getStorageAt(self): k = sha3_256( bytes.fromhex(acc1.recipient.hex().zfill(64) + "1".zfill(64)) ) - response = req("0x" + k.hex()) + response = await req("0x" + k.hex()) self.assertEqual( response, "0x000000000000000000000000000000000000000000000000000000000000162e", ) # doesn't exist - response = req("0x3") + response = await req("0x3") self.assertEqual( response, "0x0000000000000000000000000000000000000000000000000000000000000000", ) - def test_getCode(self): + async def test_getCode(self): key = bytes.fromhex( "c987d4506fb6824639f9a9e3b8834584f5165e94680501d1b0044071cd36c3b3" ) @@ -1020,7 +1015,7 @@ def test_getCode(self): acc1 = Address.create_from_identity(id1, full_shard_key=0) created_addr = "0x8531eb33bba796115f56ffa1b7df1ea3acdd8cdd00000000" - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master @@ -1034,27 +1029,27 @@ def test_getCode(self): ) self.assertTrue(slaves[0].add_tx(tx)) - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(block))) + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(block))) for using_eth_endpoint in (True, False): if using_eth_endpoint: - resp = send_request("eth_getCode", [created_addr[:-8], "0x0"]) + resp = await send_request("eth_getCode", [created_addr[:-8], "0x0"]) else: - resp = send_request("getCode", [created_addr]) + resp = await send_request("getCode", [created_addr]) self.assertEqual( resp, "0x6080604052600080fd00a165627a7a72305820a6ef942c101f06333ac35072a8ff40332c71d0e11cd0e6d86de8cae7b42696550029", ) - def test_gasPrice(self): + async def test_gasPrice(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master @@ -1072,28 +1067,28 @@ def test_gasPrice(self): ) self.assertTrue(slaves[0].add_tx(tx)) - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) self.assertTrue( - call_async(clusters[0].get_shard(2 | 0).add_block(block)) + (await clusters[0].get_shard(2 | 0).add_block(block)) ) for using_eth_endpoint in (True, False): if using_eth_endpoint: - resp = send_request("eth_gasPrice", ["0x0"]) + resp = await send_request("eth_gasPrice", ["0x0"]) else: - resp = send_request( + resp = await send_request( "gasPrice", ["0x0", quantity_encoder(token_id_encode("QKC"))] ) self.assertEqual(resp, "0xc") - def test_getWork_and_submitWork(self): + async def test_getWork_and_submitWork(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, remote_mining=True, shard_size=1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master @@ -1110,7 +1105,7 @@ def test_getWork_and_submitWork(self): self.assertTrue(slaves[0].add_tx(tx)) for shard_id in ["0x0", None]: # shard, then root - resp = send_request("getWork", [shard_id]) + resp = await send_request("getWork", [shard_id]) self.assertEqual(resp[1:], ["0x1", "0xa"]) # height and diff header_hash_hex = resp[0] @@ -1122,17 +1117,15 @@ def test_getWork_and_submitWork(self): miner_address = Address.create_from( master.env.quark_chain_config.ROOT.COINBASE_ADDRESS ) - block = call_async( - master.get_next_block_to_mine( - address=miner_address, branch_value=shard_id and 0b01 - ) + block = await master.get_next_block_to_mine( + address=miner_address, branch_value=shard_id and 0b01 ) # solve it and submit work = MiningWork(bytes.fromhex(header_hash_hex[2:]), 1, 10) solver = DoubleSHA256(work) nonce = solver.mine(0, 10000).nonce mixhash = "0x" + sha3_256(b"").hex() - resp = send_request( + resp = await send_request( "submitWork", [ shard_id, @@ -1149,11 +1142,11 @@ def test_getWork_and_submitWork(self): clusters[0].get_shard_state(1 | 0).get_tip().header.height, 1 ) - def test_getWork_with_optional_diff_divider(self): + async def test_getWork_with_optional_diff_divider(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, remote_mining=True, shard_size=1, small_coinbase=True ) as clusters, jrpc_http_server_context(clusters[0].master): master = clusters[0].master @@ -1163,10 +1156,10 @@ def test_getWork_with_optional_diff_divider(self): qkc_config.ROOT.CONSENSUS_TYPE = ConsensusType.POW_SIMULATE # add a root block first to init shard chains - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=None) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=None ) - call_async(master.add_root_block(block)) + await master.add_root_block(block) qkc_config.ROOT.POSW_CONFIG.ENABLED = True qkc_config.ROOT.POSW_CONFIG.ENABLE_TIMESTAMP = 0 @@ -1177,11 +1170,11 @@ def test_getWork_with_optional_diff_divider(self): acc1.recipient, ) - resp = send_request("getWork", [None]) + resp = await send_request("getWork", [None]) # height and diff, and returns the diff divider since it's PoSW mineable self.assertEqual(resp[1:], ["0x2", "0xa", hex(1000)]) - def test_createTransactions(self): + async def test_createTransactions(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_random_account(full_shard_key=1) @@ -1197,23 +1190,23 @@ def test_createTransactions(self): }, ] - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True, loadtest_accounts=loadtest_accounts ) as clusters, jrpc_http_server_context(clusters[0].master): slaves = clusters[0].slave_list master = clusters[0].master - block = call_async( - master.get_next_block_to_mine(address=acc2, branch_value=None) + block = await master.get_next_block_to_mine( + address=acc2, branch_value=None ) - call_async(master.add_root_block(block)) + await master.add_root_block(block) - send_request("createTransactions", {"numTxPerShard": 1, "xShardPercent": 0}) + await send_request("createTransactions", {"numTxPerShard": 1, "xShardPercent": 0}) # ------------------------------- Test for JSONRPCWebsocketServer ------------------------------- -@contextmanager -def jrpc_websocket_server_context(slave_server, port=38590): +@asynccontextmanager +async def jrpc_websocket_server_context(slave_server, port=38590): env = DEFAULT_ENV.copy() env.cluster_config = ClusterConfig() env.cluster_config.JSON_RPC_PORT = 38391 @@ -1222,27 +1215,24 @@ def jrpc_websocket_server_context(slave_server, port=38590): env.slave_config = env.cluster_config.get_slave_config("S0") env.slave_config.HOST = "0.0.0.0" env.slave_config.WEBSOCKET_JSON_RPC_PORT = port - server = JSONRPCWebsocketServer.start_websocket_server(env, slave_server) + server = await JSONRPCWebsocketServer.start_websocket_server(env, slave_server) try: yield server finally: server.shutdown() -def send_websocket_request(request, num_response=1, port=38590): +async def send_websocket_request(request, num_response=1, port=38590): responses = [] - async def __send_request(request, port): - uri = "ws://0.0.0.0:" + str(port) - async with websockets.connect(uri) as websocket: - await websocket.send(request) - while True: - response = await websocket.recv() - responses.append(response) - if len(responses) == num_response: - return responses - - return call_async(__send_request(request, port)) + uri = "ws://0.0.0.0:" + str(port) + async with websockets.connect(uri) as websocket: + await websocket.send(request) + while True: + response = await websocket.recv() + responses.append(response) + if len(responses) == num_response: + return responses async def get_websocket(port=38590): @@ -1250,12 +1240,16 @@ async def get_websocket(port=38590): return await websockets.connect(uri) -class TestJSONRPCWebsocket(unittest.TestCase): - def test_new_heads(self): +class TestJSONRPCWebsocket(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.loop = asyncio.get_event_loop() + self.loop.set_debug(False) + + async def test_new_heads(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_websocket_server_context(clusters[0].slave_list[0]): # clusters[0].slave_list[0] has two shards with full_shard_id 2 and 3 @@ -1267,20 +1261,20 @@ def test_new_heads(self): "params": ["newHeads", "0x00000002"], "id": 3, } - websocket = call_async(get_websocket()) - call_async(websocket.send(json.dumps(request))) - response = call_async(websocket.recv()) + websocket = await get_websocket() + await websocket.send(json.dumps(request)) + response = await websocket.recv() response = json.loads(response) self.assertEqual(response["id"], 3) - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(block))) + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(block))) block_hash = block.header.get_hash() block_height = block.header.height - response = call_async(websocket.recv()) + response = await websocket.recv() response = json.loads(response) self.assertEqual( response["params"]["result"]["hash"], data_encoder(block_hash) @@ -1289,16 +1283,16 @@ def test_new_heads(self): response["params"]["result"]["height"], quantity_encoder(block_height) ) - def test_new_heads_with_chain_reorg(self): + async def test_new_heads_with_chain_reorg(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True, genesis_minor_quarkash=10000000 ) as clusters, jrpc_websocket_server_context( clusters[0].slave_list[0], port=38591 ): - websocket = call_async(get_websocket(port=38591)) + websocket = await get_websocket(port=38591) request = { "jsonrpc": "2.0", @@ -1306,8 +1300,8 @@ def test_new_heads_with_chain_reorg(self): "params": ["newHeads", "0x00000002"], "id": 3, } - call_async(websocket.send(json.dumps(request))) - response = call_async(websocket.recv()) + await websocket.send(json.dumps(request)) + response = await websocket.recv() response = json.loads(response) self.assertEqual(response["id"], 3) @@ -1318,7 +1312,7 @@ def test_new_heads_with_chain_reorg(self): b0 = state.create_block_to_mine(address=acc1) state.finalize_and_add_block(b0) self.assertEqual(state.header_tip, b0.header) - response = call_async(websocket.recv()) + response = await websocket.recv() d = json.loads(response) self.assertEqual( d["params"]["result"]["hash"], data_encoder(b0.header.get_hash()) @@ -1334,28 +1328,28 @@ def test_new_heads_with_chain_reorg(self): # new heads b1, b2 emitted from new chain blocks = [b1, b2] for b in blocks: - response = call_async(websocket.recv()) + response = await websocket.recv() d = json.loads(response) self.assertEqual( d["params"]["result"]["hash"], data_encoder(b.header.get_hash()) ) - def test_new_pending_xshard_tx_sender(self): + async def test_new_pending_xshard_tx_sender(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0x0) acc2 = Address.create_from_identity(id1, full_shard_key=0x10001) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_websocket_server_context( clusters[0].slave_list[0], port=38592 ): master = clusters[0].master slaves = clusters[0].slave_list - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=None) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=None ) - call_async(master.add_root_block(block)) + await master.add_root_block(block) request = { "jsonrpc": "2.0", @@ -1364,10 +1358,10 @@ def test_new_pending_xshard_tx_sender(self): "id": 6, } - websocket = call_async(get_websocket(38592)) - call_async(websocket.send(json.dumps(request))) + websocket = await get_websocket(38592) + await websocket.send(json.dumps(request)) - sub_response = json.loads(call_async(websocket.recv())) + sub_response = json.loads(await websocket.recv()) self.assertEqual(sub_response["id"], 6) self.assertEqual(len(sub_response["result"]), 34) @@ -1381,33 +1375,33 @@ def test_new_pending_xshard_tx_sender(self): ) self.assertTrue(slaves[0].add_tx(tx)) - tx_response = json.loads(call_async(websocket.recv())) + tx_response = json.loads(await websocket.recv()) self.assertEqual( tx_response["params"]["subscription"], sub_response["result"] ) self.assertTrue(tx_response["params"]["result"], tx.get_hash()) - b1 = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + b1 = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(b1))) + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(b1))) - def test_new_pending_xshard_tx_target(self): + async def test_new_pending_xshard_tx_target(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0x10001) acc2 = Address.create_from_identity(id1, full_shard_key=0x0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_websocket_server_context( clusters[0].slave_list[0], port=38593 ): master = clusters[0].master slaves = clusters[0].slave_list - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=None) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=None ) - call_async(master.add_root_block(block)) + await master.add_root_block(block) request = { "jsonrpc": "2.0", @@ -1415,10 +1409,10 @@ def test_new_pending_xshard_tx_target(self): "params": ["newPendingTransactions", "0x00000002"], "id": 6, } - websocket = call_async(get_websocket(38593)) - call_async(websocket.send(json.dumps(request))) + websocket = await get_websocket(38593) + await websocket.send(json.dumps(request)) - sub_response = json.loads(call_async(websocket.recv())) + sub_response = json.loads(await websocket.recv()) self.assertEqual(sub_response["id"], 6) self.assertEqual(len(sub_response["result"]), 34) @@ -1432,33 +1426,33 @@ def test_new_pending_xshard_tx_target(self): ) self.assertTrue(slaves[1].add_tx(tx)) - b1 = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0x10003) + b1 = await master.get_next_block_to_mine( + address=acc1, branch_value=0x10003 ) - self.assertTrue(call_async(clusters[0].get_shard(0x10003).add_block(b1))) + self.assertTrue((await clusters[0].get_shard(0x10003).add_block(b1))) - tx_response = json.loads(call_async(websocket.recv())) + tx_response = json.loads(await websocket.recv()) self.assertEqual( tx_response["params"]["subscription"], sub_response["result"] ) self.assertTrue(tx_response["params"]["result"], tx.get_hash()) - def test_new_pending_tx_same_acc_multi_subscriptions(self): + async def test_new_pending_tx_same_acc_multi_subscriptions(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0x0) acc2 = Address.create_from_identity(id1, full_shard_key=0x10001) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_websocket_server_context( clusters[0].slave_list[0], port=38594 ): master = clusters[0].master slaves = clusters[0].slave_list - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=None) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=None ) - call_async(master.add_root_block(block)) + await master.add_root_block(block) requests = [] REQ_NUM = 5 @@ -1471,9 +1465,12 @@ def test_new_pending_tx_same_acc_multi_subscriptions(self): } requests.append(req) - websocket = call_async(get_websocket(38594)) - [call_async(websocket.send(json.dumps(req))) for req in requests] - sub_responses = [json.loads(call_async(websocket.recv())) for _ in requests] + websocket = await get_websocket(38594) + for req in requests: + await websocket.send(json.dumps(req)) + sub_responses = [] + for _ in requests: + sub_responses.append(json.loads(await websocket.recv())) for i, resp in enumerate(sub_responses): self.assertEqual(resp["id"], i) @@ -1489,34 +1486,36 @@ def test_new_pending_tx_same_acc_multi_subscriptions(self): ) self.assertTrue(slaves[0].add_tx(tx)) - tx_responses = [json.loads(call_async(websocket.recv())) for _ in requests] + tx_responses = [] + for _ in requests: + tx_responses.append(json.loads(await websocket.recv())) for i, resp in enumerate(tx_responses): self.assertEqual( resp["params"]["subscription"], sub_responses[i]["result"] ) self.assertTrue(resp["params"]["result"], tx.get_hash()) - def test_new_pending_tx_with_reorg(self): + async def test_new_pending_tx_with_reorg(self): id1 = Identity.create_random_identity() id2 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id2, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True, genesis_minor_quarkash=10000000 ) as clusters, jrpc_websocket_server_context( clusters[0].slave_list[0], port=38595 ): - websocket = call_async(get_websocket(port=38595)) + websocket = await get_websocket(port=38595) request = { "jsonrpc": "2.0", "method": "subscribe", "params": ["newPendingTransactions", "0x00000002"], "id": 3, } - call_async(websocket.send(json.dumps(request))) + await websocket.send(json.dumps(request)) - sub_response = json.loads(call_async(websocket.recv())) + sub_response = json.loads(await websocket.recv()) self.assertEqual(sub_response["id"], 3) self.assertEqual(len(sub_response["result"]), 34) @@ -1532,7 +1531,7 @@ def test_new_pending_tx_with_reorg(self): value=12345, ) self.assertTrue(state.add_tx(tx)) - tx_response1 = json.loads(call_async(websocket.recv())) + tx_response1 = json.loads(await websocket.recv()) self.assertEqual( tx_response1["params"]["subscription"], sub_response["result"] ) @@ -1545,11 +1544,11 @@ def test_new_pending_tx_with_reorg(self): b2 = b1.create_block_to_append() state.finalize_and_add_block(b2) # fork should happen, b0-b2 is picked up - tx_response2 = json.loads(call_async(websocket.recv())) + tx_response2 = json.loads(await websocket.recv()) self.assertEqual(state.header_tip, b2.header) self.assertEqual(tx_response2, tx_response1) - def test_logs(self): + async def test_logs(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -1561,14 +1560,14 @@ def test_logs(self): "data": "0x", } - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True, genesis_minor_quarkash=10000000 ) as clusters, jrpc_websocket_server_context( clusters[0].slave_list[0], port=38596 ): master = clusters[0].master slaves = clusters[0].slave_list - websocket = call_async(get_websocket(port=38596)) + websocket = await get_websocket(port=38596) # filter by contract address contract_addr = mk_contract_address(acc1.recipient, 0, acc1.full_shard_key) @@ -1586,8 +1585,8 @@ def test_logs(self): ], "id": 4, } - call_async(websocket.send(json.dumps(filter_req))) - response = call_async(websocket.recv()) + await websocket.send(json.dumps(filter_req)) + response = await websocket.recv() response = json.loads(response) self.assertEqual(response["id"], 4) @@ -1606,8 +1605,8 @@ def test_logs(self): ], "id": 5, } - call_async(websocket.send(json.dumps(filter_req))) - response = call_async(websocket.recv()) + await websocket.send(json.dumps(filter_req)) + response = await websocket.recv() response = json.loads(response) self.assertEqual(response["id"], 5) @@ -1620,16 +1619,14 @@ def test_logs(self): expected_log_parts["transactionHash"] = "0x" + tx.get_hash().hex() self.assertTrue(slaves[0].add_tx(tx)) - block = call_async( - master.get_next_block_to_mine( - address=acc1, branch_value=0b10 - ) # branch_value = 2 - ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(block))) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 + ) # branch_value = 2 + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(block))) count = 0 while count < 2: - response = call_async(websocket.recv()) + response = await websocket.recv() count += 1 d = json.loads(response) self.assertDictContainsSubset(expected_log_parts, d["params"]["result"]) @@ -1639,16 +1636,16 @@ def test_logs(self): ) self.assertEqual(count, 2) - def test_log_removed_flag_with_chain_reorg(self): + async def test_log_removed_flag_with_chain_reorg(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True, genesis_minor_quarkash=10000000 ) as clusters, jrpc_websocket_server_context( clusters[0].slave_list[0], port=38597 ): - websocket = call_async(get_websocket(port=38597)) + websocket = await get_websocket(port=38597) # a log subscriber with no-filter request request = { @@ -1657,8 +1654,8 @@ def test_log_removed_flag_with_chain_reorg(self): "params": ["logs", "0x00000002", {}], "id": 3, } - call_async(websocket.send(json.dumps(request))) - response = call_async(websocket.recv()) + await websocket.send(json.dumps(request)) + response = await websocket.recv() response = json.loads(response) self.assertEqual(response["id"], 3) @@ -1676,7 +1673,7 @@ def test_log_removed_flag_with_chain_reorg(self): self.assertEqual(state.header_tip, b0.header) tx_hash = tx.get_hash() - response = call_async(websocket.recv()) + response = await websocket.recv() d = json.loads(response) self.assertEqual( d["params"]["result"]["transactionHash"], data_encoder(tx_hash) @@ -1692,7 +1689,7 @@ def test_log_removed_flag_with_chain_reorg(self): self.assertEqual(state.header_tip, b2.header) # log emitted from old chain, flag is set to True - response = call_async(websocket.recv()) + response = await websocket.recv() d = json.loads(response) self.assertEqual( d["params"]["result"]["transactionHash"], data_encoder(tx_hash) @@ -1700,17 +1697,17 @@ def test_log_removed_flag_with_chain_reorg(self): self.assertEqual(d["params"]["result"]["removed"], True) # log emitted from new chain - response = call_async(websocket.recv()) + response = await websocket.recv() d = json.loads(response) self.assertEqual( d["params"]["result"]["transactionHash"], data_encoder(tx_hash) ) self.assertEqual(d["params"]["result"]["removed"], False) - def test_invalid_subscription(self): + async def test_invalid_subscription(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_websocket_server_context( clusters[0].slave_list[0], port=38598 @@ -1730,26 +1727,27 @@ def test_invalid_subscription(self): "id": 3, } - websocket = call_async(get_websocket(port=38598)) - [ - call_async(websocket.send(json.dumps(req))) - for req in [request1, request2] - ] - responses = [json.loads(call_async(websocket.recv())) for _ in range(2)] - [self.assertTrue(resp["error"]) for resp in responses] # emit error message + websocket = await get_websocket(port=38598) + for req in [request1, request2]: + await websocket.send(json.dumps(req)) + responses = [] + for _ in range(2): + responses.append(json.loads(await websocket.recv())) + for resp in responses: + self.assertTrue(resp["error"]) # emit error message - def test_multi_subs_with_some_unsubs_in_one_ws_conn(self): + async def test_multi_subs_with_some_unsubs_in_one_ws_conn(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_websocket_server_context( clusters[0].slave_list[0], port=38599 ): # clusters[0].slave_list[0] has two shards with full_shard_id 2 and 3 master = clusters[0].master - websocket = call_async(get_websocket(port=38599)) + websocket = await get_websocket(port=38599) # make 3 subscriptions on new heads ids = [3, 4, 5] @@ -1761,8 +1759,8 @@ def test_multi_subs_with_some_unsubs_in_one_ws_conn(self): "params": ["newHeads", "0x00000002"], "id": id, } - call_async(websocket.send(json.dumps(request))) - response = call_async(websocket.recv()) + await websocket.send(json.dumps(request)) + response = await websocket.recv() response = json.loads(response) sub_ids.append(response["result"]) self.assertEqual(response["id"], id) @@ -1774,32 +1772,32 @@ def test_multi_subs_with_some_unsubs_in_one_ws_conn(self): "params": [sub_ids[0]], "id": 3, } - call_async(websocket.send(json.dumps(request))) - response = call_async(websocket.recv()) + await websocket.send(json.dumps(request)) + response = await websocket.recv() response = json.loads(response) self.assertEqual(response["result"], True) # unsubscribed successfully # add a new block, should expect only 2 responses - root_block = call_async( - master.get_next_block_to_mine(acc1, branch_value=None) + root_block = await master.get_next_block_to_mine( + acc1, branch_value=None ) - call_async(master.add_root_block(root_block)) + await master.add_root_block(root_block) - block = call_async( - master.get_next_block_to_mine(address=acc1, branch_value=0b10) + block = await master.get_next_block_to_mine( + address=acc1, branch_value=0b10 ) - self.assertTrue(call_async(clusters[0].get_shard(2 | 0).add_block(block))) + self.assertTrue((await clusters[0].get_shard(2 | 0).add_block(block))) for sub_id in sub_ids[1:]: - response = call_async(websocket.recv()) + response = await websocket.recv() response = json.loads(response) self.assertEqual(response["params"]["subscription"], sub_id) - def test_unsubscribe(self): + async def test_unsubscribe(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) - with ClusterContext( + async with ClusterContext( 1, acc1, small_coinbase=True ) as clusters, jrpc_websocket_server_context( clusters[0].slave_list[0], port=38600 @@ -1810,9 +1808,9 @@ def test_unsubscribe(self): "params": ["newPendingTransactions", "0x00000002"], "id": 6, } - websocket = call_async(get_websocket(port=38600)) - call_async(websocket.send(json.dumps(request))) - sub_response = json.loads(call_async(websocket.recv())) + websocket = await get_websocket(port=38600) + await websocket.send(json.dumps(request)) + sub_response = json.loads(await websocket.recv()) # Check subscription response self.assertEqual(sub_response["id"], 6) @@ -1826,12 +1824,12 @@ def test_unsubscribe(self): } # Unsubscribe successfully - call_async(websocket.send(json.dumps(unsubscribe))) - response = json.loads(call_async(websocket.recv())) + await websocket.send(json.dumps(unsubscribe)) + response = json.loads(await websocket.recv()) self.assertTrue(response["result"]) self.assertEqual(response["id"], 3) # Invalid unsubscription if sub_id does not exist - call_async(websocket.send(json.dumps(unsubscribe))) - response = json.loads(call_async(websocket.recv())) + await websocket.send(json.dumps(unsubscribe)) + response = json.loads(await websocket.recv()) self.assertTrue(response["error"]) diff --git a/quarkchain/cluster/tests/test_miner.py b/quarkchain/cluster/tests/test_miner.py index 9ab761993..91ac8556b 100644 --- a/quarkchain/cluster/tests/test_miner.py +++ b/quarkchain/cluster/tests/test_miner.py @@ -63,8 +63,13 @@ async def add(block): ): miner = self.miner_gen(consensus, create, add) # should generate 5 blocks and then end - loop = asyncio.get_event_loop() - loop.run_until_complete(miner._mine_new_block_async()) + + async def go(): + task = miner._mine_new_block_async() + if task is not None: + await task + + asyncio.run(go()) self.assertEqual(len(self.added_blocks), 5) def test_simulate_mine_handle_block_exception(self): @@ -91,8 +96,13 @@ async def add(block): miner = self.miner_gen(ConsensusType.POW_SIMULATE, create, add) # only 2 blocks can be added - loop = asyncio.get_event_loop() - loop.run_until_complete(miner._mine_new_block_async()) + + async def go(): + task = miner._mine_new_block_async() + if task is not None: + await task + + asyncio.run(go()) self.assertEqual(len(self.added_blocks), 2) def test_sha3sha3(self): @@ -142,8 +152,7 @@ async def go(): with self.assertRaises(ValueError): await miner.submit_work(b"", 42, b"") - loop = asyncio.get_event_loop() - loop.run_until_complete(go()) + asyncio.run(go()) def test_get_work(self): now, height = 42, 42 @@ -206,8 +215,7 @@ async def go(): self.assertEqual(len(miner.work_map), 4) self.assertEqual(len(miner.current_works), 2) - loop = asyncio.get_event_loop() - loop.run_until_complete(go()) + asyncio.run(go()) def test_submit_work(self): now, height = 42, 42 @@ -264,8 +272,7 @@ async def go(): self.assertEqual(miner.work_map, {}) self.assertEqual(len(self.added_blocks), 1) - loop = asyncio.get_event_loop() - loop.run_until_complete(go()) + asyncio.run(go()) def test_submit_work_with_guardian(self): now = 42 @@ -303,8 +310,7 @@ async def go(): res = await miner.submit_work(work.hash, i, sha3_256(b"")) self.assertTrue(res) - loop = asyncio.get_event_loop() - loop.run_until_complete(go()) + asyncio.run(go()) def test_submit_work_with_remote_guardian(self): now = 42 @@ -354,8 +360,7 @@ async def go(): res = await miner.submit_work(work.hash, i, sha3_256(b""), bytes(65)) self.assertFalse(res) - loop = asyncio.get_event_loop() - loop.run_until_complete(go()) + asyncio.run(go()) def test_validate_seal_with_adjusted_diff(self): diff = 1000 diff --git a/quarkchain/cluster/tests/test_native_token.py b/quarkchain/cluster/tests/test_native_token.py index ed9618a9f..d802a05bf 100644 --- a/quarkchain/cluster/tests/test_native_token.py +++ b/quarkchain/cluster/tests/test_native_token.py @@ -1,3 +1,4 @@ +import asyncio import unittest from quarkchain.cluster.shard_state import ShardState @@ -24,9 +25,9 @@ def create_default_shard_state(env, shard_id=0, diff_calc=None): return shard_state -class TestNativeTokenShardState(unittest.TestCase): - def setUp(self): - super().setUp() +class TestNativeTokenShardState(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + await super().asyncSetUp() config = get_test_env().quark_chain_config self.root_coinbase = config.ROOT.COINBASE_AMOUNT self.shard_coinbase = next(iter(config.shards.values())).COINBASE_AMOUNT @@ -39,7 +40,7 @@ def setUp(self): def get_after_tax_reward(self, value: int) -> int: return value * self.tax_rate.numerator // self.tax_rate.denominator - def test_native_token_transfer(self): + async def test_native_token_transfer(self): """in-shard transfer QETH using genesis_token as gas """ QETH = token_id_encode("QETH") @@ -88,7 +89,7 @@ def test_native_token_transfer(self): self.assertEqual(tx_list[0].gas_token_id, self.genesis_token) self.assertEqual(tx_list[0].transfer_token_id, QETH) - def test_native_token_transfer_0_value_success(self): + async def test_native_token_transfer_0_value_success(self): """to prevent storage spamming, do not delta_token_balance does not take action if value is 0 """ MALICIOUS0 = token_id_encode("MALICIOUS0") @@ -115,7 +116,7 @@ def test_native_token_transfer_0_value_success(self): ) self.assertFalse(state.add_tx(tx)) - def test_disallowed_unknown_token(self): + async def test_disallowed_unknown_token(self): """do not allow tx with unknown token id """ MALICIOUS0 = token_id_encode("MALICIOUS0") @@ -153,7 +154,7 @@ def test_disallowed_unknown_token(self): self.assertFalse(state.add_tx(tx1)) @mock_pay_native_token_as_gas() - def test_native_token_gas(self): + async def test_native_token_gas(self): """in-shard transfer QETH using native token as gas """ qeth = token_id_encode("QETH") @@ -205,7 +206,7 @@ def test_native_token_gas(self): self.assertEqual(tx_list[0].gas_token_id, qeth) self.assertEqual(tx_list[0].transfer_token_id, qeth) - def test_xshard_native_token_sent(self): + async def test_xshard_native_token_sent(self): """x-shard transfer QETH using genesis_token as gas """ QETH = token_id_encode("QETHXX") @@ -279,7 +280,7 @@ def test_xshard_native_token_sent(self): self.get_after_tax_reward(opcodes.GTXCOST + self.shard_coinbase), ) - def test_xshard_native_token_received(self): + async def test_xshard_native_token_received(self): QETH = token_id_encode("QETHXX") id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -381,7 +382,7 @@ def test_xshard_native_token_received(self): ) @mock_pay_native_token_as_gas() - def test_xshard_native_token_gas_sent(self): + async def test_xshard_native_token_gas_sent(self): """x-shard transfer QETH using QETH as gas """ qeth = token_id_encode("QETHXX") @@ -453,7 +454,7 @@ def test_xshard_native_token_gas_sent(self): ) @mock_pay_native_token_as_gas() - def test_xshard_native_token_gas_received(self): + async def test_xshard_native_token_gas_received(self): qeth = token_id_encode("QETHXX") id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -548,7 +549,7 @@ def test_xshard_native_token_gas_received(self): state0.evm_state.xshard_receive_gas_used, opcodes.GTXXSHARDCOST ) - def test_contract_suicide(self): + async def test_contract_suicide(self): """ Kill Call Data: 0x41c0e1b5 """ diff --git a/quarkchain/cluster/tests/test_protocol.py b/quarkchain/cluster/tests/test_protocol.py index a6785aa58..d79dd080a 100644 --- a/quarkchain/cluster/tests/test_protocol.py +++ b/quarkchain/cluster/tests/test_protocol.py @@ -78,8 +78,11 @@ def test_forward(self): writer = MagicMock() reader.read.side_effect = [requestSizeBytes, metaBytes, rawData] - conn = DummyP2PConnection(DEFAULT_ENV, reader, writer) - asyncio.get_event_loop().run_until_complete(conn.loop_once()) + async def run(): + conn = DummyP2PConnection(DEFAULT_ENV, reader, writer) + await conn.loop_once() + return conn + conn = asyncio.run(run()) conn.mockClusterConnection.write_raw_data.assert_called_once_with( ClusterMetadata(FORWARD_BRANCH, CLUSTER_PEER_ID), rawData @@ -100,8 +103,11 @@ def test_no_forward(self): writer = MagicMock() reader.read.side_effect = [requestSizeBytes, metaBytes, rawData] - conn = DummyP2PConnection(DEFAULT_ENV, reader, writer) - asyncio.get_event_loop().run_until_complete(conn.loop_once()) + async def run(): + conn = DummyP2PConnection(DEFAULT_ENV, reader, writer) + await conn.loop_once() + return conn + conn = asyncio.run(run()) conn.mockClusterConnection.write_raw_data.assert_not_called() writer.write.assert_has_calls( @@ -125,8 +131,11 @@ def test_forward(self): writer = MagicMock() reader.read.side_effect = [requestSizeBytes, metaBytes, rawData] - conn = DummyClusterConnection(DEFAULT_ENV, reader, writer) - asyncio.get_event_loop().run_until_complete(conn.loop_once()) + async def run(): + conn = DummyClusterConnection(DEFAULT_ENV, reader, writer) + await conn.loop_once() + return conn + conn = asyncio.run(run()) conn.mockP2PConnection.write_raw_data.assert_called_once_with( P2PMetadata(FORWARD_BRANCH), rawData @@ -147,8 +156,11 @@ def test_no_forward(self): writer = MagicMock() reader.read.side_effect = [requestSizeBytes, metaBytes, rawData] - conn = DummyClusterConnection(DEFAULT_ENV, reader, writer) - asyncio.get_event_loop().run_until_complete(conn.loop_once()) + async def run(): + conn = DummyClusterConnection(DEFAULT_ENV, reader, writer) + await conn.loop_once() + return conn + conn = asyncio.run(run()) conn.mockP2PConnection.write_raw_data.assert_not_called() writer.write.assert_has_calls( diff --git a/quarkchain/cluster/tests/test_root_state.py b/quarkchain/cluster/tests/test_root_state.py index d6d9d1c05..e472c1e7a 100644 --- a/quarkchain/cluster/tests/test_root_state.py +++ b/quarkchain/cluster/tests/test_root_state.py @@ -51,32 +51,32 @@ def add_minor_block_to_cluster(s_states, block): ) -class TestRootState(unittest.TestCase): - def test_root_state_simple(self): +class TestRootState(unittest.IsolatedAsyncioTestCase): + async def test_root_state_simple(self): env = get_test_env() state = RootState(env=env) self.assertEqual(state.tip.height, 0) - def test_blocks_with_incorrect_version(self): + async def test_blocks_with_incorrect_version(self): env = get_test_env() r_state, s_states = create_default_state(env) root_block = r_state.create_block_to_mine([]) root_block.header.version = 1 - with self.assertRaisesRegexp(ValueError, "incorrect root block version"): + with self.assertRaisesRegex(ValueError, "incorrect root block version"): r_state.add_block(root_block) root_block.header.version = 0 r_state.add_block(root_block) - def test_blocks_with_incorrect_height(self): + async def test_blocks_with_incorrect_height(self): env = get_test_env() r_state, s_states = create_default_state(env) root_block = r_state.create_block_to_mine([]) root_block.header.height += 1 - with self.assertRaisesRegexp(ValueError, "incorrect block height"): + with self.assertRaisesRegex(ValueError, "incorrect block height"): r_state.add_block(root_block) - def test_blocks_with_incorrect_merkle_and_minor_block_list(self): + async def test_blocks_with_incorrect_merkle_and_minor_block_list(self): env = get_test_env() r_state, s_states = create_default_state(env) self.assertEqual(r_state.tip.total_difficulty, 2000000) @@ -96,7 +96,7 @@ def test_blocks_with_incorrect_merkle_and_minor_block_list(self): root_block0 = r_state.create_block_to_mine([b0.header, b1.header]) root_block1 = r_state.create_block_to_mine([b0.header]) - with self.assertRaisesRegexp(ValueError, "incorrect merkle root"): + with self.assertRaisesRegex(ValueError, "incorrect merkle root"): root_block1.header.hash_merkle_root = root_block0.header.hash_merkle_root r_state.add_block(root_block1) @@ -109,22 +109,22 @@ def test_blocks_with_incorrect_merkle_and_minor_block_list(self): root_block_with_incorrect_mlist2 = r_state.create_block_to_mine( [b1.header, b0.header] ) - with self.assertRaisesRegexp(ValueError, "does not link to previous block"): + with self.assertRaisesRegex(ValueError, "does not link to previous block"): r_state.add_block(root_block_with_incorrect_mlist0) - with self.assertRaisesRegexp(ValueError, "does not link to previous block"): + with self.assertRaisesRegex(ValueError, "does not link to previous block"): r_state.add_block(root_block_with_incorrect_mlist1) - with self.assertRaisesRegexp(ValueError, "shard id must be ordered"): + with self.assertRaisesRegex(ValueError, "shard id must be ordered"): r_state.add_block(root_block_with_incorrect_mlist2) - def test_blocks_with_incorrect_total_difficulty(self): + async def test_blocks_with_incorrect_total_difficulty(self): env = get_test_env() r_state, s_states = create_default_state(env) root_block = r_state.create_block_to_mine([]) root_block.header.total_difficulty += 1 - with self.assertRaisesRegexp(ValueError, "incorrect total difficulty"): + with self.assertRaisesRegex(ValueError, "incorrect total difficulty"): r_state.add_block(root_block) - def test_reorg_with_shorter_chain(self): + async def test_reorg_with_shorter_chain(self): env = get_test_env() r_state, s_states = create_default_state(env) @@ -148,7 +148,7 @@ def test_reorg_with_shorter_chain(self): self.assertIsNone(r_state.db.get_root_block_by_height(3), None) self.assertEqual(r_state.db.get_root_block_by_height(2), root_block10) - def test_root_state_and_shard_state_add_block(self): + async def test_root_state_and_shard_state_add_block(self): env = get_test_env() r_state, s_states = create_default_state(env) self.assertEqual(r_state.tip.total_difficulty, 2000000) @@ -184,7 +184,7 @@ def test_root_state_and_shard_state_add_block(self): self.assertTrue(s_state1.add_root_block(root_block)) self.assertEqual(s_state1.root_tip, root_block.header) - def test_root_state_add_block_no_proof_of_progress(self): + async def test_root_state_add_block_no_proof_of_progress(self): env = get_test_env() r_state, s_states = create_default_state(env) s_state0 = s_states[2 | 0] @@ -208,7 +208,7 @@ def test_root_state_add_block_no_proof_of_progress(self): root_block = r_state.create_block_to_mine([b1.header]) self.assertTrue(r_state.add_block(root_block)) - def test_root_state_add_two_blocks(self): + async def test_root_state_add_two_blocks(self): env = get_test_env() r_state, s_states = create_default_state(env) s_state0 = s_states[2 | 0] @@ -243,7 +243,7 @@ def test_root_state_add_two_blocks(self): self.assertTrue(r_state.add_block(root_block1)) - def test_root_state_and_shard_state_fork(self): + async def test_root_state_and_shard_state_fork(self): env = get_test_env() r_state, s_states = create_default_state(env) @@ -324,7 +324,7 @@ def test_root_state_and_shard_state_fork(self): self.assertEqual(s_state0.root_tip, root_block2.header) self.assertEqual(s_state1.root_tip, root_block2.header) - def test_root_state_difficulty_and_coinbase(self): + async def test_root_state_difficulty_and_coinbase(self): env = get_test_env() env.quark_chain_config.SKIP_ROOT_DIFFICULTY_CHECK = False env.quark_chain_config.ROOT.GENESIS.DIFFICULTY = 1000 @@ -408,7 +408,7 @@ def test_root_state_difficulty_and_coinbase(self): root_block0.header.difficulty, ) - def test_root_state_recovery(self): + async def test_root_state_recovery(self): env = get_test_env() r_state, s_states = create_default_state(env) @@ -476,8 +476,9 @@ def test_root_state_recovery(self): recovered_state.db.get_root_block_by_height(tip_height), root_block0 ) - def test_add_root_block_with_minor_block_with_wrong_root_block_hash(self): - """ Test for the following case + async def test_add_root_block_with_minor_block_with_wrong_root_block_hash(self): + """ + Test for the following case +--+ +--+ |r1|<---|r3| /+--+ +--+ @@ -559,7 +560,7 @@ def test_add_root_block_with_minor_block_with_wrong_root_block_hash(self): ) self.assertTrue(r_state.add_block(root_block4)) - def test_add_minor_block_with_wrong_root_block_hash(self): + async def test_add_minor_block_with_wrong_root_block_hash(self): """ Test for the following case +--+ |r1| @@ -634,7 +635,7 @@ def test_add_minor_block_with_wrong_root_block_hash(self): with self.assertRaises(ValueError): add_minor_block_to_cluster(s_states, m3) - def test_root_state_add_root_block_too_many_minor_blocks(self): + async def test_root_state_add_root_block_too_many_minor_blocks(self): env = get_test_env() r_state, s_states = create_default_state(env) s_state0 = s_states[2 | 0] @@ -654,7 +655,7 @@ def test_root_state_add_root_block_too_many_minor_blocks(self): root_block = r_state.create_block_to_mine( m_header_list=headers, create_time=headers[-1].create_time + 1 ) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( ValueError, "too many minor blocks in the root block for shard" ): r_state.add_block(root_block) @@ -665,7 +666,7 @@ def test_root_state_add_root_block_too_many_minor_blocks(self): ) r_state.add_block(root_block) - def test_root_chain_fork_using_largest_total_diff(self): + async def test_root_chain_fork_using_largest_total_diff(self): env = get_test_env(shard_size=1) r_state, s_states = create_default_state(env) @@ -684,7 +685,7 @@ def test_root_chain_fork_using_largest_total_diff(self): self.assertTrue(r_state.add_block(rb3)) self.assertEqual(r_state.tip.get_hash(), rb3.header.get_hash()) - def test_root_coinbase_decay(self): + async def test_root_coinbase_decay(self): env = get_test_env() r_state, s_states = create_default_state(env) coinbase = r_state._calculate_root_block_coinbase( diff --git a/quarkchain/cluster/tests/test_shard_db_operator.py b/quarkchain/cluster/tests/test_shard_db_operator.py index 5b44694d5..05f95f465 100644 --- a/quarkchain/cluster/tests/test_shard_db_operator.py +++ b/quarkchain/cluster/tests/test_shard_db_operator.py @@ -35,8 +35,8 @@ def create_default_shard_state( return shard_state -class TestShardDbOperator(unittest.TestCase): - def test_get_minor_block_by_hash(self): +class TestShardDbOperator(unittest.IsolatedAsyncioTestCase): + async def test_get_minor_block_by_hash(self): db = ShardDbOperator(InMemoryDb(), DEFAULT_ENV, Branch(2)) block = MinorBlock(MinorBlockHeader(), MinorBlockMeta()) block_hash = block.header.get_hash() @@ -47,7 +47,7 @@ def test_get_minor_block_by_hash(self): self.assertEqual(db.get_minor_block_header_by_hash(block_hash), block.header) self.assertIsNone(db.get_minor_block_header_by_hash(b"")) - def test_get_transaction_by_address(self): + async def test_get_transaction_by_address(self): id1 = Identity.create_random_identity() miner_addr = Address.create_random_account(full_shard_key=0) acc00 = Address.create_from_identity(id1, full_shard_key=0) diff --git a/quarkchain/cluster/tests/test_shard_state.py b/quarkchain/cluster/tests/test_shard_state.py index 9c8dbb486..37f4e3b9f 100644 --- a/quarkchain/cluster/tests/test_shard_state.py +++ b/quarkchain/cluster/tests/test_shard_state.py @@ -56,9 +56,9 @@ def create_default_shard_state( return shard_state -class TestShardState(unittest.TestCase): - def setUp(self): - super().setUp() +class TestShardState(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + await super().asyncSetUp() config = get_test_env().quark_chain_config self.root_coinbase = config.ROOT.COINBASE_AMOUNT self.shard_coinbase = next(iter(config.shards.values())).COINBASE_AMOUNT @@ -71,7 +71,7 @@ def setUp(self): def get_after_tax_reward(self, value: int) -> int: return value * self.tax_rate.numerator // self.tax_rate.denominator - def test_shard_state_simple(self): + async def test_shard_state_simple(self): env = get_test_env() state = create_default_shard_state(env) self.assertEqual(state.root_tip.height, 0) @@ -82,7 +82,7 @@ def test_shard_state_simple(self): {self.genesis_token: 2500000000000000000}, ) - def test_get_total_balance(self): + async def test_get_total_balance(self): acc_size = 60 id_list = [Identity.create_random_identity() for _ in range(acc_size)] acc_list = [Address.create_from_identity(i, full_shard_key=0) for i in id_list] @@ -148,7 +148,7 @@ def test_get_total_balance(self): qkc_token, state.header_tip.get_hash(), None, 1, start=urandom(32) ) - def test_init_genesis_state(self): + async def test_init_genesis_state(self): env = get_test_env() state = create_default_shard_state(env) genesis_header = state.header_tip @@ -178,12 +178,12 @@ def test_init_genesis_state(self): self.assertEqual(state.header_tip, new_genesis_block.header) self.assertEqual(new_genesis_block, state.db.get_minor_block_by_height(0)) - def test_blocks_with_incorrect_version(self): + async def test_blocks_with_incorrect_version(self): env = get_test_env() state = create_default_shard_state(env=env) root_block = state.root_tip.create_block_to_append() root_block.header.version = 1 - with self.assertRaisesRegexp(ValueError, "incorrect root block version"): + with self.assertRaisesRegex(ValueError, "incorrect root block version"): state.add_root_block(root_block.finalize()) root_block.header.version = 0 @@ -191,14 +191,14 @@ def test_blocks_with_incorrect_version(self): shard_block = state.create_block_to_mine() shard_block.header.version = 1 - with self.assertRaisesRegexp(ValueError, "incorrect minor block version"): + with self.assertRaisesRegex(ValueError, "incorrect minor block version"): state.finalize_and_add_block(shard_block) shard_block.header.version = 0 state.finalize_and_add_block(shard_block) @mock_pay_native_token_as_gas() - def test_gas_price(self): + async def test_gas_price(self): id_list = [Identity.create_random_identity() for _ in range(5)] acc_list = [Address.create_from_identity(i, full_shard_key=0) for i in id_list] env = get_test_env( @@ -286,7 +286,7 @@ def test_gas_price(self): gas_price = state.gas_price(token_id=1) self.assertEqual(gas_price, 0) - def test_estimate_gas(self): + async def test_estimate_gas(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_random_account(full_shard_key=0) @@ -317,7 +317,7 @@ def test_estimate_gas(self): estimate = state.estimate_gas(tx, acc1) self.assertEqual(estimate, 32176) - def test_execute_tx(self): + async def test_execute_tx(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_random_account(full_shard_key=0) @@ -338,7 +338,7 @@ def test_execute_tx(self): res = state.execute_tx(tx, acc1) self.assertEqual(res, b"") - def test_add_tx_incorrect_from_shard_id(self): + async def test_add_tx_incorrect_from_shard_id(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=1) acc2 = Address.create_random_account(full_shard_key=1) @@ -355,7 +355,7 @@ def test_add_tx_incorrect_from_shard_id(self): self.assertFalse(state.add_tx(tx)) self.assertIsNone(state.execute_tx(tx, acc1)) - def test_one_tx(self): + async def test_one_tx(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_random_account(full_shard_key=0) @@ -435,7 +435,7 @@ def test_one_tx(self): tx_list, _ = state.db.get_transactions_by_address(acc2) self.assertEqual(tx_list[0].value, 12345) - def test_duplicated_tx(self): + async def test_duplicated_tx(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_random_account(full_shard_key=0) @@ -496,7 +496,7 @@ def test_duplicated_tx(self): self.assertTrue(state.db.contain_transaction_hash(tx.get_hash())) self.assertFalse(state.add_tx(tx)) - def test_add_invalid_tx_fail(self): + async def test_add_invalid_tx_fail(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_random_account(full_shard_key=0) @@ -514,7 +514,7 @@ def test_add_invalid_tx_fail(self): self.assertFalse(state.add_tx(tx)) self.assertEqual(len(state.tx_queue), 0) - def test_add_non_neighbor_tx_fail(self): + async def test_add_non_neighbor_tx_fail(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_random_account(full_shard_key=3) # not acc1's neighbor @@ -551,7 +551,7 @@ def test_add_non_neighbor_tx_fail(self): self.assertTrue(state.add_tx(tx)) self.assertEqual(len(state.tx_queue), 1) - def test_exceeding_xshard_limit(self): + async def test_exceeding_xshard_limit(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_random_account(full_shard_key=1) @@ -606,7 +606,7 @@ def test_exceeding_xshard_limit(self): b1 = state.create_block_to_mine(address=acc3) self.assertEqual(len(b1.tx_list), 1) - def test_two_tx_in_one_block(self): + async def test_two_tx_in_one_block(self): id1 = Identity.create_random_identity() id2 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -721,7 +721,7 @@ def test_two_tx_in_one_block(self): state.evm_state.get_full_shard_key(acc2.recipient), acc2.full_shard_key ) - def test_fork_does_not_confirm_tx(self): + async def test_fork_does_not_confirm_tx(self): """Tx should only be confirmed and removed from tx queue by the best chain""" id1 = Identity.create_random_identity() id2 = Identity.create_random_identity() @@ -765,7 +765,7 @@ def test_fork_does_not_confirm_tx(self): state.finalize_and_add_block(b2) self.assertEqual(len(state.tx_queue), 0) - def test_revert_fork_put_tx_back_to_queue(self): + async def test_revert_fork_put_tx_back_to_queue(self): """Tx in the reverted chain should be put back to the queue""" id1 = Identity.create_random_identity() id2 = Identity.create_random_identity() @@ -818,7 +818,7 @@ def test_revert_fork_put_tx_back_to_queue(self): # b0-b3-b4 becomes the best chain self.assertEqual(len(state.tx_queue), 0) - def test_stale_block_count(self): + async def test_stale_block_count(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc3 = Address.create_random_account(full_shard_key=0) @@ -836,7 +836,7 @@ def test_stale_block_count(self): state.finalize_and_add_block(b2) self.assertEqual(state.db.get_block_count_by_height(1), 2) - def test_xshard_tx_sent(self): + async def test_xshard_tx_sent(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id1, full_shard_key=1) @@ -897,7 +897,7 @@ def test_xshard_tx_sent(self): self.get_after_tax_reward(opcodes.GTXCOST + self.shard_coinbase), ) - def test_xshard_tx_sent_old(self): + async def test_xshard_tx_sent_old(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id1, full_shard_key=1) @@ -962,7 +962,7 @@ def test_xshard_tx_sent_old(self): self.get_after_tax_reward(opcodes.GTXCOST + self.shard_coinbase), ) - def test_xshard_tx_insufficient_gas(self): + async def test_xshard_tx_insufficient_gas(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id1, full_shard_key=1) @@ -986,7 +986,7 @@ def test_xshard_tx_insufficient_gas(self): self.assertEqual(len(b1.tx_list), 0) self.assertEqual(len(state.tx_queue), 0) - def test_xshard_tx_received(self): + async def test_xshard_tx_received(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id1, full_shard_key=16) @@ -1074,7 +1074,7 @@ def test_xshard_tx_received(self): evm_state0 = state0.evm_state self.assertEqual(evm_state0.xshard_receive_gas_used, opcodes.GTXXSHARDCOST) - def test_xshard_tx_received_ddos_fix(self): + async def test_xshard_tx_received_ddos_fix(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id1, full_shard_key=16) @@ -1173,7 +1173,7 @@ def test_xshard_tx_received_ddos_fix(self): b3.meta.evm_cross_shard_receive_gas_used, opcodes.GTXXSHARDCOST ) - def test_xshard_tx_received_exclude_non_neighbor(self): + async def test_xshard_tx_received_exclude_non_neighbor(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id1, full_shard_key=3) @@ -1227,7 +1227,7 @@ def test_xshard_tx_received_exclude_non_neighbor(self): evm_state0 = state0.evm_state self.assertEqual(evm_state0.xshard_receive_gas_used, 0) - def test_xshard_from_root_block(self): + async def test_xshard_from_root_block(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -1286,7 +1286,7 @@ def _testcase_evm_enabled_coinbase_is_code(): 1000000, ) - def test_xshard_for_two_root_blocks(self): + async def test_xshard_for_two_root_blocks(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id1, full_shard_key=1) @@ -1417,7 +1417,7 @@ def test_xshard_for_two_root_blocks(self): self.assertEqual(state0.evm_state.gas_used, 18000) self.assertEqual(state0.evm_state.xshard_receive_gas_used, 18000) - def test_xshard_gas_limit(self): + async def test_xshard_gas_limit(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id1, full_shard_key=16) @@ -1569,7 +1569,7 @@ def test_xshard_gas_limit(self): gas_limit=opcodes.GTXXSHARDCOST, xshard_gas_limit=2 * opcodes.GTXXSHARDCOST, ) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( ValueError, "xshard_gas_limit \\d+ should not exceed total gas_limit" ): # xshard_gas_limit should be smaller than gas_limit @@ -1582,13 +1582,13 @@ def test_xshard_gas_limit(self): b6 = state0.create_block_to_mine( address=acc3, xshard_gas_limit=opcodes.GTXXSHARDCOST ) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( ValueError, "incorrect xshard gas limit, expected \\d+, actual \\d+" ): # xshard_gas_limit should be gas_limit // 2 state0.finalize_and_add_block(b6) - def test_xshard_gas_limit_from_multiple_shards(self): + async def test_xshard_gas_limit_from_multiple_shards(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id1, full_shard_key=16) @@ -1744,7 +1744,7 @@ def test_xshard_gas_limit_from_multiple_shards(self): 10000000 + 1000000 + 12345 + 888888 + 111111, ) - def test_xshard_root_block_coinbase(self): + async def test_xshard_root_block_coinbase(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -1795,10 +1795,10 @@ def test_xshard_root_block_coinbase(self): state1.get_token_balance(acc1.recipient, self.genesis_token), 10000000 ) - def test_xshard_smart_contract(self): + async def test_xshard_smart_contract(self): pass - def test_xshard_sender_gas_limit(self): + async def test_xshard_sender_gas_limit(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id1, full_shard_key=16) @@ -1830,7 +1830,7 @@ def test_xshard_sender_gas_limit(self): ) self.assertFalse(state0.add_tx(tx0)) b0.add_tx(tx0) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( RuntimeError, "xshard evm tx exceeds xshard gas limit" ): state0.finalize_and_add_block(b0) @@ -1850,7 +1850,7 @@ def test_xshard_sender_gas_limit(self): ) self.assertFalse(state0.add_tx(tx2, xshard_gas_limit=opcodes.GTXCOST * 9)) b2.add_tx(tx2) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( RuntimeError, "xshard evm tx exceeds xshard gas limit" ): state0.finalize_and_add_block(b2, xshard_gas_limit=opcodes.GTXCOST * 9) @@ -1869,7 +1869,7 @@ def test_xshard_sender_gas_limit(self): b1.add_tx(tx1) state0.finalize_and_add_block(b1) - def test_fork_resolve(self): + async def test_fork_resolve(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -1891,7 +1891,7 @@ def test_fork_resolve(self): state.finalize_and_add_block(b2) self.assertEqual(state.header_tip, b2.header) - def test_root_chain_first_consensus(self): + async def test_root_chain_first_consensus(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -1939,7 +1939,7 @@ def test_root_chain_first_consensus(self): self.assertGreater(b4.header.height, b00.header.height) self.assertEqual(state0.header_tip, b00.header) - def test_shard_state_add_root_block(self): + async def test_shard_state_add_root_block(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -2022,7 +2022,7 @@ def test_shard_state_add_root_block(self): self.assertEqual(state0.db.get_minor_block_by_height(2), b3) self.assertEqual(state0.db.get_minor_block_by_height(3), b4) - def test_shard_reorg_by_adding_root_block(self): + async def test_shard_reorg_by_adding_root_block(self): id1 = Identity.create_random_identity() id2 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -2066,7 +2066,7 @@ def test_shard_reorg_by_adding_root_block(self): self.assertEqual(state0.root_tip, root_block1.header) self.assertEqual(state0.evm_state.trie.root_hash, b1.meta.hash_evm_state_root) - def test_shard_state_add_root_block_too_many_minor_blocks(self): + async def test_shard_state_add_root_block_too_many_minor_blocks(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -2090,7 +2090,7 @@ def test_shard_state_add_root_block_too_many_minor_blocks(self): ) # Too many blocks - with self.assertRaisesRegexp( + with self.assertRaisesRegex( ValueError, "too many minor blocks in the root block" ): state.add_root_block(root_block) @@ -2104,7 +2104,7 @@ def test_shard_state_add_root_block_too_many_minor_blocks(self): root_block.finalize() state.add_root_block(root_block) - def test_shard_state_fork_resolve_with_higher_root_chain(self): + async def test_shard_state_fork_resolve_with_higher_root_chain(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -2138,7 +2138,7 @@ def test_shard_state_fork_resolve_with_higher_root_chain(self): state.finalize_and_add_block(b3) self.assertEqual(state.header_tip, b2.header) - def test_shard_state_difficulty(self): + async def test_shard_state_difficulty(self): env = get_test_env() for shard_config in env.quark_chain_config.shards.values(): shard_config.GENESIS.DIFFICULTY = 10000 @@ -2171,7 +2171,7 @@ def test_shard_state_difficulty(self): state.header_tip.difficulty - state.header_tip.difficulty // 2048 * 2, ) - def test_shard_state_recovery_from_root_block(self): + async def test_shard_state_recovery_from_root_block(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -2213,7 +2213,7 @@ def test_shard_state_recovery_from_root_block(self): recovered_state.evm_state.trie.root_hash, block_meta[4].hash_evm_state_root ) - def test_shard_state_recovery_from_genesis(self): + async def test_shard_state_recovery_from_genesis(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -2249,7 +2249,7 @@ def test_shard_state_recovery_from_genesis(self): recovered_state.evm_state.trie.root_hash, genesis.meta.hash_evm_state_root ) - def test_add_block_receipt_root_not_match(self): + async def test_add_block_receipt_root_not_match(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1) acc3 = Address.create_random_account(full_shard_key=0) @@ -2267,7 +2267,7 @@ def test_add_block_receipt_root_not_match(self): ) b1.meta.hash_evm_receipt_root = bytes(32) - def test_not_update_tip_on_root_fork(self): + async def test_not_update_tip_on_root_fork(self): """block's hash_prev_root_block must be on the same chain with root_tip to update tip. +--+ @@ -2319,7 +2319,7 @@ def test_not_update_tip_on_root_fork(self): # but m1 should still be the tip self.assertEqual(state.header_tip, m1.header) - def test_add_root_block_revert_header_tip(self): + async def test_add_root_block_revert_header_tip(self): """ block's hash_prev_root_block must be on the same chain with root_tip to update tip. +--+ @@ -2390,7 +2390,7 @@ def test_add_root_block_revert_header_tip(self): self.assertEqual(state.root_tip, r4.header) self.assertEqual(state.header_tip, m2.header) - def test_posw_fetch_previous_coinbase_address(self): + async def test_posw_fetch_previous_coinbase_address(self): acc = Address.create_from_identity( Identity.create_random_identity(), full_shard_key=0 ) @@ -2424,7 +2424,7 @@ def test_posw_fetch_previous_coinbase_address(self): # Cached should have certain items (>= 5) self.assertGreaterEqual(len(state.coinbase_addr_cache), 5) - def test_posw_coinbase_send_under_limit(self): + async def test_posw_coinbase_send_under_limit(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) id2 = Identity.create_random_identity() @@ -2533,7 +2533,7 @@ def test_posw_coinbase_send_under_limit(self): res = state.execute_tx(tx3, acc2) self.assertIsNotNone(res, "tx should succeed") - def test_posw_coinbase_send_equal_locked(self): + async def test_posw_coinbase_send_equal_locked(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) env = get_test_env(genesis_account=acc1, genesis_minor_quarkash=0) @@ -2586,7 +2586,7 @@ def test_posw_coinbase_send_equal_locked(self): state.shard_config.COINBASE_AMOUNT - 1, ) - def test_posw_coinbase_send_above_locked(self): + async def test_posw_coinbase_send_above_locked(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_from_identity(id1, full_shard_key=1 << 16) @@ -2657,7 +2657,7 @@ def test_posw_coinbase_send_above_locked(self): - 30000 // 2, # tax rate is 0.5 ) - def test_posw_validate_minor_block_seal(self): + async def test_posw_validate_minor_block_seal(self): acc = Address(b"\x01" * 20, full_shard_key=0) env = get_test_env(genesis_account=acc, genesis_minor_quarkash=256) state = create_default_shard_state(env=env, shard_id=0, posw_override=True) @@ -2697,7 +2697,7 @@ def test_posw_validate_minor_block_seal(self): self.assertEqual(extra1["posw_mineable_blocks"], 256) self.assertEqual(extra1["posw_mined_blocks"], i + 1) - def test_posw_window_edge_cases(self): + async def test_posw_window_edge_cases(self): acc = Address(b"\x01" * 20, full_shard_key=0) env = get_test_env(genesis_account=acc, genesis_minor_quarkash=500) state = create_default_shard_state( @@ -2725,7 +2725,7 @@ def test_posw_window_edge_cases(self): with self.assertRaises(ValueError): state.finalize_and_add_block(m) - def test_incorrect_coinbase_amount(self): + async def test_incorrect_coinbase_amount(self): env = get_test_env() state = create_default_shard_state(env=env) @@ -2748,7 +2748,7 @@ def test_incorrect_coinbase_amount(self): with self.assertRaises(ValueError): state.add_block(b) - def test_shard_coinbase_decay(self): + async def test_shard_coinbase_decay(self): env = get_test_env() state = create_default_shard_state(env=env) coinbase = state.get_coinbase_amount_map(state.shard_config.EPOCH_INTERVAL) @@ -2779,7 +2779,7 @@ def test_shard_coinbase_decay(self): }, ) - def test_enable_tx_timestamp(self): + async def test_enable_tx_timestamp(self): # whitelist acc1, make tx to acc2 # but do not whitelist acc2 and tx fails id1 = Identity.create_random_identity() @@ -2830,12 +2830,12 @@ def test_enable_tx_timestamp(self): b4 = state.create_block_to_mine() self.assertEqual(len(b4.tx_list), 0) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( RuntimeError, "unwhitelisted senders not allowed before tx is enabled" ): state.finalize_and_add_block(b3) - def test_enable_evm_timestamp_with_contract_create(self): + async def test_enable_evm_timestamp_with_contract_create(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -2858,12 +2858,12 @@ def test_enable_evm_timestamp_with_contract_create(self): b2 = state.create_block_to_mine() self.assertEqual(len(b2.tx_list), 0) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( RuntimeError, "smart contract tx is not allowed before evm is enabled" ): state.finalize_and_add_block(b1) - def test_enable_eip155_signer_timestamp(self): + async def test_enable_eip155_signer_timestamp(self): # whitelist acc1, make tx to acc2 # but do not whitelist acc2 and tx fails id1 = Identity.create_random_identity() @@ -2908,7 +2908,7 @@ def test_enable_eip155_signer_timestamp(self): self.assertEqual(len(b3.tx_list), 1) state.finalize_and_add_block(b3) - def test_eip155_signer_attack(self): + async def test_eip155_signer_attack(self): # use chain 0 signed tx to submit to chain 1 id0 = Identity.create_random_identity() id1 = Identity.create_random_identity() @@ -2969,7 +2969,7 @@ def test_eip155_signer_attack(self): ) self.assertFalse(state1.add_tx(tx2)) - def test_enable_evm_timestamp_with_contract_call(self): + async def test_enable_evm_timestamp_with_contract_call(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_random_account(full_shard_key=0) @@ -2999,12 +2999,12 @@ def test_enable_evm_timestamp_with_contract_call(self): b2 = state.create_block_to_mine() self.assertEqual(len(b2.tx_list), 0) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( RuntimeError, "smart contract tx is not allowed before evm is enabled" ): state.finalize_and_add_block(b1) - def test_qkchashx_qkchash_with_rotation_stats(self): + async def test_qkchashx_qkchash_with_rotation_stats(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -3058,7 +3058,7 @@ def _testcase_generate_and_mine_minor_block(qkchash_with_rotation_stats): ) state.finalize_and_add_block(b2) - def test_failed_transaction_gas(self): + async def test_failed_transaction_gas(self): """in-shard revert contract transaction validating the failed transaction gas used""" id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -3123,7 +3123,7 @@ def test_failed_transaction_gas(self): }, ) - def test_skip_under_priced_tx_to_block(self): + async def test_skip_under_priced_tx_to_block(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_random_account(full_shard_key=0) @@ -3171,7 +3171,7 @@ def test_skip_under_priced_tx_to_block(self): self.assertEqual(len(b1.tx_list), 1) self.assertEqual(len(state.tx_queue), 1) - def test_get_root_chain_stakes(self): + async def test_get_root_chain_stakes(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) env = get_test_env(genesis_account=acc1, genesis_minor_quarkash=10000000) @@ -3296,7 +3296,7 @@ def tx_gen(value, data: str): self.assertEqual(stakes, 42) self.assertEqual(signer, random_signer.recipient) - def test_remove_tx_from_queue_with_higher_nonce(self): + async def test_remove_tx_from_queue_with_higher_nonce(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) acc2 = Address.create_random_account(full_shard_key=0) @@ -3366,7 +3366,7 @@ def __prepare_gas_reserve_contract(evm_state, supervisor) -> bytes: evm_state.commit() return contract_addr - def test_pay_native_token_as_gas_contract_api(self): + async def test_pay_native_token_as_gas_contract_api(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) env = get_test_env(genesis_account=acc1, genesis_minor_quarkash=10000000) @@ -3473,7 +3473,7 @@ def tx_gen(data: str, value=None, transfer_token_id=None): self.assertTrue(success) self.assertEqual(int.from_bytes(output, byteorder="big"), 0) - def test_pay_native_token_as_gas_end_to_end(self): + async def test_pay_native_token_as_gas_end_to_end(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) # genesis balance: 100 ether for both QKC and QI @@ -3598,7 +3598,7 @@ def tx_gen( with self.assertRaises(InvalidNativeToken): apply_transaction(evm_state, tx_use_up_reserve, bytes(32)) - def test_mint_new_native_token(self): + async def test_mint_new_native_token(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) env = get_test_env(genesis_account=acc1, genesis_minor_quarkash=10 ** 20) @@ -3695,7 +3695,7 @@ def tx_gen(data: str, value: Optional[int] = 0): self.assertEqual(int.from_bytes(output[64:96], byteorder="big"), amount) @mock_pay_native_token_as_gas(lambda *x: (50, x[-1] * 2)) - def test_native_token_as_gas_in_shard(self): + async def test_native_token_as_gas_in_shard(self): id1 = Identity.create_random_identity() id2 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -3765,7 +3765,7 @@ def tx_gen(value, token_id, to, increment_nonce=True): # 10% refund rate, triple the gas price @mock_pay_native_token_as_gas(lambda *x: (10, x[-1] * 3)) - def test_native_token_as_gas_cross_shard(self): + async def test_native_token_as_gas_cross_shard(self): id1 = Identity.create_random_identity() id2 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) @@ -3894,7 +3894,7 @@ def tx_gen(to): self.get_after_tax_reward(self.shard_coinbase + (3 * gas_price) * 9000), ) - def test_posw_stake_by_block_decay_by_epoch(self): + async def test_posw_stake_by_block_decay_by_epoch(self): acc = Address(b"\x01" * 20, full_shard_key=0) env = get_test_env(genesis_account=acc, genesis_minor_quarkash=200) env.quark_chain_config.ENABLE_POSW_STAKING_DECAY_TIMESTAMP = 100 @@ -3920,7 +3920,7 @@ def test_posw_stake_by_block_decay_by_epoch(self): posw_info = state._posw_info(b1) self.assertEqual(posw_info.posw_mineable_blocks, 200 / 100) - def test_blockhash_in_evm(self): + async def test_blockhash_in_evm(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) diff --git a/quarkchain/cluster/tests/test_utils.py b/quarkchain/cluster/tests/test_utils.py index 566c84ff1..0a3d5c7d4 100644 --- a/quarkchain/cluster/tests/test_utils.py +++ b/quarkchain/cluster/tests/test_utils.py @@ -1,6 +1,6 @@ import asyncio import socket -from contextlib import ContextDecorator, closing +from contextlib import closing from quarkchain.cluster.cluster_config import ( ClusterConfig, @@ -22,7 +22,7 @@ from quarkchain.evm.specials import SystemContract from quarkchain.evm.transactions import Transaction as EvmTransaction from quarkchain.protocol import AbstractConnection -from quarkchain.utils import call_async, check, is_p2 +from quarkchain.utils import check, is_p2 def get_test_env( @@ -307,7 +307,7 @@ def get_next_port(): return s.getsockname()[1] -def create_test_clusters( +async def create_test_clusters( num_cluster, genesis_account, chain_size, @@ -329,7 +329,6 @@ def create_test_clusters( bootstrap_port = get_next_port() # first cluster will listen on this port cluster_list = [] - loop = asyncio.get_event_loop() for i in range(num_cluster): env = get_test_env( @@ -394,7 +393,7 @@ def create_test_clusters( master_server.start() # Wait until the cluster is ready - loop.run_until_complete(master_server.cluster_active_future) + await master_server.cluster_active_future # Substitute diff calculate with an easier one for slave in slave_server_list: @@ -402,10 +401,10 @@ def create_test_clusters( shard.state.diff_calc = easy_diff_calc # Start simple network and connect to seed host - network = SimpleNetwork(env, master_server, loop) - network.start_server() + network = SimpleNetwork(env, master_server) + await network.start_server() if connect and i != 0: - peer = call_async(network.connect("127.0.0.1", bootstrap_port)) + peer = await network.connect("127.0.0.1", bootstrap_port) else: peer = None @@ -414,34 +413,54 @@ def create_test_clusters( return cluster_list -def shutdown_clusters(cluster_list, expect_aborted_rpc_count=0): - loop = asyncio.get_event_loop() - +async def shutdown_clusters(cluster_list, expect_aborted_rpc_count=0): # allow pending RPCs to finish to avoid annoying connection reset error messages - loop.run_until_complete(asyncio.sleep(0.1)) + await asyncio.sleep(0.1) for cluster in cluster_list: # Shutdown simple network first - cluster.network.shutdown() + await cluster.network.shutdown() # Sleep 0.1 so that DESTROY_CLUSTER_PEER_ID command could be processed - loop.run_until_complete(asyncio.sleep(0.1)) - - for cluster in cluster_list: - for slave in cluster.slave_list: - slave.master.close() - loop.run_until_complete(slave.get_shutdown_future()) - - for slave in cluster.master.slave_pool: - slave.close() - - cluster.master.shutdown() - loop.run_until_complete(cluster.master.get_shutdown_future()) - - check(expect_aborted_rpc_count == AbstractConnection.aborted_rpc_count) - - -class ClusterContext(ContextDecorator): + await asyncio.sleep(0.1) + + try: + # Close all connections BEFORE calling shutdown() to ensure tasks are cancelled + for cluster in cluster_list: + for slave in cluster.slave_list: + slave.master.close() + for slave in cluster.master.slave_pool: + slave.close() + + # Give cancelled tasks a moment to clean up + await asyncio.sleep(0.05) + + # Shut down master and slaves, then wait for shutdown futures + for cluster in cluster_list: + cluster.master.shutdown() + for slave in cluster.slave_list: + slave.shutdown() + + for cluster in cluster_list: + await cluster.master.get_shutdown_future() + for slave in cluster.slave_list: + await slave.get_shutdown_future() + if hasattr(slave, 'server') and slave.server: + await slave.server.wait_closed() + + check(expect_aborted_rpc_count == AbstractConnection.aborted_rpc_count) + finally: + # Always cancel remaining tasks, even if check() fails + current = asyncio.current_task() + pending = [t for t in asyncio.all_tasks() if not t.done() and t is not current] + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + AbstractConnection.aborted_rpc_count = 0 + + +class ClusterContext: def __init__( self, num_cluster, @@ -475,8 +494,8 @@ def __init__( check(is_p2(self.num_slaves)) check(is_p2(self.shard_size)) - def __enter__(self): - self.cluster_list = create_test_clusters( + async def __aenter__(self): + self.cluster_list = await create_test_clusters( self.num_cluster, self.genesis_account, self.chain_size, @@ -493,8 +512,8 @@ def __enter__(self): ) return self.cluster_list - def __exit__(self, exc_type, exc_val, traceback): - shutdown_clusters(self.cluster_list) + async def __aexit__(self, exc_type, exc_val, traceback): + await shutdown_clusters(self.cluster_list) def mock_pay_native_token_as_gas(mock=None): @@ -502,15 +521,28 @@ def mock_pay_native_token_as_gas(mock=None): mock = mock or (lambda *x: (100, x[-1])) def decorator(f): - def wrapper(*args, **kwargs): - import quarkchain.evm.messages as m - - m.get_gas_utility_info = mock - m.pay_native_token_as_gas = mock - ret = f(*args, **kwargs) - m.get_gas_utility_info = get_gas_utility_info - m.pay_native_token_as_gas = pay_native_token_as_gas - return ret + if asyncio.iscoroutinefunction(f): + async def wrapper(*args, **kwargs): + import quarkchain.evm.messages as m + + m.get_gas_utility_info = mock + m.pay_native_token_as_gas = mock + try: + return await f(*args, **kwargs) + finally: + m.get_gas_utility_info = get_gas_utility_info + m.pay_native_token_as_gas = pay_native_token_as_gas + else: + def wrapper(*args, **kwargs): + import quarkchain.evm.messages as m + + m.get_gas_utility_info = mock + m.pay_native_token_as_gas = mock + try: + return f(*args, **kwargs) + finally: + m.get_gas_utility_info = get_gas_utility_info + m.pay_native_token_as_gas = pay_native_token_as_gas return wrapper diff --git a/quarkchain/cluster/tx_generator.py b/quarkchain/cluster/tx_generator.py index 2efacd50d..6c30a7449 100644 --- a/quarkchain/cluster/tx_generator.py +++ b/quarkchain/cluster/tx_generator.py @@ -35,7 +35,7 @@ def generate(self, num_tx, x_shard_percent, tx: TypedTransaction): return False self.running = True - asyncio.ensure_future(self.__gen(num_tx, x_shard_percent, tx)) + asyncio.create_task(self.__gen(num_tx, x_shard_percent, tx)) return True async def __gen(self, num_tx, x_shard_percent, sample_tx: TypedTransaction): diff --git a/quarkchain/p2p/auth.py b/quarkchain/p2p/auth.py index 25fcb573a..f94c6fbff 100644 --- a/quarkchain/p2p/auth.py +++ b/quarkchain/p2p/auth.py @@ -10,7 +10,7 @@ from Crypto.Hash import ( keccak as keccaklib, ) # keccak from pycryptodome; unlike eth_hash, its update() method does not leak memory -from eth_hash.preimage import BasePreImage +from eth_hash.abc import PreImageAPI import rlp from rlp import sedes @@ -47,7 +47,7 @@ async def handshake( remote: kademlia.Node, privkey: datatypes.PrivateKey, token: CancelToken ) -> Tuple[ - bytes, bytes, BasePreImage, BasePreImage, asyncio.StreamReader, asyncio.StreamWriter + bytes, bytes, PreImageAPI, PreImageAPI, asyncio.StreamReader, asyncio.StreamWriter ]: # noqa: E501 """ Perform the auth handshake with given remote. @@ -70,7 +70,7 @@ async def _handshake( reader: asyncio.StreamReader, writer: asyncio.StreamWriter, token: CancelToken, -) -> Tuple[bytes, bytes, BasePreImage, BasePreImage]: +) -> Tuple[bytes, bytes, PreImageAPI, PreImageAPI]: """See the handshake() function above. This code was factored out into this helper so that we can create Peers with directly @@ -140,7 +140,7 @@ def derive_secrets( remote_ephemeral_pubkey: datatypes.PublicKey, auth_init_ciphertext: bytes, auth_ack_ciphertext: bytes, - ) -> Tuple[bytes, bytes, BasePreImage, BasePreImage]: + ) -> Tuple[bytes, bytes, PreImageAPI, PreImageAPI]: """Derive base secrets from ephemeral key agreement.""" # ecdhe-shared-secret = ecdh.agree(ephemeral-privkey, remote-ephemeral-pubk) ecdhe_shared_secret = ecies.ecdh_agree( diff --git a/quarkchain/p2p/cancel_token/tests/test_cancel_token.py b/quarkchain/p2p/cancel_token/tests/test_cancel_token.py index 73eaaa887..3c41f7ff1 100644 --- a/quarkchain/p2p/cancel_token/tests/test_cancel_token.py +++ b/quarkchain/p2p/cancel_token/tests/test_cancel_token.py @@ -19,10 +19,12 @@ def test_token_single(): def test_token_chain_event_loop_mismatch(): + # In Python 3.10+, asyncio primitives no longer accept a loop= parameter, + # so EventLoopMismatch is no longer raised. Chaining tokens always works. token = CancelToken("token") - token2 = CancelToken("token2", loop=asyncio.new_event_loop()) - with pytest.raises(EventLoopMismatch): - token.chain(token2) + token2 = CancelToken("token2") + chain = token.chain(token2) + assert chain is not None def test_token_chain_trigger_chain(): @@ -81,65 +83,66 @@ def test_token_chain_trigger_last(): @pytest.mark.asyncio -async def test_token_wait(event_loop): +async def test_token_wait(): token = CancelToken("token") - event_loop.call_soon(token.trigger) - done, pending = await asyncio.wait([token.wait()], timeout=0.1) + asyncio.get_running_loop().call_soon(token.trigger) + done, pending = await asyncio.wait( + [asyncio.create_task(token.wait())], timeout=0.1 + ) assert len(done) == 1 assert len(pending) == 0 assert token.triggered @pytest.mark.asyncio -async def test_wait_cancel_pending_tasks_on_completion(event_loop): +async def test_wait_cancel_pending_tasks_on_completion(): token = CancelToken("token") token2 = CancelToken("token2") chain = token.chain(token2) - event_loop.call_soon(token2.trigger) + asyncio.get_running_loop().call_soon(token2.trigger) await chain.wait() await assert_only_current_task_not_done() @pytest.mark.asyncio -async def test_wait_cancel_pending_tasks_on_cancellation(event_loop): +async def test_wait_cancel_pending_tasks_on_cancellation(): """Test that cancelling a pending CancelToken.wait() coroutine doesn't leave .wait() coroutines for any chained tokens behind. """ token = ( CancelToken("token").chain(CancelToken("token2")).chain(CancelToken("token3")) ) - token_wait_coroutine = token.wait() - done, pending = await asyncio.wait([token_wait_coroutine], timeout=0.1) + token_wait_task = asyncio.create_task(token.wait()) + done, pending = await asyncio.wait([token_wait_task], timeout=0.1) assert len(done) == 0 assert len(pending) == 1 pending_task = pending.pop() - assert pending_task._coro == token_wait_coroutine pending_task.cancel() await assert_only_current_task_not_done() @pytest.mark.asyncio -async def test_cancellable_wait(event_loop): +async def test_cancellable_wait(): fut = asyncio.Future() - event_loop.call_soon(functools.partial(fut.set_result, "result")) + asyncio.get_running_loop().call_soon(functools.partial(fut.set_result, "result")) result = await CancelToken("token").cancellable_wait(fut, timeout=1) assert result == "result" await assert_only_current_task_not_done() @pytest.mark.asyncio -async def test_cancellable_wait_future_exception(event_loop): +async def test_cancellable_wait_future_exception(): fut = asyncio.Future() - event_loop.call_soon(functools.partial(fut.set_exception, Exception())) + asyncio.get_running_loop().call_soon(functools.partial(fut.set_exception, Exception())) with pytest.raises(Exception): await CancelToken("token").cancellable_wait(fut, timeout=1) await assert_only_current_task_not_done() @pytest.mark.asyncio -async def test_cancellable_wait_cancels_subtasks_when_cancelled(event_loop): +async def test_cancellable_wait_cancels_subtasks_when_cancelled(): token = CancelToken("") - future = asyncio.ensure_future(token.cancellable_wait(asyncio.sleep(2))) + future = asyncio.create_task(token.cancellable_wait(asyncio.sleep(2))) with pytest.raises(asyncio.TimeoutError): # asyncio.wait_for() will timeout and then cancel our cancellable_wait() future, but # Task.cancel() doesn't immediately cancels the task @@ -159,7 +162,7 @@ async def test_cancellable_wait_timeout(): @pytest.mark.asyncio -async def test_cancellable_wait_operation_cancelled(event_loop): +async def test_cancellable_wait_operation_cancelled(): token = CancelToken("token") token.trigger() with pytest.raises(OperationCancelled): @@ -171,8 +174,8 @@ async def assert_only_current_task_not_done(): # This sleep() is necessary because Task.cancel() doesn't immediately cancels the task: # https://docs.python.org/3/library/asyncio-task.html#asyncio.Task.cancel await asyncio.sleep(0.01) - for task in asyncio.Task.all_tasks(): - if task == asyncio.Task.current_task(): + for task in asyncio.all_tasks(): + if task == asyncio.current_task(): # This is the task for this very test, so it will be running assert not task.done() else: diff --git a/quarkchain/p2p/cancel_token/token.py b/quarkchain/p2p/cancel_token/token.py index d6e4d3e04..df7da1680 100644 --- a/quarkchain/p2p/cancel_token/token.py +++ b/quarkchain/p2p/cancel_token/token.py @@ -10,15 +10,7 @@ class CancelToken: def __init__(self, name: str, loop: asyncio.AbstractEventLoop = None) -> None: self.name = name self._chain = [] # : List['CancelToken'] - self._triggered = asyncio.Event(loop=loop) - self._loop = loop - - @property - def loop(self) -> asyncio.AbstractEventLoop: - """ - Return the `loop` that this token is bound to. - """ - return self._loop + self._triggered = asyncio.Event() def chain(self, token: "CancelToken") -> "CancelToken": """ @@ -27,12 +19,8 @@ def chain(self, token: "CancelToken") -> "CancelToken": called on either of the chained tokens, but calling trigger() on the new token has no effect on either of the chained tokens. """ - if self.loop != token._loop: - raise EventLoopMismatch( - "Chained CancelToken objects must be on the same event loop" - ) chain_name = ":".join([self.name, token.name]) - chain = CancelToken(chain_name, loop=self.loop) + chain = CancelToken(chain_name) chain._chain.extend([self, token]) return chain @@ -82,9 +70,9 @@ async def wait(self) -> None: if self.triggered_token is not None: return - futures = [asyncio.ensure_future(self._triggered.wait(), loop=self.loop)] + futures = [asyncio.create_task(self._triggered.wait())] for token in self._chain: - futures.append(asyncio.ensure_future(token.wait(), loop=self.loop)) + futures.append(asyncio.create_task(token.wait())) def cancel_not_done(fut: "asyncio.Future[None]") -> None: for future in futures: @@ -98,7 +86,7 @@ async def _wait_for_first(futures: Sequence[Awaitable[Any]]) -> None: await cast(Awaitable[Any], future) return - fut = asyncio.ensure_future(_wait_for_first(futures), loop=self.loop) + fut = asyncio.create_task(_wait_for_first(futures)) fut.add_done_callback(cancel_not_done) await fut @@ -115,7 +103,7 @@ async def cancellable_wait( All pending futures are cancelled before returning. """ futures = [ - asyncio.ensure_future(a, loop=self.loop) + asyncio.ensure_future(a) for a in awaitables + (self.wait(),) ] try: @@ -123,9 +111,8 @@ async def cancellable_wait( futures, timeout=timeout, return_when=asyncio.FIRST_COMPLETED, - loop=self.loop, ) - except asyncio.futures.CancelledError: + except asyncio.CancelledError: # Since we use return_when=asyncio.FIRST_COMPLETED above, we can be sure none of our # futures will be done here, so we don't need to check if any is done before cancelling. for future in futures: diff --git a/quarkchain/p2p/discovery.py b/quarkchain/p2p/discovery.py index 7380c933d..08af933ff 100644 --- a/quarkchain/p2p/discovery.py +++ b/quarkchain/p2p/discovery.py @@ -174,7 +174,7 @@ def update_routing_table(self, node: kademlia.Node) -> None: # with the least recently seen node on that bucket. If the bonding fails the node will # be removed from the bucket and a new one will be picked from the bucket's # replacement cache. - asyncio.ensure_future(self.bond(eviction_candidate)) + asyncio.create_task(self.bond(eviction_candidate)) async def bond(self, node: kademlia.Node) -> bool: """Bond with the given node. @@ -1164,7 +1164,7 @@ async def _prune(self) -> None: ) async def _start_udp_listener(self) -> None: - loop = asyncio.get_event_loop() + loop = asyncio.get_running_loop() # TODO: Support IPv6 addresses as well. await loop.create_datagram_endpoint( lambda: self.proto, local_addr=("0.0.0.0", self.port), family=socket.AF_INET @@ -1498,9 +1498,6 @@ def _test() -> None: from quarkchain.p2p import constants from quarkchain.p2p import ecies - loop = asyncio.get_event_loop() - loop.set_debug(True) - parser = argparse.ArgumentParser() parser.add_argument("-bootnode", type=str, help="The enode to use as bootnode") parser.add_argument("-v5", action="store_true") @@ -1537,9 +1534,12 @@ def _test() -> None: discovery = DiscoveryProtocol(privkey, addr, bootstrap_nodes, 1, cancel_token) async def run() -> None: + loop = asyncio.get_running_loop() await loop.create_datagram_endpoint( lambda: discovery, local_addr=("0.0.0.0", listen_port) ) + for sig in [signal.SIGINT, signal.SIGTERM]: + loop.add_signal_handler(sig, discovery.cancel_token.trigger) try: await discovery.bootstrap() if args.v5: @@ -1554,11 +1554,7 @@ async def run() -> None: finally: await discovery.stop() - for sig in [signal.SIGINT, signal.SIGTERM]: - loop.add_signal_handler(sig, discovery.cancel_token.trigger) - - loop.run_until_complete(run()) - loop.close() + asyncio.run(run(), debug=True) if __name__ == "__main__": diff --git a/quarkchain/p2p/ecies.py b/quarkchain/p2p/ecies.py index b37971c5e..f45744256 100644 --- a/quarkchain/p2p/ecies.py +++ b/quarkchain/p2p/ecies.py @@ -45,8 +45,7 @@ def ecdh_agree(privkey: datatypes.PrivateKey, pubkey: datatypes.PublicKey) -> by privkey_as_int = int(cast(int, privkey)) ec_privkey = ec.derive_private_key(privkey_as_int, CURVE, default_backend()) pubkey_bytes = b"\x04" + pubkey.to_bytes() - pubkey_nums = ec.EllipticCurvePublicNumbers.from_encoded_point(CURVE, pubkey_bytes) - ec_pubkey = pubkey_nums.public_key(default_backend()) + ec_pubkey = ec.EllipticCurvePublicKey.from_encoded_point(CURVE, pubkey_bytes) return ec_privkey.exchange(ec.ECDH(), ec_pubkey) diff --git a/quarkchain/p2p/p2p_manager.py b/quarkchain/p2p/p2p_manager.py index 897b683b7..b0f8c72d8 100644 --- a/quarkchain/p2p/p2p_manager.py +++ b/quarkchain/p2p/p2p_manager.py @@ -1,3 +1,4 @@ +import asyncio import ipaddress import socket from cryptography.hazmat.primitives.constant_time import bytes_eq @@ -125,7 +126,7 @@ async def _run(self) -> None: self.secure_peer.add_sync_task() if self.secure_peer.state == ConnectionState.CONNECTING: self.secure_peer.state = ConnectionState.ACTIVE - self.secure_peer.active_future.set_result(None) + self.secure_peer.active_event.set() try: while self.is_operational: metadata, raw_data = await self.secure_peer.read_metadata_and_raw_data() @@ -368,8 +369,7 @@ class P2PManager(AbstractNetwork): network.port """ - def __init__(self, env, master_server, loop): - self.loop = loop + def __init__(self, env, master_server): self.env = env self.master_server = master_server master_server.network = self # cannot say this is a good design @@ -415,8 +415,8 @@ def __init__(self, env, master_server, loop): self.ip = ipaddress.ip_address(socket.gethostbyname(socket.gethostname())) self.port = env.cluster_config.P2P_PORT - def start(self) -> None: - self.loop.create_task(self.server.run()) + async def start(self) -> None: + asyncio.create_task(self.server.run()) def iterate_peers(self): return [p.secure_peer for p in self.server.peer_pool.connected_nodes.values()] @@ -436,7 +436,7 @@ def get_peer_by_cluster_peer_id(self, cluster_peer_id): return quark_peer.secure_peer return None - def shutdown(self): + async def shutdown(self): for peer_id, peer in self.active_peer_pool.items(): peer.close() - self.loop.run_until_complete(self.server.cancel()) + await self.server.cancel() diff --git a/quarkchain/p2p/peer.py b/quarkchain/p2p/peer.py index 326dbeb19..f67eb79f1 100644 --- a/quarkchain/p2p/peer.py +++ b/quarkchain/p2p/peer.py @@ -32,7 +32,7 @@ from eth_utils import to_tuple -from eth_hash.preimage import BasePreImage +from eth_hash.abc import PreImageAPI from eth_keys import datatypes from quarkchain.utils import Logger, time_ms @@ -128,8 +128,8 @@ async def handshake(remote: Node, factory: "BasePeerFactory") -> "BasePeer": ("writer", asyncio.StreamWriter), ("aes_secret", bytes), ("mac_secret", bytes), - ("egress_mac", BasePreImage), - ("ingress_mac", BasePreImage), + ("egress_mac", PreImageAPI), + ("ingress_mac", PreImageAPI), ], ) diff --git a/quarkchain/p2p/service.py b/quarkchain/p2p/service.py index c85e4d8e4..63ed1db88 100644 --- a/quarkchain/p2p/service.py +++ b/quarkchain/p2p/service.py @@ -44,7 +44,7 @@ def __init__( self._loop = loop - base_token = CancelToken(type(self).__name__, loop=loop) + base_token = CancelToken(type(self).__name__) if token is None: self.cancel_token = base_token @@ -58,7 +58,7 @@ def logger(self): def get_event_loop(self) -> asyncio.AbstractEventLoop: if self._loop is None: - return asyncio.get_event_loop() + return asyncio.get_running_loop() else: return self._loop @@ -134,7 +134,7 @@ async def _run_task_wrapper() -> None: # self.logger.debug("Task %s finished with no errors" % awaitable) pass - self._tasks.add(asyncio.ensure_future(_run_task_wrapper())) + self._tasks.add(asyncio.create_task(_run_task_wrapper())) self.gc() def run_daemon_task(self, awaitable: Awaitable[Any]) -> None: diff --git a/quarkchain/p2p/tests/test_discovery.py b/quarkchain/p2p/tests/test_discovery.py index a520bc677..6e9075a9b 100644 --- a/quarkchain/p2p/tests/test_discovery.py +++ b/quarkchain/p2p/tests/test_discovery.py @@ -140,8 +140,9 @@ async def test_wait_ping(echo): node = random_node() # Schedule a call to proto.recv_ping() simulating a ping from the node we expect. - recv_ping_coroutine = asyncio.coroutine(lambda: proto.recv_ping_v4(node, echo, b"")) - asyncio.ensure_future(recv_ping_coroutine()) + async def do_recv_ping(): + proto.recv_ping_v4(node, echo, b"") + asyncio.create_task(do_recv_ping()) got_ping = await proto.wait_ping(node) @@ -151,8 +152,9 @@ async def test_wait_ping(echo): # If we waited for a ping from a different node, wait_ping() would timeout and thus return # false. - recv_ping_coroutine = asyncio.coroutine(lambda: proto.recv_ping_v4(node, echo, b"")) - asyncio.ensure_future(recv_ping_coroutine()) + async def do_recv_ping2(): + proto.recv_ping_v4(node, echo, b"") + asyncio.create_task(do_recv_ping2()) node2 = random_node() got_ping = await proto.wait_ping(node2) @@ -174,10 +176,9 @@ async def test_wait_pong(): token, discovery._get_msg_expiration(), ] - recv_pong_coroutine = asyncio.coroutine( - lambda: proto.recv_pong_v4(node, pong_msg_payload, b"") - ) - asyncio.ensure_future(recv_pong_coroutine()) + async def do_recv_pong(): + proto.recv_pong_v4(node, pong_msg_payload, b"") + asyncio.create_task(do_recv_pong()) got_pong = await proto.wait_pong_v4(node, token) @@ -189,15 +190,14 @@ async def test_wait_pong(): # If the remote node echoed something different than what we expected, wait_pong() would # timeout. wrong_token = b"foo" - pong_msg_payload = [ + wrong_pong_msg_payload = [ us.address.to_endpoint(), wrong_token, discovery._get_msg_expiration(), ] - recv_pong_coroutine = asyncio.coroutine( - lambda: proto.recv_pong_v4(node, pong_msg_payload, b"") - ) - asyncio.ensure_future(recv_pong_coroutine()) + async def do_recv_wrong_pong(): + proto.recv_pong_v4(node, wrong_pong_msg_payload, b"") + asyncio.create_task(do_recv_wrong_pong()) got_pong = await proto.wait_pong_v4(node, token) @@ -217,10 +217,9 @@ async def test_wait_neighbours(): [n.address.to_endpoint() + [n.pubkey.to_bytes()] for n in neighbours], discovery._get_msg_expiration(), ] - recv_neighbours_coroutine = asyncio.coroutine( - lambda: proto.recv_neighbours_v4(node, neighbours_msg_payload, b"") - ) - asyncio.ensure_future(recv_neighbours_coroutine()) + async def do_recv_neighbours(): + proto.recv_neighbours_v4(node, neighbours_msg_payload, b"") + asyncio.create_task(do_recv_neighbours()) received_neighbours = await proto.wait_neighbours(node) @@ -245,7 +244,9 @@ async def test_bond(): proto.send_ping_v4 = lambda remote: token # Pretend we get a pong from the node we are bonding with. - proto.wait_pong_v4 = asyncio.coroutine(lambda n, t: t == token and n == node) + async def mock_wait_pong_v4(n, t): + return t == token and n == node + proto.wait_pong_v4 = mock_wait_pong_v4 bonded = await proto.bond(node) @@ -274,12 +275,12 @@ async def test_update_routing_table_triggers_bond_if_eviction_candidate(): bond_called = False - def bond(node): + async def bond(node): nonlocal bond_called bond_called = True assert node == old_node - proto.bond = asyncio.coroutine(bond) + proto.bond = bond # Pretend our routing table failed to add the new node by returning the least recently seen # node for an eviction check. proto.routing.add_node = lambda n: old_node @@ -389,13 +390,13 @@ def test_find_node_neighbours_v5(): @pytest.mark.asyncio -async def test_topic_query(event_loop, short_timeout_undo): - bob = await get_listening_discovery_protocol(event_loop) +async def test_topic_query(short_timeout_undo): + bob = await get_listening_discovery_protocol() les_nodes = [random_node() for _ in range(10)] topic = b"les" for n in les_nodes: bob.topic_table.add_node(n, topic) - alice = await get_listening_discovery_protocol(event_loop) + alice = await get_listening_discovery_protocol() echo = alice.send_topic_query(bob.this_node, topic) received_nodes = await alice.wait_topic_nodes(bob.this_node, echo) @@ -405,9 +406,9 @@ async def test_topic_query(event_loop, short_timeout_undo): @pytest.mark.asyncio -async def test_topic_register(event_loop): - bob = await get_listening_discovery_protocol(event_loop) - alice = await get_listening_discovery_protocol(event_loop) +async def test_topic_register(): + bob = await get_listening_discovery_protocol() + alice = await get_listening_discovery_protocol() topics = [b"les", b"les2"] # In order to register ourselves under a given topic we need to first get a ticket. @@ -553,10 +554,11 @@ def get_discovery_protocol(seed=b"seed", address=None): ) -async def get_listening_discovery_protocol(event_loop): +async def get_listening_discovery_protocol(): addr = kademlia.Address("127.0.0.1", random.randint(1024, 9999)) proto = get_discovery_protocol(os.urandom(4), addr) - await event_loop.create_datagram_endpoint( + loop = asyncio.get_running_loop() + await loop.create_datagram_endpoint( lambda: proto, local_addr=(addr.ip, addr.udp_port), family=socket.AF_INET ) return proto diff --git a/quarkchain/p2p/tests/test_peer_collect_sub_proto_msgs.py b/quarkchain/p2p/tests/test_peer_collect_sub_proto_msgs.py index fa62de315..0db1bbd53 100644 --- a/quarkchain/p2p/tests/test_peer_collect_sub_proto_msgs.py +++ b/quarkchain/p2p/tests/test_peer_collect_sub_proto_msgs.py @@ -7,8 +7,8 @@ from quarkchain.p2p.tools.paragon.helpers import get_directly_linked_peers @pytest.mark.asyncio -async def test_peer_subscriber_filters_messages(request, event_loop): - peer, remote = await get_directly_linked_peers(request, event_loop) +async def test_peer_subscriber_filters_messages(request): + peer, remote = await get_directly_linked_peers(request) with peer.collect_sub_proto_messages() as collector: assert collector in peer._subscribers diff --git a/quarkchain/p2p/tests/test_peer_subscriber.py b/quarkchain/p2p/tests/test_peer_subscriber.py index 1253957b6..6a3be5c04 100644 --- a/quarkchain/p2p/tests/test_peer_subscriber.py +++ b/quarkchain/p2p/tests/test_peer_subscriber.py @@ -25,8 +25,8 @@ class AllSubscriber(PeerSubscriber): @pytest.mark.asyncio -async def test_peer_subscriber_filters_messages(request, event_loop): - peer, remote = await get_directly_linked_peers(request, event_loop) +async def test_peer_subscriber_filters_messages(request): + peer, remote = await get_directly_linked_peers(request) get_sum_subscriber = GetSumSubscriber() all_subscriber = AllSubscriber() diff --git a/quarkchain/p2p/tests/test_service.py b/quarkchain/p2p/tests/test_service.py index 9a72e45e4..b5f5381bf 100644 --- a/quarkchain/p2p/tests/test_service.py +++ b/quarkchain/p2p/tests/test_service.py @@ -24,7 +24,7 @@ async def _run(self): @pytest.mark.asyncio async def test_daemon_exit_causes_parent_cancellation(): service = ParentService() - asyncio.ensure_future(service.run()) + asyncio.create_task(service.run()) await asyncio.sleep(0.01) @@ -43,7 +43,7 @@ async def test_daemon_exit_causes_parent_cancellation(): @pytest.mark.asyncio async def test_service_tasks_do_not_leak_memory(): service = WaitService() - asyncio.ensure_future(service.run()) + asyncio.create_task(service.run()) end = asyncio.Event() @@ -76,7 +76,7 @@ async def run_until_end(): async def test_service_children_do_not_leak_memory(): parent = WaitService() child = WaitService() - asyncio.ensure_future(parent.run()) + asyncio.create_task(parent.run()) parent.run_child_service(child) diff --git a/quarkchain/p2p/tools/paragon/helpers.py b/quarkchain/p2p/tools/paragon/helpers.py index 8e1fb62a3..728ac8a97 100644 --- a/quarkchain/p2p/tools/paragon/helpers.py +++ b/quarkchain/p2p/tools/paragon/helpers.py @@ -126,7 +126,7 @@ async def do_handshake() -> None: f_alice.set_result(alice) handshake_finished.set() - asyncio.ensure_future(do_handshake()) + asyncio.create_task(do_handshake()) use_eip8 = False responder = auth.HandshakeResponder( @@ -169,7 +169,6 @@ async def do_handshake() -> None: async def get_directly_linked_peers( request: Any, - event_loop: asyncio.AbstractEventLoop, alice_factory: BasePeerFactory = None, bob_factory: BasePeerFactory = None, ) -> Tuple[BasePeer, BasePeer]: @@ -191,12 +190,14 @@ async def get_directly_linked_peers( # Perform the handshake for the enabled sub-protocol. await asyncio.gather(alice.do_sub_proto_handshake(), bob.do_sub_proto_handshake()) - asyncio.ensure_future(alice.run()) - asyncio.ensure_future(bob.run()) + asyncio.create_task(alice.run()) + asyncio.create_task(bob.run()) + + loop = asyncio.get_running_loop() def finalizer() -> None: - event_loop.run_until_complete( - asyncio.gather(alice.cancel(), bob.cancel(), loop=event_loop) + loop.run_until_complete( + asyncio.gather(alice.cancel(), bob.cancel()) ) request.addfinalizer(finalizer) diff --git a/quarkchain/protocol.py b/quarkchain/protocol.py index 1988344ad..815cbcb4a 100644 --- a/quarkchain/protocol.py +++ b/quarkchain/protocol.py @@ -41,7 +41,6 @@ def __init__( op_ser_map, op_non_rpc_map, op_rpc_map, - loop=None, metadata_class=Metadata, name=None, ): @@ -53,13 +52,14 @@ def __init__( self.peer_rpc_id = -1 self.rpc_id = 0 # 0 is for non-rpc (fire-and-forget) self.rpc_future_map = dict() - loop = loop if loop else asyncio.get_event_loop() - self.active_future = loop.create_future() - self.close_future = loop.create_future() + self.active_event = asyncio.Event() + self.close_event = asyncio.Event() self.metadata_class = metadata_class if name is None: name = "conn_{}".format(self.__get_next_connection_id()) self.name = name if name else "[connection name missing]" + self._loop_task = None # Track the active_and_loop_forever task + self._handler_tasks = set() # Track message handler tasks async def read_metadata_and_raw_data(self): raise NotImplementedError() @@ -182,35 +182,53 @@ async def loop_once(self): self.close_with_error("{}: error reading request: {}".format(self.name, e)) return - asyncio.ensure_future( + task = asyncio.create_task( self.__internal_handle_metadata_and_raw_data(metadata, raw_data) ) + self._handler_tasks.add(task) + task.add_done_callback(self._handler_tasks.discard) async def active_and_loop_forever(self): - if self.state == ConnectionState.CONNECTING: - self.state = ConnectionState.ACTIVE - self.active_future.set_result(None) - while self.state == ConnectionState.ACTIVE: - await self.loop_once() - - assert self.state == ConnectionState.CLOSED - - # Abort all in-flight RPCs - for rpc_id, future in self.rpc_future_map.items(): - future.set_exception(RuntimeError("{}: connection abort".format(self.name))) - AbstractConnection.aborted_rpc_count += len(self.rpc_future_map) - self.rpc_future_map.clear() + try: + if self.state == ConnectionState.CONNECTING: + self.state = ConnectionState.ACTIVE + self.active_event.set() + while self.state == ConnectionState.ACTIVE: + await self.loop_once() + finally: + # Cancel any in-flight handler tasks + for task in self._handler_tasks: + task.cancel() + self._handler_tasks.clear() + + # Ensure active_event is set so wait_until_active() callers are not stuck + # (e.g. if connection closed before it ever became active) + if not self.active_event.is_set(): + self.active_event.set() + + if self.state != ConnectionState.CLOSED: + self.state = ConnectionState.CLOSED + self.close_event.set() + + # Abort all in-flight RPCs (runs even on cancellation) + for rpc_id, future in self.rpc_future_map.items(): + if not future.done(): + future.set_exception(RuntimeError("{}: connection abort".format(self.name))) + AbstractConnection.aborted_rpc_count += len(self.rpc_future_map) + self.rpc_future_map.clear() async def wait_until_active(self): - await self.active_future + await self.active_event.wait() async def wait_until_closed(self): - await self.close_future + await self.close_event.wait() def close(self): if self.state != ConnectionState.CLOSED: self.state = ConnectionState.CLOSED - self.close_future.set_result(None) + self.close_event.set() + if self._loop_task and not self._loop_task.done(): + self._loop_task.cancel() def close_with_error(self, error): self.close() @@ -235,14 +253,12 @@ def __init__( op_ser_map, op_non_rpc_map, op_rpc_map, - loop=None, metadata_class=Metadata, name=None, command_size_limit=None, # No limit ): - loop = loop if loop else asyncio.get_event_loop() super().__init__( - op_ser_map, op_non_rpc_map, op_rpc_map, loop, metadata_class, name=name + op_ser_map, op_non_rpc_map, op_rpc_map, metadata_class, name=name ) self.env = env self.reader = reader @@ -291,5 +307,19 @@ def write_raw_data(self, metadata, raw_data): def close(self): """ Override AbstractConnection.close() """ + self.reader.feed_eof() self.writer.close() super().close() + + async def active_and_loop_forever(self): + """ Override AbstractConnection.active_and_loop_forever() to ensure the + underlying TCP socket is released even when the task is cancelled. + Without this, cancelled tasks leave file descriptors registered in epoll + indefinitely, which accumulates across many tests. + """ + try: + await super().active_and_loop_forever() + except asyncio.CancelledError: + if not self.writer.is_closing(): + self.writer.close() + raise diff --git a/quarkchain/tools/adjust_difficulty.py b/quarkchain/tools/adjust_difficulty.py index 26782ff24..6b85c220c 100644 --- a/quarkchain/tools/adjust_difficulty.py +++ b/quarkchain/tools/adjust_difficulty.py @@ -147,9 +147,9 @@ def main(): args = parser.parse_args() if args.balanced: - asyncio.get_event_loop().run_until_complete(async_adjust_difficulty(args)) + asyncio.run(async_adjust_difficulty(args)) else: - asyncio.get_event_loop().run_until_complete(adjust_imbalanced_hashpower(args)) + asyncio.run(adjust_imbalanced_hashpower(args)) if __name__ == "__main__": diff --git a/quarkchain/tools/batch_deploy_contract.py b/quarkchain/tools/batch_deploy_contract.py index f1fd1cfcf..a91c2c0ef 100644 --- a/quarkchain/tools/batch_deploy_contract.py +++ b/quarkchain/tools/batch_deploy_contract.py @@ -118,7 +118,7 @@ def main(): genesisId = Identity.create_from_key(DEFAULT_ENV.config.GENESIS_KEY) endpoint = Endpoint("http://" + args.jrpc_endpoint) - asyncio.get_event_loop().run_until_complete(deploy(endpoint, genesisId, data)) + asyncio.run(deploy(endpoint, genesisId, data)) if __name__ == "__main__": diff --git a/quarkchain/tools/client_version_poll.py b/quarkchain/tools/client_version_poll.py index 9c35e5c84..ae94bb218 100644 --- a/quarkchain/tools/client_version_poll.py +++ b/quarkchain/tools/client_version_poll.py @@ -92,4 +92,4 @@ async def main(): if __name__ == "__main__": - asyncio.get_event_loop().run_until_complete(main()) + asyncio.run(main()) diff --git a/quarkchain/tools/fund_testnet.py b/quarkchain/tools/fund_testnet.py index b550fd5a3..0c65a557b 100644 --- a/quarkchain/tools/fund_testnet.py +++ b/quarkchain/tools/fund_testnet.py @@ -175,7 +175,7 @@ def main(): endpoint = Endpoint("http://" + args.jrpc_endpoint) addrByAmount = read_addr(args.tqkc_file) - asyncio.get_event_loop().run_until_complete(fund(endpoint, genesisId, addrByAmount)) + asyncio.run(fund(endpoint, genesisId, addrByAmount)) if __name__ == "__main__": diff --git a/quarkchain/tools/monitoring.py b/quarkchain/tools/monitoring.py index 3ce3139a6..3d9e7e017 100644 --- a/quarkchain/tools/monitoring.py +++ b/quarkchain/tools/monitoring.py @@ -67,9 +67,7 @@ async def crawl_async(ip, p2p_port, jrpc_port): def crawl_bfs(ip, p2p_port, jrpc_port): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - cache = loop.run_until_complete(crawl_async(ip, p2p_port, jrpc_port)) + cache = asyncio.run(crawl_async(ip, p2p_port, jrpc_port)) res = {} # we can avoid the loop, but it will look crazy @@ -181,7 +179,7 @@ def watch_nodes_stats(ip, p2p_port, jrpc_port, ip_lookup={}): for idx, cluster in enumerate(clusters) ] ) - asyncio.get_event_loop().run_until_complete(async_watch(clusters)) + asyncio.run(async_watch(clusters)) def main(): diff --git a/quarkchain/utils.py b/quarkchain/utils.py index a0dfa7e18..a20282434 100644 --- a/quarkchain/utils.py +++ b/quarkchain/utils.py @@ -74,20 +74,33 @@ def crash(): p[0] = b"x" -def call_async(coro): - future = asyncio.ensure_future(coro) - asyncio.get_event_loop().run_until_complete(future) - return future.result() +def _get_or_create_event_loop(): + """Get the running event loop, or create and set a new one if none is running. - -def assert_true_with_timeout(f, duration=1): - async def d(): - deadline = time.time() + duration - while not f() and time.time() < deadline: - await asyncio.sleep(0.001) - assert f() - - asyncio.get_event_loop().run_until_complete(d()) + In Python 3.12+, asyncio.get_event_loop() raises DeprecationWarning when + there is no current event loop. This helper uses get_running_loop() first + and falls back to creating a new loop for sync contexts. + """ + try: + return asyncio.get_running_loop() + except RuntimeError: + pass + try: + loop = asyncio.get_event_loop() + if not loop.is_closed(): + return loop + except RuntimeError: + pass + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + return loop + + +async def async_assert_true_with_timeout(f, duration=3): + deadline = time.time() + duration + while not f() and time.time() < deadline: + await asyncio.sleep(0.001) + assert f() _LOGGING_FILE_PREFIX = os.path.join("logging", "__init__.") @@ -98,7 +111,7 @@ class QKCLogger(logging.getLoggerClass()): refer to ABSLLogger """ - def findCaller(self, stack_info=False): + def findCaller(self, stack_info=False, stacklevel=1): frame = sys._getframe(2) f_to_skip = { func for func in dir(Logger) if callable(getattr(Logger, func)) @@ -354,7 +367,13 @@ def send_log_to_kafka(cls, level_str, msg): "level": level_str, "message": msg, } - asyncio.ensure_future( + try: + loop = asyncio.get_running_loop() + except RuntimeError: + # No running event loop (e.g., during startup/shutdown). + # Silently skip Kafka logging to avoid crashing the caller. + return + loop.create_task( cls._kafka_logger.log_kafka_sample_async( cls._kafka_logger.cluster_config.MONITORING.ERRORS, sample )