test: fix dbcrash and fee_estimation extended tests

.cirrus.yml

@@ -169,7 +169,6 @@ task:
     - netsh int ipv4 set dynamicport tcp start=1025 num=64511
     - netsh int ipv6 set dynamicport tcp start=1025 num=64511
     # Heavier tests are moved to a secondary task
-    # Exclude feature_dbcrash and feature_fee_estimation, failing https://github.com/ElementsProject/elements/pull/1298
     - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 --extended --exclude wallet_avoidreuse,feature_trim_headers,feature_dbcrash,feature_fee_estimation
 
 task:
@@ -186,8 +185,7 @@ task:
     - netsh int ipv4 set dynamicport tcp start=1025 num=64511
     - netsh int ipv6 set dynamicport tcp start=1025 num=64511
     # Execute tests excluded from the main task
-    # Ignore failures for now until https://github.com/ElementsProject/elements/pull/1298 is merged
-    - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 wallet_avoidreuse feature_trim_headers feature_dbcrash feature_fee_estimation || true
+    - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 wallet_avoidreuse feature_trim_headers feature_dbcrash feature_fee_estimation
 
 task:
   name: 'ARM [unit tests, no functional tests] [bullseye]'

test/functional/feature_dbcrash.py

@@ -186,17 +186,18 @@ def verify_utxo_hash(self):
             assert_equal(nodei_utxo_hash, node3_utxo_hash)
 
     def generate_small_transactions(self, node, count, utxo_list):
-        FEE = 1000  # TODO: replace this with node relay fee based calculation
         num_transactions = 0
         random.shuffle(utxo_list)
         while len(utxo_list) >= 2 and num_transactions < count:
+            fee = 1000
             tx = CTransaction()
             input_amount = 0
             for _ in range(2):
                 utxo = utxo_list.pop()
                 tx.vin.append(CTxIn(COutPoint(int(utxo['txid'], 16), utxo['vout'])))
                 input_amount += int(utxo['amount'] * COIN)
-            output_amount = (input_amount - FEE) // 3
+            output_amount = (input_amount - fee) // 3
+            fee = input_amount - (3 * output_amount)
 
             if output_amount <= 0:
                 # Sanity check -- if we chose inputs that are too small, skip
@@ -205,6 +206,9 @@ def generate_small_transactions(self, node, count, utxo_list):
             for _ in range(3):
                 tx.vout.append(CTxOut(output_amount, bytes.fromhex(utxo['scriptPubKey'])))
 
+            # ELEMENTS: add fee output
+            tx.vout.append(CTxOut(fee))
+
             # Sign and send the transaction to get into the mempool
             tx_signed_hex = node.signrawtransactionwithwallet(tx.serialize().hex())['hex']
             node.sendrawtransaction(tx_signed_hex)
@@ -234,10 +238,13 @@ def run_test(self):
         # Main test loop:
         # each time through the loop, generate a bunch of transactions,
         # and then either mine a single new block on the tip, or some-sized reorg.
-        for i in range(40):
-            self.log.info(f"Iteration {i}, generating 2500 transactions {self.restart_counts}")
+        # ELEMENTS: modified to only run until successfully testing a node crash on restart
+        # with a maximum of 10 iterations
+        i = 0
+        while self.crashed_on_restart < 1:
+            self.log.info(f"Iteration {i}, generating 3000 transactions {self.restart_counts}")
             # Generate a bunch of small-ish transactions
-            self.generate_small_transactions(self.nodes[3], 2500, utxo_list)
+            self.generate_small_transactions(self.nodes[3], 3000, utxo_list)
             # Pick a random block between current tip, and starting tip
             current_height = self.nodes[3].getblockcount()
             random_height = random.randint(starting_tip_height, current_height)
@@ -265,6 +272,11 @@ def run_test(self):
             utxo_list = self.nodes[3].listunspent()
             self.log.debug(f"Node3 utxo count: {len(utxo_list)}")
 
+            if i >= 11:
+                raise AssertionError(f"12 iterations without node crash, this should not happen")
+            else:
+                i += 1
+
         # Check that the utxo hashes agree with node3
         # Useful side effect: each utxo cache gets flushed here, so that we
         # won't get crashes on shutdown at the end of the test.

test/functional/feature_fee_estimation.py

@@ -134,11 +134,13 @@ def send_tx(node, utxo, feerate):
     """Broadcast a 1in-1out transaction with a specific input and feerate (sat/vb)."""
     tx = CTransaction()
     tx.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), REDEEM_SCRIPT)]
-    tx.vout = [CTxOut(int(utxo["amount"] * COIN), P2SH)]
+    tx.vout = [CTxOut(int(utxo["amount"] * COIN), P2SH), CTxOut(int(utxo["amount"] * COIN))]
 
     # vbytes == bytes as we are using legacy transactions
     fee = tx.get_vsize() * feerate
-    tx.vout[0].nValue -= fee
+    amount = tx.vout[0].nValue.getAmount()
+    tx.vout[0].nValue.setToAmount(amount - fee)
+    tx.vout[1].nValue.setToAmount(fee)
 
     return node.sendrawtransaction(tx.serialize().hex())
 
@@ -208,7 +210,7 @@ def transact_and_mine(self, numblocks, mining_node):
 
     def initial_split(self, node):
         """Split two coinbase UTxOs into many small coins"""
-        utxo_count = 2048
+        utxo_count = 1450 # ELEMENTS reduced to fit into max tx weight
         self.confutxo = []
         splitted_amount = Decimal("0.04")
         fee = Decimal("0.1")
@@ -220,6 +222,7 @@ def initial_split(self, node):
         ]
         tx.vout = [CTxOut(int(splitted_amount * COIN), P2SH) for _ in range(utxo_count)]
         tx.vout.append(CTxOut(int(change * COIN), P2SH))
+        tx.vout.append(CTxOut(int(fee * COIN)))
         txhex = node.signrawtransactionwithwallet(tx.serialize().hex())["hex"]
         txid = node.sendrawtransaction(txhex)
         self.confutxo = [

test/functional/feature_trim_headers.py

@@ -246,7 +246,12 @@ def run_test(self):
         self.connect_nodes(0, 1)
         self.connect_nodes(0, 2)
 
-        self.sync_all()
+        import time
+        self.log.info("Sync blocks")
+        start = time.time()
+        self.sync_blocks(timeout=1200) # extended timeout for windows CI
+        end = time.time()
+        self.log.info(f"Block sync took {end - start:.2f} seconds")
         self.check_height(expected_height, all=True)
 
         self.log.info("Prune the pruned node")

test/functional/test_framework/util.py

@@ -519,7 +519,10 @@ def create_confirmed_utxos(test_framework, fee, node, count, **kwargs):
         inputs = []
         inputs.append({"txid": t["txid"], "vout": t["vout"]})
         send_value = t['amount'] - fee
-        outputs = [{addr1: satoshi_round(send_value / 2)}, {addr2: satoshi_round(send_value / 2)}, {"fee": fee}]
+        # ELEMENTS: ensure outputs balance with inputs
+        val1 = satoshi_round(send_value / 2)
+        val2 = send_value - val1
+        outputs = [{addr1: val1}, {addr2: val2}, {"fee": fee}]
         raw_tx = node.createrawtransaction(inputs, outputs)
         signed_tx = node.signrawtransactionwithwallet(raw_tx)["hex"]
         node.sendrawtransaction(signed_tx)