From 01af622dee2d8db1365c1b3482f3a86c8b1978d9 Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Mon, 9 Sep 2024 18:33:18 -0700 Subject: [PATCH] add SPEEDEX to calculator, rework to calculate multiple phases --- scripts/resource-calc.ipynb | 882 +++++++++++++++++++++++++----------- 1 file changed, 609 insertions(+), 273 deletions(-) diff --git a/scripts/resource-calc.ipynb b/scripts/resource-calc.ipynb index d9991fb62f..596dc00939 100644 --- a/scripts/resource-calc.ipynb +++ b/scripts/resource-calc.ipynb @@ -1,13 +1,389 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Calculator" + ] + }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ - "# inputs\n", + "def pprint(*args, **kwargs):\n", + " formatted_args = []\n", + " for arg in args:\n", + " if isinstance(arg, int):\n", + " formatted_args.append(f\"{arg:,}\")\n", + " elif isinstance(arg, float):\n", + " formatted_args.append(f\"{arg:,.2f}\")\n", + " else:\n", + " formatted_args.append(arg)\n", + " print(*formatted_args, **kwargs)\n", + "\n", + "def calculate_all(title):\n", + "\n", + " print(\"*\"*30, title)\n", + " TPL = { e: tps*ledger_time for e, tps in TPS.items() }\n", + " tpl_total = sum(TPL.values())\n", + "\n", + " LE_reads_per_op = {}\n", + " LE_writes_per_op = {}\n", + " LE_size = {}\n", + " event_size_per_op = {}\n", + "\n", + " if big_settings:\n", + " # classic\n", + " # max number of LE reads per op (vnext: limited DEX)\n", + " LE_reads_per_op['classic'] = 10 if v_next else 1000\n", + " # max number of LE writes per op\n", + " LE_writes_per_op['classic'] = LE_reads_per_op['classic']\n", + " # avg size of LE\n", + " LE_size['classic'] = 140\n", + " # models each \"write\" as a token transfer\n", + " event_size_per_op['classic'] = LE_writes_per_op['classic']*token_transfer_event_size\n", + "\n", + " # soroban\n", + " # max number of LE reads\n", + " LE_reads_per_op['soroban'] = 40\n", + " # max number of LE writes\n", + " LE_writes_per_op['soroban'] = 25\n", + " # write as much as possible, use max allowed per tx\n", + " # note that this also impacts the total of amount read per tx\n", + " # but as tx also need to load wasm blobs, in average is probably fine\n", + " LE_size['soroban'] = (65*1024)/LE_writes_per_op['soroban']\n", + " # max event produced\n", + " event_size_per_op['soroban'] = 16*1024\n", + "\n", + " # speedex\n", + " LE_reads_per_op['speedex'] = 4\n", + " LE_writes_per_op['speedex'] = 2\n", + " LE_size['speedex'] = 512\n", + " event_size_per_op['speedex'] = LE_writes_per_op['speedex']*token_transfer_event_size\n", + "\n", + " # tps getledgerentries (watcher node)\n", + " gle_tps = 5\n", + " else:\n", + " # classic\n", + " # avg number of LE reads per op \n", + " LE_reads_per_op['classic'] = 5\n", + " # avg number of LE writes per op\n", + " LE_writes_per_op['classic'] = 5\n", + " # avg size of LE\n", + " LE_size['classic'] = 140\n", + " # models each \"write\" as a token transfer\n", + " event_size_per_op['classic'] = LE_writes_per_op['classic']*token_transfer_event_size\n", + "\n", + " # soroban\n", + " # avg number of LE reads\n", + " LE_reads_per_op['soroban'] = 10\n", + " # avg number of LE writes\n", + " LE_writes_per_op['soroban'] = 5\n", + " # use a 10th of capacity\n", + " LE_size['soroban'] = (65*1024)/LE_writes_per_op['soroban']/10\n", + " # event produced is 50% of max\n", + " event_size_per_op['soroban'] = 16*1024/2\n", + "\n", + " # speedex\n", + " LE_reads_per_op['speedex'] = 4\n", + " LE_writes_per_op['speedex'] = 2\n", + " # assumes simple transfers only\n", + " LE_size['speedex'] = 140\n", + " event_size_per_op['speedex'] = LE_writes_per_op['speedex']*token_transfer_event_size\n", + "\n", + " # tps getledgerentries (watcher node)\n", + " gle_tps = 2\n", + "\n", + " ###########\n", + " # bucket list modeling\n", + "\n", + " # number of iops and bytes needed to read a single ledger entry\n", + " # need to\n", + " # 1. walk from recent to oldest buckets (up to bucket 19)\n", + " # bloom filter helps short circuit lookups\n", + " # 2. for each bucket, page size on larger buckets -> more than 1 read occurs\n", + "\n", + " # additional extra reads (overhead)\n", + " bl_avg_extra_nb_reads = 1\n", + "\n", + " if large_bl:\n", + " # more Soroban entries in max mode\n", + " bl_avg_le_size = 1024\n", + " else:\n", + " bl_avg_le_size = 300\n", + "\n", + "\n", + " bl_avg_extra_bytes_read = bl_avg_le_size*bl_avg_extra_nb_reads\n", + "\n", + " bl_avg_nb_reads = 1+bl_avg_extra_nb_reads\n", + "\n", + " def bl_bytes_read_per_le(le_size):\n", + " return le_size + bl_avg_extra_bytes_read\n", + "\n", + " ############\n", + " # bucket list activity\n", + "\n", + " # size of an account LE\n", + " le_account_bytes = 127\n", + "\n", + " ledger_LE_reads = {}\n", + " ledger_bytes_read = {}\n", + " ledger_LE_writes = {}\n", + " ledger_bytes_write = {}\n", + "\n", + " # fees/seqnum\n", + " ledger_LE_reads['fee'] = tpl_total*bl_avg_nb_reads\n", + " ledger_bytes_read['fee'] = tpl_total*bl_bytes_read_per_le(le_account_bytes)\n", + " ledger_LE_writes['fee'] = tpl_total\n", + " ledger_bytes_write['fee'] = ledger_LE_writes['fee'] * le_account_bytes\n", + "\n", + " # per phase\n", + "\n", + " # classic\n", + " ledger_LE_reads['classic'] = TPL['classic']*LE_reads_per_op['classic']\n", + " ledger_bytes_read['classic'] = ledger_LE_reads['classic']*bl_bytes_read_per_le(LE_size['classic'])\n", + " ledger_LE_reads['classic'] *= bl_avg_nb_reads\n", + " ledger_LE_writes['classic'] = TPL['classic']*LE_writes_per_op['classic']\n", + " ledger_bytes_write['classic'] = ledger_LE_writes['classic']*LE_size['classic']\n", + "\n", + " # soroban\n", + " ledger_LE_reads['soroban'] = TPL['soroban']*LE_reads_per_op['soroban']\n", + " ledger_bytes_read['soroban'] = ledger_LE_reads['soroban']*bl_bytes_read_per_le(LE_size['soroban'])\n", + " ledger_LE_reads['soroban'] *= bl_avg_nb_reads\n", + " ledger_LE_writes['soroban'] = TPL['soroban']*LE_writes_per_op['soroban']\n", + " ledger_bytes_write['soroban'] = ledger_LE_writes['soroban']*LE_size['soroban']\n", + "\n", + " # speedex\n", + " ledger_LE_reads['speedex'] = TPL['speedex']*LE_reads_per_op['speedex']\n", + " ledger_bytes_read['speedex'] = ledger_LE_reads['speedex']*bl_bytes_read_per_le(LE_size['speedex'])\n", + " ledger_LE_reads['speedex'] *= bl_avg_nb_reads\n", + " ledger_LE_writes['speedex'] = TPL['speedex']*LE_writes_per_op['speedex']\n", + " ledger_bytes_write['speedex'] = ledger_LE_writes['speedex']*LE_size['speedex']\n", + "\n", + " # totals\n", + " print('ledger_LE_reads:', ledger_LE_reads)\n", + " print('ledger_bytes_read:', ledger_bytes_read)\n", + " print('ledger_LE_writes:', ledger_LE_writes)\n", + " print('ledger_bytes_write:', ledger_bytes_write)\n", + "\n", + " ledger_total_LE_reads = sum(ledger_LE_reads.values())\n", + " ledger_total_bytes_read = sum(ledger_bytes_read.values())\n", + " ledger_total_LE_writes = sum(ledger_LE_writes.values()) \n", + " ledger_total_bytes_write = sum(ledger_bytes_write.values())\n", "\n", + " pprint(\"ledger_total_LE_reads: \", ledger_total_LE_reads)\n", + " pprint(\"ledger_total_bytes_read: \", ledger_total_bytes_read)\n", + "\n", + " pprint(\"ledger_total_LE_writes: \", ledger_total_LE_writes)\n", + " pprint(\"ledger_total_bytes_write: \", ledger_total_bytes_write)\n", + "\n", + "\n", + "\n", + " ##############\n", + " # meta\n", + "\n", + " ledger_meta_events = {}\n", + " ledger_meta_other = {}\n", + "\n", + " # classic\n", + " ledger_meta_events['classic'] = TPL['classic']*event_size_per_op['classic']\n", + " # ledger changes\n", + " ledger_meta_other['classic'] = ledger_bytes_write['classic']*2\n", + "\n", + " # soroban\n", + " ledger_meta_events['soroban'] = TPL['soroban']*event_size_per_op['soroban']\n", + " # ledger changes\n", + " ledger_meta_other['soroban'] = ledger_bytes_write['soroban']*2\n", + "\n", + " # speedex\n", + " ledger_meta_events['speedex'] = TPL['speedex']*event_size_per_op['speedex']\n", + " # ledger changes\n", + " ledger_meta_other['speedex'] = ledger_bytes_write['speedex']*2\n", + "\n", + " ledger_meta_events = sum(ledger_meta_events.values())\n", + " ledger_meta_other = sum(ledger_meta_other.values())\n", + " ledger_meta = ledger_meta_events + ledger_meta_other\n", + "\n", + " pprint(\"ledger_meta_events: \", ledger_meta_events)\n", + " pprint(\"ledger_meta_other:\", ledger_meta_other)\n", + " pprint(\"ledger_meta:\", ledger_meta)\n", + "\n", + "\n", + " pprint(\"meta_events (MB/s): \", ledger_meta_events/(1024*1024*ledger_time))\n", + " pprint(\"meta_other (MB/s):\", ledger_meta_other/(1024*1024*ledger_time))\n", + " pprint(\"meta (MB/s):\", ledger_meta/(1024*1024*ledger_time))\n", + "\n", + " # Overlay activity per ledger period\n", + " ## bucket list (overhead from validity checks from reading the account entry ie seqnum/fee)\n", + " ledger_overlay_LE_reads = tpl_total*flooding_recv_factor\n", + " ledger_overlay_LE_bytes = ledger_overlay_LE_reads*bl_bytes_read_per_le(le_account_bytes)\n", + " ledger_overlay_LE_reads *= bl_avg_nb_reads\n", + "\n", + " ## bandwdith\n", + " if v_next:\n", + " overlay_fan_out = 10\n", + " else:\n", + " overlay_fan_out = 20\n", + "\n", + " if large_tier1:\n", + " tier1_size = 100\n", + " else:\n", + " tier1_size = 25\n", + "\n", + " # how long is allocated per ledger to flooding\n", + " ledger_overlay_processing_time_ms = ledger_time*1000\n", + "\n", + " # tx size calculations\n", + " tx_bytes = {}\n", + " tx_footprint = {}\n", + " tx_no_footprint_size_bytes = {}\n", + "\n", + " ## classic\n", + " tx_bytes['classic'] = 200\n", + "\n", + " ## soroban\n", + " tx_footprint['soroban'] = LE_reads_per_op['soroban']+LE_writes_per_op['soroban']\n", + "\n", + " ### Soroban key size (as seen in footprints)\n", + " LE_key_size_bytes_soroban = 80\n", + " ### Soroban tx size without footprint\n", + " tx_no_footprint_size_bytes['soroban'] = 200\n", + "\n", + " tx_bytes['soroban'] = tx_no_footprint_size_bytes['soroban'] + (tx_footprint['soroban']*LE_key_size_bytes_soroban)\n", + "\n", + " ## speedex\n", + " tx_footprint['speedex'] = LE_reads_per_op['speedex']+LE_writes_per_op['speedex']\n", + "\n", + " ### speedex tx size without footprint\n", + " tx_no_footprint_size_bytes['speedex'] = 200\n", + "\n", + " tx_bytes['speedex'] = tx_no_footprint_size_bytes['speedex'] + (tx_footprint['speedex']*LE_key_size_bytes_soroban)\n", + "\n", + " # SCP\n", + " scp_message_bytes = 150\n", + " # number of tx sets per ledger (should be 1, but due to timing issues more than 1 leader may nominate)\n", + " ledger_scp_tx_set_count = 1.1\n", + "\n", + " ### tx flooding\n", + " ##### use the same factor than recv\n", + " ledger_tx_flooding = {e: tpl*flooding_recv_factor for e, tpl in TPL.items()}\n", + " ledger_tx_flood_bytes = sum(ledger_tx_flooding[e]*tx_bytes[e] for e in tx_bytes.keys())\n", + "\n", + " pprint(\"flooding bytes (per connection): \", ledger_tx_flood_bytes)\n", + " ledger_tx_flood_bytes *= overlay_fan_out\n", + "\n", + " ### SCP flooding\n", + " #### 6 messages per tier1 org\n", + " ledger_scp_messages = tier1_size*6\n", + " ledger_scp_flood_bytes = ledger_scp_messages*scp_message_bytes\n", + " #### tx set overhead per ledger\n", + " ledger_txset_bytes = sum(TPL[e]*tx_bytes[e] for e in tx_bytes)\n", + " \n", + " pprint(\"TxSet bytes: \", ledger_txset_bytes)\n", + " ledger_txset_flood_bytes = ledger_txset_bytes*ledger_scp_tx_set_count\n", + " ledger_scp_flood_bytes += ledger_txset_flood_bytes\n", + "\n", + " ledger_scp_flood_bytes *= overlay_fan_out\n", + "\n", + " # totals\n", + " ledger_flood_bytes = ledger_tx_flood_bytes+ledger_scp_flood_bytes\n", + "\n", + " flood_bps = ledger_flood_bytes / ledger_overlay_processing_time_ms * 1000\n", + "\n", + " pprint(\"Network bps (GBit/s): \", flood_bps*8/1024/1024/1024)\n", + "\n", + " ###############\n", + " # calculates additional activity caused by watcher nodes\n", + " # its impact is on \"overlay activity\" as it's happening outside of \"apply\"\n", + " \n", + " if not is_validator:\n", + " # watchers expose the \"getledgerentries\" endpoint\n", + " gle_reads = ledger_time*2\n", + " gle_read_bytes = gle_reads*bl_bytes_read_per_le(bl_avg_le_size)\n", + " gle_reads *= bl_avg_nb_reads\n", + "\n", + " ledger_overlay_LE_reads += gle_reads\n", + " ledger_overlay_LE_bytes += gle_read_bytes\n", + "\n", + " ################\n", + " # apply step\n", + " # apply is equivalent to the sequence [loads, classic, Soroban, writes]\n", + " apply_time_ms_classic = TPL['classic']*tx_exec_time_ms['classic']\n", + " if v_next:\n", + " # overlay can use the entire time (background apply)\n", + " ledger_overlay_time_ms = ledger_time*1000\n", + " # calculates time available to read, classic is not parallel but rest is\n", + " apply_other_exec_time_ms = (TPL['soroban']*tx_exec_time_ms['soroban'] + TPL['speedex']*tx_exec_time_ms['speedex'])/nb_exec_lanes\n", + " apply_exec_time_ms = apply_time_ms_classic+apply_other_exec_time_ms\n", + " else:\n", + " ledger_overlay_time_ms = ledger_time*1000 - apply_time_ms\n", + " apply_other_exec_time_ms = TPL['soroban']*tx_exec_time_ms['soroban']+TPL['speedex']*tx_exec_time_ms['speedex']\n", + " apply_exec_time_ms = apply_time_ms_classic+apply_other_exec_time_ms\n", + "\n", + " pprint(\"apply_exec_time_ms: \", apply_exec_time_ms)\n", + "\n", + " apply_read_time_ms = apply_time_ms - apply_write_time_ms - apply_exec_time_ms\n", + " assert apply_read_time_ms > 0, \"TPS too high (apply work exceeds target)\"\n", + " apply_read_time_ms += apply_optimistic_read_time_ms\n", + "\n", + " pprint(\"apply_read_time_ms: \", apply_read_time_ms)\n", + " pprint(\"apply_write_time_ms: \", apply_write_time_ms)\n", + " # calculates max iops for overlay and apply separately\n", + "\n", + " iops_overlay = ledger_overlay_LE_reads*1000/ledger_overlay_time_ms\n", + " iops_apply = ledger_total_LE_reads*1000/apply_read_time_ms\n", + "\n", + " rbps_overlay = ledger_overlay_LE_bytes*1000/ledger_overlay_time_ms\n", + "\n", + " rbps_apply = ledger_total_bytes_read*1000/apply_read_time_ms\n", + " wbps_apply = ledger_total_bytes_write*1000/apply_write_time_ms\n", + "\n", + " pprint(\"rbps_apply (MB): \", rbps_apply/1024/1024)\n", + " pprint(\"wbps_apply (MB): \", wbps_apply/1024/1024)\n", + "\n", + " if v_next:\n", + " # parallel -> add\n", + " max_iops = iops_overlay + iops_apply\n", + " max_rbps = rbps_overlay + rbps_apply\n", + " max_wbps = wbps_apply\n", + " max_bps = max(rbps_overlay + rbps_apply, rbps_overlay + wbps_apply)\n", + " else:\n", + " # sequential -> max\n", + " max_iops = max(iops_overlay, iops_apply)\n", + " max_rbps = max(rbps_overlay, rbps_apply)\n", + " max_wbps = wbps_apply\n", + " max_bps = max(max_rbps, max_wbps)\n", + "\n", + " pprint(\"max iops: \", max_iops)\n", + " pprint(\"max read Mbps: \", max_rbps/1024/1024)\n", + " pprint(\"max write Mbps: \", max_wbps/1024/1024)\n", + " pprint(\"max disk bandwidth Mbps: \", max_bps/1024/1024)\n", + "\n", + " # source: https://aws.amazon.com/ec2/instance-types/i3en\n", + " max_4k_IOPS_NVMe = 2000000\n", + " max_write_NVMe = 16*1024*1024*1024\n", + " max_bps_NVMe=4096*max_4k_IOPS_NVMe\n", + " assert max_iops < max_4k_IOPS_NVMe, \"disk IOPS exceeded\"\n", + " assert max_bps < max_bps_NVMe, \"disk bandwidth exceeded\"\n", + " assert max_wbps < max_write_NVMe, \"disk linear write bandwidth exceeded\"\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Main Settings" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ "# set to True when using future work\n", "# v_next = False should represent what core can do today\n", "v_next = True\n", @@ -16,35 +392,36 @@ "# watchers expose endpoints for RPC that need to be modeled\n", "is_validator = False\n", "\n", - "# set to True when comnputing limits\n", - "# this uses settings that are \"worst case\"\n", - "model_max = True\n", + "# worst case limits\n", + "big_settings = True\n", "\n", - "# ledger cycle period in seconds\n", - "ledger_time = 5\n", + "# True: BL contains large ledger entries\n", + "large_bl = True\n", "\n", - "# Transactions Per Ledger (derived from TPS as it's the common metric)\n", - "if model_max:\n", - " classic_TPL = 5000*ledger_time\n", - " soroban_TPL = 2000*ledger_time\n", - "else:\n", - " classic_TPL = 1000*ledger_time\n", - " soroban_TPL = 20*ledger_time\n", + "# True: larger \"tier1\"\n", + "large_tier1 = True\n", "\n", "# how long each tx type takes\n", - "classic_tx_exec_time_ms = 0.1\n", - "soroban_tx_exec_time_ms = 10\n", + "tx_exec_time_ms = {'classic': 0.1, 'soroban': 8, 'speedex': 1}\n", + "\n", + "# total round time\n", + "ledger_time = 5.0\n", + "\n", + "# how much time is allocated to apply phase\n", + "apply_time_ms = 3000\n", + "\n", + "# how much time can be spent reading without blocking apply\n", + "apply_optimistic_read_time_ms = 0\n", "\n", "if v_next:\n", - " # number of threads used during apply\n", - " nb_exec_lanes = 50\n", - " apply_time_ms = 3000\n", + " nb_exec_lanes = 10\n", "else:\n", - " apply_time_ms = 1000\n", + " # do not define it as current does not support lanes\n", + " 1\n", "\n", "# how long is allocated for writing\n", - "# (may want to make this a function of classic/Soroban tps&footprints)\n", - "if model_max:\n", + "# (may want to make this a function of tps&footprints instead)\n", + "if v_next:\n", " apply_write_time_ms = 250\n", "else:\n", " apply_write_time_ms = 100\n", @@ -55,74 +432,16 @@ "# other settings\n", "# apply time (in addition to source account modified for seqnum & fees)\n", "\n", - "if model_max:\n", - " # classic\n", - " # max number of LE reads per op (vnext: limited DEX)\n", - " classic_LE_reads_per_op = 10 if v_next else 1000\n", - " # max number of LE writes per op\n", - " classic_LE_writes_per_op = classic_LE_reads_per_op\n", - " # avg size of LE\n", - " classic_LE_size = 140\n", - "\n", - " # soroban\n", - " # max number of LE reads\n", - " soroban_LE_reads_per_op = 40\n", - " # max number of LE writes\n", - " soroban_LE_writes_per_op = 25\n", - " # write as much as possible, use max allowed per tx\n", - " soroban_LE_size = (65*1024)/soroban_LE_writes_per_op\n", - " # tps getledgerentries (watcher node)\n", - " gle_tps = 5\n", - "else:\n", - " # classic\n", - " # avg number of LE reads per op \n", - " classic_LE_reads_per_op = 5\n", - " # avg number of LE writes per op\n", - " classic_LE_writes_per_op = 5\n", - " # avg size of LE\n", - " classic_LE_size = 140\n", - "\n", - " # soroban\n", - " # avg number of LE reads\n", - " soroban_LE_reads_per_op = 10\n", - " # avg number of LE writes\n", - " soroban_LE_writes_per_op = 5\n", - " # use a 10th of capacity\n", - " soroban_LE_size = (65*1024)/soroban_LE_writes_per_op/10\n", - " # tps getledgerentries (watcher node)\n", - " gle_tps = 2\n" + "# how many bytes in a token transfer size\n", + "# \"transfer\", source, destination, asset issuer/contract, amount\n", + "token_transfer_event_size = 16+3*32+4*8+16\n" ] }, { - "cell_type": "code", - "execution_count": 2, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# bucket list settings\n", - "\n", - "# number of iops and bytes needed to read a single ledger entry\n", - "# need to\n", - "# 1. walk from recent to oldest buckets (up to bucket 19)\n", - "# bloom filter helps short circuit lookups\n", - "# 2. for each bucket, page size on larger buckets -> more than 1 read occurs\n", - "\n", - "# additional extra reads (overhead)\n", - "bl_avg_extra_nb_reads = 1\n", - "\n", - "if model_max:\n", - " # more Soroban entries in max mode\n", - " bl_avg_le_size = 1024\n", - "else:\n", - " bl_avg_le_size = 300\n", - "\n", - "\n", - "bl_avg_extra_bytes_read = bl_avg_le_size*bl_avg_extra_nb_reads\n", - "\n", - "bl_avg_nb_reads = 1+bl_avg_extra_nb_reads\n", - "\n", - "def bl_bytes_read_per_le(le_size):\n", - " return le_size + bl_avg_extra_bytes_read" + "# Current network" ] }, { @@ -134,73 +453,46 @@ "name": "stdout", "output_type": "stream", "text": [ - "ledger_LE_reads: 50000\n", - "ledger_bytes_read: 28775000\n", - "ledger_LE_writes: 25000\n", - "ledger_bytes_write: 3175000\n", - "ledger_classic_LE_reads: 500000\n", - "ledger_classic_bytes_read: 291000000\n", - "ledger_classic_LE_writes: 250000\n", - "ledger_classic_bytes_write: 35000000\n", - "ledger_soroban_LE_reads: 800000\n", - "ledger_soroban_bytes_read: 1474560000.0\n", - "ledger_soroban_LE_writes: 250000\n", - "ledger_soroban_bytes_write: 665600000.0\n" + "****************************** current\n", + "ledger_LE_reads: {'fee': 2100.0, 'classic': 20000.0, 'soroban': 4000.0, 'speedex': 0.0}\n", + "ledger_bytes_read: {'fee': 1208550.0, 'classic': 11640000.0, 'soroban': 7372800.0, 'speedex': 0.0}\n", + "ledger_LE_writes: {'fee': 1050.0, 'classic': 10000.0, 'soroban': 1250.0, 'speedex': 0.0}\n", + "ledger_bytes_write: {'fee': 133350.0, 'classic': 1400000.0, 'soroban': 3328000.0, 'speedex': 0.0}\n", + "ledger_total_LE_reads: 26,100.00\n", + "ledger_total_bytes_read: 20,221,350.00\n", + "ledger_total_LE_writes: 12,300.00\n", + "ledger_total_bytes_write: 4,861,350.00\n", + "ledger_meta_events: 2,419,200.00\n", + "ledger_meta_other: 9,456,000.00\n", + "ledger_meta: 11,875,200.00\n", + "meta_events (MB/s): 0.46\n", + "meta_other (MB/s): 1.80\n", + "meta (MB/s): 2.27\n", + "flooding bytes (per connection): 1,175,000.00\n", + "TxSet bytes: 470,000.00\n", + "Network bps (GBit/s): 0.03\n", + "apply_exec_time_ms: 140.00\n", + "apply_read_time_ms: 2,610.00\n", + "apply_write_time_ms: 250\n", + "rbps_apply (MB): 7.39\n", + "wbps_apply (MB): 18.54\n", + "max iops: 11,054.00\n", + "max read Mbps: 7.97\n", + "max write Mbps: 18.54\n", + "max disk bandwidth Mbps: 19.12\n" ] } ], "source": [ - "\n", - "# calculate activity per ledger (apply time)\n", - "\n", - "# size of an account LE\n", - "le_account_bytes = 127\n", - "\n", - "# fees/seqnum\n", - "ledger_LE_reads = classic_TPL*bl_avg_nb_reads\n", - "ledger_bytes_read = classic_TPL*bl_bytes_read_per_le(le_account_bytes)\n", - "ledger_LE_writes = classic_TPL\n", - "ledger_bytes_write = ledger_LE_writes * le_account_bytes\n", - "\n", - "print(\"ledger_LE_reads: \", ledger_LE_reads)\n", - "print(\"ledger_bytes_read: \", ledger_bytes_read)\n", - "\n", - "print(\"ledger_LE_writes: \", ledger_LE_writes)\n", - "print(\"ledger_bytes_write: \", ledger_bytes_write)\n", - "\n", - "# per phase\n", - "ledger_classic_LE_reads = classic_TPL*classic_LE_reads_per_op\n", - "ledger_classic_bytes_read = ledger_classic_LE_reads*bl_bytes_read_per_le(classic_LE_size)\n", - "ledger_classic_LE_reads *= bl_avg_nb_reads\n", - "\n", - "print(\"ledger_classic_LE_reads: \", ledger_classic_LE_reads)\n", - "print(\"ledger_classic_bytes_read: \", ledger_classic_bytes_read)\n", - "\n", - "ledger_classic_LE_writes = classic_TPL*classic_LE_writes_per_op\n", - "ledger_classic_bytes_write = ledger_classic_LE_writes*classic_LE_size\n", - "\n", - "print(\"ledger_classic_LE_writes: \", ledger_classic_LE_writes)\n", - "print(\"ledger_classic_bytes_write: \", ledger_classic_bytes_write)\n", - "\n", - "ledger_soroban_LE_reads = soroban_TPL*soroban_LE_reads_per_op\n", - "ledger_soroban_bytes_read = ledger_soroban_LE_reads*bl_bytes_read_per_le(soroban_LE_size)\n", - "ledger_soroban_LE_reads *= bl_avg_nb_reads\n", - "\n", - "print(\"ledger_soroban_LE_reads: \", ledger_soroban_LE_reads)\n", - "print(\"ledger_soroban_bytes_read: \", ledger_soroban_bytes_read)\n", - "\n", - "\n", - "ledger_soroban_LE_writes = soroban_TPL*soroban_LE_writes_per_op\n", - "ledger_soroban_bytes_write = ledger_soroban_LE_writes*soroban_LE_size\n", - "\n", - "print(\"ledger_soroban_LE_writes: \", ledger_soroban_LE_writes)\n", - "print(\"ledger_soroban_bytes_write: \", ledger_soroban_bytes_write)\n", - "\n", - "\n", - "ledger_LE_reads += ledger_classic_LE_reads + ledger_soroban_LE_reads\n", - "ledger_bytes_read += ledger_classic_bytes_read + ledger_soroban_bytes_read\n", - "ledger_LE_writes += ledger_classic_LE_writes + ledger_soroban_LE_writes\n", - "ledger_bytes_write += ledger_classic_bytes_write + ledger_soroban_bytes_write\n" + "TPS = {'classic': 200, 'soroban': 10, 'speedex': 0}\n", + "calculate_all(\"current\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Parallel Soroban" ] }, { @@ -212,92 +504,99 @@ "name": "stdout", "output_type": "stream", "text": [ - "flooding bytes (per connection): 147500000.0\n", - "TxSet bytes: 59000000\n", - "Network bps (GBit/s): 3.1663477420806885\n" + "****************************** phase 1\n", + "ledger_LE_reads: {'fee': 7000.0, 'classic': 20000.0, 'soroban': 200000.0, 'speedex': 0.0}\n", + "ledger_bytes_read: {'fee': 4028500.0, 'classic': 11640000.0, 'soroban': 368640000.0, 'speedex': 0.0}\n", + "ledger_LE_writes: {'fee': 3500.0, 'classic': 10000.0, 'soroban': 62500.0, 'speedex': 0.0}\n", + "ledger_bytes_write: {'fee': 444500.0, 'classic': 1400000.0, 'soroban': 166400000.0, 'speedex': 0.0}\n", + "ledger_total_LE_reads: 227,000.00\n", + "ledger_total_bytes_read: 384,308,500.00\n", + "ledger_total_LE_writes: 76,000.00\n", + "ledger_total_bytes_write: 168,244,500.00\n", + "ledger_meta_events: 42,560,000.00\n", + "ledger_meta_other: 335,600,000.00\n", + "ledger_meta: 378,160,000.00\n", + "meta_events (MB/s): 8.12\n", + "meta_other (MB/s): 64.01\n", + "meta (MB/s): 72.13\n", + "flooding bytes (per connection): 34,250,000.00\n", + "TxSet bytes: 13,700,000.00\n", + "Network bps (GBit/s): 0.74\n", + "apply_exec_time_ms: 1,433.33\n", + "apply_read_time_ms: 1,316.67\n", + "apply_write_time_ms: 250\n", + "rbps_apply (MB): 278.36\n", + "wbps_apply (MB): 641.80\n", + "max iops: 175,909.06\n", + "max read Mbps: 280.28\n", + "max write Mbps: 641.80\n", + "max disk bandwidth Mbps: 643.73\n" ] } ], "source": [ - "# Overlay activity per ledger period\n", - "\n", - "## bucket list\n", - "ledger_overlay_LE_reads = ((classic_TPL+soroban_TPL)*flooding_recv_factor)\n", - "ledger_overlay_LE_bytes = ledger_overlay_LE_reads*bl_bytes_read_per_le(le_account_bytes)\n", - "ledger_overlay_LE_reads *= bl_avg_nb_reads\n", - "\n", - "## bandwdith\n", - "\n", - "if v_next:\n", - " overlay_fan_out = 10\n", - "else:\n", - " overlay_fan_out = 20\n", - "\n", - "if model_max:\n", - " tier1_size = 100\n", - "else:\n", - " tier1_size = 25\n", - "\n", - "# how long is allocated per ledger to flooding\n", - "ledger_overlay_processing_time_ms = ledger_time*1000\n", - "\n", - "classic_tx_bytes = 200\n", - "soroban_tx_footprint = soroban_LE_reads_per_op+soroban_LE_writes_per_op\n", - "\n", - "# Soroban key size (as seen in footprints)\n", - "soroban_LE_key_size_bytes = 80\n", - "# Soroban tx size without footprint\n", - "soroban_tx_no_footprint_size_bytes = 200\n", - "\n", - "soroban_tx_bytes = soroban_tx_no_footprint_size_bytes + (soroban_tx_footprint*soroban_LE_key_size_bytes)\n", - "\n", - "scp_message_bytes = 150\n", - "# number of tx sets per ledger (should be 1, but due to timing issues more than 1 leader may nominate)\n", - "ledger_scp_tx_set_count = 1.1\n", - "\n", - "### tx flooding\n", - "##### use the same factor than recv\n", - "ledger_classic_tx_flooding = (classic_TPL*flooding_recv_factor)\n", - "ledger_soroban_tx_flooding = (soroban_TPL*flooding_recv_factor)\n", - "ledger_tx_flood_bytes = (ledger_classic_tx_flooding*classic_tx_bytes) + (ledger_soroban_tx_flooding*soroban_tx_bytes)\n", - "print(\"flooding bytes (per connection): \", ledger_tx_flood_bytes)\n", - "ledger_tx_flood_bytes *= overlay_fan_out\n", - "\n", - "### SCP flooding\n", - "#### 6 messages per tier1 org\n", - "ledger_scp_messages = tier1_size*6\n", - "ledger_scp_flood_bytes = ledger_scp_messages*scp_message_bytes\n", - "#### tx set overhead per ledger\n", - "ledger_txset_bytes = (classic_TPL*classic_tx_bytes) + (soroban_TPL*soroban_tx_bytes)\n", - "print(\"TxSet bytes: \", ledger_txset_bytes)\n", - "ledger_txset_flood_bytes = ledger_txset_bytes*ledger_scp_tx_set_count\n", - "ledger_scp_flood_bytes += ledger_txset_flood_bytes\n", - "\n", - "ledger_scp_flood_bytes *= overlay_fan_out\n", - "\n", - "# totals\n", - "ledger_flood_bytes = ledger_tx_flood_bytes+ledger_scp_flood_bytes\n", - "\n", - "flood_bps = ledger_flood_bytes / ledger_overlay_processing_time_ms * 1000\n", - "\n", - "print(\"Network bps (GBit/s): \", flood_bps*8/1024/1024/1024)\n" + "nb_exec_lanes = 15\n", + "TPS = {'classic': 200, 'soroban': 500, 'speedex': 0}\n", + "calculate_all(\"phase 1\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Faster consensus" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "****************************** phase 2\n", + "ledger_LE_reads: {'fee': 2800.0, 'classic': 8000.0, 'soroban': 80000.0, 'speedex': 0.0}\n", + "ledger_bytes_read: {'fee': 1611400.0, 'classic': 4656000.0, 'soroban': 147456000.0, 'speedex': 0.0}\n", + "ledger_LE_writes: {'fee': 1400.0, 'classic': 4000.0, 'soroban': 25000.0, 'speedex': 0.0}\n", + "ledger_bytes_write: {'fee': 177800.0, 'classic': 560000.0, 'soroban': 66560000.0, 'speedex': 0.0}\n", + "ledger_total_LE_reads: 90,800.00\n", + "ledger_total_bytes_read: 153,723,400.00\n", + "ledger_total_LE_writes: 30,400.00\n", + "ledger_total_bytes_write: 67,297,800.00\n", + "ledger_meta_events: 17,024,000.00\n", + "ledger_meta_other: 134,240,000.00\n", + "ledger_meta: 151,264,000.00\n", + "meta_events (MB/s): 8.12\n", + "meta_other (MB/s): 64.01\n", + "meta (MB/s): 72.13\n", + "flooding bytes (per connection): 13,700,000.00\n", + "TxSet bytes: 5,480,000.00\n", + "Network bps (GBit/s): 0.74\n", + "apply_exec_time_ms: 573.33\n", + "apply_read_time_ms: 176.67\n", + "apply_write_time_ms: 250\n", + "rbps_apply (MB): 829.82\n", + "wbps_apply (MB): 256.72\n", + "max iops: 517,466.26\n", + "max read Mbps: 831.75\n", + "max write Mbps: 256.72\n", + "max disk bandwidth Mbps: 831.75\n" + ] + } + ], "source": [ - "# calculates additional \"overlay activity\" caused by watcher node activity\n", - "if not is_validator:\n", - " # watchers expose the \"getledgerentries\" endpoint\n", - " gle_reads = ledger_time*2\n", - " gle_read_bytes = gle_reads*bl_bytes_read_per_le(bl_avg_le_size)\n", - " gle_reads *= bl_avg_nb_reads\n", - "\n", - " ledger_overlay_LE_reads += gle_reads\n", - " ledger_overlay_LE_bytes += gle_read_bytes" + "ledger_time = 2.0\n", + "apply_time_ms = 1000\n", + "calculate_all(\"phase 2\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Speedex++" ] }, { @@ -309,63 +608,100 @@ "name": "stdout", "output_type": "stream", "text": [ - "max iops: 1963575.4285714286\n", - "max read Mbps: 2463.800726209368\n", - "max write Mbps: 2684.6885681152344\n", - "max disk bandwidth Mbps: 2703.9018592834473\n" + "****************************** phase 3\n", + "ledger_LE_reads: {'fee': 4800.0, 'classic': 8000.0, 'soroban': 80000.0, 'speedex': 8000.0}\n", + "ledger_bytes_read: {'fee': 2762400.0, 'classic': 4656000.0, 'soroban': 147456000.0, 'speedex': 6144000.0}\n", + "ledger_LE_writes: {'fee': 2400.0, 'classic': 4000.0, 'soroban': 25000.0, 'speedex': 2000.0}\n", + "ledger_bytes_write: {'fee': 304800.0, 'classic': 560000.0, 'soroban': 66560000.0, 'speedex': 1024000.0}\n", + "ledger_total_LE_reads: 100,800.00\n", + "ledger_total_bytes_read: 161,018,400.00\n", + "ledger_total_LE_writes: 33,400.00\n", + "ledger_total_bytes_write: 68,448,800.00\n", + "ledger_meta_events: 17,344,000.00\n", + "ledger_meta_other: 136,288,000.00\n", + "ledger_meta: 153,632,000.00\n", + "meta_events (MB/s): 8.27\n", + "meta_other (MB/s): 64.99\n", + "meta (MB/s): 73.26\n", + "flooding bytes (per connection): 15,400,000.00\n", + "TxSet bytes: 6,160,000.00\n", + "Network bps (GBit/s): 0.83\n", + "apply_exec_time_ms: 640.00\n", + "apply_read_time_ms: 110.00\n", + "apply_write_time_ms: 250\n", + "rbps_apply (MB): 1,395.99\n", + "wbps_apply (MB): 261.11\n", + "max iops: 922,367.64\n", + "max read Mbps: 1,399.29\n", + "max write Mbps: 261.11\n", + "max disk bandwidth Mbps: 1,399.29\n" ] } ], "source": [ - "# apply is equivalent to the sequence [loads, classic, Soroban, writes]\n", - "\n", - "if v_next:\n", - " # overlay can use the entire time (background apply)\n", - " ledger_overlay_time_ms = ledger_time*1000\n", - " # calculates time available to read by assuming parallel execution\n", - " apply_read_time_ms = apply_time_ms - apply_write_time_ms - (classic_TPL*classic_tx_exec_time_ms+soroban_TPL*soroban_tx_exec_time_ms)/nb_exec_lanes\n", - " assert apply_read_time_ms > 0, \"TPS too high (apply_read_time_ms <= 0)\"\n", - "else:\n", - " ledger_overlay_time_ms = ledger_time*1000 - apply_time_ms\n", - " apply_read_time_ms = apply_time_ms - apply_write_time_ms - (classic_TPL*classic_tx_exec_time_ms+soroban_TPL*soroban_tx_exec_time_ms)\n", - " assert apply_read_time_ms > 0, \"TPS too high (apply_read_time_ms <= 0)\"\n", - "\n", - "# calculates max iops for overlay and apply separately\n", - "\n", - "iops_overlay = ledger_overlay_LE_reads*1000/ledger_overlay_time_ms\n", - "iops_apply = ledger_LE_reads*1000/apply_read_time_ms\n", - "\n", - "rbps_overlay = ledger_overlay_LE_bytes*1000/ledger_overlay_time_ms\n", - "\n", - "rbps_apply = ledger_bytes_read*1000/apply_read_time_ms\n", - "wbps_apply = ledger_bytes_write*1000/apply_write_time_ms\n", - "\n", + "nb_exec_lanes = 15\n", + "TPS = {'classic': 200, 'soroban': 500, 'speedex': 500}\n", + "calculate_all(\"phase 3\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Bigger SKU" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "****************************** phase 4\n", + "ledger_LE_reads: {'fee': 40800.0, 'classic': 8000.0, 'soroban': 160000.0, 'speedex': 144000.0}\n", + "ledger_bytes_read: {'fee': 23480400.0, 'classic': 4656000.0, 'soroban': 294912000.0, 'speedex': 110592000.0}\n", + "ledger_LE_writes: {'fee': 20400.0, 'classic': 4000.0, 'soroban': 50000.0, 'speedex': 36000.0}\n", + "ledger_bytes_write: {'fee': 2590800.0, 'classic': 560000.0, 'soroban': 133120000.0, 'speedex': 18432000.0}\n", + "ledger_total_LE_reads: 352,800.00\n", + "ledger_total_bytes_read: 433,640,400.00\n", + "ledger_total_LE_writes: 110,400.00\n", + "ledger_total_bytes_write: 154,702,800.00\n", + "ledger_meta_events: 39,168,000.00\n", + "ledger_meta_other: 304,224,000.00\n", + "ledger_meta: 343,392,000.00\n", + "meta_events (MB/s): 18.68\n", + "meta_other (MB/s): 145.07\n", + "meta (MB/s): 163.74\n", + "flooding bytes (per connection): 57,800,000.00\n", + "TxSet bytes: 23,120,000.00\n", + "Network bps (GBit/s): 3.10\n", + "apply_exec_time_ms: 720.00\n", + "apply_read_time_ms: 280.00\n", + "apply_write_time_ms: 250\n", + "rbps_apply (MB): 1,476.97\n", + "wbps_apply (MB): 590.14\n", + "max iops: 1,311,004.00\n", + "max read Mbps: 1,504.97\n", + "max write Mbps: 590.14\n", + "max disk bandwidth Mbps: 1,504.97\n" + ] + } + ], + "source": [ + "# c5ad.16xlarge equivalent: 64 vCPU (50+14 workers for other work)\n", + "# NVMe\n", + "# 20 GBit network\n", + "# https://aws.amazon.com/ec2/instance-types/c5/\n", + "nb_exec_lanes = 50\n", "\n", + "# optimistic reads\n", + "apply_optimistic_read_time_ms = 250\n", "\n", - "if v_next:\n", - " # parallel -> add\n", - " max_iops = iops_overlay + iops_apply\n", - " max_rbps = rbps_overlay + rbps_apply\n", - " max_wbps = wbps_apply\n", - " max_bps = max(rbps_overlay + rbps_apply, rbps_overlay + wbps_apply)\n", - "else:\n", - " # sequential -> max\n", - " max_iops = max(iops_overlay, iops_apply)\n", - " max_rbps = max(rbps_overlay, rbps_apply)\n", - " max_wbps = wbps_apply\n", - " max_bps = max(max_rbps, max_wbps)\n", - "\n", - "print(\"max iops: \", max_iops)\n", - "print(\"max read Mbps: \", max_rbps/1024/1024)\n", - "print(\"max write Mbps: \", max_wbps/1024/1024)\n", - "print(\"max disk bandwidth Mbps: \", max_bps/1024/1024)\n", - "\n", - "# source: https://aws.amazon.com/ec2/instance-types/i3en\n", - "max_4k_IOPS_NVMe = 2000000\n", - "max_write_NVMe = 16*1024*1024*1024\n", - "assert max_iops < max_4k_IOPS_NVMe, \"disk IOPS exceeded\"\n", - "assert max_bps < (4096*max_4k_IOPS_NVMe), \"disk bandwidth exceeded\"\n", - "assert max_wbps < max_write_NVMe, \"disk linear write bandwidth exceeded\"\n" + "TPS = {'classic': 200, 'soroban': 1000, 'speedex': 9000}\n", + "calculate_all(\"phase 4\")" ] } ],