feat(l1): use grafana and prometheus outside of docker

crates/blockchain/blockchain.rs

@@ -417,7 +417,7 @@ impl Blockchain {
     /// - [`BatchProcessingFailure`] (if the error was caused by block processing).
     ///
     /// Note: only the last block's state trie is stored in the db
-    pub async fn add_blocks_in_batch(
+ pub async fn add_blocks_in_batch(
         &self,
         blocks: Vec<Block>,
     ) -> Result<(), (ChainError, Option<BatchBlockProcessingFailure>)> {
@@ -447,6 +447,12 @@ impl Blockchain {
         let mut total_gas_used = 0;
         let mut transactions_count = 0;
 
+        info!(
+            "[SYNCING] Processing {} blocks starting from block number {}",
+            blocks_len,
+            first_block_header.number
+        );
+
         let interval = Instant::now();
         for (i, block) in blocks.iter().enumerate() {
             // for the first block, we need to query the store
@@ -477,17 +483,51 @@ impl Blockchain {
                     )
                 })?;
 
-            info!("Processed block {} out of {}", i, blocks.len());
             last_valid_hash = block.hash();
             total_gas_used += block.header.gas_used;
             transactions_count += block.body.transactions.len();
             all_receipts.push((block.hash(), receipts));
+
+            if i % 100 == 0 || i == blocks_len - 1 {
+                info!(
+                    "[SYNCING] Processed block {} out of {}: {}",
+                    i + 1,
+                    blocks_len,
+                    block.header.number
+                );
+
+                metrics!(
+                    let _ = METRICS_BLOCKS.set_block_number(block.header.number);
+                    METRICS_BLOCKS.set_latest_gas_used(total_gas_used as f64 / (i + 1) as f64);
+                    METRICS_BLOCKS.set_latest_block_gas_limit(block.header.gas_limit as f64);
+                    METRICS_BLOCKS.set_latest_gigagas(
+                        (total_gas_used as f64 / 10_f64.powf(9_f64))
+                            / (interval.elapsed().as_millis() as f64 / 1000_f64)
+                    );
+                );
+            }
         }
 
+        info!(
+            "[SYNCING] Processed all {} blocks, last valid hash: {}, took {} s",
+            blocks_len,
+            last_valid_hash,
+            (interval.elapsed().as_millis() as f64 / 1000_f64)
+        );
+
+        let interval_2 = Instant::now();
+
         let account_updates = vm
             .get_state_transitions()
             .map_err(|err| (ChainError::EvmError(err), None))?;
 
+        info!(
+            "[SYNCING] Get state transitions, took {} s",
+            (interval_2.elapsed().as_millis() as f64 / 1000_f64)
+        );
+
+        let interval_3 = Instant::now();
+
         let last_block = blocks
             .last()
             .ok_or_else(|| (ChainError::Custom("Last block not found".into()), None))?;
@@ -503,13 +543,27 @@ impl Blockchain {
             .map_err(|e| (e.into(), None))?
             .ok_or((ChainError::ParentStateNotFound, None))?;
 
+        info!(
+            "[SYNCING] Account updates applied, took {} s",
+            (interval_3.elapsed().as_millis() as f64 / 1000_f64)
+        );
+
+        let interval_4 = Instant::now();
+
         let new_state_root = account_updates_list.state_trie_hash;
         let state_updates = account_updates_list.state_updates;
         let accounts_updates = account_updates_list.storage_updates;
 
         // Check state root matches the one in block header
         validate_state_root(&last_block.header, new_state_root).map_err(|e| (e, None))?;
 
+        info!(
+            "[SYNCING] State root validated, took {} s",
+            (interval_4.elapsed().as_millis() as f64 / 1000_f64)
+        );
+
+        let interval_5 = Instant::now();
+
         let update_batch = UpdateBatch {
             account_updates: state_updates,
             storage_updates: accounts_updates,
@@ -522,6 +576,11 @@ impl Blockchain {
             .await
             .map_err(|e| (e.into(), None))?;
 
+        info!(
+            "[SYNCING] Blocks stored, took {} s",
+            (interval_5.elapsed().as_millis() as f64 / 1000_f64)
+        );
+
         let elapsed_seconds = interval.elapsed().as_millis() / 1000;
         let mut throughput = 0.0;
         if elapsed_seconds != 0 && total_gas_used != 0 {

crates/blockchain/metrics/docker-compose-metrics-exporter.yaml

@@ -0,0 +1,12 @@
+services:
+  ethereum-metrics-exporter:
+    image: samcm/ethereum-metrics-exporter:latest
+    command:
+      - --execution-url=http://host.docker.internal:8545
+      - --execution-modules="eth","net","web3","txpool"
+      - --consensus-url=http://host.docker.internal:5053
+      - --metrics-port=9093
+    ports:
+      - "9093:9093"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"

crates/blockchain/metrics/docker-compose-metrics-l1.overrides.yaml

@@ -17,7 +17,7 @@ services:
 
   prometheus:
     volumes:
-      - ./provisioning/prometheus/prometheus_l1_sync_docker.yaml:/etc/prometheus/prometheus.yaml
+      - ./provisioning/prometheus/prometheus_l1_sync_docker.yaml:/etc/prometheus/prometheus_docker.yaml
     ports:
       - "9091:9090"
     extra_hosts:

crates/blockchain/metrics/docker-compose-metrics-l2.overrides.yaml

@@ -1,7 +1,7 @@
 services:
   prometheus:
     volumes:
-      - ../metrics/provisioning/prometheus/prometheus_l2.yaml:/etc/prometheus/prometheus.yaml
+      - ../metrics/provisioning/prometheus/prometheus_l2.yaml:/etc/prometheus/prometheus_docker.yaml
     ports:
       - "9092:9090"
   grafana:

crates/blockchain/metrics/docker-compose-metrics.yaml

@@ -2,7 +2,7 @@
 services:
   prometheus:
     image: prom/prometheus
-    command: --config.file=/etc/prometheus/prometheus.yaml
+    command: --config.file=/etc/prometheus/prometheus_docker.yaml
     #volumes: defined in the .overrides file
     #ports: defined in the .overrides file
   grafana:

crates/blockchain/metrics/provisioning/grafana_provisioning/datasources/prometheus.yaml

@@ -5,7 +5,7 @@ datasources:
     type: prometheus
     uid: prom-001
     access: proxy
-    url: http://prometheus:9090
+    url: http://localhost:9090
     isDefault: true
     version: 2 
     jsonData:

crates/blockchain/metrics/provisioning/grafana_provisioning/datasources/prometheus_docker.yaml

@@ -0,0 +1,15 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prom-001
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    version: 2 
+    jsonData:
+      timeInterval: 10s
+      httpMethod: POST
+    readOnly: false
+    editable: true

crates/blockchain/metrics/provisioning/prometheus/prometheus_l1_sync.yaml

@@ -0,0 +1,26 @@
+global:
+  scrape_interval: 10s
+  evaluation_interval: 10s
+
+scrape_configs:
+  - job_name: 'node_exporter'
+    static_configs:
+      - targets: ['localhost:9100']
+
+  - job_name: "ethrex L1"
+    static_configs:
+      - targets: ["localhost:3701"]
+    scrape_interval: 10s
+    metrics_path: /metrics
+
+  - job_name: "lighthouse consensus"
+    static_configs:
+      - targets: ["localhost:5054"]
+    scrape_interval: 10s
+    metrics_path: /metrics
+
+  - job_name: "ethereum-metrics-exporter"
+    static_configs:
+      - targets: ["localhost:9093"]
+    scrape_interval:  5s
+    metrics_path: /metrics

crates/networking/p2p/sync.rs

@@ -184,9 +184,9 @@ impl Syncer {
 
         // TODO(#2126): To avoid modifying the current_head while backtracking we use a separate search_head
         let mut search_head = current_head;
-
+        let interval = Instant::now();
         loop {
-            debug!("Requesting Block Headers from {search_head}");
+            info!("[SYNCING] Requesting Block Headers from {search_head}");
 
             let Some(mut block_headers) = self
                 .peers
@@ -302,6 +302,11 @@ impl Syncer {
                 break;
             };
         }
+        info!(
+            "[SYNCING] Fetched {} block headers in {} seconds",
+            all_block_hashes.len(),
+            interval.elapsed().as_secs()
+        );
         match sync_mode {
             SyncMode::Snap => {
                 // snap-sync: launch tasks to fetch blocks and state in parallel
@@ -382,7 +387,7 @@ impl Syncer {
 
         let since = Instant::now();
         loop {
-            debug!("Requesting Block Bodies");
+            info!("[SYNCING] Requesting Block Bodies");
             let mut headers_iter = block_headers.iter().skip(headers_consumed);
             let block_request_result = self
                 .peers
@@ -428,6 +433,14 @@ impl Syncer {
             return Err(SyncError::BodiesNotFound);
         };
 
+        info!(
+            "[SYNCING] Received {} blocks , starting from block with number {}, until block with number {}, in {} seconds.",
+            blocks_len,
+            first_block.header.number,
+            last_block.header.number,
+            since.elapsed().as_secs()
+        );
+
         // To ensure proper execution, we set the chain as canonical before processing the blocks.
         // Some opcodes rely on previous block hashes, and due to our current setup, we only support a single chain (no sidechains).
         // As a result, we must store the headers and set the chain upfront to writing to the database during execution.

tooling/sync/Makefile

@@ -180,6 +180,14 @@ start_hoodi_metrics_docker: ## Start L1 docker compose, lighthouse in background
 	@echo "Starting ethrex..."
 	cd $(ETHREX_DIR)/tooling/sync && make start_ethrex_hoodi
 
+start_hoodi_metrics: ## Start just metrics exporter docker compose, lighthouse in background, and ethrex for hoodi
+	@echo "Starting L1 docker compose with metrics..."
+	cd $(ETHREX_DIR)/crates/blockchain/metrics && docker compose -f docker-compose-metrics-exporter.yaml  up -d
+	@echo "Starting lighthouse in background..."
+	cd $(ETHREX_DIR)/tooling/sync && nohup make hoodi-lighthouse > /dev/null 2>&1 &
+	@echo "Starting ethrex..."
+	cd $(ETHREX_DIR)/tooling/sync && make start_ethrex_hoodi
+
 start_holesky_metrics_docker: ## Start L1 docker compose, lighthouse in background, and ethrex for holesky
 	@echo "Starting L1 docker compose with metrics..."
 	cd $(ETHREX_DIR)/crates/blockchain/metrics && docker compose -f docker-compose-metrics.yaml -f docker-compose-metrics-l1.overrides.yaml up -d
@@ -188,6 +196,14 @@ start_holesky_metrics_docker: ## Start L1 docker compose, lighthouse in backgrou
 	@echo "Starting ethrex..."
 	cd $(ETHREX_DIR)/tooling/sync && make start_ethrex_holesky
 
+start_holesky_metrics: ## Start just metrics exporter docker compose, lighthouse in background, and ethrex for holesky
+	@echo "Starting L1 docker compose with metrics..."
+	cd $(ETHREX_DIR)/crates/blockchain/metrics && docker compose -f docker-compose-metrics-exporter.yaml up -d
+	@echo "Starting lighthouse in background..."
+	cd $(ETHREX_DIR)/tooling/sync && nohup make holesky-lighthouse > /dev/null 2>&1 &
+	@echo "Starting ethrex..."
+	cd $(ETHREX_DIR)/tooling/sync && make start_ethrex_holesky
+
 tail-syncing-logs: ## Tail the syncing logs for a given log file. Environment variable LOGNAME with the name of the file needs to be provided.
 ifndef LOGNAME
 	$(error "Log file not provided")