all: Add opt-in OpenTelemetry tracing

Cargo.toml

@@ -80,6 +80,11 @@ indicatif = "0.18"
 Inflector = "0.11.3"
 itertools = "0.15.0"
 lazy_static = "1.5.0"
+# Default features off: no metrics, no logs, no `tracing`-based internal logging.
+opentelemetry = { version = "0.33.1", default-features = false, features = ["trace"] }
+opentelemetry_sdk = { version = "0.33.1", default-features = false, features = ["trace"] }
+opentelemetry-http = { version = "0.33.1", default-features = false }
+opentelemetry-otlp = { version = "0.33.1", default-features = false, features = ["http-proto", "reqwest-blocking-client", "trace"] }
 prost = "0.14"
 prost-types = "0.14"
 redis = { version = "1.2.2", features = [

NEWS.md

@@ -12,6 +12,7 @@ $ docker pull graphprotocol/graph-node:v0.45.0
 
 ### What's New
 
+- **OpenTelemetry tracing.** Setting `GRAPH_OTEL_TRACING=true` exports traces over OTLP (`http/protobuf`) to any collector: one trace per processed block with the handler and, optionally, host function spans under it, one per batch written by the write queue, linked to the traces of the blocks it contains, and one per GraphQL request that joins the trace of an incoming `traceparent` header. `GRAPH_OTEL_SPANS` picks the span groups; the exporter and sampler are configured with the standard `OTEL_*` variables. Off by default, with negligible overhead when off. See [docs/tracing.md](docs/tracing.md).
 - **Postponed attribute index creation re-enabled, now built in the background.** `GRAPH_POSTPONE_ATTRIBUTE_INDEX_CREATION` (introduced in v0.44.0 but inert — hardcoded off) now actually defers non-critical attribute indexes until a new deployment nears chain head (`GRAPH_POSTPONE_INDEXES_CREATION_THRESHOLD`, default 10000 blocks from head). Index creation now runs in a background task instead of blocking block processing, and interrupted `CREATE INDEX CONCURRENTLY` runs are detected and rebuilt on retry instead of being silently skipped. ([#6608](https://github.com/graphprotocol/graph-node/pull/6608))
 - **`ethereum.decodeParams` host function.** Decodes calldata/event data whose top-level type is a dynamic tuple (e.g. Gnosis Safe's `execTransaction`), which `ethereum.decode` cannot handle since it expects a single ABI value with a leading offset word. Gated behind apiVersion 0.0.10; `ethereum.decode` itself is unchanged. ([#6649](https://github.com/graphprotocol/graph-node/pull/6649))
 

chain/ethereum/src/runtime/runtime_adapter.rs

@@ -208,6 +208,14 @@ async fn ethereum_call(
         asc_get::<_, AscUnresolvedContractCall, _>(ctx.heap, wasm_ptr.into(), &ctx.gas, 0)?
     };
 
+    ctx.metrics.stopwatch.set_attrs_with(|| {
+        [
+            ("contract", call.contract_name.clone().into()),
+            ("function", call.function_name.clone().into()),
+            ("address", format!("{:#x}", call.contract_address).into()),
+        ]
+    });
+
     let result = eth_call(
         eth_adapter,
         call_cache,

chain/ethereum/src/trigger.rs

@@ -22,6 +22,7 @@ use graph::runtime::HostExportError;
 use graph::runtime::asc_new;
 use graph::runtime::gas::GasCounter;
 use graph::semver::Version;
+use graph::telemetry::AttrValue;
 use graph_runtime_wasm::module::ToAscPtr;
 use std::{cmp::Ordering, sync::Arc};
 
@@ -71,6 +72,36 @@ impl MappingTriggerTrait for MappingTrigger {
             None => String::new(),
         }
     }
+
+    fn trace_attributes(&self) -> Vec<(&'static str, AttrValue)> {
+        match self {
+            MappingTrigger::Log { log, .. } => {
+                transaction_attributes(log.transaction_hash, log.log_index)
+            }
+            MappingTrigger::Call { call, .. } => {
+                transaction_attributes(call.transaction_hash, None)
+            }
+            MappingTrigger::Block { .. } => Vec::new(),
+        }
+    }
+}
+
+/// The span attributes that locate a trigger inside its block.
+fn transaction_attributes(
+    tx_hash: Option<B256>,
+    log_index: Option<u64>,
+) -> Vec<(&'static str, AttrValue)> {
+    let mut attrs = Vec::new();
+    if let Some(tx_hash) = tx_hash {
+        attrs.push(("tx_hash", format!("{tx_hash:#x}").into()));
+    }
+    if let Some(log_index) = log_index {
+        attrs.push((
+            "log_index",
+            i64::try_from(log_index).unwrap_or(i64::MAX).into(),
+        ));
+    }
+    attrs
 }
 
 // Logging the block is too verbose, so this strips the block from the trigger for Debug.
@@ -649,3 +680,26 @@ impl<'a> EthereumCallData<'a> {
         &self.call.to
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn transaction_attributes_name_the_transaction() {
+        let tx_hash = B256::from([0xab; 32]);
+        let attrs = transaction_attributes(Some(tx_hash), Some(7));
+        assert_eq!(
+            attrs,
+            vec![
+                ("tx_hash", AttrValue::from(format!("0x{}", "ab".repeat(32)))),
+                ("log_index", AttrValue::from(7i64)),
+            ]
+        );
+        assert_eq!(
+            transaction_attributes(Some(tx_hash), None),
+            vec![("tx_hash", AttrValue::from(format!("0x{}", "ab".repeat(32))))]
+        );
+        assert!(transaction_attributes(None, None).is_empty());
+    }
+}

core/src/subgraph/runner/mod.rs

@@ -42,6 +42,7 @@ use graph::prelude::{
 };
 use graph::schema::EntityKey;
 use graph::slog::{Logger, debug, error, info, o, trace, warn};
+use graph::telemetry::SpanGroup;
 use graph::util::lfu_cache::EvictStats;
 use graph::util::{backoff::ExponentialBackoff, lfu_cache::LfuCache};
 use std::sync::Arc;
@@ -578,9 +579,29 @@ where
         let block_start = Instant::now();
 
         let action = {
-            let stopwatch = &self.metrics.stream.stopwatch;
-            let _section = stopwatch.start_section(PROCESS_BLOCK_SECTION_NAME);
-            self.process_block(block, cursor).await
+            let stopwatch = self.metrics.stream.stopwatch.cheap_clone();
+            // Each block is its own trace.
+            let _section = stopwatch.start_root_section(
+                PROCESS_BLOCK_SECTION_NAME,
+                SpanGroup::Block,
+                || {
+                    [
+                        (
+                            "subgraph_id",
+                            self.inputs.deployment.hash.to_string().into(),
+                        ),
+                        ("block_number", block_ptr.number.into()),
+                        ("block_hash", block_ptr.hash_hex().into()),
+                        ("network", self.inputs.network.clone().into()),
+                    ]
+                },
+                &[],
+            );
+            let action = self.process_block(block, cursor).await;
+            if let Err(err) = &action {
+                stopwatch.set_error_with(|| err.to_string());
+            }
+            action
         };
 
         let action = self.handle_action(block_start, block_ptr, action).await?;
@@ -720,7 +741,11 @@ where
 
         // Transact entity operations into the store and update the
         // subgraph's block stream pointer
-        let _section = self.metrics.host.stopwatch.start_section("transact_block");
+        let _section = self
+            .metrics
+            .host
+            .stopwatch
+            .start_section_with("transact_block", || [("entity_count", mods.len().into())]);
         let start = Instant::now();
 
         // If a deterministic error has happened, make the PoI to be the only entity that'll be stored.
@@ -1098,7 +1123,7 @@ where
             vid_gen,
         );
 
-        let _section = self
+        let section = self
             .metrics
             .stream
             .stopwatch
@@ -1134,6 +1159,10 @@ where
             Err(MappingError::PossibleReorg(e)) => return Err(ProcessingError::PossibleReorg(e)),
         }
 
+        // The later stages have sections of their own and are not part
+        // of trigger processing.
+        section.end();
+
         // Check if there are any datasources that have expired in this block. ie: the end_block
         // of that data source is equal to the block number of the current block.
         let has_expired_data_sources = self.inputs.end_blocks.contains(&block_ptr.number);
@@ -1485,7 +1514,7 @@ where
             close_to_chain_head(block_ptr, &self.state.cached_head_ptr, CAUGHT_UP_DISTANCE);
         if is_caught_up {
             // Stop recording time-to-sync metrics.
-            self.metrics.stream.stopwatch.disable();
+            self.metrics.stream.stopwatch.stop_recording();
         }
         Ok(is_caught_up)
     }

docs/environment-variables.md

@@ -160,6 +160,43 @@ those.
   value will not be cached. The default is 3. A value of 0 means that there
   is no limit on the size of a cache entry.
 
+## OpenTelemetry tracing
+
+Tracing is off by default and adds one atomic load per instrumented
+section when off. [tracing.md](./tracing.md) describes what is traced and
+how to set up a collector.
+
+- `GRAPH_OTEL_TRACING`: set to `true` to export spans over OTLP. Defaults to
+  `false`.
+- `GRAPH_OTEL_SPANS`: comma-separated list of the span groups to emit:
+  `block` (one trace per processed block and the runner sections under it),
+  `handler` (one span per trigger handler), `host_fn` (one span per WASM host
+  function call, which can be thousands per handler), `store` (write batches
+  and the SQL queries that serve GraphQL) and `graphql` (one trace per
+  GraphQL request). `all` and `none` are accepted. Defaults to
+  `block,handler,store,graphql`.
+
+The exporter speaks OTLP over `http/protobuf` and is configured with the
+standard OpenTelemetry variables, which the SDK reads directly:
+
+- `OTEL_EXPORTER_OTLP_ENDPOINT`: base URL of the collector; traces go to
+  `<endpoint>/v1/traces`. Defaults to `http://localhost:4318`.
+  `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` overrides the full traces URL.
+- `OTEL_EXPORTER_OTLP_PROTOCOL`: only `http/protobuf`, the default, is
+  supported.
+- `OTEL_EXPORTER_OTLP_HEADERS`, `OTEL_EXPORTER_OTLP_TIMEOUT`: passed to the
+  collector as described in the OpenTelemetry specification.
+- `OTEL_SERVICE_NAME`: the `service.name` resource attribute. Defaults to
+  `graph-node`.
+- `OTEL_TRACES_SAMPLER` and `OTEL_TRACES_SAMPLER_ARG`: sampling is decided
+  once per trace at its root span. Use `parentbased_traceidratio` with an
+  argument between 0 and 1 to sample a fraction of blocks and requests.
+  Defaults to `parentbased_always_on`.
+
+Spans are exported in batches. On SIGINT or SIGTERM, graph-node flushes the
+buffered spans before exiting; on SIGKILL the last few seconds of spans are
+lost.
+
 ## Miscellaneous
 
 - `GRAPH_NODE_ID`: sets the node ID, allowing to run multiple Graph Nodes