feat!: trace and measure the control plane's own work, on forge v1.12.9 and grove v1.8.0

README.md

@@ -197,6 +197,12 @@ app.Use(cpext.New(
 app.Run()
 ```
 
+## Observability
+
+Ctrl Plane traces and measures its own work. In a Forge app you get it for free: the extension records on `app.Metrics()`, so `/_/metrics` carries series like `forge_ctrlplane_provider_call_duration_seconds` and `forge_ctrlplane_deployment_duration_seconds`, and spans such as `execute deployment` and `provision instance` nest under forge's HTTP server span. The reconciler, garbage collector and provider health poller each start one root span per pass that has work.
+
+Outside Forge, pass a go-utils metrics sink with `app.WithMetrics(m)` and install an OpenTelemetry tracer provider. The sink has to keep one series per name and label set, as forge's metrics do; go-utils' `NewMetricsCollector` keys by name alone and would merge them. Error messages, tenant IDs, names, URL paths and query strings stay off spans and labels, Docker's client spans included, and calls to providers never forward your callers' baggage. The full list of spans and metrics is in `docs/content/docs/subsystems/observability.mdx`.
+
 ## Documentation
 
 Full documentation is available in the `docs/` directory and covers architecture, core concepts, every subsystem, guides for writing custom providers and deployment strategies, and the complete HTTP API reference.

admin/service_impl.go

@@ -6,11 +6,16 @@ import (
 	"strings"
 	"time"
 
+	gometrics "github.com/xraph/go-utils/metrics"
+	"go.opentelemetry.io/otel/baggage"
+	"go.opentelemetry.io/otel/trace"
+
 	ctrlplane "github.com/xraph/ctrlplane"
 	"github.com/xraph/ctrlplane/auth"
 	"github.com/xraph/ctrlplane/event"
 	"github.com/xraph/ctrlplane/id"
 	"github.com/xraph/ctrlplane/instance"
+	"github.com/xraph/ctrlplane/internal/obs"
 	"github.com/xraph/ctrlplane/managed"
 	"github.com/xraph/ctrlplane/network"
 	"github.com/xraph/ctrlplane/provider"
@@ -47,6 +52,13 @@ type service struct {
 	events         event.Bus
 	auth           auth.Provider
 	providerHealth ProviderHealthGetter
+	metrics        *obs.Metrics
+}
+
+// SetMetrics times provider health tests on m. A nil m records nothing.
+// In a Forge app the ctrlplane extension passes app.Metrics() in for you.
+func (s *service) SetMetrics(m gometrics.Metrics) {
+	s.metrics = obs.NewMetrics(m)
 }
 
 // NewService creates a new admin service.
@@ -476,7 +488,29 @@ func (s *service) TestProviderHealth(ctx context.Context, providerName string) (
 
 	// If the provider implements HealthChecker, use it.
 	if hc, ok := p.(provider.HealthChecker); ok {
-		status, healthErr := hc.HealthCheck(pctx)
+		providerType := provider.TypeName(p)
+		started := time.Now()
+
+		hctx, span := obs.StartChild(baggage.ContextWithoutBaggage(pctx), "check provider health",
+			trace.WithAttributes(obs.AttrProviderType.String(providerType)))
+		status, healthErr := hc.HealthCheck(hctx)
+
+		outcome := obs.HealthHealthy
+
+		switch {
+		case healthErr != nil:
+			outcome = obs.ResultError
+
+			obs.Fail(span, healthErr)
+		case status != nil && !status.Healthy:
+			outcome = obs.HealthUnhealthy
+
+			obs.FailClass(span, "unhealthy")
+		}
+
+		span.SetAttributes(obs.AttrProviderHealthy.Bool(outcome == obs.HealthHealthy))
+		span.End()
+		s.metrics.ProviderHealthChecked(providerType, outcome, time.Since(started))
 
 		result := &ProviderHealthResult{
 			Name:      providerName,

app/controlplane.go

@@ -6,6 +6,8 @@ import (
 	"net/http"
 	"time"
 
+	gometrics "github.com/xraph/go-utils/metrics"
+
 	ctrlplane "github.com/xraph/ctrlplane"
 	"github.com/xraph/ctrlplane/admin"
 	audithook "github.com/xraph/ctrlplane/audit_hook"
@@ -20,6 +22,7 @@ import (
 	"github.com/xraph/ctrlplane/instance"
 	"github.com/xraph/ctrlplane/metrics"
 	"github.com/xraph/ctrlplane/network"
+	"github.com/xraph/ctrlplane/observability"
 	"github.com/xraph/ctrlplane/plugin"
 	"github.com/xraph/ctrlplane/provider"
 	"github.com/xraph/ctrlplane/providerhealth"
@@ -45,6 +48,14 @@ type CtrlPlane struct {
 	pendingExts []plugin.Extension
 	auditHook   *audithook.Extension
 
+	// metrics, when set by WithMetrics, is handed to every subsystem
+	// that records metrics.
+	metrics gometrics.Metrics
+
+	// gc is the garbage collector worker, kept so wireMetrics can reach
+	// it after the scheduler owns it.
+	gc *worker.GarbageCollector
+
 	// leaderElector, when set, limits background workers to the replica
 	// that holds leadership.
 	leaderElector worker.LeaderElector
@@ -330,13 +341,16 @@ func (cp *CtrlPlane) wireServices() {
 	cp.scheduler.Register(worker.NewReconciler(cp.store, cp.store, cp.Bootstraps, cp.providers, cp.events, 60*time.Second))
 	cp.scheduler.Register(worker.NewHealthRunner(cp.Health, cp.events, healthInterval))
 	cp.scheduler.Register(worker.NewTelemetryCollector(cp.Telemetry, cp.providers, telemetryInterval))
-	cp.scheduler.Register(worker.NewGarbageCollector(cp.store, cp.Instances, cp.store, cp.events, 5*time.Minute, worker.GCConfig{}))
+	cp.gc = worker.NewGarbageCollector(cp.store, cp.Instances, cp.store, cp.events, 5*time.Minute, worker.GCConfig{})
+	cp.scheduler.Register(cp.gc)
 	cp.scheduler.Register(worker.NewCertRenewer(cp.Network, cp.events, 12*time.Hour))
 
 	if cp.leaderElector != nil {
 		cp.scheduler.SetLeaderElector(cp.leaderElector)
 	}
 
+	cp.wireMetrics()
+
 	// Default audit-trail plugin: bridges every lifecycle event to
 	// admin.AuditEntry rows in the store. Without this nothing
 	// writes to the audit log table — the dashboard's Audit Log
@@ -356,6 +370,49 @@ func (cp *CtrlPlane) wireServices() {
 	}
 }
 
+// metricsSetter is satisfied by every subsystem that records metrics.
+type metricsSetter interface {
+	SetMetrics(m gometrics.Metrics)
+}
+
+// wireMetrics hands the WithMetrics sink to every subsystem that records
+// metrics and registers the lifecycle event counter, unless you registered
+// an observability.MetricsExtension yourself. Without WithMetrics it does
+// nothing, and every subsystem records nothing.
+func (cp *CtrlPlane) wireMetrics() {
+	if cp.metrics == nil {
+		return
+	}
+
+	targets := []any{
+		cp.providers, cp.scheduler, cp.gc, cp.Bootstraps, cp.Instances, cp.Deploys,
+		cp.Health, cp.Metrics, cp.ProviderHealth, cp.Admin,
+	}
+
+	for _, t := range targets {
+		if s, ok := t.(metricsSetter); ok {
+			s.SetMetrics(cp.metrics)
+		}
+	}
+
+	if cp.extensions == nil {
+		return
+	}
+
+	cp.extensions.SetMetrics(cp.metrics)
+
+	// The plugin registry doesn't dedupe, so a host that already
+	// registered its own lifecycle event plugin would count every event
+	// twice.
+	for _, ext := range cp.extensions.Extensions() {
+		if _, ok := ext.(*observability.MetricsExtension); ok {
+			return
+		}
+	}
+
+	cp.extensions.Register(observability.NewMetricsExtension(cp.metrics))
+}
+
 // storeAuditRecorder is the default audit_hook.Recorder. Translates
 // the audit_hook AuditEvent shape into an admin.AuditEntry and
 // writes via store.InsertAuditEntry. Failures are returned to the

app/options.go

@@ -3,6 +3,8 @@ package app
 import (
 	"errors"
 
+	gometrics "github.com/xraph/go-utils/metrics"
+
 	ctrlplane "github.com/xraph/ctrlplane"
 	"github.com/xraph/ctrlplane/auth"
 	"github.com/xraph/ctrlplane/bootstrap"
@@ -35,6 +37,31 @@ func WithStore(s store.Store) Option {
 	}
 }
 
+// WithMetrics records Ctrl Plane's metrics on m: worker passes and leader
+// election, provider calls and credential lookups, deployments and
+// readiness waits, provider and instance health checks, bootstrap and
+// garbage collection activity, the resource poller, lifecycle events and
+// plugin hook errors. In a Forge app pass app.Metrics(); the ctrlplane
+// extension does that for you. A nil m records nothing.
+//
+// It registers an observability.MetricsExtension on m for the lifecycle
+// event counter, unless WithExtension already registered one.
+//
+// Outside Forge, m must keep one series per name and label set: the vector
+// metrics here call m.Counter(name, metrics.WithLabels(...)) once per label
+// combination. go-utils' NewMetricsCollector keys series by name alone and
+// would merge them; forge's metrics, which app.Metrics() returns, do not.
+//
+// Spans need no option. They go to the global OpenTelemetry tracer
+// provider, which forge.New installs.
+func WithMetrics(m gometrics.Metrics) Option {
+	return func(cp *CtrlPlane) error {
+		cp.metrics = m
+
+		return nil
+	}
+}
+
 // WithAuth sets the authentication/authorization provider.
 func WithAuth(p auth.Provider) Option {
 	return func(cp *CtrlPlane) error {