From 428c9833ee1c1dd6202692919f5ef1b290e0e63d Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 20 Jul 2026 16:41:47 -0400 Subject: [PATCH 1/4] chore(telemetry): protect built-in metric compatibility Signed-off-by: Yordis Prieto --- .config/mise/tasks/semconv/check | 8 + .config/mise/tasks/semconv/generate | 14 +- otel/semconv/registry/manifest.yaml | 6 + .../registry/trogon/eventstore/metrics.yaml | 414 ++++++++++++++++++ .../registry/csharp/metric-names.cs.j2 | 25 ++ .../templates/registry/csharp/weaver.yaml | 37 +- .../Index/IndexTrackerTests.cs | 2 +- .../Metrics/AverageMetricTests.cs | 2 +- .../Metrics/CacheResourcesTrackerTests.cs | 2 +- .../Metrics/DurationMaxTrackerTests.cs | 4 +- .../Metrics/DurationTrackerTests.cs | 2 +- .../Metrics/ElectionsCounterTrackerTests.cs | 2 +- .../Metrics/ProcessMetricsTests.cs | 12 +- .../Metrics/QueueBusyTrackerTests.cs | 2 +- .../Metrics/QueueProcessingTrackerTests.cs | 2 +- .../Metrics/SystemMetricsTests.cs | 4 +- .../OpenTelemetry/MetricNamesTests.cs | 24 + .../Chunks/TFChunkTrackerTests.cs | 6 +- src/EventStore.Core/ClusterVNodeStartup.cs | 3 +- src/EventStore.Core/Metrics/AverageMetric.cs | 4 +- .../Metrics/CacheResourcesMetrics.cs | 6 +- src/EventStore.Core/Metrics/CounterMetric.cs | 7 +- .../Metrics/DurationMaxMetric.cs | 2 +- src/EventStore.Core/Metrics/DurationMetric.cs | 2 +- src/EventStore.Core/Metrics/ProcessMetrics.cs | 18 +- src/EventStore.Core/Metrics/SystemMetrics.cs | 4 +- src/EventStore.Core/MetricsBootstrapper.cs | 114 ++--- .../ProjectionsSubsystem.cs | 11 +- .../Generated/MetricNames.g.cs | 133 ++++++ .../README.md | 2 +- 30 files changed, 755 insertions(+), 119 deletions(-) create mode 100644 otel/semconv/registry/manifest.yaml create mode 100644 otel/semconv/registry/trogon/eventstore/metrics.yaml create mode 100644 otel/semconv/templates/registry/csharp/metric-names.cs.j2 create mode 100644 src/EventStore.Core.XUnit.Tests/OpenTelemetry/MetricNamesTests.cs create mode 100644 src/TrogonEventStore.SemanticConventions/Generated/MetricNames.g.cs diff --git a/.config/mise/tasks/semconv/check b/.config/mise/tasks/semconv/check index 55edb693fe..9408a4acdb 100755 --- a/.config/mise/tasks/semconv/check +++ b/.config/mise/tasks/semconv/check @@ -10,3 +10,11 @@ trap 'rm -rf "$generated"' EXIT HUP INT TERM "$root/.config/mise/tasks/semconv/generate" "$generated" diff -ru "$expected" "$generated" + +if grep -n '"eventstore-' \ + "$root/src/EventStore.Core/MetricsBootstrapper.cs" \ + "$root/src/EventStore.Projections.Core/ProjectionsSubsystem.cs" +then + echo "Built-in metric names must come from the generated MetricNames catalog." >&2 + exit 1 +fi diff --git a/.config/mise/tasks/semconv/generate b/.config/mise/tasks/semconv/generate index f808e5479e..538cebd294 100755 --- a/.config/mise/tasks/semconv/generate +++ b/.config/mise/tasks/semconv/generate @@ -6,10 +6,13 @@ set -eu root=$(CDPATH='' cd -- "$(dirname -- "$0")/../../../.." && pwd) output=${1:-"$root/src/TrogonEventStore.SemanticConventions/Generated"} registry_version=$(sed -n '1p' "$root/otel/semconv/registry-version") -registry="https://github.com/open-telemetry/semantic-conventions@${registry_version}[model]" +registry="$root/otel/semconv/registry" +official_registry="https://github.com/open-telemetry/semantic-conventions@${registry_version}[model]" staging=$(mktemp -d) trap 'rm -rf "$staging"' EXIT HUP INT TERM +grep -Fqx " registry_path: $official_registry" "$registry/manifest.yaml" + weaver registry check \ --future \ --registry "$registry" @@ -17,7 +20,14 @@ weaver registry check \ weaver registry generate csharp "$staging" \ --future \ --registry "$registry" \ - --templates "$root/otel/semconv/templates" + --templates "$root/otel/semconv/templates" \ + -D attributes=false + +weaver registry generate csharp "$staging" \ + --future \ + --registry "$official_registry" \ + --templates "$root/otel/semconv/templates" \ + -D metrics=false set -- "$staging"/*.g.cs [ -e "$1" ] || exit 1 diff --git a/otel/semconv/registry/manifest.yaml b/otel/semconv/registry/manifest.yaml new file mode 100644 index 0000000000..bace77c70f --- /dev/null +++ b/otel/semconv/registry/manifest.yaml @@ -0,0 +1,6 @@ +name: trogon_eventstore +description: Semantic conventions for TrogonEventStore telemetry. +schema_url: https://trogondb.com/schemas/0.1.0 +dependencies: + - schema_url: https://opentelemetry.io/schemas/1.43.0 + registry_path: https://github.com/open-telemetry/semantic-conventions@v1.43.0[model] diff --git a/otel/semconv/registry/trogon/eventstore/metrics.yaml b/otel/semconv/registry/trogon/eventstore/metrics.yaml new file mode 100644 index 0000000000..9e7168d604 --- /dev/null +++ b/otel/semconv/registry/trogon/eventstore/metrics.yaml @@ -0,0 +1,414 @@ +groups: + - id: metric.eventstore.statuses + type: metric + stability: stable + brief: Component status transition timestamps. + metric_name: eventstore-statuses + instrument: counter + unit: s + - id: metric.eventstore.grpc.method.duration + type: metric + stability: stable + brief: Duration of gRPC method calls. + metric_name: eventstore-grpc-method-duration-seconds + instrument: histogram + unit: s + - id: metric.eventstore.gossip.latency + type: metric + stability: stable + brief: Duration of gossip exchanges. + metric_name: eventstore-gossip-latency-seconds + instrument: histogram + unit: s + - id: metric.eventstore.gossip.processing.duration + type: metric + stability: stable + brief: Duration of gossip message processing. + metric_name: eventstore-gossip-processing-duration-seconds + instrument: histogram + unit: s + - id: metric.eventstore.queue.queueing.duration.max + type: metric + stability: stable + brief: Maximum time messages wait in queues. + metric_name: eventstore-queue-queueing-duration-max-seconds + instrument: gauge + unit: s + - id: metric.eventstore.queue.processing.duration + type: metric + stability: stable + brief: Duration of queue message processing. + metric_name: eventstore-queue-processing-duration-seconds + instrument: histogram + unit: s + - id: metric.eventstore.queue.busy + type: metric + stability: stable + brief: Average queue processing time. + metric_name: eventstore-queue-busy-seconds + instrument: counter + unit: s + - id: metric.eventstore.queue.length + type: metric + stability: stable + brief: Number of messages waiting in queues. + metric_name: eventstore-queue-length + instrument: updowncounter + unit: "{item}" + - id: metric.eventstore.io.bytes + type: metric + stability: stable + brief: Bytes read by transaction file operations. + metric_name: eventstore-io-bytes + instrument: counter + unit: By + - id: metric.eventstore.io.events + type: metric + stability: stable + brief: Events read or indexed by storage operations. + metric_name: eventstore-io-events + instrument: counter + unit: "{event}" + - id: metric.eventstore.io.record.read.duration + type: metric + stability: stable + brief: Duration of transaction file record reads. + metric_name: eventstore-io-record-read-duration-seconds + instrument: histogram + unit: s + - id: metric.eventstore.elections.count + type: metric + stability: stable + brief: Number of cluster elections. + metric_name: eventstore-elections-count + instrument: counter + unit: "{election}" + - id: metric.eventstore.current.incoming.grpc.calls + type: metric + stability: stable + brief: Number of active incoming gRPC calls. + metric_name: eventstore-current-incoming-grpc-calls + instrument: updowncounter + unit: "{call}" + - id: metric.eventstore.incoming.grpc.calls + type: metric + stability: stable + brief: Number of incoming gRPC calls by outcome. + metric_name: eventstore-incoming-grpc-calls + instrument: counter + unit: "{call}" + - id: metric.eventstore.cache.hits.misses + type: metric + stability: stable + brief: Cache hits and misses. + metric_name: eventstore-cache-hits-misses + instrument: counter + unit: "{operation}" + - id: metric.eventstore.cache.resources.bytes + type: metric + stability: stable + brief: Cache resource size and capacity in bytes. + metric_name: eventstore-cache-resources-bytes + instrument: updowncounter + unit: By + - id: metric.eventstore.cache.resources.entries + type: metric + stability: stable + brief: Cache resource size, capacity, and count in entries. + metric_name: eventstore-cache-resources-entries + instrument: updowncounter + unit: "{entry}" + - id: metric.eventstore.logical.chunk.read.distribution + type: metric + stability: stable + brief: Distance between read and current logical chunks. + metric_name: eventstore-logical-chunk-read-distribution + instrument: histogram + unit: "{chunk}" + - id: metric.eventstore.persistent.sub.connections + type: metric + stability: stable + brief: Persistent subscription connections. + metric_name: eventstore-persistent-sub-connections + instrument: updowncounter + unit: "{connection}" + - id: metric.eventstore.persistent.sub.parked.messages + type: metric + stability: stable + brief: Parked persistent subscription messages. + metric_name: eventstore-persistent-sub-parked-messages + instrument: updowncounter + unit: "{message}" + - id: metric.eventstore.persistent.sub.in.flight.messages + type: metric + stability: stable + brief: In-flight persistent subscription messages. + metric_name: eventstore-persistent-sub-in-flight-messages + instrument: updowncounter + unit: "{message}" + - id: metric.eventstore.persistent.sub.oldest.parked.message + type: metric + stability: stable + brief: Age of the oldest parked persistent subscription message. + metric_name: eventstore-persistent-sub-oldest-parked-message-seconds + instrument: updowncounter + unit: s + - id: metric.eventstore.persistent.sub.park.message.requests + type: metric + stability: stable + brief: Persistent subscription park requests. + metric_name: eventstore-persistent-sub-park-message-requests + instrument: counter + unit: "{request}" + - id: metric.eventstore.persistent.sub.parked.message.replays + type: metric + stability: stable + brief: Persistent subscription parked-message replays. + metric_name: eventstore-persistent-sub-parked-message-replays + instrument: counter + unit: "{operation}" + - id: metric.eventstore.persistent.sub.parked.message.truncates + type: metric + stability: stable + brief: Persistent subscription parked-message truncations. + metric_name: eventstore-persistent-sub-parked-message-truncates + instrument: counter + unit: "{operation}" + - id: metric.eventstore.persistent.sub.items.processed + type: metric + stability: stable + brief: Items processed by persistent subscriptions. + metric_name: eventstore-persistent-sub-items-processed + instrument: counter + unit: "{item}" + - id: metric.eventstore.persistent.sub.last.known.event.number + type: metric + stability: stable + brief: Last known event number for persistent subscriptions. + metric_name: eventstore-persistent-sub-last-known-event-number + instrument: counter + unit: "{event}" + - id: metric.eventstore.persistent.sub.last.known.event.commit.position + type: metric + stability: stable + brief: Last known commit position for persistent subscriptions. + metric_name: eventstore-persistent-sub-last-known-event-commit-position + instrument: counter + unit: "1" + - id: metric.eventstore.persistent.sub.checkpointed.event.number + type: metric + stability: stable + brief: Last checkpointed event number for persistent subscriptions. + metric_name: eventstore-persistent-sub-checkpointed-event-number + instrument: counter + unit: "{event}" + - id: metric.eventstore.persistent.sub.checkpointed.event.commit.position + type: metric + stability: stable + brief: Last checkpointed commit position for persistent subscriptions. + metric_name: eventstore-persistent-sub-checkpointed-event-commit-position + instrument: counter + unit: "1" + - id: metric.eventstore.checkpoints + type: metric + stability: stable + brief: Transaction log checkpoint positions. + metric_name: eventstore-checkpoints + instrument: updowncounter + unit: "1" + - id: metric.eventstore.writer.flush.size.max + type: metric + stability: stable + brief: Maximum writer flush size. + metric_name: eventstore-writer-flush-size-max + instrument: gauge + unit: By + - id: metric.eventstore.writer.flush.duration.max + type: metric + stability: stable + brief: Maximum writer flush duration. + metric_name: eventstore-writer-flush-duration-max-seconds + instrument: gauge + unit: s + - id: metric.eventstore.kestrel.connections + type: metric + stability: stable + brief: Active Kestrel connections. + metric_name: eventstore-kestrel-connections + instrument: updowncounter + unit: "{connection}" + - id: metric.eventstore.sys.load.avg + type: metric + stability: stable + brief: System load average. + metric_name: eventstore-sys-load-avg + instrument: gauge + unit: "1" + - id: metric.eventstore.sys.cpu + type: metric + stability: stable + brief: System CPU utilization. + metric_name: eventstore-sys-cpu + instrument: updowncounter + unit: "%" + - id: metric.eventstore.sys.mem + type: metric + stability: stable + brief: System memory capacity and usage. + metric_name: eventstore-sys-mem-bytes + instrument: gauge + unit: By + - id: metric.eventstore.sys.disk + type: metric + stability: stable + brief: System disk capacity and usage. + metric_name: eventstore-sys-disk-bytes + instrument: gauge + unit: By + - id: metric.eventstore.proc.up.time + type: metric + stability: stable + brief: Process uptime. + metric_name: eventstore-proc-up-time + instrument: counter + unit: s + - id: metric.eventstore.proc.cpu + type: metric + stability: stable + brief: Process CPU utilization. + metric_name: eventstore-proc-cpu + instrument: updowncounter + unit: "%" + - id: metric.eventstore.proc.thread.count + type: metric + stability: stable + brief: Process thread count. + metric_name: eventstore-proc-thread-count + instrument: updowncounter + unit: "{thread}" + - id: metric.eventstore.proc.thread.pool.pending.work.item.count + type: metric + stability: stable + brief: Pending thread-pool work items. + metric_name: eventstore-proc-thread-pool-pending-work-item-count + instrument: updowncounter + unit: "{item}" + - id: metric.eventstore.proc.contention.count + type: metric + stability: stable + brief: Process lock contentions. + metric_name: eventstore-proc-contention-count + instrument: counter + unit: "{contention}" + - id: metric.eventstore.proc.exception.count + type: metric + stability: stable + brief: Process exceptions. + metric_name: eventstore-proc-exception-count + instrument: counter + unit: "{exception}" + - id: metric.eventstore.gc.time.in.gc + type: metric + stability: stable + brief: Time spent in garbage collection. + metric_name: eventstore-gc-time-in-gc + instrument: updowncounter + unit: "%" + - id: metric.eventstore.gc.heap.size + type: metric + stability: stable + brief: Managed heap size. + metric_name: eventstore-gc-heap-size-bytes + instrument: updowncounter + unit: By + - id: metric.eventstore.gc.heap.fragmentation + type: metric + stability: stable + brief: Managed heap fragmentation. + metric_name: eventstore-gc-heap-fragmentation + instrument: updowncounter + unit: "%" + - id: metric.eventstore.gc.total.allocated + type: metric + stability: stable + brief: Total bytes allocated by the managed process. + metric_name: eventstore-gc-total-allocated + instrument: counter + unit: By + - id: metric.eventstore.gc.pause.duration.max + type: metric + stability: stable + brief: Maximum garbage collection pause duration. + metric_name: eventstore-gc-pause-duration-max-seconds + instrument: gauge + unit: s + - id: metric.eventstore.proc.mem + type: metric + stability: stable + brief: Process memory usage. + metric_name: eventstore-proc-mem-bytes + instrument: gauge + unit: By + - id: metric.eventstore.gc.generation.size + type: metric + stability: stable + brief: Managed heap generation size. + metric_name: eventstore-gc-generation-size-bytes + instrument: updowncounter + unit: By + - id: metric.eventstore.gc.collection.count + type: metric + stability: stable + brief: Garbage collection count by generation. + metric_name: eventstore-gc-collection-count + instrument: counter + unit: "{collection}" + - id: metric.eventstore.disk.io.bytes + type: metric + stability: stable + brief: Process disk I/O bytes. + metric_name: eventstore-disk-io-bytes + instrument: counter + unit: By + - id: metric.eventstore.disk.io.operations + type: metric + stability: stable + brief: Process disk I/O operations. + metric_name: eventstore-disk-io-operations + instrument: counter + unit: "{operation}" + - id: metric.eventstore.projection.events.processed.after.restart + type: metric + stability: stable + brief: Events processed by projections after restart. + metric_name: eventstore-projection-events-processed-after-restart-total + instrument: counter + unit: "{event}" + - id: metric.eventstore.projection.progress + type: metric + stability: stable + brief: Projection progress. + metric_name: eventstore-projection-progress + instrument: updowncounter + unit: "1" + - id: metric.eventstore.projection.running + type: metric + stability: stable + brief: Whether a projection is running. + metric_name: eventstore-projection-running + instrument: updowncounter + unit: "1" + - id: metric.eventstore.projection.status + type: metric + stability: stable + brief: Projection status. + metric_name: eventstore-projection-status + instrument: updowncounter + unit: "1" + - id: metric.eventstore.projection.state.size + type: metric + stability: stable + brief: Projection state size. + metric_name: eventstore-projection-state-size + instrument: updowncounter + unit: By diff --git a/otel/semconv/templates/registry/csharp/metric-names.cs.j2 b/otel/semconv/templates/registry/csharp/metric-names.cs.j2 new file mode 100644 index 0000000000..63066acae6 --- /dev/null +++ b/otel/semconv/templates/registry/csharp/metric-names.cs.j2 @@ -0,0 +1,25 @@ +// + +using System; +using System.Collections.Generic; + +namespace TrogonEventStore.SemanticConventions +{ + public static class MetricNames + { +{% for group in ctx %} +{% for metric in group.metrics | sort(attribute="metric_name") %} + public const string {{ metric.metric_name | pascal_case | regex_replace("^Eventstore", "") }} = "{{ metric.metric_name }}"; +{% endfor %} +{% endfor %} + + public static IReadOnlyList All { get; } = Array.AsReadOnly(new[] + { +{% for group in ctx %} +{% for metric in group.metrics | sort(attribute="metric_name") %} + {{ metric.metric_name | pascal_case | regex_replace("^Eventstore", "") }}, +{% endfor %} +{% endfor %} + }); + } +}{{ "\n" }} diff --git a/otel/semconv/templates/registry/csharp/weaver.yaml b/otel/semconv/templates/registry/csharp/weaver.yaml index 416da1351b..99e35e39f5 100644 --- a/otel/semconv/templates/registry/csharp/weaver.yaml +++ b/otel/semconv/templates/registry/csharp/weaver.yaml @@ -2,18 +2,35 @@ whitespace_control: trim_blocks: true lstrip_blocks: true +params: + attributes: true + metrics: true + templates: - template: attribute-names.cs.j2 filter: > - semconv_grouped_attributes - | map({ - root_namespace: .root_namespace, - attributes: [.attributes[] | select( - .name == "service.name" or - .name == "service.instance.id" or - .name == "service.version" - )] - }) - | map(select(.attributes | length > 0)) + if $attributes then + semconv_grouped_attributes + | map({ + root_namespace: .root_namespace, + attributes: [.attributes[] | select( + .name == "service.name" or + .name == "service.instance.id" or + .name == "service.version" + )] + }) + | map(select(.attributes | length > 0)) + else + empty + end application_mode: single file_name: AttributeNames.g.cs + - template: metric-names.cs.j2 + filter: > + if $metrics then + semconv_grouped_metrics + else + empty + end + application_mode: single + file_name: MetricNames.g.cs diff --git a/src/EventStore.Core.XUnit.Tests/Index/IndexTrackerTests.cs b/src/EventStore.Core.XUnit.Tests/Index/IndexTrackerTests.cs index a2d7e88d3c..e74a5ba501 100644 --- a/src/EventStore.Core.XUnit.Tests/Index/IndexTrackerTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Index/IndexTrackerTests.cs @@ -20,7 +20,7 @@ public IndexTrackerTests() var meter = new Meter($"{typeof(IndexTrackerTests)}"); _listener = new TestMeterListener(meter); - var eventMetric = new CounterMetric(meter, "eventstore-io", "events"); + var eventMetric = new CounterMetric(meter, "eventstore-io-events"); _sut = new IndexTracker(new CounterSubMetric(eventMetric, new[] { new KeyValuePair("activity", "written") })); } diff --git a/src/EventStore.Core.XUnit.Tests/Metrics/AverageMetricTests.cs b/src/EventStore.Core.XUnit.Tests/Metrics/AverageMetricTests.cs index ebfe0472a6..ddf0e756f0 100644 --- a/src/EventStore.Core.XUnit.Tests/Metrics/AverageMetricTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Metrics/AverageMetricTests.cs @@ -11,7 +11,7 @@ public void calculates_average() { using var meter = new Meter($"{typeof(QueueProcessingTrackerTests)}"); using var listener = new TestMeterListener(meter); - var sut = new AverageMetric(meter, "the-metric", "seconds", label => new("queue", label)); + var sut = new AverageMetric(meter, "the-metric-seconds", label => new("queue", label)); sut.Register("readers", () => 1); sut.Register("readers", () => 2); sut.Register("writer", () => 3); diff --git a/src/EventStore.Core.XUnit.Tests/Metrics/CacheResourcesTrackerTests.cs b/src/EventStore.Core.XUnit.Tests/Metrics/CacheResourcesTrackerTests.cs index 09ce8224da..935fc716d7 100644 --- a/src/EventStore.Core.XUnit.Tests/Metrics/CacheResourcesTrackerTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Metrics/CacheResourcesTrackerTests.cs @@ -27,7 +27,7 @@ public void Dispose() var listener = new TestMeterListener(meter); _disposables.RegisterForDispose(listener); - var metrics = new CacheResourcesMetrics(meter, "the-metric"); + var metrics = new CacheResourcesMetrics(meter, "the-metric-bytes", "the-metric-entries"); var sut = new CacheResourcesTracker(metrics); return (sut, listener); } diff --git a/src/EventStore.Core.XUnit.Tests/Metrics/DurationMaxTrackerTests.cs b/src/EventStore.Core.XUnit.Tests/Metrics/DurationMaxTrackerTests.cs index f43717223a..c332a37560 100644 --- a/src/EventStore.Core.XUnit.Tests/Metrics/DurationMaxTrackerTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Metrics/DurationMaxTrackerTests.cs @@ -16,7 +16,7 @@ public DurationMaxTrackerTests() { var meter = new Meter($"{typeof(DurationMaxTrackerTests)}"); _listener = new TestMeterListener(meter); - var metric = new DurationMaxMetric(meter, "the-metric"); + var metric = new DurationMaxMetric(meter, "the-metric-seconds"); _sut = new DurationMaxTracker( metric: metric, name: "the-tracker", @@ -198,7 +198,7 @@ public void no_name() using var meter = new Meter($"{typeof(DurationMaxTrackerTests)}"); using var listener = new TestMeterListener(meter); var sut = new DurationMaxTracker( - metric: new DurationMaxMetric(meter, "the-metric"), + metric: new DurationMaxMetric(meter, "the-metric-seconds"), name: null, expectedScrapeIntervalSeconds: 15); diff --git a/src/EventStore.Core.XUnit.Tests/Metrics/DurationTrackerTests.cs b/src/EventStore.Core.XUnit.Tests/Metrics/DurationTrackerTests.cs index 045f59dba4..fd87e25b7a 100644 --- a/src/EventStore.Core.XUnit.Tests/Metrics/DurationTrackerTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Metrics/DurationTrackerTests.cs @@ -15,7 +15,7 @@ public DurationTrackerTests() { var meter = new Meter($"{typeof(DurationTrackerTests)}"); _listener = new TestMeterListener(meter); - var durationMetric = new DurationMetric(meter, "the-histogram", _clock); + var durationMetric = new DurationMetric(meter, "the-histogram-seconds", _clock); _sut = new DurationTracker(durationMetric, "the-duration"); } diff --git a/src/EventStore.Core.XUnit.Tests/Metrics/ElectionsCounterTrackerTests.cs b/src/EventStore.Core.XUnit.Tests/Metrics/ElectionsCounterTrackerTests.cs index 1066300230..7f2df718cb 100644 --- a/src/EventStore.Core.XUnit.Tests/Metrics/ElectionsCounterTrackerTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Metrics/ElectionsCounterTrackerTests.cs @@ -41,7 +41,7 @@ public void Dispose() var listener = new TestMeterListener(meter); _disposables.RegisterForDispose(meter); - var metric = new CounterMetric(meter, "test-metric", unit: ""); + var metric = new CounterMetric(meter, "test-metric"); var sut = new ElectionsCounterTracker(new CounterSubMetric(metric, [])); return (sut, listener); diff --git a/src/EventStore.Core.XUnit.Tests/Metrics/ProcessMetricsTests.cs b/src/EventStore.Core.XUnit.Tests/Metrics/ProcessMetricsTests.cs index 1ecf2ab741..5dae572a76 100644 --- a/src/EventStore.Core.XUnit.Tests/Metrics/ProcessMetricsTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Metrics/ProcessMetricsTests.cs @@ -39,19 +39,19 @@ public ProcessMetricsTests() { MetricsConfiguration.ProcessTracker.LockContentionCount, "eventstore-proc-contention-count" }, { MetricsConfiguration.ProcessTracker.ExceptionCount, "eventstore-proc-exception-count" }, { MetricsConfiguration.ProcessTracker.TimeInGc, "eventstore-gc-time-in-gc" }, - { MetricsConfiguration.ProcessTracker.HeapSize, "eventstore-gc-heap-size" }, + { MetricsConfiguration.ProcessTracker.HeapSize, "eventstore-gc-heap-size-bytes" }, { MetricsConfiguration.ProcessTracker.HeapFragmentation, "eventstore-gc-heap-fragmentation" }, { MetricsConfiguration.ProcessTracker.TotalAllocatedBytes, "eventstore-gc-total-allocated" }, - { MetricsConfiguration.ProcessTracker.GcPauseDuration, "eventstore-gc-pause-duration" }, + { MetricsConfiguration.ProcessTracker.GcPauseDuration, "eventstore-gc-pause-duration-seconds" }, }); - _sut.CreateMemoryMetric("eventstore-proc-mem", new() { + _sut.CreateMemoryMetric("eventstore-proc-mem-bytes", new() { { MetricsConfiguration.ProcessTracker.MemWorkingSet, "working-set" }, { MetricsConfiguration.ProcessTracker.MemPagedBytes, "paged-bytes" }, { MetricsConfiguration.ProcessTracker.MemVirtualBytes, "virtual-bytes" }, }); - _sut.CreateGcGenerationSizeMetric("eventstore-gc-generation-size", new() { + _sut.CreateGcGenerationSizeMetric("eventstore-gc-generation-size-bytes", new() { { MetricsConfiguration.ProcessTracker.Gen0Size, "gen0" }, { MetricsConfiguration.ProcessTracker.Gen1Size, "gen1" }, { MetricsConfiguration.ProcessTracker.Gen2Size, "gen2" }, @@ -64,12 +64,12 @@ public ProcessMetricsTests() { MetricsConfiguration.ProcessTracker.Gen2CollectionCount, "gen2" }, }); - _sut.CreateDiskBytesMetric("eventstore-disk-io", new() { + _sut.CreateDiskBytesMetric("eventstore-disk-io-bytes", new() { { MetricsConfiguration.ProcessTracker.DiskReadBytes, "read" }, { MetricsConfiguration.ProcessTracker.DiskWrittenBytes, "written" }, }); - _sut.CreateDiskOpsMetric("eventstore-disk-io", new() { + _sut.CreateDiskOpsMetric("eventstore-disk-io-operations", new() { { MetricsConfiguration.ProcessTracker.DiskReadBytes, "read" }, { MetricsConfiguration.ProcessTracker.DiskWrittenBytes, "written" }, }); diff --git a/src/EventStore.Core.XUnit.Tests/Metrics/QueueBusyTrackerTests.cs b/src/EventStore.Core.XUnit.Tests/Metrics/QueueBusyTrackerTests.cs index 08155e6ab2..8afc93ed50 100644 --- a/src/EventStore.Core.XUnit.Tests/Metrics/QueueBusyTrackerTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Metrics/QueueBusyTrackerTests.cs @@ -12,7 +12,7 @@ public async Task records() { using var meter = new Meter($"{typeof(QueueProcessingTrackerTests)}"); using var listener = new TestMeterListener(meter); - var metric = new AverageMetric(meter, "the-metric", "seconds", label => new("queue", label)); + var metric = new AverageMetric(meter, "the-metric-seconds", label => new("queue", label)); var sut = new QueueBusyTracker(metric, "the-queue"); sut.EnterBusy(); diff --git a/src/EventStore.Core.XUnit.Tests/Metrics/QueueProcessingTrackerTests.cs b/src/EventStore.Core.XUnit.Tests/Metrics/QueueProcessingTrackerTests.cs index a4bc94d997..31034f9361 100644 --- a/src/EventStore.Core.XUnit.Tests/Metrics/QueueProcessingTrackerTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Metrics/QueueProcessingTrackerTests.cs @@ -16,7 +16,7 @@ public class QueueProcessingTrackerTests : IDisposable public QueueProcessingTrackerTests() { var meter = new Meter($"{typeof(QueueProcessingTrackerTests)}"); - var metric = new DurationMetric(meter, "the-metric", _clock); + var metric = new DurationMetric(meter, "the-metric-seconds", _clock); _listener = new TestMeterListener(meter); _sut = new(metric, "the-queue"); } diff --git a/src/EventStore.Core.XUnit.Tests/Metrics/SystemMetricsTests.cs b/src/EventStore.Core.XUnit.Tests/Metrics/SystemMetricsTests.cs index 54df6c337a..c92c12c2e5 100644 --- a/src/EventStore.Core.XUnit.Tests/Metrics/SystemMetricsTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Metrics/SystemMetricsTests.cs @@ -40,12 +40,12 @@ public SystemMetricsTests() _sut.CreateCpuMetric("eventstore-sys-cpu"); - _sut.CreateMemoryMetric("eventstore-sys-mem", new() { + _sut.CreateMemoryMetric("eventstore-sys-mem-bytes", new() { { MetricsConfiguration.SystemTracker.FreeMem, "free" }, { MetricsConfiguration.SystemTracker.TotalMem, "total" }, }); - _sut.CreateDiskMetric("eventstore-sys-disk", ".", new() { + _sut.CreateDiskMetric("eventstore-sys-disk-bytes", ".", new() { { MetricsConfiguration.SystemTracker.DriveTotalBytes, "total" }, { MetricsConfiguration.SystemTracker.DriveUsedBytes, "used" }, }); diff --git a/src/EventStore.Core.XUnit.Tests/OpenTelemetry/MetricNamesTests.cs b/src/EventStore.Core.XUnit.Tests/OpenTelemetry/MetricNamesTests.cs new file mode 100644 index 0000000000..7393da8b85 --- /dev/null +++ b/src/EventStore.Core.XUnit.Tests/OpenTelemetry/MetricNamesTests.cs @@ -0,0 +1,24 @@ +using System; +using System.Linq; +using System.Reflection; +using TrogonEventStore.SemanticConventions; +using Xunit; + +namespace EventStore.Core.XUnit.Tests.OpenTelemetry; + +public class MetricNamesTests +{ + [Fact] + public void all_contains_every_metric_constant_once_in_name_order() + { + var constants = typeof(MetricNames) + .GetFields(BindingFlags.Public | BindingFlags.Static) + .Where(field => field.IsLiteral && field.FieldType == typeof(string)) + .Select(field => (string)field.GetRawConstantValue()) + .OrderBy(value => value, StringComparer.Ordinal) + .ToArray(); + + Assert.Equal(constants, MetricNames.All); + Assert.Equal(constants.Length, constants.Distinct(StringComparer.Ordinal).Count()); + } +} diff --git a/src/EventStore.Core.XUnit.Tests/TransactionLog/Chunks/TFChunkTrackerTests.cs b/src/EventStore.Core.XUnit.Tests/TransactionLog/Chunks/TFChunkTrackerTests.cs index 5cfb553f9c..f80ce331a1 100644 --- a/src/EventStore.Core.XUnit.Tests/TransactionLog/Chunks/TFChunkTrackerTests.cs +++ b/src/EventStore.Core.XUnit.Tests/TransactionLog/Chunks/TFChunkTrackerTests.cs @@ -27,14 +27,14 @@ public TFChunkTrackerTests() var meter = new Meter($"{typeof(TFChunkTrackerTests)}"); _listener = new TestMeterListener(meter); _doubleListener = new TestMeterListener(meter); - var byteMetric = new CounterMetric(meter, "eventstore-io", unit: "bytes"); - var eventMetric = new CounterMetric(meter, "eventstore-io", unit: "events"); + var byteMetric = new CounterMetric(meter, "eventstore-io-bytes"); + var eventMetric = new CounterMetric(meter, "eventstore-io-events"); var writerCheckpoint = new InMemoryCheckpoint(WriterCheckpoint); var readTag = new KeyValuePair("activity", "read"); _sut = new TFChunkTracker( readDistribution: new LogicalChunkReadDistributionMetric(meter, "chunk-read-distribution", writerCheckpoint, ChunkSize), - readDurationMetric: new DurationMetric(meter, "eventstore-io-record-read-duration", _clock), + readDurationMetric: new DurationMetric(meter, "eventstore-io-record-read-duration-seconds", _clock), readBytes: new CounterSubMetric(byteMetric, [readTag]), readEvents: new CounterSubMetric(eventMetric, [readTag])); } diff --git a/src/EventStore.Core/ClusterVNodeStartup.cs b/src/EventStore.Core/ClusterVNodeStartup.cs index a6e1dbdc4e..ae388d807f 100644 --- a/src/EventStore.Core/ClusterVNodeStartup.cs +++ b/src/EventStore.Core/ClusterVNodeStartup.cs @@ -29,6 +29,7 @@ using OpenTelemetry; using OpenTelemetry.Metrics; using OpenTelemetry.Trace; +using TrogonEventStore.SemanticConventions; using ClientGossip = EventStore.Core.Services.Transport.Grpc.Gossip; using ClusterGossip = EventStore.Core.Services.Transport.Grpc.Cluster.Gossip; using GrpcOperations = EventStore.Core.Services.Transport.Grpc.Operations; @@ -304,7 +305,7 @@ private static void ConfigureMetrics( .AddMeter(TelemetryMeterInstrumentation.GetNames(metricsConfiguration.Meters)) .AddView(i => { - if (i.Name == MetricsBootstrapper.LogicalChunkReadDistributionName) + if (i.Name == MetricNames.LogicalChunkReadDistribution) { // 20 buckets, 0, 1, 2, 4, 8, ... return new ExplicitBucketHistogramConfiguration diff --git a/src/EventStore.Core/Metrics/AverageMetric.cs b/src/EventStore.Core/Metrics/AverageMetric.cs index a887f8f887..863ec1e14c 100644 --- a/src/EventStore.Core/Metrics/AverageMetric.cs +++ b/src/EventStore.Core/Metrics/AverageMetric.cs @@ -12,10 +12,10 @@ public class AverageMetric private readonly Func _genTag; private readonly Dictionary>, Tag[])> _subMetricGroups = new(); - public AverageMetric(Meter meter, string name, string unit, Func genTag) + public AverageMetric(Meter meter, string name, Func genTag) { _genTag = genTag; - meter.CreateObservableCounter(name + "-" + unit, Observe); + meter.CreateObservableCounter(name, Observe); } public void Register(string group, Func subMetric) diff --git a/src/EventStore.Core/Metrics/CacheResourcesMetrics.cs b/src/EventStore.Core/Metrics/CacheResourcesMetrics.cs index 850ca06d31..b5b7204e66 100644 --- a/src/EventStore.Core/Metrics/CacheResourcesMetrics.cs +++ b/src/EventStore.Core/Metrics/CacheResourcesMetrics.cs @@ -10,10 +10,10 @@ public class CacheResourcesMetrics private readonly ObservableUpDownMetric _bytesMetric; private readonly ObservableUpDownMetric _entriesMetric; - public CacheResourcesMetrics(Meter meter, string name) + public CacheResourcesMetrics(Meter meter, string bytesMetricName, string entriesMetricName) { - _bytesMetric = new ObservableUpDownMetric(meter, name + "-bytes"); - _entriesMetric = new ObservableUpDownMetric(meter, name + "-entries"); + _bytesMetric = new ObservableUpDownMetric(meter, bytesMetricName); + _entriesMetric = new ObservableUpDownMetric(meter, entriesMetricName); } public void Register(string cache, ResizerUnit unit, Func getStats) diff --git a/src/EventStore.Core/Metrics/CounterMetric.cs b/src/EventStore.Core/Metrics/CounterMetric.cs index b8ec1961a0..08b906315c 100644 --- a/src/EventStore.Core/Metrics/CounterMetric.cs +++ b/src/EventStore.Core/Metrics/CounterMetric.cs @@ -8,13 +8,8 @@ public class CounterMetric private readonly List _subMetrics = new(); private readonly object _lock = new(); - public CounterMetric(Meter meter, string name, string unit) + public CounterMetric(Meter meter, string name) { - if (!string.IsNullOrWhiteSpace(unit)) - { - name = name + "-" + unit; - } - meter.CreateObservableCounter(name, Observe); } diff --git a/src/EventStore.Core/Metrics/DurationMaxMetric.cs b/src/EventStore.Core/Metrics/DurationMaxMetric.cs index c23fdc1db2..95bbf5bd84 100644 --- a/src/EventStore.Core/Metrics/DurationMaxMetric.cs +++ b/src/EventStore.Core/Metrics/DurationMaxMetric.cs @@ -11,7 +11,7 @@ public DurationMaxMetric(Meter meter, string name) { // gauge rather than updowncounter because the dimensions wont make sense to sum, // because they are maxes and not necessarily from the same moment - meter.CreateObservableGauge(name + "-seconds", Observe); + meter.CreateObservableGauge(name, Observe); } public void Add(DurationMaxTracker tracker) diff --git a/src/EventStore.Core/Metrics/DurationMetric.cs b/src/EventStore.Core/Metrics/DurationMetric.cs index a9bf5f953d..a7b541887c 100644 --- a/src/EventStore.Core/Metrics/DurationMetric.cs +++ b/src/EventStore.Core/Metrics/DurationMetric.cs @@ -12,7 +12,7 @@ public class DurationMetric public DurationMetric(Meter meter, string name, IClock clock = null) { _clock = clock ?? Clock.Instance; - _histogram = meter.CreateHistogram(name + "-seconds"); + _histogram = meter.CreateHistogram(name); } public Duration Start(string durationName) => diff --git a/src/EventStore.Core/Metrics/ProcessMetrics.cs b/src/EventStore.Core/Metrics/ProcessMetrics.cs index 7e0eadf3bc..0547235304 100644 --- a/src/EventStore.Core/Metrics/ProcessMetrics.cs +++ b/src/EventStore.Core/Metrics/ProcessMetrics.cs @@ -43,13 +43,13 @@ public void CreateObservableMetrics(Dictionary metricNam CreateObservableCounter(ProcessTracker.LockContentionCount, () => Monitor.LockContentionCount); CreateObservableCounter(ProcessTracker.ExceptionCount, RuntimeStats.GetExceptionCount); - CreateObservableCounter(ProcessTracker.TotalAllocatedBytes, () => GC.GetTotalAllocatedBytes(), "bytes"); + CreateObservableCounter(ProcessTracker.TotalAllocatedBytes, () => GC.GetTotalAllocatedBytes()); CreateObservableUpDownCounter(ProcessTracker.Cpu, RuntimeStats.GetCpuUsage); CreateObservableUpDownCounter(ProcessTracker.ThreadCount, () => ThreadPool.ThreadCount); CreateObservableUpDownCounter(ProcessTracker.ThreadPoolPendingWorkItemCount, () => ThreadPool.PendingWorkItemCount); CreateObservableUpDownCounter(ProcessTracker.TimeInGc, RuntimeStats.GetLastGCPercentTimeInGC); - CreateObservableUpDownCounter(ProcessTracker.HeapSize, () => GC.GetGCMemoryInfo().HeapSizeBytes, "bytes"); + CreateObservableUpDownCounter(ProcessTracker.HeapSize, () => GC.GetGCMemoryInfo().HeapSizeBytes); CreateObservableUpDownCounter(ProcessTracker.HeapFragmentation, () => { var info = GC.GetGCMemoryInfo(); @@ -58,7 +58,7 @@ public void CreateObservableMetrics(Dictionary metricNam return; - void CreateObservableCounter(ProcessTracker tracker, Func observe, string? unit = null) where T : struct + void CreateObservableCounter(ProcessTracker tracker, Func observe) where T : struct { if (enabledNames.TryGetValue(tracker, out var name)) { @@ -66,11 +66,11 @@ void CreateObservableCounter(ProcessTracker tracker, Func observe, string? } } - void CreateObservableUpDownCounter(ProcessTracker tracker, Func observe, string? unit = null) where T : struct + void CreateObservableUpDownCounter(ProcessTracker tracker, Func observe) where T : struct { if (enabledNames.TryGetValue(tracker, out var name)) { - meter.CreateObservableUpDownCounter(unit is null ? name : $"{name}-{unit}", observe); + meter.CreateObservableUpDownCounter(name, observe); } } } @@ -85,7 +85,7 @@ public void CreateMemoryMetric(string metricName, Dictionary if (dims.AnyRegistered()) { - meter.CreateObservableCounter($"{name}-operations", dims.GenObserve()); + meter.CreateObservableCounter(name, dims.GenObserve()); } } } diff --git a/src/EventStore.Core/Metrics/SystemMetrics.cs b/src/EventStore.Core/Metrics/SystemMetrics.cs index 46f3618c0c..d0f75e9ded 100644 --- a/src/EventStore.Core/Metrics/SystemMetrics.cs +++ b/src/EventStore.Core/Metrics/SystemMetrics.cs @@ -51,7 +51,7 @@ public void CreateMemoryMetric(string metricName, Dictionary new("queue", label)); - var queueLengthMetric = new ObservableUpDownMetric(coreMeter, "eventstore-queue-length", "items"); - var byteMetric = new CounterMetric(coreMeter, "eventstore-io", unit: "bytes"); - var eventMetric = new CounterMetric(coreMeter, "eventstore-io", unit: "events"); - var recordReadDurationMetric = new DurationMetric(coreMeter, "eventstore-io-record-read-duration"); - var electionsCounterMetric = new CounterMetric(coreMeter, "eventstore-elections-count", unit: ""); + var statusMetric = new StatusMetric(coreMeter, MetricNames.Statuses); + var grpcMethodMetric = new DurationMetric(coreMeter, MetricNames.GrpcMethodDurationSeconds); + var gossipLatencyMetric = new DurationMetric(coreMeter, MetricNames.GossipLatencySeconds); + var gossipProcessingMetric = new DurationMetric(coreMeter, MetricNames.GossipProcessingDurationSeconds); + var queueQueueingDurationMaxMetric = new DurationMaxMetric(coreMeter, MetricNames.QueueQueueingDurationMaxSeconds); + var queueProcessingDurationMetric = new DurationMetric(coreMeter, MetricNames.QueueProcessingDurationSeconds); + var queueBusyMetric = new AverageMetric(coreMeter, MetricNames.QueueBusySeconds, label => new("queue", label)); + var queueLengthMetric = new ObservableUpDownMetric(coreMeter, MetricNames.QueueLength, "items"); + var byteMetric = new CounterMetric(coreMeter, MetricNames.IoBytes); + var eventMetric = new CounterMetric(coreMeter, MetricNames.IoEvents); + var recordReadDurationMetric = new DurationMetric(coreMeter, MetricNames.IoRecordReadDurationSeconds); + var electionsCounterMetric = new CounterMetric(coreMeter, MetricNames.ElectionsCount); // incoming grpc calls var enabledCalls = conf.IncomingGrpcCalls.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToArray(); @@ -104,8 +103,8 @@ public static void Bootstrap( { _ = new IncomingGrpcCallsMetric( coreMeter, - "eventstore-current-incoming-grpc-calls", - "eventstore-incoming-grpc-calls", + MetricNames.CurrentIncomingGrpcCalls, + MetricNames.IncomingGrpcCalls, enabledCalls); } @@ -113,7 +112,7 @@ public static void Bootstrap( var enabledCacheHitsMisses = conf.CacheHitsMisses.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToArray(); if (enabledCacheHitsMisses.Length > 0) { - var metric = new CacheHitsMissesMetric(coreMeter, enabledCacheHitsMisses, "eventstore-cache-hits-misses", new() { + var metric = new CacheHitsMissesMetric(coreMeter, enabledCacheHitsMisses, MetricNames.CacheHitsMisses, new() { { Conf.Cache.StreamInfo, "stream-info" }, { Conf.Cache.Chunk, "chunk" }, }); @@ -123,7 +122,10 @@ public static void Bootstrap( // dynamic cache resources if (conf.CacheResources) { - var metrics = new CacheResourcesMetrics(coreMeter, "eventstore-cache-resources"); + var metrics = new CacheResourcesMetrics( + coreMeter, + MetricNames.CacheResourcesBytes, + MetricNames.CacheResourcesEntries); trackers.CacheResourcesTracker = new CacheResourcesTracker(metrics); } @@ -140,7 +142,7 @@ public static void Bootstrap( trackers.TransactionFileTracker = new TFChunkTracker( readDistribution: new LogicalChunkReadDistributionMetric( meter: coreMeter, - name: LogicalChunkReadDistributionName, + name: MetricNames.LogicalChunkReadDistribution, writer: dbConfig.WriterCheckpoint, chunkSize: dbConfig.ChunkSize), readDurationMetric: recordReadDurationMetric, @@ -191,25 +193,25 @@ public static void Bootstrap( var tracker = new PersistentSubscriptionTracker(); trackers.PersistentSubscriptionTracker = tracker; - coreMeter.CreateObservableUpDownCounter("eventstore-persistent-sub-connections", tracker.ObserveConnectionsCount); - coreMeter.CreateObservableUpDownCounter("eventstore-persistent-sub-parked-messages", tracker.ObserveParkedMessages); - coreMeter.CreateObservableUpDownCounter("eventstore-persistent-sub-in-flight-messages", tracker.ObserveInFlightMessages); - coreMeter.CreateObservableUpDownCounter("eventstore-persistent-sub-oldest-parked-message-seconds", tracker.ObserveOldestParkedMessage); - - coreMeter.CreateObservableCounter("eventstore-persistent-sub-park-message-requests", tracker.ObserveParkMessageRequests); - coreMeter.CreateObservableCounter("eventstore-persistent-sub-parked-message-replays", tracker.ObserveParkedMessageReplays); - coreMeter.CreateObservableCounter("eventstore-persistent-sub-parked-message-truncates", tracker.ObserveParkedMessageTruncates); - coreMeter.CreateObservableCounter("eventstore-persistent-sub-items-processed", tracker.ObserveItemsProcessed); - coreMeter.CreateObservableCounter("eventstore-persistent-sub-last-known-event-number", tracker.ObserveLastKnownEvent); - coreMeter.CreateObservableCounter("eventstore-persistent-sub-last-known-event-commit-position", tracker.ObserveLastKnownEventCommitPosition); - coreMeter.CreateObservableCounter("eventstore-persistent-sub-checkpointed-event-number", tracker.ObserveLastCheckpointedEvent); - coreMeter.CreateObservableCounter("eventstore-persistent-sub-checkpointed-event-commit-position", tracker.ObserveLastCheckpointedEventCommitPosition); + coreMeter.CreateObservableUpDownCounter(MetricNames.PersistentSubConnections, tracker.ObserveConnectionsCount); + coreMeter.CreateObservableUpDownCounter(MetricNames.PersistentSubParkedMessages, tracker.ObserveParkedMessages); + coreMeter.CreateObservableUpDownCounter(MetricNames.PersistentSubInFlightMessages, tracker.ObserveInFlightMessages); + coreMeter.CreateObservableUpDownCounter(MetricNames.PersistentSubOldestParkedMessageSeconds, tracker.ObserveOldestParkedMessage); + + coreMeter.CreateObservableCounter(MetricNames.PersistentSubParkMessageRequests, tracker.ObserveParkMessageRequests); + coreMeter.CreateObservableCounter(MetricNames.PersistentSubParkedMessageReplays, tracker.ObserveParkedMessageReplays); + coreMeter.CreateObservableCounter(MetricNames.PersistentSubParkedMessageTruncates, tracker.ObserveParkedMessageTruncates); + coreMeter.CreateObservableCounter(MetricNames.PersistentSubItemsProcessed, tracker.ObserveItemsProcessed); + coreMeter.CreateObservableCounter(MetricNames.PersistentSubLastKnownEventNumber, tracker.ObserveLastKnownEvent); + coreMeter.CreateObservableCounter(MetricNames.PersistentSubLastKnownEventCommitPosition, tracker.ObserveLastKnownEventCommitPosition); + coreMeter.CreateObservableCounter(MetricNames.PersistentSubCheckpointedEventNumber, tracker.ObserveLastCheckpointedEvent); + coreMeter.CreateObservableCounter(MetricNames.PersistentSubCheckpointedEventCommitPosition, tracker.ObserveLastCheckpointedEventCommitPosition); } // checkpoints _ = new CheckpointMetric( coreMeter, - "eventstore-checkpoints", + MetricNames.Checkpoints, conf.Checkpoints.Where(x => x.Value).Select(x => x.Key switch { Conf.Checkpoint.Chaser => dbConfig.ChaserCheckpoint, @@ -260,7 +262,7 @@ public static void Bootstrap( { if (conf.Writer.TryGetValue(Conf.WriterTracker.FlushSize, out var flushSizeEnabled) && flushSizeEnabled) { - var maxMetric = new MaxMetric(coreMeter, "eventstore-writer-flush-size-max"); + var maxMetric = new MaxMetric(coreMeter, MetricNames.WriterFlushSizeMax); trackers.WriterFlushSizeTracker = new MaxTracker( metric: maxMetric, name: null, @@ -269,7 +271,7 @@ public static void Bootstrap( if (conf.Writer.TryGetValue(Conf.WriterTracker.FlushDuration, out var flushDurationEnabled) && flushDurationEnabled) { - var maxDurationmetric = new DurationMaxMetric(coreMeter, "eventstore-writer-flush-duration-max"); + var maxDurationmetric = new DurationMaxMetric(coreMeter, MetricNames.WriterFlushDurationMaxSeconds); trackers.WriterFlushDurationTracker = new DurationMaxTracker( maxDurationmetric, name: null, @@ -312,27 +314,27 @@ public static void Bootstrap( // kestrel if (conf.Kestrel.TryGetValue(Conf.KestrelTracker.ConnectionCount, out var kestrelConnections) && kestrelConnections) { - _ = new ConnectionMetric(coreMeter, "eventstore-kestrel-connections"); + _ = new ConnectionMetric(coreMeter, MetricNames.KestrelConnections); } var timeout = TimeSpan.FromSeconds(1); // system var systemMetrics = new SystemMetrics(coreMeter, timeout, conf.System); - systemMetrics.CreateLoadAverageMetric("eventstore-sys-load-avg", new() { + systemMetrics.CreateLoadAverageMetric(MetricNames.SysLoadAvg, new() { { Conf.SystemTracker.LoadAverage1m, "1m" }, { Conf.SystemTracker.LoadAverage5m, "5m" }, { Conf.SystemTracker.LoadAverage15m, "15m" }, }); - systemMetrics.CreateCpuMetric("eventstore-sys-cpu"); + systemMetrics.CreateCpuMetric(MetricNames.SysCpu); - systemMetrics.CreateMemoryMetric("eventstore-sys-mem", new() { + systemMetrics.CreateMemoryMetric(MetricNames.SysMemBytes, new() { { Conf.SystemTracker.FreeMem, "free" }, { Conf.SystemTracker.TotalMem, "total" }, }); - systemMetrics.CreateDiskMetric("eventstore-sys-disk", dbConfig.Path, new() { + systemMetrics.CreateDiskMetric(MetricNames.SysDiskBytes, dbConfig.Path, new() { { Conf.SystemTracker.DriveTotalBytes, "total" }, { Conf.SystemTracker.DriveUsedBytes, "used" }, }); @@ -340,44 +342,44 @@ public static void Bootstrap( // process var processMetrics = new ProcessMetrics(coreMeter, timeout, conf.ExpectedScrapeIntervalSeconds, conf.Process); processMetrics.CreateObservableMetrics(new() { - { Conf.ProcessTracker.UpTime, "eventstore-proc-up-time" }, - { Conf.ProcessTracker.Cpu, "eventstore-proc-cpu" }, - { Conf.ProcessTracker.ThreadCount, "eventstore-proc-thread-count" }, - { Conf.ProcessTracker.ThreadPoolPendingWorkItemCount, "eventstore-proc-thread-pool-pending-work-item-count" }, - { Conf.ProcessTracker.LockContentionCount, "eventstore-proc-contention-count" }, - { Conf.ProcessTracker.ExceptionCount, "eventstore-proc-exception-count" }, - { Conf.ProcessTracker.TimeInGc, "eventstore-gc-time-in-gc" }, - { Conf.ProcessTracker.HeapSize, "eventstore-gc-heap-size" }, - { Conf.ProcessTracker.HeapFragmentation, "eventstore-gc-heap-fragmentation" }, - { Conf.ProcessTracker.TotalAllocatedBytes, "eventstore-gc-total-allocated" }, - { Conf.ProcessTracker.GcPauseDuration, "eventstore-gc-pause-duration-max" }, + { Conf.ProcessTracker.UpTime, MetricNames.ProcUpTime }, + { Conf.ProcessTracker.Cpu, MetricNames.ProcCpu }, + { Conf.ProcessTracker.ThreadCount, MetricNames.ProcThreadCount }, + { Conf.ProcessTracker.ThreadPoolPendingWorkItemCount, MetricNames.ProcThreadPoolPendingWorkItemCount }, + { Conf.ProcessTracker.LockContentionCount, MetricNames.ProcContentionCount }, + { Conf.ProcessTracker.ExceptionCount, MetricNames.ProcExceptionCount }, + { Conf.ProcessTracker.TimeInGc, MetricNames.GcTimeInGc }, + { Conf.ProcessTracker.HeapSize, MetricNames.GcHeapSizeBytes }, + { Conf.ProcessTracker.HeapFragmentation, MetricNames.GcHeapFragmentation }, + { Conf.ProcessTracker.TotalAllocatedBytes, MetricNames.GcTotalAllocated }, + { Conf.ProcessTracker.GcPauseDuration, MetricNames.GcPauseDurationMaxSeconds }, }); - processMetrics.CreateMemoryMetric("eventstore-proc-mem", new() { + processMetrics.CreateMemoryMetric(MetricNames.ProcMemBytes, new() { { Conf.ProcessTracker.MemWorkingSet, "working-set" }, { Conf.ProcessTracker.MemPagedBytes, "paged-bytes" }, { Conf.ProcessTracker.MemVirtualBytes, "virtual-bytes" }, }); - processMetrics.CreateGcGenerationSizeMetric("eventstore-gc-generation-size", new() { + processMetrics.CreateGcGenerationSizeMetric(MetricNames.GcGenerationSizeBytes, new() { { Conf.ProcessTracker.Gen0Size, "gen0" }, { Conf.ProcessTracker.Gen1Size, "gen1" }, { Conf.ProcessTracker.Gen2Size, "gen2" }, { Conf.ProcessTracker.LohSize, "loh" }, }); - processMetrics.CreateGcCollectionCountMetric("eventstore-gc-collection-count", new() { + processMetrics.CreateGcCollectionCountMetric(MetricNames.GcCollectionCount, new() { { Conf.ProcessTracker.Gen0CollectionCount, "gen0" }, { Conf.ProcessTracker.Gen1CollectionCount, "gen1" }, { Conf.ProcessTracker.Gen2CollectionCount, "gen2" }, }); - processMetrics.CreateDiskBytesMetric("eventstore-disk-io", new() { + processMetrics.CreateDiskBytesMetric(MetricNames.DiskIoBytes, new() { { Conf.ProcessTracker.DiskReadBytes, "read" }, { Conf.ProcessTracker.DiskWrittenBytes, "written" }, }); - processMetrics.CreateDiskOpsMetric("eventstore-disk-io", new() { + processMetrics.CreateDiskOpsMetric(MetricNames.DiskIoOperations, new() { { Conf.ProcessTracker.DiskReadOps, "read" }, { Conf.ProcessTracker.DiskWrittenOps, "written" }, }); diff --git a/src/EventStore.Projections.Core/ProjectionsSubsystem.cs b/src/EventStore.Projections.Core/ProjectionsSubsystem.cs index 3223051522..4ea3fe35d0 100644 --- a/src/EventStore.Projections.Core/ProjectionsSubsystem.cs +++ b/src/EventStore.Projections.Core/ProjectionsSubsystem.cs @@ -20,6 +20,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using TrogonEventStore.SemanticConventions; using ILogger = Serilog.ILogger; namespace EventStore.Projections.Core; @@ -210,11 +211,11 @@ private void ConfigureProjectionMetrics(bool isEnabled) var tracker = new ProjectionTracker(); _projectionTracker = tracker; - projectionMeter.CreateObservableCounter("eventstore-projection-events-processed-after-restart-total", tracker.ObserveEventsProcessed); - projectionMeter.CreateObservableUpDownCounter("eventstore-projection-progress", tracker.ObserveProgress); - projectionMeter.CreateObservableUpDownCounter("eventstore-projection-running", tracker.ObserveRunning); - projectionMeter.CreateObservableUpDownCounter("eventstore-projection-status", tracker.ObserveStatus); - projectionMeter.CreateObservableUpDownCounter("eventstore-projection-state-size", tracker.ObserveStateSize); + projectionMeter.CreateObservableCounter(MetricNames.ProjectionEventsProcessedAfterRestartTotal, tracker.ObserveEventsProcessed); + projectionMeter.CreateObservableUpDownCounter(MetricNames.ProjectionProgress, tracker.ObserveProgress); + projectionMeter.CreateObservableUpDownCounter(MetricNames.ProjectionRunning, tracker.ObserveRunning); + projectionMeter.CreateObservableUpDownCounter(MetricNames.ProjectionStatus, tracker.ObserveStatus); + projectionMeter.CreateObservableUpDownCounter(MetricNames.ProjectionStateSize, tracker.ObserveStateSize); } public void ConfigureServices(IServiceCollection services, IConfiguration configuration) => diff --git a/src/TrogonEventStore.SemanticConventions/Generated/MetricNames.g.cs b/src/TrogonEventStore.SemanticConventions/Generated/MetricNames.g.cs new file mode 100644 index 0000000000..2f4753a642 --- /dev/null +++ b/src/TrogonEventStore.SemanticConventions/Generated/MetricNames.g.cs @@ -0,0 +1,133 @@ +// + +using System; +using System.Collections.Generic; + +namespace TrogonEventStore.SemanticConventions +{ + public static class MetricNames + { + public const string CacheHitsMisses = "eventstore-cache-hits-misses"; + public const string CacheResourcesBytes = "eventstore-cache-resources-bytes"; + public const string CacheResourcesEntries = "eventstore-cache-resources-entries"; + public const string Checkpoints = "eventstore-checkpoints"; + public const string CurrentIncomingGrpcCalls = "eventstore-current-incoming-grpc-calls"; + public const string DiskIoBytes = "eventstore-disk-io-bytes"; + public const string DiskIoOperations = "eventstore-disk-io-operations"; + public const string ElectionsCount = "eventstore-elections-count"; + public const string GcCollectionCount = "eventstore-gc-collection-count"; + public const string GcGenerationSizeBytes = "eventstore-gc-generation-size-bytes"; + public const string GcHeapFragmentation = "eventstore-gc-heap-fragmentation"; + public const string GcHeapSizeBytes = "eventstore-gc-heap-size-bytes"; + public const string GcPauseDurationMaxSeconds = "eventstore-gc-pause-duration-max-seconds"; + public const string GcTimeInGc = "eventstore-gc-time-in-gc"; + public const string GcTotalAllocated = "eventstore-gc-total-allocated"; + public const string GossipLatencySeconds = "eventstore-gossip-latency-seconds"; + public const string GossipProcessingDurationSeconds = "eventstore-gossip-processing-duration-seconds"; + public const string GrpcMethodDurationSeconds = "eventstore-grpc-method-duration-seconds"; + public const string IncomingGrpcCalls = "eventstore-incoming-grpc-calls"; + public const string IoBytes = "eventstore-io-bytes"; + public const string IoEvents = "eventstore-io-events"; + public const string IoRecordReadDurationSeconds = "eventstore-io-record-read-duration-seconds"; + public const string KestrelConnections = "eventstore-kestrel-connections"; + public const string LogicalChunkReadDistribution = "eventstore-logical-chunk-read-distribution"; + public const string PersistentSubCheckpointedEventCommitPosition = "eventstore-persistent-sub-checkpointed-event-commit-position"; + public const string PersistentSubCheckpointedEventNumber = "eventstore-persistent-sub-checkpointed-event-number"; + public const string PersistentSubConnections = "eventstore-persistent-sub-connections"; + public const string PersistentSubInFlightMessages = "eventstore-persistent-sub-in-flight-messages"; + public const string PersistentSubItemsProcessed = "eventstore-persistent-sub-items-processed"; + public const string PersistentSubLastKnownEventCommitPosition = "eventstore-persistent-sub-last-known-event-commit-position"; + public const string PersistentSubLastKnownEventNumber = "eventstore-persistent-sub-last-known-event-number"; + public const string PersistentSubOldestParkedMessageSeconds = "eventstore-persistent-sub-oldest-parked-message-seconds"; + public const string PersistentSubParkMessageRequests = "eventstore-persistent-sub-park-message-requests"; + public const string PersistentSubParkedMessageReplays = "eventstore-persistent-sub-parked-message-replays"; + public const string PersistentSubParkedMessageTruncates = "eventstore-persistent-sub-parked-message-truncates"; + public const string PersistentSubParkedMessages = "eventstore-persistent-sub-parked-messages"; + public const string ProcContentionCount = "eventstore-proc-contention-count"; + public const string ProcCpu = "eventstore-proc-cpu"; + public const string ProcExceptionCount = "eventstore-proc-exception-count"; + public const string ProcMemBytes = "eventstore-proc-mem-bytes"; + public const string ProcThreadCount = "eventstore-proc-thread-count"; + public const string ProcThreadPoolPendingWorkItemCount = "eventstore-proc-thread-pool-pending-work-item-count"; + public const string ProcUpTime = "eventstore-proc-up-time"; + public const string ProjectionEventsProcessedAfterRestartTotal = "eventstore-projection-events-processed-after-restart-total"; + public const string ProjectionProgress = "eventstore-projection-progress"; + public const string ProjectionRunning = "eventstore-projection-running"; + public const string ProjectionStateSize = "eventstore-projection-state-size"; + public const string ProjectionStatus = "eventstore-projection-status"; + public const string QueueBusySeconds = "eventstore-queue-busy-seconds"; + public const string QueueLength = "eventstore-queue-length"; + public const string QueueProcessingDurationSeconds = "eventstore-queue-processing-duration-seconds"; + public const string QueueQueueingDurationMaxSeconds = "eventstore-queue-queueing-duration-max-seconds"; + public const string Statuses = "eventstore-statuses"; + public const string SysCpu = "eventstore-sys-cpu"; + public const string SysDiskBytes = "eventstore-sys-disk-bytes"; + public const string SysLoadAvg = "eventstore-sys-load-avg"; + public const string SysMemBytes = "eventstore-sys-mem-bytes"; + public const string WriterFlushDurationMaxSeconds = "eventstore-writer-flush-duration-max-seconds"; + public const string WriterFlushSizeMax = "eventstore-writer-flush-size-max"; + + public static IReadOnlyList All { get; } = Array.AsReadOnly(new[] + { + CacheHitsMisses, + CacheResourcesBytes, + CacheResourcesEntries, + Checkpoints, + CurrentIncomingGrpcCalls, + DiskIoBytes, + DiskIoOperations, + ElectionsCount, + GcCollectionCount, + GcGenerationSizeBytes, + GcHeapFragmentation, + GcHeapSizeBytes, + GcPauseDurationMaxSeconds, + GcTimeInGc, + GcTotalAllocated, + GossipLatencySeconds, + GossipProcessingDurationSeconds, + GrpcMethodDurationSeconds, + IncomingGrpcCalls, + IoBytes, + IoEvents, + IoRecordReadDurationSeconds, + KestrelConnections, + LogicalChunkReadDistribution, + PersistentSubCheckpointedEventCommitPosition, + PersistentSubCheckpointedEventNumber, + PersistentSubConnections, + PersistentSubInFlightMessages, + PersistentSubItemsProcessed, + PersistentSubLastKnownEventCommitPosition, + PersistentSubLastKnownEventNumber, + PersistentSubOldestParkedMessageSeconds, + PersistentSubParkMessageRequests, + PersistentSubParkedMessageReplays, + PersistentSubParkedMessageTruncates, + PersistentSubParkedMessages, + ProcContentionCount, + ProcCpu, + ProcExceptionCount, + ProcMemBytes, + ProcThreadCount, + ProcThreadPoolPendingWorkItemCount, + ProcUpTime, + ProjectionEventsProcessedAfterRestartTotal, + ProjectionProgress, + ProjectionRunning, + ProjectionStateSize, + ProjectionStatus, + QueueBusySeconds, + QueueLength, + QueueProcessingDurationSeconds, + QueueQueueingDurationMaxSeconds, + Statuses, + SysCpu, + SysDiskBytes, + SysLoadAvg, + SysMemBytes, + WriterFlushDurationMaxSeconds, + WriterFlushSizeMax, + }); + } +} diff --git a/src/TrogonEventStore.SemanticConventions/README.md b/src/TrogonEventStore.SemanticConventions/README.md index 31380553ac..e356c804a8 100644 --- a/src/TrogonEventStore.SemanticConventions/README.md +++ b/src/TrogonEventStore.SemanticConventions/README.md @@ -1,5 +1,5 @@ # TrogonEventStore Semantic Conventions -This package gives TrogonEventStore components one generated source for OpenTelemetry attribute names. It has no runtime dependencies. +This package gives TrogonEventStore components one generated source for OpenTelemetry attribute and built-in metric names. `MetricNames.All` provides every built-in metric name exactly once in deterministic order. The package has no runtime dependencies. The pinned OpenTelemetry registry version and C# templates under `otel/semconv` are the source of truth. Regenerate the committed constants with `mise run semconv:generate` and verify them with `mise run semconv:check`. From e67b82e4b0fab7393903457b83581db77a009c09 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 20 Jul 2026 17:01:23 -0400 Subject: [PATCH 2/4] fix(metrics): keep process metric fixtures representative Signed-off-by: Yordis Prieto --- .../Metrics/ProcessMetricsTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/EventStore.Core.XUnit.Tests/Metrics/ProcessMetricsTests.cs b/src/EventStore.Core.XUnit.Tests/Metrics/ProcessMetricsTests.cs index 5dae572a76..4697e2475c 100644 --- a/src/EventStore.Core.XUnit.Tests/Metrics/ProcessMetricsTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Metrics/ProcessMetricsTests.cs @@ -42,7 +42,7 @@ public ProcessMetricsTests() { MetricsConfiguration.ProcessTracker.HeapSize, "eventstore-gc-heap-size-bytes" }, { MetricsConfiguration.ProcessTracker.HeapFragmentation, "eventstore-gc-heap-fragmentation" }, { MetricsConfiguration.ProcessTracker.TotalAllocatedBytes, "eventstore-gc-total-allocated" }, - { MetricsConfiguration.ProcessTracker.GcPauseDuration, "eventstore-gc-pause-duration-seconds" }, + { MetricsConfiguration.ProcessTracker.GcPauseDuration, "eventstore-gc-pause-duration-max-seconds" }, }); _sut.CreateMemoryMetric("eventstore-proc-mem-bytes", new() { @@ -70,8 +70,8 @@ public ProcessMetricsTests() }); _sut.CreateDiskOpsMetric("eventstore-disk-io-operations", new() { - { MetricsConfiguration.ProcessTracker.DiskReadBytes, "read" }, - { MetricsConfiguration.ProcessTracker.DiskWrittenBytes, "written" }, + { MetricsConfiguration.ProcessTracker.DiskReadOps, "read" }, + { MetricsConfiguration.ProcessTracker.DiskWrittenOps, "written" }, }); // To trigger the GC pause detection metric. @@ -383,7 +383,7 @@ public void can_detect_gc_pauses() try { Assert.Collection( - _doubleListener.RetrieveMeasurements("eventstore-gc-pause-duration-seconds"), + _doubleListener.RetrieveMeasurements("eventstore-gc-pause-duration-max-seconds"), m => { Assert.Collection( From e0f2dac90308497bc45f081c8662e057cb0d12dd Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 20 Jul 2026 20:50:56 -0400 Subject: [PATCH 3/4] feat(telemetry): align metrics with semantic conventions Signed-off-by: Yordis Prieto --- .config/mise/tasks/semconv/generate | 8 +- docs/diagnostics/metrics.md | 712 ++++------------- .../registry/trogon/eventstore/metrics.yaml | 734 ++++++++++-------- .../registry/csharp/metric-names.cs.j2 | 14 +- .../open-telemetry-metric-definitions.cs.j2 | 33 + .../csharp/trogon-attribute-names.cs.j2 | 25 + .../templates/registry/csharp/weaver.yaml | 49 +- src/EventStore.ClusterNode/metricsconfig.json | 27 - .../Configuration/MetricsConfiguration.cs | 32 +- .../Index/IndexStatusTrackerTests.cs | 79 +- .../Index/IndexTrackerTests.cs | 12 +- .../Metrics/AverageMetricTests.cs | 19 +- .../Metrics/CacheHitsMiseesTrackerTests.cs | 31 +- .../Metrics/CacheResourcesTrackerTests.cs | 14 +- .../Metrics/DurationMaxTrackerTests.cs | 21 +- .../Metrics/DurationTrackerTests.cs | 15 +- .../Metrics/ElectionsCounterTrackerTests.cs | 7 +- .../Metrics/IncomingGrpcCallsMetricTests.cs | 167 ++-- .../Metrics/MaxTrackerTests.cs | 21 +- .../MetricInstrumentDefinitionTests.cs | 156 ++++ .../PersistentSubscriptionMetricsTests.cs | 19 +- .../Metrics/ProcessMetricsTests.cs | 428 ++-------- .../Metrics/QueueBusyTrackerTests.cs | 11 +- .../Metrics/QueueProcessingTrackerTests.cs | 11 +- .../Metrics/SystemMetricsTests.cs | 278 ++++--- .../Metrics/TestMeterListener.cs | 13 +- .../OpenTelemetry/MetricNamesTests.cs | 146 +++- .../TelemetryMeterInstrumentationTests.cs | 4 + .../TelemetryServiceIdentityTests.cs | 7 + .../Scavenge/ScavengeStatusTrackerTests.cs | 29 +- .../Services/VNode/NodeStatusTrackerTests.cs | 88 +-- .../Checkpoint/CheckpointMetricTests.cs | 12 +- .../Chunks/TFChunkTrackerTests.cs | 28 +- src/EventStore.Core/ClusterVNodeStartup.cs | 33 +- .../TelemetryMeterInstrumentation.cs | 4 +- .../Diagnostics/TelemetryServiceIdentity.cs | 30 + .../Index/IndexStatusTracker.cs | 9 +- .../Metrics/ActivityStatusSubMetric.cs | 5 +- src/EventStore.Core/Metrics/AverageMetric.cs | 54 -- .../Metrics/CacheHitsMissesMetric.cs | 18 +- .../Metrics/CacheResourcesMetrics.cs | 14 +- .../Metrics/ConnectionMetric.cs | 46 -- src/EventStore.Core/Metrics/CounterMetric.cs | 10 +- src/EventStore.Core/Metrics/Duration.cs | 5 +- .../Metrics/DurationMaxMetric.cs | 10 +- .../Metrics/DurationMaxTracker.cs | 7 +- src/EventStore.Core/Metrics/DurationMetric.cs | 9 +- .../Metrics/GCSuspensionMetric.cs | 209 ----- .../Metrics/IncomingGrpcCallsMetric.cs | 96 +-- .../LogicalChunkReadDistributionMetric.cs | 13 +- src/EventStore.Core/Metrics/MaxMetric.cs | 10 +- src/EventStore.Core/Metrics/MaxTracker.cs | 7 +- .../Metrics/ObservableUpDownMetric.cs | 26 +- .../Metrics/PersistentSubscriptionTracker.cs | 57 +- src/EventStore.Core/Metrics/ProcessMetrics.cs | 191 +++-- .../Metrics/QueueBusyTracker.cs | 2 +- .../Metrics/QueueLengthTracker.cs | 3 +- .../Metrics/QueueProcessingTracker.cs | 5 +- src/EventStore.Core/Metrics/StatusMetric.cs | 22 +- .../Metrics/StatusSubMetric.cs | 63 +- .../Metrics/SummedCounterMetric.cs | 58 ++ src/EventStore.Core/Metrics/SystemMetrics.cs | 154 +++- src/EventStore.Core/MetricsBootstrapper.cs | 166 ++-- .../Services/VNode/NodeStatusTracker.cs | 11 +- .../Checkpoint/CheckpointMetric.cs | 20 +- .../Chunks/TransactionFileTracker.cs | 8 +- .../Scavenging/ScavengeStatusTracker.cs | 9 +- .../Metrics/ProjectionMetricsTests.cs | 132 +++- .../Metrics/ProjectionTracker.cs | 55 +- .../ProjectionsSubsystem.cs | 32 +- .../Generated/AttributeNames.g.cs | 10 + .../Generated/MetricDefinitions.g.cs | 267 +++++++ .../Generated/MetricNames.g.cs | 133 ---- .../OpenTelemetryMetricDefinitions.g.cs | 39 + .../Generated/TrogonAttributeNames.g.cs | 55 ++ .../MetricDefinition.cs | 62 ++ .../MetricInstrumentKind.cs | 10 + .../README.md | 27 +- ...rogonEventStore.SemanticConventions.csproj | 2 +- 79 files changed, 2757 insertions(+), 2701 deletions(-) create mode 100644 otel/semconv/templates/registry/csharp/open-telemetry-metric-definitions.cs.j2 create mode 100644 otel/semconv/templates/registry/csharp/trogon-attribute-names.cs.j2 create mode 100644 src/EventStore.Core.XUnit.Tests/Metrics/MetricInstrumentDefinitionTests.cs delete mode 100644 src/EventStore.Core/Metrics/AverageMetric.cs delete mode 100644 src/EventStore.Core/Metrics/ConnectionMetric.cs delete mode 100644 src/EventStore.Core/Metrics/GCSuspensionMetric.cs create mode 100644 src/EventStore.Core/Metrics/SummedCounterMetric.cs create mode 100644 src/TrogonEventStore.SemanticConventions/Generated/MetricDefinitions.g.cs delete mode 100644 src/TrogonEventStore.SemanticConventions/Generated/MetricNames.g.cs create mode 100644 src/TrogonEventStore.SemanticConventions/Generated/OpenTelemetryMetricDefinitions.g.cs create mode 100644 src/TrogonEventStore.SemanticConventions/Generated/TrogonAttributeNames.g.cs create mode 100644 src/TrogonEventStore.SemanticConventions/MetricDefinition.cs create mode 100644 src/TrogonEventStore.SemanticConventions/MetricInstrumentKind.cs diff --git a/.config/mise/tasks/semconv/generate b/.config/mise/tasks/semconv/generate index 538cebd294..b0ce43f4c1 100755 --- a/.config/mise/tasks/semconv/generate +++ b/.config/mise/tasks/semconv/generate @@ -21,13 +21,17 @@ weaver registry generate csharp "$staging" \ --future \ --registry "$registry" \ --templates "$root/otel/semconv/templates" \ - -D attributes=false + -D attributes=false \ + -D custom_attributes=true \ + -D official_metrics=false weaver registry generate csharp "$staging" \ --future \ --registry "$official_registry" \ --templates "$root/otel/semconv/templates" \ - -D metrics=false + -D custom_attributes=false \ + -D metrics=false \ + -D official_metrics=true set -- "$staging"/*.g.cs [ -e "$1" ] || exit 1 diff --git a/docs/diagnostics/metrics.md b/docs/diagnostics/metrics.md index 63f8c50551..ae011b7174 100644 --- a/docs/diagnostics/metrics.md +++ b/docs/diagnostics/metrics.md @@ -1,604 +1,174 @@ -# Metrics + -TrogonEventStore collects metrics in [Prometheus format](https://prometheus.io/docs/instrumenting/exposition_formats/#text-based-format), available on the `/-/metrics` endpoint. Prometheus can be configured to scrape this endpoint directly. The metrics are configured in `metricsconfig.json`. +# Metrics reference -In addition, TrogonEventStore can actively export metrics to a specified endpoint using the [OpenTelemetry Protocol](https://opentelemetry.io/docs/specs/otel/protocol/) (OTLP). +TrogonEventStore exposes metrics through the OpenTelemetry Protocol (OTLP) and the Prometheus-compatible `/-/metrics` endpoint. The canonical instrument identity is the dotted OpenTelemetry name documented here. Prometheus exporters may translate dots, units, or other name components according to their configured translation strategy. -The built-in Core and Projections meter sources are always registered. Use the `Meters` array in `metricsconfig.json` only when additional components expose their own `System.Diagnostics.Metrics` meter sources. +Configure node-specific metrics in `metricsconfig.json`. The `Meters` array adds third-party `System.Diagnostics.Metrics` meter sources; the built-in sources do not need to be listed there. -## Metrics reference +Units use UCUM notation: `s` is seconds, `By` is bytes, and `1` is dimensionless. Units such as `{event}` are annotated counts. The unit is metadata and is not repeated in the canonical instrument name. -### Caches +Telemetry resources include service identity, process creation time and PID, executable and runtime identity, and host name and architecture. Deployments should provide `host.id` through `OTEL_RESOURCE_ATTRIBUTES` when a stable machine or cloud instance identifier is available. -#### Cache hits and misses +## Built-in meter sources -TrogonEventStore tracks cache hits/misses metrics for `stream-info` and `chunk` caches. +The meter provider always subscribes to these sources: -| Time series | Type | Description | -|:---------------------------------------------------------------------------|:-------------------------|:----------------------------------------| -| `eventstore_cache_hits_misses{cache=,kind=<"hits"\|"misses">}` | [Counter](#common-types) | Total hits/misses on _CACHE_NAME_ cache | +| Meter | Source | +| --- | --- | +| `EventStore.Core` | Core TrogonEventStore instruments | +| `EventStore.Projections.Core` | Projection instruments | +| `System.Runtime` | Native .NET process, garbage collection, exception, lock, and thread pool instruments | +| `Microsoft.AspNetCore.Server.Kestrel` | Native Kestrel connection and server instruments | -Example configuration: -```json -"CacheHitsMisses": { - "StreamInfo": true, - "Chunk": false -} -``` - -Example output: -``` -# TYPE eventstore_cache_hits_misses counter -eventstore_cache_hits_misses{cache="stream-info",kind="hits"} 104329 1688157489545 -eventstore_cache_hits_misses{cache="stream-info",kind="misses"} 117 1688157489545 -``` - -#### Dynamic cache resources - -Certain caches that TrogonEventStore uses are dynamic in nature i.e. their capacity scales up/down during their lifetime. TrogonEventStore records metrics for resources being used by each such dynamic cache. - -| Time series | Type | Description | -|:---------------------------------------------------------------------------------|:-----------------------|:-----------------------------------------------------| -| `eventstore_cache_resources_bytes{cache=,kind=<"capacity"\|"size">}` | [Gauge](#common-types) | Current capacity/size of _CACHE_NAME_ cache in bytes | -| `eventstore_cache_resources_entries{cache=,kind="count"}` | [Gauge](#common-types) | Current number of entries in _CACHE_NAME_ cache | - -Example configuration: -```json -"CacheResources": true -``` - -Example output: -``` -# TYPE eventstore_cache_resources_bytes gauge -# UNIT eventstore_cache_resources_bytes bytes -eventstore_cache_resources_bytes{cache="LastEventNumber",kind="capacity"} 50000000 1688157491029 -eventstore_cache_resources_bytes{cache="LastEventNumber",kind="size"} 15804 1688157491029 - -# TYPE eventstore_cache_resources_entries gauge -# UNIT eventstore_cache_resources_entries entries -eventstore_cache_resources_entries{cache="LastEventNumber",kind="count"} 75 1688157491029 -``` - -### Checkpoints - -| Time series | Type | Description | -|:--------------------------------------------------------------------|:-----------------------|:---------------------------------------| -| `eventstore_checkpoints{name=,read="non-flushed"}` | [Gauge](#common-types) | Value for _CHECKPOINT_NAME_ checkpoint | - -Example configuration: -```json -"Checkpoints": { - "Replication": true, - "Chaser": false, - "Epoch": false, - "Index": false, - "Proposal": false, - "Truncate": false, - "Writer": false, - "StreamExistenceFilter": false -} -``` - -Example output: -``` -# TYPE eventstore_checkpoints gauge -eventstore_checkpoints{name="replication",read="non-flushed"} 613363 1688054162478 -``` - -### Elections Count - -This metric tracks the number of elections that have been completed. - -| Time series | Type | Description | -|:-----------------------------|:-------------------------|:-----------------------------| -| `eventstore_elections_count` | [Counter](#common-types) | Elections count in a cluster | - -Example configuration: -```json -"ElectionsCount": true -``` - -Example output: -``` -# TYPE eventstore_elections_count counter -eventstore_elections_count 0 1710188996949 -``` - -### Events - -These metrics track events written to and read from the server, including reads from caches. - -| Time series | Type | Description | -|:-----------------------------------------------------|:-------------------------|:--------------------| -| `eventstore_io_bytes{activity="read"}` | [Counter](#common-types) | Event bytes read | -| `eventstore_io_events{activity=<"read"\|"written">}` | [Counter](#common-types) | Events read/written | - -Example configuration: -```json -"Events": { - "Read": false, - "Written": true -} -``` - -Example output: -``` -# TYPE eventstore_io_events counter -# UNIT eventstore_io_events events -eventstore_io_events{activity="written"} 320 1687963622074 -``` - -### Gossip - -Measures the round trip latency and processing time of gossip. -Usually a node pushes new gossip to other nodes periodically or when its view of the cluster changes. Sometimes nodes pull gossip from each other if there is a suspected network problem. - -#### Gossip latency - -| Time series | Type | Description | -|:--------------------------------------------------------------------------------------------------------------------|:---------------------------|:---------------------------------------------------------------------------------------------| -| `eventstore_gossip_latency_seconds_bucket{activity="pull-from-peer",status=<"successful"\|"failed">,le=}` | [Histogram](#common-types) | Number of gossips pulled from peers with latency less than or equal to _DURATION_ in seconds | -| `eventstore_gossip_latency_seconds_bucket{activity="push-to-peer",status=<"successful"\|"failed">,le=}` | [Histogram](#common-types) | Number of gossips pushed to peers with latency less than or equal to _DURATION_ in seconds | - -#### Gossip processing - -| Time Series | Type | Description | -|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------|:-------------------------------------------------------------------------------------------------------------| -| `eventstore_gossip_processing_duration_seconds_bucket{`
`activity="push-from-peer",`
`status=<"successful"\|"failed">,`
`le=}` | [Histogram](#common-types) | Number of gossips pushed from peers that took less than or equal to _DURATION_ in seconds to process | -| `eventstore_gossip_processing_duration_seconds_bucket{`
`activity="request-from-peer",`
`status=<"successful"\|"failed">,`
`le=}` | [Histogram](#common-types) | Number of gossip requests from peers that took less than or equal to _DURATION_ in seconds to process | -| `eventstore_gossip_processing_duration_seconds_bucket{`
`activity="request-from-grpc-client",`
`status=<"successful"\|"failed">,`
`le=}` | [Histogram](#common-types) | Number of gossip requests from gRPC clients that took less than or equal to _DURATION_ in seconds to process | - -Example configuration: -```json -"Gossip": { - "PullFromPeer": false, - "PushToPeer": true, - "ProcessingPushFromPeer": false, - "ProcessingRequestFromPeer": false, - "ProcessingRequestFromGrpcClient": false -} -``` - -Example output: -``` -# TYPE eventstore_gossip_latency_seconds histogram -# UNIT eventstore_gossip_latency_seconds seconds -eventstore_gossip_latency_seconds_bucket{activity="push-to-peer",status="successful",le="0.005"} 8 1687972306948 -``` - -### Incoming gRPC calls - -| Time series | Type | Description | -|:------------------------------------------------------------------|:-------------------------|:-------------------------------------------------------------------------------------------| -| `eventstore_current_incoming_grpc_calls` | [Gauge](#common-types) | Inflight gRPC calls i.e. gRPC requests that have started on the server but not yet stopped | -| `eventstore_incoming_grpc_calls{kind="total"}` | [Counter](#common-types) | Total gRPC requests served | -| `eventstore_incoming_grpc_calls{kind="failed"}` | [Counter](#common-types) | Total gRPC requests failed | -| `eventstore_incoming_grpc_calls{`
`kind="unimplemented"}` | [Counter](#common-types) | Total gRPC requests made to unimplemented methods | -| `eventstore_incoming_grpc_calls{`
`kind="deadline-exceeded"}` | [Counter](#common-types) | Total gRPC requests for which deadline have exceeded | - -Example configuration: -```json -"IncomingGrpcCalls": { - "Current": true, - "Total": false, - "Failed": true, - "Unimplemented": false, - "DeadlineExceeded": false -} -``` - -Example output: -``` -# TYPE eventstore_current_incoming_grpc_calls gauge -eventstore_current_incoming_grpc_calls 1 1687963622074 - -# TYPE eventstore_incoming_grpc_calls counter -eventstore_incoming_grpc_calls{kind="failed"} 1 1687962877623 -``` - -#### Client protocol gRPC methods - -In addition, TrogonEventStore also records metrics for each of client protocol gRPC methods: `StreamRead`, `StreamAppend`, `StreamBatchAppend`, `StreamDelete` and `StreamTombstone`. They are grouped together according to the mapping defined in the configuration. - -| Time series | Type | Description | -|:---------------------------------------------------------------------------------------------------------------|:---------------------------|:-------------------------------------------------------------------------------------------------| -| `eventstore_grpc_method_duration_seconds_bucket{`
`activity=