From 93875e84c6012aa2b10f09c20c7975fc4741d109 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Thu, 16 Jul 2026 10:13:56 +0200 Subject: [PATCH 1/5] feat: add kpi that tracks CR count per configured cluster Signed-off-by: Markus Wieland --- helm/bundles/cortex-nova/templates/kpis.yaml | 20 +- .../multicluster_object_count_kpi.go | 142 ++++++++++ .../multicluster_object_count_kpi_test.go | 243 ++++++++++++++++++ internal/knowledge/kpis/supported_kpis.go | 2 + pkg/multicluster/client.go | 80 ++++++ pkg/multicluster/client_test.go | 97 +++++++ pkg/multicluster/testing/client.go | 6 + 7 files changed, 589 insertions(+), 1 deletion(-) create mode 100644 internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go create mode 100644 internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi_test.go create mode 100644 pkg/multicluster/testing/client.go diff --git a/helm/bundles/cortex-nova/templates/kpis.yaml b/helm/bundles/cortex-nova/templates/kpis.yaml index 5717cd62e..59b28d2f7 100644 --- a/helm/bundles/cortex-nova/templates/kpis.yaml +++ b/helm/bundles/cortex-nova/templates/kpis.yaml @@ -191,4 +191,22 @@ spec: - name: host-details - name: host-utilization description: | - This KPI tracks the capacity and utilization of VMware hosts in terms of CPU, RAM, and disk resources. \ No newline at end of file + This KPI tracks the capacity and utilization of VMware hosts in terms of CPU, RAM, and disk resources. +--- +apiVersion: cortex.cloud/v1alpha1 +kind: KPI +metadata: + name: multicluster-object-count +spec: + schedulingDomain: nova + impl: multicluster_object_count_kpi + opts: + gvks: + - cortex.cloud/v1alpha1/ReservationList + - cortex.cloud/v1alpha1/CommittedResourceList + - kvm.cloud.sap/v1/HypervisorList + description: | + This KPI reports the number of objects of each configured GVK per remote + cluster, labelled by the cluster's availability zone. It uses + PartialObjectMetadataList so only object metadata is fetched, making it + efficient even for large object counts. \ No newline at end of file diff --git a/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go new file mode 100644 index 000000000..a8b15ab45 --- /dev/null +++ b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go @@ -0,0 +1,142 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "fmt" + "log/slog" + "strings" + "unicode" + + "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins" + "github.com/cobaltcore-dev/cortex/pkg/conf" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" + "github.com/prometheus/client_golang/prometheus" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// MulticlusterObjectCountKPIOpts configures which GVKs to count per cluster. +// Each entry is a "group/version/Kind" string, e.g. +// "cortex.cloud/v1alpha1/HypervisorList". +type MulticlusterObjectCountKPIOpts struct { + GVKs []string `json:"gvks"` +} + +type gvkDesc struct { + gvk schema.GroupVersionKind + labelKeys []string // snake_cased routing label keys (stable order) + desc *prometheus.Desc +} + +// MulticlusterObjectCountKPI reports the number of objects of each configured +// GVK per remote cluster, labelled by the cluster's routing labels. +type MulticlusterObjectCountKPI struct { + plugins.BaseKPI[MulticlusterObjectCountKPIOpts] + mcl *multicluster.Client + descs []gvkDesc +} + +func (MulticlusterObjectCountKPI) GetName() string { return "multicluster_object_count_kpi" } + +func (k *MulticlusterObjectCountKPI) Init(_ *db.DB, c client.Client, opts conf.RawOpts) error { + if err := k.BaseKPI.Init(nil, c, opts); err != nil { + return err + } + mcl, ok := c.(*multicluster.Client) + if !ok { + return fmt.Errorf("multicluster_object_count_kpi requires a *multicluster.Client, got %T", c) + } + k.mcl = mcl + + for _, raw := range k.Options.GVKs { + gvk, err := parseGVK(raw) + if err != nil { + return fmt.Errorf("invalid GVK %q: %w", raw, err) + } + routeLabels := mcl.ConfiguredRouteLabels(gvk) + // Collect unique label keys across all clusters (stable order via insertion). + keySet := map[string]bool{} + var labelKeys []string + for _, lm := range routeLabels { + for k := range lm { + snake := toSnakeCase(k) + if !keySet[snake] { + keySet[snake] = true + labelKeys = append(labelKeys, snake) + } + } + } + varLabels := append([]string{"group", "version", "kind"}, labelKeys...) + desc := prometheus.NewDesc( + "cortex_multicluster_object_count", + "Number of objects of a given GVK per cluster", + varLabels, + nil, + ) + k.descs = append(k.descs, gvkDesc{gvk: gvk, labelKeys: labelKeys, desc: desc}) + } + return nil +} + +func (k *MulticlusterObjectCountKPI) Describe(ch chan<- *prometheus.Desc) { + for _, d := range k.descs { + ch <- d.desc + } +} + +func (k *MulticlusterObjectCountKPI) Collect(ch chan<- prometheus.Metric) { + ctx := context.Background() + for _, d := range k.descs { + counts, err := k.mcl.CountPerClusterByGVK(ctx, d.gvk) + if err != nil { + slog.Error("multicluster_object_count_kpi: failed to count objects", + "gvk", d.gvk, "err", err) + continue + } + for _, c := range counts { + labelVals := make([]string, 0, 3+len(d.labelKeys)) + labelVals = append(labelVals, d.gvk.Group, d.gvk.Version, d.gvk.Kind) + for _, key := range d.labelKeys { + labelVals = append(labelVals, labelValueForSnakeKey(key, c.Labels)) + } + ch <- prometheus.MustNewConstMetric(d.desc, prometheus.GaugeValue, + float64(c.Count), labelVals...) + } + } +} + +// parseGVK parses a "group/version/Kind" string. +func parseGVK(s string) (schema.GroupVersionKind, error) { + parts := strings.SplitN(s, "/", 3) + if len(parts) != 3 { + return schema.GroupVersionKind{}, fmt.Errorf("expected group/version/Kind, got: %s", s) + } + return schema.GroupVersionKind{Group: parts[0], Version: parts[1], Kind: parts[2]}, nil +} + +// toSnakeCase converts camelCase to snake_case (e.g. availabilityZone → availability_zone). +func toSnakeCase(s string) string { + var b strings.Builder + for i, r := range s { + if unicode.IsUpper(r) && i > 0 { + b.WriteByte('_') + } + b.WriteRune(unicode.ToLower(r)) + } + return b.String() +} + +// labelValueForSnakeKey finds the value in labels whose key, when snake_cased, +// matches snakeKey. Returns empty string for nil labels or no match (home cluster). +func labelValueForSnakeKey(snakeKey string, labels map[string]string) string { + for k, v := range labels { + if toSnakeCase(k) == snakeKey { + return v + } + } + return "" +} diff --git a/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi_test.go b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi_test.go new file mode 100644 index 000000000..5eff691b6 --- /dev/null +++ b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi_test.go @@ -0,0 +1,243 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/pkg/conf" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/cobaltcore-dev/cortex/pkg/multicluster" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" +) + +var ( + cmListGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMapList"} +) + +func newKPITestScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + if err := corev1.AddToScheme(s); err != nil { + t.Fatal(err) + } + if err := v1alpha1.AddToScheme(s); err != nil { + t.Fatal(err) + } + return s +} + +// FakeCluster is a minimal cluster.Cluster backed by a fake client. +type fakeCluster struct { + cluster.Cluster + FakeClient client.Client +} + +func (f *fakeCluster) GetClient() client.Client { return f.FakeClient } +func (f *fakeCluster) GetCache() cache.Cache { return nil } + +// NewFakeCluster returns a FakeCluster pre-loaded with the given objects. +func newFakeCluster(scheme *runtime.Scheme, objs ...client.Object) *fakeCluster { + return &fakeCluster{ + FakeClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build(), + } +} + +// RemoteEntry describes one remote cluster to register in the client. +type RemoteEntry struct { + Cluster cluster.Cluster + Labels map[string]string + GVKs []schema.GroupVersionKind +} + +// NewClient builds a *multicluster.Client with a home cluster and the given +// remote entries. It uses AddRemoteCluster so no real REST config is needed. +func newFakeMCLClient(scheme *runtime.Scheme, home cluster.Cluster, homeGVKs []schema.GroupVersionKind, remotes []RemoteEntry) *multicluster.Client { + c := &multicluster.Client{ + HomeScheme: scheme, + HomeCluster: home, + } + for _, gvk := range homeGVKs { + c.AddHomeGVK(gvk) + } + for _, r := range remotes { + c.AddRemoteCluster(r.Cluster, r.Labels, r.GVKs...) + } + return c +} + +func TestMulticlusterObjectCountKPI_GetName(t *testing.T) { + kpi := &MulticlusterObjectCountKPI{} + if got := kpi.GetName(); got != "multicluster_object_count_kpi" { + t.Errorf("GetName() = %q, want %q", got, "multicluster_object_count_kpi") + } +} + +func TestMulticlusterObjectCountKPI_Init_RejectsNonMCLClient(t *testing.T) { + scheme, err := v1alpha1.SchemeBuilder.Build() + if err != nil { + t.Fatal(err) + } + plainClient := fake.NewClientBuilder().WithScheme(scheme).Build() + kpi := &MulticlusterObjectCountKPI{} + if err := kpi.Init(nil, plainClient, conf.NewRawOpts(`{"gvks":[]}`)); err == nil { + t.Fatal("expected error when passing a non-*multicluster.Client") + } +} + +func TestMulticlusterObjectCountKPI_Init_RejectsInvalidGVK(t *testing.T) { + scheme := newKPITestScheme(t) + mcl := newFakeMCLClient(scheme, newFakeCluster(scheme), nil, nil) + kpi := &MulticlusterObjectCountKPI{} + opts := conf.NewRawOpts(`{"gvks":["not-a-valid-gvk"]}`) + if err := kpi.Init(nil, mcl, opts); err == nil { + t.Fatal("expected error for invalid GVK string") + } +} + +func TestMulticlusterObjectCountKPI_Describe(t *testing.T) { + scheme := newKPITestScheme(t) + remote := newFakeCluster(scheme) + mcl := newFakeMCLClient(scheme, newFakeCluster(scheme), nil, []RemoteEntry{ + { + Cluster: remote, + Labels: map[string]string{"availabilityZone": "az-1"}, + GVKs: []schema.GroupVersionKind{cmListGVK}, + }, + }) + + kpi := &MulticlusterObjectCountKPI{} + opts := conf.NewRawOpts(`{"gvks":["/v1/ConfigMapList"]}`) + if err := kpi.Init(nil, mcl, opts); err != nil { + t.Fatalf("Init failed: %v", err) + } + + ch := make(chan *prometheus.Desc, 5) + kpi.Describe(ch) + close(ch) + var count int + for range ch { + count++ + } + if count != 1 { + t.Errorf("Describe: expected 1 descriptor, got %d", count) + } +} + +func TestMulticlusterObjectCountKPI_Collect(t *testing.T) { + scheme := newKPITestScheme(t) + + az1 := newFakeCluster(scheme, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-1", Namespace: "default"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-2", Namespace: "default"}}, + ) + az2 := newFakeCluster(scheme, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-3", Namespace: "default"}}, + ) + + mcl := newFakeMCLClient(scheme, newFakeCluster(scheme), nil, []RemoteEntry{ + {Cluster: az1, Labels: map[string]string{"availabilityZone": "az-1"}, GVKs: []schema.GroupVersionKind{cmListGVK}}, + {Cluster: az2, Labels: map[string]string{"availabilityZone": "az-2"}, GVKs: []schema.GroupVersionKind{cmListGVK}}, + }) + + kpi := &MulticlusterObjectCountKPI{} + opts := conf.NewRawOpts(`{"gvks":["/v1/ConfigMapList"]}`) + if err := kpi.Init(nil, mcl, opts); err != nil { + t.Fatalf("Init failed: %v", err) + } + + ch := make(chan prometheus.Metric, 10) + kpi.Collect(ch) + close(ch) + + // Collect into a map: availabilityZone -> count value. + byAZ := map[string]float64{} + for m := range ch { + var metric dto.Metric + if err := m.Write(&metric); err != nil { + t.Fatalf("failed to write metric: %v", err) + } + var az string + for _, lp := range metric.Label { + if lp.GetName() == "availability_zone" { + az = lp.GetValue() + } + } + byAZ[az] = metric.Gauge.GetValue() + } + + if len(byAZ) != 2 { + t.Fatalf("expected metrics for 2 clusters, got %d: %v", len(byAZ), byAZ) + } + if byAZ["az-1"] != 2 { + t.Errorf("az-1: expected count 2, got %g", byAZ["az-1"]) + } + if byAZ["az-2"] != 1 { + t.Errorf("az-2: expected count 1, got %g", byAZ["az-2"]) + } +} + +func TestMulticlusterObjectCountKPI_Collect_HomeCluster(t *testing.T) { + scheme := newKPITestScheme(t) + + home := newFakeCluster(scheme, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-home", Namespace: "default"}}, + ) + + mcl := newFakeMCLClient(scheme, home, []schema.GroupVersionKind{cmListGVK}, nil) + + kpi := &MulticlusterObjectCountKPI{} + opts := conf.NewRawOpts(`{"gvks":["/v1/ConfigMapList"]}`) + if err := kpi.Init(nil, mcl, opts); err != nil { + t.Fatalf("Init failed: %v", err) + } + + ch := make(chan prometheus.Metric, 10) + kpi.Collect(ch) + close(ch) + + var collected int + var count float64 + for m := range ch { + collected++ + var metric dto.Metric + if err := m.Write(&metric); err != nil { + t.Fatalf("failed to write metric: %v", err) + } + count = metric.Gauge.GetValue() + } + if collected != 1 { + t.Fatalf("expected 1 metric (home cluster), got %d", collected) + } + if count != 1 { + t.Errorf("expected count 1, got %g", count) + } +} + +func TestMulticlusterObjectCountKPI_SnakeCaseLabels(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"availabilityZone", "availability_zone"}, + {"az", "az"}, + {"myLabelKey", "my_label_key"}, + {"alreadysnake", "alreadysnake"}, + } + for _, tt := range tests { + if got := toSnakeCase(tt.input); got != tt.want { + t.Errorf("toSnakeCase(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} diff --git a/internal/knowledge/kpis/supported_kpis.go b/internal/knowledge/kpis/supported_kpis.go index 155e3aab9..6a350d1e6 100644 --- a/internal/knowledge/kpis/supported_kpis.go +++ b/internal/knowledge/kpis/supported_kpis.go @@ -34,4 +34,6 @@ var supportedKPIs = map[string]plugins.KPI{ "decision_state_kpi": &deployment.DecisionStateKPI{}, "kpi_state_kpi": &deployment.KPIStateKPI{}, "pipeline_state_kpi": &deployment.PipelineStateKPI{}, + + "multicluster_object_count_kpi": &deployment.MulticlusterObjectCountKPI{}, } diff --git a/pkg/multicluster/client.go b/pkg/multicluster/client.go index 979258a44..eff768947 100644 --- a/pkg/multicluster/client.go +++ b/pkg/multicluster/client.go @@ -13,6 +13,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" @@ -183,6 +184,32 @@ func (c *Client) AddRemote(ctx context.Context, host, caCert string, insecureSki return cl, nil } +// AddRemoteCluster registers a pre-built cluster.Cluster as a remote for the +// given GVKs with the provided routing labels. Unlike AddRemote, it does not +// create a new cluster from a REST config — the caller supplies the cluster +// directly. This is useful for in-process wiring and for unit tests. +func (c *Client) AddRemoteCluster(cl cluster.Cluster, labels map[string]string, gvks ...schema.GroupVersionKind) { + c.remoteClustersMu.Lock() + defer c.remoteClustersMu.Unlock() + if c.remoteClusters == nil { + c.remoteClusters = make(map[schema.GroupVersionKind][]remoteCluster) + } + for _, gvk := range gvks { + c.remoteClusters[gvk] = append(c.remoteClusters[gvk], remoteCluster{ + cluster: cl, + labels: labels, + }) + } +} + +// AddHomeGVK marks the given GVK as served by the home cluster. +func (c *Client) AddHomeGVK(gvk schema.GroupVersionKind) { + if c.homeGVKs == nil { + c.homeGVKs = make(map[schema.GroupVersionKind]bool) + } + c.homeGVKs[gvk] = true +} + // Get the gvk registered for the given resource in the home cluster's scheme. func (c *Client) GVKFromHomeScheme(obj runtime.Object) (gvk schema.GroupVersionKind, err error) { gvks, unversioned, err := c.HomeScheme.ObjectKinds(obj) @@ -448,6 +475,59 @@ func (c *Client) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts return errors.New("apply operation is not supported in multicluster client") } +// ClusterObjectCount is one entry in the result of CountPerClusterByGVK. +// Labels holds the routing labels for the cluster (nil for the home cluster). +type ClusterObjectCount struct { + Labels map[string]string + Count int +} + +// CountPerClusterByGVK returns the number of objects of the given GVK in each +// configured cluster. It uses PartialObjectMetadataList so only object metadata +// crosses the wire — no spec or status — making it efficient even for large +// object counts. Clusters that return an error are logged and skipped (same +// policy as List). The home cluster is included with nil Labels. +func (c *Client) CountPerClusterByGVK(ctx context.Context, gvk schema.GroupVersionKind, opts ...client.ListOption) ([]ClusterObjectCount, error) { + log := ctrl.LoggerFrom(ctx) + + c.remoteClustersMu.RLock() + remotes := c.remoteClusters[gvk] + isHome := c.homeGVKs[gvk] + if len(remotes) == 0 && !isHome { + c.remoteClustersMu.RUnlock() + return nil, fmt.Errorf("GVK %s is not configured in home or any remote cluster", gvk) + } + type clusterEntry struct { + cl cluster.Cluster + labels map[string]string // nil for home + } + entries := make([]clusterEntry, 0, len(remotes)+1) + for _, r := range remotes { + entries = append(entries, clusterEntry{cl: r.cluster, labels: maps.Clone(r.labels)}) + } + if isHome && c.HomeCluster != nil { + entries = append(entries, clusterEntry{cl: c.HomeCluster, labels: nil}) + } + c.remoteClustersMu.RUnlock() + + results := make([]ClusterObjectCount, 0, len(entries)) + for _, e := range entries { + partialList := &metav1.PartialObjectMetadataList{} + partialList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: gvk.Group, + Version: gvk.Version, + Kind: gvk.Kind, + }) + if err := e.cl.GetClient().List(ctx, partialList, opts...); err != nil { + log.Error(err, "error counting resources from cluster", + "gvk", gvk, "host", e.cl.GetConfig().Host) + continue + } + results = append(results, ClusterObjectCount{Labels: e.labels, Count: len(partialList.Items)}) + } + return results, nil +} + // Create routes the object to the matching cluster using the ResourceRouter // and performs a Create operation. func (c *Client) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { diff --git a/pkg/multicluster/client_test.go b/pkg/multicluster/client_test.go index 6e965d440..6191109d3 100644 --- a/pkg/multicluster/client_test.go +++ b/pkg/multicluster/client_test.go @@ -1726,3 +1726,100 @@ func TestClient_ConfiguredRouteLabels(t *testing.T) { } }) } + +func TestClient_CountPerClusterByGVK(t *testing.T) { + scheme := newTestScheme(t) + ctx := context.Background() + + t.Run("counts objects per remote cluster", func(t *testing.T) { + az1 := newFakeCluster(scheme, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-1", Namespace: "default"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-2", Namespace: "default"}}, + ) + az2 := newFakeCluster(scheme, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-3", Namespace: "default"}}, + ) + c := &Client{ + HomeScheme: scheme, + remoteClusters: map[schema.GroupVersionKind][]remoteCluster{ + configMapListGVK: { + {cluster: az1, labels: map[string]string{"availabilityZone": "az-1"}}, + {cluster: az2, labels: map[string]string{"availabilityZone": "az-2"}}, + }, + }, + } + results, err := c.CountPerClusterByGVK(ctx, configMapListGVK) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(results) != 2 { + t.Fatalf("expected 2 results, got %d", len(results)) + } + byAZ := map[string]int{} + for _, r := range results { + byAZ[r.Labels["availabilityZone"]] = r.Count + } + if byAZ["az-1"] != 2 { + t.Errorf("expected 2 objects in az-1, got %d", byAZ["az-1"]) + } + if byAZ["az-2"] != 1 { + t.Errorf("expected 1 object in az-2, got %d", byAZ["az-2"]) + } + }) + + t.Run("includes home cluster with nil labels", func(t *testing.T) { + home := newFakeCluster(scheme, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-home", Namespace: "default"}}, + ) + c := &Client{ + HomeCluster: home, + HomeScheme: scheme, + homeGVKs: map[schema.GroupVersionKind]bool{configMapListGVK: true}, + } + results, err := c.CountPerClusterByGVK(ctx, configMapListGVK) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if results[0].Labels != nil { + t.Errorf("expected nil labels for home cluster, got %v", results[0].Labels) + } + if results[0].Count != 1 { + t.Errorf("expected count 1, got %d", results[0].Count) + } + }) + + t.Run("returns error for unconfigured GVK", func(t *testing.T) { + c := &Client{ + HomeScheme: scheme, + } + _, err := c.CountPerClusterByGVK(ctx, configMapListGVK) + if err == nil { + t.Fatal("expected error for unconfigured GVK") + } + }) + + t.Run("empty clusters return zero counts", func(t *testing.T) { + az1 := newFakeCluster(scheme) // no objects + c := &Client{ + HomeScheme: scheme, + remoteClusters: map[schema.GroupVersionKind][]remoteCluster{ + configMapListGVK: { + {cluster: az1, labels: map[string]string{"availabilityZone": "az-1"}}, + }, + }, + } + results, err := c.CountPerClusterByGVK(ctx, configMapListGVK) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if results[0].Count != 0 { + t.Errorf("expected count 0, got %d", results[0].Count) + } + }) +} diff --git a/pkg/multicluster/testing/client.go b/pkg/multicluster/testing/client.go new file mode 100644 index 000000000..d63105708 --- /dev/null +++ b/pkg/multicluster/testing/client.go @@ -0,0 +1,6 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +// Package multiclustertest provides helpers for building *multicluster.Client +// instances in unit tests without a real Kubernetes API server. +package multiclustertest From adeea16ea3feb49c0c6697a837e7698e670137a5 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Thu, 16 Jul 2026 10:19:08 +0200 Subject: [PATCH 2/5] feat: add multicluster object count KPI to KVM metrics Signed-off-by: Markus Wieland --- helm/bundles/cortex-nova/templates/kpis.yaml | 20 +------------------ .../cortex-nova/templates/kpis_kvm.yaml | 19 ++++++++++++++++++ 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/helm/bundles/cortex-nova/templates/kpis.yaml b/helm/bundles/cortex-nova/templates/kpis.yaml index 59b28d2f7..5717cd62e 100644 --- a/helm/bundles/cortex-nova/templates/kpis.yaml +++ b/helm/bundles/cortex-nova/templates/kpis.yaml @@ -191,22 +191,4 @@ spec: - name: host-details - name: host-utilization description: | - This KPI tracks the capacity and utilization of VMware hosts in terms of CPU, RAM, and disk resources. ---- -apiVersion: cortex.cloud/v1alpha1 -kind: KPI -metadata: - name: multicluster-object-count -spec: - schedulingDomain: nova - impl: multicluster_object_count_kpi - opts: - gvks: - - cortex.cloud/v1alpha1/ReservationList - - cortex.cloud/v1alpha1/CommittedResourceList - - kvm.cloud.sap/v1/HypervisorList - description: | - This KPI reports the number of objects of each configured GVK per remote - cluster, labelled by the cluster's availability zone. It uses - PartialObjectMetadataList so only object metadata is fetched, making it - efficient even for large object counts. \ No newline at end of file + This KPI tracks the capacity and utilization of VMware hosts in terms of CPU, RAM, and disk resources. \ No newline at end of file diff --git a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml index 10ff5c45f..a7370de76 100644 --- a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml @@ -41,4 +41,23 @@ spec: - name: nova-flavors - name: identity-projects - name: identity-domains +--- +apiVersion: cortex.cloud/v1alpha1 +kind: KPI +metadata: + name: multicluster-object-count +spec: + schedulingDomain: nova + impl: multicluster_object_count_kpi + opts: + gvks: + - cortex.cloud/v1alpha1/HistoryList + - cortex.cloud/v1alpha1/ReservationList + - cortex.cloud/v1alpha1/CommittedResourceList + - kvm.cloud.sap/v1/HypervisorList + description: | + This KPI reports the number of objects of each configured GVK per remote + cluster, labelled by the configured cluster labels. It uses + PartialObjectMetadataList so only object metadata is fetched, making it + efficient even for large object counts. {{- end }} \ No newline at end of file From 8f869844878808fcc61cf24d67c54a89b93ee48d Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Thu, 16 Jul 2026 10:23:15 +0200 Subject: [PATCH 3/5] fix: remove unused claude artifact Signed-off-by: Markus Wieland --- pkg/multicluster/testing/client.go | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 pkg/multicluster/testing/client.go diff --git a/pkg/multicluster/testing/client.go b/pkg/multicluster/testing/client.go deleted file mode 100644 index d63105708..000000000 --- a/pkg/multicluster/testing/client.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -// Package multiclustertest provides helpers for building *multicluster.Client -// instances in unit tests without a real Kubernetes API server. -package multiclustertest From 8706f6ff6b963034454c62dd06919aa1b118b6d3 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Thu, 16 Jul 2026 10:33:01 +0200 Subject: [PATCH 4/5] feat: add is_home label Signed-off-by: Markus Wieland --- .../multicluster_object_count_kpi.go | 10 +++++-- .../multicluster_object_count_kpi_test.go | 30 +++++++++++++------ pkg/multicluster/client.go | 13 ++++---- pkg/multicluster/client_test.go | 8 ++++- 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go index a8b15ab45..52ccf192b 100644 --- a/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go +++ b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go @@ -70,7 +70,7 @@ func (k *MulticlusterObjectCountKPI) Init(_ *db.DB, c client.Client, opts conf.R } } } - varLabels := append([]string{"group", "version", "kind"}, labelKeys...) + varLabels := append([]string{"group", "version", "kind", "is_home"}, labelKeys...) desc := prometheus.NewDesc( "cortex_multicluster_object_count", "Number of objects of a given GVK per cluster", @@ -98,8 +98,12 @@ func (k *MulticlusterObjectCountKPI) Collect(ch chan<- prometheus.Metric) { continue } for _, c := range counts { - labelVals := make([]string, 0, 3+len(d.labelKeys)) - labelVals = append(labelVals, d.gvk.Group, d.gvk.Version, d.gvk.Kind) + isHome := "false" + if c.IsHome { + isHome = "true" + } + labelVals := make([]string, 0, 4+len(d.labelKeys)) + labelVals = append(labelVals, d.gvk.Group, d.gvk.Version, d.gvk.Kind, isHome) for _, key := range d.labelKeys { labelVals = append(labelVals, labelValueForSnakeKey(key, c.Labels)) } diff --git a/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi_test.go b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi_test.go index 5eff691b6..fc3335dec 100644 --- a/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi_test.go +++ b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi_test.go @@ -168,12 +168,18 @@ func TestMulticlusterObjectCountKPI_Collect(t *testing.T) { if err := m.Write(&metric); err != nil { t.Fatalf("failed to write metric: %v", err) } - var az string + var az, isHome string for _, lp := range metric.Label { - if lp.GetName() == "availability_zone" { + switch lp.GetName() { + case "availability_zone": az = lp.GetValue() + case "is_home": + isHome = lp.GetValue() } } + if isHome != "false" { + t.Errorf("az %q: expected is_home=false for remote cluster, got %q", az, isHome) + } byAZ[az] = metric.Gauge.GetValue() } @@ -194,7 +200,6 @@ func TestMulticlusterObjectCountKPI_Collect_HomeCluster(t *testing.T) { home := newFakeCluster(scheme, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-home", Namespace: "default"}}, ) - mcl := newFakeMCLClient(scheme, home, []schema.GroupVersionKind{cmListGVK}, nil) kpi := &MulticlusterObjectCountKPI{} @@ -208,20 +213,27 @@ func TestMulticlusterObjectCountKPI_Collect_HomeCluster(t *testing.T) { close(ch) var collected int - var count float64 for m := range ch { collected++ var metric dto.Metric if err := m.Write(&metric); err != nil { t.Fatalf("failed to write metric: %v", err) } - count = metric.Gauge.GetValue() + var isHome string + for _, lp := range metric.Label { + if lp.GetName() == "is_home" { + isHome = lp.GetValue() + } + } + if isHome != "true" { + t.Errorf("expected is_home=true for home cluster, got %q", isHome) + } + if got := metric.Gauge.GetValue(); got != 1 { + t.Errorf("expected count 1, got %g", got) + } } if collected != 1 { - t.Fatalf("expected 1 metric (home cluster), got %d", collected) - } - if count != 1 { - t.Errorf("expected count 1, got %g", count) + t.Fatalf("expected 1 metric, got %d", collected) } } diff --git a/pkg/multicluster/client.go b/pkg/multicluster/client.go index eff768947..604921799 100644 --- a/pkg/multicluster/client.go +++ b/pkg/multicluster/client.go @@ -476,17 +476,19 @@ func (c *Client) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts } // ClusterObjectCount is one entry in the result of CountPerClusterByGVK. -// Labels holds the routing labels for the cluster (nil for the home cluster). +// Labels holds the routing labels for the cluster. IsHome is true for the home +// cluster, which has no routing labels. type ClusterObjectCount struct { Labels map[string]string Count int + IsHome bool } // CountPerClusterByGVK returns the number of objects of the given GVK in each // configured cluster. It uses PartialObjectMetadataList so only object metadata // crosses the wire — no spec or status — making it efficient even for large // object counts. Clusters that return an error are logged and skipped (same -// policy as List). The home cluster is included with nil Labels. +// policy as List). The home cluster is included with IsHome set to true. func (c *Client) CountPerClusterByGVK(ctx context.Context, gvk schema.GroupVersionKind, opts ...client.ListOption) ([]ClusterObjectCount, error) { log := ctrl.LoggerFrom(ctx) @@ -499,14 +501,15 @@ func (c *Client) CountPerClusterByGVK(ctx context.Context, gvk schema.GroupVersi } type clusterEntry struct { cl cluster.Cluster - labels map[string]string // nil for home + labels map[string]string + isHome bool } entries := make([]clusterEntry, 0, len(remotes)+1) for _, r := range remotes { entries = append(entries, clusterEntry{cl: r.cluster, labels: maps.Clone(r.labels)}) } if isHome && c.HomeCluster != nil { - entries = append(entries, clusterEntry{cl: c.HomeCluster, labels: nil}) + entries = append(entries, clusterEntry{cl: c.HomeCluster, isHome: true}) } c.remoteClustersMu.RUnlock() @@ -523,7 +526,7 @@ func (c *Client) CountPerClusterByGVK(ctx context.Context, gvk schema.GroupVersi "gvk", gvk, "host", e.cl.GetConfig().Host) continue } - results = append(results, ClusterObjectCount{Labels: e.labels, Count: len(partialList.Items)}) + results = append(results, ClusterObjectCount{Labels: e.labels, Count: len(partialList.Items), IsHome: e.isHome}) } return results, nil } diff --git a/pkg/multicluster/client_test.go b/pkg/multicluster/client_test.go index 6191109d3..bcf90e325 100644 --- a/pkg/multicluster/client_test.go +++ b/pkg/multicluster/client_test.go @@ -1757,6 +1757,9 @@ func TestClient_CountPerClusterByGVK(t *testing.T) { } byAZ := map[string]int{} for _, r := range results { + if r.IsHome { + t.Errorf("expected IsHome=false for remote cluster %v", r.Labels) + } byAZ[r.Labels["availabilityZone"]] = r.Count } if byAZ["az-1"] != 2 { @@ -1767,7 +1770,7 @@ func TestClient_CountPerClusterByGVK(t *testing.T) { } }) - t.Run("includes home cluster with nil labels", func(t *testing.T) { + t.Run("includes home cluster with IsHome set", func(t *testing.T) { home := newFakeCluster(scheme, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-home", Namespace: "default"}}, ) @@ -1783,6 +1786,9 @@ func TestClient_CountPerClusterByGVK(t *testing.T) { if len(results) != 1 { t.Fatalf("expected 1 result, got %d", len(results)) } + if !results[0].IsHome { + t.Errorf("expected IsHome=true for home cluster") + } if results[0].Labels != nil { t.Errorf("expected nil labels for home cluster, got %v", results[0].Labels) } From 1d2393d174a4874834c9d170b8ad7a7f16307090 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Thu, 16 Jul 2026 10:34:38 +0200 Subject: [PATCH 5/5] chore: add comments explaining the mcl remote cluster label to metric label mapping Signed-off-by: Markus Wieland --- .../deployment/multicluster_object_count_kpi.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go index 52ccf192b..b1c4a58a0 100644 --- a/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go +++ b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go @@ -58,7 +58,11 @@ func (k *MulticlusterObjectCountKPI) Init(_ *db.DB, c client.Client, opts conf.R return fmt.Errorf("invalid GVK %q: %w", raw, err) } routeLabels := mcl.ConfiguredRouteLabels(gvk) - // Collect unique label keys across all clusters (stable order via insertion). + // Each remote cluster is registered with a routing label map (e.g. + // {"availabilityZone": "eu-de-1a"}). We collect the union of all key names + // across every remote cluster and snake_case them to produce Prometheus label + // names (availabilityZone → availability_zone). These become variable labels + // on the descriptor so each cluster gets its own time series. keySet := map[string]bool{} var labelKeys []string for _, lm := range routeLabels { @@ -70,6 +74,9 @@ func (k *MulticlusterObjectCountKPI) Init(_ *db.DB, c client.Client, opts conf.R } } } + // Fixed labels: group/version/kind identify the resource type; is_home + // distinguishes the home cluster (no routing labels) from remote clusters. + // The routing label keys follow (e.g. availability_zone). varLabels := append([]string{"group", "version", "kind", "is_home"}, labelKeys...) desc := prometheus.NewDesc( "cortex_multicluster_object_count", @@ -102,6 +109,11 @@ func (k *MulticlusterObjectCountKPI) Collect(ch chan<- prometheus.Metric) { if c.IsHome { isHome = "true" } + // Label values must match the descriptor order: group, version, kind, + // is_home, then one value per routing label key. For remote clusters the + // routing label value comes from the cluster's registration labels (e.g. + // Labels["availabilityZone"] → label value for "availability_zone"). The + // home cluster has no routing labels, so those positions are empty strings. labelVals := make([]string, 0, 4+len(d.labelKeys)) labelVals = append(labelVals, d.gvk.Group, d.gvk.Version, d.gvk.Kind, isHome) for _, key := range d.labelKeys {