feat(capacity): improve FlavorGroupCapacity CRD observability

helm/bundles/cortex-nova/templates/alerts.yaml

@@ -588,6 +588,28 @@ spec:
           This may mean hypervisors in that AZ are fully utilized for the corresponding
           flavor group and no further committed resources can be placed there.
 
+    - alert: CortexNovaCommittedResourceCapacityNotReady
+      expr: |
+        cortex_committed_resource_capacity_ready{service="cortex-nova-metrics"} == 0
+      for: 10m
+      labels:
+        context: committed-resource-capacity
+        dashboard: cortex-status-dashboard/cortex-status-dashboard
+        service: cortex
+        severity: warning
+        support_group: workload-management
+        playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity
+      annotations:
+        summary: "FlavorGroupCapacity for {{ "{{" }} $labels.flavor_group {{ "}}" }} in {{ "{{" }} $labels.az {{ "}}" }} has been not-ready for >10 minutes"
+        description: >
+          The FlavorGroupCapacity CRD for flavor group {{ "{{" }} $labels.flavor_group {{ "}}" }}
+          in availability zone {{ "{{" }} $labels.az {{ "}}" }} has had Ready=False for more than
+          10 minutes. The capacity controller failed to complete all scheduler probes for this
+          (flavor group x AZ) pair. The capacity API (report-capacity) is serving stale total
+          capacity values for this group without usage data — Limes receives capacity but no
+          usage, causing silent staleness. Investigate the capacity controller logs for probe
+          errors and check scheduler availability.
+
     # Committed Resource Usage API
     - alert: CortexNovaCommittedResourceUsageErrors
       expr: |

internal/scheduling/reservations/capacity/controller_test.go

@@ -81,7 +81,7 @@ func newFlavorGroupKnowledge(t *testing.T, groupName string, smallestMemoryMB ui
 }
 
 // newHypervisor creates a Hypervisor CRD with a topology AZ label, memory and CPU effective capacity.
-func newHypervisor(name, az string, memoryBytes int64, instanceIDs ...string) *hv1.Hypervisor {
+func newHypervisor(name, az string, memoryBytes int64) *hv1.Hypervisor {
 	hv := &hv1.Hypervisor{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   name,
@@ -96,9 +96,6 @@ func newHypervisor(name, az string, memoryBytes int64, instanceIDs ...string) *h
 			hv1.ResourceCPU:    *cpuQty,
 		}
 	}
-	for _, id := range instanceIDs {
-		hv.Status.Instances = append(hv.Status.Instances, hv1.Instance{ID: id})
-	}
 	return hv
 }
 
@@ -200,7 +197,7 @@ func TestReconcileAZ_CreatesCRD(t *testing.T) {
 	)
 
 	scheme := newTestScheme(t)
-	hv := newHypervisor("host-1", az, memBytes, "vm1")
+	hv := newHypervisor("host-1", az, memBytes)
 	knowledge := newFlavorGroupKnowledge(t, groupName, memMB)
 
 	fakeClient := fake.NewClientBuilder().
@@ -842,7 +839,7 @@ func TestComputeVMUsage_ZerosOutWhenAllVMsRemoved(t *testing.T) {
 	)
 
 	scheme := newTestScheme(t)
-	hv := newHypervisor("host-1", az, memBytes, "vm1")
+	hv := newHypervisor("host-1", az, memBytes)
 	knowledge := newFlavorGroupKnowledge(t, groupName, memMB)
 
 	// Pre-create CRD with non-zero RunningInstances to simulate prior state.

internal/scheduling/reservations/capacity/metrics.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/cobaltcore-dev/cortex/api/v1alpha1"
 	"github.com/prometheus/client_golang/prometheus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -31,6 +32,7 @@ type Monitor struct {
 	freeCapacityGiB            *prometheus.GaugeVec
 	exclusivelyFreeCapacityGiB *prometheus.GaugeVec
 	exclusivelyFreeSlots       *prometheus.GaugeVec
+	readyGauge                 *prometheus.GaugeVec
 }
 
 // NewMonitor creates a new Monitor that reads FlavorGroupCapacity CRDs.
@@ -77,6 +79,10 @@ func NewMonitor(c client.Client) Monitor {
 			Name: "cortex_committed_resource_exclusively_free_slots",
 			Help: "Number of smallest-flavor VM slots available after the cross-group capacity split.",
 		}, capacityFlavorLabels),
+		readyGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "cortex_committed_resource_capacity_ready",
+			Help: "1 if the FlavorGroupCapacity CRD is Ready (all scheduler probes succeeded), 0 otherwise.",
+		}, capacityLabels),
 	}
 }
 
@@ -92,6 +98,7 @@ func (m *Monitor) Describe(ch chan<- *prometheus.Desc) {
 	m.freeCapacityGiB.Describe(ch)
 	m.exclusivelyFreeCapacityGiB.Describe(ch)
 	m.exclusivelyFreeSlots.Describe(ch)
+	m.readyGauge.Describe(ch)
 }
 
 // Collect implements prometheus.Collector — lists all FlavorGroupCapacity CRDs and exports gauges.
@@ -115,6 +122,7 @@ func (m *Monitor) Collect(ch chan<- prometheus.Metric) {
 	m.freeCapacityGiB.Reset()
 	m.exclusivelyFreeCapacityGiB.Reset()
 	m.exclusivelyFreeSlots.Reset()
+	m.readyGauge.Reset()
 
 	for _, crd := range list.Items {
 		groupAZLabels := prometheus.Labels{
@@ -138,6 +146,15 @@ func (m *Monitor) Collect(ch chan<- prometheus.Metric) {
 		}
 		m.exclusivelyFreeSlots.With(groupAZFlavorLabels).Set(float64(crd.Status.ExclusivelyFreeSlots))
 
+		readyVal := 0.0
+		for _, cond := range crd.Status.Conditions {
+			if cond.Type == v1alpha1.FlavorGroupCapacityConditionReady && cond.Status == metav1.ConditionTrue {
+				readyVal = 1.0
+				break
+			}
+		}
+		m.readyGauge.With(groupAZLabels).Set(readyVal)
+
 		for _, f := range crd.Status.Flavors {
 			flavorLabels := prometheus.Labels{
 				"flavor_group": crd.Spec.FlavorGroup,
@@ -161,4 +178,5 @@ func (m *Monitor) Collect(ch chan<- prometheus.Metric) {
 	m.freeCapacityGiB.Collect(ch)
 	m.exclusivelyFreeCapacityGiB.Collect(ch)
 	m.exclusivelyFreeSlots.Collect(ch)
+	m.readyGauge.Collect(ch)
 }