diff --git a/helm/bundles/cortex-nova/templates/alerts.yaml b/helm/bundles/cortex-nova/templates/alerts.yaml index 8db8bfb2e..6430f1c26 100644 --- a/helm/bundles/cortex-nova/templates/alerts.yaml +++ b/helm/bundles/cortex-nova/templates/alerts.yaml @@ -588,6 +588,28 @@ spec: This may mean hypervisors in that AZ are fully utilized for the corresponding flavor group and no further committed resources can be placed there. + - alert: CortexNovaCommittedResourceCapacityNotReady + expr: | + cortex_committed_resource_capacity_ready{service="cortex-nova-metrics"} == 0 + for: 10m + labels: + context: committed-resource-capacity + dashboard: cortex-status-dashboard/cortex-status-dashboard + service: cortex + severity: warning + support_group: workload-management + playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity + annotations: + summary: "FlavorGroupCapacity for {{ "{{" }} $labels.flavor_group {{ "}}" }} in {{ "{{" }} $labels.az {{ "}}" }} has been not-ready for >10 minutes" + description: > + The FlavorGroupCapacity CRD for flavor group {{ "{{" }} $labels.flavor_group {{ "}}" }} + in availability zone {{ "{{" }} $labels.az {{ "}}" }} has had Ready=False for more than + 10 minutes. The capacity controller failed to complete all scheduler probes for this + (flavor group x AZ) pair. The capacity API (report-capacity) is serving stale total + capacity values for this group without usage data — Limes receives capacity but no + usage, causing silent staleness. Investigate the capacity controller logs for probe + errors and check scheduler availability. + # Committed Resource Usage API - alert: CortexNovaCommittedResourceUsageErrors expr: | diff --git a/internal/scheduling/reservations/capacity/controller_test.go b/internal/scheduling/reservations/capacity/controller_test.go index d01110af0..4bdd1493f 100644 --- a/internal/scheduling/reservations/capacity/controller_test.go +++ b/internal/scheduling/reservations/capacity/controller_test.go @@ -81,7 +81,7 @@ func newFlavorGroupKnowledge(t *testing.T, groupName string, smallestMemoryMB ui } // newHypervisor creates a Hypervisor CRD with a topology AZ label, memory and CPU effective capacity. -func newHypervisor(name, az string, memoryBytes int64, instanceIDs ...string) *hv1.Hypervisor { +func newHypervisor(name, az string, memoryBytes int64) *hv1.Hypervisor { hv := &hv1.Hypervisor{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -96,9 +96,6 @@ func newHypervisor(name, az string, memoryBytes int64, instanceIDs ...string) *h hv1.ResourceCPU: *cpuQty, } } - for _, id := range instanceIDs { - hv.Status.Instances = append(hv.Status.Instances, hv1.Instance{ID: id}) - } return hv } @@ -200,7 +197,7 @@ func TestReconcileAZ_CreatesCRD(t *testing.T) { ) scheme := newTestScheme(t) - hv := newHypervisor("host-1", az, memBytes, "vm1") + hv := newHypervisor("host-1", az, memBytes) knowledge := newFlavorGroupKnowledge(t, groupName, memMB) fakeClient := fake.NewClientBuilder(). @@ -842,7 +839,7 @@ func TestComputeVMUsage_ZerosOutWhenAllVMsRemoved(t *testing.T) { ) scheme := newTestScheme(t) - hv := newHypervisor("host-1", az, memBytes, "vm1") + hv := newHypervisor("host-1", az, memBytes) knowledge := newFlavorGroupKnowledge(t, groupName, memMB) // Pre-create CRD with non-zero RunningInstances to simulate prior state. diff --git a/internal/scheduling/reservations/capacity/metrics.go b/internal/scheduling/reservations/capacity/metrics.go index 27293e0eb..375ddfe74 100644 --- a/internal/scheduling/reservations/capacity/metrics.go +++ b/internal/scheduling/reservations/capacity/metrics.go @@ -9,6 +9,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/prometheus/client_golang/prometheus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -31,6 +32,7 @@ type Monitor struct { freeCapacityGiB *prometheus.GaugeVec exclusivelyFreeCapacityGiB *prometheus.GaugeVec exclusivelyFreeSlots *prometheus.GaugeVec + readyGauge *prometheus.GaugeVec } // NewMonitor creates a new Monitor that reads FlavorGroupCapacity CRDs. @@ -77,6 +79,10 @@ func NewMonitor(c client.Client) Monitor { Name: "cortex_committed_resource_exclusively_free_slots", Help: "Number of smallest-flavor VM slots available after the cross-group capacity split.", }, capacityFlavorLabels), + readyGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "cortex_committed_resource_capacity_ready", + Help: "1 if the FlavorGroupCapacity CRD is Ready (all scheduler probes succeeded), 0 otherwise.", + }, capacityLabels), } } @@ -92,6 +98,7 @@ func (m *Monitor) Describe(ch chan<- *prometheus.Desc) { m.freeCapacityGiB.Describe(ch) m.exclusivelyFreeCapacityGiB.Describe(ch) m.exclusivelyFreeSlots.Describe(ch) + m.readyGauge.Describe(ch) } // Collect implements prometheus.Collector — lists all FlavorGroupCapacity CRDs and exports gauges. @@ -115,6 +122,7 @@ func (m *Monitor) Collect(ch chan<- prometheus.Metric) { m.freeCapacityGiB.Reset() m.exclusivelyFreeCapacityGiB.Reset() m.exclusivelyFreeSlots.Reset() + m.readyGauge.Reset() for _, crd := range list.Items { groupAZLabels := prometheus.Labels{ @@ -138,6 +146,15 @@ func (m *Monitor) Collect(ch chan<- prometheus.Metric) { } m.exclusivelyFreeSlots.With(groupAZFlavorLabels).Set(float64(crd.Status.ExclusivelyFreeSlots)) + readyVal := 0.0 + for _, cond := range crd.Status.Conditions { + if cond.Type == v1alpha1.FlavorGroupCapacityConditionReady && cond.Status == metav1.ConditionTrue { + readyVal = 1.0 + break + } + } + m.readyGauge.With(groupAZLabels).Set(readyVal) + for _, f := range crd.Status.Flavors { flavorLabels := prometheus.Labels{ "flavor_group": crd.Spec.FlavorGroup, @@ -161,4 +178,5 @@ func (m *Monitor) Collect(ch chan<- prometheus.Metric) { m.freeCapacityGiB.Collect(ch) m.exclusivelyFreeCapacityGiB.Collect(ch) m.exclusivelyFreeSlots.Collect(ch) + m.readyGauge.Collect(ch) }