OLS-3799 Add wait-for-rhokp init container to app-server deployment

internal/controller/appserver/assets_test.go

@@ -1158,6 +1158,7 @@ var _ = Describe("App server assets", func() {
 
 			Expect(deployment.Spec.Template.Spec.InitContainers).To(ConsistOf(
 				utils.GeneratePostgresWaitInitContainer(testReconcilerInstance.GetPostgresImage()),
+				utils.GenerateRHOKPWaitInitContainer(testReconcilerInstance.GetAppServerImage(), testReconcilerInstance.GetNamespace()),
 				corev1.Container{
 					Name:    "rag-0",
 					Image:   "rag-ocp-product-docs:4.19",

internal/controller/appserver/deployment.go

@@ -443,6 +443,9 @@ func GenerateOLSDeployment(r reconciler.Reconciler, cr *olsv1alpha1.OLSConfig) (
 
 	initContainers := []corev1.Container{}
 	initContainers = append(initContainers, utils.GeneratePostgresWaitInitContainer(r.GetPostgresImage()))
+	if !cr.Spec.OLSConfig.ByokRAGOnly {
+		initContainers = append(initContainers, utils.GenerateRHOKPWaitInitContainer(r.GetAppServerImage(), r.GetNamespace()))
+	}
 	if len(cr.Spec.OLSConfig.RAG) > 0 {
 		ragInitContainers := GenerateRAGInitContainers(cr)
 		initContainers = append(initContainers, ragInitContainers...)

internal/controller/appserver/deployment_test.go

@@ -761,6 +761,34 @@ var _ = Describe("App server deployment generation", func() {
 			}
 		})
 
+		It("should include wait-for-rhokp init container when ByokRAGOnly is false", func() {
+			cr.Spec.OLSConfig.ByokRAGOnly = false
+			dep, err := GenerateOLSDeployment(testReconcilerInstance, cr)
+			Expect(err).NotTo(HaveOccurred())
+
+			var found bool
+			for _, ic := range dep.Spec.Template.Spec.InitContainers {
+				if ic.Name == utils.RHOKPWaitInitContainerName {
+					found = true
+					Expect(ic.Image).To(Equal(testReconcilerInstance.GetAppServerImage()))
+					Expect(ic.VolumeMounts).ToNot(BeEmpty())
+					break
+				}
+			}
+			Expect(found).To(BeTrue(), "wait-for-rhokp init container should be present when ByokRAGOnly is false")
+		})
+
+		It("should not include wait-for-rhokp init container when ByokRAGOnly is true", func() {
+			cr.Spec.OLSConfig.ByokRAGOnly = true
+			dep, err := GenerateOLSDeployment(testReconcilerInstance, cr)
+			Expect(err).NotTo(HaveOccurred())
+
+			for _, ic := range dep.Spec.Template.Spec.InitContainers {
+				Expect(ic.Name).NotTo(Equal(utils.RHOKPWaitInitContainerName),
+					"wait-for-rhokp init container should not be present when ByokRAGOnly is true")
+			}
+		})
+
 		It("should add OLS_ROSA_PRODUCT when configured on the reconciler", func() {
 			cr.Spec.OLSConfig.IntrospectionEnabled = utils.BoolPtr(false)
 			cr.Spec.OLSConfig.UserDataCollection = olsv1alpha1.UserDataCollectionSpec{

internal/controller/utils/constants.go

@@ -209,6 +209,8 @@ const (
 	PostgresDeploymentName = "lightspeed-postgres-server"
 	// PostgresWaitInitContainerName is the name of the init container that waits for Postgres to accept connections
 	PostgresWaitInitContainerName = "wait-for-postgres"
+	// RHOKPWaitInitContainerName is the name of the init container that waits for RHOKP/Solr to be reachable
+	RHOKPWaitInitContainerName = "wait-for-rhokp"
 	// PostgresSecretKeyName is the name of the key holding Postgres server secret
 	PostgresSecretKeyName = "password"
 	// PostgresDefaultUser is the default user name for postgres

internal/controller/utils/rhokp_wait.go

@@ -0,0 +1,92 @@
+package utils
+
+import (
+	"fmt"
+	"path"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+// RHOKPWaitMaxSeconds is the maximum time the wait-for-rhokp init container
+// will poll before giving up.  Matches the RHOKP startup-probe budget
+// (20 s initial-delay + 34 failures × 10 s period ≈ 360 s).
+const RHOKPWaitMaxSeconds = 360
+
+// GenerateRHOKPWaitInitContainer returns an init container that blocks until the
+// standalone RHOKP/Solr instance is reachable on its HTTPS endpoint.
+//
+// Without this gate the app-server's SolrHybridSearch initializer may exhaust
+// its 120 s retry budget before RHOKP is ready, permanently caching a nil
+// reference for product-docs RAG (OLS-3799).
+//
+// The container uses the app-server image (which ships curl) and polls the
+// Solr admin/ping endpoint.  The RHOKP CA certificate volume must already
+// be defined on the pod (AppRHOKPCACertVolumeName).
+func GenerateRHOKPWaitInitContainer(image, namespace string) corev1.Container {
+	rhokpURL := RHOKPServiceURL(namespace) + RHOOKPReadinessHTTPPath
+
+	caPath := path.Join(OLSAppCertsMountRoot, AppRHOKPCACertDir, AppRHOKPCACertFile)
+
+	script := fmt.Sprintf(`
+if ! command -v curl >/dev/null 2>&1; then
+  echo "wait-for-rhokp: curl not found in image" >&2
+  exit 1
+fi
+
+sleep_sec=1
+max_sleep=30
+start=$(date +%%s)
+max_elapsed=%d
+url="%s"
+ca="%s"
+
+backoff() {
+  sleep "$sleep_sec"
+  sleep_sec=$((sleep_sec * 2))
+  [ "$sleep_sec" -gt "$max_sleep" ] && sleep_sec="$max_sleep"
+}
+
+while true; do
+  now=$(date +%%s)
+  elapsed=$((now - start))
+  if [ "$elapsed" -ge "$max_elapsed" ]; then
+    echo "wait-for-rhokp: timed out after ${max_elapsed}s" >&2
+    exit 1
+  fi
+
+  if curl -sf --cacert "$ca" --max-time 5 "$url" >/dev/null 2>&1; then
+    echo "wait-for-rhokp: RHOKP/Solr is reachable"
+    exit 0
+  fi
+
+  echo "wait-for-rhokp: not ready yet (elapsed=${elapsed}s)" >&2
+  backoff
+done
+`, RHOKPWaitMaxSeconds, rhokpURL, caPath)
+
+	return corev1.Container{
+		Name:            RHOKPWaitInitContainerName,
+		Image:           image,
+		ImagePullPolicy: corev1.PullIfNotPresent,
+		Command:         []string{"/bin/sh", "-c", script},
+		VolumeMounts: []corev1.VolumeMount{
+			{
+				Name:      AppRHOKPCACertVolumeName,
+				MountPath: path.Join(OLSAppCertsMountRoot, AppRHOKPCACertDir),
+				ReadOnly:  true,
+			},
+		},
+		SecurityContext: RestrictedContainerSecurityContext(),
+		Resources: corev1.ResourceRequirements{
+			Requests: corev1.ResourceList{
+				corev1.ResourceCPU:    resource.MustParse("10m"),
+				corev1.ResourceMemory: resource.MustParse("32Mi"),
+			},
+			Limits: corev1.ResourceList{
+				corev1.ResourceCPU:    resource.MustParse("100m"),
+				corev1.ResourceMemory: resource.MustParse("64Mi"),
+			},
+		},
+	}
+}

internal/controller/utils/rhokp_wait_test.go

@@ -0,0 +1,79 @@
+package utils
+
+import (
+	"fmt"
+	"path"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+var _ = Describe("RHOKP wait", func() {
+
+	Describe("GenerateRHOKPWaitInitContainer", func() {
+		const testNS = "openshift-lightspeed"
+
+		It("uses the provided image and generates correct container spec", func() {
+			c := GenerateRHOKPWaitInitContainer(OLSAppServerImageDefault, testNS)
+			Expect(c.Name).To(Equal(RHOKPWaitInitContainerName))
+			Expect(c.Image).To(Equal(OLSAppServerImageDefault))
+			Expect(c.SecurityContext).To(Equal(RestrictedContainerSecurityContext()))
+			Expect(c.Resources.Requests).To(HaveKey(corev1.ResourceCPU))
+			Expect(c.Resources.Requests).To(HaveKey(corev1.ResourceMemory))
+			Expect(c.Resources.Requests[corev1.ResourceCPU]).To(Equal(resource.MustParse("10m")))
+			Expect(c.Resources.Requests[corev1.ResourceMemory]).To(Equal(resource.MustParse("32Mi")))
+			Expect(c.Resources.Limits[corev1.ResourceCPU]).To(Equal(resource.MustParse("100m")))
+			Expect(c.Resources.Limits[corev1.ResourceMemory]).To(Equal(resource.MustParse("64Mi")))
+			Expect(c.Command).To(HaveLen(3))
+		})
+
+		It("mounts the RHOKP CA certificate volume", func() {
+			c := GenerateRHOKPWaitInitContainer(OLSAppServerImageDefault, testNS)
+			Expect(c.VolumeMounts).To(HaveLen(1))
+			Expect(c.VolumeMounts[0].Name).To(Equal(AppRHOKPCACertVolumeName))
+			Expect(c.VolumeMounts[0].MountPath).To(Equal(path.Join(OLSAppCertsMountRoot, AppRHOKPCACertDir)))
+			Expect(c.VolumeMounts[0].ReadOnly).To(BeTrue())
+		})
+
+		It("contains curl-based readiness check using admin/ping endpoint", func() {
+			c := GenerateRHOKPWaitInitContainer(OLSAppServerImageDefault, testNS)
+			script := c.Command[2]
+
+			By("using curl for connectivity check")
+			Expect(script).To(ContainSubstring("curl"))
+			Expect(script).To(ContainSubstring("--cacert"))
+
+			By("targeting the RHOKP admin/ping endpoint")
+			expectedURL := RHOKPServiceURL(testNS) + RHOOKPReadinessHTTPPath
+			Expect(script).To(ContainSubstring(expectedURL))
+		})
+
+		It("guards against missing curl binary", func() {
+			c := GenerateRHOKPWaitInitContainer(OLSAppServerImageDefault, testNS)
+			script := c.Command[2]
+			Expect(script).To(ContainSubstring("command -v curl"))
+			Expect(script).To(ContainSubstring("curl not found in image"))
+		})
+
+		It("has timeout and backoff logic", func() {
+			c := GenerateRHOKPWaitInitContainer(OLSAppServerImageDefault, testNS)
+			script := c.Command[2]
+
+			Expect(script).To(ContainSubstring(fmt.Sprintf("max_elapsed=%d", RHOKPWaitMaxSeconds)))
+			Expect(script).To(ContainSubstring("backoff"))
+			Expect(script).To(ContainSubstring("timed out"))
+			Expect(script).To(ContainSubstring("not ready yet"))
+			Expect(script).To(ContainSubstring("RHOKP/Solr is reachable"))
+		})
+
+		It("uses the correct namespace in the service URL", func() {
+			customNS := "my-custom-namespace"
+			c := GenerateRHOKPWaitInitContainer(OLSAppServerImageDefault, customNS)
+			script := c.Command[2]
+			Expect(script).To(ContainSubstring(fmt.Sprintf("%s.%s.svc", RHOKPServiceName, customNS)))
+		})
+	})
+})