Reference declarative agents by tag; digests only for sandbox agents

go/core/internal/controller/translator/agent/adk_api_translator.go

@@ -117,8 +117,12 @@ var DefaultImageConfig = ImageConfig{
 }
 
 // PythonADKImageDigest, PythonADKFullImageDigest, GoADKImageDigest, and GoADKFullImageDigest
-// are set at controller link time from the pushed runtime image manifest digests. The "full"
-// variants bundle the sandbox runtime (code execution / bash tools); the slim variants do not.
+// default to the pushed runtime image manifest digests baked in at controller link time, and
+// can be overridden at runtime via the --app[-full]-image-digest / --golang-adk[-full]-image-digest
+// flags (for mirrored registries that re-assign digests). They are only consulted for sandbox
+// agents — Substrate requires digest-pinned refs — while regular agents reference images by tag.
+// The "full" variants bundle the sandbox runtime (code execution / bash tools); the slim
+// variants do not.
 var PythonADKImageDigest string
 var PythonADKFullImageDigest string
 var GoADKImageDigest string

go/core/internal/controller/translator/agent/deployments.go

@@ -123,37 +123,53 @@ func validateExtraContainers(containers []corev1.Container) error {
 	return nil
 }
 
-func resolvePythonRuntimeImage(registry string, full bool) (string, error) {
+func resolvePythonRuntimeImage(registry string, full, pinDigest bool) (string, error) {
 	repo := DefaultImageConfig.Repository
 	digest := PythonADKImageDigest
 	imageLabel := "app"
 	if full {
 		digest = PythonADKFullImageDigest
 		imageLabel = "app-full"
 	}
-	if d := normalizeImageDigest(digest); d != "" {
-		return fmt.Sprintf("%s/%s@%s", registry, repo, d), nil
-	}
-	return "", fmt.Errorf(
-		"%s image digest is not set at link time; rebuild the controller after pushing agent runtime images",
-		imageLabel,
-	)
+	return resolveRuntimeImage(registry, repo, digest, imageLabel, full, pinDigest)
 }
 
-func resolveGoRuntimeImage(registry string, full bool) (string, error) {
+func resolveGoRuntimeImage(registry string, full, pinDigest bool) (string, error) {
 	repo := getRuntimeImageRepository(v1alpha2.DeclarativeRuntime_Go)
 	digest := GoADKImageDigest
 	imageLabel := "golang-adk"
 	if full {
 		digest = GoADKFullImageDigest
 		imageLabel = "golang-adk-full"
 	}
+	return resolveRuntimeImage(registry, repo, digest, imageLabel, full, pinDigest)
+}
+
+// resolveRuntimeImage builds the image reference for a declarative agent runtime.
+//
+// Regular agents get a tag reference (registry/repository:tag) so mirrored
+// registries that do not preserve upstream manifest digests still resolve
+// (https://github.com/kagent-dev/kagent/issues/2055). Full-variant images share
+// the repository and are published under "<tag>-full" (see APP_FULL_IMAGE_TAG /
+// GOLANG_ADK_FULL_IMAGE_TAG in the Makefile).
+//
+// Sandbox agents require pinDigest: Substrate ActorTemplate validation rejects
+// image refs without a digest, so those use the link-time (or flag-overridden)
+// runtime image digests.
+func resolveRuntimeImage(registry, repository, digest, imageLabel string, full, pinDigest bool) (string, error) {
+	if !pinDigest {
+		tag := DefaultImageConfig.Tag
+		if full {
+			tag += "-full"
+		}
+		return fmt.Sprintf("%s/%s:%s", registry, repository, tag), nil
+	}
 	if d := normalizeImageDigest(digest); d != "" {
-		return fmt.Sprintf("%s/%s@%s", registry, repo, d), nil
+		return fmt.Sprintf("%s/%s@%s", registry, repository, d), nil
 	}
 	return "", fmt.Errorf(
-		"%s image digest is not set at link time; rebuild the controller after pushing agent runtime images",
-		imageLabel,
+		"%s image digest is not set; rebuild the controller after pushing agent runtime images, or override it via --%s-image-digest",
+		imageLabel, imageLabel,
 	)
 }
 
@@ -190,16 +206,19 @@ func resolveInlineDeployment(agent v1alpha2.AgentObject, mdd *modelDeploymentDat
 
 	var image string
 	full := needsSRTSettings(agent, specRef.Sandbox)
+	// Substrate ActorTemplates reject tag refs, so sandbox agents pin by digest;
+	// everything else references by tag (resolvable in mirrored registries).
+	pinDigest := agent.GetWorkloadMode() == v1alpha2.WorkloadModeSandbox
 	switch runtime {
 	case v1alpha2.DeclarativeRuntime_Go:
 		var err error
-		image, err = resolveGoRuntimeImage(registry, full)
+		image, err = resolveGoRuntimeImage(registry, full, pinDigest)
 		if err != nil {
 			return nil, err
 		}
 	default:
 		var err error
-		image, err = resolvePythonRuntimeImage(registry, full)
+		image, err = resolvePythonRuntimeImage(registry, full, pinDigest)
 		if err != nil {
 			return nil, err
 		}

go/core/internal/controller/translator/agent/imageconfig_test.go

@@ -4,6 +4,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/kagent-dev/kagent/go/api/v1alpha2"
 )
 
 func TestImageConfigImage(t *testing.T) {
@@ -45,11 +47,11 @@ func TestResolveGoRuntimeImageWithDigest(t *testing.T) {
 	GoADKImageDigest = "sha256:go-base"
 	GoADKFullImageDigest = "sha256:go-full"
 
-	got, err := resolveGoRuntimeImage("localhost:5001", false)
+	got, err := resolveGoRuntimeImage("localhost:5001", false, true)
 	require.NoError(t, err)
 	require.Equal(t, "localhost:5001/kagent-dev/kagent/golang-adk@sha256:go-base", got)
 
-	got, err = resolveGoRuntimeImage("localhost:5001", true)
+	got, err = resolveGoRuntimeImage("localhost:5001", true, true)
 	require.NoError(t, err)
 	require.Equal(t, "localhost:5001/kagent-dev/kagent/golang-adk@sha256:go-full", got)
 }
@@ -64,11 +66,11 @@ func TestResolveGoRuntimeImageWithoutDigest(t *testing.T) {
 	GoADKImageDigest = ""
 	GoADKFullImageDigest = ""
 
-	_, err := resolveGoRuntimeImage("localhost:5001", false)
+	_, err := resolveGoRuntimeImage("localhost:5001", false, true)
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "golang-adk")
 
-	_, err = resolveGoRuntimeImage("localhost:5001", true)
+	_, err = resolveGoRuntimeImage("localhost:5001", true, true)
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "golang-adk-full")
 }
@@ -83,11 +85,11 @@ func TestResolvePythonRuntimeImageWithDigest(t *testing.T) {
 	PythonADKImageDigest = "sha256:app-digest"
 	PythonADKFullImageDigest = "sha256:app-full-digest"
 
-	got, err := resolvePythonRuntimeImage("ghcr.io", false)
+	got, err := resolvePythonRuntimeImage("ghcr.io", false, true)
 	require.NoError(t, err)
 	require.Equal(t, "ghcr.io/kagent-dev/kagent/app@sha256:app-digest", got)
 
-	gotFull, err := resolvePythonRuntimeImage("ghcr.io", true)
+	gotFull, err := resolvePythonRuntimeImage("ghcr.io", true, true)
 	require.NoError(t, err)
 	require.Equal(t, "ghcr.io/kagent-dev/kagent/app@sha256:app-full-digest", gotFull)
 }
@@ -99,7 +101,7 @@ func TestResolvePythonFullRuntimeImageWithoutDigest(t *testing.T) {
 	})
 	PythonADKFullImageDigest = ""
 
-	_, err := resolvePythonRuntimeImage("ghcr.io", true)
+	_, err := resolvePythonRuntimeImage("ghcr.io", true, true)
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "app-full")
 }
@@ -122,7 +124,60 @@ func TestResolvePythonRuntimeImageWithoutDigest(t *testing.T) {
 	})
 	PythonADKImageDigest = ""
 
-	_, err := resolvePythonRuntimeImage("ghcr.io", false)
+	_, err := resolvePythonRuntimeImage("ghcr.io", false, true)
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "app")
 }
+
+func TestResolveRuntimeImageByTag(t *testing.T) {
+	originalTag := DefaultImageConfig.Tag
+	t.Cleanup(func() { DefaultImageConfig.Tag = originalTag })
+	DefaultImageConfig.Tag = "v9.9.9"
+
+	got, err := resolvePythonRuntimeImage("my-registry.example.com", false, false)
+	require.NoError(t, err)
+	require.Equal(t, "my-registry.example.com/kagent-dev/kagent/app:v9.9.9", got)
+
+	got, err = resolvePythonRuntimeImage("my-registry.example.com", true, false)
+	require.NoError(t, err)
+	require.Equal(t, "my-registry.example.com/kagent-dev/kagent/app:v9.9.9-full", got)
+
+	got, err = resolveGoRuntimeImage("my-registry.example.com", false, false)
+	require.NoError(t, err)
+	require.Equal(t, "my-registry.example.com/kagent-dev/kagent/golang-adk:v9.9.9", got)
+
+	got, err = resolveGoRuntimeImage("my-registry.example.com", true, false)
+	require.NoError(t, err)
+	require.Equal(t, "my-registry.example.com/kagent-dev/kagent/golang-adk:v9.9.9-full", got)
+}
+
+func TestResolveRuntimeImageByTagIgnoresMissingDigest(t *testing.T) {
+	original := PythonADKImageDigest
+	t.Cleanup(func() { PythonADKImageDigest = original })
+	PythonADKImageDigest = ""
+
+	_, err := resolvePythonRuntimeImage("ghcr.io", false, false)
+	require.NoError(t, err)
+}
+
+func TestResolveInlineDeploymentImagePinning(t *testing.T) {
+	original := PythonADKImageDigest
+	t.Cleanup(func() { PythonADKImageDigest = original })
+	PythonADKImageDigest = "sha256:pin-test"
+
+	spec := v1alpha2.AgentSpec{
+		Type:        v1alpha2.AgentType_Declarative,
+		Declarative: &v1alpha2.DeclarativeAgentSpec{SystemMessage: "test", ModelConfig: "test-model"},
+	}
+
+	regular := &v1alpha2.Agent{Spec: spec}
+	dep, err := resolveInlineDeployment(regular, &modelDeploymentData{})
+	require.NoError(t, err)
+	require.NotContains(t, dep.Image, "@sha256:", "regular agents reference images by tag")
+	require.Contains(t, dep.Image, ":"+DefaultImageConfig.Tag)
+
+	sandbox := &v1alpha2.SandboxAgent{Spec: v1alpha2.SandboxAgentSpec{AgentSpec: spec}}
+	sdep, err := resolveInlineDeployment(sandbox, &modelDeploymentData{})
+	require.NoError(t, err)
+	require.Contains(t, sdep.Image, "@sha256:pin-test", "sandbox agents require digest-pinned images (Substrate rejects tag refs)")
+}

go/core/internal/controller/translator/agent/runtime_test.go

@@ -105,7 +105,8 @@ func TestRuntime_GoRuntime(t *testing.T) {
 	require.Len(t, deployment.Spec.Template.Spec.Containers, 1)
 	container := deployment.Spec.Template.Spec.Containers[0]
 	assert.Contains(t, container.Image, "golang-adk", "Image should use golang-adk repository")
-	assert.Contains(t, container.Image, "@sha256:test-go-base", "Go runtime should use digest-pinned golang-adk image")
+	assert.Contains(t, container.Image, ":"+translator.DefaultImageConfig.Tag, "Go runtime should reference the golang-adk image by tag")
+	assert.NotContains(t, container.Image, "@sha256:", "regular agents must not use digest-pinned images")
 
 	// Verify Go runtime readiness probe timings (fast startup)
 	require.NotNil(t, container.ReadinessProbe)
@@ -178,7 +179,8 @@ func TestRuntime_GoRuntimeWithSkillsUsesFullImageTag(t *testing.T) {
 	require.Len(t, deployment.Spec.Template.Spec.Containers, 1)
 	container := deployment.Spec.Template.Spec.Containers[0]
 	assert.Contains(t, container.Image, "golang-adk", "Image should use golang-adk repository")
-	assert.Contains(t, container.Image, "@sha256:test-go-full", "Go runtime with skills should use digest-pinned golang-adk-full image")
+	assert.Contains(t, container.Image, ":"+translator.DefaultImageConfig.Tag+"-full", "Go runtime with skills should reference the golang-adk full image by tag")
+	assert.NotContains(t, container.Image, "@sha256:", "regular agents must not use digest-pinned images")
 }
 
 func TestRuntime_PythonRuntime(t *testing.T) {
@@ -245,11 +247,12 @@ func TestRuntime_PythonRuntime(t *testing.T) {
 	}
 	require.NotNil(t, deployment, "Deployment should be in manifest")
 
-	// Verify container image uses digest-pinned app (Python ADK)
+	// Verify container image uses the tag-referenced app image (Python ADK)
 	require.Len(t, deployment.Spec.Template.Spec.Containers, 1)
 	container := deployment.Spec.Template.Spec.Containers[0]
-	assert.Contains(t, container.Image, "/app@", "Image should use app repository")
-	assert.Contains(t, container.Image, "@sha256:test-app", "Python runtime should use digest-pinned app image")
+	assert.Contains(t, container.Image, "/app:", "Image should use app repository")
+	assert.Contains(t, container.Image, ":"+translator.DefaultImageConfig.Tag, "Python runtime should reference the app image by tag")
+	assert.NotContains(t, container.Image, "@sha256:", "regular agents must not use digest-pinned images")
 
 	// Verify Python runtime readiness probe timings (slower startup)
 	require.NotNil(t, container.ReadinessProbe)
@@ -322,11 +325,12 @@ func TestRuntime_DefaultToPython(t *testing.T) {
 	}
 	require.NotNil(t, deployment, "Deployment should be in manifest")
 
-	// Verify container image uses digest-pinned app (Python ADK) by default
+	// Verify container image uses the tag-referenced app image (Python ADK) by default
 	require.Len(t, deployment.Spec.Template.Spec.Containers, 1)
 	container := deployment.Spec.Template.Spec.Containers[0]
-	assert.Contains(t, container.Image, "/app@", "Image should default to app repository")
-	assert.Contains(t, container.Image, "@sha256:test-app", "Default Python runtime should use digest-pinned app image")
+	assert.Contains(t, container.Image, "/app:", "Image should default to app repository")
+	assert.Contains(t, container.Image, ":"+translator.DefaultImageConfig.Tag, "Default Python runtime should reference the app image by tag")
+	assert.NotContains(t, container.Image, "@sha256:", "regular agents must not use digest-pinned images")
 
 	// Verify Python runtime readiness probe timings
 	require.NotNil(t, container.ReadinessProbe)
@@ -412,7 +416,8 @@ func TestRuntime_CustomRepositoryPath(t *testing.T) {
 	require.Len(t, deployment.Spec.Template.Spec.Containers, 1)
 	container := deployment.Spec.Template.Spec.Containers[0]
 	assert.Contains(t, container.Image, "my-registry.com/custom/golang-adk", "Image should use custom repository with golang-adk")
-	assert.Contains(t, container.Image, "@sha256:test-go-base", "Go runtime should use digest-pinned golang-adk image")
+	assert.Contains(t, container.Image, ":"+translator.DefaultImageConfig.Tag, "Go runtime should reference the golang-adk image by tag")
+	assert.NotContains(t, container.Image, "@sha256:", "regular agents must not use digest-pinned images")
 }
 
 func TestRuntime_CustomRepositoryPath_WithSkillsUsesFullTag(t *testing.T) {
@@ -485,5 +490,6 @@ func TestRuntime_CustomRepositoryPath_WithSkillsUsesFullTag(t *testing.T) {
 	require.Len(t, deployment.Spec.Template.Spec.Containers, 1)
 	container := deployment.Spec.Template.Spec.Containers[0]
 	assert.Contains(t, container.Image, "my-registry.com/custom/golang-adk", "Image should use custom repository with golang-adk")
-	assert.Contains(t, container.Image, "@sha256:test-go-full", "Go runtime with skills should use digest-pinned golang-adk-full image")
+	assert.Contains(t, container.Image, ":"+translator.DefaultImageConfig.Tag+"-full", "Go runtime with skills should reference the golang-adk full image by tag")
+	assert.NotContains(t, container.Image, "@sha256:", "regular agents must not use digest-pinned images")
 }

go/core/internal/controller/translator/agent/testdata/outputs/agent_with_a2a_config.json

@@ -186,7 +186,7 @@
                     "value": "http://kagent-controller.kagent:8083"
                   }
                 ],
-                "image": "ghcr.io/kagent-dev/kagent/app@sha256:test-app",
+                "image": "ghcr.io/kagent-dev/kagent/app:dev",
                 "imagePullPolicy": "IfNotPresent",
                 "name": "kagent",
                 "ports": [

go/core/internal/controller/translator/agent/testdata/outputs/agent_with_allowed_headers.json

@@ -196,7 +196,7 @@
                     "value": "http://kagent-controller.kagent:8083"
                   }
                 ],
-                "image": "ghcr.io/kagent-dev/kagent/app@sha256:test-app",
+                "image": "ghcr.io/kagent-dev/kagent/app:dev",
                 "imagePullPolicy": "IfNotPresent",
                 "name": "kagent",
                 "ports": [

go/core/internal/controller/translator/agent/testdata/outputs/agent_with_code.json

@@ -193,7 +193,7 @@
                     "value": "/config/srt-settings.json"
                   }
                 ],
-                "image": "ghcr.io/kagent-dev/kagent/app@sha256:test-app-full",
+                "image": "ghcr.io/kagent-dev/kagent/app:dev-full",
                 "imagePullPolicy": "IfNotPresent",
                 "name": "kagent",
                 "ports": [

go/core/internal/controller/translator/agent/testdata/outputs/agent_with_context_config.json

@@ -207,7 +207,7 @@
                     "value": "http://kagent-controller.kagent:8083"
                   }
                 ],
-                "image": "ghcr.io/kagent-dev/kagent/app@sha256:test-app",
+                "image": "ghcr.io/kagent-dev/kagent/app:dev",
                 "imagePullPolicy": "IfNotPresent",
                 "name": "kagent",
                 "ports": [

go/core/internal/controller/translator/agent/testdata/outputs/agent_with_cross_namespace_tools.json

@@ -202,7 +202,7 @@
                     "value": "http://kagent-controller.kagent:8083"
                   }
                 ],
-                "image": "ghcr.io/kagent-dev/kagent/app@sha256:test-app",
+                "image": "ghcr.io/kagent-dev/kagent/app:dev",
                 "imagePullPolicy": "IfNotPresent",
                 "name": "kagent",
                 "ports": [

go/core/internal/controller/translator/agent/testdata/outputs/agent_with_custom_sa.json

@@ -155,7 +155,7 @@
                     "value": "http://kagent-controller.kagent:8083"
                   }
                 ],
-                "image": "ghcr.io/kagent-dev/kagent/app@sha256:test-app",
+                "image": "ghcr.io/kagent-dev/kagent/app:dev",
                 "imagePullPolicy": "IfNotPresent",
                 "name": "kagent",
                 "ports": [

go/core/internal/controller/translator/agent/testdata/outputs/agent_with_default_sa.json

@@ -155,7 +155,7 @@
                     "value": "http://kagent-controller.kagent:8083"
                   }
                 ],
-                "image": "ghcr.io/kagent-dev/kagent/app@sha256:test-app",
+                "image": "ghcr.io/kagent-dev/kagent/app:dev",
                 "imagePullPolicy": "IfNotPresent",
                 "name": "kagent",
                 "ports": [

go/core/internal/controller/translator/agent/testdata/outputs/agent_with_embedding_provider.json

@@ -202,7 +202,7 @@
                     "value": "http://kagent-controller.kagent:8083"
                   }
                 ],
-                "image": "ghcr.io/kagent-dev/kagent/app@sha256:test-app",
+                "image": "ghcr.io/kagent-dev/kagent/app:dev",
                 "imagePullPolicy": "IfNotPresent",
                 "name": "kagent",
                 "ports": [

go/core/internal/controller/translator/agent/testdata/outputs/agent_with_extra_containers.json

@@ -180,7 +180,7 @@
                     "value": "http://kagent-controller.kagent:8083"
                   }
                 ],
-                "image": "ghcr.io/kagent-dev/kagent/app@sha256:test-app",
+                "image": "ghcr.io/kagent-dev/kagent/app:dev",
                 "imagePullPolicy": "IfNotPresent",
                 "name": "kagent",
                 "ports": [

go/core/internal/controller/translator/agent/testdata/outputs/agent_with_git_skills.json

@@ -215,7 +215,7 @@
                     "value": "/config/srt-settings.json"
                   }
                 ],
-                "image": "ghcr.io/kagent-dev/kagent/app@sha256:test-app-full",
+                "image": "ghcr.io/kagent-dev/kagent/app:dev-full",
                 "imagePullPolicy": "IfNotPresent",
                 "name": "kagent",
                 "ports": [

go/core/internal/controller/translator/agent/testdata/outputs/agent_with_http_toolserver.json

@@ -195,7 +195,7 @@
                     "value": "http://kagent-controller.kagent:8083"
                   }
                 ],
-                "image": "ghcr.io/kagent-dev/kagent/app@sha256:test-app",
+                "image": "ghcr.io/kagent-dev/kagent/app:dev",
                 "imagePullPolicy": "IfNotPresent",
                 "name": "kagent",
                 "ports": [