Skip to content

e2e: add rollback, pause/resume, and non-existing image tests #115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ptalgulk01 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

e2e: add rollback, pause/resume, and non-existing image tests #115

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
348 changes: 336 additions & 12 deletions test/e2e/bootcnode_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,35 @@ func TestUpdateReboot(t *testing.T) {
fmt.Sprintf("expected update-marker to exist on host, kubectl exec output: %s", string(out)))

t.Logf("Verified update-marker exists on host via daemon pod")

// Phase 7: Rollback to original image.
originalRef := env.NodeImageDigestedPullSpec()

modified = pool.DeepCopy()
modified.Spec.Image.Ref = originalRef
g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed())
*pool = *modified

t.Logf("Patched pool to rollback to original image %s", originalRef)

// Phase 8: Wait for Idle with the original digest — proves rollback succeeded.
g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) {
var bn2 bootcv1alpha1.BootcNode
err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn2)
return bn2.Status, err
}).WithTimeout(5*time.Minute).Should(And(
HaveField("Booted", And(
Not(BeNil()),
HaveField("ImageDigest", Equal(env.NodeImageDigest())),
)),
HaveField("Conditions", ContainElement(And(
HaveField("Type", bootcv1alpha1.NodeIdle),
HaveField("Status", metav1.ConditionTrue),
HaveField("Reason", bootcv1alpha1.NodeReasonIdle),
))),
), "expected node to reach Idle with original image after rollback")

t.Logf("Node %q successfully rolled back to original image", nodeName)
}

// TestTagResolution creates a pool with a tag-based image ref, verifies
Expand All @@ -220,19 +249,19 @@ func TestTagResolution(t *testing.T) {
g.Expect(env.Client.Create(ctx, pool)).To(Succeed())

// Verify targetDigest is resolved to the original image digest.
g.Eventually(func(g Gomega) string {
g.Eventually(func() (string, error) {
var p bootcv1alpha1.BootcNodePool
g.Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(pool), &p)).To(Succeed())
return p.Status.TargetDigest
err := env.Client.Get(ctx, client.ObjectKeyFromObject(pool), &p)
return p.Status.TargetDigest, err
}).WithTimeout(1 * time.Minute).Should(Equal(env.NodeImageDigest()))

t.Logf("Tag resolved to original digest %s", env.NodeImageDigest())

// Wait for node to reach Idle with the original image.
g.Eventually(func(g Gomega) bootcv1alpha1.BootcNodeStatus {
g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) {
var bn bootcv1alpha1.BootcNode
g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed())
return bn.Status
err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)
return bn.Status, err
}).WithTimeout(3 * time.Minute).Should(And(
HaveField("Booted", And(
Not(BeNil()),
Expand All @@ -251,23 +280,30 @@ func TestTagResolution(t *testing.T) {
"localhost:5000/node@"+env.NodeImageUpdateDigest(),
"localhost:5000/node:latest",
)
// Restore the original tag on cleanup so later tests are not affected.
t.Cleanup(func() {
e2eutil.RetagImage(t,
"localhost:5000/node@"+env.NodeImageDigest(),
"localhost:5000/node:latest",
)
})
Comment thread
jlebon marked this conversation as resolved.

t.Logf("Retagged node:latest to update digest %s", env.NodeImageUpdateDigest())

// Wait for the controller to re-resolve and pick up the new digest.
g.Eventually(func(g Gomega) string {
g.Eventually(func() (string, error) {
var p bootcv1alpha1.BootcNodePool
g.Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(pool), &p)).To(Succeed())
return p.Status.TargetDigest
err := env.Client.Get(ctx, client.ObjectKeyFromObject(pool), &p)
return p.Status.TargetDigest, err
}).WithTimeout(1 * time.Minute).Should(Equal(env.NodeImageUpdateDigest()))

t.Logf("Tag re-resolved to update digest %s", env.NodeImageUpdateDigest())

// Wait for node to reach Idle with the update image.
g.Eventually(func(g Gomega) bootcv1alpha1.BootcNodeStatus {
g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) {
var bn bootcv1alpha1.BootcNode
g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed())
return bn.Status
err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)
return bn.Status, err
}).WithTimeout(5 * time.Minute).Should(And(
HaveField("Booted", And(
Not(BeNil()),
Expand All @@ -281,3 +317,291 @@ func TestTagResolution(t *testing.T) {

t.Logf("Node %q is Idle with update image", nodeName)
}

// TestPauseResume provisions a worker node, starts an update with the
// pool paused, verifies the node stages but does not reboot, then resumes
// and verifies the update completes.
func TestPauseResume(t *testing.T) {
g := NewWithT(t)
g.SetDefaultEventuallyTimeout(pollTimeout)
g.SetDefaultEventuallyPollingInterval(pollInterval)

env := e2eutil.New(t)
nodeName := env.AddNode(t)

ctx := context.Background()

// Phase 1: Create pool with original image and wait for Idle.
pool := env.NewPool("bnp-pause", env.NodeImageDigestedPullSpec())
g.Expect(env.Client.Create(ctx, pool)).To(Succeed())

var bn bootcv1alpha1.BootcNode
g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) {
err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)
return bn.Status, err
}).WithTimeout(3 * time.Minute).Should(And(
HaveField("Booted", Not(BeNil())),
HaveField("Conditions", ContainElement(And(
HaveField("Type", bootcv1alpha1.NodeIdle),
HaveField("Status", metav1.ConditionTrue),
HaveField("Reason", bootcv1alpha1.NodeReasonIdle),
))),
))

t.Logf("Node %q is Idle with original image", nodeName)

// Phase 2: Patch pool to update image with paused=true.
updateRef := env.NodeImageUpdateDigestedPullSpec()

modified := pool.DeepCopy()
modified.Spec.Image.Ref = updateRef
if modified.Spec.Rollout == nil {
modified.Spec.Rollout = &bootcv1alpha1.RolloutSpec{}
}
modified.Spec.Rollout.Paused = true
g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed())
*pool = *modified

t.Logf("Patched pool to update image %s with paused=true", updateRef)

// Phase 3: Wait for node to stage the image. The node should reach
// Staged state but not proceed to reboot because the pool is paused.
g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) {
err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)
return bn.Status, err
}).WithTimeout(5 * time.Minute).Should(And(
HaveField("Staged", And(
Not(BeNil()),
HaveField("ImageDigest", Equal(env.NodeImageUpdateDigest())),
)),
HaveField("Conditions", ContainElement(And(
HaveField("Type", bootcv1alpha1.NodeIdle),
HaveField("Status", metav1.ConditionFalse),
HaveField("Reason", bootcv1alpha1.NodeReasonStaged),
))),
HaveField("Booted", And(
Not(BeNil()),
HaveField("ImageDigest", Equal(env.NodeImageDigest())),
)),
))

t.Logf("Node %q staged update but did not reboot (paused)", nodeName)

// Verify the node stays Staged and does not proceed to reboot.
g.Consistently(func() ([]metav1.Condition, error) {
var bn2 bootcv1alpha1.BootcNode
err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn2)
return bn2.Status.Conditions, err
}).WithTimeout(10*time.Second).WithPolling(2*time.Second).Should(ContainElement(And(
HaveField("Type", bootcv1alpha1.NodeIdle),
HaveField("Status", metav1.ConditionFalse),
HaveField("Reason", bootcv1alpha1.NodeReasonStaged),
)), "node should remain Staged while paused")

// Phase 4: Resume the rollout.
modified = pool.DeepCopy()
modified.Spec.Rollout.Paused = false
g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed())
*pool = *modified

t.Logf("Resumed rollout (paused=false)")

// Phase 5: Wait for node to complete the update — proves the full
// update lifecycle completed after resume (reboot, boot into new image).
g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) {
err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)
return bn.Status, err
}).WithTimeout(5*time.Minute).Should(And(
HaveField("Booted", And(
Not(BeNil()),
HaveField("ImageDigest", Equal(env.NodeImageUpdateDigest())),
)),
HaveField("Conditions", ContainElement(And(
HaveField("Type", bootcv1alpha1.NodeIdle),
HaveField("Status", metav1.ConditionTrue),
HaveField("Reason", bootcv1alpha1.NodeReasonIdle),
))),
), "expected node to reach Idle with update image after resume")

t.Logf("Node %q completed update after resume", nodeName)
}

// TestMidRolloutImageChange provisions a worker node, starts a rollout
// to the update image with paused=true so the node stages but does not
// reboot, then changes the target back to the original image and resumes.
// The node should converge to the final target (original), proving that
// the controller and daemon state machines do not deadlock when the
// target image changes mid-rollout.
func TestMidRolloutImageChange(t *testing.T) {
g := NewWithT(t)
g.SetDefaultEventuallyTimeout(pollTimeout)
g.SetDefaultEventuallyPollingInterval(pollInterval)

env := e2eutil.New(t)
nodeName := env.AddNode(t)

ctx := context.Background()

// Phase 1: Create pool with original image and wait for Idle.
pool := env.NewPool("bnp-midroll", env.NodeImageDigestedPullSpec())
g.Expect(env.Client.Create(ctx, pool)).To(Succeed())

var bn bootcv1alpha1.BootcNode
g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) {
err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)
return bn.Status, err
}).WithTimeout(3 * time.Minute).Should(And(
HaveField("Booted", Not(BeNil())),
HaveField("Conditions", ContainElement(And(
HaveField("Type", bootcv1alpha1.NodeIdle),
HaveField("Status", metav1.ConditionTrue),
HaveField("Reason", bootcv1alpha1.NodeReasonIdle),
))),
))

t.Logf("Node %q is Idle with original image", nodeName)

// Phase 2: Start rollout to update image with paused=true.
updateRef := env.NodeImageUpdateDigestedPullSpec()

modified := pool.DeepCopy()
modified.Spec.Image.Ref = updateRef
if modified.Spec.Rollout == nil {
modified.Spec.Rollout = &bootcv1alpha1.RolloutSpec{}
}
modified.Spec.Rollout.Paused = true
g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed())
*pool = *modified

t.Logf("Started rollout to update image %s with paused=true", updateRef)

// Phase 3: Wait for the node to stage the update image.
g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) {
err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)
return bn.Status, err
}).WithTimeout(5 * time.Minute).Should(And(
HaveField("Staged", And(
Not(BeNil()),
HaveField("ImageDigest", Equal(env.NodeImageUpdateDigest())),
)),
HaveField("Conditions", ContainElement(And(
HaveField("Type", bootcv1alpha1.NodeIdle),
HaveField("Status", metav1.ConditionFalse),
HaveField("Reason", bootcv1alpha1.NodeReasonStaged),
))),
HaveField("Booted", And(
Not(BeNil()),
HaveField("ImageDigest", Equal(env.NodeImageDigest())),
)),
))

t.Logf("Node %q staged update image (paused, not rebooted)", nodeName)

// Phase 4: Change the target back to the original image while still

@jlebon jlebon Jul 27, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is not very different from TestPauseResume. Not quite what I had in mind for #69 (comment). Instead we do something like:

  • provision two nodes (let's say A and B)
  • set the new updated image
  • wait for node A to be Rebooting
  • immediately switch the target image to another updated image
  • wait until both node A and node B eventually are running the updated image
  • verify that node B did not incur an additional reboot into the first update image (e.g. check journalctl --list-boots perhaps?)

@ptalgulk01 ptalgulk01 Jul 28, 2026 •
edited
Loading

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

immediately switch the target image to another updated image

here you mean to update the node A when it is rebooting to new image already we have patched to other new image?
And keep node B as it is

@jlebon jlebon Jul 29, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

immediately switch the target image to another updated image

here you mean to update the node A when it is rebooting to new image already we have patched to other new image? And keep node B as it is

I mean changing the pool targetImage to another image. We don't mutate the nodes directly. Node A will reboot into the wrong image, which is fine and expected, but it should then realize that it's not in sync and immediately restage and reboot again. Node B should be able to avoid that reboot and directly stage the new image (abandoning the staged deployment it already had for the previous targetImage).

// paused. This simulates changing the desired image mid-rollout.
originalRef := env.NodeImageDigestedPullSpec()

modified = pool.DeepCopy()
modified.Spec.Image.Ref = originalRef
g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed())
*pool = *modified

t.Logf("Changed target back to original image %s (still paused)", originalRef)

// Phase 5: Resume the rollout.
modified = pool.DeepCopy()
modified.Spec.Rollout.Paused = false
g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed())
*pool = *modified

t.Logf("Resumed rollout (paused=false)")

// Phase 6: The node should converge to Idle with the original image.
// The staged update image should be discarded since the target changed.
g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) {
err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)
return bn.Status, err
}).WithTimeout(5*time.Minute).Should(And(
HaveField("Booted", And(
Not(BeNil()),
HaveField("ImageDigest", Equal(env.NodeImageDigest())),
)),
HaveField("Conditions", ContainElement(And(
HaveField("Type", bootcv1alpha1.NodeIdle),
HaveField("Status", metav1.ConditionTrue),
HaveField("Reason", bootcv1alpha1.NodeReasonIdle),
))),
), "expected node to reach Idle with original image after mid-rollout target change")

t.Logf("Node %q converged to original image after mid-rollout change", nodeName)
}

// TestNonExistingImage provisions a worker node, creates a pool with the
// original image, then updates to a non-existing image and verifies the
// node enters degraded state and the update does not proceed.
func TestNonExistingImage(t *testing.T) {
g := NewWithT(t)
g.SetDefaultEventuallyTimeout(pollTimeout)
g.SetDefaultEventuallyPollingInterval(pollInterval)

env := e2eutil.New(t)
nodeName := env.AddNode(t)

ctx := context.Background()

// Phase 1: Create pool with original image and wait for Idle.
pool := env.NewPool("bnp-noimg", env.NodeImageDigestedPullSpec())
g.Expect(env.Client.Create(ctx, pool)).To(Succeed())

var bn bootcv1alpha1.BootcNode
g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) {
err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)
return bn.Status, err
}).WithTimeout(3 * time.Minute).Should(And(
HaveField("Booted", Not(BeNil())),
HaveField("Conditions", ContainElement(And(
HaveField("Type", bootcv1alpha1.NodeIdle),
HaveField("Status", metav1.ConditionTrue),
HaveField("Reason", bootcv1alpha1.NodeReasonIdle),
))),
))

t.Logf("Node %q is Idle with original image", nodeName)

// Phase 2: Patch pool to update to a non-existing image.
nonExistingRef := "localhost:5000/node@sha256:0000000000000000000000000000000000000000000000000000000000000000"

modified := pool.DeepCopy()
modified.Spec.Image.Ref = nonExistingRef
g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed())
*pool = *modified

t.Logf("Patched pool to non-existing image %s", nonExistingRef)

// Phase 3: Wait for node to enter degraded state.
// The daemon should fail to pull the image and report an error.
g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) {
err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)
return bn.Status, err
}).WithTimeout(5*time.Minute).Should(And(
HaveField("Conditions", ContainElement(And(
HaveField("Type", bootcv1alpha1.NodeDegraded),
HaveField("Status", metav1.ConditionTrue),
HaveField("Reason", bootcv1alpha1.NodeReasonError),
Comment thread
jlebon marked this conversation as resolved.
HaveField("Message", ContainSubstring("stage failed")),
))),
HaveField("Booted", And(
Not(BeNil()),
HaveField("ImageDigest", Equal(env.NodeImageDigest())),
)),
), "expected node to enter degraded state when pulling non-existing image")

t.Logf("Node %q entered degraded state as expected", nodeName)

// Phase 4: Verify the node did not stage the non-existing image.
g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed())
g.Expect(bn.Status.Staged).To(BeNil(),
"node should not have staged the non-existing image")

t.Logf("Verified node %q did not stage non-existing image", nodeName)
}