e2e: add mid-rollout image change test - #129
Open
ptalgulk01 wants to merge 1 commit into
Open
Conversation
Signed-off-by: Prachiti Talgulkar <ptalgulk01@users.noreply.github.com>
alicefr
reviewed
Aug 7, 2026
Comment on lines
+266
to
+269
| if e.nodeImageRegistry == "" || e.nodeImageUpdate2Digest == "" { | ||
| return "" | ||
| } | ||
| return e.nodeImageRegistry + "@" + e.nodeImageUpdate2Digest |
Collaborator
There was a problem hiding this comment.
this is a duplication of the function NodeImageUpdateDigestedPullSpec can you define a local common function which takes the image as input
Comment on lines
+484
to
+510
| var daemonPod corev1.Pod | ||
| g.Eventually(func(g Gomega) { | ||
| var pods corev1.PodList | ||
| g.Expect(env.Client.List(ctx, &pods, | ||
| client.InNamespace("bootc-operator"), | ||
| client.MatchingLabels{ | ||
| "app.kubernetes.io/name": "bootc-operator", | ||
| "app.kubernetes.io/component": "daemon", | ||
| }, | ||
| )).To(Succeed()) | ||
| var matched []corev1.Pod | ||
| for _, p := range pods.Items { | ||
| if p.Spec.NodeName == nodeName && p.Status.Phase == corev1.PodRunning { | ||
| matched = append(matched, p) | ||
| } | ||
| } | ||
| g.Expect(matched).To(HaveLen(1)) | ||
| daemonPod = matched[0] | ||
| }).WithTimeout(1 * time.Minute).Should(Succeed()) | ||
|
|
||
| kubeconfigPath := os.Getenv("KUBECONFIG") | ||
| cmd := exec.CommandContext(ctx, "kubectl", "--kubeconfig", kubeconfigPath, | ||
| "-n", "bootc-operator", "exec", daemonPod.Name, "--", | ||
| "nsenter", "-m/proc/1/ns/mnt", "--", "journalctl", "--list-boots") | ||
| out, err := cmd.CombinedOutput() | ||
| g.Expect(err).NotTo(HaveOccurred(), | ||
| fmt.Sprintf("journalctl --list-boots failed: %s", string(out))) |
Collaborator
There was a problem hiding this comment.
This is very similar to what we do already here, would you mind to refactor and create a common function to reduce duplication
| HaveField("Status", metav1.ConditionTrue), | ||
| HaveField("Reason", bootcv1alpha1.NodeReasonIdle), | ||
| ))) | ||
| }).WithTimeout(3 * time.Minute).Should(Succeed()) |
Collaborator
There was a problem hiding this comment.
Please check the REVIEW_GOLANG.md for the test assertions
Comment on lines
+405
to
+419
| g.Eventually(func(g Gomega) { | ||
| for _, nodeName := range []string{nodeA, nodeB} { | ||
| var bn bootcv1alpha1.BootcNode | ||
| g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed()) | ||
| for _, c := range bn.Status.Conditions { | ||
| if c.Type == bootcv1alpha1.NodeIdle && | ||
| c.Status == metav1.ConditionFalse && | ||
| c.Reason == bootcv1alpha1.NodeReasonRebooting { | ||
| rebootingNode = nodeName | ||
| return | ||
| } | ||
| } | ||
| } | ||
| g.Expect(rebootingNode).NotTo(BeEmpty(), "expected at least one node to be Rebooting") | ||
| }).WithTimeout(5 * time.Minute).Should(Succeed()) |
Collaborator
There was a problem hiding this comment.
Suggested change
| g.Eventually(func(g Gomega) { | |
| for _, nodeName := range []string{nodeA, nodeB} { | |
| var bn bootcv1alpha1.BootcNode | |
| g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed()) | |
| for _, c := range bn.Status.Conditions { | |
| if c.Type == bootcv1alpha1.NodeIdle && | |
| c.Status == metav1.ConditionFalse && | |
| c.Reason == bootcv1alpha1.NodeReasonRebooting { | |
| rebootingNode = nodeName | |
| return | |
| } | |
| } | |
| } | |
| g.Expect(rebootingNode).NotTo(BeEmpty(), "expected at least one node to be Rebooting") | |
| }).WithTimeout(5 * time.Minute).Should(Succeed()) | |
| rebootingNode := g.Eventually(func(g Gomega) string { | |
| for _, name := range []string{nodeA, nodeB} { | |
| var bn bootcv1alpha1.BootcNode | |
| g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: name}, &bn)).To(Succeed()) | |
| cond := meta.FindStatusCondition(bn.Status.Conditions, bootcv1alpha1.NodeIdle) | |
| if cond != nil && | |
| cond.Status == metav1.ConditionFalse && | |
| cond.Reason == bootcv1alpha1.NodeReasonRebooting { | |
| return name | |
| } | |
| } | |
| return "" | |
| }).WithTimeout(5 * time.Minute).ShouldNot(BeEmpty()) |
Comment on lines
+439
to
+454
| // Phase 6: Wait for both nodes to be Idle with the second update image. | ||
| for _, nodeName := range []string{nodeA, nodeB} { | ||
| g.Eventually(func(g Gomega) { | ||
| var bn bootcv1alpha1.BootcNode | ||
| g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed()) | ||
| g.Expect(bn.Status.Booted).NotTo(BeNil()) | ||
| g.Expect(bn.Status.Booted.ImageDigest).To(Equal(env.NodeImageUpdate2Digest()), | ||
| "expected booted digest to match second update image") | ||
| g.Expect(bn.Status.Conditions).To(ContainElement(And( | ||
| HaveField("Type", bootcv1alpha1.NodeIdle), | ||
| HaveField("Status", metav1.ConditionTrue), | ||
| HaveField("Reason", bootcv1alpha1.NodeReasonIdle), | ||
| ))) | ||
| }).WithTimeout(8 * time.Minute).Should(Succeed(), | ||
| "expected node %s to reach Idle with second update image", nodeName) | ||
| } |
Collaborator
There was a problem hiding this comment.
Suggested change
| // Phase 6: Wait for both nodes to be Idle with the second update image. | |
| for _, nodeName := range []string{nodeA, nodeB} { | |
| g.Eventually(func(g Gomega) { | |
| var bn bootcv1alpha1.BootcNode | |
| g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed()) | |
| g.Expect(bn.Status.Booted).NotTo(BeNil()) | |
| g.Expect(bn.Status.Booted.ImageDigest).To(Equal(env.NodeImageUpdate2Digest()), | |
| "expected booted digest to match second update image") | |
| g.Expect(bn.Status.Conditions).To(ContainElement(And( | |
| HaveField("Type", bootcv1alpha1.NodeIdle), | |
| HaveField("Status", metav1.ConditionTrue), | |
| HaveField("Reason", bootcv1alpha1.NodeReasonIdle), | |
| ))) | |
| }).WithTimeout(8 * time.Minute).Should(Succeed(), | |
| "expected node %s to reach Idle with second update image", nodeName) | |
| } | |
| for _, nodeName := range []string{nodeA, nodeB} { | |
| g.Eventually(func() (bootcv1alpha1.BootcNode, error) { | |
| var bn bootcv1alpha1.BootcNode | |
| err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) | |
| return bn, err | |
| }).WithTimeout(8 * time.Minute).Should(SatisfyAll( | |
| HaveField("Status.Booted", Not(BeNil())), | |
| HaveField("Status.Booted.ImageDigest", Equal(env.NodeImageUpdate2Digest())), | |
| HaveField("Status.Conditions", ContainElement(And( | |
| HaveField("Type", bootcv1alpha1.NodeIdle), | |
| HaveField("Status", metav1.ConditionTrue), | |
| HaveField("Reason", bootcv1alpha1.NodeReasonIdle), | |
| ))), | |
| ), "expected node %s to reach Idle with second update image", nodeName) | |
| } |
Collaborator
|
@ptalgulk01 the ci is timing out, you need to increase the overall timeout. I hit the same in #128 , the PR already increases it. So, either way, we wait for mine to be merged or try to increase it here, and I will rebase if this land first |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
2 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Summary
TestMidRolloutImageChangee2e test that verifies changing the target image mid-rollout does not cause unnecessary rebootsnode:update2) for testing distinct image transitionsNodeImageUpdate2DigestedPullSpec()andNodeImageUpdate2Digest()helpers to the e2e test environmentThe test provisions two worker nodes, starts a rollout to one image, the switches the target to a different image while one node is rebooting. It verifies both nodes converge to the final image and uses
journalctl --list-bootsto confirm the non-rebooting node did not wastefully reboot into the first image.Partial progress on #69
Test plan
make e2e V=1 RUN=TestMidRolloutImageChangepasses (183s)make -n build-update-imageandmake -n e2edry-runs confirmMakefile correctness
latest,update,update2) produce distinctdigests