feat(cli): optimize and unify pod readiness check logic

cli/pkg/terminus/tasks.go

@@ -97,25 +97,8 @@ func (t *CheckKeyPodsRunning) Execute(runtime connector.Runtime) error {
 		if !strings.HasPrefix(pod.Namespace, "user-") && !strings.HasPrefix(pod.Namespace, "os-") {
 			continue
 		}
-		if pod.Status.Phase != corev1.PodRunning {
-			return fmt.Errorf("pod %s/%s is not running", pod.Namespace, pod.Name)
-		}
-		if len(pod.Status.ContainerStatuses) != len(pod.Spec.Containers) {
-			return fmt.Errorf("pod %s/%s has not started all containers yet", pod.Namespace, pod.Name)
-		}
-		for _, cStatus := range pod.Status.ContainerStatuses {
-			if cStatus.State.Terminated != nil {
-				if cStatus.State.Terminated.ExitCode != 0 {
-					return fmt.Errorf("container %s in pod %s/%s is terminated", cStatus.Name, pod.Namespace, pod.Name)
-				}
-				continue
-			}
-			if cStatus.State.Running == nil {
-				return fmt.Errorf("container %s in pod %s/%s is not running", cStatus.Name, pod.Namespace, pod.Name)
-			}
-			if !cStatus.Ready {
-				return fmt.Errorf("container %s in pod %s/%s is not ready", cStatus.Name, pod.Namespace, pod.Name)
-			}
+		if err := utils.AssertPodReady(&pod); err != nil {
+			return err
 		}
 	}
 	return nil
@@ -126,29 +109,39 @@ type CheckPodsRunning struct {
 	labels map[string][]string
 }
 
-func (c *CheckPodsRunning) Execute(runtime connector.Runtime) error {
+func (c *CheckPodsRunning) Execute(_ connector.Runtime) error {
 	if c.labels == nil {
 		return nil
 	}
 
-	kubectl, err := util.GetCommand(common.CommandKubectl)
-	if err != nil {
-		return errors.Wrap(errors.WithStack(err), "kubectl not found")
-	}
-
 	var ctx, cancel = context.WithTimeout(context.Background(), 3*time.Minute)
 	defer cancel()
+
+	kubeConfig, err := ctrl.GetConfig()
+	if err != nil {
+		return errors.Wrap(err, "failed to load kubeconfig")
+	}
+	kubeClient, err := kubernetes.NewForConfig(kubeConfig)
+	if err != nil {
+		return errors.Wrap(err, "failed to create kube client")
+	}
+
 	for ns, labels := range c.labels {
 		for _, label := range labels {
-			var cmd = fmt.Sprintf("%s get pod -n %s -l '%s' -o jsonpath='{.items[*].status.phase}'", kubectl, ns, label)
-			phase, err := runtime.GetRunner().SudoCmdContext(ctx, cmd, false, false)
+			podList, err := kubeClient.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: label})
 			if err != nil {
 				return fmt.Errorf("pod status invalid, namespace: %s, label: %s, waiting ...", ns, label)
 			}
 
-			if phase != "Running" {
-				logger.Infof("pod in namespace: %s, label: %s, current phase: %s, waiting ...", ns, label, phase)
-				return fmt.Errorf("pod is %s, namespace: %s, label: %s, waiting ...", phase, ns, label)
+			if podList == nil || len(podList.Items) == 0 {
+				return fmt.Errorf("no pod found, namespace: %s, label: %s, waiting ...", ns, label)
+			}
+
+			for i := range podList.Items {
+				pod := &podList.Items[i]
+				if err := utils.AssertPodReady(pod); err != nil {
+					return err
+				}
 			}
 		}
 	}

cli/pkg/utils/pod.go

@@ -0,0 +1,111 @@
+package utils
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+func AssertPodReady(pod *corev1.Pod) error {
+	if pod == nil {
+		return fmt.Errorf("pod is nil")
+	}
+
+	podKey := fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)
+	if pod.DeletionTimestamp != nil {
+		return fmt.Errorf("pod %s is terminating", podKey)
+	}
+	if pod.Status.Phase != corev1.PodRunning {
+		return fmt.Errorf("pod %s is not running (phase=%s)", podKey, pod.Status.Phase)
+	}
+
+	if len(pod.Spec.InitContainers) > 0 {
+		initStatusByName := make(map[string]corev1.ContainerStatus, len(pod.Status.InitContainerStatuses))
+		for i := range pod.Status.InitContainerStatuses {
+			s := pod.Status.InitContainerStatuses[i]
+			initStatusByName[s.Name] = s
+		}
+		for _, ic := range pod.Spec.InitContainers {
+			s, ok := initStatusByName[ic.Name]
+			if !ok {
+				return fmt.Errorf("pod %s has not started init container %s yet", podKey, ic.Name)
+			}
+			if t := s.State.Terminated; t != nil {
+				if t.ExitCode != 0 {
+					return fmt.Errorf(
+						"init container %s in pod %s terminated (exitCode=%d, reason=%s, message=%s)",
+						s.Name, podKey, t.ExitCode, t.Reason, t.Message,
+					)
+				}
+				continue
+			}
+			if w := s.State.Waiting; w != nil {
+				return fmt.Errorf(
+					"init container %s in pod %s is waiting (reason=%s, message=%s)",
+					s.Name, podKey, w.Reason, w.Message,
+				)
+			}
+			return fmt.Errorf("pod %s init container %s is still running", podKey, s.Name)
+		}
+	}
+
+	readyCondFound := false
+	for i := range pod.Status.Conditions {
+		cond := pod.Status.Conditions[i]
+		if cond.Type != corev1.PodReady {
+			continue
+		}
+		readyCondFound = true
+		if cond.Status != corev1.ConditionTrue {
+			if cond.Reason != "" || cond.Message != "" {
+				return fmt.Errorf("pod %s is not ready (reason=%s, message=%s)", podKey, cond.Reason, cond.Message)
+			}
+			return fmt.Errorf("pod %s is not ready", podKey)
+		}
+		break
+	}
+	if !readyCondFound {
+		return fmt.Errorf("pod %s is not ready (missing Ready condition)", podKey)
+	}
+
+	statusByName := make(map[string]corev1.ContainerStatus, len(pod.Status.ContainerStatuses))
+	for i := range pod.Status.ContainerStatuses {
+		s := pod.Status.ContainerStatuses[i]
+		statusByName[s.Name] = s
+	}
+
+	for _, c := range pod.Spec.Containers {
+		cStatus, ok := statusByName[c.Name]
+		if !ok {
+			return fmt.Errorf("pod %s has not started container %s yet", podKey, c.Name)
+		}
+
+		if t := cStatus.State.Terminated; t != nil {
+			return fmt.Errorf(
+				"container %s in pod %s terminated (exitCode=%d, reason=%s, message=%s)",
+				cStatus.Name,
+				podKey,
+				t.ExitCode,
+				t.Reason,
+				t.Message,
+			)
+		}
+
+		if cStatus.State.Running == nil {
+			if w := cStatus.State.Waiting; w != nil {
+				return fmt.Errorf(
+					"container %s in pod %s is waiting (reason=%s, message=%s)",
+					cStatus.Name,
+					podKey,
+					w.Reason,
+					w.Message,
+				)
+			}
+			return fmt.Errorf("container %s in pod %s is not running", cStatus.Name, podKey)
+		}
+		if !cStatus.Ready {
+			return fmt.Errorf("container %s in pod %s is not ready", cStatus.Name, podKey)
+		}
+	}
+	return nil
+}