A Terminating pod is not ready (#528)

* terminating is not ready

* tests

* remove blackboxing from tests

changelog/v0.25.2/terminating-is-not-ready.yaml

@@ -0,0 +1,6 @@
+changelog:
+- type: FIX
+  issueLink: https://github.com/solo-io/go-utils/issues/529
+  resolvesIssue: true
+  description: >-
+    Revise kubectl tool to not report `Terminating` pods as `Running`.

testutils/errors_test.go

@@ -1,10 +1,9 @@
-package testutils_test
+package testutils
 
 import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/rotisserie/eris"
-	. "github.com/solo-io/go-utils/testutils"
 )
 
 var _ = Describe("eris errors", func() {

testutils/get_current_file_test.go

@@ -1,10 +1,8 @@
-package testutils_test
+package testutils
 
 import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-
-	. "github.com/solo-io/go-utils/testutils"
 )
 
 var _ = Describe("GetCurrentFileDirectory", func() {

testutils/kubernetes.go

@@ -17,6 +17,12 @@ import (
 	"github.com/pkg/errors"
 )
 
+// Helper vars that allow us to mock the kubectl call with static echo text
+var (
+	kubectlExecutable          = "kubectl"
+	kubectlArgs       []string = nil
+)
+
 // Deprecated: this function is incredibly slow, use CreateNamespacesInParallel instead
 func SetupKubeForTest(namespace string) error {
 	context := os.Getenv("KUBECTL_CONTEXT")
@@ -45,7 +51,10 @@ func DeleteCrd(crd string) error {
 }
 
 func kubectl(args ...string) *exec.Cmd {
-	cmd := exec.Command("kubectl", args...)
+	if kubectlArgs != nil {
+		args = kubectlArgs
+	}
+	cmd := exec.Command(kubectlExecutable, args...)
 	cmd.Env = os.Environ()
 	// disable DEBUG=1 from getting through to kube
 	for i, pair := range cmd.Env {
@@ -137,18 +146,10 @@ func KubectlOutChan(r io.Reader, args ...string) (<-chan io.Reader, chan struct{
 // WaitPodsRunning waits for all pods to be running
 func WaitPodsRunning(ctx context.Context, interval time.Duration, namespace string, labels ...string) error {
 	finished := func(output string) bool {
-		return strings.Contains(output, "Running") || strings.Contains(output, "ContainerCreating")
-	}
-	for _, label := range labels {
-		if err := WaitPodStatus(ctx, interval, namespace, label, "Running or ContainerCreating", finished); err != nil {
-			return err
-		}
-	}
-	finished = func(output string) bool {
-		return strings.Contains(output, "Running")
+		return strings.Contains(output, "Running") && !strings.Contains(output, "Terminating")
 	}
 	for _, label := range labels {
-		if err := WaitPodStatus(ctx, interval, namespace, label, "Running", finished); err != nil {
+		if err := WaitPodStatus(ctx, interval, namespace, label, "Running, not Terminating", finished); err != nil {
 			return err
 		}
 	}

testutils/kubernetes_test.go

@@ -0,0 +1,44 @@
+package testutils
+
+import (
+	"context"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Kubernetes", func() {
+	Context("WaitPodsRunning", func() {
+		var (
+			ctx    context.Context
+			cancel context.CancelFunc
+		)
+		BeforeEach(func() {
+			kubectlExecutable = `echo`
+			ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(time.Second))
+		})
+		AfterEach(func() {
+			cancel()
+		})
+		It("Reports running pods as running", func() {
+			kubectlArgs = []string{`"\
+NAMESPACE            NAME                                              READY   STATUS        RESTARTS   AGE \
+kube-system          coredns-5dd5756b68-62pwz                          1/1     Running       0          3d18h" #`}
+
+			// The parameters here don't really matter since we're not actually calling kubectl.
+			// We do need a label however to enter the loop that calls the command.
+			Expect(WaitPodsRunning(ctx, time.Millisecond, "", "foo=bar")).ToNot(HaveOccurred())
+		})
+		It("Doesn't report terminating pods as running", func() {
+			kubectlArgs = []string{`"\
+NAMESPACE            NAME                                              READY   STATUS        RESTARTS   AGE \
+kube-system          coredns-5dd5756b68-jlm7r                          1/1     Terminating   0          3d18h \
+kube-system          coredns-5dd5756b68-62pwz                          1/1     Running       0          3d18h" #`}
+
+			// The parameters here don't really matter since we're not actually calling kubectl.
+			// We do need a label however to enter the loop that calls the command.
+			Expect(WaitPodsRunning(ctx, time.Millisecond, "", "foo=bar")).To(MatchError(ContainSubstring("timed out waiting")))
+		})
+	})
+})

testutils/testutils_suite_test.go

@@ -1,4 +1,4 @@
-package testutils_test
+package testutils
 
 import (
 	"testing"