fixup! test: Add tests to ensure new certificate is being used

docker/registry/registry.go

@@ -148,8 +148,6 @@ func (r Registry) Shutdown(ctx context.Context) error {
 }
 
 func (r Registry) ListenAndServe(log logr.Logger) error {
-	log = log.WithName("registry")
-
 	var err error
 	if r.config.HTTP.TLS.Certificate != "" && r.config.HTTP.TLS.Key != "" {
 		watcher, cwErr := certwatcher.New(r.config.HTTP.TLS.Certificate, r.config.HTTP.TLS.Key)
@@ -162,7 +160,7 @@ func (r Registry) ListenAndServe(log logr.Logger) error {
 		}
 		go func() {
 			if startErr := watcher.Start(context.TODO()); startErr != nil {
-				log.Error(startErr, "certwatcher Start failed")
+				panic(fmt.Sprintf("certwatcher Start failed: %v", startErr))
 			}
 		}()
 

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/docker/docker-credential-helpers v0.8.2
 	github.com/docker/go-connections v0.5.0
 	github.com/elazarl/goproxy v0.0.0-20230731152917-f99041a5c027
+	github.com/go-logr/logr v1.4.2
 	github.com/google/go-containerregistry v0.20.2
 	github.com/hashicorp/go-getter v1.7.6
 	github.com/mesosphere/dkp-cli-runtime/core v0.7.3
@@ -114,7 +115,6 @@ require (
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect

tasks/test.yaml

@@ -81,7 +81,7 @@ tasks:
             {{if .E2E_FOCUS}}--focus="{{.E2E_FOCUS}}"{{end}} \
             {{if .E2E_SKIP}}--skip="{{.E2E_SKIP}}"{{end}} \
             {{if .E2E_LABEL}}--label-filter="{{.E2E_LABEL}}"{{end}} \
-            {{if .E2E_LE2E_GINKGO_FLAGSABEL}}{{.E2E_GINKGO_FLAGS}}{{end}} \
+            {{if .E2E_GINKGO_FLAGS}}{{.E2E_GINKGO_FLAGS}}{{end}} \
             --junit-report=junit-e2e.xml \
             --json-report=report-e2e.json \
             --tags e2e \

test/e2e/helmbundle/helpers/helpers.go

@@ -116,20 +116,26 @@ func GenerateCertificateAndKeyWithIPSAN(
 	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
 
 	caCertFile = filepath.Join(destDir, "ca.crt")
-	caCertF, err := os.Create(caCertFile)
+	tmpCACertFile := caCertFile + ".tmp"
+	caCertF, err := os.Create(tmpCACertFile)
 	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
 	defer caCertF.Close()
 	gomega.ExpectWithOffset(1, pem.Encode(caCertF, &pem.Block{Type: "CERTIFICATE", Bytes: caDerBytes})).
 		To(gomega.Succeed())
+	gomega.ExpectWithOffset(1, caCertF.Close()).To(gomega.Succeed())
+	gomega.ExpectWithOffset(1, os.Rename(tmpCACertFile, caCertFile)).To(gomega.Succeed())
 
 	b, err := x509.MarshalECPrivateKey(caPriv)
 	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
 	pemBlock := pem.Block{Type: "EC PRIVATE KEY", Bytes: b}
 	caKeyFile = filepath.Join(destDir, "ca.key")
-	caKeyF, err := os.Create(caKeyFile)
+	tmpCAKeyFile := caKeyFile + ".tmp"
+	caKeyF, err := os.Create(tmpCAKeyFile)
 	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
 	defer caKeyF.Close()
 	gomega.ExpectWithOffset(1, pem.Encode(caKeyF, &pemBlock)).To(gomega.Succeed())
+	gomega.ExpectWithOffset(1, caKeyF.Close()).To(gomega.Succeed())
+	gomega.ExpectWithOffset(1, os.Rename(tmpCAKeyFile, caKeyFile)).To(gomega.Succeed())
 
 	priv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
 	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
@@ -158,20 +164,26 @@ func GenerateCertificateAndKeyWithIPSAN(
 	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
 
 	certFile = filepath.Join(destDir, "tls.crt")
-	certF, err := os.Create(certFile)
+	tmpCertFile := certFile + ".tmp"
+	certF, err := os.Create(tmpCertFile)
 	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
 	defer certF.Close()
 	gomega.ExpectWithOffset(1, pem.Encode(certF, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})).
 		To(gomega.Succeed())
+	gomega.ExpectWithOffset(1, certF.Close()).To(gomega.Succeed())
+	gomega.ExpectWithOffset(1, os.Rename(tmpCertFile, certFile)).To(gomega.Succeed())
 
 	b, err = x509.MarshalECPrivateKey(priv)
 	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
 	pemBlock = pem.Block{Type: "EC PRIVATE KEY", Bytes: b}
 	keyFile = filepath.Join(destDir, "tls.key")
-	keyF, err := os.Create(keyFile)
+	tmpKeyFile := keyFile + ".tmp"
+	keyF, err := os.Create(tmpKeyFile)
 	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
 	defer keyF.Close()
 	gomega.ExpectWithOffset(1, pem.Encode(keyF, &pemBlock)).To(gomega.Succeed())
+	gomega.ExpectWithOffset(1, keyF.Close()).To(gomega.Succeed())
+	gomega.ExpectWithOffset(1, os.Rename(tmpKeyFile, keyFile)).To(gomega.Succeed())
 
 	return caCertFile, caKeyFile, certFile, keyFile
 }

test/e2e/helmbundle/serve_bundle_test.go

@@ -6,7 +6,10 @@
 package helmbundle_test
 
 import (
+	"crypto/x509"
+	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
 	"strconv"
 	"time"
@@ -96,7 +99,7 @@ var _ = Describe("Serve Helm Bundle", func() {
 		ipAddr := helpers.GetFirstNonLoopbackIP(GinkgoT())
 
 		tempCertDir := GinkgoT().TempDir()
-		caCertFile, _, certFile, keyFile := helpers.GenerateCertificateAndKeyWithIPSAN(
+		originalCACertFile, _, certFile, keyFile := helpers.GenerateCertificateAndKeyWithIPSAN(
 			GinkgoT(),
 			tempCertDir,
 			ipAddr,
@@ -129,27 +132,51 @@ var _ = Describe("Serve Helm Bundle", func() {
 
 		helpers.WaitForTCPPort(GinkgoT(), ipAddr.String(), port)
 
-		// First check mindthegap is // First check that the helm chart is accessible with the old certificate.
-		helpers.ValidateChartIsAvailable(GinkgoT(), Default, ipAddr.String(), port, "podinfo", "6.2.0", helm.CAFileOpt(caCertFile))
+		// First check that the helm chart is accessible with the old certificate.
+		helpers.ValidateChartIsAvailable(GinkgoT(), Default, ipAddr.String(), port, "podinfo", "6.2.0", helm.CAFileOpt(originalCACertFile))
 
-		helpers.ValidateChartIsAvailable(GinkgoT(), Default, ipAddr.String(), port, "node-feature-discovery", "0.15.2", helm.CAFileOpt(caCertFile))
+		helpers.ValidateChartIsAvailable(GinkgoT(), Default, ipAddr.String(), port, "node-feature-discovery", "0.15.2", helm.CAFileOpt(originalCACertFile))
+
+		// Backup the original CA file to be used after checking the new CA file works.
+		// This is to ensure that the server is definitely using the new certificate.
+		backupDir := GinkgoT().TempDir()
+		caCertFileName := filepath.Base(originalCACertFile)
+		Expect(os.Rename(originalCACertFile, filepath.Join(backupDir, caCertFileName))).To(Succeed())
+		originalCACertFile = filepath.Join(backupDir, caCertFileName)
 
 		// Create a new certificate. This can happen at any time the server is running,
 		// and the server is expected to eventually use the new certificate.
 		// This also generates a new CA file which is even better because we can check
 		// that the server is using the certificate issued by the new CA.
-		caCertFile, _, _, _ = helpers.GenerateCertificateAndKeyWithIPSAN(
+		newCACertFile, _, _, _ := helpers.GenerateCertificateAndKeyWithIPSAN(
 			GinkgoT(),
 			tempCertDir,
 			ipAddr,
 		)
 
 		Eventually(func(g Gomega) {
-			helpers.ValidateChartIsAvailable(GinkgoT(), g, ipAddr.String(), port, "podinfo", "6.2.0", helm.CAFileOpt(caCertFile))
+			helpers.ValidateChartIsAvailable(GinkgoT(), Default, ipAddr.String(), port, "podinfo", "6.2.0", helm.CAFileOpt(newCACertFile))
 
-			helpers.ValidateChartIsAvailable(GinkgoT(), g, ipAddr.String(), port, "node-feature-discovery", "0.15.2", helm.CAFileOpt(caCertFile))
+			helpers.ValidateChartIsAvailable(GinkgoT(), Default, ipAddr.String(), port, "node-feature-discovery", "0.15.2", helm.CAFileOpt(newCACertFile))
 		}).WithTimeout(time.Second * 5).WithPolling(time.Second * 1).Should(Succeed())
 
+		// Now check that the original CA file is now no longer valid, ensuring that the new
+		// certificate is being used by mindthegap serve.
+		h, cleanup := helm.NewClient(
+			output.NewNonInteractiveShell(GinkgoWriter, GinkgoWriter, 10),
+		)
+		DeferCleanup(cleanup)
+		helmTmpDir := GinkgoT().TempDir()
+
+		_, err = h.GetChartFromRepo(
+			helmTmpDir,
+			"",
+			fmt.Sprintf("%s://%s:%d/charts/%s", helm.OCIScheme, ipAddr.String(), port, "podinfo"),
+			"6.2.0",
+			helm.CAFileOpt(originalCACertFile),
+		)
+		Expect(errors.As(err, &x509.UnknownAuthorityError{})).To(BeTrue())
+
 		close(stopCh)
 
 		Eventually(done).Should(BeClosed())

test/e2e/imagebundle/helpers/helpers.go

@@ -30,6 +30,7 @@ import (
 	"github.com/onsi/gomega"
 	"github.com/onsi/gomega/gstruct"
 	"github.com/spf13/cobra"
+	ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/mesosphere/dkp-cli-runtime/core/output"
 
@@ -55,6 +56,7 @@ func NewCommand(
 	newFn func(out output.Output) *cobra.Command,
 ) *cobra.Command {
 	t.Helper()
+	ctrllog.SetLogger(ginkgo.GinkgoLogr)
 	cmd := newFn(output.NewNonInteractiveShell(ginkgo.GinkgoWriter, ginkgo.GinkgoWriter, 10))
 	cmd.SilenceUsage = true
 	return cmd