test(main): add e2e test and controller test (#24)

Signed-off-by: cuisongliu <[email protected]>

.github/workflows/test.yml

@@ -23,3 +23,35 @@ jobs:
       - name: build
         run: |
           make test
+  job1:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@master
+
+      - name: Set up Go
+        uses: actions/setup-go@master
+        with:
+          go-version: 1.23.x
+
+      - name: Verify sealos
+        run: |
+          wget https://github.com/labring/sealos/releases/download/v4.3.7/sealos_4.3.7_linux_amd64.tar.gz
+          tar -zxvf sealos_4.3.7_linux_amd64.tar.gz sealos
+          sudo chmod a+x sealos
+          sudo mv sealos /usr/bin/
+          sudo sealos version
+      - name: prune os
+        run: |
+          sudo systemctl unmask containerd
+          sudo systemctl unmask docker
+          sudo apt-get remove -y moby-buildx moby-cli moby-compose moby-containerd moby-engine
+          sudo apt-get remove docker docker-engine docker.io containerd runc
+          sudo apt-get purge docker-ce docker-ce-cli containerd.io # docker-compose-plugin
+          sudo apt-get remove -y moby-engine moby-cli moby-buildx moby-compose
+          sudo rm -rf /var/run/docker.sock
+          sudo rm -rf /run/containerd/containerd.sock
+          sudo sealos run labring/kubernetes:v1.25.0 labring/helm:v3.8.2 labring/calico:v3.26.5  labring/openebs:v3.9.0
+      - name: build
+        run: |
+          sudo make e2e

Makefile

@@ -62,7 +62,7 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: manifests generate fmt vet envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... --ginkgo.v -v --ginkgo.trace
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./api/... --ginkgo.v -v --ginkgo.trace
 
 ##@ Build
 
@@ -168,3 +168,6 @@ info:
 	@cat deploy/charts/automq-operator/Chart.yaml
 	@cat deploy/images/shim/image.txt
 
+.PHONY: e2e
+e2e: manifests generate fmt vet envtest ## Run tests.
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./e2e/... --ginkgo.v -v --ginkgo.trace

api/v1beta1/webhook_suite_test.go

@@ -208,9 +208,6 @@ var _ = BeforeSuite(func() {
 
 		WebhookInstallOptions: envtest.WebhookInstallOptions{
 			Paths: []string{filepath.Join("..", "..", "config", "webhook")},
-			//MutatingWebhooks:         nil,
-			//ValidatingWebhooks:       nil,
-			IgnoreErrorIfPathMissing: false,
 		},
 	}
 

e2e/automq_cluster_test.go

@@ -0,0 +1,165 @@
+/*
+Copyright 2024 cuisongliu.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	infrav1beta1 "github.com/cuisongliu/automq-operator/api/v1beta1"
+	//+kubebuilder:scaffold:imports
+)
+
+// These tests use Ginkgo (BDD-style Go testing framework). Refer to
+// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
+
+var cfg *rest.Config
+var k8sClient client.Client
+var testEnv *envtest.Environment
+var kubeconfigPath string
+
+func TestControllers(t *testing.T) {
+	RegisterFailHandler(Fail)
+
+	RunSpecs(t, "Controller Suite")
+}
+
+var _ = Describe("automq_controller", func() {
+	Context("install", func() {
+		It("PrometheusOperator", func() {
+			var err error
+			err = InstallPrometheusOperator()
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+	})
+	Context("automq_controller tests", func() {
+		ctx := context.Background()
+		namespaceName := "automq-operator"
+		namespace := &v1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      namespaceName,
+				Namespace: namespaceName,
+			},
+		}
+		automq := &infrav1beta1.AutoMQ{}
+		automq.Name = "automq-s1"
+		automq.Namespace = namespaceName
+		automq.Spec.ClusterID = "rZdE0DjZSrqy96PXrMUZVw"
+
+		BeforeEach(func() {
+			By("Creating the Namespace to perform the tests")
+			err := k8sClient.Create(ctx, namespace)
+			Expect(err).To(Not(HaveOccurred()))
+
+			By("Setting the NAMESPACE_NAME ENV VAR which stores the Operand image")
+			err = os.Setenv("NAMESPACE_NAME", namespaceName)
+			Expect(err).To(Not(HaveOccurred()))
+		})
+		It("Update Endpoint", func() {
+			By("creating the custom resource for the automq")
+			err := k8sClient.Get(ctx, client.ObjectKeyFromObject(automq), automq)
+			if err != nil && errors.IsNotFound(err) {
+				// Let's mock our custom resource at the same way that we would
+				// apply on the cluster the manifest under config/samples
+				automq.Spec.S3.Endpoint = "http://localhost:9000"
+				automq.Spec.S3.Bucket = "ko3"
+				automq.Spec.S3.AccessKeyID = "min"
+				automq.Spec.S3.SecretAccessKey = "test"
+				automq.Spec.S3.Region = "us-east-1"
+				err = k8sClient.Create(ctx, automq)
+				Expect(err).To(Not(HaveOccurred()))
+			}
+		})
+		AfterEach(func() {
+			By("removing the custom resource for the automq")
+			found := &infrav1beta1.AutoMQ{}
+			err := k8sClient.Get(ctx, client.ObjectKeyFromObject(automq), found)
+			Expect(err).To(Not(HaveOccurred()))
+
+			Eventually(func() error {
+				return k8sClient.Delete(context.TODO(), found)
+			}, 2*time.Minute, time.Second).Should(Succeed())
+
+			// TODO(user): Attention if you improve this code by adding other context test you MUST
+			// be aware of the current delete namespace limitations.
+			// More info: https://book.kubebuilder.io/reference/envtest.html#testing-considerations
+			By("Deleting the Namespace to perform the tests")
+			_ = k8sClient.Delete(ctx, namespace)
+
+			By("Removing the Image ENV VAR which stores the Operand image")
+			_ = os.Unsetenv("NAMESPACE_NAME")
+		})
+	})
+
+})
+
+var _ = BeforeSuite(func() {
+	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
+	usingCluster := true
+	By("bootstrapping test environment")
+	testEnv = &envtest.Environment{
+		CRDDirectoryPaths:     []string{filepath.Join("..", "config", "crd", "bases")},
+		ErrorIfCRDPathMissing: true,
+		UseExistingCluster:    &usingCluster,
+		// The BinaryAssetsDirectory is only required if you want to run the tests directly
+		// without call the makefile target test. If not informed it will look for the
+		// default path defined in controller-runtime which is /usr/local/kubebuilder/.
+		// Note that you must have the required binaries setup under the bin directory to perform
+		// the tests directly. When we run make test it will be setup and used automatically.
+		BinaryAssetsDirectory: filepath.Join("..", "bin", "k8s",
+			fmt.Sprintf("1.28.0-%s-%s", runtime.GOOS, runtime.GOARCH)),
+	}
+
+	// cfg is defined in this file globally.
+	cfg, err := testEnv.Start()
+	Expect(err).NotTo(HaveOccurred())
+	Expect(cfg).NotTo(BeNil())
+
+	err = infrav1beta1.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+
+	//+kubebuilder:scaffold:scheme
+
+	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
+	Expect(err).NotTo(HaveOccurred())
+	Expect(k8sClient).NotTo(BeNil())
+})
+
+var _ = AfterSuite(func() {
+	By("tearing down the test environment")
+	err := testEnv.Stop()
+	Expect(err).NotTo(HaveOccurred())
+})

e2e/utils.go

@@ -0,0 +1,105 @@
+/*
+Copyright 2024 [email protected].
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"os"
+	"os/exec"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2" //nolint:golint,revive
+)
+
+var (
+	prometheusOperatorURL = "config/deps/prometheus-operator.yaml"
+)
+
+// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics.
+func InstallPrometheusOperator() error {
+	cmd := exec.Command("kubectl", "apply", "-f", prometheusOperatorURL, "--server-side=true", "--overwrite=false", "--force-conflicts")
+	out, err := Run(cmd)
+	print(out)
+	return err
+}
+
+//docker.io/labring/minio:RELEASE.2024-01-11T07-46-16Z
+
+// Run executes the provided command within this context
+func Run(cmd *exec.Cmd) ([]byte, error) {
+	dir, _ := GetProjectDir()
+	cmd.Dir = dir
+	fmt.Fprintf(GinkgoWriter, "running dir: %s\n", cmd.Dir)
+
+	// To allow make commands be executed from the project directory which is subdir on SDK repo
+	// TODO:(user) You might not need the following code
+	if err := os.Chdir(cmd.Dir); err != nil {
+		fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err)
+	}
+
+	cmd.Env = append(os.Environ(), "GO111MODULE=on")
+	command := strings.Join(cmd.Args, " ")
+	fmt.Fprintf(GinkgoWriter, "running: %s\n", command)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return output, fmt.Errorf("%s failed with error: (%v) %s", command, err, string(output))
+	}
+
+	return output, nil
+}
+
+func GetProjectDir() (string, error) {
+	wd, err := os.Getwd()
+	if err != nil {
+		return wd, err
+	}
+	wd = strings.Replace(wd, "/internal/controller", "", -1)
+	wd = strings.Replace(wd, "/e2e", "", -1)
+	return wd, nil
+}
+
+func CreateKubeconfigFileForRestConfig(restConfig *rest.Config) string {
+	clusters := make(map[string]*clientcmdapi.Cluster)
+	clusters["default-cluster"] = &clientcmdapi.Cluster{
+		Server:                   restConfig.Host,
+		CertificateAuthorityData: restConfig.CAData,
+	}
+	contexts := make(map[string]*clientcmdapi.Context)
+	contexts["default-context"] = &clientcmdapi.Context{
+		Cluster:  "default-cluster",
+		AuthInfo: "default-user",
+	}
+	authinfos := make(map[string]*clientcmdapi.AuthInfo)
+	authinfos["default-user"] = &clientcmdapi.AuthInfo{
+		ClientCertificateData: restConfig.CertData,
+		ClientKeyData:         restConfig.KeyData,
+	}
+	clientConfig := clientcmdapi.Config{
+		Kind:           "Config",
+		APIVersion:     "v1",
+		Clusters:       clusters,
+		Contexts:       contexts,
+		CurrentContext: "default-context",
+		AuthInfos:      authinfos,
+	}
+	kubeConfigFile, _ := os.CreateTemp("", "kubeconfig")
+	_ = clientcmd.WriteToFile(clientConfig, kubeConfigFile.Name())
+	return kubeConfigFile.Name()
+}