Skip to content

Commit

Permalink
e2e: use helm to install out-of-tree cloud-provider-azure
Browse files Browse the repository at this point in the history
  • Loading branch information
@jackfrancis
jackfrancis committed Apr 2, 2022
1 parent d120f2d commit 1fc368f
Show file tree
Hide file tree
Showing 11 changed files with 534 additions and 735 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ bazel-*
/tilt.d
tilt-settings.json
tilt_config.json
*.kubeconfig

# e2e output
test/e2e/junit.e2e_suite.*.xml
Expand Down
16 changes: 15 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,9 @@ KUBECTL_VER := v1.22.4
KUBECTL_BIN := kubectl
KUBECTL := $(TOOLS_BIN_DIR)/$(KUBECTL_BIN)-$(KUBECTL_VER)

HELM_BIN := helm
HELM := $(TOOLS_BIN_DIR)/$(HELM_BIN)

YQ_VER := v4.14.2
YQ_BIN := yq
YQ := $(TOOLS_BIN_DIR)/$(YQ_BIN)-$(YQ_VER)
Expand Down Expand Up @@ -668,7 +671,7 @@ kind-create: $(KUBECTL) ## Create capz kind cluster if needed.
./scripts/kind-with-registry.sh

.PHONY: tilt-up
tilt-up: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) kind-create ## Start tilt and build kind cluster if needed.
tilt-up: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) kind-create ## Start tilt and build kind cluster if needed.
EXP_CLUSTER_RESOURCE_SET=true EXP_AKS=true EXP_MACHINE_POOL=true tilt up

.PHONY: delete-cluster
Expand Down Expand Up @@ -740,12 +743,23 @@ $(KUBECTL): ## Build kubectl from tools folder.
ln -sf $(KUBECTL) $(TOOLS_BIN_DIR)/$(KUBECTL_BIN)
chmod +x $(KUBECTL) $(TOOLS_BIN_DIR)/$(KUBECTL_BIN)

$(HELM): ## Put helm into tools folder.
mkdir -p $(TOOLS_BIN_DIR)
rm -f "$(TOOLS_BIN_DIR)/$(HELM_BIN)*"
curl -fsSL -o $(TOOLS_BIN_DIR)/get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
chmod 700 $(TOOLS_BIN_DIR)/get_helm.sh
USE_SUDO=false HELM_INSTALL_DIR=$(TOOLS_BIN_DIR) $(TOOLS_BIN_DIR)/get_helm.sh
rm -f $(TOOLS_BIN_DIR)/get_helm.sh

.PHONY: $(ENVSUBST_BIN)
$(ENVSUBST_BIN): $(ENVSUBST)

.PHONY: $(KUBECTL_BIN)
$(KUBECTL_BIN): $(KUBECTL)

.PHONY: $(HELM_BIN)
$(HELM_BIN): $(HELM)

.PHONY: $(GO_APIDIFF_BIN)
$(GO_APIDIFF_BIN): $(GO_APIDIFF)

Expand Down
9 changes: 7 additions & 2 deletions Tiltfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

envsubst_cmd = "./hack/tools/bin/envsubst"
kubectl_cmd = "./hack/tools/bin/kubectl"
helm_cmd = "./hack/tools/bin/helm"
tools_bin = "./hack/tools/bin"

#Add tools to path
Expand Down Expand Up @@ -331,9 +332,13 @@ def deploy_worker_templates(template, substitutions):
yaml = yaml.replace("${" + substitution + "}", value)

yaml = yaml.replace('"', '\\"') # add escape character to double quotes in yaml
flavor_name = os.path.basename(flavor)
flavor_cmd = "RANDOM=$(bash -c 'echo $RANDOM'); CLUSTER_NAME=" + flavor.replace("windows", "win") + "-$RANDOM; make generate-flavors; echo \"" + yaml + "\" > ./.tiltbuild/" + flavor + "; cat ./.tiltbuild/" + flavor + " | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f - && echo \"Cluster \'$CLUSTER_NAME\' created, don't forget to delete\""
if "external-cloud-provider" in flavor_name:
flavor_cmd += "; until " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig; do sleep 5; done; " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig -o jsonpath={.data.value} | base64 --decode > ./${CLUSTER_NAME}.kubeconfig; until " + kubectl_cmd + " --kubeconfig=./${CLUSTER_NAME}.kubeconfig get nodes; do sleep 5; done; " + helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure --generate-name --set infra.clusterName=${CLUSTER_NAME}"
local_resource(
name = os.path.basename(flavor),
cmd = "RANDOM=$(bash -c 'echo $RANDOM'); CLUSTER_NAME=" + flavor.replace("windows", "win") + "-$RANDOM; make generate-flavors; echo \"" + yaml + "\" > ./.tiltbuild/" + flavor + "; cat ./.tiltbuild/" + flavor + " | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f - && echo \"Cluster \'$CLUSTER_NAME\' created, don't forget to delete\"",
name = flavor_name,
cmd = flavor_cmd,
auto_init = False,
trigger_mode = TRIGGER_MODE_MANUAL,
labels = ["flavors"],
Expand Down
14 changes: 13 additions & 1 deletion docs/book/src/topics/cloud-provider-config.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,15 @@ All cloud provider config values can be customized by creating the `${RESOURCE}-

To deploy a cluster using [external cloud provider](https://github.com/kubernetes-sigs/cloud-provider-azure), create a cluster configuration with the [external cloud provider template](https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-azure/main/templates/cluster-template-external-cloud-provider.yaml).

After deploying the cluster, you should eventually see a set of pods like the following in a `Running` state:
After the cluster has provisioned, install the `cloud-provider-azure` components using the official helm chart:

```bash
helm install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure --generate-name --set infra.clusterName=${CLUSTER_NAME}
```

The Helm chart will pick the right version of `cloud-controller-manager` and `cloud-node-manager` to work with the version of Kubernetes your cluster is running.

After running `helm install`, you should eventually see a set of pods like these in a `Running` state:

```bash
kube-system cloud-controller-manager 1/1 Running 0 41s
Expand All @@ -81,6 +89,10 @@ kube-system cloud-node-manager-mfsdg
kube-system cloud-node-manager-qrz74 1/1 Running 0 24s
```

For more information see the official [`cloud-provider-azure` helm chart documentation](https://github.com/kubernetes-sigs/cloud-provider-azure/tree/master/helm/cloud-provider-azure).

If you're not familiar with using Helm to manage Kubernetes applications as packages, there's lots of good [Helm documentation on the official website](https://helm.sh/docs/).

## Storage Drivers

### Azure File CSI Driver
Expand Down
76 changes: 55 additions & 21 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,16 @@ require (
go.opentelemetry.io/otel/sdk v1.4.0
go.opentelemetry.io/otel/sdk/metric v0.27.0
go.opentelemetry.io/otel/trace v1.4.0
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871
golang.org/x/mod v0.5.1
k8s.io/api v0.23.0
k8s.io/apimachinery v0.23.0
k8s.io/client-go v0.23.0
k8s.io/component-base v0.23.0
helm.sh/helm/v3 v3.8.1
k8s.io/api v0.23.4
k8s.io/apimachinery v0.23.4
k8s.io/client-go v0.23.4
k8s.io/component-base v0.23.4
k8s.io/klog/v2 v2.30.0
k8s.io/kubectl v0.23.0
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b
k8s.io/kubectl v0.23.4
k8s.io/utils v0.0.0-20211116205334-6203023598ed
sigs.k8s.io/cluster-api v1.1.1
sigs.k8s.io/cluster-api/test v1.1.2
sigs.k8s.io/controller-runtime v0.11.1
Expand All @@ -54,9 +55,13 @@ require (
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/BurntSushi/toml v0.3.1 // indirect
github.com/BurntSushi/toml v0.4.1 // indirect
github.com/MakeNowJust/heredoc v1.0.0 // indirect
github.com/Microsoft/go-winio v0.5.0 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.1.1 // indirect
github.com/Masterminds/sprig/v3 v3.2.2 // indirect
github.com/Masterminds/squirrel v1.5.2 // indirect
github.com/Microsoft/go-winio v0.5.1 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/alessio/shellescape v1.4.1 // indirect
Expand All @@ -68,16 +73,21 @@ require (
github.com/containerd/containerd v1.5.9 // indirect
github.com/coredns/caddy v1.1.0 // indirect
github.com/coredns/corefile-migration v1.0.14 // indirect
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/docker/cli v20.10.11+incompatible // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v20.10.12+incompatible // indirect
github.com/docker/docker-credential-helpers v0.6.4 // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/felixge/httpsnoop v1.0.2 // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-errors/errors v1.0.1 // indirect
Expand All @@ -86,6 +96,7 @@ require (
github.com/go-openapi/jsonreference v0.19.5 // indirect
github.com/go-openapi/swag v0.19.14 // indirect
github.com/gobuffalo/flect v0.2.4 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.0.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
Expand All @@ -96,27 +107,41 @@ require (
github.com/google/go-querystring v1.0.0 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gosuri/uitable v0.0.4 // indirect
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jmoiron/sqlx v1.3.4 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
github.com/lib/pq v1.10.4 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/magiconair/properties v1.8.5 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.0 // indirect
github.com/mitchellh/mapstructure v1.4.2 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/locker v1.0.1 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
Expand All @@ -126,49 +151,58 @@ require (
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc // indirect
github.com/russross/blackfriday v1.5.2 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/spf13/afero v1.6.0 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/cobra v1.2.1 // indirect
github.com/spf13/cobra v1.3.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/viper v1.9.0 // indirect
github.com/spf13/viper v1.10.0 // indirect
github.com/stoewer/go-strcase v1.2.0 // indirect
github.com/stretchr/testify v1.7.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/valyala/fastjson v1.6.3 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.4.0 // indirect
go.opentelemetry.io/otel/internal/metric v0.27.0 // indirect
go.opentelemetry.io/proto/otlp v0.12.0 // indirect
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
golang.org/x/net v0.0.0-20210825183410-e898025ed96a // indirect
golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 // indirect
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
google.golang.org/grpc v1.44.0 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/gorp.v1 v1.7.2 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.63.2 // indirect
gopkg.in/ini.v1 v1.66.2 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
k8s.io/apiextensions-apiserver v0.23.0 // indirect
k8s.io/apiserver v0.23.0 // indirect
k8s.io/cli-runtime v0.23.0 // indirect
k8s.io/apiextensions-apiserver v0.23.4 // indirect
k8s.io/apiserver v0.23.4 // indirect
k8s.io/cli-runtime v0.23.4 // indirect
k8s.io/cluster-bootstrap v0.23.0 // indirect
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
oras.land/oras-go v1.1.0 // indirect
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
sigs.k8s.io/kustomize/api v0.10.1 // indirect
sigs.k8s.io/kustomize/kyaml v0.13.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

Expand Down
Loading

0 comments on commit 1fc368f

Please sign in to comment.