diff --git a/bootstrap/api/v1beta1/kthreesconfig_types.go b/bootstrap/api/v1beta1/kthreesconfig_types.go index ff890fa7..59df6c19 100644 --- a/bootstrap/api/v1beta1/kthreesconfig_types.go +++ b/bootstrap/api/v1beta1/kthreesconfig_types.go @@ -103,6 +103,10 @@ type KThreesServerConfig struct { // DisableComponents specifies extra commands to run before k3s setup runs // +optional DisableComponents []string `json:"disableComponents,omitempty"` + + // DisableExternalCloudProvider suppresses the 'cloud-provider=external' kubelet argument. (default: false) + // +optional + DisableExternalCloudProvider bool `json:"disableExternalCloudProvider,omitempty"` } type KThreesAgentConfig struct { diff --git a/bootstrap/api/v1beta1/zz_generated.deepcopy.go b/bootstrap/api/v1beta1/zz_generated.deepcopy.go index 5314bf7c..d15c02cb 100644 --- a/bootstrap/api/v1beta1/zz_generated.deepcopy.go +++ b/bootstrap/api/v1beta1/zz_generated.deepcopy.go @@ -325,6 +325,11 @@ func (in *KThreesServerConfig) DeepCopyInto(out *KThreesServerConfig) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.KubeSchedulerArgs != nil { + in, out := &in.KubeSchedulerArgs, &out.KubeSchedulerArgs + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.TLSSan != nil { in, out := &in.TLSSan, &out.TLSSan *out = make([]string, len(*in)) diff --git a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml index d460c976..1f65748b 100644 --- a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml +++ b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml @@ -166,6 +166,10 @@ spec: items: type: string type: array + disableExternalCloudProvider: + description: 'DisableExternalCloudProvider suppresses the ''cloud-provider=external'' + kubelet argument. (default: false)' + type: boolean httpsListenPort: description: 'HTTPSListenPort HTTPS listen port (default: 6443)' type: string diff --git a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml index 26d64972..0817275d 100644 --- a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml +++ b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml @@ -180,6 +180,11 @@ spec: items: type: string type: array + disableExternalCloudProvider: + description: 'DisableExternalCloudProvider suppresses + the ''cloud-provider=external'' kubelet argument. (default: + false)' + type: boolean httpsListenPort: description: 'HTTPSListenPort HTTPS listen port (default: 6443)' diff --git a/bootstrap/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml b/bootstrap/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml index 296bd5ef..ab498d29 100644 --- a/bootstrap/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml +++ b/bootstrap/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml @@ -246,6 +246,11 @@ spec: items: type: string type: array + disableExternalCloudProvider: + description: 'DisableExternalCloudProvider suppresses the + ''cloud-provider=external'' kubelet argument. (default: + false)' + type: boolean httpsListenPort: description: 'HTTPSListenPort HTTPS listen port (default: 6443)' diff --git a/bootstrap/config/manager/kustomization.yaml b/bootstrap/config/manager/kustomization.yaml index 9beeff2d..a57bd0fa 100644 --- a/bootstrap/config/manager/kustomization.yaml +++ b/bootstrap/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: ghcr.io/cluster-api-provider-k3s/cluster-api-k3s/bootstrap-controller - newTag: v0.2.0 + newTag: v0.1.7 diff --git a/bootstrap/controllers/kthreesconfig_controller.go b/bootstrap/controllers/kthreesconfig_controller.go index d44780dd..5b360b9b 100644 --- a/bootstrap/controllers/kthreesconfig_controller.go +++ b/bootstrap/controllers/kthreesconfig_controller.go @@ -290,7 +290,7 @@ func (r *KThreesConfigReconciler) joinWorker(ctx context.Context, scope *Scope) return err } - configStruct := k3s.GenerateWorkerConfig(serverURL, tokn, scope.Config.Spec.AgentConfig) + configStruct := k3s.GenerateWorkerConfig(serverURL, tokn, scope.Config.Spec.ServerConfig, scope.Config.Spec.AgentConfig) b, err := kubeyaml.Marshal(configStruct) if err != nil { diff --git a/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml b/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml index 7e770a8a..1f65748b 100644 --- a/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml +++ b/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml @@ -166,6 +166,10 @@ spec: items: type: string type: array + disableExternalCloudProvider: + description: 'DisableExternalCloudProvider suppresses the ''cloud-provider=external'' + kubelet argument. (default: false)' + type: boolean httpsListenPort: description: 'HTTPSListenPort HTTPS listen port (default: 6443)' type: string @@ -181,6 +185,12 @@ spec: items: type: string type: array + kubeSchedulerArgs: + description: KubeSchedulerArgs is a customized flag for kube-scheduler + process + items: + type: string + type: array serviceCidr: description: 'ServiceCidr Network CIDR to use for services IPs (default: "10.43.0.0/16")' diff --git a/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml b/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml index 37dcc7f8..0817275d 100644 --- a/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml +++ b/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml @@ -180,6 +180,11 @@ spec: items: type: string type: array + disableExternalCloudProvider: + description: 'DisableExternalCloudProvider suppresses + the ''cloud-provider=external'' kubelet argument. (default: + false)' + type: boolean httpsListenPort: description: 'HTTPSListenPort HTTPS listen port (default: 6443)' @@ -196,6 +201,12 @@ spec: items: type: string type: array + kubeSchedulerArgs: + description: KubeSchedulerArgs is a customized flag for + kube-scheduler process + items: + type: string + type: array serviceCidr: description: 'ServiceCidr Network CIDR to use for services IPs (default: "10.43.0.0/16")' diff --git a/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml b/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml index 88519ed1..ab498d29 100644 --- a/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml +++ b/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml @@ -246,6 +246,11 @@ spec: items: type: string type: array + disableExternalCloudProvider: + description: 'DisableExternalCloudProvider suppresses the + ''cloud-provider=external'' kubelet argument. (default: + false)' + type: boolean httpsListenPort: description: 'HTTPSListenPort HTTPS listen port (default: 6443)' @@ -262,6 +267,12 @@ spec: items: type: string type: array + kubeSchedulerArgs: + description: KubeSchedulerArgs is a customized flag for kube-scheduler + process + items: + type: string + type: array serviceCidr: description: 'ServiceCidr Network CIDR to use for services IPs (default: "10.43.0.0/16")' diff --git a/controlplane/config/manager/kustomization.yaml b/controlplane/config/manager/kustomization.yaml index 2da61e4c..85d5a44b 100644 --- a/controlplane/config/manager/kustomization.yaml +++ b/controlplane/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: ghcr.io/cluster-api-provider-k3s/cluster-api-k3s/controlplane-controller - newTag: v0.2.0 + newTag: v0.1.7 diff --git a/pkg/k3s/config.go b/pkg/k3s/config.go index e1013ab8..1119051c 100644 --- a/pkg/k3s/config.go +++ b/pkg/k3s/config.go @@ -40,12 +40,13 @@ type K3sAgentConfig struct { } func GenerateInitControlPlaneConfig(controlPlaneEndpoint string, token string, serverConfig bootstrapv1.KThreesServerConfig, agentConfig bootstrapv1.KThreesAgentConfig) K3sServerConfig { + kubeletExtraArgs := getKubeletExtraArgs(serverConfig) k3sServerConfig := K3sServerConfig{ - DisableCloudController: true, + DisableCloudController: !serverConfig.DisableExternalCloudProvider, ClusterInit: true, KubeAPIServerArgs: append(serverConfig.KubeAPIServerArgs, "anonymous-auth=true", getTLSCipherSuiteArg()), TLSSan: append(serverConfig.TLSSan, controlPlaneEndpoint), - KubeControllerManagerArgs: append(serverConfig.KubeControllerManagerArgs, "cloud-provider=external"), + KubeControllerManagerArgs: append(serverConfig.KubeControllerManagerArgs, kubeletExtraArgs...), KubeSchedulerArgs: serverConfig.KubeSchedulerArgs, BindAddress: serverConfig.BindAddress, HTTPSListenPort: serverConfig.HTTPSListenPort, @@ -60,7 +61,7 @@ func GenerateInitControlPlaneConfig(controlPlaneEndpoint string, token string, s k3sServerConfig.K3sAgentConfig = K3sAgentConfig{ Token: token, - KubeletArgs: append(agentConfig.KubeletArgs, "cloud-provider=external"), + KubeletArgs: append(agentConfig.KubeletArgs, kubeletExtraArgs...), NodeLabels: agentConfig.NodeLabels, NodeTaints: agentConfig.NodeTaints, PrivateRegistry: agentConfig.PrivateRegistry, @@ -72,11 +73,12 @@ func GenerateInitControlPlaneConfig(controlPlaneEndpoint string, token string, s } func GenerateJoinControlPlaneConfig(serverURL string, token string, controlplaneendpoint string, serverConfig bootstrapv1.KThreesServerConfig, agentConfig bootstrapv1.KThreesAgentConfig) K3sServerConfig { + kubeletExtraArgs := getKubeletExtraArgs(serverConfig) k3sServerConfig := K3sServerConfig{ - DisableCloudController: true, + DisableCloudController: !serverConfig.DisableExternalCloudProvider, KubeAPIServerArgs: append(serverConfig.KubeAPIServerArgs, "anonymous-auth=true", getTLSCipherSuiteArg()), TLSSan: append(serverConfig.TLSSan, controlplaneendpoint), - KubeControllerManagerArgs: append(serverConfig.KubeControllerManagerArgs, "cloud-provider=external"), + KubeControllerManagerArgs: append(serverConfig.KubeControllerManagerArgs, kubeletExtraArgs...), KubeSchedulerArgs: serverConfig.KubeSchedulerArgs, BindAddress: serverConfig.BindAddress, HTTPSListenPort: serverConfig.HTTPSListenPort, @@ -92,7 +94,7 @@ func GenerateJoinControlPlaneConfig(serverURL string, token string, controlplane k3sServerConfig.K3sAgentConfig = K3sAgentConfig{ Token: token, Server: serverURL, - KubeletArgs: append(agentConfig.KubeletArgs, "cloud-provider=external"), + KubeletArgs: append(agentConfig.KubeletArgs, kubeletExtraArgs...), NodeLabels: agentConfig.NodeLabels, NodeTaints: agentConfig.NodeTaints, PrivateRegistry: agentConfig.PrivateRegistry, @@ -103,11 +105,12 @@ func GenerateJoinControlPlaneConfig(serverURL string, token string, controlplane return k3sServerConfig } -func GenerateWorkerConfig(serverURL string, token string, agentConfig bootstrapv1.KThreesAgentConfig) K3sAgentConfig { +func GenerateWorkerConfig(serverURL string, token string, serverConfig bootstrapv1.KThreesServerConfig, agentConfig bootstrapv1.KThreesAgentConfig) K3sAgentConfig { + kubeletExtraArgs := getKubeletExtraArgs(serverConfig) return K3sAgentConfig{ Server: serverURL, Token: token, - KubeletArgs: append(agentConfig.KubeletArgs, "cloud-provider=external"), + KubeletArgs: append(agentConfig.KubeletArgs, kubeletExtraArgs...), NodeLabels: agentConfig.NodeLabels, NodeTaints: agentConfig.NodeTaints, PrivateRegistry: agentConfig.PrivateRegistry, @@ -152,3 +155,11 @@ func getTLSCipherSuiteArg() string { return fmt.Sprintf("tls-cipher-suites=%s", ciphersList) } + +func getKubeletExtraArgs(serverConfig bootstrapv1.KThreesServerConfig) []string { + kubeletExtraArgs := []string{} + if !serverConfig.DisableExternalCloudProvider { + kubeletExtraArgs = append(kubeletExtraArgs, "cloud-provider=external") + } + return kubeletExtraArgs +}