diff --git a/artifacts/agent/karmada-agent.yaml b/artifacts/agent/karmada-agent.yaml index 4f4eb5801763..0fcd1a579c28 100644 --- a/artifacts/agent/karmada-agent.yaml +++ b/artifacts/agent/karmada-agent.yaml @@ -25,7 +25,7 @@ spec: imagePullPolicy: {{image_pull_policy}} command: - /bin/karmada-agent - - --karmada-kubeconfig=/etc/kubeconfig/karmada-kubeconfig + - --karmada-kubeconfig=/etc/karmada/config/karmada.config - --karmada-context={{karmada_context}} - --cluster-name={{member_cluster_name}} - --cluster-api-endpoint={{member_cluster_api_endpoint}} @@ -48,9 +48,9 @@ spec: name: metrics protocol: TCP volumeMounts: - - name: kubeconfig - mountPath: /etc/kubeconfig + - name: karmada-config + mountPath: /etc/karmada/config volumes: - - name: kubeconfig + - name: karmada-config secret: - secretName: karmada-kubeconfig + secretName: karmada-config diff --git a/artifacts/deploy/karmada-aggregated-apiserver.yaml b/artifacts/deploy/karmada-aggregated-apiserver.yaml index 58493c5ceff0..1f37b9d50ab7 100644 --- a/artifacts/deploy/karmada-aggregated-apiserver.yaml +++ b/artifacts/deploy/karmada-aggregated-apiserver.yaml @@ -25,23 +25,26 @@ spec: image: docker.io/karmada/karmada-aggregated-apiserver:latest imagePullPolicy: IfNotPresent volumeMounts: - - name: karmada-certs - mountPath: /etc/karmada/pki + - name: server-cert + mountPath: /etc/karmada/pki/server + readOnly: true + - name: etcd-client-cert + mountPath: /etc/karmada/pki/etcd-client + readOnly: true + - name: karmada-config + mountPath: /etc/karmada/config readOnly: true - - name: kubeconfig - subPath: kubeconfig - mountPath: /etc/kubeconfig command: - /bin/karmada-aggregated-apiserver - - --kubeconfig=/etc/kubeconfig - - --authentication-kubeconfig=/etc/kubeconfig - - --authorization-kubeconfig=/etc/kubeconfig + - --kubeconfig=/etc/karmada/config/karmada.config + - --authentication-kubeconfig=/etc/karmada/config/karmada.config + - --authorization-kubeconfig=/etc/karmada/config/karmada.config - --etcd-servers=https://etcd-client.karmada-system.svc.cluster.local:2379 - - --etcd-cafile=/etc/karmada/pki/etcd-ca.crt - - --etcd-certfile=/etc/karmada/pki/etcd-client.crt - - --etcd-keyfile=/etc/karmada/pki/etcd-client.key - - --tls-cert-file=/etc/karmada/pki/karmada.crt - - --tls-private-key-file=/etc/karmada/pki/karmada.key + - --etcd-cafile=/etc/karmada/pki/etcd-client/ca.crt + - --etcd-certfile=/etc/karmada/pki/etcd-client/tls.crt + - --etcd-keyfile=/etc/karmada/pki/etcd-client/tls.key + - --tls-cert-file=/etc/karmada/pki/server/tls.crt + - --tls-private-key-file=/etc/karmada/pki/server/tls.key - --audit-log-path=- - --audit-log-maxage=0 - --audit-log-maxbackup=0 @@ -66,12 +69,15 @@ spec: periodSeconds: 10 timeoutSeconds: 15 volumes: - - name: karmada-certs + - name: server-cert + secret: + secretName: server-cert + - name: etcd-client-cert secret: - secretName: karmada-cert-secret - - name: kubeconfig + secretName: etcd-client-cert + - name: karmada-config secret: - secretName: kubeconfig + secretName: karmada-config --- apiVersion: v1 kind: Service diff --git a/artifacts/deploy/karmada-apiserver.yaml b/artifacts/deploy/karmada-apiserver.yaml index 5d5c2f458b5f..aba045ed64d3 100644 --- a/artifacts/deploy/karmada-apiserver.yaml +++ b/artifacts/deploy/karmada-apiserver.yaml @@ -36,29 +36,29 @@ spec: - kube-apiserver - --allow-privileged=true - --authorization-mode=Node,RBAC - - --client-ca-file=/etc/karmada/pki/ca.crt + - --client-ca-file=/etc/karmada/pki/server/ca.crt - --enable-bootstrap-token-auth=true - - --etcd-cafile=/etc/karmada/pki/etcd-ca.crt - - --etcd-certfile=/etc/karmada/pki/etcd-client.crt - - --etcd-keyfile=/etc/karmada/pki/etcd-client.key + - --etcd-cafile=/etc/karmada/pki/etcd-client/ca.crt + - --etcd-certfile=/etc/karmada/pki/etcd-client/tls.crt + - --etcd-keyfile=/etc/karmada/pki/etcd-client/tls.key - --etcd-servers=https://etcd-client.karmada-system.svc.cluster.local:2379 - --bind-address=0.0.0.0 - --disable-admission-plugins=StorageObjectInUseProtection,ServiceAccount - --runtime-config= - --secure-port=5443 - --service-account-issuer=https://kubernetes.default.svc.cluster.local - - --service-account-key-file=/etc/karmada/pki/karmada.key - - --service-account-signing-key-file=/etc/karmada/pki/karmada.key + - --service-account-key-file=/etc/karmada/pki/server/tls.key + - --service-account-signing-key-file=/etc/karmada/pki/server/tls.key - --service-cluster-ip-range=10.96.0.0/12 - - --proxy-client-cert-file=/etc/karmada/pki/front-proxy-client.crt - - --proxy-client-key-file=/etc/karmada/pki/front-proxy-client.key + - --proxy-client-cert-file=/etc/karmada/pki/front-proxy-client/tls.crt + - --proxy-client-key-file=/etc/karmada/pki/front-proxy-client/tls.key - --requestheader-allowed-names=front-proxy-client - - --requestheader-client-ca-file=/etc/karmada/pki/front-proxy-ca.crt + - --requestheader-client-ca-file=/etc/karmada/pki/front-proxy-client/ca.crt - --requestheader-extra-headers-prefix=X-Remote-Extra- - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User - - --tls-cert-file=/etc/karmada/pki/apiserver.crt - - --tls-private-key-file=/etc/karmada/pki/apiserver.key + - --tls-cert-file=/etc/karmada/pki/server/tls.crt + - --tls-private-key-file=/etc/karmada/pki/server/tls.key - --tls-min-version=VersionTLS13 name: karmada-apiserver image: registry.k8s.io/kube-apiserver:{{karmada_apiserver_version}} @@ -88,8 +88,14 @@ spec: terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - - mountPath: /etc/karmada/pki - name: karmada-certs + - mountPath: /etc/karmada/pki/server + name: server-cert + readOnly: true + - mountPath: /etc/karmada/pki/front-proxy-client + name: front-proxy-client-cert + readOnly: true + - mountPath: /etc/karmada/pki/etcd-client + name: etcd-client-cert readOnly: true dnsPolicy: ClusterFirstWithHostNet enableServiceLinks: true @@ -105,9 +111,18 @@ spec: - effect: NoExecute operator: Exists volumes: - - name: karmada-certs + - name: server-cert + secret: + secretName: server-cert + - name: client-cert + secret: + secretName: client-cert + - name: front-proxy-client-cert + secret: + secretName: front-proxy-client-cert + - name: etcd-client-cert secret: - secretName: karmada-cert-secret + secretName: etcd-client-cert --- apiVersion: v1 kind: Service diff --git a/artifacts/deploy/karmada-cert-secret.yaml b/artifacts/deploy/karmada-cert-secret.yaml deleted file mode 100644 index 2a32ae4b2eb5..000000000000 --- a/artifacts/deploy/karmada-cert-secret.yaml +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: karmada-cert-secret - namespace: karmada-system -type: Opaque -data: - ca.crt: | - {{ca_crt}} - ca.key: | - {{ca_key}} - karmada.crt: | - {{client_crt}} - karmada.key: | - {{client_key}} - apiserver.crt: | - {{apiserver_crt}} - apiserver.key: | - {{apiserver_key}} - front-proxy-ca.crt: | - {{front_proxy_ca_crt}} - front-proxy-client.crt: | - {{front_proxy_client_crt}} - front-proxy-client.key: | - {{front_proxy_client_key}} - etcd-ca.crt: | - {{etcd_ca_crt}} - etcd-server.crt: | - {{etcd_server_crt}} - etcd-server.key: | - {{etcd_server_key}} - etcd-client.crt: | - {{etcd_client_crt}} - etcd-client.key: | - {{etcd_client_key}} diff --git a/artifacts/deploy/karmada-controller-manager.yaml b/artifacts/deploy/karmada-controller-manager.yaml index 6e5afc50856d..66871ff4da9e 100644 --- a/artifacts/deploy/karmada-controller-manager.yaml +++ b/artifacts/deploy/karmada-controller-manager.yaml @@ -25,7 +25,7 @@ spec: imagePullPolicy: IfNotPresent command: - /bin/karmada-controller-manager - - --kubeconfig=/etc/kubeconfig + - --kubeconfig=/etc/karmada/config/karmada.config - --metrics-bind-address=:8080 - --cluster-status-update-frequency=10s - --failover-eviction-timeout=30s @@ -47,10 +47,9 @@ spec: name: metrics protocol: TCP volumeMounts: - - name: kubeconfig - subPath: kubeconfig - mountPath: /etc/kubeconfig + - name: karmada-config + mountPath: /etc/karmada/config volumes: - - name: kubeconfig + - name: karmada-config secret: - secretName: kubeconfig + secretName: karmada-config diff --git a/artifacts/deploy/karmada-descheduler.yaml b/artifacts/deploy/karmada-descheduler.yaml index 696507be4b2c..25b4606144d9 100644 --- a/artifacts/deploy/karmada-descheduler.yaml +++ b/artifacts/deploy/karmada-descheduler.yaml @@ -25,12 +25,12 @@ spec: imagePullPolicy: IfNotPresent command: - /bin/karmada-descheduler - - --kubeconfig=/etc/kubeconfig + - --kubeconfig=/etc/karmada/config/karmada.config - --metrics-bind-address=0.0.0.0:8080 - --health-probe-bind-address=0.0.0.0:10358 - - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt - - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt - - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key + - --scheduler-estimator-ca-file=/etc/karmada/pki/client/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/client/tls.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/client/tls.key - --v=4 livenessProbe: httpGet: @@ -46,16 +46,16 @@ spec: name: metrics protocol: TCP volumeMounts: - - name: karmada-certs - mountPath: /etc/karmada/pki + - name: client-cert + mountPath: /etc/karmada/pki/client + readOnly: true + - name: karmada-config + mountPath: /etc/karmada/config/ readOnly: true - - name: kubeconfig - subPath: kubeconfig - mountPath: /etc/kubeconfig volumes: - - name: karmada-certs + - name: client-cert secret: - secretName: karmada-cert-secret - - name: kubeconfig + secretName: client-cert + - name: karmada-config secret: - secretName: kubeconfig + secretName: karmada-config diff --git a/artifacts/deploy/karmada-etcd.yaml b/artifacts/deploy/karmada-etcd.yaml index d429700b0ebf..cd377e5d46af 100644 --- a/artifacts/deploy/karmada-etcd.yaml +++ b/artifacts/deploy/karmada-etcd.yaml @@ -40,7 +40,7 @@ spec: command: - /bin/sh - -ec - - 'etcdctl get /registry --prefix --keys-only --endpoints https://127.0.0.1:2379 --cacert /etc/karmada/pki/etcd-ca.crt --cert /etc/karmada/pki/etcd-server.crt --key /etc/karmada/pki/etcd-server.key' + - 'etcdctl get /registry --prefix --keys-only --endpoints https://127.0.0.1:2379 --cacert /etc/karmada/pki/etcd-client/ca.crt --cert /etc/karmada/pki/etcd-client/tls.crt --key /etc/karmada/pki/etcd-client/tls.key' failureThreshold: 3 initialDelaySeconds: 600 periodSeconds: 60 @@ -56,8 +56,10 @@ spec: volumeMounts: - mountPath: /var/lib/etcd name: etcd-data - - mountPath: /etc/karmada/pki - name: etcd-certs + - mountPath: /etc/karmada/pki/etcd-server/ + name: etcd-server-cert + - mountPath: /etc/karmada/pki/etcd-client/ + name: etcd-client-cert resources: requests: cpu: 100m @@ -76,10 +78,10 @@ spec: - etcd0=http://etcd-0.etcd.karmada-system.svc.cluster.local:2380 - --initial-cluster-state - new - - --cert-file=/etc/karmada/pki/etcd-server.crt + - --cert-file=/etc/karmada/pki/etcd-server/tls.crt - --client-cert-auth=true - - --key-file=/etc/karmada/pki/etcd-server.key - - --trusted-ca-file=/etc/karmada/pki/etcd-ca.crt + - --key-file=/etc/karmada/pki/etcd-server/tls.key + - --trusted-ca-file=/etc/karmada/pki/etcd-server/ca.crt - --data-dir=/var/lib/etcd - --snapshot-count=10000 # Setting Golang's secure cipher suites as etcd's cipher suites. @@ -91,9 +93,12 @@ spec: path: /var/lib/karmada-etcd type: DirectoryOrCreate name: etcd-data - - name: etcd-certs + - name: etcd-server-cert secret: - secretName: karmada-cert-secret + secretName: etcd-server-cert + - name: etcd-client-cert + secret: + secretName: etcd-client-cert --- apiVersion: v1 diff --git a/artifacts/deploy/karmada-metrics-adapter.yaml b/artifacts/deploy/karmada-metrics-adapter.yaml index 437b18c419b0..03124ae25fac 100644 --- a/artifacts/deploy/karmada-metrics-adapter.yaml +++ b/artifacts/deploy/karmada-metrics-adapter.yaml @@ -25,20 +25,20 @@ spec: image: docker.io/karmada/karmada-metrics-adapter:latest imagePullPolicy: IfNotPresent volumeMounts: - - name: karmada-certs - mountPath: /etc/karmada/pki + - name: server-cert + mountPath: /etc/karmada/pki/server + readOnly: true + - name: karmada-config + mountPath: /etc/karmada/config readOnly: true - - name: kubeconfig - subPath: kubeconfig - mountPath: /etc/kubeconfig command: - /bin/karmada-metrics-adapter - - --kubeconfig=/etc/kubeconfig - - --authentication-kubeconfig=/etc/kubeconfig - - --authorization-kubeconfig=/etc/kubeconfig - - --client-ca-file=/etc/karmada/pki/ca.crt - - --tls-cert-file=/etc/karmada/pki/karmada.crt - - --tls-private-key-file=/etc/karmada/pki/karmada.key + - --kubeconfig=/etc/karmada/config/karmada.config + - --authentication-kubeconfig=/etc/karmada/config/karmada.config + - --authorization-kubeconfig=/etc/karmada/config/karmada.config + - --client-ca-file=/etc/karmada/pki/server/ca.crt + - --tls-cert-file=/etc/karmada/pki/server/tls.crt + - --tls-private-key-file=/etc/karmada/pki/server/tls.key - --audit-log-path=- - --audit-log-maxage=0 - --audit-log-maxbackup=0 @@ -65,12 +65,12 @@ spec: requests: cpu: 100m volumes: - - name: karmada-certs + - name: server-cert secret: - secretName: karmada-cert-secret - - name: kubeconfig + secretName: server-cert + - name: karmada-config secret: - secretName: kubeconfig + secretName: karmada-config --- apiVersion: v1 kind: Service diff --git a/artifacts/deploy/karmada-scheduler-estimator.yaml b/artifacts/deploy/karmada-scheduler-estimator.yaml index b1ed2a3abaed..79d4af5aa2e4 100644 --- a/artifacts/deploy/karmada-scheduler-estimator.yaml +++ b/artifacts/deploy/karmada-scheduler-estimator.yaml @@ -27,9 +27,9 @@ spec: - /bin/karmada-scheduler-estimator - --kubeconfig=/etc/{{member_cluster_name}}-kubeconfig - --cluster-name={{member_cluster_name}} - - --grpc-auth-cert-file=/etc/karmada/pki/karmada.crt - - --grpc-auth-key-file=/etc/karmada/pki/karmada.key - - --grpc-client-ca-file=/etc/karmada/pki/ca.crt + - --grpc-auth-cert-file=/etc/karmada/pki/server/tls.crt + - --grpc-auth-key-file=/etc/karmada/pki/server/tls.key + - --grpc-client-ca-file=/etc/karmada/pki/server/ca.crt - --metrics-bind-address=0.0.0.0:8080 - --health-probe-bind-address=0.0.0.0:10351 livenessProbe: @@ -46,16 +46,16 @@ spec: name: metrics protocol: TCP volumeMounts: - - name: karmada-certs - mountPath: /etc/karmada/pki + - name: server-cert + mountPath: /etc/karmada/pki/server readOnly: true - name: member-kubeconfig subPath: {{member_cluster_name}}-kubeconfig mountPath: /etc/{{member_cluster_name}}-kubeconfig volumes: - - name: karmada-certs + - name: server-cert secret: - secretName: karmada-cert-secret + secretName: server-cert - name: member-kubeconfig secret: secretName: {{member_cluster_name}}-kubeconfig diff --git a/artifacts/deploy/karmada-scheduler.yaml b/artifacts/deploy/karmada-scheduler.yaml index 78ea39224650..8a99c00292df 100644 --- a/artifacts/deploy/karmada-scheduler.yaml +++ b/artifacts/deploy/karmada-scheduler.yaml @@ -38,25 +38,25 @@ spec: protocol: TCP command: - /bin/karmada-scheduler - - --kubeconfig=/etc/kubeconfig + - --kubeconfig=/etc/karmada/config/karmada.config - --metrics-bind-address=0.0.0.0:8080 - --health-probe-bind-address=0.0.0.0:10351 - --enable-scheduler-estimator=true - - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt - - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt - - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key + - --scheduler-estimator-ca-file=/etc/karmada/pki/client/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/client/tls.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/client/tls.key - --v=4 volumeMounts: - - name: karmada-certs - mountPath: /etc/karmada/pki + - name: client-cert + mountPath: /etc/karmada/pki/client + readOnly: true + - name: karmada-config + mountPath: /etc/karmada/config/ readOnly: true - - name: kubeconfig - subPath: kubeconfig - mountPath: /etc/kubeconfig volumes: - - name: karmada-certs + - name: client-cert secret: - secretName: karmada-cert-secret - - name: kubeconfig + secretName: client-cert + - name: karmada-config secret: - secretName: kubeconfig + secretName: karmada-config diff --git a/artifacts/deploy/karmada-search.yaml b/artifacts/deploy/karmada-search.yaml index b972096f05dc..a6adf8f51ddc 100644 --- a/artifacts/deploy/karmada-search.yaml +++ b/artifacts/deploy/karmada-search.yaml @@ -25,23 +25,26 @@ spec: image: docker.io/karmada/karmada-search:latest imagePullPolicy: IfNotPresent volumeMounts: - - name: karmada-certs - mountPath: /etc/karmada/pki + - name: server-cert + mountPath: /etc/karmada/pki/server + readOnly: true + - name: etcd-client-cert + mountPath: /etc/karmada/pki/etcd-client + readOnly: true + - name: karmada-config + mountPath: /etc/karmada/config readOnly: true - - name: kubeconfig - subPath: kubeconfig - mountPath: /etc/kubeconfig command: - /bin/karmada-search - - --kubeconfig=/etc/kubeconfig - - --authentication-kubeconfig=/etc/kubeconfig - - --authorization-kubeconfig=/etc/kubeconfig + - --kubeconfig=/etc/karmada/config/karmada.config + - --authentication-kubeconfig=/etc/karmada/config/karmada.config + - --authorization-kubeconfig=/etc/karmada/config/karmada.config - --etcd-servers=https://etcd-client.karmada-system.svc.cluster.local:2379 - - --etcd-cafile=/etc/karmada/pki/etcd-ca.crt - - --etcd-certfile=/etc/karmada/pki/etcd-client.crt - - --etcd-keyfile=/etc/karmada/pki/etcd-client.key - - --tls-cert-file=/etc/karmada/pki/karmada.crt - - --tls-private-key-file=/etc/karmada/pki/karmada.key + - --etcd-cafile=/etc/karmada/pki/etcd-client/ca.crt + - --etcd-certfile=/etc/karmada/pki/etcd-client/tls.crt + - --etcd-keyfile=/etc/karmada/pki/etcd-client/tls.key + - --tls-cert-file=/etc/karmada/pki/server/tls.crt + - --tls-private-key-file=/etc/karmada/pki/server/tls.key - --audit-log-path=- - --audit-log-maxage=0 - --audit-log-maxbackup=0 @@ -59,12 +62,15 @@ spec: requests: cpu: 100m volumes: - - name: karmada-certs + - name: server-cert + secret: + secretName: server-cert + - name: etcd-client-cert secret: - secretName: karmada-cert-secret - - name: kubeconfig + secretName: etcd-client-cert + - name: karmada-config secret: - secretName: kubeconfig + secretName: karmada-config --- apiVersion: v1 kind: Service diff --git a/artifacts/deploy/karmada-secret-cert.yaml b/artifacts/deploy/karmada-secret-cert.yaml new file mode 100644 index 000000000000..e4fa812ac461 --- /dev/null +++ b/artifacts/deploy/karmada-secret-cert.yaml @@ -0,0 +1,97 @@ +apiVersion: v1 +kind: Secret +metadata: + name: ca-key-pair + namespace: karmada-system +type: kubernetes.io/tls +data: + tls.crt: | + {{ca_crt}} + tls.key: | + {{ca_key}} +--- +# server.crt: mainly used as server certificate for karmada components, such as karmada-apiserver, karmada-search, karmada-metrics-adapter, etc. +apiVersion: v1 +kind: Secret +metadata: + name: server-cert + namespace: karmada-system +type: kubernetes.io/tls +data: + ca.crt: | + {{ca_crt}} + tls.crt: | + {{server_crt}} + tls.key: | + {{server_key}} +--- +# client.crt: mainly used to construct kubeconfig for other components to access karmada-apiserver. +apiVersion: v1 +kind: Secret +metadata: + name: client-cert + namespace: karmada-system +type: kubernetes.io/tls +data: + ca.crt: | + {{ca_crt}} + tls.crt: | + {{client_crt}} + tls.key: | + {{client_key}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: front-proxy-client-cert + namespace: karmada-system +type: kubernetes.io/tls +data: + ca.crt: | + {{ca_crt}} + tls.crt: | + {{front_proxy_client_crt}} + tls.key: | + {{front_proxy_client_key}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: etcd-server-cert + namespace: karmada-system +type: kubernetes.io/tls +data: + ca.crt: | + {{ca_crt}} + tls.crt: | + {{etcd_server_crt}} + tls.key: | + {{etcd_server_key}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: etcd-client-cert + namespace: karmada-system +type: kubernetes.io/tls +data: + ca.crt: | + {{ca_crt}} + tls.crt: | + {{etcd_client_crt}} + tls.key: | + {{etcd_client_key}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: webhook-server-cert + namespace: karmada-system +type: kubernetes.io/tls +data: + ca.crt: | + {{ca_crt}} + tls.crt: | + {{webhook_server_crt}} + tls.key: | + {{webhook_server_key}} diff --git a/artifacts/deploy/secret.yaml b/artifacts/deploy/karmada-secret-config.yaml similarity index 93% rename from artifacts/deploy/secret.yaml rename to artifacts/deploy/karmada-secret-config.yaml index be55726f8a81..5638e61b4f5e 100644 --- a/artifacts/deploy/secret.yaml +++ b/artifacts/deploy/karmada-secret-config.yaml @@ -1,6 +1,6 @@ apiVersion: v1 stringData: - kubeconfig: |- + karmada.config: |- apiVersion: v1 clusters: - cluster: @@ -22,5 +22,5 @@ stringData: client-key-data: {{client_key}} kind: Secret metadata: - name: kubeconfig + name: karmada-config namespace: karmada-system diff --git a/artifacts/deploy/karmada-webhook-cert-secret.yaml b/artifacts/deploy/karmada-webhook-cert-secret.yaml deleted file mode 100644 index aabdeedc2ef2..000000000000 --- a/artifacts/deploy/karmada-webhook-cert-secret.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: webhook-cert - namespace: karmada-system -type: kubernetes.io/tls -data: - tls.crt: | - {{server_certificate}} - tls.key: | - {{server_key}} diff --git a/artifacts/deploy/karmada-webhook.yaml b/artifacts/deploy/karmada-webhook.yaml index bd54acec983c..ec09aa0ed25a 100644 --- a/artifacts/deploy/karmada-webhook.yaml +++ b/artifacts/deploy/karmada-webhook.yaml @@ -25,13 +25,13 @@ spec: imagePullPolicy: IfNotPresent command: - /bin/karmada-webhook - - --kubeconfig=/etc/kubeconfig + - --kubeconfig=/etc/karmada/config/karmada.config - --bind-address=0.0.0.0 - --metrics-bind-address=:8080 - --default-not-ready-toleration-seconds=30 - --default-unreachable-toleration-seconds=30 - --secure-port=8443 - - --cert-dir=/var/serving-cert + - --cert-dir=/etc/karmada/pki/webhook-server - --v=4 ports: - containerPort: 8443 @@ -39,11 +39,11 @@ spec: name: metrics protocol: TCP volumeMounts: - - name: kubeconfig - subPath: kubeconfig - mountPath: /etc/kubeconfig - - name: cert - mountPath: /var/serving-cert + - name: karmada-config + mountPath: /etc/karmada/config + readOnly: true + - name: webhook-server-cert + mountPath: /etc/karmada/pki/webhook-server readOnly: true readinessProbe: httpGet: @@ -51,12 +51,12 @@ spec: port: 8443 scheme: HTTPS volumes: - - name: kubeconfig + - name: karmada-config secret: - secretName: kubeconfig - - name: cert + secretName: karmada-config + - name: webhook-server-cert secret: - secretName: webhook-cert + secretName: webhook-server-cert --- apiVersion: v1 kind: Service diff --git a/artifacts/deploy/kube-controller-manager.yaml b/artifacts/deploy/kube-controller-manager.yaml index 205759193f3c..4163b46fba31 100644 --- a/artifacts/deploy/kube-controller-manager.yaml +++ b/artifacts/deploy/kube-controller-manager.yaml @@ -35,20 +35,20 @@ spec: - command: - kube-controller-manager - --allocate-node-cidrs=true - - --authentication-kubeconfig=/etc/kubeconfig - - --authorization-kubeconfig=/etc/kubeconfig + - --authentication-kubeconfig=/etc/karmada/config/karmada.config + - --authorization-kubeconfig=/etc/karmada/config/karmada.config - --bind-address=0.0.0.0 - - --client-ca-file=/etc/karmada/pki/ca.crt + - --client-ca-file=/etc/karmada/pki/ca/tls.crt - --cluster-cidr=10.244.0.0/16 - --cluster-name=karmada - - --cluster-signing-cert-file=/etc/karmada/pki/ca.crt - - --cluster-signing-key-file=/etc/karmada/pki/ca.key + - --cluster-signing-cert-file=/etc/karmada/pki/ca/tls.crt + - --cluster-signing-key-file=/etc/karmada/pki/ca/tls.key - --controllers=namespace,garbagecollector,serviceaccount-token,ttl-after-finished,bootstrapsigner,tokencleaner,csrapproving,csrcleaner,csrsigning,clusterrole-aggregation - - --kubeconfig=/etc/kubeconfig + - --kubeconfig=/etc/karmada/config/karmada.config - --leader-elect=true - --node-cidr-mask-size=24 - - --root-ca-file=/etc/karmada/pki/ca.crt - - --service-account-private-key-file=/etc/karmada/pki/karmada.key + - --root-ca-file=/etc/karmada/pki/ca/tls.crt + - --service-account-private-key-file=/etc/karmada/pki/server/tls.key - --service-cluster-ip-range=10.96.0.0/12 - --use-service-account-credentials=true - --v=4 @@ -69,17 +69,23 @@ spec: requests: cpu: 200m volumeMounts: - - mountPath: /etc/karmada/pki - name: karmada-certs + - mountPath: /etc/karmada/pki/ca + name: ca-key-pair + readOnly: true + - mountPath: /etc/karmada/pki/server + name: server-cert + readOnly: true + - mountPath: /etc/karmada/config + name: karmada-config readOnly: true - - mountPath: /etc/kubeconfig - subPath: kubeconfig - name: kubeconfig priorityClassName: system-node-critical volumes: - - name: karmada-certs + - name: ca-key-pair + secret: + secretName: ca-key-pair + - name: server-cert secret: - secretName: karmada-cert-secret - - name: kubeconfig + secretName: server-cert + - name: karmada-config secret: - secretName: kubeconfig + secretName: karmada-config diff --git a/examples/customresourceinterpreter/karmada-interpreter-webhook-example.yaml b/examples/customresourceinterpreter/karmada-interpreter-webhook-example.yaml index 317065268ae0..8042ba5f76e5 100644 --- a/examples/customresourceinterpreter/karmada-interpreter-webhook-example.yaml +++ b/examples/customresourceinterpreter/karmada-interpreter-webhook-example.yaml @@ -25,19 +25,19 @@ spec: imagePullPolicy: IfNotPresent command: - /bin/karmada-interpreter-webhook-example - - --kubeconfig=/etc/kubeconfig + - --kubeconfig=/etc/karmada/config/karmada.config - --bind-address=0.0.0.0 - --secure-port=8445 - - --cert-dir=/var/serving-cert + - --cert-dir=/etc/karmada/pki/webhook-server - --v=4 ports: - containerPort: 8445 volumeMounts: - - name: kubeconfig - subPath: kubeconfig - mountPath: /etc/kubeconfig - - name: cert - mountPath: /var/serving-cert + - name: karmada-config + mountPath: /etc/karmada/config + readOnly: true + - name: webhook-server-cert + mountPath: /etc/karmada/pki/webhook-server readOnly: true readinessProbe: httpGet: @@ -45,12 +45,12 @@ spec: port: 8445 scheme: HTTPS volumes: - - name: kubeconfig + - name: karmada-config secret: - secretName: kubeconfig - - name: cert + secretName: karmada-config + - name: webhook-server-cert secret: - secretName: webhook-cert + secretName: webhook-server-cert --- apiVersion: v1 kind: Service diff --git a/hack/deploy-karmada-agent.sh b/hack/deploy-karmada-agent.sh index deb6cb55fb8c..a11c4fc607db 100755 --- a/hack/deploy-karmada-agent.sh +++ b/hack/deploy-karmada-agent.sh @@ -83,7 +83,7 @@ kubectl --context="${MEMBER_CLUSTER_NAME}" apply -f "${REPO_ROOT}/artifacts/agen kubectl --context="${MEMBER_CLUSTER_NAME}" apply -f "${REPO_ROOT}/artifacts/agent/clusterrolebinding.yaml" # create secret -kubectl --context="${MEMBER_CLUSTER_NAME}" create secret generic karmada-kubeconfig --from-file=karmada-kubeconfig="${KARMADA_APISERVER_KUBECONFIG}" -n "${KARMADA_SYSTEM_NAMESPACE}" +kubectl --context="${MEMBER_CLUSTER_NAME}" create secret generic karmada-config --from-file=karmada.config="${KARMADA_APISERVER_KUBECONFIG}" -n "${KARMADA_SYSTEM_NAMESPACE}" # extract api endpoint of member cluster MEMBER_CLUSTER=$(kubectl config view -o jsonpath='{.contexts[?(@.name == "'${MEMBER_CLUSTER_NAME}'")].context.cluster}') diff --git a/hack/deploy-karmada.sh b/hack/deploy-karmada.sh index 12879dee89ce..d4cf28058567 100755 --- a/hack/deploy-karmada.sh +++ b/hack/deploy-karmada.sh @@ -95,37 +95,36 @@ function generate_cert_secret { local TEMP_PATH TEMP_PATH=$(mktemp -d) - cp -rf "${REPO_ROOT}"/artifacts/deploy/karmada-cert-secret.yaml "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - cp -rf "${REPO_ROOT}"/artifacts/deploy/secret.yaml "${TEMP_PATH}"/secret-tmp.yaml - cp -rf "${REPO_ROOT}"/artifacts/deploy/karmada-webhook-cert-secret.yaml "${TEMP_PATH}"/karmada-webhook-cert-secret-tmp.yaml - - sed -i'' -e "s/{{ca_crt}}/${karmada_ca}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{ca_key}}/${karmada_ca_key}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{client_crt}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{client_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{apiserver_crt}}/${KARMADA_APISERVER_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{apiserver_key}}/${KARMADA_APISERVER_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - - sed -i'' -e "s/{{front_proxy_ca_crt}}/${FRONT_PROXY_CA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{front_proxy_client_crt}}/${FRONT_PROXY_CLIENT_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{front_proxy_client_key}}/${FRONT_PROXY_CLIENT_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - - sed -i'' -e "s/{{etcd_ca_crt}}/${ETCD_CA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{etcd_server_crt}}/${ETCD_SERVER_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{etcd_server_key}}/${ETCD_SERVER_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{etcd_client_crt}}/${ETCD_CLIENT_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{etcd_client_key}}/${ETCD_CLIENT_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - - sed -i'' -e "s/{{ca_crt}}/${karmada_ca}/g" "${TEMP_PATH}"/secret-tmp.yaml - sed -i'' -e "s/{{client_crt}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/secret-tmp.yaml - sed -i'' -e "s/{{client_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/secret-tmp.yaml - - sed -i'' -e "s/{{server_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/karmada-webhook-cert-secret-tmp.yaml - sed -i'' -e "s/{{server_certificate}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/karmada-webhook-cert-secret-tmp.yaml - - kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${TEMP_PATH}"/secret-tmp.yaml - kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${TEMP_PATH}"/karmada-webhook-cert-secret-tmp.yaml + cp -rf "${REPO_ROOT}"/artifacts/deploy/karmada-secret-cert.yaml "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + cp -rf "${REPO_ROOT}"/artifacts/deploy/karmada-secret-config.yaml "${TEMP_PATH}"/karmada-secret-config-tmp.yaml + + sed -i'' -e "s/{{ca_crt}}/${karmada_ca}/g" "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + sed -i'' -e "s/{{ca_key}}/${karmada_ca_key}/g" "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + sed -i'' -e "s/{{ca_crt}}/${karmada_ca}/g" "${TEMP_PATH}"/karmada-secret-config-tmp.yaml + + sed -i'' -e "s/{{server_crt}}/${SERVER_CRT}/g" "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + sed -i'' -e "s/{{server_key}}/${SERVER_KEY}/g" "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + + sed -i'' -e "s/{{client_crt}}/${CLIENT_CRT}/g" "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + sed -i'' -e "s/{{client_key}}/${CLIENT_KEY}/g" "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + + sed -i'' -e "s/{{front_proxy_client_crt}}/${FRONT_PROXY_CLIENT_CRT}/g" "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + sed -i'' -e "s/{{front_proxy_client_key}}/${FRONT_PROXY_CLIENT_KEY}/g" "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + + sed -i'' -e "s/{{etcd_server_crt}}/${ETCD_SERVER_CRT}/g" "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + sed -i'' -e "s/{{etcd_server_key}}/${ETCD_SERVER_KEY}/g" "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + + sed -i'' -e "s/{{etcd_client_crt}}/${ETCD_CLIENT_CRT}/g" "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + sed -i'' -e "s/{{etcd_client_key}}/${ETCD_CLIENT_KEY}/g" "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + + sed -i'' -e "s/{{client_crt}}/${CLIENT_CRT}/g" "${TEMP_PATH}"/karmada-secret-config-tmp.yaml + sed -i'' -e "s/{{client_key}}/${CLIENT_KEY}/g" "${TEMP_PATH}"/karmada-secret-config-tmp.yaml + + sed -i'' -e "s/{{webhook_server_crt}}/${SERVER_CRT}/g" "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + sed -i'' -e "s/{{webhook_server_key}}/${SERVER_KEY}/g" "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + + kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${TEMP_PATH}"/karmada-secret-cert-tmp.yaml + kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${TEMP_PATH}"/karmada-secret-config-tmp.yaml rm -rf "${TEMP_PATH}" } @@ -149,26 +148,25 @@ util::cmd_must_exist "openssl" util::cmd_must_exist_cfssl ${CFSSL_VERSION} # create CA signers util::create_signing_certkey "" "${CERT_DIR}" ca karmada '"client auth","server auth"' -util::create_signing_certkey "" "${CERT_DIR}" front-proxy-ca front-proxy-ca '"client auth","server auth"' -util::create_signing_certkey "" "${CERT_DIR}" etcd-ca etcd-ca '"client auth","server auth"' # signs a certificate -util::create_certkey "" "${CERT_DIR}" "ca" karmada system:admin "system:masters" kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" "${interpreter_webhook_example_service_external_ip_address}" -util::create_certkey "" "${CERT_DIR}" "ca" apiserver karmada-apiserver "" "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" $(util::get_apiserver_ip_from_kubeconfig "${HOST_CLUSTER_NAME}") -util::create_certkey "" "${CERT_DIR}" "front-proxy-ca" front-proxy-client front-proxy-client "" kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" -util::create_certkey "" "${CERT_DIR}" "etcd-ca" etcd-server etcd-server "" kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" -util::create_certkey "" "${CERT_DIR}" "etcd-ca" etcd-client etcd-client "" "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" +karmadaAltNames=("*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" "kubernetes.default.svc" "*.etcd.karmada-system.svc.cluster.local" $(util::get_apiserver_ip_from_kubeconfig "${HOST_CLUSTER_NAME}") "${interpreter_webhook_example_service_external_ip_address}") +# create client.crt (CN=system:admin, O=system:masters), mainly used to construct kubeconfig for other components to access karmada-apiserver +util::create_certkey "" "${CERT_DIR}" "ca" client system:admin system:masters "${karmadaAltNames[@]}" +# create server.crt (CN=server, O=''), mainly used as server certificate for karmada components, such as karmada-apiserver, karmada-search, karmada-metrics-adapter, etc. +util::create_certkey "" "${CERT_DIR}" "ca" server server "" "${karmadaAltNames[@]}" +util::create_certkey "" "${CERT_DIR}" "ca" front-proxy-client front-proxy-client "" "${karmadaAltNames[@]}" +util::create_certkey "" "${CERT_DIR}" "ca" etcd-server etcd-server "" "${karmadaAltNames[@]}" +util::create_certkey "" "${CERT_DIR}" "ca" etcd-client etcd-client "" "${karmadaAltNames[@]}" # create namespace for control plane components kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/namespace.yaml" -KARMADA_CRT=$(base64 < "${CERT_DIR}/karmada.crt" | tr -d '\r\n') -KARMADA_KEY=$(base64 < "${CERT_DIR}/karmada.key" | tr -d '\r\n') -KARMADA_APISERVER_CRT=$(base64 < "${CERT_DIR}/apiserver.crt" | tr -d '\r\n') -KARMADA_APISERVER_KEY=$(base64 < "${CERT_DIR}/apiserver.key" | tr -d '\r\n') -FRONT_PROXY_CA_CRT=$(base64 < "${CERT_DIR}/front-proxy-ca.crt" | tr -d '\r\n') +SERVER_CRT=$(base64 < "${CERT_DIR}/server.crt" | tr -d '\r\n') +SERVER_KEY=$(base64 < "${CERT_DIR}/server.key" | tr -d '\r\n') +CLIENT_CRT=$(base64 < "${CERT_DIR}/client.crt" | tr -d '\r\n') +CLIENT_KEY=$(base64 < "${CERT_DIR}/client.key" | tr -d '\r\n') FRONT_PROXY_CLIENT_CRT=$(base64 < "${CERT_DIR}/front-proxy-client.crt" | tr -d '\r\n') FRONT_PROXY_CLIENT_KEY=$(base64 < "${CERT_DIR}/front-proxy-client.key" | tr -d '\r\n') -ETCD_CA_CRT=$(base64 < "${CERT_DIR}/etcd-ca.crt" | tr -d '\r\n') ETCD_SERVER_CRT=$(base64 < "${CERT_DIR}/etcd-server.crt" | tr -d '\r\n') ETCD_SERVER_KEY=$(base64 < "${CERT_DIR}/etcd-server.key" | tr -d '\r\n') ETCD_CLIENT_CRT=$(base64 < "${CERT_DIR}/etcd-client.crt" | tr -d '\r\n') @@ -237,7 +235,7 @@ else fi # write karmada api server config to kubeconfig file -util::append_client_kubeconfig "${HOST_CLUSTER_KUBECONFIG}" "${CERT_DIR}/karmada.crt" "${CERT_DIR}/karmada.key" "${KARMADA_APISERVER_IP}" "${KARMADA_APISERVER_SECURE_PORT}" karmada-apiserver +util::append_client_kubeconfig "${HOST_CLUSTER_KUBECONFIG}" "${CERT_DIR}/client.crt" "${CERT_DIR}/client.key" "${KARMADA_APISERVER_IP}" "${KARMADA_APISERVER_SECURE_PORT}" karmada-apiserver # deploy kube controller manager cp "${REPO_ROOT}"/artifacts/deploy/kube-controller-manager.yaml "${TEMP_PATH_APISERVER}"/kube-controller-manager.yaml