diff --git a/.github/workflows/test_workflows.yml b/.github/workflows/test_workflows.yml
index d87161f6..d6244cbe 100644
--- a/.github/workflows/test_workflows.yml
+++ b/.github/workflows/test_workflows.yml
@@ -27,4 +27,4 @@ jobs:
- name: run
uses: ./.github/actions/e2e
with:
- mo_version: "1.0.0-rc1"
+ mo_version: "1.2.3"
diff --git a/Makefile b/Makefile
index dc7bc376..0d001a9e 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@ SHELL=/usr/bin/env bash -o pipefail
REPO ?= "matrixorigin/matrixone-operator"
TAG ?= "latest"
GOPROXY ?= "https://proxy.golang.org,direct"
-MO_VERSION ?= "nightly-d98832bb"
+MO_VERSION ?= "1.2.3"
MO_IMAGE_REPO ?= "matrixorigin/matrixone"
BRANCH ?= main
ENVTEST_K8S_VERSION = 1.24.1
diff --git a/api/core/v1alpha1/common_helpers.go b/api/core/v1alpha1/common_helpers.go
index 66ebbe03..c5b09065 100644
--- a/api/core/v1alpha1/common_helpers.go
+++ b/api/core/v1alpha1/common_helpers.go
@@ -33,6 +33,10 @@ const (
reasonEmpty = "empty"
)
+var (
+ LatestOpVersion = semver.MustParse("1.3.0")
+)
+
func (c *ConditionalStatus) SetCondition(condition metav1.Condition) {
if c.Conditions == nil {
c.Conditions = []metav1.Condition{}
@@ -293,3 +297,14 @@ func (p *PodSet) GetSemVer() (*semver.Version, bool) {
}
return &v, true
}
+
+func (p *PodSet) GetOperatorVersion() semver.Version {
+ if p.OperatorVersion == nil {
+ return LatestOpVersion
+ }
+ v, err := semver.ParseTolerant(*p.OperatorVersion)
+ if err != nil {
+ return LatestOpVersion
+ }
+ return v
+}
diff --git a/api/core/v1alpha1/common_types.go b/api/core/v1alpha1/common_types.go
index a0cd4422..d913c7a8 100644
--- a/api/core/v1alpha1/common_types.go
+++ b/api/core/v1alpha1/common_types.go
@@ -114,6 +114,10 @@ type PodSet struct {
// reconciliations
// +optional
SemanticVersion *string `json:"semanticVersion,omitempty"`
+
+ // OperatorVersion is the controller version of mo-operator that should be used to
+ // reconcile this set
+ OperatorVersion *string `json:"operatorVersion,omitempty"`
}
// MainContainer is the description of the main container of a Pod
diff --git a/api/core/v1alpha1/controller_version.go b/api/core/v1alpha1/controller_version.go
new file mode 100644
index 00000000..ae605bfa
--- /dev/null
+++ b/api/core/v1alpha1/controller_version.go
@@ -0,0 +1,32 @@
+// Copyright 2024 Matrix Origin
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1alpha1
+
+import "github.com/blang/semver/v4"
+
+type Gate string
+
+var (
+ gateVersions = map[Gate][]semver.Version{}
+)
+
+func (g Gate) Enabled(v semver.Version) bool {
+ for _, minVersion := range gateVersions[g] {
+ if versionPrecedes(minVersion, v) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/api/core/v1alpha1/matrixonecluster_types.go b/api/core/v1alpha1/matrixonecluster_types.go
index fb963796..68dbb43a 100644
--- a/api/core/v1alpha1/matrixonecluster_types.go
+++ b/api/core/v1alpha1/matrixonecluster_types.go
@@ -97,6 +97,8 @@ type MatrixOneClusterSpec struct {
// reconciliations
// +optional
SemanticVersion *string `json:"semanticVersion,omitempty"`
+
+ OperatorVersion *string `json:"operatorVersion,omitempty"`
}
func (m *MatrixOneCluster) GetMetricReaderEnabled() bool {
diff --git a/api/core/v1alpha1/semver.go b/api/core/v1alpha1/semver.go
index 0eea847b..604d3558 100644
--- a/api/core/v1alpha1/semver.go
+++ b/api/core/v1alpha1/semver.go
@@ -22,13 +22,16 @@ const (
MOFeaturePipelineInfo MOFeature = "PipelineInfo"
MOFeatureSessionSource MOFeature = "SessionSource"
MOFeatureLockMigration MOFeature = "LockMigration"
+
+ MOFeatureDiscoveryFixed MOFeature = "DiscoveryFixed"
)
var (
featureVersions = map[MOFeature][]semver.Version{
- MOFeaturePipelineInfo: {semver.MustParse("1.1.2"), semver.MustParse("1.2.0")},
- MOFeatureSessionSource: {semver.MustParse("1.1.2"), semver.MustParse("1.2.0")},
- MOFeatureLockMigration: {semver.MustParse("1.2.0")},
+ MOFeaturePipelineInfo: {semver.MustParse("1.1.2"), semver.MustParse("1.2.0")},
+ MOFeatureSessionSource: {semver.MustParse("1.1.2"), semver.MustParse("1.2.0")},
+ MOFeatureLockMigration: {semver.MustParse("1.2.0")},
+ MOFeatureDiscoveryFixed: {semver.MustParse("2.0.0")},
}
MinimalVersion = semver.Version{Major: 0, Minor: 0, Patch: 0}
diff --git a/api/core/v1alpha1/zz_generated.deepcopy.go b/api/core/v1alpha1/zz_generated.deepcopy.go
index 41e94f52..be3be3e6 100644
--- a/api/core/v1alpha1/zz_generated.deepcopy.go
+++ b/api/core/v1alpha1/zz_generated.deepcopy.go
@@ -1704,6 +1704,11 @@ func (in *MatrixOneClusterSpec) DeepCopyInto(out *MatrixOneClusterSpec) {
*out = new(string)
**out = **in
}
+ if in.OperatorVersion != nil {
+ in, out := &in.OperatorVersion, &out.OperatorVersion
+ *out = new(string)
+ **out = **in
+ }
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MatrixOneClusterSpec.
@@ -1964,6 +1969,11 @@ func (in *PodSet) DeepCopyInto(out *PodSet) {
*out = new(string)
**out = **in
}
+ if in.OperatorVersion != nil {
+ in, out := &in.OperatorVersion, &out.OperatorVersion
+ *out = new(string)
+ **out = **in
+ }
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodSet.
diff --git a/charts/kruise/templates/manager.yaml b/charts/kruise/templates/manager.yaml
index 88f7486b..9a2f9037 100644
--- a/charts/kruise/templates/manager.yaml
+++ b/charts/kruise/templates/manager.yaml
@@ -62,7 +62,7 @@ spec:
command:
- /manager
image: {{ .Values.manager.image.repository }}:{{ .Values.manager.image.tag }}
- imagePullPolicy: Always
+ imagePullPolicy: {{ .Values.imagePullPolicy }}
name: manager
env:
{{- if .Values.enableKubeCacheMutationDetector }}
@@ -193,7 +193,7 @@ spec:
- --feature-gates={{ .Values.featureGates }}
- --socket-file={{ .Values.daemon.socketFile }}
image: {{ .Values.manager.image.repository }}:{{ .Values.manager.image.tag }}
- imagePullPolicy: Always
+ imagePullPolicy: {{ .Values.imagePullPolicy }}
name: daemon
env:
{{- if .Values.enableKubeCacheMutationDetector }}
diff --git a/charts/kruise/values.yaml b/charts/kruise/values.yaml
index 98f4f050..b4fae9ee 100644
--- a/charts/kruise/values.yaml
+++ b/charts/kruise/values.yaml
@@ -95,3 +95,5 @@ daemon:
serviceAccount:
annotations: {}
+
+imagePullPolicy: IfNotPresent
diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnpools.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnpools.yaml
index cfc433cb..3e7abbbd 100644
--- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnpools.yaml
+++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnpools.yaml
@@ -172,6 +172,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
pauseUpdate:
diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml
index 7990aa2a..f8433194 100644
--- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml
+++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml
@@ -146,6 +146,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
pauseUpdate:
diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_dnsets.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_dnsets.yaml
index 155947d3..1623155e 100644
--- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_dnsets.yaml
+++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_dnsets.yaml
@@ -121,6 +121,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_logsets.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_logsets.yaml
index 4af4e2a6..996e3e53 100644
--- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_logsets.yaml
+++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_logsets.yaml
@@ -112,6 +112,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml
index 3fb44671..81aa1a85 100644
--- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml
+++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml
@@ -138,6 +138,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
pauseUpdate:
@@ -489,6 +494,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
pauseUpdate:
@@ -824,6 +834,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
@@ -1021,6 +1036,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
@@ -1247,6 +1267,8 @@ spec:
NodeSelector specifies default node selector for all components,
this will be overridden by component-level config
type: object
+ operatorVersion:
+ type: string
proxy:
description: Proxy defines an optional MO Proxy of this cluster
properties:
@@ -1288,6 +1310,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
@@ -1462,6 +1489,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
@@ -1670,6 +1702,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
pauseUpdate:
@@ -1979,6 +2016,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_proxysets.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_proxysets.yaml
index 4991bcb1..dfe3e701 100644
--- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_proxysets.yaml
+++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_proxysets.yaml
@@ -106,6 +106,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_webuis.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_webuis.yaml
index f1c08228..6f67020c 100644
--- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_webuis.yaml
+++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_webuis.yaml
@@ -90,6 +90,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
diff --git a/deploy/crds/core.matrixorigin.io_cnpools.yaml b/deploy/crds/core.matrixorigin.io_cnpools.yaml
index cfc433cb..3e7abbbd 100644
--- a/deploy/crds/core.matrixorigin.io_cnpools.yaml
+++ b/deploy/crds/core.matrixorigin.io_cnpools.yaml
@@ -172,6 +172,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
pauseUpdate:
diff --git a/deploy/crds/core.matrixorigin.io_cnsets.yaml b/deploy/crds/core.matrixorigin.io_cnsets.yaml
index 7990aa2a..f8433194 100644
--- a/deploy/crds/core.matrixorigin.io_cnsets.yaml
+++ b/deploy/crds/core.matrixorigin.io_cnsets.yaml
@@ -146,6 +146,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
pauseUpdate:
diff --git a/deploy/crds/core.matrixorigin.io_dnsets.yaml b/deploy/crds/core.matrixorigin.io_dnsets.yaml
index 155947d3..1623155e 100644
--- a/deploy/crds/core.matrixorigin.io_dnsets.yaml
+++ b/deploy/crds/core.matrixorigin.io_dnsets.yaml
@@ -121,6 +121,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
diff --git a/deploy/crds/core.matrixorigin.io_logsets.yaml b/deploy/crds/core.matrixorigin.io_logsets.yaml
index 4af4e2a6..996e3e53 100644
--- a/deploy/crds/core.matrixorigin.io_logsets.yaml
+++ b/deploy/crds/core.matrixorigin.io_logsets.yaml
@@ -112,6 +112,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
diff --git a/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml b/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml
index 3fb44671..81aa1a85 100644
--- a/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml
+++ b/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml
@@ -138,6 +138,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
pauseUpdate:
@@ -489,6 +494,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
pauseUpdate:
@@ -824,6 +834,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
@@ -1021,6 +1036,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
@@ -1247,6 +1267,8 @@ spec:
NodeSelector specifies default node selector for all components,
this will be overridden by component-level config
type: object
+ operatorVersion:
+ type: string
proxy:
description: Proxy defines an optional MO Proxy of this cluster
properties:
@@ -1288,6 +1310,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
@@ -1462,6 +1489,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
@@ -1670,6 +1702,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
pauseUpdate:
@@ -1979,6 +2016,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
diff --git a/deploy/crds/core.matrixorigin.io_proxysets.yaml b/deploy/crds/core.matrixorigin.io_proxysets.yaml
index 4991bcb1..dfe3e701 100644
--- a/deploy/crds/core.matrixorigin.io_proxysets.yaml
+++ b/deploy/crds/core.matrixorigin.io_proxysets.yaml
@@ -106,6 +106,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
diff --git a/deploy/crds/core.matrixorigin.io_webuis.yaml b/deploy/crds/core.matrixorigin.io_webuis.yaml
index f1c08228..6f67020c 100644
--- a/deploy/crds/core.matrixorigin.io_webuis.yaml
+++ b/deploy/crds/core.matrixorigin.io_webuis.yaml
@@ -90,6 +90,11 @@ spec:
additionalProperties:
type: string
type: object
+ operatorVersion:
+ description: |-
+ OperatorVersion is the controller version of mo-operator that should be used to
+ reconcile this set
+ type: string
overlay:
x-kubernetes-preserve-unknown-fields: true
promDiscoveryScheme:
diff --git a/docs/reference/api-reference.md b/docs/reference/api-reference.md
index 6212be5c..d3abd509 100644
--- a/docs/reference/api-reference.md
+++ b/docs/reference/api-reference.md
@@ -421,6 +421,7 @@ _Appears in:_
| `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | |
| `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | |
| `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | |
+| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | |
| `cacheVolume` _[Volume](#volume)_ | CacheVolume is the desired local cache volume for CNSet,
node storage will be used if not specified | | |
| `sharedStorageCache` _[SharedStorageCache](#sharedstoragecache)_ | SharedStorageCache is the configuration of the S3 sharedStorageCache | | |
| `pythonUdfSidecar` _[PythonUdfSidecar](#pythonudfsidecar)_ | PythonUdfSidecar is the python udf server in CN | | |
@@ -621,6 +622,7 @@ _Appears in:_
| `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | |
| `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | |
| `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | |
+| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | |
| `cacheVolume` _[Volume](#volume)_ | CacheVolume is the desired local cache volume for CNSet,
node storage will be used if not specified | | |
| `sharedStorageCache` _[SharedStorageCache](#sharedstoragecache)_ | SharedStorageCache is the configuration of the S3 sharedStorageCache | | |
| `pythonUdfSidecar` _[PythonUdfSidecar](#pythonudfsidecar)_ | PythonUdfSidecar is the python udf server in CN | | |
@@ -817,6 +819,7 @@ _Appears in:_
| `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | |
| `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | |
| `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | |
+| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | |
| `cacheVolume` _[Volume](#volume)_ | CacheVolume is the desired local cache volume for DNSet,
node storage will be used if not specified | | |
| `sharedStorageCache` _[SharedStorageCache](#sharedstoragecache)_ | | | |
@@ -891,6 +894,8 @@ _Appears in:_
| `path` _string_ | Path the path that the shared fileSystem mounted to | | |
+
+
#### InitialConfig
@@ -981,6 +986,7 @@ _Appears in:_
| `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | |
| `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | |
| `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | |
+| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | |
| `volume` _[Volume](#volume)_ | Volume is the local persistent volume for each LogService instance | | |
| `sharedStorage` _[SharedStorageProvider](#sharedstorageprovider)_ | SharedStorage is an external shared storage shared by all LogService instances | | |
| `initialConfig` _[InitialConfig](#initialconfig)_ | InitialConfig is the initial configuration of HAKeeper
InitialConfig is immutable | | |
@@ -1090,6 +1096,7 @@ _Appears in:_
| `restoreFrom` _string_ | | | |
| `metricReaderEnabled` _boolean_ | MetricReaderEnabled enables metric reader for operator and other apps to query
metric from MO cluster | | |
| `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | |
+| `operatorVersion` _string_ | | | |
#### MigrateStatus
@@ -1208,6 +1215,7 @@ _Appears in:_
| `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | |
| `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | |
| `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | |
+| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | |
#### PoolScaleStrategy
@@ -1362,6 +1370,7 @@ _Appears in:_
| `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | |
| `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | |
| `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | |
+| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | |
| `serviceType` _[ServiceType](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#servicetype-v1-core)_ | ServiceType is the service type of proxy service | ClusterIP | Enum: [ClusterIP NodePort LoadBalancer]
|
| `serviceAnnotations` _object (keys:string, values:string)_ | ServiceAnnotations are the annotations for the proxy service | | |
| `nodePort` _integer_ | NodePort specifies the node port to use when ServiceType is NodePort or LoadBalancer,
reconciling will fail if the node port is not available. | | |
@@ -1704,6 +1713,7 @@ _Appears in:_
| `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | |
| `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | |
| `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | |
+| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | |
| `serviceType` _[ServiceType](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#servicetype-v1-core)_ | ServiceType is the service type of cn service | ClusterIP | Enum: [ClusterIP NodePort LoadBalancer]
|
| `updateStrategy` _[RollingUpdateStrategy](#rollingupdatestrategy)_ | UpdateStrategy rolling update strategy | | |
| `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#pullpolicy-v1-core)_ | | | |
diff --git a/pkg/controllers/cnset/resource.go b/pkg/controllers/cnset/resource.go
index aad39758..3196400a 100644
--- a/pkg/controllers/cnset/resource.go
+++ b/pkg/controllers/cnset/resource.go
@@ -259,8 +259,13 @@ func buildCNSetConfigMap(cn *v1alpha1.CNSet, ls *v1alpha1.LogSet) (*corev1.Confi
}
cfg.Merge(common.FileServiceConfig(fmt.Sprintf("%s/%s", common.DataPath, common.DataDir), ls.Spec.SharedStorage, &cn.Spec.SharedStorageCache))
cfg.Set([]string{"service-type"}, "CN")
- cfg.Set([]string{"hakeeper-client", "service-addresses"}, logset.HaKeeperAdds(ls))
- // cfg.Set([]string{"hakeeper-client", "discovery-address"}, ls.Status.Discovery.String())
+ if sv, ok := cn.Spec.GetSemVer(); ok && v1alpha1.HasMOFeature(*sv, v1alpha1.MOFeatureDiscoveryFixed) {
+ // issue: https://github.com/matrixorigin/MO-Cloud/issues/4158
+ // via discovery-address, operator can take off unhealthy logstores without restart CN/TN
+ cfg.Set([]string{"hakeeper-client", "discovery-address"}, ls.Status.Discovery.String())
+ } else {
+ cfg.Set([]string{"hakeeper-client", "service-addresses"}, logset.HaKeeperAdds(ls))
+ }
cfg.Set([]string{"cn", "role"}, cn.Spec.Role)
cfg.Set([]string{"cn", "lockservice", "listen-address"}, fmt.Sprintf("0.0.0.0:%d", common.LockServicePort))
cfg.Set([]string{"cn", "port-base"}, cnPortBase)
diff --git a/pkg/controllers/cnset/resource_test.go b/pkg/controllers/cnset/resource_test.go
index 18b52121..f2fadeba 100644
--- a/pkg/controllers/cnset/resource_test.go
+++ b/pkg/controllers/cnset/resource_test.go
@@ -15,12 +15,13 @@
package cnset
import (
+ "testing"
+
"github.com/google/go-cmp/cmp"
"github.com/matrixorigin/matrixone-operator/api/core/v1alpha1"
. "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/pointer"
- "testing"
)
func Test_buildCNSetConfigMap(t *testing.T) {
@@ -154,6 +155,70 @@ name = "ETL"
[fileservice.cache]
memory-capacity = "1B"
+[hakeeper-client]
+service-addresses = []
+`,
+ },
+ {
+ name: "before 1.3.0",
+ args: args{
+ cn: &v1alpha1.CNSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Namespace: "test",
+ Name: "test",
+ },
+ Spec: v1alpha1.CNSetSpec{
+ PodSet: v1alpha1.PodSet{
+ OperatorVersion: pointer.String("1.2.3"),
+ },
+ },
+ },
+ ls: &v1alpha1.LogSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Namespace: "test",
+ Name: "test",
+ },
+ Spec: v1alpha1.LogSetSpec{SharedStorage: v1alpha1.SharedStorageProvider{
+ FileSystem: &v1alpha1.FileSystemProvider{
+ Path: "/test",
+ },
+ }},
+ Status: v1alpha1.LogSetStatus{
+ Discovery: &v1alpha1.LogSetDiscovery{
+ Port: 6001,
+ Address: "test",
+ },
+ },
+ },
+ },
+ wantConfig: `data-dir = "/var/lib/matrixone/data"
+service-type = "CN"
+
+[cn]
+port-base = 6002
+role = ""
+
+[cn.lockservice]
+listen-address = "0.0.0.0:6003"
+
+[[fileservice]]
+backend = "DISK"
+data-dir = "/var/lib/matrixone/data"
+name = "LOCAL"
+
+[[fileservice]]
+backend = "DISK"
+data-dir = "/test"
+name = "S3"
+
+[[fileservice]]
+backend = "DISK-ETL"
+data-dir = "/test"
+name = "ETL"
+
+[fileservice.cache]
+memory-capacity = "1B"
+
[hakeeper-client]
service-addresses = []
`,
diff --git a/pkg/controllers/dnset/resource.go b/pkg/controllers/dnset/resource.go
index 5046006b..7abfd875 100644
--- a/pkg/controllers/dnset/resource.go
+++ b/pkg/controllers/dnset/resource.go
@@ -20,12 +20,12 @@ import (
"text/template"
"github.com/go-errors/errors"
- "github.com/matrixorigin/matrixone-operator/pkg/controllers/logset"
recon "github.com/matrixorigin/controller-runtime/pkg/reconciler"
"github.com/matrixorigin/controller-runtime/pkg/util"
"github.com/matrixorigin/matrixone-operator/api/core/v1alpha1"
"github.com/matrixorigin/matrixone-operator/pkg/controllers/common"
+ "github.com/matrixorigin/matrixone-operator/pkg/controllers/logset"
"github.com/openkruise/kruise-api/apps/pub"
kruise "github.com/openkruise/kruise-api/apps/v1beta1"
corev1 "k8s.io/api/core/v1"
@@ -190,8 +190,13 @@ func buildDNSetConfigMap(dn *v1alpha1.DNSet, ls *v1alpha1.LogSet) (*corev1.Confi
// [tn] is configured, all config items should go to the [tn] toml table
configAlias = aliasTN
}
- conf.Set([]string{"hakeeper-client", "service-addresses"}, logset.HaKeeperAdds(ls))
- // conf.Set([]string{"hakeeper-client", "discovery-address"}, ls.Status.Discovery.String())
+ if sv, ok := dn.Spec.GetSemVer(); ok && v1alpha1.HasMOFeature(*sv, v1alpha1.MOFeatureDiscoveryFixed) {
+ // issue: https://github.com/matrixorigin/MO-Cloud/issues/4158
+ // via discovery-address, operator can take off unhealthy logstores without restart CN/TN
+ conf.Set([]string{"hakeeper-client", "discovery-address"}, ls.Status.Discovery.String())
+ } else {
+ conf.Set([]string{"hakeeper-client", "service-addresses"}, logset.HaKeeperAdds(ls))
+ }
conf.Merge(common.FileServiceConfig(fmt.Sprintf("%s/%s", common.DataPath, common.DataDir), ls.Spec.SharedStorage, &dn.Spec.SharedStorageCache))
conf.Set([]string{"service-type"}, serviceType)
conf.Set([]string{configAlias, "listen-address"}, getListenAddress())
diff --git a/pkg/controllers/dnset/resource_test.go b/pkg/controllers/dnset/resource_test.go
index 500dab78..e4a45358 100644
--- a/pkg/controllers/dnset/resource_test.go
+++ b/pkg/controllers/dnset/resource_test.go
@@ -15,11 +15,12 @@
package dnset
import (
+ "testing"
+
"github.com/google/go-cmp/cmp"
"github.com/matrixorigin/matrixone-operator/api/core/v1alpha1"
. "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "testing"
)
func Test_buildDNSetConfigMap(t *testing.T) {
@@ -169,6 +170,75 @@ memory-capacity = "1B"
[hakeeper-client]
service-addresses = []
+`,
+ },
+ {
+ name: "2.0",
+ args: args{
+ dn: &v1alpha1.DNSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Namespace: "test",
+ Name: "test",
+ },
+ Spec: v1alpha1.DNSetSpec{
+ PodSet: v1alpha1.PodSet{
+ MainContainer: v1alpha1.MainContainer{
+ Image: "test:v2.0.0",
+ },
+ },
+ },
+ },
+ ls: &v1alpha1.LogSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Namespace: "test",
+ Name: "test",
+ },
+ Spec: v1alpha1.LogSetSpec{SharedStorage: v1alpha1.SharedStorageProvider{
+ FileSystem: &v1alpha1.FileSystemProvider{
+ Path: "/test",
+ },
+ }},
+ Status: v1alpha1.LogSetStatus{
+ Discovery: &v1alpha1.LogSetDiscovery{
+ Port: 6001,
+ Address: "test",
+ },
+ },
+ },
+ },
+ wantConfig: `data-dir = "/var/lib/matrixone/data"
+service-type = "DN"
+
+[dn]
+listen-address = "0.0.0.0:41010"
+port-base = 41010
+
+[dn.LogtailServer]
+listen-address = "0.0.0.0:32003"
+
+[dn.lockservice]
+listen-address = "0.0.0.0:6003"
+
+[[fileservice]]
+backend = "DISK"
+data-dir = "/var/lib/matrixone/data"
+name = "LOCAL"
+
+[[fileservice]]
+backend = "DISK"
+data-dir = "/test"
+name = "S3"
+
+[[fileservice]]
+backend = "DISK-ETL"
+data-dir = "/test"
+name = "ETL"
+
+[fileservice.cache]
+memory-capacity = "1B"
+
+[hakeeper-client]
+discovery-address = "test:6001"
`,
},
}
diff --git a/pkg/controllers/logset/configmap.go b/pkg/controllers/logset/configmap.go
index 719b2326..b881982f 100644
--- a/pkg/controllers/logset/configmap.go
+++ b/pkg/controllers/logset/configmap.go
@@ -43,6 +43,7 @@ const (
// Since HA requires instance-based heterogeneous configuration (e.g. instance UUID and advertised addresses), we need a start script to build these configurations based on
// the instance meta injected by k8s downward API
// TODO(aylei): add logservice topology labels
+// Deprecated: use startScriptTplV2.sh instead
var startScriptTpl = template.Must(template.New("logservice-start-script").Parse(`
#!/bin/sh
set -eu
@@ -99,6 +100,62 @@ echo "/mo-service -cfg ${conf} $@"
exec /mo-service -cfg ${conf} $@
`))
+var startScriptTplV2 = template.Must(template.New("logservice-start-script-v2").Parse(`
+#!/bin/sh
+set -eu
+
+POD_NAME=${POD_NAME:-$HOSTNAME}
+ADDR="${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc"
+ORDINAL=${POD_NAME##*-}
+if [ -z "${HOSTNAME_UUID+guard}" ]; then
+ UUID=$(printf '00000000-0000-0000-0000-0%011x' ${ORDINAL})
+else
+ UUID=$(echo ${ADDR} | sha256sum | od -x | head -1 | awk '{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}')
+fi
+conf=$(mktemp)
+
+bc=$(mktemp)
+cat < ${bc}
+uuid = "${UUID}"
+service-host = "${ADDR}"
+raft-port = {{ .RaftPort }}
+logservice-port = {{ .LogServicePort }}
+gossip-port = {{ .GossipPort }}
+EOF
+
+# build instance config
+sed "/\[logservice\]/r ${bc}" {{ .ConfigFilePath }} > ${conf}
+
+# insert gossip config
+gossipTmp=$(mktemp)
+sed "/\[logservice\]/d" {{ .GossipFilePath }} > ${gossipTmp}
+sed -i "/\[logservice\]/r ${gossipTmp}" ${conf}
+
+# append bootstrap config
+sed "/\[logservice\]/d" {{ .BootstrapFilePath }} >> ${conf}
+
+# there is a chance that the dns is not yet added to kubedns and the
+# server will crash, wait before myself to be resolvable
+elapseTime=0
+period=1
+threshold=30
+while true; do
+ sleep ${period}
+ elapseTime=$(( elapseTime+period ))
+ if [ ${elapseTime} -ge ${threshold} ]; then
+ echo "waiting for dns resolvable timeout" >&2 && exit 1
+ fi
+ if nslookup ${ADDR} >/dev/null; then
+ break
+ else
+ echo "waiting pod dns name ${ADDR} resolvable" >&2
+ fi
+done
+
+echo "/mo-service -cfg ${conf} $@"
+exec /mo-service -cfg ${conf} $@
+`))
+
type model struct {
RaftPort int
LogServicePort int
@@ -158,7 +215,12 @@ func buildConfigMap(ls *v1alpha1.LogSet) (*corev1.ConfigMap, error) {
// 2. build the start script
buff := new(bytes.Buffer)
- err = startScriptTpl.Execute(buff, &model{
+ tpl := startScriptTpl
+
+ if sv, ok := ls.Spec.GetSemVer(); ok && v1alpha1.HasMOFeature(*sv, v1alpha1.MOFeatureDiscoveryFixed) {
+ tpl = startScriptTplV2
+ }
+ err = tpl.Execute(buff, &model{
RaftPort: raftPort,
LogServicePort: logServicePort,
GossipPort: gossipPort,
@@ -166,6 +228,7 @@ func buildConfigMap(ls *v1alpha1.LogSet) (*corev1.ConfigMap, error) {
BootstrapFilePath: fmt.Sprintf("%s/%s", bootstrapPath, bootstrapFile),
GossipFilePath: fmt.Sprintf("%s/%s", gossipPath, gossipFile),
})
+
if err != nil {
return nil, err
}
diff --git a/pkg/controllers/mocluster/controller.go b/pkg/controllers/mocluster/controller.go
index 0f8d1b86..8f15a679 100644
--- a/pkg/controllers/mocluster/controller.go
+++ b/pkg/controllers/mocluster/controller.go
@@ -19,6 +19,8 @@ import (
"crypto/rand"
"encoding/base64"
"fmt"
+ "time"
+
"github.com/go-errors/errors"
recon "github.com/matrixorigin/controller-runtime/pkg/reconciler"
"github.com/matrixorigin/controller-runtime/pkg/util"
@@ -34,7 +36,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/manager"
- "time"
)
const (
@@ -349,6 +350,9 @@ func setPodSetDefault(ps *v1alpha1.PodSet, mo *v1alpha1.MatrixOneCluster) {
if ps.SemanticVersion == nil {
ps.SemanticVersion = mo.Spec.SemanticVersion
}
+ if ps.OperatorVersion == nil {
+ ps.OperatorVersion = mo.Spec.OperatorVersion
+ }
}
func setOverlay(o **v1alpha1.Overlay, mo *v1alpha1.MatrixOneCluster) {
diff --git a/test/e2e/matrixonecluster_test.go b/test/e2e/matrixonecluster_test.go
index 12b448df..9b1f7847 100644
--- a/test/e2e/matrixonecluster_test.go
+++ b/test/e2e/matrixonecluster_test.go
@@ -16,6 +16,9 @@ package e2e
import (
"context"
"fmt"
+ "strings"
+ "time"
+
recon "github.com/matrixorigin/controller-runtime/pkg/reconciler"
"github.com/matrixorigin/controller-runtime/pkg/util"
"github.com/matrixorigin/matrixone-operator/api/core/v1alpha1"
@@ -32,8 +35,6 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
- "strings"
- "time"
)
const (
@@ -42,7 +43,7 @@ const (
teardownClusterTimeout = 10 * time.Minute
pollInterval = 15 * time.Second
portForwardTimeout = 10 * time.Second
- sqlTestTimeout = 10 * time.Minute
+ sqlTestTimeout = 15 * time.Minute
)
var _ = Describe("MatrixOneCluster test", func() {
@@ -66,6 +67,7 @@ var _ = Describe("MatrixOneCluster test", func() {
Name: "test",
},
Spec: v1alpha1.MatrixOneClusterSpec{
+ OperatorVersion: pointer.String("1.3.0"),
CNGroups: []v1alpha1.CNGroup{{
Name: "tp",
CNSetSpec: v1alpha1.CNSetSpec{