feat: v0.5.0.dev231215 (#198)

* repo-sync-2023-12-15T17:04:07+0800 * repo-sync-2023-12-15T17:47:55+0800
secretflow · Dec 15, 2023 · c4b58da · c4b58da
1 parent afbd526
commit c4b58da
Show file tree

Hide file tree

Showing 134 changed files with 11,493 additions and 617 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 `Fixed` for any bug fixes.
 `Security` in case of vulnerabilities.
 
+## [0.5.0.dev231215] - 2023-12-15
+### Added
+- Add document for deploying Kuscia on k8s.
+### Changed
+- Optimize log output.
+
 ## [0.5.0.dev231205] - 2023-12-5
 ### Changed
 - Optimize Kuscia deployment configuration and add configuration documentation.

diff --git a/cmd/kuscia/autonomy/autonomy.go b/cmd/kuscia/autonomy/autonomy.go
@@ -51,17 +51,14 @@ func Run(ctx context.Context, configFile string, onlyControllers bool) error {
 	defer cancel()
 
 	kusciaConf := confloader.ReadConfig(configFile, common.RunModeAutonomy)
-	nlog.Debugf("Read kuscia config: %+v", kusciaConf)
+	conf := modules.InitDependencies(ctx, kusciaConf)
+	defer conf.Close()
 
-	// dns must start before dependencies because that dependencies init process may access network.
 	var coreDnsModule modules.Module
 	if !onlyControllers {
 		coreDnsModule = modules.RunCoreDNS(runCtx, cancel, &kusciaConf)
 	}
 
-	conf := modules.InitDependencies(ctx, kusciaConf, onlyControllers)
-	defer conf.Close()
-
 	if onlyControllers {
 		conf.MakeClients()
 		modules.RunOperatorsAllinOne(runCtx, cancel, conf, true)

diff --git a/cmd/kuscia/confloader/config.go b/cmd/kuscia/confloader/config.go
@@ -30,11 +30,6 @@ import (
 var (
 	defaultRootDir           = "/home/kuscia/"
 	defaultEndpointForMaster = "https://127.0.0.1:6443"
-	CertPrefix               = "etc/certs/"
-	LogPrefix                = "var/logs/"
-	StdoutPrefix             = "var/stdout/"
-	TmpPrefix                = "var/tmp/"
-	ConfPrefix               = "etc/conf/"
 )
 
 type KusciaConfig struct {
@@ -129,10 +124,10 @@ func defaultKusciaConfig(rootDir string) KusciaConfig {
 	}
 	return KusciaConfig{
 		RootDir:        rootDir,
-		CAKeyFile:      filepath.Join(rootDir, TmpPrefix, "ca.key"),
-		CACertFile:     filepath.Join(rootDir, TmpPrefix, "ca.crt"),
-		DomainKeyFile:  filepath.Join(rootDir, TmpPrefix, "domain.key"),
-		DomainCertFile: filepath.Join(rootDir, TmpPrefix, "domain.crt"),
+		CAKeyFile:      filepath.Join(rootDir, common.CertPrefix, "ca.key"),
+		CACertFile:     filepath.Join(rootDir, common.CertPrefix, "ca.crt"),
+		DomainKeyFile:  filepath.Join(rootDir, common.CertPrefix, "domain.key"),
+		DomainCertFile: filepath.Join(rootDir, common.CertPrefix, "domain.crt"),
 		EnvoyIP:        hostIP,
 		KusciaAPI:      kaconfig.NewDefaultKusciaAPIConfig(rootDir),
 	}

diff --git a/cmd/kuscia/confloader/secretbackendloader.go b/cmd/kuscia/confloader/secretbackendloader.go
@@ -22,7 +22,6 @@ import (
 	"github.com/secretflow/kuscia/pkg/secretbackend"
 	// register driver
 	_ "github.com/secretflow/kuscia/pkg/secretbackend/mem"
-	_ "github.com/secretflow/kuscia/pkg/secretbackend/rfile"
 )
 
 type SecretBackendParams struct {

diff --git a/cmd/kuscia/lite/lite.go b/cmd/kuscia/lite/lite.go
@@ -26,7 +26,6 @@ import (
 	"github.com/secretflow/kuscia/cmd/kuscia/modules"
 	"github.com/secretflow/kuscia/cmd/kuscia/utils"
 	"github.com/secretflow/kuscia/pkg/common"
-	"github.com/secretflow/kuscia/pkg/utils/nlog"
 )
 
 func NewLiteCommand(ctx context.Context) *cobra.Command {
@@ -48,14 +47,11 @@ func Run(ctx context.Context, configFile string) error {
 	runCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	kusciaConf := confloader.ReadConfig(configFile, common.RunModeLite)
-	nlog.Debugf("Read kuscia config: %+v", kusciaConf)
+	conf := modules.InitDependencies(ctx, kusciaConf)
+	defer conf.Close()
 
-	// dns must start before dependencies because that dependencies init process may access network.
 	coreDnsModule := modules.RunCoreDNS(runCtx, cancel, &kusciaConf)
 
-	conf := modules.InitDependencies(ctx, kusciaConf, false)
-	defer conf.Close()
-
 	conf.MakeClients()
 
 	if conf.EnableContainerd {

diff --git a/cmd/kuscia/master/master.go b/cmd/kuscia/master/master.go
@@ -59,16 +59,14 @@ func Run(ctx context.Context, configFile string, onlyControllers bool) error {
 	defer cancel()
 
 	kusciaConf := confloader.ReadConfig(configFile, common.RunModeMaster)
-	nlog.Debugf("Read kuscia config: %+v", kusciaConf)
+	conf := modules.InitDependencies(ctx, kusciaConf)
+	defer conf.Close()
 
-	// dns must start before dependencies because that dependencies init process may access network.
 	var coreDnsModule modules.Module
 	if !onlyControllers {
 		coreDnsModule = modules.RunCoreDNS(runCtx, cancel, &kusciaConf)
 	}
 
-	conf := modules.InitDependencies(ctx, kusciaConf, onlyControllers)
-
 	if onlyControllers {
 		conf.MakeClients()
 		modules.RunOperatorsAllinOne(runCtx, cancel, conf, false)

diff --git a/cmd/kuscia/modules/agent.go b/cmd/kuscia/modules/agent.go
@@ -22,9 +22,9 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/secretflow/kuscia/cmd/kuscia/confloader"
 	"github.com/secretflow/kuscia/pkg/agent/commands"
 	"github.com/secretflow/kuscia/pkg/agent/config"
+	"github.com/secretflow/kuscia/pkg/common"
 	"github.com/secretflow/kuscia/pkg/utils/kubeconfig"
 	"github.com/secretflow/kuscia/pkg/utils/meta"
 	"github.com/secretflow/kuscia/pkg/utils/nlog"
@@ -47,7 +47,7 @@ func NewAgent(i *Dependencies) Module {
 	if err != nil {
 		nlog.Fatalf("Get hostname fail: %v", err)
 	}
-	conf.StdoutPath = filepath.Join(i.RootDir, confloader.StdoutPrefix)
+	conf.StdoutPath = filepath.Join(i.RootDir, common.StdoutPrefix)
 	if conf.Node.NodeName == "" {
 		conf.Node.NodeName = hostname
 	}

diff --git a/cmd/kuscia/modules/allinone_operator.go b/cmd/kuscia/modules/allinone_operator.go
@@ -34,6 +34,7 @@ func RunOperatorsAllinOne(runctx context.Context, cancel context.CancelFunc, con
 
 	if startAgent {
 		RunAgent(runctx, cancel, conf)
+		RunConfManager(runctx, cancel, conf)
 		RunDataMesh(runctx, cancel, conf)
 		RunTransport(runctx, cancel, conf)
 	}

diff --git a/cmd/kuscia/modules/confmanager.go b/cmd/kuscia/modules/confmanager.go
@@ -21,8 +21,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"sync/atomic"
 	"time"
 
+	"github.com/secretflow/kuscia/pkg/common"
 	"github.com/secretflow/kuscia/pkg/confmanager/commands"
 	"github.com/secretflow/kuscia/pkg/confmanager/config"
 	"github.com/secretflow/kuscia/pkg/confmanager/service"
@@ -77,14 +79,20 @@ func NewConfManager(ctx context.Context, d *Dependencies) (Module, error) {
 	conf.DomainKey = d.DomainKey
 	conf.TLS.RootCA = d.CACert
 	conf.TLS.RootCAKey = d.CAKey
-	conf.DomainCertValue = &d.DomainCertByMasterValue
+	switch d.RunMode {
+	case common.RunModeLite:
+		conf.DomainCertValue = &d.DomainCertByMasterValue
+	case common.RunModeAutonomy:
+		conf.DomainCertValue = &atomic.Value{}
+		conf.DomainCertValue.Store(d.DomainCert)
+	}
 	secretBackend := findSecretBackend(d.SecretBackendHolder, conf.Backend)
 	if secretBackend == nil {
 		return nil, fmt.Errorf("failed to find secret backend %s for cm", conf.Backend)
 	}
 	conf.BackendDriver = secretBackend
 
-	nlog.Infof("Conf manager config is %+v", conf)
+	nlog.Debugf("Conf manager config is %+v", conf)
 
 	if err := conf.TLS.GenerateServerKeyCerts(serverCertsCommonName, nil, []string{defaultServerCertsSanDNSName}); err != nil {
 		return nil, err
@@ -110,7 +118,9 @@ func (m confManagerModule) Run(ctx context.Context) error {
 
 func (m confManagerModule) WaitReady(ctx context.Context) error {
 	timeoutTicker := time.NewTicker(30 * time.Second)
+	defer timeoutTicker.Stop()
 	checkTicker := time.NewTicker(1 * time.Second)
+	defer checkTicker.Stop()
 	for {
 		select {
 		case <-checkTicker.C:

diff --git a/cmd/kuscia/modules/containerd.go b/cmd/kuscia/modules/containerd.go
@@ -22,7 +22,7 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/secretflow/kuscia/cmd/kuscia/confloader"
+	pkgcom "github.com/secretflow/kuscia/pkg/common"
 	"github.com/secretflow/kuscia/pkg/utils/common"
 	"github.com/secretflow/kuscia/pkg/utils/nlog"
 	"github.com/secretflow/kuscia/pkg/utils/nlog/ljwriter"
@@ -44,8 +44,8 @@ func NewContainerd(i *Dependencies) Module {
 }
 
 func (s *containerdModule) Run(ctx context.Context) error {
-	configPath := filepath.Join(s.Root, confloader.ConfPrefix, "containerd.toml")
-	configPathTmpl := filepath.Join(s.Root, confloader.ConfPrefix, "containerd.toml.tmpl")
+	configPath := filepath.Join(s.Root, pkgcom.ConfPrefix, "containerd.toml")
+	configPathTmpl := filepath.Join(s.Root, pkgcom.ConfPrefix, "containerd.toml.tmpl")
 	if err := common.RenderConfig(configPathTmpl, configPath, s); err != nil {
 		return err
 	}
@@ -54,7 +54,7 @@ func (s *containerdModule) Run(ctx context.Context) error {
 	crictlFile := "/etc/crictl.yaml"
 	if _, err := os.Stat(crictlFile); err != nil {
 		if os.IsNotExist(err) {
-			if err = os.Link(filepath.Join(s.Root, confloader.ConfPrefix, "crictl.yaml"), crictlFile); err != nil {
+			if err = os.Link(filepath.Join(s.Root, pkgcom.ConfPrefix, "crictl.yaml"), crictlFile); err != nil {
 				return err
 			}
 		} else {
@@ -72,7 +72,7 @@ func (s *containerdModule) Run(ctx context.Context) error {
 	}
 
 	sp := supervisor.NewSupervisor("containerd", nil, -1)
-	s.LogConfig.LogPath = filepath.Join(s.Root, confloader.LogPrefix, "containerd.log")
+	s.LogConfig.LogPath = filepath.Join(s.Root, pkgcom.LogPrefix, "containerd.log")
 	lj, _ := ljwriter.New(&s.LogConfig)
 	n := nlog.NewNLog(nlog.SetWriter(lj))
 	return sp.Run(ctx, func(ctx context.Context) supervisor.Cmd {

diff --git a/cmd/kuscia/modules/coredns.go b/cmd/kuscia/modules/coredns.go
@@ -28,6 +28,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 
 	"github.com/secretflow/kuscia/cmd/kuscia/confloader"
+	"github.com/secretflow/kuscia/pkg/common"
 	"github.com/secretflow/kuscia/pkg/coredns"
 	"github.com/secretflow/kuscia/pkg/utils/network"
 	"github.com/secretflow/kuscia/pkg/utils/nlog"
@@ -120,7 +121,7 @@ func (s *CorednsModule) Run(ctx context.Context) error {
 	)
 	dnsserver.Directives = directives
 
-	contents, err := os.ReadFile(filepath.Join(s.rootDir, confloader.ConfPrefix, "corefile"))
+	contents, err := os.ReadFile(filepath.Join(s.rootDir, common.ConfPrefix, "corefile"))
 	if err != nil {
 		return err
 	}
@@ -183,7 +184,7 @@ func prepareResolvConf(rootDir string) error {
 	}
 
 	resolvConf := "/etc/resolv.conf"
-	backupResolvConf := filepath.Join(rootDir, confloader.TmpPrefix, "resolv.conf")
+	backupResolvConf := filepath.Join(rootDir, common.TmpPrefix, "resolv.conf")
 	exist := paths.CheckFileExist(backupResolvConf)
 	if !exist {
 		if err = paths.CopyFile(resolvConf, backupResolvConf); err != nil {

diff --git a/cmd/kuscia/modules/domainroute.go b/cmd/kuscia/modules/domainroute.go
@@ -21,7 +21,6 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/secretflow/kuscia/cmd/kuscia/confloader"
 	"github.com/secretflow/kuscia/pkg/common"
 	"github.com/secretflow/kuscia/pkg/gateway/commands"
 	"github.com/secretflow/kuscia/pkg/gateway/config"
@@ -42,7 +41,7 @@ type domainRouteModule struct {
 func NewDomainRoute(i *Dependencies) Module {
 	conf := config.DefaultStaticGatewayConfig()
 	conf.RootDir = i.RootDir
-	conf.ConfBasedir = filepath.Join(i.RootDir, confloader.ConfPrefix, "domainroute")
+	conf.ConfBasedir = filepath.Join(i.RootDir, common.ConfPrefix, "domainroute")
 	conf.DomainID = i.DomainID
 	conf.DomainKey = i.DomainKey
 	conf.MasterConfig = &i.Master

diff --git a/cmd/kuscia/modules/envoy.go b/cmd/kuscia/modules/envoy.go
@@ -27,7 +27,7 @@ import (
 
 	"gopkg.in/yaml.v3"
 
-	"github.com/secretflow/kuscia/cmd/kuscia/confloader"
+	"github.com/secretflow/kuscia/pkg/common"
 	"github.com/secretflow/kuscia/pkg/gateway/utils"
 	"github.com/secretflow/kuscia/pkg/utils/nlog"
 	"github.com/secretflow/kuscia/pkg/utils/supervisor"
@@ -87,7 +87,7 @@ func NewEnvoy(i *Dependencies) Module {
 }
 
 func (s *envoyModule) Run(ctx context.Context) error {
-	if err := os.MkdirAll(filepath.Join(s.rootDir, confloader.LogPrefix, "envoy/"), 0755); err != nil {
+	if err := os.MkdirAll(filepath.Join(s.rootDir, common.LogPrefix, "envoy/"), 0755); err != nil {
 		return err
 	}
 	deltaArgs, err := s.readCommandArgs()
@@ -97,13 +97,13 @@ func (s *envoyModule) Run(ctx context.Context) error {
 
 	args := []string{
 		"-c",
-		filepath.Join(s.rootDir, confloader.ConfPrefix, "envoy/envoy.yaml"),
+		filepath.Join(s.rootDir, common.ConfPrefix, "envoy/envoy.yaml"),
 		"--service-cluster",
 		s.cluster,
 		"--service-node",
 		s.id,
 		"--log-path",
-		filepath.Join(s.rootDir, confloader.LogPrefix, "envoy/envoy.log"),
+		filepath.Join(s.rootDir, common.LogPrefix, "envoy/envoy.log"),
 	}
 	args = append(args, deltaArgs.Args...)
 	sp := supervisor.NewSupervisor("envoy", nil, -1)
@@ -129,7 +129,7 @@ func (s *envoyModule) logRotate(ctx context.Context) {
 
 		time.Sleep(d)
 
-		cmd := exec.Command("logrotate", filepath.Join(s.rootDir, confloader.ConfPrefix, "logrotate.conf"))
+		cmd := exec.Command("logrotate", filepath.Join(s.rootDir, common.ConfPrefix, "logrotate.conf"))
 		if err := cmd.Run(); err != nil {
 			nlog.Errorf("Logrotate run error: %v", err)
 		}
@@ -158,7 +158,7 @@ func (s *envoyModule) Name() string {
 }
 
 func (s *envoyModule) readCommandArgs() (*EnvoyCommandLineConfig, error) {
-	configPath := filepath.Join(s.rootDir, confloader.ConfPrefix, s.commandLineConfigFile)
+	configPath := filepath.Join(s.rootDir, common.ConfPrefix, s.commandLineConfigFile)
 	data, err := os.ReadFile(configPath)
 	if err != nil {
 		return nil, err

diff --git a/cmd/kuscia/modules/k3s.go b/cmd/kuscia/modules/k3s.go
@@ -35,7 +35,7 @@ import (
 
 	"github.com/google/uuid"
 
-	"github.com/secretflow/kuscia/cmd/kuscia/confloader"
+	pkgcom "github.com/secretflow/kuscia/pkg/common"
 	"github.com/secretflow/kuscia/pkg/utils/common"
 	"github.com/secretflow/kuscia/pkg/utils/network"
 	"github.com/secretflow/kuscia/pkg/utils/nlog/ljwriter"
@@ -128,7 +128,6 @@ func NewK3s(i *Dependencies) Module {
 	if clusterToken == "" {
 		clusterToken = fmt.Sprintf("%x", md5.Sum([]byte(i.DomainID)))
 	}
-	nlog.Infof("ClusterToken is: %s", clusterToken)
 	hostIP, err := network.GetHostIP()
 	if err != nil {
 		nlog.Fatal(err)
@@ -176,15 +175,15 @@ func (s *k3sModule) Run(ctx context.Context) error {
 	}
 	if s.enableAudit {
 		args = append(args,
-			"--kube-apiserver-arg=audit-log-path="+filepath.Join(s.rootDir, confloader.LogPrefix, "k3s-audit.log"),
-			"--kube-apiserver-arg=audit-policy-file="+filepath.Join(s.rootDir, confloader.ConfPrefix, "k3s/k3s-audit-policy.yaml"),
+			"--kube-apiserver-arg=audit-log-path="+filepath.Join(s.rootDir, pkgcom.LogPrefix, "k3s-audit.log"),
+			"--kube-apiserver-arg=audit-policy-file="+filepath.Join(s.rootDir, pkgcom.ConfPrefix, "k3s/k3s-audit-policy.yaml"),
 			"--kube-apiserver-arg=audit-log-maxbackup=10",
 			"--kube-apiserver-arg=audit-log-maxsize=300",
 		)
 	}
 
 	sp := supervisor.NewSupervisor("k3s", nil, -1)
-	s.LogConfig.LogPath = filepath.Join(s.rootDir, confloader.LogPrefix, "k3s.log")
+	s.LogConfig.LogPath = filepath.Join(s.rootDir, pkgcom.LogPrefix, "k3s.log")
 	lj, _ := ljwriter.New(&s.LogConfig)
 	n := nlog.NewNLog(nlog.SetWriter(lj))
 
@@ -274,9 +273,9 @@ func genKusciaKubeConfig(conf *Dependencies) error {
 		serverCertFile:         filepath.Join(conf.RootDir, k3sDataDirPrefix, "server/tls/server-ca.crt"),
 		clientKeyFile:          filepath.Join(conf.RootDir, k3sDataDirPrefix, "server/tls/client-ca.key"),
 		clientCertFile:         filepath.Join(conf.RootDir, k3sDataDirPrefix, "server/tls/client-ca.crt"),
-		clusterRoleFile:        filepath.Join(conf.RootDir, confloader.ConfPrefix, "kuscia-clusterrole.yaml"),
-		clusterRoleBindingFile: filepath.Join(conf.RootDir, confloader.ConfPrefix, "kuscia-clusterrolebinding.yaml"),
-		kubeConfigTmplFile:     filepath.Join(conf.RootDir, confloader.ConfPrefix, "kuscia.kubeconfig.tmpl"),
+		clusterRoleFile:        filepath.Join(conf.RootDir, pkgcom.ConfPrefix, "kuscia-clusterrole.yaml"),
+		clusterRoleBindingFile: filepath.Join(conf.RootDir, pkgcom.ConfPrefix, "kuscia-clusterrolebinding.yaml"),
+		kubeConfigTmplFile:     filepath.Join(conf.RootDir, pkgcom.ConfPrefix, "kuscia.kubeconfig.tmpl"),
 		kubeConfig:             conf.KusciaKubeConfig,
 	}
 
@@ -341,7 +340,7 @@ func genKusciaKubeConfig(conf *Dependencies) error {
 func applyKusciaResources(conf *Dependencies) error {
 	// apply kuscia clusterRole
 	resourceFiles := []string{
-		filepath.Join(conf.RootDir, confloader.ConfPrefix, "domain-cluster-res.yaml"),
+		filepath.Join(conf.RootDir, pkgcom.ConfPrefix, "domain-cluster-res.yaml"),
 	}
 	sw := sync.WaitGroup{}
 	for _, file := range resourceFiles {