feat: support buildx moby worker in docker 23.0.0 onward to accelerat…

…ing building process by skipping docker load (#1472) * feat: support buildx moby worker in docker 23.0.0 to accelerating building process by skipping docker load Signed-off-by: Kaiyang-Chen <[email protected]> * fix: lint error Signed-off-by: Kaiyang-Chen <[email protected]> * fix: lint error Signed-off-by: Kaiyang Chen <[email protected]> * feat: relocate getDockerVersion in pkg/driver/docker Signed-off-by: Kaiyang Chen <[email protected]> --------- Signed-off-by: Kaiyang-Chen <[email protected]> Signed-off-by: Kaiyang Chen <[email protected]>
tensorchord · Feb 14, 2023 · 0929661 · 0929661
1 parent 34ca5b6
commit 0929661
Show file tree

Hide file tree

Showing 10 changed files with 137 additions and 21 deletions.
diff --git a/pkg/app/bootstrap.go b/pkg/app/bootstrap.go
@@ -29,6 +29,7 @@ import (
 	"github.com/tensorchord/envd/pkg/buildkitd"
 	"github.com/tensorchord/envd/pkg/home"
 	sshconfig "github.com/tensorchord/envd/pkg/ssh/config"
+	"github.com/tensorchord/envd/pkg/types"
 	"github.com/tensorchord/envd/pkg/util/fileutil"
 )
 
@@ -205,10 +206,19 @@ func buildkit(clicontext *cli.Context) error {
 	}
 
 	logrus.Debug("bootstrap the buildkitd container")
-	bkClient, err := buildkitd.NewClient(clicontext.Context,
-		c.Builder, c.BuilderAddress, clicontext.String("dockerhub-mirror"))
-	if err != nil {
-		return errors.Wrap(err, "failed to create buildkit client")
+	var bkClient buildkitd.Client
+	if c.Builder == types.BuilderTypeMoby {
+		bkClient, err = buildkitd.NewMobyClient(clicontext.Context,
+			c.Builder, c.BuilderAddress, clicontext.String("dockerhub-mirror"))
+		if err != nil {
+			return errors.Wrap(err, "failed to create moby buildkit client")
+		}
+	} else {
+		bkClient, err = buildkitd.NewClient(clicontext.Context,
+			c.Builder, c.BuilderAddress, clicontext.String("dockerhub-mirror"))
+		if err != nil {
+			return errors.Wrap(err, "failed to create buildkit client")
+		}
 	}
 	defer bkClient.Close()
 	logrus.Infof("The buildkit is running at %s", bkClient.BuildkitdAddr())

diff --git a/pkg/app/prune.go b/pkg/app/prune.go
@@ -23,6 +23,7 @@ import (
 	"github.com/tensorchord/envd/pkg/app/telemetry"
 	"github.com/tensorchord/envd/pkg/buildkitd"
 	"github.com/tensorchord/envd/pkg/home"
+	"github.com/tensorchord/envd/pkg/types"
 )
 
 var CommandPrune = &cli.Command{
@@ -77,10 +78,19 @@ func prune(clicontext *cli.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to get the current context")
 	}
-	bkClient, err := buildkitd.NewClient(clicontext.Context,
-		c.Builder, c.BuilderAddress, "")
-	if err != nil {
-		return errors.Wrap(err, "failed to create buildkit client")
+	var bkClient buildkitd.Client
+	if c.Builder == types.BuilderTypeMoby {
+		bkClient, err = buildkitd.NewMobyClient(clicontext.Context,
+			c.Builder, c.BuilderAddress, "")
+		if err != nil {
+			return errors.Wrap(err, "failed to create moby buildkit client")
+		}
+	} else {
+		bkClient, err = buildkitd.NewClient(clicontext.Context,
+			c.Builder, c.BuilderAddress, "")
+		if err != nil {
+			return errors.Wrap(err, "failed to create buildkit client")
+		}
 	}
 	if err := bkClient.Prune(clicontext.Context,
 		keepDuration, keepStorage, filter, verbose, cleanAll); err != nil {

diff --git a/pkg/builder/build.go b/pkg/builder/build.go
@@ -45,12 +45,21 @@ func New(ctx context.Context, opt Options) (Builder, error) {
 		return nil, errors.Wrap(err, "failed to parse output")
 	}
 
+	c, err := home.GetManager().ContextGetCurrent()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get the current context")
+	}
+
 	logrus.WithField("entry", entries).Debug("getting exporter entry")
 	// Build docker image by default
 	if len(entries) == 0 {
+		exportType := client.ExporterDocker
+		if c.Builder == types.BuilderTypeMoby {
+			exportType = "moby"
+		}
 		entries = []client.ExportEntry{
 			{
-				Type: client.ExporterDocker,
+				Type: exportType,
 			},
 		}
 	} else if len(entries) > 1 {
@@ -76,13 +85,19 @@ func New(ctx context.Context, opt Options) (Builder, error) {
 		GetDepsFilesHandler: vc.GetDefaultGraph().GetDepsFiles,
 	}
 
-	c, err := home.GetManager().ContextGetCurrent()
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get the current context")
-	}
-	cli, err := buildkitd.NewClient(ctx, c.Builder, c.BuilderAddress, "")
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to create buildkit client")
+	var cli buildkitd.Client
+	if c.Builder == types.BuilderTypeMoby {
+		cli, err = buildkitd.NewMobyClient(ctx,
+			c.Builder, c.BuilderAddress, "")
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create moby buildkit client")
+		}
+	} else {
+		cli, err = buildkitd.NewClient(ctx,
+			c.Builder, c.BuilderAddress, "")
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create buildkit client")
+		}
 	}
 	b.Client = cli
 
@@ -294,6 +309,15 @@ func (b generalBuilder) build(ctx context.Context, pw progresswriter.Writer) err
 
 func constructSolveOpt(ce []client.CacheOptionsEntry, entry client.ExportEntry,
 	b generalBuilder, attachable []session.Attachable) client.SolveOpt {
+	c, _ := home.GetManager().ContextGetCurrent()
+	if entry.Attrs == nil && c.Builder == types.BuilderTypeMoby {
+		entry = client.ExportEntry{
+			Type: "moby",
+			Attrs: map[string]string{
+				"name": b.Tag,
+			},
+		}
+	}
 	opt := client.SolveOpt{
 		CacheExports: ce,
 		Exports:      []client.ExportEntry{entry},

diff --git a/pkg/buildkitd/buildkitd.go b/pkg/buildkitd/buildkitd.go
@@ -17,11 +17,13 @@ package buildkitd
 import (
 	"context"
 	"fmt"
+	"net"
 	"os"
 	"text/tabwriter"
 	"time"
 
 	"github.com/cockroachdb/errors"
+	dockerclient "github.com/docker/docker/client"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/client/llb"
 	gateway "github.com/moby/buildkit/frontend/gateway/client"
@@ -69,6 +71,41 @@ type generalClient struct {
 	logger *logrus.Entry
 }
 
+func NewMobyClient(ctx context.Context, driver types.BuilderType,
+	socket, mirror string) (Client, error) {
+	logrus.Debug("getting moby buildkit client")
+	c := &generalClient{
+		containerName: socket,
+		image:         viper.GetString(flag.FlagBuildkitdImage),
+		mirror:        mirror,
+	}
+	c.socket = socket
+	c.driver = driver
+	c.logger = logrus.WithFields(logrus.Fields{
+		"container": c.containerName,
+		"image":     c.image,
+		"socket":    c.socket,
+		"driver":    c.driver,
+	})
+	dockerCli, err := dockerclient.NewClientWithOpts(dockerclient.FromEnv, dockerclient.WithAPIVersionNegotiation())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create the client")
+	}
+	bkcli, err := client.New(ctx, c.BuildkitdAddr(),
+		client.WithFailFast(),
+		client.WithContextDialer(func(context.Context, string) (net.Conn, error) {
+			return dockerCli.DialHijack(ctx, "/grpc", "h2c", nil)
+		}), client.WithSessionDialer(func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error) {
+			return dockerCli.DialHijack(ctx, "/session", proto, meta)
+		}),
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create buildkit clientt")
+	}
+	c.Client = bkcli
+	return c, nil
+}
+
 func NewClient(ctx context.Context, driver types.BuilderType,
 	socket, mirror string) (Client, error) {
 	c := &generalClient{

diff --git a/pkg/driver/docker/docker.go b/pkg/driver/docker/docker.go
@@ -21,6 +21,7 @@ import (
 	"io"
 	"os"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
@@ -380,3 +381,23 @@ func (c dockerClient) waitUntilRunning(ctx context.Context,
 		}
 	}
 }
+
+func GetDockerVersion() (int, error) {
+
+	ctx := context.Background()
+	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	if err != nil {
+		return -1, err
+	}
+	defer cli.Close()
+
+	info, err := cli.Info(ctx)
+	if err != nil {
+		return -1, err
+	}
+	version, err := strconv.Atoi(strings.Split(info.ServerVersion, ".")[0])
+	if err != nil {
+		return -1, err
+	}
+	return version, nil
+}
diff --git a/pkg/home/context.go b/pkg/home/context.go
@@ -95,6 +95,7 @@ func (m *generalManager) ContextCreate(ctx types.Context, use bool) error {
 	}
 	switch ctx.Builder {
 	case types.BuilderTypeDocker,
+		types.BuilderTypeMoby,
 		types.BuilderTypeNerdctl,
 		types.BuilderTypeKubernetes,
 		types.BuilderTypeUNIXDomainSocket,

diff --git a/pkg/home/manager.go b/pkg/home/manager.go
@@ -20,6 +20,7 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/sirupsen/logrus"
 
+	"github.com/tensorchord/envd/pkg/driver/docker"
 	sshconfig "github.com/tensorchord/envd/pkg/ssh/config"
 	"github.com/tensorchord/envd/pkg/types"
 )
@@ -53,6 +54,11 @@ var (
 )
 
 func Initialize() error {
+	builder := types.BuilderTypeDocker
+	dockerVersion, err := docker.GetDockerVersion()
+	if err == nil && dockerVersion > 22 {
+		builder = types.BuilderTypeMoby
+	}
 	once.Do(func() {
 		defaultManager = &generalManager{
 			cacheMap: make(map[string]bool),
@@ -61,7 +67,7 @@ func Initialize() error {
 				Contexts: []types.Context{
 					{
 						Name:           "default",
-						Builder:        types.BuilderTypeDocker,
+						Builder:        builder,
 						BuilderAddress: "envd_buildkitd",
 						Runner:         types.RunnerTypeDocker,
 						RunnerAddress:  nil,

diff --git a/pkg/lang/ir/v0/supervisor.go b/pkg/lang/ir/v0/supervisor.go
@@ -58,8 +58,11 @@ func (g generalGraph) installHorust(root llb.State) llb.State {
 		File(llb.Mkdir(types.HorustServiceDir, 0755, llb.WithParents(true)),
 			llb.WithCustomNamef("[internal] mkdir for horust service: %s", types.HorustServiceDir)).
 		File(llb.Mkdir(types.HorustLogDir, 0777, llb.WithParents(true)),
-			llb.WithCustomNamef("[internal] mkdir for horust log: %s", types.HorustLogDir))
-	return horust
+			llb.WithCustomNamef("[internal] mkdir for horust log: %s", types.HorustLogDir)).
+		Run(llb.Shlexf(`sudo chmod 777 %s`, types.HorustLogDir),
+			llb.WithCustomName("[internal] change dirctory permission for logging"))
+
+	return horust.Root()
 }
 
 func (g generalGraph) addNewProcess(root llb.State, name, command string, depends []string) llb.State {

diff --git a/pkg/lang/ir/v1/supervisor.go b/pkg/lang/ir/v1/supervisor.go
@@ -58,8 +58,11 @@ func (g generalGraph) installHorust(root llb.State) llb.State {
 		File(llb.Mkdir(types.HorustServiceDir, 0755, llb.WithParents(true)),
 			llb.WithCustomNamef("[internal] mkdir for horust service: %s", types.HorustServiceDir)).
 		File(llb.Mkdir(types.HorustLogDir, 0777, llb.WithParents(true)),
-			llb.WithCustomNamef("[internal] mkdir for horust log: %s", types.HorustLogDir))
-	return horust
+			llb.WithCustomNamef("[internal] mkdir for horust log: %s", types.HorustLogDir)).
+		Run(llb.Shlexf(`sudo chmod 777 %s`, types.HorustLogDir),
+			llb.WithCustomName("[internal] change dirctory permission for logging"))
+
+	return horust.Root()
 }
 
 func (g generalGraph) addNewProcess(root llb.State, name, command string, depends []string) llb.State {

diff --git a/pkg/types/envd.go b/pkg/types/envd.go
@@ -122,6 +122,7 @@ type Context struct {
 type BuilderType string
 
 const (
+	BuilderTypeMoby             BuilderType = "moby-worker"
 	BuilderTypeDocker           BuilderType = "docker-container"
 	BuilderTypeNerdctl          BuilderType = "nerdctl-container"
 	BuilderTypeKubernetes       BuilderType = "kube-pod"