From 09296619283ce568866df414bf26db3bda24d9c7 Mon Sep 17 00:00:00 2001 From: Kaiyang Chen <48289729+Kaiyang-Chen@users.noreply.github.com> Date: Mon, 13 Feb 2023 22:03:24 -0600 Subject: [PATCH] feat: support buildx moby worker in docker 23.0.0 onward to accelerating building process by skipping docker load (#1472) * feat: support buildx moby worker in docker 23.0.0 to accelerating building process by skipping docker load Signed-off-by: Kaiyang-Chen * fix: lint error Signed-off-by: Kaiyang-Chen * fix: lint error Signed-off-by: Kaiyang Chen * feat: relocate getDockerVersion in pkg/driver/docker Signed-off-by: Kaiyang Chen --------- Signed-off-by: Kaiyang-Chen Signed-off-by: Kaiyang Chen --- pkg/app/bootstrap.go | 18 ++++++++++++---- pkg/app/prune.go | 18 ++++++++++++---- pkg/builder/build.go | 40 ++++++++++++++++++++++++++++-------- pkg/buildkitd/buildkitd.go | 37 +++++++++++++++++++++++++++++++++ pkg/driver/docker/docker.go | 21 +++++++++++++++++++ pkg/home/context.go | 1 + pkg/home/manager.go | 8 +++++++- pkg/lang/ir/v0/supervisor.go | 7 +++++-- pkg/lang/ir/v1/supervisor.go | 7 +++++-- pkg/types/envd.go | 1 + 10 files changed, 137 insertions(+), 21 deletions(-) diff --git a/pkg/app/bootstrap.go b/pkg/app/bootstrap.go index f238ace65..3a6fa80b0 100644 --- a/pkg/app/bootstrap.go +++ b/pkg/app/bootstrap.go @@ -29,6 +29,7 @@ import ( "github.com/tensorchord/envd/pkg/buildkitd" "github.com/tensorchord/envd/pkg/home" sshconfig "github.com/tensorchord/envd/pkg/ssh/config" + "github.com/tensorchord/envd/pkg/types" "github.com/tensorchord/envd/pkg/util/fileutil" ) @@ -205,10 +206,19 @@ func buildkit(clicontext *cli.Context) error { } logrus.Debug("bootstrap the buildkitd container") - bkClient, err := buildkitd.NewClient(clicontext.Context, - c.Builder, c.BuilderAddress, clicontext.String("dockerhub-mirror")) - if err != nil { - return errors.Wrap(err, "failed to create buildkit client") + var bkClient buildkitd.Client + if c.Builder == types.BuilderTypeMoby { + bkClient, err = buildkitd.NewMobyClient(clicontext.Context, + c.Builder, c.BuilderAddress, clicontext.String("dockerhub-mirror")) + if err != nil { + return errors.Wrap(err, "failed to create moby buildkit client") + } + } else { + bkClient, err = buildkitd.NewClient(clicontext.Context, + c.Builder, c.BuilderAddress, clicontext.String("dockerhub-mirror")) + if err != nil { + return errors.Wrap(err, "failed to create buildkit client") + } } defer bkClient.Close() logrus.Infof("The buildkit is running at %s", bkClient.BuildkitdAddr()) diff --git a/pkg/app/prune.go b/pkg/app/prune.go index 555fe35a8..aedce9bde 100644 --- a/pkg/app/prune.go +++ b/pkg/app/prune.go @@ -23,6 +23,7 @@ import ( "github.com/tensorchord/envd/pkg/app/telemetry" "github.com/tensorchord/envd/pkg/buildkitd" "github.com/tensorchord/envd/pkg/home" + "github.com/tensorchord/envd/pkg/types" ) var CommandPrune = &cli.Command{ @@ -77,10 +78,19 @@ func prune(clicontext *cli.Context) error { if err != nil { return errors.Wrap(err, "failed to get the current context") } - bkClient, err := buildkitd.NewClient(clicontext.Context, - c.Builder, c.BuilderAddress, "") - if err != nil { - return errors.Wrap(err, "failed to create buildkit client") + var bkClient buildkitd.Client + if c.Builder == types.BuilderTypeMoby { + bkClient, err = buildkitd.NewMobyClient(clicontext.Context, + c.Builder, c.BuilderAddress, "") + if err != nil { + return errors.Wrap(err, "failed to create moby buildkit client") + } + } else { + bkClient, err = buildkitd.NewClient(clicontext.Context, + c.Builder, c.BuilderAddress, "") + if err != nil { + return errors.Wrap(err, "failed to create buildkit client") + } } if err := bkClient.Prune(clicontext.Context, keepDuration, keepStorage, filter, verbose, cleanAll); err != nil { diff --git a/pkg/builder/build.go b/pkg/builder/build.go index ab0b32c4b..aa454c1fd 100644 --- a/pkg/builder/build.go +++ b/pkg/builder/build.go @@ -45,12 +45,21 @@ func New(ctx context.Context, opt Options) (Builder, error) { return nil, errors.Wrap(err, "failed to parse output") } + c, err := home.GetManager().ContextGetCurrent() + if err != nil { + return nil, errors.Wrap(err, "failed to get the current context") + } + logrus.WithField("entry", entries).Debug("getting exporter entry") // Build docker image by default if len(entries) == 0 { + exportType := client.ExporterDocker + if c.Builder == types.BuilderTypeMoby { + exportType = "moby" + } entries = []client.ExportEntry{ { - Type: client.ExporterDocker, + Type: exportType, }, } } else if len(entries) > 1 { @@ -76,13 +85,19 @@ func New(ctx context.Context, opt Options) (Builder, error) { GetDepsFilesHandler: vc.GetDefaultGraph().GetDepsFiles, } - c, err := home.GetManager().ContextGetCurrent() - if err != nil { - return nil, errors.Wrap(err, "failed to get the current context") - } - cli, err := buildkitd.NewClient(ctx, c.Builder, c.BuilderAddress, "") - if err != nil { - return nil, errors.Wrap(err, "failed to create buildkit client") + var cli buildkitd.Client + if c.Builder == types.BuilderTypeMoby { + cli, err = buildkitd.NewMobyClient(ctx, + c.Builder, c.BuilderAddress, "") + if err != nil { + return nil, errors.Wrap(err, "failed to create moby buildkit client") + } + } else { + cli, err = buildkitd.NewClient(ctx, + c.Builder, c.BuilderAddress, "") + if err != nil { + return nil, errors.Wrap(err, "failed to create buildkit client") + } } b.Client = cli @@ -294,6 +309,15 @@ func (b generalBuilder) build(ctx context.Context, pw progresswriter.Writer) err func constructSolveOpt(ce []client.CacheOptionsEntry, entry client.ExportEntry, b generalBuilder, attachable []session.Attachable) client.SolveOpt { + c, _ := home.GetManager().ContextGetCurrent() + if entry.Attrs == nil && c.Builder == types.BuilderTypeMoby { + entry = client.ExportEntry{ + Type: "moby", + Attrs: map[string]string{ + "name": b.Tag, + }, + } + } opt := client.SolveOpt{ CacheExports: ce, Exports: []client.ExportEntry{entry}, diff --git a/pkg/buildkitd/buildkitd.go b/pkg/buildkitd/buildkitd.go index bd4f4edde..adaaf6c47 100644 --- a/pkg/buildkitd/buildkitd.go +++ b/pkg/buildkitd/buildkitd.go @@ -17,11 +17,13 @@ package buildkitd import ( "context" "fmt" + "net" "os" "text/tabwriter" "time" "github.com/cockroachdb/errors" + dockerclient "github.com/docker/docker/client" "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" gateway "github.com/moby/buildkit/frontend/gateway/client" @@ -69,6 +71,41 @@ type generalClient struct { logger *logrus.Entry } +func NewMobyClient(ctx context.Context, driver types.BuilderType, + socket, mirror string) (Client, error) { + logrus.Debug("getting moby buildkit client") + c := &generalClient{ + containerName: socket, + image: viper.GetString(flag.FlagBuildkitdImage), + mirror: mirror, + } + c.socket = socket + c.driver = driver + c.logger = logrus.WithFields(logrus.Fields{ + "container": c.containerName, + "image": c.image, + "socket": c.socket, + "driver": c.driver, + }) + dockerCli, err := dockerclient.NewClientWithOpts(dockerclient.FromEnv, dockerclient.WithAPIVersionNegotiation()) + if err != nil { + return nil, errors.Wrap(err, "failed to create the client") + } + bkcli, err := client.New(ctx, c.BuildkitdAddr(), + client.WithFailFast(), + client.WithContextDialer(func(context.Context, string) (net.Conn, error) { + return dockerCli.DialHijack(ctx, "/grpc", "h2c", nil) + }), client.WithSessionDialer(func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error) { + return dockerCli.DialHijack(ctx, "/session", proto, meta) + }), + ) + if err != nil { + return nil, errors.Wrap(err, "failed to create buildkit clientt") + } + c.Client = bkcli + return c, nil +} + func NewClient(ctx context.Context, driver types.BuilderType, socket, mirror string) (Client, error) { c := &generalClient{ diff --git a/pkg/driver/docker/docker.go b/pkg/driver/docker/docker.go index a564297f5..c1f37dfb7 100644 --- a/pkg/driver/docker/docker.go +++ b/pkg/driver/docker/docker.go @@ -21,6 +21,7 @@ import ( "io" "os" "regexp" + "strconv" "strings" "time" @@ -380,3 +381,23 @@ func (c dockerClient) waitUntilRunning(ctx context.Context, } } } + +func GetDockerVersion() (int, error) { + + ctx := context.Background() + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return -1, err + } + defer cli.Close() + + info, err := cli.Info(ctx) + if err != nil { + return -1, err + } + version, err := strconv.Atoi(strings.Split(info.ServerVersion, ".")[0]) + if err != nil { + return -1, err + } + return version, nil +} diff --git a/pkg/home/context.go b/pkg/home/context.go index 97735bb5f..b3dcf16d0 100644 --- a/pkg/home/context.go +++ b/pkg/home/context.go @@ -95,6 +95,7 @@ func (m *generalManager) ContextCreate(ctx types.Context, use bool) error { } switch ctx.Builder { case types.BuilderTypeDocker, + types.BuilderTypeMoby, types.BuilderTypeNerdctl, types.BuilderTypeKubernetes, types.BuilderTypeUNIXDomainSocket, diff --git a/pkg/home/manager.go b/pkg/home/manager.go index 518c8ac42..cc41a9f05 100644 --- a/pkg/home/manager.go +++ b/pkg/home/manager.go @@ -20,6 +20,7 @@ import ( "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" + "github.com/tensorchord/envd/pkg/driver/docker" sshconfig "github.com/tensorchord/envd/pkg/ssh/config" "github.com/tensorchord/envd/pkg/types" ) @@ -53,6 +54,11 @@ var ( ) func Initialize() error { + builder := types.BuilderTypeDocker + dockerVersion, err := docker.GetDockerVersion() + if err == nil && dockerVersion > 22 { + builder = types.BuilderTypeMoby + } once.Do(func() { defaultManager = &generalManager{ cacheMap: make(map[string]bool), @@ -61,7 +67,7 @@ func Initialize() error { Contexts: []types.Context{ { Name: "default", - Builder: types.BuilderTypeDocker, + Builder: builder, BuilderAddress: "envd_buildkitd", Runner: types.RunnerTypeDocker, RunnerAddress: nil, diff --git a/pkg/lang/ir/v0/supervisor.go b/pkg/lang/ir/v0/supervisor.go index 550500be8..1e492f40f 100644 --- a/pkg/lang/ir/v0/supervisor.go +++ b/pkg/lang/ir/v0/supervisor.go @@ -58,8 +58,11 @@ func (g generalGraph) installHorust(root llb.State) llb.State { File(llb.Mkdir(types.HorustServiceDir, 0755, llb.WithParents(true)), llb.WithCustomNamef("[internal] mkdir for horust service: %s", types.HorustServiceDir)). File(llb.Mkdir(types.HorustLogDir, 0777, llb.WithParents(true)), - llb.WithCustomNamef("[internal] mkdir for horust log: %s", types.HorustLogDir)) - return horust + llb.WithCustomNamef("[internal] mkdir for horust log: %s", types.HorustLogDir)). + Run(llb.Shlexf(`sudo chmod 777 %s`, types.HorustLogDir), + llb.WithCustomName("[internal] change dirctory permission for logging")) + + return horust.Root() } func (g generalGraph) addNewProcess(root llb.State, name, command string, depends []string) llb.State { diff --git a/pkg/lang/ir/v1/supervisor.go b/pkg/lang/ir/v1/supervisor.go index 1ce7d87f6..bc7c7284d 100644 --- a/pkg/lang/ir/v1/supervisor.go +++ b/pkg/lang/ir/v1/supervisor.go @@ -58,8 +58,11 @@ func (g generalGraph) installHorust(root llb.State) llb.State { File(llb.Mkdir(types.HorustServiceDir, 0755, llb.WithParents(true)), llb.WithCustomNamef("[internal] mkdir for horust service: %s", types.HorustServiceDir)). File(llb.Mkdir(types.HorustLogDir, 0777, llb.WithParents(true)), - llb.WithCustomNamef("[internal] mkdir for horust log: %s", types.HorustLogDir)) - return horust + llb.WithCustomNamef("[internal] mkdir for horust log: %s", types.HorustLogDir)). + Run(llb.Shlexf(`sudo chmod 777 %s`, types.HorustLogDir), + llb.WithCustomName("[internal] change dirctory permission for logging")) + + return horust.Root() } func (g generalGraph) addNewProcess(root llb.State, name, command string, depends []string) llb.State { diff --git a/pkg/types/envd.go b/pkg/types/envd.go index c41155ee6..32b988e64 100644 --- a/pkg/types/envd.go +++ b/pkg/types/envd.go @@ -122,6 +122,7 @@ type Context struct { type BuilderType string const ( + BuilderTypeMoby BuilderType = "moby-worker" BuilderTypeDocker BuilderType = "docker-container" BuilderTypeNerdctl BuilderType = "nerdctl-container" BuilderTypeKubernetes BuilderType = "kube-pod"