From 1ae191a936322cd2a388cad7b148e4c6e765b8b5 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Tue, 18 Jul 2023 23:53:10 -0400 Subject: [PATCH] trace: instrument `compose up` at a high-level * Image pull * Image build * Service apply * Scale down/up (event) * Recreate container (event) * Scale up (event) * Container start (event) Signed-off-by: Milas Bowman --- internal/tracing/attributes.go | 152 +++++++++++++++++++++++++++++++++ internal/tracing/wrap.go | 91 ++++++++++++++++++++ pkg/compose/build.go | 30 +++++-- pkg/compose/convergence.go | 44 ++++++---- pkg/compose/up.go | 6 +- 5 files changed, 295 insertions(+), 28 deletions(-) create mode 100644 internal/tracing/attributes.go create mode 100644 internal/tracing/wrap.go diff --git a/internal/tracing/attributes.go b/internal/tracing/attributes.go new file mode 100644 index 0000000000..76327fe400 --- /dev/null +++ b/internal/tracing/attributes.go @@ -0,0 +1,152 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package tracing + +import ( + "strings" + "time" + + "github.com/compose-spec/compose-go/types" + moby "github.com/docker/docker/api/types" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// SpanOptions is a small helper type to make it easy to share the options helpers between +// downstream functions that accept slices of trace.SpanStartOption and trace.EventOption. +type SpanOptions []trace.SpanStartEventOption + +func (s SpanOptions) SpanStartOptions() []trace.SpanStartOption { + out := make([]trace.SpanStartOption, len(s)) + for i := range s { + out[i] = s[i] + } + return out +} + +func (s SpanOptions) EventOptions() []trace.EventOption { + out := make([]trace.EventOption, len(s)) + for i := range s { + out[i] = s[i] + } + return out +} + +// ProjectOptions returns common attributes from a Compose project. +// +// For convenience, it's returned as a SpanOptions object to allow it to be +// passed directly to the wrapping helper methods in this package such as +// SpanWrapFunc. +func ProjectOptions(proj *types.Project) SpanOptions { + if proj == nil { + return nil + } + + disabledServiceNames := make([]string, len(proj.DisabledServices)) + for i := range proj.DisabledServices { + disabledServiceNames[i] = proj.DisabledServices[i].Name + } + + attrs := []attribute.KeyValue{ + attribute.String("project.name", proj.Name), + attribute.String("project.dir", proj.WorkingDir), + attribute.StringSlice("project.compose_files", proj.ComposeFiles), + attribute.StringSlice("project.services.active", proj.ServiceNames()), + attribute.StringSlice("project.services.disabled", disabledServiceNames), + attribute.StringSlice("project.profiles", proj.Profiles), + attribute.StringSlice("project.volumes", proj.VolumeNames()), + attribute.StringSlice("project.networks", proj.NetworkNames()), + attribute.StringSlice("project.secrets", proj.SecretNames()), + attribute.StringSlice("project.configs", proj.ConfigNames()), + attribute.StringSlice("project.extensions", keys(proj.Extensions)), + } + return []trace.SpanStartEventOption{ + trace.WithAttributes(attrs...), + } +} + +// ServiceOptions returns common attributes from a Compose service. +// +// For convenience, it's returned as a SpanOptions object to allow it to be +// passed directly to the wrapping helper methods in this package such as +// SpanWrapFunc. +func ServiceOptions(service types.ServiceConfig) SpanOptions { + attrs := []attribute.KeyValue{ + attribute.String("service.name", service.Name), + attribute.String("service.image", service.Image), + attribute.StringSlice("service.networks", keys(service.Networks)), + } + + configNames := make([]string, len(service.Configs)) + for i := range service.Configs { + configNames[i] = service.Configs[i].Source + } + attrs = append(attrs, attribute.StringSlice("service.configs", configNames)) + + secretNames := make([]string, len(service.Secrets)) + for i := range service.Secrets { + secretNames[i] = service.Secrets[i].Source + } + attrs = append(attrs, attribute.StringSlice("service.secrets", secretNames)) + + volNames := make([]string, len(service.Volumes)) + for i := range service.Volumes { + volNames[i] = service.Volumes[i].Source + } + attrs = append(attrs, attribute.StringSlice("service.volumes", volNames)) + + return []trace.SpanStartEventOption{ + trace.WithAttributes(attrs...), + } +} + +// ContainerOptions returns common attributes from a Moby container. +// +// For convenience, it's returned as a SpanOptions object to allow it to be +// passed directly to the wrapping helper methods in this package such as +// SpanWrapFunc. +func ContainerOptions(container moby.Container) SpanOptions { + attrs := []attribute.KeyValue{ + attribute.String("container.id", container.ID), + attribute.String("container.image", container.Image), + unixTimeAttr("container.created_at", container.Created), + } + + if len(container.Names) != 0 { + attrs = append(attrs, attribute.String("container.name", strings.TrimPrefix(container.Names[0], "/"))) + } + + return []trace.SpanStartEventOption{ + trace.WithAttributes(attrs...), + } +} + +func keys[T any](m map[string]T) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} + +func timeAttr(key string, value time.Time) attribute.KeyValue { + return attribute.String(key, value.Format(time.RFC3339)) +} + +func unixTimeAttr(key string, value int64) attribute.KeyValue { + return timeAttr(key, time.Unix(value, 0).UTC()) +} diff --git a/internal/tracing/wrap.go b/internal/tracing/wrap.go new file mode 100644 index 0000000000..8de84036f8 --- /dev/null +++ b/internal/tracing/wrap.go @@ -0,0 +1,91 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package tracing + +import ( + "context" + + "go.opentelemetry.io/otel/codes" + semconv "go.opentelemetry.io/otel/semconv/v1.18.0" + "go.opentelemetry.io/otel/trace" +) + +// SpanWrapFunc wraps a function that takes a context with a trace.Span, marking the status as codes.Error if the +// wrapped function returns an error. +// +// The context passed to the function is created from the span to ensure correct propagation. +// +// NOTE: This function is nearly identical to SpanWrapFuncForErrGroup, except the latter is designed specially for +// convenience with errgroup.Group due to its prevalence throughout the codebase. The code is duplicated to avoid +// adding even more levels of function wrapping/indirection. +func SpanWrapFunc(spanName string, opts SpanOptions, fn func(ctx context.Context) error) func(context.Context) error { + return func(ctx context.Context) error { + ctx, span := Tracer.Start(ctx, spanName, opts.SpanStartOptions()...) + defer span.End() + + if err := fn(ctx); err != nil { + span.SetStatus(codes.Error, err.Error()) + return err + } + + span.SetStatus(codes.Ok, "") + return nil + } +} + +// SpanWrapFuncForErrGroup wraps a function that takes a context with a trace.Span, marking the status as codes.Error +// if the wrapped function returns an error. +// +// The context passed to the function is created from the span to ensure correct propagation. +// +// NOTE: This function is nearly identical to SpanWrapFunc, except this function is designed specially for +// convenience with errgroup.Group due to its prevalence throughout the codebase. The code is duplicated to avoid +// adding even more levels of function wrapping/indirection. +func SpanWrapFuncForErrGroup(ctx context.Context, spanName string, opts SpanOptions, fn func(ctx context.Context) error) func() error { + return func() error { + ctx, span := Tracer.Start(ctx, spanName, opts.SpanStartOptions()...) + defer span.End() + + if err := fn(ctx); err != nil { + span.SetStatus(codes.Error, err.Error()) + return err + } + + span.SetStatus(codes.Ok, "") + return nil + } +} + +// EventWrapFuncForErrGroup invokes a function and records an event, optionally including the returned +// error as the "exception message" on the event. +// +// This is intended for lightweight usage to wrap errgroup.Group calls where a full span is not desired. +func EventWrapFuncForErrGroup(ctx context.Context, eventName string, opts SpanOptions, fn func(ctx context.Context) error) func() error { + return func() error { + span := trace.SpanFromContext(ctx) + eventOpts := opts.EventOptions() + + err := fn(ctx) + + if err != nil { + eventOpts = append(eventOpts, trace.WithAttributes(semconv.ExceptionMessage(err.Error()))) + } + span.AddEvent(eventName, eventOpts...) + + return err + } +} diff --git a/pkg/compose/build.go b/pkg/compose/build.go index 9840b79c90..c057af7ecf 100644 --- a/pkg/compose/build.go +++ b/pkg/compose/build.go @@ -22,6 +22,8 @@ import ( "os" "path/filepath" + "github.com/docker/compose/v2/internal/tracing" + "github.com/docker/buildx/controller/pb" "github.com/compose-spec/compose-go/types" @@ -170,7 +172,11 @@ func (s *composeService) ensureImagesExists(ctx context.Context, project *types. return err } - err = s.pullRequiredImages(ctx, project, images, quietPull) + err = tracing.SpanWrapFunc("project/pull", tracing.ProjectOptions(project), + func(ctx context.Context) error { + return s.pullRequiredImages(ctx, project, images, quietPull) + }, + )(ctx) if err != nil { return err } @@ -186,16 +192,24 @@ func (s *composeService) ensureImagesExists(ctx context.Context, project *types. } if buildRequired { - builtImages, err := s.build(ctx, project, api.BuildOptions{ - Progress: mode, - }) + err = tracing.SpanWrapFunc("project/build", tracing.ProjectOptions(project), + func(ctx context.Context) error { + builtImages, err := s.build(ctx, project, api.BuildOptions{ + Progress: mode, + }) + if err != nil { + return err + } + + for name, digest := range builtImages { + images[name] = digest + } + return nil + }, + )(ctx) if err != nil { return err } - - for name, digest := range builtImages { - images[name] = digest - } } // set digest as com.docker.compose.image label so we can detect outdated containers diff --git a/pkg/compose/convergence.go b/pkg/compose/convergence.go index 046119c5c3..179d6fffc8 100644 --- a/pkg/compose/convergence.go +++ b/pkg/compose/convergence.go @@ -25,8 +25,12 @@ import ( "sync" "time" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "github.com/compose-spec/compose-go/types" "github.com/containerd/containerd/platforms" + "github.com/docker/compose/v2/internal/tracing" moby "github.com/docker/docker/api/types" containerType "github.com/docker/docker/api/types/container" specs "github.com/opencontainers/image-spec/specs-go/v1" @@ -93,17 +97,19 @@ func (c *convergence) apply(ctx context.Context, project *types.Project, options return err } - strategy := options.RecreateDependencies - if utils.StringContains(options.Services, name) { - strategy = options.Recreate - } - err = c.ensureService(ctx, project, service, strategy, options.Inherit, options.Timeout) - if err != nil { - return err - } + return tracing.SpanWrapFunc("service/apply", tracing.ServiceOptions(service), func(ctx context.Context) error { + strategy := options.RecreateDependencies + if utils.StringContains(options.Services, name) { + strategy = options.Recreate + } + err = c.ensureService(ctx, project, service, strategy, options.Inherit, options.Timeout) + if err != nil { + return err + } - c.updateProject(project, name) - return nil + c.updateProject(project, name) + return nil + })(ctx) }) } @@ -179,7 +185,8 @@ func (c *convergence) ensureService(ctx context.Context, project *types.Project, if i >= expected { // Scale Down container := container - eg.Go(func() error { + traceOpts := append(tracing.ServiceOptions(service), tracing.ContainerOptions(container)...) + eg.Go(tracing.SpanWrapFuncForErrGroup(ctx, "service/scale/down", traceOpts, func(ctx context.Context) error { timeoutInSecond := utils.DurationSecondToInt(timeout) err := c.service.apiClient().ContainerStop(ctx, container.ID, containerType.StopOptions{ Timeout: timeoutInSecond, @@ -188,7 +195,7 @@ func (c *convergence) ensureService(ctx context.Context, project *types.Project, return err } return c.service.apiClient().ContainerRemove(ctx, container.ID, moby.ContainerRemoveOptions{}) - }) + })) continue } @@ -198,11 +205,11 @@ func (c *convergence) ensureService(ctx context.Context, project *types.Project, } if mustRecreate { i, container := i, container - eg.Go(func() error { + eg.Go(tracing.SpanWrapFuncForErrGroup(ctx, "container/recreate", tracing.ContainerOptions(container), func(ctx context.Context) error { recreated, err := c.service.recreateContainer(ctx, project, service, container, inherit, timeout) updated[i] = recreated return err - }) + })) continue } @@ -218,9 +225,9 @@ func (c *convergence) ensureService(ctx context.Context, project *types.Project, w.Event(progress.CreatedEvent(name)) default: container := container - eg.Go(func() error { + eg.Go(tracing.EventWrapFuncForErrGroup(ctx, "service/start", tracing.ContainerOptions(container), func(ctx context.Context) error { return c.service.startContainer(ctx, container) - }) + })) } updated[i] = container } @@ -231,7 +238,8 @@ func (c *convergence) ensureService(ctx context.Context, project *types.Project, number := next + i name := getContainerName(project.Name, service, number) i := i - eg.Go(func() error { + eventOpts := tracing.SpanOptions{trace.WithAttributes(attribute.String("container.name", name))} + eg.Go(tracing.EventWrapFuncForErrGroup(ctx, "service/scale/up", eventOpts, func(ctx context.Context) error { opts := createOptions{ AutoRemove: false, AttachStdin: false, @@ -241,7 +249,7 @@ func (c *convergence) ensureService(ctx context.Context, project *types.Project, container, err := c.service.createContainer(ctx, project, service, name, number, opts) updated[actual+i] = container return err - }) + })) continue } diff --git a/pkg/compose/up.go b/pkg/compose/up.go index 98066cd615..a0225c45a8 100644 --- a/pkg/compose/up.go +++ b/pkg/compose/up.go @@ -23,6 +23,8 @@ import ( "os/signal" "syscall" + "github.com/docker/compose/v2/internal/tracing" + "github.com/compose-spec/compose-go/types" "github.com/docker/cli/cli" "github.com/docker/compose/v2/pkg/api" @@ -31,7 +33,7 @@ import ( ) func (s *composeService) Up(ctx context.Context, project *types.Project, options api.UpOptions) error { - err := progress.Run(ctx, func(ctx context.Context) error { + err := progress.Run(ctx, tracing.SpanWrapFunc("project/up", tracing.ProjectOptions(project), func(ctx context.Context) error { err := s.create(ctx, project, options.Create) if err != nil { return err @@ -40,7 +42,7 @@ func (s *composeService) Up(ctx context.Context, project *types.Project, options return s.start(ctx, project.Name, options.Start, nil) } return nil - }, s.stdinfo()) + }), s.stdinfo()) if err != nil { return err }