diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml index 25e6ef22fbc1f..f45f6b463462b 100644 --- a/.github/workflows/ci-build.yaml +++ b/.github/workflows/ci-build.yaml @@ -370,11 +370,11 @@ jobs: path: test-results - name: combine-go-coverage # We generate coverage reports for all Argo CD components, but only the applicationset-controller, - # app-controller, and repo-server report contain coverage data. The other components currently don't shut down - # gracefully, so no coverage data is produced. Once those components are fixed, we can add references to their - # coverage output directories. + # app-controller, repo-server, and commit-server report contain coverage data. The other components currently + # don't shut down gracefully, so no coverage data is produced. Once those components are fixed, we can add + # references to their coverage output directories. run: | - go tool covdata percent -i=test-results,e2e-code-coverage/applicationset-controller,e2e-code-coverage/repo-server,e2e-code-coverage/app-controller -o test-results/full-coverage.out + go tool covdata percent -i=test-results,e2e-code-coverage/applicationset-controller,e2e-code-coverage/repo-server,e2e-code-coverage/app-controller,e2e-code-coverage/commit-server -o test-results/full-coverage.out - name: Upload code coverage information to codecov.io uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 with: diff --git a/.mockery.yaml b/.mockery.yaml index 3a8b437ef347d..a2af16e826166 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -26,6 +26,13 @@ packages: github.com/argoproj/argo-cd/v2/applicationset/utils: interfaces: Renderer: + github.com/argoproj/argo-cd/v2/commitserver/commit: + interfaces: + RepoClientFactory: + github.com/argoproj/argo-cd/v2/commitserver/apiclient: + interfaces: + CommitServiceClient: + Clientset: github.com/argoproj/argo-cd/v2/controller/cache: interfaces: LiveStateCache: diff --git a/Dockerfile b/Dockerfile index 8132cedbdca6d..484d7dde34548 100644 --- a/Dockerfile +++ b/Dockerfile @@ -140,7 +140,8 @@ RUN ln -s /usr/local/bin/argocd /usr/local/bin/argocd-server && \ ln -s /usr/local/bin/argocd /usr/local/bin/argocd-dex && \ ln -s /usr/local/bin/argocd /usr/local/bin/argocd-notifications && \ ln -s /usr/local/bin/argocd /usr/local/bin/argocd-applicationset-controller && \ - ln -s /usr/local/bin/argocd /usr/local/bin/argocd-k8s-auth + ln -s /usr/local/bin/argocd /usr/local/bin/argocd-k8s-auth && \ + ln -s /usr/local/bin/argocd /usr/local/bin/argocd-commit-server USER $ARGOCD_USER_ID ENTRYPOINT ["/usr/bin/tini", "--"] diff --git a/Makefile b/Makefile index c1ef27163cc60..4dd04ff66348a 100644 --- a/Makefile +++ b/Makefile @@ -472,6 +472,7 @@ start-e2e-local: mod-vendor-local dep-ui-local cli-local mkdir -p /tmp/coverage/repo-server mkdir -p /tmp/coverage/applicationset-controller mkdir -p /tmp/coverage/notification + mkdir -p /tmp/coverage/commit-server # set paths for locally managed ssh known hosts and tls certs data ARGOCD_SSH_DATA_PATH=/tmp/argo-e2e/app/config/ssh \ ARGOCD_TLS_DATA_PATH=/tmp/argo-e2e/app/config/tls \ diff --git a/Procfile b/Procfile index fd955a39ac416..d0f834f70490b 100644 --- a/Procfile +++ b/Procfile @@ -4,6 +4,7 @@ dex: sh -c "ARGOCD_BINARY_NAME=argocd-dex go run github.com/argoproj/argo-cd/v2/ redis: hack/start-redis-with-password.sh repo-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "GOCOVERDIR=${ARGOCD_COVERAGE_DIR:-/tmp/coverage/repo-server} FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_GNUPGHOME=${ARGOCD_GNUPGHOME:-/tmp/argocd-local/gpg/keys} ARGOCD_PLUGINSOCKFILEPATH=${ARGOCD_PLUGINSOCKFILEPATH:-./test/cmp} ARGOCD_GPG_DATA_PATH=${ARGOCD_GPG_DATA_PATH:-/tmp/argocd-local/gpg/source} ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-repo-server ARGOCD_GPG_ENABLED=${ARGOCD_GPG_ENABLED:-false} $COMMAND --loglevel debug --port ${ARGOCD_E2E_REPOSERVER_PORT:-8081} --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --otlp-address=${ARGOCD_OTLP_ADDRESS}" cmp-server: [ "$ARGOCD_E2E_TEST" = 'true' ] && exit 0 || [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_BINARY_NAME=argocd-cmp-server ARGOCD_PLUGINSOCKFILEPATH=${ARGOCD_PLUGINSOCKFILEPATH:-./test/cmp} $COMMAND --config-dir-path ./test/cmp --loglevel debug --otlp-address=${ARGOCD_OTLP_ADDRESS}" +commit-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "GOCOVERDIR=${ARGOCD_COVERAGE_DIR:-/tmp/coverage/commit-server} FORCE_LOG_COLORS=1 ARGOCD_BINARY_NAME=argocd-commit-server $COMMAND --loglevel debug --port ${ARGOCD_E2E_COMMITSERVER_PORT:-8086}" ui: sh -c 'cd ui && ${ARGOCD_E2E_YARN_CMD:-yarn} start' git-server: test/fixture/testrepos/start-git.sh helm-registry: test/fixture/testrepos/start-helm-registry.sh diff --git a/cmd/argocd-commit-server/commands/argocd_commit_server.go b/cmd/argocd-commit-server/commands/argocd_commit_server.go new file mode 100644 index 0000000000000..95fd8ba762114 --- /dev/null +++ b/cmd/argocd-commit-server/commands/argocd_commit_server.go @@ -0,0 +1,117 @@ +package commands + +import ( + "fmt" + "net" + "net/http" + "os" + "os/signal" + "sync" + "syscall" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "google.golang.org/grpc/health/grpc_health_v1" + + cmdutil "github.com/argoproj/argo-cd/v2/cmd/util" + "github.com/argoproj/argo-cd/v2/commitserver" + "github.com/argoproj/argo-cd/v2/commitserver/apiclient" + "github.com/argoproj/argo-cd/v2/commitserver/metrics" + "github.com/argoproj/argo-cd/v2/common" + "github.com/argoproj/argo-cd/v2/reposerver/askpass" + "github.com/argoproj/argo-cd/v2/util/cli" + "github.com/argoproj/argo-cd/v2/util/env" + "github.com/argoproj/argo-cd/v2/util/errors" + "github.com/argoproj/argo-cd/v2/util/healthz" + ioutil "github.com/argoproj/argo-cd/v2/util/io" +) + +// NewCommand returns a new instance of an argocd-commit-server command +func NewCommand() *cobra.Command { + var ( + listenHost string + listenPort int + metricsPort int + metricsHost string + ) + command := &cobra.Command{ + Use: "argocd-commit-server", + Short: "Run Argo CD Commit Server", + Long: "Argo CD Commit Server is an internal service which commits and pushes hydrated manifests to git. This command runs Commit Server in the foreground.", + RunE: func(cmd *cobra.Command, args []string) error { + vers := common.GetVersion() + vers.LogStartupInfo( + "Argo CD Commit Server", + map[string]any{ + "port": listenPort, + }, + ) + + cli.SetLogFormat(cmdutil.LogFormat) + cli.SetLogLevel(cmdutil.LogLevel) + + metricsServer := metrics.NewMetricsServer() + http.Handle("/metrics", metricsServer.GetHandler()) + go func() { errors.CheckError(http.ListenAndServe(fmt.Sprintf("%s:%d", metricsHost, metricsPort), nil)) }() + + askPassServer := askpass.NewServer(askpass.CommitServerSocketPath) + go func() { errors.CheckError(askPassServer.Run()) }() + + server := commitserver.NewServer(askPassServer, metricsServer) + grpc := server.CreateGRPC() + + listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", listenHost, listenPort)) + errors.CheckError(err) + + healthz.ServeHealthCheck(http.DefaultServeMux, func(r *http.Request) error { + if val, ok := r.URL.Query()["full"]; ok && len(val) > 0 && val[0] == "true" { + // connect to itself to make sure commit server is able to serve connection + // used by liveness probe to auto restart commit server + conn, err := apiclient.NewConnection(fmt.Sprintf("localhost:%d", listenPort)) + if err != nil { + return err + } + defer ioutil.Close(conn) + client := grpc_health_v1.NewHealthClient(conn) + res, err := client.Check(r.Context(), &grpc_health_v1.HealthCheckRequest{}) + if err != nil { + return err + } + if res.Status != grpc_health_v1.HealthCheckResponse_SERVING { + return fmt.Errorf("grpc health check status is '%v'", res.Status) + } + return nil + } + return nil + }) + + // Graceful shutdown code adapted from here: https://gist.github.com/embano1/e0bf49d24f1cdd07cffad93097c04f0a + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM) + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + s := <-sigCh + log.Printf("got signal %v, attempting graceful shutdown", s) + grpc.GracefulStop() + wg.Done() + }() + + log.Println("starting grpc server") + err = grpc.Serve(listener) + errors.CheckError(err) + wg.Wait() + log.Println("clean shutdown") + + return nil + }, + } + command.Flags().StringVar(&cmdutil.LogFormat, "logformat", env.StringFromEnv("ARGOCD_COMMIT_SERVER_LOGFORMAT", "text"), "Set the logging format. One of: text|json") + command.Flags().StringVar(&cmdutil.LogLevel, "loglevel", env.StringFromEnv("ARGOCD_COMMIT_SERVER_LOGLEVEL", "info"), "Set the logging level. One of: debug|info|warn|error") + command.Flags().StringVar(&listenHost, "address", env.StringFromEnv("ARGOCD_COMMIT_SERVER_LISTEN_ADDRESS", common.DefaultAddressCommitServer), "Listen on given address for incoming connections") + command.Flags().IntVar(&listenPort, "port", common.DefaultPortCommitServer, "Listen on given port for incoming connections") + command.Flags().StringVar(&metricsHost, "metrics-address", env.StringFromEnv("ARGOCD_COMMIT_SERVER_METRICS_LISTEN_ADDRESS", common.DefaultAddressCommitServerMetrics), "Listen on given address for metrics") + command.Flags().IntVar(&metricsPort, "metrics-port", common.DefaultPortCommitServerMetrics, "Start metrics server on given port") + + return command +} diff --git a/cmd/main.go b/cmd/main.go index fcf771cc1512c..92eb27049c9fc 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -11,6 +11,7 @@ import ( appcontroller "github.com/argoproj/argo-cd/v2/cmd/argocd-application-controller/commands" applicationset "github.com/argoproj/argo-cd/v2/cmd/argocd-applicationset-controller/commands" cmpserver "github.com/argoproj/argo-cd/v2/cmd/argocd-cmp-server/commands" + commitserver "github.com/argoproj/argo-cd/v2/cmd/argocd-commit-server/commands" dex "github.com/argoproj/argo-cd/v2/cmd/argocd-dex/commands" gitaskpass "github.com/argoproj/argo-cd/v2/cmd/argocd-git-ask-pass/commands" k8sauth "github.com/argoproj/argo-cd/v2/cmd/argocd-k8s-auth/commands" @@ -46,6 +47,8 @@ func main() { case "argocd-cmp-server": command = cmpserver.NewCommand() isCLI = true + case "argocd-commit-server": + command = commitserver.NewCommand() case "argocd-dex": command = dex.NewCommand() case "argocd-notifications": diff --git a/commitserver/apiclient/clientset.go b/commitserver/apiclient/clientset.go new file mode 100644 index 0000000000000..795766e54e3db --- /dev/null +++ b/commitserver/apiclient/clientset.go @@ -0,0 +1,49 @@ +package apiclient + +import ( + "fmt" + + log "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "github.com/argoproj/argo-cd/v2/util/io" +) + +// Clientset represents commit server api clients +type Clientset interface { + NewCommitServerClient() (io.Closer, CommitServiceClient, error) +} + +type clientSet struct { + address string +} + +// NewCommitServerClient creates new instance of commit server client +func (c *clientSet) NewCommitServerClient() (io.Closer, CommitServiceClient, error) { + conn, err := NewConnection(c.address) + if err != nil { + return nil, nil, fmt.Errorf("failed to open a new connection to commit server: %w", err) + } + return conn, NewCommitServiceClient(conn), nil +} + +// NewConnection creates new connection to commit server +func NewConnection(address string) (*grpc.ClientConn, error) { + var opts []grpc.DialOption + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + + // TODO: switch to grpc.NewClient. + // nolint:staticcheck + conn, err := grpc.Dial(address, opts...) + if err != nil { + log.Errorf("Unable to connect to commit service with address %s", address) + return nil, err + } + return conn, nil +} + +// NewCommitServerClientset creates new instance of commit server Clientset +func NewCommitServerClientset(address string) Clientset { + return &clientSet{address: address} +} diff --git a/commitserver/apiclient/commit.pb.go b/commitserver/apiclient/commit.pb.go new file mode 100644 index 0000000000000..3e371575827ac --- /dev/null +++ b/commitserver/apiclient/commit.pb.go @@ -0,0 +1,1382 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: commitserver/commit/commit.proto + +package apiclient + +import ( + context "context" + fmt "fmt" + v1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + proto "github.com/gogo/protobuf/proto" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +// CommitHydratedManifestsRequest is the request to commit hydrated manifests to a repository. +type CommitHydratedManifestsRequest struct { + // Repo contains repository information including, at minimum, the URL of the repository. Generally it will contain + // repo credentials. + Repo *v1alpha1.Repository `protobuf:"bytes,1,opt,name=repo,proto3" json:"repo,omitempty"` + // SyncBranch is the branch Argo CD syncs from, i.e. the hydrated branch. + SyncBranch string `protobuf:"bytes,2,opt,name=syncBranch,proto3" json:"syncBranch,omitempty"` + // TargetBranch is the branch Argo CD is committing to, i.e. the branch that will be updated. + TargetBranch string `protobuf:"bytes,3,opt,name=targetBranch,proto3" json:"targetBranch,omitempty"` + // DrySha is the commit SHA from the dry branch, i.e. pre-rendered manifest branch. + DrySha string `protobuf:"bytes,4,opt,name=drySha,proto3" json:"drySha,omitempty"` + // CommitMessage is the commit message to use when committing changes. + CommitMessage string `protobuf:"bytes,5,opt,name=commitMessage,proto3" json:"commitMessage,omitempty"` + // Paths contains the paths to write hydrated manifests to, along with the manifests and commands to execute. + Paths []*PathDetails `protobuf:"bytes,6,rep,name=paths,proto3" json:"paths,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CommitHydratedManifestsRequest) Reset() { *m = CommitHydratedManifestsRequest{} } +func (m *CommitHydratedManifestsRequest) String() string { return proto.CompactTextString(m) } +func (*CommitHydratedManifestsRequest) ProtoMessage() {} +func (*CommitHydratedManifestsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_cf3a3abbc35e3069, []int{0} +} +func (m *CommitHydratedManifestsRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *CommitHydratedManifestsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_CommitHydratedManifestsRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *CommitHydratedManifestsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CommitHydratedManifestsRequest.Merge(m, src) +} +func (m *CommitHydratedManifestsRequest) XXX_Size() int { + return m.Size() +} +func (m *CommitHydratedManifestsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_CommitHydratedManifestsRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_CommitHydratedManifestsRequest proto.InternalMessageInfo + +func (m *CommitHydratedManifestsRequest) GetRepo() *v1alpha1.Repository { + if m != nil { + return m.Repo + } + return nil +} + +func (m *CommitHydratedManifestsRequest) GetSyncBranch() string { + if m != nil { + return m.SyncBranch + } + return "" +} + +func (m *CommitHydratedManifestsRequest) GetTargetBranch() string { + if m != nil { + return m.TargetBranch + } + return "" +} + +func (m *CommitHydratedManifestsRequest) GetDrySha() string { + if m != nil { + return m.DrySha + } + return "" +} + +func (m *CommitHydratedManifestsRequest) GetCommitMessage() string { + if m != nil { + return m.CommitMessage + } + return "" +} + +func (m *CommitHydratedManifestsRequest) GetPaths() []*PathDetails { + if m != nil { + return m.Paths + } + return nil +} + +// PathDetails holds information about hydrated manifests to be written to a particular path in the hydrated manifests +// commit. +type PathDetails struct { + // Path is the path to write the hydrated manifests to. + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + // Manifests contains the manifests to write to the path. + Manifests []*HydratedManifestDetails `protobuf:"bytes,2,rep,name=manifests,proto3" json:"manifests,omitempty"` + // Commands contains the commands executed when hydrating the manifests. + Commands []string `protobuf:"bytes,3,rep,name=commands,proto3" json:"commands,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PathDetails) Reset() { *m = PathDetails{} } +func (m *PathDetails) String() string { return proto.CompactTextString(m) } +func (*PathDetails) ProtoMessage() {} +func (*PathDetails) Descriptor() ([]byte, []int) { + return fileDescriptor_cf3a3abbc35e3069, []int{1} +} +func (m *PathDetails) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PathDetails) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_PathDetails.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *PathDetails) XXX_Merge(src proto.Message) { + xxx_messageInfo_PathDetails.Merge(m, src) +} +func (m *PathDetails) XXX_Size() int { + return m.Size() +} +func (m *PathDetails) XXX_DiscardUnknown() { + xxx_messageInfo_PathDetails.DiscardUnknown(m) +} + +var xxx_messageInfo_PathDetails proto.InternalMessageInfo + +func (m *PathDetails) GetPath() string { + if m != nil { + return m.Path + } + return "" +} + +func (m *PathDetails) GetManifests() []*HydratedManifestDetails { + if m != nil { + return m.Manifests + } + return nil +} + +func (m *PathDetails) GetCommands() []string { + if m != nil { + return m.Commands + } + return nil +} + +// ManifestDetails contains the hydrated manifests. +type HydratedManifestDetails struct { + // ManifestJSON is the hydrated manifest as JSON. + ManifestJSON string `protobuf:"bytes,1,opt,name=manifestJSON,proto3" json:"manifestJSON,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *HydratedManifestDetails) Reset() { *m = HydratedManifestDetails{} } +func (m *HydratedManifestDetails) String() string { return proto.CompactTextString(m) } +func (*HydratedManifestDetails) ProtoMessage() {} +func (*HydratedManifestDetails) Descriptor() ([]byte, []int) { + return fileDescriptor_cf3a3abbc35e3069, []int{2} +} +func (m *HydratedManifestDetails) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *HydratedManifestDetails) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_HydratedManifestDetails.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *HydratedManifestDetails) XXX_Merge(src proto.Message) { + xxx_messageInfo_HydratedManifestDetails.Merge(m, src) +} +func (m *HydratedManifestDetails) XXX_Size() int { + return m.Size() +} +func (m *HydratedManifestDetails) XXX_DiscardUnknown() { + xxx_messageInfo_HydratedManifestDetails.DiscardUnknown(m) +} + +var xxx_messageInfo_HydratedManifestDetails proto.InternalMessageInfo + +func (m *HydratedManifestDetails) GetManifestJSON() string { + if m != nil { + return m.ManifestJSON + } + return "" +} + +// ManifestsResponse is the response to the ManifestsRequest. +type CommitHydratedManifestsResponse struct { + // HydratedSha is the commit SHA of the hydrated manifests commit. + HydratedSha string `protobuf:"bytes,1,opt,name=hydratedSha,proto3" json:"hydratedSha,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CommitHydratedManifestsResponse) Reset() { *m = CommitHydratedManifestsResponse{} } +func (m *CommitHydratedManifestsResponse) String() string { return proto.CompactTextString(m) } +func (*CommitHydratedManifestsResponse) ProtoMessage() {} +func (*CommitHydratedManifestsResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_cf3a3abbc35e3069, []int{3} +} +func (m *CommitHydratedManifestsResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *CommitHydratedManifestsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_CommitHydratedManifestsResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *CommitHydratedManifestsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CommitHydratedManifestsResponse.Merge(m, src) +} +func (m *CommitHydratedManifestsResponse) XXX_Size() int { + return m.Size() +} +func (m *CommitHydratedManifestsResponse) XXX_DiscardUnknown() { + xxx_messageInfo_CommitHydratedManifestsResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_CommitHydratedManifestsResponse proto.InternalMessageInfo + +func (m *CommitHydratedManifestsResponse) GetHydratedSha() string { + if m != nil { + return m.HydratedSha + } + return "" +} + +func init() { + proto.RegisterType((*CommitHydratedManifestsRequest)(nil), "CommitHydratedManifestsRequest") + proto.RegisterType((*PathDetails)(nil), "PathDetails") + proto.RegisterType((*HydratedManifestDetails)(nil), "HydratedManifestDetails") + proto.RegisterType((*CommitHydratedManifestsResponse)(nil), "CommitHydratedManifestsResponse") +} + +func init() { proto.RegisterFile("commitserver/commit/commit.proto", fileDescriptor_cf3a3abbc35e3069) } + +var fileDescriptor_cf3a3abbc35e3069 = []byte{ + // 446 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x93, 0xc1, 0x6e, 0xd3, 0x40, + 0x10, 0x86, 0xe5, 0x24, 0x8d, 0xc8, 0xa4, 0xbd, 0xec, 0x81, 0x5a, 0x39, 0xb8, 0x96, 0xc5, 0x21, + 0x17, 0xd6, 0xaa, 0x11, 0xdc, 0xb8, 0x34, 0x1c, 0x2a, 0x44, 0x01, 0x39, 0x37, 0x54, 0x09, 0x6d, + 0xd7, 0x83, 0xbd, 0x34, 0xf6, 0x2e, 0xbb, 0x1b, 0x4b, 0x79, 0x1f, 0x1e, 0x86, 0x23, 0x8f, 0x80, + 0xf2, 0x24, 0xc8, 0x6b, 0x9b, 0xc6, 0x48, 0x69, 0x4f, 0x9e, 0xf9, 0x67, 0xf4, 0xcd, 0xe8, 0xf7, + 0x2c, 0x84, 0x5c, 0x96, 0xa5, 0xb0, 0x06, 0x75, 0x8d, 0x3a, 0x6e, 0x93, 0xee, 0x43, 0x95, 0x96, + 0x56, 0x2e, 0x3e, 0xe4, 0xc2, 0x16, 0xdb, 0x3b, 0xca, 0x65, 0x19, 0x33, 0x9d, 0x4b, 0xa5, 0xe5, + 0x77, 0x17, 0xbc, 0xe4, 0x59, 0x5c, 0x27, 0xb1, 0xba, 0xcf, 0x63, 0xa6, 0x84, 0x89, 0x99, 0x52, + 0x1b, 0xc1, 0x99, 0x15, 0xb2, 0x8a, 0xeb, 0x4b, 0xb6, 0x51, 0x05, 0xbb, 0x8c, 0x73, 0xac, 0x50, + 0x33, 0x8b, 0x59, 0x4b, 0x8b, 0x7e, 0x8e, 0x20, 0x58, 0x39, 0xfc, 0xf5, 0x2e, 0x73, 0x85, 0x1b, + 0x56, 0x89, 0x6f, 0x68, 0xac, 0x49, 0xf1, 0xc7, 0x16, 0x8d, 0x25, 0xb7, 0x30, 0xd1, 0xa8, 0xa4, + 0xef, 0x85, 0xde, 0x72, 0x9e, 0x5c, 0xd3, 0x87, 0xf9, 0xb4, 0x9f, 0xef, 0x82, 0xaf, 0x3c, 0xa3, + 0x75, 0x42, 0xd5, 0x7d, 0x4e, 0x9b, 0xf9, 0xf4, 0x60, 0x3e, 0xed, 0xe7, 0xd3, 0x14, 0x95, 0x34, + 0xc2, 0x4a, 0xbd, 0x4b, 0x1d, 0x95, 0x04, 0x00, 0x66, 0x57, 0xf1, 0x2b, 0xcd, 0x2a, 0x5e, 0xf8, + 0xa3, 0xd0, 0x5b, 0xce, 0xd2, 0x03, 0x85, 0x44, 0x70, 0x6a, 0x99, 0xce, 0xd1, 0x76, 0x1d, 0x63, + 0xd7, 0x31, 0xd0, 0xc8, 0x73, 0x98, 0x66, 0x7a, 0xb7, 0x2e, 0x98, 0x3f, 0x71, 0xd5, 0x2e, 0x23, + 0x2f, 0xe0, 0xac, 0xb5, 0xee, 0x06, 0x8d, 0x61, 0x39, 0xfa, 0x27, 0xae, 0x3c, 0x14, 0x49, 0x04, + 0x27, 0x8a, 0xd9, 0xc2, 0xf8, 0xd3, 0x70, 0xbc, 0x9c, 0x27, 0xa7, 0xf4, 0x33, 0xb3, 0xc5, 0x3b, + 0xb4, 0x4c, 0x6c, 0x4c, 0xda, 0x96, 0xa2, 0x2d, 0xcc, 0x0f, 0x54, 0x42, 0x60, 0xd2, 0xe8, 0xce, + 0x92, 0x59, 0xea, 0x62, 0xf2, 0x06, 0x66, 0x65, 0x6f, 0x9d, 0x3f, 0x72, 0x28, 0x9f, 0xfe, 0x6f, + 0x6a, 0x8f, 0x7d, 0x68, 0x25, 0x0b, 0x78, 0xd6, 0xec, 0xc3, 0xaa, 0xcc, 0xf8, 0xe3, 0x70, 0xbc, + 0x9c, 0xa5, 0xff, 0xf2, 0xe8, 0x2d, 0x9c, 0x1f, 0x21, 0x34, 0xbe, 0xf4, 0x8c, 0xf7, 0xeb, 0x4f, + 0x1f, 0xbb, 0x55, 0x06, 0x5a, 0xb4, 0x82, 0x8b, 0xa3, 0xff, 0xd6, 0x28, 0x59, 0x19, 0x24, 0x21, + 0xcc, 0x8b, 0xae, 0xd8, 0xf8, 0xd7, 0x52, 0x0e, 0xa5, 0xa4, 0x84, 0xb3, 0x16, 0xb2, 0x46, 0x5d, + 0x0b, 0x8e, 0xe4, 0x16, 0xce, 0x8f, 0x50, 0xc9, 0x05, 0x7d, 0xfc, 0x96, 0x16, 0x21, 0x7d, 0x62, + 0xa1, 0xab, 0xd5, 0xaf, 0x7d, 0xe0, 0xfd, 0xde, 0x07, 0xde, 0x9f, 0x7d, 0xe0, 0x7d, 0x79, 0xfd, + 0xc4, 0xb1, 0x0f, 0x5e, 0x0b, 0x53, 0x82, 0x6f, 0x04, 0x56, 0xf6, 0x6e, 0xea, 0x8e, 0xfb, 0xd5, + 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x89, 0xb8, 0xdf, 0x48, 0x4e, 0x03, 0x00, 0x00, +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// CommitServiceClient is the client API for CommitService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type CommitServiceClient interface { + // Commit commits hydrated manifests to a repository. + CommitHydratedManifests(ctx context.Context, in *CommitHydratedManifestsRequest, opts ...grpc.CallOption) (*CommitHydratedManifestsResponse, error) +} + +type commitServiceClient struct { + cc *grpc.ClientConn +} + +func NewCommitServiceClient(cc *grpc.ClientConn) CommitServiceClient { + return &commitServiceClient{cc} +} + +func (c *commitServiceClient) CommitHydratedManifests(ctx context.Context, in *CommitHydratedManifestsRequest, opts ...grpc.CallOption) (*CommitHydratedManifestsResponse, error) { + out := new(CommitHydratedManifestsResponse) + err := c.cc.Invoke(ctx, "/CommitService/CommitHydratedManifests", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// CommitServiceServer is the server API for CommitService service. +type CommitServiceServer interface { + // Commit commits hydrated manifests to a repository. + CommitHydratedManifests(context.Context, *CommitHydratedManifestsRequest) (*CommitHydratedManifestsResponse, error) +} + +// UnimplementedCommitServiceServer can be embedded to have forward compatible implementations. +type UnimplementedCommitServiceServer struct { +} + +func (*UnimplementedCommitServiceServer) CommitHydratedManifests(ctx context.Context, req *CommitHydratedManifestsRequest) (*CommitHydratedManifestsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CommitHydratedManifests not implemented") +} + +func RegisterCommitServiceServer(s *grpc.Server, srv CommitServiceServer) { + s.RegisterService(&_CommitService_serviceDesc, srv) +} + +func _CommitService_CommitHydratedManifests_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CommitHydratedManifestsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CommitServiceServer).CommitHydratedManifests(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/CommitService/CommitHydratedManifests", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CommitServiceServer).CommitHydratedManifests(ctx, req.(*CommitHydratedManifestsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _CommitService_serviceDesc = grpc.ServiceDesc{ + ServiceName: "CommitService", + HandlerType: (*CommitServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "CommitHydratedManifests", + Handler: _CommitService_CommitHydratedManifests_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "commitserver/commit/commit.proto", +} + +func (m *CommitHydratedManifestsRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CommitHydratedManifestsRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CommitHydratedManifestsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Paths) > 0 { + for iNdEx := len(m.Paths) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Paths[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintCommit(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x32 + } + } + if len(m.CommitMessage) > 0 { + i -= len(m.CommitMessage) + copy(dAtA[i:], m.CommitMessage) + i = encodeVarintCommit(dAtA, i, uint64(len(m.CommitMessage))) + i-- + dAtA[i] = 0x2a + } + if len(m.DrySha) > 0 { + i -= len(m.DrySha) + copy(dAtA[i:], m.DrySha) + i = encodeVarintCommit(dAtA, i, uint64(len(m.DrySha))) + i-- + dAtA[i] = 0x22 + } + if len(m.TargetBranch) > 0 { + i -= len(m.TargetBranch) + copy(dAtA[i:], m.TargetBranch) + i = encodeVarintCommit(dAtA, i, uint64(len(m.TargetBranch))) + i-- + dAtA[i] = 0x1a + } + if len(m.SyncBranch) > 0 { + i -= len(m.SyncBranch) + copy(dAtA[i:], m.SyncBranch) + i = encodeVarintCommit(dAtA, i, uint64(len(m.SyncBranch))) + i-- + dAtA[i] = 0x12 + } + if m.Repo != nil { + { + size, err := m.Repo.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintCommit(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *PathDetails) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PathDetails) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PathDetails) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Commands) > 0 { + for iNdEx := len(m.Commands) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Commands[iNdEx]) + copy(dAtA[i:], m.Commands[iNdEx]) + i = encodeVarintCommit(dAtA, i, uint64(len(m.Commands[iNdEx]))) + i-- + dAtA[i] = 0x1a + } + } + if len(m.Manifests) > 0 { + for iNdEx := len(m.Manifests) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Manifests[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintCommit(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } + if len(m.Path) > 0 { + i -= len(m.Path) + copy(dAtA[i:], m.Path) + i = encodeVarintCommit(dAtA, i, uint64(len(m.Path))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *HydratedManifestDetails) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *HydratedManifestDetails) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *HydratedManifestDetails) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.ManifestJSON) > 0 { + i -= len(m.ManifestJSON) + copy(dAtA[i:], m.ManifestJSON) + i = encodeVarintCommit(dAtA, i, uint64(len(m.ManifestJSON))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *CommitHydratedManifestsResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CommitHydratedManifestsResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CommitHydratedManifestsResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.HydratedSha) > 0 { + i -= len(m.HydratedSha) + copy(dAtA[i:], m.HydratedSha) + i = encodeVarintCommit(dAtA, i, uint64(len(m.HydratedSha))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func encodeVarintCommit(dAtA []byte, offset int, v uint64) int { + offset -= sovCommit(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *CommitHydratedManifestsRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Repo != nil { + l = m.Repo.Size() + n += 1 + l + sovCommit(uint64(l)) + } + l = len(m.SyncBranch) + if l > 0 { + n += 1 + l + sovCommit(uint64(l)) + } + l = len(m.TargetBranch) + if l > 0 { + n += 1 + l + sovCommit(uint64(l)) + } + l = len(m.DrySha) + if l > 0 { + n += 1 + l + sovCommit(uint64(l)) + } + l = len(m.CommitMessage) + if l > 0 { + n += 1 + l + sovCommit(uint64(l)) + } + if len(m.Paths) > 0 { + for _, e := range m.Paths { + l = e.Size() + n += 1 + l + sovCommit(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *PathDetails) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Path) + if l > 0 { + n += 1 + l + sovCommit(uint64(l)) + } + if len(m.Manifests) > 0 { + for _, e := range m.Manifests { + l = e.Size() + n += 1 + l + sovCommit(uint64(l)) + } + } + if len(m.Commands) > 0 { + for _, s := range m.Commands { + l = len(s) + n += 1 + l + sovCommit(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *HydratedManifestDetails) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.ManifestJSON) + if l > 0 { + n += 1 + l + sovCommit(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *CommitHydratedManifestsResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.HydratedSha) + if l > 0 { + n += 1 + l + sovCommit(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovCommit(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozCommit(x uint64) (n int) { + return sovCommit(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *CommitHydratedManifestsRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCommit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CommitHydratedManifestsRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CommitHydratedManifestsRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Repo", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCommit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthCommit + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthCommit + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Repo == nil { + m.Repo = &v1alpha1.Repository{} + } + if err := m.Repo.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SyncBranch", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCommit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthCommit + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthCommit + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SyncBranch = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TargetBranch", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCommit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthCommit + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthCommit + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TargetBranch = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DrySha", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCommit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthCommit + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthCommit + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DrySha = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CommitMessage", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCommit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthCommit + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthCommit + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CommitMessage = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Paths", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCommit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthCommit + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthCommit + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Paths = append(m.Paths, &PathDetails{}) + if err := m.Paths[len(m.Paths)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipCommit(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthCommit + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PathDetails) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCommit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PathDetails: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PathDetails: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Path", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCommit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthCommit + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthCommit + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Path = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Manifests", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCommit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthCommit + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthCommit + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Manifests = append(m.Manifests, &HydratedManifestDetails{}) + if err := m.Manifests[len(m.Manifests)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Commands", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCommit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthCommit + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthCommit + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Commands = append(m.Commands, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipCommit(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthCommit + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *HydratedManifestDetails) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCommit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: HydratedManifestDetails: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: HydratedManifestDetails: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ManifestJSON", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCommit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthCommit + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthCommit + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ManifestJSON = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipCommit(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthCommit + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *CommitHydratedManifestsResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCommit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CommitHydratedManifestsResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CommitHydratedManifestsResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field HydratedSha", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCommit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthCommit + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthCommit + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.HydratedSha = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipCommit(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthCommit + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipCommit(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCommit + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCommit + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCommit + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthCommit + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupCommit + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthCommit + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthCommit = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowCommit = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupCommit = fmt.Errorf("proto: unexpected end of group") +) diff --git a/commitserver/apiclient/mocks/Clientset.go b/commitserver/apiclient/mocks/Clientset.go new file mode 100644 index 0000000000000..bb51a52c9a623 --- /dev/null +++ b/commitserver/apiclient/mocks/Clientset.go @@ -0,0 +1,68 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + +package mocks + +import ( + apiclient "github.com/argoproj/argo-cd/v2/commitserver/apiclient" + io "github.com/argoproj/argo-cd/v2/util/io" + + mock "github.com/stretchr/testify/mock" +) + +// Clientset is an autogenerated mock type for the Clientset type +type Clientset struct { + mock.Mock +} + +// NewCommitServerClient provides a mock function with given fields: +func (_m *Clientset) NewCommitServerClient() (io.Closer, apiclient.CommitServiceClient, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for NewCommitServerClient") + } + + var r0 io.Closer + var r1 apiclient.CommitServiceClient + var r2 error + if rf, ok := ret.Get(0).(func() (io.Closer, apiclient.CommitServiceClient, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() io.Closer); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(io.Closer) + } + } + + if rf, ok := ret.Get(1).(func() apiclient.CommitServiceClient); ok { + r1 = rf() + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(apiclient.CommitServiceClient) + } + } + + if rf, ok := ret.Get(2).(func() error); ok { + r2 = rf() + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// NewClientset creates a new instance of Clientset. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewClientset(t interface { + mock.TestingT + Cleanup(func()) +}) *Clientset { + mock := &Clientset{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/commitserver/apiclient/mocks/CommitServiceClient.go b/commitserver/apiclient/mocks/CommitServiceClient.go new file mode 100644 index 0000000000000..d122aa1a710c1 --- /dev/null +++ b/commitserver/apiclient/mocks/CommitServiceClient.go @@ -0,0 +1,69 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + +package mocks + +import ( + context "context" + + apiclient "github.com/argoproj/argo-cd/v2/commitserver/apiclient" + + grpc "google.golang.org/grpc" + + mock "github.com/stretchr/testify/mock" +) + +// CommitServiceClient is an autogenerated mock type for the CommitServiceClient type +type CommitServiceClient struct { + mock.Mock +} + +// CommitHydratedManifests provides a mock function with given fields: ctx, in, opts +func (_m *CommitServiceClient) CommitHydratedManifests(ctx context.Context, in *apiclient.CommitHydratedManifestsRequest, opts ...grpc.CallOption) (*apiclient.CommitHydratedManifestsResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for CommitHydratedManifests") + } + + var r0 *apiclient.CommitHydratedManifestsResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *apiclient.CommitHydratedManifestsRequest, ...grpc.CallOption) (*apiclient.CommitHydratedManifestsResponse, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *apiclient.CommitHydratedManifestsRequest, ...grpc.CallOption) *apiclient.CommitHydratedManifestsResponse); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*apiclient.CommitHydratedManifestsResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *apiclient.CommitHydratedManifestsRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewCommitServiceClient creates a new instance of CommitServiceClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewCommitServiceClient(t interface { + mock.TestingT + Cleanup(func()) +}) *CommitServiceClient { + mock := &CommitServiceClient{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/commitserver/commit/commit.go b/commitserver/commit/commit.go new file mode 100644 index 0000000000000..9682680d7af62 --- /dev/null +++ b/commitserver/commit/commit.go @@ -0,0 +1,218 @@ +package commit + +import ( + "context" + "fmt" + "os" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/argoproj/argo-cd/v2/commitserver/apiclient" + "github.com/argoproj/argo-cd/v2/commitserver/metrics" + "github.com/argoproj/argo-cd/v2/util/git" + "github.com/argoproj/argo-cd/v2/util/io/files" +) + +// Service is the service that handles commit requests. +type Service struct { + gitCredsStore git.CredsStore + metricsServer *metrics.Server + repoClientFactory RepoClientFactory +} + +// NewService returns a new instance of the commit service. +func NewService(gitCredsStore git.CredsStore, metricsServer *metrics.Server) *Service { + return &Service{ + gitCredsStore: gitCredsStore, + metricsServer: metricsServer, + repoClientFactory: NewRepoClientFactory(gitCredsStore, metricsServer), + } +} + +// CommitHydratedManifests handles a commit request. It clones the repository, checks out the sync branch, checks out +// the target branch, clears the repository contents, writes the manifests to the repository, commits the changes, and +// pushes the changes. It returns the hydrated revision SHA and an error if one occurred. +func (s *Service) CommitHydratedManifests(ctx context.Context, r *apiclient.CommitHydratedManifestsRequest) (*apiclient.CommitHydratedManifestsResponse, error) { + // This method is intentionally short. It's a wrapper around handleCommitRequest that adds metrics and logging. + // Keep logic here minimal and put most of the logic in handleCommitRequest. + startTime := time.Now() + + // We validate for a nil repo in handleCommitRequest, but we need to check for a nil repo here to get the repo URL + // for metrics. + var repoURL string + if r.Repo != nil { + repoURL = r.Repo.Repo + } + + s.metricsServer.IncPendingCommitRequest(repoURL) + defer s.metricsServer.DecPendingCommitRequest(repoURL) + + logCtx := log.WithFields(log.Fields{"branch": r.TargetBranch, "drySHA": r.DrySha}) + + out, sha, err := s.handleCommitRequest(ctx, logCtx, r) + if err != nil { + logCtx.WithError(err).WithField("output", out).Error("failed to handle commit request") + s.metricsServer.IncCommitRequest(repoURL, metrics.CommitResponseTypeFailure) + s.metricsServer.ObserveCommitRequestDuration(repoURL, metrics.CommitResponseTypeFailure, time.Since(startTime)) + + // No need to wrap this error, sufficient context is build in handleCommitRequest. + return &apiclient.CommitHydratedManifestsResponse{}, err + } + + logCtx.Info("Successfully handled commit request") + s.metricsServer.IncCommitRequest(repoURL, metrics.CommitResponseTypeSuccess) + s.metricsServer.ObserveCommitRequestDuration(repoURL, metrics.CommitResponseTypeSuccess, time.Since(startTime)) + return &apiclient.CommitHydratedManifestsResponse{ + HydratedSha: sha, + }, nil +} + +// handleCommitRequest handles the commit request. It clones the repository, checks out the sync branch, checks out the +// target branch, clears the repository contents, writes the manifests to the repository, commits the changes, and pushes +// the changes. It returns the output of the git commands and an error if one occurred. +func (s *Service) handleCommitRequest(ctx context.Context, logCtx *log.Entry, r *apiclient.CommitHydratedManifestsRequest) (string, string, error) { + if r.Repo == nil { + return "", "", fmt.Errorf("repo is required") + } + if r.Repo.Repo == "" { + return "", "", fmt.Errorf("repo URL is required") + } + if r.TargetBranch == "" { + return "", "", fmt.Errorf("target branch is required") + } + if r.SyncBranch == "" { + return "", "", fmt.Errorf("sync branch is required") + } + + logCtx = logCtx.WithField("repo", r.Repo.Repo) + logCtx.Debug("Initiating git client") + gitClient, dirPath, cleanup, err := s.initGitClient(ctx, logCtx, r) + if err != nil { + return "", "", fmt.Errorf("failed to init git client: %w", err) + } + defer cleanup() + + logCtx.Debugf("Checking out sync branch %s", r.SyncBranch) + var out string + out, err = gitClient.CheckoutOrOrphan(r.SyncBranch, false) + if err != nil { + return out, "", fmt.Errorf("failed to checkout sync branch: %w", err) + } + + logCtx.Debugf("Checking out target branch %s", r.TargetBranch) + out, err = gitClient.CheckoutOrNew(r.TargetBranch, r.SyncBranch, false) + if err != nil { + return out, "", fmt.Errorf("failed to checkout target branch: %w", err) + } + + logCtx.Debug("Clearing repo contents") + out, err = gitClient.RemoveContents() + if err != nil { + return out, "", fmt.Errorf("failed to clear repo: %w", err) + } + + logCtx.Debug("Writing manifests") + err = WriteForPaths(dirPath, r.Repo.Repo, r.DrySha, r.Paths) + if err != nil { + return "", "", fmt.Errorf("failed to write manifests: %w", err) + } + + logCtx.Debug("Committing and pushing changes") + out, err = gitClient.CommitAndPush(r.TargetBranch, r.CommitMessage) + if err != nil { + return out, "", fmt.Errorf("failed to commit and push: %w", err) + } + + logCtx.Debug("Getting commit SHA") + sha, err := gitClient.CommitSHA() + if err != nil { + return "", "", fmt.Errorf("failed to get commit SHA: %w", err) + } + + return "", sha, nil +} + +// initGitClient initializes a git client for the given repository and returns the client, the path to the directory where +// the repository is cloned, a cleanup function that should be called when the directory is no longer needed, and an error +// if one occurred. +func (s *Service) initGitClient(ctx context.Context, logCtx *log.Entry, r *apiclient.CommitHydratedManifestsRequest) (git.Client, string, func(), error) { + dirPath, err := files.CreateTempDir("/tmp/_commit-service") + if err != nil { + return nil, "", nil, fmt.Errorf("failed to create temp dir: %w", err) + } + // Call cleanupOrLog in this function if an error occurs to ensure the temp dir is cleaned up. + cleanupOrLog := func() { + err := os.RemoveAll(dirPath) + if err != nil { + logCtx.WithError(err).Error("failed to cleanup temp dir") + } + } + + gitClient, err := s.repoClientFactory.NewClient(r.Repo, dirPath) + if err != nil { + cleanupOrLog() + return nil, "", nil, fmt.Errorf("failed to create git client: %w", err) + } + + logCtx.Debugf("Initializing repo %s", r.Repo.Repo) + err = gitClient.Init() + if err != nil { + cleanupOrLog() + return nil, "", nil, fmt.Errorf("failed to init git client: %w", err) + } + + logCtx.Debugf("Fetching repo %s", r.Repo.Repo) + err = gitClient.Fetch("") + if err != nil { + cleanupOrLog() + return nil, "", nil, fmt.Errorf("failed to clone repo: %w", err) + } + + logCtx.Debugf("Getting user info for repo credentials") + gitCreds := r.Repo.GetGitCreds(s.gitCredsStore) + startTime := time.Now() + authorName, authorEmail, err := gitCreds.GetUserInfo(ctx) + s.metricsServer.ObserveUserInfoRequestDuration(r.Repo.Repo, getCredentialType(r.Repo), time.Since(startTime)) + if err != nil { + cleanupOrLog() + return nil, "", nil, fmt.Errorf("failed to get github app info: %w", err) + } + + if authorName == "" { + authorName = "Argo CD" + } + if authorEmail == "" { + logCtx.Warnf("Author email not available, using 'argo-cd@example.com'.") + authorEmail = "argo-cd@example.com" + } + + logCtx.Debugf("Setting author %s <%s>", authorName, authorEmail) + _, err = gitClient.SetAuthor(authorName, authorEmail) + if err != nil { + cleanupOrLog() + return nil, "", nil, fmt.Errorf("failed to set author: %w", err) + } + + return gitClient, dirPath, cleanupOrLog, nil +} + +type hydratorMetadataFile struct { + RepoURL string `json:"repoURL"` + DrySHA string `json:"drySha"` + Commands []string `json:"commands"` +} + +// TODO: make this configurable via ConfigMap. +var manifestHydrationReadmeTemplate = ` +# Manifest Hydration + +To hydrate the manifests in this repository, run the following commands: + +` + "```shell\n" + ` +git clone {{ .RepoURL }} +# cd into the cloned directory +git checkout {{ .DrySHA }} +{{ range $command := .Commands -}} +{{ $command }} +{{ end -}}` + "```" diff --git a/commitserver/commit/commit.proto b/commitserver/commit/commit.proto new file mode 100644 index 0000000000000..fdf8b23c0d00e --- /dev/null +++ b/commitserver/commit/commit.proto @@ -0,0 +1,50 @@ +syntax = "proto3"; +option go_package = "github.com/argoproj/argo-cd/v2/commitserver/apiclient"; + +import "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1/generated.proto"; + +// CommitHydratedManifestsRequest is the request to commit hydrated manifests to a repository. +message CommitHydratedManifestsRequest { + // Repo contains repository information including, at minimum, the URL of the repository. Generally it will contain + // repo credentials. + github.com.argoproj.argo_cd.v2.pkg.apis.application.v1alpha1.Repository repo = 1; + // SyncBranch is the branch Argo CD syncs from, i.e. the hydrated branch. + string syncBranch = 2; + // TargetBranch is the branch Argo CD is committing to, i.e. the branch that will be updated. + string targetBranch = 3; + // DrySha is the commit SHA from the dry branch, i.e. pre-rendered manifest branch. + string drySha = 4; + // CommitMessage is the commit message to use when committing changes. + string commitMessage = 5; + // Paths contains the paths to write hydrated manifests to, along with the manifests and commands to execute. + repeated PathDetails paths = 6; +} + +// PathDetails holds information about hydrated manifests to be written to a particular path in the hydrated manifests +// commit. +message PathDetails { + // Path is the path to write the hydrated manifests to. + string path = 1; + // Manifests contains the manifests to write to the path. + repeated HydratedManifestDetails manifests = 2; + // Commands contains the commands executed when hydrating the manifests. + repeated string commands = 3; +} + +// ManifestDetails contains the hydrated manifests. +message HydratedManifestDetails { + // ManifestJSON is the hydrated manifest as JSON. + string manifestJSON = 1; +} + +// ManifestsResponse is the response to the ManifestsRequest. +message CommitHydratedManifestsResponse { + // HydratedSha is the commit SHA of the hydrated manifests commit. + string hydratedSha = 1; +} + +// CommitService is the service for committing hydrated manifests to a repository. +service CommitService { + // Commit commits hydrated manifests to a repository. + rpc CommitHydratedManifests (CommitHydratedManifestsRequest) returns (CommitHydratedManifestsResponse); +} diff --git a/commitserver/commit/commit_test.go b/commitserver/commit/commit_test.go new file mode 100644 index 0000000000000..77bb9b53482a2 --- /dev/null +++ b/commitserver/commit/commit_test.go @@ -0,0 +1,125 @@ +package commit + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/argoproj/argo-cd/v2/commitserver/apiclient" + "github.com/argoproj/argo-cd/v2/commitserver/commit/mocks" + "github.com/argoproj/argo-cd/v2/commitserver/metrics" + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "github.com/argoproj/argo-cd/v2/util/git" + gitmocks "github.com/argoproj/argo-cd/v2/util/git/mocks" +) + +func Test_CommitHydratedManifests(t *testing.T) { + t.Parallel() + + validRequest := &apiclient.CommitHydratedManifestsRequest{ + Repo: &v1alpha1.Repository{ + Repo: "https://github.com/argoproj/argocd-example-apps.git", + }, + TargetBranch: "main", + SyncBranch: "env/test", + CommitMessage: "test commit message", + } + + t.Run("missing repo", func(t *testing.T) { + t.Parallel() + + service, _ := newServiceWithMocks(t) + request := &apiclient.CommitHydratedManifestsRequest{} + _, err := service.CommitHydratedManifests(context.Background(), request) + require.Error(t, err) + assert.ErrorContains(t, err, "repo is required") + }) + + t.Run("missing repo URL", func(t *testing.T) { + t.Parallel() + + service, _ := newServiceWithMocks(t) + request := &apiclient.CommitHydratedManifestsRequest{ + Repo: &v1alpha1.Repository{}, + } + _, err := service.CommitHydratedManifests(context.Background(), request) + require.Error(t, err) + assert.ErrorContains(t, err, "repo URL is required") + }) + + t.Run("missing target branch", func(t *testing.T) { + t.Parallel() + + service, _ := newServiceWithMocks(t) + request := &apiclient.CommitHydratedManifestsRequest{ + Repo: &v1alpha1.Repository{ + Repo: "https://github.com/argoproj/argocd-example-apps.git", + }, + } + _, err := service.CommitHydratedManifests(context.Background(), request) + require.Error(t, err) + assert.ErrorContains(t, err, "target branch is required") + }) + + t.Run("missing sync branch", func(t *testing.T) { + t.Parallel() + + service, _ := newServiceWithMocks(t) + request := &apiclient.CommitHydratedManifestsRequest{ + Repo: &v1alpha1.Repository{ + Repo: "https://github.com/argoproj/argocd-example-apps.git", + }, + TargetBranch: "main", + } + _, err := service.CommitHydratedManifests(context.Background(), request) + require.Error(t, err) + assert.ErrorContains(t, err, "sync branch is required") + }) + + t.Run("failed to create git client", func(t *testing.T) { + t.Parallel() + + service, mockRepoClientFactory := newServiceWithMocks(t) + mockRepoClientFactory.On("NewClient", mock.Anything, mock.Anything).Return(nil, assert.AnError).Once() + + _, err := service.CommitHydratedManifests(context.Background(), validRequest) + require.Error(t, err) + assert.ErrorIs(t, err, assert.AnError) + }) + + t.Run("happy path", func(t *testing.T) { + t.Parallel() + + service, mockRepoClientFactory := newServiceWithMocks(t) + mockGitClient := gitmocks.NewClient(t) + mockGitClient.On("Init").Return(nil).Once() + mockGitClient.On("Fetch", mock.Anything).Return(nil).Once() + mockGitClient.On("SetAuthor", "Argo CD", "argo-cd@example.com").Return("", nil).Once() + mockGitClient.On("CheckoutOrOrphan", "env/test", false).Return("", nil).Once() + mockGitClient.On("CheckoutOrNew", "main", "env/test", false).Return("", nil).Once() + mockGitClient.On("RemoveContents").Return("", nil).Once() + mockGitClient.On("CommitAndPush", "main", "test commit message").Return("", nil).Once() + mockGitClient.On("CommitSHA").Return("it-worked!", nil).Once() + mockRepoClientFactory.On("NewClient", mock.Anything, mock.Anything).Return(mockGitClient, nil).Once() + + resp, err := service.CommitHydratedManifests(context.Background(), validRequest) + require.NoError(t, err) + require.NotNil(t, resp) + assert.Equal(t, "it-worked!", resp.HydratedSha) + }) +} + +func newServiceWithMocks(t *testing.T) (*Service, *mocks.RepoClientFactory) { + t.Helper() + + metricsServer := metrics.NewMetricsServer() + mockCredsStore := git.NoopCredsStore{} + service := NewService(mockCredsStore, metricsServer) + mockRepoClientFactory := mocks.NewRepoClientFactory(t) + service.repoClientFactory = mockRepoClientFactory + + return service, mockRepoClientFactory +} diff --git a/commitserver/commit/credentialtypehelper.go b/commitserver/commit/credentialtypehelper.go new file mode 100644 index 0000000000000..eda3b8040d497 --- /dev/null +++ b/commitserver/commit/credentialtypehelper.go @@ -0,0 +1,23 @@ +package commit + +import "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + +// getCredentialType returns the type of credential used by the repository. +func getCredentialType(repo *v1alpha1.Repository) string { + if repo == nil { + return "" + } + if repo.Password != "" { + return "https" + } + if repo.SSHPrivateKey != "" { + return "ssh" + } + if repo.GithubAppPrivateKey != "" && repo.GithubAppId != 0 && repo.GithubAppInstallationId != 0 { + return "github-app" + } + if repo.GCPServiceAccountKey != "" { + return "cloud-source-repositories" + } + return "" +} diff --git a/commitserver/commit/credentialtypehelper_test.go b/commitserver/commit/credentialtypehelper_test.go new file mode 100644 index 0000000000000..45a013410c20d --- /dev/null +++ b/commitserver/commit/credentialtypehelper_test.go @@ -0,0 +1,62 @@ +package commit + +import ( + "testing" + + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" +) + +func TestRepository_GetCredentialType(t *testing.T) { + tests := []struct { + name string + repo *v1alpha1.Repository + want string + }{ + { + name: "Empty Repository", + repo: nil, + want: "", + }, + { + name: "HTTPS Repository", + repo: &v1alpha1.Repository{ + Repo: "foo", + Password: "some-password", + }, + want: "https", + }, + { + name: "SSH Repository", + repo: &v1alpha1.Repository{ + Repo: "foo", + SSHPrivateKey: "some-key", + }, + want: "ssh", + }, + { + name: "GitHub App Repository", + repo: &v1alpha1.Repository{ + Repo: "foo", + GithubAppPrivateKey: "some-key", + GithubAppId: 1, + GithubAppInstallationId: 1, + }, + want: "github-app", + }, + { + name: "Google Cloud Repository", + repo: &v1alpha1.Repository{ + Repo: "foo", + GCPServiceAccountKey: "some-key", + }, + want: "cloud-source-repositories", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getCredentialType(tt.repo); got != tt.want { + t.Errorf("Repository.GetCredentialType() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/commitserver/commit/hydratorhelper.go b/commitserver/commit/hydratorhelper.go new file mode 100644 index 0000000000000..a4fbeb591b53c --- /dev/null +++ b/commitserver/commit/hydratorhelper.go @@ -0,0 +1,145 @@ +package commit + +import ( + "encoding/json" + "fmt" + "os" + "path" + "text/template" + + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/argoproj/argo-cd/v2/commitserver/apiclient" + "github.com/argoproj/argo-cd/v2/util/io/files" +) + +// WriteForPaths writes the manifests, hydrator.metadata, and README.md files for each path in the provided paths. It +// also writes a root-level hydrator.metadata file containing the repo URL and dry SHA. +func WriteForPaths(rootPath string, repoUrl string, drySha string, paths []*apiclient.PathDetails) error { + // Write the top-level readme. + err := writeMetadata(rootPath, hydratorMetadataFile{DrySHA: drySha, RepoURL: repoUrl}) + if err != nil { + return fmt.Errorf("failed to write top-level hydrator metadata: %w", err) + } + + for _, p := range paths { + hydratePath := p.Path + if hydratePath == "." { + hydratePath = "" + } + + var fullHydratePath string + fullHydratePath, err = files.SecureMkdirAll(rootPath, hydratePath, os.ModePerm) + if err != nil { + return fmt.Errorf("failed to create path: %w", err) + } + + // Write the manifests + err = writeManifests(fullHydratePath, p.Manifests) + if err != nil { + return fmt.Errorf("failed to write manifests: %w", err) + } + + // Write hydrator.metadata containing information about the hydration process. + hydratorMetadata := hydratorMetadataFile{ + Commands: p.Commands, + DrySHA: drySha, + RepoURL: repoUrl, + } + err = writeMetadata(fullHydratePath, hydratorMetadata) + if err != nil { + return fmt.Errorf("failed to write hydrator metadata: %w", err) + } + + // Write README + err = writeReadme(fullHydratePath, hydratorMetadata) + if err != nil { + return fmt.Errorf("failed to write readme: %w", err) + } + } + return nil +} + +// writeMetadata writes the metadata to the hydrator.metadata file. +func writeMetadata(dirPath string, metadata hydratorMetadataFile) error { + hydratorMetadataJson, err := json.MarshalIndent(metadata, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal hydrator metadata: %w", err) + } + // No need to use SecureJoin here, as the path is already sanitized. + hydratorMetadataPath := path.Join(dirPath, "hydrator.metadata") + err = os.WriteFile(hydratorMetadataPath, hydratorMetadataJson, os.ModePerm) + if err != nil { + return fmt.Errorf("failed to write hydrator metadata: %w", err) + } + return nil +} + +// writeReadme writes the readme to the README.md file. +func writeReadme(dirPath string, metadata hydratorMetadataFile) error { + readmeTemplate := template.New("readme") + readmeTemplate, err := readmeTemplate.Parse(manifestHydrationReadmeTemplate) + if err != nil { + return fmt.Errorf("failed to parse readme template: %w", err) + } + // Create writer to template into + // No need to use SecureJoin here, as the path is already sanitized. + readmePath := path.Join(dirPath, "README.md") + readmeFile, err := os.Create(readmePath) + if err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to create README file: %w", err) + } + err = readmeTemplate.Execute(readmeFile, metadata) + closeErr := readmeFile.Close() + if closeErr != nil { + log.WithError(closeErr).Error("failed to close README file") + } + if err != nil { + return fmt.Errorf("failed to execute readme template: %w", err) + } + return nil +} + +// writeManifests writes the manifests to the manifest.yaml file, truncating the file if it exists and appending the +// manifests in the order they are provided. +func writeManifests(dirPath string, manifests []*apiclient.HydratedManifestDetails) error { + // If the file exists, truncate it. + // No need to use SecureJoin here, as the path is already sanitized. + manifestPath := path.Join(dirPath, "manifest.yaml") + + file, err := os.OpenFile(manifestPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) + if err != nil { + return fmt.Errorf("failed to open manifest file: %w", err) + } + defer func() { + err := file.Close() + if err != nil { + log.WithError(err).Error("failed to close file") + } + }() + + enc := yaml.NewEncoder(file) + defer func() { + err := enc.Close() + if err != nil { + log.WithError(err).Error("failed to close yaml encoder") + } + }() + enc.SetIndent(2) + + for _, m := range manifests { + obj := &unstructured.Unstructured{} + err = json.Unmarshal([]byte(m.ManifestJSON), obj) + if err != nil { + return fmt.Errorf("failed to unmarshal manifest: %w", err) + } + err = enc.Encode(&obj.Object) + if err != nil { + return fmt.Errorf("failed to encode manifest: %w", err) + } + } + + return nil +} diff --git a/commitserver/commit/hydratorhelper_test.go b/commitserver/commit/hydratorhelper_test.go new file mode 100644 index 0000000000000..51e8adf0c69a5 --- /dev/null +++ b/commitserver/commit/hydratorhelper_test.go @@ -0,0 +1,135 @@ +package commit + +import ( + "encoding/json" + "os" + "path" + "testing" + + securejoin "github.com/cyphar/filepath-securejoin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/argoproj/argo-cd/v2/commitserver/apiclient" +) + +func TestWriteForPaths(t *testing.T) { + dir := t.TempDir() + + repoUrl := "https://github.com/example/repo" + drySha := "abc123" + paths := []*apiclient.PathDetails{ + { + Path: "path1", + Manifests: []*apiclient.HydratedManifestDetails{ + {ManifestJSON: `{"kind":"Pod","apiVersion":"v1"}`}, + }, + Commands: []string{"command1", "command2"}, + }, + { + Path: "path2", + Manifests: []*apiclient.HydratedManifestDetails{ + {ManifestJSON: `{"kind":"Service","apiVersion":"v1"}`}, + }, + Commands: []string{"command3"}, + }, + } + + err := WriteForPaths(dir, repoUrl, drySha, paths) + require.NoError(t, err) + + // Check if the top-level hydrator.metadata exists and contains the repo URL and dry SHA + topMetadataPath := path.Join(dir, "hydrator.metadata") + topMetadataBytes, err := os.ReadFile(topMetadataPath) + require.NoError(t, err) + + var topMetadata hydratorMetadataFile + err = json.Unmarshal(topMetadataBytes, &topMetadata) + require.NoError(t, err) + assert.Equal(t, repoUrl, topMetadata.RepoURL) + assert.Equal(t, drySha, topMetadata.DrySHA) + + for _, p := range paths { + fullHydratePath, err := securejoin.SecureJoin(dir, p.Path) + require.NoError(t, err) + + // Check if each path directory exists + assert.DirExists(t, fullHydratePath) + + // Check if each path contains a hydrator.metadata file and contains the repo URL + metadataPath := path.Join(fullHydratePath, "hydrator.metadata") + metadataBytes, err := os.ReadFile(metadataPath) + require.NoError(t, err) + + var readMetadata hydratorMetadataFile + err = json.Unmarshal(metadataBytes, &readMetadata) + require.NoError(t, err) + assert.Equal(t, repoUrl, readMetadata.RepoURL) + + // Check if each path contains a README.md file and contains the repo URL + readmePath := path.Join(fullHydratePath, "README.md") + readmeBytes, err := os.ReadFile(readmePath) + require.NoError(t, err) + assert.Contains(t, string(readmeBytes), repoUrl) + + // Check if each path contains a manifest.yaml file and contains the word Pod + manifestPath := path.Join(fullHydratePath, "manifest.yaml") + manifestBytes, err := os.ReadFile(manifestPath) + require.NoError(t, err) + assert.Contains(t, string(manifestBytes), "kind") + } +} + +func TestWriteMetadata(t *testing.T) { + dir := t.TempDir() + + metadata := hydratorMetadataFile{ + RepoURL: "https://github.com/example/repo", + DrySHA: "abc123", + } + + err := writeMetadata(dir, metadata) + require.NoError(t, err) + + metadataPath := path.Join(dir, "hydrator.metadata") + metadataBytes, err := os.ReadFile(metadataPath) + require.NoError(t, err) + + var readMetadata hydratorMetadataFile + err = json.Unmarshal(metadataBytes, &readMetadata) + require.NoError(t, err) + assert.Equal(t, metadata, readMetadata) +} + +func TestWriteReadme(t *testing.T) { + dir := t.TempDir() + + metadata := hydratorMetadataFile{ + RepoURL: "https://github.com/example/repo", + DrySHA: "abc123", + } + + err := writeReadme(dir, metadata) + require.NoError(t, err) + + readmePath := path.Join(dir, "README.md") + readmeBytes, err := os.ReadFile(readmePath) + require.NoError(t, err) + assert.Contains(t, string(readmeBytes), metadata.RepoURL) +} + +func TestWriteManifests(t *testing.T) { + dir := t.TempDir() + + manifests := []*apiclient.HydratedManifestDetails{ + {ManifestJSON: `{"kind":"Pod","apiVersion":"v1"}`}, + } + + err := writeManifests(dir, manifests) + require.NoError(t, err) + + manifestPath := path.Join(dir, "manifest.yaml") + manifestBytes, err := os.ReadFile(manifestPath) + require.NoError(t, err) + assert.Contains(t, string(manifestBytes), "kind") +} diff --git a/commitserver/commit/mocks/RepoClientFactory.go b/commitserver/commit/mocks/RepoClientFactory.go new file mode 100644 index 0000000000000..020c78fdf5f85 --- /dev/null +++ b/commitserver/commit/mocks/RepoClientFactory.go @@ -0,0 +1,59 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + +package mocks + +import ( + git "github.com/argoproj/argo-cd/v2/util/git" + mock "github.com/stretchr/testify/mock" + + v1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" +) + +// RepoClientFactory is an autogenerated mock type for the RepoClientFactory type +type RepoClientFactory struct { + mock.Mock +} + +// NewClient provides a mock function with given fields: repo, rootPath +func (_m *RepoClientFactory) NewClient(repo *v1alpha1.Repository, rootPath string) (git.Client, error) { + ret := _m.Called(repo, rootPath) + + if len(ret) == 0 { + panic("no return value specified for NewClient") + } + + var r0 git.Client + var r1 error + if rf, ok := ret.Get(0).(func(*v1alpha1.Repository, string) (git.Client, error)); ok { + return rf(repo, rootPath) + } + if rf, ok := ret.Get(0).(func(*v1alpha1.Repository, string) git.Client); ok { + r0 = rf(repo, rootPath) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(git.Client) + } + } + + if rf, ok := ret.Get(1).(func(*v1alpha1.Repository, string) error); ok { + r1 = rf(repo, rootPath) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewRepoClientFactory creates a new instance of RepoClientFactory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewRepoClientFactory(t interface { + mock.TestingT + Cleanup(func()) +}) *RepoClientFactory { + mock := &RepoClientFactory{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/commitserver/commit/repo_client_factory.go b/commitserver/commit/repo_client_factory.go new file mode 100644 index 0000000000000..f0f3b5c75dbd8 --- /dev/null +++ b/commitserver/commit/repo_client_factory.go @@ -0,0 +1,32 @@ +package commit + +import ( + "github.com/argoproj/argo-cd/v2/commitserver/metrics" + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "github.com/argoproj/argo-cd/v2/util/git" +) + +// RepoClientFactory is a factory for creating git clients for a repository. +type RepoClientFactory interface { + NewClient(repo *v1alpha1.Repository, rootPath string) (git.Client, error) +} + +type repoClientFactory struct { + gitCredsStore git.CredsStore + metricsServer *metrics.Server +} + +// NewRepoClientFactory returns a new instance of the repo client factory. +func NewRepoClientFactory(gitCredsStore git.CredsStore, metricsServer *metrics.Server) RepoClientFactory { + return &repoClientFactory{ + gitCredsStore: gitCredsStore, + metricsServer: metricsServer, + } +} + +// NewClient creates a new git client for the repository. +func (r *repoClientFactory) NewClient(repo *v1alpha1.Repository, rootPath string) (git.Client, error) { + gitCreds := repo.GetGitCreds(r.gitCredsStore) + opts := git.WithEventHandlers(metrics.NewGitClientEventHandlers(r.metricsServer)) + return git.NewClientExt(repo.Repo, rootPath, gitCreds, repo.IsInsecure(), repo.IsLFSEnabled(), repo.Proxy, repo.NoProxy, opts) +} diff --git a/commitserver/metrics/githandlers.go b/commitserver/metrics/githandlers.go new file mode 100644 index 0000000000000..4a960ebd54f34 --- /dev/null +++ b/commitserver/metrics/githandlers.go @@ -0,0 +1,34 @@ +package metrics + +import ( + "time" + + "github.com/argoproj/argo-cd/v2/util/git" +) + +// NewGitClientEventHandlers creates event handlers that update Git related metrics +func NewGitClientEventHandlers(metricsServer *Server) git.EventHandlers { + return git.EventHandlers{ + OnFetch: func(repo string) func() { + startTime := time.Now() + metricsServer.IncGitRequest(repo, GitRequestTypeFetch) + return func() { + metricsServer.ObserveGitRequestDuration(repo, GitRequestTypeFetch, time.Since(startTime)) + } + }, + OnLsRemote: func(repo string) func() { + startTime := time.Now() + metricsServer.IncGitRequest(repo, GitRequestTypeLsRemote) + return func() { + metricsServer.ObserveGitRequestDuration(repo, GitRequestTypeLsRemote, time.Since(startTime)) + } + }, + OnPush: func(repo string) func() { + startTime := time.Now() + metricsServer.IncGitRequest(repo, GitRequestTypePush) + return func() { + metricsServer.ObserveGitRequestDuration(repo, GitRequestTypePush, time.Since(startTime)) + } + }, + } +} diff --git a/commitserver/metrics/metrics.go b/commitserver/metrics/metrics.go new file mode 100644 index 0000000000000..e6d291e063892 --- /dev/null +++ b/commitserver/metrics/metrics.go @@ -0,0 +1,157 @@ +package metrics + +import ( + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// Server is a prometheus server which collects application metrics. +type Server struct { + handler http.Handler + commitPendingRequestsGauge *prometheus.GaugeVec + gitRequestCounter *prometheus.CounterVec + gitRequestHistogram *prometheus.HistogramVec + commitRequestHistogram *prometheus.HistogramVec + userInfoRequestHistogram *prometheus.HistogramVec + commitRequestCounter *prometheus.CounterVec +} + +// GitRequestType is the type of git request +type GitRequestType string + +const ( + // GitRequestTypeLsRemote is a request to list remote refs + GitRequestTypeLsRemote = "ls-remote" + // GitRequestTypeFetch is a request to fetch from remote + GitRequestTypeFetch = "fetch" + // GitRequestTypePush is a request to push to remote + GitRequestTypePush = "push" +) + +// CommitResponseType is the type of response for a commit request +type CommitResponseType string + +const ( + // CommitResponseTypeSuccess is a successful commit request + CommitResponseTypeSuccess = "success" + // CommitResponseTypeFailure is a failed commit request + CommitResponseTypeFailure = "failure" +) + +// NewMetricsServer returns a new prometheus server which collects application metrics. +func NewMetricsServer() *Server { + registry := prometheus.NewRegistry() + registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + registry.MustRegister(collectors.NewGoCollector()) + + commitPendingRequestsGauge := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "argocd_commitserver_commit_pending_request_total", + Help: "Number of pending commit requests", + }, + []string{"repo"}, + ) + registry.MustRegister(commitPendingRequestsGauge) + + gitRequestCounter := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "argocd_commitserver_git_request_total", + Help: "Number of git requests performed by repo server", + }, + []string{"repo", "request_type"}, + ) + registry.MustRegister(gitRequestCounter) + + gitRequestHistogram := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "argocd_commitserver_git_request_duration_seconds", + Help: "Git requests duration seconds.", + Buckets: []float64{0.1, 0.25, .5, 1, 2, 4, 10, 20}, + }, + []string{"repo", "request_type"}, + ) + registry.MustRegister(gitRequestHistogram) + + commitRequestHistogram := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "argocd_commitserver_commit_request_duration_seconds", + Help: "Commit request duration seconds.", + Buckets: []float64{0.1, 0.25, .5, 1, 2, 4, 10, 20}, + }, + []string{"repo", "response_type"}, + ) + registry.MustRegister(commitRequestHistogram) + + userInfoRequestHistogram := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "argocd_commitserver_userinfo_request_duration_seconds", + Help: "Userinfo request duration seconds.", + Buckets: []float64{0.1, 0.25, .5, 1, 2, 4, 10, 20}, + }, + []string{"repo", "credential_type"}, + ) + registry.MustRegister(userInfoRequestHistogram) + + commitRequestCounter := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "argocd_commitserver_commit_request_total", + Help: "Number of commit requests performed handled", + }, + []string{"repo", "response_type"}, + ) + registry.MustRegister(commitRequestCounter) + + return &Server{ + handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}), + commitPendingRequestsGauge: commitPendingRequestsGauge, + gitRequestCounter: gitRequestCounter, + gitRequestHistogram: gitRequestHistogram, + commitRequestHistogram: commitRequestHistogram, + userInfoRequestHistogram: userInfoRequestHistogram, + commitRequestCounter: commitRequestCounter, + } +} + +// GetHandler returns the http.Handler for the prometheus server +func (m *Server) GetHandler() http.Handler { + return m.handler +} + +// IncPendingCommitRequest increments the pending commit requests gauge +func (m *Server) IncPendingCommitRequest(repo string) { + m.commitPendingRequestsGauge.WithLabelValues(repo).Inc() +} + +// DecPendingCommitRequest decrements the pending commit requests gauge +func (m *Server) DecPendingCommitRequest(repo string) { + m.commitPendingRequestsGauge.WithLabelValues(repo).Dec() +} + +// IncGitRequest increments the git requests counter +func (m *Server) IncGitRequest(repo string, requestType GitRequestType) { + m.gitRequestCounter.WithLabelValues(repo, string(requestType)).Inc() +} + +// ObserveGitRequestDuration observes the duration of a git request +func (m *Server) ObserveGitRequestDuration(repo string, requestType GitRequestType, duration time.Duration) { + m.gitRequestHistogram.WithLabelValues(repo, string(requestType)).Observe(duration.Seconds()) +} + +// ObserveCommitRequestDuration observes the duration of a commit request +func (m *Server) ObserveCommitRequestDuration(repo string, rt CommitResponseType, duration time.Duration) { + m.commitRequestHistogram.WithLabelValues(repo, string(rt)).Observe(duration.Seconds()) +} + +// ObserveUserInfoRequestDuration observes the duration of a userinfo request +func (m *Server) ObserveUserInfoRequestDuration(repo string, credentialType string, duration time.Duration) { + m.userInfoRequestHistogram.WithLabelValues(repo, credentialType).Observe(duration.Seconds()) +} + +// IncCommitRequest increments the commit request counter +func (m *Server) IncCommitRequest(repo string, rt CommitResponseType) { + m.commitRequestCounter.WithLabelValues(repo, string(rt)).Inc() +} diff --git a/commitserver/server.go b/commitserver/server.go new file mode 100644 index 0000000000000..5e5b63324ca17 --- /dev/null +++ b/commitserver/server.go @@ -0,0 +1,38 @@ +package commitserver + +import ( + "google.golang.org/grpc" + "google.golang.org/grpc/health" + "google.golang.org/grpc/health/grpc_health_v1" + + "github.com/argoproj/argo-cd/v2/commitserver/apiclient" + "github.com/argoproj/argo-cd/v2/commitserver/commit" + "github.com/argoproj/argo-cd/v2/commitserver/metrics" + versionpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/version" + "github.com/argoproj/argo-cd/v2/server/version" + "github.com/argoproj/argo-cd/v2/util/git" +) + +// ArgoCDCommitServer is the server that handles commit requests. +type ArgoCDCommitServer struct { + commitService *commit.Service +} + +// NewServer returns a new instance of the commit server. +func NewServer(gitCredsStore git.CredsStore, metricsServer *metrics.Server) *ArgoCDCommitServer { + return &ArgoCDCommitServer{commitService: commit.NewService(gitCredsStore, metricsServer)} +} + +// CreateGRPC creates a new gRPC server. +func (a *ArgoCDCommitServer) CreateGRPC() *grpc.Server { + server := grpc.NewServer() + versionpkg.RegisterVersionServiceServer(server, version.NewServer(nil, func() (bool, error) { + return true, nil + })) + apiclient.RegisterCommitServiceServer(server, a.commitService) + + healthService := health.NewServer() + grpc_health_v1.RegisterHealthServer(server, healthService) + + return server +} diff --git a/common/common.go b/common/common.go index d2e47aa5b1607..eb5db25af2615 100644 --- a/common/common.go +++ b/common/common.go @@ -62,15 +62,19 @@ const ( DefaultPortArgoCDMetrics = 8082 DefaultPortArgoCDAPIServerMetrics = 8083 DefaultPortRepoServerMetrics = 8084 + DefaultPortCommitServer = 8086 + DefaultPortCommitServerMetrics = 8087 ) // DefaultAddressAPIServer for ArgoCD components const ( - DefaultAddressAdminDashboard = "localhost" - DefaultAddressAPIServer = "0.0.0.0" - DefaultAddressAPIServerMetrics = "0.0.0.0" - DefaultAddressRepoServer = "0.0.0.0" - DefaultAddressRepoServerMetrics = "0.0.0.0" + DefaultAddressAdminDashboard = "localhost" + DefaultAddressAPIServer = "0.0.0.0" + DefaultAddressAPIServerMetrics = "0.0.0.0" + DefaultAddressRepoServer = "0.0.0.0" + DefaultAddressRepoServerMetrics = "0.0.0.0" + DefaultAddressCommitServer = "0.0.0.0" + DefaultAddressCommitServerMetrics = "0.0.0.0" ) // Default paths on the pod's file system diff --git a/docs/operator-manual/metrics.md b/docs/operator-manual/metrics.md index 02a490998307a..b18ef8c4fb224 100644 --- a/docs/operator-manual/metrics.md +++ b/docs/operator-manual/metrics.md @@ -128,6 +128,20 @@ Scraped at the `argocd-repo-server:8084/metrics` endpoint. | `argocd_redis_request_total` | counter | Number of Kubernetes requests executed during application reconciliation. | | `argocd_repo_pending_request_total` | gauge | Number of pending requests requiring repository lock | +## Commit Server Metrics + +Metrics about the Commit Server. +Scraped at the `argocd-commit-server:8087/metrics` endpoint. + +| Metric | Type | Description | +|---------------------------------------------------------|:---------:|------------------------------------------------------| +| `argocd_commitserver_commit_pending_request_total` | guage | Number of pending commit requests. | +| `argocd_commitserver_git_request_duration_seconds` | histogram | Git requests duration seconds. | +| `argocd_commitserver_git_request_total` | counter | Number of git requests performed by commit server | +| `argocd_commitserver_commit_request_duration_seconds` | histogram | Commit requests duration seconds. | +| `argocd_commitserver_userinfo_request_duration_seconds` | histogram | Userinfo requests duration seconds. | +| `argocd_commitserver_commit_request_total` | counter | Number of commit requests performed by commit server | + ## Prometheus Operator If using Prometheus Operator, the following ServiceMonitor example manifests can be used. diff --git a/go.mod b/go.mod index 0c5c7b03ff357..f79922246875f 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/golang/protobuf v1.5.4 github.com/google/btree v1.1.3 github.com/google/go-cmp v0.6.0 + github.com/google/go-github/v62 v62.0.0 github.com/google/go-github/v63 v63.0.0 github.com/google/go-jsonnet v0.20.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 @@ -139,7 +140,6 @@ require ( github.com/go-jose/go-jose/v4 v4.0.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-github/v62 v62.0.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect diff --git a/hack/generate-proto.sh b/hack/generate-proto.sh index 83f542a9d21ab..d4a6f991664d0 100755 --- a/hack/generate-proto.sh +++ b/hack/generate-proto.sh @@ -95,7 +95,7 @@ MOD_ROOT=${GOPATH}/pkg/mod grpc_gateway_version=$(go list -m github.com/grpc-ecosystem/grpc-gateway | awk '{print $NF}' | head -1) GOOGLE_PROTO_API_PATH=${MOD_ROOT}/github.com/grpc-ecosystem/grpc-gateway@${grpc_gateway_version}/third_party/googleapis GOGO_PROTOBUF_PATH=${PROJECT_ROOT}/vendor/github.com/gogo/protobuf -PROTO_FILES=$(find "$PROJECT_ROOT" \( -name "*.proto" -and -path '*/server/*' -or -path '*/reposerver/*' -and -name "*.proto" -or -path '*/cmpserver/*' -and -name "*.proto" \) | sort) +PROTO_FILES=$(find "$PROJECT_ROOT" \( -name "*.proto" -and -path '*/server/*' -or -path '*/reposerver/*' -and -name "*.proto" -or -path '*/cmpserver/*' -and -name "*.proto" -or -path '*/commitserver/*' -and -name "*.proto" \) | sort) for i in ${PROTO_FILES}; do protoc \ -I"${PROJECT_ROOT}" \ @@ -162,3 +162,4 @@ clean_swagger server clean_swagger reposerver clean_swagger controller clean_swagger cmpserver +clean_swagger commitserver diff --git a/manifests/base/commit-server/argocd-commit-server-deployment.yaml b/manifests/base/commit-server/argocd-commit-server-deployment.yaml new file mode 100644 index 0000000000000..d80d0ba62a01c --- /dev/null +++ b/manifests/base/commit-server/argocd-commit-server-deployment.yaml @@ -0,0 +1,159 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: argocd-commit-server + app.kubernetes.io/part-of: argocd + app.kubernetes.io/component: commit-server + name: argocd-commit-server +spec: + selector: + matchLabels: + app.kubernetes.io/name: argocd-commit-server + template: + metadata: + labels: + app.kubernetes.io/name: argocd-commit-server + spec: + serviceAccountName: argocd-commit-server + automountServiceAccountToken: false + containers: + - name: argocd-commit-server + image: quay.io/argoproj/argocd:latest + imagePullPolicy: Always + args: + - /usr/local/bin/argocd-commit-server + env: + - name: ARGOCD_COMMIT_SERVER_LISTEN_ADDRESS + valueFrom: + configMapKeyRef: + name: argocd-cmd-params-cm + key: commitserver.listen.address + optional: true + - name: ARGOCD_COMMIT_SERVER_METRICS_LISTEN_ADDRESS + valueFrom: + configMapKeyRef: + name: argocd-cmd-params-cm + key: commitserver.metrics.listen.address + optional: true + - name: ARGOCD_COMMIT_SERVER_LOGFORMAT + valueFrom: + configMapKeyRef: + name: argocd-cmd-params-cm + key: commitserver.log.format + optional: true + - name: ARGOCD_COMMIT_SERVER_LOGLEVEL + valueFrom: + configMapKeyRef: + name: argocd-cmd-params-cm + key: commitserver.log.level + optional: true + ports: + - containerPort: 8086 + - containerPort: 8087 + livenessProbe: + httpGet: + path: /healthz?full=true + port: 8087 + initialDelaySeconds: 30 + periodSeconds: 30 + failureThreshold: 3 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /healthz + port: 8087 + initialDelaySeconds: 5 + periodSeconds: 10 + securityContext: + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: ssh-known-hosts + mountPath: /app/config/ssh + - name: tls-certs + mountPath: /app/config/tls + - name: gpg-keys + mountPath: /app/config/gpg/source + - name: gpg-keyring + mountPath: /app/config/gpg/keys + - name: argocd-commit-server-tls + mountPath: /app/config/reposerver/tls + - name: tmp + mountPath: /tmp + - mountPath: /helm-working-dir + name: helm-working-dir + - mountPath: /home/argocd/cmp-server/plugins + name: plugins + initContainers: + - command: + - /bin/cp + - -n + - /usr/local/bin/argocd + - /var/run/argocd/argocd-cmp-server + image: quay.io/argoproj/argocd:latest + name: copyutil + securityContext: + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /var/run/argocd + name: var-files + volumes: + - name: ssh-known-hosts + configMap: + name: argocd-ssh-known-hosts-cm + - name: tls-certs + configMap: + name: argocd-tls-certs-cm + - name: gpg-keys + configMap: + name: argocd-gpg-keys-cm + - name: gpg-keyring + emptyDir: {} + - name: tmp + emptyDir: {} + - name: helm-working-dir + emptyDir: {} + - name: argocd-commit-server-tls + secret: + secretName: argocd-commit-server-tls + optional: true + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + - key: ca.crt + path: ca.crt + - emptyDir: {} + name: var-files + - emptyDir: {} + name: plugins + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/name: argocd-commit-server + topologyKey: kubernetes.io/hostname + - weight: 5 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/part-of: argocd + topologyKey: kubernetes.io/hostname diff --git a/manifests/base/commit-server/argocd-commit-server-network-policy.yaml b/manifests/base/commit-server/argocd-commit-server-network-policy.yaml new file mode 100644 index 0000000000000..119bf985d5ddd --- /dev/null +++ b/manifests/base/commit-server/argocd-commit-server-network-policy.yaml @@ -0,0 +1,22 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: argocd-commit-server-network-policy +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: argocd-commit-server + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: argocd-application-controller + ports: + - protocol: TCP + port: 8086 + - from: + - namespaceSelector: { } + ports: + - port: 8087 diff --git a/manifests/base/commit-server/argocd-commit-server-sa.yaml b/manifests/base/commit-server/argocd-commit-server-sa.yaml new file mode 100644 index 0000000000000..802d484ed3ffe --- /dev/null +++ b/manifests/base/commit-server/argocd-commit-server-sa.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: argocd-commit-server + app.kubernetes.io/part-of: argocd + app.kubernetes.io/component: commit-server + name: argocd-commit-server diff --git a/manifests/base/commit-server/argocd-commit-server-service.yaml b/manifests/base/commit-server/argocd-commit-server-service.yaml new file mode 100644 index 0000000000000..75e761a10e666 --- /dev/null +++ b/manifests/base/commit-server/argocd-commit-server-service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: argocd-commit-server + app.kubernetes.io/part-of: argocd + app.kubernetes.io/component: commit-server + name: argocd-commit-server +spec: + ports: + - name: server + protocol: TCP + port: 8086 + targetPort: 8086 + - name: metrics + protocol: TCP + port: 8087 + targetPort: 8087 + selector: + app.kubernetes.io/name: argocd-commit-server diff --git a/manifests/base/commit-server/kustomization.yaml b/manifests/base/commit-server/kustomization.yaml new file mode 100644 index 0000000000000..1bdee4f2c1430 --- /dev/null +++ b/manifests/base/commit-server/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- argocd-commit-server-sa.yaml +- argocd-commit-server-deployment.yaml +- argocd-commit-server-service.yaml +- argocd-commit-server-network-policy.yaml diff --git a/reposerver/askpass/common.go b/reposerver/askpass/common.go index c9757f5878956..2a34cca52c84c 100644 --- a/reposerver/askpass/common.go +++ b/reposerver/askpass/common.go @@ -11,6 +11,8 @@ const ( ASKPASS_NONCE_ENV = "ARGOCD_GIT_ASKPASS_NONCE" // AKSPASS_SOCKET_PATH_ENV is the environment variable that is used to pass the socket path to the askpass script AKSPASS_SOCKET_PATH_ENV = "ARGOCD_ASK_PASS_SOCK" + // CommitServerSocketPath is the path to the socket used by the commit server to communicate with the askpass server + CommitServerSocketPath = "/tmp/commit-server-ask-pass.sock" ) func init() { diff --git a/reposerver/repository/repository.go b/reposerver/repository/repository.go index 71fa431d557bc..df390ce3bde75 100644 --- a/reposerver/repository/repository.go +++ b/reposerver/repository/repository.go @@ -2510,7 +2510,7 @@ func checkoutRevision(gitClient git.Client, revision string, submoduleEnabled bo } } - err = gitClient.Checkout(revision, submoduleEnabled) + _, err = gitClient.Checkout(revision, submoduleEnabled) if err != nil { // When fetching with no revision, only refs/heads/* and refs/remotes/origin/* are fetched. If checkout fails // for the given revision, try explicitly fetching it. @@ -2522,7 +2522,7 @@ func checkoutRevision(gitClient git.Client, revision string, submoduleEnabled bo return status.Errorf(codes.Internal, "Failed to checkout revision %s: %v", revision, err) } - err = gitClient.Checkout("FETCH_HEAD", submoduleEnabled) + _, err = gitClient.Checkout("FETCH_HEAD", submoduleEnabled) if err != nil { return status.Errorf(codes.Internal, "Failed to checkout FETCH_HEAD: %v", err) } diff --git a/reposerver/repository/repository_test.go b/reposerver/repository/repository_test.go index 57256dce7468e..80f155e6fedad 100644 --- a/reposerver/repository/repository_test.go +++ b/reposerver/repository/repository_test.go @@ -109,7 +109,7 @@ func newServiceWithMocks(t *testing.T, root string, signed bool) (*Service, *git gitClient.On("Init").Return(nil) gitClient.On("IsRevisionPresent", mock.Anything).Return(false) gitClient.On("Fetch", mock.Anything).Return(nil) - gitClient.On("Checkout", mock.Anything, mock.Anything).Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Return("", nil) gitClient.On("LsRemote", mock.Anything).Return(mock.Anything, nil) gitClient.On("CommitSHA").Return(mock.Anything, nil) gitClient.On("Root").Return(root) @@ -188,7 +188,7 @@ func newServiceWithCommitSHA(t *testing.T, root, revision string) *Service { gitClient.On("Init").Return(nil) gitClient.On("IsRevisionPresent", mock.Anything).Return(false) gitClient.On("Fetch", mock.Anything).Return(nil) - gitClient.On("Checkout", mock.Anything, mock.Anything).Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Return("", nil) gitClient.On("LsRemote", revision).Return(revision, revisionErr) gitClient.On("CommitSHA").Return("632039659e542ed7de0c170a4fcc1c571b288fc0", nil) gitClient.On("Root").Return(root) @@ -3070,7 +3070,7 @@ func TestCheckoutRevisionPresentSkipFetch(t *testing.T) { gitClient := &gitmocks.Client{} gitClient.On("Init").Return(nil) gitClient.On("IsRevisionPresent", revision).Return(true) - gitClient.On("Checkout", revision, mock.Anything).Return(nil) + gitClient.On("Checkout", revision, mock.Anything).Return("", nil) err := checkoutRevision(gitClient, revision, false) require.NoError(t, err) @@ -3083,7 +3083,7 @@ func TestCheckoutRevisionNotPresentCallFetch(t *testing.T) { gitClient.On("Init").Return(nil) gitClient.On("IsRevisionPresent", revision).Return(false) gitClient.On("Fetch", "").Return(nil) - gitClient.On("Checkout", revision, mock.Anything).Return(nil) + gitClient.On("Checkout", revision, mock.Anything).Return("", nil) err := checkoutRevision(gitClient, revision, false) require.NoError(t, err) @@ -3409,7 +3409,7 @@ func TestErrorGetGitDirectories(t *testing.T) { }, want: nil, wantErr: assert.Error}, {name: "InvalidResolveRevision", fields: fields{service: func() *Service { s, _, _ := newServiceWithOpt(t, func(gitClient *gitmocks.Client, helmClient *helmmocks.Client, paths *iomocks.TempPaths) { - gitClient.On("Checkout", mock.Anything, mock.Anything).Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Return("", nil) gitClient.On("LsRemote", mock.Anything).Return("", fmt.Errorf("ah error")) gitClient.On("Root").Return(root) paths.On("GetPath", mock.Anything).Return(".", nil) @@ -3426,7 +3426,7 @@ func TestErrorGetGitDirectories(t *testing.T) { }, want: nil, wantErr: assert.Error}, {name: "ErrorVerifyCommit", fields: fields{service: func() *Service { s, _, _ := newServiceWithOpt(t, func(gitClient *gitmocks.Client, helmClient *helmmocks.Client, paths *iomocks.TempPaths) { - gitClient.On("Checkout", mock.Anything, mock.Anything).Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Return("", nil) gitClient.On("LsRemote", mock.Anything).Return("", fmt.Errorf("ah error")) gitClient.On("VerifyCommitSignature", mock.Anything).Return("", fmt.Errorf("revision %s is not signed", "sadfsadf")) gitClient.On("Root").Return(root) @@ -3463,7 +3463,7 @@ func TestGetGitDirectories(t *testing.T) { gitClient.On("Init").Return(nil) gitClient.On("IsRevisionPresent", mock.Anything).Return(false) gitClient.On("Fetch", mock.Anything).Return(nil) - gitClient.On("Checkout", mock.Anything, mock.Anything).Once().Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Once().Return("", nil) gitClient.On("LsRemote", "HEAD").Return("632039659e542ed7de0c170a4fcc1c571b288fc0", nil) gitClient.On("Root").Return(root) paths.On("GetPath", mock.Anything).Return(root, nil) @@ -3496,7 +3496,7 @@ func TestGetGitDirectoriesWithHiddenDirSupported(t *testing.T) { gitClient.On("Init").Return(nil) gitClient.On("IsRevisionPresent", mock.Anything).Return(false) gitClient.On("Fetch", mock.Anything).Return(nil) - gitClient.On("Checkout", mock.Anything, mock.Anything).Once().Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Once().Return("", nil) gitClient.On("LsRemote", "HEAD").Return("632039659e542ed7de0c170a4fcc1c571b288fc0", nil) gitClient.On("Root").Return(root) paths.On("GetPath", mock.Anything).Return(root, nil) @@ -3551,7 +3551,7 @@ func TestErrorGetGitFiles(t *testing.T) { }, want: nil, wantErr: assert.Error}, {name: "InvalidResolveRevision", fields: fields{service: func() *Service { s, _, _ := newServiceWithOpt(t, func(gitClient *gitmocks.Client, helmClient *helmmocks.Client, paths *iomocks.TempPaths) { - gitClient.On("Checkout", mock.Anything, mock.Anything).Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Return("", nil) gitClient.On("LsRemote", mock.Anything).Return("", fmt.Errorf("ah error")) gitClient.On("Root").Return(root) paths.On("GetPath", mock.Anything).Return(".", nil) @@ -3590,7 +3590,7 @@ func TestGetGitFiles(t *testing.T) { gitClient.On("Init").Return(nil) gitClient.On("IsRevisionPresent", mock.Anything).Return(false) gitClient.On("Fetch", mock.Anything).Return(nil) - gitClient.On("Checkout", mock.Anything, mock.Anything).Once().Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Once().Return("", nil) gitClient.On("LsRemote", "HEAD").Return("632039659e542ed7de0c170a4fcc1c571b288fc0", nil) gitClient.On("Root").Return(root) gitClient.On("LsFiles", mock.Anything, mock.Anything).Once().Return(files, nil) @@ -3654,7 +3654,7 @@ func TestErrorUpdateRevisionForPaths(t *testing.T) { }, want: nil, wantErr: assert.Error}, {name: "InvalidResolveRevision", fields: fields{service: func() *Service { s, _, _ := newServiceWithOpt(t, func(gitClient *gitmocks.Client, helmClient *helmmocks.Client, paths *iomocks.TempPaths) { - gitClient.On("Checkout", mock.Anything, mock.Anything).Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Return("", nil) gitClient.On("LsRemote", mock.Anything).Return("", fmt.Errorf("ah error")) gitClient.On("Root").Return(root) paths.On("GetPath", mock.Anything).Return(".", nil) @@ -3672,7 +3672,7 @@ func TestErrorUpdateRevisionForPaths(t *testing.T) { }, want: nil, wantErr: assert.Error}, {name: "InvalidResolveSyncedRevision", fields: fields{service: func() *Service { s, _, _ := newServiceWithOpt(t, func(gitClient *gitmocks.Client, helmClient *helmmocks.Client, paths *iomocks.TempPaths) { - gitClient.On("Checkout", mock.Anything, mock.Anything).Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Return("", nil) gitClient.On("LsRemote", "HEAD").Once().Return("632039659e542ed7de0c170a4fcc1c571b288fc0", nil) gitClient.On("LsRemote", mock.Anything).Return("", fmt.Errorf("ah error")) gitClient.On("Root").Return(root) @@ -3725,7 +3725,7 @@ func TestUpdateRevisionForPaths(t *testing.T) { }{ {name: "NoPathAbort", fields: func() fields { s, _, c := newServiceWithOpt(t, func(gitClient *gitmocks.Client, helmClient *helmmocks.Client, paths *iomocks.TempPaths) { - gitClient.On("Checkout", mock.Anything, mock.Anything).Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Return("", nil) }, ".") return fields{ service: s, @@ -3740,7 +3740,7 @@ func TestUpdateRevisionForPaths(t *testing.T) { }, want: &apiclient.UpdateRevisionForPathsResponse{}, wantErr: assert.NoError}, {name: "SameResolvedRevisionAbort", fields: func() fields { s, _, c := newServiceWithOpt(t, func(gitClient *gitmocks.Client, helmClient *helmmocks.Client, paths *iomocks.TempPaths) { - gitClient.On("Checkout", mock.Anything, mock.Anything).Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Return("", nil) gitClient.On("LsRemote", "HEAD").Once().Return("632039659e542ed7de0c170a4fcc1c571b288fc0", nil) gitClient.On("LsRemote", "SYNCEDHEAD").Once().Return("632039659e542ed7de0c170a4fcc1c571b288fc0", nil) paths.On("GetPath", mock.Anything).Return(".", nil) @@ -3766,7 +3766,7 @@ func TestUpdateRevisionForPaths(t *testing.T) { gitClient.On("Init").Return(nil) gitClient.On("IsRevisionPresent", mock.Anything).Return(false) gitClient.On("Fetch", mock.Anything).Return(nil) - gitClient.On("Checkout", mock.Anything, mock.Anything).Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Return("", nil) gitClient.On("LsRemote", "HEAD").Once().Return("632039659e542ed7de0c170a4fcc1c571b288fc0", nil) gitClient.On("LsRemote", "SYNCEDHEAD").Once().Return("1e67a504d03def3a6a1125d934cb511680f72555", nil) paths.On("GetPath", mock.Anything).Return(".", nil) @@ -3795,7 +3795,7 @@ func TestUpdateRevisionForPaths(t *testing.T) { gitClient.On("Init").Return(nil) gitClient.On("IsRevisionPresent", mock.Anything).Return(false) gitClient.On("Fetch", mock.Anything).Return(nil) - gitClient.On("Checkout", mock.Anything, mock.Anything).Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Return("", nil) gitClient.On("LsRemote", "HEAD").Once().Return("632039659e542ed7de0c170a4fcc1c571b288fc0", nil) gitClient.On("LsRemote", "SYNCEDHEAD").Once().Return("1e67a504d03def3a6a1125d934cb511680f72555", nil) paths.On("GetPath", mock.Anything).Return(".", nil) @@ -3833,7 +3833,7 @@ func TestUpdateRevisionForPaths(t *testing.T) { gitClient.On("Init").Return(nil) gitClient.On("IsRevisionPresent", mock.Anything).Return(false) gitClient.On("Fetch", mock.Anything).Return(nil) - gitClient.On("Checkout", mock.Anything, mock.Anything).Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Return("", nil) gitClient.On("LsRemote", "HEAD").Once().Return("632039659e542ed7de0c170a4fcc1c571b288fc0", nil) gitClient.On("LsRemote", "SYNCEDHEAD").Once().Return("1e67a504d03def3a6a1125d934cb511680f72555", nil) paths.On("GetPath", mock.Anything).Return(".", nil) diff --git a/test/container/Procfile b/test/container/Procfile index 4cebac203f76d..5048fff289ed7 100644 --- a/test/container/Procfile +++ b/test/container/Procfile @@ -3,6 +3,7 @@ api-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run dex: sh -c "test $ARGOCD_IN_CI = true && exit 0; ARGOCD_BINARY_NAME=argocd-dex go run github.com/argoproj/argo-cd/cmd gendexcfg -o `pwd`/dist/dex.yaml && docker run --rm -p ${ARGOCD_E2E_DEX_PORT:-5556}:${ARGOCD_E2E_DEX_PORT:-5556} -v `pwd`/dist/dex.yaml:/dex.yaml ghcr.io/dexidp/dex:v2.41.1 serve /dex.yaml" redis: sh -c "/usr/local/bin/redis-server --save "" --appendonly no --port ${ARGOCD_E2E_REDIS_PORT:-6379}" repo-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_GNUPGHOME=${ARGOCD_GNUPGHOME:-/tmp/argocd-local/gpg/keys} ARGOCD_PLUGINSOCKFILEPATH=${ARGOCD_PLUGINSOCKFILEPATH:-./test/cmp} ARGOCD_GPG_DATA_PATH=${ARGOCD_GPG_DATA_PATH:-/tmp/argocd-local/gpg/source} ARGOCD_BINARY_NAME=argocd-repo-server $COMMAND --loglevel debug --port ${ARGOCD_E2E_REPOSERVER_PORT:-8081} --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379}" +commit-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_BINARY_NAME=argocd-commit-server $COMMAND --loglevel debug --port ${ARGOCD_E2E_COMMITSERVER_PORT:-8086}" ui: sh -c "test $ARGOCD_IN_CI = true && exit 0; cd ui && ARGOCD_E2E_YARN_HOST=0.0.0.0 ${ARGOCD_E2E_YARN_CMD:-yarn} start" reaper: ./test/container/reaper.sh sshd: sudo sh -c "test $ARGOCD_E2E_TEST = true && /usr/sbin/sshd -p 2222 -D -e" diff --git a/util/git/client.go b/util/git/client.go index 81bc87f081e43..6521f578d0d06 100644 --- a/util/git/client.go +++ b/util/git/client.go @@ -69,7 +69,7 @@ type Client interface { Init() error Fetch(revision string) error Submodule() error - Checkout(revision string, submoduleEnabled bool) error + Checkout(revision string, submoduleEnabled bool) (string, error) LsRefs() (*Refs, error) LsRemote(revision string) (string, error) LsFiles(path string, enableNewGitFileGlobbing bool) ([]string, error) @@ -80,11 +80,23 @@ type Client interface { IsAnnotatedTag(string) bool ChangedFiles(revision string, targetRevision string) ([]string, error) IsRevisionPresent(revision string) bool + // SetAuthor sets the author name and email in the git configuration. + SetAuthor(name, email string) (string, error) + // CheckoutOrOrphan checks out the branch. If the branch does not exist, it creates an orphan branch. + CheckoutOrOrphan(branch string, submoduleEnabled bool) (string, error) + // CheckoutOrNew checks out the given branch. If the branch does not exist, it creates an empty branch based on + // the base branch. + CheckoutOrNew(branch, base string, submoduleEnabled bool) (string, error) + // RemoveContents removes all files from the git repository. + RemoveContents() (string, error) + // CommitAndPush commits and pushes changes to the target branch. + CommitAndPush(branch, message string) (string, error) } type EventHandlers struct { OnLsRemote func(repo string) func() OnFetch func(repo string) func() + OnPush func(repo string) func() } // nativeGitClient implements Client interface using git CLI @@ -459,43 +471,43 @@ func (m *nativeGitClient) Submodule() error { return nil } -// Checkout checkout specified revision -func (m *nativeGitClient) Checkout(revision string, submoduleEnabled bool) error { +// Checkout checks out the specified revision +func (m *nativeGitClient) Checkout(revision string, submoduleEnabled bool) (string, error) { if revision == "" || revision == "HEAD" { revision = "origin/HEAD" } - if _, err := m.runCmd("checkout", "--force", revision); err != nil { - return err + if out, err := m.runCmd("checkout", "--force", revision); err != nil { + return out, fmt.Errorf("failed to checkout %s: %w", revision, err) } // We must populate LFS content by using lfs checkout, if we have at least // one LFS reference in the current revision. if m.IsLFSEnabled() { if largeFiles, err := m.LsLargeFiles(); err == nil { if len(largeFiles) > 0 { - if _, err := m.runCmd("lfs", "checkout"); err != nil { - return err + if out, err := m.runCmd("lfs", "checkout"); err != nil { + return out, fmt.Errorf("failed to checkout LFS files: %w", err) } } } else { - return err + return "", fmt.Errorf("failed to list LFS files: %w", err) } } if _, err := os.Stat(m.root + "/.gitmodules"); !os.IsNotExist(err) { if submoduleEnabled { if err := m.Submodule(); err != nil { - return err + return "", fmt.Errorf("failed to update submodules: %w", err) } } } // NOTE // The double “f” in the arguments is not a typo: the first “f” tells // `git clean` to delete untracked files and directories, and the second “f” - // tells it to clean untractked nested Git repositories (for example a + // tells it to clean untracked nested Git repositories (for example a // submodule which has since been removed). - if _, err := m.runCmd("clean", "-ffdx"); err != nil { - return err + if out, err := m.runCmd("clean", "-ffdx"); err != nil { + return out, fmt.Errorf("failed to clean: %w", err) } - return nil + return "", nil } func (m *nativeGitClient) getRefs() ([]*plumbing.Reference, error) { @@ -811,6 +823,123 @@ func (m *nativeGitClient) ChangedFiles(revision string, targetRevision string) ( return files, nil } +// config runs a git config command. +func (m *nativeGitClient) config(args ...string) (string, error) { + args = append([]string{"config"}, args...) + out, err := m.runCmd(args...) + if err != nil { + return out, fmt.Errorf("failed to run git config: %w", err) + } + return out, nil +} + +// SetAuthor sets the author name and email in the git configuration. +func (m *nativeGitClient) SetAuthor(name, email string) (string, error) { + if name != "" { + out, err := m.config("--local", "user.name", name) + if err != nil { + return out, err + } + } + if email != "" { + out, err := m.config("--local", "user.email", email) + if err != nil { + return out, err + } + } + return "", nil +} + +// CheckoutOrOrphan checks out the branch. If the branch does not exist, it creates an orphan branch. +func (m *nativeGitClient) CheckoutOrOrphan(branch string, submoduleEnabled bool) (string, error) { + out, err := m.Checkout(branch, submoduleEnabled) + if err != nil { + // If the branch doesn't exist, create it as an orphan branch. + if strings.Contains(err.Error(), "did not match any file(s) known to git") { + out, err = m.runCmd("switch", "--orphan", branch) + if err != nil { + return out, fmt.Errorf("failed to create orphan branch: %w", err) + } + } else { + return out, fmt.Errorf("failed to checkout branch: %w", err) + } + + // Make an empty initial commit. + out, err = m.runCmd("commit", "--allow-empty", "-m", "Initial commit") + if err != nil { + return out, fmt.Errorf("failed to commit initial commit: %w", err) + } + + // Push the commit. + err = m.runCredentialedCmd("push", "origin", branch) + if err != nil { + return "", fmt.Errorf("failed to push to branch: %w", err) + } + } + return "", nil +} + +// CheckoutOrNew checks out the given branch. If the branch does not exist, it creates an empty branch based on +// the base branch. +func (m *nativeGitClient) CheckoutOrNew(branch, base string, submoduleEnabled bool) (string, error) { + out, err := m.Checkout(branch, submoduleEnabled) + if err != nil { + if strings.Contains(err.Error(), "did not match any file(s) known to git") { + // If the branch does not exist, create any empty branch based on the sync branch + // First, checkout the sync branch. + out, err = m.Checkout(base, submoduleEnabled) + if err != nil { + return out, fmt.Errorf("failed to checkout sync branch: %w", err) + } + + out, err = m.runCmd("checkout", "-b", branch) + if err != nil { + return out, fmt.Errorf("failed to create branch: %w", err) + } + } else { + return out, fmt.Errorf("failed to checkout branch: %w", err) + } + } + return "", nil +} + +// RemoveContents removes all files from the git repository. +func (m *nativeGitClient) RemoveContents() (string, error) { + out, err := m.runCmd("rm", "-r", "--ignore-unmatch", ".") + if err != nil { + return out, fmt.Errorf("failed to clear repo contents: %w", err) + } + return "", nil +} + +// CommitAndPush commits and pushes changes to the target branch. +func (m *nativeGitClient) CommitAndPush(branch, message string) (string, error) { + out, err := m.runCmd("add", ".") + if err != nil { + return out, fmt.Errorf("failed to add files: %w", err) + } + + out, err = m.runCmd("commit", "-m", message) + if err != nil { + if strings.Contains(out, "nothing to commit, working tree clean") { + return out, nil + } + return out, fmt.Errorf("failed to commit: %w", err) + } + + if m.OnPush != nil { + done := m.OnPush(m.repoURL) + defer done() + } + + err = m.runCredentialedCmd("push", "origin", branch) + if err != nil { + return "", fmt.Errorf("failed to push: %w", err) + } + + return "", nil +} + // runWrapper runs a custom command with all the semantics of running the Git client func (m *nativeGitClient) runGnuPGWrapper(wrapper string, args ...string) (string, error) { cmd := exec.Command(wrapper, args...) @@ -850,7 +979,7 @@ func (m *nativeGitClient) runCredentialedCmd(args ...string) error { func (m *nativeGitClient) runCmdOutput(cmd *exec.Cmd, ropts runOpts) (string, error) { cmd.Dir = m.root cmd.Env = append(os.Environ(), cmd.Env...) - // Set $HOME to nowhere, so we can be execute Git regardless of any external + // Set $HOME to nowhere, so we can execute Git regardless of any external // authentication keys (e.g. in ~/.ssh) -- this is especially important for // running tests on local machines and/or CircleCI. cmd.Env = append(cmd.Env, "HOME=/dev/null") diff --git a/util/git/client_test.go b/util/git/client_test.go index a953e8bf214e1..a335d4a1bd6c8 100644 --- a/util/git/client_test.go +++ b/util/git/client_test.go @@ -6,6 +6,7 @@ import ( "os/exec" "path" "path/filepath" + "strings" "testing" "time" @@ -21,6 +22,13 @@ func runCmd(workingDir string, name string, args ...string) error { return cmd.Run() } +func outputCmd(workingDir string, name string, args ...string) ([]byte, error) { + cmd := exec.Command(name, args...) + cmd.Dir = workingDir + cmd.Stderr = os.Stderr + return cmd.Output() +} + func _createEmptyGitRepo() (string, error) { tempDir, err := os.MkdirTemp("", "") if err != nil { @@ -365,7 +373,7 @@ func Test_nativeGitClient_Submodule(t *testing.T) { require.NoError(t, err) // Call Checkout() with submoduleEnabled=false. - err = client.Checkout(commitSHA, false) + _, err = client.Checkout(commitSHA, false) require.NoError(t, err) // Check if submodule url does not exist in .git/config @@ -373,7 +381,7 @@ func Test_nativeGitClient_Submodule(t *testing.T) { require.Error(t, err) // Call Submodule() via Checkout() with submoduleEnabled=true. - err = client.Checkout(commitSHA, true) + _, err = client.Checkout(commitSHA, true) require.NoError(t, err) // Check if the .gitmodule URL is reflected in .git/config @@ -479,3 +487,353 @@ func Test_nativeGitClient_RevisionMetadata(t *testing.T) { Message: "| Initial commit |\n\n(╯°□°)╯︵ ┻━┻", }, metadata) } + +func Test_nativeGitClient_SetAuthor(t *testing.T) { + expectedName := "Tester" + expectedEmail := "test@example.com" + + tempDir, err := _createEmptyGitRepo() + require.NoError(t, err) + + client, err := NewClient(fmt.Sprintf("file://%s", tempDir), NopCreds{}, true, false, "", "") + require.NoError(t, err) + + err = client.Init() + require.NoError(t, err) + + out, err := client.SetAuthor(expectedName, expectedEmail) + require.NoError(t, err, "error output: ", out) + + // Check git user.name + gitUserName, err := outputCmd(client.Root(), "git", "config", "--local", "user.name") + require.NoError(t, err) + actualName := strings.TrimSpace(string(gitUserName)) + require.Equal(t, expectedName, actualName) + + // Check git user.email + gitUserEmail, err := outputCmd(client.Root(), "git", "config", "--local", "user.email") + require.NoError(t, err) + actualEmail := strings.TrimSpace(string(gitUserEmail)) + require.Equal(t, expectedEmail, actualEmail) +} + +func Test_nativeGitClient_CheckoutOrOrphan(t *testing.T) { + t.Run("checkout to an existing branch", func(t *testing.T) { + // not main or master + expectedBranch := "feature" + + tempDir, err := _createEmptyGitRepo() + require.NoError(t, err) + + client, err := NewClientExt(fmt.Sprintf("file://%s", tempDir), tempDir, NopCreds{}, true, false, "", "") + require.NoError(t, err) + + err = client.Init() + require.NoError(t, err) + + // set the author for the initial commit of the orphan branch + out, err := client.SetAuthor("test", "test@example.com") + require.NoError(t, err, "error output: %s", out) + + // get base branch + gitCurrentBranch, err := outputCmd(tempDir, "git", "rev-parse", "--abbrev-ref", "HEAD") + require.NoError(t, err) + baseBranch := strings.TrimSpace(string(gitCurrentBranch)) + + // get base commit + gitCurrentCommitHash, err := outputCmd(tempDir, "git", "rev-parse", "HEAD") + require.NoError(t, err) + expectedCommitHash := strings.TrimSpace(string(gitCurrentCommitHash)) + + // make expected branch + err = runCmd(tempDir, "git", "checkout", "-b", expectedBranch) + require.NoError(t, err) + + // checkout to base branch, ready to test + err = runCmd(tempDir, "git", "checkout", baseBranch) + require.NoError(t, err) + + out, err = client.CheckoutOrOrphan(expectedBranch, false) + require.NoError(t, err, "error output: ", out) + + // get current branch, verify current branch + gitCurrentBranch, err = outputCmd(tempDir, "git", "rev-parse", "--abbrev-ref", "HEAD") + require.NoError(t, err) + actualBranch := strings.TrimSpace(string(gitCurrentBranch)) + require.Equal(t, expectedBranch, actualBranch) + + // get current commit hash, verify current commit hash + // equal -> not orphan + gitCurrentCommitHash, err = outputCmd(tempDir, "git", "rev-parse", "HEAD") + require.NoError(t, err) + actualCommitHash := strings.TrimSpace(string(gitCurrentCommitHash)) + require.Equal(t, expectedCommitHash, actualCommitHash) + }) + + t.Run("orphan", func(t *testing.T) { + // not main or master + expectedBranch := "feature" + + // make origin git repository + tempDir, err := _createEmptyGitRepo() + require.NoError(t, err) + originGitRepoUrl := fmt.Sprintf("file://%s", tempDir) + err = runCmd(tempDir, "git", "commit", "-m", "Second commit", "--allow-empty") + require.NoError(t, err) + + // get base branch + gitCurrentBranch, err := outputCmd(tempDir, "git", "rev-parse", "--abbrev-ref", "HEAD") + require.NoError(t, err) + baseBranch := strings.TrimSpace(string(gitCurrentBranch)) + + // make test dir + tempDir, err = os.MkdirTemp("", "") + require.NoError(t, err) + + client, err := NewClientExt(originGitRepoUrl, tempDir, NopCreds{}, true, false, "", "") + require.NoError(t, err) + + err = client.Init() + require.NoError(t, err) + + // set the author for the initial commit of the orphan branch + out, err := client.SetAuthor("test", "test@example.com") + require.NoError(t, err, "error output: %s", out) + + err = client.Fetch("") + require.NoError(t, err) + + // checkout to origin base branch + err = runCmd(tempDir, "git", "checkout", baseBranch) + require.NoError(t, err) + + // get base commit + gitCurrentCommitHash, err := outputCmd(tempDir, "git", "rev-parse", "HEAD") + require.NoError(t, err) + baseCommitHash := strings.TrimSpace(string(gitCurrentCommitHash)) + + out, err = client.CheckoutOrOrphan(expectedBranch, false) + require.NoError(t, err, "error output: ", out) + + // get current branch, verify current branch + gitCurrentBranch, err = outputCmd(tempDir, "git", "rev-parse", "--abbrev-ref", "HEAD") + require.NoError(t, err) + actualBranch := strings.TrimSpace(string(gitCurrentBranch)) + require.Equal(t, expectedBranch, actualBranch) + + // check orphan branch + + // get current commit hash, verify current commit hash + // not equal -> orphan + gitCurrentCommitHash, err = outputCmd(tempDir, "git", "rev-parse", "HEAD") + require.NoError(t, err) + currentCommitHash := strings.TrimSpace(string(gitCurrentCommitHash)) + require.NotEqual(t, baseCommitHash, currentCommitHash) + + // get commit count on current branch, verify 1 -> orphan + gitCommitCount, err := outputCmd(tempDir, "git", "rev-list", "--count", actualBranch) + require.NoError(t, err) + require.Equal(t, "1", strings.TrimSpace(string(gitCommitCount))) + }) +} + +func Test_nativeGitClient_CheckoutOrNew(t *testing.T) { + t.Run("checkout to an existing branch", func(t *testing.T) { + // Example status + // * 57aef63 (feature) Second commit + // * a4fad22 (main) Initial commit + + // Test scenario + // given : main branch (w/ Initial commit) + // when : try to check out [main -> feature] + // then : feature branch (w/ Second commit) + + // not main or master + expectedBranch := "feature" + + tempDir, err := _createEmptyGitRepo() + require.NoError(t, err) + + client, err := NewClientExt(fmt.Sprintf("file://%s", tempDir), tempDir, NopCreds{}, true, false, "", "") + require.NoError(t, err) + + err = client.Init() + require.NoError(t, err) + + out, err := client.SetAuthor("test", "test@example.com") + require.NoError(t, err, "error output: %s", out) + + // get base branch + gitCurrentBranch, err := outputCmd(tempDir, "git", "rev-parse", "--abbrev-ref", "HEAD") + require.NoError(t, err) + baseBranch := strings.TrimSpace(string(gitCurrentBranch)) + + // make expected branch + err = runCmd(tempDir, "git", "checkout", "-b", expectedBranch) + require.NoError(t, err) + + // make expected commit + err = runCmd(tempDir, "git", "commit", "-m", "Second commit", "--allow-empty") + require.NoError(t, err) + + // get expected commit + expectedCommitHash, err := client.CommitSHA() + require.NoError(t, err) + + // checkout to base branch, ready to test + err = runCmd(tempDir, "git", "checkout", baseBranch) + require.NoError(t, err) + + out, err = client.CheckoutOrNew(expectedBranch, baseBranch, false) + require.NoError(t, err, "error output: ", out) + + // get current branch, verify current branch + gitCurrentBranch, err = outputCmd(tempDir, "git", "rev-parse", "--abbrev-ref", "HEAD") + require.NoError(t, err) + actualBranch := strings.TrimSpace(string(gitCurrentBranch)) + require.Equal(t, expectedBranch, actualBranch) + + // get current commit hash, verify current commit hash + actualCommitHash, err := client.CommitSHA() + require.NoError(t, err) + require.Equal(t, expectedCommitHash, actualCommitHash) + }) + + t.Run("new", func(t *testing.T) { + // Test scenario + // given : main branch (w/ Initial commit) + // * a4fad22 (main) Initial commit + // when : try to check out [main -> feature] + // then : feature branch (w/ Initial commit) + // * a4fad22 (feature, main) Initial commit + + // not main or master + expectedBranch := "feature" + + tempDir, err := _createEmptyGitRepo() + require.NoError(t, err) + + client, err := NewClientExt(fmt.Sprintf("file://%s", tempDir), tempDir, NopCreds{}, true, false, "", "") + require.NoError(t, err) + + err = client.Init() + require.NoError(t, err) + + out, err := client.SetAuthor("test", "test@example.com") + require.NoError(t, err, "error output: %s", out) + + // get base branch + gitCurrentBranch, err := outputCmd(tempDir, "git", "rev-parse", "--abbrev-ref", "HEAD") + require.NoError(t, err) + baseBranch := strings.TrimSpace(string(gitCurrentBranch)) + + // get expected commit + expectedCommitHash, err := client.CommitSHA() + require.NoError(t, err) + + out, err = client.CheckoutOrNew(expectedBranch, baseBranch, false) + require.NoError(t, err, "error output: ", out) + + // get current branch, verify current branch + gitCurrentBranch, err = outputCmd(tempDir, "git", "rev-parse", "--abbrev-ref", "HEAD") + require.NoError(t, err) + actualBranch := strings.TrimSpace(string(gitCurrentBranch)) + require.Equal(t, expectedBranch, actualBranch) + + // get current commit hash, verify current commit hash + actualCommitHash, err := client.CommitSHA() + require.NoError(t, err) + require.Equal(t, expectedCommitHash, actualCommitHash) + }) +} + +func Test_nativeGitClient_RemoveContents(t *testing.T) { + // Example status + // 2 files : + // * /README.md + // * /scripts/startup.sh + + // given + tempDir, err := _createEmptyGitRepo() + require.NoError(t, err) + + client, err := NewClient(fmt.Sprintf("file://%s", tempDir), NopCreds{}, true, false, "", "") + require.NoError(t, err) + + err = client.Init() + require.NoError(t, err) + + out, err := client.SetAuthor("test", "test@example.com") + require.NoError(t, err, "error output: ", out) + + err = runCmd(client.Root(), "touch", "README.md") + require.NoError(t, err) + + err = runCmd(client.Root(), "mkdir", "scripts") + require.NoError(t, err) + + err = runCmd(client.Root(), "touch", "scripts/startup.sh") + require.NoError(t, err) + + err = runCmd(client.Root(), "git", "add", "--all") + require.NoError(t, err) + + err = runCmd(client.Root(), "git", "commit", "-m", "Make files") + require.NoError(t, err) + + // when + out, err = client.RemoveContents() + require.NoError(t, err, "error output: ", out) + + // then + ls, err := outputCmd(client.Root(), "ls", "-l") + require.NoError(t, err) + require.Equal(t, "total 0", strings.TrimSpace(string(ls))) +} + +func Test_nativeGitClient_CommitAndPush(t *testing.T) { + tempDir, err := _createEmptyGitRepo() + require.NoError(t, err) + + // config receive.denyCurrentBranch updateInstead + // because local git init make a non-bare repository which cannot be pushed normally + err = runCmd(tempDir, "git", "config", "--local", "receive.denyCurrentBranch", "updateInstead") + require.NoError(t, err) + + // get branch + gitCurrentBranch, err := outputCmd(tempDir, "git", "rev-parse", "--abbrev-ref", "HEAD") + require.NoError(t, err) + branch := strings.TrimSpace(string(gitCurrentBranch)) + + client, err := NewClient(fmt.Sprintf("file://%s", tempDir), NopCreds{}, true, false, "", "") + require.NoError(t, err) + + err = client.Init() + require.NoError(t, err) + + out, err := client.SetAuthor("test", "test@example.com") + require.NoError(t, err, "error output: ", out) + + err = client.Fetch(branch) + require.NoError(t, err) + + out, err = client.Checkout(branch, false) + require.NoError(t, err, "error output: ", out) + + // make a file then commit and push + err = runCmd(client.Root(), "touch", "README.md") + require.NoError(t, err) + + out, err = client.CommitAndPush(branch, "docs: README") + require.NoError(t, err, "error output: %s", out) + + // get current commit hash of the cloned repository + expectedCommitHash, err := client.CommitSHA() + require.NoError(t, err) + + // get origin repository's current commit hash + gitCurrentCommitHash, err := outputCmd(tempDir, "git", "rev-parse", "HEAD") + require.NoError(t, err) + actualCommitHash := strings.TrimSpace(string(gitCurrentCommitHash)) + require.Equal(t, expectedCommitHash, actualCommitHash) +} diff --git a/util/git/creds.go b/util/git/creds.go index 5715925dcef9e..e9611785913e0 100644 --- a/util/git/creds.go +++ b/util/git/creds.go @@ -8,12 +8,15 @@ import ( "errors" "fmt" "io" + "net/http" "net/url" "os" "strconv" "strings" "time" + "github.com/google/go-github/v62/github" + "golang.org/x/oauth2" "golang.org/x/oauth2/google" @@ -77,6 +80,8 @@ type CredsStore interface { type Creds interface { Environ() (io.Closer, []string, error) + // GetUserInfo gets the username and email address for the credentials, if they're available. + GetUserInfo(ctx context.Context) (string, string, error) } // nop implementation @@ -94,16 +99,24 @@ func (c NopCreds) Environ() (io.Closer, []string, error) { return NopCloser{}, nil, nil } +// GetUserInfo returns empty strings for user info +func (c NopCreds) GetUserInfo(ctx context.Context) (name string, email string, err error) { + return "", "", nil +} + var _ io.Closer = NopCloser{} type GenericHTTPSCreds interface { HasClientCert() bool GetClientCertData() string GetClientCertKey() string - Environ() (io.Closer, []string, error) + Creds } -var _ GenericHTTPSCreds = HTTPSCreds{} +var ( + _ GenericHTTPSCreds = HTTPSCreds{} + _ Creds = HTTPSCreds{} +) // HTTPS creds implementation type HTTPSCreds struct { @@ -141,6 +154,12 @@ func NewHTTPSCreds(username string, password string, clientCertData string, clie } } +// GetUserInfo returns the username and email address for the credentials, if they're available. +func (c HTTPSCreds) GetUserInfo(ctx context.Context) (string, string, error) { + // Email not implemented for HTTPS creds. + return c.username, "", nil +} + func (c HTTPSCreds) BasicAuthHeader() string { h := "Authorization: Basic " t := c.username + ":" + c.password @@ -231,6 +250,8 @@ func (c HTTPSCreds) GetClientCertKey() string { return c.clientCertKey } +var _ Creds = SSHCreds{} + // SSH implementation type SSHCreds struct { sshPrivateKey string @@ -245,6 +266,13 @@ func NewSSHCreds(sshPrivateKey string, caPath string, insecureIgnoreHostKey bool return SSHCreds{sshPrivateKey, caPath, insecureIgnoreHostKey, store, proxy, noProxy} } +// GetUserInfo returns empty strings for user info. +// TODO: Implement this method to return the username and email address for the credentials, if they're available. +func (c SSHCreds) GetUserInfo(ctx context.Context) (string, string, error) { + // User info not implemented for SSH creds. + return "", "", nil +} + type sshPrivateKeyFile string type authFilePaths []string @@ -414,6 +442,37 @@ func (g GitHubAppCreds) Environ() (io.Closer, []string, error) { }), env, nil } +// GetUserInfo returns the username and email address for the credentials, if they're available. +func (g GitHubAppCreds) GetUserInfo(ctx context.Context) (string, string, error) { + // We use the apps transport to get the app slug. + appTransport, err := g.getAppTransport() + if err != nil { + return "", "", fmt.Errorf("failed to create GitHub app transport: %w", err) + } + appClient := github.NewClient(&http.Client{Transport: appTransport}) + app, _, err := appClient.Apps.Get(ctx, "") + if err != nil { + return "", "", fmt.Errorf("failed to get app info: %w", err) + } + + // Then we use the installation transport to get the installation info. + appInstallTransport, err := g.getInstallationTransport() + if err != nil { + return "", "", fmt.Errorf("failed to get app installation: %w", err) + } + httpClient := http.Client{Transport: appInstallTransport} + client := github.NewClient(&httpClient) + + appLogin := fmt.Sprintf("%s[bot]", app.GetSlug()) + user, _, err := client.Users.Get(ctx, appLogin) + if err != nil { + return "", "", fmt.Errorf("failed to get app user info: %w", err) + } + authorName := user.GetLogin() + authorEmail := fmt.Sprintf("%d+%s@users.noreply.github.com", user.GetID(), user.GetLogin()) + return authorName, authorEmail, nil +} + // getAccessToken fetches GitHub token using the app id, install id, and private key. // the token is then cached for re-use. func (g GitHubAppCreds) getAccessToken() (string, error) { @@ -421,11 +480,44 @@ func (g GitHubAppCreds) getAccessToken() (string, error) { ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() + itr, err := g.getInstallationTransport() + if err != nil { + return "", fmt.Errorf("failed to create GitHub app installation transport: %w", err) + } + + return itr.Token(ctx) +} + +// getAppTransport creates a new GitHub transport for the app +func (g GitHubAppCreds) getAppTransport() (*ghinstallation.AppsTransport, error) { + // GitHub API url + baseUrl := "https://api.github.com" + if g.baseURL != "" { + baseUrl = strings.TrimSuffix(g.baseURL, "/") + } + + // Create a new GitHub transport + c := GetRepoHTTPClient(baseUrl, g.insecure, g, g.proxy, g.noProxy) + itr, err := ghinstallation.NewAppsTransport(c.Transport, + g.appID, + []byte(g.privateKey), + ) + if err != nil { + return nil, fmt.Errorf("failed to initialize GitHub installation transport: %w", err) + } + + itr.BaseURL = baseUrl + + return itr, nil +} + +// getInstallationTransport creates a new GitHub transport for the app installation +func (g GitHubAppCreds) getInstallationTransport() (*ghinstallation.Transport, error) { // Compute hash of creds for lookup in cache h := sha256.New() _, err := h.Write([]byte(fmt.Sprintf("%s %d %d %s", g.privateKey, g.appID, g.appInstallId, g.baseURL))) if err != nil { - return "", err + return nil, fmt.Errorf("failed to get get SHA256 hash for GitHub app credentials: %w", err) } key := fmt.Sprintf("%x", h.Sum(nil)) @@ -434,7 +526,7 @@ func (g GitHubAppCreds) getAccessToken() (string, error) { if found { itr := t.(*ghinstallation.Transport) // This method caches the token and if it's expired retrieves a new one - return itr.Token(ctx) + return itr, nil } // GitHub API url @@ -451,7 +543,7 @@ func (g GitHubAppCreds) getAccessToken() (string, error) { []byte(g.privateKey), ) if err != nil { - return "", err + return nil, fmt.Errorf("failed to initialize GitHub installation transport: %w", err) } itr.BaseURL = baseUrl @@ -459,7 +551,7 @@ func (g GitHubAppCreds) getAccessToken() (string, error) { // Add transport to cache githubAppTokenCache.Set(key, itr, time.Minute*60) - return itr.Token(ctx) + return itr, nil } func (g GitHubAppCreds) HasClientCert() bool { @@ -474,6 +566,8 @@ func (g GitHubAppCreds) GetClientCertKey() string { return g.clientCertKey } +var _ Creds = GoogleCloudCreds{} + // GoogleCloudCreds to authenticate to Google Cloud Source repositories type GoogleCloudCreds struct { creds *google.Credentials @@ -489,6 +583,16 @@ func NewGoogleCloudCreds(jsonData string, store CredsStore) GoogleCloudCreds { return GoogleCloudCreds{creds, store} } +// GetUserInfo returns the username and email address for the credentials, if they're available. +// TODO: implement getting email instead of just username. +func (c GoogleCloudCreds) GetUserInfo(ctx context.Context) (string, string, error) { + username, err := c.getUsername() + if err != nil { + return "", "", fmt.Errorf("failed to get username from creds: %w", err) + } + return username, "", nil +} + func (c GoogleCloudCreds) Environ() (io.Closer, []string, error) { username, err := c.getUsername() if err != nil { diff --git a/util/git/git_test.go b/util/git/git_test.go index d507c728c0fde..f06df14c070ea 100644 --- a/util/git/git_test.go +++ b/util/git/git_test.go @@ -320,7 +320,7 @@ func TestLFSClient(t *testing.T) { err = client.Fetch("") require.NoError(t, err) - err = client.Checkout(commitSHA, true) + _, err = client.Checkout(commitSHA, true) require.NoError(t, err) largeFiles, err := client.LsLargeFiles() @@ -358,7 +358,7 @@ func TestVerifyCommitSignature(t *testing.T) { commitSHA, err := client.LsRemote("HEAD") require.NoError(t, err) - err = client.Checkout(commitSHA, true) + _, err = client.Checkout(commitSHA, true) require.NoError(t, err) // 28027897aad1262662096745f2ce2d4c74d02b7f is a commit that is signed in the repo @@ -415,7 +415,7 @@ func TestNewFactory(t *testing.T) { err = client.Fetch("") require.NoError(t, err) - err = client.Checkout(commitSHA, true) + _, err = client.Checkout(commitSHA, true) require.NoError(t, err) revisionMetadata, err := client.RevisionMetadata(commitSHA) diff --git a/util/git/mocks/Client.go b/util/git/mocks/Client.go index 490aa4e99b90d..9357264e3bdd6 100644 --- a/util/git/mocks/Client.go +++ b/util/git/mocks/Client.go @@ -43,21 +43,115 @@ func (_m *Client) ChangedFiles(revision string, targetRevision string) ([]string } // Checkout provides a mock function with given fields: revision, submoduleEnabled -func (_m *Client) Checkout(revision string, submoduleEnabled bool) error { +func (_m *Client) Checkout(revision string, submoduleEnabled bool) (string, error) { ret := _m.Called(revision, submoduleEnabled) if len(ret) == 0 { panic("no return value specified for Checkout") } - var r0 error - if rf, ok := ret.Get(0).(func(string, bool) error); ok { + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(string, bool) (string, error)); ok { + return rf(revision, submoduleEnabled) + } + if rf, ok := ret.Get(0).(func(string, bool) string); ok { r0 = rf(revision, submoduleEnabled) } else { - r0 = ret.Error(0) + r0 = ret.Get(0).(string) } - return r0 + if rf, ok := ret.Get(1).(func(string, bool) error); ok { + r1 = rf(revision, submoduleEnabled) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// CheckoutOrNew provides a mock function with given fields: branch, base, submoduleEnabled +func (_m *Client) CheckoutOrNew(branch string, base string, submoduleEnabled bool) (string, error) { + ret := _m.Called(branch, base, submoduleEnabled) + + if len(ret) == 0 { + panic("no return value specified for CheckoutOrNew") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(string, string, bool) (string, error)); ok { + return rf(branch, base, submoduleEnabled) + } + if rf, ok := ret.Get(0).(func(string, string, bool) string); ok { + r0 = rf(branch, base, submoduleEnabled) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(string, string, bool) error); ok { + r1 = rf(branch, base, submoduleEnabled) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// CheckoutOrOrphan provides a mock function with given fields: branch, submoduleEnabled +func (_m *Client) CheckoutOrOrphan(branch string, submoduleEnabled bool) (string, error) { + ret := _m.Called(branch, submoduleEnabled) + + if len(ret) == 0 { + panic("no return value specified for CheckoutOrOrphan") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(string, bool) (string, error)); ok { + return rf(branch, submoduleEnabled) + } + if rf, ok := ret.Get(0).(func(string, bool) string); ok { + r0 = rf(branch, submoduleEnabled) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(string, bool) error); ok { + r1 = rf(branch, submoduleEnabled) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// CommitAndPush provides a mock function with given fields: branch, message +func (_m *Client) CommitAndPush(branch string, message string) (string, error) { + ret := _m.Called(branch, message) + + if len(ret) == 0 { + panic("no return value specified for CommitAndPush") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(string, string) (string, error)); ok { + return rf(branch, message) + } + if rf, ok := ret.Get(0).(func(string, string) string); ok { + r0 = rf(branch, message) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(string, string) error); ok { + r1 = rf(branch, message) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } // CommitSHA provides a mock function with given fields: @@ -278,6 +372,34 @@ func (_m *Client) LsRemote(revision string) (string, error) { return r0, r1 } +// RemoveContents provides a mock function with given fields: +func (_m *Client) RemoveContents() (string, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for RemoveContents") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func() (string, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // RevisionMetadata provides a mock function with given fields: revision func (_m *Client) RevisionMetadata(revision string) (*git.RevisionMetadata, error) { ret := _m.Called(revision) @@ -326,6 +448,34 @@ func (_m *Client) Root() string { return r0 } +// SetAuthor provides a mock function with given fields: name, email +func (_m *Client) SetAuthor(name string, email string) (string, error) { + ret := _m.Called(name, email) + + if len(ret) == 0 { + panic("no return value specified for SetAuthor") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(string, string) (string, error)); ok { + return rf(name, email) + } + if rf, ok := ret.Get(0).(func(string, string) string); ok { + r0 = rf(name, email) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(string, string) error); ok { + r1 = rf(name, email) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Submodule provides a mock function with given fields: func (_m *Client) Submodule() error { ret := _m.Called() diff --git a/util/io/files/secure_mkdir_default.go b/util/io/files/secure_mkdir_default.go new file mode 100644 index 0000000000000..fe7733e2d071f --- /dev/null +++ b/util/io/files/secure_mkdir_default.go @@ -0,0 +1,25 @@ +//go:build !linux + +package files + +import ( + "fmt" + "os" + + securejoin "github.com/cyphar/filepath-securejoin" +) + +// SecureMkdirAll creates a directory with the given mode and returns the full path to the directory. It prevents +// directory traversal attacks by ensuring the path is within the root directory. The path is constructed as if the +// given root is the root of the filesystem. So anything traversing outside the root is simply removed from the path. +func SecureMkdirAll(root, unsafePath string, mode os.FileMode) (string, error) { + fullPath, err := securejoin.SecureJoin(root, unsafePath) + if err != nil { + return "", fmt.Errorf("failed to construct secure path: %w", err) + } + err = os.MkdirAll(fullPath, mode) + if err != nil { + return "", fmt.Errorf("failed to create directory: %w", err) + } + return fullPath, nil +} diff --git a/util/io/files/secure_mkdir_linux.go b/util/io/files/secure_mkdir_linux.go new file mode 100644 index 0000000000000..14f727dda480d --- /dev/null +++ b/util/io/files/secure_mkdir_linux.go @@ -0,0 +1,25 @@ +//go:build linux + +package files + +import ( + "fmt" + "os" + + securejoin "github.com/cyphar/filepath-securejoin" +) + +// SecureMkdirAll creates a directory with the given mode and returns the full path to the directory. It prevents +// directory traversal attacks by ensuring the path is within the root directory. The path is constructed as if the +// given root is the root of the filesystem. So anything traversing outside the root is simply removed from the path. +func SecureMkdirAll(root, unsafePath string, mode os.FileMode) (string, error) { + err := securejoin.MkdirAll(root, unsafePath, int(mode)) + if err != nil { + return "", fmt.Errorf("failed to make directory: %w", err) + } + fullPath, err := securejoin.SecureJoin(root, unsafePath) + if err != nil { + return "", fmt.Errorf("failed to construct secure path: %w", err) + } + return fullPath, nil +} diff --git a/util/io/files/secure_mkdir_test.go b/util/io/files/secure_mkdir_test.go new file mode 100644 index 0000000000000..e696629235404 --- /dev/null +++ b/util/io/files/secure_mkdir_test.go @@ -0,0 +1,67 @@ +package files + +import ( + "os" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSecureMkdirAllDefault(t *testing.T) { + root := t.TempDir() + + unsafePath := "test/dir" + fullPath, err := SecureMkdirAll(root, unsafePath, os.ModePerm) + require.NoError(t, err) + + expectedPath := path.Join(root, unsafePath) + assert.Equal(t, expectedPath, fullPath) +} + +func TestSecureMkdirAllWithExistingDir(t *testing.T) { + root := t.TempDir() + unsafePath := "existing/dir" + + fullPath, err := SecureMkdirAll(root, unsafePath, os.ModePerm) + require.NoError(t, err) + + newPath, err := SecureMkdirAll(root, unsafePath, os.ModePerm) + require.NoError(t, err) + assert.Equal(t, fullPath, newPath) +} + +func TestSecureMkdirAllWithFile(t *testing.T) { + root := t.TempDir() + unsafePath := "file.txt" + + filePath := filepath.Join(root, unsafePath) + err := os.WriteFile(filePath, []byte("test"), os.ModePerm) + require.NoError(t, err) + + // Should fail because there is a file at the path + _, err = SecureMkdirAll(root, unsafePath, os.ModePerm) + require.Error(t, err) +} + +func TestSecureMkdirAllDotDotPath(t *testing.T) { + root := t.TempDir() + unsafePath := "../outside" + + fullPath, err := SecureMkdirAll(root, unsafePath, os.ModePerm) + require.NoError(t, err) + + expectedPath := filepath.Join(root, "outside") + assert.Equal(t, expectedPath, fullPath) + + info, err := os.Stat(fullPath) + require.NoError(t, err) + assert.True(t, info.IsDir()) + + relPath, err := filepath.Rel(root, fullPath) + require.NoError(t, err) + assert.False(t, strings.HasPrefix(relPath, "..")) +}