From 2ca36a3cdf29aae45fec0eeb2515e213b71bc025 Mon Sep 17 00:00:00 2001 From: David Archer Date: Tue, 25 Jun 2024 14:21:38 -0400 Subject: [PATCH 01/10] Support dynamic pipeline generation in-process --- go.mod | 16 +- go.sum | 17 ++ pipeline.go | 103 ++++----- pipeline_test.go | 558 +++++++++++++++++++++++++---------------------- plugin.go | 263 +--------------------- plugin.yml | 80 +------ plugin_test.go | 220 ++----------------- 7 files changed, 382 insertions(+), 875 deletions(-) diff --git a/go.mod b/go.mod index 7a86b58..39c68d1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/buildkite-plugins/monorepo-diff-buildkite-plugin -go 1.19 +go 1.22 + +toolchain go1.22.4 require ( github.com/bmatcuk/doublestar/v2 v2.0.4 @@ -10,8 +12,12 @@ require ( ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + github.com/buildkite/go-pipeline v0.9.0 // indirect + github.com/buildkite/interpolate v0.1.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/oleiade/reflections v1.0.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + golang.org/x/sys v0.18.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d784ee9..50fea0e 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,34 @@ github.com/bmatcuk/doublestar/v2 v2.0.4 h1:6I6oUiT/sU27eE2OFcWqBhL1SwjyvQuOssxT4a1yidI= github.com/bmatcuk/doublestar/v2 v2.0.4/go.mod h1:QMmcs3H2AUQICWhfzLXz+IYln8lRQmTZRptLie8RgRw= +github.com/buildkite/go-pipeline v0.9.0 h1:2a2bibJ9dCCyyNReH73jkQVUYyUnhYAxISyf3+mrQNs= +github.com/buildkite/go-pipeline v0.9.0/go.mod h1:4aqMzJ3iagc0wcI5h8NQpON9xfyq27QGDi4xfnKiCUs= +github.com/buildkite/interpolate v0.1.2 h1:mVbMCphpu2MHUr1qLdjq9xc3NjNWYg/w/CbrGS5ckzg= +github.com/buildkite/interpolate v0.1.2/go.mod h1:UNVe6A+UfiBNKbhAySrBbZFZFxQ+DXr9nWen6WVt/A8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/oleiade/reflections v1.0.1 h1:D1XO3LVEYroYskEsoSiGItp9RUxG6jWnCVvrqH0HHQM= +github.com/oleiade/reflections v1.0.1/go.mod h1:rdFxbxq4QXVZWj0F+e9jqjDkc7dbp97vkRixKo2JR60= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -21,3 +36,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pipeline.go b/pipeline.go index 6b2b3e9..ad42b60 100644 --- a/pipeline.go +++ b/pipeline.go @@ -8,39 +8,13 @@ import ( "strings" "github.com/bmatcuk/doublestar/v2" + "github.com/buildkite/go-pipeline" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) -// WaitStep represents a Buildkite Wait Step -// https://buildkite.com/docs/pipelines/wait-step -// We can't use Step here since the value for Wait is always nil -// regardless of whether or not we want to include the key. -type WaitStep struct{} - -func (WaitStep) MarshalYAML() (interface{}, error) { - return map[string]interface{}{ - "wait": nil, - }, nil -} - -func (s Step) MarshalYAML() (interface{}, error) { - if s.Group == "" { - type Alias Step - return (Alias)(s), nil - } - - label := s.Group - s.Group = "" - return Group{Label: label, Steps: []Step{s}}, nil -} - -func (n PluginNotify) MarshalYAML() (interface{}, error) { - return n, nil -} - // PipelineGenerator generates pipeline file -type PipelineGenerator func(steps []Step, plugin Plugin) (*os.File, error) +type PipelineGenerator func(steps []pipeline.Step, plugin Plugin) (*os.File, bool, error) func uploadPipeline(plugin Plugin, generatePipeline PipelineGenerator) (string, []string, error) { diffOutput, err := diff(plugin.Diff) @@ -61,21 +35,22 @@ func uploadPipeline(plugin Plugin, generatePipeline PipelineGenerator) (string, return "", []string{}, err } - pipeline, err := generatePipeline(steps, plugin) - defer os.Remove(pipeline.Name()) + p, hasSteps, err := generatePipeline(steps, plugin) + defer os.Remove(p.Name()) if err != nil { log.Error(err) return "", []string{}, err } - cmd := "buildkite-agent" - args := []string{"pipeline", "upload", pipeline.Name()} - - if !plugin.Interpolation { - args = append(args, "--no-interpolation") + if !hasSteps { + // Handle the case where no steps were provided + log.Info("No steps generated. Skipping pipeline upload.") + return "", []string{}, nil } + cmd := "buildkite-agent" + args := []string{"pipeline", "upload", p.Name()} _, err = executeCommand("buildkite-agent", args) return cmd, args, err @@ -96,8 +71,8 @@ func diff(command string) ([]string, error) { return strings.Fields(strings.TrimSpace(output)), nil } -func stepsToTrigger(files []string, watch []WatchConfig) ([]Step, error) { - steps := []Step{} +func stepsToTrigger(files []string, watch []WatchConfig) ([]pipeline.Step, error) { + steps := []pipeline.Step{} for _, w := range watch { for _, p := range w.Paths { @@ -107,7 +82,20 @@ func stepsToTrigger(files []string, watch []WatchConfig) ([]Step, error) { return nil, err } if match { - steps = append(steps, w.Step) + out, err := executeCommand("bash", []string{"-c", w.Generator}) + if err != nil { + return nil, err + } + + p, err := pipeline.Parse(strings.NewReader(out)) + if err != nil { + return nil, err + } + + for _, step := range p.Steps { + steps = append(steps, step) + } + break } } @@ -137,8 +125,8 @@ func matchPath(p string, f string) (bool, error) { return false, nil } -func dedupSteps(steps []Step) []Step { - unique := []Step{} +func dedupSteps(steps []pipeline.Step) []pipeline.Step { + unique := []pipeline.Step{} for _, p := range steps { duplicate := false for _, t := range unique { @@ -156,42 +144,25 @@ func dedupSteps(steps []Step) []Step { return unique } -func generatePipeline(steps []Step, plugin Plugin) (*os.File, error) { +func generatePipeline(steps []pipeline.Step, plugin Plugin) (*os.File, bool, error) { tmp, err := ioutil.TempFile(os.TempDir(), "bmrd-") if err != nil { - return nil, fmt.Errorf("could not create temporary pipeline file: %v", err) - } - - yamlSteps := make([]yaml.Marshaler, len(steps)) - - for i, step := range steps { - yamlSteps[i] = step - } - - if plugin.Wait { - yamlSteps = append(yamlSteps, WaitStep{}) + return nil, false, fmt.Errorf("could not create temporary pipeline file: %v", err) } - for _, cmd := range plugin.Hooks { - yamlSteps = append(yamlSteps, Step{Command: cmd.Command}) - } + yamlSteps := make([]interface{}, 0) - yamlNotify := make([]yaml.Marshaler, len(plugin.Notify)) - for i, n := range plugin.Notify { - yamlNotify[i] = n + for _, step := range steps { + yamlSteps = append(yamlSteps, step) } - pipeline := map[string][]yaml.Marshaler{ + pipeline := map[string]interface{}{ "steps": yamlSteps, } - if len(yamlNotify) > 0 { - pipeline["notify"] = yamlNotify - } - data, err := yaml.Marshal(&pipeline) if err != nil { - return nil, fmt.Errorf("could not serialize the pipeline: %v", err) + return nil, false, fmt.Errorf("could not serialize the pipeline: %v", err) } // Disable logging in context of go tests. @@ -200,8 +171,8 @@ func generatePipeline(steps []Step, plugin Plugin) (*os.File, error) { } if err = ioutil.WriteFile(tmp.Name(), data, 0644); err != nil { - return nil, fmt.Errorf("could not write step to temporary file: %v", err) + return nil, false, fmt.Errorf("could not write step to temporary file: %v", err) } - return tmp, nil + return tmp, len(yamlSteps) > 0, nil } diff --git a/pipeline_test.go b/pipeline_test.go index 83d6043..7f0a4c7 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -1,23 +1,23 @@ package main import ( - "io/ioutil" "os" "testing" + "github.com/buildkite/go-pipeline" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" ) -func mockGeneratePipeline(steps []Step, plugin Plugin) (*os.File, error) { +func mockGeneratePipeline(steps []pipeline.Step, plugin Plugin) (*os.File, bool, error) { mockFile, _ := os.Create("pipeline.txt") defer mockFile.Close() - return mockFile, nil + return mockFile, true, nil } func TestUploadPipelineCallsBuildkiteAgentCommand(t *testing.T) { - plugin := Plugin{Diff: "echo ./foo-service", Interpolation: true} + plugin := Plugin{Diff: "echo ./foo-service"} cmd, args, err := uploadPipeline(plugin, mockGeneratePipeline) assert.Equal(t, "buildkite-agent", cmd) @@ -25,15 +25,6 @@ func TestUploadPipelineCallsBuildkiteAgentCommand(t *testing.T) { assert.Equal(t, err.Error(), "command `buildkite-agent` failed: exec: \"buildkite-agent\": executable file not found in $PATH") } -func TestUploadPipelineCallsBuildkiteAgentCommandWithInterpolation(t *testing.T) { - plugin := Plugin{Diff: "echo ./foo-service", Interpolation: false} - cmd, args, err := uploadPipeline(plugin, mockGeneratePipeline) - - assert.Equal(t, "buildkite-agent", cmd) - assert.Equal(t, []string{"pipeline", "upload", "pipeline.txt", "--no-interpolation"}, args) - assert.Equal(t, err.Error(), "command `buildkite-agent` failed: exec: \"buildkite-agent\": executable file not found in $PATH") -} - func TestUploadPipelineCancelsIfThereIsNoDiffOutput(t *testing.T) { plugin := Plugin{Diff: "echo"} cmd, args, err := uploadPipeline(plugin, mockGeneratePipeline) @@ -71,278 +62,319 @@ func TestDiffWithSubshell(t *testing.T) { assert.Equal(t, want, got) } -func TestPipelinesToTriggerGetsListOfPipelines(t *testing.T) { - want := []string{"service-1", "service-2", "service-4"} - - watch := []WatchConfig{ - { - Paths: []string{"watch-path-1"}, - Step: Step{Trigger: "service-1"}, - }, - { - Paths: []string{"watch-path-2/", "watch-path-3/", "watch-path-4"}, - Step: Step{Trigger: "service-2"}, - }, - { - Paths: []string{"watch-path-5"}, - Step: Step{Trigger: "service-3"}, - }, - { - Paths: []string{"watch-path-2"}, - Step: Step{Trigger: "service-4"}, - }, - } - - changedFiles := []string{ - "watch-path-1/text.txt", - "watch-path-2/.gitignore", - "watch-path-2/src/index.go", - "watch-path-4/test/index_test.go", - } - - pipelines, err := stepsToTrigger(changedFiles, watch) - assert.NoError(t, err) - var got []string - - for _, v := range pipelines { - got = append(got, v.Trigger) - } - - assert.Equal(t, want, got) -} - -func TestPipelinesStepsToTrigger(t *testing.T) { +// func TestPipelinesToTriggerGetsListOfPipelines(t *testing.T) { +// want := []string{"service-1", "service-2", "service-4"} + +// watch := []WatchConfig{ +// { +// Paths: []string{"watch-path-1"}, +// Generator: "echo steps: [ trigger: 'service-1' ]", +// }, +// { +// Paths: []string{"watch-path-2/", "watch-path-3/", "watch-path-4"}, +// // Step: pipeline.TriggerStep{Trigger: "service-2"}, +// }, +// { +// Paths: []string{"watch-path-5"}, +// // Step: pipeline.TriggerStep{Trigger: "service-3"}, +// }, +// { +// Paths: []string{"watch-path-2"}, +// // Step: pipeline.TriggerStep{Trigger: "service-4"}, +// }, +// } + +// changedFiles := []string{ +// "watch-path-1/text.txt", +// "watch-path-2/.gitignore", +// "watch-path-2/src/index.go", +// "watch-path-4/test/index_test.go", +// } + +// pipelines, err := stepsToTrigger(changedFiles, watch) +// assert.NoError(t, err) +// var got []string + +// for _, v := range pipelines { +// got = append(got, v.Trigger) +// } + +// assert.Equal(t, want, got) +// } + +// func TestPipelinesStepsToTrigger(t *testing.T) { + +// testCases := map[string]struct { +// ChangedFiles []string +// WatchConfigs []WatchConfig +// Expected []Step +// }{ +// "service-1": { +// ChangedFiles: []string{ +// "watch-path-1/text.txt", +// "watch-path-2/.gitignore", +// }, +// WatchConfigs: []WatchConfig{{ +// Paths: []string{"watch-path-1"}, +// Step: Step{Trigger: "service-1"}, +// }}, +// Expected: []Step{ +// {Trigger: "service-1"}, +// }, +// }, +// "service-1-2": { +// ChangedFiles: []string{ +// "watch-path-1/text.txt", +// "watch-path-2/.gitignore", +// }, +// WatchConfigs: []WatchConfig{ +// { +// Paths: []string{"watch-path-1"}, +// Step: Step{Trigger: "service-1"}, +// }, +// { +// Paths: []string{"watch-path-2"}, +// Step: Step{Trigger: "service-2"}, +// }, +// }, +// Expected: []Step{ +// {Trigger: "service-1"}, +// {Trigger: "service-2"}, +// }, +// }, +// "extension wildcard": { +// ChangedFiles: []string{ +// "text.txt", +// ".gitignore", +// }, +// WatchConfigs: []WatchConfig{ +// { +// Paths: []string{"*.txt"}, +// Step: Step{Trigger: "txt"}, +// }, +// }, +// Expected: []Step{ +// {Trigger: "txt"}, +// }, +// }, +// "extension wildcard in subdir": { +// ChangedFiles: []string{ +// "docs/text.txt", +// }, +// WatchConfigs: []WatchConfig{ +// { +// Paths: []string{"docs/*.txt"}, +// Step: Step{Trigger: "txt"}, +// }, +// }, +// Expected: []Step{ +// {Trigger: "txt"}, +// }, +// }, +// "directory wildcard": { +// ChangedFiles: []string{ +// "docs/text.txt", +// }, +// WatchConfigs: []WatchConfig{ +// { +// Paths: []string{"**/text.txt"}, +// Step: Step{Trigger: "txt"}, +// }, +// }, +// Expected: []Step{ +// {Trigger: "txt"}, +// }, +// }, +// "directory and extension wildcard": { +// ChangedFiles: []string{ +// "package/other.txt", +// }, +// WatchConfigs: []WatchConfig{ +// { +// Paths: []string{"*/*.txt"}, +// Step: Step{Trigger: "txt"}, +// }, +// }, +// Expected: []Step{ +// {Trigger: "txt"}, +// }, +// }, +// "double directory and extension wildcard": { +// ChangedFiles: []string{ +// "package/docs/other.txt", +// }, +// WatchConfigs: []WatchConfig{ +// { +// Paths: []string{"**/*.txt"}, +// Step: Step{Trigger: "txt"}, +// }, +// }, +// Expected: []Step{ +// {Trigger: "txt"}, +// }, +// }, +// } +// for name, tc := range testCases { +// t.Run(name, func(t *testing.T) { +// steps, err := stepsToTrigger(tc.ChangedFiles, tc.WatchConfigs) +// assert.NoError(t, err) +// assert.Equal(t, tc.Expected, steps) +// }) +// } +// } + +// func TestGeneratePipeline(t *testing.T) { +// steps := []Step{ +// { +// Trigger: "foo-service-pipeline", +// Build: Build{Message: "build message"}, +// SoftFail: true, +// Notify: []StepNotify{ +// {Slack: "@adikari"}, +// }, +// }, +// { +// Trigger: "notification-test", +// Command: "command-to-run", +// Notify: []StepNotify{ +// {Basecamp: "https://basecamp-url"}, +// {GithubStatus: GithubStatusNotification{Context: "my-custom-status"}}, +// {Slack: "@someuser", Condition: "build.state === \"passed\""}, +// }, +// }, +// { +// Group: "my group", +// Trigger: "foo-service-pipeline", +// Build: Build{Message: "build message"}, +// }, +// } + +// plugin := Plugin{ +// Wait: true, +// Notify: []PluginNotify{ +// {Email: "foo@gmail.com"}, +// {Email: "bar@gmail.com"}, +// {Basecamp: "https://basecamp"}, +// {Webhook: "https://webhook"}, +// {Slack: "@adikari"}, +// {GithubStatus: GithubStatusNotification{Context: "github-context"}}, +// }, +// Hooks: []HookConfig{ +// {Command: "echo \"hello world\""}, +// {Command: "cat ./file.txt"}, +// }, +// } + +// pipeline, _, err := generatePipeline(steps, plugin) + +// require.NoError(t, err) +// defer os.Remove(pipeline.Name()) + +// got, err := ioutil.ReadFile(pipeline.Name()) +// require.NoError(t, err) + +// want := +// `notify: +// - email: foo@gmail.com +// - email: bar@gmail.com +// - basecamp_campfire: https://basecamp +// - webhook: https://webhook +// - slack: '@adikari' +// - github_commit_status: +// context: github-context +// steps: +// - trigger: foo-service-pipeline +// build: +// message: build message +// soft_fail: true +// notify: +// - slack: '@adikari' +// - trigger: notification-test +// command: command-to-run +// notify: +// - basecamp_campfire: https://basecamp-url +// - github_commit_status: +// context: my-custom-status +// - slack: '@someuser' +// if: build.state === "passed" +// - group: my group +// steps: +// - trigger: foo-service-pipeline +// build: +// message: build message +// - wait: null +// - command: echo "hello world" +// - command: cat ./file.txt +// ` + +// assert.Equal(t, want, string(got)) +// } + +// func TestGeneratePipelineWithNoSteps(t *testing.T) { +// steps := []Step{} + +// want := +// `steps: +// - wait: null +// - command: echo "hello world" +// - command: cat ./file.txt +// ` + +// plugin := Plugin{ +// Wait: true, +// Hooks: []HookConfig{ +// {Command: "echo \"hello world\""}, +// {Command: "cat ./file.txt"}, +// }, +// } + +// pipeline, _, err := generatePipeline(steps, plugin) +// require.NoError(t, err) +// defer os.Remove(pipeline.Name()) + +// got, err := ioutil.ReadFile(pipeline.Name()) +// require.NoError(t, err) + +// assert.Equal(t, want, string(got)) +// } + +func TestPipelinesDynamicStepsToTrigger(t *testing.T) { testCases := map[string]struct { ChangedFiles []string WatchConfigs []WatchConfig - Expected []Step + Expected []pipeline.Step }{ "service-1": { ChangedFiles: []string{ "watch-path-1/text.txt", "watch-path-2/.gitignore", }, - WatchConfigs: []WatchConfig{{ - Paths: []string{"watch-path-1"}, - Step: Step{Trigger: "service-1"}, - }}, - Expected: []Step{ - {Trigger: "service-1"}, - }, - }, - "service-1-2": { - ChangedFiles: []string{ - "watch-path-1/text.txt", - "watch-path-2/.gitignore", - }, - WatchConfigs: []WatchConfig{ - { - Paths: []string{"watch-path-1"}, - Step: Step{Trigger: "service-1"}, - }, - { - Paths: []string{"watch-path-2"}, - Step: Step{Trigger: "service-2"}, - }, - }, - Expected: []Step{ - {Trigger: "service-1"}, - {Trigger: "service-2"}, - }, - }, - "extension wildcard": { - ChangedFiles: []string{ - "text.txt", - ".gitignore", - }, - WatchConfigs: []WatchConfig{ - { - Paths: []string{"*.txt"}, - Step: Step{Trigger: "txt"}, - }, - }, - Expected: []Step{ - {Trigger: "txt"}, - }, - }, - "extension wildcard in subdir": { - ChangedFiles: []string{ - "docs/text.txt", - }, WatchConfigs: []WatchConfig{ { - Paths: []string{"docs/*.txt"}, - Step: Step{Trigger: "txt"}, + Paths: []string{"watch-path-1"}, + Generator: "bash -c 'echo steps: [ command: foo ]'", }, }, - Expected: []Step{ - {Trigger: "txt"}, - }, - }, - "directory wildcard": { - ChangedFiles: []string{ - "docs/text.txt", - }, - WatchConfigs: []WatchConfig{ - { - Paths: []string{"**/text.txt"}, - Step: Step{Trigger: "txt"}, + Expected: []pipeline.Step{ + &pipeline.CommandStep{ + Command: "foo", }, }, - Expected: []Step{ - {Trigger: "txt"}, - }, - }, - "directory and extension wildcard": { - ChangedFiles: []string{ - "package/other.txt", - }, - WatchConfigs: []WatchConfig{ - { - Paths: []string{"*/*.txt"}, - Step: Step{Trigger: "txt"}, - }, - }, - Expected: []Step{ - {Trigger: "txt"}, - }, - }, - "double directory and extension wildcard": { - ChangedFiles: []string{ - "package/docs/other.txt", - }, - WatchConfigs: []WatchConfig{ - { - Paths: []string{"**/*.txt"}, - Step: Step{Trigger: "txt"}, - }, - }, - Expected: []Step{ - {Trigger: "txt"}, - }, }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { - steps, err := stepsToTrigger(tc.ChangedFiles, tc.WatchConfigs) + expectedOut, err := yaml.Marshal(tc.Expected) assert.NoError(t, err) - assert.Equal(t, tc.Expected, steps) - }) - } -} - -func TestGeneratePipeline(t *testing.T) { - steps := []Step{ - { - Trigger: "foo-service-pipeline", - Build: Build{Message: "build message"}, - SoftFail: true, - Notify: []StepNotify{ - {Slack: "@adikari"}, - }, - }, - { - Trigger: "notification-test", - Command: "command-to-run", - Notify: []StepNotify{ - {Basecamp: "https://basecamp-url"}, - {GithubStatus: GithubStatusNotification{Context: "my-custom-status"}}, - {Slack: "@someuser", Condition: "build.state === \"passed\""}, - }, - }, - { - Group: "my group", - Trigger: "foo-service-pipeline", - Build: Build{Message: "build message"}, - }, - } - plugin := Plugin{ - Wait: true, - Notify: []PluginNotify{ - {Email: "foo@gmail.com"}, - {Email: "bar@gmail.com"}, - {Basecamp: "https://basecamp"}, - {Webhook: "https://webhook"}, - {Slack: "@adikari"}, - {GithubStatus: GithubStatusNotification{Context: "github-context"}}, - }, - Hooks: []HookConfig{ - {Command: "echo \"hello world\""}, - {Command: "cat ./file.txt"}, - }, - } + steps, err := stepsToTrigger(tc.ChangedFiles, tc.WatchConfigs) + assert.NoError(t, err) - pipeline, err := generatePipeline(steps, plugin) - - require.NoError(t, err) - defer os.Remove(pipeline.Name()) - - got, err := ioutil.ReadFile(pipeline.Name()) - require.NoError(t, err) - - want := - `notify: -- email: foo@gmail.com -- email: bar@gmail.com -- basecamp_campfire: https://basecamp -- webhook: https://webhook -- slack: '@adikari' -- github_commit_status: - context: github-context -steps: -- trigger: foo-service-pipeline - build: - message: build message - soft_fail: true - notify: - - slack: '@adikari' -- trigger: notification-test - command: command-to-run - notify: - - basecamp_campfire: https://basecamp-url - - github_commit_status: - context: my-custom-status - - slack: '@someuser' - if: build.state === "passed" -- group: my group - steps: - - trigger: foo-service-pipeline - build: - message: build message -- wait: null -- command: echo "hello world" -- command: cat ./file.txt -` - - assert.Equal(t, want, string(got)) -} + out, err := yaml.Marshal(steps) + assert.NoError(t, err) -func TestGeneratePipelineWithNoSteps(t *testing.T) { - steps := []Step{} - - want := - `steps: -- wait: null -- command: echo "hello world" -- command: cat ./file.txt -` - - plugin := Plugin{ - Wait: true, - Hooks: []HookConfig{ - {Command: "echo \"hello world\""}, - {Command: "cat ./file.txt"}, - }, + assert.Equal(t, expectedOut, out) + }) } - - pipeline, err := generatePipeline(steps, plugin) - require.NoError(t, err) - defer os.Remove(pipeline.Name()) - - got, err := ioutil.ReadFile(pipeline.Name()) - require.NoError(t, err) - - assert.Equal(t, want, string(got)) } diff --git a/plugin.go b/plugin.go index afbd1fe..f41ecc6 100644 --- a/plugin.go +++ b/plugin.go @@ -12,88 +12,16 @@ const pluginName = "github.com/buildkite-plugins/monorepo-diff" // Plugin buildkite monorepo diff plugin structure type Plugin struct { - Diff string - Wait bool - LogLevel string `json:"log_level"` - Interpolation bool - Hooks []HookConfig - Watch []WatchConfig - RawEnv interface{} `json:"env"` - Env map[string]string - RawNotify []map[string]interface{} `json:"notify" yaml:",omitempty"` - Notify []PluginNotify `yaml:"notify,omitempty"` -} - -// HookConfig Plugin hook configuration -type HookConfig struct { - Command string + Diff string + LogLevel string `json:"log_level"` + Watch []WatchConfig } // WatchConfig Plugin watch configuration type WatchConfig struct { - RawPath interface{} `json:"path"` - Paths []string - Step Step `json:"config"` -} - -type Group struct { - Label string `yaml:"group"` - Steps []Step `yaml:"steps"` -} - -// GithubStatusNotification is notification config for github_commit_status -type GithubStatusNotification struct { - Context string `yaml:"context,omitempty"` -} - -// PluginNotify is notify configuration for pipeline -type PluginNotify struct { - Slack string `yaml:"slack,omitempty"` - Email string `yaml:"email,omitempty"` - PagerDuty string `yaml:"pagerduty_change_event,omitempty"` - Webhook string `yaml:"webhook,omitempty"` - Basecamp string `yaml:"basecamp_campfire,omitempty"` - GithubStatus GithubStatusNotification `yaml:"github_commit_status,omitempty"` - Condition string `yaml:"if,omitempty"` -} - -// Notify is Buildkite notification definition -type StepNotify struct { - Slack string `yaml:"slack,omitempty"` - Basecamp string `yaml:"basecamp_campfire,omitempty"` - GithubStatus GithubStatusNotification `yaml:"github_commit_status,omitempty"` - Condition string `yaml:"if,omitempty"` -} - -// Step is buildkite pipeline definition -type Step struct { - Group string `yaml:"group,omitempty"` - Trigger string `yaml:"trigger,omitempty"` - Label string `yaml:"label,omitempty"` - Build Build `yaml:"build,omitempty"` - Command interface{} `yaml:"command,omitempty"` - Commands interface{} `yaml:"commands,omitempty"` - Agents Agent `yaml:"agents,omitempty"` - Artifacts []string `yaml:"artifacts,omitempty"` - RawEnv interface{} `json:"env" yaml:",omitempty"` - Env map[string]string `yaml:"env,omitempty"` - Async bool `yaml:"async,omitempty"` - SoftFail interface{} `json:"soft_fail" yaml:"soft_fail,omitempty"` - RawNotify []map[string]interface{} `json:"notify" yaml:",omitempty"` - Notify []StepNotify `yaml:"notify,omitempty"` -} - -// Agent is Buildkite agent definition -type Agent map[string]string - -// Build is buildkite build definition -type Build struct { - Message string `yaml:"message,omitempty"` - Branch string `yaml:"branch,omitempty"` - Commit string `yaml:"commit,omitempty"` - RawEnv interface{} `json:"env" yaml:",omitempty"` - Env map[string]string `yaml:"env,omitempty"` - // Notify []Notify `yaml:"notify,omitempty"` + RawPath interface{} `json:"path"` + Paths []string + Generator string `json:"generator"` } // UnmarshalJSON set defaults properties @@ -101,26 +29,14 @@ func (plugin *Plugin) UnmarshalJSON(data []byte) error { type plain Plugin def := &plain{ - Diff: "git diff --name-only HEAD~1", - Wait: false, - LogLevel: "info", - Interpolation: true, + Diff: "git diff --name-only HEAD~1", + LogLevel: "info", } _ = json.Unmarshal(data, def) *plugin = Plugin(*def) - parseResult, err := parseEnv(plugin.RawEnv) - if err != nil { - return errors.New("failed to parse plugin configuration") - } - - plugin.Env = parseResult - plugin.RawEnv = nil - - setPluginNotify(&plugin.Notify, &plugin.RawNotify) - // Path can be string or an array of strings, // handle both cases and create an array of paths. for i, p := range plugin.Watch { @@ -133,16 +49,6 @@ func (plugin *Plugin) UnmarshalJSON(data []byte) error { } } - if plugin.Watch[i].Step.Trigger != "" { - setBuild(&plugin.Watch[i].Step.Build) - } - - if plugin.Watch[i].Step.RawNotify != nil { - setNotify(&plugin.Watch[i].Step.Notify, &plugin.Watch[i].Step.RawNotify) - } - - appendEnv(&plugin.Watch[i], plugin.Env) - p.RawPath = nil } @@ -176,156 +82,3 @@ func initializePlugin(data string) (Plugin, error) { return Plugin{}, errors.New("could not initialize plugin") } - -func setPluginNotify(notifications *[]PluginNotify, rawNotify *[]map[string]interface{}) { - for _, v := range *rawNotify { - var notify PluginNotify - - if condition, ok := isString(v["if"]); ok { - notify.Condition = condition - } - - if email, ok := isString(v["email"]); ok { - notify.Email = email - *notifications = append(*notifications, notify) - continue - } - - if basecamp, ok := isString(v["basecamp_campfire"]); ok { - notify.Basecamp = basecamp - *notifications = append(*notifications, notify) - continue - } - - if webhook, ok := isString(v["webhook"]); ok { - notify.Webhook = webhook - *notifications = append(*notifications, notify) - continue - } - - if pagerduty, ok := isString(v["pagerduty_change_event"]); ok { - notify.PagerDuty = pagerduty - *notifications = append(*notifications, notify) - continue - } - - if slack, ok := isString(v["slack"]); ok { - notify.Slack = slack - *notifications = append(*notifications, notify) - continue - } - - if github, ok := v["github_commit_status"].(map[string]interface{}); ok { - if context, ok := isString(github["context"]); ok { - notify.GithubStatus = GithubStatusNotification{Context: context} - *notifications = append(*notifications, notify) - } - continue - } - } - - *rawNotify = nil -} - -func setNotify(notifications *[]StepNotify, rawNotify *[]map[string]interface{}) { - for _, v := range *rawNotify { - var notify StepNotify - - if condition, ok := isString(v["if"]); ok { - notify.Condition = condition - } - - if basecamp, ok := isString(v["basecamp_campfire"]); ok { - notify.Basecamp = basecamp - *notifications = append(*notifications, notify) - continue - } - - if slack, ok := isString(v["slack"]); ok { - notify.Slack = slack - *notifications = append(*notifications, notify) - continue - } - - if github, ok := v["github_commit_status"].(map[string]interface{}); ok { - if context, ok := isString(github["context"]); ok { - notify.GithubStatus = GithubStatusNotification{Context: context} - *notifications = append(*notifications, notify) - } - continue - } - } - - *rawNotify = nil -} - -func setBuild(build *Build) { - if build.Message == "" { - build.Message = env("BUILDKITE_MESSAGE", "") - } - - if build.Branch == "" { - build.Branch = env("BUILDKITE_BRANCH", "") - } - - if build.Commit == "" { - build.Commit = env("BUILDKITE_COMMIT", "") - } -} - -// appends top level env to Step.Env and Step.Build.Env -func appendEnv(watch *WatchConfig, env map[string]string) { - watch.Step.Env, _ = parseEnv(watch.Step.RawEnv) - watch.Step.Build.Env, _ = parseEnv(watch.Step.Build.RawEnv) - - for key, value := range env { - if watch.Step.Command != nil || watch.Step.Commands != nil { - if watch.Step.Env == nil { - watch.Step.Env = make(map[string]string) - } - - watch.Step.Env[key] = value - continue - } - if watch.Step.Trigger != "" { - if watch.Step.Build.Env == nil { - watch.Step.Build.Env = make(map[string]string) - } - - watch.Step.Build.Env[key] = value - continue - } - } - - watch.Step.RawEnv = nil - watch.Step.Build.RawEnv = nil - watch.RawPath = nil -} - -// parse env in format from env=env-value to map[env] = env-value -func parseEnv(raw interface{}) (map[string]string, error) { - if raw == nil { - return nil, nil - } - - if _, ok := raw.([]interface{}); ok != true { - return nil, errors.New("failed to parse plugin configuration") - } - - result := make(map[string]string) - for _, v := range raw.([]interface{}) { - split := strings.Split(v.(string), "=") - key, value := strings.TrimSpace(split[0]), split[1:] - - // only key exists. set value from env - if len(key) > 0 && len(value) == 0 { - result[key] = env(key, "") - } - - if len(value) > 0 { - result[key] = strings.TrimSpace(value[0]) - } - } - - return result, nil -} diff --git a/plugin.yml b/plugin.yml index 7f751bb..ea7367f 100644 --- a/plugin.yml +++ b/plugin.yml @@ -9,91 +9,13 @@ configuration: type: string log_level: type: string - interpolation: - type: boolean - env: - type: array - notify: - type: [array] - properties: - email: - type: string - webhook: - type: string - pagerduty_change_event: - type: string - basecamp_campfire: - type: string - github_commit_status: - type: object - properties: - context: - type: string - slack: - type: string - if: - type: string watch: type: array properties: path: type: [string, array] minimum: 1 - config: - type: object - properties: - command: - type: string - trigger: - type: string - soft_fail: - type: [object, boolean] - notify: - type: [array] - properties: - basecamp_campfire: - type: string - github_commit_status: - type: object - properties: - context: - type: string - slack: - type: string - if: - type: string - async: - type: boolean - label: - type: string - build: - type: object - properties: - message: - type: string - commit: - type: string - branch: - type: string - env: - type: array - agents: - type: object - properties: - queue: - type: string - additionalProperties: - type: string - artifacts: - type: array - env: - type: array - wait: - type: boolean - hooks: - type: array - properties: - command: + generator: type: string required: - watch diff --git a/plugin_test.go b/plugin_test.go index b9f27d5..0e96717 100644 --- a/plugin_test.go +++ b/plugin_test.go @@ -26,10 +26,8 @@ func TestPluginShouldHaveDefaultValues(t *testing.T) { got, _ := initializePlugin(param) expected := Plugin{ - Diff: "git diff --name-only HEAD~1", - Wait: false, - LogLevel: "info", - Interpolation: true, + Diff: "git diff --name-only HEAD~1", + LogLevel: "info", } assert.Equal(t, expected, got) @@ -48,96 +46,15 @@ func TestPluginShouldUnmarshallCorrectly(t *testing.T) { param := `[{ "github.com/buildkite-plugins/monorepo-diff-buildkite-plugin#commit": { "diff": "cat ./hello.txt", - "wait": true, "log_level": "debug", - "interpolation": true, - "hooks": [ - { "command": "some-hook-command" }, - { "command": "another-hook-command" } - ], - "env": [ - "env1=env-1", - "env2=env-2", - "env3" - ], - "notify": [ - { "email": "foo@gmail.com" }, - { "email": "bar@gmail.com" }, - { "basecamp_campfire": "https://basecamp-url" }, - { "webhook": "https://webhook-url", "if": "build.state === 'failed'" }, - { "pagerduty_change_event": "636d22Yourc0418Key3b49eee3e8" }, - { "github_commit_status": { "context" : "my-custom-status" } }, - { "slack": "@someuser", "if": "build.state === 'passed'" } - ], "watch": [ { "path": "watch-path-1", - "config": { - "trigger": "service-2", - "build": { - "message": "some message" - } - } + "generator": "generate-service-1" }, { - "path": "watch-path-1", - "config": { - "command": "echo hello-world", - "env": [ - "env4", "hi= bye" - ], - "soft_fail": [{ - "exit_status": "*" - }], - "notify": [ - { "email": "foo@gmail.com" }, - { "email": "bar@gmail.com" }, - { "basecamp_campfire": "https://basecamp-url" }, - { "webhook": "https://webhook-url", "if": "build.state === 'failed'" }, - { "pagerduty_change_event": "636d22Yourc0418Key3b49eee3e8" }, - { "github_commit_status": { "context" : "my-custom-status" } }, - { "slack": "@someuser", "if": "build.state === 'passed'" } - ] - } - }, - { - "path": [ - "watch-path-1", - "watch-path-2" - ], - "config": { - "trigger": "service-1", - "label": "hello", - "build": { - "message": "build message", - "branch": "current branch", - "commit": "commit-hash", - "env": [ - "foo =bar", - "bar= foo" - ] - }, - "async": true, - "agents": { - "queue": "queue-1", - "database": "postgres" - }, - "artifacts": [ "artifiact-1" ], - "soft_fail": [{ - "exit_status": 127 - }] - } - }, - { - "path": "watch-path-1", - "config": { - "group": "my group", - "command": "echo hello-group", - "env": [ - "env4", "hi= bye" - ], - "soft_fail": true - } + "path": "watch-path-2", + "generator": "generate-service-2" } ] } @@ -146,103 +63,18 @@ func TestPluginShouldUnmarshallCorrectly(t *testing.T) { got, _ := initializePlugin(param) expected := Plugin{ - Diff: "cat ./hello.txt", - Wait: true, - LogLevel: "debug", - Interpolation: true, - Hooks: []HookConfig{ - {Command: "some-hook-command"}, - {Command: "another-hook-command"}, - }, - Env: map[string]string{ - "env1": "env-1", - "env2": "env-2", - "env3": "env-3", - }, - Notify: []PluginNotify{ - {Email: "foo@gmail.com"}, - {Email: "bar@gmail.com"}, - {Basecamp: "https://basecamp-url"}, - {Webhook: "https://webhook-url", Condition: "build.state === 'failed'"}, - {PagerDuty: "636d22Yourc0418Key3b49eee3e8"}, - {GithubStatus: GithubStatusNotification{Context: "my-custom-status"}}, - {Slack: "@someuser", Condition: "build.state === 'passed'"}, - }, + Diff: "cat ./hello.txt", + LogLevel: "debug", Watch: []WatchConfig{ { - Paths: []string{"watch-path-1"}, - Step: Step{ - Trigger: "service-2", - Build: Build{ - Message: "some message", - Branch: "go-rewrite", - Commit: "123", - Env: map[string]string{ - "env1": "env-1", - "env2": "env-2", - "env3": "env-3", - }, - }, - }, - }, - { - Paths: []string{"watch-path-1"}, - Step: Step{ - Command: "echo hello-world", - Env: map[string]string{ - "env1": "env-1", - "env2": "env-2", - "env3": "env-3", - "env4": "env-4", - "hi": "bye", - }, - SoftFail: []interface{}{map[string]interface{}{"exit_status": "*"}}, - Notify: []StepNotify{ - {Basecamp: "https://basecamp-url"}, - {GithubStatus: GithubStatusNotification{Context: "my-custom-status"}}, - {Slack: "@someuser", Condition: "build.state === 'passed'"}, - }, - }, + RawPath: "watch-path-1", + Paths: []string{"watch-path-1"}, + Generator: "generate-service-1", }, { - Paths: []string{"watch-path-1", "watch-path-2"}, - Step: Step{ - Trigger: "service-1", - Label: "hello", - Build: Build{ - Message: "build message", - Branch: "current branch", - Commit: "commit-hash", - Env: map[string]string{ - "foo": "bar", - "bar": "foo", - "env1": "env-1", - "env2": "env-2", - "env3": "env-3", - }, - }, - Async: true, - Agents: map[string]string{"queue": "queue-1", "database": "postgres"}, - Artifacts: []string{"artifiact-1"}, - SoftFail: []interface{}{map[string]interface{}{ - "exit_status": float64(127), - }}, - }, - }, - { - Paths: []string{"watch-path-1"}, - Step: Step{ - Group: "my group", - Command: "echo hello-group", - Env: map[string]string{ - "env1": "env-1", - "env2": "env-2", - "env3": "env-3", - "env4": "env-4", - "hi": "bye", - }, - SoftFail: true, - }, + RawPath: "watch-path-2", + Paths: []string{"watch-path-2"}, + Generator: "generate-service-2", }, }, } @@ -285,29 +117,3 @@ func TestPluginShouldOnlyFullyUnmarshallItselfAndNotOtherPlugins(t *testing.T) { _, err := initializePlugin(param) assert.NoError(t, err) } - -func TestPluginShouldErrorIfPluginConfigIsInvalid(t *testing.T) { - param := `[ - { - "github.com/buildkite-plugins/monorepo-diff-buildkite-plugin#commit": { - "env": { - "anInvalidKey": "An Invalid Value" - }, - "watch": [ - { - "path": [ - ".buildkite/**/*" - ], - "config": { - "label": "Example label", - "command": "echo hello world\\n" - } - } - ] - } - } - ] - ` - _, err := initializePlugin(param) - assert.Error(t, err) -} From 6efa88d4bb8b476769128fb9d4b392dbd525e785 Mon Sep 17 00:00:00 2001 From: David Archer Date: Tue, 25 Jun 2024 14:34:26 -0400 Subject: [PATCH 02/10] fix repo name --- plugin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin.go b/plugin.go index f41ecc6..a9622a1 100644 --- a/plugin.go +++ b/plugin.go @@ -8,7 +8,7 @@ import ( log "github.com/sirupsen/logrus" ) -const pluginName = "github.com/buildkite-plugins/monorepo-diff" +const pluginName = "github.com/davidarcher/monorepo-diff" // Plugin buildkite monorepo diff plugin structure type Plugin struct { From 76e221f5aaf8b046b860e640e842e7e0691991ba Mon Sep 17 00:00:00 2001 From: David Archer Date: Tue, 25 Jun 2024 14:39:17 -0400 Subject: [PATCH 03/10] default to debug level --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 59afb46..18f6076 100644 --- a/main.go +++ b/main.go @@ -12,7 +12,7 @@ func setupLogger(logLevel string) { ll, err := log.ParseLevel(logLevel) if err != nil { - ll = log.InfoLevel + ll = log.DebugLevel } log.SetLevel(ll) From 7da002cd0074326ea24a8c3d3c9428ce95ed9ed6 Mon Sep 17 00:00:00 2001 From: David Archer Date: Tue, 25 Jun 2024 14:49:07 -0400 Subject: [PATCH 04/10] fix plugin name --- plugin.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin.go b/plugin.go index a9622a1..5db7bdd 100644 --- a/plugin.go +++ b/plugin.go @@ -8,7 +8,7 @@ import ( log "github.com/sirupsen/logrus" ) -const pluginName = "github.com/davidarcher/monorepo-diff" +const pluginName = "monorepo-diff" // Plugin buildkite monorepo diff plugin structure type Plugin struct { @@ -67,7 +67,7 @@ func initializePlugin(data string) (Plugin, error) { for _, p := range pluginConfigs { for key, pluginConfig := range p { - if strings.HasPrefix(key, pluginName) { + if strings.Contains(key, pluginName) { var plugin Plugin if err := json.Unmarshal(pluginConfig, &plugin); err != nil { From e4d858fabd66344da8cc9806688e42daf19951c6 Mon Sep 17 00:00:00 2001 From: David Archer Date: Tue, 25 Jun 2024 14:50:35 -0400 Subject: [PATCH 05/10] info level --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 18f6076..c1b99d3 100644 --- a/main.go +++ b/main.go @@ -26,7 +26,7 @@ func main() { plugins := env("BUILDKITE_PLUGINS", "") - log.Debugf("received plugin: \n%v", plugins) + log.Infof("received plugin: \n%v", plugins) plugin, err := initializePlugin(plugins) From 7f3b3419ad34956adba35cd5a6dfbe748d231ad2 Mon Sep 17 00:00:00 2001 From: David Archer Date: Tue, 25 Jun 2024 15:03:42 -0400 Subject: [PATCH 06/10] wip --- plugin.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin.yml b/plugin.yml index ea7367f..bcede3d 100644 --- a/plugin.yml +++ b/plugin.yml @@ -19,5 +19,6 @@ configuration: type: string required: - watch + additionalProperties: false # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite-plugins/buildkite-plugin-linter/master/lib/plugin-yaml-schema.yml From 412181a19672a9bf9ed49c9dbb344e3873034f2c Mon Sep 17 00:00:00 2001 From: David Archer Date: Tue, 25 Jun 2024 15:32:37 -0400 Subject: [PATCH 07/10] wip --- hooks/command | 122 ++++++++++---------------------------------------- 1 file changed, 23 insertions(+), 99 deletions(-) diff --git a/hooks/command b/hooks/command index 405da9e..0b40e95 100755 --- a/hooks/command +++ b/hooks/command @@ -1,104 +1,28 @@ #!/bin/bash set -euo pipefail -check_cmd() { - command -v "$1" > /dev/null 2>&1 - return $? -} - -say() { - echo "$1" -} - -err() { - red=$(tput setaf 1 2>/dev/null || echo '') - reset=$(tput sgr0 2>/dev/null || echo '') - say "${red}ERROR${reset}: $1" >&2 - exit 1 -} - -get_architecture() { - _ostype="$(uname -s)" - _arch="$(uname -m)" - _arm=("arm armhf aarch64 aarch64_be armv6l armv7l armv8l arm64e") # arm64 - _amd=("x86 x86pc i386 i686 i686-64 x64 x86_64 x86_64h athlon") # amd64 - - if [[ "${_arm[*]}" =~ ${_arch} ]]; then - _arch="arm64" - elif [[ "${_amd[*]}" =~ ${_arch} ]]; then - _arch="amd64" - elif [[ "${_arch}" != "ppc64le" ]]; then - echo -e "ERROR: unsupported architecture \"${_arch}\"" >&2 - exit 2 - fi - - RETVAL="${_ostype}_${_arch}" -} - -need_cmd() { - if ! check_cmd "$1"; then - err "need '$1' (command not found)" - fi -} - -# This wraps curl or wget. -# Try curl first, if not installed, use wget instead. -downloader() { - if check_cmd curl; then - _dld=curl - elif check_cmd wget; then - _dld=wget - else - _dld='curl or wget' # to be used in error message of need_cmd - fi +ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")/..")" +PLUGIN_NAME=monorepo-diff +COMPILE_IMAGE=public.ecr.aws/docker/library/golang:1.22.4-alpine - if [ "$1" = --check ]; then - need_cmd "$_dld" - elif [ "$_dld" = curl ]; then - curl -sSfL "$1" -o "$2" - elif [ "$_dld" = wget ]; then - wget "$1" -O "$2" - else - err "Unknown downloader" - fi -} - -get_version() { - local _plugin=${BUILDKITE_PLUGINS:-""} - _version=$(echo "$_plugin" | sed -e 's/.*monorepo-diff-buildkite-plugin//' -e 's/.*#//' -e 's/\".*//') - if [[ "$_version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then true; else _version=""; fi - - RETVAL="$_version" -} - -download_binary_and_run() { - get_architecture || return 1 - local _arch="$RETVAL" - local _executable="monorepo-diff-buildkite-plugin" - local _repo="https://github.com/buildkite-plugins/monorepo-diff-buildkite-plugin" - - get_version || return 1 - local _version="$RETVAL" - - if [ -z "${_version}" ]; then - _url=${_repo}/releases/latest/download/${_executable}_${_arch} - else - _url=${_repo}/releases/download/${_version}/${_executable}_${_arch} - fi - - local test_mode="${BUILDKITE_PLUGIN_MONOREPO_DIFF_BUILDKITE_PLUGIN_TEST_MODE:-false}" - - if [[ "$test_mode" == "false" ]]; then - if ! downloader "$_url" "$_executable"; then - say "failed to download $_url" - exit 1 +compile() { + if command -v ${PLUGIN_NAME}-buildkite-plugin-bin > /dev/null 2>&1; then + return 0; fi - fi - - # todo: move it to a more secure place - chmod +x ${_executable} - - ./${_executable} -} - -download_binary_and_run "$@" || exit 1 + echo "~~~ Compiling ${PLUGIN_NAME} plugin" + local tempdir + tempdir="$(mktemp -d "/tmp/${PLUGIN_NAME}.XXXXXX")" + docker run --name="compile-$BUILDKITE_JOB_ID" \ + -e CGO_ENABLED=0 \ + -v "$ROOT:/usr/src/app:ro" \ + -w /usr/src/app \ + "$COMPILE_IMAGE" \ + go build -v -o /tmp/${PLUGIN_NAME}-buildkite-plugin-bin + docker cp "compile-$BUILDKITE_JOB_ID:/tmp/${PLUGIN_NAME}-buildkite-plugin-bin" "$tempdir/" + docker rm "compile-$BUILDKITE_JOB_ID" + export PATH="$tempdir:$PATH" +} + +compile + +monorepo-diff-buildkite-plugin-bin "$@" \ No newline at end of file From 2e1effa1117e440d25017b3714f157ae69c227a8 Mon Sep 17 00:00:00 2001 From: David Archer Date: Tue, 25 Jun 2024 15:50:18 -0400 Subject: [PATCH 08/10] yaml v3 --- go.mod | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 39c68d1..04132e0 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/bmatcuk/doublestar/v2 v2.0.4 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.1 - gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -19,5 +19,4 @@ require ( github.com/oleiade/reflections v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect golang.org/x/sys v0.18.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) From 4479c331bbcf58af5e3fd9af4e92f165b198d9e5 Mon Sep 17 00:00:00 2001 From: David Archer Date: Tue, 25 Jun 2024 15:51:52 -0400 Subject: [PATCH 09/10] yamlv3 --- go.mod | 3 ++- go.sum | 21 ++++++++++++--------- pipeline.go | 2 +- pipeline_test.go | 2 +- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 04132e0..72c900f 100644 --- a/go.mod +++ b/go.mod @@ -6,16 +6,17 @@ toolchain go1.22.4 require ( github.com/bmatcuk/doublestar/v2 v2.0.4 + github.com/buildkite/go-pipeline v0.9.0 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.1 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/buildkite/go-pipeline v0.9.0 // indirect github.com/buildkite/interpolate v0.1.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/oleiade/reflections v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect golang.org/x/sys v0.18.0 // indirect diff --git a/go.sum b/go.sum index 50fea0e..0437721 100644 --- a/go.sum +++ b/go.sum @@ -4,19 +4,25 @@ github.com/buildkite/go-pipeline v0.9.0 h1:2a2bibJ9dCCyyNReH73jkQVUYyUnhYAxISyf3 github.com/buildkite/go-pipeline v0.9.0/go.mod h1:4aqMzJ3iagc0wcI5h8NQpON9xfyq27QGDi4xfnKiCUs= github.com/buildkite/interpolate v0.1.2 h1:mVbMCphpu2MHUr1qLdjq9xc3NjNWYg/w/CbrGS5ckzg= github.com/buildkite/interpolate v0.1.2/go.mod h1:UNVe6A+UfiBNKbhAySrBbZFZFxQ+DXr9nWen6WVt/A8= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/oleiade/reflections v1.0.1 h1:D1XO3LVEYroYskEsoSiGItp9RUxG6jWnCVvrqH0HHQM= github.com/oleiade/reflections v1.0.1/go.mod h1:rdFxbxq4QXVZWj0F+e9jqjDkc7dbp97vkRixKo2JR60= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -25,16 +31,13 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= diff --git a/pipeline.go b/pipeline.go index ad42b60..881eaeb 100644 --- a/pipeline.go +++ b/pipeline.go @@ -10,7 +10,7 @@ import ( "github.com/bmatcuk/doublestar/v2" "github.com/buildkite/go-pipeline" log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // PipelineGenerator generates pipeline file diff --git a/pipeline_test.go b/pipeline_test.go index 7f0a4c7..446aa41 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -6,7 +6,7 @@ import ( "github.com/buildkite/go-pipeline" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func mockGeneratePipeline(steps []pipeline.Step, plugin Plugin) (*os.File, bool, error) { From 1a4d8187b374d0e9a04a1aefff761b2f1ee68af6 Mon Sep 17 00:00:00 2001 From: David Archer Date: Tue, 25 Jun 2024 16:10:45 -0400 Subject: [PATCH 10/10] Initial commit --- README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..bd72d48 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# monorepo-diff-buildkite-plugin +Generate dynamic Buildkite pipeline based on file changes