From c441785445d3d142bb1355300b74a0ae55967fa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa?= Date: Fri, 8 Dec 2023 11:27:13 -0600 Subject: [PATCH] Hotload support (#31) * pgx migrate test (#29) * add hotload support (#15) * import fsync * Upgrade alpine 3.13 -> 3.18 (#21) * upgrade alpine 3.13 -> 3.18 * add --pull to docker build * adding pgx as the supported hotload base driver (#26) * adding pgx as the supported hotload base driver * updating go.mod file * removing the pgx register in migrate (#27) * register hotload with pgx driver --------- Co-authored-by: Daniel Garcia Co-authored-by: Ying Chen <108433798+ychen-bloxer@users.noreply.github.com> Co-authored-by: Supriya Gowda <105773269+Gowdasupriya@users.noreply.github.com> * register pgx in main cli, not internal (#30) * register pgx in main cli, not internal * go fmt --------- Co-authored-by: Sujay Kumar Suman Co-authored-by: Ying Chen <108433798+ychen-bloxer@users.noreply.github.com> Co-authored-by: Supriya Gowda <105773269+Gowdasupriya@users.noreply.github.com> --- Dockerfile | 2 +- Dockerfile.github-actions | 2 +- cmd/migrate/.gitignore | 1 + cmd/migrate/main.go | 8 ++++++ database/driver.go | 26 +++++++++---------- database/mongodb/mongodb.go | 4 +-- internal/cli/main.go | 15 ----------- migrate_test.go | 5 ++-- source/driver.go | 24 ++++++++--------- .../go_bindata/examples/migrations/bindata.go | 12 +++++---- source/parse.go | 5 ++-- source/testing/testing.go | 5 ++-- util.go | 1 - 13 files changed, 54 insertions(+), 56 deletions(-) create mode 100644 cmd/migrate/.gitignore diff --git a/Dockerfile b/Dockerfile index 85dfff79d..ae23613a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY . ./ RUN make build-docker -FROM alpine:latest +FROM alpine:3.18 COPY --from=builder /go/src/github.com/infobloxopen/migrate/cmd/migrate/config /cli/config/ COPY --from=builder /go/src/github.com/infobloxopen/migrate/build/migrate.linux-386 /usr/local/bin/migrate diff --git a/Dockerfile.github-actions b/Dockerfile.github-actions index 2d211cbc3..b4726d1c8 100644 --- a/Dockerfile.github-actions +++ b/Dockerfile.github-actions @@ -1,4 +1,4 @@ -FROM alpine:latest +FROM alpine:3.18 RUN apk add --no-cache ca-certificates diff --git a/cmd/migrate/.gitignore b/cmd/migrate/.gitignore new file mode 100644 index 000000000..219f6a587 --- /dev/null +++ b/cmd/migrate/.gitignore @@ -0,0 +1 @@ +migrate diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index a91a3650e..b6188d95f 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -5,6 +5,10 @@ import ( "strings" "github.com/golang-migrate/migrate/v4/internal/cli" + "github.com/infobloxopen/hotload" + _ "github.com/infobloxopen/hotload/fsnotify" + "github.com/jackc/pgx/v4/stdlib" + "github.com/lib/pq" "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -25,6 +29,10 @@ func init() { // logrus formatter customFormatter := new(logrus.JSONFormatter) logrus.SetFormatter(customFormatter) + + hotload.RegisterSQLDriver("pgx", stdlib.GetDefaultDriver()) + hotload.RegisterSQLDriver("postgres", pq.Driver{}) + hotload.RegisterSQLDriver("postgresql", pq.Driver{}) } func main() { diff --git a/database/driver.go b/database/driver.go index fa80f455c..984d29144 100644 --- a/database/driver.go +++ b/database/driver.go @@ -25,22 +25,22 @@ var drivers = make(map[string]Driver) // Driver is the interface every database driver must implement. // // How to implement a database driver? -// 1. Implement this interface. -// 2. Optionally, add a function named `WithInstance`. -// This function should accept an existing DB instance and a Config{} struct -// and return a driver instance. -// 3. Add a test that calls database/testing.go:Test() -// 4. Add own tests for Open(), WithInstance() (when provided) and Close(). -// All other functions are tested by tests in database/testing. -// Saves you some time and makes sure all database drivers behave the same way. -// 5. Call Register in init(). -// 6. Create a internal/cli/build_.go file -// 7. Add driver name in 'DATABASE' variable in Makefile +// 1. Implement this interface. +// 2. Optionally, add a function named `WithInstance`. +// This function should accept an existing DB instance and a Config{} struct +// and return a driver instance. +// 3. Add a test that calls database/testing.go:Test() +// 4. Add own tests for Open(), WithInstance() (when provided) and Close(). +// All other functions are tested by tests in database/testing. +// Saves you some time and makes sure all database drivers behave the same way. +// 5. Call Register in init(). +// 6. Create a internal/cli/build_.go file +// 7. Add driver name in 'DATABASE' variable in Makefile // // Guidelines: -// * Don't try to correct user input. Don't assume things. +// - Don't try to correct user input. Don't assume things. // When in doubt, return an error and explain the situation to the user. -// * All configuration input must come from the URL string in func Open() +// - All configuration input must come from the URL string in func Open() // or the Config{} struct in WithInstance. Don't os.Getenv(). type Driver interface { // Open returns a new driver instance configured with parameters diff --git a/database/mongodb/mongodb.go b/database/mongodb/mongodb.go index 00f0fd1be..4c0415760 100644 --- a/database/mongodb/mongodb.go +++ b/database/mongodb/mongodb.go @@ -182,7 +182,7 @@ func (m *Mongo) Open(dsn string) (database.Driver, error) { return mc, nil } -//Parse the url param, convert it to boolean +// Parse the url param, convert it to boolean // returns error if param invalid. returns defaultValue if param not present func parseBoolean(urlParam string, defaultValue bool) (bool, error) { @@ -199,7 +199,7 @@ func parseBoolean(urlParam string, defaultValue bool) (bool, error) { return defaultValue, nil } -//Parse the url param, convert it to int +// Parse the url param, convert it to int // returns error if param invalid. returns defaultValue if param not present func parseInt(urlParam string, defaultValue int) (int, error) { diff --git a/internal/cli/main.go b/internal/cli/main.go index 985696312..5167a09a0 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -11,9 +11,6 @@ import ( "syscall" "time" - "github.com/infobloxopen/hotload" - _ "github.com/infobloxopen/hotload/fsnotify" - "github.com/lib/pq" flag "github.com/spf13/pflag" "github.com/spf13/viper" @@ -41,10 +38,6 @@ const ( forceUsage = `force V Set version V but don't run migration (ignores dirty state)` ) -func init() { - hotload.RegisterSQLDriver("postgres", pq.Driver{}) -} - func handleSubCmdHelp(help bool, usage string, flagSet *flag.FlagSet) { if help { fmt.Fprintln(os.Stderr, usage) @@ -161,14 +154,6 @@ Database drivers: `+strings.Join(database.List(), ", ")+"\n", createUsage, gotoU var migraterErr error if driver := viper.GetString("database.driver"); driver == "hotload" { - u, err := url.Parse(databasePtr) - if err != nil { - log.fatalErr(fmt.Errorf("could not parse hotload dsn %v: %s", databasePtr, err)) - } - hostname := u.Hostname() - if !(hostname == "postgres" || hostname == "pgx") { - log.fatalErr(fmt.Errorf("unsupported hotload base driver: %s", hostname)) - } db, err := sql.Open(driver, databasePtr) if err != nil { log.fatalErr(fmt.Errorf("could not open hotload dsn %s: %s", databasePtr, err)) diff --git a/migrate_test.go b/migrate_test.go index 5ed2d5e64..8a6e22ff7 100644 --- a/migrate_test.go +++ b/migrate_test.go @@ -19,8 +19,9 @@ import ( // sourceStubMigrations hold the following migrations: // u = up migration, d = down migration, n = version -// | 1 | - | 3 | 4 | 5 | - | 7 | -// | u d | - | u | u d | d | - | u d | +// +// | 1 | - | 3 | 4 | 5 | - | 7 | +// | u d | - | u | u d | d | - | u d | var sourceStubMigrations *source.Migrations const ( diff --git a/source/driver.go b/source/driver.go index 94bdef317..396eabfae 100644 --- a/source/driver.go +++ b/source/driver.go @@ -17,21 +17,21 @@ var drivers = make(map[string]Driver) // Driver is the interface every source driver must implement. // // How to implement a source driver? -// 1. Implement this interface. -// 2. Optionally, add a function named `WithInstance`. -// This function should accept an existing source instance and a Config{} struct -// and return a driver instance. -// 3. Add a test that calls source/testing.go:Test() -// 4. Add own tests for Open(), WithInstance() (when provided) and Close(). -// All other functions are tested by tests in source/testing. -// Saves you some time and makes sure all source drivers behave the same way. -// 5. Call Register in init(). +// 1. Implement this interface. +// 2. Optionally, add a function named `WithInstance`. +// This function should accept an existing source instance and a Config{} struct +// and return a driver instance. +// 3. Add a test that calls source/testing.go:Test() +// 4. Add own tests for Open(), WithInstance() (when provided) and Close(). +// All other functions are tested by tests in source/testing. +// Saves you some time and makes sure all source drivers behave the same way. +// 5. Call Register in init(). // // Guidelines: -// * All configuration input must come from the URL string in func Open() +// - All configuration input must come from the URL string in func Open() // or the Config{} struct in WithInstance. Don't os.Getenv(). -// * Drivers are supposed to be read only. -// * Ideally don't load any contents (into memory) in Open or WithInstance. +// - Drivers are supposed to be read only. +// - Ideally don't load any contents (into memory) in Open or WithInstance. type Driver interface { // Open returns a new driver instance configured with parameters // coming from the URL string. Migrate will call this function diff --git a/source/go_bindata/examples/migrations/bindata.go b/source/go_bindata/examples/migrations/bindata.go index d0a57e24e..0b74b4c73 100644 --- a/source/go_bindata/examples/migrations/bindata.go +++ b/source/go_bindata/examples/migrations/bindata.go @@ -213,11 +213,13 @@ var _bindata = map[string]func() (*asset, error){ // directory embedded in the file by go-bindata. // For example if you run go-bindata on data/... and data contains the // following hierarchy: -// data/ -// foo.txt -// img/ -// a.png -// b.png +// +// data/ +// foo.txt +// img/ +// a.png +// b.png +// // then AssetDir("data") would return []string{"foo.txt", "img"} // AssetDir("data/img") would return []string{"a.png", "b.png"} // AssetDir("foo.txt") and AssetDir("notexist") would return an error diff --git a/source/parse.go b/source/parse.go index 2f888fe75..df085ae29 100644 --- a/source/parse.go +++ b/source/parse.go @@ -16,8 +16,9 @@ var ( ) // Regex matches the following pattern: -// 123_name.up.ext -// 123_name.down.ext +// +// 123_name.up.ext +// 123_name.down.ext var Regex = regexp.MustCompile(`^([0-9]+)_(.*)\.(` + string(Down) + `|` + string(Up) + `)\.(.*)$`) // Parse returns Migration for matching Regex pattern. diff --git a/source/testing/testing.go b/source/testing/testing.go index 6c148df1e..2b5505c4b 100644 --- a/source/testing/testing.go +++ b/source/testing/testing.go @@ -15,8 +15,9 @@ import ( // It assumes that the driver tests has access to the following migrations: // // u = up migration, d = down migration, n = version -// | 1 | - | 3 | 4 | 5 | - | 7 | -// | u d | - | u | u d | d | - | u d | +// +// | 1 | - | 3 | 4 | 5 | - | 7 | +// | u d | - | u | u d | d | - | u d | // // See source/stub/stub_test.go or source/file/file_test.go for an example. func Test(t *testing.T, d source.Driver) { diff --git a/util.go b/util.go index bcf90077d..663d68f16 100644 --- a/util.go +++ b/util.go @@ -16,7 +16,6 @@ type MultiError struct { // NewMultiError returns an error type holding multiple errors. // // Deprecated: Use github.com/hashicorp/go-multierror instead -// func NewMultiError(errs ...error) MultiError { compactErrs := make([]error, 0) for _, e := range errs {