diff --git a/.circleci/config.yml b/.circleci/config.yml index 5323bbb..245cc2c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,6 +7,7 @@ setup_env: &setup_env command: | env | sort > /tmp/env.old + export DOCKERIZE_VER=0.12.0 export HADOLINT_VER=1.18.0 export SHELLCHECK_VER=0.7.1 export GOLANGCI_LINT_VER=1.31.0 @@ -32,6 +33,13 @@ jobs: environment: GOFLAGS: "-mod=readonly" EXAMPLE_APIKEY_ADMIN: "admin" + EXAMPLE_MYSQL_ADDR_HOST: "localhost" + EXAMPLE_MYSQL_AUTH_LOGIN: "root" + EXAMPLE_MYSQL_AUTH_PASS: "" + NARADA4D_TEST_MYSQL: "goose-mysql://root@127.0.0.1" + - image: "mysql:5.6" + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" steps: - checkout - *setup_env @@ -46,6 +54,8 @@ jobs: GO111MODULE: "on" command: | cd /tmp # Protect go.mod for modifications by `go get`. + dockerize --version | tee /dev/stderr | grep -wq v$DOCKERIZE_VER || + curl -sSfL https://github.com/powerman/dockerize/releases/download/v${DOCKERIZE_VER}/dockerize-$(uname)-x86_64 | install /dev/stdin $(go env GOPATH)/bin/dockerize hadolint --version | tee /dev/stderr | grep -wq v$HADOLINT_VER || curl -sSfL https://github.com/hadolint/hadolint/releases/download/v${HADOLINT_VER}/hadolint-$(uname)-x86_64 | install /dev/stdin $(go env GOPATH)/bin/hadolint shellcheck --version | tee /dev/stderr | grep -wq $SHELLCHECK_VER || diff --git a/.dockerignore b/.dockerignore index bdf7129..8ae1cc3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,4 @@ # a*b?c/d[0-9]e[^a-z\]]f\[g - pattern * !bin +!internal/migrations/mysql/*.sql diff --git a/.github/workflows/CI&CD.yml b/.github/workflows/CI&CD.yml index a2a0f64..56ea798 100644 --- a/.github/workflows/CI&CD.yml +++ b/.github/workflows/CI&CD.yml @@ -16,7 +16,15 @@ jobs: test: runs-on: ubuntu-latest timeout-minutes: 30 + services: + mysql: + image: 'mysql:5.6' + env: + MYSQL_ALLOW_EMPTY_PASSWORD: 'yes' + ports: + - 3306:3306 env: + DOCKERIZE_VER: '0.12.0' HADOLINT_VER: '1.18.0' SHELLCHECK_VER: '0.7.1' GOLANGCI_LINT_VER: '1.31.0' @@ -24,6 +32,10 @@ jobs: GOSWAGGER_VER: '0.25.0' GOVERALLS_VER: '0.0.7' EXAMPLE_APIKEY_ADMIN: 'admin' + EXAMPLE_MYSQL_ADDR_HOST: 'localhost' + EXAMPLE_MYSQL_AUTH_LOGIN: 'root' + EXAMPLE_MYSQL_AUTH_PASS: '' + NARADA4D_TEST_MYSQL: 'goose-mysql://root@127.0.0.1' steps: - uses: actions/setup-go@v2 with: @@ -52,6 +64,8 @@ jobs: GO111MODULE: 'on' run: | cd /tmp # Protect go.mod for modifications by `go get`. + dockerize --version | tee /dev/stderr | grep -wq v$DOCKERIZE_VER || + curl -sSfL https://github.com/powerman/dockerize/releases/download/v${DOCKERIZE_VER}/dockerize-$(uname)-x86_64 | install /dev/stdin $(go env GOPATH)/bin/dockerize hadolint --version | tee /dev/stderr | grep -wq v$HADOLINT_VER || curl -sSfL https://github.com/hadolint/hadolint/releases/download/v${HADOLINT_VER}/hadolint-$(uname)-x86_64 | install /dev/stdin $(go env GOPATH)/bin/hadolint shellcheck --version | tee /dev/stderr | grep -wq $SHELLCHECK_VER || diff --git a/.golangci.yml b/.golangci.yml index 4065c50..01b4260 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -41,6 +41,7 @@ run: # on Windows. skip-files: - "\\.[\\w-]+\\.go$" + - "/migrations/(.*/)?[0-9]{5}_" # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules": # If invoked with -mod=readonly, the go command is disallowed from the implicit diff --git a/README.md b/README.md index 2319587..18f1cba 100644 --- a/README.md +++ b/README.md @@ -271,7 +271,6 @@ $ ./bin/address-book serve ## TODO - [ ] Update JSON Schema support cheatsheet to latest go-swagger version. -- [ ] Replace trivial in-memory DAL with more complete one based on - Postgresql with metrics and migrations support. +- [ ] Add alternative DAL implementation for Postgresql. - [ ] Add cookie-based auth with CSRF middleware. - [ ] Add an example of adapter for external service in `svc/something`. diff --git a/cmd/address-book/init_test.go b/cmd/address-book/init_test.go index c8c22d0..880ae2a 100644 --- a/cmd/address-book/init_test.go +++ b/cmd/address-book/init_test.go @@ -10,6 +10,7 @@ import ( "github.com/powerman/go-service-example/api/openapi/model" "github.com/powerman/go-service-example/internal/app" "github.com/powerman/go-service-example/internal/config" + dal "github.com/powerman/go-service-example/internal/dal/mysql" "github.com/powerman/go-service-example/internal/srv/openapi" "github.com/powerman/go-service-example/pkg/def" ) @@ -17,6 +18,7 @@ import ( func TestMain(m *testing.M) { def.Init() initMetrics(reg, "test") + dal.InitMetrics(reg, "test") app.InitMetrics(reg) openapi.InitMetrics(reg, "test") cfg = config.MustGetServeTest() @@ -32,3 +34,7 @@ var ( apiKeyUser = oapiclient.APIKeyAuth("API-Key", "header", "user") apiContact1 = &model.Contact{ID: 1, Name: swag.String("A")} ) + +type tLogger check.C + +func (t tLogger) Print(args ...interface{}) { t.Log(args...) } diff --git a/cmd/address-book/service.go b/cmd/address-book/service.go index b0dabec..c80db58 100644 --- a/cmd/address-book/service.go +++ b/cmd/address-book/service.go @@ -7,8 +7,10 @@ import ( "github.com/powerman/go-service-example/api/openapi/restapi" "github.com/powerman/go-service-example/internal/app" "github.com/powerman/go-service-example/internal/config" - dal "github.com/powerman/go-service-example/internal/dal/memory" + dal "github.com/powerman/go-service-example/internal/dal/mysql" + migrations_mysql "github.com/powerman/go-service-example/internal/migrations/mysql" "github.com/powerman/go-service-example/internal/srv/openapi" + "github.com/powerman/go-service-example/pkg/cobrax" "github.com/powerman/go-service-example/pkg/concurrent" "github.com/powerman/go-service-example/pkg/def" "github.com/powerman/go-service-example/pkg/serve" @@ -29,14 +31,19 @@ type service struct { srv *restapi.Server } -func initService(_, serveCmd *cobra.Command) error { +func initService(cmd, serveCmd *cobra.Command) error { namespace := regexp.MustCompile(`[^a-zA-Z0-9]+`).ReplaceAllString(def.ProgName, "_") initMetrics(reg, namespace) + dal.InitMetrics(reg, namespace) app.InitMetrics(reg) openapi.InitMetrics(reg, namespace) + gooseMySQLCmd := cobrax.NewGooseMySQLCmd(migrations_mysql.Goose(), config.GetGooseMySQL) + cmd.AddCommand(gooseMySQLCmd) + return config.Init(config.FlagSets{ - Serve: serveCmd.Flags(), + Serve: serveCmd.Flags(), + GooseMySQL: gooseMySQLCmd.Flags(), }) } @@ -78,7 +85,7 @@ func (s *service) runServe(ctxStartup, ctxShutdown Ctx, shutdown func()) (err er } func (s *service) connectRepo(ctx Ctx) (interface{}, error) { - return dal.New(ctx) + return dal.New(ctx, s.cfg.MySQLGooseDir, s.cfg.MySQL) } func (s *service) serveMetrics(ctx Ctx) error { diff --git a/cmd/address-book/service_integration_test.go b/cmd/address-book/service_integration_test.go index d57b048..d47efb6 100644 --- a/cmd/address-book/service_integration_test.go +++ b/cmd/address-book/service_integration_test.go @@ -14,11 +14,17 @@ import ( "github.com/powerman/go-service-example/internal/srv/openapi" "github.com/powerman/go-service-example/pkg/def" "github.com/powerman/go-service-example/pkg/netx" + "github.com/powerman/mysqlx" ) func TestSmoke(tt *testing.T) { t := check.T(tt) + tempDBCfg, cleanup, err := mysqlx.EnsureTempDB(tLogger(*t), "", cfg.MySQL) + cfg.MySQL = tempDBCfg + t.Must(t.Nil(err)) + defer cleanup() + s := &service{cfg: cfg} ctxStartup, cancel := context.WithTimeout(ctx, def.TestTimeout) @@ -29,6 +35,9 @@ func TestSmoke(tt *testing.T) { defer func() { shutdown() t.Nil(<-errc, "RunServe") + if s.repo != nil { + s.repo.Close() + } }() t.Must(t.Nil(netx.WaitTCPPort(ctxStartup, cfg.Addr), "connect to service")) diff --git a/docker-compose.yml b/docker-compose.yml index 336f746..a28dd3b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,21 @@ version: "3.8" +volumes: + mysql: + services: + mysql: + image: "mysql:5.6" + container_name: example_mysql + restart: always + ports: + - "${example_mysql_addr_port:-0}:3306" + volumes: + - "mysql:/var/lib/mysql" + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + address-book: build: context: . @@ -14,3 +28,6 @@ services: - "${EXAMPLE_METRICS_ADDR_PORT:-0}:9000" environment: EXAMPLE_APIKEY_ADMIN: "${EXAMPLE_APIKEY_ADMIN:?}" + EXAMPLE_MYSQL_ADDR_HOST: "mysql" + EXAMPLE_MYSQL_AUTH_LOGIN: "root" + EXAMPLE_MYSQL_AUTH_PASS: "" diff --git a/env.sh.dist b/env.sh.dist index 3b2e969..bbf8463 100644 --- a/env.sh.dist +++ b/env.sh.dist @@ -5,11 +5,18 @@ # - Set all _PORT vars to port numbers not used by your system. +export example_mysql_addr_port="3306" + export EXAMPLE_APIKEY_ADMIN="admin" export EXAMPLE_ADDR_HOST="localhost" export EXAMPLE_ADDR_PORT="8000" export EXAMPLE_METRICS_ADDR_PORT="9000" +export EXAMPLE_MYSQL_ADDR_HOST="127.0.0.1" +export EXAMPLE_MYSQL_ADDR_PORT="${example_mysql_addr_port}" +export EXAMPLE_MYSQL_AUTH_LOGIN="root" +export EXAMPLE_MYSQL_AUTH_PASS="" export GO_TEST_TIME_FACTOR="1.0" # Increase if tests fail because of slow CPU. +export NARADA4D_TEST_MYSQL="goose-mysql://root@127.0.0.1:${example_mysql_addr_port}" # DO NOT MODIFY BELOW THIS LINE! env1="$(sed -e '/^$/d' -e '/^#/d' -e 's/=.*//' env.sh.dist)" diff --git a/go.mod b/go.mod index db643be..9fc33d4 100644 --- a/go.mod +++ b/go.mod @@ -11,12 +11,18 @@ require ( github.com/go-openapi/strfmt v0.19.5 github.com/go-openapi/swag v0.19.9 github.com/go-openapi/validate v0.19.11 + github.com/go-sql-driver/mysql v1.5.0 github.com/golang/mock v1.4.4 github.com/jessevdk/go-flags v1.4.0 + github.com/jmoiron/sqlx v1.2.0 github.com/powerman/appcfg v0.5.0 github.com/powerman/check v1.2.1 github.com/powerman/getenv v0.1.0 + github.com/powerman/goose/v2 v2.7.0 github.com/powerman/must v0.1.0 + github.com/powerman/mysqlx v0.3.3 + github.com/powerman/narada4d v1.7.0 + github.com/powerman/sqlxx v0.2.0 github.com/powerman/structlog v0.7.1 github.com/prometheus/client_golang v1.7.1 github.com/rs/cors v1.7.0 diff --git a/go.sum b/go.sum index 3245d9f..49cd680 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.0.0-20200206145737-bbfc9a55622e/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -123,6 +124,10 @@ github.com/go-openapi/validate v0.19.10 h1:tG3SZ5DC5KF4cyt7nqLVcQXGj5A7mpaYkAcNP github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8= github.com/go-openapi/validate v0.19.11 h1:8lCr0b9lNWKjVjW/hSZZvltUy+bULl7vbnCTsOzlhPo= github.com/go-openapi/validate v0.19.11/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= @@ -151,6 +156,7 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -178,8 +184,12 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= +github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -189,10 +199,13 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -211,6 +224,10 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -220,6 +237,10 @@ github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8 github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= +github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -246,12 +267,30 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/powerman/appcfg v0.5.0 h1:89ITgn7/nTEO+pn92PSqT9hzgZZ6BUTC+rTo3Z7WMew= github.com/powerman/appcfg v0.5.0/go.mod h1:jsBjFCRGB54w0EAUtDkbSnuxG6Y5YI//0v+Dq/LUgSY= +github.com/powerman/check v1.0.1/go.mod h1:wOYaLZeuhk6i79TDfPcqkDx70tbaVLEiE0aqMhMKojc= +github.com/powerman/check v1.1.0/go.mod h1:nX1qs/UsVgnQrkebazvKUJzMrtOItv4O0+Y79nhVi3Y= github.com/powerman/check v1.2.1 h1:x1io+fh3njxQEmbvONx5a9QWZXVwFP1BDPKNqK0uZAo= github.com/powerman/check v1.2.1/go.mod h1:IW+hYd9ihaKn7ri1+NGS8WWwrKdytbU8hgNC1wbcurc= github.com/powerman/getenv v0.1.0 h1:GTqwBYwtjoxjK/kB+qDyfUiqSz7vZ5431x002ftW13s= github.com/powerman/getenv v0.1.0/go.mod h1:kTSy/ckmNA/gYTbH6+XOUaGWUCeS9k7QzSol+LsOsZM= +github.com/powerman/goose v2.7.0-rc4.0.20200329145851-5c15923690fa+incompatible h1:50EFIy0MsWcUXblSAkgRxbJRA67gz+LPSmwLaqgMHhI= +github.com/powerman/goose v2.7.0-rc4.0.20200329145851-5c15923690fa+incompatible/go.mod h1:BJwHhs3GM6/vR+Cp2hI3l0gDsCSOeMyOIo4PLzXUNb4= +github.com/powerman/goose/v2 v2.7.0 h1:IxQY65kQiCj3u7SgrFM2pVOIR3kFrRkK0vQFxccJ7v8= +github.com/powerman/goose/v2 v2.7.0/go.mod h1:xhmEJVY4uJKRs76TzeDcoPhF8w4A8M98+E29eNdivHA= +github.com/powerman/gotest v0.2.0/go.mod h1:vR8gUJorGFev+TucLyhbbx566IelaJFxCSAcDYD4fdY= +github.com/powerman/gotest v0.3.0 h1:3HdlkyT8XoTIw220BvCP+yy7OhB1AxZ11Qfzn00l3YQ= +github.com/powerman/gotest v0.3.0/go.mod h1:vR8gUJorGFev+TucLyhbbx566IelaJFxCSAcDYD4fdY= github.com/powerman/must v0.1.0 h1:/Romq/snkUx7jU0RVxbaQWo8eoGV5LEFP7lMY2Paq7I= github.com/powerman/must v0.1.0/go.mod h1:agQ0zQd+ZBTI+I+KUL9G1l9ZPXTcjzOUfXvZUQpOVwo= +github.com/powerman/mysqlx v0.3.2 h1:OnRrIVR44wbdfaJqlgD6XoLN4sxMSdaCUUHtNno4VT4= +github.com/powerman/mysqlx v0.3.2/go.mod h1:ZSl17PWU2DgwM2KjN8OrdAq9tYC5p4Xm3lTN6W7j3SA= +github.com/powerman/mysqlx v0.3.3 h1:eBTeJvrLY+mcacJZekzM3VKCUT8gj/5KKG8f2KYXqzg= +github.com/powerman/mysqlx v0.3.3/go.mod h1:ZSl17PWU2DgwM2KjN8OrdAq9tYC5p4Xm3lTN6W7j3SA= +github.com/powerman/narada4d v1.7.0 h1:Mq4cTkzaMaLzHM2d2uy+6nuRHso+UdSkSWCjUUtN/z8= +github.com/powerman/narada4d v1.7.0/go.mod h1:l3ih9ucyUYBCaHqPz0pAcN/6NQ3cThGzYQAnXY5zRjs= +github.com/powerman/pqx v0.6.1/go.mod h1:EJGFN0+vvTWFqwT1bTrnVWrJ8hO7RAW6rp2JVhYwrVE= +github.com/powerman/sqlxx v0.2.0 h1:NA2Z+o7Q6Lu9UqwGFqqPePKWoPYZ5doOsBnfstCU1uU= +github.com/powerman/sqlxx v0.2.0/go.mod h1:dsfudBeSTo0U61OEAC/WS0Ea7qityww7jvFvWt3U1Fw= github.com/powerman/structlog v0.7.1 h1:68xN/GdxU7SYbXu/7P4RWyK7lKaN7FXjncarf7wO+Wk= github.com/powerman/structlog v0.7.1/go.mod h1:ksFcAlZ6WcmwQScaECqbijO3pEAKiSVKmJo/Vtpw8aM= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -292,6 +331,12 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 h1:hBSHahWMEgzwRyS6dRpxY0XyjZsHyQ61s084wo5PJe0= github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= +github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= +github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -323,6 +368,7 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -335,8 +381,10 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -350,6 +398,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -374,12 +423,16 @@ golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -399,11 +452,13 @@ golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/internal/config/config.go b/internal/config/config.go index 9fa95d7..10e2e5d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -8,7 +8,9 @@ package config import ( + "github.com/go-sql-driver/mysql" "github.com/powerman/appcfg" + "github.com/powerman/go-service-example/pkg/cobrax" "github.com/powerman/go-service-example/pkg/def" "github.com/powerman/go-service-example/pkg/netx" "github.com/spf13/pflag" @@ -26,15 +28,26 @@ var all = &struct { //nolint:gochecknoglobals // Config is global anyway. AddrHost appcfg.NotEmptyString `env:"ADDR_HOST"` AddrPort appcfg.Port `env:"ADDR_PORT"` MetricsAddrPort appcfg.Port `env:"METRICS_ADDR_PORT"` + MySQLAddrHost appcfg.NotEmptyString `env:"MYSQL_ADDR_HOST"` + MySQLAddrPort appcfg.Port `env:"MYSQL_ADDR_PORT"` + MySQLAuthLogin appcfg.NotEmptyString `env:"MYSQL_AUTH_LOGIN"` + MySQLAuthPass appcfg.String `env:"MYSQL_AUTH_PASS"` + MySQLDBName appcfg.NotEmptyString `env:"MYSQL_DB"` + MySQLGooseDir appcfg.NotEmptyString }{ // Defaults, if any: AddrHost: appcfg.MustNotEmptyString(def.Hostname), AddrPort: appcfg.MustPort("8000"), MetricsAddrPort: appcfg.MustPort("9000"), + MySQLAddrPort: appcfg.MustPort("3306"), + MySQLAuthLogin: appcfg.MustNotEmptyString(def.ProgName), + MySQLDBName: appcfg.MustNotEmptyString(def.ProgName), + MySQLGooseDir: appcfg.MustNotEmptyString("internal/migrations/mysql"), } // FlagSets for all CLI subcommands which use flags to set config values. type FlagSets struct { - Serve *pflag.FlagSet + Serve *pflag.FlagSet + GooseMySQL *pflag.FlagSet } var fs FlagSets //nolint:gochecknoglobals // Flags are global anyway. @@ -54,15 +67,28 @@ func Init(flagsets FlagSets) error { appcfg.AddPFlag(fs.Serve, &all.AddrHost, "host", "host to serve OpenAPI") appcfg.AddPFlag(fs.Serve, &all.AddrPort, "port", "port to serve OpenAPI") appcfg.AddPFlag(fs.Serve, &all.MetricsAddrPort, "metrics.port", "port to serve Prometheus metrics") + appcfg.AddPFlag(fs.Serve, &all.MySQLAddrHost, "mysql.host", "host to connect to MySQL") + appcfg.AddPFlag(fs.Serve, &all.MySQLAddrPort, "mysql.port", "port to connect to MySQL") + appcfg.AddPFlag(fs.Serve, &all.MySQLAuthLogin, "mysql.user", "MySQL username") + appcfg.AddPFlag(fs.Serve, &all.MySQLAuthPass, "mysql.pass", "MySQL password") + appcfg.AddPFlag(fs.Serve, &all.MySQLDBName, "mysql.dbname", "MySQL database name") + + appcfg.AddPFlag(fs.GooseMySQL, &all.MySQLAddrHost, "mysql.host", "host to connect to MySQL") + appcfg.AddPFlag(fs.GooseMySQL, &all.MySQLAddrPort, "mysql.port", "port to connect to MySQL") + appcfg.AddPFlag(fs.GooseMySQL, &all.MySQLAuthLogin, "mysql.user", "MySQL username") + appcfg.AddPFlag(fs.GooseMySQL, &all.MySQLAuthPass, "mysql.pass", "MySQL password") + appcfg.AddPFlag(fs.GooseMySQL, &all.MySQLDBName, "mysql.dbname", "MySQL database name") return nil } // ServeConfig contains configuration for subcommand. type ServeConfig struct { - APIKeyAdmin string - Addr netx.Addr - MetricsAddr netx.Addr + APIKeyAdmin string + Addr netx.Addr + MetricsAddr netx.Addr + MySQL *mysql.Config + MySQLGooseDir string } // GetServe validates and returns configuration for subcommand. @@ -73,6 +99,13 @@ func GetServe() (c *ServeConfig, err error) { APIKeyAdmin: all.APIKeyAdmin.Value(&err), Addr: netx.NewAddr(all.AddrHost.Value(&err), all.AddrPort.Value(&err)), MetricsAddr: netx.NewAddr(all.AddrHost.Value(&err), all.MetricsAddrPort.Value(&err)), + MySQL: def.NewMySQLConfig(def.MySQLConfig{ + Addr: netx.NewAddr(all.MySQLAddrHost.Value(&err), all.MySQLAddrPort.Value(&err)), + User: all.MySQLAuthLogin.Value(&err), + Pass: all.MySQLAuthPass.Value(&err), + DB: all.MySQLDBName.Value(&err), + }), + MySQLGooseDir: all.MySQLGooseDir.Value(&err), } if err != nil { return nil, appcfg.WrapPErr(err, fs.Serve, all) @@ -80,6 +113,24 @@ func GetServe() (c *ServeConfig, err error) { return c, nil } +func GetGooseMySQL() (c *cobrax.GooseMySQLConfig, err error) { + defer cleanup() + + c = &cobrax.GooseMySQLConfig{ + MySQL: def.NewMySQLConfig(def.MySQLConfig{ + Addr: netx.NewAddr(all.MySQLAddrHost.Value(&err), all.MySQLAddrPort.Value(&err)), + User: all.MySQLAuthLogin.Value(&err), + Pass: all.MySQLAuthPass.Value(&err), + DB: all.MySQLDBName.Value(&err), + }), + MySQLGooseDir: all.MySQLGooseDir.Value(&err), + } + if err != nil { + return nil, appcfg.WrapPErr(err, fs.GooseMySQL, all) + } + return c, nil +} + // Cleanup must be called by all Get* functions to ensure second call to // any of them will panic. func cleanup() { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4afd03a..315fdfb 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -14,10 +14,21 @@ func Test(t *testing.T) { APIKeyAdmin: "admin", Addr: netx.NewAddr(def.Hostname, 8000), MetricsAddr: netx.NewAddr(def.Hostname, 9000), + MySQL: def.NewMySQLConfig(def.MySQLConfig{ + Addr: netx.NewAddr("localhost", 3306), + User: "config", + Pass: "", + DB: "config", + }), + MySQLGooseDir: "internal/migrations/mysql", } t.Run("required", func(tt *testing.T) { t := check.T(tt) + require(t, "MySQLAuthPass") + os.Setenv("EXAMPLE_MYSQL_AUTH_PASS", "") + require(t, "MySQLAddrHost") + os.Setenv("EXAMPLE_MYSQL_ADDR_HOST", "localhost") require(t, "APIKeyAdmin") os.Setenv("EXAMPLE_APIKEY_ADMIN", "admin") }) @@ -31,6 +42,10 @@ func Test(t *testing.T) { t := check.T(tt) constraint(t, "EXAMPLE_ADDR_PORT", "x", `^AddrPort .* invalid syntax`) constraint(t, "EXAMPLE_METRICS_ADDR_PORT", "x", `^MetricsAddrPort .* invalid syntax`) + constraint(t, "EXAMPLE_MYSQL_ADDR_HOST", "", `^MySQLAddrHost .* empty`) + constraint(t, "EXAMPLE_MYSQL_ADDR_PORT", "x", `^MySQLAddrPort .* invalid syntax`) + constraint(t, "EXAMPLE_MYSQL_AUTH_LOGIN", "", `^MySQLAuthLogin .* empty`) + constraint(t, "EXAMPLE_MYSQL_DB", "", `^MySQLDBName .* empty`) }) t.Run("env", func(tt *testing.T) { t := check.T(tt) @@ -38,11 +53,22 @@ func Test(t *testing.T) { os.Setenv("EXAMPLE_ADDR_HOST", "localhost3") os.Setenv("EXAMPLE_ADDR_PORT", "8003") os.Setenv("EXAMPLE_METRICS_ADDR_PORT", "9003") + os.Setenv("EXAMPLE_MYSQL_ADDR_HOST", "mysql3") + os.Setenv("EXAMPLE_MYSQL_ADDR_PORT", "33306") + os.Setenv("EXAMPLE_MYSQL_AUTH_LOGIN", "user3") + os.Setenv("EXAMPLE_MYSQL_AUTH_PASS", "pass3") + os.Setenv("EXAMPLE_MYSQL_DB", "db3") c, err := testGetServe() t.Nil(err) want.APIKeyAdmin = "admin3" want.Addr = netx.NewAddr("localhost3", 8003) want.MetricsAddr = netx.NewAddr("localhost3", 9003) + want.MySQL = def.NewMySQLConfig(def.MySQLConfig{ + Addr: netx.NewAddr("mysql3", 33306), + User: "user3", + Pass: "pass3", + DB: "db3", + }) t.DeepEqual(c, want) }) t.Run("flag", func(tt *testing.T) { @@ -51,10 +77,21 @@ func Test(t *testing.T) { "--host=localhost4", "--port=8004", "--metrics.port=9004", + "--mysql.host=mysql4", + "--mysql.port=43306", + "--mysql.user=user4", + "--mysql.pass=pass4", + "--mysql.dbname=db4", ) t.Nil(err) want.Addr = netx.NewAddr("localhost4", 8004) want.MetricsAddr = netx.NewAddr("localhost4", 9004) + want.MySQL = def.NewMySQLConfig(def.MySQLConfig{ + Addr: netx.NewAddr("mysql4", 43306), + User: "user4", + Pass: "pass4", + DB: "db4", + }) t.DeepEqual(c, want) }) t.Run("cleanup", func(tt *testing.T) { diff --git a/internal/config/init_test.go b/internal/config/init_test.go index 1f98540..9a7fc8e 100644 --- a/internal/config/init_test.go +++ b/internal/config/init_test.go @@ -13,7 +13,8 @@ import ( var ( testAll = all testFlagsets = FlagSets{ - Serve: pflag.NewFlagSet("", 0), + Serve: pflag.NewFlagSet("", 0), + GooseMySQL: pflag.NewFlagSet("", 0), } ) diff --git a/internal/config/testing.go b/internal/config/testing.go index ac59d44..67229b1 100644 --- a/internal/config/testing.go +++ b/internal/config/testing.go @@ -1,6 +1,10 @@ package config import ( + "os" + "path/filepath" + + "github.com/powerman/go-service-example/pkg/def" "github.com/powerman/go-service-example/pkg/netx" "github.com/powerman/must" "github.com/spf13/pflag" @@ -9,15 +13,26 @@ import ( // MustGetServeTest returns config suitable for use in tests. func MustGetServeTest() *ServeConfig { err := Init(FlagSets{ - Serve: pflag.NewFlagSet("", pflag.ContinueOnError), + Serve: pflag.NewFlagSet("", pflag.ContinueOnError), + GooseMySQL: pflag.NewFlagSet("", pflag.ContinueOnError), }) must.NoErr(err) cfg, err := GetServe() must.NoErr(err) + cfg.MySQL.Timeout = def.TestTimeout + const host = "localhost" cfg.Addr = netx.NewAddr(host, netx.UnusedTCPPort(host)) cfg.MetricsAddr = netx.NewAddr(host, 0) + rootDir, err := os.Getwd() + must.NoErr(err) + for _, err := os.Stat(filepath.Join(rootDir, "go.mod")); os.IsNotExist(err) && filepath.Dir(rootDir) != rootDir; _, err = os.Stat(filepath.Join(rootDir, "go.mod")) { + rootDir = filepath.Dir(rootDir) + } + + cfg.MySQLGooseDir = filepath.Join(rootDir, cfg.MySQLGooseDir) + return cfg } diff --git a/internal/dal/mysql/dal.go b/internal/dal/mysql/dal.go new file mode 100644 index 0000000..7895578 --- /dev/null +++ b/internal/dal/mysql/dal.go @@ -0,0 +1,49 @@ +// Package dal implements Data Access Layer using MySQL DB. +package dal + +import ( + "context" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/powerman/go-service-example/internal/app" + migrations "github.com/powerman/go-service-example/internal/migrations/mysql" + "github.com/powerman/go-service-example/pkg/repo" +) + +const ( + schemaVersion = 3 + dbMaxOpenConns = 0 // Unlimited. + dbMaxIdleConns = 5 // A bit more than default (2). +) + +type Ctx = context.Context + +// Repo provides access to storage. +type Repo struct { + *repo.Repo +} + +// New creates and returns new Repo. +// It will also run required DB migrations and connects to DB. +func New(ctx Ctx, dir string, cfg *mysql.Config) (_ *Repo, err error) { + returnErrs := []error{ // List of app.Err… returned by Repo methods. + app.ErrContactExists, + } + + r := &Repo{} + r.Repo, err = repo.New(ctx, migrations.Goose(), repo.Config{ + MySQL: cfg, + GooseDir: dir, + SchemaVersion: schemaVersion, + Metric: metric, + ReturnErrs: returnErrs, + }) + if err != nil { + return nil, err + } + r.DB.SetMaxOpenConns(dbMaxOpenConns) + r.DB.SetMaxIdleConns(dbMaxIdleConns) + r.SchemaVer.HoldSharedLock(ctx, time.Second) + return r, nil +} diff --git a/internal/dal/mysql/init_integration_test.go b/internal/dal/mysql/init_integration_test.go new file mode 100644 index 0000000..21e5f5d --- /dev/null +++ b/internal/dal/mysql/init_integration_test.go @@ -0,0 +1,56 @@ +// +build integration + +package dal_test + +import ( + "context" + "runtime" + "strings" + "testing" + + "github.com/powerman/check" + "github.com/powerman/go-service-example/internal/app" + "github.com/powerman/go-service-example/internal/config" + dal "github.com/powerman/go-service-example/internal/dal/mysql" + "github.com/powerman/go-service-example/pkg/def" + "github.com/powerman/mysqlx" + "github.com/prometheus/client_golang/prometheus" +) + +func TestMain(m *testing.M) { + def.Init() + reg := prometheus.NewPedanticRegistry() + app.InitMetrics(reg) + dal.InitMetrics(reg, "test") + cfg = config.MustGetServeTest() + check.TestMain(m) +} + +var ( + ctx = context.Background() + cfg *config.ServeConfig +) + +type tLogger check.C + +func (t tLogger) Print(args ...interface{}) { t.Log(args...) } + +func newTestRepo(t *check.C) (cleanup func(), r *dal.Repo) { + t.Helper() + + pc, _, _, _ := runtime.Caller(1) + suffix := runtime.FuncForPC(pc).Name() + suffix = suffix[:strings.LastIndex(suffix, ".")] + suffix += "_" + t.Name() + + tempDBCfg, cleanupDB, err := mysqlx.EnsureTempDB(tLogger(*t), suffix, cfg.MySQL) + t.Must(t.Nil(err)) + r, err = dal.New(ctx, cfg.MySQLGooseDir, tempDBCfg) + t.Must(t.Nil(err)) + + cleanup = func() { + r.Close() + cleanupDB() + } + return cleanup, r +} diff --git a/internal/dal/mysql/methods.go b/internal/dal/mysql/methods.go new file mode 100644 index 0000000..8c12ab4 --- /dev/null +++ b/internal/dal/mysql/methods.go @@ -0,0 +1,44 @@ +package dal + +import ( + "github.com/powerman/go-service-example/internal/app" + "github.com/powerman/go-service-example/pkg/repo" +) + +func (r *Repo) AddContact(ctx Ctx, name string) (id int, err error) { + err = r.NoTx(func() error { + res, err := r.DB.NamedExecContext(ctx, sqlContactAdd, argContactAdd{ + Name: name, + }) + switch { + case repo.DuplicateEntry(err): + return app.ErrContactExists + case err != nil: + return err + default: + insertID, err := res.LastInsertId() + id = int(insertID) + return err + } + }) + return +} + +func (r *Repo) LstContacts(ctx Ctx, page app.SeekPage) (contacts []app.Contact, err error) { + err = r.NoTx(func() error { + var rows []rowContactLst + err := r.DB.NamedSelectContext(ctx, &rows, sqlContactLst, argContactLst{ + SinceID: page.SinceID, + Limit: page.Limit, + }) + if err != nil { + return err + } + contacts = make([]app.Contact, len(rows)) + for i := range rows { + contacts[i] = appContact(rows[i]) + } + return nil + }) + return +} diff --git a/internal/dal/mysql/methods_integration_test.go b/internal/dal/mysql/methods_integration_test.go new file mode 100644 index 0000000..3c5792d --- /dev/null +++ b/internal/dal/mysql/methods_integration_test.go @@ -0,0 +1,68 @@ +// +build integration + +package dal_test + +import ( + "testing" + + "github.com/powerman/check" + "github.com/powerman/go-service-example/internal/app" +) + +func TestContact(tt *testing.T) { + t := check.T(tt) + t.Parallel() + cleanup, r := newTestRepo(t) + defer cleanup() + + var ( + c1 = app.Contact{ID: 1, Name: "A"} + c3 = app.Contact{ID: 3, Name: "B"} + c4 = app.Contact{ID: 4, Name: "C"} + ) + + contacts, err := r.LstContacts(ctx, app.SeekPage{SinceID: 0, Limit: 2}) + t.Nil(err) + t.Len(contacts, 0) + + testsAdd := []struct { + name string + want int + wantErr error + }{ + {c1.Name, c1.ID, nil}, + {c1.Name, 0, app.ErrContactExists}, + {c3.Name, c3.ID, nil}, + {c4.Name, c4.ID, nil}, + } + for _, tc := range testsAdd { + tc := tc + t.Run("", func(tt *testing.T) { + t := check.T(tt) //nolint:govet // False positive. + res, err := r.AddContact(ctx, tc.name) + t.Err(err, tc.wantErr) + t.Equal(res, tc.want) + }) + } + + testsLst := []struct { + page app.SeekPage + want []app.Contact + wantErr error + }{ + {app.SeekPage{SinceID: 0, Limit: 0}, []app.Contact{}, nil}, + {app.SeekPage{SinceID: 0, Limit: 2}, []app.Contact{c1, c3}, nil}, + {app.SeekPage{SinceID: 2, Limit: 5}, []app.Contact{c3, c4}, nil}, + {app.SeekPage{SinceID: c3.ID, Limit: 2}, []app.Contact{c4}, nil}, + {app.SeekPage{SinceID: c4.ID, Limit: 2}, []app.Contact{}, nil}, + } + for _, tc := range testsLst { + tc := tc + t.Run("", func(tt *testing.T) { + t := check.T(tt) + res, err := r.LstContacts(ctx, tc.page) + t.Err(err, tc.wantErr) + t.DeepEqual(res, tc.want) + }) + } +} diff --git a/internal/dal/mysql/metrics.go b/internal/dal/mysql/metrics.go new file mode 100644 index 0000000..b38809a --- /dev/null +++ b/internal/dal/mysql/metrics.go @@ -0,0 +1,15 @@ +package dal + +import ( + "github.com/powerman/go-service-example/internal/app" + "github.com/powerman/go-service-example/pkg/repo" + "github.com/prometheus/client_golang/prometheus" +) + +var metric repo.Metrics //nolint:gochecknoglobals // Metrics are global anyway. + +func InitMetrics(reg *prometheus.Registry, namespace string) { + const subsystem = "dal_mysql" + + metric = repo.NewMetrics(reg, namespace, subsystem, new(app.Repo)) +} diff --git a/internal/dal/mysql/models.go b/internal/dal/mysql/models.go new file mode 100644 index 0000000..fa470c8 --- /dev/null +++ b/internal/dal/mysql/models.go @@ -0,0 +1,10 @@ +package dal + +import "github.com/powerman/go-service-example/internal/app" + +func appContact(v rowContactLst) app.Contact { + return app.Contact{ + ID: v.ID, + Name: v.Name, + } +} diff --git a/internal/dal/mysql/sql.go b/internal/dal/mysql/sql.go new file mode 100644 index 0000000..60cf9bc --- /dev/null +++ b/internal/dal/mysql/sql.go @@ -0,0 +1,34 @@ +package dal + +import ( + "time" +) + +const ( + sqlContactAdd = ` +INSERT INTO Contact (name) VALUES (:name) + ` + sqlContactLst = ` +SELECT id, name, ctime +FROM Contact +WHERE id > :since_id +ORDER BY id ASC +LIMIT :limit + ` +) + +type ( + argContactAdd struct { + Name string + } + + argContactLst struct { + SinceID int + Limit int + } + rowContactLst struct { + ID int + Name string + Ctime time.Time + } +) diff --git a/internal/dal/mysql/test.goconvey b/internal/dal/mysql/test.goconvey new file mode 100644 index 0000000..1cff4fb --- /dev/null +++ b/internal/dal/mysql/test.goconvey @@ -0,0 +1 @@ +-tags=integration diff --git a/internal/migrations/mysql/00001_down_not_supported.sql b/internal/migrations/mysql/00001_down_not_supported.sql new file mode 100644 index 0000000..d781a46 --- /dev/null +++ b/internal/migrations/mysql/00001_down_not_supported.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- SQL in this section is executed when the migration is applied. +CREATE PROCEDURE down_not_supported() + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='downgrade is not supported, restore from backup instead'; + +-- +goose Down +-- SQL in this section is executed when the migration is rolled back. +-- Example usage: CALL down_not_supported(); +DROP PROCEDURE down_not_supported; diff --git a/internal/migrations/mysql/00002_noop.go b/internal/migrations/mysql/00002_noop.go new file mode 100644 index 0000000..6a93bd9 --- /dev/null +++ b/internal/migrations/mysql/00002_noop.go @@ -0,0 +1,20 @@ +package migrations + +import ( + "database/sql" +) + +// This is just an example how to define Go migrations. +func init() { + goose.AddMigration(upNoop, downNoop) +} + +func upNoop(tx *sql.Tx) error { + // This code is executed when the migration is applied. + return nil +} + +func downNoop(tx *sql.Tx) error { + // This code is executed when the migration is rolled back. + return nil // migrate.ErrDownNotSupported +} diff --git a/internal/migrations/mysql/00003_create_contact.sql b/internal/migrations/mysql/00003_create_contact.sql new file mode 100644 index 0000000..69503eb --- /dev/null +++ b/internal/migrations/mysql/00003_create_contact.sql @@ -0,0 +1,13 @@ +-- +goose Up +-- SQL in this section is executed when the migration is applied. +CREATE TABLE IF NOT EXISTS Contact ( + id INT(11) NOT NULL AUTO_INCREMENT, + name VARCHAR(191) NOT NULL, + ctime DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE KEY (name), + PRIMARY KEY (id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- +goose Down +-- SQL in this section is executed when the migration is rolled back. +DROP TABLE Contact; diff --git a/internal/migrations/mysql/goose.go b/internal/migrations/mysql/goose.go new file mode 100644 index 0000000..6c28438 --- /dev/null +++ b/internal/migrations/mysql/goose.go @@ -0,0 +1,13 @@ +// Package migrations provides goose migrations. +package migrations + +import ( + "github.com/powerman/go-service-example/pkg/def" + goosepkg "github.com/powerman/goose/v2" +) + +//nolint:gochecknoglobals // Force code generated by `goose create` to use instance. +var goose = def.NewGoose() + +// Goose returns goose instance with Go migrations defined in the package. +func Goose() *goosepkg.Instance { return goose } diff --git a/internal/migrations/mysql/integration_test.go b/internal/migrations/mysql/integration_test.go new file mode 100644 index 0000000..957efd0 --- /dev/null +++ b/internal/migrations/mysql/integration_test.go @@ -0,0 +1,28 @@ +// +build integration + +package migrations_test + +import ( + "context" + "testing" + + "github.com/powerman/check" + "github.com/powerman/go-service-example/internal/config" + migrations "github.com/powerman/go-service-example/internal/migrations/mysql" + "github.com/powerman/go-service-example/pkg/def" + "github.com/powerman/go-service-example/pkg/migrate" +) + +var cfg *config.ServeConfig + +func TestMain(m *testing.M) { + def.Init() + cfg = config.MustGetServeTest() + check.TestMain(m) +} + +func Test(tt *testing.T) { + t := check.T(tt) + ctx := context.Background() + migrate.UpDownTest(t, ctx, migrations.Goose(), ".", cfg.MySQL) +} diff --git a/internal/migrations/mysql/test.goconvey b/internal/migrations/mysql/test.goconvey new file mode 100644 index 0000000..1cff4fb --- /dev/null +++ b/internal/migrations/mysql/test.goconvey @@ -0,0 +1 @@ +-tags=integration diff --git a/pkg/cobrax/goose-mysql.go b/pkg/cobrax/goose-mysql.go new file mode 100644 index 0000000..1891085 --- /dev/null +++ b/pkg/cobrax/goose-mysql.go @@ -0,0 +1,44 @@ +package cobrax + +import ( + "context" + "fmt" + "strings" + + "github.com/go-sql-driver/mysql" + "github.com/powerman/go-service-example/pkg/migrate" + goosepkg "github.com/powerman/goose/v2" + "github.com/spf13/cobra" +) + +// GooseMySQLConfig contain configuration for goose command. +type GooseMySQLConfig struct { + MySQL *mysql.Config + MySQLGooseDir string +} + +// NewGooseMySQLCmd creates new goose command executed by run. +func NewGooseMySQLCmd(goose *goosepkg.Instance, getCfg func() (*GooseMySQLConfig, error)) *cobra.Command { + cmd := &cobra.Command{ + Use: "goose-mysql", + Short: "Migrate MySQL database schema", + Args: gooseArgs, + RunE: func(cmd *cobra.Command, args []string) error { + gooseCmd := strings.Join(args, " ") + + ctx := context.Background() + cfg, err := getCfg() + if err != nil { + return fmt.Errorf("failed to get config: %w", err) + } + + err = migrate.Run(ctx, goose, cfg.MySQLGooseDir, gooseCmd, cfg.MySQL) + if err != nil { + return fmt.Errorf("failed to run goose %s: %w", gooseCmd, err) + } + return nil + }, + } + cmd.SetUsageTemplate(gooseUsageTemplate) + return cmd +} diff --git a/pkg/cobrax/goose.go b/pkg/cobrax/goose.go new file mode 100644 index 0000000..7a67dc4 --- /dev/null +++ b/pkg/cobrax/goose.go @@ -0,0 +1,63 @@ +package cobrax + +import ( + "errors" + "fmt" + "regexp" + "strings" + + "github.com/spf13/cobra" +) + +// gooseUsageTemplate is cobra usage template for goose commands. +const gooseUsageTemplate = `Usage: + {{.CommandPath}} [command]{{if gt (len .Aliases) 0}} + +Aliases: + {{.NameAndAliases}}{{end}}{{if .HasExample}} + +Examples: +{{.Example}}{{end}} + +Available Commands: + up Migrate the DB to the most recent version available + up-by-one Migrate the DB up by 1 + up-to VERSION Migrate the DB to a specific VERSION + down Roll back the version by 1 + down-to VERSION Roll back to a specific VERSION + redo Re-run the latest migration + reset Roll back all migrations + status Dump the migration status for the current DB + version Print the current version of the database + create NAME [sql|go] Creates new migration file with the current timestamp + fix Apply sequential ordering to migrations{{if .HasAvailableLocalFlags}} + +Flags: +{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} + +Global Flags: +{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}} + +Additional help topics:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}} + {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}} +` + +var ( + reGooseCommand = regexp.MustCompile(`^(?:up|up-by-one|up-to\s+\d+|down|down-to\s+\d+|redo|reset|status|version|create\s+\S+\s+(?:go|sql)|fix)$`) //nolint:gochecknoglobals // Regexp. + errInvalidCommand = errors.New("invalid goose command") +) + +// validGooseCommand returns true if command is a valid goose command. +func validGooseCommand(command string) bool { + return reGooseCommand.MatchString(command) +} + +func gooseArgs(cmd *cobra.Command, args []string) error { + gooseCmd := strings.Join(args, " ") + if gooseCmd == "" { + return ErrRequireFlagOrCommand + } else if !validGooseCommand(gooseCmd) { + return fmt.Errorf("%w: %s", errInvalidCommand, gooseCmd) + } + return nil +} diff --git a/pkg/def/def.go b/pkg/def/def.go index 49f2c56..4d883e5 100644 --- a/pkg/def/def.go +++ b/pkg/def/def.go @@ -6,8 +6,10 @@ import ( "net/http" "time" + "github.com/jmoiron/sqlx" "github.com/powerman/getenv" "github.com/powerman/must" + "github.com/powerman/sqlxx" "github.com/prometheus/client_golang/prometheus" ) @@ -26,6 +28,8 @@ func Init() error { must.AbortIf = must.PanicIf + sqlx.NameMapper = sqlxx.ToSnake + setupLog() if hostnameErr != nil { diff --git a/pkg/def/goose.go b/pkg/def/goose.go new file mode 100644 index 0000000..0506dce --- /dev/null +++ b/pkg/def/goose.go @@ -0,0 +1,17 @@ +package def + +import ( + "github.com/powerman/goose/v2" + "github.com/powerman/structlog" +) + +// NewGoose creates a goose instance with configured logger. +func NewGoose() *goose.Instance { + log := structlog.New(structlog.KeyUnit, "goose"). + SetKeysFormat(map[string]string{ + structlog.KeyMessage: " %[2]s", + }) + g := goose.NewInstance() + g.SetLogger(log) + return g +} diff --git a/pkg/def/mysql.go b/pkg/def/mysql.go new file mode 100644 index 0000000..84d382f --- /dev/null +++ b/pkg/def/mysql.go @@ -0,0 +1,31 @@ +package def + +import ( + "github.com/go-sql-driver/mysql" + "github.com/powerman/go-service-example/pkg/netx" +) + +// MySQLConfig contains MySQL connection and authentication details. +type MySQLConfig struct { + Addr netx.Addr + User string + Pass string + DB string +} + +// NewMySQLConfig creates a new default config for MySQL. +func NewMySQLConfig(cfg MySQLConfig) *mysql.Config { + c := mysql.NewConfig() + c.User = cfg.User + c.Passwd = cfg.Pass + c.Net = "tcp" + c.Addr = cfg.Addr.String() + c.DBName = cfg.DB + c.Params = map[string]string{ + "sql_mode": "'TRADITIONAL'", // 5.6 defaults + all strict modes. + } + c.Collation = "utf8mb4_unicode_ci" + c.ParseTime = true + c.RejectReadOnly = true + return c +} diff --git a/pkg/migrate/error_down_not_supported.go b/pkg/migrate/error_down_not_supported.go new file mode 100644 index 0000000..f5e46c8 --- /dev/null +++ b/pkg/migrate/error_down_not_supported.go @@ -0,0 +1,7 @@ +package migrate + +import "errors" + +// ErrDownNotSupported must be returned from goose Down function in case +// this migration does not support downgrade. +var ErrDownNotSupported = errors.New("downgrade is not supported, restore from backup instead") diff --git a/pkg/migrate/goose.go b/pkg/migrate/goose.go new file mode 100644 index 0000000..59aa148 --- /dev/null +++ b/pkg/migrate/goose.go @@ -0,0 +1,137 @@ +// Package migrate manage DB migrations. +package migrate + +import ( + "context" + "database/sql" + "errors" + "fmt" + "regexp" + "strings" + + "github.com/go-sql-driver/mysql" + goosepkg "github.com/powerman/goose/v2" + "github.com/powerman/must" + + // Driver. + _ "github.com/powerman/narada4d/protocol/goose-mysql" + "github.com/powerman/narada4d/schemaver" + "github.com/powerman/structlog" +) + +// Ctx is a synonym for convenience. +type Ctx = context.Context + +var errSelfCheck = errors.New("unexpected db schema version") + +//nolint:gochecknoglobals // Regexp. +var reTCP = regexp.MustCompile(`(^|@)tcp[(]([^)]*)[)]`) + +func connect(ctx Ctx, goose *goosepkg.Instance, cfg *mysql.Config) (db *sql.DB, ver *schemaver.SchemaVer, err error) { + log := structlog.FromContext(ctx, nil) + + cfg = cfg.Clone() + cfg.MaxAllowedPacket = 0 + cfg.MultiStatements = true // https://github.com/pressly/goose/issues/190 + + db, err = sql.Open("mysql", cfg.FormatDSN()) + if err != nil { + return nil, nil, fmt.Errorf("sql.Open: %w", err) + } + defer func(dbClose func() error) { + if err != nil { + log.WarnIfFail(dbClose) + } + }(db.Close) + + if cfg.Timeout != 0 { + var cancel func() + ctx, cancel = context.WithTimeout(ctx, cfg.Timeout) + defer cancel() + } + err = db.PingContext(ctx) + if err2 := new(mysql.MySQLError); errors.As(err, &err2) && err2.Number == 1049 { + cfgNoDB := cfg.Clone() + cfgNoDB.DBName = "" + db2, err := sql.Open("mysql", cfgNoDB.FormatDSN()) + if err != nil { + return nil, nil, fmt.Errorf("sql.Open: %w", err) + } + _, err = db2.ExecContext(ctx, fmt.Sprintf( + "CREATE DATABASE IF NOT EXISTS `%s` COLLATE %s", cfg.DBName, cfg.Collation)) + log.WarnIfFail(db2.Close) + if err != nil { + return nil, nil, fmt.Errorf("create database %q: %w", cfg.DBName, err) + } + } + for err != nil { + nextErr := db.PingContext(ctx) + if errors.Is(nextErr, context.DeadlineExceeded) || errors.Is(nextErr, context.Canceled) { + return nil, nil, fmt.Errorf("db.Ping: %w", err) + } + err = nextErr + } + + must.NoErr(goose.SetDialect("mysql")) + _, _ = goose.EnsureDBVersion(db) // Race on CREATE TABLE, so allowed to fail. + + ver, err = schemaver.NewAt("goose-mysql://" + reTCP.ReplaceAllString(cfg.FormatDSN(), "$1$2")) + if err != nil { + return nil, nil, err + } + + return db, ver, nil +} + +// UpTo migrates up to a specific version. +// +// Unlike goose.UpTo it will return error is current version doesn't match +// requested one after migration. +func UpTo(ctx Ctx, goose *goosepkg.Instance, dir string, version int64, cfg *mysql.Config) (*schemaver.SchemaVer, error) { + log := structlog.FromContext(ctx, nil) + + db, ver, err := connect(ctx, goose, cfg) + if err != nil { + return nil, err + } + defer log.WarnIfFail(db.Close) + + _ = ver.ExclusiveLock() + defer ver.Unlock() + + err = goose.UpTo(db, dir, version) + if err != nil { + return nil, fmt.Errorf("goose.UpTo %d: %w", version, err) + } + if v, _ := goose.GetDBVersion(db); v != version { + return nil, fmt.Errorf("%w: %d (should be %d)", errSelfCheck, v, version) + } + + return ver, nil +} + +// Run executes goose command. It also enforce "fix" after "create". +func Run(ctx Ctx, goose *goosepkg.Instance, dir string, command string, cfg *mysql.Config) error { + log := structlog.FromContext(ctx, nil) + + db, ver, err := connect(ctx, goose, cfg) + if err != nil { + return err + } + defer log.WarnIfFail(db.Close) + defer log.WarnIfFail(ver.Close) + + _ = ver.ExclusiveLock() + defer ver.Unlock() + + cmdArgs := strings.Fields(command) + cmd, args := cmdArgs[0], cmdArgs[1:] + err = goose.Run(cmd, db, dir, args...) + if err == nil && cmd == "create" { + err = goose.Run("fix", db, dir) + } + if err != nil { + return fmt.Errorf("goose.Run %q: %w", command, err) + } + return nil +} diff --git a/pkg/migrate/testing_integration.go b/pkg/migrate/testing_integration.go new file mode 100644 index 0000000..eab39d2 --- /dev/null +++ b/pkg/migrate/testing_integration.go @@ -0,0 +1,51 @@ +// +build integration + +package migrate + +import ( + "runtime" + "strings" + + "github.com/go-sql-driver/mysql" + "github.com/powerman/check" + goosepkg "github.com/powerman/goose/v2" + "github.com/powerman/mysqlx" +) + +type tLogger check.C + +func (t tLogger) Print(args ...interface{}) { t.Log(args...) } + +// UpDownTest creates temporary database, test given migrations, and removes +// temporary database. +func UpDownTest(t *check.C, ctx Ctx, goose *goosepkg.Instance, dir string, cfg *mysql.Config) { + pc, _, _, _ := runtime.Caller(1) + suffix := runtime.FuncForPC(pc).Name() + suffix = suffix[:strings.LastIndex(suffix, ".")] + + cfg, cleanup, err := mysqlx.EnsureTempDB(tLogger(*t), suffix, cfg) + t.Must(t.Nil(err)) + defer cleanup() + + db, _, err := connect(ctx, goose, cfg) + t.Must(t.Nil(err)) + defer db.Close() + + t.Must(t.Nil(Run(ctx, goose, dir, "up", cfg))) + for v, _ := goose.GetDBVersion(db); v > 0; v, _ = goose.GetDBVersion(db) { + err := Run(ctx, goose, dir, "down", cfg) + if err != nil && t.Contains(err.Error(), ErrDownNotSupported.Error()) { + t.Logf("downgrade from version %d is not supported", v) + t.Nil(Run(ctx, goose, dir, "up", cfg)) + return + } + t.Must(t.Nil(err)) + v2, err := goose.GetDBVersion(db) + t.Nil(err) + t.Less(v2, v) + } + v, err := goose.GetDBVersion(db) + t.Nil(err) + t.Zero(v) + t.Nil(Run(ctx, goose, dir, "up", cfg)) +} diff --git a/pkg/reflectx/reflectx.go b/pkg/reflectx/reflectx.go new file mode 100644 index 0000000..79e49ba --- /dev/null +++ b/pkg/reflectx/reflectx.go @@ -0,0 +1,30 @@ +// Package reflectx provide helpers for reflect. +package reflectx + +import ( + "reflect" + "runtime" + "strings" +) + +// MethodsOf require pointer to interface (e.g.: new(app.Appl)) and +// returns all it methods. +func MethodsOf(v interface{}) []string { + typ := reflect.TypeOf(v) + if typ.Kind() != reflect.Ptr || typ.Elem().Kind() != reflect.Interface { + panic("require pointer to interface") + } + typ = typ.Elem() + methods := make([]string, typ.NumMethod()) + for i := 0; i < typ.NumMethod(); i++ { + methods[i] = typ.Method(i).Name + } + return methods +} + +// CallerMethodName returns caller's method name for given stack depth. +func CallerMethodName(skip int) string { + pc, _, _, _ := runtime.Caller(1 + skip) + names := strings.Split(runtime.FuncForPC(pc).Name(), ".") + return names[len(names)-1] +} diff --git a/pkg/repo/metrics.go b/pkg/repo/metrics.go new file mode 100644 index 0000000..a21e1b9 --- /dev/null +++ b/pkg/repo/metrics.go @@ -0,0 +1,68 @@ +package repo + +import ( + "time" + + "github.com/powerman/go-service-example/pkg/reflectx" + "github.com/prometheus/client_golang/prometheus" +) + +// Metrics contains general metrics for DAL methods. +type Metrics struct { + callErrTotal *prometheus.CounterVec + callDuration *prometheus.HistogramVec +} + +const methodLabel = "method" + +// NewMetrics registers and returns common DAL metrics used by all +// services (namespace). +func NewMetrics(reg *prometheus.Registry, namespace, subsystem string, methodsFrom interface{}) (metric Metrics) { + metric.callErrTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "errors_total", + Help: "Amount of DAL errors.", + }, + []string{methodLabel}, + ) + reg.MustRegister(metric.callErrTotal) + metric.callDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "call_duration_seconds", + Help: "DAL call latency.", + }, + []string{methodLabel}, + ) + reg.MustRegister(metric.callDuration) + + for _, methodName := range reflectx.MethodsOf(methodsFrom) { + l := prometheus.Labels{ + methodLabel: methodName, + } + metric.callErrTotal.With(l) + metric.callDuration.With(l) + } + + return metric +} + +func (m Metrics) instrument(method string, f func() error) func() error { + return func() (err error) { + start := time.Now() + l := prometheus.Labels{methodLabel: method} + defer func() { + m.callDuration.With(l).Observe(time.Since(start).Seconds()) + if err != nil { + m.callErrTotal.With(l).Inc() + } else if err := recover(); err != nil { + m.callErrTotal.With(l).Inc() + panic(err) + } + }() + return f() + } +} diff --git a/pkg/repo/repo.go b/pkg/repo/repo.go new file mode 100644 index 0000000..dd471c1 --- /dev/null +++ b/pkg/repo/repo.go @@ -0,0 +1,190 @@ +// Package repo provide helpers for Data Access Layer. +package repo + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strconv" + + "github.com/go-sql-driver/mysql" + "github.com/jmoiron/sqlx" + "github.com/powerman/go-service-example/pkg/migrate" + "github.com/powerman/go-service-example/pkg/reflectx" + goosepkg "github.com/powerman/goose/v2" + "github.com/powerman/narada4d/schemaver" + "github.com/powerman/sqlxx" + "github.com/powerman/structlog" +) + +// Ctx is a synonym for convenience. +type Ctx = context.Context + +// MaxKeySize for indexed MySQL utf8mb4 CHAR/VARCHAR column. +const MaxKeySize = 191 + +// Errors. +var ( + ErrSchemaVer = errors.New("unsupported DB schema version") +) + +// DuplicateEntry returns true if err is mysql error "Duplicate entry…". +func DuplicateEntry(err error) bool { + const duplicateEntry = 1062 + if errMySQL := new(mysql.MySQLError); errors.As(err, &errMySQL) { + return errMySQL.Number == duplicateEntry + } + return false +} + +// Config contains repo configuration. +type Config struct { + MySQL *mysql.Config + GooseDir string + SchemaVersion int64 + Metric Metrics + ReturnErrs []error // List of app.Err… returned by DAL methods. +} + +// Repo provides access to storage. +type Repo struct { + DB *sqlxx.DB + SchemaVer *schemaver.SchemaVer + schemaVersion string + returnErrs []error + metric Metrics + log *structlog.Logger +} + +// New creates and returns new Repo. +// It will also run required DB migrations and connects to DB. +func New(ctx Ctx, goose *goosepkg.Instance, cfg Config) (*Repo, error) { + log := structlog.FromContext(ctx, nil) + + schemaVer, err := migrate.UpTo(ctx, goose, cfg.GooseDir, cfg.SchemaVersion, cfg.MySQL) + if err != nil { + return nil, fmt.Errorf("migration: %w", err) + } + + db, err := sql.Open("mysql", cfg.MySQL.FormatDSN()) + if err != nil { + log.WarnIfFail(schemaVer.Close) + return nil, fmt.Errorf("sql.Open: %w", err) + } + + if cfg.MySQL.Timeout != 0 { + var cancel func() + ctx, cancel = context.WithTimeout(ctx, cfg.MySQL.Timeout) + defer cancel() + } + err = db.PingContext(ctx) + for err != nil { + nextErr := db.PingContext(ctx) + if errors.Is(nextErr, context.DeadlineExceeded) || errors.Is(nextErr, context.Canceled) { + log.WarnIfFail(db.Close) + log.WarnIfFail(schemaVer.Close) + return nil, fmt.Errorf("db.Ping: %w", err) + } + err = nextErr + } + + r := &Repo{ + DB: sqlxx.NewDB(sqlx.NewDb(db, "mysql")), + SchemaVer: schemaVer, + schemaVersion: strconv.Itoa(int(cfg.SchemaVersion)), + returnErrs: cfg.ReturnErrs, + metric: cfg.Metric, + log: log, + } + return r, nil +} + +// Close closes connection to DB. +func (r *Repo) Close() { + r.log.WarnIfFail(r.DB.Close) + r.log.WarnIfFail(r.SchemaVer.Close) +} + +// Turn sqlx errors like `missing destination …` into panics +// https://github.com/jmoiron/sqlx/issues/529. As we can't distinguish +// between sqlx and other errors except driver ones, let's hope filtering +// driver errors is enough and there are no other non-driver regular errors. +func (r *Repo) strict(err error) error { + switch { + case err == nil: + case errors.As(err, new(*mysql.MySQLError)): + case errors.Is(err, ErrSchemaVer): + case errors.Is(err, sql.ErrNoRows): + case errors.Is(err, context.Canceled): + case errors.Is(err, context.DeadlineExceeded): + default: + for i := range r.returnErrs { + if errors.Is(err, r.returnErrs[i]) { + return err + } + } + panic(err) + } + return err +} + +func (r *Repo) schemaLock(f func() error) error { + ver := r.SchemaVer.SharedLock() + defer r.SchemaVer.Unlock() + if ver != r.schemaVersion { + return fmt.Errorf("schema version %s, need %s: %w", ver, r.schemaVersion, ErrSchemaVer) + } + return f() +} + +// NoTx provides DAL method wrapper with: +// - converting sqlx errors which are actually bugs into panics, +// - ensure valid schema version while accessing DB, +// - general metrics for DAL methods, +// - wrapping errors with DAL method name. +func (r *Repo) NoTx(f func() error) (err error) { + methodName := reflectx.CallerMethodName(1) + return r.strict(r.schemaLock(r.metric.instrument(methodName, func() error { + err := f() + if err != nil { + err = fmt.Errorf("%s: %w", methodName, err) + } + return err + }))) +} + +// Tx provides DAL method wrapper with: +// - converting sqlx errors which are actually bugs into panics, +// - ensure valid schema version while accessing DB, +// - general metrics for DAL methods, +// - wrapping errors with DAL method name, +// - transaction. +func (r *Repo) Tx(ctx Ctx, opts *sql.TxOptions, f func(*sqlxx.Tx) error) (err error) { + methodName := reflectx.CallerMethodName(1) + return r.strict(r.schemaLock(r.metric.instrument(methodName, func() error { + tx, err := r.DB.BeginTxx(ctx, opts) + if err == nil { //nolint:nestif // No idea how to simplify. + defer func() { + if err := recover(); err != nil { + if err := tx.Rollback(); err != nil { + log := structlog.FromContext(ctx, nil) + log.Warn("failed to tx.Rollback", "method", methodName, "err", err) + } + panic(err) + } + }() + err = f(tx) + if err == nil { + err = tx.Commit() + } else if err := tx.Rollback(); err != nil { + log := structlog.FromContext(ctx, nil) + log.Warn("failed to tx.Rollback", "method", methodName, "err", err) + } + } + if err != nil { + err = fmt.Errorf("%s: %w", methodName, err) + } + return err + }))) +} diff --git a/scripts/test b/scripts/test index c71d0af..81cb10d 100755 --- a/scripts/test +++ b/scripts/test @@ -13,4 +13,8 @@ mod="$(go list -m)" golangci-lint run +if which dockerize &>/dev/null; then + dockerize -timeout 30s -wait "tcp://${EXAMPLE_MYSQL_ADDR_HOST}:${EXAMPLE_MYSQL_ADDR_PORT:-3306}" +fi + gotestsum -- -race -tags=integration "$@" ./...