Vulns (#32)

* clickhouse: Quote db name in ensureVersionTable * Quote in drop as well * Leverage quoteIdentifier from pgx * Add to clickhouse README.md database creation * Remove cluster adaptation for tables to pass tests * Update README.md * Correct a spelling mistake * Move supported go version to standard place * add tests for scylladb. add scylladb to docs * small changes to retry failed by timeout CI * Update aws-sdk-go to adress vulerabilitiy * Update lib/pq to fix cert permissions issues When attempting to connect to a PostgreSQL database using certificate credentials, authentication may fail due to permissions issues on the certificate files. When using migrate in Kubernetes, this issue may be unavoidable when using secrets. The github.com/lib/pq library has resolved this issue as of v1.10.6, so this commit updates that library to the latest release version (v1.10.9) to resolve the issue in migrate. * Added support for pgx locking table In order to support running migrations through PgBouncer which does not support advisory locks. * Added documentation and test for lock strategy * Add syntax highlighting to Postgres example * Updated version of spanner to support sequences and generate uuid * fix: typo Signed-off-by: guoguangwu <[email protected]> * feature: add rqlite support * Add rqlite 8.0.0 to tested database versions * patch vulns * update rqlite 8 container version to 8.0.6 * New release prep * Update dktest from v0.3.16 to v0.4.0 to fix docker vulnerability * Fix linter issues * Update outdated GitHub Actions * Update from alpine 3.18 to 3.19 * Bump golang.org/x/crypto from 0.14.0 to 0.17.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.14.0 to 0.17.0. - [Commits](golang/crypto@v0.14.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: indirect ... Signed-off-by: dependabot[bot] <[email protected]> * add 8.11 and 8.12 versions and remove debug logging * Bump google.golang.org/grpc from 1.55.0 to 1.56.3 Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.55.0 to 1.56.3. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](grpc/grpc-go@v1.55.0...v1.56.3) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: indirect ... Signed-off-by: dependabot[bot] <[email protected]> * Clean up require directive grouping * Reformat ScyllaDB/Cassandra docs * Quote locktable from config in queries Defer rollback of transactions * Update aws-sdk-go from v1.44.301 to v1.49.6 * Drop support for Go 1.19 and add support for Go 1.21 * Only test against YugabyteDB LTS releases * Fix GoReleaser deprecations https://goreleaser.com/deprecations/#-rm-dist https://goreleaser.com/deprecations/#sourcerlcp --------- Signed-off-by: guoguangwu <[email protected]> Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: Luca Nardelli <[email protected]> Co-authored-by: Viktoria <[email protected]> Co-authored-by: Viktoria <[email protected]> Co-authored-by: Sandro <[email protected]> Co-authored-by: Maxim Korolyov <[email protected]> Co-authored-by: Jan Śreniawski <[email protected]> Co-authored-by: Rob Prentiss <[email protected]> Co-authored-by: Andreas Emilsson <[email protected]> Co-authored-by: Tommy Karlsson <[email protected]> Co-authored-by: Toby Scott <[email protected]> Co-authored-by: Jacopo Sabbatini <[email protected]> Co-authored-by: guoguangwu <[email protected]> Co-authored-by: Erik Swenson <[email protected]> Co-authored-by: Dale Hui <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
infobloxopen · Dec 21, 2023 · 9e6d75a · 9e6d75a
1 parent d6166ab
commit 9e6d75a
Show file tree

Hide file tree

Showing 32 changed files with 1,224 additions and 303 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -9,10 +9,10 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
-          go-version: "1.20.x"
-      - uses: actions/checkout@v3
+          go-version: "1.21.x"
+      - uses: actions/checkout@v4
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
@@ -22,11 +22,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: ["1.19.x", "1.20.x"]
+        go: ["1.20.x", "1.21.x"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go }}
 
@@ -60,30 +60,30 @@ jobs:
     # 3. When the workflow is triggered by a tag with `v` prefix
     if: ${{ success() && github.repository == 'golang-migrate/migrate' && startsWith(github.ref, 'refs/tags/v') }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: 2.7
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
-          go-version: "1.19.x"
+          go-version: "1.21.x"
 
-      - uses: docker/setup-qemu-action@v1
-      - uses: docker/setup-buildx-action@v1
-      - uses: docker/login-action@v1
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
         with:
           username: golangmigrate
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - run: echo "SOURCE=$(make echo-source)" >> $GITHUB_ENV
       - run: echo "DATABASE=$(make echo-database)" >> $GITHUB_ENV
 
-      - uses: goreleaser/goreleaser-action@v2
+      - uses: goreleaser/goreleaser-action@v5
         with:
           version: latest
-          args: release --rm-dist
+          args: release --clean
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -87,7 +87,6 @@ release:
   prerelease: auto
 source:
   enabled: true
-  rlcp: true
   format: zip
 changelog:
   skip: false

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.20-alpine3.18 AS builder
+FROM golang:1.21-alpine3.19 AS builder
 ARG VERSION
 
 RUN apk add --no-cache git gcc musl-dev make
@@ -15,7 +15,7 @@ COPY . ./
 
 RUN make build-docker
 
-FROM alpine:3.18
+FROM alpine:3.19
 
 COPY --from=builder /go/src/github.com/infobloxopen/migrate/cmd/migrate/config /cli/config/
 COPY --from=builder /go/src/github.com/infobloxopen/migrate/build/migrate.linux-386 /usr/local/bin/migrate

diff --git a/Dockerfile.github-actions b/Dockerfile.github-actions
@@ -1,4 +1,4 @@
-FROM alpine:3.18
+FROM alpine:3.19
 
 RUN apk add --no-cache ca-certificates
 

diff --git a/FAQ.md b/FAQ.md
@@ -50,7 +50,7 @@
   and whenever we want, not just once at the beginning of all tests.
 
 #### Can I maintain my driver in my own repository?
-  Yes, technically thats possible. We want to encourage you to contribute your driver to this respository though.
+  Yes, technically thats possible. We want to encourage you to contribute your driver to this repository though.
   The driver's functionality is dictated by migrate's interfaces. That means there should really
   just be one driver for a database/ source. We want to prevent a future where several drivers doing the exact same thing,
   just implemented a bit differently, co-exist somewhere on GitHub. If users have to do research first to find the

diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md
@@ -28,7 +28,7 @@ migrate -database YOUR_DATABASE_URL -path PATH_TO_YOUR_MIGRATIONS up
 
 Just add the code to your app and you're ready to go!
 
-Before commiting your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways.
+Before committing your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways.
 (e.g. if you created a table in a migration but reverse migration did not delete it, you will encounter an error when running the forward migration again)
 It's also worth checking your migrations in a separate, containerized environment. You can find some tools at the [end of this document](#further-reading).
 

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -7,7 +7,7 @@ pipeline {
     label 'ubuntu_docker_label'
   }
   tools {
-    go "Go 1.17"
+    go "Go 1.20"
   }
   options {
     checkoutToSubdirectory('src/github.com/infobloxopen/migrate')

diff --git a/Makefile b/Makefile
@@ -1,5 +1,5 @@
 SOURCE ?= file go_bindata github github_ee bitbucket aws_s3 google_cloud_storage godoc_vfs gitlab
-DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5
+DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5 rqlite
 DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher
 BUILD_NUMBER ?= 0
 VERSION ?= $(shell git describe --tags --long --dirty=-unsupported 2>/dev/null | cut -c 2-)-j$(BUILD_NUMBER)

diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@
 [![Coverage Status](https://img.shields.io/coveralls/github/golang-migrate/migrate/master.svg)](https://coveralls.io/github/golang-migrate/migrate?branch=master)
 [![packagecloud.io](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/golang-migrate/migrate?filter=debs)
 [![Docker Pulls](https://img.shields.io/docker/pulls/migrate/migrate.svg)](https://hub.docker.com/r/migrate/migrate/)
-![Supported Go Versions](https://img.shields.io/badge/Go-1.19%2C%201.20-lightgrey.svg)
+![Supported Go Versions](https://img.shields.io/badge/Go-1.20%2C%201.21-lightgrey.svg)
 [![GitHub Release](https://img.shields.io/github/release/golang-migrate/migrate.svg)](https://github.com/golang-migrate/migrate/releases)
 [![Go Report Card](https://goreportcard.com/badge/github.com/golang-migrate/migrate/v4)](https://goreportcard.com/report/github.com/golang-migrate/migrate/v4)
 
@@ -28,11 +28,11 @@ Database drivers run migrations. [Add a new database?](database/driver.go)
 * [PGX v5](database/pgx/v5)
 * [Redshift](database/redshift)
 * [Ql](database/ql)
-* [Cassandra](database/cassandra)
+* [Cassandra / ScyllaDB](database/cassandra)
 * [SQLite](database/sqlite)
 * [SQLite3](database/sqlite3) ([todo #165](https://github.com/mattes/migrate/issues/165))
 * [SQLCipher](database/sqlcipher)
-* [MySQL/ MariaDB](database/mysql)
+* [MySQL / MariaDB](database/mysql)
 * [Neo4j](database/neo4j)
 * [MongoDB](database/mongodb)
 * [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170))
@@ -43,6 +43,7 @@ Database drivers run migrations. [Add a new database?](database/driver.go)
 * [ClickHouse](database/clickhouse)
 * [Firebird](database/firebird)
 * [MS SQL Server](database/sqlserver)
+* [RQLite](database/rqlite)
 
 ### Database URLs
 

diff --git a/database/cassandra/README.md b/database/cassandra/README.md
@@ -1,12 +1,17 @@
-# Cassandra
+# Cassandra / ScyllaDB
 
-* Drop command will not work on Cassandra 2.X because it rely on
+* `Drop()` method will not work on Cassandra 2.X because it rely on
 system_schema table which comes with 3.X
-* Other commands should work properly but are **not tested**
+* Other methods should work properly but are **not tested**
 * The Cassandra driver (gocql) does not natively support executing multiple statements in a single query. To allow for multiple statements in a single migration, you can use the `x-multi-statement` param. There are two important caveats:
   * This mode splits the migration text into separately-executed statements by a semi-colon `;`. Thus `x-multi-statement` cannot be used when a statement in the migration contains a string with a semi-colon.
   * The queries are not executed in any sort of transaction/batch, meaning you are responsible for fixing partial migrations.
 
+**ScyllaDB**
+
+* No additional configuration is required since it is a drop-in replacement for Cassandra.
+* The `Drop()` method` works for ScyllaDB 5.1
+
 
 ## Usage
 `cassandra://host:port/keyspace?param1=value&param2=value2`

diff --git a/database/cassandra/cassandra_test.go b/database/cassandra/cassandra_test.go
@@ -25,9 +25,11 @@ var (
 	// Although Cassandra 2.x is supported by the Apache Foundation,
 	// the migrate db driver only supports Cassandra 3.x since it uses
 	// the system_schema keyspace.
+	// last ScyllaDB version tested is 5.1.11
 	specs = []dktesting.ContainerSpec{
 		{ImageName: "cassandra:3.0", Options: opts},
 		{ImageName: "cassandra:3.11", Options: opts},
+		{ImageName: "scylladb/scylla:5.1.11", Options: opts},
 	}
 )
 

diff --git a/database/clickhouse/README.md b/database/clickhouse/README.md
@@ -23,3 +23,4 @@
 * Clickhouse cluster mode is not officially supported, since it's not tested right now, but you can try enabling `schema_migrations` table replication by specifying a `x-cluster-name`:
   * When `x-cluster-name` is specified, `x-migrations-table-engine` also should be specified. See the docs regarding [replicated table engines](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/#table_engines-replication).
   * When `x-cluster-name` is specified, only the `schema_migrations` table is replicated across the cluster. You still need to write your migrations so that the application tables are replicated within the cluster.
+* If you want to create database inside the migration, you should know, that table which will manage migrations `schema-migrations table` will be in `default` table, so you can't use `USE <database_name>` inside migration. In this case you may not specify the database in the connection string (example you can find [here](examples/migrations/003_create_database.up.sql))
diff --git a/database/clickhouse/clickhouse.go b/database/clickhouse/clickhouse.go
@@ -220,7 +220,7 @@ func (ch *ClickHouse) ensureVersionTable() (err error) {
 
 	var (
 		table string
-		query = "SHOW TABLES FROM " + ch.config.DatabaseName + " LIKE '" + ch.config.MigrationsTable + "'"
+		query = "SHOW TABLES FROM " + quoteIdentifier(ch.config.DatabaseName) + " LIKE '" + ch.config.MigrationsTable + "'"
 	)
 	// check if migration table exists
 	if err := ch.conn.QueryRow(query).Scan(&table); err != nil {
@@ -259,7 +259,7 @@ func (ch *ClickHouse) ensureVersionTable() (err error) {
 }
 
 func (ch *ClickHouse) Drop() (err error) {
-	query := "SHOW TABLES FROM " + ch.config.DatabaseName
+	query := "SHOW TABLES FROM " + quoteIdentifier(ch.config.DatabaseName)
 	tables, err := ch.conn.Query(query)
 
 	if err != nil {
@@ -277,7 +277,7 @@ func (ch *ClickHouse) Drop() (err error) {
 			return err
 		}
 
-		query = "DROP TABLE IF EXISTS " + ch.config.DatabaseName + "." + table
+		query = "DROP TABLE IF EXISTS " + quoteIdentifier(ch.config.DatabaseName) + "." + quoteIdentifier(table)
 
 		if _, err := ch.conn.Exec(query); err != nil {
 			return &database.Error{OrigErr: err, Query: []byte(query)}
@@ -305,3 +305,12 @@ func (ch *ClickHouse) Unlock() error {
 	return nil
 }
 func (ch *ClickHouse) Close() error { return ch.conn.Close() }
+
+// Copied from lib/pq implementation: https://github.com/lib/pq/blob/v1.9.0/conn.go#L1611
+func quoteIdentifier(name string) string {
+	end := strings.IndexRune(name, 0)
+	if end > -1 {
+		name = name[:end]
+	}
+	return `"` + strings.Replace(name, `"`, `""`, -1) + `"`
+}
diff --git a/database/clickhouse/examples/migrations/003_create_database.down.sql b/database/clickhouse/examples/migrations/003_create_database.down.sql
@@ -0,0 +1,10 @@
+DROP TABLE IF EXISTS driver_ratings;
+DROP TABLE IF EXISTS user_ratings;
+DROP TABLE IF EXISTS orders;
+DROP TABLE IF EXISTS driver_ratings_queue;
+DROP TABLE IF EXISTS user_ratings_queue;
+DROP TABLE IF EXISTS orders_queue;
+DROP VIEW IF EXISTS user_ratings_queue_mv;
+DROP VIEW IF EXISTS driver_ratings_queue_mv;
+DROP VIEW IF EXISTS orders_queue_mv;
+DROP DATABASE IF EXISTS analytics;
diff --git a/database/clickhouse/examples/migrations/003_create_database.up.sql b/database/clickhouse/examples/migrations/003_create_database.up.sql
@@ -0,0 +1,81 @@
+CREATE DATABASE IF NOT EXISTS analytics;
+
+CREATE TABLE IF NOT EXISTS analytics.driver_ratings(
+    rate UInt8,
+    userID Int64,
+    driverID String,
+    orderID String,
+    inserted_time DateTime DEFAULT now()
+) ENGINE = MergeTree
+PARTITION BY driverID
+ORDER BY (inserted_time);
+
+CREATE TABLE analytics.driver_ratings_queue(
+    rate UInt8,
+    userID Int64,
+    driverID String,
+    orderID String
+) ENGINE = Kafka
+SETTINGS kafka_broker_list = 'broker:9092',
+    kafka_topic_list = 'driver-ratings',
+    kafka_group_name = 'rating_readers',
+    kafka_format = 'Avro',
+    kafka_max_block_size = 1048576;
+
+CREATE MATERIALIZED VIEW analytics.driver_ratings_queue_mv TO analytics.driver_ratings AS
+SELECT rate, userID, driverID, orderID
+FROM analytics.driver_ratings_queue;
+
+CREATE TABLE IF NOT EXISTS analytics.user_ratings(
+    rate UInt8,
+    userID Int64,
+    driverID String,
+    orderID String,
+    inserted_time DateTime DEFAULT now()
+) ENGINE = MergeTree
+    PARTITION BY userID
+    ORDER BY (inserted_time);
+
+CREATE TABLE analytics.user_ratings_queue(
+    rate UInt8,
+    userID Int64,
+    driverID String,
+    orderID String
+) ENGINE = Kafka
+SETTINGS kafka_broker_list = 'broker:9092',
+    kafka_topic_list = 'user-ratings',
+    kafka_group_name = 'rating_readers',
+    kafka_format = 'JSON',
+    kafka_max_block_size = 1048576;
+
+CREATE MATERIALIZED VIEW analytics.user_ratings_queue_mv TO analytics.user_ratings AS
+SELECT rate, userID, driverID, orderID
+FROM analytics.user_ratings_queue;
+
+CREATE TABLE IF NOT EXISTS analytics.orders(
+    from_place String,
+    to_place String,
+    userID Int64,
+    driverID String,
+    orderID String,
+    inserted_time DateTime DEFAULT now()
+) ENGINE = MergeTree
+    PARTITION BY driverID
+    ORDER BY (inserted_time);
+
+CREATE TABLE analytics.orders_queue(
+    from_place String,
+    to_place String,
+    userID Int64,
+    driverID String,
+    orderID String
+) ENGINE = Kafka
+SETTINGS kafka_broker_list = 'broker:9092',
+    kafka_topic_list = 'orders',
+    kafka_group_name = 'order_readers',
+    kafka_format = 'Avro',
+    kafka_max_block_size = 1048576;
+
+CREATE MATERIALIZED VIEW analytics.orders_queue_mv TO orders AS
+SELECT from_place, to_place, userID, driverID, orderID
+FROM analytics.orders_queue;