feat(migrations): support for running database migrations manually (#…

…2622) * feat(auth): support for running database migrations manually #2621 * feat(backend): support for running database migrations manually #2621 * docs(deployment): add enable manual migrations environment variable --------- Co-authored-by: tadejgolobic <[email protected]>
interledger · Apr 2, 2024 · ae859a9 · ae859a9
1 parent 0b526f2
commit ae859a9
Show file tree

Hide file tree

Showing 9 changed files with 31 additions and 24 deletions.
diff --git a/packages/auth/Dockerfile b/packages/auth/Dockerfile
@@ -17,6 +17,7 @@ RUN --mount=type=cache,id=pnpm,target=/pnpm/store \
 FROM base AS prod-deps
 
 COPY package.json pnpm-workspace.yaml .npmrc ./
+COPY packages/auth/knexfile.js ./packages/auth/knexfile.js
 COPY packages/auth/package.json ./packages/auth/package.json
 COPY packages/token-introspection/package.json ./packages/token-introspection/package.json
 
@@ -51,10 +52,10 @@ COPY --from=prod-deps /home/rafiki/packages/auth/node_modules ./packages/auth/no
 COPY --from=prod-deps /home/rafiki/packages/auth/package.json ./packages/auth/package.json
 COPY --from=prod-deps /home/rafiki/packages/token-introspection/node_modules ./packages/token-introspection/node_modules
 COPY --from=prod-deps /home/rafiki/packages/token-introspection/package.json ./packages/token-introspection/package.json
+COPY --from=prod-deps /home/rafiki/packages/auth/knexfile.js ./packages/auth/knexfile.js
 
 COPY --from=builder /home/rafiki/packages/auth/migrations/ ./packages/auth/migrations
 COPY --from=builder /home/rafiki/packages/auth/dist ./packages/auth/dist
 COPY --from=builder /home/rafiki/packages/token-introspection/dist ./packages/token-introspection/dist
 
-
 CMD ["node", "/home/rafiki/packages/auth/dist/index.js"]
diff --git a/packages/auth/knexfile.js b/packages/auth/knexfile.js
@@ -59,11 +59,7 @@ module.exports = {
 
   production: {
     client: 'postgresql',
-    connection: {
-      database: 'my_auth_db',
-      user: 'username',
-      password: 'password'
-    },
+    connection: process.env.AUTH_DATABASE_URL,
     pool: {
       min: 2,
       max: 10

diff --git a/packages/auth/src/config/app.ts b/packages/auth/src/config/app.ts
@@ -28,6 +28,7 @@ export const Config = {
   introspectionPort: envInt('INTROSPECTION_PORT', 3007),
   env: envString('NODE_ENV', 'development'),
   trustProxy: envBool('TRUST_PROXY', false),
+  enableManualMigrations: envBool('ENABLE_MANUAL_MIGRATIONS', false),
   databaseUrl:
     process.env.NODE_ENV === 'test'
       ? `${process.env.AUTH_DATABASE_URL}_${process.env.JEST_WORKER_ID}`

diff --git a/packages/auth/src/index.ts b/packages/auth/src/index.ts
@@ -263,18 +263,22 @@ export const start = async (
     }
   })
 
+  const config = await container.use('config')
+
   // Do migrations
   const knex = await container.use('knex')
-  // Needs a wrapped inline function
-  await callWithRetry(async () => {
-    await knex.migrate.latest({
-      directory: __dirname + '/../migrations'
+
+  if (!config.enableManualMigrations) {
+    // Needs a wrapped inline function
+    await callWithRetry(async () => {
+      await knex.migrate.latest({
+        directory: __dirname + '/../migrations'
+      })
     })
-  })
+  }
 
   Model.knex(knex)
 
-  const config = await container.use('config')
   await app.boot()
 
   await app.startAdminServer(config.adminPort)

diff --git a/packages/backend/Dockerfile b/packages/backend/Dockerfile
@@ -17,6 +17,7 @@ RUN --mount=type=cache,id=pnpm,target=/pnpm/store \
 FROM base AS prod-deps
 
 COPY package.json pnpm-workspace.yaml .npmrc ./
+COPY packages/backend/knexfile.js ./packages/backend/knexfile.js
 COPY packages/backend/package.json ./packages/backend/package.json
 COPY packages/token-introspection/package.json ./packages/token-introspection/package.json
 
@@ -51,10 +52,11 @@ COPY --from=prod-deps /home/rafiki/packages/backend/node_modules ./packages/back
 COPY --from=prod-deps /home/rafiki/packages/backend/package.json ./packages/backend/package.json
 COPY --from=prod-deps /home/rafiki/packages/token-introspection/node_modules ./packages/token-introspection/node_modules
 COPY --from=prod-deps /home/rafiki/packages/token-introspection/package.json ./packages/token-introspection/package.json
+COPY --from=prod-deps /home/rafiki/packages/backend/knexfile.js ./packages/backend/knexfile.js
 
 COPY --from=builder /home/rafiki/packages/backend/migrations/ ./packages/backend/migrations
 COPY --from=builder /home/rafiki/packages/backend/dist ./packages/backend/dist
 COPY --from=builder /home/rafiki/packages/token-introspection/dist ./packages/token-introspection/dist
-
+COPY --from=builder /home/rafiki/packages/backend/knexfile.js ./packages/backend/knexfile.js
 
 CMD ["node", "/home/rafiki/packages/backend/dist/index.js"]
diff --git a/packages/backend/knexfile.js b/packages/backend/knexfile.js
@@ -35,11 +35,7 @@ module.exports = {
 
   production: {
     client: 'postgresql',
-    connection: {
-      database: 'my_db',
-      user: 'username',
-      password: 'password'
-    },
+    connection: process.env.DATABASE_URL,
     pool: {
       min: 2,
       max: 10

diff --git a/packages/backend/src/config/app.ts b/packages/backend/src/config/app.ts
@@ -64,6 +64,7 @@ export const Config = {
   connectorPort: envInt('CONNECTOR_PORT', 3002),
   autoPeeringServerPort: envInt('AUTO_PEERING_SERVER_PORT', 3005),
   enableAutoPeering: envBool('ENABLE_AUTO_PEERING', false),
+  enableManualMigrations: envBool('ENABLE_MANUAL_MIGRATIONS', false),
   databaseUrl:
     process.env.NODE_ENV === 'test'
       ? `${process.env.DATABASE_URL}_${process.env.JEST_WORKER_ID}`

diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts
@@ -571,18 +571,22 @@ export const start = async (
     }
   })
 
+  const config = await container.use('config')
+
   // Do migrations
   const knex = await container.use('knex')
-  // Needs a wrapped inline function
-  await callWithRetry(async () => {
-    await knex.migrate.latest({
-      directory: __dirname + '/../migrations'
+
+  if (!config.enableManualMigrations) {
+    // Needs a wrapped inline function
+    await callWithRetry(async () => {
+      await knex.migrate.latest({
+        directory: __dirname + '/../migrations'
+      })
     })
-  })
+  }
 
   Model.knex(knex)
 
-  const config = await container.use('config')
   await app.boot()
   await app.startAdminServer(config.adminPort)
   logger.info(`Admin listening on ${app.getAdminPort()}`)

diff --git a/packages/documentation/src/content/docs/integration/deployment.md b/packages/documentation/src/content/docs/integration/deployment.md
@@ -105,6 +105,7 @@ Now, the Admin UI can be found on localhost:3010.
 | `WEBHOOK_WORKERS`                                     | backend.workers.webhook                                                                                                                 | `1`                                                         | number of workers processing webhook events                                                                                                                                                                |
 | `WEBHOOK_WORKER_IDLE`                                 | backend.workerIdle                                                                                                                      | `200`                                                       | time in milliseconds that WEBHOOK_WORKERS will wait until they check an empty webhook event queue again                                                                                                    |
 | `WITHDRAWAL_THROTTLE_DELAY`                           | backend.withdrawalThrottleDelay                                                                                                         | `undefined`                                                 | delay in liquidity withdrawal processing                                                                                                                                                                   |
+| `ENABLE_MANUAL_MIGRATIONS`                            | backend.enableManualMigrations                                                                                                          | `false`                                                     | When set to true, user needs to run database manually with command `npm run knex -- migrate:latest --env production`                                                                                       |
 
 #### Auth
 
@@ -129,6 +130,7 @@ Now, the Admin UI can be found on localhost:3010.
 | `QUOTE_INTERACTION`            | auth.interaction.quote                                                                                                   | `false`                                                          | flag - quote grants are interactive or not                                                                                                                                         |
 | `TRUST_PROXY`                  |                                                                                                                          | `false`                                                          | flag to use X-Forwarded-Proto header to determine if connections is secure                                                                                                         |
 | `WAIT_SECONDS`                 | auth.grant.waitSeconds                                                                                                   | `5`                                                              | wait time included in grant request response (`grant.continue`)                                                                                                                    |
+| `ENABLE_MANUAL_MIGRATIONS`     | auth.enableManualMigrations                                                                                              | `false`                                                          | When set to true, user needs to run database manually with command `npm run knex -- migrate:latest --env production`                                                               |
 
 #### Frontend