From 073450e0a096edb11576ffe028ba0494df58b861 Mon Sep 17 00:00:00 2001 From: brontolosone <177225737+brontolosone@users.noreply.github.com> Date: Thu, 1 Aug 2024 14:07:06 +0100 Subject: [PATCH] improve: add support for database connection URIs - Unifies connection configuration of Slonik/Knex - Allows connections to PostgreSQL over Unix domain sockets --- README.md | 13 +++++ lib/model/knexfile.js | 4 +- lib/model/migrate.js | 4 +- lib/util/db.js | 71 +++++++++++++++----------- test/unit/util/db.js | 113 +++++------------------------------------- 5 files changed, 71 insertions(+), 134 deletions(-) diff --git a/README.md b/README.md index 8759bd908..d325773cd 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ You can also run `make debug` to run the server with a standard node inspector p ### Setting up the database manually +#### Basic configuration First, create a database and user in Postgres. Either use the same settings as the [default configuration file](config/default.json), or update your local configuration file to match the settings you choose. For example: ```sql @@ -59,6 +60,18 @@ CREATE EXTENSION IF NOT EXISTS CITEXT; CREATE EXTENSION IF NOT EXISTS pg_trgm; ``` +#### Advanced configuration +Rather than specifying username/host/password/database etc in separate fields, you can also use a "connection URI". This allows for many more options, eg for accessing your database over Unix domain sockets. A database configuration block that does that may look like this: +```javascript + "database": { + "uri": "postgresql://%2Frun%2Fpostgresql/jubilant" + }, +``` +which will connect to the server using the socket at `/run/postgresql/.s.PGSQL.5432`, using your current user (which must have access to the `jubilant` database), using passwordless "peer authentication" (which must be enabled in your PostgreSQL server configuration, usually in `pg_hba.conf`.) + +For details on the URI syntax, see [Postgres' documentation](https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-CONNSTRING-URIS) and the [parser documentation](https://www.npmjs.com/package/pg-connection-string?activeTab=readme#connection-strings). + +#### With Docker If you are using Docker, you may find it easiest to run the database in Docker by running `make run-docker-postgres`. ### Creating an admin user diff --git a/lib/model/knexfile.js b/lib/model/knexfile.js index 84622389b..f7bc03379 100644 --- a/lib/model/knexfile.js +++ b/lib/model/knexfile.js @@ -28,10 +28,10 @@ NODE_CONFIG_DIR=../../config DEBUG=knex:query,knex:bindings npx knex migrate:up */ const config = require('config'); -const { connectionObject } = require('../util/db'); +const { connectionString } = require('../util/db'); module.exports = { client: 'pg', - connection: connectionObject(config.get('default.database')) + connection: connectionString(config.get('default.database')) }; diff --git a/lib/model/migrate.js b/lib/model/migrate.js index fe18ed2f5..954101374 100644 --- a/lib/model/migrate.js +++ b/lib/model/migrate.js @@ -11,10 +11,10 @@ // top-level operations with a database, like migrations. const knex = require('knex'); -const { connectionObject } = require('../util/db'); +const { connectionString } = require('../util/db'); // Connects to the postgres database specified in configuration and returns it. -const connect = (config) => knex({ client: 'pg', connection: connectionObject(config) }); +const connect = (config) => knex({ client: 'pg', connection: connectionString(config) }); // Connects to a database, passes it to a function for operations, then ensures its closure. const withDatabase = (config) => (mutator) => { diff --git a/lib/util/db.js b/lib/util/db.js index 6e096bcd1..13d536167 100644 --- a/lib/util/db.js +++ b/lib/util/db.js @@ -24,47 +24,58 @@ const { Transform } = require('stream'); // DATABASE CONFIG const validateConfig = (config) => { - const { host, port, database, user, password, ssl, maximumPoolSize, ...unsupported } = config; - - if (ssl != null && ssl !== true) - return Problem.internal.invalidDatabaseConfig({ reason: 'If ssl is specified, its value can only be true.' }); - - const unsupportedKeys = Object.keys(unsupported); - if (unsupportedKeys.length !== 0) - return Problem.internal.invalidDatabaseConfig({ - reason: `'${unsupportedKeys[0]}' is unknown or is not supported.` - }); - - return null; + // There's two ways of specifying connection details: + // a) original style: separate fields for components + // b) new style: using a single 'uri' field, which holds a connection URI (see https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-CONNSTRING-URIS ) + // This new style allows for connecting passwordless (but authenticated) over domain sockets, among other things. + // Example: `postgresql://%2Frun%2Fpostgresql/odkcentral` which will connect to the socket at /run/postgresql/.s.PGSQL.5432 + // When an uri is supplied, we treat it as opaque and simply pass it on verbatim to Slonk/Knex etc, which both will in turn hopefully pass it on unmolested + // to node-postgres, which uses https://nodei.co/npm/pg-connection-string/ to parse it. + // + // Thus we support the legacy fields — people do not have to adapt their configs or habits, yet we automatically support more advanced use cases without further + // code changes through anything that is supported through pg-connection-string, as it becomes available. + + // case 1: a connection URI is supplied. + if (config.uri) { + const { uri, maximumPoolSize, ...unsupported } = config; + if (uri != null) { + // ideally we'd validate this uri for suitability for `pg-connection-string.parse()`. But: + // a) how would we target the exact version of pg-connection-string used by the pg module; it's not re-exported so that would require some acrobatics + // b) that parse() doesn't actually do any validation anyway — it just makes a best effort + // c) we can't use node's URL parser to test basic wellformedness either, because only a subset of URIs are URLs. + // Thus we have to treat the uri as completely opaque, which means the user may end up with an unintelligable error from somewhere deep down in the call stack. + + // reject any keys other than uri and maximumPoolSize + if (Object.keys(unsupported).length) + return Problem.internal.invalidDatabaseConfig({ reason: 'When specifying a `uri`, no configuration field other than (optionally) `maximumPoolsize` may be specified.' }); + return null; + } + } else { + // case 2: DB config is broken down + const { host, port, database, user, password, ssl, maximumPoolSize, ...unsupported } = config; + + if (ssl != null && ssl !== true) + return Problem.internal.invalidDatabaseConfig({ reason: 'If ssl is specified, its value can only be true.' }); + const unsupportedKeys = Object.keys(unsupported); + if (unsupportedKeys.length !== 0) + return Problem.internal.invalidDatabaseConfig({ + reason: `'${unsupportedKeys[0]}' is unknown or is not supported.` + }); + return null; + } }; // Returns a connection string that will be passed to Slonik. const connectionString = (config) => { const problem = validateConfig(config); if (problem != null) throw problem; + if (config.uri) return config.uri; const encodedPassword = encodeURIComponent(config.password); const hostWithPort = config.port == null ? config.host : `${config.host}:${config.port}`; const queryString = config.ssl == null ? '' : `?ssl=${config.ssl}`; return `postgres://${config.user}:${encodedPassword}@${hostWithPort}/${config.database}${queryString}`; }; -// Returns an object that Knex will use to connect to the database. -const connectionObject = (config) => { - const problem = validateConfig(config); - if (problem != null) throw problem; - // We ignore maximumPoolSize when using Knex. - const { maximumPoolSize, ...knexConfig } = config; - if (knexConfig.ssl === true) { - // Slonik seems to specify `false` for `rejectUnauthorized` whenever SSL is - // specified: - // https://github.com/gajus/slonik/issues/159#issuecomment-891089466. We do - // the same here so that Knex will connect to the database in the same way - // as Slonik. - knexConfig.ssl = { rejectUnauthorized: false }; - } - return knexConfig; -}; - //////////////////////////////////////////////////////////////////////////////// // SLONIK UTIL @@ -567,7 +578,7 @@ const postgresErrorToProblem = (x) => { }; module.exports = { - connectionString, connectionObject, + connectionString, unjoiner, extender, equals, page, queryFuncs, insert, insertMany, updater, markDeleted, markUndeleted, QueryOptions, diff --git a/test/unit/util/db.js b/test/unit/util/db.js index 0dcf86df0..da9d565a7 100644 --- a/test/unit/util/db.js +++ b/test/unit/util/db.js @@ -10,6 +10,14 @@ describe('util/db', () => { describe('connectionString', () => { const { connectionString } = util; + it('should accept a connection URI and return it intact', () => { + const uri = 'postgres://bar:baz@localhost/foo'; + const result = connectionString({ + uri, + }); + result.should.equal(uri); + }); + it('should return a string with the required options', () => { const result = connectionString({ host: 'localhost', @@ -94,108 +102,13 @@ describe('util/db', () => { encoding: 'latin1' }); result.should.throw(); - }); - }); - - describe('connectionObject', () => { - const { connectionObject } = util; - - it('should return an object with the required options', () => { - const result = connectionObject({ - host: 'localhost', - database: 'foo', - user: 'bar', - password: 'baz' - }); - result.should.eql({ - host: 'localhost', - database: 'foo', - user: 'bar', - password: 'baz' - }); - }); - - it('should include the port if one is specified', () => { - const result = connectionObject({ - host: 'localhost', - database: 'foo', - user: 'bar', - password: 'baz', - port: 1234 - }); - result.should.eql({ - host: 'localhost', - database: 'foo', - user: 'bar', - password: 'baz', - port: 1234 - }); - }); - - it('should return the correct object if ssl is true', () => { - const result = connectionObject({ - host: 'localhost', - database: 'foo', - user: 'bar', - password: 'baz', - ssl: true - }); - result.should.eql({ - host: 'localhost', - database: 'foo', - user: 'bar', - password: 'baz', - ssl: { rejectUnauthorized: false } - }); - }); - - it('should throw if ssl is false', () => { - const result = () => connectionObject({ - host: 'localhost', - database: 'foo', - user: 'bar', - password: 'baz', - ssl: false - }); - result.should.throw(); - }); - it('should throw if ssl is an object', () => { - const result = () => connectionObject({ - host: 'localhost', - database: 'foo', - user: 'bar', - password: 'baz', - ssl: { rejectUnauthorized: false } + const resultForUri = () => connectionString({ + uri: 'postgresql://bar:baz@localhost/foo?encoding=latin1', + maximumPoolSize: 42, + unsupportedoption: 'boo!', }); - result.should.throw(); - }); - - it('should allow (but ignore) maximumPoolSize', () => { - const result = connectionObject({ - host: 'localhost', - database: 'foo', - user: 'bar', - password: 'baz', - maximumPoolSize: 42 - }); - result.should.eql({ - host: 'localhost', - database: 'foo', - user: 'bar', - password: 'baz' - }); - }); - - it('should throw for an unsupported option', () => { - const result = () => connectionObject({ - host: 'localhost', - database: 'foo', - user: 'bar', - password: 'baz', - encoding: 'latin1' - }); - result.should.throw(); + resultForUri.should.throw(); }); });