From c3cdb65a1a91dea276d1fe20c9ba0d643a0cc89f Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 26 Sep 2024 20:04:22 +0300 Subject: [PATCH 1/7] fix(dapi): getStatus cache invalidation (#2155) --- .../tenderdash/BlockchainListener.js | 36 +++++++++++++------ .../platform/getStatusHandlerFactory.js | 27 ++++++++++++-- .../tenderdash/BlockchainListener.spec.js | 24 ++++++++----- 3 files changed, 64 insertions(+), 23 deletions(-) diff --git a/packages/dapi/lib/externalApis/tenderdash/BlockchainListener.js b/packages/dapi/lib/externalApis/tenderdash/BlockchainListener.js index 6a0e1f5dc5..f6e3e7549f 100644 --- a/packages/dapi/lib/externalApis/tenderdash/BlockchainListener.js +++ b/packages/dapi/lib/externalApis/tenderdash/BlockchainListener.js @@ -13,7 +13,12 @@ class BlockchainListener extends EventEmitter { */ constructor(tenderdashWsClient) { super(); + this.wsClient = tenderdashWsClient; + + this.processLogger = logger.child({ + process: 'BlockchainListener', + }); } /** @@ -30,14 +35,7 @@ class BlockchainListener extends EventEmitter { * Subscribe to blocks and transaction results */ start() { - const processLogger = logger.child({ - process: 'BlockchainListener', - }); - - processLogger.info('Subscribed to state transition results'); - // Emit transaction results - this.wsClient.subscribe(TX_QUERY); this.wsClient.on(TX_QUERY, (message) => { const [hashString] = (message.events || []).map((event) => { const hashAttribute = event.attributes.find((attribute) => attribute.key === 'hash'); @@ -53,15 +51,31 @@ class BlockchainListener extends EventEmitter { return; } - processLogger.trace(`received transaction result for ${hashString}`); + this.processLogger.trace(`Received transaction result for ${hashString}`); this.emit(BlockchainListener.getTransactionEventName(hashString), message); }); - // TODO: It's not using // Emit blocks and contained transactions - // this.wsClient.subscribe(NEW_BLOCK_QUERY); - // this.wsClient.on(NEW_BLOCK_QUERY, (message) => this.emit(EVENTS.NEW_BLOCK, message)); + this.wsClient.on(NEW_BLOCK_QUERY, (message) => { + this.processLogger.trace('Received new platform block'); + + this.emit(EVENTS.NEW_BLOCK, message); + }); + + this.wsClient.on('connect', () => { + this.#subscribe(); + }); + + if (this.wsClient.isConnected) { + this.#subscribe(); + } + } + + #subscribe() { + this.wsClient.subscribe(TX_QUERY); + this.wsClient.subscribe(NEW_BLOCK_QUERY); + this.processLogger.debug('Subscribed to platform blockchain events'); } } diff --git a/packages/dapi/lib/grpcServer/handlers/platform/getStatusHandlerFactory.js b/packages/dapi/lib/grpcServer/handlers/platform/getStatusHandlerFactory.js index 24249b044e..af86ebb312 100644 --- a/packages/dapi/lib/grpcServer/handlers/platform/getStatusHandlerFactory.js +++ b/packages/dapi/lib/grpcServer/handlers/platform/getStatusHandlerFactory.js @@ -9,6 +9,7 @@ const { } = require('@dashevo/dapi-grpc'); const BlockchainListener = require('../../../externalApis/tenderdash/BlockchainListener'); +const logger = require('../../../logger'); /** * @param {BlockchainListener} blockchainListener @@ -17,12 +18,23 @@ const BlockchainListener = require('../../../externalApis/tenderdash/BlockchainL * @return {getStatusHandler} */ function getStatusHandlerFactory(blockchainListener, driveClient, tenderdashRpcClient) { - // Clean cache when new platform block committed let cachedResponse = null; + let cleanCacheTimeout = null; - blockchainListener.on(BlockchainListener.EVENTS.NEW_BLOCK, () => { + function cleanCache() { cachedResponse = null; - }); + + // cancel scheduled cache cleanup + if (cleanCacheTimeout !== null) { + clearTimeout(cleanCacheTimeout); + cleanCacheTimeout = null; + } + + logger.trace({ endpoint: 'getStatus' }, 'cleanup cache'); + } + + // Clean cache when new platform block committed + blockchainListener.on(BlockchainListener.EVENTS.NEW_BLOCK, cleanCache); // DAPI Software version const packageJsonPath = path.resolve(__dirname, '..', '..', '..', '..', 'package.json'); @@ -210,6 +222,15 @@ function getStatusHandlerFactory(blockchainListener, driveClient, tenderdashRpcC cachedResponse = new GetStatusResponse(); cachedResponse.setV0(v0); + // Cancel any existing scheduled cache cleanup + if (cleanCacheTimeout !== null) { + clearTimeout(cleanCacheTimeout); + cleanCacheTimeout = null; + } + + // Clean cache in 3 minutes + cleanCacheTimeout = setTimeout(cleanCache, 3 * 60 * 1000); + return cachedResponse; } diff --git a/packages/dapi/test/integration/externalApis/tenderdash/BlockchainListener.spec.js b/packages/dapi/test/integration/externalApis/tenderdash/BlockchainListener.spec.js index cc2c5fdd7c..b31b79e32b 100644 --- a/packages/dapi/test/integration/externalApis/tenderdash/BlockchainListener.spec.js +++ b/packages/dapi/test/integration/externalApis/tenderdash/BlockchainListener.spec.js @@ -15,8 +15,8 @@ describe('BlockchainListener', () => { ({ sinon } = this); wsClientMock = new EventEmitter(); wsClientMock.subscribe = sinon.stub(); + blockchainListener = new BlockchainListener(wsClientMock); - blockchainListener.start(); sinon.spy(blockchainListener, 'on'); sinon.spy(blockchainListener, 'off'); @@ -84,19 +84,23 @@ describe('BlockchainListener', () => { }); describe('#start', () => { - it('should subscribe to transaction events from WS client', () => { - // TODO: We don't use it for now - // expect(wsClientMock.subscribe).to.be.calledTwice(); - expect(wsClientMock.subscribe).to.be.calledOnce(); + it('should subscribe to transaction events from WS client if it is connected', () => { + wsClientMock.isConnected = true; + + blockchainListener.start(); + + expect(wsClientMock.subscribe).to.be.calledTwice(); expect(wsClientMock.subscribe.firstCall).to.be.calledWithExactly( BlockchainListener.TX_QUERY, ); - // expect(wsClientMock.subscribe.secondCall).to.be.calledWithExactly( - // BlockchainListener.NEW_BLOCK_QUERY, - // ); + expect(wsClientMock.subscribe.secondCall).to.be.calledWithExactly( + BlockchainListener.NEW_BLOCK_QUERY, + ); }); - it.skip('should emit block when new block is arrived', (done) => { + it('should emit block when new block is arrived', (done) => { + blockchainListener.start(); + blockchainListener.on(BlockchainListener.EVENTS.NEW_BLOCK, (message) => { expect(message).to.be.deep.equal(blockMessageMock); @@ -107,6 +111,8 @@ describe('BlockchainListener', () => { }); it('should emit transaction when transaction is arrived', (done) => { + blockchainListener.start(); + const topic = BlockchainListener.getTransactionEventName(transactionHash); blockchainListener.on(topic, (message) => { From 2366dd578748598fbfdfce050a85e1dcac4bf7e7 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 26 Sep 2024 22:56:17 +0300 Subject: [PATCH 2/7] fix(dapi): invalid mainnet seed ports (#2173) --- packages/js-dapi-client/lib/networkConfigs.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/js-dapi-client/lib/networkConfigs.js b/packages/js-dapi-client/lib/networkConfigs.js index 7308016ee5..4bc7c83213 100644 --- a/packages/js-dapi-client/lib/networkConfigs.js +++ b/packages/js-dapi-client/lib/networkConfigs.js @@ -53,10 +53,10 @@ module.exports = { }, mainnet: { seeds: [ - 'seed-1.mainnet.networks.dash.org:1443', - 'seed-2.mainnet.networks.dash.org:1443', - 'seed-3.mainnet.networks.dash.org:1443', - 'seed-4.mainnet.networks.dash.org:1443', + 'seed-1.mainnet.networks.dash.org', + 'seed-2.mainnet.networks.dash.org', + 'seed-3.mainnet.networks.dash.org', + 'seed-4.mainnet.networks.dash.org', ], network: 'mainnet', }, From 5e6bb35961e1f85b93b5ef220bf5e87f3af6af4f Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 26 Sep 2024 20:06:18 +0300 Subject: [PATCH 3/7] fix(dashmate): cannot read properties of undefined (reading 'expires') (#2164) --- .../doctor/analyse/analyseConfigFactory.js | 4 +++ .../obtainZeroSSLCertificateTaskFactory.js | 11 ++++++++ .../validateZeroSslCertificateFactory.js | 27 +++++++++++++------ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js b/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js index 7396668d43..4bb65d0ffb 100644 --- a/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js +++ b/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js @@ -134,6 +134,10 @@ and revoke the previous certificate in the ZeroSSL dashboard`, description: chalk`ZeroSSL certificate is not valid.`, solution: chalk`Please run {bold.cyanBright dashmate ssl zerossl obtain} to get a new one.`, }, + [ERRORS.ZERO_SSL_API_ERROR]: { + description: ssl?.data?.error?.message, + solution: chalk`Please contact ZeroSSL support if needed.`, + }, }[ssl.error] ?? {}; if (description) { diff --git a/packages/dashmate/src/listr/tasks/ssl/zerossl/obtainZeroSSLCertificateTaskFactory.js b/packages/dashmate/src/listr/tasks/ssl/zerossl/obtainZeroSSLCertificateTaskFactory.js index 86bb3db5ca..ca679233d0 100644 --- a/packages/dashmate/src/listr/tasks/ssl/zerossl/obtainZeroSSLCertificateTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/ssl/zerossl/obtainZeroSSLCertificateTaskFactory.js @@ -64,6 +64,9 @@ export default function obtainZeroSSLCertificateTaskFactory( case ERRORS.CERTIFICATE_ID_IS_NOT_SET: // eslint-disable-next-line no-param-reassign task.output = 'Certificate is not configured yet, creating a new one'; + + // We need to create a new certificate + ctx.certificate = null; break; case ERRORS.PRIVATE_KEY_IS_NOT_PRESENT: // If certificate exists but private key does not, then we can't set up TLS connection @@ -85,6 +88,9 @@ export default function obtainZeroSSLCertificateTaskFactory( case ERRORS.CERTIFICATE_EXPIRES_SOON: // eslint-disable-next-line no-param-reassign task.output = `Certificate exists but expires in less than ${ctx.expirationDays} days at ${ctx.certificate.expires}. Obtain a new one`; + + // We need to create a new certificate + ctx.certificate = null; break; case ERRORS.CERTIFICATE_IS_NOT_VALIDATED: // eslint-disable-next-line no-param-reassign @@ -93,7 +99,12 @@ export default function obtainZeroSSLCertificateTaskFactory( case ERRORS.CERTIFICATE_IS_NOT_VALID: // eslint-disable-next-line no-param-reassign task.output = 'Certificate is not valid. Create a new one'; + + // We need to create a new certificate + ctx.certificate = null; break; + case ERRORS.ZERO_SSL_API_ERROR: + throw ctx.error; default: throw new Error(`Unknown error: ${error}`); } diff --git a/packages/dashmate/src/ssl/zerossl/validateZeroSslCertificateFactory.js b/packages/dashmate/src/ssl/zerossl/validateZeroSslCertificateFactory.js index f4d9956a9d..20b221216c 100644 --- a/packages/dashmate/src/ssl/zerossl/validateZeroSslCertificateFactory.js +++ b/packages/dashmate/src/ssl/zerossl/validateZeroSslCertificateFactory.js @@ -11,6 +11,7 @@ export const ERRORS = { CERTIFICATE_EXPIRES_SOON: 'CERTIFICATE_EXPIRES_SOON', CERTIFICATE_IS_NOT_VALIDATED: 'CERTIFICATE_IS_NOT_VALIDATED', CERTIFICATE_IS_NOT_VALID: 'CERTIFICATE_IS_NOT_VALID', + ZERO_SSL_API_ERROR: 'ZERO_SSL_API_ERROR', }; /** @@ -68,9 +69,22 @@ export default function validateZeroSslCertificateFactory(homeDir, getCertificat data.isBundleFilePresent = fs.existsSync(data.bundleFilePath); // This function will throw an error if certificate with specified ID is not present - const certificate = await getCertificate(data.apiKey, certificateId); + try { + data.certificate = await getCertificate(data.apiKey, certificateId); + } catch (e) { + if (e.code) { + data.error = e; - data.isExpiresSoon = certificate.isExpiredInDays(expirationDays); + return { + error: ERRORS.ZERO_SSL_API_ERROR, + data, + }; + } + + throw e; + } + + data.isExpiresSoon = data.certificate.isExpiredInDays(expirationDays); // If certificate exists but private key does not, then we can't setup TLS connection // In this case we need to regenerate a certificate or put back this private key @@ -82,17 +96,16 @@ export default function validateZeroSslCertificateFactory(homeDir, getCertificat } // We need to make sure that external IP and certificate IP match - if (certificate.common_name !== data.externalIp) { + if (data.certificate.common_name !== data.externalIp) { return { error: ERRORS.EXTERNAL_IP_MISMATCH, data, }; } - if (['pending_validation', 'draft'].includes(certificate.status)) { + if (['pending_validation', 'draft'].includes(data.certificate.status)) { // Certificate is already created, so we just need to pass validation // and download certificate file - data.certificate = certificate; // We need to download new certificate bundle data.isBundleFilePresent = false; @@ -103,7 +116,7 @@ export default function validateZeroSslCertificateFactory(homeDir, getCertificat }; } - if (certificate.status !== 'issued' || data.isExpiresSoon) { + if (data.certificate.status !== 'issued' || data.isExpiresSoon) { // Certificate is going to expire soon, or current certificate is not valid // we need to obtain a new one @@ -128,8 +141,6 @@ export default function validateZeroSslCertificateFactory(homeDir, getCertificat } // Certificate is valid, so we might need only to download certificate bundle - data.certificate = certificate; - return { data, }; From 769a93663b68e8bfef46c04d04a21a6497f69077 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 25 Sep 2024 19:43:40 +0300 Subject: [PATCH 4/7] fix(dashmate): colors[updated] is not a function (#2157) --- packages/dashmate/src/update/updateNodeFactory.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dashmate/src/update/updateNodeFactory.js b/packages/dashmate/src/update/updateNodeFactory.js index 13cf1ef83c..0f50d44acb 100644 --- a/packages/dashmate/src/update/updateNodeFactory.js +++ b/packages/dashmate/src/update/updateNodeFactory.js @@ -31,7 +31,7 @@ export default function updateNodeFactory(getServiceList, docker) { name, title, image, updated: 'error', }); } else { - let updated = null; + let updated = 'error'; stream.on('data', (data) => { // parse all stdout and gather Status message From 72bedbe9f1df0c92d966b2d679aaed85a1e98d02 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 26 Sep 2024 12:14:14 +0300 Subject: [PATCH 5/7] fix(dashmate): doctor fails collecting to big logs (#2158) --- packages/dashmate/src/docker/DockerCompose.js | 16 +++++++++++++--- .../tasks/doctor/collectSamplesTaskFactory.js | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/dashmate/src/docker/DockerCompose.js b/packages/dashmate/src/docker/DockerCompose.js index c9231128be..8a50800805 100644 --- a/packages/dashmate/src/docker/DockerCompose.js +++ b/packages/dashmate/src/docker/DockerCompose.js @@ -487,13 +487,23 @@ export default class DockerCompose { * Logs * * @param {Config} config - * @return {Promise} + * @param {string[]} services + * @param {Object} options + * @param {number} options.tail + * @return {Promise<{exitCode: number | null, out: string, err: string}>} */ - async logs(config, services = []) { + async logs(config, services = [], options = {}) { await this.throwErrorIfNotInstalled(); + const args = [...services]; + if (options.tail) { + args.unshift('--tail', options.tail.toString()); + } + + const commandOptions = this.#createOptions(config); + try { - return dockerCompose.logs(services, this.#createOptions(config)); + return await dockerCompose.logs(args, commandOptions); } catch (e) { throw new DockerComposeError(e); } diff --git a/packages/dashmate/src/listr/tasks/doctor/collectSamplesTaskFactory.js b/packages/dashmate/src/listr/tasks/doctor/collectSamplesTaskFactory.js index 1c4d7540be..8e4ae5794a 100644 --- a/packages/dashmate/src/listr/tasks/doctor/collectSamplesTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/doctor/collectSamplesTaskFactory.js @@ -323,7 +323,7 @@ export default function collectSamplesTaskFactory( services.map(async (service) => { const [inspect, logs] = (await Promise.allSettled([ dockerCompose.inspectService(config, service.name), - dockerCompose.logs(config, [service.name]), + dockerCompose.logs(config, [service.name], { tail: 300000 }), ])).map((e) => e.value || e.reason); if (logs?.out) { From 726a93e6ce28f59e08788c8cb2e671e920dac5ad Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 26 Sep 2024 20:05:46 +0300 Subject: [PATCH 6/7] fix(dashmate): port marks as closed if ipv6 is not disabled (#2162) --- packages/dashmate/src/status/providers.js | 83 +++++++++++++++++++---- 1 file changed, 68 insertions(+), 15 deletions(-) diff --git a/packages/dashmate/src/status/providers.js b/packages/dashmate/src/status/providers.js index a3a08b94b6..2eee8a72f5 100644 --- a/packages/dashmate/src/status/providers.js +++ b/packages/dashmate/src/status/providers.js @@ -1,6 +1,8 @@ +import https from 'https'; import { PortStateEnum } from './enums/portState.js'; const MAX_REQUEST_TIMEOUT = 5000; +const MAX_RESPONSE_SIZE = 1 * 1024 * 1024; // 1 MB const request = async (url) => { try { @@ -29,12 +31,6 @@ const requestJSON = async (url) => { return response; }; -const requestText = async (url) => { - const response = await request(url); - - return response.text(); -}; - const insightURLs = { testnet: 'https://testnet-insight.dashevo.org/insight-api', mainnet: 'https://insight.dash.org/insight-api', @@ -68,15 +64,72 @@ export default { }, mnowatch: { checkPortStatus: async (port) => { - try { - return requestText(`https://mnowatch.org/${port}/`); - } catch (e) { - if (process.env.DEBUG) { - // eslint-disable-next-line no-console - console.warn(e); - } - return PortStateEnum.ERROR; - } + // We use http request instead fetch function to force + // using IPv4 otherwise mnwatch could try to connect to IPv6 node address + // and fail (Core listens for IPv4 only) + // https://github.com/dashpay/platform/issues/2100 + + const options = { + hostname: 'mnowatch.org', + port: 443, + path: `/${port}/`, + method: 'GET', + family: 4, // Force IPv4 + timeout: MAX_REQUEST_TIMEOUT, + }; + + return new Promise((resolve) => { + const req = https.request(options, (res) => { + let data = ''; + + // Check if the status code is 200 + if (res.statusCode !== 200) { + if (process.env.DEBUG) { + // eslint-disable-next-line no-console + console.warn(`Port check request failed with status code ${res.statusCode}`); + } + // Consume response data to free up memory + res.resume(); + resolve(PortStateEnum.ERROR); + return; + } + + // Optionally set the encoding to receive strings directly + res.setEncoding('utf8'); + + // Collect data chunks + res.on('data', (chunk) => { + data += chunk; + + if (data.length > MAX_RESPONSE_SIZE) { + resolve(PortStateEnum.ERROR); + + if (process.env.DEBUG) { + // eslint-disable-next-line no-console + console.warn('Port check response size exceeded'); + } + + req.destroy(); + } + }); + + // Handle the end of the response + res.on('end', () => { + resolve(data); + }); + }); + + req.on('error', (e) => { + if (process.env.DEBUG) { + // eslint-disable-next-line no-console + console.warn(`Port check request failed: ${e}`); + } + + resolve(PortStateEnum.ERROR); + }); + + req.end(); + }); }, }, }; From 394c702f993094cb41a1fe2061b1d00d25498312 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 26 Sep 2024 18:32:38 +0300 Subject: [PATCH 7/7] test(dashmate): e2e tests failing due to DKG interval check (#2171) --- packages/dashmate/test/e2e/testnetFullnode.spec.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/dashmate/test/e2e/testnetFullnode.spec.js b/packages/dashmate/test/e2e/testnetFullnode.spec.js index 12329f3407..01d36b61db 100644 --- a/packages/dashmate/test/e2e/testnetFullnode.spec.js +++ b/packages/dashmate/test/e2e/testnetFullnode.spec.js @@ -141,6 +141,7 @@ describe('Testnet Fullnode', function main() { await task.run({ isVerbose: true, + isSafe: true, }); await assertServiceRunning(config, 'core'); @@ -160,6 +161,7 @@ describe('Testnet Fullnode', function main() { await task.run({ isVerbose: true, + isSafe: true, }); await assertServiceRunning(config, 'core', false);