diff --git a/bfx-report-ui b/bfx-report-ui index b14ccb940..ef3faa57d 160000 --- a/bfx-report-ui +++ b/bfx-report-ui @@ -1 +1 @@ -Subproject commit b14ccb94004977332f8f6838bda0d43c6f0d3914 +Subproject commit ef3faa57d8919ca6833f85736a8911583d58a2b9 diff --git a/package.json b/package.json index 62b46ba7d..6c56bf857 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bfx-reports-framework", - "version": "4.17.2", + "version": "4.18.0", "description": "Bitfinex reports framework", "main": "worker.js", "license": "Apache-2.0", @@ -10,6 +10,7 @@ "bfx-facs-scheduler": "git+https://github.com:bitfinexcom/bfx-facs-scheduler.git", "bfx-report": "git+https://github.com/bitfinexcom/bfx-report.git", "bfx-svc-boot-js": "https://github.com/bitfinexcom/bfx-svc-boot-js.git", + "bignumber.js": "9.1.2", "csv": "5.5.3", "grenache-nodejs-ws": "git+https://github.com:bitfinexcom/grenache-nodejs-ws.git", "html-pdf": "3.0.1", diff --git a/test/7-interrupt-operations.spec.js b/test/7-interrupt-operations.spec.js new file mode 100644 index 000000000..e75ad4ae1 --- /dev/null +++ b/test/7-interrupt-operations.spec.js @@ -0,0 +1,267 @@ +'use strict' + +const { + setTimeout +} = require('node:timers/promises') +const path = require('node:path') +const request = require('supertest') +const { assert } = require('chai') + +const { + stopEnvironment +} = require('bfx-report/test/helpers/helpers.boot') +const { + rmDB, + rmAllFiles +} = require('bfx-report/test/helpers/helpers.core') + +const { + startEnvironment +} = require('./helpers/helpers.boot') +const { + getRServiceProxy, + emptyDB, + rmRf +} = require('./helpers/helpers.core') +const { + createMockRESTv2SrvWithDate +} = require('./helpers/helpers.mock-rest-v2') + +process.env.NODE_CONFIG_DIR = path.join(__dirname, 'config') +const { app } = require('bfx-report-express') +const agent = request.agent(app) + +const { + signUpTestCase, + getSyncProgressTestCase +} = require('./test-cases') + +let wrkReportServiceApi = null +let mockRESTv2Srv = null + +const basePath = '/api' +const tempDirPath = path.join(__dirname, '..', 'workers/loc.api/queue/temp') +const dbDirPath = path.join(__dirname, '..', 'db') +const reportFolderPath = path.join(__dirname, '..', 'report-files') +const date = new Date() +const end = date.getTime() +const start = (new Date()).setDate(date.getDate() - 90) + +const apiKeys = { + apiKey: 'fake', + apiSecret: 'fake' +} +const email = 'fake@email.fake' +const password = '123Qwerty' +const isSubAccount = false + +describe('Interrupt operations', () => { + const params = { + processorQueue: null, + aggregatorQueue: null, + basePath, + auth: { + email, + password, + isSubAccount + }, + apiKeys, + date, + end, + start + } + const auth = { token: 'user-token' } + + before(async function () { + this.timeout(20000) + + mockRESTv2Srv = createMockRESTv2SrvWithDate(start, end, 100) + + await rmRf(reportFolderPath) + await rmAllFiles(tempDirPath, ['README.md']) + await rmDB(dbDirPath) + const env = await startEnvironment(false, false, 1) + + wrkReportServiceApi = env.wrksReportServiceApi[0] + const rService = wrkReportServiceApi.grc_bfx.api + const rServiceProxy = getRServiceProxy(rService, { + async _getPublicTrades (targetMethod, context, argsList) { + await setTimeout(5000) + + return Reflect.apply(...arguments) + } + }) + rService._transactionTaxReport.rService = rServiceProxy + params.processorQueue = wrkReportServiceApi.lokue_processor.q + params.aggregatorQueue = wrkReportServiceApi.lokue_aggregator.q + + await emptyDB() + }) + + after(async function () { + this.timeout(20000) + + await stopEnvironment() + await rmDB(dbDirPath) + await rmAllFiles(tempDirPath, ['README.md']) + await rmRf(reportFolderPath) + + try { + await mockRESTv2Srv.close() + } catch (err) { } + }) + + signUpTestCase(agent, params, (token) => { auth.token = token }) + + it('it should be successfully performed by the syncNow method', async function () { + this.timeout(60000) + + const res = await agent + .post(`${basePath}/json-rpc`) + .type('json') + .send({ + auth, + method: 'syncNow', + id: 5 + }) + .expect('Content-Type', /json/) + .expect(200) + + assert.isObject(res.body) + assert.propertyVal(res.body, 'id', 5) + assert.isOk( + typeof res.body.result === 'number' || + res.body.result === 'SYNCHRONIZATION_IS_STARTED' + ) + }) + + getSyncProgressTestCase(agent, { basePath, auth }) + + it('it should interrupt transaction tax report', async function () { + this.timeout(60000) + + const trxTaxReportPromise = agent + .post(`${basePath}/json-rpc`) + .type('json') + .send({ + auth, + method: 'getTransactionTaxReport', + params: { + end, + start: start + (45 * 24 * 60 * 60 * 1000), + strategy: 'LIFO' + }, + id: 5 + }) + .expect('Content-Type', /json/) + .expect(200) + const interruptOperationsPromise = setTimeout(1000).then(() => { + return agent + .post(`${basePath}/json-rpc`) + .type('json') + .send({ + auth, + method: 'interruptOperations', + params: { + names: ['TRX_TAX_REPORT_INTERRUPTER'] + }, + id: 5 + }) + .expect('Content-Type', /json/) + .expect(200) + }) + + const [ + trxTaxReport, + interruptOperations + ] = await Promise.all([ + trxTaxReportPromise, + interruptOperationsPromise + ]) + + assert.isObject(interruptOperations.body) + assert.propertyVal(interruptOperations.body, 'id', 5) + assert.isBoolean(interruptOperations.body.result) + assert.isOk(interruptOperations.body.result) + + assert.isObject(trxTaxReport.body) + assert.propertyVal(trxTaxReport.body, 'id', 5) + assert.isArray(trxTaxReport.body.result) + assert.lengthOf(trxTaxReport.body.result, 0) + }) + + it('it should not be successfully performed by the interruptOperations method', async function () { + const res = await agent + .post(`${basePath}/json-rpc`) + .type('json') + .send({ + auth, + method: 'interruptOperations', + params: { + names: [ + 'FAKE_INTERRUPTER', + 'TRX_TAX_REPORT_INTERRUPTER' + ] + }, + id: 5 + }) + .expect('Content-Type', /json/) + .expect(400) + + assert.isObject(res.body) + assert.isObject(res.body.error) + assert.propertyVal(res.body.error, 'code', 400) + assert.propertyVal(res.body.error, 'message', 'Args params is not valid') + assert.propertyVal(res.body, 'id', 5) + }) + + it('it should interrupt transaction tax report after sign-out', async function () { + this.timeout(60000) + + const trxTaxReportPromise = agent + .post(`${basePath}/json-rpc`) + .type('json') + .send({ + auth, + method: 'getTransactionTaxReport', + params: { + end, + start: start + (45 * 24 * 60 * 60 * 1000), + strategy: 'LIFO' + }, + id: 5 + }) + .expect('Content-Type', /json/) + .expect(200) + const signOutPromise = setTimeout(1000).then(() => { + return agent + .post(`${basePath}/json-rpc`) + .type('json') + .send({ + auth, + method: 'signOut', + id: 5 + }) + .expect('Content-Type', /json/) + .expect(200) + }) + + const [ + trxTaxReport, + signOut + ] = await Promise.all([ + trxTaxReportPromise, + signOutPromise + ]) + + assert.isObject(signOut.body) + assert.propertyVal(signOut.body, 'id', 5) + assert.isBoolean(signOut.body.result) + assert.isOk(signOut.body.result) + + assert.isObject(trxTaxReport.body) + assert.propertyVal(trxTaxReport.body, 'id', 5) + assert.isArray(trxTaxReport.body.result) + assert.lengthOf(trxTaxReport.body.result, 0) + }) +}) diff --git a/test/config/default.json b/test/config/default.json index bc2254b82..226849f7c 100644 --- a/test/config/default.json +++ b/test/config/default.json @@ -1,7 +1,9 @@ { "app": { "port": 31339, - "host": "127.0.0.1" + "host": "127.0.0.1", + "httpRpcTimeout": 600000, + "wsRpcTimeout": 3600000 }, "grenacheClient": { "query": "rest:report:api", diff --git a/test/helpers/mock-data.js b/test/helpers/mock-data.js index dd927b363..8e423591e 100644 --- a/test/helpers/mock-data.js +++ b/test/helpers/mock-data.js @@ -130,7 +130,7 @@ module.exports = new Map([ 'trades', [ [ - 12345, + 100012345, 'tBTCUSD', _ms, 12345, @@ -143,7 +143,33 @@ module.exports = new Map([ 'USD' ], [ - 10012346, + 110012345, + 'tETHUSD', + _ms, + 12345, + 0.12345, + 12345, + null, + null, + false, + -3.0001, + 'USD' + ], + [ + 120012345, + 'tETHUSD', + _ms, + 12345, + -0.011, + 12355, + null, + null, + false, + -3.0001, + 'USD' + ], + [ + 130012345, 'tBTCEUR', _ms, 12345, @@ -156,7 +182,20 @@ module.exports = new Map([ 'BTC' ], [ - 20012347, + 140012345, + 'tBTCEUR', + _ms, + 12345, + -0.011, + 12355, + null, + null, + false, + -0.0001, + 'BTC' + ], + [ + 150012347, 'tBTCGBP', _ms, 12345, @@ -169,7 +208,20 @@ module.exports = new Map([ 'BTC' ], [ - 30012345, + 160012347, + 'tBTCGBP', + _ms, + 12345, + -0.011, + 12355, + null, + null, + false, + -0.0001, + 'BTC' + ], + [ + 170012345, 'tBTCJPY', _ms, 12345, @@ -180,6 +232,58 @@ module.exports = new Map([ false, -0.0001, 'BTC' + ], + [ + 180012345, + 'tBTCJPY', + _ms, + 12345, + -0.011, + 12355, + null, + null, + false, + -0.0001, + 'BTC' + ], + [ + 190012345, + 'tETHBTC', + _ms, + 12345, + 0.01, + 12345, + null, + null, + false, + -0.0001, + 'BTC' + ], + [ + 200012345, + 'tETHBTC', + _ms, + 12345, + -0.01, + 12355, + null, + null, + false, + -0.0001, + 'BTC' + ], + [ + 210012345, + 'tBTCUSD', + _ms, + 12345, + 0.12345, + 12345, + null, + null, + false, + -3.0001, + 'USD' ] ] ], diff --git a/test/test-cases/additional-api-sync-mode-sqlite-test-cases.js b/test/test-cases/additional-api-sync-mode-sqlite-test-cases.js index 768dc5d2e..d7f82f647 100644 --- a/test/test-cases/additional-api-sync-mode-sqlite-test-cases.js +++ b/test/test-cases/additional-api-sync-mode-sqlite-test-cases.js @@ -413,6 +413,84 @@ module.exports = ( } }) + it('it should be successfully performed by the getTransactionTaxReport method, LIFO strategy', async function () { + this.timeout(60000) + + const res = await agent + .post(`${basePath}/json-rpc`) + .type('json') + .send({ + auth, + method: 'getTransactionTaxReport', + params: { + end, + start: start + (45 * 24 * 60 * 60 * 1000), + strategy: 'LIFO' + }, + id: 5 + }) + .expect('Content-Type', /json/) + .expect(200) + + assert.isObject(res.body) + assert.propertyVal(res.body, 'id', 5) + assert.isArray(res.body.result) + assert.isAtLeast(res.body.result.length, 1) + + res.body.result.forEach((item) => { + assert.isObject(item) + assert.containsAllKeys(item, [ + 'asset', + 'amount', + 'mtsAcquired', + 'mtsSold', + 'proceeds', + 'cost', + 'gainOrLoss', + 'type' + ]) + }) + }) + + it('it should be successfully performed by the getTransactionTaxReport method, FIFO strategy', async function () { + this.timeout(60000) + + const res = await agent + .post(`${basePath}/json-rpc`) + .type('json') + .send({ + auth, + method: 'getTransactionTaxReport', + params: { + end, + start: start + (45 * 24 * 60 * 60 * 1000), + strategy: 'FIFO' + }, + id: 5 + }) + .expect('Content-Type', /json/) + .expect(200) + + assert.isObject(res.body) + assert.propertyVal(res.body, 'id', 5) + assert.isArray(res.body.result) + assert.isAtLeast(res.body.result.length, 1) + + res.body.result.forEach((item) => { + assert.isObject(item) + assert.containsAllKeys(item, [ + 'asset', + 'amount', + 'mtsAcquired', + 'mtsSold', + 'proceeds', + 'cost', + 'gainOrLoss', + 'type' + ]) + }) + }) + it('it should be successfully performed by the getTradedVolume method', async function () { this.timeout(60000) @@ -1034,6 +1112,33 @@ module.exports = ( ) }) + it('it should be successfully performed by the getTransactionTaxReportFile method', async function () { + this.timeout(60000) + + const procPromise = queueToPromise(params.processorQueue) + const aggrPromise = queueToPromise(params.aggregatorQueue) + + const res = await agent + .post(`${basePath}/json-rpc`) + .type('json') + .send({ + auth, + method: 'getTransactionTaxReportFile', + params: { + isPDFRequired, + end, + start, + strategy: 'LIFO', + email + }, + id: 5 + }) + .expect('Content-Type', /json/) + .expect(200) + + await testMethodOfGettingReportFile(procPromise, aggrPromise, res) + }) + it('it should be successfully performed by the getTradedVolumeFile method', async function () { this.timeout(60000) diff --git a/test/test-cases/api-sync-mode-sqlite-test-cases.js b/test/test-cases/api-sync-mode-sqlite-test-cases.js index ae03d76f5..644b29f5f 100644 --- a/test/test-cases/api-sync-mode-sqlite-test-cases.js +++ b/test/test-cases/api-sync-mode-sqlite-test-cases.js @@ -1808,7 +1808,7 @@ module.exports = ( symbol: ['tBTCUSD', 'tETHUSD'], start: 0, end, - limit: 10 + limit: 2 }, id: 5 }) diff --git a/workers/loc.api/bfx.api.router/index.js b/workers/loc.api/bfx.api.router/index.js index 5a011cd60..cabbba25f 100644 --- a/workers/loc.api/bfx.api.router/index.js +++ b/workers/loc.api/bfx.api.router/index.js @@ -3,6 +3,9 @@ const BaseBfxApiRouter = require( 'bfx-report/workers/loc.api/bfx.api.router' ) +const Interrupter = require( + 'bfx-report/workers/loc.api/interrupter' +) const RateLimitChecker = require('./rate.limit.checker') @@ -62,7 +65,7 @@ class BfxApiRouter extends BaseBfxApiRouter { /** * @override */ - route (methodName, method) { + route (methodName, method, interrupter) { if ( !methodName || methodName.startsWith('_') @@ -83,9 +86,27 @@ class BfxApiRouter extends BaseBfxApiRouter { if (rateLimitChecker.check()) { // Cool down delay - return new Promise((resolve) => ( - setTimeout(resolve, this._coolDownDelayMs)) - ).then(() => { + return new Promise((resolve) => { + const onceInterruptHandler = () => { + clearTimeout(timeout) + resolve() + } + const setTimeoutHandler = () => { + if (interrupter instanceof Interrupter) { + interrupter.offInterrupt(onceInterruptHandler) + } + + resolve() + } + + const timeout = setTimeout(setTimeoutHandler, this._coolDownDelayMs) + + if (!(interrupter instanceof Interrupter)) { + return + } + + interrupter.onceInterrupt(onceInterruptHandler) + }).then(() => { rateLimitChecker.add() return method() diff --git a/workers/loc.api/di/app.deps.js b/workers/loc.api/di/app.deps.js index db53474f0..ccf75527f 100644 --- a/workers/loc.api/di/app.deps.js +++ b/workers/loc.api/di/app.deps.js @@ -97,6 +97,7 @@ const { fullTaxReportCsvWriter } = require('../generate-report-file/csv-writer') const FullTaxReport = require('../sync/full.tax.report') +const TransactionTaxReport = require('../sync/transaction.tax.report') const WeightedAveragesReport = require('../sync/weighted.averages.report') const SqliteDbMigrator = require( '../sync/dao/db-migrations/sqlite.db.migrator' @@ -111,7 +112,8 @@ const { syncFactory, processMessageManagerFactory, syncUserStepDataFactory, - wsEventEmitterFactory + wsEventEmitterFactory, + interrupterFactory } = require('./factories') const Crypto = require('../sync/crypto') const Authenticator = require('../sync/authenticator') @@ -151,6 +153,7 @@ module.exports = ({ ['_positionsSnapshot', TYPES.PositionsSnapshot], ['_fullSnapshotReport', TYPES.FullSnapshotReport], ['_fullTaxReport', TYPES.FullTaxReport], + ['_transactionTaxReport', TYPES.TransactionTaxReport], ['_tradedVolume', TYPES.TradedVolume], ['_totalFeesReport', TYPES.TotalFeesReport], ['_performingLoan', TYPES.PerformingLoan], @@ -341,6 +344,8 @@ module.exports = ({ bind(TYPES.SyncInterrupter) .to(SyncInterrupter) .inSingletonScope() + bind(TYPES.InterrupterFactory) + .toFactory(interrupterFactory) bind(TYPES.Movements) .to(Movements) bind(TYPES.WinLossVSAccountBalance) @@ -393,6 +398,9 @@ module.exports = ({ ) bind(TYPES.FullTaxReport) .to(FullTaxReport) + bind(TYPES.TransactionTaxReport) + .to(TransactionTaxReport) + .inSingletonScope() rebind(TYPES.WeightedAveragesReport) .to(WeightedAveragesReport) rebind(TYPES.ReportFileJobData) diff --git a/workers/loc.api/di/factories/index.js b/workers/loc.api/di/factories/index.js index e0c1d5945..d0cde3b95 100644 --- a/workers/loc.api/di/factories/index.js +++ b/workers/loc.api/di/factories/index.js @@ -7,6 +7,7 @@ const syncFactory = require('./sync-factory') const processMessageManagerFactory = require('./process-message-manager-factory') const syncUserStepDataFactory = require('./sync-user-step-data-factory') const wsEventEmitterFactory = require('./ws-event-emitter-factory') +const interrupterFactory = require('./interrupter-factory') module.exports = { migrationsFactory, @@ -15,5 +16,6 @@ module.exports = { syncFactory, processMessageManagerFactory, syncUserStepDataFactory, - wsEventEmitterFactory + wsEventEmitterFactory, + interrupterFactory } diff --git a/workers/loc.api/di/factories/interrupter-factory.js b/workers/loc.api/di/factories/interrupter-factory.js new file mode 100644 index 000000000..866644474 --- /dev/null +++ b/workers/loc.api/di/factories/interrupter-factory.js @@ -0,0 +1,29 @@ +'use strict' + +const { + AuthError +} = require('bfx-report/workers/loc.api/errors') + +const TYPES = require('../types') + +module.exports = (ctx) => { + const authenticator = ctx.container.get(TYPES.Authenticator) + + return (params) => { + const { user, name } = params ?? {} + + if (!user) { + throw new AuthError() + } + + const interrupter = ctx.container.get( + TYPES.Interrupter + ).setName(name) + + authenticator.setInterrupterToUserSession( + user, interrupter + ) + + return interrupter + } +} diff --git a/workers/loc.api/di/types.js b/workers/loc.api/di/types.js index ab72b5cd5..316ac85fa 100644 --- a/workers/loc.api/di/types.js +++ b/workers/loc.api/di/types.js @@ -69,5 +69,7 @@ module.exports = { SyncUserStepData: Symbol.for('SyncUserStepData'), SyncUserStepDataFactory: Symbol.for('SyncUserStepDataFactory'), HTTPRequest: Symbol.for('HTTPRequest'), - SummaryByAsset: Symbol.for('SummaryByAsset') + SummaryByAsset: Symbol.for('SummaryByAsset'), + TransactionTaxReport: Symbol.for('TransactionTaxReport'), + InterrupterFactory: Symbol.for('InterrupterFactory') } diff --git a/workers/loc.api/errors/index.js b/workers/loc.api/errors/index.js index 6cd45c546..a57ac54e1 100644 --- a/workers/loc.api/errors/index.js +++ b/workers/loc.api/errors/index.js @@ -248,6 +248,30 @@ class AuthTokenTTLSettingError extends ArgsParamsError { } } +class CurrencyConversionError extends BaseError { + constructor (data, message = 'ERR_CURRENCY_HAS_NOT_BEEN_CONVERTED_TO_USD') { + super({ data, message }) + } +} + +class CurrencyPairSeparationError extends BaseError { + constructor (data, message = 'ERR_CURRENCY_PAIR_HAS_NOT_BEEN_SEPARATED_CORRECTLY') { + super({ data, message }) + } +} + +class PubTradeFindForTrxTaxError extends BaseError { + constructor (data, message = 'ERR_NO_PUBLIC_TRADES_FOR_TRX_TAX') { + super({ data, message }) + } +} + +class PubTradePriceFindForTrxTaxError extends BaseError { + constructor (message = 'ERR_NO_PUBLIC_TRADE_PRICE_FOR_TRX_TAX') { + super(message) + } +} + module.exports = { BaseError, CollSyncPermissionError, @@ -284,5 +308,9 @@ module.exports = { LastSyncedInfoGettingError, SyncInfoUpdatingError, AuthTokenGenerationError, - AuthTokenTTLSettingError + AuthTokenTTLSettingError, + CurrencyConversionError, + CurrencyPairSeparationError, + PubTradeFindForTrxTaxError, + PubTradePriceFindForTrxTaxError } diff --git a/workers/loc.api/generate-report-file/csv-writer/full-snapshot-report-csv-writer.js b/workers/loc.api/generate-report-file/csv-writer/full-snapshot-report-csv-writer.js index 044d8f04a..9bc1fc10a 100644 --- a/workers/loc.api/generate-report-file/csv-writer/full-snapshot-report-csv-writer.js +++ b/workers/loc.api/generate-report-file/csv-writer/full-snapshot-report-csv-writer.js @@ -47,7 +47,8 @@ module.exports = ( const res = await getDataFromApi({ getData: rService[name].bind(rService), args, - callerName: 'REPORT_FILE_WRITER' + callerName: 'REPORT_FILE_WRITER', + shouldNotInterrupt: true }) const { diff --git a/workers/loc.api/generate-report-file/csv-writer/full-tax-report-csv-writer.js b/workers/loc.api/generate-report-file/csv-writer/full-tax-report-csv-writer.js index c120c4547..15578c0e5 100644 --- a/workers/loc.api/generate-report-file/csv-writer/full-tax-report-csv-writer.js +++ b/workers/loc.api/generate-report-file/csv-writer/full-tax-report-csv-writer.js @@ -48,7 +48,8 @@ module.exports = ( const res = await getDataFromApi({ getData: rService[name].bind(rService), args, - callerName: 'REPORT_FILE_WRITER' + callerName: 'REPORT_FILE_WRITER', + shouldNotInterrupt: true }) const { timestamps, diff --git a/workers/loc.api/generate-report-file/report.file.job.data.js b/workers/loc.api/generate-report-file/report.file.job.data.js index 430725f1c..38e2271d5 100644 --- a/workers/loc.api/generate-report-file/report.file.job.data.js +++ b/workers/loc.api/generate-report-file/report.file.job.data.js @@ -528,6 +528,52 @@ class ReportFileJobData extends BaseReportFileJobData { return jobData } + async getTransactionTaxReportFileJobData ( + args, + uId, + uInfo + ) { + checkParams(args, 'paramsSchemaForTransactionTaxReportFile') + + const { + userId, + userInfo + } = await checkJobAndGetUserData( + this.rService, + uId, + uInfo + ) + + const reportFileArgs = getReportFileArgs(args) + + const jobData = { + userInfo, + userId, + name: 'getTransactionTaxReport', + fileNamesMap: [['getTransactionTaxReport', 'transaction-tax-report']], + args: reportFileArgs, + propNameForPagination: null, + columnsCsv: { + asset: 'CURRENCY', + type: 'SOURCE', + amount: 'AMOUNT', + mtsAcquired: 'DATE ACQUIRED', + mtsSold: 'DATE SOLD', + proceeds: 'PROCEEDS', + cost: 'COST', + gainOrLoss: 'GAIN OR LOSS' + }, + formatSettings: { + asset: 'symbol', + type: 'lowerCaseWithUpperFirst', + mtsAcquired: 'date', + mtsSold: 'date' + } + } + + return jobData + } + async getTradedVolumeFileJobData ( args, uId, diff --git a/workers/loc.api/helpers/index.js b/workers/loc.api/helpers/index.js index 677615245..528f54f6f 100644 --- a/workers/loc.api/helpers/index.js +++ b/workers/loc.api/helpers/index.js @@ -11,7 +11,8 @@ const { pickLowerObjectsNumbers, sumAllObjectsNumbers, pickAllLowerObjectsNumbers, - sumArrayVolumes + sumArrayVolumes, + pushLargeArr } = require('./utils') const { isSubAccountApiKeys, @@ -33,5 +34,6 @@ module.exports = { pickLowerObjectsNumbers, sumAllObjectsNumbers, pickAllLowerObjectsNumbers, - sumArrayVolumes + sumArrayVolumes, + pushLargeArr } diff --git a/workers/loc.api/helpers/schema.js b/workers/loc.api/helpers/schema.js index db81e9197..da5b45021 100644 --- a/workers/loc.api/helpers/schema.js +++ b/workers/loc.api/helpers/schema.js @@ -49,6 +49,22 @@ const paramsSchemaForUpdateSubAccount = { } } +const paramsSchemaForInterruptOperations = { + type: 'object', + properties: { + names: { + type: 'array', + minItems: 1, + items: { + type: 'string', + enum: [ + 'TRX_TAX_REPORT_INTERRUPTER' + ] + } + } + } +} + const paramsSchemaForCandlesApi = { ...cloneDeep(baseParamsSchemaForCandlesApi), properties: { @@ -214,6 +230,25 @@ const paramsSchemaForFullTaxReportApi = { } } +const paramsSchemaForTransactionTaxReportApi = { + type: 'object', + properties: { + end: { + type: 'integer' + }, + start: { + type: 'integer' + }, + strategy: { + type: 'string', + enum: [ + 'FIFO', + 'LIFO' + ] + } + } +} + const paramsSchemaForWinLossApi = { type: 'object', properties: { @@ -412,6 +447,15 @@ const paramsSchemaForFullTaxReportFile = { } } +const paramsSchemaForTransactionTaxReportFile = { + type: 'object', + properties: { + ...cloneDeep(paramsSchemaForTransactionTaxReportApi.properties), + timezone, + dateFormat + } +} + const paramsSchemaForTradedVolumeFile = { type: 'object', properties: { @@ -454,12 +498,14 @@ module.exports = { paramsSchemaForEditCandlesConf, paramsSchemaForCreateSubAccount, paramsSchemaForUpdateSubAccount, + paramsSchemaForInterruptOperations, paramsSchemaForBalanceHistoryApi, paramsSchemaForWinLossApi, paramsSchemaForWinLossVSAccountBalanceApi, paramsSchemaForPositionsSnapshotApi, paramsSchemaForFullSnapshotReportApi, paramsSchemaForFullTaxReportApi, + paramsSchemaForTransactionTaxReportApi, paramsSchemaForTradedVolumeApi, paramsSchemaForTotalFeesReportApi, paramsSchemaForPerformingLoanApi, @@ -471,6 +517,7 @@ module.exports = { paramsSchemaForPositionsSnapshotFile, paramsSchemaForFullSnapshotReportFile, paramsSchemaForFullTaxReportFile, + paramsSchemaForTransactionTaxReportFile, paramsSchemaForTradedVolumeFile, paramsSchemaForTotalFeesReportFile, paramsSchemaForPerformingLoanFile, diff --git a/workers/loc.api/helpers/utils.js b/workers/loc.api/helpers/utils.js index 541bb2591..75a8a374e 100644 --- a/workers/loc.api/helpers/utils.js +++ b/workers/loc.api/helpers/utils.js @@ -266,6 +266,12 @@ const sumArrayVolumes = (propName, objects = []) => { }, []) } +const pushLargeArr = (dest, src) => { + for (const item of src) { + dest.push(item) + } +} + module.exports = { checkParamsAuth, tryParseJSON, @@ -276,5 +282,6 @@ module.exports = { pickLowerObjectsNumbers, sumAllObjectsNumbers, pickAllLowerObjectsNumbers, - sumArrayVolumes + sumArrayVolumes, + pushLargeArr } diff --git a/workers/loc.api/process.message.manager/process.messages.js b/workers/loc.api/process.message.manager/process.messages.js index 07aaad4e3..7ba879a8e 100644 --- a/workers/loc.api/process.message.manager/process.messages.js +++ b/workers/loc.api/process.message.manager/process.messages.js @@ -7,6 +7,9 @@ module.exports = { READY_MIGRATIONS: 'ready:migrations', ERROR_MIGRATIONS: 'error:migrations', + READY_TRX_TAX_REPORT: 'ready:trx-tax-report', + ERROR_TRX_TAX_REPORT: 'error:trx-tax-report', + ALL_TABLE_HAVE_BEEN_CLEARED: 'all-tables-have-been-cleared', ALL_TABLE_HAVE_NOT_BEEN_CLEARED: 'all-tables-have-not-been-cleared', ALL_TABLE_HAVE_BEEN_REMOVED: 'all-tables-have-been-removed', diff --git a/workers/loc.api/service.report.framework.js b/workers/loc.api/service.report.framework.js index 057c195a8..c6631ddb9 100644 --- a/workers/loc.api/service.report.framework.js +++ b/workers/loc.api/service.report.framework.js @@ -256,7 +256,9 @@ class FrameworkReportService extends ReportService { getPlatformStatus (space, args, cb) { return this._responder(async () => { - const rest = this._getREST({}) + const rest = this._getREST({}, { + interrupter: args?.interrupter + }) const res = await rest.status() const isMaintenance = !Array.isArray(res) || !res[0] @@ -463,6 +465,14 @@ class FrameworkReportService extends ReportService { }, 'stopSyncNow', args, cb) } + interruptOperations (space, args = {}, cb) { + return this._privResponder(() => { + checkParams(args, 'paramsSchemaForInterruptOperations', ['names']) + + return this._authenticator.interruptOperations(args) + }, 'interruptOperations', args, cb) + } + getPublicTradesConf (space, args = {}, cb) { return this._privResponder(() => { return this._publicCollsConfAccessors @@ -720,7 +730,8 @@ class FrameworkReportService extends ReportService { (args) => this._getDataFromApi({ getData: (space, args) => super.getActivePositions(space, args), args, - callerName: 'ACTIVE_POSITIONS_GETTER' + callerName: 'ACTIVE_POSITIONS_GETTER', + shouldNotInterrupt: true }), args, { @@ -760,7 +771,8 @@ class FrameworkReportService extends ReportService { return super.getPositionsAudit(space, args) }, args, - callerName: 'POSITIONS_AUDIT_GETTER' + callerName: 'POSITIONS_AUDIT_GETTER', + shouldNotInterrupt: true }), args, { @@ -1409,6 +1421,28 @@ class FrameworkReportService extends ReportService { }, 'getFullTaxReport', args, cb) } + getTransactionTaxReport (space, args, cb) { + return this._privResponder(async () => { + await this._dataConsistencyChecker + .check(this._CHECKER_NAMES.TRANSACTION_TAX_REPORT, args) + + checkParams(args, 'paramsSchemaForTransactionTaxReportApi') + + return this._transactionTaxReport.getTransactionTaxReport(args) + }, 'getTransactionTaxReport', args, cb) + } + + makeTrxTaxReportInBackground (space, args, cb) { + return this._privResponder(async () => { + await this._dataConsistencyChecker + .check(this._CHECKER_NAMES.TRANSACTION_TAX_REPORT, args) + + checkParams(args, 'paramsSchemaForTransactionTaxReportApi') + + return this._transactionTaxReport.makeTrxTaxReportInBackground(args) + }, 'makeTrxTaxReportInBackground', args, cb) + } + getTradedVolume (space, args, cb) { return this._privResponder(async () => { await this._dataConsistencyChecker @@ -1553,6 +1587,20 @@ class FrameworkReportService extends ReportService { }, 'getFullTaxReportFile', args, cb) } + /** + * @deprecated + */ + getTransactionTaxReportCsv (...args) { return this.getTransactionTaxReportFile(...args) } + + getTransactionTaxReportFile (space, args, cb) { + return this._responder(() => { + return this._generateReportFile( + 'getTransactionTaxReportFileJobData', + args + ) + }, 'getTransactionTaxReportFile', args, cb) + } + /** * @deprecated */ diff --git a/workers/loc.api/sync/authenticator/index.js b/workers/loc.api/sync/authenticator/index.js index 79f98a06d..31ededdaf 100644 --- a/workers/loc.api/sync/authenticator/index.js +++ b/workers/loc.api/sync/authenticator/index.js @@ -10,6 +10,9 @@ const { isENetError, isAuthError } = require('bfx-report/workers/loc.api/helpers') +const Interrupter = require( + 'bfx-report/workers/loc.api/interrupter' +) const { serializeVal } = require('../dao/helpers') const { @@ -501,6 +504,7 @@ class Authenticator { throw new AuthError() } + await this._processInterrupters(user) this.removeUserSession({ email, isSubAccount, @@ -750,7 +754,7 @@ class Authenticator { ) { const session = this.getUserSessionByToken( token, - isReturnedPassword + { isReturnedPassword } ) const { authToken, apiKey, apiSecret } = session ?? {} @@ -1105,6 +1109,40 @@ class Authenticator { return true } + setInterrupterToUserSession (user, interrupter) { + const userSession = this.getUserSessionByEmail( + user, + { shouldOrigObjRefBeReturned: true } + ) + + if (!userSession) { + throw new AuthError() + } + + userSession.interrupters = userSession.interrupters instanceof Set + ? userSession.interrupters + : new Set() + + if (!(interrupter instanceof Interrupter)) { + return + } + + interrupter.onceInterrupted(() => { + userSession.interrupters.delete(interrupter) + }) + + userSession.interrupters.add(interrupter) + } + + async interruptOperations (args) { + await this._processInterrupters( + args?.auth, + args?.params?.names + ) + + return true + } + setUserSession (user) { const { token, @@ -1135,17 +1173,35 @@ class Authenticator { this.userTokenMapByEmail.set(tokenKey, token) } - getUserSessionByToken (token, isReturnedPassword) { + getUserSessionByToken (token, opts) { + const { + isReturnedPassword, + shouldOrigObjRefBeReturned + } = opts ?? {} + const session = this.userSessions.get(token) + if (shouldOrigObjRefBeReturned) { + return session + } + return pickSessionProps(session, isReturnedPassword) } - getUserSessionByEmail (user, isReturnedPassword) { + getUserSessionByEmail (user, opts) { + const { + isReturnedPassword, + shouldOrigObjRefBeReturned + } = opts ?? {} + const tokenKey = this._getTokenKeyByEmailField(user) const token = this.userTokenMapByEmail.get(tokenKey) const session = this.userSessions.get(token) + if (shouldOrigObjRefBeReturned) { + return session + } + return pickSessionProps(session, isReturnedPassword) } @@ -1518,6 +1574,46 @@ class Authenticator { ) ) } + + async _processInterrupters (user, names) { + const _names = Array.isArray(names) + ? names + : [names] + const filteredName = _names.filter((name) => ( + name && + typeof name === 'string' + )) + const userSession = this.getUserSessionByEmail( + user, + { shouldOrigObjRefBeReturned: true } + ) + + if ( + !(userSession?.interrupters instanceof Set) || + userSession.interrupters.size === 0 + ) { + return [] + } + + const promises = [] + const interrupters = [...userSession.interrupters] + .filter((int) => ( + filteredName.length === 0 || + filteredName.some((name) => name === int.name) + )) + + for (const interrupter of interrupters) { + userSession.interrupters.delete(interrupter) + + if (!(interrupter instanceof Interrupter)) { + continue + } + + promises.push(interrupter.interrupt()) + } + + return await Promise.all(promises) + } } decorateInjectable(Authenticator, depsTypes) diff --git a/workers/loc.api/sync/dao/dao.better.sqlite.js b/workers/loc.api/sync/dao/dao.better.sqlite.js index b2b735c60..6c2a2a9cc 100644 --- a/workers/loc.api/sync/dao/dao.better.sqlite.js +++ b/workers/loc.api/sync/dao/dao.better.sqlite.js @@ -988,6 +988,7 @@ class BetterSqliteDAO extends DAO { filter = {}, sort = [], subQuery = { + filter: {}, sort: [] }, groupFns = [], @@ -1003,7 +1004,10 @@ class BetterSqliteDAO extends DAO { group, groupProj } = getGroupQuery({ groupFns, groupResBy }) - const _subQuery = getSubQuery({ name: collName, subQuery }) + const { + subQuery: _subQuery, + subQueryValues + } = getSubQuery({ name: collName, subQuery }) const _sort = getOrderQuery(sort) const { where, @@ -1035,7 +1039,7 @@ class BetterSqliteDAO extends DAO { return this.query({ action: MAIN_DB_WORKER_ACTIONS.ALL, sql, - params: { ...values, ...limitVal } + params: { ...values, ...subQueryValues, ...limitVal } }, { withWorkerThreads: true }) } diff --git a/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v31.js b/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v31.js deleted file mode 100644 index 10b4ecc0f..000000000 --- a/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v31.js +++ /dev/null @@ -1,339 +0,0 @@ -'use strict' - -const AbstractMigration = require('./abstract.migration') -const { getSqlArrToModifyColumns } = require('./helpers') - -const { - CONSTR_FIELD_NAME, - TRIGGER_FIELD_NAME, - ID_PRIMARY_KEY -} = require('./helpers/const') - -const CREATE_UPDATE_MTS_TRIGGERS = [ - `insert_#{tableName}_createdAt_and_updatedAt - AFTER INSERT ON #{tableName} - FOR EACH ROW - BEGIN - UPDATE #{tableName} - SET createdAt = CAST((julianday('now') - 2440587.5) * 86400000.0 as INT), - updatedAt = CAST((julianday('now') - 2440587.5) * 86400000.0 as INT) - WHERE _id = NEW._id; - END`, - `update_#{tableName}_updatedAt - AFTER UPDATE ON #{tableName} - FOR EACH ROW - BEGIN - UPDATE #{tableName} - SET updatedAt = CAST((julianday('now') - 2440587.5) * 86400000.0 as INT) - WHERE _id = NEW._id; - END` -] -const QUERY_TO_SET_FRESH_MTS = `UPDATE #{tableName} -SET createdAt = CAST((julianday('now') - 2440587.5) * 86400000.0 as INT), - updatedAt = CAST((julianday('now') - 2440587.5) * 86400000.0 as INT)` -const QUERIES_TO_DELETE_DATA = [ - 'DELETE FROM ledgers', - 'DELETE FROM trades', - 'DELETE FROM fundingTrades', - 'DELETE FROM publicTrades', - 'DELETE FROM orders', - 'DELETE FROM movements', - 'DELETE FROM fundingOfferHistory', - 'DELETE FROM fundingLoanHistory', - 'DELETE FROM fundingCreditHistory', - 'DELETE FROM positionsHistory', - 'DELETE FROM positionsSnapshot', - 'DELETE FROM logins', - 'DELETE FROM changeLogs', - 'DELETE FROM payInvoiceList', - 'DELETE FROM tickersHistory', - 'DELETE FROM statusMessages', - 'DELETE FROM symbols', - 'DELETE FROM mapSymbols', - 'DELETE FROM inactiveCurrencies', - 'DELETE FROM inactiveSymbols', - 'DELETE FROM futures', - 'DELETE FROM currencies', - 'DELETE FROM candles' -] -const QUERY_TO_SET_INITIAL_SYNC_PROGRESS_STATE = `\ -UPDATE progress - SET value = 'SYNCHRONIZATION_HAS_NOT_STARTED_YET'` - -const _replacePlaceholder = (sql, tableName) => { - return sql.replace(/#{tableName\}/g, tableName) -} - -const _getCreateUpdateMtsTriggers = (tableName) => { - return CREATE_UPDATE_MTS_TRIGGERS.map((item) => { - const sql = _replacePlaceholder(item, tableName) - - return `CREATE TRIGGER IF NOT EXISTS ${sql}` - }) -} - -const _getQueryToSetFreshMts = (tableName) => { - return _replacePlaceholder( - QUERY_TO_SET_FRESH_MTS, - tableName - ) -} - -class MigrationV31 extends AbstractMigration { - /** - * @override - */ - before () { return this.dao.disableForeignKeys() } - - /** - * @override - */ - up () { - const sqlArr = [ - 'ALTER TABLE users ADD COLUMN createdAt BIGINT', - 'ALTER TABLE users ADD COLUMN updatedAt BIGINT', - ..._getCreateUpdateMtsTriggers('users'), - _getQueryToSetFreshMts('users'), - - 'ALTER TABLE subAccounts ADD COLUMN createdAt BIGINT', - 'ALTER TABLE subAccounts ADD COLUMN updatedAt BIGINT', - ..._getCreateUpdateMtsTriggers('subAccounts'), - _getQueryToSetFreshMts('subAccounts'), - - 'DROP INDEX IF EXISTS statusMessages_timestamp_key__type', - 'DROP INDEX IF EXISTS mapSymbols_key_value', - - 'ALTER TABLE publicСollsСonf ADD COLUMN createdAt BIGINT', - 'ALTER TABLE publicСollsСonf ADD COLUMN updatedAt BIGINT', - ..._getCreateUpdateMtsTriggers('publicСollsСonf'), - _getQueryToSetFreshMts('publicСollsСonf'), - - 'ALTER TABLE scheduler ADD COLUMN createdAt BIGINT', - 'ALTER TABLE scheduler ADD COLUMN updatedAt BIGINT', - ..._getCreateUpdateMtsTriggers('scheduler'), - _getQueryToSetFreshMts('scheduler'), - - 'ALTER TABLE syncMode ADD COLUMN createdAt BIGINT', - 'ALTER TABLE syncMode ADD COLUMN updatedAt BIGINT', - ..._getCreateUpdateMtsTriggers('syncMode'), - _getQueryToSetFreshMts('syncMode'), - - 'ALTER TABLE progress ADD COLUMN createdAt BIGINT', - 'ALTER TABLE progress ADD COLUMN updatedAt BIGINT', - ..._getCreateUpdateMtsTriggers('progress'), - _getQueryToSetFreshMts('progress'), - - 'DROP TABLE IF EXISTS syncQueue', - `CREATE TABLE syncQueue ( - _id ${ID_PRIMARY_KEY}, - collName VARCHAR(255) NOT NULL, - state VARCHAR(255), - createdAt BIGINT, - updatedAt BIGINT, - ownerUserId INT, - isOwnerScheduler INT, - CONSTRAINT syncQueue_fk_ownerUserId - FOREIGN KEY(ownerUserId) - REFERENCES users(_id) - ON UPDATE CASCADE - ON DELETE CASCADE - )`, - - 'DROP TABLE IF EXISTS completedOnFirstSyncColls', - `CREATE TABLE syncUserSteps ( - _id ${ID_PRIMARY_KEY}, - collName VARCHAR(255) NOT NULL, - syncedAt BIGINT, - baseStart BIGINT, - baseEnd BIGINT, - isBaseStepReady INT, - currStart BIGINT, - currEnd BIGINT, - isCurrStepReady INT, - createdAt BIGINT, - updatedAt BIGINT, - subUserId INT, - user_id INT, - syncQueueId INT, - CONSTRAINT syncUserSteps_fk_user_id - FOREIGN KEY(user_id) - REFERENCES users(_id) - ON UPDATE CASCADE - ON DELETE CASCADE, - CONSTRAINT syncUserSteps_fk_subUserId - FOREIGN KEY(subUserId) - REFERENCES users(_id) - ON UPDATE CASCADE - ON DELETE CASCADE - )`, - - /* - * Delete data to start the sync from scratch to avoid inconsistency - * if the previous sync step has not been finished successfully - */ - ...QUERIES_TO_DELETE_DATA, - QUERY_TO_SET_INITIAL_SYNC_PROGRESS_STATE - ] - - this.addSql(sqlArr) - } - - /** - * @override - */ - down () { - const sqlArr = [ - 'DROP TRIGGER insert_users_createdAt_and_updatedAt', - 'DROP TRIGGER update_users_updatedAt', - ...getSqlArrToModifyColumns( - 'users', - { - _id: ID_PRIMARY_KEY, - id: 'BIGINT', - email: 'VARCHAR(255)', - apiKey: 'VARCHAR(255) NOT NULL', - apiSecret: 'VARCHAR(255) NOT NULL', - active: 'INT', - isDataFromDb: 'INT', - timezone: 'VARCHAR(255)', - username: 'VARCHAR(255)', - passwordHash: 'VARCHAR(255)', - isNotProtected: 'INT', - isSubAccount: 'INT', - isSubUser: 'INT' - } - ), - - 'DROP TRIGGER insert_subAccounts_createdAt_and_updatedAt', - 'DROP TRIGGER update_subAccounts_updatedAt', - ...getSqlArrToModifyColumns( - 'subAccounts', - { - _id: ID_PRIMARY_KEY, - masterUserId: 'INT NOT NULL', - subUserId: 'INT NOT NULL', - - [CONSTR_FIELD_NAME]: [ - `CONSTRAINT #{tableName}_fk_masterUserId - FOREIGN KEY (masterUserId) - REFERENCES users(_id) - ON UPDATE CASCADE - ON DELETE CASCADE`, - `CONSTRAINT #{tableName}_fk_subUserId - FOREIGN KEY (subUserId) - REFERENCES users(_id) - ON UPDATE CASCADE - ON DELETE CASCADE` - ], - [TRIGGER_FIELD_NAME]: `delete_#{tableName}_subUsers_from_users - AFTER DELETE ON #{tableName} - FOR EACH ROW - BEGIN - DELETE FROM users - WHERE _id = OLD.subUserId; - END` - } - ), - - 'DROP INDEX IF EXISTS statusMessages_key__type', - 'DROP INDEX IF EXISTS symbols_pairs', - 'DROP INDEX IF EXISTS mapSymbols_key', - 'DROP INDEX IF EXISTS inactiveCurrencies_pairs', - 'DROP INDEX IF EXISTS inactiveSymbols_pairs', - 'DROP INDEX IF EXISTS futures_pairs', - - 'DROP TRIGGER insert_publicСollsСonf_createdAt_and_updatedAt', - 'DROP TRIGGER update_publicСollsСonf_updatedAt', - ...getSqlArrToModifyColumns( - 'publicСollsСonf', - { - _id: ID_PRIMARY_KEY, - confName: 'VARCHAR(255)', - symbol: 'VARCHAR(255)', - start: 'BIGINT', - timeframe: 'VARCHAR(255)', - user_id: 'INT NOT NULL', - - [CONSTR_FIELD_NAME]: `\ - CONSTRAINT #{tableName}_fk_user_id - FOREIGN KEY (user_id) - REFERENCES users(_id) - ON UPDATE CASCADE - ON DELETE CASCADE` - } - ), - - 'DROP TRIGGER insert_scheduler_createdAt_and_updatedAt', - 'DROP TRIGGER update_scheduler_updatedAt', - ...getSqlArrToModifyColumns( - 'scheduler', - { - _id: ID_PRIMARY_KEY, - isEnable: 'INT' - } - ), - - 'DROP TRIGGER insert_syncMode_createdAt_and_updatedAt', - 'DROP TRIGGER update_syncMode_updatedAt', - ...getSqlArrToModifyColumns( - 'syncMode', - { - _id: ID_PRIMARY_KEY, - isEnable: 'INT' - } - ), - - 'DROP TRIGGER insert_progress_createdAt_and_updatedAt', - 'DROP TRIGGER update_progress_updatedAt', - ...getSqlArrToModifyColumns( - 'progress', - { - _id: ID_PRIMARY_KEY, - value: 'VARCHAR(255)' - } - ), - - 'DROP TABLE IF EXISTS syncQueue', - `CREATE TABLE syncQueue ( - _id ${ID_PRIMARY_KEY}, - collName VARCHAR(255), - state VARCHAR(255) - )`, - - 'DROP TABLE IF EXISTS syncUserSteps', - `CREATE TABLE completedOnFirstSyncColls ( - _id ${ID_PRIMARY_KEY}, - collName VARCHAR(255) NOT NULL, - mts BIGINT, - subUserId INT, - user_id INT, - CONSTRAINT completedOnFirstSyncColls_fk_user_id - FOREIGN KEY(user_id) - REFERENCES users(_id) - ON UPDATE CASCADE - ON DELETE CASCADE, - CONSTRAINT completedOnFirstSyncColls_fk_subUserId - FOREIGN KEY(subUserId) - REFERENCES users(_id) - ON UPDATE CASCADE - ON DELETE CASCADE - )`, - - /* - * Delete data to start the sync from scratch to avoid inconsistency - * if the previous sync step has not been finished successfully - */ - ...QUERIES_TO_DELETE_DATA, - QUERY_TO_SET_INITIAL_SYNC_PROGRESS_STATE - ] - - this.addSql(sqlArr) - } - - /** - * @override - */ - after () { return this.dao.enableForeignKeys() } -} - -module.exports = MigrationV31 diff --git a/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v41.1716385152034.js b/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v41.1716385152034.js new file mode 100644 index 000000000..32316f81a --- /dev/null +++ b/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v41.1716385152034.js @@ -0,0 +1,148 @@ +'use strict' + +/* + * CREATED_AT: 2024-05-22T13:39:12.034Z + * VERSION: v41 + */ + +const { + ID_PRIMARY_KEY +} = require('./helpers/const') +const { + getSqlArrToModifyColumns +} = require('./helpers') + +const AbstractMigration = require('./abstract.migration') + +class MigrationV41 extends AbstractMigration { + /** + * @override + */ + up () { + const sqlArr = [ + 'ALTER TABLE ledgers ADD COLUMN exactUsdValue DECIMAL(22,12)', + 'ALTER TABLE trades ADD COLUMN exactUsdValue DECIMAL(22,12)', + 'ALTER TABLE movements ADD COLUMN exactUsdValue DECIMAL(22,12)', + + 'ALTER TABLE ledgers ADD COLUMN _isInvoicePayOrder INT', + `UPDATE ledgers SET _isInvoicePayOrder = ( + SELECT 1 FROM ( + SELECT * FROM ledgers AS l + WHERE l.description COLLATE NOCASE LIKE '%InvoicePay Order%' + AND l._id = ledgers._id + ) + )`, + + 'ALTER TABLE ledgers ADD COLUMN _isAirdropOnWallet INT', + `UPDATE ledgers SET _isAirdropOnWallet = ( + SELECT 1 FROM ( + SELECT * FROM ledgers AS l + WHERE l.description COLLATE NOCASE LIKE '%Airdrop on wallet%' + AND l._id = ledgers._id + ) + )`, + + 'ALTER TABLE trades ADD COLUMN _isExchange INT', + `UPDATE trades SET _isExchange = ( + SELECT 1 FROM ( + SELECT * FROM trades AS t + WHERE t.orderType COLLATE NOCASE LIKE '%EXCHANGE%' + AND t._id = trades._id + ) + )` + ] + + this.addSql(sqlArr) + } + + /** + * @override + */ + down () { + const sqlArr = [ + ...getSqlArrToModifyColumns( + 'ledgers', + { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + currency: 'VARCHAR(255)', + mts: 'BIGINT', + amount: 'DECIMAL(22,12)', + amountUsd: 'DECIMAL(22,12)', + exactUsdValue: 'DECIMAL(22,12)', + balance: 'DECIMAL(22,12)', + _nativeBalance: 'DECIMAL(22,12)', + balanceUsd: 'DECIMAL(22,12)', + _nativeBalanceUsd: 'DECIMAL(22,12)', + description: 'TEXT', + wallet: 'VARCHAR(255)', + _category: 'INT', + _isMarginFundingPayment: 'INT', + _isAffiliateRebate: 'INT', + _isStakingPayments: 'INT', + _isSubAccountsTransfer: 'INT', + _isBalanceRecalced: 'INT', + subUserId: 'INT', + user_id: 'INT NOT NULL' + } + ), + + ...getSqlArrToModifyColumns( + 'trades', + { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + symbol: 'VARCHAR(255)', + mtsCreate: 'BIGINT', + orderID: 'BIGINT', + execAmount: 'DECIMAL(22,12)', + execPrice: 'DECIMAL(22,12)', + exactUsdValue: 'DECIMAL(22,12)', + orderType: 'VARCHAR(255)', + orderPrice: 'DECIMAL(22,12)', + maker: 'INT', + fee: 'DECIMAL(22,12)', + feeCurrency: 'VARCHAR(255)', + subUserId: 'INT', + user_id: 'INT NOT NULL' + } + ), + + ...getSqlArrToModifyColumns( + 'movements', + { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + currency: 'VARCHAR(255)', + currencyName: 'VARCHAR(255)', + mtsStarted: 'BIGINT', + mtsUpdated: 'BIGINT', + status: 'VARCHAR(255)', + amount: 'DECIMAL(22,12)', + amountUsd: 'DECIMAL(22,12)', + exactUsdValue: 'DECIMAL(22,12)', + fees: 'DECIMAL(22,12)', + destinationAddress: 'VARCHAR(255)', + transactionId: 'VARCHAR(255)', + note: 'TEXT', + subUserId: 'INT', + user_id: 'INT NOT NULL' + } + ) + ] + + this.addSql(sqlArr) + } + + /** + * @override + */ + beforeDown () { return this.dao.disableForeignKeys() } + + /** + * @override + */ + afterDown () { return this.dao.enableForeignKeys() } +} + +module.exports = MigrationV41 diff --git a/workers/loc.api/sync/dao/helpers/find-in-coll-by/get-query.js b/workers/loc.api/sync/dao/helpers/find-in-coll-by/get-query.js index 6c6745764..08d11a503 100644 --- a/workers/loc.api/sync/dao/helpers/find-in-coll-by/get-query.js +++ b/workers/loc.api/sync/dao/helpers/find-in-coll-by/get-query.js @@ -44,7 +44,7 @@ module.exports = (args, methodColl, opts) => { group, groupProj } = getGroupQuery(methodColl) - const subQuery = getSubQuery(methodColl) + const { subQuery, subQueryValues } = getSubQuery(methodColl) const projection = getProjectionQuery( _model, exclude, @@ -65,6 +65,6 @@ module.exports = (args, methodColl, opts) => { return { sql, - sqlParams: { ...values, ...limitVal } + sqlParams: { ...values, ...subQueryValues, ...limitVal } } } diff --git a/workers/loc.api/sync/dao/helpers/get-sub-query.js b/workers/loc.api/sync/dao/helpers/get-sub-query.js index 43f69cd9a..b536f5024 100644 --- a/workers/loc.api/sync/dao/helpers/get-sub-query.js +++ b/workers/loc.api/sync/dao/helpers/get-sub-query.js @@ -1,16 +1,39 @@ 'use strict' +const getWhereQuery = require('./get-where-query') const getOrderQuery = require('./get-order-query') -module.exports = ({ - name, - subQuery: { sort = [] } = {} -} = {}) => { +module.exports = (params) => { + const { + name + } = params ?? {} + const filter = params?.subQuery?.filter ?? {} + const sort = params?.subQuery?.sort ?? [] + + const alias = `${name}_sub_q` + const { + where, + values + } = getWhereQuery( + filter, + { alias } + ) const _sort = getOrderQuery(sort) - if (!_sort) { - return name + if ( + !_sort && + !where + ) { + return { + subQuery: name, + subQueryValues: {} + } } - return `(SELECT * FROM ${name} ${_sort})` + const subQuery = `(SELECT * FROM ${name} AS ${alias} ${where} ${_sort})` + + return { + subQuery, + subQueryValues: values + } } diff --git a/workers/loc.api/sync/dao/helpers/serialization/deserialize-val.js b/workers/loc.api/sync/dao/helpers/serialization/deserialize-val.js index e34febd4c..26d52e079 100644 --- a/workers/loc.api/sync/dao/helpers/serialization/deserialize-val.js +++ b/workers/loc.api/sync/dao/helpers/serialization/deserialize-val.js @@ -16,7 +16,10 @@ module.exports = ( '_isMarginFundingPayment', '_isAffiliateRebate', '_isStakingPayments', - '_isSubAccountsTransfer' + '_isSubAccountsTransfer', + '_isInvoicePayOrder', + '_isAirdropOnWallet', + '_isExchange' ] ) => { if ( diff --git a/workers/loc.api/sync/data.consistency.checker/checker.names.js b/workers/loc.api/sync/data.consistency.checker/checker.names.js index db644c065..f4dff2943 100644 --- a/workers/loc.api/sync/data.consistency.checker/checker.names.js +++ b/workers/loc.api/sync/data.consistency.checker/checker.names.js @@ -7,6 +7,7 @@ module.exports = { POSITIONS_SNAPSHOT: 'getPositionsSnapshot', FULL_SNAPSHOT_REPORT: 'getFullSnapshotReport', FULL_TAX_REPORT: 'getFullTaxReport', + TRANSACTION_TAX_REPORT: 'getTransactionTaxReport', TRADED_VOLUME: 'getTradedVolume', TOTAL_FEES_REPORT: 'getTotalFeesReport', PERFORMING_LOAN: 'getPerformingLoan', diff --git a/workers/loc.api/sync/data.consistency.checker/checkers.js b/workers/loc.api/sync/data.consistency.checker/checkers.js index 041ed2cbf..ff5b666e4 100644 --- a/workers/loc.api/sync/data.consistency.checker/checkers.js +++ b/workers/loc.api/sync/data.consistency.checker/checkers.js @@ -103,6 +103,20 @@ class Checkers { }) } + [CHECKER_NAMES.TRANSACTION_TAX_REPORT] (auth) { + return this.syncCollsManager + .haveCollsBeenSyncedUpToDate({ + auth, + params: { + schema: [ + this.SYNC_API_METHODS.TRADES, + this.SYNC_API_METHODS.LEDGERS, + this.SYNC_API_METHODS.MOVEMENTS + ] + } + }) + } + [CHECKER_NAMES.TRADED_VOLUME] (auth) { return this.syncCollsManager .haveCollsBeenSyncedUpToDate({ diff --git a/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js b/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js index 56f18e8db..57830ae32 100644 --- a/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js +++ b/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js @@ -3,7 +3,7 @@ const SYNC_API_METHODS = require('../../schema/sync.api.methods') const { addPropsToResIfExist, - getFlagsFromLedgerDescription, + getFlagsFromStringProp, getCategoryFromDescription, convertArrayMapToObjectMap } = require('./helpers') @@ -106,12 +106,13 @@ class ApiMiddlewareHandlerAfter { [SYNC_API_METHODS.LEDGERS] (args, apiRes) { const res = apiRes.res.map(item => { - const { balance } = { ...item } + const { balance } = item ?? {} return { ...item, - ...getFlagsFromLedgerDescription( + ...getFlagsFromStringProp( item, + 'description', [ { fieldName: '_isMarginFundingPayment', @@ -132,6 +133,14 @@ class ApiMiddlewareHandlerAfter { { fieldName: '_isSubAccountsTransfer', pattern: '^transfer.+sa[(].+[)]' + }, + { + fieldName: '_isInvoicePayOrder', + pattern: 'InvoicePay Order' + }, + { + fieldName: '_isAirdropOnWallet', + pattern: 'Airdrop on wallet' } ] ), @@ -145,6 +154,29 @@ class ApiMiddlewareHandlerAfter { } } + [SYNC_API_METHODS.TRADES] (args, apiRes) { + const res = apiRes.res.map((item) => { + return { + ...item, + ...getFlagsFromStringProp( + item, + 'orderType', + [ + { + fieldName: '_isExchange', + pattern: 'EXCHANGE' + } + ] + ) + } + }) + + return { + ...apiRes, + res + } + } + [SYNC_API_METHODS.CANDLES] (args, apiRes) { return addPropsToResIfExist( args, diff --git a/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-flags-from-ledger-description.js b/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-flags-from-string-prop.js similarity index 64% rename from workers/loc.api/sync/data.inserter/api.middleware/helpers/get-flags-from-ledger-description.js rename to workers/loc.api/sync/data.inserter/api.middleware/helpers/get-flags-from-string-prop.js index f2f5ad520..ada7a5479 100644 --- a/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-flags-from-ledger-description.js +++ b/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-flags-from-string-prop.js @@ -1,14 +1,13 @@ 'use strict' module.exports = ( - ledger, + item, + stringPropName, schemas = [] ) => { - if ( - !ledger || - typeof ledger !== 'object' || - typeof ledger.description !== 'string' - ) { + const stringProp = item?.[stringPropName] + + if (typeof stringProp !== 'string') { return {} } @@ -18,12 +17,12 @@ module.exports = ( pattern, handler, isCaseSensitivity - } = { ...schema } + } = schema ?? {} if (typeof handler === 'function') { return { ...accum, - [fieldName]: handler(ledger.description) + [fieldName]: handler(stringProp) } } @@ -34,7 +33,7 @@ module.exports = ( return { ...accum, - [fieldName]: regExp.test(ledger.description) + [fieldName]: regExp.test(stringProp) } }, {}) } diff --git a/workers/loc.api/sync/data.inserter/api.middleware/helpers/index.js b/workers/loc.api/sync/data.inserter/api.middleware/helpers/index.js index 2ad42b071..2118a78bc 100644 --- a/workers/loc.api/sync/data.inserter/api.middleware/helpers/index.js +++ b/workers/loc.api/sync/data.inserter/api.middleware/helpers/index.js @@ -3,8 +3,8 @@ const addPropsToResIfExist = require( './add-props-to-res-if-exist' ) -const getFlagsFromLedgerDescription = require( - './get-flags-from-ledger-description' +const getFlagsFromStringProp = require( + './get-flags-from-string-prop' ) const getCategoryFromDescription = require( './get-category-from-description' @@ -15,7 +15,7 @@ const convertArrayMapToObjectMap = require( module.exports = { addPropsToResIfExist, - getFlagsFromLedgerDescription, + getFlagsFromStringProp, getCategoryFromDescription, convertArrayMapToObjectMap } diff --git a/workers/loc.api/sync/helpers/get-back-iterable.js b/workers/loc.api/sync/helpers/get-back-iterable.js index 18eee8180..2e31f278b 100644 --- a/workers/loc.api/sync/helpers/get-back-iterable.js +++ b/workers/loc.api/sync/helpers/get-back-iterable.js @@ -5,15 +5,18 @@ module.exports = (array) => { [Symbol.iterator] (areEntriesReturned) { return { index: array.length, + res: { + done: false, + value: undefined + }, next () { this.index -= 1 + this.res.done = this.index < 0 + this.res.value = areEntriesReturned + ? [this.index, array[this.index]] + : array[this.index] - return { - done: this.index < 0, - value: areEntriesReturned - ? [this.index, array[this.index]] - : array[this.index] - } + return this.res } } }, diff --git a/workers/loc.api/sync/movements/index.js b/workers/loc.api/sync/movements/index.js index 1576d5b11..2040c91f2 100644 --- a/workers/loc.api/sync/movements/index.js +++ b/workers/loc.api/sync/movements/index.js @@ -3,6 +3,8 @@ const { orderBy } = require('lodash') const { merge } = require('lib-js-util-base') +const { pushLargeArr } = require('../../helpers/utils') + const { decorateInjectable } = require('../../di/utils') const depsTypes = (TYPES) => [ @@ -41,7 +43,8 @@ class Movements { isExcludePrivate = true, isWithdrawals = false, isDeposits = false, - isMovementsWithoutSATransferLedgers = false + isMovementsWithoutSATransferLedgers = false, + areExtraPaymentsIncluded = false } = params ?? {} const user = await this.authenticator @@ -86,32 +89,69 @@ class Movements { } ) + const ledgersOrder = this._getLedgersOrder(sort) + const extraMovementsPromise = this.getExtraMovements({ + auth: user, + start, + end, + sort: ledgersOrder, + isWithdrawals, + isDeposits, + isExcludePrivate, + areExtraPaymentsIncluded + }) + if (isMovementsWithoutSATransferLedgers) { - return movementsPromise + const [ + movements, + extraMovements + ] = await Promise.all([ + movementsPromise, + extraMovementsPromise + ]) + + const remapedLedgers = this._remapLedgersToMovements( + extraMovements + ) + pushLargeArr(movements, remapedLedgers) + const { + propNames, + orders + } = this._getLodashOrder(sort) + const orderedRes = orderBy( + movements, + propNames, + orders + ) + + return orderedRes } - const ledgersOrder = this._getLedgersOrder(sort) const ledgersPromise = this.getSubAccountsTransferLedgers({ auth: user, start, end, sort: ledgersOrder, isWithdrawals, - isDeposits + isDeposits, + isExcludePrivate }) const [ movements, + extraMovements, ledgers ] = await Promise.all([ movementsPromise, + extraMovementsPromise, ledgersPromise ]) + pushLargeArr(extraMovements, ledgers) const remapedLedgers = this._remapLedgersToMovements( - ledgers + extraMovements ) - movements.push(...remapedLedgers) + pushLargeArr(movements, remapedLedgers) const { propNames, @@ -175,6 +215,72 @@ class Movements { ) } + /* + * Considers the following additional movements from ledgers: + * - `InvoicePay Order` + * - `Airdrop on wallet` + */ + getExtraMovements (params = {}) { + const { + auth = {}, + start = 0, + end = Date.now(), + filter: _filter, + sort = [['mts', -1], ['id', -1]], + projection = this.ledgersModel, + exclude = ['user_id'], + isExcludePrivate = true, + isWithdrawals = false, + isDeposits = false, + areExtraPaymentsIncluded = false + } = params ?? {} + + const withdrawalsFilter = isWithdrawals + ? { $lt: { amount: 0 } } + : {} + const depositsFilter = isDeposits + ? { $gt: { amount: 0 } } + : {} + const filter = merge( + {}, + withdrawalsFilter, + depositsFilter, + _filter + ) + const extraPaymentsFilter = areExtraPaymentsIncluded + ? { + $or: { + $eq: { + _isInvoicePayOrder: 1, + _isAirdropOnWallet: 1, + _isMarginFundingPayment: 1, + _isAffiliateRebate: 1, + _isStakingPayments: 1 + } + } + } + : { $or: { $eq: { _isInvoicePayOrder: 1 } } } + + return this.dao.getElemsInCollBy( + this.ALLOWED_COLLS.LEDGERS, + { + subQuery: { + filter: extraPaymentsFilter + }, + filter: { + $lte: { mts: end }, + $gte: { mts: start }, + user_id: auth._id, + ...filter + }, + sort, + projection, + exclude, + isExcludePrivate + } + ) + } + _remapLedgersToMovements (ledgers) { return ledgers.map((ledger) => { const { @@ -182,7 +288,14 @@ class Movements { currency, amount, amountUsd, - subUserId + subUserId, + _id, + exactUsdValue, + + _isAirdropOnWallet, + _isMarginFundingPayment, + _isAffiliateRebate, + _isStakingPayments } = ledger return { @@ -199,7 +312,14 @@ class Movements { transactionId: '', note: '', subUserId, - _isFromLedgers: true + isLedgers: true, + _id, + exactUsdValue, + + _isAirdropOnWallet, + _isMarginFundingPayment, + _isAffiliateRebate, + _isStakingPayments } }) } diff --git a/workers/loc.api/sync/schema/models.js b/workers/loc.api/sync/schema/models.js deleted file mode 100644 index d1b75e573..000000000 --- a/workers/loc.api/sync/schema/models.js +++ /dev/null @@ -1,815 +0,0 @@ -'use strict' - -/* - * The version must be increased when DB schema is changed - * - * For each new DB version need to implement new migration - * in the `workers/loc.api/sync/dao/db-migrations/sqlite-migrations` folder, - * e.g. `migration.v1.js`, where `v1` is `SUPPORTED_DB_VERSION` - */ - -const SUPPORTED_DB_VERSION = 40 - -const TABLES_NAMES = require('./tables-names') -const { - CONSTR_FIELD_NAME, - TRIGGER_FIELD_NAME, - INDEX_FIELD_NAME, - UNIQUE_INDEX_FIELD_NAME, - ID_PRIMARY_KEY -} = require('./const') -const { - CREATE_UPDATE_API_KEYS_TRIGGERS, - CREATE_UPDATE_MTS_TRIGGERS, - DELETE_SUB_USERS_TRIGGER -} = require('./common.triggers') -const { - USER_ID_CONSTRAINT, - MASTER_USER_ID_CONSTRAINT, - OWNER_USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT -} = require('./common.constraints') -const { - getModelsMap: _getModelsMap, - getModelOf: _getModelOf -} = require('./helpers') - -const getModelsMap = (params = {}) => { - return _getModelsMap({ - ...params, - models: params?.models ?? _models - }) -} - -const getModelOf = (tableName) => { - return _getModelOf(tableName, _models) -} - -const _models = new Map([ - [ - TABLES_NAMES.USERS, - { - _id: ID_PRIMARY_KEY, - id: 'BIGINT', - email: 'VARCHAR(255)', - apiKey: 'VARCHAR(255)', - apiSecret: 'VARCHAR(255)', - authToken: 'VARCHAR(255)', - active: 'INT', - isDataFromDb: 'INT', - timezone: 'VARCHAR(255)', - username: 'VARCHAR(255)', - localUsername: 'VARCHAR(255)', - passwordHash: 'VARCHAR(255)', - isNotProtected: 'INT', - isSubAccount: 'INT', - isSubUser: 'INT', - shouldNotSyncOnStartupAfterUpdate: 'INT', - isSyncOnStartupRequired: 'INT', - authTokenTTLSec: 'INT', - isStagingBfxApi: 'INT', - createdAt: 'BIGINT', - updatedAt: 'BIGINT', - - [UNIQUE_INDEX_FIELD_NAME]: ['email', 'username'], - [TRIGGER_FIELD_NAME]: [ - ...CREATE_UPDATE_API_KEYS_TRIGGERS, - ...CREATE_UPDATE_MTS_TRIGGERS - ] - } - ], - [ - TABLES_NAMES.SUB_ACCOUNTS, - { - _id: ID_PRIMARY_KEY, - masterUserId: 'INT NOT NULL', - subUserId: 'INT NOT NULL', - createdAt: 'BIGINT', - updatedAt: 'BIGINT', - - [CONSTR_FIELD_NAME]: [ - MASTER_USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT - ], - [TRIGGER_FIELD_NAME]: [ - DELETE_SUB_USERS_TRIGGER, - ...CREATE_UPDATE_MTS_TRIGGERS - ] - } - ], - [ - TABLES_NAMES.LEDGERS, - { - _id: ID_PRIMARY_KEY, - id: 'BIGINT', - currency: 'VARCHAR(255)', - mts: 'BIGINT', - amount: 'DECIMAL(22,12)', - amountUsd: 'DECIMAL(22,12)', - balance: 'DECIMAL(22,12)', - _nativeBalance: 'DECIMAL(22,12)', - balanceUsd: 'DECIMAL(22,12)', - _nativeBalanceUsd: 'DECIMAL(22,12)', - description: 'TEXT', - wallet: 'VARCHAR(255)', - _category: 'INT', - _isMarginFundingPayment: 'INT', - _isAffiliateRebate: 'INT', - _isStakingPayments: 'INT', - _isSubAccountsTransfer: 'INT', - _isBalanceRecalced: 'INT', - subUserId: 'INT', - user_id: 'INT NOT NULL', - - [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], - [INDEX_FIELD_NAME]: [ - ['user_id', 'wallet', 'currency', 'mts'], - ['user_id', 'wallet', 'mts'], - ['user_id', 'currency', 'mts'], - ['user_id', '_isMarginFundingPayment', 'mts'], - ['user_id', '_isAffiliateRebate', 'mts'], - ['user_id', '_isStakingPayments', 'mts'], - ['user_id', '_isSubAccountsTransfer', 'mts'], - ['user_id', '_category', 'mts'], - ['user_id', 'mts'], - ['currency', 'mts'], - ['user_id', 'subUserId', 'mts', - 'WHERE subUserId IS NOT NULL'], - ['subUserId', 'mts', '_id', - 'WHERE subUserId IS NOT NULL'] - ], - [CONSTR_FIELD_NAME]: [ - USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT - ] - } - ], - [ - TABLES_NAMES.TRADES, - { - _id: ID_PRIMARY_KEY, - id: 'BIGINT', - symbol: 'VARCHAR(255)', - mtsCreate: 'BIGINT', - orderID: 'BIGINT', - execAmount: 'DECIMAL(22,12)', - execPrice: 'DECIMAL(22,12)', - orderType: 'VARCHAR(255)', - orderPrice: 'DECIMAL(22,12)', - maker: 'INT', - fee: 'DECIMAL(22,12)', - feeCurrency: 'VARCHAR(255)', - subUserId: 'INT', - user_id: 'INT NOT NULL', - - [UNIQUE_INDEX_FIELD_NAME]: ['id', 'symbol', 'user_id'], - [INDEX_FIELD_NAME]: [ - ['user_id', 'symbol', 'mtsCreate'], - ['user_id', 'orderID', 'mtsCreate'], - ['user_id', 'mtsCreate'], - ['user_id', 'subUserId', 'mtsCreate', - 'WHERE subUserId IS NOT NULL'], - ['subUserId', 'orderID', - 'WHERE subUserId IS NOT NULL'] - ], - [CONSTR_FIELD_NAME]: [ - USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT - ] - } - ], - [ - TABLES_NAMES.FUNDING_TRADES, - { - _id: ID_PRIMARY_KEY, - id: 'BIGINT', - symbol: 'VARCHAR(255)', - mtsCreate: 'BIGINT', - offerID: 'BIGINT', - amount: 'DECIMAL(22,12)', - rate: 'DECIMAL(22,12)', - period: 'BIGINT', - maker: 'INT', - subUserId: 'INT', - user_id: 'INT NOT NULL', - - [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], - [INDEX_FIELD_NAME]: [ - ['user_id', 'symbol', 'mtsCreate'], - ['user_id', 'mtsCreate'], - ['user_id', 'subUserId', 'mtsCreate', - 'WHERE subUserId IS NOT NULL'] - ], - [CONSTR_FIELD_NAME]: [ - USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT - ] - } - ], - [ - TABLES_NAMES.PUBLIC_TRADES, - { - _id: ID_PRIMARY_KEY, - id: 'BIGINT', - mts: 'BIGINT', - rate: 'DECIMAL(22,12)', - period: 'BIGINT', - amount: 'DECIMAL(22,12)', - price: 'DECIMAL(22,12)', - _symbol: 'VARCHAR(255)', - - [UNIQUE_INDEX_FIELD_NAME]: ['id', '_symbol'], - [INDEX_FIELD_NAME]: [ - ['_symbol', 'mts'], - ['mts'] - ] - } - ], - [ - TABLES_NAMES.ORDERS, - { - _id: ID_PRIMARY_KEY, - id: 'BIGINT', - gid: 'BIGINT', - cid: 'BIGINT', - symbol: 'VARCHAR(255)', - mtsCreate: 'BIGINT', - mtsUpdate: 'BIGINT', - amount: 'DECIMAL(22,12)', - amountOrig: 'DECIMAL(22,12)', - type: 'VARCHAR(255)', - typePrev: 'VARCHAR(255)', - flags: 'INT', - status: 'VARCHAR(255)', - price: 'DECIMAL(22,12)', - priceAvg: 'DECIMAL(22,12)', - priceTrailing: 'DECIMAL(22,12)', - priceAuxLimit: 'DECIMAL(22,12)', - notify: 'INT', - placedId: 'BIGINT', - _lastAmount: 'DECIMAL(22,12)', - amountExecuted: 'DECIMAL(22,12)', - routing: 'VARCHAR(255)', - meta: 'TEXT', // JSON - subUserId: 'INT', - user_id: 'INT NOT NULL', - - [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], - [INDEX_FIELD_NAME]: [ - ['user_id', 'symbol', 'mtsUpdate'], - ['user_id', 'type', 'mtsUpdate'], - ['user_id', 'mtsUpdate'], - ['user_id', 'subUserId', 'mtsUpdate', - 'WHERE subUserId IS NOT NULL'] - ], - [CONSTR_FIELD_NAME]: [ - USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT - ] - } - ], - [ - TABLES_NAMES.MOVEMENTS, - { - _id: ID_PRIMARY_KEY, - id: 'BIGINT', - currency: 'VARCHAR(255)', - currencyName: 'VARCHAR(255)', - mtsStarted: 'BIGINT', - mtsUpdated: 'BIGINT', - status: 'VARCHAR(255)', - amount: 'DECIMAL(22,12)', - amountUsd: 'DECIMAL(22,12)', - fees: 'DECIMAL(22,12)', - destinationAddress: 'VARCHAR(255)', - transactionId: 'VARCHAR(255)', - note: 'TEXT', - subUserId: 'INT', - user_id: 'INT NOT NULL', - - [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], - [INDEX_FIELD_NAME]: [ - ['user_id', 'status', 'mtsStarted'], - ['user_id', 'status', 'mtsUpdated'], - ['user_id', 'currency', 'mtsUpdated'], - ['user_id', 'mtsUpdated'], - ['user_id', 'subUserId', 'mtsUpdated', - 'WHERE subUserId IS NOT NULL'] - ], - [CONSTR_FIELD_NAME]: [ - USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT - ] - } - ], - [ - TABLES_NAMES.FUNDING_OFFER_HISTORY, - { - _id: ID_PRIMARY_KEY, - id: 'BIGINT', - symbol: 'VARCHAR(255)', - mtsCreate: 'BIGINT', - mtsUpdate: 'BIGINT', - amount: 'DECIMAL(22,12)', - amountOrig: 'DECIMAL(22,12)', - type: 'VARCHAR(255)', - flags: 'TEXT', - status: 'TEXT', - rate: 'VARCHAR(255)', - period: 'INT', - notify: 'INT', - hidden: 'INT', - renew: 'INT', - rateReal: 'INT', - amountExecuted: 'DECIMAL(22,12)', - subUserId: 'INT', - user_id: 'INT NOT NULL', - - [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], - [INDEX_FIELD_NAME]: [ - ['user_id', 'symbol', 'mtsUpdate'], - ['user_id', 'status', 'mtsUpdate'], - ['user_id', 'mtsUpdate'], - ['user_id', 'subUserId', 'mtsUpdate', - 'WHERE subUserId IS NOT NULL'] - ], - [CONSTR_FIELD_NAME]: [ - USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT - ] - } - ], - [ - TABLES_NAMES.FUNDING_LOAN_HISTORY, - { - _id: ID_PRIMARY_KEY, - id: 'BIGINT', - symbol: 'VARCHAR(255)', - side: 'INT', - mtsCreate: 'BIGINT', - mtsUpdate: 'BIGINT', - amount: 'DECIMAL(22,12)', - flags: 'TEXT', - status: 'TEXT', - rate: 'VARCHAR(255)', - period: 'INT', - mtsOpening: 'BIGINT', - mtsLastPayout: 'BIGINT', - notify: 'INT', - hidden: 'INT', - renew: 'INT', - rateReal: 'INT', - noClose: 'INT', - subUserId: 'INT', - user_id: 'INT NOT NULL', - - [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], - [INDEX_FIELD_NAME]: [ - ['user_id', 'symbol', 'mtsUpdate'], - ['user_id', 'status', 'mtsUpdate'], - ['user_id', 'mtsUpdate'], - ['user_id', 'subUserId', 'mtsUpdate', - 'WHERE subUserId IS NOT NULL'] - ], - [CONSTR_FIELD_NAME]: [ - USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT - ] - } - ], - [ - TABLES_NAMES.FUNDING_CREDIT_HISTORY, - { - _id: ID_PRIMARY_KEY, - id: 'BIGINT', - symbol: 'VARCHAR(255)', - side: 'INT', - mtsCreate: 'BIGINT', - mtsUpdate: 'BIGINT', - amount: 'DECIMAL(22,12)', - flags: 'TEXT', - status: 'TEXT', - rate: 'VARCHAR(255)', - period: 'INT', - mtsOpening: 'BIGINT', - mtsLastPayout: 'BIGINT', - notify: 'INT', - hidden: 'INT', - renew: 'INT', - rateReal: 'INT', - noClose: 'INT', - positionPair: 'VARCHAR(255)', - subUserId: 'INT', - user_id: 'INT NOT NULL', - - [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], - [INDEX_FIELD_NAME]: [ - ['user_id', 'symbol', 'mtsUpdate'], - ['user_id', 'status', 'mtsUpdate'], - ['user_id', 'mtsUpdate'], - ['user_id', 'subUserId', 'mtsUpdate', - 'WHERE subUserId IS NOT NULL'] - ], - [CONSTR_FIELD_NAME]: [ - USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT - ] - } - ], - [ - TABLES_NAMES.POSITIONS_HISTORY, - { - _id: ID_PRIMARY_KEY, - id: 'BIGINT', - symbol: 'VARCHAR(255)', - status: 'VARCHAR(255)', - amount: 'DECIMAL(22,12)', - basePrice: 'DECIMAL(22,12)', - closePrice: 'DECIMAL(22,12)', - marginFunding: 'DECIMAL(22,12)', - marginFundingType: 'INT', - pl: 'DECIMAL(22,12)', - plPerc: 'DECIMAL(22,12)', - liquidationPrice: 'DECIMAL(22,12)', - leverage: 'DECIMAL(22,12)', - placeholder: 'TEXT', - mtsCreate: 'BIGINT', - mtsUpdate: 'BIGINT', - subUserId: 'INT', - user_id: 'INT NOT NULL', - - [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], - [INDEX_FIELD_NAME]: [ - ['user_id', 'symbol', 'mtsUpdate'], - ['user_id', 'mtsUpdate', 'mtsCreate'], - ['user_id', 'mtsUpdate'], - ['user_id', 'subUserId', 'mtsUpdate', - 'WHERE subUserId IS NOT NULL'], - ['subUserId', 'id', - 'WHERE subUserId IS NOT NULL'] - ], - [CONSTR_FIELD_NAME]: [ - USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT - ] - } - ], - [ - TABLES_NAMES.POSITIONS_SNAPSHOT, - { - _id: ID_PRIMARY_KEY, - id: 'BIGINT', - symbol: 'VARCHAR(255)', - status: 'VARCHAR(255)', - amount: 'DECIMAL(22,12)', - basePrice: 'DECIMAL(22,12)', - closePrice: 'DECIMAL(22,12)', - marginFunding: 'DECIMAL(22,12)', - marginFundingType: 'INT', - pl: 'DECIMAL(22,12)', - plPerc: 'DECIMAL(22,12)', - liquidationPrice: 'DECIMAL(22,12)', - leverage: 'DECIMAL(22,12)', - placeholder: 'TEXT', - mtsCreate: 'BIGINT', - mtsUpdate: 'BIGINT', - subUserId: 'INT', - user_id: 'INT NOT NULL', - - // The API returns a lot of data with the same values, - // that cause unique indexes are not included - [INDEX_FIELD_NAME]: [ - ['user_id', 'mtsUpdate'], - ['user_id', 'symbol', 'mtsUpdate'], - ['user_id', 'subUserId', 'mtsUpdate', - 'WHERE subUserId IS NOT NULL'] - ], - [CONSTR_FIELD_NAME]: [ - USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT - ] - } - ], - [ - TABLES_NAMES.LOGINS, - { - _id: ID_PRIMARY_KEY, - id: 'BIGINT', - time: 'BIGINT', - ip: 'VARCHAR(255)', - extraData: 'TEXT', - subUserId: 'INT', - user_id: 'INT NOT NULL', - - [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], - [INDEX_FIELD_NAME]: [ - ['user_id', 'time'], - ['user_id', 'subUserId', 'time', - 'WHERE subUserId IS NOT NULL'] - ], - [CONSTR_FIELD_NAME]: [ - USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT - ] - } - ], - [ - TABLES_NAMES.CHANGE_LOGS, - { - _id: ID_PRIMARY_KEY, - mtsCreate: 'BIGINT', - log: 'VARCHAR(255)', - ip: 'VARCHAR(255)', - userAgent: 'TEXT', - subUserId: 'INT', - user_id: 'INT NOT NULL', - - [UNIQUE_INDEX_FIELD_NAME]: ['mtsCreate', 'log', 'user_id'], - [INDEX_FIELD_NAME]: [ - ['user_id', 'mtsCreate'], - ['user_id', 'subUserId', 'mtsCreate', - 'WHERE subUserId IS NOT NULL'] - ], - [CONSTR_FIELD_NAME]: [ - USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT - ] - } - ], - [ - TABLES_NAMES.PAY_INVOICE_LIST, - { - _id: ID_PRIMARY_KEY, - id: 'VARCHAR(255)', - t: 'BIGINT', - duration: 'INT', - amount: 'DECIMAL(22,12)', - currency: 'VARCHAR(255)', - orderId: 'VARCHAR(255)', - payCurrencies: 'TEXT', // JSON - webhook: 'VARCHAR(255)', - redirectUrl: 'VARCHAR(255)', - status: 'VARCHAR(255)', - customerInfo: 'TEXT', // JSON - invoices: 'TEXT', // JSON - payment: 'TEXT', // JSON - merchantName: 'VARCHAR(255)', - subUserId: 'INT', - user_id: 'INT NOT NULL', - - [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], - [INDEX_FIELD_NAME]: [ - ['user_id', 'currency', 't'], - ['user_id', 'id', 't'], - ['user_id', 't'], - ['user_id', 'subUserId', 't', - 'WHERE subUserId IS NOT NULL'], - ['subUserId', 'id', - 'WHERE subUserId IS NOT NULL'] - ], - [CONSTR_FIELD_NAME]: [ - USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT - ] - } - ], - [ - TABLES_NAMES.TICKERS_HISTORY, - { - _id: ID_PRIMARY_KEY, - symbol: 'VARCHAR(255)', - bid: 'DECIMAL(22,12)', - bidPeriod: 'INT', - ask: 'DECIMAL(22,12)', - mtsUpdate: 'BIGINT', - - [UNIQUE_INDEX_FIELD_NAME]: ['mtsUpdate', 'symbol'], - [INDEX_FIELD_NAME]: [ - ['symbol', 'mtsUpdate'] - ] - } - ], - [ - TABLES_NAMES.STATUS_MESSAGES, - { - _id: ID_PRIMARY_KEY, - key: 'VARCHAR(255)', - timestamp: 'BIGINT', - price: 'DECIMAL(22,12)', - priceSpot: 'DECIMAL(22,12)', - fundBal: 'DECIMAL(22,12)', - fundingAccrued: 'DECIMAL(22,12)', - fundingStep: 'DECIMAL(22,12)', - clampMin: 'DECIMAL(22,12)', - clampMax: 'DECIMAL(22,12)', - _type: 'VARCHAR(255)', - - [UNIQUE_INDEX_FIELD_NAME]: ['key', '_type'], - [INDEX_FIELD_NAME]: [ - ['key', 'timestamp'] - ] - } - ], - [ - TABLES_NAMES.PUBLIC_COLLS_CONF, - { - _id: ID_PRIMARY_KEY, - confName: 'VARCHAR(255)', - symbol: 'VARCHAR(255)', - start: 'BIGINT', - timeframe: 'VARCHAR(255)', - createdAt: 'BIGINT', - updatedAt: 'BIGINT', - user_id: 'INT NOT NULL', - - [UNIQUE_INDEX_FIELD_NAME]: [ - 'symbol', 'user_id', 'confName', 'timeframe' - ], - [CONSTR_FIELD_NAME]: USER_ID_CONSTRAINT, - [TRIGGER_FIELD_NAME]: CREATE_UPDATE_MTS_TRIGGERS - } - ], - [ - TABLES_NAMES.SYMBOLS, - { - _id: ID_PRIMARY_KEY, - pairs: 'VARCHAR(255)', - - [UNIQUE_INDEX_FIELD_NAME]: ['pairs'] - } - ], - [ - TABLES_NAMES.MAP_SYMBOLS, - { - _id: ID_PRIMARY_KEY, - key: 'VARCHAR(255)', - value: 'VARCHAR(255)', - - [UNIQUE_INDEX_FIELD_NAME]: ['key'] - } - ], - [ - TABLES_NAMES.INACTIVE_CURRENCIES, - { - _id: ID_PRIMARY_KEY, - pairs: 'VARCHAR(255)', - - [UNIQUE_INDEX_FIELD_NAME]: ['pairs'] - } - ], - [ - TABLES_NAMES.INACTIVE_SYMBOLS, - { - _id: ID_PRIMARY_KEY, - pairs: 'VARCHAR(255)', - - [UNIQUE_INDEX_FIELD_NAME]: ['pairs'] - } - ], - [ - TABLES_NAMES.FUTURES, - { - _id: ID_PRIMARY_KEY, - pairs: 'VARCHAR(255)', - - [UNIQUE_INDEX_FIELD_NAME]: ['pairs'] - } - ], - [ - TABLES_NAMES.CURRENCIES, - { - _id: ID_PRIMARY_KEY, - id: 'VARCHAR(255)', - name: 'VARCHAR(255)', - pool: 'VARCHAR(255)', - explorer: 'TEXT', - symbol: 'VARCHAR(255)', - walletFx: 'TEXT', - - [UNIQUE_INDEX_FIELD_NAME]: ['id'] - } - ], - [ - TABLES_NAMES.MARGIN_CURRENCY_LIST, - { - _id: ID_PRIMARY_KEY, - symbol: 'VARCHAR(255)', - - [UNIQUE_INDEX_FIELD_NAME]: ['symbol'] - } - ], - [ - TABLES_NAMES.CANDLES, - { - _id: ID_PRIMARY_KEY, - mts: 'BIGINT', - open: 'DECIMAL(22,12)', - close: 'DECIMAL(22,12)', - high: 'DECIMAL(22,12)', - low: 'DECIMAL(22,12)', - volume: 'DECIMAL(22,12)', - _symbol: 'VARCHAR(255)', - _timeframe: 'VARCHAR(255)', - - [UNIQUE_INDEX_FIELD_NAME]: ['_symbol', '_timeframe', 'mts'], - [INDEX_FIELD_NAME]: [ - ['_timeframe', '_symbol', 'mts'], - ['_timeframe', 'mts'], - ['_symbol', 'mts'], - ['close', 'mts'] - ] - } - ], - [ - TABLES_NAMES.SCHEDULER, - { - _id: ID_PRIMARY_KEY, - isEnable: 'INT', - createdAt: 'BIGINT', - updatedAt: 'BIGINT', - - [TRIGGER_FIELD_NAME]: CREATE_UPDATE_MTS_TRIGGERS - } - ], - [ - TABLES_NAMES.SYNC_MODE, - { - _id: ID_PRIMARY_KEY, - isEnable: 'INT', - createdAt: 'BIGINT', - updatedAt: 'BIGINT', - - [TRIGGER_FIELD_NAME]: CREATE_UPDATE_MTS_TRIGGERS - } - ], - [ - TABLES_NAMES.PROGRESS, - { - _id: ID_PRIMARY_KEY, - error: 'VARCHAR(255)', - value: 'DECIMAL(22,12)', - state: 'VARCHAR(255)', - createdAt: 'BIGINT', - updatedAt: 'BIGINT', - - [TRIGGER_FIELD_NAME]: CREATE_UPDATE_MTS_TRIGGERS - } - ], - [ - TABLES_NAMES.SYNC_QUEUE, - { - _id: ID_PRIMARY_KEY, - collName: 'VARCHAR(255) NOT NULL', - state: 'VARCHAR(255)', - createdAt: 'BIGINT', - updatedAt: 'BIGINT', - ownerUserId: 'INT', - isOwnerScheduler: 'INT', - - [CONSTR_FIELD_NAME]: OWNER_USER_ID_CONSTRAINT, - [TRIGGER_FIELD_NAME]: CREATE_UPDATE_MTS_TRIGGERS - } - ], - [ - TABLES_NAMES.SYNC_USER_STEPS, - { - _id: ID_PRIMARY_KEY, - collName: 'VARCHAR(255) NOT NULL', - syncedAt: 'BIGINT', - baseStart: 'BIGINT', - baseEnd: 'BIGINT', - isBaseStepReady: 'INT', - currStart: 'BIGINT', - currEnd: 'BIGINT', - isCurrStepReady: 'INT', - createdAt: 'BIGINT', - updatedAt: 'BIGINT', - subUserId: 'INT', - user_id: 'INT', - syncQueueId: 'INT', - - [UNIQUE_INDEX_FIELD_NAME]: [ - // It needs to cover public collections - ['collName', - 'WHERE user_id IS NULL'], - // It needs to cover private collections - ['user_id', 'collName', - 'WHERE user_id IS NOT NULL AND subUserId IS NULL'], - // It needs to cover private collections of sub-account - ['user_id', 'subUserId', 'collName', - 'WHERE user_id IS NOT NULL AND subUserId IS NOT NULL'] - ], - [CONSTR_FIELD_NAME]: [ - USER_ID_CONSTRAINT, - SUB_USER_ID_CONSTRAINT - ], - [TRIGGER_FIELD_NAME]: CREATE_UPDATE_MTS_TRIGGERS - } - ] -]) - -module.exports = { - SUPPORTED_DB_VERSION, - getModelsMap, - getModelOf -} diff --git a/workers/loc.api/sync/schema/models/candles.js b/workers/loc.api/sync/schema/models/candles.js new file mode 100644 index 000000000..b4d352cc6 --- /dev/null +++ b/workers/loc.api/sync/schema/models/candles.js @@ -0,0 +1,27 @@ +'use strict' + +const { + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') + +module.exports = { + _id: ID_PRIMARY_KEY, + mts: 'BIGINT', + open: 'DECIMAL(22,12)', + close: 'DECIMAL(22,12)', + high: 'DECIMAL(22,12)', + low: 'DECIMAL(22,12)', + volume: 'DECIMAL(22,12)', + _symbol: 'VARCHAR(255)', + _timeframe: 'VARCHAR(255)', + + [UNIQUE_INDEX_FIELD_NAME]: ['_symbol', '_timeframe', 'mts'], + [INDEX_FIELD_NAME]: [ + ['_timeframe', '_symbol', 'mts'], + ['_timeframe', 'mts'], + ['_symbol', 'mts'], + ['close', 'mts'] + ] +} diff --git a/workers/loc.api/sync/schema/models/change-logs.js b/workers/loc.api/sync/schema/models/change-logs.js new file mode 100644 index 000000000..bee9fe677 --- /dev/null +++ b/workers/loc.api/sync/schema/models/change-logs.js @@ -0,0 +1,33 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + mtsCreate: 'BIGINT', + log: 'VARCHAR(255)', + ip: 'VARCHAR(255)', + userAgent: 'TEXT', + subUserId: 'INT', + user_id: 'INT NOT NULL', + + [UNIQUE_INDEX_FIELD_NAME]: ['mtsCreate', 'log', 'user_id'], + [INDEX_FIELD_NAME]: [ + ['user_id', 'mtsCreate'], + ['user_id', 'subUserId', 'mtsCreate', + 'WHERE subUserId IS NOT NULL'] + ], + [CONSTR_FIELD_NAME]: [ + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT + ] +} diff --git a/workers/loc.api/sync/schema/models/currencies.js b/workers/loc.api/sync/schema/models/currencies.js new file mode 100644 index 000000000..2086de4b9 --- /dev/null +++ b/workers/loc.api/sync/schema/models/currencies.js @@ -0,0 +1,18 @@ +'use strict' + +const { + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') + +module.exports = { + _id: ID_PRIMARY_KEY, + id: 'VARCHAR(255)', + name: 'VARCHAR(255)', + pool: 'VARCHAR(255)', + explorer: 'TEXT', + symbol: 'VARCHAR(255)', + walletFx: 'TEXT', + + [UNIQUE_INDEX_FIELD_NAME]: ['id'] +} diff --git a/workers/loc.api/sync/schema/models/funding-credit-history.js b/workers/loc.api/sync/schema/models/funding-credit-history.js new file mode 100644 index 000000000..61a6daea5 --- /dev/null +++ b/workers/loc.api/sync/schema/models/funding-credit-history.js @@ -0,0 +1,49 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + symbol: 'VARCHAR(255)', + side: 'INT', + mtsCreate: 'BIGINT', + mtsUpdate: 'BIGINT', + amount: 'DECIMAL(22,12)', + flags: 'TEXT', + status: 'TEXT', + rate: 'VARCHAR(255)', + period: 'INT', + mtsOpening: 'BIGINT', + mtsLastPayout: 'BIGINT', + notify: 'INT', + hidden: 'INT', + renew: 'INT', + rateReal: 'INT', + noClose: 'INT', + positionPair: 'VARCHAR(255)', + subUserId: 'INT', + user_id: 'INT NOT NULL', + + [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], + [INDEX_FIELD_NAME]: [ + ['user_id', 'symbol', 'mtsUpdate'], + ['user_id', 'status', 'mtsUpdate'], + ['user_id', 'mtsUpdate'], + ['user_id', 'subUserId', 'mtsUpdate', + 'WHERE subUserId IS NOT NULL'] + ], + [CONSTR_FIELD_NAME]: [ + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT + ] +} diff --git a/workers/loc.api/sync/schema/models/funding-loan-history.js b/workers/loc.api/sync/schema/models/funding-loan-history.js new file mode 100644 index 000000000..9f5c146f9 --- /dev/null +++ b/workers/loc.api/sync/schema/models/funding-loan-history.js @@ -0,0 +1,48 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + symbol: 'VARCHAR(255)', + side: 'INT', + mtsCreate: 'BIGINT', + mtsUpdate: 'BIGINT', + amount: 'DECIMAL(22,12)', + flags: 'TEXT', + status: 'TEXT', + rate: 'VARCHAR(255)', + period: 'INT', + mtsOpening: 'BIGINT', + mtsLastPayout: 'BIGINT', + notify: 'INT', + hidden: 'INT', + renew: 'INT', + rateReal: 'INT', + noClose: 'INT', + subUserId: 'INT', + user_id: 'INT NOT NULL', + + [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], + [INDEX_FIELD_NAME]: [ + ['user_id', 'symbol', 'mtsUpdate'], + ['user_id', 'status', 'mtsUpdate'], + ['user_id', 'mtsUpdate'], + ['user_id', 'subUserId', 'mtsUpdate', + 'WHERE subUserId IS NOT NULL'] + ], + [CONSTR_FIELD_NAME]: [ + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT + ] +} diff --git a/workers/loc.api/sync/schema/models/funding-offer-history.js b/workers/loc.api/sync/schema/models/funding-offer-history.js new file mode 100644 index 000000000..8be9da194 --- /dev/null +++ b/workers/loc.api/sync/schema/models/funding-offer-history.js @@ -0,0 +1,47 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + symbol: 'VARCHAR(255)', + mtsCreate: 'BIGINT', + mtsUpdate: 'BIGINT', + amount: 'DECIMAL(22,12)', + amountOrig: 'DECIMAL(22,12)', + type: 'VARCHAR(255)', + flags: 'TEXT', + status: 'TEXT', + rate: 'VARCHAR(255)', + period: 'INT', + notify: 'INT', + hidden: 'INT', + renew: 'INT', + rateReal: 'INT', + amountExecuted: 'DECIMAL(22,12)', + subUserId: 'INT', + user_id: 'INT NOT NULL', + + [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], + [INDEX_FIELD_NAME]: [ + ['user_id', 'symbol', 'mtsUpdate'], + ['user_id', 'status', 'mtsUpdate'], + ['user_id', 'mtsUpdate'], + ['user_id', 'subUserId', 'mtsUpdate', + 'WHERE subUserId IS NOT NULL'] + ], + [CONSTR_FIELD_NAME]: [ + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT + ] +} diff --git a/workers/loc.api/sync/schema/models/funding-trades.js b/workers/loc.api/sync/schema/models/funding-trades.js new file mode 100644 index 000000000..32585bbb1 --- /dev/null +++ b/workers/loc.api/sync/schema/models/funding-trades.js @@ -0,0 +1,38 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + symbol: 'VARCHAR(255)', + mtsCreate: 'BIGINT', + offerID: 'BIGINT', + amount: 'DECIMAL(22,12)', + rate: 'DECIMAL(22,12)', + period: 'BIGINT', + maker: 'INT', + subUserId: 'INT', + user_id: 'INT NOT NULL', + + [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], + [INDEX_FIELD_NAME]: [ + ['user_id', 'symbol', 'mtsCreate'], + ['user_id', 'mtsCreate'], + ['user_id', 'subUserId', 'mtsCreate', + 'WHERE subUserId IS NOT NULL'] + ], + [CONSTR_FIELD_NAME]: [ + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT + ] +} diff --git a/workers/loc.api/sync/schema/models/futures.js b/workers/loc.api/sync/schema/models/futures.js new file mode 100644 index 000000000..95479f039 --- /dev/null +++ b/workers/loc.api/sync/schema/models/futures.js @@ -0,0 +1,13 @@ +'use strict' + +const { + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') + +module.exports = { + _id: ID_PRIMARY_KEY, + pairs: 'VARCHAR(255)', + + [UNIQUE_INDEX_FIELD_NAME]: ['pairs'] +} diff --git a/workers/loc.api/sync/schema/models/inactive-currencies.js b/workers/loc.api/sync/schema/models/inactive-currencies.js new file mode 100644 index 000000000..95479f039 --- /dev/null +++ b/workers/loc.api/sync/schema/models/inactive-currencies.js @@ -0,0 +1,13 @@ +'use strict' + +const { + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') + +module.exports = { + _id: ID_PRIMARY_KEY, + pairs: 'VARCHAR(255)', + + [UNIQUE_INDEX_FIELD_NAME]: ['pairs'] +} diff --git a/workers/loc.api/sync/schema/models/inactive-symbols.js b/workers/loc.api/sync/schema/models/inactive-symbols.js new file mode 100644 index 000000000..95479f039 --- /dev/null +++ b/workers/loc.api/sync/schema/models/inactive-symbols.js @@ -0,0 +1,13 @@ +'use strict' + +const { + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') + +module.exports = { + _id: ID_PRIMARY_KEY, + pairs: 'VARCHAR(255)', + + [UNIQUE_INDEX_FIELD_NAME]: ['pairs'] +} diff --git a/workers/loc.api/sync/schema/models/index.js b/workers/loc.api/sync/schema/models/index.js new file mode 100644 index 000000000..31e2994ac --- /dev/null +++ b/workers/loc.api/sync/schema/models/index.js @@ -0,0 +1,102 @@ +'use strict' + +/* + * The version must be increased when DB schema is changed + * + * For each new DB version need to implement new migration + * in the `workers/loc.api/sync/dao/db-migrations/sqlite-migrations` folder, + * e.g. `migration.v1.js`, where `v1` is `SUPPORTED_DB_VERSION` + */ +const SUPPORTED_DB_VERSION = 41 + +const TABLES_NAMES = require('../tables-names') + +const users = require('./users') +const subAccounts = require('./sub-accounts') +const ledgers = require('./ledgers') +const trades = require('./trades') +const fundingTrades = require('./funding-trades') +const publicTrades = require('./public-trades') +const orders = require('./orders') +const movements = require('./movements') +const fundingOfferHistory = require('./funding-offer-history') +const fundingLoanHistory = require('./funding-loan-history') +const fundingCreditHistory = require('./funding-credit-history') +const positionsHistory = require('./positions-history') +const positionsSnapshot = require('./positions-snapshot') +const logins = require('./logins') +const changeLogs = require('./change-logs') +const payInvoiceList = require('./pay-invoice-list') +const tickersHistory = require('./tickers-history') +const statusMessages = require('./status-messages') +const publicCollsConf = require('./public-colls-conf') +const symbols = require('./symbols') +const mapSymbols = require('./map-symbols') +const inactiveCurrencies = require('./inactive-currencies') +const inactiveSymbols = require('./inactive-symbols') +const futures = require('./futures') +const currencies = require('./currencies') +const marginCurrencyList = require('./margin-currency-list') +const candles = require('./candles') +const scheduler = require('./scheduler') +const syncMode = require('./sync-mode') +const progress = require('./progress') +const syncQueue = require('./sync-queue') +const syncUserSteps = require('./sync-user-steps') + +const _models = new Map([ + [TABLES_NAMES.USERS, users], + [TABLES_NAMES.SUB_ACCOUNTS, subAccounts], + [TABLES_NAMES.LEDGERS, ledgers], + [TABLES_NAMES.TRADES, trades], + [TABLES_NAMES.FUNDING_TRADES, fundingTrades], + [TABLES_NAMES.PUBLIC_TRADES, publicTrades], + [TABLES_NAMES.ORDERS, orders], + [TABLES_NAMES.MOVEMENTS, movements], + [TABLES_NAMES.FUNDING_OFFER_HISTORY, fundingOfferHistory], + [TABLES_NAMES.FUNDING_LOAN_HISTORY, fundingLoanHistory], + [TABLES_NAMES.FUNDING_CREDIT_HISTORY, fundingCreditHistory], + [TABLES_NAMES.POSITIONS_HISTORY, positionsHistory], + [TABLES_NAMES.POSITIONS_SNAPSHOT, positionsSnapshot], + [TABLES_NAMES.LOGINS, logins], + [TABLES_NAMES.CHANGE_LOGS, changeLogs], + [TABLES_NAMES.PAY_INVOICE_LIST, payInvoiceList], + [TABLES_NAMES.TICKERS_HISTORY, tickersHistory], + [TABLES_NAMES.STATUS_MESSAGES, statusMessages], + [TABLES_NAMES.PUBLIC_COLLS_CONF, publicCollsConf], + [TABLES_NAMES.SYMBOLS, symbols], + [TABLES_NAMES.MAP_SYMBOLS, mapSymbols], + [TABLES_NAMES.INACTIVE_CURRENCIES, inactiveCurrencies], + [TABLES_NAMES.INACTIVE_SYMBOLS, inactiveSymbols], + [TABLES_NAMES.FUTURES, futures], + [TABLES_NAMES.CURRENCIES, currencies], + [TABLES_NAMES.MARGIN_CURRENCY_LIST, marginCurrencyList], + [TABLES_NAMES.CANDLES, candles], + [TABLES_NAMES.SCHEDULER, scheduler], + [TABLES_NAMES.SYNC_MODE, syncMode], + [TABLES_NAMES.PROGRESS, progress], + [TABLES_NAMES.SYNC_QUEUE, syncQueue], + [TABLES_NAMES.SYNC_USER_STEPS, syncUserSteps] +]) + +const { + getModelsMap: _getModelsMap, + getModelOf: _getModelOf +} = require('../helpers') + +const getModelsMap = (params = {}) => { + return _getModelsMap({ + ...params, + models: params?.models ?? _models + }) +} + +const getModelOf = (tableName) => { + return _getModelOf(tableName, _models) +} + +module.exports = { + SUPPORTED_DB_VERSION, + getModelsMap, + getModelOf +} diff --git a/workers/loc.api/sync/schema/models/ledgers.js b/workers/loc.api/sync/schema/models/ledgers.js new file mode 100644 index 000000000..59905db7a --- /dev/null +++ b/workers/loc.api/sync/schema/models/ledgers.js @@ -0,0 +1,62 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + currency: 'VARCHAR(255)', + mts: 'BIGINT', + amount: 'DECIMAL(22,12)', + amountUsd: 'DECIMAL(22,12)', + exactUsdValue: 'DECIMAL(22,12)', + balance: 'DECIMAL(22,12)', + _nativeBalance: 'DECIMAL(22,12)', + balanceUsd: 'DECIMAL(22,12)', + _nativeBalanceUsd: 'DECIMAL(22,12)', + description: 'TEXT', + wallet: 'VARCHAR(255)', + _category: 'INT', + _isMarginFundingPayment: 'INT', + _isAffiliateRebate: 'INT', + _isStakingPayments: 'INT', + _isSubAccountsTransfer: 'INT', + _isBalanceRecalced: 'INT', + _isInvoicePayOrder: 'INT', + _isAirdropOnWallet: 'INT', + subUserId: 'INT', + user_id: 'INT NOT NULL', + + [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], + [INDEX_FIELD_NAME]: [ + ['user_id', 'wallet', 'currency', 'mts'], + ['user_id', 'wallet', 'mts'], + ['user_id', 'currency', 'mts'], + ['user_id', '_isMarginFundingPayment', 'mts'], + ['user_id', '_isAffiliateRebate', 'mts'], + ['user_id', '_isStakingPayments', 'mts'], + ['user_id', '_isSubAccountsTransfer', 'mts'], + ['user_id', '_category', 'mts'], + ['user_id', 'mts'], + ['currency', 'mts'], + ['_isInvoicePayOrder'], + ['_isAirdropOnWallet'], + ['user_id', 'subUserId', 'mts', + 'WHERE subUserId IS NOT NULL'], + ['subUserId', 'mts', '_id', + 'WHERE subUserId IS NOT NULL'] + ], + [CONSTR_FIELD_NAME]: [ + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT + ] +} diff --git a/workers/loc.api/sync/schema/models/logins.js b/workers/loc.api/sync/schema/models/logins.js new file mode 100644 index 000000000..a3c54ee9b --- /dev/null +++ b/workers/loc.api/sync/schema/models/logins.js @@ -0,0 +1,33 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + time: 'BIGINT', + ip: 'VARCHAR(255)', + extraData: 'TEXT', + subUserId: 'INT', + user_id: 'INT NOT NULL', + + [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], + [INDEX_FIELD_NAME]: [ + ['user_id', 'time'], + ['user_id', 'subUserId', 'time', + 'WHERE subUserId IS NOT NULL'] + ], + [CONSTR_FIELD_NAME]: [ + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT + ] +} diff --git a/workers/loc.api/sync/schema/models/map-symbols.js b/workers/loc.api/sync/schema/models/map-symbols.js new file mode 100644 index 000000000..5f9b6c29d --- /dev/null +++ b/workers/loc.api/sync/schema/models/map-symbols.js @@ -0,0 +1,14 @@ +'use strict' + +const { + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') + +module.exports = { + _id: ID_PRIMARY_KEY, + key: 'VARCHAR(255)', + value: 'VARCHAR(255)', + + [UNIQUE_INDEX_FIELD_NAME]: ['key'] +} diff --git a/workers/loc.api/sync/schema/models/margin-currency-list.js b/workers/loc.api/sync/schema/models/margin-currency-list.js new file mode 100644 index 000000000..649087e19 --- /dev/null +++ b/workers/loc.api/sync/schema/models/margin-currency-list.js @@ -0,0 +1,13 @@ +'use strict' + +const { + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') + +module.exports = { + _id: ID_PRIMARY_KEY, + symbol: 'VARCHAR(255)', + + [UNIQUE_INDEX_FIELD_NAME]: ['symbol'] +} diff --git a/workers/loc.api/sync/schema/models/movements.js b/workers/loc.api/sync/schema/models/movements.js new file mode 100644 index 000000000..4fcde946e --- /dev/null +++ b/workers/loc.api/sync/schema/models/movements.js @@ -0,0 +1,45 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + currency: 'VARCHAR(255)', + currencyName: 'VARCHAR(255)', + mtsStarted: 'BIGINT', + mtsUpdated: 'BIGINT', + status: 'VARCHAR(255)', + amount: 'DECIMAL(22,12)', + amountUsd: 'DECIMAL(22,12)', + exactUsdValue: 'DECIMAL(22,12)', + fees: 'DECIMAL(22,12)', + destinationAddress: 'VARCHAR(255)', + transactionId: 'VARCHAR(255)', + note: 'TEXT', + subUserId: 'INT', + user_id: 'INT NOT NULL', + + [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], + [INDEX_FIELD_NAME]: [ + ['user_id', 'status', 'mtsStarted'], + ['user_id', 'status', 'mtsUpdated'], + ['user_id', 'currency', 'mtsUpdated'], + ['user_id', 'mtsUpdated'], + ['user_id', 'subUserId', 'mtsUpdated', + 'WHERE subUserId IS NOT NULL'] + ], + [CONSTR_FIELD_NAME]: [ + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT + ] +} diff --git a/workers/loc.api/sync/schema/models/orders.js b/workers/loc.api/sync/schema/models/orders.js new file mode 100644 index 000000000..987850b5d --- /dev/null +++ b/workers/loc.api/sync/schema/models/orders.js @@ -0,0 +1,53 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + gid: 'BIGINT', + cid: 'BIGINT', + symbol: 'VARCHAR(255)', + mtsCreate: 'BIGINT', + mtsUpdate: 'BIGINT', + amount: 'DECIMAL(22,12)', + amountOrig: 'DECIMAL(22,12)', + type: 'VARCHAR(255)', + typePrev: 'VARCHAR(255)', + flags: 'INT', + status: 'VARCHAR(255)', + price: 'DECIMAL(22,12)', + priceAvg: 'DECIMAL(22,12)', + priceTrailing: 'DECIMAL(22,12)', + priceAuxLimit: 'DECIMAL(22,12)', + notify: 'INT', + placedId: 'BIGINT', + _lastAmount: 'DECIMAL(22,12)', + amountExecuted: 'DECIMAL(22,12)', + routing: 'VARCHAR(255)', + meta: 'TEXT', // JSON + subUserId: 'INT', + user_id: 'INT NOT NULL', + + [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], + [INDEX_FIELD_NAME]: [ + ['user_id', 'symbol', 'mtsUpdate'], + ['user_id', 'type', 'mtsUpdate'], + ['user_id', 'mtsUpdate'], + ['user_id', 'subUserId', 'mtsUpdate', + 'WHERE subUserId IS NOT NULL'] + ], + [CONSTR_FIELD_NAME]: [ + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT + ] +} diff --git a/workers/loc.api/sync/schema/models/pay-invoice-list.js b/workers/loc.api/sync/schema/models/pay-invoice-list.js new file mode 100644 index 000000000..4f8954adf --- /dev/null +++ b/workers/loc.api/sync/schema/models/pay-invoice-list.js @@ -0,0 +1,47 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + id: 'VARCHAR(255)', + t: 'BIGINT', + duration: 'INT', + amount: 'DECIMAL(22,12)', + currency: 'VARCHAR(255)', + orderId: 'VARCHAR(255)', + payCurrencies: 'TEXT', // JSON + webhook: 'VARCHAR(255)', + redirectUrl: 'VARCHAR(255)', + status: 'VARCHAR(255)', + customerInfo: 'TEXT', // JSON + invoices: 'TEXT', // JSON + payment: 'TEXT', // JSON + merchantName: 'VARCHAR(255)', + subUserId: 'INT', + user_id: 'INT NOT NULL', + + [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], + [INDEX_FIELD_NAME]: [ + ['user_id', 'currency', 't'], + ['user_id', 'id', 't'], + ['user_id', 't'], + ['user_id', 'subUserId', 't', + 'WHERE subUserId IS NOT NULL'], + ['subUserId', 'id', + 'WHERE subUserId IS NOT NULL'] + ], + [CONSTR_FIELD_NAME]: [ + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT + ] +} diff --git a/workers/loc.api/sync/schema/models/positions-history.js b/workers/loc.api/sync/schema/models/positions-history.js new file mode 100644 index 000000000..9edbcf1cb --- /dev/null +++ b/workers/loc.api/sync/schema/models/positions-history.js @@ -0,0 +1,48 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + symbol: 'VARCHAR(255)', + status: 'VARCHAR(255)', + amount: 'DECIMAL(22,12)', + basePrice: 'DECIMAL(22,12)', + closePrice: 'DECIMAL(22,12)', + marginFunding: 'DECIMAL(22,12)', + marginFundingType: 'INT', + pl: 'DECIMAL(22,12)', + plPerc: 'DECIMAL(22,12)', + liquidationPrice: 'DECIMAL(22,12)', + leverage: 'DECIMAL(22,12)', + placeholder: 'TEXT', + mtsCreate: 'BIGINT', + mtsUpdate: 'BIGINT', + subUserId: 'INT', + user_id: 'INT NOT NULL', + + [UNIQUE_INDEX_FIELD_NAME]: ['id', 'user_id'], + [INDEX_FIELD_NAME]: [ + ['user_id', 'symbol', 'mtsUpdate'], + ['user_id', 'mtsUpdate', 'mtsCreate'], + ['user_id', 'mtsUpdate'], + ['user_id', 'subUserId', 'mtsUpdate', + 'WHERE subUserId IS NOT NULL'], + ['subUserId', 'id', + 'WHERE subUserId IS NOT NULL'] + ], + [CONSTR_FIELD_NAME]: [ + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT + ] +} diff --git a/workers/loc.api/sync/schema/models/positions-snapshot.js b/workers/loc.api/sync/schema/models/positions-snapshot.js new file mode 100644 index 000000000..3db0a4e5f --- /dev/null +++ b/workers/loc.api/sync/schema/models/positions-snapshot.js @@ -0,0 +1,45 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + symbol: 'VARCHAR(255)', + status: 'VARCHAR(255)', + amount: 'DECIMAL(22,12)', + basePrice: 'DECIMAL(22,12)', + closePrice: 'DECIMAL(22,12)', + marginFunding: 'DECIMAL(22,12)', + marginFundingType: 'INT', + pl: 'DECIMAL(22,12)', + plPerc: 'DECIMAL(22,12)', + liquidationPrice: 'DECIMAL(22,12)', + leverage: 'DECIMAL(22,12)', + placeholder: 'TEXT', + mtsCreate: 'BIGINT', + mtsUpdate: 'BIGINT', + subUserId: 'INT', + user_id: 'INT NOT NULL', + + // The API returns a lot of data with the same values, + // that cause unique indexes are not included + [INDEX_FIELD_NAME]: [ + ['user_id', 'mtsUpdate'], + ['user_id', 'symbol', 'mtsUpdate'], + ['user_id', 'subUserId', 'mtsUpdate', + 'WHERE subUserId IS NOT NULL'] + ], + [CONSTR_FIELD_NAME]: [ + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT + ] +} diff --git a/workers/loc.api/sync/schema/models/progress.js b/workers/loc.api/sync/schema/models/progress.js new file mode 100644 index 000000000..aeaf27796 --- /dev/null +++ b/workers/loc.api/sync/schema/models/progress.js @@ -0,0 +1,20 @@ +'use strict' + +const { + TRIGGER_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + CREATE_UPDATE_MTS_TRIGGERS +} = require('../common.triggers') + +module.exports = { + _id: ID_PRIMARY_KEY, + error: 'VARCHAR(255)', + value: 'DECIMAL(22,12)', + state: 'VARCHAR(255)', + createdAt: 'BIGINT', + updatedAt: 'BIGINT', + + [TRIGGER_FIELD_NAME]: CREATE_UPDATE_MTS_TRIGGERS +} diff --git a/workers/loc.api/sync/schema/models/public-colls-conf.js b/workers/loc.api/sync/schema/models/public-colls-conf.js new file mode 100644 index 000000000..b694d663e --- /dev/null +++ b/workers/loc.api/sync/schema/models/public-colls-conf.js @@ -0,0 +1,31 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + TRIGGER_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + CREATE_UPDATE_MTS_TRIGGERS +} = require('../common.triggers') +const { + USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + confName: 'VARCHAR(255)', + symbol: 'VARCHAR(255)', + start: 'BIGINT', + timeframe: 'VARCHAR(255)', + createdAt: 'BIGINT', + updatedAt: 'BIGINT', + user_id: 'INT NOT NULL', + + [UNIQUE_INDEX_FIELD_NAME]: [ + 'symbol', 'user_id', 'confName', 'timeframe' + ], + [CONSTR_FIELD_NAME]: USER_ID_CONSTRAINT, + [TRIGGER_FIELD_NAME]: CREATE_UPDATE_MTS_TRIGGERS +} diff --git a/workers/loc.api/sync/schema/models/public-trades.js b/workers/loc.api/sync/schema/models/public-trades.js new file mode 100644 index 000000000..7038b5371 --- /dev/null +++ b/workers/loc.api/sync/schema/models/public-trades.js @@ -0,0 +1,24 @@ +'use strict' + +const { + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') + +module.exports = { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + mts: 'BIGINT', + rate: 'DECIMAL(22,12)', + period: 'BIGINT', + amount: 'DECIMAL(22,12)', + price: 'DECIMAL(22,12)', + _symbol: 'VARCHAR(255)', + + [UNIQUE_INDEX_FIELD_NAME]: ['id', '_symbol'], + [INDEX_FIELD_NAME]: [ + ['_symbol', 'mts'], + ['mts'] + ] +} diff --git a/workers/loc.api/sync/schema/models/scheduler.js b/workers/loc.api/sync/schema/models/scheduler.js new file mode 100644 index 000000000..834802f2d --- /dev/null +++ b/workers/loc.api/sync/schema/models/scheduler.js @@ -0,0 +1,18 @@ +'use strict' + +const { + TRIGGER_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + CREATE_UPDATE_MTS_TRIGGERS +} = require('../common.triggers') + +module.exports = { + _id: ID_PRIMARY_KEY, + isEnable: 'INT', + createdAt: 'BIGINT', + updatedAt: 'BIGINT', + + [TRIGGER_FIELD_NAME]: CREATE_UPDATE_MTS_TRIGGERS +} diff --git a/workers/loc.api/sync/schema/models/status-messages.js b/workers/loc.api/sync/schema/models/status-messages.js new file mode 100644 index 000000000..40bf26369 --- /dev/null +++ b/workers/loc.api/sync/schema/models/status-messages.js @@ -0,0 +1,26 @@ +'use strict' + +const { + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') + +module.exports = { + _id: ID_PRIMARY_KEY, + key: 'VARCHAR(255)', + timestamp: 'BIGINT', + price: 'DECIMAL(22,12)', + priceSpot: 'DECIMAL(22,12)', + fundBal: 'DECIMAL(22,12)', + fundingAccrued: 'DECIMAL(22,12)', + fundingStep: 'DECIMAL(22,12)', + clampMin: 'DECIMAL(22,12)', + clampMax: 'DECIMAL(22,12)', + _type: 'VARCHAR(255)', + + [UNIQUE_INDEX_FIELD_NAME]: ['key', '_type'], + [INDEX_FIELD_NAME]: [ + ['key', 'timestamp'] + ] +} diff --git a/workers/loc.api/sync/schema/models/sub-accounts.js b/workers/loc.api/sync/schema/models/sub-accounts.js new file mode 100644 index 000000000..5692c7d3f --- /dev/null +++ b/workers/loc.api/sync/schema/models/sub-accounts.js @@ -0,0 +1,32 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + TRIGGER_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + CREATE_UPDATE_MTS_TRIGGERS, + DELETE_SUB_USERS_TRIGGER +} = require('../common.triggers') +const { + MASTER_USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + masterUserId: 'INT NOT NULL', + subUserId: 'INT NOT NULL', + createdAt: 'BIGINT', + updatedAt: 'BIGINT', + + [CONSTR_FIELD_NAME]: [ + MASTER_USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT + ], + [TRIGGER_FIELD_NAME]: [ + DELETE_SUB_USERS_TRIGGER, + ...CREATE_UPDATE_MTS_TRIGGERS + ] +} diff --git a/workers/loc.api/sync/schema/models/symbols.js b/workers/loc.api/sync/schema/models/symbols.js new file mode 100644 index 000000000..95479f039 --- /dev/null +++ b/workers/loc.api/sync/schema/models/symbols.js @@ -0,0 +1,13 @@ +'use strict' + +const { + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') + +module.exports = { + _id: ID_PRIMARY_KEY, + pairs: 'VARCHAR(255)', + + [UNIQUE_INDEX_FIELD_NAME]: ['pairs'] +} diff --git a/workers/loc.api/sync/schema/models/sync-mode.js b/workers/loc.api/sync/schema/models/sync-mode.js new file mode 100644 index 000000000..834802f2d --- /dev/null +++ b/workers/loc.api/sync/schema/models/sync-mode.js @@ -0,0 +1,18 @@ +'use strict' + +const { + TRIGGER_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + CREATE_UPDATE_MTS_TRIGGERS +} = require('../common.triggers') + +module.exports = { + _id: ID_PRIMARY_KEY, + isEnable: 'INT', + createdAt: 'BIGINT', + updatedAt: 'BIGINT', + + [TRIGGER_FIELD_NAME]: CREATE_UPDATE_MTS_TRIGGERS +} diff --git a/workers/loc.api/sync/schema/models/sync-queue.js b/workers/loc.api/sync/schema/models/sync-queue.js new file mode 100644 index 000000000..b40cbcdc8 --- /dev/null +++ b/workers/loc.api/sync/schema/models/sync-queue.js @@ -0,0 +1,26 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + TRIGGER_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + CREATE_UPDATE_MTS_TRIGGERS +} = require('../common.triggers') +const { + OWNER_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + collName: 'VARCHAR(255) NOT NULL', + state: 'VARCHAR(255)', + createdAt: 'BIGINT', + updatedAt: 'BIGINT', + ownerUserId: 'INT', + isOwnerScheduler: 'INT', + + [CONSTR_FIELD_NAME]: OWNER_USER_ID_CONSTRAINT, + [TRIGGER_FIELD_NAME]: CREATE_UPDATE_MTS_TRIGGERS +} diff --git a/workers/loc.api/sync/schema/models/sync-user-steps.js b/workers/loc.api/sync/schema/models/sync-user-steps.js new file mode 100644 index 000000000..a69dba5d6 --- /dev/null +++ b/workers/loc.api/sync/schema/models/sync-user-steps.js @@ -0,0 +1,49 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + TRIGGER_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + CREATE_UPDATE_MTS_TRIGGERS +} = require('../common.triggers') +const { + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + collName: 'VARCHAR(255) NOT NULL', + syncedAt: 'BIGINT', + baseStart: 'BIGINT', + baseEnd: 'BIGINT', + isBaseStepReady: 'INT', + currStart: 'BIGINT', + currEnd: 'BIGINT', + isCurrStepReady: 'INT', + createdAt: 'BIGINT', + updatedAt: 'BIGINT', + subUserId: 'INT', + user_id: 'INT', + syncQueueId: 'INT', + + [UNIQUE_INDEX_FIELD_NAME]: [ + // It needs to cover public collections + ['collName', + 'WHERE user_id IS NULL'], + // It needs to cover private collections + ['user_id', 'collName', + 'WHERE user_id IS NOT NULL AND subUserId IS NULL'], + // It needs to cover private collections of sub-account + ['user_id', 'subUserId', 'collName', + 'WHERE user_id IS NOT NULL AND subUserId IS NOT NULL'] + ], + [CONSTR_FIELD_NAME]: [ + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT + ], + [TRIGGER_FIELD_NAME]: CREATE_UPDATE_MTS_TRIGGERS +} diff --git a/workers/loc.api/sync/schema/models/tickers-history.js b/workers/loc.api/sync/schema/models/tickers-history.js new file mode 100644 index 000000000..5a5b07833 --- /dev/null +++ b/workers/loc.api/sync/schema/models/tickers-history.js @@ -0,0 +1,21 @@ +'use strict' + +const { + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') + +module.exports = { + _id: ID_PRIMARY_KEY, + symbol: 'VARCHAR(255)', + bid: 'DECIMAL(22,12)', + bidPeriod: 'INT', + ask: 'DECIMAL(22,12)', + mtsUpdate: 'BIGINT', + + [UNIQUE_INDEX_FIELD_NAME]: ['mtsUpdate', 'symbol'], + [INDEX_FIELD_NAME]: [ + ['symbol', 'mtsUpdate'] + ] +} diff --git a/workers/loc.api/sync/schema/models/trades.js b/workers/loc.api/sync/schema/models/trades.js new file mode 100644 index 000000000..556a3996f --- /dev/null +++ b/workers/loc.api/sync/schema/models/trades.js @@ -0,0 +1,47 @@ +'use strict' + +const { + CONSTR_FIELD_NAME, + INDEX_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT +} = require('../common.constraints') + +module.exports = { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + symbol: 'VARCHAR(255)', + mtsCreate: 'BIGINT', + orderID: 'BIGINT', + execAmount: 'DECIMAL(22,12)', + execPrice: 'DECIMAL(22,12)', + exactUsdValue: 'DECIMAL(22,12)', + orderType: 'VARCHAR(255)', + orderPrice: 'DECIMAL(22,12)', + maker: 'INT', + fee: 'DECIMAL(22,12)', + feeCurrency: 'VARCHAR(255)', + subUserId: 'INT', + _isExchange: 'INT', + user_id: 'INT NOT NULL', + + [UNIQUE_INDEX_FIELD_NAME]: ['id', 'symbol', 'user_id'], + [INDEX_FIELD_NAME]: [ + ['user_id', 'symbol', 'mtsCreate'], + ['user_id', 'orderID', 'mtsCreate'], + ['user_id', '_isExchange', 'mtsCreate'], + ['user_id', 'mtsCreate'], + ['user_id', 'subUserId', 'mtsCreate', + 'WHERE subUserId IS NOT NULL'], + ['subUserId', 'orderID', + 'WHERE subUserId IS NOT NULL'] + ], + [CONSTR_FIELD_NAME]: [ + USER_ID_CONSTRAINT, + SUB_USER_ID_CONSTRAINT + ] +} diff --git a/workers/loc.api/sync/schema/models/users.js b/workers/loc.api/sync/schema/models/users.js new file mode 100644 index 000000000..89cf3cf23 --- /dev/null +++ b/workers/loc.api/sync/schema/models/users.js @@ -0,0 +1,41 @@ +'use strict' + +const { + TRIGGER_FIELD_NAME, + UNIQUE_INDEX_FIELD_NAME, + ID_PRIMARY_KEY +} = require('../const') +const { + CREATE_UPDATE_API_KEYS_TRIGGERS, + CREATE_UPDATE_MTS_TRIGGERS +} = require('../common.triggers') + +module.exports = { + _id: ID_PRIMARY_KEY, + id: 'BIGINT', + email: 'VARCHAR(255)', + apiKey: 'VARCHAR(255)', + apiSecret: 'VARCHAR(255)', + authToken: 'VARCHAR(255)', + active: 'INT', + isDataFromDb: 'INT', + timezone: 'VARCHAR(255)', + username: 'VARCHAR(255)', + localUsername: 'VARCHAR(255)', + passwordHash: 'VARCHAR(255)', + isNotProtected: 'INT', + isSubAccount: 'INT', + isSubUser: 'INT', + shouldNotSyncOnStartupAfterUpdate: 'INT', + isSyncOnStartupRequired: 'INT', + authTokenTTLSec: 'INT', + isStagingBfxApi: 'INT', + createdAt: 'BIGINT', + updatedAt: 'BIGINT', + + [UNIQUE_INDEX_FIELD_NAME]: ['email', 'username'], + [TRIGGER_FIELD_NAME]: [ + ...CREATE_UPDATE_API_KEYS_TRIGGERS, + ...CREATE_UPDATE_MTS_TRIGGERS + ] +} diff --git a/workers/loc.api/sync/schema/sync-schema.js b/workers/loc.api/sync/schema/sync-schema.js deleted file mode 100644 index 705714bca..000000000 --- a/workers/loc.api/sync/schema/sync-schema.js +++ /dev/null @@ -1,405 +0,0 @@ -'use strict' - -const TABLES_NAMES = require('./tables-names') -const ALLOWED_COLLS = require('./allowed.colls') -const SYNC_API_METHODS = require('./sync.api.methods') -const COLLS_TYPES = require('./colls.types') -const { cloneSchema } = require('./helpers') -const { getModelOf } = require('./models') - -const getMethodCollMap = (methodCollMap = _methodCollMap) => { - return cloneSchema(methodCollMap) -} - -const _methodCollMap = new Map([ - [ - SYNC_API_METHODS.LEDGERS, - { - name: ALLOWED_COLLS.LEDGERS, - maxLimit: 2500, - dateFieldName: 'mts', - symbolFieldName: 'currency', - sort: [['mts', -1], ['id', -1]], - hasNewData: false, - start: [], - isSyncRequiredAtLeastOnce: true, - type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.LEDGERS) - } - ], - [ - SYNC_API_METHODS.TRADES, - { - name: ALLOWED_COLLS.TRADES, - maxLimit: 2500, - dateFieldName: 'mtsCreate', - symbolFieldName: 'symbol', - sort: [['mtsCreate', -1]], - hasNewData: false, - start: [], - isSyncRequiredAtLeastOnce: true, - type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.TRADES) - } - ], - [ - SYNC_API_METHODS.FUNDING_TRADES, - { - name: ALLOWED_COLLS.FUNDING_TRADES, - maxLimit: 1000, - dateFieldName: 'mtsCreate', - symbolFieldName: 'symbol', - sort: [['mtsCreate', -1]], - hasNewData: false, - start: [], - isSyncRequiredAtLeastOnce: false, - type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.FUNDING_TRADES) - } - ], - [ - SYNC_API_METHODS.PUBLIC_TRADES, - { - name: ALLOWED_COLLS.PUBLIC_TRADES, - maxLimit: 5000, - dateFieldName: 'mts', - symbolFieldName: '_symbol', - sort: [['mts', -1]], - hasNewData: false, - start: [], - confName: 'publicTradesConf', - isSyncRequiredAtLeastOnce: false, - additionalApiCallArgs: { isNotMoreThanInnerMax: true }, - type: COLLS_TYPES.PUBLIC_INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.PUBLIC_TRADES) - } - ], - [ - SYNC_API_METHODS.STATUS_MESSAGES, - { - name: ALLOWED_COLLS.STATUS_MESSAGES, - maxLimit: 5000, - dateFieldName: 'timestamp', - symbolFieldName: 'key', - sort: [['timestamp', -1]], - hasNewData: false, - confName: 'statusMessagesConf', - isSyncRequiredAtLeastOnce: false, - type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.STATUS_MESSAGES) - } - ], - [ - SYNC_API_METHODS.ORDERS, - { - name: ALLOWED_COLLS.ORDERS, - maxLimit: 2500, - dateFieldName: 'mtsUpdate', - symbolFieldName: 'symbol', - sort: [['mtsUpdate', -1]], - hasNewData: false, - start: [], - isSyncRequiredAtLeastOnce: false, - type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.ORDERS) - } - ], - [ - SYNC_API_METHODS.MOVEMENTS, - { - name: ALLOWED_COLLS.MOVEMENTS, - maxLimit: 250, - dateFieldName: 'mtsUpdated', - symbolFieldName: 'currency', - sort: [['mtsUpdated', -1]], - hasNewData: false, - start: [], - isSyncRequiredAtLeastOnce: true, - type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.MOVEMENTS) - } - ], - [ - SYNC_API_METHODS.FUNDING_OFFER_HISTORY, - { - name: ALLOWED_COLLS.FUNDING_OFFER_HISTORY, - maxLimit: 10000, - dateFieldName: 'mtsUpdate', - symbolFieldName: 'symbol', - sort: [['mtsUpdate', -1]], - hasNewData: false, - start: [], - isSyncRequiredAtLeastOnce: false, - type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.FUNDING_OFFER_HISTORY) - } - ], - [ - SYNC_API_METHODS.FUNDING_LOAN_HISTORY, - { - name: ALLOWED_COLLS.FUNDING_LOAN_HISTORY, - maxLimit: 10000, - dateFieldName: 'mtsUpdate', - symbolFieldName: 'symbol', - sort: [['mtsUpdate', -1]], - hasNewData: false, - start: [], - isSyncRequiredAtLeastOnce: false, - type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.FUNDING_LOAN_HISTORY) - } - ], - [ - SYNC_API_METHODS.FUNDING_CREDIT_HISTORY, - { - name: ALLOWED_COLLS.FUNDING_CREDIT_HISTORY, - maxLimit: 10000, - dateFieldName: 'mtsUpdate', - symbolFieldName: 'symbol', - sort: [['mtsUpdate', -1]], - hasNewData: false, - start: [], - isSyncRequiredAtLeastOnce: false, - type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.FUNDING_CREDIT_HISTORY) - } - ], - [ - SYNC_API_METHODS.POSITIONS_HISTORY, - { - name: ALLOWED_COLLS.POSITIONS_HISTORY, - maxLimit: 10000, - dateFieldName: 'mtsUpdate', - symbolFieldName: 'symbol', - sort: [['mtsUpdate', -1]], - hasNewData: false, - start: [], - isSyncRequiredAtLeastOnce: false, - shouldNotApiMiddlewareBeLaunched: true, - type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.POSITIONS_HISTORY) - } - ], - [ - SYNC_API_METHODS.POSITIONS_SNAPSHOT, - { - name: ALLOWED_COLLS.POSITIONS_SNAPSHOT, - maxLimit: 10000, - dateFieldName: 'mtsUpdate', - symbolFieldName: 'symbol', - sort: [['mtsUpdate', -1]], - hasNewData: false, - start: [], - isSyncRequiredAtLeastOnce: false, - type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.POSITIONS_SNAPSHOT) - } - ], - [ - SYNC_API_METHODS.LOGINS, - { - name: ALLOWED_COLLS.LOGINS, - maxLimit: 10000, - dateFieldName: 'time', - symbolFieldName: null, - sort: [['time', -1]], - hasNewData: false, - start: [], - isSyncRequiredAtLeastOnce: true, - type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.LOGINS) - } - ], - [ - SYNC_API_METHODS.CHANGE_LOGS, - { - name: ALLOWED_COLLS.CHANGE_LOGS, - maxLimit: 10000, - dateFieldName: 'mtsCreate', - symbolFieldName: null, - sort: [['mtsCreate', -1]], - hasNewData: false, - start: [], - isSyncRequiredAtLeastOnce: false, - type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.CHANGE_LOGS) - } - ], - [ - SYNC_API_METHODS.PAY_INVOICE_LIST, - { - name: ALLOWED_COLLS.PAY_INVOICE_LIST, - maxLimit: 100, - dateFieldName: 't', - symbolFieldName: 'currency', - sort: [['t', -1]], - hasNewData: false, - start: [], - isSyncRequiredAtLeastOnce: false, - type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.PAY_INVOICE_LIST) - } - ], - [ - SYNC_API_METHODS.TICKERS_HISTORY, - { - name: ALLOWED_COLLS.TICKERS_HISTORY, - maxLimit: 10000, - dateFieldName: 'mtsUpdate', - symbolFieldName: 'symbol', - sort: [['mtsUpdate', -1]], - hasNewData: false, - start: [], - confName: 'tickersHistoryConf', - isSyncRequiredAtLeastOnce: false, - additionalApiCallArgs: { isNotMoreThanInnerMax: true }, - type: COLLS_TYPES.PUBLIC_INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.TICKERS_HISTORY) - } - ], - [ - SYNC_API_METHODS.WALLETS, - { - name: ALLOWED_COLLS.LEDGERS, - dateFieldName: 'mts', - symbolFieldName: 'currency', - sort: [['mts', -1]], - groupFns: ['max(mts)', 'max(id)'], - groupResBy: ['wallet', 'currency'], - isSyncRequiredAtLeastOnce: true, - type: COLLS_TYPES.HIDDEN_INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.LEDGERS), - dataStructureConverter: (accum, { - wallet: type, - currency, - balance, - mts: mtsUpdate - } = {}) => { - accum.push({ - type, - currency, - balance, - unsettledInterest: null, - balanceAvailable: null, - placeHolder: null, - mtsUpdate - }) - - return accum - } - } - ], - [ - SYNC_API_METHODS.SYMBOLS, - { - name: ALLOWED_COLLS.SYMBOLS, - maxLimit: 10000, - projection: 'pairs', - sort: [['pairs', 1]], - hasNewData: false, - isSyncRequiredAtLeastOnce: true, - type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY, - model: getModelOf(TABLES_NAMES.SYMBOLS) - } - ], - [ - SYNC_API_METHODS.MAP_SYMBOLS, - { - name: ALLOWED_COLLS.MAP_SYMBOLS, - maxLimit: 10000, - projection: ['key', 'value'], - sort: [['key', 1]], - hasNewData: false, - isSyncRequiredAtLeastOnce: true, - type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.MAP_SYMBOLS) - } - ], - [ - SYNC_API_METHODS.INACTIVE_CURRENCIES, - { - name: ALLOWED_COLLS.INACTIVE_CURRENCIES, - maxLimit: 10000, - projection: 'pairs', - sort: [['pairs', 1]], - hasNewData: false, - isSyncRequiredAtLeastOnce: true, - type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY, - model: getModelOf(TABLES_NAMES.INACTIVE_CURRENCIES) - } - ], - [ - SYNC_API_METHODS.INACTIVE_SYMBOLS, - { - name: ALLOWED_COLLS.INACTIVE_SYMBOLS, - maxLimit: 10000, - projection: 'pairs', - sort: [['pairs', 1]], - hasNewData: false, - isSyncRequiredAtLeastOnce: true, - type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY, - model: getModelOf(TABLES_NAMES.INACTIVE_SYMBOLS) - } - ], - [ - SYNC_API_METHODS.FUTURES, - { - name: ALLOWED_COLLS.FUTURES, - maxLimit: 10000, - projection: 'pairs', - sort: [['pairs', 1]], - hasNewData: false, - isSyncRequiredAtLeastOnce: true, - type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY, - model: getModelOf(TABLES_NAMES.FUTURES) - } - ], - [ - SYNC_API_METHODS.CURRENCIES, - { - name: ALLOWED_COLLS.CURRENCIES, - maxLimit: 10000, - projection: null, - sort: [['name', 1]], - hasNewData: false, - isSyncRequiredAtLeastOnce: true, - type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.CURRENCIES) - } - ], - [ - SYNC_API_METHODS.MARGIN_CURRENCY_LIST, - { - name: ALLOWED_COLLS.MARGIN_CURRENCY_LIST, - maxLimit: 10000, - projection: 'symbol', - sort: [['symbol', 1]], - hasNewData: false, - isSyncRequiredAtLeastOnce: true, - type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY, - model: getModelOf(TABLES_NAMES.MARGIN_CURRENCY_LIST) - } - ], - [ - SYNC_API_METHODS.CANDLES, - { - name: ALLOWED_COLLS.CANDLES, - maxLimit: 10000, - dateFieldName: 'mts', - symbolFieldName: '_symbol', - timeframeFieldName: '_timeframe', - sort: [['mts', -1]], - hasNewData: false, - start: [], - confName: 'candlesConf', - isSyncRequiredAtLeastOnce: true, - additionalApiCallArgs: { isNotMoreThanInnerMax: true }, - type: COLLS_TYPES.PUBLIC_INSERTABLE_ARRAY_OBJECTS, - model: getModelOf(TABLES_NAMES.CANDLES) - } - ] -]) - -module.exports = { - getMethodCollMap -} diff --git a/workers/loc.api/sync/schema/sync-schema/candles.js b/workers/loc.api/sync/schema/sync-schema/candles.js new file mode 100644 index 000000000..1d8fde700 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/candles.js @@ -0,0 +1,23 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.CANDLES, + maxLimit: 10000, + dateFieldName: 'mts', + symbolFieldName: '_symbol', + timeframeFieldName: '_timeframe', + sort: [['mts', -1]], + hasNewData: false, + start: [], + confName: 'candlesConf', + isSyncRequiredAtLeastOnce: true, + additionalApiCallArgs: { isNotMoreThanInnerMax: true }, + type: COLLS_TYPES.PUBLIC_INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.CANDLES) +} diff --git a/workers/loc.api/sync/schema/sync-schema/change-logs.js b/workers/loc.api/sync/schema/sync-schema/change-logs.js new file mode 100644 index 000000000..d2db78f01 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/change-logs.js @@ -0,0 +1,20 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.CHANGE_LOGS, + maxLimit: 10000, + dateFieldName: 'mtsCreate', + symbolFieldName: null, + sort: [['mtsCreate', -1]], + hasNewData: false, + start: [], + isSyncRequiredAtLeastOnce: false, + type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.CHANGE_LOGS) +} diff --git a/workers/loc.api/sync/schema/sync-schema/currencies.js b/workers/loc.api/sync/schema/sync-schema/currencies.js new file mode 100644 index 000000000..3efc28d06 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/currencies.js @@ -0,0 +1,18 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.CURRENCIES, + maxLimit: 10000, + projection: null, + sort: [['name', 1]], + hasNewData: false, + isSyncRequiredAtLeastOnce: true, + type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.CURRENCIES) +} diff --git a/workers/loc.api/sync/schema/sync-schema/funding-credit-history.js b/workers/loc.api/sync/schema/sync-schema/funding-credit-history.js new file mode 100644 index 000000000..a39e4add9 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/funding-credit-history.js @@ -0,0 +1,20 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.FUNDING_CREDIT_HISTORY, + maxLimit: 10000, + dateFieldName: 'mtsUpdate', + symbolFieldName: 'symbol', + sort: [['mtsUpdate', -1]], + hasNewData: false, + start: [], + isSyncRequiredAtLeastOnce: false, + type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.FUNDING_CREDIT_HISTORY) +} diff --git a/workers/loc.api/sync/schema/sync-schema/funding-loan-history.js b/workers/loc.api/sync/schema/sync-schema/funding-loan-history.js new file mode 100644 index 000000000..2a20834cb --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/funding-loan-history.js @@ -0,0 +1,20 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.FUNDING_LOAN_HISTORY, + maxLimit: 10000, + dateFieldName: 'mtsUpdate', + symbolFieldName: 'symbol', + sort: [['mtsUpdate', -1]], + hasNewData: false, + start: [], + isSyncRequiredAtLeastOnce: false, + type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.FUNDING_LOAN_HISTORY) +} diff --git a/workers/loc.api/sync/schema/sync-schema/funding-offer-history.js b/workers/loc.api/sync/schema/sync-schema/funding-offer-history.js new file mode 100644 index 000000000..46187c773 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/funding-offer-history.js @@ -0,0 +1,20 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.FUNDING_OFFER_HISTORY, + maxLimit: 10000, + dateFieldName: 'mtsUpdate', + symbolFieldName: 'symbol', + sort: [['mtsUpdate', -1]], + hasNewData: false, + start: [], + isSyncRequiredAtLeastOnce: false, + type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.FUNDING_OFFER_HISTORY) +} diff --git a/workers/loc.api/sync/schema/sync-schema/funding-trades.js b/workers/loc.api/sync/schema/sync-schema/funding-trades.js new file mode 100644 index 000000000..913d0b751 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/funding-trades.js @@ -0,0 +1,20 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.FUNDING_TRADES, + maxLimit: 1000, + dateFieldName: 'mtsCreate', + symbolFieldName: 'symbol', + sort: [['mtsCreate', -1]], + hasNewData: false, + start: [], + isSyncRequiredAtLeastOnce: false, + type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.FUNDING_TRADES) +} diff --git a/workers/loc.api/sync/schema/sync-schema/futures.js b/workers/loc.api/sync/schema/sync-schema/futures.js new file mode 100644 index 000000000..e2d1c6766 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/futures.js @@ -0,0 +1,18 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.FUTURES, + maxLimit: 10000, + projection: 'pairs', + sort: [['pairs', 1]], + hasNewData: false, + isSyncRequiredAtLeastOnce: true, + type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY, + model: getModelOf(TABLES_NAMES.FUTURES) +} diff --git a/workers/loc.api/sync/schema/sync-schema/inactive-currencies.js b/workers/loc.api/sync/schema/sync-schema/inactive-currencies.js new file mode 100644 index 000000000..37889f257 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/inactive-currencies.js @@ -0,0 +1,18 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.INACTIVE_CURRENCIES, + maxLimit: 10000, + projection: 'pairs', + sort: [['pairs', 1]], + hasNewData: false, + isSyncRequiredAtLeastOnce: true, + type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY, + model: getModelOf(TABLES_NAMES.INACTIVE_CURRENCIES) +} diff --git a/workers/loc.api/sync/schema/sync-schema/inactive-symbols.js b/workers/loc.api/sync/schema/sync-schema/inactive-symbols.js new file mode 100644 index 000000000..a86be9b82 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/inactive-symbols.js @@ -0,0 +1,18 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.INACTIVE_SYMBOLS, + maxLimit: 10000, + projection: 'pairs', + sort: [['pairs', 1]], + hasNewData: false, + isSyncRequiredAtLeastOnce: true, + type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY, + model: getModelOf(TABLES_NAMES.INACTIVE_SYMBOLS) +} diff --git a/workers/loc.api/sync/schema/sync-schema/index.js b/workers/loc.api/sync/schema/sync-schema/index.js new file mode 100644 index 000000000..773ae4e1e --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/index.js @@ -0,0 +1,67 @@ +'use strict' + +const SYNC_API_METHODS = require('../sync.api.methods') + +const ledgers = require('./ledgers') +const trades = require('./trades') +const fundingTrades = require('./funding-trades') +const publicTrades = require('./public-trades') +const statusMessages = require('./status-messages') +const orders = require('./orders') +const movements = require('./movements') +const fundingOfferHistory = require('./funding-offer-history') +const fundingLoanHistory = require('./funding-loan-history') +const fundingCreditHistory = require('./funding-credit-history') +const positionsHistory = require('./positions-history') +const positionsSnapshot = require('./positions-snapshot') +const logins = require('./logins') +const changeLogs = require('./change-logs') +const payInvoiceList = require('./pay-invoice-list') +const tickersHistory = require('./tickers-history') +const wallets = require('./wallets') +const symbols = require('./symbols') +const mapSymbols = require('./map-symbols') +const inactiveCurrencies = require('./inactive-currencies') +const inactiveSymbols = require('./inactive-symbols') +const futures = require('./futures') +const currencies = require('./currencies') +const marginCurrencyList = require('./margin-currency-list') +const candles = require('./candles') + +const _methodCollMap = new Map([ + [SYNC_API_METHODS.LEDGERS, ledgers], + [SYNC_API_METHODS.TRADES, trades], + [SYNC_API_METHODS.FUNDING_TRADES, fundingTrades], + [SYNC_API_METHODS.PUBLIC_TRADES, publicTrades], + [SYNC_API_METHODS.STATUS_MESSAGES, statusMessages], + [SYNC_API_METHODS.ORDERS, orders], + [SYNC_API_METHODS.MOVEMENTS, movements], + [SYNC_API_METHODS.FUNDING_OFFER_HISTORY, fundingOfferHistory], + [SYNC_API_METHODS.FUNDING_LOAN_HISTORY, fundingLoanHistory], + [SYNC_API_METHODS.FUNDING_CREDIT_HISTORY, fundingCreditHistory], + [SYNC_API_METHODS.POSITIONS_HISTORY, positionsHistory], + [SYNC_API_METHODS.POSITIONS_SNAPSHOT, positionsSnapshot], + [SYNC_API_METHODS.LOGINS, logins], + [SYNC_API_METHODS.CHANGE_LOGS, changeLogs], + [SYNC_API_METHODS.PAY_INVOICE_LIST, payInvoiceList], + [SYNC_API_METHODS.TICKERS_HISTORY, tickersHistory], + [SYNC_API_METHODS.WALLETS, wallets], + [SYNC_API_METHODS.SYMBOLS, symbols], + [SYNC_API_METHODS.MAP_SYMBOLS, mapSymbols], + [SYNC_API_METHODS.INACTIVE_CURRENCIES, inactiveCurrencies], + [SYNC_API_METHODS.INACTIVE_SYMBOLS, inactiveSymbols], + [SYNC_API_METHODS.FUTURES, futures], + [SYNC_API_METHODS.CURRENCIES, currencies], + [SYNC_API_METHODS.MARGIN_CURRENCY_LIST, marginCurrencyList], + [SYNC_API_METHODS.CANDLES, candles] +]) + +const { cloneSchema } = require('../helpers') + +const getMethodCollMap = (methodCollMap = _methodCollMap) => { + return cloneSchema(methodCollMap) +} + +module.exports = { + getMethodCollMap +} diff --git a/workers/loc.api/sync/schema/sync-schema/ledgers.js b/workers/loc.api/sync/schema/sync-schema/ledgers.js new file mode 100644 index 000000000..7f158006a --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/ledgers.js @@ -0,0 +1,20 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.LEDGERS, + maxLimit: 2500, + dateFieldName: 'mts', + symbolFieldName: 'currency', + sort: [['mts', -1], ['id', -1]], + hasNewData: false, + start: [], + isSyncRequiredAtLeastOnce: true, + type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.LEDGERS) +} diff --git a/workers/loc.api/sync/schema/sync-schema/logins.js b/workers/loc.api/sync/schema/sync-schema/logins.js new file mode 100644 index 000000000..9b20fba99 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/logins.js @@ -0,0 +1,20 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.LOGINS, + maxLimit: 10000, + dateFieldName: 'time', + symbolFieldName: null, + sort: [['time', -1]], + hasNewData: false, + start: [], + isSyncRequiredAtLeastOnce: true, + type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.LOGINS) +} diff --git a/workers/loc.api/sync/schema/sync-schema/map-symbols.js b/workers/loc.api/sync/schema/sync-schema/map-symbols.js new file mode 100644 index 000000000..0a8a67aa6 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/map-symbols.js @@ -0,0 +1,18 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.MAP_SYMBOLS, + maxLimit: 10000, + projection: ['key', 'value'], + sort: [['key', 1]], + hasNewData: false, + isSyncRequiredAtLeastOnce: true, + type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.MAP_SYMBOLS) +} diff --git a/workers/loc.api/sync/schema/sync-schema/margin-currency-list.js b/workers/loc.api/sync/schema/sync-schema/margin-currency-list.js new file mode 100644 index 000000000..a3ec1b8e4 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/margin-currency-list.js @@ -0,0 +1,18 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.MARGIN_CURRENCY_LIST, + maxLimit: 10000, + projection: 'symbol', + sort: [['symbol', 1]], + hasNewData: false, + isSyncRequiredAtLeastOnce: true, + type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY, + model: getModelOf(TABLES_NAMES.MARGIN_CURRENCY_LIST) +} diff --git a/workers/loc.api/sync/schema/sync-schema/movements.js b/workers/loc.api/sync/schema/sync-schema/movements.js new file mode 100644 index 000000000..8e59c5f82 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/movements.js @@ -0,0 +1,20 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.MOVEMENTS, + maxLimit: 250, + dateFieldName: 'mtsUpdated', + symbolFieldName: 'currency', + sort: [['mtsUpdated', -1]], + hasNewData: false, + start: [], + isSyncRequiredAtLeastOnce: true, + type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.MOVEMENTS) +} diff --git a/workers/loc.api/sync/schema/sync-schema/orders.js b/workers/loc.api/sync/schema/sync-schema/orders.js new file mode 100644 index 000000000..b241f8113 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/orders.js @@ -0,0 +1,20 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.ORDERS, + maxLimit: 2500, + dateFieldName: 'mtsUpdate', + symbolFieldName: 'symbol', + sort: [['mtsUpdate', -1]], + hasNewData: false, + start: [], + isSyncRequiredAtLeastOnce: false, + type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.ORDERS) +} diff --git a/workers/loc.api/sync/schema/sync-schema/pay-invoice-list.js b/workers/loc.api/sync/schema/sync-schema/pay-invoice-list.js new file mode 100644 index 000000000..788cf41e3 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/pay-invoice-list.js @@ -0,0 +1,20 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.PAY_INVOICE_LIST, + maxLimit: 100, + dateFieldName: 't', + symbolFieldName: 'currency', + sort: [['t', -1]], + hasNewData: false, + start: [], + isSyncRequiredAtLeastOnce: false, + type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.PAY_INVOICE_LIST) +} diff --git a/workers/loc.api/sync/schema/sync-schema/positions-history.js b/workers/loc.api/sync/schema/sync-schema/positions-history.js new file mode 100644 index 000000000..560d173c6 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/positions-history.js @@ -0,0 +1,21 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.POSITIONS_HISTORY, + maxLimit: 10000, + dateFieldName: 'mtsUpdate', + symbolFieldName: 'symbol', + sort: [['mtsUpdate', -1]], + hasNewData: false, + start: [], + isSyncRequiredAtLeastOnce: false, + shouldNotApiMiddlewareBeLaunched: true, + type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.POSITIONS_HISTORY) +} diff --git a/workers/loc.api/sync/schema/sync-schema/positions-snapshot.js b/workers/loc.api/sync/schema/sync-schema/positions-snapshot.js new file mode 100644 index 000000000..fd2ff6613 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/positions-snapshot.js @@ -0,0 +1,20 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.POSITIONS_SNAPSHOT, + maxLimit: 10000, + dateFieldName: 'mtsUpdate', + symbolFieldName: 'symbol', + sort: [['mtsUpdate', -1]], + hasNewData: false, + start: [], + isSyncRequiredAtLeastOnce: false, + type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.POSITIONS_SNAPSHOT) +} diff --git a/workers/loc.api/sync/schema/sync-schema/public-trades.js b/workers/loc.api/sync/schema/sync-schema/public-trades.js new file mode 100644 index 000000000..0de4fc4e8 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/public-trades.js @@ -0,0 +1,22 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.PUBLIC_TRADES, + maxLimit: 5000, + dateFieldName: 'mts', + symbolFieldName: '_symbol', + sort: [['mts', -1]], + hasNewData: false, + start: [], + confName: 'publicTradesConf', + isSyncRequiredAtLeastOnce: false, + additionalApiCallArgs: { isNotMoreThanInnerMax: true }, + type: COLLS_TYPES.PUBLIC_INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.PUBLIC_TRADES) +} diff --git a/workers/loc.api/sync/schema/sync-schema/status-messages.js b/workers/loc.api/sync/schema/sync-schema/status-messages.js new file mode 100644 index 000000000..2af584039 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/status-messages.js @@ -0,0 +1,20 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.STATUS_MESSAGES, + maxLimit: 5000, + dateFieldName: 'timestamp', + symbolFieldName: 'key', + sort: [['timestamp', -1]], + hasNewData: false, + confName: 'statusMessagesConf', + isSyncRequiredAtLeastOnce: false, + type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.STATUS_MESSAGES) +} diff --git a/workers/loc.api/sync/schema/sync-schema/symbols.js b/workers/loc.api/sync/schema/sync-schema/symbols.js new file mode 100644 index 000000000..922faea08 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/symbols.js @@ -0,0 +1,18 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.SYMBOLS, + maxLimit: 10000, + projection: 'pairs', + sort: [['pairs', 1]], + hasNewData: false, + isSyncRequiredAtLeastOnce: true, + type: COLLS_TYPES.PUBLIC_UPDATABLE_ARRAY, + model: getModelOf(TABLES_NAMES.SYMBOLS) +} diff --git a/workers/loc.api/sync/schema/sync-schema/tickers-history.js b/workers/loc.api/sync/schema/sync-schema/tickers-history.js new file mode 100644 index 000000000..5053198c7 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/tickers-history.js @@ -0,0 +1,22 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.TICKERS_HISTORY, + maxLimit: 10000, + dateFieldName: 'mtsUpdate', + symbolFieldName: 'symbol', + sort: [['mtsUpdate', -1]], + hasNewData: false, + start: [], + confName: 'tickersHistoryConf', + isSyncRequiredAtLeastOnce: false, + additionalApiCallArgs: { isNotMoreThanInnerMax: true }, + type: COLLS_TYPES.PUBLIC_INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.TICKERS_HISTORY) +} diff --git a/workers/loc.api/sync/schema/sync-schema/trades.js b/workers/loc.api/sync/schema/sync-schema/trades.js new file mode 100644 index 000000000..d6479e896 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/trades.js @@ -0,0 +1,20 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.TRADES, + maxLimit: 2500, + dateFieldName: 'mtsCreate', + symbolFieldName: 'symbol', + sort: [['mtsCreate', -1]], + hasNewData: false, + start: [], + isSyncRequiredAtLeastOnce: true, + type: COLLS_TYPES.INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.TRADES) +} diff --git a/workers/loc.api/sync/schema/sync-schema/wallets.js b/workers/loc.api/sync/schema/sync-schema/wallets.js new file mode 100644 index 000000000..54c2d19d8 --- /dev/null +++ b/workers/loc.api/sync/schema/sync-schema/wallets.js @@ -0,0 +1,37 @@ +'use strict' + +const TABLES_NAMES = require('../tables-names') +const ALLOWED_COLLS = require('../allowed.colls') +const COLLS_TYPES = require('../colls.types') + +const { getModelOf } = require('../models') + +module.exports = { + name: ALLOWED_COLLS.LEDGERS, + dateFieldName: 'mts', + symbolFieldName: 'currency', + sort: [['mts', -1]], + groupFns: ['max(mts)', 'max(id)'], + groupResBy: ['wallet', 'currency'], + isSyncRequiredAtLeastOnce: true, + type: COLLS_TYPES.HIDDEN_INSERTABLE_ARRAY_OBJECTS, + model: getModelOf(TABLES_NAMES.LEDGERS), + dataStructureConverter: (accum, { + wallet: type, + currency, + balance, + mts: mtsUpdate + } = {}) => { + accum.push({ + type, + currency, + balance, + unsettledInterest: null, + balanceAvailable: null, + placeHolder: null, + mtsUpdate + }) + + return accum + } +} diff --git a/workers/loc.api/sync/sub.account.api.data/index.js b/workers/loc.api/sync/sub.account.api.data/index.js index 1570ac7a7..839ff2634 100644 --- a/workers/loc.api/sync/sub.account.api.data/index.js +++ b/workers/loc.api/sync/sub.account.api.data/index.js @@ -189,7 +189,8 @@ class SubAccountApiData { const res = await this.getDataFromApi({ getData: (space, args) => method(args), args, - callerName: 'SUB_ACCOUNT_API_DATA' + callerName: 'SUB_ACCOUNT_API_DATA', + shouldNotInterrupt: true }) resArr.push(res) diff --git a/workers/loc.api/sync/sync.interrupter/index.js b/workers/loc.api/sync/sync.interrupter/index.js index 5f28d4b52..865c0dd60 100644 --- a/workers/loc.api/sync/sync.interrupter/index.js +++ b/workers/loc.api/sync/sync.interrupter/index.js @@ -3,6 +3,9 @@ const Interrupter = require( 'bfx-report/workers/loc.api/interrupter' ) +const INTERRUPTER_NAMES = require( + 'bfx-report/workers/loc.api/interrupter/interrupter.names' +) const SYNC_PROGRESS_STATES = require('../progress/sync.progress.states') @@ -20,6 +23,7 @@ class SyncInterrupter extends Interrupter { this.INITIAL_PROGRESS = 'SYNCHRONIZATION_HAS_NOT_BEEN_STARTED_TO_INTERRUPT' this.INTERRUPTED_PROGRESS = SYNC_PROGRESS_STATES.INTERRUPTED_PROGRESS + this.setName(INTERRUPTER_NAMES.SYNC_INTERRUPTER) this._init() } diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/__test__/helpers/get-mocked-trades.js b/workers/loc.api/sync/transaction.tax.report/helpers/__test__/helpers/get-mocked-trades.js new file mode 100644 index 000000000..a657fa136 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/__test__/helpers/get-mocked-trades.js @@ -0,0 +1,56 @@ +'use strict' + +const splitSymbolPairs = require( + 'bfx-report/workers/loc.api/helpers/split-symbol-pairs' +) + +module.exports = (mockTrades, opts) => { + const missingFields = { + _id: 1, + id: 1, + orderID: 1, + orderType: 'EXCHANGE LIMIT', + orderPrice: null, + maker: 1, + fee: -0.5, + feeCurrency: 'USD', + subUserId: null, + user_id: 1, + + firstSymb: null, + lastSymb: null, + firstSymbPriceUsd: null, + lastSymbPriceUsd: null + } + + return mockTrades.map((trade, i) => { + const isAdditionalTrxMovements = opts?.isAdditionalTrxMovements ?? + trade?.isAdditionalTrxMovements ?? + trade?.isMovements ?? + trade?.isLedgers + const [firstSymb, lastSymb] = splitSymbolPairs(trade.symbol) + + return { + ...missingFields, + + isAdditionalTrxMovements, + isMovements: !!trade?.isMovements, + isLedgers: !!trade?.isLedgers, + isTrades: !isAdditionalTrxMovements, + _id: i + 1, + id: i + 1, + orderID: i + 1, + orderPrice: trade.execPrice, + firstSymb, + lastSymb, + firstSymbPriceUsd: lastSymb === 'USD' ? trade.execPrice : null, + lastSymbPriceUsd: lastSymb === 'USD' ? 1 : null, + + ...trade, + + mtsCreate: opts?.year + ? new Date(trade.mtsCreate).setUTCFullYear(opts?.year) + : trade.mtsCreate + } + }) +} diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/__test__/helpers/index.js b/workers/loc.api/sync/transaction.tax.report/helpers/__test__/helpers/index.js new file mode 100644 index 000000000..2013b36b1 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/__test__/helpers/index.js @@ -0,0 +1,13 @@ +'use strict' + +const { + mockTradesForNextYear, + mockTrades +} = require('./mock-trades') +const getMockedTrades = require('./get-mocked-trades') + +module.exports = { + mockTradesForNextYear, + mockTrades, + getMockedTrades +} diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/__test__/helpers/mock-trades.js b/workers/loc.api/sync/transaction.tax.report/helpers/__test__/helpers/mock-trades.js new file mode 100644 index 000000000..b85d85b06 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/__test__/helpers/mock-trades.js @@ -0,0 +1,124 @@ +'use strict' + +const mockTradesForNextYear = [ + { + symbol: 'tUSTUSD', + mtsCreate: Date.UTC(2024, 3, 27), + execAmount: -200, + execPrice: 0.98 + }, + { + isAdditionalTrxMovements: true, + symbol: 'tBTCUSD', + mtsCreate: Date.UTC(2024, 3, 1), + execAmount: -1, + execPrice: 41_000 + }, + { + symbol: 'tBTCUSD', + mtsCreate: Date.UTC(2024, 2, 17), + execAmount: -5, + execPrice: 61_000 + }, + { + isAdditionalTrxMovements: true, + symbol: 'tBTCUSD', + mtsCreate: Date.UTC(2024, 1, 8), + execAmount: -3, + execPrice: 44_000 + }, + { + symbol: 'tBTCUSD', + mtsCreate: Date.UTC(2024, 0, 14), + execAmount: 2, + execPrice: 48_000 + } +] +const mockTrades = [ + { + isAdditionalTrxMovements: true, + isTaxablePayment: true, + isMarginFundingPayment: true, + symbol: 'tEURUSD', + mtsCreate: Date.UTC(2023, 6, 23), + execAmount: 2.11, + execPrice: 1.05, + firstSymbPriceUsd: 1.05, + lastSymbPriceUsd: 1 + }, + { + symbol: 'tUSTEUR', + mtsCreate: Date.UTC(2023, 6, 21), + execAmount: -100, + execPrice: 0.9, + firstSymbPriceUsd: 1.05, + lastSymbPriceUsd: 0.95 + }, + { + symbol: 'tETHUST', + mtsCreate: Date.UTC(2023, 5, 11), + execAmount: -1, + execPrice: 2800, + firstSymbPriceUsd: 3_110, + lastSymbPriceUsd: 1.11 + }, + { + symbol: 'tETHBTC', + mtsCreate: Date.UTC(2023, 4, 22), + execAmount: -1, + execPrice: 0.055, + firstSymbPriceUsd: 2_650, + lastSymbPriceUsd: 48_000 + }, + { + symbol: 'tETHUSD', + mtsCreate: Date.UTC(2023, 4, 10), + execAmount: -1, + execPrice: 2_000 + }, + { + symbol: 'tETHUSD', + mtsCreate: Date.UTC(2023, 3, 10), + execAmount: -2, + execPrice: 3_200 + }, + { + isAdditionalTrxMovements: true, + symbol: 'tETHUSD', + mtsCreate: Date.UTC(2023, 3, 2), + execAmount: -2, + execPrice: 3000 + }, + { + symbol: 'tETHBTC', + mtsCreate: Date.UTC(2023, 2, 23), + execAmount: 10, + execPrice: 0.05, + firstSymbPriceUsd: 2_601, + lastSymbPriceUsd: 50_000 + }, + { + symbol: 'tBTCUSD', + mtsCreate: Date.UTC(2023, 2, 3), + execAmount: -2, + execPrice: 33_000 + }, + { + isAdditionalTrxMovements: true, + symbol: 'tBTCUSD', + mtsCreate: Date.UTC(2023, 1, 5), + execAmount: 20, + execPrice: 43_000 + }, + { + symbol: 'tBTCUSD', + mtsCreate: Date.UTC(2023, 0, 10), + execAmount: 3, + execPrice: 20_000 + } +] + +module.exports = { + mockTradesForNextYear, + mockTrades +} diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/__test__/look-up-trades.spec.js b/workers/loc.api/sync/transaction.tax.report/helpers/__test__/look-up-trades.spec.js new file mode 100644 index 000000000..00133a00d --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/__test__/look-up-trades.spec.js @@ -0,0 +1,442 @@ +'use strict' + +const { assert } = require('chai') + +const lookUpTrades = require('../look-up-trades') +const { + mockTradesForNextYear, + mockTrades, + getMockedTrades +} = require('./helpers') +const { + testBuyTradesWithUnrealizedProfit, + testSaleTradesWithRealizedProfit +} = require('./test-cases') + +describe('lookUpTrades helper for trx tax report', () => { + it('Lookup buy trx with unrealized profit, LIFO strategy', async function () { + const { + buyTradesWithUnrealizedProfit + } = await lookUpTrades( + getMockedTrades(mockTrades), + { + isBackIterativeSaleLookUp: false, + isBackIterativeBuyLookUp: false, + isBuyTradesWithUnrealizedProfitRequired: true, + isNotGainOrLossRequired: true + } + ) + + assert.isArray(buyTradesWithUnrealizedProfit) + assert.equal(buyTradesWithUnrealizedProfit.length, 5) + + testBuyTradesWithUnrealizedProfit(buyTradesWithUnrealizedProfit, 0, { + isAdditionalTrxMovements: false, + mtsCreate: Date.UTC(2023, 5, 11), + firstSymb: 'ETH', + lastSymb: 'UST', + execAmount: -1, + execPrice: 2_800, + buyFilledAmount: 100 + }) + testBuyTradesWithUnrealizedProfit(buyTradesWithUnrealizedProfit, 1, { + isAdditionalTrxMovements: false, + mtsCreate: Date.UTC(2023, 4, 22), + firstSymb: 'ETH', + lastSymb: 'BTC', + execAmount: -1, + execPrice: 0.055, + buyFilledAmount: 0 + }) + testBuyTradesWithUnrealizedProfit(buyTradesWithUnrealizedProfit, 2, { + isAdditionalTrxMovements: false, + mtsCreate: Date.UTC(2023, 2, 23), + firstSymb: 'ETH', + lastSymb: 'BTC', + execAmount: 10, + execPrice: 0.05, + buyFilledAmount: 7 + }) + testBuyTradesWithUnrealizedProfit(buyTradesWithUnrealizedProfit, 3, { + isAdditionalTrxMovements: true, + mtsCreate: Date.UTC(2023, 1, 5), + firstSymb: 'BTC', + lastSymb: 'USD', + execAmount: 20, + execPrice: 43_000, + buyFilledAmount: 2.5 + }) + testBuyTradesWithUnrealizedProfit(buyTradesWithUnrealizedProfit, 4, { + isAdditionalTrxMovements: false, + mtsCreate: Date.UTC(2023, 0, 10), + firstSymb: 'BTC', + lastSymb: 'USD', + execAmount: 3, + execPrice: 20_000, + buyFilledAmount: 0 + }) + }) + + it('Lookup sale trx with realized profit, LIFO strategy', async function () { + const { + saleTradesWithRealizedProfit + } = await lookUpTrades( + getMockedTrades(mockTrades), + { + isBackIterativeSaleLookUp: false, + isBackIterativeBuyLookUp: false, + isBuyTradesWithUnrealizedProfitRequired: false, + isNotGainOrLossRequired: false + } + ) + + assert.isArray(saleTradesWithRealizedProfit) + assert.equal(saleTradesWithRealizedProfit.length, 8) + + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 0, { + asset: 'EUR', + amount: 2.11, + mtsAcquired: Date.UTC(2023, 6, 23), + mtsSold: null, + proceeds: 2.2155, + cost: null, + gainOrLoss: 2.2155, + type: 'MARGIN_FUNDING_PAYMENT' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 1, { + asset: 'UST', + amount: 100, + mtsAcquired: Date.UTC(2023, 5, 11), + mtsSold: Date.UTC(2023, 6, 21), + proceeds: 105, + cost: 111, + gainOrLoss: -6, + type: 'EXCHANGE' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 2, { + asset: 'ETH', + amount: 1, + mtsAcquired: Date.UTC(2023, 2, 23), + mtsSold: Date.UTC(2023, 5, 11), + proceeds: 3_110, + cost: 2_601, + gainOrLoss: 509, + type: 'EXCHANGE' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 3, { + asset: 'ETH', + amount: 1, + mtsAcquired: Date.UTC(2023, 2, 23), + mtsSold: Date.UTC(2023, 4, 22), + proceeds: 2_650, + cost: 2_601, + gainOrLoss: 49, + type: 'EXCHANGE' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 4, { + asset: 'ETH', + amount: 1, + mtsAcquired: Date.UTC(2023, 2, 23), + mtsSold: Date.UTC(2023, 4, 10), + proceeds: 2_000, + cost: 2_601, + gainOrLoss: -601, + type: 'EXCHANGE' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 5, { + asset: 'ETH', + amount: 2, + mtsAcquired: Date.UTC(2023, 2, 23), + mtsSold: Date.UTC(2023, 3, 10), + proceeds: 6_400, + cost: 5_202, + gainOrLoss: 1_198, + type: 'EXCHANGE' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 6, { + asset: 'BTC', + amount: 0.5, + mtsAcquired: Date.UTC(2023, 1, 5), + mtsSold: Date.UTC(2023, 2, 23), + proceeds: 25_000, + cost: 21_500, + gainOrLoss: 3_500, + type: 'EXCHANGE' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 7, { + asset: 'BTC', + amount: 2, + mtsAcquired: Date.UTC(2023, 1, 5), + mtsSold: Date.UTC(2023, 2, 3), + proceeds: 66_000, + cost: 86_000, + gainOrLoss: -20_000, + type: 'EXCHANGE' + }) + }) + + it('Lookup buy trx with unrealized profit, FIFO strategy', async function () { + const { + buyTradesWithUnrealizedProfit + } = await lookUpTrades( + getMockedTrades(mockTrades), + { + isBackIterativeSaleLookUp: true, + isBackIterativeBuyLookUp: true, + isBuyTradesWithUnrealizedProfitRequired: true, + isNotGainOrLossRequired: true + } + ) + + assert.isArray(buyTradesWithUnrealizedProfit) + assert.equal(buyTradesWithUnrealizedProfit.length, 5) + + testBuyTradesWithUnrealizedProfit(buyTradesWithUnrealizedProfit, 0, { + isAdditionalTrxMovements: false, + mtsCreate: Date.UTC(2023, 5, 11), + firstSymb: 'ETH', + lastSymb: 'UST', + execAmount: -1, + execPrice: 2_800, + buyFilledAmount: 100 + }) + testBuyTradesWithUnrealizedProfit(buyTradesWithUnrealizedProfit, 1, { + isAdditionalTrxMovements: false, + mtsCreate: Date.UTC(2023, 4, 22), + firstSymb: 'ETH', + lastSymb: 'BTC', + execAmount: -1, + execPrice: 0.055, + buyFilledAmount: 0 + }) + testBuyTradesWithUnrealizedProfit(buyTradesWithUnrealizedProfit, 2, { + isAdditionalTrxMovements: false, + mtsCreate: Date.UTC(2023, 2, 23), + firstSymb: 'ETH', + lastSymb: 'BTC', + execAmount: 10, + execPrice: 0.05, + buyFilledAmount: 7 + }) + testBuyTradesWithUnrealizedProfit(buyTradesWithUnrealizedProfit, 3, { + isAdditionalTrxMovements: true, + mtsCreate: Date.UTC(2023, 1, 5), + firstSymb: 'BTC', + lastSymb: 'USD', + execAmount: 20, + execPrice: 43_000, + buyFilledAmount: 0 + }) + testBuyTradesWithUnrealizedProfit(buyTradesWithUnrealizedProfit, 4, { + isAdditionalTrxMovements: false, + mtsCreate: Date.UTC(2023, 0, 10), + firstSymb: 'BTC', + lastSymb: 'USD', + execAmount: 3, + execPrice: 20_000, + buyFilledAmount: 2.5 + }) + }) + + it('Lookup sale trx with realized profit, FIFO strategy', async function () { + const { + saleTradesWithRealizedProfit + } = await lookUpTrades( + getMockedTrades(mockTrades), + { + isBackIterativeSaleLookUp: true, + isBackIterativeBuyLookUp: true, + isBuyTradesWithUnrealizedProfitRequired: false, + isNotGainOrLossRequired: false + } + ) + + assert.isArray(saleTradesWithRealizedProfit) + assert.equal(saleTradesWithRealizedProfit.length, 8) + + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 0, { + asset: 'EUR', + amount: 2.11, + mtsAcquired: Date.UTC(2023, 6, 23), + mtsSold: null, + proceeds: 2.2155, + cost: null, + gainOrLoss: 2.2155, + type: 'MARGIN_FUNDING_PAYMENT' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 1, { + asset: 'UST', + amount: 100, + mtsAcquired: Date.UTC(2023, 5, 11), + mtsSold: Date.UTC(2023, 6, 21), + proceeds: 105, + cost: 111, + gainOrLoss: -6, + type: 'EXCHANGE' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 2, { + asset: 'ETH', + amount: 1, + mtsAcquired: Date.UTC(2023, 2, 23), + mtsSold: Date.UTC(2023, 5, 11), + proceeds: 3_110, + cost: 2_601, + gainOrLoss: 509, + type: 'EXCHANGE' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 3, { + asset: 'ETH', + amount: 1, + mtsAcquired: Date.UTC(2023, 2, 23), + mtsSold: Date.UTC(2023, 4, 22), + proceeds: 2_650, + cost: 2_601, + gainOrLoss: 49, + type: 'EXCHANGE' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 4, { + asset: 'ETH', + amount: 1, + mtsAcquired: Date.UTC(2023, 2, 23), + mtsSold: Date.UTC(2023, 4, 10), + proceeds: 2_000, + cost: 2_601, + gainOrLoss: -601, + type: 'EXCHANGE' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 5, { + asset: 'ETH', + amount: 2, + mtsAcquired: Date.UTC(2023, 2, 23), + mtsSold: Date.UTC(2023, 3, 10), + proceeds: 6_400, + cost: 5_202, + gainOrLoss: 1_198, + type: 'EXCHANGE' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 6, { + asset: 'BTC', + amount: 0.5, + mtsAcquired: Date.UTC(2023, 0, 10), + mtsSold: Date.UTC(2023, 2, 23), + proceeds: 25_000, + cost: 10_000, + gainOrLoss: 15_000, + type: 'EXCHANGE' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 7, { + asset: 'BTC', + amount: 2, + mtsAcquired: Date.UTC(2023, 0, 10), + mtsSold: Date.UTC(2023, 2, 3), + proceeds: 66_000, + cost: 40_000, + gainOrLoss: 26_000, + type: 'EXCHANGE' + }) + }) + + it('Lookup sale trx with realized profit considering prev year, LIFO strategy', async function () { + const { + buyTradesWithUnrealizedProfit + } = await lookUpTrades( + getMockedTrades(mockTrades), + { + isBackIterativeSaleLookUp: false, + isBackIterativeBuyLookUp: false, + isBuyTradesWithUnrealizedProfitRequired: true, + isNotGainOrLossRequired: true + } + ) + const _mockTradesForNextYear = getMockedTrades(mockTradesForNextYear) + _mockTradesForNextYear.push(...buyTradesWithUnrealizedProfit) + + const { + saleTradesWithRealizedProfit + } = await lookUpTrades( + _mockTradesForNextYear, + { + isBackIterativeSaleLookUp: false, + isBackIterativeBuyLookUp: false, + isBuyTradesWithUnrealizedProfitRequired: false, + isNotGainOrLossRequired: false + } + ) + + assert.isArray(saleTradesWithRealizedProfit) + assert.equal(saleTradesWithRealizedProfit.length, 2) + + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 0, { + asset: 'UST', + amount: 200, + mtsAcquired: Date.UTC(2023, 5, 11), + mtsSold: Date.UTC(2024, 3, 27), + proceeds: 196, + cost: 222, + gainOrLoss: -26, + type: 'EXCHANGE' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 1, { + asset: 'BTC', + amount: 5, + mtsAcquired: Date.UTC(2023, 1, 5), + mtsSold: Date.UTC(2024, 2, 17), + proceeds: 305_000, + cost: 220_275, + gainOrLoss: 84_725, + type: 'EXCHANGE' + }) + }) + + it('Lookup sale trx with realized profit considering prev year, FIFO strategy', async function () { + const { + buyTradesWithUnrealizedProfit + } = await lookUpTrades( + getMockedTrades(mockTrades), + { + isBackIterativeSaleLookUp: true, + isBackIterativeBuyLookUp: true, + isBuyTradesWithUnrealizedProfitRequired: true, + isNotGainOrLossRequired: true + } + ) + const _mockTradesForNextYear = getMockedTrades(mockTradesForNextYear) + _mockTradesForNextYear.push(...buyTradesWithUnrealizedProfit) + + const { + saleTradesWithRealizedProfit + } = await lookUpTrades( + _mockTradesForNextYear, + { + isBackIterativeSaleLookUp: true, + isBackIterativeBuyLookUp: true, + isBuyTradesWithUnrealizedProfitRequired: false, + isNotGainOrLossRequired: false + } + ) + + assert.isArray(saleTradesWithRealizedProfit) + assert.equal(saleTradesWithRealizedProfit.length, 2) + + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 0, { + asset: 'UST', + amount: 200, + mtsAcquired: Date.UTC(2023, 5, 11), + mtsSold: Date.UTC(2024, 3, 27), + proceeds: 196, + cost: 222, + gainOrLoss: -26, + type: 'EXCHANGE' + }) + testSaleTradesWithRealizedProfit(saleTradesWithRealizedProfit, 1, { + asset: 'BTC', + amount: 5, + mtsAcquired: Date.UTC(2023, 1, 5), + mtsSold: Date.UTC(2024, 2, 17), + proceeds: 305_000, + cost: 215_000, + gainOrLoss: 90_000, + type: 'EXCHANGE' + }) + }) +}) diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/__test__/test-cases/index.js b/workers/loc.api/sync/transaction.tax.report/helpers/__test__/test-cases/index.js new file mode 100644 index 000000000..b88351255 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/__test__/test-cases/index.js @@ -0,0 +1,13 @@ +'use strict' + +const testBuyTradesWithUnrealizedProfit = require( + './test-buy-trades-with-unrealized-profit' +) +const testSaleTradesWithRealizedProfit = require( + './test-sale-trades-with-realized-profit' +) + +module.exports = { + testBuyTradesWithUnrealizedProfit, + testSaleTradesWithRealizedProfit +} diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/__test__/test-cases/test-buy-trades-with-unrealized-profit.js b/workers/loc.api/sync/transaction.tax.report/helpers/__test__/test-cases/test-buy-trades-with-unrealized-profit.js new file mode 100644 index 000000000..bfc776e35 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/__test__/test-cases/test-buy-trades-with-unrealized-profit.js @@ -0,0 +1,49 @@ +'use strict' + +const { assert } = require('chai') +const BigNumber = require('bignumber.js') + +module.exports = (arr, index, props) => { + const trade = arr[index] + const { + isAdditionalTrxMovements, + mtsCreate, + firstSymb, + lastSymb, + execAmount, + execPrice, + + buyFilledAmount + } = props ?? {} + + assert.isObject(trade) + assert.isBoolean(trade.isBuyTradesWithUnrealizedProfitForPrevPeriod) + assert.isOk(trade.isBuyTradesWithUnrealizedProfitForPrevPeriod) + assert.isBoolean(trade.isBuyTrx) + assert.isOk(trade.isBuyTrx) + assert.isBoolean(trade.isBuyTrxHistFilled) + assert.isNotOk(trade.isBuyTrxHistFilled) + assert.instanceOf(trade.proceedsForBuyTrxUsd, BigNumber) + assert.equal(trade.proceedsForBuyTrxUsd.toNumber(), 0) + assert.instanceOf(trade.proceedsForBuyTrxUsd, BigNumber) + assert.equal(trade.proceedsForBuyTrxUsd.toNumber(), 0) + assert.isNumber(trade.firstSymbPriceUsd) + assert.isNumber(trade.lastSymbPriceUsd) + assert.isArray(trade.saleTrxsForRealizedProfit) + + assert.isBoolean(trade.isAdditionalTrxMovements) + assert.equal(trade.isAdditionalTrxMovements, isAdditionalTrxMovements) + assert.isNumber(trade.mtsCreate) + assert.equal(trade.mtsCreate, mtsCreate) + assert.isString(trade.firstSymb) + assert.equal(trade.firstSymb, firstSymb) + assert.isString(trade.lastSymb) + assert.equal(trade.lastSymb, lastSymb) + assert.isNumber(trade.execAmount) + assert.equal(trade.execAmount, execAmount) + assert.isNumber(trade.execPrice) + assert.equal(trade.execPrice, execPrice) + + assert.instanceOf(trade.buyFilledAmount, BigNumber) + assert.equal(trade.buyFilledAmount.toNumber(), buyFilledAmount) +} diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/__test__/test-cases/test-sale-trades-with-realized-profit.js b/workers/loc.api/sync/transaction.tax.report/helpers/__test__/test-cases/test-sale-trades-with-realized-profit.js new file mode 100644 index 000000000..e88152d12 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/__test__/test-cases/test-sale-trades-with-realized-profit.js @@ -0,0 +1,48 @@ +'use strict' + +const { assert } = require('chai') + +module.exports = (arr, index, props) => { + const trade = arr[index] + const { + asset, + amount, + mtsAcquired, + mtsSold, + proceeds, + cost, + gainOrLoss, + type + } = props ?? {} + + assert.isObject(trade) + + assert.isString(trade.asset) + assert.equal(trade.asset, asset) + assert.isNumber(trade.amount) + assert.equal(trade.amount, amount) + assert.isNumber(trade.mtsAcquired) + assert.equal(trade.mtsAcquired, mtsAcquired) + + if (mtsSold === null) { + assert.isNull(trade.mtsSold) + } else { + assert.isNumber(trade.mtsSold) + assert.equal(trade.mtsSold, mtsSold) + } + + assert.isNumber(trade.proceeds) + assert.equal(trade.proceeds, proceeds) + + if (cost === null) { + assert.isNull(trade.cost) + } else { + assert.isNumber(trade.cost) + assert.equal(trade.cost, cost) + } + + assert.isNumber(trade.gainOrLoss) + assert.equal(trade.gainOrLoss, gainOrLoss) + assert.isString(trade.type) + assert.equal(trade.type, type) +} diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/find-public-trade.js b/workers/loc.api/sync/transaction.tax.report/helpers/find-public-trade.js new file mode 100644 index 000000000..91f214ef2 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/find-public-trade.js @@ -0,0 +1,23 @@ +'use strict' + +module.exports = (pubTrades, mts) => { + let startIndex = 0 + let endIndex = pubTrades.length - 1 + let middleIndex = null + + while (startIndex <= endIndex) { + middleIndex = Math.floor((startIndex + endIndex) / 2) + + if (pubTrades[middleIndex]?.mts === mts) { + return pubTrades[middleIndex] + } + if (mts < pubTrades[middleIndex]?.mts) { + endIndex = middleIndex - 1 + } + if (mts > pubTrades[middleIndex]?.mts) { + startIndex = middleIndex + 1 + } + } + + return pubTrades[middleIndex] +} diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/get-ccy-pair-for-conversion.js b/workers/loc.api/sync/transaction.tax.report/helpers/get-ccy-pair-for-conversion.js new file mode 100644 index 000000000..76ca4f70b --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/get-ccy-pair-for-conversion.js @@ -0,0 +1,22 @@ +'use strict' + +const TrxPriceCalculator = require('./trx.price.calculator') + +module.exports = (symbol, trxPriceCalculator) => { + if (trxPriceCalculator?.kindOfCcyForTriangulation === TrxPriceCalculator.IS_FOREX_CCY_FOR_TRIANGULATION) { + const symbSeparator = ( + symbol.length > 3 || + TrxPriceCalculator.CRYPTO_CCY_FOR_TRIANGULATION > 3 + ) + ? ':' + : '' + + return `t${TrxPriceCalculator.CRYPTO_CCY_FOR_TRIANGULATION}${symbSeparator}${symbol}` + } + + const symbSeparator = symbol.length > 3 + ? ':' + : '' + + return `t${symbol}${symbSeparator}USD` +} diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/get-trx-map-by-ccy.js b/workers/loc.api/sync/transaction.tax.report/helpers/get-trx-map-by-ccy.js new file mode 100644 index 000000000..f2a2fe68f --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/get-trx-map-by-ccy.js @@ -0,0 +1,146 @@ +'use strict' + +const { isForexSymb } = require('../../helpers') +const PRIORITY_CURRENCY_LIST = require('./priority.currency.list') +const TrxPriceCalculator = require('./trx.price.calculator') + +// Handle tETHF0:USTF0 symbols +const symbRegExpNormalizer = /F0$/i + +const setCcyCalculator = (map, symb, trxPriceCalculator) => { + if (!map.has(symb)) { + map.set(symb, []) + } + + map.get(symb).push(trxPriceCalculator) +} + +const calcTotalTrxAmount = (trxMapArray) => { + return trxMapArray.reduce((accum, [s, calcs]) => { + return accum + calcs.length + }, 0) +} + +const placeTriangulationCcyAtStart = (map) => { + if (!map.has(TrxPriceCalculator.CRYPTO_CCY_FOR_TRIANGULATION)) { + return { + trxMapByCcy: map, + totalTrxAmount: calcTotalTrxAmount([...map]) + } + } + + const triangulationCcyCalculators = map.get(TrxPriceCalculator.CRYPTO_CCY_FOR_TRIANGULATION) + map.delete(TrxPriceCalculator.CRYPTO_CCY_FOR_TRIANGULATION) + + const trxMapArray = [ + [TrxPriceCalculator.CRYPTO_CCY_FOR_TRIANGULATION, triangulationCcyCalculators], + ...map + ] + + return { + trxMapByCcy: new Map(trxMapArray), + totalTrxAmount: calcTotalTrxAmount(trxMapArray) + } +} + +module.exports = (trxs) => { + const trxMapByCcy = new Map() + + for (const trx of trxs) { + const firstSymb = trx.firstSymb.replace(symbRegExpNormalizer, '') + const lastSymb = trx.lastSymb.replace(symbRegExpNormalizer, '') + const isFirstSymbForex = isForexSymb(trx.firstSymb) + const isLastSymbForex = isForexSymb(trx.lastSymb) + const priorCcyListIndexForFirstSymb = PRIORITY_CURRENCY_LIST + .indexOf(firstSymb) + const priorCcyListIndexForLastSymb = PRIORITY_CURRENCY_LIST + .indexOf(lastSymb) + const isFirstSymbInPriorCcyList = priorCcyListIndexForFirstSymb >= 0 + const isLastSymbInPriorCcyList = priorCcyListIndexForLastSymb >= 0 + + // To Handle tEURUSD etc cases for `_isMarginFundingPayment` etc + if ( + isFirstSymbForex && + lastSymb === 'USD' + ) { + const triangulationCcyCalculator = new TrxPriceCalculator( + trx, + TrxPriceCalculator.FIRST_SYMB_PRICE_PROP_NAME, + TrxPriceCalculator.LAST_SYMB_PRICE_PROP_NAME, + TrxPriceCalculator.IS_CRYPTO_CCY_FOR_TRIANGULATION + ) + + setCcyCalculator( + trxMapByCcy, + TrxPriceCalculator.CRYPTO_CCY_FOR_TRIANGULATION, + triangulationCcyCalculator + ) + setCcyCalculator(trxMapByCcy, firstSymb, new TrxPriceCalculator( + trx, + TrxPriceCalculator.FIRST_SYMB_PRICE_PROP_NAME, + TrxPriceCalculator.LAST_SYMB_PRICE_PROP_NAME, + TrxPriceCalculator.IS_FOREX_CCY_FOR_TRIANGULATION, + triangulationCcyCalculator + )) + + continue + } + if (isFirstSymbForex) { + setCcyCalculator(trxMapByCcy, lastSymb, new TrxPriceCalculator( + trx, + TrxPriceCalculator.LAST_SYMB_PRICE_PROP_NAME, + TrxPriceCalculator.FIRST_SYMB_PRICE_PROP_NAME + )) + + continue + } + if (isLastSymbForex) { + setCcyCalculator(trxMapByCcy, firstSymb, new TrxPriceCalculator( + trx, + TrxPriceCalculator.FIRST_SYMB_PRICE_PROP_NAME, + TrxPriceCalculator.LAST_SYMB_PRICE_PROP_NAME + )) + + continue + } + if ( + isFirstSymbInPriorCcyList && + ( + !isLastSymbInPriorCcyList || + priorCcyListIndexForFirstSymb <= priorCcyListIndexForLastSymb + ) + ) { + setCcyCalculator(trxMapByCcy, firstSymb, new TrxPriceCalculator( + trx, + TrxPriceCalculator.FIRST_SYMB_PRICE_PROP_NAME, + TrxPriceCalculator.LAST_SYMB_PRICE_PROP_NAME + )) + + continue + } + if ( + isLastSymbInPriorCcyList && + ( + !isFirstSymbInPriorCcyList || + priorCcyListIndexForLastSymb <= priorCcyListIndexForFirstSymb + ) + ) { + setCcyCalculator(trxMapByCcy, lastSymb, new TrxPriceCalculator( + trx, + TrxPriceCalculator.LAST_SYMB_PRICE_PROP_NAME, + TrxPriceCalculator.FIRST_SYMB_PRICE_PROP_NAME + )) + + continue + } + + setCcyCalculator(trxMapByCcy, firstSymb, new TrxPriceCalculator( + trx, + TrxPriceCalculator.FIRST_SYMB_PRICE_PROP_NAME, + TrxPriceCalculator.LAST_SYMB_PRICE_PROP_NAME + )) + } + + // To Handle tEURUSD etc cases, first get price for `BTCUSD`, then rest eg `BTCEUR` + return placeTriangulationCcyAtStart(trxMapByCcy) +} diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/get-trx-tax-type.js b/workers/loc.api/sync/transaction.tax.report/helpers/get-trx-tax-type.js new file mode 100644 index 000000000..6556bd7b7 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/get-trx-tax-type.js @@ -0,0 +1,20 @@ +'use strict' + +const TRX_TAX_TYPES = require('./trx.tax.types') + +module.exports = (trx) => { + if (trx?.isAirdropOnWallet) { + return TRX_TAX_TYPES.AIRDROP_ON_WALLET + } + if (trx?.isMarginFundingPayment) { + return TRX_TAX_TYPES.MARGIN_FUNDING_PAYMENT + } + if (trx?.isAffiliateRebate) { + return TRX_TAX_TYPES.AFFILIATE_REBATE + } + if (trx?.isStakingPayments) { + return TRX_TAX_TYPES.STAKING_PAYMENT + } + + return TRX_TAX_TYPES.EXCHANGE +} diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/index.js b/workers/loc.api/sync/transaction.tax.report/helpers/index.js new file mode 100644 index 000000000..680903d1e --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/index.js @@ -0,0 +1,29 @@ +'use strict' + +const TRX_TAX_TYPES = require('./trx.tax.types') +const TRX_TAX_STRATEGIES = require('./trx.tax.strategies') +const PRIORITY_CURRENCY_LIST = require('./priority.currency.list') +const PROGRESS_STATES = require('./progress.states') +const remapTrades = require('./remap-trades') +const remapMovements = require('./remap-movements') +const lookUpTrades = require('./look-up-trades') +const getTrxMapByCcy = require('./get-trx-map-by-ccy') +const findPublicTrade = require('./find-public-trade') +const TrxPriceCalculator = require('./trx.price.calculator') +const getCcyPairForConversion = require('./get-ccy-pair-for-conversion') +const getTrxTaxType = require('./get-trx-tax-type') + +module.exports = { + TRX_TAX_TYPES, + TRX_TAX_STRATEGIES, + PRIORITY_CURRENCY_LIST, + PROGRESS_STATES, + remapTrades, + remapMovements, + lookUpTrades, + getTrxMapByCcy, + findPublicTrade, + TrxPriceCalculator, + getCcyPairForConversion, + getTrxTaxType +} diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/look-up-trades.js b/workers/loc.api/sync/transaction.tax.report/helpers/look-up-trades.js new file mode 100644 index 000000000..c98bb4ea5 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/look-up-trades.js @@ -0,0 +1,411 @@ +'use strict' + +const BigNumber = require('bignumber.js') +const { setImmediate } = require('node:timers/promises') +const splitSymbolPairs = require( + 'bfx-report/workers/loc.api/helpers/split-symbol-pairs' +) + +const { + isForexSymb, + getBackIterable +} = require('../../helpers') + +const { + CurrencyConversionError, + CurrencyPairSeparationError +} = require('../../../errors') + +const getTrxTaxType = require('./get-trx-tax-type') + +module.exports = async (trades, opts) => { + const { + isBackIterativeSaleLookUp = false, + isBackIterativeBuyLookUp = false, + isBuyTradesWithUnrealizedProfitRequired = false, + isNotGainOrLossRequired = false, + interrupter + } = opts ?? {} + + const saleTradesWithRealizedProfit = [] + const buyTradesWithUnrealizedProfit = [] + + if ( + !Array.isArray(trades) || + trades.length === 0 + ) { + return { + saleTradesWithRealizedProfit, + buyTradesWithUnrealizedProfit + } + } + + let lastLoopUnlockMts = Date.now() + const tradeIterator = isBackIterativeSaleLookUp + ? getBackIterable(trades) + : trades + + for (const [i, trade] of tradeIterator.entries()) { + if (interrupter?.hasInterrupted()) { + return { + saleTradesWithRealizedProfit, + buyTradesWithUnrealizedProfit + } + } + + const currentLoopUnlockMts = Date.now() + + /* + * Trx hist restoring is a hard sync operation, + * to prevent EventLoop locking more than 1sec + * it needs to resolve async queue + */ + if ((currentLoopUnlockMts - lastLoopUnlockMts) > 1000) { + await setImmediate() + + lastLoopUnlockMts = currentLoopUnlockMts + } + + trade.isAdditionalTrxMovements = trade + .isAdditionalTrxMovements ?? false + + trade.isSaleTrx = trade.isSaleTrx ?? false + trade.isSaleTrxHistFilled = trade.isSaleTrxHistFilled ?? false + trade.saleFilledAmount = trade + .saleFilledAmount ?? new BigNumber(0) + trade.costForSaleTrxUsd = trade + .costForSaleTrxUsd ?? new BigNumber(0) + trade.buyTrxsForRealizedProfit = trade + .buyTrxsForRealizedProfit ?? [] + + trade.isBuyTrx = trade.isBuyTrx ?? false + trade.isBuyTrxHistFilled = trade.isBuyTrxHistFilled ?? false + trade.buyFilledAmount = trade + .buyFilledAmount ?? new BigNumber(0) + trade.proceedsForBuyTrxUsd = trade + .proceedsForBuyTrxUsd ?? new BigNumber(0) + trade.saleTrxsForRealizedProfit = trade + .saleTrxsForRealizedProfit ?? [] + + if ( + !trade?.symbol || + !Number.isFinite(trade?.execPrice) || + ( + !isBuyTradesWithUnrealizedProfitRequired && + trade.execPrice === 0 + ) || + !Number.isFinite(trade?.execAmount) || + trade.execAmount === 0 + ) { + continue + } + + const [firstSymb, lastSymb] = ( + trade?.firstSymb && + trade?.lastSymb + ) + ? [trade?.firstSymb, trade?.lastSymb] + : splitSymbolPairs(trade.symbol) + trade.firstSymb = firstSymb + trade.lastSymb = lastSymb + + /* + * Exapmle of considered trxs as sale: + * - buy ETC:BTC -> amount 5, price 0.5 (here needs to be considered as 2 trxs: buy ETC and sale BTC) + * - sale ETC:BTC -> amount -2, price 0.6 (here needs to be considered as 2 trxs: sale ETC and buy BTC) + * - sale ETC:USD -> amount -3, price 4000 + * - sale UST:EUR - > amount -3, price 0.9 (here needs to be considered EUR price and converted to USD) + */ + const isLastSymbForex = isForexSymb(lastSymb) + const isDistinctSale = trade.execAmount < 0 + const isSaleBetweenCrypto = ( + trade.execAmount > 0 && + !isLastSymbForex + ) + trade.isSaleTrx = isDistinctSale || isSaleBetweenCrypto + trade.isBuyTrx = ( + ( + trade.execAmount > 0 || + !isLastSymbForex + ) && + ( + !trade.isTaxablePayment || + !isForexSymb(firstSymb) + ) + ) + + if ( + !trade.isSaleTrx || + trade.isBuyTradesWithUnrealizedProfitForPrevPeriod + ) { + continue + } + if ( + !firstSymb || + !lastSymb + ) { + throw new CurrencyPairSeparationError({ + symbol: trade.symbol, + firstSymb, + lastSymb + }) + } + + const saleAmount = trade.execAmount < 0 + ? new BigNumber(trade.execAmount).abs() + : new BigNumber(trade.execAmount) + .times(trade.execPrice) + .abs() + const _salePriceUsd = isDistinctSale + ? trade.firstSymbPriceUsd + : trade.lastSymbPriceUsd + const salePriceUsd = isNotGainOrLossRequired ? 0 : _salePriceUsd + const saleAsset = isDistinctSale + ? firstSymb + : lastSymb + + if (!Number.isFinite(salePriceUsd)) { + throw new CurrencyConversionError({ + symbol: saleAsset, + priceUsd: salePriceUsd + }) + } + + const startPoint = isBackIterativeBuyLookUp + ? trades.length - 1 + : i + 1 + const checkPoint = (j) => ( + isBackIterativeBuyLookUp + ? i < j + : trades.length > j + ) + const shiftPoint = (j) => ( + isBackIterativeBuyLookUp + ? j - 1 + : j + 1 + ) + + for (let j = startPoint; checkPoint(j); j = shiftPoint(j)) { + if (interrupter?.hasInterrupted()) { + return { + saleTradesWithRealizedProfit, + buyTradesWithUnrealizedProfit + } + } + if (trade.isSaleTrxHistFilled) { + break + } + + const tradeForLookup = trades[j] + + if ( + tradeForLookup?.isBuyTrxHistFilled || + !tradeForLookup?.symbol || + !Number.isFinite(tradeForLookup?.execAmount) || + tradeForLookup.execAmount === 0 || + !Number.isFinite(tradeForLookup?.execPrice) || + ( + !isBuyTradesWithUnrealizedProfitRequired && + tradeForLookup.execPrice === 0 + ) + ) { + continue + } + + tradeForLookup.isBuyTrx = tradeForLookup.isBuyTrx ?? false + tradeForLookup.isBuyTrxHistFilled = tradeForLookup + .isBuyTrxHistFilled ?? false + tradeForLookup.buyFilledAmount = tradeForLookup + .buyFilledAmount ?? new BigNumber(0) + tradeForLookup.proceedsForBuyTrxUsd = tradeForLookup + .proceedsForBuyTrxUsd ?? new BigNumber(0) + tradeForLookup.saleTrxsForRealizedProfit = tradeForLookup + .saleTrxsForRealizedProfit ?? [] + + const [firstSymbForLookup, lastSymbForLookup] = ( + tradeForLookup?.firstSymb && + tradeForLookup?.lastSymb + ) + ? [tradeForLookup?.firstSymb, tradeForLookup?.lastSymb] + : splitSymbolPairs(tradeForLookup.symbol) + tradeForLookup.firstSymb = firstSymbForLookup + tradeForLookup.lastSymb = lastSymbForLookup + + if ( + !firstSymbForLookup || + !lastSymbForLookup + ) { + throw new CurrencyPairSeparationError({ + symbol: tradeForLookup.symbol, + firstSymb: firstSymbForLookup, + lastSymb: lastSymbForLookup + }) + } + + if ( + ( + tradeForLookup.execAmount < 0 && + isForexSymb(lastSymbForLookup) + ) || + ( + tradeForLookup.isTaxablePayment && + isForexSymb(firstSymbForLookup) + ) + ) { + continue + } + + tradeForLookup.isBuyTrx = true + + const buyAsset = tradeForLookup.execAmount > 0 + ? firstSymbForLookup + : lastSymbForLookup + + if (saleAsset !== buyAsset) { + continue + } + + tradeForLookup.saleTrxsForRealizedProfit.push(trade) + trade.buyTrxsForRealizedProfit.push(tradeForLookup) + + const buyAmount = tradeForLookup.execAmount > 0 + ? new BigNumber(tradeForLookup.execAmount).abs() + : new BigNumber(tradeForLookup.execAmount) + .times(tradeForLookup.execPrice) + .abs() + const _buyPriceUsd = tradeForLookup.execAmount > 0 + ? tradeForLookup.firstSymbPriceUsd + : tradeForLookup.lastSymbPriceUsd + const buyPriceUsd = isNotGainOrLossRequired ? 0 : _buyPriceUsd + const buyRestAmount = buyAmount + .minus(tradeForLookup.buyFilledAmount) + const saleRestAmount = saleAmount + .minus(trade.saleFilledAmount) + + if (!Number.isFinite(buyPriceUsd)) { + throw new CurrencyConversionError({ + symbol: buyAsset, + priceUsd: buyPriceUsd + }) + } + + if (buyRestAmount.lt(saleRestAmount)) { + tradeForLookup.buyFilledAmount = buyAmount + trade.saleFilledAmount = trade.saleFilledAmount + .plus(buyRestAmount) + tradeForLookup.proceedsForBuyTrxUsd = tradeForLookup + .proceedsForBuyTrxUsd + .plus(buyRestAmount.times(salePriceUsd)) + trade.costForSaleTrxUsd = trade.costForSaleTrxUsd + .plus(buyRestAmount.times(buyPriceUsd)) + tradeForLookup.isBuyTrxHistFilled = true + } + if (buyRestAmount.gt(saleRestAmount)) { + tradeForLookup.buyFilledAmount = tradeForLookup + .buyFilledAmount + .plus(saleRestAmount) + trade.saleFilledAmount = saleAmount + tradeForLookup.proceedsForBuyTrxUsd = tradeForLookup + .proceedsForBuyTrxUsd + .plus(saleRestAmount.times(salePriceUsd)) + trade.costForSaleTrxUsd = trade.costForSaleTrxUsd + .plus(saleRestAmount.times(buyPriceUsd)) + trade.isSaleTrxHistFilled = true + } + if (buyRestAmount.eq(saleRestAmount)) { + tradeForLookup.buyFilledAmount = buyAmount + trade.saleFilledAmount = saleAmount + tradeForLookup.proceedsForBuyTrxUsd = tradeForLookup + .proceedsForBuyTrxUsd + .plus(buyRestAmount.times(salePriceUsd)) + trade.costForSaleTrxUsd = trade.costForSaleTrxUsd + .plus(buyRestAmount.times(buyPriceUsd)) + tradeForLookup.isBuyTrxHistFilled = true + trade.isSaleTrxHistFilled = true + } + + if (tradeForLookup.isBuyTrxHistFilled) { + tradeForLookup.buyAsset = buyAsset + tradeForLookup.buyAmount = buyAmount + tradeForLookup.mtsAcquiredForBuyTrx = tradeForLookup.mtsCreate + tradeForLookup.mtsSoldForBuyTrx = trade.mtsCreate + tradeForLookup.costForBuyTrxUsd = buyAmount.times(buyPriceUsd) + tradeForLookup.gainOrLossForBuyTrxUsd = tradeForLookup + .proceedsForBuyTrxUsd + .minus(tradeForLookup.costForBuyTrxUsd) + } + } + + trade.saleAsset = saleAsset + trade.saleAmount = saleAmount + trade.mtsAcquiredForSaleTrx = ( + trade.buyTrxsForRealizedProfit[0]?.mtsCreate > + trade.buyTrxsForRealizedProfit[trade.buyTrxsForRealizedProfit.length - 1]?.mtsCreate + ) + ? trade.buyTrxsForRealizedProfit[trade.buyTrxsForRealizedProfit.length - 1]?.mtsCreate + : trade.buyTrxsForRealizedProfit[0]?.mtsCreate + trade.mtsSoldForSaleTrx = trade.mtsCreate + trade.proceedsForSaleTrxUsd = saleAmount.times(salePriceUsd) + trade.gainOrLossUsd = trade.proceedsForSaleTrxUsd + .minus(trade.costForSaleTrxUsd) + } + + for (const trade of trades) { + if ( + isBuyTradesWithUnrealizedProfitRequired && + trade?.isBuyTrx && + !trade?.isBuyTrxHistFilled + ) { + trade.isBuyTradesWithUnrealizedProfitForPrevPeriod = true + buyTradesWithUnrealizedProfit.push(trade) + } + + if ( + isBuyTradesWithUnrealizedProfitRequired || + trade?.isBuyTradesWithUnrealizedProfitForPrevPeriod || + ( + !trade?.isTaxablePayment && + ( + !trade?.isSaleTrx || + trade?.isAdditionalTrxMovements + ) + ) + ) { + continue + } + if (trade.isTaxablePayment) { + const proceeds = new BigNumber(trade.execAmount) + .times(trade.firstSymbPriceUsd) + .toNumber() + + saleTradesWithRealizedProfit.push({ + asset: trade.firstSymb, + amount: trade.execAmount, + mtsAcquired: trade.mtsCreate, + mtsSold: null, + proceeds, + cost: null, + gainOrLoss: proceeds, + type: getTrxTaxType(trade) + }) + + continue + } + + saleTradesWithRealizedProfit.push({ + asset: trade.saleAsset, + amount: trade.saleAmount.toNumber(), + mtsAcquired: trade.mtsAcquiredForSaleTrx, + mtsSold: trade.mtsSoldForSaleTrx, + proceeds: trade.proceedsForSaleTrxUsd.toNumber(), + cost: trade.costForSaleTrxUsd.toNumber(), + gainOrLoss: trade.gainOrLossUsd.toNumber(), + type: getTrxTaxType(trade) + }) + } + + return { + saleTradesWithRealizedProfit, + buyTradesWithUnrealizedProfit + } +} diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/priority.currency.list.js b/workers/loc.api/sync/transaction.tax.report/helpers/priority.currency.list.js new file mode 100644 index 000000000..ab85d6460 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/priority.currency.list.js @@ -0,0 +1,13 @@ +'use strict' + +/* + * The order is important: + * the smaller the index, the more priority currency + */ +const PRIORITY_CURRENCY_LIST = [ + 'BTC', + 'ETH', + 'UST' +] + +module.exports = PRIORITY_CURRENCY_LIST diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/progress.states.js b/workers/loc.api/sync/transaction.tax.report/helpers/progress.states.js new file mode 100644 index 000000000..654a39b39 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/progress.states.js @@ -0,0 +1,12 @@ +'use strict' + +const PROGRESS_STATES = { + GENERATION_STARTED: 'GENERATION_STARTED', + OBTAINING_CURRENCY_PRICES: 'OBTAINING_CURRENCY_PRICES', + TRANSACTION_HISTORY_GENERATION: 'TRANSACTION_HISTORY_GENERATION', + GENERATION_COMPLETED: 'GENERATION_COMPLETED', + + GENERATION_INTERRUPTED: 'GENERATION_INTERRUPTED' +} + +module.exports = PROGRESS_STATES diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/remap-movements.js b/workers/loc.api/sync/transaction.tax.report/helpers/remap-movements.js new file mode 100644 index 000000000..90e46f832 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/remap-movements.js @@ -0,0 +1,99 @@ +'use strict' + +const BigNumber = require('bignumber.js') + +const { + isForexSymb +} = require('../../helpers') + +module.exports = (movements, params) => { + const { + remappedTrxs, + remappedTrxsForConvToUsd + } = params + + for (const movement of movements) { + const isTaxablePayment = !!( + movement?._isAirdropOnWallet || + movement?._isMarginFundingPayment || + movement?._isAffiliateRebate || + movement?._isStakingPayments + ) + + if ( + !movement?.currency || + ( + !isTaxablePayment && + isForexSymb(movement.currency) + ) || + !Number.isFinite(movement?.amount) || + movement.amount === 0 || + !Number.isFinite(movement?.mtsUpdated) + ) { + continue + } + + const firstSymb = movement.currency + const lastSymb = 'USD' + const symbSeparator = firstSymb.length > 3 + ? ':' + : '' + + const remappedMovement = { + _id: movement._id, + // NOTE: it means entries are not taken from trades table + isAdditionalTrxMovements: true, + // NOTE: movements can have sub-account transfer entries from ledgers table + isMovements: !movement.isLedgers, + isLedgers: !!movement.isLedgers, + isTrades: false, + isTaxablePayment, + symbol: `t${firstSymb}${symbSeparator}${lastSymb}`, + mtsCreate: movement.mtsUpdated, + firstSymb, + lastSymb, + firstSymbPriceUsd: null, + lastSymbPriceUsd: 1, + execAmount: movement.amount, + // NOTE: execPrice = firstSymbPriceUsd and should be set when converting currencies + execPrice: 0, + // NOTE: exactUsdValue can be null on the first launch, for warm-up it's filling from pub-trades + exactUsdValue: movement.exactUsdValue, + + isAirdropOnWallet: !!movement?._isAirdropOnWallet, + isMarginFundingPayment: !!movement?._isMarginFundingPayment, + isAffiliateRebate: !!movement?._isAffiliateRebate, + isStakingPayments: !!movement?._isStakingPayments + } + + remappedTrxs.push(remappedMovement) + + if ( + isTaxablePayment && + firstSymb === 'USD' + ) { + remappedMovement.firstSymbPriceUsd = 1 + remappedMovement.execPrice = 1 + + continue + } + if ( + Number.isFinite(movement.exactUsdValue) && + movement.exactUsdValue !== 0 + ) { + const price = new BigNumber(movement.exactUsdValue) + .div(movement.amount) + .abs() + .toNumber() + + remappedMovement.firstSymbPriceUsd = price + remappedMovement.execPrice = price + + continue + } + + remappedTrxsForConvToUsd.push(remappedMovement) + } + + return params +} diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/remap-trades.js b/workers/loc.api/sync/transaction.tax.report/helpers/remap-trades.js new file mode 100644 index 000000000..bf73a5486 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/remap-trades.js @@ -0,0 +1,69 @@ +'use strict' + +const BigNumber = require('bignumber.js') +const splitSymbolPairs = require( + 'bfx-report/workers/loc.api/helpers/split-symbol-pairs' +) + +module.exports = (trades, params) => { + const { + remappedTrxs, + remappedTrxsForConvToUsd + } = params + + for (const trade of trades) { + if ( + !trade?.symbol || + !Number.isFinite(trade?.execAmount) || + trade.execAmount === 0 || + !Number.isFinite(trade?.execPrice) || + trade.execPrice === 0 || + !Number.isFinite(trade?.mtsCreate) + ) { + continue + } + + const [firstSymb, lastSymb] = splitSymbolPairs(trade.symbol) + trade.firstSymb = firstSymb + trade.lastSymb = lastSymb + trade.firstSymbPriceUsd = null + trade.lastSymbPriceUsd = null + trade.isAdditionalTrxMovements = false + trade.isTaxablePayment = false + trade.isAirdropOnWallet = false + trade.isMarginFundingPayment = false + trade.isAffiliateRebate = false + trade.isStakingPayments = false + trade.isMovements = false + trade.isLedgers = false + trade.isTrades = true + + remappedTrxs.push(trade) + + if (lastSymb === 'USD') { + trade.firstSymbPriceUsd = trade.execPrice + trade.lastSymbPriceUsd = 1 + + continue + } + if ( + Number.isFinite(trade.exactUsdValue) && + trade.exactUsdValue !== 0 + ) { + trade.firstSymbPriceUsd = new BigNumber(trade.exactUsdValue) + .div(trade.execAmount) + .abs() + .toNumber() + trade.lastSymbPriceUsd = new BigNumber(trade.exactUsdValue) + .div(new BigNumber(trade.execAmount).times(trade.execPrice)) + .abs() + .toNumber() + + continue + } + + remappedTrxsForConvToUsd.push(trade) + } + + return params +} diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/trx.price.calculator.js b/workers/loc.api/sync/transaction.tax.report/helpers/trx.price.calculator.js new file mode 100644 index 000000000..77fb09375 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/trx.price.calculator.js @@ -0,0 +1,107 @@ +'use strict' + +const BigNumber = require('bignumber.js') + +const { PubTradePriceFindForTrxTaxError } = require('../../../errors') + +class TrxPriceCalculator { + static FIRST_SYMB_PRICE_PROP_NAME = 'firstSymbPriceUsd' + static LAST_SYMB_PRICE_PROP_NAME = 'lastSymbPriceUsd' + + static CRYPTO_CCY_FOR_TRIANGULATION = 'BTC' + static IS_CRYPTO_CCY_FOR_TRIANGULATION = 'IS_CRYPTO_CCY_FOR_TRIANGULATION' + static IS_FOREX_CCY_FOR_TRIANGULATION = 'IS_FOREX_CCY_FOR_TRIANGULATION' + + pubTradePrice = null + + constructor ( + trx, + convPricePropName, + calcPricePropName, + kindOfCcyForTriangulation, + triangulationCcyCalculator + ) { + this.trx = trx + this.convPricePropName = convPricePropName + this.calcPricePropName = calcPricePropName + this.kindOfCcyForTriangulation = kindOfCcyForTriangulation + this.triangulationCcyCalculator = triangulationCcyCalculator + } + + /* + * Example in case `this.kindOfCcyForTriangulation` exists: + * - eg. tEURUSD + * - if `this.kindOfCcyForTriangulation === IS_CRYPTO_CCY_FOR_TRIANGULATION`, `pubTradePrice` is price for BTC:USD, eg 21_000 + * - if `this.kindOfCcyForTriangulation === IS_FOREX_CCY_FOR_TRIANGULATION`,`pubTradePrice` is price for BTC:EUR, eg 20_000 + * - then price EUR:USD will be `20_000 / 21_000` + */ + calcPrice (pubTradePrice) { + if ( + !Number.isFinite(pubTradePrice) || + pubTradePrice === 0 + ) { + throw new PubTradePriceFindForTrxTaxError() + } + + this.pubTradePrice = pubTradePrice + + if (this.kindOfCcyForTriangulation === this.constructor.IS_CRYPTO_CCY_FOR_TRIANGULATION) { + return + } + if ( + this.kindOfCcyForTriangulation === this.constructor.IS_FOREX_CCY_FOR_TRIANGULATION && + ( + !(this.triangulationCcyCalculator instanceof TrxPriceCalculator) || + !Number.isFinite(this.triangulationCcyCalculator.pubTradePrice) || + this.triangulationCcyCalculator.pubTradePrice === 0 + ) + ) { + throw new PubTradePriceFindForTrxTaxError() + } + + this.trx[this.convPricePropName] = this.kindOfCcyForTriangulation === this.constructor.IS_FOREX_CCY_FOR_TRIANGULATION + ? new BigNumber(this.triangulationCcyCalculator.pubTradePrice) + .div(pubTradePrice) + .toNumber() + : pubTradePrice + + if (this.trx.isAdditionalTrxMovements) { + this.trx.execPrice = this.trx[this.convPricePropName] + } + if ( + !Number.isFinite(this.trx.execPrice) || + this.trx.execPrice === 0 + ) { + return + } + if (this.constructor.FIRST_SYMB_PRICE_PROP_NAME === this.calcPricePropName) { + const priceUsd = this.#calcFirstSymbPrice(this.trx[this.convPricePropName]) + this.trx[this.calcPricePropName] = priceUsd + this.trx.exactUsdValue = new BigNumber(this.trx.execAmount) + .times(priceUsd) + .toNumber() + + return + } + + const priceUsd = this.#calcLastSymbPrice(this.trx[this.convPricePropName]) + this.trx[this.calcPricePropName] = priceUsd + this.trx.exactUsdValue = new BigNumber(this.trx.execAmount) + .times(this.trx[this.convPricePropName]) + .toNumber() + } + + #calcFirstSymbPrice (pubTradePrice) { + return new BigNumber(pubTradePrice) + .times(this.trx.execPrice) + .toNumber() + } + + #calcLastSymbPrice (pubTradePrice) { + return new BigNumber(pubTradePrice) + .div(this.trx.execPrice) + .toNumber() + } +} + +module.exports = TrxPriceCalculator diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/trx.tax.strategies.js b/workers/loc.api/sync/transaction.tax.report/helpers/trx.tax.strategies.js new file mode 100644 index 000000000..087b2bbf7 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/trx.tax.strategies.js @@ -0,0 +1,8 @@ +'use strict' + +const TRX_TAX_STRATEGIES = { + FIFO: 'FIFO', + LIFO: 'LIFO' +} + +module.exports = TRX_TAX_STRATEGIES diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/trx.tax.types.js b/workers/loc.api/sync/transaction.tax.report/helpers/trx.tax.types.js new file mode 100644 index 000000000..3a95b1df6 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/trx.tax.types.js @@ -0,0 +1,12 @@ +'use strict' + +const TRX_TAX_TYPES = { + AIRDROP_ON_WALLET: 'AIRDROP_ON_WALLET', + MARGIN_FUNDING_PAYMENT: 'MARGIN_FUNDING_PAYMENT', + AFFILIATE_REBATE: 'AFFILIATE_REBATE', + STAKING_PAYMENT: 'STAKING_PAYMENT', + + EXCHANGE: 'EXCHANGE' +} + +module.exports = TRX_TAX_TYPES diff --git a/workers/loc.api/sync/transaction.tax.report/index.js b/workers/loc.api/sync/transaction.tax.report/index.js new file mode 100644 index 000000000..c0b662df4 --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/index.js @@ -0,0 +1,576 @@ +'use strict' + +const INTERRUPTER_NAMES = require( + 'bfx-report/workers/loc.api/interrupter/interrupter.names' +) + +const { pushLargeArr } = require('../../helpers/utils') +const { getBackIterable } = require('../helpers') +const { PubTradeFindForTrxTaxError } = require('../../errors') + +const { + TRX_TAX_STRATEGIES, + PROGRESS_STATES, + remapTrades, + remapMovements, + lookUpTrades, + getTrxMapByCcy, + findPublicTrade, + getCcyPairForConversion +} = require('./helpers') + +const isTestEnv = process.env.NODE_ENV === 'test' + +const { decorateInjectable } = require('../../di/utils') + +const depsTypes = (TYPES) => [ + TYPES.DAO, + TYPES.Authenticator, + TYPES.SyncSchema, + TYPES.ALLOWED_COLLS, + TYPES.SYNC_API_METHODS, + TYPES.Movements, + TYPES.RService, + TYPES.GetDataFromApi, + TYPES.WSEventEmitterFactory, + TYPES.Logger, + TYPES.InterrupterFactory, + TYPES.CurrencyConverter, + TYPES.ProcessMessageManager +] +class TransactionTaxReport { + constructor ( + dao, + authenticator, + syncSchema, + ALLOWED_COLLS, + SYNC_API_METHODS, + movements, + rService, + getDataFromApi, + wsEventEmitterFactory, + logger, + interrupterFactory, + currencyConverter, + processMessageManager + ) { + this.dao = dao + this.authenticator = authenticator + this.syncSchema = syncSchema + this.ALLOWED_COLLS = ALLOWED_COLLS + this.SYNC_API_METHODS = SYNC_API_METHODS + this.movements = movements + this.rService = rService + this.getDataFromApi = getDataFromApi + this.wsEventEmitterFactory = wsEventEmitterFactory + this.logger = logger + this.interrupterFactory = interrupterFactory + this.currencyConverter = currencyConverter + this.processMessageManager = processMessageManager + + this.tradesModel = this.syncSchema.getModelsMap() + .get(this.ALLOWED_COLLS.TRADES) + } + + async makeTrxTaxReportInBackground (args = {}) { + const { auth, params } = args ?? {} + const user = await this.authenticator + .verifyRequestUser({ auth }) + const _args = { auth: user, params } + + const trxTaxReportPromise = this.getTransactionTaxReport(_args) + + this.wsEventEmitterFactory() + .emitTrxTaxReportGenerationInBackgroundToOne(() => { + return trxTaxReportPromise + }, user) + .then(() => {}, (err) => { + this.logger.error(`TRX_TAX_REPORT_GEN_FAILED: ${err.stack || err}`) + }) + + trxTaxReportPromise.catch(() => { + this.processMessageManager.sendState( + this.processMessageManager.PROCESS_MESSAGES.ERROR_TRX_TAX_REPORT + ) + }) + + return true + } + + async getTransactionTaxReport (args = {}) { + const { auth, params } = args ?? {} + const start = params.start ?? 0 + const end = params.end ?? Date.now() + const strategy = params.strategy ?? TRX_TAX_STRATEGIES.LIFO + const user = await this.authenticator + .verifyRequestUser({ auth }) + const interrupter = this.interrupterFactory({ + user, + name: INTERRUPTER_NAMES.TRX_TAX_REPORT_INTERRUPTER + }) + await this.#emitProgress( + user, + { progress: 0, state: PROGRESS_STATES.GENERATION_STARTED } + ) + + const isFIFO = strategy === TRX_TAX_STRATEGIES.FIFO + const isLIFO = strategy === TRX_TAX_STRATEGIES.LIFO + + const { + trxs: trxsForCurrPeriod, + trxsForConvToUsd + } = await this.#getTrxs({ + user, + start, + end + }) + + if ( + !Array.isArray(trxsForCurrPeriod) || + trxsForCurrPeriod.length === 0 + ) { + interrupter.emitInterrupted() + await this.#emitProgress( + user, + { progress: 100, state: PROGRESS_STATES.GENERATION_COMPLETED } + ) + + return [] + } + + const { + trxs: trxsForPrevPeriod + } = start > 0 + ? await this.#getTrxs({ + user, + start: 0, + end: start - 1 + }) + : { trxs: [] } + + const isBackIterativeSaleLookUp = isFIFO && !isLIFO + const isBackIterativeBuyLookUp = isFIFO && !isLIFO + + const { buyTradesWithUnrealizedProfit } = await lookUpTrades( + trxsForPrevPeriod, + { + isBackIterativeSaleLookUp, + isBackIterativeBuyLookUp, + isBuyTradesWithUnrealizedProfitRequired: true, + isNotGainOrLossRequired: true, + interrupter + } + ) + + pushLargeArr(trxsForCurrPeriod, buyTradesWithUnrealizedProfit) + pushLargeArr( + trxsForConvToUsd, + buyTradesWithUnrealizedProfit + .filter((trx) => ( + !Number.isFinite(trx?.firstSymbPriceUsd) || + !Number.isFinite(trx?.lastSymbPriceUsd) + )) + ) + await this.#convertCurrencies( + trxsForConvToUsd, + { interrupter, user } + ) + + const { saleTradesWithRealizedProfit } = await lookUpTrades( + trxsForCurrPeriod, + { + isBackIterativeSaleLookUp, + isBackIterativeBuyLookUp, + interrupter + } + ) + + interrupter.emitInterrupted() + + if (interrupter.hasInterrupted()) { + await this.#emitProgress( + user, + { progress: null, state: PROGRESS_STATES.GENERATION_INTERRUPTED } + ) + + return [] + } + + await this.#emitProgress( + user, + { progress: 100, state: PROGRESS_STATES.GENERATION_COMPLETED } + ) + + return saleTradesWithRealizedProfit + } + + async #getTrxs (params) { + const { + user, + start, + end + } = params ?? {} + + const tradesPromise = this.#getTrades(params) + const withdrawalsPromise = this.movements.getMovements({ + auth: user, + start, + end, + isWithdrawals: true, + isExcludePrivate: false, + areExtraPaymentsIncluded: true + }) + const depositsPromise = this.movements.getMovements({ + auth: user, + start, + end, + isDeposits: true, + isExcludePrivate: false, + areExtraPaymentsIncluded: true + }) + + const [ + trades, + withdrawals, + deposits + ] = await Promise.all([ + tradesPromise, + withdrawalsPromise, + depositsPromise + ]) + + const movements = [...withdrawals, ...deposits] + const remappedTrxs = [] + const remappedTrxsForConvToUsd = [] + + remapTrades( + trades, + { remappedTrxs, remappedTrxsForConvToUsd } + ) + remapMovements( + movements, + { remappedTrxs, remappedTrxsForConvToUsd } + ) + + const trxs = remappedTrxs + .sort((a, b) => b?.mtsCreate - a?.mtsCreate) + const trxsForConvToUsd = remappedTrxsForConvToUsd + .sort((a, b) => b?.mtsCreate - a?.mtsCreate) + + return { + trxs, + trxsForConvToUsd + } + } + + async #convertCurrencies (trxs, opts) { + const { interrupter, user } = opts + const { + trxMapByCcy, + totalTrxAmount + } = getTrxMapByCcy(trxs) + let count = 0 + let progress = 0 + + for (const [symbol, trxPriceCalculators] of trxMapByCcy.entries()) { + if (interrupter.hasInterrupted()) { + return + } + + const trxPriceCalculatorIterator = getBackIterable(trxPriceCalculators) + + let pubTrades = [] + let pubTradeStart = pubTrades[0]?.mts + let pubTradeEnd = pubTrades[pubTrades.length - 1]?.mts + + for (const trxPriceCalculator of trxPriceCalculatorIterator) { + count += 1 + + if (interrupter.hasInterrupted()) { + return + } + + const { trx } = trxPriceCalculator + + if ( + pubTrades.length === 0 || + pubTradeStart > trx.mtsCreate || + pubTradeEnd < trx.mtsCreate + ) { + const start = trx.mtsCreate - 1 + + pubTrades = await this.#getPublicTrades( + { + symbol: getCcyPairForConversion(symbol, trxPriceCalculator), + start + }, + opts + ) + + if ( + !Array.isArray(pubTrades) || + pubTrades.length === 0 + ) { + const ccySynonymous = await this.currencyConverter + .getCurrenciesSynonymous() + const synonymous = ccySynonymous.get(symbol) + + if ( + !synonymous || + trxPriceCalculator.kindOfCcyForTriangulation + ) { + throw new PubTradeFindForTrxTaxError({ + symbol, + pubTradeStart, + pubTradeEnd, + requiredMts: trx.mtsCreate + }) + } + + for (const [symbol, conversion] of synonymous) { + const res = await this.#getPublicTrades( + { + symbol: getCcyPairForConversion(symbol, trxPriceCalculator), + start + }, + opts + ) + + if ( + !Array.isArray(res) || + res.length === 0 + ) { + continue + } + + pubTrades = res.map((item) => { + if (Number.isFinite(item?.price)) { + item.price = item.price * conversion + } + + return item + }) + + break + } + } + + pubTradeStart = start ?? pubTrades[0]?.mts + pubTradeEnd = pubTrades[pubTrades.length - 1]?.mts + } + + if ( + !Array.isArray(pubTrades) || + pubTrades.length === 0 || + !Number.isFinite(pubTradeStart) || + !Number.isFinite(pubTradeEnd) || + pubTradeStart > trx.mtsCreate || + pubTradeEnd < trx.mtsCreate + ) { + throw new PubTradeFindForTrxTaxError({ + symbol, + pubTradeStart, + pubTradeEnd, + requiredMts: trx.mtsCreate + }) + } + + const pubTrade = findPublicTrade(pubTrades, trx.mtsCreate) + trxPriceCalculator.calcPrice(pubTrade?.price) + const _progress = (count / totalTrxAmount) * 100 + + if ( + _progress <= 0 || + _progress >= 100 + ) { + continue + } + + progress = _progress + await this.#emitProgress( + user, + { progress, state: PROGRESS_STATES.OBTAINING_CURRENCY_PRICES } + ) + } + } + + await this.#updateExactUsdValueInColls(trxs) + await this.#emitProgress( + user, + { progress, state: PROGRESS_STATES.TRANSACTION_HISTORY_GENERATION } + ) + } + + async #getTrades ({ + user, + start, + end, + symbol + }) { + const symbFilter = ( + Array.isArray(symbol) && + symbol.length !== 0 + ) + ? { $in: { symbol } } + : {} + + return this.dao.getElemsInCollBy( + this.ALLOWED_COLLS.TRADES, + { + filter: { + user_id: user._id, + $eq: { + user_id: user._id, + _isExchange: 1 + }, + $lte: { mtsCreate: end }, + $gte: { mtsCreate: start }, + ...symbFilter + }, + sort: [['mtsCreate', -1]], + projection: this.tradesModel, + exclude: ['user_id'], + isExcludePrivate: false + } + ) + } + + async #getPublicTrades (params, opts) { + const { + symbol, + start = 0, + end = Date.now(), + sort = 1, + limit = 10000 + } = params ?? {} + const { interrupter } = opts ?? {} + const args = { + isNotMoreThanInnerMax: true, + params: { + symbol, + start, + end, + sort, + limit, + notCheckNextPage: true, + notThrowError: true + } + } + + const getDataFn = this.rService[this.SYNC_API_METHODS.PUBLIC_TRADES] + .bind(this.rService) + + const { res } = await this.getDataFromApi({ + getData: (s, args) => getDataFn(args), + args, + callerName: 'TRANSACTION_TAX_REPORT', + eNetErrorAttemptsTimeframeMin: 10, + eNetErrorAttemptsTimeoutMs: 10000, + interrupter + }) + + const pubTrades = Array.isArray(res) + ? res + : [] + + if (isTestEnv) { + /* + * Need to reverse pub-trades array for test env + * as mocked test server return data in desc order + */ + return pubTrades.reverse() + } + + return pubTrades + } + + async #updateExactUsdValueInColls (trxs) { + let trades = [] + let movements = [] + let ledgers = [] + + for (const [i, trx] of trxs.entries()) { + const isLast = (i + 1) === trxs.length + + if (trx.isTrades) { + trades.push(trx) + } + if (trx.isMovements) { + movements.push(trx) + } + if (trx.isLedgers) { + ledgers.push(trx) + } + + if ( + trades.length >= 20_000 || + isLast + ) { + await this.dao.updateElemsInCollBy( + this.ALLOWED_COLLS.TRADES, + trades, + ['_id'], + ['exactUsdValue'] + ) + + trades = [] + } + if ( + movements.length >= 20_000 || + isLast + ) { + await this.dao.updateElemsInCollBy( + this.ALLOWED_COLLS.MOVEMENTS, + movements, + ['_id'], + ['exactUsdValue'] + ) + + movements = [] + } + if ( + ledgers.length >= 20_000 || + isLast + ) { + await this.dao.updateElemsInCollBy( + this.ALLOWED_COLLS.LEDGERS, + ledgers, + ['_id'], + ['exactUsdValue'] + ) + + ledgers = [] + } + } + } + + async #emitProgress (user, params) { + const { + progress = null, + state = null + } = params ?? {} + + await this.wsEventEmitterFactory() + .emitTrxTaxReportGenerationProgressToOne( + { + progress: Number.isFinite(progress) + ? Math.floor(progress) + : progress, + state + }, + user + ) + + if (state !== PROGRESS_STATES.GENERATION_COMPLETED) { + return + } + + this.processMessageManager.sendState( + this.processMessageManager.PROCESS_MESSAGES.READY_TRX_TAX_REPORT + ) + } +} + +decorateInjectable(TransactionTaxReport, depsTypes) + +module.exports = TransactionTaxReport diff --git a/workers/loc.api/ws-transport/ws.event.emitter.js b/workers/loc.api/ws-transport/ws.event.emitter.js index c1ea2481a..ff4ab8213 100644 --- a/workers/loc.api/ws-transport/ws.event.emitter.js +++ b/workers/loc.api/ws-transport/ws.event.emitter.js @@ -138,6 +138,36 @@ class WSEventEmitter extends AbstractWSEventEmitter { }, 'emitBfxUnamePwdAuthRequired') } + emitTrxTaxReportGenerationInBackgroundToOne ( + handler = () => {}, + auth = {} + ) { + return this.emit(async (user, ...args) => { + if (this.isNotTargetUser(auth, user)) { + return { isNotEmitted: true } + } + + return typeof handler === 'function' + ? await handler(user, ...args) + : handler + }, 'emitTrxTaxReportGenerationInBackgroundToOne') + } + + emitTrxTaxReportGenerationProgressToOne ( + handler = () => {}, + auth = {} + ) { + return this.emit(async (user, ...args) => { + if (this.isNotTargetUser(auth, user)) { + return { isNotEmitted: true } + } + + return typeof handler === 'function' + ? await handler(user, ...args) + : handler + }, 'emitTrxTaxReportGenerationProgressToOne') + } + async emitRedirectingRequestsStatusToApi ( handler = () => {} ) {