diff --git a/bun.lockb b/bun.lockb index b25e9fd..55218c3 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/crawler/index.ts b/crawler/index.ts index ea737b6..240cb7d 100644 --- a/crawler/index.ts +++ b/crawler/index.ts @@ -356,7 +356,7 @@ program THEN SUBSTR(ud.asset, 1, INSTR(ud.asset, '_') - 1) ELSE ud.asset END AS xasset_id, - FLOOR(SUM(p.price * ud.balance * ${tsKf})) points + CAST(SUM(p.price * ud.balance * ${tsKf}) AS INTEGER) points FROM user_data ud LEFT JOIN @@ -380,7 +380,9 @@ program points: number; }, [number] - >('SELECT * FROM changes WHERE batch_id = ?'); + >( + 'SELECT address, batch_id, points FROM changes WHERE batch_id = ? AND points <> 0', + ); const batchChanges = batchChangesQuery.all(batchId); if (batchChanges) { batchChanges.forEach((change) => { @@ -392,7 +394,7 @@ program `); }); logger.debug( - `${batchChanges.length} addresses were updated manually while recalculating`, + `${batchChanges.length} addresses user points were updated manually while recalculating`, ); } } @@ -413,14 +415,13 @@ program logger.debug('Batch IDs: %s', batchIds.join(',')); let firstTs = all[0].ts; - if (options.recalculate) { - const lastBatchId = batchIds.at(-1); - if ( - lastBatchId && - lastBatchId <= RECALCULATE.LAST_NOT_MARKED_AS_PROCESSED_BATCH - ) { - firstTs = all[lastBatchId - 1].ts; - } + const lastBatchId = batchIds.at(-1); + if ( + options.recalculate && + lastBatchId && + lastBatchId <= RECALCULATE.LAST_NOT_MARKED_AS_PROCESSED_BATCH + ) { + firstTs = all[lastBatchId - 1].ts; logger.debug('Last batch id is %s', lastBatchId); logger.debug('First ts is %s', firstTs); } @@ -471,7 +472,7 @@ program ); // calc L1, L2 points - const stmt = db.prepare( + let stmt = db.prepare( ` UPDATE user_points_public @@ -480,7 +481,7 @@ program prev_points_l2 = points_l2, points_l1 = COALESCE(points_l1,0) + COALESCE(( SELECT - FLOOR(SUM(upp1.change) * ${config.l1_percent / 100}) + CAST(SUM(upp1.change) * ${config.l1_percent / 100} AS INTEGER) FROM referrals r LEFT JOIN user_points_public upp1 ON (upp1.address = r.referral AND r.ts <= $ts) @@ -491,7 +492,7 @@ program ),0), points_l2 = COALESCE(points_l2,0) + COALESCE(( SELECT - FLOOR(SUM(upp2.change) * ${config.l2_percent / 100}) + CAST(SUM(upp2.change) * ${config.l2_percent / 100} AS INTEGER) FROM referrals r2 LEFT JOIN referrals r3 ON (r3.referrer = r2.referral AND r3.ts <= $ts) @@ -503,6 +504,47 @@ program ),0) `, ); + + if (options.recalculate) { + const query = db.query<{ height: number }, number>( + `SELECT height FROM tasks WHERE ts = $ts LIMIT 1`, + ); + const batchHeight = query.all(firstTs)[0].height; + + stmt = db.prepare( + ` + UPDATE + user_points_public + SET + prev_points_l1 = points_l1, + prev_points_l2 = points_l2, + points_l1 = COALESCE(points_l1,0) + COALESCE(( + SELECT + CAST(SUM(upp1.change) * ${config.l1_percent / 100} AS INTEGER) + FROM + referrals r + LEFT JOIN user_points_public upp1 ON (upp1.address = r.referral AND r.height <= ${batchHeight}) + LEFT JOIN user_kyc k ON (k.address = r.referrer AND k.ts <= $ts) + WHERE + r.referrer = user_points_public.address AND + k.address IS NOT NULL + ),0), + points_l2 = COALESCE(points_l2,0) + COALESCE(( + SELECT + CAST(SUM(upp2.change) * ${config.l2_percent / 100} AS INTEGER) + FROM + referrals r2 + LEFT JOIN referrals r3 ON (r3.referrer = r2.referral AND r3.height <= ${batchHeight}) + LEFT JOIN user_points_public upp2 ON (upp2.address = r3.referral AND r3.height <= ${batchHeight}) + LEFT JOIN user_kyc k2 ON (k2.address = r2.referrer AND k2.ts <= $ts) + WHERE + r2.referrer = user_points_public.address AND + k2.address IS NOT NULL + ),0) + `, + ); + } + stmt.run({ $ts: firstTs }); db.exec( @@ -542,6 +584,37 @@ program db.exec( `UPDATE batches SET status="processed" WHERE batch_id IN (${processedBatchesIds})`, ); + + if (options.recalculate) { + const batchChangesQuery = db.query< + { + address: string; + batch_id: number; + points_l1: number; + points_l2: number; + }, + [number] + >( + 'SELECT address, batch_id, points_l1, points_l2 FROM changes WHERE batch_id = ? AND (points_l1 <> 0 OR points_l2 <> 0)', + ); + const batchChanges = batchChangesQuery.all(batchId); + if (batchChanges) { + batchChanges.forEach((change) => { + db.exec(` + UPDATE + user_points_public + SET + points_l1 = COALESCE(points_l1,0) + COALESCE(${change.points_l1}, 0), + points_l2 = COALESCE(points_l2,0) + COALESCE(${change.points_l2}, 0) + WHERE + address = '${change.address}' + `); + }); + logger.debug( + `${batchChanges.length} addresses referral points were updated manually while recalculating`, + ); + } + } } }); diff --git a/lib/sources/mars/index.ts b/lib/sources/mars/index.ts index ac97b2f..7eb4af5 100644 --- a/lib/sources/mars/index.ts +++ b/lib/sources/mars/index.ts @@ -5,7 +5,7 @@ import pLimit from 'p-limit'; import { SourceInterface } from '../../../types/sources/source'; import { MarsPositionResponse } from '../../../types/sources/marsPositionResponse'; import { Logger } from 'pino'; -import { RECALCULATE } from "../../../constants"; +import { RECALCULATE } from '../../../constants'; export default class MarsSource implements SourceInterface { rpc: string; diff --git a/package.json b/package.json index c66e305..a0f9251 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "cosmjs-types": "^0.9.0", "dotenv": "^16.4.5", "express": "^4.19.2", + "graphql": "^16.9.0", "graphql-request": "^7.0.1", "guid-typescript": "^1.0.9", "lodash": "^4.17.21", @@ -50,4 +51,4 @@ "typescript-eslint": "^7.9.0" }, "type": "module" -} \ No newline at end of file +} diff --git a/recalculate.ts b/recalculate.ts index 6bfc9d0..f1d30d4 100644 --- a/recalculate.ts +++ b/recalculate.ts @@ -3,7 +3,7 @@ import * as path from 'path'; import { execSync } from 'child_process'; import { constants, Database } from 'bun:sqlite'; -const OLD_DATABASE = 'recalculate/old_data.db'; +const OLD_DATABASE = 'recalculate/36-data.db'; const NEW_DATABASE = 'recalculate/new_data.db'; const CHANGES_FILE = 'recalculate/changes.csv'; @@ -15,7 +15,7 @@ async function parseCsv(filePath: string): Promise { const [headerLine, ...dataLines] = lines; const headers = headerLine.split(','); - if (headers.length !== 3) { + if (headers.length !== 5) { throw new Error('CSV file does not have correct number of columns'); } @@ -60,20 +60,223 @@ async function main() { const batches = batchesQuery.all(null); newDb.exec( - `CREATE TABLE IF NOT EXISTS changes (address TEXT, batch_id TEXT, points INTEGER, PRIMARY KEY(address, batch_id DESC));`, + `CREATE TABLE IF NOT EXISTS changes (address TEXT, batch_id TEXT, points INTEGER, points_l1 INTEGER, points_l2 INTEGER, PRIMARY KEY(address, batch_id DESC));`, ); const changesLines = await parseCsv(CHANGES_FILE); changesLines.forEach((line) => { - const [address, batchId, points] = line.split(','); + const [address, batchId, points, points_l1, points_l2] = line.split(','); newDb.exec(` - INSERT INTO changes (address, batch_id, points) - VALUES ('${address}', '${batchId}', ${points}) + INSERT INTO changes (address, batch_id, points, points_l1, points_l2) + VALUES ('${address}', '${batchId}', ${points}, ${points_l1}, ${points_l2}) ON CONFLICT (address, batch_id) DO UPDATE SET - points = changes.points + excluded.points + points = changes.points + excluded.points, + points_l1 = changes.points_l1 + excluded.points_l1, + points_l2 = changes.points_l2 + excluded.points_l2 `); }); + newDb.exec( + `CREATE TABLE IF NOT EXISTS batch_heights (batch_id INTEGER, height INTEGER, ts INTEGER);`, + ); + + const thirdBatchTsQuery = oldDb.query<{ batch_3_ts: number }, null>( + `SELECT ts AS batch_3_ts FROM batches WHERE batch_id = 3`, + ); + const thirdBatchTs = thirdBatchTsQuery.all(null)[0].batch_3_ts; + + const thirdBatchHeightQuery = oldDb.query<{ height: number }, null>( + `SELECT height FROM tasks WHERE batch_id = 3 LIMIT 1`, + ); + const thirdBatchHeight = thirdBatchHeightQuery.all(null)[0].height; + + const checkLevelOneQueryText = ` + WITH batch_heights AS ( + SELECT DISTINCT + t.batch_id AS batch_id, + t.height AS height, + b.ts AS ts + FROM + tasks t + JOIN + batches b + ON + t.batch_id = b.batch_id + ), + + ReferrerPoints AS ( + SELECT + r.referrer AS address, + SUM(CASE + WHEN up.batch_id IN (1, 2) AND r.height <= ${thirdBatchHeight} THEN up.points + ELSE 0 + END) AS points_from_earlier_batches, + SUM(CASE + WHEN k.ts <= bh.ts AND r.height <= bh.height THEN up.points + ELSE 0 + END) AS total_points + FROM + referrals r + JOIN + user_points up + ON + r.referral = up.address + JOIN + batch_heights bh + ON + up.batch_id = bh.batch_id + LEFT JOIN user_kyc k + ON k.address = r.referrer + GROUP BY + r.referrer + ), + + AdjustedPoints AS ( + SELECT + rp.address, + CASE + WHEN kyc.ts < ${thirdBatchTs} THEN rp.points_from_earlier_batches + rp.total_points + ELSE rp.total_points + END AS adjusted_points + FROM + ReferrerPoints rp + LEFT JOIN + user_kyc kyc + ON + rp.address = kyc.address + ), + + ReferralPointsPublic AS ( + SELECT + upp.address AS public_address, + upp.points_l1 + FROM + user_points_public upp + ), + + Results AS ( + SELECT + ap.address, + (ap.adjusted_points * 0.25) / 1000000 AS referral_points_25_percent, + CAST(rpp.points_l1 AS REAL) / 1000000 AS points_l1, + ABS(((ap.adjusted_points * 0.25) / 1000000) - (CAST(rpp.points_l1 AS REAL) / 1000000)) AS diff + FROM + AdjustedPoints ap + LEFT JOIN + ReferralPointsPublic rpp + ON + ap.address = rpp.public_address + ) + + SELECT + count(*) as row_count + FROM + Results + WHERE + diff >= 1; + `; + + const checkLevelTwoQueryText = ` + WITH batch_heights AS ( + SELECT DISTINCT + t.batch_id AS batch_id, + t.height AS height, + b.ts AS ts + FROM + tasks t + JOIN + batches b + ON + t.batch_id = b.batch_id + ), + + LevelTwoReferrals AS ( + SELECT + r1.referrer AS referrer, + r2.referral AS l2_referral, + r2.height AS height + FROM + referrals r1 + JOIN + referrals r2 + ON + r1.referral = r2.referrer + ), + + LevelTwoReferrerPoints AS ( + SELECT + l2r.referrer AS referrer, + SUM(CASE + WHEN up.batch_id IN (1, 2) AND l2r.height <= ${thirdBatchHeight} THEN up.points + ELSE 0 + END) AS points_from_earlier_batches, + SUM(CASE + WHEN k.ts <= bh.ts AND l2r.height <= bh.height THEN up.points + ELSE 0 + END) AS total_points + FROM + LevelTwoReferrals l2r + JOIN + user_points up + ON + l2r.l2_referral = up.address + JOIN + batch_heights bh + ON + up.batch_id = bh.batch_id + LEFT JOIN + user_kyc k + ON + k.address = l2r.referrer + GROUP BY + l2r.referrer + ), + + AdjustedPointsLevelTwo AS ( + SELECT + rp.referrer as address, + CASE + WHEN kyc.ts < ${thirdBatchTs} THEN rp.points_from_earlier_batches + rp.total_points + ELSE rp.total_points + END AS adjusted_points + FROM + LevelTwoReferrerPoints rp + LEFT JOIN + user_kyc kyc + ON + rp.referrer = kyc.address + ), + + ReferralPointsPublic AS ( + SELECT + upp.address AS public_address, + upp.points_l2 + FROM + user_points_public upp + ), + + Results AS ( + SELECT + ap.address, + (ap.adjusted_points * 0.125) / 1000000 AS referral_points_12_5_percent, + CAST(rpp.points_l2 AS REAL) / 1000000 AS points_l2, + ABS(((ap.adjusted_points * 0.125) / 1000000) - (CAST(rpp.points_l2 AS REAL) / 1000000)) AS diff + FROM + AdjustedPointsLevelTwo ap + LEFT JOIN + ReferralPointsPublic rpp + ON + ap.address = rpp.public_address + ) + + SELECT + count(*) as row_count + FROM + Results + WHERE + diff >= 1; + `; + for (const batch of batches) { console.log(`Processing batch ${batch.batch_id}...`); @@ -147,108 +350,27 @@ async function main() { console.error(`Error executing finish command: ${error}`); } - const checkLevelOneQuery = newDb.query< - { - row_count: number; - }, - [null] - >(` - WITH batch_3_ts AS ( - SELECT ts - FROM batches - WHERE batch_id = 3 - ), - level_one_referral_points AS ( - SELECT - r.referrer AS referrer, - SUM( - CASE - WHEN up.batch_id IN (1, 2) AND kyc.ts < (SELECT ts FROM batch_3_ts) THEN up.points - ELSE up.points - END - ) * 0.25 AS referral_points_total - FROM - referrals r - LEFT JOIN - user_points up ON r.referral = up.address - LEFT JOIN - batches b ON up.batch_id = b.batch_id - LEFT JOIN - user_kyc kyc ON r.referrer = kyc.address - WHERE - (b.ts >= kyc.ts AND up.batch_id NOT IN (1, 2)) - OR - (up.batch_id IN (1, 2) AND kyc.ts < (SELECT ts FROM batch_3_ts)) - GROUP BY - r.referrer - ) - SELECT - COUNT(*) AS row_count - FROM - level_one_referral_points l1rp - LEFT JOIN - user_points_public upp ON l1rp.referrer = upp.address - WHERE - ABS((l1rp.referral_points_total / 1000000.0) - (COALESCE(upp.points_l1, 0) / 1000000.0)) >= 1; - `); + const checkLevelOneQuery = newDb.query<{ row_count: number }, [null]>( + checkLevelOneQueryText, + ); const checkLevelOne = checkLevelOneQuery.get(null); if (checkLevelOne) { console.log(`${checkLevelOne.row_count} addresses are broken with l1`); } - const checkLevelTwoQuery = newDb.query< - { - row_count: number; - }, - [null] - >(` - WITH batch_3_ts AS ( - SELECT ts - FROM batches - WHERE batch_id = 3 - ), - level_two_referral_points AS ( - SELECT - r1.referrer AS level_two_referrer, - SUM( - CASE - WHEN up.batch_id IN (1, 2) AND kyc.ts < (SELECT ts FROM batch_3_ts) THEN up.points - ELSE up.points - END - ) * 0.125 AS referral_points_total_l2 - FROM - referrals r1 - JOIN - referrals r2 ON r1.referral = r2.referrer -- Join to get the referrals of referrals (level two) - LEFT JOIN - user_points up ON r2.referral = up.address - LEFT JOIN - batches b ON up.batch_id = b.batch_id - LEFT JOIN - user_kyc kyc ON r1.referrer = kyc.address - WHERE - (b.ts >= kyc.ts AND up.batch_id NOT IN (1, 2)) - OR - (up.batch_id IN (1, 2) AND kyc.ts < (SELECT ts FROM batch_3_ts)) - GROUP BY - r1.referrer - ) - SELECT - COUNT(*) AS row_count - FROM - level_two_referral_points l2rp - LEFT JOIN - user_points_public upp ON l2rp.level_two_referrer = upp.address - WHERE - ABS((l2rp.referral_points_total_l2 / 1000000.0) - (COALESCE(upp.points_l2, 0) / 1000000.0)) >= 1; - `); + const checkLevelTwoQuery = newDb.query<{ row_count: number }, [null]>( + checkLevelTwoQueryText, + ); const checkLevelTwo = checkLevelTwoQuery.get(null); if (checkLevelTwo) { console.log(`${checkLevelTwo.row_count} addresses are broken with l2`); } - // if (batch.batch_id === 3) break; + // if (batch.batch_id == 4) break; } + + console.log(checkLevelOneQueryText); + console.log(checkLevelTwoQueryText); } main();