diff --git a/crawler/index.ts b/crawler/index.ts index 240cb7d..23c8d2f 100644 --- a/crawler/index.ts +++ b/crawler/index.ts @@ -356,7 +356,7 @@ program THEN SUBSTR(ud.asset, 1, INSTR(ud.asset, '_') - 1) ELSE ud.asset END AS xasset_id, - CAST(SUM(p.price * ud.balance * ${tsKf}) AS INTEGER) points + FLOOR(SUM(p.price * ud.balance * ${tsKf})) points FROM user_data ud LEFT JOIN @@ -452,7 +452,7 @@ program INSERT OR IGNORE INTO user_points_public (address, asset_id, points, "change", prev_points_l1, prev_points_l2, points_l1, points_l2, place, prev_place) SELECT - r.referrer address, + COALESCE(als.address, r.referrer) address, CASE WHEN INSTR(s.asset_id, '_') > 0 THEN SUBSTR(s.asset_id, 1, INSTR(s.asset_id, '_') - 1) @@ -460,12 +460,13 @@ program 0 points, 0 change, 0 prev_points_l1, - 0 prev_points_l2, - 0 points_l1, - 0 points_l2, - 0 place, - 0 prev_place + 0 prev_points_l2, + 0 points_l1, + 0 points_l2, + 0 place, + 0 prev_place FROM referrals r + LEFT JOIN aliases als ON (als.alias = r.referrer) LEFT JOIN schedule s GROUP BY address; `, @@ -481,25 +482,27 @@ program prev_points_l2 = points_l2, points_l1 = COALESCE(points_l1,0) + COALESCE(( SELECT - CAST(SUM(upp1.change) * ${config.l1_percent / 100} AS INTEGER) + FLOOR(SUM(upp1.change) * ${config.l1_percent / 100}) FROM referrals r + LEFT JOIN aliases als ON (als.alias = r.referrer) LEFT JOIN user_points_public upp1 ON (upp1.address = r.referral AND r.ts <= $ts) - LEFT JOIN user_kyc k ON (k.address = r.referrer AND k.ts <= $ts) + LEFT JOIN user_kyc k ON (k.address = COALESCE(als.address, r.referrer) AND k.ts <= $ts) WHERE - r.referrer = user_points_public.address AND + COALESCE(als.address, r.referrer) = user_points_public.address AND k.address IS NOT NULL ),0), points_l2 = COALESCE(points_l2,0) + COALESCE(( SELECT - CAST(SUM(upp2.change) * ${config.l2_percent / 100} AS INTEGER) + FLOOR(SUM(upp2.change) * ${config.l2_percent / 100}) FROM referrals r2 LEFT JOIN referrals r3 ON (r3.referrer = r2.referral AND r3.ts <= $ts) + LEFT JOIN aliases als ON (als.alias = r2.referrer) LEFT JOIN user_points_public upp2 ON (upp2.address = r3.referral AND r3.ts <= $ts) - LEFT JOIN user_kyc k2 ON (k2.address = r2.referrer AND k2.ts <= $ts) + LEFT JOIN user_kyc k2 ON (k2.address = COALESCE(als.address, r2.referrer) AND k2.ts <= $ts) WHERE - r2.referrer = user_points_public.address AND + COALESCE(als.address, r2.referrer) = user_points_public.address AND k2.address IS NOT NULL ),0) `, @@ -520,25 +523,27 @@ program prev_points_l2 = points_l2, points_l1 = COALESCE(points_l1,0) + COALESCE(( SELECT - CAST(SUM(upp1.change) * ${config.l1_percent / 100} AS INTEGER) + FLOOR(SUM(upp1.change) * ${config.l1_percent / 100}) FROM referrals r + LEFT JOIN aliases als ON (als.alias = r.referrer) LEFT JOIN user_points_public upp1 ON (upp1.address = r.referral AND r.height <= ${batchHeight}) - LEFT JOIN user_kyc k ON (k.address = r.referrer AND k.ts <= $ts) + LEFT JOIN user_kyc k ON (k.address = COALESCE(als.address, r.referrer) AND k.ts <= $ts) WHERE - r.referrer = user_points_public.address AND + COALESCE(als.address, r.referrer) = user_points_public.address AND k.address IS NOT NULL ),0), points_l2 = COALESCE(points_l2,0) + COALESCE(( SELECT - CAST(SUM(upp2.change) * ${config.l2_percent / 100} AS INTEGER) + FLOOR(SUM(upp2.change) * ${config.l2_percent / 100}) FROM referrals r2 LEFT JOIN referrals r3 ON (r3.referrer = r2.referral AND r3.height <= ${batchHeight}) + LEFT JOIN aliases als ON (als.alias = r2.referrer) LEFT JOIN user_points_public upp2 ON (upp2.address = r3.referral AND r3.height <= ${batchHeight}) - LEFT JOIN user_kyc k2 ON (k2.address = r2.referrer AND k2.ts <= $ts) + LEFT JOIN user_kyc k2 ON (k2.address = COALESCE(als.address, r2.referrer) AND k2.ts <= $ts) WHERE - r2.referrer = user_points_public.address AND + COALESCE(als.address, r2.referrer) = user_points_public.address AND k2.address IS NOT NULL ),0) `, diff --git a/lib/sources/astroport/index.ts b/lib/sources/astroport/index.ts index 2b1bd04..ffe7a93 100644 --- a/lib/sources/astroport/index.ts +++ b/lib/sources/astroport/index.ts @@ -43,10 +43,7 @@ export default class AstroportSource implements SourceInterface { } this.assets = params.assets; - if (!params.recalculate) { - throw new Error('No recalculation flag configured'); - } - this.recalculate = params.recalculate; + this.recalculate = params.recalculate || false; this.rpc = rpc; this.concurrencyLimit = parseInt(params.concurrency_limit || '3', 10); diff --git a/lib/sources/mars/index.ts b/lib/sources/mars/index.ts index 7eb4af5..c814421 100644 --- a/lib/sources/mars/index.ts +++ b/lib/sources/mars/index.ts @@ -40,10 +40,7 @@ export default class MarsSource implements SourceInterface { } this.assets = params.assets; - if (!params.recalculate) { - throw new Error('No recalculation flag configured'); - } - this.recalculate = params.recalculate; + this.recalculate = params.recalculate || false; if (!params.nft_contract) { throw new Error('No mars nft contract configured in params'); diff --git a/recalculate.ts b/recalculate.ts index f1d30d4..77c8685 100644 --- a/recalculate.ts +++ b/recalculate.ts @@ -3,19 +3,24 @@ import * as path from 'path'; import { execSync } from 'child_process'; import { constants, Database } from 'bun:sqlite'; -const OLD_DATABASE = 'recalculate/36-data.db'; +const OLD_DATABASE = 'recalculate/old_data.db'; const NEW_DATABASE = 'recalculate/new_data.db'; const CHANGES_FILE = 'recalculate/changes.csv'; +const ALIASES_FILE = 'recalculate/aliases.csv'; -async function parseCsv(filePath: string): Promise { +async function parseCsv( + filePath: string, + separator: string, + numberOfColumns: number, +): Promise { const data = await fsPromises.readFile(filePath, 'utf8'); const lines = data.trim().split('\n'); const [headerLine, ...dataLines] = lines; - const headers = headerLine.split(','); + const headers = headerLine.split(separator); - if (headers.length !== 5) { + if (headers.length !== numberOfColumns) { throw new Error('CSV file does not have correct number of columns'); } @@ -59,26 +64,43 @@ async function main() { >('SELECT * FROM batches'); const batches = batchesQuery.all(null); - newDb.exec( - `CREATE TABLE IF NOT EXISTS changes (address TEXT, batch_id TEXT, points INTEGER, points_l1 INTEGER, points_l2 INTEGER, PRIMARY KEY(address, batch_id DESC));`, - ); + const createChangesQuery = `CREATE TABLE IF NOT EXISTS changes (address TEXT, batch_id TEXT, points INTEGER, points_l1 INTEGER, points_l2 INTEGER, PRIMARY KEY(address, batch_id DESC));`; + oldDb.exec(createChangesQuery); + newDb.exec(createChangesQuery); + + const clearChangesQuery = `DELETE FROM changes;`; + oldDb.exec(clearChangesQuery); + newDb.exec(clearChangesQuery); - const changesLines = await parseCsv(CHANGES_FILE); + const changesLines = await parseCsv(CHANGES_FILE, ',', 5); changesLines.forEach((line) => { const [address, batchId, points, points_l1, points_l2] = line.split(','); - newDb.exec(` + const insertChangeQuery = ` INSERT INTO changes (address, batch_id, points, points_l1, points_l2) VALUES ('${address}', '${batchId}', ${points}, ${points_l1}, ${points_l2}) ON CONFLICT (address, batch_id) DO UPDATE SET points = changes.points + excluded.points, points_l1 = changes.points_l1 + excluded.points_l1, points_l2 = changes.points_l2 + excluded.points_l2 - `); + `; + oldDb.exec(insertChangeQuery); + newDb.exec(insertChangeQuery); }); - newDb.exec( - `CREATE TABLE IF NOT EXISTS batch_heights (batch_id INTEGER, height INTEGER, ts INTEGER);`, - ); + const createAliasesQuery = `CREATE TABLE IF NOT EXISTS aliases (alias TEXT, address TEXT, code TEXT, PRIMARY KEY(alias));`; + oldDb.exec(createAliasesQuery); + newDb.exec(createAliasesQuery); + + const aliasesLines = await parseCsv(ALIASES_FILE, ';', 3); + aliasesLines.forEach((line) => { + const [alias, address, code] = line.split(';'); + const insertAliasQuery = ` + INSERT OR IGNORE INTO aliases (alias, address, code) + VALUES ('${alias}', '${address}', '${code}') + `; + oldDb.exec(insertAliasQuery); + newDb.exec(insertAliasQuery); + }); const thirdBatchTsQuery = oldDb.query<{ batch_3_ts: number }, null>( `SELECT ts AS batch_3_ts FROM batches WHERE batch_id = 3`, @@ -365,8 +387,6 @@ async function main() { if (checkLevelTwo) { console.log(`${checkLevelTwo.row_count} addresses are broken with l2`); } - - // if (batch.batch_id == 4) break; } console.log(checkLevelOneQueryText);