Skip to content

Commit

Permalink
And more fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@deniszagumennov
deniszagumennov committed Sep 25, 2024
1 parent cdbc02c commit ad87796
Show file tree
Hide file tree
Showing 4 changed files with 61 additions and 42 deletions.
43 changes: 24 additions & 19 deletions crawler/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ program
THEN SUBSTR(ud.asset, 1, INSTR(ud.asset, '_') - 1)
ELSE ud.asset
END AS xasset_id,
CAST(SUM(p.price * ud.balance * ${tsKf}) AS INTEGER) points
FLOOR(SUM(p.price * ud.balance * ${tsKf})) points
FROM
user_data ud
LEFT JOIN
Expand Down Expand Up @@ -452,20 +452,21 @@ program
INSERT OR IGNORE INTO user_points_public
(address, asset_id, points, "change", prev_points_l1, prev_points_l2, points_l1, points_l2, place, prev_place)
SELECT
r.referrer address,
COALESCE(als.address, r.referrer) address,
CASE
WHEN INSTR(s.asset_id, '_') > 0
THEN SUBSTR(s.asset_id, 1, INSTR(s.asset_id, '_') - 1)
ELSE s.asset_id END AS asset_id,
0 points,
0 change,
0 prev_points_l1,
0 prev_points_l2,
0 points_l1,
0 points_l2,
0 place,
0 prev_place
0 prev_points_l2,
0 points_l1,
0 points_l2,
0 place,
0 prev_place
FROM referrals r
LEFT JOIN aliases als ON (als.alias = r.referrer)
LEFT JOIN schedule s
GROUP BY address;
`,
Expand All @@ -481,25 +482,27 @@ program
prev_points_l2 = points_l2,
points_l1 = COALESCE(points_l1,0) + COALESCE((
SELECT
CAST(SUM(upp1.change) * ${config.l1_percent / 100} AS INTEGER)
FLOOR(SUM(upp1.change) * ${config.l1_percent / 100})
FROM
referrals r
LEFT JOIN aliases als ON (als.alias = r.referrer)
LEFT JOIN user_points_public upp1 ON (upp1.address = r.referral AND r.ts <= $ts)
LEFT JOIN user_kyc k ON (k.address = r.referrer AND k.ts <= $ts)
LEFT JOIN user_kyc k ON (k.address = COALESCE(als.address, r.referrer) AND k.ts <= $ts)
WHERE
r.referrer = user_points_public.address AND
COALESCE(als.address, r.referrer) = user_points_public.address AND
k.address IS NOT NULL
),0),
points_l2 = COALESCE(points_l2,0) + COALESCE((
SELECT
CAST(SUM(upp2.change) * ${config.l2_percent / 100} AS INTEGER)
FLOOR(SUM(upp2.change) * ${config.l2_percent / 100})
FROM
referrals r2
LEFT JOIN referrals r3 ON (r3.referrer = r2.referral AND r3.ts <= $ts)
LEFT JOIN aliases als ON (als.alias = r2.referrer)
LEFT JOIN user_points_public upp2 ON (upp2.address = r3.referral AND r3.ts <= $ts)
LEFT JOIN user_kyc k2 ON (k2.address = r2.referrer AND k2.ts <= $ts)
LEFT JOIN user_kyc k2 ON (k2.address = COALESCE(als.address, r2.referrer) AND k2.ts <= $ts)
WHERE
r2.referrer = user_points_public.address AND
COALESCE(als.address, r2.referrer) = user_points_public.address AND
k2.address IS NOT NULL
),0)
`,
Expand All @@ -520,25 +523,27 @@ program
prev_points_l2 = points_l2,
points_l1 = COALESCE(points_l1,0) + COALESCE((
SELECT
CAST(SUM(upp1.change) * ${config.l1_percent / 100} AS INTEGER)
FLOOR(SUM(upp1.change) * ${config.l1_percent / 100})
FROM
referrals r
LEFT JOIN aliases als ON (als.alias = r.referrer)
LEFT JOIN user_points_public upp1 ON (upp1.address = r.referral AND r.height <= ${batchHeight})
LEFT JOIN user_kyc k ON (k.address = r.referrer AND k.ts <= $ts)
LEFT JOIN user_kyc k ON (k.address = COALESCE(als.address, r.referrer) AND k.ts <= $ts)
WHERE
r.referrer = user_points_public.address AND
COALESCE(als.address, r.referrer) = user_points_public.address AND
k.address IS NOT NULL
),0),
points_l2 = COALESCE(points_l2,0) + COALESCE((
SELECT
CAST(SUM(upp2.change) * ${config.l2_percent / 100} AS INTEGER)
FLOOR(SUM(upp2.change) * ${config.l2_percent / 100})
FROM
referrals r2
LEFT JOIN referrals r3 ON (r3.referrer = r2.referral AND r3.height <= ${batchHeight})
LEFT JOIN aliases als ON (als.alias = r2.referrer)
LEFT JOIN user_points_public upp2 ON (upp2.address = r3.referral AND r3.height <= ${batchHeight})
LEFT JOIN user_kyc k2 ON (k2.address = r2.referrer AND k2.ts <= $ts)
LEFT JOIN user_kyc k2 ON (k2.address = COALESCE(als.address, r2.referrer) AND k2.ts <= $ts)
WHERE
r2.referrer = user_points_public.address AND
COALESCE(als.address, r2.referrer) = user_points_public.address AND
k2.address IS NOT NULL
),0)
`,
Expand Down
5 changes: 1 addition & 4 deletions lib/sources/astroport/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,7 @@ export default class AstroportSource implements SourceInterface {
}
this.assets = params.assets;

if (!params.recalculate) {
throw new Error('No recalculation flag configured');
}
this.recalculate = params.recalculate;
this.recalculate = params.recalculate || false;

this.rpc = rpc;
this.concurrencyLimit = parseInt(params.concurrency_limit || '3', 10);
Expand Down
5 changes: 1 addition & 4 deletions lib/sources/mars/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,7 @@ export default class MarsSource implements SourceInterface {
}
this.assets = params.assets;

if (!params.recalculate) {
throw new Error('No recalculation flag configured');
}
this.recalculate = params.recalculate;
this.recalculate = params.recalculate || false;

if (!params.nft_contract) {
throw new Error('No mars nft contract configured in params');
Expand Down
50 changes: 35 additions & 15 deletions recalculate.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,24 @@ import * as path from 'path';
import { execSync } from 'child_process';
import { constants, Database } from 'bun:sqlite';

const OLD_DATABASE = 'recalculate/36-data.db';
const OLD_DATABASE = 'recalculate/old_data.db';
const NEW_DATABASE = 'recalculate/new_data.db';
const CHANGES_FILE = 'recalculate/changes.csv';
const ALIASES_FILE = 'recalculate/aliases.csv';

async function parseCsv(filePath: string): Promise<string[]> {
async function parseCsv(
filePath: string,
separator: string,
numberOfColumns: number,
): Promise<string[]> {
const data = await fsPromises.readFile(filePath, 'utf8');

const lines = data.trim().split('\n');

const [headerLine, ...dataLines] = lines;
const headers = headerLine.split(',');
const headers = headerLine.split(separator);

if (headers.length !== 5) {
if (headers.length !== numberOfColumns) {
throw new Error('CSV file does not have correct number of columns');
}

Expand Down Expand Up @@ -59,26 +64,43 @@ async function main() {
>('SELECT * FROM batches');
const batches = batchesQuery.all(null);

newDb.exec(
`CREATE TABLE IF NOT EXISTS changes (address TEXT, batch_id TEXT, points INTEGER, points_l1 INTEGER, points_l2 INTEGER, PRIMARY KEY(address, batch_id DESC));`,
);
const createChangesQuery = `CREATE TABLE IF NOT EXISTS changes (address TEXT, batch_id TEXT, points INTEGER, points_l1 INTEGER, points_l2 INTEGER, PRIMARY KEY(address, batch_id DESC));`;
oldDb.exec(createChangesQuery);
newDb.exec(createChangesQuery);

const clearChangesQuery = `DELETE FROM changes;`;
oldDb.exec(clearChangesQuery);
newDb.exec(clearChangesQuery);

const changesLines = await parseCsv(CHANGES_FILE);
const changesLines = await parseCsv(CHANGES_FILE, ',', 5);
changesLines.forEach((line) => {
const [address, batchId, points, points_l1, points_l2] = line.split(',');
newDb.exec(`
const insertChangeQuery = `
INSERT INTO changes (address, batch_id, points, points_l1, points_l2)
VALUES ('${address}', '${batchId}', ${points}, ${points_l1}, ${points_l2})
ON CONFLICT (address, batch_id) DO UPDATE SET
points = changes.points + excluded.points,
points_l1 = changes.points_l1 + excluded.points_l1,
points_l2 = changes.points_l2 + excluded.points_l2
`);
`;
oldDb.exec(insertChangeQuery);
newDb.exec(insertChangeQuery);
});

newDb.exec(
`CREATE TABLE IF NOT EXISTS batch_heights (batch_id INTEGER, height INTEGER, ts INTEGER);`,
);
const createAliasesQuery = `CREATE TABLE IF NOT EXISTS aliases (alias TEXT, address TEXT, code TEXT, PRIMARY KEY(alias));`;
oldDb.exec(createAliasesQuery);
newDb.exec(createAliasesQuery);

const aliasesLines = await parseCsv(ALIASES_FILE, ';', 3);
aliasesLines.forEach((line) => {
const [alias, address, code] = line.split(';');
const insertAliasQuery = `
INSERT OR IGNORE INTO aliases (alias, address, code)
VALUES ('${alias}', '${address}', '${code}')
`;
oldDb.exec(insertAliasQuery);
newDb.exec(insertAliasQuery);
});

const thirdBatchTsQuery = oldDb.query<{ batch_3_ts: number }, null>(
`SELECT ts AS batch_3_ts FROM batches WHERE batch_id = 3`,
Expand Down Expand Up @@ -365,8 +387,6 @@ async function main() {
if (checkLevelTwo) {
console.log(`${checkLevelTwo.row_count} addresses are broken with l2`);
}

// if (batch.batch_id == 4) break;
}

console.log(checkLevelOneQueryText);
Expand Down

0 comments on commit ad87796

Please sign in to comment.