Skip to content

Commit

Permalink
chore: update fetch script
Browse files Browse the repository at this point in the history
  • Loading branch information
yjl9903 committed Jan 9, 2024
1 parent e2e05e9 commit 61580b6
Showing 1 changed file with 14 additions and 1 deletion.
15 changes: 14 additions & 1 deletion scripts/fetch.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import fs from 'node:fs';
import fs from 'fs-extra';
import path from 'node:path';
import { fetch } from 'undici';

Expand All @@ -22,6 +22,7 @@ const ufetch = async (url: RequestInfo, init?: RequestInit): Promise<Response> =
const env = process?.env ?? {};
const list = ['HTTPS_PROXY', 'https_proxy', 'HTTP_PROXY', 'http_proxy'];
for (const l of list) {
// @ts-ignore
const t = env[l];
if (!!t) {
return t;
Expand All @@ -32,6 +33,9 @@ const ufetch = async (url: RequestInfo, init?: RequestInit): Promise<Response> =
};

async function main(start: number, dist: string) {
await fs.mkdir(dist, { recursive: true });

let empty = 0;
for (let page = start; ; page++) {
console.log(`Fetch page ${page}`);

Expand All @@ -45,6 +49,15 @@ async function main(start: number, dist: string) {
JSON.stringify(r, null, 2),
'utf-8'
);

if (r.length === 0) {
empty++;
if (empty >= 10) {
break;
}
} else {
empty = 0;
}
}
}

Expand Down

0 comments on commit 61580b6

Please sign in to comment.