Skip to content

Commit

Permalink
Fix code and make up to date for soul fighter event
Browse files Browse the repository at this point in the history
  • Loading branch information
Hi-Ray committed Jul 22, 2023
1 parent 0c43b2b commit 36a782c
Show file tree
Hide file tree
Showing 7 changed files with 62 additions and 41 deletions.
2 changes: 0 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@
"update-deps": "npx npm-check-updates -u"
},
"devDependencies": {
"@types/delete-empty": "^3.0.2",
"@types/download": "^8.0.2",
"@types/fs-extra": "^11.0.1",
"@types/node": "^20.4.2",
Expand All @@ -68,7 +67,6 @@
"axios": "^1.4.0",
"cheerio": "^1.0.0-rc.12",
"colorette": "^2.0.20",
"delete-empty": "^3.0.0",
"dotenv": "^16.3.1",
"download": "^8.0.0",
"finder": "^0.1.4",
Expand Down
3 changes: 1 addition & 2 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import { version } from '../package.json';
import { greenBright, yellowBright } from 'colorette';
import Tracer from 'tracer';
import dotenv from 'dotenv';
import deleteEmpty from 'delete-empty';

// Main logger
const logger = Tracer.colorConsole();
Expand Down Expand Up @@ -47,7 +46,7 @@ const main = async () => {
logger.info('Downloading event: ' + dist.event);
await handle(dist.url, `events/${dist.game}/${dist.event}`);
}
await deleteEmpty('./events');

// Sync to Samba Share
if (samba) {
await sync();
Expand Down
7 changes: 3 additions & 4 deletions src/scraper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,13 +122,12 @@ export const scraper = async (json = false, name = 'events.json') => {

logger.info('Fetching riot-client assets');

// for (const riotClientManifest of riotClientManifests) {
// await processManifest(riotClientManifest, 'events');
// }
for (const riotClientManifest of riotClientManifests) {
await processManifest(riotClientManifest, 'events');
}

for (const homePage of homePages) {
if (homePage.game === 'lol') {
continue;
const page = <HomepageJson>homePage.data;
page.npe.navigation.forEach((value) => {
if (!value.isPlugin && value.url?.includes('embed.rgpub.io') && !idBlacklist.includes(value.id)) {
Expand Down
6 changes: 4 additions & 2 deletions src/stormrazor/lol/finder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@ export const findFiles = async (exportDir: string, potentialFiles: string[]): Pr
const foundFiles = potentialFiles.filter(
(file) =>
file.includes('lib-embed') ||
(!potentialFiles.includes(`_/lib-embed/images${file}`) &&
!potentialFiles.includes(`_/lib-embed/videos${file}`)),
file.match(/svgs?/gimu) ||
file.match(/videos?/gimu) ||
file.match(/images?/gimu) ||
file.match(/sounds?/gimu),
);

await fs.writeFile(path.join(exportDir, 'files.txt'), foundFiles.join('\n'), { flag: 'a' });
Expand Down
17 changes: 17 additions & 0 deletions src/stormrazor/lol/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import download from 'download';
import Tracer from 'tracer';
import fs from 'fs-extra';
import path from 'path';
import { removeEmptyDirectories } from '../../util/fs';

const logger = Tracer.colorConsole();

Expand Down Expand Up @@ -56,6 +57,13 @@ const downloadFiles = async (foundFiles: string[], tmpDir: string, basePath: str
const downloadPath = path.join(basePath, foundFile).replace(':/', '://').replace(':\\', '://');

logger.info(`Attempting to download ${foundFile}`);

try {
await download(foundFile, tmpDir);
logger.info(`Downloaded ${foundFile}`);
continue;
} catch (e) {}

try {
// Downloads the file.
await download(downloadPath, exportDir);
Expand Down Expand Up @@ -146,6 +154,8 @@ export const handle = async (distURL: string, tmpDir: string) => {
// Regex for finding paths.
const pathRegex = /"\.?([\w\.\/-]*\.(?:jpg|png|gif|webm|svg|webp))/g;

const altPathRegex = /"https:\/\/?([\w\.\/-]*\.(?:jpg|png|gif|webm|svg|webp))"/gimu;

const externalFiles = /"(.*?)"/gimu;

// The base path without the file.
Expand All @@ -170,6 +180,9 @@ export const handle = async (distURL: string, tmpDir: string) => {
const potentialFiles: string[] =
[...content.matchAll(pathRegex)].map((m) => m[1]).map((m) => m.replaceAll('/vendor', '')) ?? [];

const remainingFiles: string[] =
[...content.matchAll(altPathRegex)].map((m) => m[1]).map((m) => m?.replaceAll('/vendor', '')) ?? [];

// Find potential external files.
const potentialExternalFiles: string[] = [...content.matchAll(externalFiles)].map((m) => m[1]) ?? [];

Expand All @@ -181,6 +194,7 @@ export const handle = async (distURL: string, tmpDir: string) => {

// Finds the files from the potential files.
const foundFiles: string[] = await findFiles(tmpDir, potentialFiles);
const remainingFilesFound: string[] = await findFiles(tmpDir, remainingFiles);

// Finds the inline HTML SVGs from the file files.
const foundSvgs: string[] = await findSvgs(tmpDir, content);
Expand All @@ -192,10 +206,13 @@ export const handle = async (distURL: string, tmpDir: string) => {

// Downloads the found files.
await downloadFiles(foundFiles, tmpDir, basePath);
await downloadFiles(remainingFilesFound, tmpDir, basePath);

// Saves the found SVGs.
await saveSvgs(foundSvgs, tmpDir);

// Downloads the external urls found.
await downloadURL(cleanedExternal, tmpDir);

await removeEmptyDirectories('events');
};
36 changes: 36 additions & 0 deletions src/util/fs.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import { promises as fsPromises } from 'fs';
import path from 'path';

/**
* Recursively removes empty directories from the given directory.
*
* If the directory itself is empty, it is also removed.
*
* Async code taken from: https://gist.github.com/fixpunkt/fe32afe14fbab99d9feb4e8da7268445
* Code taken from: https://gist.github.com/jakub-g/5903dc7e4028133704a4
*
* @param {string} directory Path to the directory to clean up
*/
export async function removeEmptyDirectories(directory: string): Promise<void> {
// lstat does not follow symlinks (in contrast to stat)
const fileStats = await fsPromises.lstat(directory);
if (!fileStats.isDirectory()) {
return;
}
let fileNames = await fsPromises.readdir(directory);
if (fileNames.length > 0) {
const recursiveRemovalPromises = fileNames.map((fileName) =>
removeEmptyDirectories(path.join(directory, fileName)),
);
await Promise.all(recursiveRemovalPromises);

// re-evaluate fileNames; after deleting subdirectory
// we may have parent directory empty now
fileNames = await fsPromises.readdir(directory);
}

if (fileNames.length === 0) {
console.log('Removing: ', directory);
await fsPromises.rmdir(directory);
}
}
32 changes: 1 addition & 31 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -292,11 +292,6 @@
dependencies:
"@types/node" "*"

"@types/delete-empty@^3.0.2":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@types/delete-empty/-/delete-empty-3.0.2.tgz#95677440adf049277dedb75bcd89b3e939ec4a90"
integrity sha512-HYLzzaRBwk30YwGlSYQhgeiSfcLDc6ksSpONtdZrehvxhwGdwJPxvYsO6vsWrRjRlymj2BBVBgniE8qPz88EMA==

"@types/download@^8.0.2":
version "8.0.2"
resolved "https://registry.yarnpkg.com/@types/download/-/download-8.0.2.tgz#2ef0a8b19caec152b51a2efe2e99a6795dcf1ec2"
Expand Down Expand Up @@ -524,11 +519,6 @@ ajv@^6.10.0, ajv@^6.12.4:
json-schema-traverse "^0.4.1"
uri-js "^4.2.2"

ansi-colors@^4.1.0:
version "4.1.3"
resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.3.tgz#37611340eb2243e70cc604cad35d63270d48781b"
integrity sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==

ansi-regex@^5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304"
Expand Down Expand Up @@ -987,16 +977,6 @@ delayed-stream@~1.0.0:
resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==

delete-empty@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/delete-empty/-/delete-empty-3.0.0.tgz#f8040f2669f26fa7060bc2304e9859c593b685e8"
integrity sha512-ZUyiwo76W+DYnKsL3Kim6M/UOavPdBJgDYWOmuQhYaZvJH0AXAHbUNyEDtRbBra8wqqr686+63/0azfEk1ebUQ==
dependencies:
ansi-colors "^4.1.0"
minimist "^1.2.0"
path-starts-with "^2.0.0"
rimraf "^2.6.2"

diff-sequences@^29.4.3:
version "29.4.3"
resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.4.3.tgz#9314bc1fabe09267ffeca9cbafc457d8499a13f2"
Expand Down Expand Up @@ -1979,11 +1959,6 @@ minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2:
dependencies:
brace-expansion "^1.1.7"

minimist@^1.2.0:
version "1.2.8"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==

minimist@^1.2.6:
version "1.2.7"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18"
Expand Down Expand Up @@ -2204,11 +2179,6 @@ path-parse@^1.0.7:
resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735"
integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==

path-starts-with@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/path-starts-with/-/path-starts-with-2.0.0.tgz#ffd6d51926cd497022b44d392196033d5451892f"
integrity sha512-3UHTHbJz5+NLkPafFR+2ycJOjoc4WV2e9qCZCnm71zHiWaFrm1XniLVTkZXvaRgxr1xFh9JsTdicpH2yM03nLA==

path-type@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b"
Expand Down Expand Up @@ -2408,7 +2378,7 @@ reusify@^1.0.4:
resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76"
integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==

rimraf@^2.6.1, rimraf@^2.6.2:
rimraf@^2.6.1:
version "2.7.1"
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec"
integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==
Expand Down

0 comments on commit 36a782c

Please sign in to comment.