From 85ee150b84e866f6f7c63b6983215d184e8b2c13 Mon Sep 17 00:00:00 2001 From: Kamil Charkiewicz Date: Fri, 17 Nov 2023 17:42:33 +0100 Subject: [PATCH] duplicate search init --- packages/landing-friend-core/src/analyzer.ts | 29 +++--- packages/landing-friend-core/src/config.ts | 4 + .../landing-friend-core/src/data/filePath.ts | 1 + .../src/data/forbiddenChar.ts | 20 ++++ .../landing-friend-core/src/data/index.ts | 5 +- .../src/data/staticTags.ts | 73 ++++++++++++++ .../src/data/{exclude.ts => unicode.ts} | 95 ------------------- packages/landing-friend-core/src/index.ts | 1 + .../src/searchDuplicated.ts | 74 +++++++++++++++ .../src/types/analyzerTypes.ts | 22 ++--- .../src/types/configTypes.ts | 3 +- .../src/types/duplicatedTypes.ts | 19 ++++ .../landing-friend-core/src/types/index.ts | 1 + .../analyzer/analyzers/advancedAnalyzer.ts | 16 +--- .../utils/analyzer/analyzers/basicAnalyzer.ts | 64 +++++-------- .../src/utils/analyzer/functions/checkFile.ts | 6 +- .../src/utils/analyzer/index.ts | 2 +- .../src/utils/analyzer/prepareHTML.ts | 4 +- .../analyzer/sections/generateMainSection.ts | 15 +-- .../src/utils/config/index.ts | 82 ---------------- .../src/utils/functions/clearContent.ts | 17 ++++ .../src/utils/functions/filePath.ts | 10 ++ .../src/utils/functions/getFromFile.ts | 31 ++++++ .../src/utils/functions/index.ts | 5 + .../src/utils/functions/match.ts | 18 ++++ .../src/utils/functions/saveFile.ts | 33 +++++++ .../landing-friend-core/src/utils/index.ts | 3 +- .../searchDuplicated/functions/getContent.ts | 54 +++++++++++ .../utils/searchDuplicated/functions/index.ts | 1 + .../src/utils/searchDuplicated/index.ts | 1 + packages/landing-friend/src/CLI.ts | 85 ++++++++++++++++- .../src/functions/configInit.ts | 20 +++- 32 files changed, 542 insertions(+), 272 deletions(-) create mode 100644 packages/landing-friend-core/src/data/filePath.ts create mode 100644 packages/landing-friend-core/src/data/forbiddenChar.ts create mode 100644 packages/landing-friend-core/src/data/staticTags.ts rename packages/landing-friend-core/src/data/{exclude.ts => unicode.ts} (68%) create mode 100644 packages/landing-friend-core/src/searchDuplicated.ts create mode 100644 packages/landing-friend-core/src/types/duplicatedTypes.ts delete mode 100644 packages/landing-friend-core/src/utils/config/index.ts create mode 100644 packages/landing-friend-core/src/utils/functions/clearContent.ts create mode 100644 packages/landing-friend-core/src/utils/functions/filePath.ts create mode 100644 packages/landing-friend-core/src/utils/functions/getFromFile.ts create mode 100644 packages/landing-friend-core/src/utils/functions/index.ts create mode 100644 packages/landing-friend-core/src/utils/functions/match.ts create mode 100644 packages/landing-friend-core/src/utils/functions/saveFile.ts create mode 100644 packages/landing-friend-core/src/utils/searchDuplicated/functions/getContent.ts create mode 100644 packages/landing-friend-core/src/utils/searchDuplicated/functions/index.ts create mode 100644 packages/landing-friend-core/src/utils/searchDuplicated/index.ts diff --git a/packages/landing-friend-core/src/analyzer.ts b/packages/landing-friend-core/src/analyzer.ts index 34f93e0..7505906 100644 --- a/packages/landing-friend-core/src/analyzer.ts +++ b/packages/landing-friend-core/src/analyzer.ts @@ -3,20 +3,25 @@ import open, { apps } from "open"; import path from "path"; import { + AdditionalTagsName, AdvancedTagsName, AllTagsName, checkFiles, CombinedPatterns, CombineTagsWithReason, ConfigFile, + fileLocation, + FileName, + fileName, getHtmlFiles, matchedSetting, message, + pathName, prepareHTMLWithTables, - saveAnalyze, + saveFile, } from "@/index.js"; -export const websiteAnalyzer = async (config: ConfigFile, interval: NodeJS.Timer) => { +export const websiteAnalyzer = async (config: ConfigFile, interval?: NodeJS.Timer) => { const { input, analyzer, advancedAnalyzer, excludedPage, sitemap, domain } = config; if (!analyzer) { return message("Define analyzer in config", "redBright"); @@ -38,8 +43,8 @@ export const websiteAnalyzer = async (config: ConfigFile, interval: NodeJS.Timer ) { combinedTagsPatternsArray.push( await checkFiles({ - file, - input, + file: file.replace("\\", "/"), + input: input.replace(/\.\//g, ""), tags: analyzer, advancedTags: advancedAnalyzer, domain, @@ -73,7 +78,8 @@ export const websiteAnalyzer = async (config: ConfigFile, interval: NodeJS.Timer quantity: value.quantity, content: value.content, forbiddenCharacters: value.forbiddenCharacters, - keywordsIncluded: tag !== "keywords" ? value.keywordsIncluded : undefined, + keywordsIncluded: + tag !== AdditionalTagsName.Keywords ? value.keywordsIncluded : undefined, multipleTags: value.multipleTags, tagAmount: tag in AdvancedTagsName ? value.tagAmount : undefined, listOfFoundMeta: value.listOfFoundMeta, @@ -87,13 +93,10 @@ export const websiteAnalyzer = async (config: ConfigFile, interval: NodeJS.Timer }); }); - const location = "./SEO"; - const fileName = (extension: ".json" | ".html") => `seo-analyze${extension}`; - const pathname = (extension: ".json" | ".html") => `${location}/${fileName(extension)}`; clearTimeout(interval); try { - saveAnalyze(pathname(".json"), JSON.stringify(cleanedTagsPatterns, null, 2)); - saveAnalyze(pathname(".html"), htmlWithTablesAndCharts); + saveFile(pathName(FileName.analyze, ".json"), JSON.stringify(cleanedTagsPatterns, null, 2)); + saveFile(pathName(FileName.analyze, ".html"), htmlWithTablesAndCharts); message( "Your website has been analyzed, JSON and html files have been generated in ./SEO", "green" @@ -101,9 +104,11 @@ export const websiteAnalyzer = async (config: ConfigFile, interval: NodeJS.Timer } catch { message("Failed to create files", "red"); } finally { - if (fs.existsSync(path.join(process.cwd(), location, fileName(".html")))) { + if ( + fs.existsSync(path.join(process.cwd(), fileLocation, fileName(FileName.analyze, ".html"))) + ) { try { - await open(path.join(process.cwd(), location, fileName(".html")), { + await open(path.join(process.cwd(), fileLocation, fileName(FileName.analyze, ".html")), { app: { name: apps.browser }, }); message("The analysis file has been opened in your browser.", "green"); diff --git a/packages/landing-friend-core/src/config.ts b/packages/landing-friend-core/src/config.ts index 82cd3e6..3fa3934 100644 --- a/packages/landing-friend-core/src/config.ts +++ b/packages/landing-friend-core/src/config.ts @@ -47,6 +47,10 @@ export const EXTENDED_ADVANCED_ANALYZER_GLOBAL_CONFIG_FILE: Pick = { + searchDuplicated: true, +}; + export const readConfig = (filePath: string, option: "init" | "generate") => { if (!fs.existsSync(filePath)) { if (option === "generate") { diff --git a/packages/landing-friend-core/src/data/filePath.ts b/packages/landing-friend-core/src/data/filePath.ts new file mode 100644 index 0000000..6a87508 --- /dev/null +++ b/packages/landing-friend-core/src/data/filePath.ts @@ -0,0 +1 @@ +export const fileLocation = "./SEO"; diff --git a/packages/landing-friend-core/src/data/forbiddenChar.ts b/packages/landing-friend-core/src/data/forbiddenChar.ts new file mode 100644 index 0000000..764bb40 --- /dev/null +++ b/packages/landing-friend-core/src/data/forbiddenChar.ts @@ -0,0 +1,20 @@ +export const forbiddenCharacters = [ + "！", + "｜", + "：", + "ı", + "＆", + "＜", + "＞", + "＂", + "／", + "｀", + "‘", + "’", + "‹", + "›", + " ", + "­", + "´", + "¸", +]; diff --git a/packages/landing-friend-core/src/data/index.ts b/packages/landing-friend-core/src/data/index.ts index b06e45d..66513d1 100644 --- a/packages/landing-friend-core/src/data/index.ts +++ b/packages/landing-friend-core/src/data/index.ts @@ -1 +1,4 @@ -export * from "./exclude.js"; +export * from "./filePath.js"; +export * from "./forbiddenChar.js"; +export * from "./staticTags.js"; +export * from "./unicode.js"; diff --git a/packages/landing-friend-core/src/data/staticTags.ts b/packages/landing-friend-core/src/data/staticTags.ts new file mode 100644 index 0000000..03c7069 --- /dev/null +++ b/packages/landing-friend-core/src/data/staticTags.ts @@ -0,0 +1,73 @@ +export const staticTags = [ + "a", + "abbr", + "acronym", + "address", + "audio", + "b", + "bdi", + "bdo", + "blockquote", + "br", + "canvas", + "caption", + "cite", + "code", + "col", + "colgroup", + "dd", + "del", + "dfn", + "div", + "dl", + "dt", + "em", + "embed", + "figure", + "figcaption", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "iframe", + "ins", + "kbd", + "li", + "mark", + "noscript", + "object", + "ol", + "p", + "param", + "path", + "picture", + "pre", + "q", + "rp", + "rt", + "ruby", + "samp", + "script", + "small", + "span", + "strong", + "style", + "sub", + "sup", + "svg", + "table", + "tbody", + "td", + "tfoot", + "th", + "thead", + "time", + "track", + "tr", + "ul", + "var", + "video", + "wbr", +]; diff --git a/packages/landing-friend-core/src/data/exclude.ts b/packages/landing-friend-core/src/data/unicode.ts similarity index 68% rename from packages/landing-friend-core/src/data/exclude.ts rename to packages/landing-friend-core/src/data/unicode.ts index 6dea232..75c3ffc 100644 --- a/packages/landing-friend-core/src/data/exclude.ts +++ b/packages/landing-friend-core/src/data/unicode.ts @@ -104,98 +104,3 @@ export const unicode = { "þ": "þ", "ÿ": "ÿ", }; - -export const staticTags = [ - "a", - "abbr", - "acronym", - "address", - "audio", - "b", - "bdi", - "bdo", - "blockquote", - "br", - "canvas", - "caption", - "cite", - "code", - "col", - "colgroup", - "dd", - "del", - "dfn", - "div", - "dl", - "dt", - "em", - "embed", - "figure", - "figcaption", - "h1", - "h2", - "h3", - "h4", - "h5", - "h6", - "iframe", - "ins", - "kbd", - "li", - "mark", - "noscript", - "object", - "ol", - "p", - "param", - "path", - "picture", - "pre", - "q", - "rp", - "rt", - "ruby", - "samp", - "script", - "small", - "span", - "strong", - "style", - "sub", - "sup", - "svg", - "table", - "tbody", - "td", - "tfoot", - "th", - "thead", - "time", - "track", - "tr", - "ul", - "var", - "video", - "wbr", -]; - -export const forbiddenCharacters = [ - "！", - "｜", - "：", - "ı", - "＆", - "＜", - "＞", - "＂", - "／", - "｀", - "‘", - "’", - "‹", - "›", - " ", - "­", - "´", - "¸", -]; diff --git a/packages/landing-friend-core/src/index.ts b/packages/landing-friend-core/src/index.ts index 7fad3ce..3d64f93 100644 --- a/packages/landing-friend-core/src/index.ts +++ b/packages/landing-friend-core/src/index.ts @@ -2,6 +2,7 @@ export * from "./analyzer.js"; export * from "./config.js"; export * from "./console.js"; export * from "./data/index.js"; +export * from "./searchDuplicated.js"; export * from "./sitemap.js"; export * from "./types/index.js"; export * from "./utils/index.js"; diff --git a/packages/landing-friend-core/src/searchDuplicated.ts b/packages/landing-friend-core/src/searchDuplicated.ts new file mode 100644 index 0000000..b7646eb --- /dev/null +++ b/packages/landing-friend-core/src/searchDuplicated.ts @@ -0,0 +1,74 @@ +import fs from "fs"; +import open, { apps } from "open"; +import path from "path"; + +import { + ConfigFile, + fileLocation, + FileName, + fileName, + FileWithDuplicateContent, + getContent, + getHtmlFiles, + matchedSetting, + message, + pathName, + saveFile, +} from "@/index.js"; + +export const searchDuplicated = async (config: ConfigFile, interval?: NodeJS.Timer) => { + const { input, searchDuplicated, excludedPage } = config; + + if (!searchDuplicated) { + return message("Define analyzer in config", "redBright"); + } + + const contentArray: FileWithDuplicateContent[] = []; + + const allHtmlFiles = getHtmlFiles(input, false); + + for (const file of allHtmlFiles) { + if ( + !matchedSetting( + file + .replace("\\", "/") + .replace(/\.html|\.php/g, "") + .replace(/index/g, "") + .replace(/\/$/g, ""), + excludedPage + ) + ) { + contentArray.push( + await getContent({ + file: file.replace("\\", "/"), + input: input.replace(/\.\//g, ""), + }) + ); + } + } + + clearTimeout(interval); + try { + saveFile(pathName(FileName.duplicated, ".json"), JSON.stringify(contentArray, null, 2)); + // saveFile(pathName(FileName.duplicated, ".html"), htmlWithTablesAndCharts); + message( + "Your website has been analyzed, JSON and html files have been generated in ./SEO", + "green" + ); + } catch { + message("Failed to create files", "red"); + } finally { + if ( + fs.existsSync(path.join(process.cwd(), fileLocation, fileName(FileName.duplicated, ".html"))) + ) { + try { + await open(path.join(process.cwd(), fileLocation, fileName(FileName.duplicated, ".html")), { + app: { name: apps.browser }, + }); + message("The analysis file has been opened in your browser.", "green"); + } catch { + message("Cannot open browser. Please open file manually", "red"); + } + } + } +}; diff --git a/packages/landing-friend-core/src/types/analyzerTypes.ts b/packages/landing-friend-core/src/types/analyzerTypes.ts index 83f24ed..0757202 100644 --- a/packages/landing-friend-core/src/types/analyzerTypes.ts +++ b/packages/landing-friend-core/src/types/analyzerTypes.ts @@ -1,25 +1,25 @@ export enum AdvancedTagsName { - og = "og", - twitter = "twitter", + Og = "og", + Twitter = "twitter", } -export type AdvancedTagsNameType = keyof typeof AdvancedTagsName; +export type AdvancedTagsNameType = `${AdvancedTagsName}`; export enum TagsName { - h1 = "h1", - title = "title", - description = "description", + H1 = "h1", + Title = "title", + Description = "description", } -export type TagsNameType = keyof typeof TagsName; +export type TagsNameType = `${TagsName}`; export enum AdditionalTagsName { - lastSentence = "lastSentence", - keywords = "keywords", - canonical = "canonical", + LastSentence = "lastSentence", + Keywords = "keywords", + Canonical = "canonical", } -export type AdditionalTagsNameType = keyof typeof AdditionalTagsName; +export type AdditionalTagsNameType = `${AdditionalTagsName}`; export type BasicTagsName = TagsNameType | AdditionalTagsNameType; diff --git a/packages/landing-friend-core/src/types/configTypes.ts b/packages/landing-friend-core/src/types/configTypes.ts index e9027ad..fe679d4 100644 --- a/packages/landing-friend-core/src/types/configTypes.ts +++ b/packages/landing-friend-core/src/types/configTypes.ts @@ -1,6 +1,6 @@ import { LanguageCode } from "iso-639-1"; -import { AdvancedTagsProps, TagsProps } from "./index.js"; +import { AdvancedTagsProps, TagsProps } from "@/index.js"; export type SitemapSettings = { locale: { @@ -20,4 +20,5 @@ export type ConfigFile = { sitemap?: SitemapSettings; analyzer?: TagsProps; advancedAnalyzer?: AdvancedTagsProps; + searchDuplicated?: boolean; }; diff --git a/packages/landing-friend-core/src/types/duplicatedTypes.ts b/packages/landing-friend-core/src/types/duplicatedTypes.ts new file mode 100644 index 0000000..ac297b7 --- /dev/null +++ b/packages/landing-friend-core/src/types/duplicatedTypes.ts @@ -0,0 +1,19 @@ +export enum DuplicatedSearchName { + SamePage = "samePage", + SameTitle = "sameTitle", + SameMetaDesc = "sameMetaDesc", +} + +export interface DuplicatedContent { + content?: string; + numberOfDuplicates?: number; + duplicatesOnSite?: string[]; +} + +export type DuplicatedSearchNameTypes = `${DuplicatedSearchName}`; + +export type DuplicatedContentWithName = { + [name in DuplicatedSearchName]?: DuplicatedContent; +}; + +export type FileWithDuplicateContent = Record; diff --git a/packages/landing-friend-core/src/types/index.ts b/packages/landing-friend-core/src/types/index.ts index 80bcb93..7eb886b 100644 --- a/packages/landing-friend-core/src/types/index.ts +++ b/packages/landing-friend-core/src/types/index.ts @@ -1,2 +1,3 @@ export * from "./analyzerTypes.js"; export * from "./configTypes.js"; +export * from "./duplicatedTypes.js"; diff --git a/packages/landing-friend-core/src/utils/analyzer/analyzers/advancedAnalyzer.ts b/packages/landing-friend-core/src/utils/analyzer/analyzers/advancedAnalyzer.ts index f218b1c..3522adb 100644 --- a/packages/landing-friend-core/src/utils/analyzer/analyzers/advancedAnalyzer.ts +++ b/packages/landing-friend-core/src/utils/analyzer/analyzers/advancedAnalyzer.ts @@ -2,11 +2,10 @@ import { AdvancedTagsNameType, AdvancedTagsPatterns, AdvancedTagsProps, + clearContent, forbiddenCharacters as _forbiddenCharacters, MetaNameTagsProps, MetaNameWithProps, - staticTags, - unicode, } from "@/index.js"; interface MatchedArrayProps { @@ -68,19 +67,10 @@ export const checkFileToAdvanceAnalyzer = async ({ for (const match of matches) { for (const [metaName, value] of Object.entries(match)) { - let content: string | undefined; + let content: string | undefined = undefined; let status: string | undefined; - for (const staticTag of staticTags) { - const _content = value.content; - const staticTagRegex = new RegExp(`<${staticTag}.*?>|`, "g"); - for (const [_unicode, replacement] of Object.entries(unicode)) { - const unicodeRegex = new RegExp(`${_unicode}`, "g"); - content = _content.replace(unicodeRegex, replacement); - } - - content = _content.replace(staticTagRegex, ""); - } + content = clearContent(value.content); const forbiddenCharacters = _forbiddenCharacters.filter( char => content && content.includes(char) diff --git a/packages/landing-friend-core/src/utils/analyzer/analyzers/basicAnalyzer.ts b/packages/landing-friend-core/src/utils/analyzer/analyzers/basicAnalyzer.ts index 9da652f..eb1e6a5 100644 --- a/packages/landing-friend-core/src/utils/analyzer/analyzers/basicAnalyzer.ts +++ b/packages/landing-friend-core/src/utils/analyzer/analyzers/basicAnalyzer.ts @@ -1,12 +1,12 @@ import { + AdditionalTagsName, BasicTagsName, + clearContent, forbiddenCharacters as _forbiddenCharacters, - staticTags, TagsName, TagsPatterns, TagsProps, TagsWithReason, - unicode, } from "@/index.js"; const arrayFilter = (firstArray: string[], secondArray: string[]) => { @@ -15,15 +15,15 @@ const arrayFilter = (firstArray: string[], secondArray: string[]) => { const checkContent = (tagName: BasicTagsName, fileContent: string) => { let regex: RegExp | undefined; - let matches: RegExpMatchArray | null = null; + let matches: string[] | RegExpMatchArray | null = null; if (tagName === "description") { regex = new RegExp(`(.*?)`, "g"); } else { regex = new RegExp(`<${tagName}.*?>(.*?)`, "g"); @@ -31,7 +31,7 @@ const checkContent = (tagName: BasicTagsName, fileContent: string) => { matches = regex && fileContent.match(regex); - if (tagName === "lastSentence" && matches !== null) { + if (tagName === AdditionalTagsName.LastSentence && matches !== null) { const lastMatch = matches[matches.length - 1]; matches = lastMatch !== undefined ? [lastMatch] : null; } @@ -41,25 +41,11 @@ const checkContent = (tagName: BasicTagsName, fileContent: string) => { updatedMatches.forEach((match, index) => { const captureGroups = regex!.exec(match); if (captureGroups) { - let content = captureGroups[1]; - staticTags.forEach(staticTag => { - const staticTagRegex = new RegExp( - `<${staticTag}.*?>||\\.css.*?}|@media.*?}|{|}`, - "g" - ); - Object.entries(unicode).forEach(([unicode, replacement]) => { - const unicodeRegex = new RegExp(`${unicode}`, "g"); - content = content.replace(unicodeRegex, replacement); - }); - - content = content.replace(staticTagRegex, ""); - }); - - updatedMatches[index] = content; + updatedMatches[index] = clearContent(captureGroups[1])!; } }); - matches = updatedMatches as RegExpMatchArray; + matches = updatedMatches; } return matches; @@ -89,8 +75,8 @@ export const checkFileToBasicAnalyzer = ({ const url = domain + file.replace("index.html", ""); if (tags.keywords.count) { - const keywordsMatch = checkContent("keywords", fileContent); - const h1Match = checkContent("h1", fileContent); + const keywordsMatch = checkContent(AdditionalTagsName.Keywords, fileContent); + const h1Match = checkContent(TagsName.H1, fileContent); if (keywordsMatch && keywordsMatch.length > 0) { const keywords = keywordsMatch[0].split(", "); @@ -107,8 +93,8 @@ export const checkFileToBasicAnalyzer = ({ const tag = _tag as BasicTagsName; const value = _value as TagsWithReason; - if (!countKeywords && tag === "keywords") return; - if (!countWordsInLast && tag === "lastSentence") return; + if (!countKeywords && tag === AdditionalTagsName.Keywords) return; + if (!countWordsInLast && tag === AdditionalTagsName.LastSentence) return; const matches = checkContent(tag, fileContent); if (matches) { @@ -130,11 +116,11 @@ export const checkFileToBasicAnalyzer = ({ let toMuchKeywords: string[] = []; const tagKeywords = - tag === "canonical" + tag === AdditionalTagsName.Canonical ? undefined : !tags.keywords.count ? undefined - : tag === "keywords" + : tag === AdditionalTagsName.Keywords ? mainKeywordsArray : mainKeywordsArray.length > 0 ? mainKeywordsArray.filter(keyword => @@ -143,7 +129,7 @@ export const checkFileToBasicAnalyzer = ({ : undefined; if (h1Keywords.length > 0) { - if (tag in TagsName || tag === "lastSentence") { + if (tag in TagsName || tag === AdditionalTagsName.LastSentence) { missingKeywords = arrayFilter(h1Keywords, tagKeywords ? tagKeywords : []); toMuchKeywords = arrayFilter(tagKeywords ? tagKeywords : [], h1Keywords); } @@ -155,7 +141,7 @@ export const checkFileToBasicAnalyzer = ({ minLength && maxLength && (match.length < minLength || match.length > maxLength); const isH1Exist = countKeywords && mainKeywordsArray.length > 0 - ? tag === "h1" && h1Keywords.length === 0 + ? tag === TagsName.H1 && h1Keywords.length === 0 : false; const areKeywordsMissingOrExcessive = countKeywords || isH1Exist || mainKeywordsArray.length === 0 @@ -166,7 +152,7 @@ export const checkFileToBasicAnalyzer = ({ isLengthInvalid || areKeywordsMissingOrExcessive || isH1Exist || - (tag === "canonical" && match !== url) || + (tag === AdditionalTagsName.Canonical && match !== url) || forbiddenCharacters.length > 0 ); }; @@ -179,15 +165,15 @@ export const checkFileToBasicAnalyzer = ({ maxLength: value.maxLength, minLength: value.minLength, requirement: - tag === "keywords" + tag === AdditionalTagsName.Keywords ? undefined - : tag === "lastSentence" + : tag === AdditionalTagsName.LastSentence ? "Tag should contain the same keywords as upper tags" - : tag === "canonical" + : tag === AdditionalTagsName.Canonical ? "The canonical link must be the same as the URL." : `Tag length should be between ${value.minLength} and ${value.maxLength}`, quantity: match.length, - content: tag === "keywords" ? tagKeywords : match, + content: tag === AdditionalTagsName.Keywords ? tagKeywords : match, multipleTags: undefined, keywordsIncluded: tagKeywords, forbiddenCharacters: @@ -205,11 +191,11 @@ export const checkFileToBasicAnalyzer = ({ [tag]: { ...value, requirement: - tag === "keywords" + tag === AdditionalTagsName.Keywords ? "At least one keyword required" - : tag === "lastSentence" + : tag === AdditionalTagsName.LastSentence ? "Tag should contain the same keywords as upper tags" - : tag === "canonical" + : tag === AdditionalTagsName.Canonical ? "The canonical link must be the same as the URL." : `Tag length should be between ${value.minLength} and ${value.maxLength}`, quantity: 0, diff --git a/packages/landing-friend-core/src/utils/analyzer/functions/checkFile.ts b/packages/landing-friend-core/src/utils/analyzer/functions/checkFile.ts index ff32939..2079574 100644 --- a/packages/landing-friend-core/src/utils/analyzer/functions/checkFile.ts +++ b/packages/landing-friend-core/src/utils/analyzer/functions/checkFile.ts @@ -55,11 +55,11 @@ export const checkFiles = async ({ countKeywords: boolean; countWordsInLast: boolean; }): Promise => { - const _fileContent = readFile(path.join(process.cwd(), input.replace(/\.\//g, ""), file)); + const _fileContent = readFile(path.join(process.cwd(), input, file)); const fileContent = _fileContent.replace(/\r?\n\s*/g, " "); const firstPatternsPromise = checkFileToBasicAnalyzer({ - file: file.replace("\\", "/"), + file, fileContent, tags, domain, @@ -68,7 +68,7 @@ export const checkFiles = async ({ }); const secondPatternsPromise = checkFileToAdvanceAnalyzer({ - file: file.replace("\\", "/"), + file, fileContent, advancedTags, }); diff --git a/packages/landing-friend-core/src/utils/analyzer/index.ts b/packages/landing-friend-core/src/utils/analyzer/index.ts index b2a9b43..6bf18af 100644 --- a/packages/landing-friend-core/src/utils/analyzer/index.ts +++ b/packages/landing-friend-core/src/utils/analyzer/index.ts @@ -1,4 +1,4 @@ -export * from "../config/index.js"; +export * from "../functions/index.js"; export * from "./analyzers/index.js"; export * from "./functions/index.js"; export * from "./prepareHTML.js"; diff --git a/packages/landing-friend-core/src/utils/analyzer/prepareHTML.ts b/packages/landing-friend-core/src/utils/analyzer/prepareHTML.ts index da5a6db..c354c85 100644 --- a/packages/landing-friend-core/src/utils/analyzer/prepareHTML.ts +++ b/packages/landing-friend-core/src/utils/analyzer/prepareHTML.ts @@ -42,10 +42,10 @@ const generateTableRows = ({ let h1Keywords: string[] | undefined; let mainKeywords: string[] | undefined; - if (TagsName.h1 in tagData) { + if (TagsName.H1 in tagData) { h1Keywords = tagData.h1.keywordsIncluded; } - if (AdditionalTagsName.keywords in tagData) { + if (AdditionalTagsName.Keywords in tagData) { mainKeywords = tagData.keywords.keywordsIncluded; } diff --git a/packages/landing-friend-core/src/utils/analyzer/sections/generateMainSection.ts b/packages/landing-friend-core/src/utils/analyzer/sections/generateMainSection.ts index 39bfa8a..621b2a2 100644 --- a/packages/landing-friend-core/src/utils/analyzer/sections/generateMainSection.ts +++ b/packages/landing-friend-core/src/utils/analyzer/sections/generateMainSection.ts @@ -19,8 +19,8 @@ export const generateMainSection = ({ pathname: __pathname, domain, }: Props) => { - if (!countWordsInLast && tag === "lastSentence") return ""; - if (!countKeywords && tag === "keywords") return ""; + if (!countWordsInLast && tag === AdditionalTagsName.LastSentence) return ""; + if (!countKeywords && tag === AdditionalTagsName.Keywords) return ""; if (value.multipleTags) { return `Warning! Number of multiple ${tag} on the page: ${value.quantity}Check the code`; } else { @@ -29,8 +29,8 @@ export const generateMainSection = ({ const url = domain + pathname; const firstCell = ` - ${tag === "keywords" ? "Length of " : "List of "} - ${tag === "lastSentence" ? "last sentence" : tag}: + ${tag === AdditionalTagsName.Keywords ? "Length of " : "List of "} + ${tag === AdditionalTagsName.LastSentence ? "last sentence" : tag}: ${ value.minLength && value.maxLength ? ` ${ typeof value.content === "string" @@ -56,7 +56,10 @@ export const generateMainSection = ({ : `No characters detected/strong>` } ${ - value.quantity > 0 && countKeywords && tag !== "keywords" && tag !== "canonical" + value.quantity > 0 && + countKeywords && + tag !== AdditionalTagsName.Keywords && + tag !== AdditionalTagsName.Canonical ? value.keywordsIncluded && value.keywordsIncluded.length > 0 ? ` | Keywords included: ${value.keywordsIncluded.join( ", " diff --git a/packages/landing-friend-core/src/utils/config/index.ts b/packages/landing-friend-core/src/utils/config/index.ts deleted file mode 100644 index 9bb19cc..0000000 --- a/packages/landing-friend-core/src/utils/config/index.ts +++ /dev/null @@ -1,82 +0,0 @@ -import fs from "fs"; -import path from "path"; - -import { message } from "../../index.js"; - -export const getHtmlFiles = (base: string, deleteFileExtension: boolean) => { - const baseWithoutDot = base.replace(/\.\//g, ""); - - return getDirectories(base) - .map(file => { - const relativePath = file.replace(baseWithoutDot, ""); - return relativePath.replace(/\\/g, "/"); - }) - .filter(file => file.endsWith(".html") || file.endsWith(".php")) - .map(file => (deleteFileExtension ? file.replace(/\.html|\.php/g, "") : file)); -}; - -export const getDirectories = (dir: string, fileList = [] as string[]) => { - const files = fs.readdirSync(dir); - files.forEach((file: string) => { - const filePath = path.join(dir, file); - const isDirectory = fs.statSync(filePath).isDirectory(); - - if (isDirectory) { - getDirectories(filePath, fileList); - } else fileList.push(filePath); - }); - - return fileList; -}; - -export const readFile = (filePath: string) => { - return fs.readFileSync(filePath, "utf8"); -}; - -export const saveSitemap = (filePath: string, content: string) => { - const directory = path.dirname(filePath); - if (!fs.existsSync(directory)) { - fs.mkdirSync(directory, { recursive: true }); - } - - fs.writeFileSync(filePath, content); -}; - -export const saveAnalyze = (filePath: string, content: string) => { - const directory = path.dirname(filePath); - if (!fs.existsSync(directory)) { - fs.mkdirSync(directory, { recursive: true }); - message(`Folder SEO was created in ${directory}`, "yellow"); - } - fs.writeFileSync(filePath, content); -}; - -export const saveOldSitemap = (filePath: string, newFilePath: string) => { - if (fs.existsSync(filePath)) { - if (!fs.existsSync(newFilePath)) { - fs.mkdirSync(newFilePath, { recursive: true }); - message(`Folder SEO was created in ${newFilePath}`, "yellow"); - } - fs.copyFileSync(filePath, `${newFilePath}/sitemapOld.xml`); - message(`Old sitemap detected. Moved to ${newFilePath}`, "green"); - } -}; - -export const matchedSetting = (file: string, paths: string[]) => { - file = file.endsWith("/") ? file : file + "/"; - if (paths.length > 0) { - if ( - paths.find(path => { - const regexPattern = path - .replace(/\/$/g, "/$") - .replace(/^\.\//g, "^/") - .replace("*/", "/") - .replace("/*", "/"); - return file.match(new RegExp(regexPattern, "g")) !== null; - }) - ) { - return true; - } - } - return false; -}; diff --git a/packages/landing-friend-core/src/utils/functions/clearContent.ts b/packages/landing-friend-core/src/utils/functions/clearContent.ts new file mode 100644 index 0000000..580ac05 --- /dev/null +++ b/packages/landing-friend-core/src/utils/functions/clearContent.ts @@ -0,0 +1,17 @@ +import { staticTags, unicode } from "@/index.js"; + +export const clearContent = (content: string | undefined) => { + if (!content) return; + let finalContent = content; + staticTags.forEach(staticTag => { + const staticTagRegex = new RegExp(`<${staticTag}.*?>|`, "g"); + Object.entries(unicode).forEach(([unicode, replacement]) => { + const unicodeRegex = new RegExp(`${unicode}`, "g"); + finalContent = content.replace(unicodeRegex, replacement); + }); + + finalContent = content.replace(staticTagRegex, ""); + }); + + return finalContent; +}; diff --git a/packages/landing-friend-core/src/utils/functions/filePath.ts b/packages/landing-friend-core/src/utils/functions/filePath.ts new file mode 100644 index 0000000..5c8ecc6 --- /dev/null +++ b/packages/landing-friend-core/src/utils/functions/filePath.ts @@ -0,0 +1,10 @@ +import { fileLocation } from "@/index.js"; + +export enum FileName { + analyze = "seo-analyze", + duplicated = "duplicated-analyze", +} + +export const fileName = (name: FileName, extension: ".json" | ".html") => `${name}${extension}`; +export const pathName = (name: FileName, extension: ".json" | ".html") => + `${fileLocation}/${fileName(name, extension)}`; diff --git a/packages/landing-friend-core/src/utils/functions/getFromFile.ts b/packages/landing-friend-core/src/utils/functions/getFromFile.ts new file mode 100644 index 0000000..21aa702 --- /dev/null +++ b/packages/landing-friend-core/src/utils/functions/getFromFile.ts @@ -0,0 +1,31 @@ +import fs from "fs"; +import path from "path"; + +export const readFile = (filePath: string) => { + return fs.readFileSync(filePath, "utf8"); +}; +export const getHtmlFiles = (base: string, deleteFileExtension: boolean) => { + const baseWithoutDot = base.replace(/\.\//g, ""); + + return getDirectories(base) + .map(file => { + const relativePath = file.replace(baseWithoutDot, ""); + return relativePath.replace(/\\/g, "/"); + }) + .filter(file => file.endsWith(".html") || file.endsWith(".php")) + .map(file => (deleteFileExtension ? file.replace(/\.html|\.php/g, "") : file)); +}; + +export const getDirectories = (dir: string, fileList = [] as string[]) => { + const files = fs.readdirSync(dir); + files.forEach((file: string) => { + const filePath = path.join(dir, file); + const isDirectory = fs.statSync(filePath).isDirectory(); + + if (isDirectory) { + getDirectories(filePath, fileList); + } else fileList.push(filePath); + }); + + return fileList; +}; diff --git a/packages/landing-friend-core/src/utils/functions/index.ts b/packages/landing-friend-core/src/utils/functions/index.ts new file mode 100644 index 0000000..19a8809 --- /dev/null +++ b/packages/landing-friend-core/src/utils/functions/index.ts @@ -0,0 +1,5 @@ +export * from "./clearContent.js"; +export * from "./filePath.js"; +export * from "./getFromFile.js"; +export * from "./match.js"; +export * from "./saveFile.js"; diff --git a/packages/landing-friend-core/src/utils/functions/match.ts b/packages/landing-friend-core/src/utils/functions/match.ts new file mode 100644 index 0000000..d68109b --- /dev/null +++ b/packages/landing-friend-core/src/utils/functions/match.ts @@ -0,0 +1,18 @@ +export const matchedSetting = (file: string, paths: string[]) => { + file = file.endsWith("/") ? file : file + "/"; + if (paths.length > 0) { + if ( + paths.find(path => { + const regexPattern = path + .replace(/\/$/g, "/$") + .replace(/^\.\//g, "^/") + .replace("*/", "/") + .replace("/*", "/"); + return file.match(new RegExp(regexPattern, "g")) !== null; + }) + ) { + return true; + } + } + return false; +}; diff --git a/packages/landing-friend-core/src/utils/functions/saveFile.ts b/packages/landing-friend-core/src/utils/functions/saveFile.ts new file mode 100644 index 0000000..b08a481 --- /dev/null +++ b/packages/landing-friend-core/src/utils/functions/saveFile.ts @@ -0,0 +1,33 @@ +import fs from "fs"; +import path from "path"; + +import { message } from "@/console.js"; + +export const saveSitemap = (filePath: string, content: string) => { + const directory = path.dirname(filePath); + if (!fs.existsSync(directory)) { + fs.mkdirSync(directory, { recursive: true }); + } + + fs.writeFileSync(filePath, content); +}; + +export const saveFile = (filePath: string, content: string) => { + const directory = path.dirname(filePath); + if (!fs.existsSync(directory)) { + fs.mkdirSync(directory, { recursive: true }); + message(`Folder SEO was created in ${directory}`, "yellow"); + } + fs.writeFileSync(filePath, content); +}; + +export const saveOldSitemap = (filePath: string, newFilePath: string) => { + if (fs.existsSync(filePath)) { + if (!fs.existsSync(newFilePath)) { + fs.mkdirSync(newFilePath, { recursive: true }); + message(`Folder SEO was created in ${newFilePath}`, "yellow"); + } + fs.copyFileSync(filePath, `${newFilePath}/sitemapOld.xml`); + message(`Old sitemap detected. Moved to ${newFilePath}`, "green"); + } +}; diff --git a/packages/landing-friend-core/src/utils/index.ts b/packages/landing-friend-core/src/utils/index.ts index e4fa104..d031198 100644 --- a/packages/landing-friend-core/src/utils/index.ts +++ b/packages/landing-friend-core/src/utils/index.ts @@ -1,2 +1,3 @@ export * from "./analyzer/index.js"; -export * from "./config/index.js"; +export * from "./functions/index.js"; +export * from "./searchDuplicated/index.js"; diff --git a/packages/landing-friend-core/src/utils/searchDuplicated/functions/getContent.ts b/packages/landing-friend-core/src/utils/searchDuplicated/functions/getContent.ts new file mode 100644 index 0000000..8fcc635 --- /dev/null +++ b/packages/landing-friend-core/src/utils/searchDuplicated/functions/getContent.ts @@ -0,0 +1,54 @@ +import path from "path"; + +import { + clearContent, + DuplicatedContent, + DuplicatedContentWithName, + FileWithDuplicateContent, + readFile, +} from "@/index.js"; + +interface Props { + file: string; + input: string; +} + +const checkContent = (fileContent: string): DuplicatedContentWithName => { + let samePage: DuplicatedContent | undefined; + let sameTitle: DuplicatedContent | undefined; + let sameMetaDesc: DuplicatedContent | undefined; + + const fullContentRegex = new RegExp(`(.*?)`, "g"); + const titleRegex = new RegExp(`(.*?)`, "g"); + const descRegex = new RegExp(``, "g"); + + if (fileContent.match(fullContentRegex)) { + // const match = fileContent.match(fullContentRegex)?.[0]; + samePage = { + content: undefined, + }; + } + if (fileContent.match(titleRegex)) { + const match = titleRegex.exec(fileContent)?.[1]; + sameTitle = { + content: clearContent(match), + }; + } + if (fileContent.match(descRegex)) { + const match = fileContent.match(descRegex)?.[1]; + sameMetaDesc = { + content: clearContent(match), + }; + } + + return { sameMetaDesc, samePage, sameTitle }; +}; + +export const getContent = async ({ file, input }: Props) => { + const _fileContent = readFile(path.join(process.cwd(), input, file)); + const fileContent = _fileContent.replace(/\r?\n\s*/g, " "); + + const content: FileWithDuplicateContent = { [file]: checkContent(fileContent) }; + + return content; +}; diff --git a/packages/landing-friend-core/src/utils/searchDuplicated/functions/index.ts b/packages/landing-friend-core/src/utils/searchDuplicated/functions/index.ts new file mode 100644 index 0000000..9872275 --- /dev/null +++ b/packages/landing-friend-core/src/utils/searchDuplicated/functions/index.ts @@ -0,0 +1 @@ +export * from "./getContent.js"; diff --git a/packages/landing-friend-core/src/utils/searchDuplicated/index.ts b/packages/landing-friend-core/src/utils/searchDuplicated/index.ts new file mode 100644 index 0000000..96322e2 --- /dev/null +++ b/packages/landing-friend-core/src/utils/searchDuplicated/index.ts @@ -0,0 +1 @@ +export * from "./functions/index.js"; diff --git a/packages/landing-friend/src/CLI.ts b/packages/landing-friend/src/CLI.ts index 1e9b4d6..0eeb8f9 100644 --- a/packages/landing-friend/src/CLI.ts +++ b/packages/landing-friend/src/CLI.ts @@ -1,5 +1,11 @@ // #!/usr/bin/env node -import { message, readConfig, sitemapGenerator, websiteAnalyzer } from "@landing-friend/core"; +import { + message, + readConfig, + searchDuplicated, + sitemapGenerator, + websiteAnalyzer, +} from "@landing-friend/core"; import yargs from "yargs"; import { hideBin } from "yargs/helpers"; @@ -68,6 +74,7 @@ yargs(hideBin(process.argv)) // }, // }, // async () => { + // console.clear(); // const config = readConfig("landing-friend-config.ts"); // if (!config) { // message("Config not found", "red"); @@ -95,6 +102,7 @@ yargs(hideBin(process.argv)) }, }, async () => { + console.clear(); const config = readConfig("landing-friend-config.ts", "generate"); if (!config) { message( @@ -126,6 +134,81 @@ yargs(hideBin(process.argv)) } } ) + //!!!!!!! + // USE IT INSTEAD OF BELOW COMMAND IF U WANT CONSOLE.LOG SOMETHING + // + .command( + "duplicated", + "Find duplicated content", + { + help: { + describe: "Search the out file to find duplicated content on your site.", + }, + }, + async () => { + console.clear(); + const config = readConfig("landing-friend-config.ts", "generate"); + if (!config) { + message( + "No config detected. Please create one using init command or create it manually", + "red" + ); + return; + } + + try { + message("Searching for duplicates...", "yellow"); + await searchDuplicated(config); + } catch (e) { + const error = e as Error; + message(error.message, "red"); + return; + } finally { + process.exit(); + } + } + ) + // .command( + // "duplicated", + // "Find duplicated content", + // { + // help: { + // describe: "Search the out file to find duplicated content on your site.", + // }, + // }, + // async () => { + // console.clear(); + // const config = readConfig("landing-friend-config.ts", "generate"); + // if (!config) { + // message( + // "No config detected. Please create one using init command or create it manually", + // "red" + // ); + // return; + // } + + // const char = "."; + // const maxChar = 3; + // let progress = ""; + // const interval = setInterval(() => { + // if (progress.length < maxChar) { + // progress += char; + // console.clear(); + // message(`Searching for duplicates${progress}`, "yellow"); + // } else progress = ""; + // }, 500); + + // try { + // await searchDuplicated(config, interval); + // } catch (e) { + // const error = e as Error; + // message(error.message, "red"); + // return; + // } finally { + // process.exit(); + // } + // } + // ) .help() .showHelpOnFail(true) .strict() diff --git a/packages/landing-friend/src/functions/configInit.ts b/packages/landing-friend/src/functions/configInit.ts index 72241fc..18c4005 100644 --- a/packages/landing-friend/src/functions/configInit.ts +++ b/packages/landing-friend/src/functions/configInit.ts @@ -2,6 +2,7 @@ import { ConfigFile, EXTENDED_ADVANCED_ANALYZER_GLOBAL_CONFIG_FILE, EXTENDED_ANALYZER_GLOBAL_CONFIG_FILE, + EXTENDED_DUPLICATED_ANALYZER_CONFIG_FILE, EXTENDED_SITEMAP_GLOBAL_CONFIG_FILE, GLOBAL_CONFIG_FILE, initConfig, @@ -17,6 +18,11 @@ export const configInit = async () => { return; } + let extendResponseBySitemap: typeof EXTENDED_SITEMAP_GLOBAL_CONFIG_FILE = {}; + let extendResponseByAnalyzer: typeof EXTENDED_ANALYZER_GLOBAL_CONFIG_FILE = {}; + let extendResponseByAdvanceAnalyzer: typeof EXTENDED_ADVANCED_ANALYZER_GLOBAL_CONFIG_FILE = {}; + let extendResponseByDuplicatedAnalyzer: typeof EXTENDED_DUPLICATED_ANALYZER_CONFIG_FILE = {}; + const directories: ConfigFile = await inquirer.prompt([ { type: "input", @@ -49,7 +55,6 @@ export const configInit = async () => { default: GLOBAL_CONFIG_FILE.excludedPage, }, ]); - let extendResponseBySitemap: Pick = {}; const { extendConfigBySitemap } = await inquirer.prompt<{ extendConfigBySitemap: boolean; @@ -89,9 +94,7 @@ export const configInit = async () => { }, ]); } - let extendResponseByAnalyzer: Pick = {}; - let extendResponseByAdvanceAnalyzer: Pick = {}; const { extendConfigByAnalyzer } = await inquirer.prompt<{ extendConfigByAnalyzer: boolean; }>({ @@ -190,7 +193,7 @@ export const configInit = async () => { }>({ type: "confirm", name: "extendAdvanceAnalyzer", - message: "Does you want to enable advance analyzer?", + message: "Do you want to enable advance analyzer?", default: true, }); @@ -211,11 +214,20 @@ export const configInit = async () => { ]); } } + + extendResponseByDuplicatedAnalyzer = await inquirer.prompt({ + type: "confirm", + name: "searchDuplicated", + message: "Do you want to search for duplicated pages and content?", + default: EXTENDED_DUPLICATED_ANALYZER_CONFIG_FILE.searchDuplicated, + }); + const completeConfig: ConfigFile = { ...directories, ...extendResponseBySitemap, ...extendResponseByAnalyzer, ...extendResponseByAdvanceAnalyzer, + ...extendResponseByDuplicatedAnalyzer, }; await initConfig(completeConfig); };