Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add tests for build post list script #3284

Open
wants to merge 83 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
83 commits
Select commit Hold shift + click to select a range
7ae14bd
fefewg
vishvamsinh28 Oct 22, 2024
d1cb179
added logs for debugging
vishvamsinh28 Oct 22, 2024
e5d6aab
Merge branch 'master' into buildPostTest
anshgoyalevil Oct 26, 2024
cc57d88
Merge branch 'master' into buildPostTest
anshgoyalevil Oct 31, 2024
fbcd76b
lint fix
anshgoyalevil Oct 31, 2024
17dedfe
Merge branch 'master' into buildPostTest
vishvamsinh28 Nov 4, 2024
23a5c98
feqf
vishvamsinh28 Nov 4, 2024
ec110dc
feqfeq
vishvamsinh28 Nov 4, 2024
b653ecf
ffqef
vishvamsinh28 Nov 4, 2024
accec3e
path update
vishvamsinh28 Nov 4, 2024
bb4ffe0
path update
vishvamsinh28 Nov 4, 2024
c24eb8e
logs added
vishvamsinh28 Nov 4, 2024
63edd8d
fwf
vishvamsinh28 Nov 4, 2024
b87b00d
logs added
vishvamsinh28 Nov 4, 2024
0f15784
posix
vishvamsinh28 Nov 4, 2024
8f26f50
fwqfqw
vishvamsinh28 Nov 4, 2024
10453f4
path chanegs
vishvamsinh28 Nov 4, 2024
323cdc5
fqwefe
vishvamsinh28 Nov 5, 2024
bb2d6b6
used fs extra
vishvamsinh28 Nov 5, 2024
cf53980
Merge branch 'master' into buildPostTest
vishvamsinh28 Nov 8, 2024
abfea38
Merge branch 'master' into buildPostTest
vishvamsinh28 Nov 9, 2024
a9c95d9
path fix
vishvamsinh28 Nov 9, 2024
a3a8dca
qwfeg
vishvamsinh28 Nov 9, 2024
70a0cd1
remove logs
vishvamsinh28 Nov 9, 2024
1383ea6
apply coderabbit suggetions
vishvamsinh28 Nov 9, 2024
fd129fe
fge
vishvamsinh28 Nov 9, 2024
e63203f
update tests
vishvamsinh28 Nov 9, 2024
b7e8c77
tests updated
vishvamsinh28 Nov 9, 2024
4d7b5a0
fef
vishvamsinh28 Nov 9, 2024
fa51639
Merge branch 'master' into buildPostTest
vishvamsinh28 Nov 10, 2024
09b530d
updated details.slug
vishvamsinh28 Nov 10, 2024
69ca91a
quick fix
vishvamsinh28 Nov 10, 2024
b935133
Merge branch 'master' into buildPostTest
vishvamsinh28 Nov 11, 2024
0c420ef
Merge branch 'master' into buildPostTest
vishvamsinh28 Nov 13, 2024
4615938
nkn
vishvamsinh28 Nov 13, 2024
281a8c8
Merge branch 'master' into buildPostTest
vishvamsinh28 Nov 15, 2024
5ba7c2d
Merge branch 'master' into buildPostTest
vishvamsinh28 Nov 22, 2024
a1e6a45
Merge branch 'master' into buildPostTest
vishvamsinh28 Nov 23, 2024
f7cf1b6
applied suggested changes
vishvamsinh28 Nov 23, 2024
a0fe58d
Merge branch 'master' into buildPostTest
vishvamsinh28 Nov 28, 2024
dc0998f
Merge branch 'master' into buildPostTest
vishvamsinh28 Nov 29, 2024
5357992
Merge branch 'master' into buildPostTest
vishvamsinh28 Nov 29, 2024
3c6618d
Merge branch 'master' into buildPostTest
vishvamsinh28 Nov 30, 2024
cf9920e
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 2, 2024
e2868d9
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 4, 2024
49722d0
apply nitpick
vishvamsinh28 Dec 4, 2024
ba08b4c
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 5, 2024
e0ac1e1
Merge branch 'master' into buildPostTest
akshatnema Dec 7, 2024
dc7e215
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 8, 2024
f8e7dfd
format code
vishvamsinh28 Dec 8, 2024
bb4da1d
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 9, 2024
45adee8
added helper function for setup
vishvamsinh28 Dec 9, 2024
d175e5c
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 9, 2024
98231ea
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 10, 2024
4893712
apply coderabbit suggestions
vishvamsinh28 Dec 10, 2024
10d1b72
apply coderabbit suggestion
vishvamsinh28 Dec 10, 2024
4a38f98
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 12, 2024
80b04b1
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 13, 2024
8b004d2
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 14, 2024
524a23c
Merge branch 'master' into buildPostTest
akshatnema Dec 14, 2024
5bed511
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 14, 2024
89f7fbc
apply nitpicks
vishvamsinh28 Dec 14, 2024
ccc22cb
apply nitpicks
vishvamsinh28 Dec 14, 2024
c951b91
apply nitpick 2
vishvamsinh28 Dec 14, 2024
8ffb9b9
apply nitpicks 3
vishvamsinh28 Dec 14, 2024
19f7c4f
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 16, 2024
96f821b
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 16, 2024
893ac66
apply nitpicks
vishvamsinh28 Dec 16, 2024
069128a
apply nitpicks 2
vishvamsinh28 Dec 16, 2024
778b257
apply nitpicks 3
vishvamsinh28 Dec 16, 2024
a4403bd
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 17, 2024
fcff3ed
apply nitpicks
vishvamsinh28 Dec 17, 2024
48010df
apply nitpicks again
vishvamsinh28 Dec 17, 2024
440670c
apply nitpicks
vishvamsinh28 Dec 17, 2024
b47cb4b
apply nitpicks once again
vishvamsinh28 Dec 17, 2024
5d01e26
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 18, 2024
eb0e387
apply nitpicks
vishvamsinh28 Dec 18, 2024
a795ce0
apply nitpicks
vishvamsinh28 Dec 18, 2024
8f00a15
apply actionable commet
vishvamsinh28 Dec 18, 2024
3de7141
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 19, 2024
dba34ee
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 20, 2024
39b1a54
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 23, 2024
0469f4f
Merge branch 'master' into buildPostTest
vishvamsinh28 Dec 24, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
195 changes: 118 additions & 77 deletions scripts/build-post-list.js
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
const { readdirSync, statSync, existsSync, readFileSync, writeFileSync } = require('fs')
const { resolve, basename } = require('path')
const { readdir, stat, pathExists, readFile, writeFile } = require('fs-extra')
const { basename, join, normalize, sep, posix, relative, parse } = require('path')
akshatnema marked this conversation as resolved.
Show resolved Hide resolved
const frontMatter = require('gray-matter')
const toc = require('markdown-toc')
const { slugify } = require('markdown-toc/lib/utils')
const readingTime = require('reading-time')
const { markdownToTxt } = require('markdown-to-txt')
const { buildNavTree, addDocButtons } = require('./build-docs')
Expand All @@ -15,51 +14,88 @@ const result = {
docsTree: {}
}
const releaseNotes = []
const basePath = 'pages'
const postDirectories = [
// order of these directories is important, as the blog should come before docs, to create a list of available release notes, which will later be used to release-note-link for spec docs
[`${basePath}/blog`, '/blog'],
[`${basePath}/docs`, '/docs'],
[`${basePath}/about`, '/about']
];

const addItem = (details) => {
if(details.slug.startsWith('/docs'))
result["docs"].push(details)
else if(details.slug.startsWith('/blog'))
result["blog"].push(details)
else if(details.slug.startsWith('/about'))
result["about"].push(details)
else {}
const sectionMap = {
'/docs': 'docs',
'/blog': 'blog',
'/about': 'about'
};
const section = Object.keys(sectionMap).find(key => details.slug.startsWith(key));
if (section) {
result[sectionMap[section]].push(details);
}
};

function getVersionDetails(slug, weight) {
const fileBaseName = basename(slug);
const versionName = fileBaseName.split('-')[0];
return {
title: versionName.startsWith('v')
? capitalize(versionName.slice(1))
: capitalize(versionName),
weight
};
}

module.exports = async function buildPostList() {
walkDirectories(postDirectories, result)
const treePosts = buildNavTree(result["docs"].filter((p) => p.slug.startsWith('/docs/')))
result["docsTree"] = treePosts
result["docs"] = addDocButtons(result["docs"], treePosts)
if (process.env.NODE_ENV === 'production') {
// console.log(inspect(result, { depth: null, colors: true }))
/**
* Builds a list of posts from the specified directories and writes it to a file
* @param {Array<Array<string>>} postDirectories - Array of [directory, slug] tuples
* @param {string} basePath - Base path for resolving relative paths
* @param {string} writeFilePath - Path where the output JSON will be written
* @throws {Error} If required parameters are missing or if any operation fails
* @returns {Promise<void>}
*/
async function buildPostList(postDirectories, basePath, writeFilePath) {
try {

if (!basePath) {
throw new Error('Error while building post list: basePath is required');
}

if (!writeFilePath) {
throw new Error('Error while building post list: writeFilePath is required');
}

if (postDirectories.length === 0) {
throw new Error('Error while building post list: postDirectories array is empty');
}
const normalizedBasePath = normalize(basePath)
await walkDirectories(postDirectories, result, normalizedBasePath)
const treePosts = buildNavTree(result.docs.filter((p) => p.slug.startsWith('/docs/')))
result.docsTree = treePosts
result.docs = addDocButtons(result.docs, treePosts)
await writeFile(writeFilePath, JSON.stringify(result, null, ' '))
} catch (error) {
throw new Error(`Error while building post list: ${error.message}`, { cause: error });
}
writeFileSync(resolve(__dirname, '..', 'config', 'posts.json'), JSON.stringify(result, null, ' '))
}

function walkDirectories(directories, result, sectionWeight = 0, sectionTitle, sectionId, rootSectionId) {
async function walkDirectories(
directories,
resultObj,
basePath,
sectionTitle,
sectionId,
rootSectionId,
sectionWeight = 0
) {
for (let dir of directories) {
let directory = dir[0]
let sectionSlug = dir[1] || ''
let files = readdirSync(directory);
const directory = posix.normalize(dir[0]);
const sectionSlug = dir[1] || '';
const files = await readdir(directory)

for (let file of files) {
let details
const fileName = [directory, file].join('/')
const fileNameWithSection = [fileName, '_section.mdx'].join('/')
const slug = fileName.replace(new RegExp(`^${basePath}`), '')
const slugElements = slug.split('/');
if (isDirectory(fileName)) {
if (existsSync(fileNameWithSection)) {
let details;
const fileName = normalize(join(directory, file));
const fileNameWithSection = normalize(join(fileName, '_section.mdx'))
const slug = `/${normalize(relative(basePath, fileName)).replace(/\\/g, '/')}`
const slugElements = slug.split('/')

if (await isDirectory(fileName)) {
if (await pathExists(fileNameWithSection)) {
// Passing a second argument to frontMatter disables cache. See https://github.com/asyncapi/website/issues/1057
details = frontMatter(readFileSync(fileNameWithSection, 'utf-8'), {}).data
details = frontMatter(await readFile(fileNameWithSection, 'utf-8'), {}).data
details.title = details.title || capitalize(basename(fileName))
} else {
details = {
Expand All @@ -68,8 +104,8 @@ function walkDirectories(directories, result, sectionWeight = 0, sectionTitle, s
}
details.isSection = true
if (slugElements.length > 3) {
details.parent = slugElements[slugElements.length - 2]
details.sectionId = slugElements[slugElements.length - 1]
details.parent = slugElements[slugElements.length - 2]
details.sectionId = slugElements[slugElements.length - 1]
}
if (!details.parent) {
details.isRootSection = true
Expand All @@ -79,9 +115,9 @@ function walkDirectories(directories, result, sectionWeight = 0, sectionTitle, s
details.slug = slug
addItem(details)
const rootId = details.parent || details.rootSectionId
walkDirectories([[fileName, slug]], result, details.weight, details.title, details.sectionId, rootId)
} else if (file.endsWith('.mdx') && !fileName.endsWith('/_section.mdx')) {
const fileContent = readFileSync(fileName, 'utf-8')
await walkDirectories([[fileName, slug]], resultObj, basePath, details.title, details.sectionId, rootId, details.sectionWeight)
} else if (file.endsWith('.mdx') && !fileName.endsWith(sep + '_section.mdx')) {
const fileContent = await readFile(fileName, 'utf-8')
// Passing a second argument to frontMatter disables cache. See https://github.com/asyncapi/website/issues/1057
const { data, content } = frontMatter(fileContent, {})
details = data
Expand All @@ -93,21 +129,16 @@ function walkDirectories(directories, result, sectionWeight = 0, sectionTitle, s
details.sectionTitle = sectionTitle
details.sectionId = sectionId
details.rootSectionId = rootSectionId
details.id = fileName
details.isIndex = fileName.endsWith('/index.mdx')
details.id = fileName.replace(/\\/g, '/')
details.isIndex = fileName.endsWith(join('index.mdx'))
details.slug = details.isIndex ? sectionSlug : slug.replace(/\.mdx$/, '')
if(details.slug.includes('/reference/specification/') && !details.title) {
const fileBaseName = basename(data.slug) // ex. v2.0.0 | v2.1.0-next-spec.1
const fileName = fileBaseName.split('-')[0] // v2.0.0 | v2.1.0
details.weight = specWeight--

if (fileName.startsWith('v')) {
details.title = capitalize(fileName.slice(1))
} else {
details.title = capitalize(fileName)
}
if (details.slug.includes('/reference/specification/') && !details.title) {
const fileBaseName = basename(details.slug)
const versionDetails = getVersionDetails(details.slug, specWeight--);
details.title = versionDetails.title;
details.weight = versionDetails.weight;

if(releaseNotes.includes(details.title)){
if (releaseNotes.includes(details.title)) {
details.releaseNoteLink = `/blog/release-notes-${details.title}`
}

Expand All @@ -122,14 +153,10 @@ function walkDirectories(directories, result, sectionWeight = 0, sectionTitle, s
}

// To create a list of available ReleaseNotes list, which will be used to add details.releaseNoteLink attribute.
if(file.startsWith("release-notes") && dir[1] === "/blog"){
const fileName_without_extension = file.slice(0,-4)
// removes the file extension. For example, release-notes-2.1.0.md -> release-notes-2.1.0
const version = fileName_without_extension.slice(fileName_without_extension.lastIndexOf("-")+1)

// gets the version from the name of the releaseNote .md file (from /blog). For example, version = 2.1.0 if fileName_without_extension = release-notes-2.1.0
releaseNotes.push(version)
// releaseNotes is the list of all available releaseNotes
if (file.startsWith('release-notes') && dir[1] === '/blog') {
const { name } = parse(file);
const version = name.split('-').pop();
releaseNotes.push(version);
}

addItem(details)
Expand All @@ -138,24 +165,38 @@ function walkDirectories(directories, result, sectionWeight = 0, sectionTitle, s
}
}

// Matches heading IDs in two formats:
// 1. {#my-heading-id}
// 2. <a name="my-heading-id">
const HEADING_ID_REGEX = /[\s]*(?:\{#([a-zA-Z0-9\-_]+)\}|<a[\s]+name="([a-zA-Z0-9\-_]+)")/;

/**
* Extracts heading IDs from markdown headings
* @param {string} str - The heading text containing potential ID
* @returns {string} The extracted ID or empty string if no valid ID found
*/
function slugifyToC(str) {
let slug
// Try to match heading ids like {# myHeadingId}
const headingIdMatch = str.match(/[\s]?\{\#([\w\d\-_]+)\}/)
if (headingIdMatch && headingIdMatch.length >= 2) {
slug = headingIdMatch[1]
} else {
// Try to match heading ids like {<a name="myHeadingId"/>}
const anchorTagMatch = str.match(/[\s]*<a[\s]+name="([\w\d\s\-_]+)"/)
if (anchorTagMatch && anchorTagMatch.length >= 2) slug = anchorTagMatch[1]
}
return slug || slugify(str, { firsth1: true, maxdepth: 6 })
if (typeof str !== 'string') return '';
if (!str.trim()) return '';
let slug = '';

// Match heading IDs like {# myHeadingId}
const idMatch = str.match(HEADING_ID_REGEX);
const [, headingId, anchorId] = idMatch || [];
slug = (headingId || anchorId || '').trim();

// If no valid ID is found, return an empty string
return slug;
}

function isDirectory(dir) {
return statSync(dir).isDirectory()
async function isDirectory(dir) {
return (await stat(dir)).isDirectory()
}

function capitalize(text) {
return text.split(/[\s\-]/g).map(word => `${word[0].toUpperCase()}${word.substr(1)}`).join(' ')
return text.split(/[\s-]/)
.map(word => word.charAt(0).toUpperCase() + word.slice(1))
.join(' ');
}

module.exports = { slugifyToC, buildPostList }
14 changes: 12 additions & 2 deletions scripts/index.js
Original file line number Diff line number Diff line change
@@ -1,13 +1,23 @@
const { resolve } = require('path');
const fs = require('fs');
const rssFeed = require('./build-rss');
const buildPostList = require('./build-post-list');
const { buildPostList } = require('./build-post-list');
const buildCaseStudiesList = require('./casestudies');
const buildAdoptersList = require('./adopters');
const buildFinanceInfoList = require('./finance');

async function start() {
await buildPostList();

const postDirectories = [
['pages/blog', '/blog'],
['pages/docs', '/docs'],
['pages/about', '/about']
];
const basePath = 'pages';
const writeFilePath = resolve(__dirname, '../config', 'posts.json');

await buildPostList(postDirectories, basePath, writeFilePath);

rssFeed(
'blog',
'AsyncAPI Initiative Blog RSS Feed',
Expand Down
Loading
Loading