From d7ec670335bc69897f007014b1b2e68feec10f8e Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Thu, 3 Apr 2025 09:42:25 +0100 Subject: [PATCH 1/5] chore: refactor the `generateBlogData` function (#7607) --- apps/site/next-data/generators/blogData.mjs | 107 ++++++++++---------- 1 file changed, 52 insertions(+), 55 deletions(-) diff --git a/apps/site/next-data/generators/blogData.mjs b/apps/site/next-data/generators/blogData.mjs index c03e23c62a48a..0d34f9d348548 100644 --- a/apps/site/next-data/generators/blogData.mjs +++ b/apps/site/next-data/generators/blogData.mjs @@ -11,11 +11,6 @@ import { getMarkdownFiles } from '../../next.helpers.mjs'; // gets the current blog path based on local module path const blogPath = join(process.cwd(), 'pages/en/blog'); -/** - * This contains the metadata of all available blog categories - */ -const blogCategories = new Set(['all']); - /** * This method parses the source (raw) Markdown content into Frontmatter * and returns basic information for blog posts @@ -39,12 +34,6 @@ const getFrontMatter = (filename, source) => { // all = (all blog posts), publish year and the actual blog category const categories = [category, `year-${publishYear}`, 'all']; - // we add the year to the categories set - blogCategories.add(`year-${publishYear}`); - - // we add the category to the categories set - blogCategories.add(category); - // this is the url used for the blog post it based on the category and filename const slug = `/blog/${category}/${basename(filename, extname(filename))}`; @@ -63,50 +52,58 @@ const generateBlogData = async () => { '**/index.md', ]); - return new Promise(resolve => { - const posts = []; - const rawFrontmatter = []; - - filenames.forEach(filename => { - // We create a stream for reading a file instead of reading the files - const _stream = createReadStream(join(blogPath, filename)); - - // We create a readline interface to read the file line-by-line - const _readLine = readline.createInterface({ input: _stream }); - - // Creates an array of the metadata based on the filename - // This prevents concurrency issues since the for-loop is synchronous - // and these event listeners are not - rawFrontmatter[filename] = [0, '']; - - // We read line by line - _readLine.on('line', line => { - rawFrontmatter[filename][1] += `${line}\n`; - - // We observe the frontmatter separators - if (line === '---') { - rawFrontmatter[filename][0] += 1; - } - - // Once we have two separators we close the readLine and the stream - if (rawFrontmatter[filename][0] === 2) { - _readLine.close(); - _stream.close(); - } - }); - - // Then we parse gray-matter on the frontmatter - // This allows us to only read the frontmatter part of each file - // and optimise the read-process as we have thousands of markdown files - _readLine.on('close', () => { - posts.push(getFrontMatter(filename, rawFrontmatter[filename][1])); - - if (posts.length === filenames.length) { - resolve({ categories: [...blogCategories], posts }); - } - }); - }); - }); + /** + * This contains the metadata of all available blog categories + */ + const blogCategories = new Set(['all']); + + const posts = await Promise.all( + filenames.map( + filename => + new Promise(resolve => { + // We create a stream for reading a file instead of reading the files + const _stream = createReadStream(join(blogPath, filename)); + + // We create a readline interface to read the file line-by-line + const _readLine = readline.createInterface({ input: _stream }); + + let rawFrontmatter = ''; + let frontmatterSeparatorsEncountered = 0; + + // We read line by line + _readLine.on('line', line => { + rawFrontmatter += `${line}\n`; + + // We observe the frontmatter separators + if (line === '---') { + frontmatterSeparatorsEncountered++; + } + + // Once we have two separators we close the readLine and the stream + if (frontmatterSeparatorsEncountered === 2) { + _readLine.close(); + _stream.close(); + } + }); + + // Then we parse gray-matter on the frontmatter + // This allows us to only read the frontmatter part of each file + // and optimise the read-process as we have thousands of markdown files + _readLine.on('close', () => { + const frontMatterData = getFrontMatter(filename, rawFrontmatter); + + frontMatterData.categories.forEach(category => { + // we add the category to the categories set + blogCategories.add(category); + }); + + resolve(frontMatterData); + }); + }) + ) + ); + + return { categories: [...blogCategories], posts }; }; export default generateBlogData; From 07837092b2f9f53bae9531622160541915e2861b Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Sat, 5 Apr 2025 23:41:17 +0100 Subject: [PATCH 2/5] add unit tests for `generateBlogData` --- .../generators/__tests__/blogData.test.mjs | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 apps/site/next-data/generators/__tests__/blogData.test.mjs diff --git a/apps/site/next-data/generators/__tests__/blogData.test.mjs b/apps/site/next-data/generators/__tests__/blogData.test.mjs new file mode 100644 index 0000000000000..5b113ed3af1f5 --- /dev/null +++ b/apps/site/next-data/generators/__tests__/blogData.test.mjs @@ -0,0 +1,189 @@ +import { Readable } from 'node:stream'; + +import generateBlogData from '@/next-data/generators/blogData.mjs'; + +let files = []; + +jest.mock('node:path', () => ({ + ...jest.requireActual('node:path'), + join: jest.fn((_base, filePath) => filePath), +})); + +jest.mock('node:fs', () => { + const originalFs = jest.requireActual('node:fs'); + return { + ...originalFs, + createReadStream: jest.fn(filename => { + const readable = new Readable(); + const file = files.find(f => f.path === filename); + readable.push(`---\n`); + file.frontMatterContent.forEach(line => readable.push(`${line}\n`)); + readable.push(`---\n`); + readable.push(null); + readable.close = () => {}; + return readable; + }), + }; +}); + +jest.mock('../../../next.helpers.mjs', () => { + const originalHelpers = jest.requireActual('../../../next.helpers.mjs'); + return { + ...originalHelpers, + getMarkdownFiles: () => Promise.resolve(files.map(file => file.path)), + }; +}); + +describe('generateBlogData', () => { + it('should return zero posts and only the default "all" category is no md file is found', async () => { + files = []; + + const blogData = await generateBlogData(); + + expect(blogData.categories).toStrictEqual(['all']); + expect(blogData.posts).toStrictEqual([]); + }); + + it('should collect the data from a single md file if only one is found', async () => { + files = [ + { + path: 'pages/en/blog/post1.md', + frontMatterContent: [ + `date: '2020-01-01T00:00:00.000Z'`, + `title: POST 1`, + `author: author`, + ], + }, + ]; + + const blogData = await generateBlogData(); + + expect(blogData.posts.length).toBe(1); + const post = blogData.posts[0]; + expect(post.title).toEqual('POST 1'); + expect(post.date).toEqual(new Date('2020-01-01T00:00:00.000Z')); + expect(post.author).toEqual('author'); + }); + + it('should collect the data from multiple md files', async () => { + const currentDate = new Date(); + + files = [ + { + path: 'pages/en/blog/post1.md', + frontMatterContent: [ + `date: '2020-01-01T00:00:00.000Z'`, + `title: POST 1`, + `author: author-a`, + ], + }, + { + path: 'pages/en/blog/post2.md', + frontMatterContent: [ + `date: '2020-01-02T00:00:00.000Z'`, + `title: POST 2`, + `author: author-b`, + ], + }, + { + path: 'pages/en/blog/post3.md', + frontMatterContent: [ + // no date specified (the date defaults to the current date) + `title: POST 3`, + `author: author-c`, + ], + }, + ]; + + const blogData = await generateBlogData(); + + expect(blogData.posts.length).toBe(3); + expect(blogData.posts[0].title).toEqual('POST 1'); + expect(blogData.posts[0].date).toEqual( + new Date('2020-01-01T00:00:00.000Z') + ); + expect(blogData.posts[0].author).toEqual('author-a'); + expect(blogData.posts[1].title).toEqual('POST 2'); + expect(blogData.posts[1].date).toEqual( + new Date('2020-01-02T00:00:00.000Z') + ); + expect(blogData.posts[1].author).toEqual('author-b'); + expect(blogData.posts[2].title).toEqual('POST 3'); + expect(blogData.posts[2].date.setMilliseconds(0)).toEqual( + currentDate.setMilliseconds(0) + ); + expect(blogData.posts[2].author).toEqual('author-c'); + }); + + it('should generate categories based on the categories of md files and their years', async () => { + files = [ + { + path: 'pages/en/blog/post1.md', + frontMatterContent: [ + "date: '2020-01-01T00:00:00.000Z'", + 'category: category-a', + ], + }, + { + path: 'pages/en/blog/sub-dir/post2.md', + frontMatterContent: [ + "date: '2020-01-02T00:00:00.000Z'", + 'category: category-b', + ], + }, + { + path: 'pages/en/blog/post3.md', + frontMatterContent: [ + "date: '2021-03-13T00:00:00.000Z'", + // no category specified (it should be "uncategorized") + ], + }, + { + path: 'pages/en/blog/post4.md', + frontMatterContent: [ + // no date specified (the date defaults to the current date) + 'category: category-b', + ], + }, + ]; + + const blogData = await generateBlogData(); + + expect(blogData.categories.sort()).toStrictEqual([ + 'all', + 'category-a', + 'category-b', + 'uncategorized', + 'year-2020', + 'year-2021', + `year-${new Date().getUTCFullYear()}`, + ]); + }); + + it('should generate slugs based on the md filenames and categories', async () => { + files = [ + { + path: 'pages/en/blog/post1.md', + frontMatterContent: ['category: category-a'], + }, + { + path: 'pages/en/blog/post2.md', + frontMatterContent: ['category: category-b'], + }, + { + path: 'pages/en/blog/post3.md', + frontMatterContent: [ + // no category specified + ], + }, + ]; + + const blogData = await generateBlogData(); + + expect(blogData.posts.map(p => p.slug).sort()).toStrictEqual([ + '/blog/category-a/post1', + '/blog/category-b/post2', + '/blog/uncategorized/post3', + ]); + }); +}); From 489e604a8862c1338cd9c3d2e92e8b81efdec5ff Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Sun, 6 Apr 2025 00:37:46 +0100 Subject: [PATCH 3/5] avoid mocking node:path --- .../site/next-data/generators/__tests__/blogData.test.mjs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/apps/site/next-data/generators/__tests__/blogData.test.mjs b/apps/site/next-data/generators/__tests__/blogData.test.mjs index 5b113ed3af1f5..aace818c0266e 100644 --- a/apps/site/next-data/generators/__tests__/blogData.test.mjs +++ b/apps/site/next-data/generators/__tests__/blogData.test.mjs @@ -1,21 +1,17 @@ +import { normalize } from 'node:path'; import { Readable } from 'node:stream'; import generateBlogData from '@/next-data/generators/blogData.mjs'; let files = []; -jest.mock('node:path', () => ({ - ...jest.requireActual('node:path'), - join: jest.fn((_base, filePath) => filePath), -})); - jest.mock('node:fs', () => { const originalFs = jest.requireActual('node:fs'); return { ...originalFs, createReadStream: jest.fn(filename => { const readable = new Readable(); - const file = files.find(f => f.path === filename); + const file = files.find(f => filename.endsWith(normalize(f.path))); readable.push(`---\n`); file.frontMatterContent.forEach(line => readable.push(`${line}\n`)); readable.push(`---\n`); From 39dbb13b1a39ac6ca513351ea4825ae2f92e29c6 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Sun, 6 Apr 2025 00:41:58 +0100 Subject: [PATCH 4/5] remove original modules --- .../generators/__tests__/blogData.test.mjs | 40 ++++++++----------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/apps/site/next-data/generators/__tests__/blogData.test.mjs b/apps/site/next-data/generators/__tests__/blogData.test.mjs index aace818c0266e..450aba17dcc0c 100644 --- a/apps/site/next-data/generators/__tests__/blogData.test.mjs +++ b/apps/site/next-data/generators/__tests__/blogData.test.mjs @@ -5,30 +5,22 @@ import generateBlogData from '@/next-data/generators/blogData.mjs'; let files = []; -jest.mock('node:fs', () => { - const originalFs = jest.requireActual('node:fs'); - return { - ...originalFs, - createReadStream: jest.fn(filename => { - const readable = new Readable(); - const file = files.find(f => filename.endsWith(normalize(f.path))); - readable.push(`---\n`); - file.frontMatterContent.forEach(line => readable.push(`${line}\n`)); - readable.push(`---\n`); - readable.push(null); - readable.close = () => {}; - return readable; - }), - }; -}); - -jest.mock('../../../next.helpers.mjs', () => { - const originalHelpers = jest.requireActual('../../../next.helpers.mjs'); - return { - ...originalHelpers, - getMarkdownFiles: () => Promise.resolve(files.map(file => file.path)), - }; -}); +jest.mock('node:fs', () => ({ + createReadStream: jest.fn(filename => { + const readable = new Readable(); + const file = files.find(f => filename.endsWith(normalize(f.path))); + readable.push(`---\n`); + file.frontMatterContent.forEach(line => readable.push(`${line}\n`)); + readable.push(`---\n`); + readable.push(null); + readable.close = () => {}; + return readable; + }), +})); + +jest.mock('../../../next.helpers.mjs', () => ({ + getMarkdownFiles: () => Promise.resolve(files.map(file => file.path)), +})); describe('generateBlogData', () => { it('should return zero posts and only the default "all" category is no md file is found', async () => { From 7090c70dda17ec5baec13cb08431967ad53a4678 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Thu, 24 Apr 2025 01:47:15 +0200 Subject: [PATCH 5/5] migrate to node test runner --- .../generators/__tests__/blogData.test.mjs | 106 ++++++++++++------ 1 file changed, 69 insertions(+), 37 deletions(-) diff --git a/apps/site/next-data/generators/__tests__/blogData.test.mjs b/apps/site/next-data/generators/__tests__/blogData.test.mjs index 450aba17dcc0c..0825eac35f248 100644 --- a/apps/site/next-data/generators/__tests__/blogData.test.mjs +++ b/apps/site/next-data/generators/__tests__/blogData.test.mjs @@ -1,35 +1,64 @@ +import assert from 'node:assert/strict'; import { normalize } from 'node:path'; import { Readable } from 'node:stream'; - -import generateBlogData from '@/next-data/generators/blogData.mjs'; +import { describe, it, mock } from 'node:test'; let files = []; -jest.mock('node:fs', () => ({ - createReadStream: jest.fn(filename => { - const readable = new Readable(); - const file = files.find(f => filename.endsWith(normalize(f.path))); - readable.push(`---\n`); - file.frontMatterContent.forEach(line => readable.push(`${line}\n`)); - readable.push(`---\n`); - readable.push(null); - readable.close = () => {}; - return readable; - }), -})); - -jest.mock('../../../next.helpers.mjs', () => ({ - getMarkdownFiles: () => Promise.resolve(files.map(file => file.path)), -})); +mock.module('node:fs', { + namedExports: { + createReadStream: filename => { + const readable = new Readable(); + const file = files.find(f => filename.endsWith(normalize(f.path))); + readable.push(`---\n`); + file.frontMatterContent.forEach(line => readable.push(`${line}\n`)); + readable.push(`---\n`); + readable.push(null); + readable.close = () => {}; + return readable; + }, + }, +}); + +mock.module('../../../next.helpers.mjs', { + namedExports: { + getMarkdownFiles: () => { + return Promise.resolve(files.map(file => file.path)); + }, + }, +}); + +const generateBlogData = (await import('../blogData.mjs')).default; describe('generateBlogData', () => { - it('should return zero posts and only the default "all" category is no md file is found', async () => { + it('should return zero posts and only the default "all" category if no md file is found', async () => { files = []; const blogData = await generateBlogData(); - expect(blogData.categories).toStrictEqual(['all']); - expect(blogData.posts).toStrictEqual([]); + assert.deepEqual(blogData.categories, ['all']); + assert.deepEqual(blogData.posts, []); + }); + + it('should collect the data from a single md file if only one is found', async () => { + files = [ + { + path: 'pages/en/blog/post1.md', + frontMatterContent: [ + `date: '2020-01-01T00:00:00.000Z'`, + `title: POST 1`, + `author: author`, + ], + }, + ]; + + const blogData = await generateBlogData(); + + assert.equal(blogData.posts.length, 1); + const post = blogData.posts[0]; + assert.equal(post.title, 'POST 1'); + assert.deepEqual(post.date, new Date('2020-01-01T00:00:00.000Z')); + assert.equal(post.author, 'author'); }); it('should collect the data from a single md file if only one is found', async () => { @@ -46,11 +75,11 @@ describe('generateBlogData', () => { const blogData = await generateBlogData(); - expect(blogData.posts.length).toBe(1); + assert.equal(blogData.posts.length, 1); const post = blogData.posts[0]; - expect(post.title).toEqual('POST 1'); - expect(post.date).toEqual(new Date('2020-01-01T00:00:00.000Z')); - expect(post.author).toEqual('author'); + assert.equal(post.title, 'POST 1'); + assert.deepEqual(post.date, new Date('2020-01-01T00:00:00.000Z')); + assert.equal(post.author, 'author'); }); it('should collect the data from multiple md files', async () => { @@ -85,22 +114,25 @@ describe('generateBlogData', () => { const blogData = await generateBlogData(); - expect(blogData.posts.length).toBe(3); - expect(blogData.posts[0].title).toEqual('POST 1'); - expect(blogData.posts[0].date).toEqual( + assert(blogData.posts.length, 3); + assert.equal(blogData.posts[0].title, 'POST 1'); + assert.deepEqual( + blogData.posts[0].date, new Date('2020-01-01T00:00:00.000Z') ); - expect(blogData.posts[0].author).toEqual('author-a'); - expect(blogData.posts[1].title).toEqual('POST 2'); - expect(blogData.posts[1].date).toEqual( + assert.equal(blogData.posts[0].author, 'author-a'); + assert.equal(blogData.posts[1].title, 'POST 2'); + assert.deepEqual( + blogData.posts[1].date, new Date('2020-01-02T00:00:00.000Z') ); - expect(blogData.posts[1].author).toEqual('author-b'); - expect(blogData.posts[2].title).toEqual('POST 3'); - expect(blogData.posts[2].date.setMilliseconds(0)).toEqual( + assert.equal(blogData.posts[1].author, 'author-b'); + assert.equal(blogData.posts[2].title, 'POST 3'); + assert.equal( + blogData.posts[2].date.setMilliseconds(0), currentDate.setMilliseconds(0) ); - expect(blogData.posts[2].author).toEqual('author-c'); + assert.equal(blogData.posts[2].author, 'author-c'); }); it('should generate categories based on the categories of md files and their years', async () => { @@ -137,7 +169,7 @@ describe('generateBlogData', () => { const blogData = await generateBlogData(); - expect(blogData.categories.sort()).toStrictEqual([ + assert.deepEqual(blogData.categories.sort(), [ 'all', 'category-a', 'category-b', @@ -168,7 +200,7 @@ describe('generateBlogData', () => { const blogData = await generateBlogData(); - expect(blogData.posts.map(p => p.slug).sort()).toStrictEqual([ + assert.deepEqual(blogData.posts.map(p => p.slug).sort(), [ '/blog/category-a/post1', '/blog/category-b/post2', '/blog/uncategorized/post3',