From da487805b2a85dc9ddc4e802db55272f413e986d Mon Sep 17 00:00:00 2001 From: luojiyin Date: Sat, 20 Jun 2026 23:02:37 +0800 Subject: [PATCH 1/2] perf(batch-lint): process files in batches to reduce memory usage --- src/utils/batch-lint.ts | 76 +++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 41 deletions(-) diff --git a/src/utils/batch-lint.ts b/src/utils/batch-lint.ts index 12058b2..698ab7e 100644 --- a/src/utils/batch-lint.ts +++ b/src/utils/batch-lint.ts @@ -3,7 +3,6 @@ import { readFile } from 'fs/promises'; import type { LintMdRulesConfig, lintMarkdown } from '@lint-md/core'; import { Piscina } from 'piscina'; import type { LintWorkerOptions } from '../types'; -import { averagedGroup } from './averaged-group'; export async function runTasksWithLimit( tasks: (() => Promise)[], @@ -38,50 +37,45 @@ export const batchLint = async ( maxThreads: concurrency, }); - const fileContentList = await runTasksWithLimit( - mdFilePaths.map((filePath) => { - return async () => { - const res = await readFile(filePath); - return { - path: filePath, - content: res.toString(), - }; - }; - }), - concurrency - ); - - // 将 md 文件内容进行分组,供各个线程分配执行 - const markdownContentGroup = averagedGroup(fileContentList, concurrency, (item) => { - return item.content.length; - }); - - const res = await Promise.all( - markdownContentGroup.map((groupItem) => { - const asyncCall = async () => { - const batchLintResult: ReturnType[] - = await lintWorkerPool.run({ - contentList: groupItem.items.map((value) => { - return value.content; - }), - isFixMode, - rules, - isDev, - } as LintWorkerOptions); + const problemResults: { + path: string + lintResult: ReturnType['lintResult'] + fixedResult?: ReturnType['fixedResult'] + }[] = []; - return batchLintResult.map((lintResult, index) => { + for (let start = 0; start < mdFilePaths.length; start += concurrency) { + const currentBatchPaths = mdFilePaths.slice(start, start + concurrency); + const fileContentList = await runTasksWithLimit( + currentBatchPaths.map((filePath) => { + return async () => { + const content = await readFile(filePath, 'utf8'); return { - ...lintResult, - path: groupItem.items[index].path, + path: filePath, + content, }; - }); + }; + }), + concurrency + ); + + const batchLintResult = await lintWorkerPool.run({ + contentList: fileContentList.map(item => item.content), + isFixMode, + rules, + isDev, + } as LintWorkerOptions) as ReturnType[]; + + const batchResults = batchLintResult.map((lintResult, index) => { + return { + ...lintResult, + path: fileContentList[index].path, }; + }); - return asyncCall(); - }) - ); + problemResults.push(...batchResults.filter((item) => { + return item.lintResult.length > 0; + })); + } - return res.flat().filter((item) => { - return item.lintResult.length > 0; - }); + return problemResults; }; From 8224a91d7ce22c7392b6e4274e70f7c1482f1eb4 Mon Sep 17 00:00:00 2001 From: luojiyin Date: Sat, 20 Jun 2026 23:02:47 +0800 Subject: [PATCH 2/2] test(lint-worker): add batch content list test --- __tests__/lint-worker.spec.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 __tests__/lint-worker.spec.ts diff --git a/__tests__/lint-worker.spec.ts b/__tests__/lint-worker.spec.ts new file mode 100644 index 0000000..cbd027c --- /dev/null +++ b/__tests__/lint-worker.spec.ts @@ -0,0 +1,13 @@ +import lintWorker from '../src/utils/lint-worker'; + +describe('lintWorker', () => { + test('returns lint results for the provided content list', async () => { + const [result] = lintWorker({ + contentList: ['中English\n'], + isFixMode: false, + rules: {}, + }); + + expect(result.lintResult.length).toBeGreaterThan(0); + }); +});