Convert repo IO and processing to async/await

package.json

@@ -19,8 +19,8 @@
         "luxon": "^3.7.2",
         "semver": "^7.7.3",
         "shelljs": "^0.10.0",
-        "simple-git": "^3.32.3",
-        "uuid": "11.1.0",
+        "simple-git": "^3.36.0",
+        "uuid": "11.1.1",
         "yargs": "^17.7.2"
     },
     "devDependencies": {

src/common/engine.js

@@ -137,7 +137,7 @@ const versionStringToNumber = (version) => {
         // eslint-disable-next-line no-magic-numbers
         _expanded += i === 0 ? s : s.padStart(NumbersPadding, '0');
     });
-    return parseInt(_expanded, 10);
+    return Number.parseInt(_expanded, 10);
 };
 
 /**
@@ -153,7 +153,7 @@ const versionStringToObject = (v, versions, oldest = false) => {
     const matchingVersions = versions.filter(({ version }) => rx.test(version));
     // versions array is sorted in version descending order
     if(oldest) {
-        return matchingVersions[matchingVersions.length - 1];
+        return matchingVersions.at(-1);
     }
     return matchingVersions[0];
 };
@@ -248,23 +248,23 @@ const maxInstalledSatisfyingVersion = (requiredRange) =>
  */
 const minInstalledSatisfyingVersion = (requiredRange) => {
     const version = module.exports.satisfyingVersions(requiredRange);
-    return version[version.length - 1];
+    return version.at(-1);
 };
 
 /**
  * Obtains node engine range
  *
  * @param {String} repoPath - to repository
- * @returns {String} engines | default engine
+ * @returns {Promise<string>} engines | default engine
  * @throws RangeError
  */
-const repositoryEngines = (repoPath) => {
+const repositoryEngines = async (repoPath) => {
     const file = repoPath.endsWith('package.json') ? repoPath : resolve(join(repoPath, 'package.json'));
-    const { error, engines } = getPackage(file);
+    const { error, engines } = await getPackage(file);
     if(error) {
         throw new RangeError(`package file not found in ${repoPath}`);
     }
-    if(engines && engines.node) {
+    if(engines?.node) {
         return engines.node;
     }
     return defaultVersion;
@@ -277,11 +277,11 @@ const repositoryEngines = (repoPath) => {
  * @param {String=} param0.version - version number x.y.z
  * @param {Boolean=} param0.oldest - choose oldest acceptable version
  * @param {Boolean=} noPackage - path does not have package.json
- * @returns {Version}
+ * @returns {Promise<Version>} version object with path to executables
  * @throws RangeError
  */
-const versionToUseValidator = ({ path, version, oldest }, noPackage) => {
-    const repoEngines = noPackage ? version : module.exports.repositoryEngines(path);
+const versionToUseValidator = async ({ path, version, oldest }, noPackage) => {
+    const repoEngines = noPackage ? version : await module.exports.repositoryEngines(path);
     const repoName = basename(path);
 
     if(version) {
@@ -292,9 +292,9 @@ const versionToUseValidator = ({ path, version, oldest }, noPackage) => {
             oldest
         );
         // _version is undefined if version is not in satisfies
-        const found = satisfies.filter(
-            (v) => _version && v.version === _version.version
-        )[0];
+        const found = satisfies.find(
+            (v) => v.version === _version?.version
+        );
         if(!found) {
             throw new RangeError(
                 `${repoName} requires NodeJS version(s) '${repoEngines}' but got '${version}'`

src/common/repos.js

@@ -1,7 +1,7 @@
 /* eslint-disable camelcase */
 
-const { readFileSync, readdirSync } = require('fs');
-const { basename, join } = require('path');
+const { readFile, readdir } = require('node:fs/promises');
+const { basename, join } = require('node:path');
 const { fileExists, folderExists } = require('./files');
 require('dotenv').config();
 
@@ -11,12 +11,12 @@ const { DEVROOT } = process.env;
 /**
  * Obtains the package.json file from repo path
  *
- * @param {String} pkgFile file - path/package.json
- * @returns {Object} JSON object
+ * @param {string} pkgFile file - path/package.json
+ * @returns {Promise<object>} JSON object
  */
-const getPackage = (pkgFile) => {
+const getPackage = async (pkgFile) => {
     if(fileExists(pkgFile)) {
-        const data = readFileSync(pkgFile).toString();
+        const data = (await readFile(pkgFile)).toString();
         return JSON.parse(data);
     }
     return { error: true };
@@ -25,42 +25,46 @@ const getPackage = (pkgFile) => {
 /**
  * Determine if a folder contains a .git folder
  *  - does not check that git can process the repo so we can still get false positives
- * @param  {String}  path
- * @return {Boolean}
+ * @param  {string}  path
+ * @return {boolean}
  */
 const isGitRepo = (path) => (basename(path) === '.git') && folderExists(path);
 
 /**
  * Obtains paths of all git repositories
  *  - only search down one folder
- * @param {String} folder
- * @param {Array} foldersToInclude
- * @return {Array<string>}  path strings
+ * @param {string} folder
+ * @param {string[]} foldersToInclude - limit the folders to  return to those in this list. If empty, return all repos
+ * @return {Promise<string[]>}  path strings
  */
-const allRepoPaths = (folder = DEVROOT, foldersToInclude = []) => {
+const allRepoPaths = async (folder = DEVROOT, foldersToInclude = []) => {
     // get files in root
-    return readdirSync(folder)
-        .filter(name => {
-            if(foldersToInclude.length) {
-                return foldersToInclude.includes(name);
-            }
-            return true;
-        })
-        .map(name => join(folder, name))
-        .filter(folderExists)
-        .filter(path => {
-            // returns array of path that end a folder named .git
-            const result = readdirSync(path)
-                .reduce((accumulator, current) => {
-                    const subFolder = join(path, current);
-                    // this is not fool-proof ideally we want to skip bogus repos
-                    if(isGitRepo(subFolder)) {
-                        accumulator.push(subFolder);
-                    }
-                    return accumulator;
-                }, [])[0];
-            return result !== undefined;
+
+    const initialNames = await readdir(folder);
+    const filtered = initialNames.reduce((acc, name) => {
+        if(foldersToInclude.length && !foldersToInclude.includes(name)) {
+            return acc;
+        }
+        const path = join(folder, name);
+        if(folderExists(path)) {
+            acc.push(path);
+        }
+        return acc;
+    }, []);
+
+    const result = [];
+    for(const path of filtered) {
+        const items = await readdir(path);
+        const hasGitRepo = items.some(current => {
+            const subFolder = join(path, current);
+            return isGitRepo(subFolder);
         });
+        if(hasGitRepo) {
+            result.push(path);
+        }
+    }
+
+    return result;
 };
 
 

src/grefplus/cmdline.js

@@ -59,9 +59,9 @@ const validateDate = (checkDate, msg) => {
         return true;
     }
     catch (e) {
-        const error = new Error();
-        error.message = `${msg} not recognized as a date [${checkDate}]. Valid formats are: ${options.allowedFormat}`;
-        error.cause = e.message;
+        const message = `${msg} not recognized as a date [${checkDate}]. Valid formats are: ${options.allowedFormat}`;
+        const error = new Error(message);
+        error.cause = e;
         throw error;
     }
 };

src/grefplus/index.js

@@ -1,9 +1,9 @@
 /* eslint no-console:off */
 require('./cmdline').setOptions();
-const { basename } = require('path');
+const { basename } = require('node:path');
 const { allRepoPaths } = require('../common/repos');
-const { promisify } = require('util');
-const _exec = promisify(require('child_process').exec);
+const { promisify } = require('node:util');
+const _exec = promisify(require('node:child_process').exec);
 const { DateTime } = require('luxon');
 const { options } = require('./cmdline');
 const DateLength = 6;
@@ -22,10 +22,10 @@ const gitCommand = (repo) => {
 /**
  * determines if item falls within range
  *
- * @param {Object} item
+ * @param {object} item
  * @param {DateTime | undefined} item.fromDate
  * @param {DateTime | undefined} item.toDate
- * @returns {Boolean}
+ * @returns {boolean}
  * @private
  *
  */
@@ -48,38 +48,39 @@ const filterPeriod = (item) => {
 
 /**
  * Obtains the git reflogs result
- * @param  {String} repo - full path to a repository
- * @param  {Array}  errors - place to store skippable errors
- * @return {Array}  objects containing date, body, and the repository base name
+ * @param  {string} repo - full path to a repository
+ * @param  {{repo:string, error:string}[]}  errors - place to store skippable errors
+ * @return {Promise<{date: DateTime, body: string, repo: string}[]>}  objects containing date, body, and the repository base name
  */
-const processRepo = (repo, errors) => {
-    return new Promise((resolve) => {
+async function processRepo(repo, errors) {
+    try {
         const cmd = gitCommand(repo);
-        _exec(cmd, { encoding:'utf8' })
-            .then(out => out.stdout.trim())
-            .then(stdout => {
-                const results = stdout.split(' +++')
-                    .filter(item => {
-                        return item.trim().length > 0;
-                    })
-                    .map(item => {
-                        return item.trim();
-                    })
-                    .map(item => {
-                        const date = DateTime.fromFormat(item.substring(DateLength, item.search(/[=]{2}/)), options.dateOptions);
-                        const body = item.substring(item.search(/[=]{2}/) + options.offset);
-                        return { date, body, repo: basename(repo) };
-                    })
-                    .filter(filterPeriod);
-                return resolve(results);
-            })
-            .catch(err => {
-                errors.push({ repo: repo, error: err });
-                // continue to next repo but be sure to return empty array
-                return resolve([]);
-            });
-    });
-};
+        const { stdout } = await _exec(cmd, { encoding:'utf8' });
+        const lines = [];
+        const repoName = basename(repo);
+        for(const item of stdout.trim().split(' +++')) {
+            const _item = item.trim();
+            if(_item.length === 0) {
+                continue;
+            }
+
+            const markerIndex = _item.indexOf('==');
+            const date = DateTime.fromFormat(_item.substring(DateLength, markerIndex), options.dateOptions);
+            if(!filterPeriod({ date })) {
+                continue;
+            }
+
+            const body = _item.substring(markerIndex + options.offset);
+            lines.push({ date, body, repo: repoName });
+        }
+        return lines;
+    }
+    catch (err) {
+        errors.push({ repo, error: err ? err.message : 'Unknown error' });
+        // continue to next repo but be sure to return empty array
+        return [];
+    }
+}
 
 /**
  * writes errors to console if in debug mode
@@ -102,43 +103,44 @@ const logErrors = (errors, isDebug, err) => {
 /**
  * Entry point
  */
-const main = () => {
-    if(options.devRoot.length) {
-        const errors = [];
-        let maxRepoLength = 0;
-        const repos = [];
-        options.devRoot.forEach(root => {
-            allRepoPaths(root, options.folderNames).forEach(repo => {
-                repos.push(repo);
-            });
-        });
-        const promises = repos.map(repo => processRepo(repo, errors));
+async function main() {
+    if(options.devRoot.length === 0) {
+        console.log(`bash variable DEVROOT is required`);
+        process.exitCode = 1;
+        return;
+    }
 
-        return Promise
-            .all(promises)
-            .then(results => {
-                results
-                    .reduce((acc, item) => acc.concat(item), [])
-                    .sort((a, b) => a.date.valueOf() - b.date.valueOf())
-                    .map(item => {
-                        if(item.repo.length > maxRepoLength) {
-                            maxRepoLength = item.repo.length;
-                        }
-                        return item;
-                    })
-                    .map(item => {
-                        let name = item.repo;
-                        name = name.padEnd(maxRepoLength);
-                        console.log(`${item.date.toFormat(options.dateOptions)}  ${name}  ${item.body}`);
-                    });
-            })
-            .catch(err => {
-                logErrors(errors, options.debug, err);
-            });
+    const errors = [];
+    let maxRepoLength = 0;
+    const repos = [];
+    for(const root of options.devRoot) {
+        const paths = await allRepoPaths(root, options.folderNames);
+        repos.push(...paths);
     }
 
-    console.log(`bash variable DEVROOT is required`);
-    process.exitCode = 1;
-};
+    try {
+        const result = [];
+        const concurrency = 8;
+        for(let i = 0; i < repos.length; i += concurrency) {
+            const batch = repos
+                .slice(i, i + concurrency)
+                .map(repo => processRepo(repo, errors));
+            result.push(...await Promise.all(batch));
+        }
+        const sorted = result
+            .flat()
+            .sort((a, b) => a.date.valueOf() - b.date.valueOf());
+        sorted.forEach(item => {
+            maxRepoLength = Math.max(maxRepoLength, item.repo.length);
+        });
+        sorted.forEach(item => {
+            console.log(`${item.date.toFormat(options.dateOptions)}  ${item.repo.padEnd(maxRepoLength)}  ${item.body}`);
+        });
+    }
+    catch (err) {
+        logErrors(errors, options.debug, err);
+    }
+
+}
 
 main().catch(console.error);

src/spawner/index.js

@@ -89,9 +89,9 @@ yargs
                     }
                     return true;
                 })
-                .check(({ version, path, oldest }) => {
+                .check(async ({ version, path, oldest }) => {
                     const hasPackage = path.endsWith('package.json') || fileExists(join(path, 'package.json'));
-                    versionToUse = versionToUseValidator({ path, version, oldest }, !hasPackage);
+                    versionToUse = await versionToUseValidator({ path, version, oldest }, !hasPackage);
                     return Boolean(versionToUse);
                 });
         },

src/yarn/index.js

@@ -70,8 +70,8 @@ yargs
                     }
                     return true;
                 })
-                .check(({ version, path, oldest }) => {
-                    versionToUse = versionToUseValidator({ path, version, oldest });
+                .check(async ({ version, path, oldest }) => {
+                    versionToUse = await versionToUseValidator({ path, version, oldest });
                     return Boolean(versionToUse);
                 });
         },