diff --git a/.gitignore b/.gitignore index e6accce..8f66d35 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ node_modules/ dist/ *.log +promptless-skill-report-card.html .DS_Store .env .env.* diff --git a/README.md b/README.md index 50cfdfa..7c6e97d 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,9 @@ Slack

-A CLI for helping tech writers. Right now it only detects the rhetorical and structural tells of LLM-generated prose and surfaces diagnostics for people to fix. +A CLI for helping tech writers and agent-instruction maintainers. It can detect +the rhetorical and structural tells of LLM-generated prose, and it can generate +a local report card for agent skills and instruction files. ## Install @@ -35,8 +37,8 @@ npm install -g git+ssh://git@github.com/Promptless/promptless-cli.git This exposes two binaries on your `PATH`: `promptless` and `pless`. ```sh -promptless sample.md -pless --diagnostic sample.md +promptless slop-cop sample.md +promptless skill-report-card ``` > The repo is currently internal — you'll need GitHub access to `Promptless/promptless-cli` for the install to fetch. @@ -47,18 +49,68 @@ pless --diagnostic sample.md git clone git@github.com:Promptless/promptless-cli.git cd promptless-cli npm install -npm run promptless -- path/to/file.md +npm run promptless -- slop-cop path/to/file.md +npm run promptless -- skill-report-card ``` ## Usage ``` -promptless [options] ... +promptless [options] ``` -Run `promptless --help` for the full option list (input format, color, diagnostic vs. human mode, pandoc passthrough). +Run `promptless --help` for the command list. -Exit codes: `0` clean, `1` violations found, `2` argument error. +### `promptless slop-cop` + +Detect LLM prose tells in text files and print editor-friendly diagnostics. + +```sh +promptless slop-cop docs/page.md +promptless slop-cop --format mdx docs/page.mdx +``` + +### `promptless skill-report-card` + +Generate a local report card for agent instruction files and skills. + +```sh +promptless skill-report-card +``` + +The default flow is interactive: + +1. Searches the current directory, common home config locations, and bounded + machine locations for `AGENTS.md`, `CLAUDE.md`, `.agents/`, `.claude/`, and + `.codex/` instruction assets. Generated copies under Codex worktrees, temp + plugin storage, customer-repo clones, and plugin caches are skipped during + broad scans. +2. Groups discovered files by skill-bearing Git repo or global skill location, + then lets you check or uncheck those groups. Instruction-only groups are + shown only when no skills are found. +3. Detects whether `claude` or `codex` CLIs are installed. +4. Shows an estimated LLM token count if a provider is available. +5. Runs deterministic local checks, uses `skill-validator` when it is installed, + prints an inline summary, and writes `promptless-skill-report-card.html`. + +Useful noninteractive examples: + +```sh +promptless skill-report-card . --yes --llm off +promptless skill-report-card ~/work/docs-agent --llm codex --out report.html +promptless skill-report-card . --yes --fail-under 80 +``` + +Privacy: deterministic checks are local. File contents are only sent to an LLM +when you explicitly choose `claude`, `codex`, or pass `--llm auto|claude|codex`. + +Attribution: `skill-report-card` uses +[`skill-validator`](https://github.com/agent-ecosystem/skill-validator) as an +internal component when the binary is available on `PATH`. Promptless adds the +interactive report-card UX, governance scoring, and HTML report. + +Exit codes: `0` clean or report generated, `1` runtime/input error, `2` +argument error or `--fail-under` threshold failure. ## License diff --git a/package.json b/package.json index 6be659c..5b57eb6 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "@promptless/promptless-cli", "version": "0.1.0", "license": "MPL-2.0", - "description": "CLI for helping tech writers. Right now it only detects the rhetorical and structural tells of LLM-generated prose and surfaces diagnostics for people to fix.", + "description": "CLI for helping tech writers and agent-instruction maintainers detect prose issues and score local agent skills.", "type": "module", "bin": { "promptless": "./dist/cli.js", @@ -18,6 +18,7 @@ "prepare": "npm run build", "promptless": "tsx src/cli.ts", "build": "tsup && chmod +x dist/cli.js", + "test": "node --import tsx --test \"src/**/*.test.ts\"", "typecheck": "tsc --noEmit" }, "devDependencies": { diff --git a/src/cli.ts b/src/cli.ts index 5df740b..7f85a6f 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -12,13 +12,15 @@ import { COMMANDS } from './commands/registry' // Naming conventions: lowercase, kebab-case, noun-ish when the command names the artifact // it operates on (traces, event), verb-noun when the operation is the point (docs-audit). +const commandNameWidth = Math.max(...COMMANDS.map((command) => command.name.length)) + 2 + const TOP_HELP = `promptless — CLI for tech writers usage: promptless [options] commands: -${COMMANDS.map((c) => ` ${c.name.padEnd(12)} ${c.summary}`).join('\n')} +${COMMANDS.map((c) => ` ${c.name.padEnd(commandNameWidth)} ${c.summary}`).join('\n')} run \`promptless --help\` for command-specific options. ` @@ -43,7 +45,7 @@ async function main(): Promise { process.exit(2) } - cmd.run(argv.slice(1)) + await cmd.run(argv.slice(1)) } main() diff --git a/src/commands/completion.ts b/src/commands/completion.ts index 67887d7..57d480b 100644 --- a/src/commands/completion.ts +++ b/src/commands/completion.ts @@ -72,6 +72,19 @@ case $state in '(-h --help)'{-h,--help}'[show help]' \\ '*:file:_files' ;; + skill-report-card) + _arguments \\ + '(-y --yes)'{-y,--yes}'[run without prompts]' \\ + '--llm[LLM provider]:provider:(off auto claude codex)' \\ + '--out[HTML report path]:file:_files' \\ + '--no-open[do not open the generated HTML report]' \\ + '--no-machine-scan[skip broader home scan]' \\ + '--max-depth[recursion depth]:number:' \\ + '--max-directories[directory scan cap]:number:' \\ + '--fail-under[minimum passing score]:score:' \\ + '(-h --help)'{-h,--help}'[show help]' \\ + '*:path:_files' + ;; completion) _arguments '1:shell:(zsh bash fish)' ;; @@ -120,6 +133,13 @@ _promptless_complete() { COMPREPLY=($(compgen -f -- "$cur")) fi ;; + skill-report-card) + if [[ "$cur" == -* ]]; then + COMPREPLY=($(compgen -W "--yes -y --llm --out --no-open --no-machine-scan --max-depth --max-directories --fail-under -h --help" -- "$cur")) + else + COMPREPLY=($(compgen -f -- "$cur")) + fi + ;; completion) COMPREPLY=($(compgen -W "zsh bash fish" -- "$cur")) ;; @@ -180,6 +200,34 @@ function fishScript(): string { `complete -c ${bin} -n '__fish_seen_subcommand_from slop-cop' -s h -l help -d 'Show help'`, ) lines.push(`complete -c ${bin} -n '__fish_seen_subcommand_from slop-cop' -F`) + lines.push( + `complete -c ${bin} -n '__fish_seen_subcommand_from skill-report-card' -s y -l yes -d 'Run without prompts'`, + ) + lines.push( + `complete -c ${bin} -n '__fish_seen_subcommand_from skill-report-card' -l llm -x -a 'off auto claude codex' -d 'LLM provider'`, + ) + lines.push( + `complete -c ${bin} -n '__fish_seen_subcommand_from skill-report-card' -l out -r -d 'HTML report path'`, + ) + lines.push( + `complete -c ${bin} -n '__fish_seen_subcommand_from skill-report-card' -l no-open -d 'Do not open HTML report'`, + ) + lines.push( + `complete -c ${bin} -n '__fish_seen_subcommand_from skill-report-card' -l no-machine-scan -d 'Skip broader home scan'`, + ) + lines.push( + `complete -c ${bin} -n '__fish_seen_subcommand_from skill-report-card' -l max-depth -x -d 'Recursion depth'`, + ) + lines.push( + `complete -c ${bin} -n '__fish_seen_subcommand_from skill-report-card' -l max-directories -x -d 'Directory scan cap'`, + ) + lines.push( + `complete -c ${bin} -n '__fish_seen_subcommand_from skill-report-card' -l fail-under -x -d 'Minimum passing score'`, + ) + lines.push( + `complete -c ${bin} -n '__fish_seen_subcommand_from skill-report-card' -s h -l help -d 'Show help'`, + ) + lines.push(`complete -c ${bin} -n '__fish_seen_subcommand_from skill-report-card' -F`) lines.push( `complete -c ${bin} -n '__fish_seen_subcommand_from completion' -a 'zsh bash fish'`, ) diff --git a/src/commands/registry.ts b/src/commands/registry.ts index 236a5cd..874fab4 100644 --- a/src/commands/registry.ts +++ b/src/commands/registry.ts @@ -1,4 +1,5 @@ import { runSlopCop } from './slop-cop' +import { runSkillReportCard } from './skill-report-card' import { runCompletion } from './completion' import { runAgentView } from './agentview' import { runLogin } from './login' @@ -8,7 +9,7 @@ import { runWhoami } from './whoami' export interface CommandEntry { name: string summary: string - run: (argv: string[]) => never + run: (argv: string[]) => void | Promise } export const COMMANDS: CommandEntry[] = [ @@ -32,6 +33,11 @@ export const COMMANDS: CommandEntry[] = [ summary: 'Detect LLM prose tells in text files', run: runSlopCop, }, + { + name: 'skill-report-card', + summary: 'Score local agent instructions and skills', + run: runSkillReportCard, + }, { name: 'agentview', summary: 'Fetch a docs page and show the markdown a coding agent sees', diff --git a/src/commands/skill-report-card.ts b/src/commands/skill-report-card.ts new file mode 100644 index 0000000..1085eb8 --- /dev/null +++ b/src/commands/skill-report-card.ts @@ -0,0 +1,352 @@ +import { spawnSync } from 'node:child_process' +import { resolve } from 'node:path' +import prompts from 'prompts' +import { discoverInstructions } from '../lib/skill-report-card/discovery' +import { detectProviders, estimateLlmUsage, resolveRequestedProvider, runLlmReview } from '../lib/skill-report-card/providers' +import { formatTerminalSummary, writeHtmlReport } from '../lib/skill-report-card/render' +import { loadInstructions, scoreInstructions, summarizeScores } from '../lib/skill-report-card/scoring' +import { + groupInstructionsForSelection, + groupsForInteractivePrompt, +} from '../lib/skill-report-card/selectionGroups' +import { runSkillValidator } from '../lib/skill-report-card/validator' +import type { + DiscoveredInstruction, + LlmProvider, + LlmReviewResult, + LoadedInstruction, + ProviderAvailability, + ResolvedLlmProvider, + SkillReportCard, +} from '../lib/skill-report-card/types' + +type PromptLlmProvider = LlmProvider | 'prompt' + +interface Args { + targetPath: string + showHelp: boolean + yes: boolean + llm: PromptLlmProvider + htmlPath: string + openReport: boolean + includeMachineScan: boolean + maxDepth: number + maxDirectories: number + failUnder: number | null +} + +const DEFAULT_HTML_FILE = 'promptless-skill-report-card.html' + +const HELP = `promptless skill-report-card — score local agent instructions and skills + +usage: + promptless skill-report-card [path] [options] + +options: + --yes, -y Run without prompts. Selects all discovered files and skips LLM review unless --llm is set. + --llm LLM review provider: off, auto, claude, codex. Default is an interactive choice. + --out HTML report path. Default: ./${DEFAULT_HTML_FILE} + --no-open Do not open the generated HTML report. + --no-machine-scan Only scan the provided/current path and common home agent config locations. + --max-depth Directory recursion depth per scan root. Default: 8. + --max-directories Stop discovery after visiting this many directories. Default: 25000. + --fail-under Exit 2 if the overall score is below this 0-100 threshold. + -h, --help Show this help + +examples: + promptless skill-report-card + promptless skill-report-card ~/work/docs-agent --llm codex + promptless skill-report-card . --yes --llm off --fail-under 80 +` + +export async function runSkillReportCard(argv: string[]): Promise { + const args = parseArgs(argv) + if (args.showHelp) { + process.stdout.write(HELP) + return + } + + const interactive = args.yes ? false : process.stdin.isTTY === true && process.stdout.isTTY === true + process.stdout.write('\nPromptless Skill Report Card\n\nLooking for agent instructions...\n') + + const discovery = await discoverInstructions(args.targetPath, { + includeMachineScan: args.includeMachineScan, + maxDepth: args.maxDepth, + maxDirectories: args.maxDirectories, + }) + + if (discovery.items.length === 0) { + process.stderr.write('promptless skill-report-card: no AGENTS.md, CLAUDE.md, or SKILL.md files found.\n') + process.exitCode = 1 + return + } + + process.stdout.write( + `Found ${discovery.items.length} instruction file${plural(discovery.items.length)} across ${discovery.scannedRoots.length} scan root${plural(discovery.scannedRoots.length)}${discovery.truncated ? ' before hitting the scan cap' : ''}.\n`, + ) + + const selectionGroups = groupInstructionsForSelection(discovery.items) + const promptGroups = groupsForInteractivePrompt(selectionGroups) + if (interactive) { + process.stdout.write( + `Grouped into ${promptGroups.length} repo/location option${plural(promptGroups.length)}.\n`, + ) + } + + const selectedItems = await selectInstructions(discovery.items, promptGroups, interactive) + if (selectedItems.length === 0) { + process.stderr.write('promptless skill-report-card: no instruction files selected.\n') + process.exitCode = 1 + return + } + + const loadedInstructions = loadInstructions(selectedItems) + const providers = detectProviders() + const selectedProvider = await chooseProvider(args.llm, providers, loadedInstructions, interactive) + + process.stdout.write('\nGenerating report card...\n') + process.stdout.write('✓ Reading selected instruction files\n') + + const validator = runSkillValidator(loadedInstructions) + if (validator.skippedReason) { + process.stdout.write(`• Skipped skill-validator: ${validator.skippedReason}\n`) + } else if (validator.available) { + process.stdout.write(`✓ Ran skill-validator on ${validator.checkedCount} skill${plural(validator.checkedCount)}\n`) + } else { + process.stdout.write('• skill-validator not found; continuing with Promptless checks\n') + } + + const llm = selectedProvider ? runOptionalLlmReview(selectedProvider, loadedInstructions) : null + if (llm) { + process.stdout.write(`✓ Added ${llm.provider} review findings\n`) + } + + const scores = scoreInstructions(loadedInstructions, [...validator.findings, ...(llm?.findings ?? [])]) + const report: SkillReportCard = { + schemaVersion: 'promptless.skill-report-card.v1', + generatedAt: new Date().toISOString(), + targetPath: resolve(args.targetPath), + htmlPath: resolve(args.htmlPath), + discovery, + selectedInstructions: loadedInstructions, + scores, + summary: summarizeScores(scores), + validator, + llm, + } + + writeHtmlReport(report) + process.stdout.write('✓ Wrote HTML report\n') + process.stdout.write(formatTerminalSummary(report)) + + if (interactive && args.openReport) openHtmlReport(report.htmlPath) + if (args.failUnder !== null && report.summary.score < args.failUnder) process.exitCode = 2 +} + +function parseArgs(argv: string[]): Args { + let showHelp = false + let yes = false + let llm: PromptLlmProvider = 'prompt' + let htmlPath = DEFAULT_HTML_FILE + let openReport = true + let includeMachineScan = true + let maxDepth = 8 + let maxDirectories = 25000 + let failUnder: number | null = null + const positional: string[] = [] + + for (let index = 0; index < argv.length; index++) { + const arg = argv[index] + if (arg === '-h' || arg === '--help') showHelp = true + else if (arg === '--yes' || arg === '-y') yes = true + else if (arg === '--no-open') openReport = false + else if (arg === '--no-machine-scan') includeMachineScan = false + else if (arg === '--llm' || arg.startsWith('--llm=')) { + const value = readFlagValue(argv, index, '--llm') + if (!arg.includes('=')) index += 1 + if (value !== 'off' && value !== 'auto' && value !== 'claude' && value !== 'codex') { + exitArgError('--llm expects one of: off, auto, claude, codex') + } + llm = value + } else if (arg === '--out' || arg.startsWith('--out=')) { + htmlPath = readFlagValue(argv, index, '--out') + if (!arg.includes('=')) index += 1 + } else if (arg === '--max-depth' || arg.startsWith('--max-depth=')) { + maxDepth = readPositiveIntegerFlag(argv, index, '--max-depth') + if (!arg.includes('=')) index += 1 + } else if (arg === '--max-directories' || arg.startsWith('--max-directories=')) { + maxDirectories = readPositiveIntegerFlag(argv, index, '--max-directories') + if (!arg.includes('=')) index += 1 + } else if (arg === '--fail-under' || arg.startsWith('--fail-under=')) { + failUnder = readScoreFlag(argv, index, '--fail-under') + if (!arg.includes('=')) index += 1 + } else if (arg.startsWith('-')) { + exitArgError(`unknown flag ${arg}`) + } else { + positional.push(arg) + } + } + + if (positional.length > 1) exitArgError('expected at most one path') + + return { + targetPath: positional[0] ?? process.cwd(), + showHelp, + yes, + llm, + htmlPath, + openReport, + includeMachineScan, + maxDepth, + maxDirectories, + failUnder, + } +} + +async function selectInstructions( + items: DiscoveredInstruction[], + groups: ReturnType, + interactive: boolean, +): Promise { + if (!interactive) return items + + const response = await prompts( + { + type: 'multiselect', + name: 'selectedIds', + message: 'Select repos and skill locations to include in this report', + hint: 'Space to toggle, enter to continue', + choices: groups.map((group) => ({ + title: group.label, + value: group.id, + selected: true, + })), + min: 1, + }, + { + onCancel: () => { + process.stdout.write('\nCancelled.\n') + process.exit(1) + }, + }, + ) + + const selectedIds = readSelectedIds(response) + return groups + .filter((group) => selectedIds.has(group.id)) + .flatMap((group) => group.items) +} + +async function chooseProvider( + requested: PromptLlmProvider, + availability: ProviderAvailability, + instructions: LoadedInstruction[], + interactive: boolean, +): Promise { + if (requested === 'off') return null + + const estimate = estimateLlmUsage(instructions) + if (requested !== 'prompt') { + const provider = resolveRequestedProvider(requested, availability) + if (!provider) { + process.stdout.write( + `• Requested LLM provider unavailable. Found claude: ${availability.claude ? 'yes' : 'no'}, codex: ${availability.codex ? 'yes' : 'no'}.\n`, + ) + } + return provider + } + + if (!interactive) return null + + const choices = [] + if (availability.claude) choices.push({ title: 'Use Claude CLI', value: 'claude' }) + if (availability.codex) choices.push({ title: 'Use Codex CLI', value: 'codex' }) + choices.push({ title: 'Skip LLM review - only deterministic checks will run', value: 'off' }) + + if (choices.length === 1) { + process.stdout.write('\nLLM review\nNo claude or codex CLI detected; skipping LLM review.\n') + return null + } + + process.stdout.write( + `\nLLM review\nFound: claude ${availability.claude ? '✓' : 'not found'}, codex ${availability.codex ? '✓' : 'not found'}\nEstimated usage: ~${estimate.inputTokens.toLocaleString()} input tokens, ~${estimate.outputTokens.toLocaleString()} output tokens.\n`, + ) + + const response = await prompts( + { + type: 'select', + name: 'provider', + message: 'Choose review mode', + choices, + initial: 0, + }, + { + onCancel: () => { + process.stdout.write('\nCancelled.\n') + process.exit(1) + }, + }, + ) + + const provider = readProviderChoice(response) + return provider === 'off' ? null : provider +} + +function runOptionalLlmReview( + provider: ResolvedLlmProvider, + instructions: LoadedInstruction[], +): LlmReviewResult | null { + try { + return runLlmReview(provider, instructions) + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + process.stderr.write(`promptless skill-report-card: ${provider} review failed: ${message}\n`) + return null + } +} + +function openHtmlReport(htmlPath: string): void { + const opener = process.platform === 'darwin' ? 'open' : process.platform === 'win32' ? 'start' : 'xdg-open' + spawnSync(opener, [htmlPath], { stdio: 'ignore' }) +} + +function readFlagValue(argv: string[], index: number, flag: string): string { + const arg = argv[index] + const value = arg.includes('=') ? arg.slice(flag.length + 1) : argv[index + 1] + if (!value || value.startsWith('-')) exitArgError(`${flag} expects a value`) + return value +} + +function readPositiveIntegerFlag(argv: string[], index: number, flag: string): number { + const value = Number(readFlagValue(argv, index, flag)) + if (!Number.isInteger(value) || value < 1) exitArgError(`${flag} expects a positive integer`) + return value +} + +function readScoreFlag(argv: string[], index: number, flag: string): number { + const value = Number(readFlagValue(argv, index, flag)) + if (!Number.isInteger(value) || value < 0 || value > 100) exitArgError(`${flag} expects an integer from 0 to 100`) + return value +} + +function readSelectedIds(response: unknown): Set { + if (!response || typeof response !== 'object' || !('selectedIds' in response)) return new Set() + const selectedIds = (response as { selectedIds: unknown }).selectedIds + if (!Array.isArray(selectedIds)) return new Set() + return new Set(selectedIds.filter((value): value is string => typeof value === 'string')) +} + +function readProviderChoice(response: unknown): 'off' | ResolvedLlmProvider { + if (!response || typeof response !== 'object' || !('provider' in response)) return 'off' + const provider = (response as { provider: unknown }).provider + return provider === 'claude' || provider === 'codex' ? provider : 'off' +} + +function plural(count: number): string { + return count === 1 ? '' : 's' +} + +function exitArgError(message: string): never { + process.stderr.write(`promptless skill-report-card: ${message}\n\n${HELP}`) + process.exit(2) +} diff --git a/src/lib/skill-report-card/discovery.ts b/src/lib/skill-report-card/discovery.ts new file mode 100644 index 0000000..a63cc02 --- /dev/null +++ b/src/lib/skill-report-card/discovery.ts @@ -0,0 +1,314 @@ +import { lstat, readdir, realpath, stat } from 'node:fs/promises' +import { homedir } from 'node:os' +import { basename, dirname, join, relative, resolve, sep } from 'node:path' +import type { + DiscoveredInstruction, + DiscoveryResult, + InstructionKind, +} from './types' + +interface SearchRoot { + path: string + maxDepth: number + includeGeneratedPaths: boolean +} + +export interface DiscoveryOptions { + includeMachineScan: boolean + maxDepth: number + maxDirectories: number + homeDirectory?: string +} + +interface DiscoveryState { + itemsByRealPath: Map + scannedRoots: string[] + seenDirectories: Set + visitedDirectoryCount: number + truncated: boolean + homeDirectory: string + maxDirectories: number +} + +const ROOT_INSTRUCTION_FILES = new Set(['AGENTS.md', 'CLAUDE.md']) +const AGENT_DIRECTORY_NAMES = new Set(['.agents', '.claude', '.codex']) +const EXCLUDED_DIRECTORY_NAMES = new Set([ + '.cache', + '.git', + '.hg', + '.next', + '.npm', + '.pnpm-store', + '.svn', + '.Trash', + '.venv', + 'Applications', + 'build', + 'dist', + 'Library', + 'Movies', + 'Music', + 'node_modules', + 'Pictures', + 'target', + 'venv', +]) + +export async function discoverInstructions( + targetPath: string, + options: DiscoveryOptions, +): Promise { + const homeDirectory = await realHomeDirectory(options.homeDirectory ?? homedir()) + const state: DiscoveryState = { + itemsByRealPath: new Map(), + scannedRoots: [], + seenDirectories: new Set(), + visitedDirectoryCount: 0, + truncated: false, + homeDirectory, + maxDirectories: options.maxDirectories, + } + + for (const root of await buildSearchRoots(targetPath, options, homeDirectory)) { + if (state.truncated) break + await scanPath(root.path, root.maxDepth, root.includeGeneratedPaths, state) + } + + const items = [...state.itemsByRealPath.values()].sort((left, right) => + left.displayPath.localeCompare(right.displayPath), + ) + return { items, scannedRoots: state.scannedRoots, truncated: state.truncated } +} + +async function realHomeDirectory(path: string): Promise { + try { + return await realpath(path) + } catch { + return resolve(path) + } +} + +async function buildSearchRoots( + targetPath: string, + options: DiscoveryOptions, + homeDirectory: string, +): Promise { + const roots: SearchRoot[] = [] + const seen = new Set() + + const addRoot = async ( + path: string, + maxDepth: number, + includeGeneratedPaths: boolean, + ): Promise => { + try { + const resolved = await realpath(resolve(path)) + if (seen.has(resolved)) return + seen.add(resolved) + roots.push({ path: resolved, maxDepth, includeGeneratedPaths }) + } catch { + return + } + } + + await addRoot(targetPath, options.maxDepth, true) + + for (const path of [ + join(homeDirectory, '.agents'), + join(homeDirectory, '.claude'), + join(homeDirectory, '.codex'), + join(homeDirectory, 'work'), + join(homeDirectory, 'src'), + join(homeDirectory, 'code'), + join(homeDirectory, 'Code'), + join(homeDirectory, 'Projects'), + join(homeDirectory, 'Developer'), + ]) { + await addRoot(path, options.maxDepth, false) + } + + if (options.includeMachineScan) { + await addRoot(homeDirectory, Math.min(options.maxDepth, 6), false) + } + + return roots +} + +async function scanPath( + path: string, + maxDepth: number, + includeGeneratedPaths: boolean, + state: DiscoveryState, +): Promise { + if (state.truncated) return + + let pathStat + try { + pathStat = await stat(path) + } catch { + return + } + + if (pathStat.isFile()) { + await maybeAddInstruction(path, state) + return + } + + if (!pathStat.isDirectory()) return + state.scannedRoots.push(path) + await scanDirectory(path, maxDepth, 0, includeGeneratedPaths, state) +} + +async function scanDirectory( + directory: string, + maxDepth: number, + depth: number, + includeGeneratedPaths: boolean, + state: DiscoveryState, +): Promise { + if (state.truncated || depth > maxDepth) return + + let realDirectory: string + try { + realDirectory = await realpath(directory) + } catch { + return + } + if (state.seenDirectories.has(realDirectory)) return + state.seenDirectories.add(realDirectory) + + state.visitedDirectoryCount += 1 + if (state.visitedDirectoryCount > state.maxDirectories) { + state.truncated = true + return + } + + let entries + try { + entries = await readdir(directory, { withFileTypes: true }) + } catch { + return + } + + for (const entry of entries) { + if (state.truncated) return + const childPath = join(directory, entry.name) + if (entry.isFile()) { + await maybeAddInstruction(childPath, state) + } else if (entry.isDirectory()) { + if (shouldSkipDirectory(entry.name, childPath, includeGeneratedPaths, state.homeDirectory)) continue + await scanDirectory(childPath, maxDepth, depth + 1, includeGeneratedPaths, state) + } else if (entry.isSymbolicLink()) { + await maybeScanSymbolicFile(childPath, state) + } + } +} + +async function maybeScanSymbolicFile(path: string, state: DiscoveryState): Promise { + try { + const linkStat = await lstat(path) + if (!linkStat.isSymbolicLink()) return + const target = await stat(path) + if (target.isFile()) await maybeAddInstruction(path, state) + } catch { + return + } +} + +function shouldSkipDirectory( + name: string, + path: string, + includeGeneratedPaths: boolean, + homeDirectory: string, +): boolean { + if (EXCLUDED_DIRECTORY_NAMES.has(name)) return true + return !includeGeneratedPaths && isGeneratedAgentPath(path, homeDirectory) +} + +function isGeneratedAgentPath(path: string, homeDirectory: string): boolean { + const relativeToHome = relative(homeDirectory, path).split(sep) + return ( + startsWithSegments(relativeToHome, ['.codex', '.tmp']) || + startsWithSegments(relativeToHome, ['.codex', 'customer-repos']) || + startsWithSegments(relativeToHome, ['.codex', 'worktrees']) || + startsWithSegments(relativeToHome, ['.codex', 'plugins', 'cache']) || + startsWithSegments(relativeToHome, ['.claude', 'plugins', 'cache']) + ) +} + +function startsWithSegments(segments: string[], prefix: string[]): boolean { + return prefix.every((segment, index) => segments[index] === segment) +} + +async function maybeAddInstruction(path: string, state: DiscoveryState): Promise { + const name = basename(path) + const kind = instructionKind(path, name) + if (!kind) return + + let realFilePath: string + let sizeBytes = 0 + try { + realFilePath = await realpath(path) + const fileStat = await stat(path) + sizeBytes = fileStat.size + } catch { + return + } + if (state.itemsByRealPath.has(realFilePath)) return + + const rootPath = rootPathForInstruction(realFilePath, kind) + state.itemsByRealPath.set(realFilePath, { + id: realFilePath, + path: realFilePath, + rootPath, + displayPath: displayPath(realFilePath, state.homeDirectory), + title: titleForInstruction(realFilePath, kind), + kind, + sizeBytes, + }) +} + +function instructionKind(path: string, name: string): InstructionKind | null { + if (ROOT_INSTRUCTION_FILES.has(name)) { + return name === 'AGENTS.md' ? 'agents-md' : 'claude-md' + } + if (name !== 'SKILL.md') return null + + const segments = path.split(sep) + const agentRootIndex = agentSkillRootIndex(segments) + if (agentRootIndex === -1) return null + const rootSegment = segments[agentRootIndex] + if (rootSegment === '.agents') return 'agent-skill' + if (rootSegment === '.claude') return 'claude-skill' + if (rootSegment === '.codex') return 'codex-skill' + return null +} + +function rootPathForInstruction(path: string, kind: InstructionKind): string { + if (kind === 'agents-md' || kind === 'claude-md') return dirname(path) + + const segments = path.split(sep) + const agentRootIndex = agentSkillRootIndex(segments) + if (agentRootIndex === -1) return dirname(path) + return segments.slice(0, agentRootIndex + 1).join(sep) || sep +} + +function agentSkillRootIndex(segments: string[]): number { + return segments.findIndex((segment, index) => { + return AGENT_DIRECTORY_NAMES.has(segment) && segments[index + 1] === 'skills' + }) +} + +function titleForInstruction(path: string, kind: InstructionKind): string { + if (kind === 'agents-md') return 'AGENTS.md' + if (kind === 'claude-md') return 'CLAUDE.md' + return basename(dirname(path)) +} + +function displayPath(path: string, homeDirectory: string): string { + const relativeToHome = relative(homeDirectory, path) + if (relativeToHome && !relativeToHome.startsWith('..') && !relativeToHome.startsWith(sep)) { + return join('~', relativeToHome) + } + return path +} diff --git a/src/lib/skill-report-card/providers.ts b/src/lib/skill-report-card/providers.ts new file mode 100644 index 0000000..8dc425f --- /dev/null +++ b/src/lib/skill-report-card/providers.ts @@ -0,0 +1,279 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { spawnSync } from 'node:child_process' +import type { + Finding, + LlmReviewResult, + LlmUsageEstimate, + LoadedInstruction, + ProviderAvailability, + ResolvedLlmProvider, + ScoreDimension, +} from './types' +import { commandExists } from './validator' + +interface RawLlmFinding { + severity: string + dimension: string + title: string + message: string + path: string + remediation: string | null +} + +interface ParsedLlmOutput { + findings: RawLlmFinding[] + summary: string | null +} + +const REVIEW_SCHEMA = { + type: 'object', + additionalProperties: false, + properties: { + summary: { type: ['string', 'null'] }, + findings: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + properties: { + severity: { type: 'string', enum: ['blocker', 'should-fix', 'nice-to-have'] }, + dimension: { + type: 'string', + enum: ['structure', 'triggering', 'specificity', 'governance', 'safety', 'evaluation', 'portability'], + }, + title: { type: 'string' }, + message: { type: 'string' }, + path: { type: 'string' }, + remediation: { type: ['string', 'null'] }, + }, + required: ['severity', 'dimension', 'title', 'message', 'path', 'remediation'], + }, + }, + }, + required: ['summary', 'findings'], +} satisfies Record + +export function detectProviders(): ProviderAvailability { + return { claude: commandExists('claude'), codex: commandExists('codex') } +} + +export function estimateLlmUsage(instructions: LoadedInstruction[]): LlmUsageEstimate { + const contentTokens = instructions.reduce((total, instruction) => total + instruction.estimatedTokens, 0) + const promptOverhead = 900 + instructions.length * 80 + return { + inputTokens: contentTokens + promptOverhead, + outputTokens: Math.max(1200, Math.min(6000, Math.round((contentTokens + promptOverhead) * 0.12))), + } +} + +export function resolveRequestedProvider( + requested: 'auto' | 'claude' | 'codex', + availability: ProviderAvailability, +): ResolvedLlmProvider | null { + if (requested === 'claude') return availability.claude ? 'claude' : null + if (requested === 'codex') return availability.codex ? 'codex' : null + if (availability.claude) return 'claude' + if (availability.codex) return 'codex' + return null +} + +export function runLlmReview( + provider: ResolvedLlmProvider, + instructions: LoadedInstruction[], +): LlmReviewResult { + const prompt = buildReviewPrompt(instructions) + const output = provider === 'claude' ? runClaude(prompt) : runCodex(prompt) + const parsed = parseProviderOutput(output) + const findings = parsed.findings.map((finding) => normalizeFinding(finding, instructions)) + return { provider, findings, rawSummary: parsed.summary } +} + +function runClaude(prompt: string): string { + const result = spawnSync( + 'claude', + ['--print', '--output-format', 'json', '--json-schema', JSON.stringify(REVIEW_SCHEMA), '--no-session-persistence'], + { input: prompt, encoding: 'utf-8', timeout: 240000 }, + ) + if (result.error) throw result.error + if (result.status !== 0) throw new Error(firstUsefulLine(result.stderr) ?? 'claude exited with a non-zero status') + return result.stdout +} + +function runCodex(prompt: string): string { + const tempDirectory = mkdtempSync(join(tmpdir(), 'promptless-skill-report-card-')) + const schemaPath = join(tempDirectory, 'schema.json') + try { + writeFileSync(schemaPath, JSON.stringify(REVIEW_SCHEMA), 'utf-8') + const result = spawnSync( + 'codex', + ['exec', '--ephemeral', '--skip-git-repo-check', '--sandbox', 'read-only', '--output-schema', schemaPath, '-'], + { input: prompt, encoding: 'utf-8', timeout: 240000 }, + ) + if (result.error) throw result.error + if (result.status !== 0) throw new Error(firstUsefulLine(result.stderr) ?? 'codex exited with a non-zero status') + return result.stdout + } finally { + rmSync(tempDirectory, { recursive: true, force: true }) + } +} + +function buildReviewPrompt(instructions: LoadedInstruction[]): string { + const files = instructions + .map((instruction) => { + return [ + `Path: ${instruction.path}`, + `Kind: ${instruction.kind}`, + 'Content:', + '```markdown', + instruction.content, + '```', + ].join('\n') + }) + .join('\n\n---\n\n') + + return `You are reviewing agent instruction files for Promptless Skill Report Card. + +Return only JSON that matches the provided schema. Focus on concrete issues that would reduce agent reliability or governance readiness. + +Severity guidance: +- blocker: unsafe, contradictory, non-portable, or likely to cause harmful agent behavior. +- should-fix: unclear triggers, broken references, stale guidance, or missing workflows. +- nice-to-have: polish or maintainability improvements. + +Dimensions: structure, triggering, specificity, governance, safety, evaluation, portability. + +Selected files: + +${files} +` +} + +function parseProviderOutput(stdout: string): ParsedLlmOutput { + const parsed = parseJson(stdout) + if (!parsed) throw new Error('LLM provider did not return parseable JSON') + + const direct = parsedOutputFromObject(parsed) + if (direct) return direct + + const structuredOutput = readRecordProperty(parsed, 'structured_output') + if (structuredOutput) { + const structured = parsedOutputFromObject(structuredOutput) + if (structured) return structured + } + + const resultText = readStringProperty(parsed, 'result') ?? readStringProperty(parsed, 'message') + if (resultText) { + const nested = parseJsonFromText(resultText) + if (nested) { + const nestedOutput = parsedOutputFromObject(nested) + if (nestedOutput) return nestedOutput + } + } + + throw new Error('LLM provider JSON did not match the expected report schema') +} + +function parsedOutputFromObject(object: Record): ParsedLlmOutput | null { + const rawFindings = object.findings + if (!Array.isArray(rawFindings)) return null + + const findings: RawLlmFinding[] = [] + for (const entry of rawFindings) { + if (!isRecord(entry)) continue + const severity = readStringProperty(entry, 'severity') + const dimension = readStringProperty(entry, 'dimension') + const title = readStringProperty(entry, 'title') + const message = readStringProperty(entry, 'message') + const path = readStringProperty(entry, 'path') + if (!severity || !dimension || !title || !message || !path) continue + findings.push({ + severity, + dimension, + title, + message, + path, + remediation: readStringProperty(entry, 'remediation'), + }) + } + + return { findings, summary: readStringProperty(object, 'summary') } +} + +function normalizeFinding(rawFinding: RawLlmFinding, instructions: LoadedInstruction[]): Finding { + const matchedInstruction = instructions.find((instruction) => { + return ( + instruction.path === rawFinding.path || + instruction.displayPath === rawFinding.path || + instruction.path.endsWith(rawFinding.path) + ) + }) + return { + severity: + rawFinding.severity === 'blocker' || rawFinding.severity === 'should-fix' + ? rawFinding.severity + : 'nice-to-have', + source: 'llm', + dimension: normalizeDimension(rawFinding.dimension), + title: rawFinding.title, + message: rawFinding.message, + path: matchedInstruction?.path ?? rawFinding.path, + remediation: rawFinding.remediation ?? undefined, + } +} + +function normalizeDimension(value: string): ScoreDimension { + if ( + value === 'structure' || + value === 'triggering' || + value === 'specificity' || + value === 'governance' || + value === 'safety' || + value === 'evaluation' || + value === 'portability' + ) { + return value + } + return 'governance' +} + +function parseJsonFromText(value: string): Record | null { + return parseJson(value) ?? parseJson(extractFencedJson(value)) +} + +function extractFencedJson(value: string): string { + const match = value.match(/```(?:json)?\s*([\s\S]*?)```/i) + return match?.[1]?.trim() ?? value +} + +function parseJson(value: string): Record | null { + try { + const parsed: unknown = JSON.parse(value) + return isRecord(parsed) ? parsed : null + } catch { + return null + } +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} + +function readStringProperty(object: Record, property: string): string | null { + const value = object[property] + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null +} + +function readRecordProperty(object: Record, property: string): Record | null { + const value = object[property] + return isRecord(value) ? value : null +} + +function firstUsefulLine(output: string): string | null { + const line = output + .split(/\r?\n/) + .map((value) => value.trim()) + .find((value) => value.length > 0) + return line ?? null +} diff --git a/src/lib/skill-report-card/render.ts b/src/lib/skill-report-card/render.ts new file mode 100644 index 0000000..e8d6650 --- /dev/null +++ b/src/lib/skill-report-card/render.ts @@ -0,0 +1,297 @@ +import { writeFileSync } from 'node:fs' +import type { Finding, InstructionScore, SkillReportCard } from './types' + +const SEVERITY_ORDER: Record = { + blocker: 0, + 'should-fix': 1, + 'nice-to-have': 2, +} + +export function writeHtmlReport(report: SkillReportCard): void { + writeFileSync(report.htmlPath, renderHtmlReport(report), 'utf-8') +} + +export function formatTerminalSummary(report: SkillReportCard): string { + const topFindings = allFindings(report.scores).slice(0, 5) + const lines = [ + '', + 'Promptless Skill Report Card', + '', + `Overall: ${report.summary.grade} ${report.summary.score}/100`, + '', + 'Reviewed:', + ` ${report.summary.skillCount} skills`, + ` ${report.summary.rootInstructionCount} root instruction files`, + ` ${report.discovery.scannedRoots.length} scanned roots${report.discovery.truncated ? ' (scan capped)' : ''}`, + '', + 'Findings:', + ` ${report.summary.blockerCount} blocker${plural(report.summary.blockerCount)}`, + ` ${report.summary.shouldFixCount} should fix`, + ` ${report.summary.niceToHaveCount} nice to have`, + ] + + if (topFindings.length > 0) { + lines.push('', 'Top issues:') + topFindings.forEach((finding, index) => { + lines.push( + `${index + 1}. ${severityLabel(finding.severity).padEnd(10)} ${finding.path}`, + ` ${finding.title}`, + ) + }) + } else { + lines.push('', 'No findings above the report-card threshold.') + } + + lines.push('', `HTML report: ${report.htmlPath}`, '') + return lines.join('\n') +} + +export function renderHtmlReport(report: SkillReportCard): string { + const findings = allFindings(report.scores) + return ` + + + + + Promptless Skill Report Card + + + +

+
+
+

Promptless Skill Report Card

+

Generated ${escapeHtml(report.generatedAt)} for ${escapeHtml(report.targetPath)}

+
+
+ ${escapeHtml(report.summary.grade)} + ${report.summary.score}/100 +
+
+ +
+ ${metric('Skills', String(report.summary.skillCount))} + ${metric('Root Files', String(report.summary.rootInstructionCount))} + ${metric('Blockers', String(report.summary.blockerCount))} + ${metric('Should Fix', String(report.summary.shouldFixCount))} +
+ +
+

Instruction Scores

+ + + + + + ${report.scores.map(renderScoreRow).join('\n')} + +
InstructionKindScoreVerdictFindings
+
+ +
+

Findings

+ ${findings.length === 0 ? '
No findings above the report-card threshold.
' : findings.map(renderFinding).join('\n')} +
+ +
+

Run Details

+
+

Scanned roots: ${report.discovery.scannedRoots.length}${report.discovery.truncated ? ' (scan capped)' : ''}

+

skill-validator: ${ + report.validator.skippedReason + ? `skipped, ${escapeHtml(report.validator.skippedReason)}` + : report.validator.available + ? `available, checked ${report.validator.checkedCount} skills` + : 'not found on PATH; skipped' + }

+

LLM review: ${ + report.llm ? `${escapeHtml(report.llm.provider)}${report.llm.rawSummary ? ` — ${escapeHtml(report.llm.rawSummary)}` : ''}` : 'not run' + }

+

skill-validator is used as an internal component when available. Promptless scoring adds governance, safety, portability, and report-card presentation.

+
+
+
+ +` +} + +function metric(label: string, value: string): string { + return `
${escapeHtml(label)}${escapeHtml(value)}
` +} + +function renderScoreRow(score: InstructionScore): string { + return ` + ${escapeHtml(score.instruction.displayPath)} + ${escapeHtml(score.instruction.kind)} + ${escapeHtml(score.grade)} ${score.score}/100 + ${escapeHtml(score.verdict)} + ${score.findings.length} + ` +} + +function renderFinding(finding: Finding): string { + const line = finding.line ? `:${finding.line}` : '' + return `
+
${escapeHtml(severityLabel(finding.severity))}${escapeHtml(finding.source)} · ${escapeHtml(finding.dimension)}
+

${escapeHtml(finding.title)}

+

${escapeHtml(finding.message)}

+

${escapeHtml(finding.path)}${line}

+ ${finding.remediation ? `

Fix: ${escapeHtml(finding.remediation)}

` : ''} +
` +} + +function allFindings(scores: InstructionScore[]): Finding[] { + return scores + .flatMap((score) => score.findings) + .sort((left, right) => { + const severitySort = SEVERITY_ORDER[left.severity] - SEVERITY_ORDER[right.severity] + if (severitySort !== 0) return severitySort + return left.path.localeCompare(right.path) + }) +} + +function severityLabel(severity: Finding['severity']): string { + if (severity === 'should-fix') return 'Should fix' + if (severity === 'nice-to-have') return 'Nice' + return 'Blocker' +} + +function plural(count: number): string { + return count === 1 ? '' : 's' +} + +function escapeHtml(value: string): string { + return value + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, ''') +} diff --git a/src/lib/skill-report-card/scoring.ts b/src/lib/skill-report-card/scoring.ts new file mode 100644 index 0000000..43ee779 --- /dev/null +++ b/src/lib/skill-report-card/scoring.ts @@ -0,0 +1,460 @@ +import { existsSync, readFileSync } from 'node:fs' +import { dirname, resolve } from 'node:path' +import type { + DimensionScore, + DiscoveredInstruction, + Finding, + FindingSeverity, + InstructionScore, + LoadedInstruction, + ReportSummary, + ScoreDimension, +} from './types' + +const SCORE_DIMENSIONS: ScoreDimension[] = [ + 'structure', + 'triggering', + 'specificity', + 'governance', + 'safety', + 'evaluation', + 'portability', +] + +const DIMENSION_WEIGHTS: Record = { + structure: 1, + triggering: 1.2, + specificity: 1, + governance: 0.9, + safety: 1.35, + evaluation: 0.8, + portability: 0.9, +} + +const SEVERITY_PENALTY: Record = { + blocker: 36, + 'should-fix': 18, + 'nice-to-have': 7, +} + +export function estimateTokens(content: string): number { + return Math.max(1, Math.ceil(content.length / 4)) +} + +export function loadInstructions(items: DiscoveredInstruction[]): LoadedInstruction[] { + return items.map((item) => { + const content = readFileSync(item.path, 'utf-8') + return { + ...item, + content, + lineCount: content.split(/\r?\n/).length, + estimatedTokens: estimateTokens(content), + } + }) +} + +export function scoreInstructions( + instructions: LoadedInstruction[], + externalFindings: Finding[], +): InstructionScore[] { + const conflictFindings = findCrossFileConflictFindings(instructions) + + return instructions.map((instruction) => { + const findings = [ + ...deterministicFindings(instruction), + ...conflictFindings.filter((finding) => finding.path === instruction.path), + ...externalFindings.filter((finding) => finding.path === instruction.path), + ] + const dimensions = scoreDimensions(findings) + const score = weightedScore(dimensions) + const blockerCount = findings.filter((finding) => finding.severity === 'blocker').length + const shouldFixCount = findings.filter((finding) => finding.severity === 'should-fix').length + return { + instruction, + score, + grade: gradeForScore(score), + verdict: blockerCount > 0 || score < 65 ? 'block' : shouldFixCount > 0 || score < 90 ? 'revise' : 'ship', + dimensions, + findings, + } + }) +} + +export function summarizeScores(scores: InstructionScore[]): ReportSummary { + const allFindings = scores.flatMap((score) => score.findings) + const totalScore = scores.reduce((total, score) => total + score.score, 0) + const averageScore = scores.length === 0 ? 0 : Math.round(totalScore / scores.length) + const blockerCount = allFindings.filter((finding) => finding.severity === 'blocker').length + const shouldFixCount = allFindings.filter((finding) => finding.severity === 'should-fix').length + const adjustedScore = adjustedSummaryScore(averageScore, blockerCount, shouldFixCount) + return { + score: adjustedScore, + grade: gradeForScore(adjustedScore), + blockerCount, + shouldFixCount, + niceToHaveCount: allFindings.filter((finding) => finding.severity === 'nice-to-have').length, + instructionCount: scores.length, + skillCount: scores.filter((score) => score.instruction.kind.endsWith('skill')).length, + rootInstructionCount: scores.filter((score) => !score.instruction.kind.endsWith('skill')).length, + } +} + +function adjustedSummaryScore( + averageScore: number, + blockerCount: number, + shouldFixCount: number, +): number { + if (blockerCount >= 5) return Math.min(averageScore, 74) + if (blockerCount > 0) return Math.min(averageScore, 84) + if (shouldFixCount > 0) return Math.min(averageScore, 89) + return averageScore +} + +export function gradeForScore(score: number): string { + if (score >= 97) return 'A+' + if (score >= 93) return 'A' + if (score >= 90) return 'A-' + if (score >= 87) return 'B+' + if (score >= 83) return 'B' + if (score >= 80) return 'B-' + if (score >= 77) return 'C+' + if (score >= 73) return 'C' + if (score >= 70) return 'C-' + if (score >= 67) return 'D+' + if (score >= 63) return 'D' + return 'F' +} + +function deterministicFindings(instruction: LoadedInstruction): Finding[] { + const findings: Finding[] = [] + findings.push(...linkFindings(instruction)) + findings.push(...safetyFindings(instruction)) + findings.push(...portabilityFindings(instruction)) + findings.push(...stalenessFindings(instruction)) + + if (instruction.kind.endsWith('skill')) { + findings.push(...skillShapeFindings(instruction)) + } else { + findings.push(...rootInstructionFindings(instruction)) + } + + if (instruction.estimatedTokens > 12000) { + findings.push({ + severity: 'blocker', + source: 'promptless', + dimension: 'specificity', + title: 'Instruction file is too large to use reliably', + message: `This file is roughly ${instruction.estimatedTokens.toLocaleString()} tokens. Agents are likely to ignore or compress important details.`, + path: instruction.path, + remediation: 'Split this into smaller, purpose-specific skills or linked reference files.', + }) + } else if (instruction.estimatedTokens > 6000) { + findings.push({ + severity: 'should-fix', + source: 'promptless', + dimension: 'specificity', + title: 'Instruction file is large', + message: `This file is roughly ${instruction.estimatedTokens.toLocaleString()} tokens, which makes activation and retention less predictable.`, + path: instruction.path, + remediation: 'Move durable references into separate files and keep the activation instructions compact.', + }) + } + + return findings +} + +function skillShapeFindings(instruction: LoadedInstruction): Finding[] { + const findings: Finding[] = [] + const frontmatterDescription = frontmatterField(instruction.content, 'description') + const hasUseWhen = /use when/i.test(frontmatterDescription ?? instruction.content.slice(0, 600)) + const headingCount = (instruction.content.match(/^##\s+/gm) ?? []).length + + if (!frontmatterDescription) { + findings.push({ + severity: 'should-fix', + source: 'promptless', + dimension: 'triggering', + title: 'Skill is missing a frontmatter description', + message: 'Agents need a concise trigger description to decide when this skill applies.', + path: instruction.path, + remediation: 'Add a frontmatter description that starts with "Use when..." and names the activation condition.', + }) + } else if (!hasUseWhen) { + findings.push({ + severity: 'should-fix', + source: 'promptless', + dimension: 'triggering', + title: 'Skill trigger is not phrased as an activation condition', + message: 'The description should tell the agent when to use the skill, not summarize all implementation details.', + path: instruction.path, + remediation: 'Rewrite the description as a short "Use when..." trigger.', + }) + } + + if (frontmatterDescription && frontmatterDescription.length > 260) { + findings.push({ + severity: 'nice-to-have', + source: 'promptless', + dimension: 'specificity', + title: 'Skill description is long', + message: 'Long trigger descriptions are harder for agents to match precisely.', + path: instruction.path, + remediation: 'Keep the description to one sentence and move details into the body.', + }) + } + + if (headingCount < 2) { + findings.push({ + severity: 'should-fix', + source: 'promptless', + dimension: 'structure', + title: 'Skill body has little visible structure', + message: 'A skill should separate activation, workflow, and output expectations so agents can follow it quickly.', + path: instruction.path, + remediation: 'Add short sections such as Workflow, Output, and Boundaries.', + }) + } + + if (!/\b(workflow|steps|process|procedure|instructions)\b/i.test(instruction.content)) { + findings.push({ + severity: 'should-fix', + source: 'promptless', + dimension: 'evaluation', + title: 'Skill lacks an explicit workflow', + message: 'The skill describes a capability but does not clearly tell the agent what to do.', + path: instruction.path, + remediation: 'Add a compact ordered workflow for the agent to execute.', + }) + } + + if (!/\b(output|deliverable|report|response|result|acceptance|verify|test)\b/i.test(instruction.content)) { + findings.push({ + severity: 'nice-to-have', + source: 'promptless', + dimension: 'evaluation', + title: 'Skill lacks output or verification criteria', + message: 'The skill does not say how success should be recognized.', + path: instruction.path, + remediation: 'Add expected output shape, verification steps, or acceptance criteria.', + }) + } + + return findings +} + +function rootInstructionFindings(instruction: LoadedInstruction): Finding[] { + const findings: Finding[] = [] + if (!/\b(test|lint|typecheck|verify|build)\b/i.test(instruction.content)) { + findings.push({ + severity: 'nice-to-have', + source: 'promptless', + dimension: 'evaluation', + title: 'Root instructions do not mention verification', + message: 'Agents get better outcomes when the root policy names the expected local checks.', + path: instruction.path, + remediation: 'Add the project-specific verification commands or point to the source of truth.', + }) + } + if (!/\b(owner|maintainer|escalate|approval|confirm)\b/i.test(instruction.content)) { + findings.push({ + severity: 'nice-to-have', + source: 'promptless', + dimension: 'governance', + title: 'Root instructions do not define escalation boundaries', + message: 'The file does not say when an agent should ask a human before taking risky action.', + path: instruction.path, + remediation: 'Document approval boundaries for production, credentials, data writes, and destructive commands.', + }) + } + return findings +} + +function linkFindings(instruction: LoadedInstruction): Finding[] { + const findings: Finding[] = [] + const linkPattern = /\[[^\]]+\]\(([^)]+)\)/g + let match: RegExpExecArray | null + + while ((match = linkPattern.exec(instruction.content)) !== null) { + const rawTarget = match[1]?.trim() + if (!rawTarget || shouldIgnoreLink(rawTarget)) continue + + const withoutAnchor = rawTarget.split('#')[0] ?? rawTarget + if (!withoutAnchor) continue + + const resolved = resolve(dirname(instruction.path), withoutAnchor) + if (!existsSync(resolved)) { + findings.push({ + severity: 'should-fix', + source: 'promptless', + dimension: 'portability', + title: 'Relative link target is missing', + message: `The link target "${rawTarget}" does not exist from this instruction file.`, + path: instruction.path, + line: lineForOffset(instruction.content, match.index), + remediation: 'Update the link or remove the stale reference.', + }) + } + } + + return findings +} + +function shouldIgnoreLink(target: string): boolean { + return ( + /^[a-z][a-z0-9+.-]*:/i.test(target) || + target.startsWith('#') || + target.startsWith('/') || + target.startsWith('~') + ) +} + +function safetyFindings(instruction: LoadedInstruction): Finding[] { + const findings: Finding[] = [] + const lines = instruction.content.split(/\r?\n/) + + lines.forEach((line, index) => { + if (isProtectiveLine(line)) return + if (/secretsmanager\s+(put-secret-value|create-secret|update-secret|delete-secret)/i.test(line)) { + findings.push({ + severity: 'blocker', + source: 'promptless', + dimension: 'safety', + title: 'Instruction includes a Secrets Manager write command', + message: 'Agents should not be instructed to write secrets directly.', + path: instruction.path, + line: index + 1, + remediation: 'Replace this with a human-run command and an explicit approval boundary.', + }) + } + if (/\b(git\s+reset\s+--hard|git\s+checkout\s+--|rm\s+-rf|push\s+--force|force-push)\b/i.test(line)) { + findings.push({ + severity: 'blocker', + source: 'promptless', + dimension: 'safety', + title: 'Instruction includes destructive command guidance', + message: 'Destructive commands need an explicit prohibition or human approval path.', + path: instruction.path, + line: index + 1, + remediation: 'State the approval requirement or remove the command from agent-executable instructions.', + }) + } + }) + + if ( + instruction.kind.endsWith('skill') && + /\b(production|prod|deploy|database|secret|credential)\b/i.test(instruction.content) && + /\b(write|update|delete|insert|deploy|push|send|post|create)\b/i.test(instruction.content) && + !/\b(approval|confirm|consent|read-only|ask the user|human)\b/i.test(instruction.content) + ) { + findings.push({ + severity: 'blocker', + source: 'promptless', + dimension: 'safety', + title: 'Risky skill lacks an approval boundary', + message: 'This skill appears able to mutate production, credentials, or durable data without saying when to ask a human.', + path: instruction.path, + remediation: 'Add an explicit approval boundary before production writes, credential changes, deploys, or destructive actions.', + }) + } + + return findings +} + +function isProtectiveLine(line: string): boolean { + return /\b(never|do not|don't|must not|without explicit|only after approval|ask before)\b/i.test(line) +} + +function portabilityFindings(instruction: LoadedInstruction): Finding[] { + const findings: Finding[] = [] + const localPathPattern = /(?:^|[\s"`'])\/Users\/[A-Za-z0-9._-]+\//gm + let match: RegExpExecArray | null + + while ((match = localPathPattern.exec(instruction.content)) !== null) { + findings.push({ + severity: 'should-fix', + source: 'promptless', + dimension: 'portability', + title: 'Instruction contains a user-specific absolute path', + message: 'Hardcoded local paths make skills fail on other machines.', + path: instruction.path, + line: lineForOffset(instruction.content, match.index), + remediation: 'Use repo-relative paths, environment variables, or documented placeholders.', + }) + } + + return findings +} + +function stalenessFindings(instruction: LoadedInstruction): Finding[] { + const findings: Finding[] = [] + const stalePattern = /\b(todo|fixme|deprecated|obsolete|old workflow|stale)\b/i + const lines = instruction.content.split(/\r?\n/) + lines.forEach((line, index) => { + if (!stalePattern.test(line)) return + findings.push({ + severity: /deprecated|obsolete|stale/i.test(line) ? 'should-fix' : 'nice-to-have', + source: 'promptless', + dimension: 'governance', + title: 'Instruction includes stale-maintenance language', + message: 'Stale markers make it unclear whether the instruction is still authoritative.', + path: instruction.path, + line: index + 1, + remediation: 'Resolve the marker or move speculative notes out of active instructions.', + }) + }) + return findings +} + +function findCrossFileConflictFindings(instructions: LoadedInstruction[]): Finding[] { + const prohibitsForcePush = instructions.filter((instruction) => + /\b(never|do not|don't|must not)\b[^.\n]*(force-push|push\s+--force)/i.test(instruction.content), + ) + const allowsForcePush = instructions.filter((instruction) => + /\b(use|run|perform|allow)\b[^.\n]*(force-push|push\s+--force)/i.test(instruction.content), + ) + + if (prohibitsForcePush.length === 0 || allowsForcePush.length === 0) return [] + + return [...prohibitsForcePush, ...allowsForcePush].map((instruction) => ({ + severity: 'blocker', + source: 'promptless', + dimension: 'governance', + title: 'Selected instructions conflict on force-push policy', + message: 'One selected instruction prohibits force-push while another appears to allow it.', + path: instruction.path, + remediation: 'Make the policy consistent across selected instruction roots.', + })) +} + +function scoreDimensions(findings: Finding[]): DimensionScore[] { + return SCORE_DIMENSIONS.map((dimension) => { + const penalty = findings + .filter((finding) => finding.dimension === dimension) + .reduce((total, finding) => total + SEVERITY_PENALTY[finding.severity], 0) + return { dimension, score: Math.max(0, 100 - penalty) } + }) +} + +function weightedScore(dimensions: DimensionScore[]): number { + const weightedTotal = dimensions.reduce( + (total, dimension) => total + dimension.score * DIMENSION_WEIGHTS[dimension.dimension], + 0, + ) + const weightTotal = dimensions.reduce((total, dimension) => total + DIMENSION_WEIGHTS[dimension.dimension], 0) + return Math.round(weightedTotal / weightTotal) +} + +function frontmatterField(content: string, fieldName: string): string | null { + if (!content.startsWith('---')) return null + const end = content.indexOf('\n---', 3) + if (end === -1) return null + const frontmatter = content.slice(3, end) + const pattern = new RegExp(`^${fieldName}:\\s*(.+)$`, 'im') + const match = frontmatter.match(pattern) + if (!match?.[1]) return null + return match[1].trim().replace(/^['"]|['"]$/g, '') +} + +function lineForOffset(content: string, offset: number): number { + return content.slice(0, offset).split(/\r?\n/).length +} diff --git a/src/lib/skill-report-card/selectionGroups.ts b/src/lib/skill-report-card/selectionGroups.ts new file mode 100644 index 0000000..fd5761b --- /dev/null +++ b/src/lib/skill-report-card/selectionGroups.ts @@ -0,0 +1,188 @@ +import { spawnSync } from 'node:child_process' +import { realpathSync } from 'node:fs' +import { homedir } from 'node:os' +import { dirname, join, relative, resolve, sep } from 'node:path' +import type { DiscoveredInstruction } from './types' + +export type SelectionGroupKind = 'git-repo' | 'global-location' | 'directory' + +export interface SelectionGroup { + id: string + kind: SelectionGroupKind + label: string + sortLabel: string + path: string + items: DiscoveredInstruction[] + skillCount: number + rootInstructionCount: number +} + +interface GroupSeed { + id: string + kind: SelectionGroupKind + labelPrefix: string | null + path: string + sortLabel: string +} + +interface KnownLocation { + id: string + label: string + pathSegments: string[] +} + +const KNOWN_GLOBAL_LOCATIONS: KnownLocation[] = [ + { id: 'codex-user-skills', label: 'Codex user skills', pathSegments: ['.codex', 'skills'] }, + { id: 'codex-user-instructions', label: 'Codex user instructions', pathSegments: ['.codex'] }, + { id: 'claude-code-user-skills', label: 'Claude Code user skills', pathSegments: ['.claude', 'skills'] }, + { id: 'claude-code-user-instructions', label: 'Claude Code user instructions', pathSegments: ['.claude'] }, + { id: 'agent-user-skills', label: 'Agent user skills', pathSegments: ['.agents', 'skills'] }, + { id: 'agent-user-instructions', label: 'Agent user instructions', pathSegments: ['.agents'] }, +] + +export function groupInstructionsForSelection( + items: DiscoveredInstruction[], + homeDirectory = homedir(), +): SelectionGroup[] { + const realHomeDirectory = realPathOrResolved(homeDirectory) + const gitRootCache = new Map() + const groupsById = new Map() + + for (const item of items) { + const seed = groupSeedForItem(item, realHomeDirectory, gitRootCache) + const existing = groupsById.get(seed.id) + if (existing) { + existing.items.push(item) + } else { + groupsById.set(seed.id, { seed, items: [item] }) + } + } + + return [...groupsById.values()] + .map(({ seed, items: groupItems }) => { + const skillCount = groupItems.filter((item) => item.kind.endsWith('skill')).length + const rootInstructionCount = groupItems.length - skillCount + return { + id: seed.id, + kind: seed.kind, + label: groupLabel(seed, skillCount, rootInstructionCount, realHomeDirectory), + sortLabel: seed.sortLabel, + path: seed.path, + items: groupItems.sort((left, right) => left.displayPath.localeCompare(right.displayPath)), + skillCount, + rootInstructionCount, + } + }) + .sort((left, right) => left.sortLabel.localeCompare(right.sortLabel)) +} + +export function groupsForInteractivePrompt(groups: SelectionGroup[]): SelectionGroup[] { + const skillBearingGroups = groups.filter((group) => group.skillCount > 0) + return skillBearingGroups.length > 0 ? skillBearingGroups : groups +} + +function groupSeedForItem( + item: DiscoveredInstruction, + homeDirectory: string, + gitRootCache: Map, +): GroupSeed { + const knownLocation = knownGlobalLocationForItem(item.path, homeDirectory) + if (knownLocation) { + const locationPath = join(homeDirectory, ...knownLocation.pathSegments) + return { + id: `global:${knownLocation.id}`, + kind: 'global-location', + labelPrefix: knownLocation.label, + path: locationPath, + sortLabel: `0:${knownLocation.id}`, + } + } + + const gitRoot = gitRootForItem(item, gitRootCache) + if (gitRoot) { + return { + id: `repo:${gitRoot}`, + kind: 'git-repo', + labelPrefix: null, + path: gitRoot, + sortLabel: `1:${displayPath(gitRoot, homeDirectory)}`, + } + } + + return { + id: `directory:${item.rootPath}`, + kind: 'directory', + labelPrefix: null, + path: item.rootPath, + sortLabel: `2:${displayPath(item.rootPath, homeDirectory)}`, + } +} + +function knownGlobalLocationForItem(path: string, homeDirectory: string): KnownLocation | null { + const relativeSegments = relative(homeDirectory, path).split(sep) + if (relativeSegments[0]?.startsWith('..') || relativeSegments[0] === '') return null + + return ( + KNOWN_GLOBAL_LOCATIONS.find((location) => + startsWithSegments(relativeSegments, location.pathSegments), + ) ?? null + ) +} + +function gitRootForItem( + item: DiscoveredInstruction, + gitRootCache: Map, +): string | null { + const directory = dirname(item.path) + const cached = gitRootCache.get(directory) + if (cached !== undefined) return cached + + const result = spawnSync('git', ['-C', directory, 'rev-parse', '--show-toplevel'], { + encoding: 'utf-8', + timeout: 5000, + }) + const gitRoot = result.status === 0 ? realPathOrResolved(result.stdout.trim()) : null + gitRootCache.set(directory, gitRoot) + return gitRoot +} + +function groupLabel( + seed: GroupSeed, + skillCount: number, + rootInstructionCount: number, + homeDirectory: string, +): string { + const counts = [ + skillCount > 0 ? `${skillCount} skill${plural(skillCount)}` : null, + rootInstructionCount > 0 + ? `${rootInstructionCount} instruction file${plural(rootInstructionCount)}` + : null, + ].filter((value): value is string => value !== null) + const pathLabel = displayPath(seed.path, homeDirectory) + const name = seed.labelPrefix ? `${seed.labelPrefix} (${pathLabel})` : pathLabel + return `${name} (${counts.join(', ')})` +} + +function displayPath(path: string, homeDirectory: string): string { + const relativeToHome = relative(homeDirectory, path) + if (relativeToHome && !relativeToHome.startsWith('..') && !relativeToHome.startsWith(sep)) { + return join('~', relativeToHome) + } + return path +} + +function startsWithSegments(segments: string[], prefix: string[]): boolean { + return prefix.every((segment, index) => segments[index] === segment) +} + +function realPathOrResolved(path: string): string { + try { + return realpathSync(path) + } catch { + return resolve(path) + } +} + +function plural(count: number): string { + return count === 1 ? '' : 's' +} diff --git a/src/lib/skill-report-card/skill-report-card.test.ts b/src/lib/skill-report-card/skill-report-card.test.ts new file mode 100644 index 0000000..f328995 --- /dev/null +++ b/src/lib/skill-report-card/skill-report-card.test.ts @@ -0,0 +1,469 @@ +import { strict as assert } from 'node:assert' +import { spawnSync } from 'node:child_process' +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' +import { discoverInstructions } from './discovery' +import { estimateLlmUsage, runLlmReview } from './providers' +import { formatTerminalSummary, writeHtmlReport } from './render' +import { loadInstructions, scoreInstructions, summarizeScores } from './scoring' +import { groupInstructionsForSelection, groupsForInteractivePrompt } from './selectionGroups' +import type { SkillReportCard } from './types' +import { runSkillValidator } from './validator' + +test('discovers root instructions and hidden skill directories', async () => { + const root = makeTempDirectory() + try { + writeFileSync(join(root, 'AGENTS.md'), 'Run tests before committing.\n', 'utf-8') + mkdirSync(join(root, '.agents', 'skills', 'review-docs'), { recursive: true }) + writeFileSync(join(root, '.agents', 'skills', 'review-docs', 'SKILL.md'), goodSkill(), 'utf-8') + + const discovery = await discoverInstructions(root, { + includeMachineScan: false, + maxDepth: 5, + maxDirectories: 100, + homeDirectory: join(root, 'home'), + }) + + assert.equal(discovery.items.length, 2) + assert.deepEqual( + discovery.items.map((item) => item.kind).sort(), + ['agent-skill', 'agents-md'], + ) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) + +test('skips generated worktree and plugin cache copies during broad scans', async () => { + const root = makeTempDirectory() + try { + const homeDirectory = join(root, 'home') + const projectDirectory = join(homeDirectory, 'project') + const codexSkillDirectory = join(homeDirectory, '.codex', 'skills', 'personal') + const worktreeSkillDirectory = join( + homeDirectory, + '.codex', + 'worktrees', + 'c5f4', + 'project', + '.agents', + 'skills', + 'copied', + ) + const pluginSkillDirectory = join( + homeDirectory, + '.claude', + 'plugins', + 'cache', + 'vendor', + '1.0.0', + '.claude', + 'skills', + 'cached', + ) + const codexTempSkillDirectory = join( + homeDirectory, + '.codex', + '.tmp', + 'plugins', + 'vendor', + 'skills', + 'cached', + ) + const customerRepoSkillDirectory = join( + homeDirectory, + '.codex', + 'customer-repos', + 'acme', + '.agents', + 'skills', + 'copied', + ) + mkdirSync(projectDirectory, { recursive: true }) + mkdirSync(codexSkillDirectory, { recursive: true }) + mkdirSync(worktreeSkillDirectory, { recursive: true }) + mkdirSync(pluginSkillDirectory, { recursive: true }) + mkdirSync(codexTempSkillDirectory, { recursive: true }) + mkdirSync(customerRepoSkillDirectory, { recursive: true }) + writeFileSync(join(projectDirectory, 'AGENTS.md'), 'Run tests before committing.\n', 'utf-8') + writeFileSync(join(codexSkillDirectory, 'SKILL.md'), goodSkill(), 'utf-8') + writeFileSync(join(worktreeSkillDirectory, 'SKILL.md'), unsafeSkill(), 'utf-8') + writeFileSync(join(pluginSkillDirectory, 'SKILL.md'), unsafeSkill(), 'utf-8') + writeFileSync(join(codexTempSkillDirectory, 'SKILL.md'), unsafeSkill(), 'utf-8') + writeFileSync(join(customerRepoSkillDirectory, 'SKILL.md'), unsafeSkill(), 'utf-8') + + const discovery = await discoverInstructions(projectDirectory, { + includeMachineScan: true, + maxDepth: 8, + maxDirectories: 1000, + homeDirectory, + }) + + assert.equal(discovery.items.some((item) => item.path.includes('/.codex/worktrees/')), false) + assert.equal(discovery.items.some((item) => item.path.includes('/.codex/.tmp/')), false) + assert.equal(discovery.items.some((item) => item.path.includes('/.codex/customer-repos/')), false) + assert.equal(discovery.items.some((item) => item.path.includes('/.claude/plugins/cache/')), false) + assert.ok(discovery.items.some((item) => item.path.endsWith('/project/AGENTS.md'))) + assert.ok(discovery.items.some((item) => item.path.endsWith('/.codex/skills/personal/SKILL.md'))) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) + +test('keeps explicit targets inside generated worktree paths', async () => { + const root = makeTempDirectory() + try { + const homeDirectory = join(root, 'home') + const worktreeProject = join(homeDirectory, '.codex', 'worktrees', 'c5f4', 'project') + mkdirSync(worktreeProject, { recursive: true }) + writeFileSync(join(worktreeProject, 'AGENTS.md'), 'Run tests before committing.\n', 'utf-8') + + const discovery = await discoverInstructions(worktreeProject, { + includeMachineScan: false, + maxDepth: 4, + maxDirectories: 100, + homeDirectory, + }) + + assert.deepEqual( + discovery.items.map((item) => item.displayPath), + ['~/.codex/worktrees/c5f4/project/AGENTS.md'], + ) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) + +test('groups interactive selection by git repo and global skill locations', async () => { + const root = makeTempDirectory() + try { + const homeDirectory = join(root, 'home') + const repoDirectory = join(homeDirectory, 'promptless', 'promptless') + const firstSkillDirectory = join(repoDirectory, '.agents', 'skills', 'review-docs') + const secondSkillDirectory = join(repoDirectory, '.agents', 'skills', 'ship-docs') + const codexSkillDirectory = join(homeDirectory, '.codex', 'skills', 'personal') + const claudeSkillDirectory = join(homeDirectory, '.claude', 'skills', 'yc-cli') + mkdirSync(firstSkillDirectory, { recursive: true }) + mkdirSync(secondSkillDirectory, { recursive: true }) + mkdirSync(codexSkillDirectory, { recursive: true }) + mkdirSync(claudeSkillDirectory, { recursive: true }) + initializeGitRepo(repoDirectory) + writeFileSync(join(repoDirectory, 'AGENTS.md'), 'Run tests before committing.\n', 'utf-8') + writeFileSync(join(firstSkillDirectory, 'SKILL.md'), goodSkill(), 'utf-8') + writeFileSync(join(secondSkillDirectory, 'SKILL.md'), goodSkill(), 'utf-8') + writeFileSync(join(codexSkillDirectory, 'SKILL.md'), goodSkill(), 'utf-8') + writeFileSync(join(claudeSkillDirectory, 'SKILL.md'), goodSkill(), 'utf-8') + + const discovery = await discoverInstructions(repoDirectory, { + includeMachineScan: true, + maxDepth: 8, + maxDirectories: 1000, + homeDirectory, + }) + const groups = groupInstructionsForSelection(discovery.items, homeDirectory) + const promptGroups = groupsForInteractivePrompt(groups) + const labels = groups.map((group) => group.label) + const promptLabels = promptGroups.map((group) => group.label) + + assert.ok(groups.length < discovery.items.length) + assert.ok(labels.some((label) => label.includes('~/promptless/promptless (2 skills, 1 instruction file)'))) + assert.ok(labels.some((label) => label.includes('Codex user skills (~/.codex/skills) (1 skill)'))) + assert.ok(labels.some((label) => label.includes('Claude Code user skills (~/.claude/skills) (1 skill)'))) + assert.deepEqual(promptLabels, labels) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) + +test('hides instruction-only repos from the interactive prompt when skills exist', async () => { + const root = makeTempDirectory() + try { + const homeDirectory = join(root, 'home') + const skillRepo = join(homeDirectory, 'skill-repo') + const rootOnlyRepo = join(homeDirectory, 'root-only-repo') + const skillDirectory = join(skillRepo, '.agents', 'skills', 'review-docs') + mkdirSync(skillDirectory, { recursive: true }) + mkdirSync(rootOnlyRepo, { recursive: true }) + initializeGitRepo(skillRepo) + initializeGitRepo(rootOnlyRepo) + writeFileSync(join(skillRepo, 'AGENTS.md'), 'Run tests before committing.\n', 'utf-8') + writeFileSync(join(skillDirectory, 'SKILL.md'), goodSkill(), 'utf-8') + writeFileSync(join(rootOnlyRepo, 'AGENTS.md'), 'Run tests before committing.\n', 'utf-8') + + const discovery = await discoverInstructions(homeDirectory, { + includeMachineScan: false, + maxDepth: 5, + maxDirectories: 1000, + homeDirectory, + }) + const promptGroups = groupsForInteractivePrompt(groupInstructionsForSelection(discovery.items, homeDirectory)) + const labels = promptGroups.map((group) => group.label) + + assert.equal(labels.some((label) => label.includes('skill-repo')), true) + assert.equal(labels.some((label) => label.includes('root-only-repo')), false) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) + +test('scores unsafe and stale skills below strong skills', () => { + const root = makeTempDirectory() + try { + mkdirSync(join(root, '.agents', 'skills', 'good'), { recursive: true }) + mkdirSync(join(root, '.agents', 'skills', 'unsafe'), { recursive: true }) + const goodPath = join(root, '.agents', 'skills', 'good', 'SKILL.md') + const unsafePath = join(root, '.agents', 'skills', 'unsafe', 'SKILL.md') + writeFileSync(goodPath, goodSkill(), 'utf-8') + writeFileSync(unsafePath, unsafeSkill(), 'utf-8') + + const loaded = loadInstructions([ + { + id: goodPath, + path: goodPath, + rootPath: join(root, '.agents'), + displayPath: goodPath, + title: 'good', + kind: 'agent-skill', + sizeBytes: 1, + }, + { + id: unsafePath, + path: unsafePath, + rootPath: join(root, '.agents'), + displayPath: unsafePath, + title: 'unsafe', + kind: 'agent-skill', + sizeBytes: 1, + }, + ]) + + const scores = scoreInstructions(loaded, []) + const goodScore = scores.find((score) => score.instruction.path === goodPath) + const unsafeScore = scores.find((score) => score.instruction.path === unsafePath) + + assert.ok(goodScore) + assert.ok(unsafeScore) + assert.equal(goodScore.verdict, 'ship') + assert.equal(unsafeScore.verdict, 'block') + assert.ok(unsafeScore.score < goodScore.score) + assert.ok(unsafeScore.findings.some((finding) => finding.dimension === 'safety')) + assert.ok(summarizeScores(scores).score <= 84) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) + +test('renders terminal and HTML reports', () => { + const root = makeTempDirectory() + try { + const skillPath = join(root, 'SKILL.md') + const htmlPath = join(root, 'report.html') + writeFileSync(skillPath, goodSkill(), 'utf-8') + const loaded = loadInstructions([ + { + id: skillPath, + path: skillPath, + rootPath: root, + displayPath: skillPath, + title: 'good', + kind: 'agent-skill', + sizeBytes: 1, + }, + ]) + const scores = scoreInstructions(loaded, []) + const report: SkillReportCard = { + schemaVersion: 'promptless.skill-report-card.v1', + generatedAt: '2026-06-05T00:00:00.000Z', + targetPath: root, + htmlPath, + discovery: { items: [], scannedRoots: [root], truncated: false }, + selectedInstructions: loaded, + scores, + summary: summarizeScores(scores), + validator: { available: false, checkedCount: 0, findings: [] }, + llm: null, + } + + const terminal = formatTerminalSummary(report) + writeHtmlReport(report) + const html = readFileSync(htmlPath, 'utf-8') + + assert.match(terminal, /Promptless Skill Report Card/) + assert.match(terminal, /HTML report:/) + assert.match(html, /Instruction Scores/) + assert.match(html, /skill-validator/) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) + +test('estimates LLM usage from selected content', () => { + const root = makeTempDirectory() + try { + const skillPath = join(root, 'SKILL.md') + writeFileSync(skillPath, goodSkill(), 'utf-8') + const loaded = loadInstructions([ + { + id: skillPath, + path: skillPath, + rootPath: root, + displayPath: skillPath, + title: 'good', + kind: 'agent-skill', + sizeBytes: 1, + }, + ]) + + const estimate = estimateLlmUsage(loaded) + assert.ok(estimate.inputTokens > loaded[0].estimatedTokens) + assert.ok(estimate.outputTokens >= 1200) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) + +test('parses skill-validator level fields without turning passes into findings', () => { + const root = makeTempDirectory() + const originalPath = process.env.PATH + try { + const binDirectory = join(root, 'bin') + const skillDirectory = join(root, '.agents', 'skills', 'good') + mkdirSync(binDirectory, { recursive: true }) + mkdirSync(skillDirectory, { recursive: true }) + const validatorPath = join(binDirectory, 'skill-validator') + const skillPath = join(skillDirectory, 'SKILL.md') + writeFileSync(skillPath, goodSkill(), 'utf-8') + writeFileSync( + validatorPath, + `#!/usr/bin/env node +console.log(JSON.stringify({ + results: [ + { level: "pass", category: "Structure", message: "SKILL.md found" }, + { level: "warning", category: "Links", message: "optional reference missing" } + ] +})) +`, + 'utf-8', + ) + chmodSync(validatorPath, 0o755) + process.env.PATH = `${binDirectory}:${originalPath ?? ''}` + + const loaded = loadInstructions([ + { + id: skillPath, + path: skillPath, + rootPath: join(root, '.agents'), + displayPath: skillPath, + title: 'good', + kind: 'agent-skill', + sizeBytes: 1, + }, + ]) + + const summary = runSkillValidator(loaded) + assert.equal(summary.available, true) + assert.equal(summary.checkedCount, 1) + assert.equal(summary.findings.length, 1) + assert.equal(summary.findings[0].title, 'Links') + } finally { + process.env.PATH = originalPath + rmSync(root, { recursive: true, force: true }) + } +}) + +test('reads Claude structured_output wrapper for LLM review results', () => { + const root = makeTempDirectory() + const originalPath = process.env.PATH + try { + const binDirectory = join(root, 'bin') + const skillPath = join(root, 'SKILL.md') + mkdirSync(binDirectory, { recursive: true }) + const claudePath = join(binDirectory, 'claude') + writeFileSync(skillPath, goodSkill(), 'utf-8') + writeFileSync( + claudePath, + `#!/usr/bin/env node +console.log(JSON.stringify({ + type: "result", + subtype: "success", + result: "", + structured_output: { + summary: "ok", + findings: [ + { + severity: "should-fix", + dimension: "triggering", + title: "Trigger is vague", + message: "The trigger could be more specific.", + path: process.argv[1] ?? "SKILL.md", + remediation: null + } + ] + } +})) +`, + 'utf-8', + ) + chmodSync(claudePath, 0o755) + process.env.PATH = `${binDirectory}:${originalPath ?? ''}` + + const loaded = loadInstructions([ + { + id: skillPath, + path: skillPath, + rootPath: root, + displayPath: skillPath, + title: 'good', + kind: 'agent-skill', + sizeBytes: 1, + }, + ]) + + const review = runLlmReview('claude', loaded) + assert.equal(review.rawSummary, 'ok') + assert.equal(review.findings.length, 1) + assert.equal(review.findings[0].source, 'llm') + } finally { + process.env.PATH = originalPath + rmSync(root, { recursive: true, force: true }) + } +}) + +function makeTempDirectory(): string { + return mkdtempSync(join(tmpdir(), 'promptless-skill-report-card-test-')) +} + +function initializeGitRepo(path: string): void { + const result = spawnSync('git', ['init'], { cwd: path, encoding: 'utf-8' }) + assert.equal(result.status, 0) +} + +function goodSkill(): string { + return `--- +description: Use when reviewing documentation changes for accuracy and style. +--- + +## Workflow + +1. Read the changed documentation. +2. Check claims against linked sources. +3. Verify formatting and local commands. + +## Output + +Return concrete findings with file paths, severity, and remediation. +` +} + +function unsafeSkill(): string { + return `--- +description: Deploy production fixes. +--- + +Run rm -rf dist and push --force when deployment gets stuck. + +TODO: replace this old workflow later. +` +} diff --git a/src/lib/skill-report-card/types.ts b/src/lib/skill-report-card/types.ts new file mode 100644 index 0000000..394ff8b --- /dev/null +++ b/src/lib/skill-report-card/types.ts @@ -0,0 +1,117 @@ +export type InstructionKind = + | 'agents-md' + | 'claude-md' + | 'agent-skill' + | 'claude-skill' + | 'codex-skill' + +export type FindingSeverity = 'blocker' | 'should-fix' | 'nice-to-have' + +export type FindingSource = 'promptless' | 'skill-validator' | 'llm' + +export type ScoreDimension = + | 'structure' + | 'triggering' + | 'specificity' + | 'governance' + | 'safety' + | 'evaluation' + | 'portability' + +export type LlmProvider = 'off' | 'auto' | 'claude' | 'codex' + +export type ResolvedLlmProvider = 'claude' | 'codex' + +export interface DiscoveredInstruction { + id: string + path: string + rootPath: string + displayPath: string + title: string + kind: InstructionKind + sizeBytes: number +} + +export interface DiscoveryResult { + items: DiscoveredInstruction[] + scannedRoots: string[] + truncated: boolean +} + +export interface LoadedInstruction extends DiscoveredInstruction { + content: string + lineCount: number + estimatedTokens: number +} + +export interface Finding { + severity: FindingSeverity + source: FindingSource + dimension: ScoreDimension + title: string + message: string + path: string + line?: number + remediation?: string +} + +export interface DimensionScore { + dimension: ScoreDimension + score: number +} + +export interface InstructionScore { + instruction: LoadedInstruction + score: number + grade: string + verdict: 'ship' | 'revise' | 'block' + dimensions: DimensionScore[] + findings: Finding[] +} + +export interface ProviderAvailability { + claude: boolean + codex: boolean +} + +export interface LlmUsageEstimate { + inputTokens: number + outputTokens: number +} + +export interface LlmReviewResult { + provider: ResolvedLlmProvider + findings: Finding[] + rawSummary: string | null +} + +export interface SkillValidatorSummary { + available: boolean + checkedCount: number + findings: Finding[] + skippedReason?: string +} + +export interface ReportSummary { + score: number + grade: string + blockerCount: number + shouldFixCount: number + niceToHaveCount: number + instructionCount: number + skillCount: number + rootInstructionCount: number +} + +export interface SkillReportCard { + schemaVersion: 'promptless.skill-report-card.v1' + generatedAt: string + targetPath: string + htmlPath: string + discovery: DiscoveryResult + selectedInstructions: LoadedInstruction[] + scores: InstructionScore[] + summary: ReportSummary + validator: SkillValidatorSummary + llm: LlmReviewResult | null +} diff --git a/src/lib/skill-report-card/validator.ts b/src/lib/skill-report-card/validator.ts new file mode 100644 index 0000000..b290c7b --- /dev/null +++ b/src/lib/skill-report-card/validator.ts @@ -0,0 +1,141 @@ +import { spawnSync } from 'node:child_process' +import { dirname } from 'node:path' +import type { Finding, LoadedInstruction, SkillValidatorSummary } from './types' + +const DEFAULT_MAX_VALIDATOR_CHECKS = 40 + +interface ValidatorResultEntry { + name: string | null + status: string | null + message: string | null +} + +export function runSkillValidator( + instructions: LoadedInstruction[], + maxChecks = DEFAULT_MAX_VALIDATOR_CHECKS, +): SkillValidatorSummary { + if (!commandExists('skill-validator')) { + return { available: false, checkedCount: 0, findings: [] } + } + + const skillInstructions = instructions.filter((instruction) => instruction.kind.endsWith('skill')) + if (skillInstructions.length > maxChecks) { + return { + available: true, + checkedCount: 0, + findings: [], + skippedReason: `selection includes ${skillInstructions.length} skills; validator cap is ${maxChecks}`, + } + } + + const findings: Finding[] = [] + let checkedCount = 0 + + for (const instruction of skillInstructions) { + const result = spawnSync( + 'skill-validator', + ['check', dirname(instruction.path), '--output', 'json', '--allow-extra-frontmatter'], + { encoding: 'utf-8', timeout: 30000 }, + ) + + checkedCount += 1 + if (result.error) { + findings.push({ + severity: 'should-fix', + source: 'skill-validator', + dimension: 'structure', + title: 'skill-validator could not run', + message: result.error.message, + path: instruction.path, + remediation: 'Run skill-validator manually for more detail.', + }) + continue + } + + const parsedEntries = parseValidatorEntries(result.stdout) + if (parsedEntries.length === 0 && result.status !== 0) { + findings.push({ + severity: 'should-fix', + source: 'skill-validator', + dimension: 'structure', + title: 'skill-validator reported an issue', + message: firstUsefulLine(result.stderr) ?? firstUsefulLine(result.stdout) ?? 'The validator exited with a non-zero status.', + path: instruction.path, + remediation: 'Run skill-validator manually for the full validation output.', + }) + continue + } + + for (const entry of parsedEntries) { + const status = entry.status?.toLowerCase() + if (status === 'passed' || status === 'pass' || status === 'ok') continue + findings.push({ + severity: status === 'failed' || status === 'fail' || status === 'error' ? 'should-fix' : 'nice-to-have', + source: 'skill-validator', + dimension: 'structure', + title: entry.name ?? 'skill-validator finding', + message: entry.message ?? 'skill-validator flagged this skill.', + path: instruction.path, + remediation: 'Run skill-validator manually for detailed remediation guidance.', + }) + } + } + + return { available: true, checkedCount, findings } +} + +export function commandExists(command: string): boolean { + const result = spawnSync('which', [command], { encoding: 'utf-8' }) + return result.status === 0 && result.stdout.trim().length > 0 +} + +function parseValidatorEntries(stdout: string): ValidatorResultEntry[] { + const parsed = parseJsonObject(stdout) + if (!parsed) return [] + + const results = readArrayProperty(parsed, 'results') + if (results.length === 0) return [] + + return results.map((entry) => ({ + name: + readStringProperty(entry, 'name') ?? + readStringProperty(entry, 'check') ?? + readStringProperty(entry, 'title') ?? + readStringProperty(entry, 'category'), + status: readStringProperty(entry, 'status') ?? readStringProperty(entry, 'level'), + message: readStringProperty(entry, 'message') ?? readStringProperty(entry, 'details'), + })) +} + +function parseJsonObject(value: string): Record | null { + try { + const parsed: unknown = JSON.parse(value) + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record + } + return null + } catch { + return null + } +} + +function readArrayProperty(object: Record, property: string): Record[] { + const value = object[property] + if (!Array.isArray(value)) return [] + return value.filter((entry): entry is Record => { + return entry !== null && typeof entry === 'object' && !Array.isArray(entry) + }) +} + +function readStringProperty(object: Record, property: string): string | null { + const value = object[property] + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null +} + +function firstUsefulLine(output: string): string | null { + const line = output + .split(/\r?\n/) + .map((value) => value.trim()) + .find((value) => value.length > 0) + return line ?? null +}