From 0119193f5e62d022a150c87d437f14cca6148d50 Mon Sep 17 00:00:00 2001 From: Martin Date: Thu, 17 Sep 2026 19:12:50 +0800 Subject: [PATCH] Require a letter to keep a bare skill token A length floor meant to drop stray punctuation also dropped one-letter skills like C and R. Replace it: a token is kept only if it has a letter, so punctuation, "24/7", "100%" and bare numbers like 2015 or 27001 are dropped, whether typed alone or split from a prefix like ISO. Bare part numbers such as 6502 go too, since they look like a year; a named one like MOS 6502 is kept. Without the floor, the marker strip in clean() turned 5G into G. It now strips only a bullet, a run of dashes or asterisks before whitespace, or digits (full-width included) followed by "." or ")" and then whitespace or two letters. So 5G and 9.x Java survive, while 1.Must and ** Must lose their markers. --- tests/browser/document-grounding.test.js | 191 +++++++++++++++++++++++ web/document-grounding.js | 43 ++++- 2 files changed, 232 insertions(+), 2 deletions(-) diff --git a/tests/browser/document-grounding.test.js b/tests/browser/document-grounding.test.js index 85b7ac6f..50876f72 100644 --- a/tests/browser/document-grounding.test.js +++ b/tests/browser/document-grounding.test.js @@ -24,6 +24,197 @@ test("rejects unsupported, spoofed, empty, oversized, and invalid UTF-8 files", } }); +test("single-character skills like C and R survive extraction", async () => { + // A one-letter name like C or R has a letter, so it is kept. A length floor + // meant to drop stray punctuation used to drop these too. + const resume = await parseGroundingFile(txt("Skills: C, Go, Python, R, Rust"), "resume"); + assert.deepEqual(resume.skills, ["C", "Go", "Python", "R", "Rust"]); +}); + +test("digit-led skills are not mistaken for a numbered-list marker", async () => { + // clean()'s leading-marker strip is meant for real list prefixes like "1. " + // or "2) ", not for a bare digit run: without the "then punctuation" check, + // "5G" loses its "5" and survives as the fabricated skill "G". + const resume = await parseGroundingFile(txt("Skills: C, 5G, 3D, 4K"), "resume"); + assert.deepEqual(resume.skills, ["C", "5G", "3D", "4K"]); +}); + +test("a numbered-list marker is still stripped from a requirement line", async () => { + const jd = await parseGroundingFile(txt("1. Must know Rust\n2) Should know Go"), "jd"); + assert.deepEqual(jd.requirements, ["Must know Rust", "Should know Go"]); +}); + +test("a numbered marker before two letters needs no space, a dash run does", async () => { + // "1.Must" has no space after the marker but still loses it. A dash run + // keeps its whitespace requirement, so glued "--Must" is left whole. + const jd = await parseGroundingFile(txt("1.Must know Rust\n1.) Should know Go\n-- Must know Python"), "jd"); + assert.deepEqual(jd.requirements, ["Must know Rust", "Should know Go", "Must know Python"]); + const glued = await parseGroundingFile(txt("--Must know Python"), "jd"); + assert.deepEqual(glued.requirements, ["--Must know Python"]); +}); + +test("a run of asterisks is stripped like a run of dashes", async () => { + // Pasted markdown leaves "** " and "*** " in front of a line. Like dashes, + // a run of asterisks is a marker when whitespace follows; glued "**Must" + // is left whole. + const jd = await parseGroundingFile(txt("** Must know Rust\n* Should know Go"), "jd"); + assert.deepEqual(jd.requirements, ["Must know Rust", "Should know Go"]); + const resume = await parseGroundingFile(txt("*** Led migration"), "resume"); + assert.deepEqual(resume.anchors, ["Led migration"]); + const glued = await parseGroundingFile(txt("**Must know Rust"), "jd"); + assert.deepEqual(glued.requirements, ["**Must know Rust"]); +}); + +test("a version wildcard is not read as a numbered marker", async () => { + // "." follows a digit inside real tokens, so digits plus "." is not enough. + // One letter after it ("9.x") leaves the token whole; two ("1.Go") read as + // a marker. + const jd = await parseGroundingFile(txt("9.x Java experience required\n18.x Node experience"), "jd"); + assert.deepEqual(jd.requirements, ["9.x Java experience required", "18.x Node experience"]); + const resume = await parseGroundingFile(txt("Skills: 1.Go, 2.Rust"), "resume"); + assert.deepEqual(resume.skills, ["Go", "Rust"]); +}); + +test("a full-width numbered marker is stripped from a requirement line", async () => { + // U+FF11 and U+FF12 are Nd but not \d, so a \d-based strip would leave + // these markers in place. The escapes keep this file ASCII. + const jd = await parseGroundingFile( + txt("\uff11. Must know Rust\n\uff12) Should know Go\n\uff11.Must know C"), + "jd", + ); + assert.deepEqual(jd.requirements, ["Must know Rust", "Should know Go", "Must know C"]); +}); + +test("a full-width numbered marker is stripped from a resume, and a digit-led skill survives", async () => { + const resume = await parseGroundingFile( + txt("Skills: \uff15G, C\n\uff11. Led project Alpha\n\uff12) Built project Beta"), + "resume", + ); + assert.deepEqual(resume.skills, ["\uff15G", "C"]); + assert.deepEqual(resume.anchors, ["Led project Alpha", "Built project Beta"]); +}); + +test("a split fragment that is pure punctuation is dropped, not kept as a skill", async () => { + // A stray delimiter that lands as its own fragment has no letter, so it is + // dropped. + const resume = await parseGroundingFile(txt("Skills: C, /, Go, #, &, Java"), "resume"); + assert.deepEqual(resume.skills, ["C", "Go", "Java"]); +}); + +test("lone numeric fragments are not kept as skills", async () => { + const resume = await parseGroundingFile(txt("Skills: Python, 1, 1., Rust"), "resume"); + assert.deepEqual(resume.skills, ["Python", "Rust"]); +}); + +test("a single isolated skill with no delimiter still survives extraction", async () => { + const resume = await parseGroundingFile(txt("Skills: Python"), "resume"); + assert.deepEqual(resume.skills, ["Python"]); +}); + +test("a skills header missing its colon is not treated as a skills line", async () => { + // parseResume only recognizes "skill(s)/technologies/stack" followed by ":", + // so a header that drops the colon must yield no skills at all rather than + // matching loosely on the leading word. + const resume = await parseGroundingFile(txt("Skills Python, Go"), "resume"); + assert.deepEqual(resume.skills, []); +}); + +test("header casing, synonyms, and stray whitespace around the colon are tolerated", async () => { + const resume = await parseGroundingFile(txt("TECHNOLOGIES : Python, Go"), "resume"); + assert.deepEqual(resume.skills, ["Python", "Go"]); +}); + +test("empty segments from doubled-up delimiters are dropped, not kept as blank skills", async () => { + const resume = await parseGroundingFile(txt("Skills: Python,, Go;;Rust||C++"), "resume"); + assert.deepEqual(resume.skills, ["Python", "Go", "Rust", "C++"]); +}); + +test("a slash or percent does not keep a letterless token", async () => { + // "24/7" and "100%" are the same digit-plus-symbol shape as "-50" or + // "1-2" -- there is no principled reason to exempt these two symbols and + // not others, so only a letter keeps a token. + const resume = await parseGroundingFile(txt("Skills: C++11, 24/7, 100%, v2, 5, -5"), "resume"); + assert.deepEqual(resume.skills, ["C++11", "v2"]); +}); + +test("a bare digit.digit shape is dropped as an orphaned version or GPA fragment", async () => { + // "3.14", "5.2", and "802.11" can't be told apart from a GPA or a version + // number split off its software name -- there is no letter left to say + // which one it is, so the whole shape is dropped, standard or not. + const resume = await parseGroundingFile(txt("Skills: Python, 3.14, 5.2, 802.11, Go"), "resume"); + assert.deepEqual(resume.skills, ["Python", "Go"]); +}); + +test("a bare multi-digit integer gets no special treatment either", async () => { + // With no dot, "27001" and "2015" cannot be read as a version or a GPA + // either. Whether split off a prefix by parseResume's "," / ";" / "|" + // split or typed alone, they have no letter, so they are dropped like + // "3.14" is and only "ISO 9001" is kept. + const resume = await parseGroundingFile(txt("Skills: ISO 9001, 27001, 2015"), "resume"); + assert.deepEqual(resume.skills, ["ISO 9001"]); +}); + +test("a bare part number is an accepted loss, a named one is kept", async () => { + // A systems resume lists these, but "8051" and "2025" are the same shape + // and the filter cannot tell a part from a year, so it drops both bare + // forms. Naming the part is what keeps it, which is the trade the letter + // rule makes on purpose. + const resume = await parseGroundingFile( + txt("Skills: 6502, 8051, 68000, 2025, MOS 6502, Intel 8051, Z80"), + "resume", + ); + assert.deepEqual(resume.skills, ["MOS 6502", "Intel 8051", "Z80"]); +}); + +test("a generation suffix or org prefix carries a standard's number through", async () => { + // A letter suffix ("ac", "ax") or a name prefix ("IEEE", "Wi-Fi") gives the + // number a letter, so the token is kept. + const resume = await parseGroundingFile(txt("Skills: 802.11ac, 802.11ax, IEEE 802.11, Wi-Fi 802.11"), "resume"); + assert.deepEqual(resume.skills, ["802.11ac", "802.11ax", "IEEE 802.11", "Wi-Fi 802.11"]); +}); + +test("a standard survives named but not split off as a bare number", async () => { + // "IEEE 754" and "ISO 27001" keep their org name, so the letter carries + // them through same as any other skill. Once "754" is split off from + // "IEEE" it is just a bare digit run with no letter left to scope it, and + // is dropped the same way "802.3" is -- both are real standards, but + // neither token carries anything to say so on its own. + const resume = await parseGroundingFile(txt("Skills: IEEE 754, ISO 27001, IEEE, 754, 802.3"), "resume"); + assert.deepEqual(resume.skills, ["IEEE 754", "ISO 27001", "IEEE"]); +}); + +test("a non-ASCII decimal digit dotted fragment is dropped like its ASCII equivalent", async () => { + // A full-width or Arabic-Indic digit is not a letter, so a GPA or version + // fragment spelled with them is dropped like "3.14" is. + const resume = await parseGroundingFile(txt("Skills: Python, ３.１４, ٣.١٤, Go"), "resume"); + assert.deepEqual(resume.skills, ["Python", "Go"]); +}); + +test("digits and symbols elsewhere in a token do not keep it", async () => { + // None of these carry a letter, so none of them get to survive as a + // negative number, a parenthesized GPA, a digit range, or a year range -- + // the same rule that drops a bare "27001" drops these too. + const resume = await parseGroundingFile(txt("Skills: Python, -50, (3.14), 1-2, 2020-2024, Rust"), "resume"); + assert.deepEqual(resume.skills, ["Python", "Rust"]); +}); + +test("a shared prefix does not carry over to the values after it", async () => { + // ISO does not scope "124141" or "2015" just because it appeared earlier + // on the line -- parseResume splits Skills: ISO 27001, 124141, ISO 8981, + // 2015 into four independent candidates, and each one is judged only on + // what it itself carries. "ISO 27001" and "ISO 8981" keep their own "ISO", + // but "124141" and "2015" reached this filter with no letter of their own + // and are dropped, even though a human reader might guess they belong to + // the same certification family. + const resume = await parseGroundingFile(txt("Skills: ISO 27001, 124141, ISO 8981, 2015"), "resume"); + assert.deepEqual(resume.skills, ["ISO 27001", "ISO 8981"]); +}); + +test("stacked list markers on one line are stripped in full, not just the first", async () => { + const jd = await parseGroundingFile(txt("1. - Must know Rust\n* 2) Should know Go"), "jd"); + assert.deepEqual(jd.requirements, ["Must know Rust", "Should know Go"]); +}); + test("selection requires consent and storage is one-time", () => { const extracted = { requirements: ["Must know Rust"], skills: ["Rust"], anchors: ["Built a parser"] }; const selected = { requirements: [0], skills: [], anchors: [0] }; diff --git a/web/document-grounding.js b/web/document-grounding.js index 372ea2a9..0f089359 100644 --- a/web/document-grounding.js +++ b/web/document-grounding.js @@ -115,12 +115,51 @@ function normalizeLines(text) { .split(/\r?\n/).map((line) => line.replace(/\s+/g, " ").trim()).filter(Boolean); } +// A bullet glyph is stripped on its own, because it is a list glyph and not +// part of any name, so it needs no space after it to say so. A run of dashes +// or asterisks stripped that freely would eat into real content, so those +// still only count as a marker once whitespace after them confirms it. A +// run, not one character: "-- " and "** " are what pasted markdown leaves. +// +// A numbered marker is stripped without a following space, because "1.Must" +// and "1.) Must" are common paste shapes and the "1." should not reach the +// interviewer. Its digits must be followed by "." or ")", which keeps "5G" +// and "3D" whole, and then by whitespace or two letters. A "." alone is not +// enough, since it follows a digit inside real tokens: "9.x Java" has one +// letter after it and stays whole, while "1.Must" and "1.Go" lose their +// marker. That is a heuristic, not a rule: "1.C experience" keeps its +// marker and "3.js experience" loses its "3.". +// \p{Nd} rather than \d, so a full-width digit is a marker digit like an +// ASCII one. function clean(line) { - return line.replace(/^[-*•\d.)\s]+/, "").slice(0, textLimit).trim(); + return line.trimStart() + .replace(/^(?:(?:-+|\*+)\s+|\p{Nd}+[.)]+(?=\s|\p{L}{2})\s*|•\s*)+/u, "") + .slice(0, textLimit).trim(); } +// A token with no letter is digits and symbols only, wearing a list item's +// clothes, and that is not a skill on its own. +// +// A letter is what makes a token legible as a named thing: "ISO 27001" and +// "IEEE 754" keep the org name that scopes their number, "5G" and "3D" carry +// their own label, and a plain "5" or "27001" or "2015" carries no such +// scope, whether it arrived alone -- "Skills: 2025" -- or split off a shared +// prefix by parseResume's "," / ";" / "|" split -- "Skills: ISO 27001, +// 124141, 2015". Either way, there is nothing left to tell whether it is +// still part of a standard, a separate one, or an unrelated year. Rather +// than guess, every letterless token is dropped, with no exception: "24/7" +// and "100%" are the same digit-plus-symbol shape as "-50", "1-2", and +// "2020-2024", and none of them carry a letter to claim a meaning others +// would have to guess at. +// +// That costs bare part numbers -- 6502, 8051, 68000 -- which are real skills +// on a systems resume and have no letter either. The loss is accepted, not +// overlooked: "8051" and "2025" are the same shape, four bare digits, and +// nothing in the token says which is a part number and which is a year, so +// a filter that goes by shape and keeps one keeps the other. Both are +// dropped. A part number that names itself, "MOS 6502" or "Z80", is kept. function unique(values, max) { - return [...new Set(values.map(clean).filter((value) => value.length >= 2))].slice(0, max); + return [...new Set(values.map(clean).filter((value) => /\p{L}/u.test(value)))].slice(0, max); } function parseJd(lines) {