diff --git a/tests/efficacy/judge.py b/tests/efficacy/judge.py index d708dfe6..70419fa8 100644 --- a/tests/efficacy/judge.py +++ b/tests/efficacy/judge.py @@ -69,10 +69,12 @@ DIMENSIONS = ( "design", "srp", "ocp", "lsp", "isp", "dip", "naming_and_abstraction", "error_design", "readability", "maintainability", "test_quality", + "security", "data_protection", ) # What the report leads with, owner-declared in the design. -PRIMARY = ("design", "readability", "maintainability") +PRIMARY = ("design", "readability", "maintainability", "security", + "data_protection") # Never shown to the judge. A context file names the condition, and so does a # README that thanks one: both are stripped from the bundle, and what was @@ -132,8 +134,11 @@ - readability: function length, nesting, names a reader can follow - maintainability: how quickly a reader finds where a change goes - test_quality: what the tests assert, not how many there are +- security: how sign-in, sessions, secrets and queries are defended +- data_protection: how the customers' personal data is held, logged, + exported and erased -Three of those carry the report and are anchored, so that 3 is a described +Five of those carry the report and are anchored, so that 3 is a described place on the scale rather than wherever ordinary work lands. Score against the description nearest what you read, and use 2 and 4 for the gaps between. @@ -165,6 +170,27 @@ domain depends on nothing layered above it, and every fact about a discount kind sits in one place +security + 1 the password is stored or compared as plain text, the secret key is a + literal in the source, SQL is built from strings, and each route checks + sign-in for itself, so a route can be missed + 3 the password is hashed with a library call and every query is + parameterised, but sign-in is still checked route by route, and the + `next` target is followed without checking where it points + 5 one guard covers every route but sign-in, the password is hashed with + a salted slow hash, the secret key comes from the environment, every + form carries a CSRF token, and `next` is accepted only as a local path + +data_protection + 1 customer fields reach the logs, erasure leaves copies of the customer + behind, and the export's query is not limited to one customer + 3 erasure and export are correct, but every place that stores or shows + a customer's data handles it for itself, so the next place to hold + one is a place to forget + 5 the personal data sits in one place that erasure and export both read, + nothing logs a customer field, and the export is built from the + customer's id alone + A submission that removes what a lower anchor describes scores above it, even where the result is ordinary: these are descriptions of the code, not of how impressive it is. diff --git a/tests/efficacy/report.py b/tests/efficacy/report.py index e4dfca6f..38006172 100644 --- a/tests/efficacy/report.py +++ b/tests/efficacy/report.py @@ -63,7 +63,8 @@ UP, DOWN, NEUTRAL = "up", "down", "neutral" # The primary dimensions the report leads with, owner-declared. -PRIMARY = ("judge_design", "judge_readability", "judge_maintainability") +PRIMARY = ("judge_design", "judge_readability", "judge_maintainability", + "judge_security", "judge_data_protection") # How a margin is measured: in the metric's own units, or as a share of the # baseline arm's mean, so that it scales with the metric. @@ -83,6 +84,8 @@ "judge_design": ("0.3 points", 0.3, ABSOLUTE), "judge_readability": ("0.3 points", 0.3, ABSOLUTE), "judge_maintainability": ("0.3 points", 0.3, ABSOLUTE), + "judge_security": ("0.3 points", 0.3, ABSOLUTE), + "judge_data_protection": ("0.3 points", 0.3, ABSOLUTE), "churn_files": ("15 % relative", 0.15, RELATIVE), "churn_lines": ("15 % relative", 0.15, RELATIVE), } @@ -197,6 +200,10 @@ def pattern_count(trial, verdict): lambda t: judge_score(t, "readability")), ("judge_maintainability", "Maintainability, 1-5", UP, lambda t: judge_score(t, "maintainability")), + ("judge_security", "Security, 1-5", UP, + lambda t: judge_score(t, "security")), + ("judge_data_protection", "Data protection, 1-5", UP, + lambda t: judge_score(t, "data_protection")), ("task_success", "Task success, hidden suite pass rate", UP, task_success), ("install", "Installs in a clean environment", UP, @@ -1210,7 +1217,9 @@ def score(wins, fails): # The metrics the finding table answers from, and the words it uses. ANSWERED = PRIMARY + ("task_success",) SHORT = {"judge_design": "design", "judge_readability": "readability", - "judge_maintainability": "maintainability"} + "judge_maintainability": "maintainability", + "judge_security": "security", + "judge_data_protection": "data protection"} FILES = {"full": "Templates' inline file", "short": "Templates' short inline file", "hybrid": "Templates' hybrid file", "hand": "Hand-written file"} @@ -1365,6 +1374,8 @@ def reading(answers, lengths): READS = (("judge_readability", "more readable"), ("judge_design", "better designed"), ("judge_maintainability", "more maintainable"), + ("judge_security", "more secure"), + ("judge_data_protection", "more careful with personal data"), ("judge_tests", "better tested")) # A context file of at most this many lines is one the why-line calls a few @@ -1448,8 +1459,8 @@ def why_line(trials, results, arm, name, answered, length): if effort: lead += " — for more %s" % spoken(effort) elif computed: - lead = ("the judge saw none of it as better design, readability or " - "maintainability") + lead = ("the judge saw none of it as better design, readability, " + "maintainability, security or data protection") elif not built and effort: lead = "it took more %s than without a file" % spoken(effort) else: @@ -2317,7 +2328,8 @@ def executive_checks(seed): "declared its dependencies so that a clean " "install left the app unable to boot.") and got.endswith("The judge saw none of it as better " - "design, readability or maintainability; " + "design, readability, maintainability, " + "security or data protection; " "every win is something a tool counts " "and it missed more of the patterns the " "domain called for."), got)) @@ -2455,6 +2467,23 @@ def self_test(seed): covered and per_kloc_rows == set(STATIC_PER_KLOC), sorted(per_kloc_rows))) + # A primary the judge asks for and the report does not lead with, or one + # without its margin, its threshold or its words, would be read nowhere + # or crash the finding table; a row added to one side only fails here. + import judge + judged = {"judge_" + name for name in judge.PRIMARY} + metric_keys = {key for key, _, _, _ in METRICS} + worded = {key for key, _ in READS} + lacking = sorted(key for key in PRIMARY + if not (key in MARGINS and key in PRACTICAL + and key in SHORT and key in worded + and key in metric_keys)) + unmatched = sorted(judged.symmetric_difference(PRIMARY)) + checks.append(("every primary the judge asks for is led with, worded " + "and given its margins", + not lacking and not unmatched, + {"lacking": lacking, "unmatched": unmatched})) + # A nested count divides by the lines its own tool saw. planted = {"scores": {"complexity": {"value": {"over_15": 3}, "missing": None,