Define thresholds, p0 PR suite

.gitignore

@@ -3,3 +3,6 @@
 **/__pycache__/
 **/*.pyc
 **/*.pyo
+
+# Vally
+vally-results/

.vally.yaml

@@ -1,9 +1,8 @@
 paths:
-  evals: evals
+  evals: evals/
 
 suites:
   pr:
-    description: Run all plugin skill evals for pull requests
-    evals:
-      - evals/linting/eval.yaml
-      - evals/security/eval.yaml
+    description: Run p0 plugin skill evals for pull requests
+    filter:
+      priority: p0

evals/linting/eval.yaml

@@ -1,11 +1,16 @@
 name: linting skills evals
 version: 1
 description: Evaluates the linting plugin skills against representative file fixes
+tags:
+  priority: p0
 
 defaults:
   runs: 1
   timeout: 5m
 
+scoring:
+  threshold: 1.0
+
 stimuli:
   - name: check-spelling-fixes-typos-and-updates-dictionary
     prompt: |

evals/security/eval.yaml

@@ -1,11 +1,16 @@
 name: security skills evals
 version: 1
 description: Evaluates the security plugin skills against representative workflow hardening tasks
+tags:
+  priority: p0
 
 defaults:
   runs: 1
   timeout: 5m
 
+scoring:
+  threshold: 1.0
+
 stimuli:
   - name: pin-github-actions-to-shas
     prompt: |