Consolidate e2e test workflows, add additional coverage

Signed-off-by: lelia <2418071+lelia@users.noreply.github.com>

Author: lelia

.github/workflows/e2e-test.yml

@@ -1,4 +1,4 @@
-name: E2E Test
+name: E2E Tests
 
 on:
   push:
@@ -10,93 +10,58 @@ permissions:
   contents: read
 
 jobs:
-  e2e-scan:
-    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
-      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
-        with:
-          python-version: '3.12'
-
-      - name: Install CLI from local repo
-        run: |
-          python -m pip install --upgrade pip
-          pip install .
-
-      - name: Run Socket CLI scan
-        env:
-          SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }}
-        run: |
-          set -o pipefail
-          socketcli \
-            --target-path tests/e2e/fixtures/simple-npm \
-            --disable-blocking \
-            --enable-debug \
-            2>&1 | tee /tmp/scan-output.log
-
-      - name: Verify scan produced a report
-        run: |
-          if grep -q "Full scan report URL: https://socket.dev/" /tmp/scan-output.log; then
-            echo "PASS: Full scan report URL found"
-            grep "Full scan report URL:" /tmp/scan-output.log
-          elif grep -q "Diff Url: https://socket.dev/" /tmp/scan-output.log; then
-            echo "PASS: Diff URL found"
-            grep "Diff Url:" /tmp/scan-output.log
-          else
-            echo "FAIL: No report URL found in scan output"
-            cat /tmp/scan-output.log
-            exit 1
-          fi
-
-  e2e-sarif:
-    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
-      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
-        with:
-          python-version: '3.12'
-
-      - name: Install CLI from local repo
-        run: |
-          python -m pip install --upgrade pip
-          pip install .
-
-      - name: Run Socket CLI scan with --sarif-file
-        env:
-          SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }}
-        run: |
-          set -o pipefail
-          socketcli \
-            --target-path tests/e2e/fixtures/simple-npm \
-            --sarif-file /tmp/results.sarif \
-            --disable-blocking \
-            2>&1 | tee /tmp/sarif-output.log
-
-      - name: Verify SARIF file is valid
-        run: |
-          python3 -c "
-          import json, sys
-          with open('/tmp/results.sarif') as f:
-              data = json.load(f)
-          assert data['version'] == '2.1.0', f'Invalid version: {data[\"version\"]}'
-          assert '\$schema' in data, 'Missing \$schema'
-          count = len(data['runs'][0]['results'])
-          print(f'PASS: Valid SARIF 2.1.0 with {count} result(s)')
-          "
-
-  e2e-reachability:
+  e2e:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: scan
+            args: >-
+              --target-path tests/e2e/fixtures/simple-npm
+              --disable-blocking
+              --enable-debug
+            validate: tests/e2e/validate-scan.sh
+
+          - name: sarif
+            args: >-
+              --target-path tests/e2e/fixtures/simple-npm
+              --sarif-file /tmp/results.sarif
+              --disable-blocking
+            validate: tests/e2e/validate-sarif.sh
+
+          - name: reachability
+            args: >-
+              --target-path tests/e2e/fixtures/simple-npm
+              --reach
+              --disable-blocking
+              --enable-debug
+            validate: tests/e2e/validate-reachability.sh
+            setup-node: "true"
+
+          - name: gitlab
+            args: >-
+              --target-path tests/e2e/fixtures/simple-npm
+              --enable-gitlab-security
+              --disable-blocking
+            validate: tests/e2e/validate-gitlab.sh
+
+          - name: json
+            args: >-
+              --target-path tests/e2e/fixtures/simple-npm
+              --enable-json
+              --disable-blocking
+            validate: tests/e2e/validate-json.sh
+
+          - name: pypi
+            args: >-
+              --target-path tests/e2e/fixtures/simple-pypi
+              --disable-blocking
+              --enable-debug
+            validate: tests/e2e/validate-scan.sh
+
+    name: e2e-${{ matrix.name }}
     steps:
       - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
         with:
@@ -108,6 +73,7 @@ jobs:
           python-version: '3.12'
 
       - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af
+        if: matrix.setup-node == 'true'
         with:
           node-version: '20'
 
@@ -117,85 +83,17 @@ jobs:
           pip install .
 
       - name: Install uv
+        if: matrix.setup-node == 'true'
         run: pip install uv
 
-      - name: Run Socket CLI with reachability
+      - name: Run Socket CLI
         env:
           SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }}
         run: |
           set -o pipefail
-          socketcli \
-            --target-path tests/e2e/fixtures/simple-npm \
-            --reach \
-            --disable-blocking \
-            --enable-debug \
-            2>&1 | tee /tmp/reach-output.log
-
-      - name: Verify reachability analysis completed
-        run: |
-          if grep -q "Reachability analysis completed successfully" /tmp/reach-output.log; then
-            echo "PASS: Reachability analysis completed"
-            grep "Reachability analysis completed successfully" /tmp/reach-output.log
-            grep "Results written to:" /tmp/reach-output.log || true
-          else
-            echo "FAIL: Reachability analysis did not complete successfully"
-            cat /tmp/reach-output.log
-            exit 1
-          fi
-
-      - name: Verify scan produced a report
-        run: |
-          if grep -q "Full scan report URL: https://socket.dev/" /tmp/reach-output.log; then
-            echo "PASS: Full scan report URL found"
-            grep "Full scan report URL:" /tmp/reach-output.log
-          elif grep -q "Diff Url: https://socket.dev/" /tmp/reach-output.log; then
-            echo "PASS: Diff URL found"
-            grep "Diff Url:" /tmp/reach-output.log
-          else
-            echo "FAIL: No report URL found in scan output"
-            cat /tmp/reach-output.log
-            exit 1
-          fi
+          socketcli ${{ matrix.args }} 2>&1 | tee /tmp/e2e-output.log
 
-      - name: Run scan with --sarif-file (all results)
+      - name: Validate results
         env:
           SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }}
-        run: |
-          socketcli \
-            --target-path tests/e2e/fixtures/simple-npm \
-            --reach \
-            --sarif-file /tmp/sarif-all.sarif \
-            --sarif-scope full \
-            --sarif-reachability all \
-            --disable-blocking \
-            2>/dev/null
-
-      - name: Run scan with --sarif-file --sarif-reachability reachable (filtered results)
-        env:
-          SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }}
-        run: |
-          socketcli \
-            --target-path tests/e2e/fixtures/simple-npm \
-            --reach \
-            --sarif-file /tmp/sarif-reachable.sarif \
-            --sarif-scope full \
-            --sarif-reachability reachable \
-            --disable-blocking \
-            2>/dev/null
-
-      - name: Verify reachable-only results are a subset of all results
-        run: |
-          test -f /tmp/sarif-all.sarif
-          test -f /tmp/sarif-reachable.sarif
-          python3 -c "
-          import json
-          with open('/tmp/sarif-all.sarif') as f:
-              all_data = json.load(f)
-          with open('/tmp/sarif-reachable.sarif') as f:
-              reach_data = json.load(f)
-          all_count = len(all_data['runs'][0]['results'])
-          reach_count = len(reach_data['runs'][0]['results'])
-          print(f'All results: {all_count}, Reachable-only results: {reach_count}')
-          assert reach_count <= all_count, f'FAIL: reachable ({reach_count}) > all ({all_count})'
-          print('PASS: Reachable-only results is a subset of all results')
-          "
+        run: bash ${{ matrix.validate }}

tests/e2e/fixtures/simple-npm/package.json

@@ -4,7 +4,7 @@
   "description": "Test fixture for reachability analysis",
   "main": "index.js",
   "dependencies": {
-    "lodash": "4.17.23",
+    "lodash": "4.18.1",
     "express": "4.22.0",
     "axios": "1.13.5"
   },

tests/e2e/fixtures/simple-pypi/requirements.txt

@@ -0,0 +1,3 @@
+requests==2.31.0
+flask==3.0.0
+pyyaml==6.0.1

tests/e2e/validate-gitlab.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPORT="gl-dependency-scanning-report.json"
+
+if [ ! -f "$REPORT" ]; then
+  echo "FAIL: GitLab report not found at $REPORT"
+  exit 1
+fi
+
+python3 -c "
+import json, re, sys
+
+with open('$REPORT') as f:
+    data = json.load(f)
+
+errors = []
+
+# v15.0.0 required root-level keys
+for key in ('version', 'scan', 'vulnerabilities', 'dependency_files'):
+    if key not in data:
+        errors.append(f'Missing required root key: {key}')
+
+if 'scan' in data:
+    scan = data['scan']
+
+    # Timestamp format: YYYY-MM-DDTHH:MM:SS (no microseconds, no trailing Z)
+    ts_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$')
+    for field in ('start_time', 'end_time'):
+        val = scan.get(field, '')
+        if not ts_pattern.match(val):
+            errors.append(f'scan.{field} \"{val}\" does not match pattern YYYY-MM-DDTHH:MM:SS')
+
+    if scan.get('type') != 'dependency_scanning':
+        errors.append(f'scan.type is \"{scan.get(\"type\")}\" expected \"dependency_scanning\"')
+
+    analyzer_id = scan.get('analyzer', {}).get('id', '')
+    if analyzer_id != 'socket-security':
+        errors.append(f'scan.analyzer.id is \"{analyzer_id}\" expected \"socket-security\"')
+
+    scanner_id = scan.get('scanner', {}).get('id', '')
+    if scanner_id != 'socket-cli':
+        errors.append(f'scan.scanner.id is \"{scanner_id}\" expected \"socket-cli\"')
+
+    if scan.get('status') != 'success':
+        errors.append(f'scan.status is \"{scan.get(\"status\")}\" expected \"success\"')
+
+# dependency_files structure check
+if 'dependency_files' in data:
+    for i, df in enumerate(data['dependency_files']):
+        for req in ('path', 'package_manager', 'dependencies'):
+            if req not in df:
+                errors.append(f'dependency_files[{i}] missing required key: {req}')
+
+if errors:
+    for e in errors:
+        print(f'FAIL: {e}')
+    sys.exit(1)
+
+vuln_count = len(data.get('vulnerabilities', []))
+dep_file_count = len(data.get('dependency_files', []))
+print(f'PASS: Valid GitLab v15.0.0 report with {vuln_count} vulnerability(ies) and {dep_file_count} dependency file(s)')
+"

tests/e2e/validate-json.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+LOG="/tmp/e2e-output.log"
+
+python3 -c "
+import json, sys
+
+# The JSON output is on stdout; the log may also contain stderr debug lines.
+# Find the first line that parses as valid JSON.
+found = False
+with open('$LOG') as f:
+    for line in f:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            data = json.loads(line)
+            if isinstance(data, dict):
+                found = True
+                print(f'PASS: Valid JSON output with {len(data)} top-level key(s)')
+                break
+        except json.JSONDecodeError:
+            continue
+
+if not found:
+    print('FAIL: No valid JSON object found in output')
+    sys.exit(1)
+"