e2e: add testing framework for templates

.github/workflows/e2e-tests.yaml

@@ -0,0 +1,92 @@
+name: E2E Tests
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  workflow_dispatch:
+
+jobs:
+  discover:
+    runs-on: ubuntu-latest
+    outputs:
+      tests: ${{ steps.find-tests.outputs.tests }}
+      has_tests: ${{ steps.find-tests.outputs.has_tests }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Find e2e tests to run
+      id: find-tests
+      run: |
+        # Get changed files vs base branch
+        if [ "${{ github.event_name }}" = "pull_request" ]; then
+          git fetch origin ${{ github.base_ref }} --depth=1
+          changed=$(git diff --name-only FETCH_HEAD...HEAD)
+        else
+          # workflow_dispatch: run all
+          changed="RUN_ALL"
+        fi
+
+        # Collect all available e2e tests
+        all_tests=$(find .testing -mindepth 3 -maxdepth 3 -name "test.sh" | while read -r f; do
+          rel="${f#.testing/}"
+          dirname "$rel"
+        done | sort)
+
+        if [ "$changed" = "RUN_ALL" ]; then
+          matched="$all_tests"
+        else
+          matched=""
+          for test in $all_tests; do
+            # Check if any changed file is under the template dir or the test dir
+            if echo "$changed" | grep -qE "^${test}/|^\.testing/${test}/"; then
+              matched="$matched $test"
+            fi
+          done
+        fi
+
+        # Build JSON array
+        if [ -n "$matched" ]; then
+          json=$(echo "$matched" | xargs -n1 | jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "tests=$json" >> $GITHUB_OUTPUT
+          echo "has_tests=true" >> $GITHUB_OUTPUT
+          echo "Will run e2e tests: $matched"
+        else
+          echo "tests=[]" >> $GITHUB_OUTPUT
+          echo "has_tests=false" >> $GITHUB_OUTPUT
+          echo "No e2e tests to run"
+        fi
+
+  e2e:
+    needs: discover
+    if: needs.discover.outputs.has_tests == 'true'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ${{ fromJSON(needs.discover.outputs.tests) }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Install func
+      uses: functions-dev/action@main
+      with:
+        version: 'latest'
+        name: func
+
+    - name: Setup Ollama
+      run: |
+        curl -fsSL https://ollama.com/install.sh | sh
+        ollama serve &
+        # Wait for server
+        for i in $(seq 1 30); do
+          curl -sf http://localhost:11434/api/tags && break
+          sleep 2
+        done
+
+    - name: Run e2e test
+      run: |
+        ./.testing/run-e2e.sh ${{ matrix.test }} --verbose

.testing/README.md

@@ -0,0 +1,102 @@
+# E2E Testing Framework
+
+End-to-end tests for templates that need external services (Ollama, databases, etc.)
+and can't be tested with a simple `func invoke`.
+
+## Usage
+
+```bash
+# Run all e2e tests
+make e2e
+
+# Run a specific test
+make e2e ARGS="python/mcp-ollama-rag"
+
+# Verbose output
+make e2e ARGS="--verbose"
+
+# Or run directly
+./.testing/run-e2e.sh python/mcp-ollama-rag --verbose
+```
+
+## Structure
+
+```
+.testing/
+  run-e2e.sh                              # runner script
+  README.md
+  <language>/
+    <template>/                           # must match a template in the repo
+      test.sh                             # entry point (required)
+      test_mcp_client.py                  # helper scripts (optional)
+```
+
+## Adding a new e2e test
+
+1. Create `.testing/<language>/<template>/test.sh`
+2. The directory name **must** match an existing template (e.g. `python/mcp-ollama-rag`)
+3. The runner discovers it automatically
+
+## Writing test.sh
+
+Each `test.sh` is self-contained. It receives these environment variables
+from `run-e2e.sh`:
+
+| Variable | Description |
+|---|---|
+| `REPO_ROOT` | Absolute path to the repo root |
+| `LANGUAGE` | Template language (e.g. `python`) |
+| `TEMPLATE` | Template name (e.g. `mcp-ollama-rag`) |
+| `FUNC_BIN` | Path to `func` binary |
+| `TEMPLATE_REPO` | Git URI or `file://` path to the templates repo |
+| `VERBOSE` | `true` or `false` |
+
+### Conventions
+
+- Use `set -euo pipefail` — fail fast on errors.
+- Use a temp directory for `func create` and clean up via `trap cleanup EXIT`.
+- Redirect stdout with `>"$OUT"` where `OUT` is `/dev/stdout` (verbose) or `/dev/null` (quiet). stderr always prints so errors are always visible.
+- Print step progress: `echo "[1/N] Step name..."` and `echo "  OK"`.
+- Exit 0 on success, non-zero on failure.
+- Print `=== PASS ===` at the end on success.
+
+### Example
+
+```bash
+#!/bin/bash
+set -euo pipefail
+
+WORKDIR=$(mktemp -d)
+if [[ "$VERBOSE" == "true" ]]; then OUT=/dev/stdout; else OUT=/dev/null; fi
+trap "rm -rf $WORKDIR" EXIT
+
+echo "[1/3] Creating function..."
+cd "$WORKDIR"
+$FUNC_BIN create test -r "$TEMPLATE_REPO" -l "$LANGUAGE" -t "$TEMPLATE" >"$OUT"
+cd test
+echo "  OK"
+
+echo "[2/3] Starting server..."
+$FUNC_BIN run --builder=host >"$OUT" &
+RUN_PID=$!
+trap "kill $RUN_PID 2>/dev/null; rm -rf $WORKDIR" EXIT
+sleep 10
+echo "  OK"
+
+echo "[3/3] Testing..."
+curl -sf http://localhost:8080/ >"$OUT"
+echo "  OK"
+
+echo "=== PASS ==="
+```
+
+## Future improvements
+
+- Extract common helpers (step counter, verbose redirection, cleanup) into a
+  `common.sh` once there are 2-3 tests sharing the same patterns.
+
+## CI
+
+The `e2e-tests.yaml` workflow runs automatically on PRs when template files or
+test files change. Each test runs in a separate parallel job. Manual trigger
+via `workflow_dispatch` runs all tests.

.testing/python/mcp-ollama-rag/test.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+#
+# E2E test for python/mcp-ollama-rag
+#
+# Expects these vars from run-e2e.sh:
+#   REPO_ROOT, LANGUAGE, TEMPLATE, FUNC_BIN, TEMPLATE_REPO, VERBOSE
+#
+# Prerequisites:
+#   - ollama installed and running
+#   - func CLI installed
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+WORKDIR=$(mktemp -d)
+LISTEN_ADDRESS="${LISTEN_ADDRESS:-127.0.0.1:8080}"
+RUN_PID=""
+
+# stdout: verbose shows it, quiet sends to /dev/null
+# stderr: always shown (errors always visible)
+if [[ "$VERBOSE" == "true" ]]; then
+    OUT=/dev/stdout
+else
+    OUT=/dev/null
+fi
+
+cleanup() {
+    if [[ -n "$RUN_PID" ]] && ps -p "$RUN_PID" > /dev/null 2>&1; then
+        kill "$RUN_PID" 2>/dev/null || true
+        wait "$RUN_PID" 2>/dev/null || true
+    fi
+    lsof -ti ":${LISTEN_ADDRESS##*:}" 2>/dev/null | xargs kill 2>/dev/null || true
+    rm -rf "$WORKDIR"
+}
+trap cleanup EXIT
+
+# ─── 1. Preflight ────────────────────────────────────────────
+echo "[1/5] Preflight checks..."
+
+if ! command -v ollama &>/dev/null; then
+    echo "FAIL: ollama is not installed"
+    exit 1
+fi
+
+if ! curl -sf http://localhost:11434/api/tags &>/dev/null; then
+    echo "FAIL: ollama server not reachable at localhost:11434"
+    echo "      Start it with: ollama serve"
+    exit 1
+fi
+
+for model in "mxbai-embed-large" "llama3.2:3b"; do
+    if ! ollama list 2>/dev/null | grep -q "$model"; then
+        echo "  Pulling model: $model ..."
+        ollama pull "$model" >"$OUT"
+    else
+        echo "  Model ready: $model"
+    fi
+done
+
+echo "  OK"
+
+# ─── 2. Create function from template ────────────────────────
+echo "[2/5] Creating function from template..."
+
+cd "$WORKDIR"
+$FUNC_BIN create e2e-test -r "$TEMPLATE_REPO" -l "$LANGUAGE" -t "$TEMPLATE" >"$OUT"
+cd e2e-test
+
+echo "  OK"
+
+# ─── 3. Install test client deps ─────────────────────────────
+echo "[3/5] Installing test client dependencies..."
+
+python -m venv "$WORKDIR/.venv" >"$OUT"
+source "$WORKDIR/.venv/bin/activate"
+pip install mcp httpx >"$OUT"
+
+echo "  OK"
+
+# ─── 4. Start function server ────────────────────────────────
+echo "[4/5] Starting function server..."
+
+if lsof -ti ":${LISTEN_ADDRESS##*:}" &>/dev/null; then
+    echo "FAIL: Port ${LISTEN_ADDRESS##*:} is already in use"
+    exit 1
+fi
+
+LISTEN_ADDRESS="$LISTEN_ADDRESS" $FUNC_BIN run --builder=host >"$OUT" &
+RUN_PID=$!
+
+for i in $(seq 1 30); do
+    if curl -sf "http://$LISTEN_ADDRESS/" &>/dev/null; then
+        break
+    fi
+    if ! ps -p "$RUN_PID" > /dev/null 2>&1; then
+        echo "FAIL: func run process died"
+        exit 1
+    fi
+    sleep 2
+done
+
+if ! curl -sf "http://$LISTEN_ADDRESS/" &>/dev/null; then
+    echo "FAIL: Server did not start within 60s"
+    exit 1
+fi
+
+echo "  OK"
+
+# ─── 5. Run MCP client tests ─────────────────────────────────
+echo "[5/5] Running MCP client tests..."
+
+python "$SCRIPT_DIR/test_mcp_client.py" "http://$LISTEN_ADDRESS/mcp" >"$OUT"
+
+echo "  OK"
+echo ""
+echo "=== PASS ==="