From 4a63df4aba1a834bfbd997c55d1f061897915c5c Mon Sep 17 00:00:00 2001
From: David Fridrich <fridrich.david19@gmail.com>
Date: Fri, 3 Apr 2026 19:39:47 +0200
Subject: [PATCH] e2e: add testing framework for templates

---
 .github/workflows/e2e-tests.yaml              |  92 ++++++++++++++
 .testing/README.md                            | 102 +++++++++++++++
 .testing/python/mcp-ollama-rag/test.sh        | 116 ++++++++++++++++++
 .../python/mcp-ollama-rag/test_mcp_client.py  | 103 ++++++++++++++++
 .testing/run-e2e.sh                           | 116 ++++++++++++++++++
 Makefile                                      |   5 +-
 README.md                                     |  11 ++
 python/mcp-ollama-rag/client/client.py        |   4 +-
 8 files changed, 546 insertions(+), 3 deletions(-)
 create mode 100644 .github/workflows/e2e-tests.yaml
 create mode 100644 .testing/README.md
 create mode 100755 .testing/python/mcp-ollama-rag/test.sh
 create mode 100644 .testing/python/mcp-ollama-rag/test_mcp_client.py
 create mode 100755 .testing/run-e2e.sh

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
new file mode 100644
index 0000000..b146523
--- /dev/null
+++ b/.github/workflows/e2e-tests.yaml
@@ -0,0 +1,92 @@
+name: E2E Tests
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  workflow_dispatch:
+
+jobs:
+  discover:
+    runs-on: ubuntu-latest
+    outputs:
+      tests: ${{ steps.find-tests.outputs.tests }}
+      has_tests: ${{ steps.find-tests.outputs.has_tests }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Find e2e tests to run
+      id: find-tests
+      run: |
+        # Get changed files vs base branch
+        if [ "${{ github.event_name }}" = "pull_request" ]; then
+          git fetch origin ${{ github.base_ref }} --depth=1
+          changed=$(git diff --name-only FETCH_HEAD...HEAD)
+        else
+          # workflow_dispatch: run all
+          changed="RUN_ALL"
+        fi
+
+        # Collect all available e2e tests
+        all_tests=$(find .testing -mindepth 3 -maxdepth 3 -name "test.sh" | while read -r f; do
+          rel="${f#.testing/}"
+          dirname "$rel"
+        done | sort)
+
+        if [ "$changed" = "RUN_ALL" ]; then
+          matched="$all_tests"
+        else
+          matched=""
+          for test in $all_tests; do
+            # Check if any changed file is under the template dir or the test dir
+            if echo "$changed" | grep -qE "^${test}/|^\.testing/${test}/"; then
+              matched="$matched $test"
+            fi
+          done
+        fi
+
+        # Build JSON array
+        if [ -n "$matched" ]; then
+          json=$(echo "$matched" | xargs -n1 | jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "tests=$json" >> $GITHUB_OUTPUT
+          echo "has_tests=true" >> $GITHUB_OUTPUT
+          echo "Will run e2e tests: $matched"
+        else
+          echo "tests=[]" >> $GITHUB_OUTPUT
+          echo "has_tests=false" >> $GITHUB_OUTPUT
+          echo "No e2e tests to run"
+        fi
+
+  e2e:
+    needs: discover
+    if: needs.discover.outputs.has_tests == 'true'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ${{ fromJSON(needs.discover.outputs.tests) }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Install func
+      uses: functions-dev/action@main
+      with:
+        version: 'latest'
+        name: func
+
+    - name: Setup Ollama
+      run: |
+        curl -fsSL https://ollama.com/install.sh | sh
+        ollama serve &
+        # Wait for server
+        for i in $(seq 1 30); do
+          curl -sf http://localhost:11434/api/tags && break
+          sleep 2
+        done
+
+    - name: Run e2e test
+      run: |
+        ./.testing/run-e2e.sh ${{ matrix.test }} --verbose
diff --git a/.testing/README.md b/.testing/README.md
new file mode 100644
index 0000000..9951f47
--- /dev/null
+++ b/.testing/README.md
@@ -0,0 +1,102 @@
+# E2E Testing Framework
+
+End-to-end tests for templates that need external services (Ollama, databases, etc.)
+and can't be tested with a simple `func invoke`.
+
+## Usage
+
+```bash
+# Run all e2e tests
+make e2e
+
+# Run a specific test
+make e2e ARGS="python/mcp-ollama-rag"
+
+# Verbose output
+make e2e ARGS="--verbose"
+
+# Or run directly
+./.testing/run-e2e.sh python/mcp-ollama-rag --verbose
+```
+
+## Structure
+
+```
+.testing/
+  run-e2e.sh                              # runner script
+  README.md
+  <language>/
+    <template>/                           # must match a template in the repo
+      test.sh                             # entry point (required)
+      test_mcp_client.py                  # helper scripts (optional)
+```
+
+## Adding a new e2e test
+
+1. Create `.testing/<language>/<template>/test.sh`
+2. The directory name **must** match an existing template (e.g. `python/mcp-ollama-rag`)
+3. The runner discovers it automatically
+
+## Writing test.sh
+
+Each `test.sh` is self-contained. It receives these environment variables
+from `run-e2e.sh`:
+
+| Variable | Description |
+|---|---|
+| `REPO_ROOT` | Absolute path to the repo root |
+| `LANGUAGE` | Template language (e.g. `python`) |
+| `TEMPLATE` | Template name (e.g. `mcp-ollama-rag`) |
+| `FUNC_BIN` | Path to `func` binary |
+| `TEMPLATE_REPO` | Git URI or `file://` path to the templates repo |
+| `VERBOSE` | `true` or `false` |
+
+### Conventions
+
+- Use `set -euo pipefail` — fail fast on errors.
+- Use a temp directory for `func create` and clean up via `trap cleanup EXIT`.
+- Redirect stdout with `>"$OUT"` where `OUT` is `/dev/stdout` (verbose) or `/dev/null` (quiet). stderr always prints so errors are always visible.
+- Print step progress: `echo "[1/N] Step name..."` and `echo "  OK"`.
+- Exit 0 on success, non-zero on failure.
+- Print `=== PASS ===` at the end on success.
+
+### Example
+
+```bash
+#!/bin/bash
+set -euo pipefail
+
+WORKDIR=$(mktemp -d)
+if [[ "$VERBOSE" == "true" ]]; then OUT=/dev/stdout; else OUT=/dev/null; fi
+trap "rm -rf $WORKDIR" EXIT
+
+echo "[1/3] Creating function..."
+cd "$WORKDIR"
+$FUNC_BIN create test -r "$TEMPLATE_REPO" -l "$LANGUAGE" -t "$TEMPLATE" >"$OUT"
+cd test
+echo "  OK"
+
+echo "[2/3] Starting server..."
+$FUNC_BIN run --builder=host >"$OUT" &
+RUN_PID=$!
+trap "kill $RUN_PID 2>/dev/null; rm -rf $WORKDIR" EXIT
+sleep 10
+echo "  OK"
+
+echo "[3/3] Testing..."
+curl -sf http://localhost:8080/ >"$OUT"
+echo "  OK"
+
+echo "=== PASS ==="
+```
+
+## Future improvements
+
+- Extract common helpers (step counter, verbose redirection, cleanup) into a
+  `common.sh` once there are 2-3 tests sharing the same patterns.
+
+## CI
+
+The `e2e-tests.yaml` workflow runs automatically on PRs when template files or
+test files change. Each test runs in a separate parallel job. Manual trigger
+via `workflow_dispatch` runs all tests.
diff --git a/.testing/python/mcp-ollama-rag/test.sh b/.testing/python/mcp-ollama-rag/test.sh
new file mode 100755
index 0000000..85fe57c
--- /dev/null
+++ b/.testing/python/mcp-ollama-rag/test.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+#
+# E2E test for python/mcp-ollama-rag
+#
+# Expects these vars from run-e2e.sh:
+#   REPO_ROOT, LANGUAGE, TEMPLATE, FUNC_BIN, TEMPLATE_REPO, VERBOSE
+#
+# Prerequisites:
+#   - ollama installed and running
+#   - func CLI installed
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+WORKDIR=$(mktemp -d)
+LISTEN_ADDRESS="${LISTEN_ADDRESS:-127.0.0.1:8080}"
+RUN_PID=""
+
+# stdout: verbose shows it, quiet sends to /dev/null
+# stderr: always shown (errors always visible)
+if [[ "$VERBOSE" == "true" ]]; then
+    OUT=/dev/stdout
+else
+    OUT=/dev/null
+fi
+
+cleanup() {
+    if [[ -n "$RUN_PID" ]] && ps -p "$RUN_PID" > /dev/null 2>&1; then
+        kill "$RUN_PID" 2>/dev/null || true
+        wait "$RUN_PID" 2>/dev/null || true
+    fi
+    lsof -ti ":${LISTEN_ADDRESS##*:}" 2>/dev/null | xargs kill 2>/dev/null || true
+    rm -rf "$WORKDIR"
+}
+trap cleanup EXIT
+
+# ─── 1. Preflight ────────────────────────────────────────────
+echo "[1/5] Preflight checks..."
+
+if ! command -v ollama &>/dev/null; then
+    echo "FAIL: ollama is not installed"
+    exit 1
+fi
+
+if ! curl -sf http://localhost:11434/api/tags &>/dev/null; then
+    echo "FAIL: ollama server not reachable at localhost:11434"
+    echo "      Start it with: ollama serve"
+    exit 1
+fi
+
+for model in "mxbai-embed-large" "llama3.2:3b"; do
+    if ! ollama list 2>/dev/null | grep -q "$model"; then
+        echo "  Pulling model: $model ..."
+        ollama pull "$model" >"$OUT"
+    else
+        echo "  Model ready: $model"
+    fi
+done
+
+echo "  OK"
+
+# ─── 2. Create function from template ────────────────────────
+echo "[2/5] Creating function from template..."
+
+cd "$WORKDIR"
+$FUNC_BIN create e2e-test -r "$TEMPLATE_REPO" -l "$LANGUAGE" -t "$TEMPLATE" >"$OUT"
+cd e2e-test
+
+echo "  OK"
+
+# ─── 3. Install test client deps ─────────────────────────────
+echo "[3/5] Installing test client dependencies..."
+
+python -m venv "$WORKDIR/.venv" >"$OUT"
+source "$WORKDIR/.venv/bin/activate"
+pip install mcp httpx >"$OUT"
+
+echo "  OK"
+
+# ─── 4. Start function server ────────────────────────────────
+echo "[4/5] Starting function server..."
+
+if lsof -ti ":${LISTEN_ADDRESS##*:}" &>/dev/null; then
+    echo "FAIL: Port ${LISTEN_ADDRESS##*:} is already in use"
+    exit 1
+fi
+
+LISTEN_ADDRESS="$LISTEN_ADDRESS" $FUNC_BIN run --builder=host >"$OUT" &
+RUN_PID=$!
+
+for i in $(seq 1 30); do
+    if curl -sf "http://$LISTEN_ADDRESS/" &>/dev/null; then
+        break
+    fi
+    if ! ps -p "$RUN_PID" > /dev/null 2>&1; then
+        echo "FAIL: func run process died"
+        exit 1
+    fi
+    sleep 2
+done
+
+if ! curl -sf "http://$LISTEN_ADDRESS/" &>/dev/null; then
+    echo "FAIL: Server did not start within 60s"
+    exit 1
+fi
+
+echo "  OK"
+
+# ─── 5. Run MCP client tests ─────────────────────────────────
+echo "[5/5] Running MCP client tests..."
+
+python "$SCRIPT_DIR/test_mcp_client.py" "http://$LISTEN_ADDRESS/mcp" >"$OUT"
+
+echo "  OK"
+echo ""
+echo "=== PASS ==="
diff --git a/.testing/python/mcp-ollama-rag/test_mcp_client.py b/.testing/python/mcp-ollama-rag/test_mcp_client.py
new file mode 100644
index 0000000..cf1fafc
--- /dev/null
+++ b/.testing/python/mcp-ollama-rag/test_mcp_client.py
@@ -0,0 +1,103 @@
+# pyright: reportUnknownMemberType=false
+"""
+E2E test for the mcp-ollama-rag template.
+
+Connects to the running MCP server, exercises all tools, and validates responses.
+
+Usage: python test_mcp_client.py <mcp-url>
+       python test_mcp_client.py http://localhost:8080/mcp
+"""
+import asyncio
+import sys
+
+from mcp import ClientSession
+from mcp.client.streamable_http import streamable_http_client
+from mcp.types import CallToolResult, TextContent
+
+
+def get_text(result: CallToolResult) -> str:
+    """Extract text from the first content item of a tool result."""
+    item = result.content[0]
+    assert isinstance(item, TextContent), f"Expected TextContent, got {type(item)}"
+    return item.text
+
+
+async def run_tests(url: str):
+    errors = []
+
+    async with streamable_http_client(url) as streams:
+        read_stream, write_stream = streams[0], streams[1]
+
+        async with ClientSession(read_stream, write_stream) as sess:
+            _ = await sess.initialize()
+
+            # ── Test 1: list_models ──────────────────────────
+            print("  [test] list_models...", end="", flush=True)
+            result = await sess.call_tool(name="list_models", arguments={})
+            if result.isError:
+                errors.append(f"list_models returned error: {result.content}")
+                print(" FAIL")
+            else:
+                print(result)
+                print(" OK")
+
+            # ── Test 2: embed_document ───────────────────────
+            print("  [test] embed_document...", end="", flush=True)
+            result = await sess.call_tool(
+                name="embed_document",
+                arguments={
+                    "data": [
+                        "https://raw.githubusercontent.com/knative/func/main/docs/function-templates/python.md",
+                    ],
+                },
+            )
+            if result.isError:
+                errors.append(f"embed_document returned error: {result.content}")
+                print(" FAIL")
+            else:
+                text = get_text(result)
+                if "ok - Embedded" not in text:
+                    errors.append(f"embed_document unexpected response: {text}")
+                    print(f" FAIL ({text})")
+                else:
+                    print(f" OK ({text})")
+
+            # ── Test 3: call_model (RAG query) ───────────────
+            print("  [test] call_model...", end="", flush=True)
+            result = await sess.call_tool(
+                name="call_model",
+                arguments={"prompt": "What is a Knative Function?"},
+            )
+            if result.isError:
+                errors.append(f"call_model returned error: {result.content}")
+                print(" FAIL")
+            else:
+                text = get_text(result)
+                if len(text) < 50:
+                    errors.append(f"call_model response too short: {text}")
+                    print(f" FAIL (response too short: {len(text)} chars)")
+                else:
+                    print(f" OK ({len(text)} chars)")
+
+    return errors
+
+
+def main():
+    if len(sys.argv) != 2:
+        print(f"Usage: {sys.argv[0]} <mcp-url>")
+        sys.exit(1)
+
+    url = sys.argv[1]
+    print(f"  Connecting to {url}")
+
+    errors = asyncio.run(run_tests(url))
+
+    if errors:
+        print(f"\n  FAILED ({len(errors)} error(s)):")
+        for e in errors:
+            print(f"    - {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/.testing/run-e2e.sh b/.testing/run-e2e.sh
new file mode 100755
index 0000000..ec69a19
--- /dev/null
+++ b/.testing/run-e2e.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+#
+# run-e2e.sh — Run e2e tests for templates
+#
+# Usage:
+#   ./.testing/run-e2e.sh                              # run all e2e tests
+#   ./.testing/run-e2e.sh python/mcp-ollama-rag        # run one test
+#   ./.testing/run-e2e.sh --verbose                    # run all, verbose
+#   ./.testing/run-e2e.sh python/mcp-ollama-rag --verbose
+#
+# Each test is a self-contained test.sh in .testing/<language>/<template>/.
+#
+# Options:
+#   --verbose  Show all stdout from test commands (stderr always shown)
+#
+# Environment variables:
+#   FUNC_BIN       - Path to func binary (default: func)
+#   TEMPLATE_REPO  - Git URI or file:// path to templates repo (default: file://<repo-root>)
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+VERBOSE=false
+
+# Parse args
+TARGETS=()
+for arg in "$@"; do
+    case "$arg" in
+        --verbose) VERBOSE=true ;;
+        -*) echo "Unknown flag: $arg"; exit 1 ;;
+        *) TARGETS+=("${arg%/}") ;;
+    esac
+done
+
+export REPO_ROOT
+export VERBOSE
+export FUNC_BIN="${FUNC_BIN:-func}"
+export TEMPLATE_REPO="${TEMPLATE_REPO:-file://$REPO_ROOT}"
+
+# Discover all available tests
+discover_tests() {
+    find "$SCRIPT_DIR" -mindepth 3 -maxdepth 3 -name "test.sh" | sort | while read -r f; do
+        rel="${f#$SCRIPT_DIR/}"
+        dirname "$rel"
+    done
+}
+
+# If no targets, run all
+if [[ ${#TARGETS[@]} -eq 0 ]]; then
+    while IFS= read -r t; do
+        TARGETS+=("$t")
+    done < <(discover_tests)
+fi
+
+if [[ ${#TARGETS[@]} -eq 0 ]]; then
+    echo "No e2e tests found in $SCRIPT_DIR"
+    exit 0
+fi
+
+# Run tests and collect results
+PASSED=()
+FAILED=()
+TOTAL_START=$(date +%s)
+
+for target in "${TARGETS[@]}"; do
+    LANGUAGE="${target%%/*}"
+    TEMPLATE="${target#*/}"
+
+    export LANGUAGE
+    export TEMPLATE
+
+    # Validate template exists in repo
+    if [[ ! -d "$REPO_ROOT/$LANGUAGE/$TEMPLATE" ]]; then
+        echo "ERROR: Template not found: $LANGUAGE/$TEMPLATE"
+        echo "  Expected: $REPO_ROOT/$LANGUAGE/$TEMPLATE"
+        FAILED+=("$target")
+        continue
+    fi
+
+    # Validate test exists
+    TEST_SCRIPT="$SCRIPT_DIR/$LANGUAGE/$TEMPLATE/test.sh"
+    if [[ ! -f "$TEST_SCRIPT" ]]; then
+        echo "ERROR: No test.sh found for $target"
+        FAILED+=("$target")
+        continue
+    fi
+
+    echo "=== $target ==="
+    START=$(date +%s)
+
+    if bash "$TEST_SCRIPT"; then
+        ELAPSED=$(( $(date +%s) - START ))
+        PASSED+=("$target (${ELAPSED}s)")
+    else
+        ELAPSED=$(( $(date +%s) - START ))
+        FAILED+=("$target (${ELAPSED}s)")
+    fi
+    echo ""
+done
+
+# Summary
+TOTAL_ELAPSED=$(( $(date +%s) - TOTAL_START ))
+echo "=== E2E Summary ==="
+for p in "${PASSED[@]}"; do
+    printf "\033[32mPASS\033[0m %s\n" "$p"
+done
+for f in "${FAILED[@]}"; do
+    printf "\033[31mFAIL\033[0m %s\n" "$f"
+done
+echo ""
+echo "${#PASSED[@]} passed, ${#FAILED[@]} failed in ${TOTAL_ELAPSED}s"
+
+if [[ ${#FAILED[@]} -gt 0 ]]; then
+    exit 1
+fi
diff --git a/Makefile b/Makefile
index 52aa746..80bc452 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: test test-go test-node test-python test-quarkus test-rust test-springboot test-typescript
+.PHONY: test test-go test-node test-python test-quarkus test-rust test-springboot test-typescript e2e
 
 LOG := test-results.log
 START_TIME := $(shell date +%s%3N)
@@ -58,3 +58,6 @@ summary:
 	echo "Full log: $(LOG)"; \
 	if [ "$$failed" -gt 0 ]; then echo ""; echo "=== Failed ==="; grep "^FAIL" $(LOG); exit 1; fi
 
+e2e:
+	@./.testing/run-e2e.sh $(ARGS)
+
diff --git a/README.md b/README.md
index b486123..d087916 100644
--- a/README.md
+++ b/README.md
@@ -253,6 +253,17 @@ github.com/function-dev/templates <---[root]
 ├── ...
 ```
 
+## E2E Tests
+
+Templates that require external services (e.g. Ollama) have self-contained e2e
+tests in `.testing/<language>/<template>/`.
+
+```bash
+make e2e                                          # run all e2e tests
+make e2e ARGS="python/mcp-ollama-rag"             # run a specific test
+make e2e ARGS="--verbose"                         # run all with full output
+```
+
 # Contact
 You can contact us on CNCF Slack
 [knative-functions](https://cloud-native.slack.com/archives/C04LKEZUXEE) channel
diff --git a/python/mcp-ollama-rag/client/client.py b/python/mcp-ollama-rag/client/client.py
index f167a30..1cce014 100644
--- a/python/mcp-ollama-rag/client/client.py
+++ b/python/mcp-ollama-rag/client/client.py
@@ -1,11 +1,11 @@
 import asyncio
 from mcp import ClientSession
-from mcp.client.streamable_http import streamablehttp_client
+from mcp.client.streamable_http import streamable_http_client
 
 async def main():
     # check your running Function MCP Server, it will output where its available
     # at during initialization.
-    async with streamablehttp_client("http://localhost:8080/mcp") as streams:
+    async with streamable_http_client("http://localhost:8080/mcp") as streams:
         read_stream,write_stream = streams[0],streams[1]
 
         async with ClientSession(read_stream,write_stream) as sess: