Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
name: test
on:
push:
branches: [main]
pull_request:
permissions:
contents: read
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
# 1.27.4 or newer: older 1.27 patches carry net/http, crypto/x509 and http2 fixes the
# council's hub and model clients need (govulncheck, 2026-11-11).
go-version: '1.27'
check-latest: true
cache: true
- name: Check the Go toolchain
run: |
v="$(go env GOVERSION)"
echo "building with $v"
case "$v" in
go1.27|go1.27.[0-3]|go1.27rc*) echo "::error::$v is older than go1.27.4"; exit 1 ;;
esac
# The step fails on gofmt's own exit status too (a file it cannot parse). It is the only step
# here that parses pkg/council/coder/proc_other_test.go (//go:build !unix): vet and test run
# on linux, and the cross-build compiles no test files.
- name: Format
run: |
out="$(gofmt -l main.go cmd pkg agentkit test)"
if [ -n "$out" ]; then echo "$out"; echo "::error::run gofmt -w on the files above"; exit 1; fi
- name: Vet
run: go vet ./...
# One target per OS goreleaser builds (.goreleaser.yaml), plus 32-bit linux/arm. No file is
# specific to an architecture, so darwin/amd64 and linux/arm64 would add nothing. A file only
# these compile breaks here and not at tag time. Build, not vet: cmd/review_fixes_test.go calls syscall.Kill untagged, so a
# windows vet cannot compile the tests.
- name: Cross-build release targets
run: |
for t in windows/amd64 windows/arm64 darwin/arm64 linux/arm; do
echo "building $t"
CGO_ENABLED=0 GOOS="${t%/*}" GOARCH="${t#*/}" go build -o /dev/null ./...
done
- name: Test
run: go test -race -count=1 ./...
# pkg/templates is its own module (go.mod replace): ./... above does not reach it.
- name: Templates module
working-directory: pkg/templates
run: go vet ./... && go test -race -count=1 ./...
eval:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
# 1.27.4 or newer: the same toolchain the test job checks, because the eval runs the
# council the release builds.
go-version: '1.27'
check-latest: true
cache: true
- name: Check the Go toolchain
run: |
v="$(go env GOVERSION)"
echo "building with $v"
case "$v" in
go1.27|go1.27.[0-3]|go1.27rc*) echo "::error::$v is older than go1.27.4"; exit 1 ;;
esac
- name: Council eval harness
run: go test -count=1 -run 'TestEval' -v ./pkg/council/
2 changes: 1 addition & 1 deletion cmd/domain.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import (
"fmt"
"github.com/ktr0731/go-fuzzyfinder"
"github.com/spf13/cobra"
"slices"
"github.com/teknoir/cli/pkg/config"
"slices"
)

var domainCmd = &cobra.Command{
Expand Down
98 changes: 98 additions & 0 deletions pkg/council/eval_platform_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package council

import (
"bytes"
"context"
"encoding/json"
"sync"

"github.com/modelcontextprotocol/go-sdk/mcp"
)

// evalPlatform is the fixture platform of the council's eval: an in-process MCP server whose tools
// answer fixed bodies, registered under the names the council's tools.Set sees in production, so a
// scenario runs a whole turn through the real Set (the cut, the header, the untrusted block and the
// level refusal) without a cluster, a network or the model proxy.
type evalPlatform struct {
server *mcp.Server
mu sync.Mutex
calls map[string]int

// The bodies the tools answer. A scenario sets the ones it needs; an unset tool answers "".
devices string // list_devices
sample string // mqtt_get_sample
preview string // mqtt_sample
logs string // inspect_logs
pod string // inspect_pod
}

// newEvalPlatform registers the fixture tools on a server that has not run yet: the harness
// connects the real tools.Set to it.
func newEvalPlatform() *evalPlatform {
p := &evalPlatform{
server: mcp.NewServer(&mcp.Implementation{Name: "tnctl", Version: "eval"}, nil),
calls: map[string]int{},
}
object := map[string]any{"type": "object", "properties": map[string]any{}}
pod := map[string]any{
"type": "object",
"properties": map[string]any{
"name": map[string]any{"type": "string"},
"tail": map[string]any{"type": "integer"},
},
"required": []any{"name"},
"additionalProperties": false,
}
p.add("whoami", "Who this tnctl works as.", object, func() string {
return harnessFacts
})
p.add("list_devices", "The devices of the teamspace.", object, func() string {
return p.devices
})
p.add("mqtt_get_sample", "A sample of the teamspace's MQTT messages.", object, func() string {
return p.sample
})
p.add("mqtt_sample", "A short preview of the teamspace's MQTT messages.", object, func() string {
return p.preview
})
p.add("inspect_pod", "The newest events of a pod on a device.", pod, func() string {
return p.pod
})
p.add("inspect_logs", "The log of a pod on a device.", pod, func() string {
return p.logs
})
return p
}

// add registers one fixture tool: it counts its calls and answers with what answer returns, indented
// as the council's server indents every result (evalIndent).
func (p *evalPlatform) add(name, desc string, schema any, answer func() string) {
p.server.AddTool(&mcp.Tool{Name: name, Description: desc, InputSchema: schema},
func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) {
p.mu.Lock()
p.calls[name]++
p.mu.Unlock()
return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: evalIndent(answer())}}}, nil
})
}

// count is how many times the tool's handler ran.
func (p *evalPlatform) count(name string) int {
p.mu.Lock()
defer p.mu.Unlock()
return p.calls[name]
}

// evalIndent is text as cmd/mcp.go's mcpIndent gives it to the council: JSON indented two spaces,
// anything that does not parse as it is, and an empty answer as the empty object. json.Indent keeps
// string literals byte for byte, so a payload's escapes reach the model as the encoder wrote them.
func evalIndent(text string) string {
if len(bytes.TrimSpace([]byte(text))) == 0 {
return "{}"
}
var b bytes.Buffer
if json.Indent(&b, []byte(text), "", " ") != nil {
return text
}
return b.String()
}
Loading
Loading