diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c8cd5db --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,113 @@ +name: CI + +on: + push: + pull_request: + +permissions: + contents: read + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v5 + + - name: Setup Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Run linter + uses: golangci/golangci-lint-action@v9 + with: + version: v2.4.0 + + test: + name: Test + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v5 + + - name: Setup Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Running Tests + run: | + go mod tidy + make test + + # The matrix is DISCOVERED, so a new component is picked up by existing here rather than by + # someone remembering to add a line below. A component added without CI would never be built or + # tested by anything -- root `./...` stops at a nested go.mod. See components/README.md. + # + # Lint and test share this one job, which is why they live in one workflow: `needs` cannot reach a + # job in another file, so splitting them would mean two copies of the discovery rule -- and a rule + # changed in only one copy would lint a set of components and test a different set, silently. + discover: + name: Discover components + runs-on: ubuntu-latest + outputs: + components: ${{ steps.list.outputs.components }} + steps: + - name: Clone the code + uses: actions/checkout@v5 + + # Keyed on the Makefile, matching the root Makefile's own discovery. + - id: list + run: | + names=$(ls -d components/*/Makefile | xargs -n1 dirname | xargs -n1 basename | jq -R . | jq -sc .) + echo "components=$names" >> "$GITHUB_OUTPUT" + + lint-components: + name: Lint components + runs-on: ubuntu-latest + needs: discover + strategy: + fail-fast: false + matrix: + component: ${{ fromJSON(needs.discover.outputs.components) }} + steps: + - name: Clone the code + uses: actions/checkout@v5 + + - name: Setup Go + uses: actions/setup-go@v6 + with: + go-version-file: components/${{ matrix.component }}/go.mod + + - name: Run linter + uses: golangci/golangci-lint-action@v9 + with: + version: v2.4.0 + working-directory: components/${{ matrix.component }} + args: --config ${{ github.workspace }}/.golangci.yml + + test-components: + name: Test components + runs-on: ubuntu-latest + needs: discover + strategy: + fail-fast: false + matrix: + component: ${{ fromJSON(needs.discover.outputs.components) }} + steps: + - name: Clone the code + uses: actions/checkout@v5 + + - name: Setup Go + uses: actions/setup-go@v6 + with: + go-version-file: components/${{ matrix.component }}/go.mod + + - name: Running Tests + working-directory: components/${{ matrix.component }} + run: | + go mod tidy + git diff --exit-code go.mod go.sum + make test diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 67ff2bf..0000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Lint - -on: - push: - pull_request: - -jobs: - lint: - name: Run on Ubuntu - runs-on: ubuntu-latest - steps: - - name: Clone the code - uses: actions/checkout@v4 - - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - - - name: Run linter - uses: golangci/golangci-lint-action@v8 - with: - version: v2.1.6 diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index da887e1..4709541 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -10,10 +10,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Clone the code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version-file: go.mod diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index fc2e80d..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Tests - -on: - push: - pull_request: - -jobs: - test: - name: Run on Ubuntu - runs-on: ubuntu-latest - steps: - - name: Clone the code - uses: actions/checkout@v4 - - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - - - name: Running Tests - run: | - go mod tidy - make test diff --git a/.gitignore b/.gitignore index d0ee133..79626d9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ bin/* Dockerfile.cross +# A pattern containing a slash is anchored to this directory, so `bin/*` above does +# not reach a component's build output (see components/README.md). +components/*/bin/ + # Locally built command binaries. `go build ./cmd/...` drops these in the repo # ROOT, not bin/, so they are not covered above — and they are multi-MB, which is # exactly the kind of thing that sneaks into a commit unnoticed. diff --git a/Dockerfile b/Dockerfile index bbac4a8..a783a44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.24 AS builder +FROM golang:1.25 AS builder ARG TARGETOS ARG TARGETARCH # VERSION is stamped into the binary (pkg/version) and surfaces as the virtual diff --git a/Makefile b/Makefile index 0585110..463071e 100644 --- a/Makefile +++ b/Makefile @@ -140,6 +140,24 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes lint-config: golangci-lint ## Verify golangci-lint linter configuration $(GOLANGCI_LINT) config verify +##@ Components + +# Each component under components/ is its own Go module, so `./...` here cannot see any of it -- +# not for build, test, vet or lint. This delegates any target to every component by name: +# `make components-lint`, `make components-test`. Adding a component needs no edit here. +# +# Without this, a nested module is simply never checked, which is the usual way one rots. +# Discovered by Makefile rather than by directory, which is both the contract (see +# components/README.md) and immune to make 3.81's wildcard not filtering on a trailing slash. +COMPONENTS := $(patsubst %/,%,$(dir $(wildcard components/*/Makefile))) + +.PHONY: components-% +components-%: ## Run the named target in every component, e.g. make components-test. + @for c in $(COMPONENTS); do \ + echo "==> $$c: $*"; \ + $(MAKE) -C $$c $* || exit 1; \ + done + ##@ Build .PHONY: build @@ -238,7 +256,7 @@ CONTROLLER_TOOLS_VERSION ?= v0.18.0 ENVTEST_VERSION ?= $(shell go list -m -f "{{ .Version }}" sigs.k8s.io/controller-runtime | awk -F'[v.]' '{printf "release-%d.%d", $$2, $$3}') #ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31) ENVTEST_K8S_VERSION ?= $(shell go list -m -f "{{ .Version }}" k8s.io/api | awk -F'[v.]' '{printf "1.%d", $$3}') -GOLANGCI_LINT_VERSION ?= v2.1.6 +GOLANGCI_LINT_VERSION ?= v2.4.0 .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. diff --git a/README.md b/README.md index 3e697c5..d937e9e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ **The Control Plane for GPUaaS** [![Discord](https://img.shields.io/badge/Discord-Join%20us-5865F2?logo=discord&logoColor=white)](https://discord.gg/7WTUuFqyS6) -![Go Version](https://img.shields.io/badge/go-1.24-00ADD8?logo=go&logoColor=white) +![Go Version](https://img.shields.io/badge/go-1.25-00ADD8?logo=go&logoColor=white) [![Go Reference](https://pkg.go.dev/badge/github.com/InftyAI/Nebula.svg)](https://pkg.go.dev/github.com/InftyAI/Nebula) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE) diff --git a/components/README.md b/components/README.md new file mode 100644 index 0000000..46c2514 --- /dev/null +++ b/components/README.md @@ -0,0 +1,47 @@ +# Components + +Optional pieces that ship separately from the manager. A cluster that deploys none of them is a +complete Nebula install. + +Each subdirectory is its **own Go module**, deliberately: the root module cannot import a +component even by accident, so "the manager does not depend on this" is enforced by the module +boundary rather than by review. The dependency only runs the other way — a component may import +`github.com/InftyAI/Nebula` for the API types, via a `replace` on the repo root. + +## The contract + +Held to so that the root plumbing keeps working and the second component does not invent its own +conventions: + +- **Its own `go.mod`**, with `replace github.com/InftyAI/Nebula => ../..` when it needs the API + types. The replace is what makes an API change break the component's build in the same PR + instead of drifting until someone bumps a tag. CI also runs `go mod tidy` and fails on a diff, + so an untidy `go.mod` is a red build rather than a surprise later. +- **Its own `Makefile` — which is what makes the component visible at all.** The root's + `components-%` rule and the CI matrix both discover components by globbing + `components/*/Makefile`, so a directory without one is checked by nothing, silently. `test` is + the target CI calls; `lint`, `build` and `docker-build` are called through the root by name + (`make components-lint`). +- **Its own `Dockerfile`, built with the repo root as context** — `docker build -f + components//Dockerfile .` — because the `replace` reaches outside the component + directory. A Dockerfile that assumes its own directory as context cannot resolve it. +- **Its own deployment, kept inside the component.** The root `config/` is the manager's install + and stays that way. A `config/` of its own or a script both work — logship uses + [`logship/hack/deploy.sh`](logship/hack/deploy.sh). +- **`internal/` for everything but `cmd/`**, because nothing outside the component should import + it. + +## Why the root tooling does not reach inside + +`go build ./...` skips directories containing their own `go.mod`, so the root's `lint`, `test` +and `vet` targets do not see any of this — silently, which is the usual way a nested module rots. +Two things prevent it: the `components-%` pattern rule in the root `Makefile`, and the `discover` +job in `.github/workflows/ci.yml` that fans lint and test out over one job per component. Both +build the list from `components/*/Makefile`, so adding a component needs no edit to either. + +## Components + +- [`logship/`](logship/) — copies the logs of the instances Nebula runs outside the cluster to + CloudWatch Logs, so a run can still be explained after the provider's own retention window + closes. Modal is the first backend, behind a port. [How to run it](logship/README.md); + [why it is shaped this way](logship/design.md). diff --git a/components/logship/Dockerfile b/components/logship/Dockerfile new file mode 100644 index 0000000..64688d0 --- /dev/null +++ b/components/logship/Dockerfile @@ -0,0 +1,42 @@ +# Build the logship binary. +# +# THE BUILD CONTEXT IS THE REPO ROOT, not this directory: +# +# docker build -f components/logship/Dockerfile . +# +# go.mod carries `replace github.com/InftyAI/Nebula => ../..` for the API types, so a context +# rooted at this directory cannot resolve the replacement and `go mod download` fails. +# +# --platform=${BUILDPLATFORM} pins the builder stage to the machine doing the building and leaves the +# target to GOOS/GOARCH below. Without it, buildx runs an arm64 builder under emulation to produce the +# arm64 image — minutes of QEMU per architecture for a cross-compile Go does for free. It is also why +# this needs no Dockerfile.cross sed dance like the root's docker-buildx. +FROM --platform=${BUILDPLATFORM} golang:1.25 AS builder +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /workspace + +# The replaced module's go.mod must be present for the component's own resolution to work, even +# where no package from it is imported yet. +COPY go.mod go.mod +COPY go.sum go.sum +COPY api/ api/ + +WORKDIR /workspace/components/logship +COPY components/logship/go.mod go.mod +COPY components/logship/go.sum go.sum +RUN go mod download + +COPY components/logship/cmd/ cmd/ +COPY components/logship/internal/ internal/ + +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -a -o logship ./cmd + +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/components/logship/logship . +USER 65532:65532 + +ENTRYPOINT ["/logship"] diff --git a/components/logship/Makefile b/components/logship/Makefile new file mode 100644 index 0000000..08303c7 --- /dev/null +++ b/components/logship/Makefile @@ -0,0 +1,108 @@ +# Image URL to use all building/pushing image targets +IMG ?= inftyai/nebula-logship:latest + +CONTAINER_TOOL ?= docker + +# What `docker-push` publishes: a manifest list covering both, so the same tag runs on Graviton and +# x86 nodes. A single-platform image is not a build failure and not a run failure either — the kubelet +# rejects it at pull time with "no match for platform in manifest", which reads like a missing tag. +PLATFORMS ?= linux/amd64,linux/arm64 + +# Shared with the root's docker-buildx on purpose, so both reuse one builder and its layer cache. +BUILDX_BUILDER ?= nebula-builder + +# The repo root. The Docker build context and the shared linter config both live there; see the +# Dockerfile's header for why the context cannot be this directory. +ROOT := ../.. + +# Reuses the root's bin/ so a component does not download its own copy of every tool. +LOCALBIN ?= $(abspath $(ROOT)/bin) +GOLANGCI_LINT = $(LOCALBIN)/golangci-lint + +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +.PHONY: all +all: build + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Development + +.PHONY: fmt +fmt: ## Run go fmt against code. + go fmt ./... + +.PHONY: vet +vet: ## Run go vet against code. + go vet ./... + +.PHONY: test +test: fmt vet ## Run tests. + go test ./... -coverprofile cover.out + +# One config for the whole repo, run separately per module: `./...` at the root cannot see a nested +# module at all, which is the only reason this target exists. Fork the config here if the rules +# ever need to differ. +.PHONY: lint +lint: golangci-lint ## Run golangci-lint linter. + $(GOLANGCI_LINT) run --config $(abspath $(ROOT)/.golangci.yml) + +.PHONY: lint-fix +lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes. + $(GOLANGCI_LINT) run --config $(abspath $(ROOT)/.golangci.yml) --fix + +##@ Build + +.PHONY: build +build: fmt vet ## Build the logship binary. + mkdir -p bin + go build -o bin/logship ./cmd + +.PHONY: run +run: fmt vet ## Run logship against your current kubectl context. It takes no flags. + go run ./cmd + +# Host platform, not PLATFORMS: --load takes a single image, and the one worth having locally is the +# one this machine can run — a kind node on Apple silicon wants the arm64 build, an x86 CI box the amd64. +.PHONY: docker-build +docker-build: ## Build for this machine, into the local image store. Context is the repo root; see the Dockerfile. + $(CONTAINER_TOOL) buildx build --load -f Dockerfile -t ${IMG} $(ROOT) + +# Builds again instead of pushing what docker-build made, and has to: a manifest list cannot live in +# the local image store, so there is nothing there to push. Pushing the single-platform image that IS +# there is what makes a node of the other architecture fail the pull with "no match for platform in +# manifest" — at pull time, so it reads as a missing tag rather than as the wrong build. +# +# Passing --builder rather than `buildx use` leaves the shell's own builder selection alone. The +# docker-container driver is required: the default driver cannot build two platforms at once. +.PHONY: docker-push +docker-push: ## Build every platform in PLATFORMS and push them as one tag. + $(CONTAINER_TOOL) buildx inspect $(BUILDX_BUILDER) >/dev/null 2>&1 \ + || $(CONTAINER_TOOL) buildx create --name $(BUILDX_BUILDER) --driver docker-container >/dev/null + $(CONTAINER_TOOL) buildx build --builder $(BUILDX_BUILDER) --push \ + --platform $(PLATFORMS) -f Dockerfile -t ${IMG} $(ROOT) + +##@ Deployment + +# Takes no instance: it watches the cluster and ships every Pod Nebula placed, through whichever +# provider placed it, reading each record's identity off that Pod. hack/deploy.sh has the details. +.PHONY: deploy +deploy: ## Ship every Nebula instance in the cluster. + IMG=$(IMG) hack/deploy.sh + +DEPLOY_NAME = $(or $(NAME),logship) + +.PHONY: undeploy +undeploy: ## Remove the Deployment and its RBAC. + kubectl -n $(or $(DEPLOY_NS),nebula-system) delete deployment $(DEPLOY_NAME) --ignore-not-found + kubectl delete clusterrolebinding $(DEPLOY_NAME) clusterrole $(DEPLOY_NAME) --ignore-not-found + kubectl -n $(or $(DEPLOY_NS),nebula-system) delete serviceaccount $(DEPLOY_NAME) --ignore-not-found + +##@ Dependencies + +.PHONY: golangci-lint +golangci-lint: ## Install golangci-lint into the root's bin/ if necessary. + $(MAKE) -C $(ROOT) golangci-lint diff --git a/components/logship/README.md b/components/logship/README.md new file mode 100644 index 0000000..c34d074 --- /dev/null +++ b/components/logship/README.md @@ -0,0 +1,145 @@ +# logship + +Copies the logs of the instances Nebula runs outside the cluster to CloudWatch Logs, so a run can +still be explained after the provider's own retention window closes. Optional: a cluster that does +not deploy it is unaffected, and the manager does not link a byte of it. + +Modal is the one provider implemented so far, and it is one backend behind a port rather than the +subject of this component — see [Providers](#providers). The design and the reasoning live in +[design.md](design.md); this file is how to run what exists. + +## Layout + +| path | what is in it | +| --- | --- | +| `cmd/main.go` | the entrypoint, and all of it: signals, the provider set, the watch. No flags | +| `cmd/fleet.go` | the wiring — an instance to its provider, a Pod's labels to a record, one sink for the process. `register` is the only place a provider is named | +| `internal/provider/` | the port a backend implements, and the registry that opens one when the first Pod needs it | +| `internal/modal/` | the Modal backend: its own `SandboxGetLogs` client, the cursor-aware reader and re-open loop, a gRPC connection pool | +| `internal/ship/` | one stream's mechanism: chunks to lines, lines to a record, records to batches, and the `Pipeline` that runs them | +| `internal/emit/` | the sink — one record per line on stdout, shared by every stream, plus the size limits that keep a line collectable | +| `internal/supervise/` | lifetimes: a changing set of instances becomes a running set of pipelines, with the restart budget | +| `internal/watch/` | the cluster half — which Pods to ship, and the instance each one names | +| `hack/deploy.sh` | what `make deploy` runs: ServiceAccount, ClusterRole, Deployment, and the checks to run afterwards in the order they can fail | +| `Makefile`, `Dockerfile`, `go.mod` | its own module and image, so the manager cannot depend on this even by accident | + +Tests sit beside the code they cover. `internal/modal/fake_server_test.go` is a fake +`SandboxGetLogs` server, which is why `make test` needs no Modal account. + +## Running + +The reader, the pipeline, the sink, the supervisor and the Pod watch exist; the cursor checkpoint and +the drain finalizer do not. So one process discovers and ships every instance in the cluster, but a +restart replays each one from the beginning. + +It takes no arguments, and there are none to take: it lists Pods labelled +`nebula.inftyai.com/enabled=true`, reads which provider each was placed on off its `spec.nodeSelector` +(`nebula.inftyai.com/provider`), and reads the instance ID off the `nebula.inftyai.com/instance-id` +annotation the virtual kubelet writes back once `Provision` returns one. A Pod without it is skipped, +not an error. + +```console +$ export MODAL_TOKEN_ID=ak-... MODAL_TOKEN_SECRET=as-... +$ make run +logship: watching Pods enabled nebula.inftyai.com/enabled instanceID nebula.inftyai.com/instance-id providers [modal] +``` + +There is deliberately no way to name an instance — not to ship one, and not to read one either. A +record's identity has one source, the Pod that owns the instance, because a second source can only +disagree with it, and a record with the wrong identity is delivered, retained, billed and invisible. + +Records go to **stdout**, one per line; every diagnostic goes to stderr. Each record carries the +*instance's* pod name and labels, because logship prints another pod's output while the log agent +stamps only its own identity: + +```console +{"time":"...","log":"{\"level\":\"INFO\",\"category\":\"user\",\"id\":\"1-7\",\"message\":\"training step 1\"}","kubernetes":{"pod_name":"exp-1-sandbox-0","labels":{...}}} +``` + +`log` is itself JSON because the consumer parses it and hides anything it cannot — see design.md. A +sandbox line that is *already* such an envelope keeps its own `level` and `category` and only gains the +`id`; the `INFO`/`user` above are fallbacks for plain output. + +Name and labels are the whole of it — no namespace, because the consumer identifies a line by pod name +and labels alone. + +Modal's credentials come from the environment only: `MODAL_TOKEN_ID`, `MODAL_TOKEN_SECRET`, and +optionally `MODAL_SERVER_URL`. The SDK merges `~/.modal.toml` too, but its loader is unexported, so a +local profile has to be exported as variables. They are needed only when something is actually on +Modal — a backend opens when the first Pod on it arrives, not at startup. + +## Providers + +A provider is one implementation of `internal/provider.Provider`: which streams an instance has, a +`ship.Source` for one of them, and `Reserve` for a backend whose transport caps concurrent streams per +connection. Everything above it — the batching, the record shape, the restart budget, the watch — is +written against that port and names no provider. Adding one is a package plus one line in +`cmd/fleet.go`'s `register`. + +Which provider an instance needs is the Pod's to say, and is deliberately not configurable: a +configured provider would have to be kept in agreement with the cluster's node pools, and being wrong +about it is indistinguishable from an idle cluster — every Pod skipped, nothing shipped, no error. +Routing per Pod also means a cluster split across two providers ships from both with no extra wiring. +A Pod on a provider this build has no backend for is logged and skipped, because Nebula gaining a +provider before logship does is a real state that must not look like silence. + +## Deploying + +One Deployment for the whole cluster. It needs no instance, no pod and no labels, because the watch +takes all three off each Pod: + +```console +make docker-push IMG=inftyai/nebula-logship:latest # amd64 + arm64, pushed as one tag +make deploy IMG=inftyai/nebula-logship:latest +make undeploy # also removes the ServiceAccount and ClusterRole +``` + +`docker-push` does its own build and does not run `docker-build`: a manifest list cannot sit in the +local image store, so there is nothing there to push. Pushing a single-platform image instead fails on +a node of the other architecture at *pull* time — `no match for platform in manifest`, which reads like +a missing tag rather than the wrong build. Override `PLATFORMS` to publish just one. + +Four things about a deployed one, in the order they bite: + +- **`replicas: 1` and `strategy: Recreate` are both load-bearing.** Two processes on one instance read + from the same cursor and ship every line twice, and there is no leader election to prevent it — + `RollingUpdate` would do exactly that for the seconds its `maxSurge` pod overlaps the old one. The + single replica is therefore the whole fleet's collection and a single point of failure, deliberately: + scaling out needs the lease first. +- **Restarts replay, and replays are billed.** A fresh process resumes from the cursor it was given, + which is the beginning, so the provider re-sends the instance's whole history and CloudWatch stores + it again. `supervise` bounds this in-process, but a *container* restart escapes that bound, which + makes a CrashLoopBackOff a billing loop rather than an outage. Check `RESTARTS` on a long run. +- **The namespace decides whether records arrive at all**, since they reach CloudWatch through the log + agent of the cluster logship runs in. The default, `nebula-system`, is where Nebula's manager already + runs in the dev cluster, and the manager's own container log is a live stream in the group the + consumer reads — so collection there is checked, not assumed. +- **A shipped instance and a findable one are not the same thing.** The consumer filters on `app` and + its three tenant IDs, so an instance's Pod missing one of them ships records that are delivered, + retained, billed and invisible, with nothing reporting a failure. Nothing here can fix that; the + labels are the Pod's. The fields the watch reads — note the two domains, Nebula's markers on + `nebula.inftyai.com` and the tenant IDs on the consumer's own: + + ```console + kubectl -n get pod exp-1-sandbox-0 -o json | jq -r ' + .metadata.annotations["nebula.inftyai.com/instance-id"], + .spec.nodeSelector["nebula.inftyai.com/provider"], (.metadata.labels)' + ``` + +In CloudWatch the record arrives one level down, under `log_processed`, with the agent's own identity +at the top. That nesting is the contract the consumer's filter patterns are written against — and a +pattern matching nothing is not an error, so a filter written against the top level alone reads as +"nothing arrived". There is no fleet size to configure either: each provider makes room for an instance +before its streams open, and one that cannot skips the instance with a log. + +## Building + +```console +make test # unit tests, no provider account needed +make lint # the repo's shared golangci config, run against this module +make build # bin/logship +make docker-build # this machine's platform only; context is the repo ROOT, see the Dockerfile header +``` + +`make -C ../.. components-test` runs the same from the root. Note that the root's own `make test` and +`make lint` do **not** cover this module: `./...` stops at a nested `go.mod`. diff --git a/components/logship/cmd/fleet.go b/components/logship/cmd/fleet.go new file mode 100644 index 0000000..4fb6887 --- /dev/null +++ b/components/logship/cmd/fleet.go @@ -0,0 +1,144 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/InftyAI/Nebula/components/logship/internal/emit" + "github.com/InftyAI/Nebula/components/logship/internal/modal" + "github.com/InftyAI/Nebula/components/logship/internal/provider" + "github.com/InftyAI/Nebula/components/logship/internal/ship" + "github.com/InftyAI/Nebula/components/logship/internal/supervise" +) + +// register wires the log providers this build has. Adding one is this line plus its package: nothing +// else here names a provider, and nothing has to be told which one a cluster uses. +func register(set *provider.Set) { + set.Register(provider.ProviderModal, func() (provider.Provider, error) { + p, err := modal.Open() + if err != nil { + return nil, err + } + return p, nil + }) +} + +// fleet routes each instance to the provider that can read it, and is both halves of what the +// supervisor needs — the stream list and the pipeline — plus the watch's Fleet. +// +// One type because all three are the same lookup: the Pod said which provider it is on, and +// everything else follows from resolving that once. +type fleet struct { + set *provider.Set + sup *supervise.Supervisor + + // sink is shared by every stream: there is one stdout. See emit.Sink. + sink *emit.Sink + + // cursor resolves where a stream resumes. Nil means the beginning; phase 3 replaces it with the + // checkpoint, which is why it is a function rather than a string. + cursor func(inst supervise.Instance, stream string) string + + // Only failures are reported from here, hence errf rather than a general logger: every one of + // them means an instance's logs are not being shipped. + errf func(msg string, keysAndValues ...any) +} + +// Ensure starts copying an instance, after making room for it. +// +// Ordering is the point: a provider has to reserve capacity BEFORE the streams it accounts for open. +// For Modal that is a connection, and a connection added afterwards cannot take over streams already +// queued behind HTTP/2's concurrency cap — which gRPC-go does silently rather than failing, the worst +// available failure for a log shipper. Reserving here rather than at a configured ceiling is why +// there is no fleet size to guess. +// +// Either failure skips the instance rather than shipping it: with no backend there is nothing to read +// it, and with no capacity its streams would wait forever with nothing said. +func (f *fleet) Ensure(inst supervise.Instance) { + if !f.reserve(inst.Provider, f.sup.InstanceCount()+1) { + f.errf("NOT shipping this instance", "provider", inst.Provider, "instance", inst.ID, "pod", inst.Pod) + return + } + f.sup.Ensure(inst) +} + +func (f *fleet) Forget(ref supervise.Ref) { f.sup.Forget(ref) } + +// streams is supervise.Config.Streams: the provider's own stream names, because they are the names it +// will have to recognise again in Source. +func (f *fleet) streams(inst supervise.Instance) []string { + p, err := f.set.Get(inst.Provider) + if err != nil { + // Unreachable via Ensure, which resolved the same provider first. Nothing is logged here + // because an instance with no streams is not silent — the supervisor says so. + return nil + } + return p.Streams() +} + +// build is supervise.Builder: one instance and one stream into a running pipeline. Called again on +// every restart, so it holds no per-run state. +func (f *fleet) build(inst supervise.Instance, stream string) (*ship.Pipeline, error) { + p, err := f.set.Get(inst.Provider) + if err != nil { + return nil, err + } + src, err := p.Source(inst.ID, stream) + if err != nil { + return nil, err + } + + // The Pod's labels wholesale, which is both less code than picking the tenant keys out and a + // closer match to what the agent writes for every other pod in the group. Nothing here has to + // know a key: the consumer reads them from the record. + record := ship.Record{Pod: inst.Pod, Labels: inst.Labels} + + cursor := "" + if f.cursor != nil { + cursor = f.cursor(inst, stream) + } + + return ship.New(ship.Config{ + Source: src, + Sink: f.sink, + Format: record.Formatter(), + Limits: emit.Limits(0), + Cursor: cursor, + Log: f.errf, + // On because off loses the bar entirely: an un-collapsed progress bar never sends a newline, + // so the assembler holds every frame until maxFragment and emits one 64 KiB line, which the + // batcher then drops for exceeding the per-event cap. Collapsed, each frame replaces the last, + // so the line stays small and arrives — as the bar's final state rather than its history. + CollapseFrames: true, + // No Throttled: a write to stdout has no rate to back off from. The agent owns retrying the + // one hop that does, which is the point of shipping this way. + }), nil +} + +// reserve resolves the instance's provider and makes room for instances of them, reporting whether +// the fleet now has somewhere to put them. +func (f *fleet) reserve(name string, instances int) bool { + p, err := f.set.Get(name) + if err != nil { + f.errf("no log provider for this instance", "provider", name, "err", err) + return false + } + if err := p.Reserve(instances); err != nil { + f.errf("cannot make room for this instance", "provider", name, "instances", instances, "err", err) + return false + } + return true +} diff --git a/components/logship/cmd/fleet_test.go b/components/logship/cmd/fleet_test.go new file mode 100644 index 0000000..963478f --- /dev/null +++ b/components/logship/cmd/fleet_test.go @@ -0,0 +1,161 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "testing" + "time" + + "github.com/InftyAI/Nebula/components/logship/internal/emit" + "github.com/InftyAI/Nebula/components/logship/internal/provider" + "github.com/InftyAI/Nebula/components/logship/internal/ship" + "github.com/InftyAI/Nebula/components/logship/internal/supervise" +) + +func podLabels() map[string]string { + return map[string]string{ + "app": "sandbox", + "example.com/org-id": "org-1", + "example.com/team-id": "team-2", + "example.com/experiment-id": "exp-3", + "pod-template-hash": "7d9f", + "example.com/something-else": "kept", + } +} + +func TestRecordCarriesEveryPodLabel(t *testing.T) { + // The tenant keys are never read here — they are the consumer's, and it reads them off the record. + // Copying the map wholesale is what keeps that true without this package knowing a key. + inst := supervise.Instance{Pod: "pod-a", Labels: podLabels()} + format := ship.Record{Pod: inst.Pod, Labels: inst.Labels}.Formatter() + + msg := format(ship.Line{Data: "hello", At: time.Unix(0, 0), Cursor: "1-0"}) + var rec struct { + Log string `json:"log"` + Kubernetes struct { + PodName string `json:"pod_name"` + Labels map[string]string `json:"labels"` + } `json:"kubernetes"` + } + if err := json.Unmarshal([]byte(msg), &rec); err != nil { + t.Fatalf("record is not JSON: %v\n%s", err, msg) + } + if rec.Kubernetes.PodName != "pod-a" { + t.Errorf("pod_name = %q", rec.Kubernetes.PodName) + } + if len(rec.Kubernetes.Labels) != len(podLabels()) { + t.Fatalf("%d labels, want all %d of the Pod's", len(rec.Kubernetes.Labels), len(podLabels())) + } + if rec.Kubernetes.Labels["pod-template-hash"] != "7d9f" { + t.Error("a label unrelated to tenancy was dropped") + } + + // The nested envelope is what keeps the line out of the hidden `private` category. + var inner struct { + Level string `json:"level"` + Category string `json:"category"` + Message string `json:"message"` + } + if err := json.Unmarshal([]byte(rec.Log), &inner); err != nil { + t.Fatalf("log is not nested JSON: %v", err) + } + if inner.Category != ship.CategoryUser || inner.Message != "hello" { + t.Fatalf("inner = %+v", inner) + } +} + +// stub is a log provider with one stream, whose instances have already ended. +type stub struct{ reserved int } + +func (s *stub) Streams() []string { return []string{"stdout"} } + +func (s *stub) Source(_, stream string) (ship.Source, error) { + if stream != "stdout" { + return nil, fmt.Errorf("no stream %q", stream) + } + return endedStream{}, nil +} + +func (s *stub) Reserve(instances int) error { s.reserved = instances; return nil } +func (s *stub) Close() error { return nil } + +type endedStream struct{} + +func (endedStream) Follow(context.Context, string, func(ship.Batch) error) error { return nil } + +// newTestFleet wires a fleet exactly as newFleet does, so what is under test is the routing rather +// than a rebuilt copy of it. +func newTestFleet(t *testing.T, set *provider.Set) (*fleet, *[]string) { + t.Helper() + + var errs []string + f := &fleet{ + set: set, + sink: emit.New(io.Discard), + errf: func(msg string, _ ...any) { errs = append(errs, msg) }, + } + f.sup = supervise.New(context.Background(), supervise.Config{Streams: f.streams, Build: f.build}) + t.Cleanup(f.sup.Shutdown) + return f, &errs +} + +func TestAnInstanceOnAnUnknownProviderIsNotShipped(t *testing.T) { + // Nebula gaining a provider before this build does. Nothing can read the instance, and the only + // other symptom is a run whose logs never arrive. + f, errs := newTestFleet(t, provider.NewSet()) + + f.Ensure(supervise.Instance{Provider: "aws", ID: "i-0abc", Pod: "exp-1-sandbox-0"}) + + if len(*errs) == 0 { + t.Error("an unreadable instance was skipped without a word") + } + if st := f.sup.Stats(); st.Started != 0 { + t.Errorf("the fleet started %d instances it has no provider for", st.Started) + } +} + +func TestRoomIsMadeBeforeTheStreamsOpen(t *testing.T) { + // Reserve has to run before Ensure, not after: for Modal a connection added once the streams are + // queued cannot take them over, and gRPC-go queues them silently. See fleet.Ensure. + p := &stub{} + set := provider.NewSet() + set.Register(provider.ProviderModal, func() (provider.Provider, error) { return p, nil }) + f, _ := newTestFleet(t, set) + + f.Ensure(supervise.Instance{Provider: provider.ProviderModal, ID: "sb-1", Pod: "p"}) + + if p.reserved != 1 { + t.Errorf("reserved for %d instances, want the one being added", p.reserved) + } +} + +func TestBuildRejectsAStreamTheProviderDoesNotHave(t *testing.T) { + // The supervisor names streams with strings, so a typo has to fail loudly rather than open + // something that ships nothing. + set := provider.NewSet() + set.Register(provider.ProviderModal, func() (provider.Provider, error) { return &stub{}, nil }) + f, _ := newTestFleet(t, set) + + inst := supervise.Instance{Provider: provider.ProviderModal, ID: "sb-1"} + if _, err := f.build(inst, "stdlog"); err == nil { + t.Fatal("build accepted a stream the provider does not have") + } +} diff --git a/components/logship/cmd/main.go b/components/logship/cmd/main.go new file mode 100644 index 0000000..487f785 --- /dev/null +++ b/components/logship/cmd/main.go @@ -0,0 +1,130 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Command logship copies the logs of Nebula's externally-run instances onto its own stdout, from +// which the cluster's log agent takes them the rest of the way. See the component's design.md. +// +// It takes no arguments and has one mode: discover every Nebula Pod in the cluster and ship it +// through the provider that Pod was placed on. Which providers this build can read is in fleet.go's +// register; which one an instance needs is the Pod's to say, never a flag's — see internal/provider. +// +// There is deliberately no way to ship, or to read, an instance named by hand. A record's identity — +// the pod name and the tenant labels — has exactly one source, the Pod the instance belongs to, +// because a second source can only ever disagree with it, and a record carrying the wrong identity is +// delivered, retained, billed and invisible, with nothing reporting a failure. +// +// stdout is the data channel and nothing else may write to it: every diagnostic goes to stderr, +// deliberately, all the way down to the stats line at the end. +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/InftyAI/Nebula/components/logship/internal/emit" + "github.com/InftyAI/Nebula/components/logship/internal/provider" + "github.com/InftyAI/Nebula/components/logship/internal/supervise" + "github.com/InftyAI/Nebula/components/logship/internal/watch" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "logship:", err) + os.Exit(1) + } +} + +// run is main's body so that os.Exit cannot skip the providers' Close, which is what ends their +// connections rather than leaving the server to time them out. +func run() error { + // No flags to define, and Parse stays regardless: it rejects an argument instead of ignoring one, + // so an out-of-date invocation fails loudly rather than quietly shipping the whole cluster. + flag.Parse() + if flag.NArg() != 0 { + return fmt.Errorf("unexpected arguments: %v", flag.Args()) + } + + // A running instance never ends its stream, so Ctrl-C is the ordinary way out. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + set := provider.NewSet() + register(set) + defer func() { + if err := set.Close(); err != nil { + logf("closing the log providers", "err", err) + } + }() + + return shipCluster(ctx, set) +} + +// shipCluster ships every Nebula Pod in the cluster, discovering them rather than being told one — +// which is the whole of what the Deployment does. See this package's doc comment for why there is no +// by-hand alternative. +func shipCluster(ctx context.Context, set *provider.Set) error { + client, err := watch.NewClient() + if err != nil { + return err + } + f := newFleet(ctx, set) + + logf("watching Pods", "enabled", watch.EnabledLabel, "instanceID", watch.InstanceIDAnnotation, + "providers", set.Names()) + w := &watch.Watcher{ + Client: client, + Fleet: f, + Log: logf, + } + if err := w.Run(ctx); err != nil { + return err + } + f.shutdown() + return nil +} + +// newFleet wires the supervisor to the providers. Nothing is dialled here: a provider opens when the +// first Pod on it arrives, so a cluster using one of them needs no credentials for the others. +func newFleet(ctx context.Context, set *provider.Set) *fleet { + f := &fleet{set: set, sink: emit.New(os.Stdout), errf: errf} + f.sup = supervise.New(ctx, supervise.Config{ + Streams: f.streams, + Build: f.build, + Log: logf, + }) + return f +} + +func (f *fleet) shutdown() { + f.sup.Shutdown() + fmt.Fprintf(os.Stderr, "logship: %+v\n", f.sup.Stats()) +} + +// logf prints an operational message, on stderr because stdout is the record channel: a diagnostic +// printed there would reach CloudWatch as a record the consumer cannot parse. +func logf(msg string, kv ...any) { + fmt.Fprintln(os.Stderr, append([]any{"logship:", msg}, kv...)...) +} + +// errf marks a message as a failure. Same channel as logf; what it adds is the ERROR token, which is +// what an operator greps for -- these are the paths where shipping stops and nothing else says so. +func errf(msg string, kv ...any) { + logf("ERROR "+msg, kv...) +} diff --git a/components/logship/design.md b/components/logship/design.md new file mode 100644 index 0000000..e916113 --- /dev/null +++ b/components/logship/design.md @@ -0,0 +1,439 @@ +# Log shipping + +Copies an externally-run instance's stdout and stderr to CloudWatch Logs while it runs, so the output +survives the provider's retention window. + +Modal is the one provider implemented, and the sections below about it describe that implementation, +not the component: reading a backend is a port (`internal/provider`), and which backend an instance +needs comes off its Pod. See [How it works](#how-it-works). + +**Status.** The reader, the pipeline, the sink, the supervisor, the Pod watch and the wiring between +them are implemented and tested, so one process finds the fleet itself and ships all of it. What is +missing is durability and scale-out: no cursor checkpoint, so a restart replays; no drain finalizer, +so a deletion can cut the tail; no lease, so there can only be one replica. See +[What is left](#what-is-left). + +**It does not talk to AWS, and it writes no files.** A record goes to logship's own stdout and the +cluster's existing log agent collects it exactly as it collects every other pod's. No AWS +credentials, no IAM grant, no hostPath — see [The handoff](#the-handoff) for what that costs. + +- [Why](#why) +- [What Modal gives us](#what-modal-gives-us) +- [How it works](#how-it-works) +- [The handoff](#the-handoff) +- [Event format](#event-format) +- [Duplicates](#duplicates) +- [Scale](#scale) +- [Failure modes](#failure-modes) +- [What is left](#what-is-left) +- [Permissions](#permissions) +- [Open decisions](#open-decisions) +- [Known gaps](#known-gaps) + +## Why + +Modal deletes logs on a clock: **1 day on Starter, 30 days on Team**, custom on Enterprise. Once +the window passes, a training run's output is gone and nothing in this repo can recover it. + +So this is durability, not convenience. `kubectl logs` still reads live from Modal and is +unaffected; what changes is whether last week's run can still be explained. + +## What Modal gives us + +`SandboxGetLogs` streams batches, each carrying a cursor — `entry_id`, formatted `-`. +Passing one back as `LastEntryId` returns strictly what followed it, so resuming is exact. +Verified against a live sandbox: + +| cursor | returned | +| --- | --- | +| `"0-0"` | the whole history | +| first line's `entry_id` | only the lines after it | +| last line's `entry_id` | nothing but `eof` | +| unrecognized (`"1-0"`) | the whole history, **no error** | + +Four things about that stream shape the code: + +- **The SDK throws the cursor away.** `sb.Stdout`/`sb.Stderr` bottom out in `outputStreamSb`, + which tracks `entry_id` for its own 55-second re-open loop and then writes only `item.GetData()` + into a pipe. The entry ID, per-item timestamp and file descriptor are gone before a caller sees + a byte. Hence the direct RPC: not for the call, but to keep what the call already returns. +- **stdout and stderr have independent ID spaces.** Two lines in the same millisecond got + `1788477379229-0` and `1788477379282-0`. Everything downstream is per-descriptor. +- **An item is a chunk, not a line.** `data` can hold several newlines or half of one, so line + assembly is ours. Items also arrive with a `task_state` or `task_progress` and no data; those + are not output, and `PutLogEvents` rejects an empty message, so the reader drops them while + still advancing the cursor past them. +- **Prefer `timestamp_ns` over `timestamp`.** The float64 seconds field cannot represent every + millisecond exactly, so a millisecond-resolution sink reorders same-millisecond lines without + it. + +Two stream-level signals are easy to confuse: `io.EOF` means the 55-second window closed and +should be re-opened, while `batch.GetEof()` means the sandbox finished. The `eof` batch's +`entry_id` is empty, so it must not reach the cursor — an empty cursor replays everything. + +## How it works + +Six packages, one direction of dependency. `internal/ship` declares the ports and knows nothing +about a provider or about where a line lands; `cmd` is where they meet. + +| package | role | +| --- | --- | +| `internal/provider` | port: which streams an instance has, a `Source` for one, capacity ahead of them, and the set of registered backends | +| `internal/modal` | one backend: cursor-aware `Follow`, the re-open loop, a `Source` per descriptor, a connection `Pool` | +| `internal/ship` | mechanism: `Assembler` (chunks → lines), `Record` (the format), `Batcher`, `Pipeline` | +| `internal/emit` | sink: one line per record on stdout, shared by every stream, plus the `Limits` that keep it collectable | +| `internal/supervise` | lifetimes: a changing set of instances → a running set of pipelines | +| `cmd` | wiring: an instance → its provider, a Pod's labels → a `Record`, one sink for the process. No flags — there is nothing to configure | + +**The provider is the Pod's, not a flag's.** `nebula.inftyai.com/provider` on the Pod's nodeSelector +is what Nebula placed the instance with, and it travels through `supervise.Instance` to the backend +that can read that instance's id — the same field holds a Modal sandbox id or an EC2 instance id, so +handing one to the wrong reader fails every attempt and spends the whole restart budget doing it. A +configured provider would instead have to be kept in agreement with the cluster's node pools, and +being wrong about it looks exactly like an idle cluster. A backend opens when the first Pod on it +arrives, so its credentials are only required if something is actually running there. + +**A pipeline is one stream.** `Pipeline.Run` reads from the source inline and ships from a second +goroutine. Two goroutines rather than one because the source's callback is synchronous — a stalled +callback stalls the cursor — and the interval flush needs a `select`. It flushes every 5 seconds +or when the batch reaches 64 KiB, and calls `Shipped(cursor)` after each successful put. Retrying is +not configured: a write to stdout has no rate to back off from, and the hop that does is the +agent's. + +**One sink, not one per stream.** There is a single stdout, so `emit.Sink` is shared by all 1,000 +pipelines, and that shapes its contract: `Put` must be concurrency-safe, must write +a batch in one `Write` — two interleaved batches are indistinguishable afterwards — and must never be +closed, since one stream ending says nothing about the descriptor the rest are still using. `Put` +also ignores its context on purpose: cancellation is how shutdown reaches the pipelines, and the last +batch of a stopping process is exactly the one that still has to be printed. + +**Backpressure drops, it does not block.** The queue is bounded at 64 KiB and drops the newest +lines on overflow, counting them. Blocking the reader would stall the cursor, and a stalled cursor +risks Modal aging the logs out underneath us — trading a visible loss for a permanent one. + +**A supervisor owns the goroutines.** `Ensure` starts one pipeline per stream the instance's +provider declares, and is idempotent on the instance ID, so a watch re-delivering the same Pod does not start a second +replay. A stream that ends is never restarted; one that fails restarts with capped backoff up to +five times and is then abandoned with a log line, because every restart replays from the last +durable cursor. + +**Its own Go module**, at `components/logship`, so the manager cannot depend on it even by +accident. Own `Makefile`, `Dockerfile` and CI job; the dependency runs one way, via a `replace` on the +repo root for `api/v1alpha1`. + +**Its own gRPC connection to Modal.** `proto/modal_proto` is public, so the request types are ours +to call, and a breaking change there can only reach us when we bump the SDK. The SDK's own +connection cannot be reused (`Client.cpClient` is unexported), and is cheap to replace here: a +`SandboxGetLogs` stream authenticates with five static metadata headers and needs none of the +rotating-JWT machinery the unary interceptors provide. Three consequences: + +- Credentials are env-only (`MODAL_TOKEN_ID`, `MODAL_TOKEN_SECRET`, `MODAL_SERVER_URL`), because + the SDK's profile loader is unexported. A local run against `~/.modal.toml` diverges. +- The SDK is **pinned twice**, so `kubectl logs` and the shipper can compile against different + proto revisions. Harmless with one server, but not obvious. +- The five headers include a client type and version a server-side minimum-version check would + read, so they are reproduced verbatim rather than rebranded. A test asserts the version const + matches the module pin, which is what a dependency bump forgets. + +`SandboxLogs` stays on the SDK rather than being rebuilt on this reader. They are different shapes +— a byte stream for a terminal against an entry stream for a sink — and collapsing them would put +`kubectl logs` behind the raw gRPC surface for no gain. + +**Its own Deployment**, one active replica, the lease still to come. Provisioning is critical and log +shipping is best-effort, so the manager's memory must not become a function of how much a tenant +prints, and shipping a log fix must not bounce the control plane. Two replicas reading one sandbox +would duplicate everything, and no cursor helps when both are live. + +**Draining will be enforced by the shipper's own finalizer**, `logship.nebula.inftyai.com/drain`, added +to the NodeClaim and removed once the sandbox has shipped to `eof`. Finalizers compose, so this +needs no protocol with the manager: each drops its own, and the object survives until both are +gone. Without it every run loses its last lines, which is the part people read. If the shipper is +dead the manager still tears the instance down, so **nothing keeps billing** and only the API +object lingers — but it still needs a bounded escape so a dead shipper does not accumulate objects +forever. + +## The handoff + +The sink prints each record as one line on logship's own stdout. Nothing else happens here. kubelet +writes that line to `/var/log/containers/logship-*.log`, the cluster's Fluent Bit DaemonSet already +tails that glob, and the record reaches +`/aws/containerinsights//application` because logship is a pod on the node — not because +anything was configured for it. + +Two earlier designs were dropped, and both reasons are worth keeping. Direct `PutLogEvents`: the +agent already runs on every node, already holds the credentials and already does batching, retry and +throttle backoff, so handing it the last hop deleted an AWS SDK dependency tree, the throttle-retry +path and the entire IAM grant. A spool under `/var/log/logship/`: **it needed a config change in a +shared, `eks`-managed DaemonSet.** Fluent Bit's `tail` inputs take no recursive `**`, so a spool is +addressed by no default input and needs an `[INPUT]` added to `fluent-bit-config` — a ConfigMap +server-side-applied by the add-on, so `kubectl edit` is not durable and one syntax error stops +collection on every node. Stdout needs neither. + +**The cost is the envelope**, and it is the one thing the handoff is not free of. Our record travels +*through* the agent's `[FILTER] kubernetes` rather than past it, and that filter does two things to +it: + +- It stamps the **emitting** pod's metadata — logship's, not the sandbox's. So the outer + `kubernetes` block is useless for identity, and always was going to be. +- `Merge_Log On` with `Merge_Log_Key log_processed` parses our line as JSON and places the result + under `log_processed` as real nested JSON, not a string. + +So the record arrives intact but one level down: what the consumer used to read at +`$.kubernetes.labels.app` is now at `$.log_processed.kubernetes.labels.app`, while the node's own +logs stay at the top level. Same field names, two depths, and the consumer has to ask for both. + +Four things about that were checked against the live group rather than reasoned about: + +- `Merge_Log` **is** on, and so is `Keep_Log` — a record carries both `log` (the raw string) and + `log_processed` (the parsed object). +- **Filter-pattern wildcards work at depth.** `{ $.log_processed.message = "*build*" }` matches, which + is what keeps the pod-name clause expressible; a three-level path with no matches returns zero + events rather than an error. +- Insights reads the same fields as dotted names with no `$`, so `coalesce(log_processed.…, …)` + covers both roots in one query. Run against the real group, it returns the ordered sandbox pod names + the consumer's discovery query needs. +- **Today's sandbox records already have a `log_processed`** — their own envelope, parsed, with no + `kubernetes` block, because the pods print structured JSON. So a consumer that descended into + `log_processed` unconditionally would break every sandbox log that works now. It has to descend only + when the nested record carries an identity of its own. + +**The handoff is still unacknowledged.** Nothing tells us the agent read a byte, so +`Pipeline.Shipped(cursor)` means "written to stdout", not "in CloudWatch", and the drain finalizer +releases on that weaker signal. What holds unread lines is kubelet's rotation of the container log +(typically 10Mi × 5), which is not ours to size. + +## Event format + +Not ours to choose. The consumer already reads sandbox logs from this group +(`internal/experimentservice/cwlogclient.go`), so the schema is the one it parses: Fluent Bit's +Kubernetes-filter output, which the observability add-on already writes for every node in the +cluster. + +```json +{"time":"","log":"","kubernetes":{"pod_name":"...","labels":{...}}} +``` + +`log` is itself JSON, holding `level`, `category`, `message` and our `id`. **That nesting is load +bearing.** `buildLine` defaults a `log` it cannot parse to category `private`, and `private` is +hidden from every caller without the developer-view role — so a line shipped as bare text lands +durably and is *invisible* in the UI. A wrong-but-valid envelope fails the same way, silently. + +**A line that is already an envelope is adopted, not nested.** `buildLine` unwraps `log` exactly +once, so wrapping a workload's own `{level, category, message}` in ours made ours the only one read: +every sandbox line arrived as `INFO`/`user` with the real envelope stranded inside `message` as text — +missing from a `category=system` query, and rendered in the UI as raw JSON. So `id` is spliced into +the workload's object instead, with `category` added only when it has none, and `level` never, since +the consumer defaults that itself. The bar for adopting is the consumer's own decode rather than +validity: an object it cannot decode into three strings, or one carrying no `message`, is wrapped as +before — adopting it would land it in `private` and hide a line the wrap shows. + +That is what logship writes. What arrives in CloudWatch is that object nested under `log_processed` +inside one of the agent's own, per [The handoff](#the-handoff) — so the same JSON, read one level +deeper. The double encoding survives: `$.log_processed.log` is still a *string* of JSON, because +`Merge_Log` parses only the one level it was handed. + +Identity is in every event, never in a stream name, because the consumer filters on the labels and the pod +name — and because a deleted Pod leaves nothing to join a stream name against anyway. The labels cost +~250 bytes a line and are captured at ship time or not at all. + +Every one of the Pod's labels is copied, not a curated few: that is what Fluent Bit's filter emits for +the node logs already in this group, and it means nothing here has to know a label key. The tenant +triple (the consumer's org, team and experiment IDs) is read only by the consumer, off the +record, so nothing here is configured with a label key at all. + +**All of logship's output shares one CloudWatch stream**, named for logship's own pod. Nothing of +the consumer regresses on that: it already reads with a whole-group `FilterLogEvents` plus an Insights +query for discovery and never used a stream-name prefix. One consequence is worth naming: +`PutLogEvents` allows 5 requests/sec **per stream** and that quota is not raisable, so the whole +fleet's output funnels through one stream's share of it. The agent batches, so this is a throughput +ceiling to watch rather than a known problem. + +Three rules of the format rather than of the API. A split falls on a rune boundary, since half a rune +reaches CloudWatch as a replacement character and corrupts the durable copy; the split is of the +*data*, never of the formatted message, which would cut an envelope in half, and each piece repeats the +`id` so concatenating same-`id` messages reassembles the line. And **a record contains no raw +newline** — one line is one record, so an unescaped `\n` would become two half-records that parse as +neither. `Assembler` produces a `Line` by splitting on newlines in the first place, and +`Record.Formatter` escapes any that a line's data still holds; the two together are what +`emit.Sink.Put` relies on. + +Labels are encoded in sorted order so a stream is byte-identical run to run. + +## Duplicates + +At-least-once, bounded. There is no exactly-once path into CloudWatch: `PutLogEvents` has no +idempotency key and no content de-duplication. + +| situation | duplicates | +| --- | --- | +| running normally | none — the cursor advances in memory | +| clean shutdown | none — the drain ships to `eof` | +| crash, or any restart | full replay of whatever Modal still holds for each live sandbox | +| unrecognized cursor | the same replay, silently — the server returns no error | +| the **agent** restarting without its tail `DB` | re-reads whatever kubelet still holds of our container log | + +The order is read → put → record. Crash between the put and the record and a line ships twice; +reverse it and the line is lost instead. + +Today those duplicates are user-visible: the consumer de-duplicates on CloudWatch's `EventId`, so a +replay of the same text arrives as new IDs and shows up as duplicated lines. The `id` in each +record is what a reader *could* collapse on. Until something does, the cursor checkpoint is what +bounds a restart's blast radius — and the cursor currently lives only in memory, so a restart +replays from `"0-0"`. `Pipeline.Shipped` is the seam a checkpoint plugs into; nothing calls it +yet. + +## Scale + +The target is **500 concurrent instances**, which is **1,000 streams**. Five numbers follow from +it. It is a design target and nothing more: no measurement supports it, and it is deliberately not a +configured limit anywhere — a provider makes room as instances arrive (`modal.Pool` widens), so +exceeding 500 costs connections rather than dropped instances. What has not been established is where the fleet-wide path +actually saturates, and the pool is unlikely to be it; the single mutex-guarded descriptor below and +the one container log the agent tails are the better suspects. + +- **One goroutine per stream, no worker pool.** A pool smaller than the stream count does not + slow the fleet, it starves part of it — and an unread stream is one whose logs age out of Modal. + 2,000 goroutines is ~16 MiB of stacks. Bytes are worth bounding here; goroutines are not. +- **A pool of gRPC connections, not one.** HTTP/2 caps concurrent streams per connection and + gRPC-go *queues* RPCs past the cap rather than failing them, so 1,000 streams on one + `ClientConn` would stall most of them with no error anywhere. 100 streams per connection. +- **The batch threshold is 64 KiB, not the API's 1 MiB.** Per-stream bounds multiply: 1 MiB would + be 1 GiB of pending events across the fleet. Same reason the assembler's `maxFragment` is 64 KiB + — two such bounds per stream puts the fleet near 128 MiB, which is a Deployment that can be + sized. +- **200 write syscalls per second** at a 5-second interval, one per stream per flush, all of them on + one mutex-guarded descriptor. The interval is a latency-versus-syscalls choice, not a fleet-size + one; the mutex is held only for the `Write`, with the batch formatted outside it. +- **No files and no disk bound at all.** kubelet rotates our + container log and the node was already sized for that; a chatty fleet costs pipe backpressure, + which the queue's 64 KiB bound turns into counted drops rather than a stalled reader. + +The other cost is the envelope: ~250 bytes per line plus CloudWatch's own 26, so ingest is +dominated by metadata for short lines — a 40-byte print costs ~320 bytes. Worth measuring against +real verbosity before quoting a monthly number. + +**The per-event cap is now 16 KiB, not CloudWatch's 256.** The binding constraint moved: the +container runtime reads a container's stdout in 16 KiB pieces and writes each as its own partial- +flagged line, leaving the agent's `cri` multiline parser to glue them back. Staying inside one chunk +means that rejoin never runs — and a rejoin that failed would deliver two halves of a JSON object, +which the consumer drops without reporting. Splitting here is cheap; getting that wrong is silent. The +**26 bytes of per-event overhead** are still reserved, both because CloudWatch still charges them and +because they double as headroom under the chunk for the runtime's own line prefix. + +With no request to size, `MaxBytes` is only a memory-and-syscall bound. The batcher clamps a backwards +timestamp rather than sorting, which keeps the output in the order the lines were printed. + +## Failure modes + +| failure | consequence | +| --- | --- | +| shipper down longer than Modal retention | logs permanently lost; the reason this exists | +| unrecognized cursor | silent full replay — the server returns no error | +| an empty `eof` entry ID reaching the cursor | replays the whole sandbox — guarded in the reader | +| sink saturated | dropped events plus a counter, never a stalled reader | +| finalizer removed before drain | the tail of that run is lost | +| a stream failing five times | abandoned and logged; that instance's remaining logs are not copied | +| a diagnostic printed to stdout instead of stderr | it ships as a record; unparseable, so it lands nowhere the consumer looks | +| `Merge_Log` turned off in the agent's config | our record becomes a *string* under `log`; every consumer clause misses | +| the consumer's filter losing the `log_processed` depth | **nothing matches, and nothing errors** | +| a record over 16 KiB reaching the runtime unsplit | split into partial chunks; correct only if the `cri` multiline parser rejoins them | +| node dies with lines kubelet has not been read out of | those lines are lost; the finalizer already released on the write | +| stdout pipe full | the queue drops newest lines and counts them; the reader keeps draining the provider | +| a Pod on a provider this build has no backend for | logged and skipped, once per event; nothing of that instance ships | +| a Pod re-provisioned onto a new instance | the replaced instance is forgotten on the update that rewrote the annotation, losing its unshipped tail; leaving it running would ship two instances under one pod name | + +## What is left + +1. **`internal/drain`** — add and remove the finalizer, with the bounded escape and a startup + sweep for stale ones. The first thing here to need `api/v1alpha1`. +2. **Leader election and metrics** — the Deployment and its RBAC exist; the lease does not. At + `replicas: 1` with `Recreate` a single reader is still enforced by the shape, which now means the + one replica is the whole fleet's collection. Metrics over `supervise.Stats` are still nothing but + a line on stderr at shutdown. No volumes and no agent-side prerequisite: being a pod is the whole + integration. +3. **The end-to-end check.** Nothing in the repo proves a record survives the agent's filter chain, + and every way it can fail is silent. It was confirmed by hand against a live cluster, which is not + a check anything re-runs. Whatever replaces it has to assert the pre-fix query returns *nothing* + as well: a pattern selecting the wrong path is otherwise indistinguishable from an empty window. +4. **The cursor checkpoint** — the one item here that costs money rather than confidence. Both halves + of the seam exist and neither is wired: `Pipeline.Shipped` is called with each accepted batch's + last cursor and nothing listens; `fleet.cursor` is a `func(inst, stream) string` nothing assigns, + so every stream resumes from `"0-0"`. Persisting per *(instance, stream)* — never per instance, + since the two streams' IDs are independent and one handed to the other replays it silently — takes + a restart from re-shipping whatever the provider still holds down to what was printed since the + last flush. Two things must hold: the order stays read → put → record (reversed, a crash loses + lines instead of repeating them), and the final flush on SIGTERM cannot use the signal's own + context, which is already cancelled by the time `shutdown` runs — it would fail instantly while + the clean path looked fine. + +## Permissions + +**Kubernetes, and nothing else.** Its own ServiceAccount and ClusterRole, narrower than the manager's: +`get`/`list`/`watch` on Pods, read-only, no status writes, no CRDs — cluster-scoped only because +instance Pods live one namespace per org. The drain finalizer will add NodeClaims and +`patch` on `nodeclaims/finalizers`, leader election `coordination.k8s.io` leases, and the checkpoint +whatever it stores cursors in; none of the three exists yet, so none is granted. + +**AWS and the node: none, structurally.** Writing to your own stdout is the least privilege a +container has: no credentials to mount, no hostPath a path bug could point at another tenant's +collector, and no CloudWatch client at all — so a compromised shipper cannot read back what it has +written. That is worth keeping in mind before adding a read of the destination, which is the one thing +that would undo it (see [Open decisions](#open-decisions)). + +## Open decisions + +Two things are settled and not by us: the log group is +`/aws/containerinsights//application`, the one the consumer is already configured with, +because a second group would be a second query no consumer makes; and retention is Terraform's, at +365 days. + +- **Where the resume point comes from.** Deferred, with the options costed. A local checkpoint is the + cheap one: a single ConfigMap in logship's own namespace holding the whole fleet's cursors is one + write per flush regardless of fleet size, needs `get`/`update` on one named object, and works in + kind. Per-Pod or per-NodeClaim annotations are the same idea at 500× the writes, each bumping a + `resourceVersion` that Nebula's own controllers watch, and the Pod variant also turns a read-only + watcher into something that patches tenant objects. + + Reading it back out of CloudWatch instead is more attractive than it sounds, because **the cursor is + already in every durable record**: `Record.Formatter` ships `Line.Cursor` as the `id` inside `log`, + so one Insights query (`stats latest(...) by pod`, descending — `FilterLogEvents` only pages + forward) reconstructs the whole fleet's resume table with nothing stored anywhere. It is also the + only option that notices the loss above: a local checkpoint records what we *printed*, so lines + kubelet never handed over are gone silently, while the destination's own last line re-ships them. + + What it costs is why it is not the default. The record carries no stream marker — `fleet.build` sets + no `Level` — so `latest` by pod alone cannot tell stdout from stderr, and guessing wrong is the + silent full replay. It reintroduces an AWS client, IRSA and the group name into a component whose + whole permission story is *none* (see [Permissions](#permissions)), which also means every start in + a cluster without that group replays. The query has to regex into `log_processed.log`, still a + *string* of JSON, whose escaping is the consumer's to change. And it needs a time window: too narrow + after a long outage returns no rows, which reads as no prior state. If it is ever built, the shape + that keeps the properties without the coupling is a verification step outside the Deployment — + "did the agent deliver everything we printed?", which nothing can answer today. +- **The 16 KiB per-event cap.** Chosen to stay inside one CRI chunk, which is a mechanism rather + than a measurement: the agent's `cri` multiline parser probably does rejoin a longer record + correctly, and if a probe shows it does, this can go back up toward CloudWatch's 256 KiB and split + fewer long lines. Erring small is the direction where being wrong is visible. +- **A slash-free identity field in the record.** the consumer filters the experiment out of the pod name + with a wildcard because a CloudWatch filter pattern cannot select a key containing `/` — confirmed + against the live group, so the experiment ID label is only usable client-side. A top-level + `experiment_id` would make that narrowing server-side and decouple the query from pod naming. Still + a two-repo change, and now a deeper one: it would sit at `$.log_processed.experiment_id`. +- **Whether to collapse progress-bar frames by default.** Implemented either way; the assembler + takes it as a flag. Two arguments point the same way: a `tqdm` bar emits a carriage-returned + frame per update and ingest is billed per GB, and since a bar emits no newline until it + finishes, an un-collapsed one is a pending line that grows for the length of the run. That + second half is what forced `maxFragment`, now load-bearing for any source that never terminates + a line. What collapsing costs is the timing of a run's progress. + +## Known gaps + +**Modal retention is not discoverable at runtime.** `TaskLogsBatch.ttl_days` exists in the proto +and came back `0` on every batch — unset for sandbox logs, and read by no client, Go or Python. +Retention has to be configuration we are told. Which plan the workspace is on therefore matters: 1 +day makes an outage lossy in hours. + +**Backfill is capped at 14 days.** `PutLogEvents` rejects events older than that, so on a 30-day +Team plan the older half of Modal's retention cannot be copied after the fact by this path. Live +shipping is unaffected. The rejection happens inside the agent, whose counters are not ours to read, +so a backfill that silently drops its older half has nowhere here to show up. diff --git a/components/logship/go.mod b/components/logship/go.mod new file mode 100644 index 0000000..01bbae7 --- /dev/null +++ b/components/logship/go.mod @@ -0,0 +1,57 @@ +module github.com/InftyAI/Nebula/components/logship + +go 1.25.0 + +require ( + github.com/modal-labs/modal-client/go v0.9.0 + google.golang.org/grpc v1.78.0 + k8s.io/api v0.33.4 + k8s.io/apimachinery v0.33.4 + k8s.io/client-go v0.33.4 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/net v0.50.0 // indirect + golang.org/x/oauth2 v0.32.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/term v0.40.0 // indirect + golang.org/x/text v0.34.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) + +// The API types are the one thing this borrows from the controller, for the NodeClaim watch. A +// replace rather than a version so an API change breaks this build in the same PR; see +// components/README.md. +replace github.com/InftyAI/Nebula => ../.. diff --git a/components/logship/go.sum b/components/logship/go.sum new file mode 100644 index 0000000..d659bea --- /dev/null +++ b/components/logship/go.sum @@ -0,0 +1,185 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY= +github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modal-labs/modal-client/go v0.9.0 h1:tZ7JFCRzNRrfC3RA6e+4V/KwtGEliEOzEfJ9Xvb+4TM= +github.com/modal-labs/modal-client/go v0.9.0/go.mod h1:dMBMTXVQ6ReY/XFXAB3Tic0IuGtC755UoHbDcqkCgcc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= +golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= +golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= +golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.33.4 h1:oTzrFVNPXBjMu0IlpA2eDDIU49jsuEorGHB4cvKupkk= +k8s.io/api v0.33.4/go.mod h1:VHQZ4cuxQ9sCUMESJV5+Fe8bGnqAARZ08tSTdHWfeAc= +k8s.io/apimachinery v0.33.4 h1:SOf/JW33TP0eppJMkIgQ+L6atlDiP/090oaX0y9pd9s= +k8s.io/apimachinery v0.33.4/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/client-go v0.33.4 h1:TNH+CSu8EmXfitntjUPwaKVPN0AYMbc9F1bBS8/ABpw= +k8s.io/client-go v0.33.4/go.mod h1:LsA0+hBG2DPwovjd931L/AoaezMPX9CmBgyVyBZmbCY= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/components/logship/hack/deploy.sh b/components/logship/hack/deploy.sh new file mode 100755 index 0000000..eaff646 --- /dev/null +++ b/components/logship/hack/deploy.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# +# deploy.sh — run logship in the cluster. +# +# One deployment for the whole cluster, and it finds its own work: it lists Pods labelled +# nebula.inftyai.com/enabled=true, reads which provider each was placed on off its nodeSelector, and +# ships its instance through that one. No instance, pod or labels to pass — see "the metadata is the +# product". The Modal secret below is one provider's; it is only read if the cluster has Modal Pods. +# +# Usage: +# hack/deploy.sh # or: make deploy +# make docker-build docker-push deploy IMG=inftyai/nebula-logship:0906-01 +# +# Inputs (environment or make flags): +# IMG image to run (default inftyai/nebula-logship:latest) +# DEPLOY_NS namespace to run in (default nebula-system) +# NAME Pod name (default logship) +# FOLLOW 0 to skip tailing the Pod's output afterwards +# +# A Deployment, so a drained or lost node reschedules instead of ending collection for the +# rest of the run — silently, since nothing downstream can tell "no logs" from "no output". +# +# But strictly ONE reader, enforced twice: replicas 1, and strategy Recreate. RollingUpdate +# is the trap — its default maxSurge starts the new pod before the old one goes, and for the +# seconds they overlap both read the same instance from the same cursor and every line is +# shipped, stored and billed twice. There is no leader election to fall back on. +# +# Restarts DO replay until the checkpoint lands (phase 3): a fresh process resumes from the +# cursor it was given, which is the beginning, so the provider re-sends the whole history. +# In-process this is already bounded — supervise caps restarts at DefaultMaxRestarts for +# exactly this reason — but a container restart escapes that bound, and CrashLoopBackOff is +# then a billing loop, not just an outage. Watch RESTARTS on a long run. +# +# Why the deploy namespace matters: with the stdout handoff, logship's records reach +# CloudWatch through the log agent of the cluster it runs in. The default is nebula-system +# because that is where Nebula's manager already runs in the dev cluster, and the +# manager's own container log is a live stream in the group the consumer reads — so collection +# there is verified, not assumed. It is also where nebula-modal-credentials lives, and a +# Secret cannot be referenced across namespaces. Moving this Pod elsewhere means checking +# both again. +# +# The metadata is the product: logship stamps the instance Pod's name and labels onto each +# record because the log agent stamps ITS OWN identity, not the instance's. The consumer then +# filters on the app label, the consumer's org/team/experiment IDs and pod_name, so a record +# shipped with the wrong ones is delivered, charged for, and invisible in the UI — no error +# anywhere, just an empty log view. Which is why none of it can be passed in here: it is read +# off the Pod that owns the instance, one object, so the parts cannot disagree. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +log() { printf '\033[36m==>\033[0m %s\n' "$*"; } +die() { printf '\033[31mERROR:\033[0m %s\n' "$*" >&2; exit 1; } + +NAME="${NAME:-logship}" +IMG="${IMG:-inftyai/nebula-logship:latest}" +DEPLOY_NS="${DEPLOY_NS:-nebula-system}" +MODAL_SECRET="${MODAL_SECRET:-nebula-modal-credentials}" +FOLLOW="${FOLLOW:-1}" +KUBECTL="${KUBECTL:-kubectl}" + +command -v "${KUBECTL}" >/dev/null 2>&1 || die "kubectl not found on PATH" + +# Unreachable and absent are separated on purpose: this cluster is reached through an SSM +# tunnel, and when it is down every kubectl fails as "not found", which reads like a missing +# namespace and sends you looking in the wrong place. +"${KUBECTL}" version --request-timeout=10s >/dev/null 2>&1 \ + || die "cannot reach the cluster ($("${KUBECTL}" config current-context 2>/dev/null || echo 'no context')) — tunnel down?" +"${KUBECTL}" get namespace "${DEPLOY_NS}" >/dev/null 2>&1 \ + || die "namespace ${DEPLOY_NS} not found — is this the cluster whose agent ships to the group the consumer reads?" +# A warning rather than fatal: a provider is only opened when a Pod on it arrives, so a cluster with +# no Modal Pods needs no Modal tokens. If some do arrive, the process says so once per Pod. +"${KUBECTL}" -n "${DEPLOY_NS}" get secret "${MODAL_SECRET}" >/dev/null 2>&1 \ + || log "WARNING: Secret ${MODAL_SECRET} not found in ${DEPLOY_NS} — Modal instances will not ship (other providers are unaffected)" + +log "granting ${NAME} read access to Pods" +# Cluster-scoped because instance Pods live in one namespace per org, and the set of orgs is not +# known here. Read-only and Pods-only: the watcher writes nothing, and the drain finalizer that +# will need NodeClaims does not exist yet. +"${KUBECTL}" apply -f - <&2 < --filter-pattern '{ \$.log_processed.kubernetes.pod_name = "*" }' + 4. no restarts (each one replays) ${KUBECTL} -n ${DEPLOY_NS} get pods -l app.kubernetes.io/instance=${NAME} + 5. tear down ${KUBECTL} -n ${DEPLOY_NS} delete deployment/${NAME} clusterrolebinding/${NAME} clusterrole/${NAME} sa/${NAME} + +EOF diff --git a/components/logship/internal/emit/emit.go b/components/logship/internal/emit/emit.go new file mode 100644 index 0000000..d6b71ab --- /dev/null +++ b/components/logship/internal/emit/emit.go @@ -0,0 +1,120 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package emit ships a record by printing it: one line on the process's own stdout, and nothing else. +// +// The destination is then whatever already collects container logs — here, the cluster's shared +// Fluent Bit DaemonSet, which this repo neither configures nor may change. It tails +// /var/log/containers/*.log, so logship is collected by virtue of being a pod. No spool, no glob to +// keep in sync with a config we do not own, no AWS credentials, and kubelet's rotation of that file +// is the buffer a restart reads back. +// +// What it costs is the envelope. The agent's `kubernetes` filter wraps our record in one of its own, +// stamped with logship's pod rather than the sandbox's, and `Merge_Log On` parses ours into a nested +// object under `log_processed`. So the identity the consumer filters on sits one level down, at +// $.log_processed.kubernetes.labels — see design.md and ship.Record. +package emit + +import ( + "bytes" + "context" + "io" + "sync" + + "github.com/InftyAI/Nebula/components/logship/internal/ship" +) + +const ( + // maxEventBytes keeps a record inside a single CRI chunk. The container runtime reads a + // container's stdout in 16 KiB pieces and writes each as its own line in the container log, + // flagged partial, leaving Fluent Bit's cri multiline parser to glue them back together. Staying + // under the chunk means that rejoin never runs: a rejoin that failed would deliver two halves of + // a JSON object, and the consumer drops an unparseable record without reporting it. + maxEventBytes = 16 << 10 + + // perEventOverhead is CloudWatch's 26 bytes per event, and it is still ours to reserve even + // though we no longer call PutLogEvents — the agent turns one line into one event and has no way + // to split what we hand it. Here it doubles as headroom under maxEventBytes for the timestamp and + // stream prefix the runtime puts in front of every line. + perEventOverhead = 26 + + // DefaultBatchBytes bounds one Write, not one request; there is no request. Batching survives + // only to keep the syscall off the per-line path. + DefaultBatchBytes = 64 << 10 +) + +// Limits are the caps a ship.Batcher must respect for a record to survive the trip. batchBytes <= 0 +// means DefaultBatchBytes. +// +// MaxEvents stays unset: the 10,000-per-request cap belonged to the API call we no longer make. +func Limits(batchBytes int) ship.Limits { + if batchBytes <= 0 { + batchBytes = DefaultBatchBytes + } + return ship.Limits{ + MaxBytes: batchBytes, + MaxEventBytes: maxEventBytes, + PerEventOverhead: perEventOverhead, + } +} + +// Sink writes each event as one line on w. +// +// One Sink serves every stream in the process, which is the difference from the per-stream sinks that +// came before it: stdout is one file descriptor, and two batches interleaved on it corrupt both. The +// mutex is what makes that sharing safe, and it is why a batch is assembled into a single buffer and +// written once rather than a line at a time. +type Sink struct { + w io.Writer + + mu sync.Mutex +} + +func New(w io.Writer) *Sink { return &Sink{w: w} } + +// Put writes the batch. Every message must be newline-free or one record becomes two lines and +// neither parses — which ship.Assembler guarantees, since it splits on newlines to produce a Line at +// all, and Record.Formatter escapes any that a line's data still holds. +// +// ctx is ignored on purpose. Cancellation is how shutdown reaches the pipelines, and the last batch +// of a stopping process is precisely the one that still has to be printed; there is also nothing to +// cancel, since a write to stdout either completes or fails. +func (s *Sink) Put(_ context.Context, events []ship.Event) error { + if len(events) == 0 { + return nil + } + + // Built outside the lock: at fleet scale a thousand streams contend for it, and formatting is the + // part that does not have to be serialised. + n := len(events) + for _, e := range events { + n += len(e.Message) + } + var b bytes.Buffer + b.Grow(n) + for _, e := range events { + b.WriteString(e.Message) + b.WriteByte('\n') + } + + s.mu.Lock() + defer s.mu.Unlock() + n, err := s.w.Write(b.Bytes()) + if err == nil && n != b.Len() { + return io.ErrShortWrite + } + return err +} diff --git a/components/logship/internal/emit/emit_test.go b/components/logship/internal/emit/emit_test.go new file mode 100644 index 0000000..4c2d710 --- /dev/null +++ b/components/logship/internal/emit/emit_test.go @@ -0,0 +1,204 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package emit + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/InftyAI/Nebula/components/logship/internal/ship" +) + +func events(msgs ...string) []ship.Event { + out := make([]ship.Event, 0, len(msgs)) + for _, m := range msgs { + out = append(out, ship.Event{Message: m, At: time.Unix(0, 0)}) + } + return out +} + +func put(t *testing.T, msgs ...string) string { + t.Helper() + var buf bytes.Buffer + if err := New(&buf).Put(t.Context(), events(msgs...)); err != nil { + t.Fatalf("Put: %v", err) + } + return buf.String() +} + +func TestPutWritesOneNewlineTerminatedLinePerEvent(t *testing.T) { + if got := put(t, `{"a":1}`, `{"b":2}`); got != `{"a":1}`+"\n"+`{"b":2}`+"\n" { + t.Fatalf("stdout = %q", got) + } +} + +// A record has to reach the agent byte for byte: the agent re-emits the line as it found it, so +// anything this layer adds or escapes lands in the consumer's parser. +func TestPutDoesNotRewriteTheRecord(t *testing.T) { + const msg = `{"time":"t","log":"{\"category\":\"user\"}","kubernetes":{"pod_name":"p"}}` + if got := strings.TrimSuffix(put(t, msg), "\n"); got != msg { + t.Fatalf("record changed in transit:\n got %s\nwant %s", got, msg) + } +} + +func TestPutOfNothingWritesNothing(t *testing.T) { + var buf bytes.Buffer + if err := New(&buf).Put(t.Context(), nil); err != nil { + t.Fatalf("Put(nil): %v", err) + } + if buf.Len() != 0 { + t.Fatalf("empty Put wrote %q", buf.String()) + } +} + +// Cancellation is how Shutdown reaches the pipelines, and Pipeline.Run drains what it has already read +// afterwards. A sink that honoured ctx would discard exactly the tail of a finishing sandbox. +func TestPutIgnoresACancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + var buf bytes.Buffer + if err := New(&buf).Put(ctx, events("last line")); err != nil { + t.Fatalf("Put on a cancelled ctx: %v", err) + } + if buf.String() != "last line\n" { + t.Fatalf("stdout = %q", buf.String()) + } +} + +// Every stream in the process shares one sink and one file descriptor. A write that interleaved would +// not lose a record, it would splice two together — an unparseable line the consumer drops in silence. +func TestConcurrentPutsDoNotInterleave(t *testing.T) { + const streams, batches = 16, 32 + + // A writer that keeps each Write separate, so "the batch arrived in one piece" is checkable at all + // — a plain buffer would hide it. + w := &chunkWriter{} + s := New(w) + + var wg sync.WaitGroup + for stream := range streams { + wg.Add(1) + go func() { + defer wg.Done() + for i := range batches { + // Two events per Put: a batch is the unit that must not be broken up, not a line. + msgs := []string{ + fmt.Sprintf(`{"stream":%d,"n":%d,"half":"a"}`, stream, i), + fmt.Sprintf(`{"stream":%d,"n":%d,"half":"b"}`, stream, i), + } + if err := s.Put(t.Context(), events(msgs...)); err != nil { + t.Errorf("Put: %v", err) + return + } + } + }() + } + wg.Wait() + + if len(w.chunks) != streams*batches { + t.Fatalf("%d writes, want one per batch (%d)", len(w.chunks), streams*batches) + } + seen := map[string]bool{} + for _, chunk := range w.chunks { + lines := strings.Split(strings.TrimSuffix(chunk, "\n"), "\n") + if len(lines) != 2 { + t.Fatalf("a batch reached the writer as %d lines: %q", len(lines), chunk) + } + for _, l := range lines { + if !strings.HasPrefix(l, `{"stream":`) || !strings.HasSuffix(l, `"}`) { + t.Fatalf("spliced line: %q", l) + } + if seen[l] { + t.Fatalf("duplicated line: %q", l) + } + seen[l] = true + } + } + if len(seen) != streams*batches*2 { + t.Fatalf("%d distinct lines, want %d", len(seen), streams*batches*2) + } +} + +// The batcher splits a line by MaxEventBytes and nothing downstream splits again, so this cap is the +// only thing standing between a long line and the multiline rejoin the agent would otherwise have to +// get right. See maxEventBytes. +func TestLimitsKeepARecordInsideOneCRIChunk(t *testing.T) { + got := Limits(0) + if got.MaxEventBytes != maxEventBytes || got.MaxEventBytes > 16<<10 { + t.Fatalf("MaxEventBytes = %d, want at most one 16 KiB chunk", got.MaxEventBytes) + } + if got.PerEventOverhead != perEventOverhead { + t.Fatalf("PerEventOverhead = %d, want CloudWatch's per-event charge reserved", got.PerEventOverhead) + } + if got.MaxBytes != DefaultBatchBytes { + t.Fatalf("MaxBytes = %d, want the default write bound", got.MaxBytes) + } + if got.MaxEvents != 0 { + t.Fatalf("MaxEvents = %d, want unset", got.MaxEvents) + } + if n := Limits(1234).MaxBytes; n != 1234 { + t.Fatalf("Limits(1234).MaxBytes = %d", n) + } +} + +// One line per record is the contract, and a record's own text is arbitrary bytes. The escaping that +// keeps the two compatible lives in ship, so this is the check that the two packages still agree. +func TestARecordWithAnEmbeddedNewlineStaysOneLine(t *testing.T) { + format := ship.Record{Pod: "p"}.Formatter() + msg := format(ship.Line{Data: "first\nsecond\r\ttab", At: time.Unix(0, 0), Cursor: "1-0"}) + + body := put(t, msg) + if n := strings.Count(body, "\n"); n != 1 { + t.Fatalf("%d newlines in one record's output, want the terminator only:\n%s", n, body) + } +} + +func TestSinkIsAShipSink(t *testing.T) { + var _ ship.Sink = (*Sink)(nil) + + // And it must NOT be a Closer. Pipeline used to close a sink at the end of its stream; one stream + // ending must not close stdout for the thousand still running. + if _, ok := any(New(&bytes.Buffer{})).(interface{ Close() error }); ok { + t.Fatal("*Sink implements Close, which a shared stdout must not") + } +} + +func TestPutReportsAWriteFailure(t *testing.T) { + want := errors.New("broken pipe") + if err := New(failWriter{want}).Put(t.Context(), events("x")); !errors.Is(err, want) { + t.Fatalf("Put = %v, want %v", err, want) + } +} + +// chunkWriter records each Write as it arrived, without a lock of its own. +type chunkWriter struct{ chunks []string } + +func (w *chunkWriter) Write(p []byte) (int, error) { + w.chunks = append(w.chunks, string(p)) + return len(p), nil +} + +type failWriter struct{ err error } + +func (w failWriter) Write([]byte) (int, error) { return 0, w.err } diff --git a/components/logship/internal/modal/client.go b/components/logship/internal/modal/client.go new file mode 100644 index 0000000..9e8e47c --- /dev/null +++ b/components/logship/internal/modal/client.go @@ -0,0 +1,143 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modal + +import ( + "context" + "crypto/tls" + "fmt" + "os" + "strconv" + "strings" + "time" + + pb "github.com/modal-labs/modal-client/go/proto/modal_proto" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/metadata" +) + +// sdkVersion is the modal-client version this mimics in the headers below. Kept in sync with +// go.mod by TestSDKVersionMatchesModulePin rather than by remembering. +const sdkVersion = "0.9.0" + +const defaultServerURL = "https://api.modal.com:443" + +// keepalive matches the SDK's. A log stream is idle for its whole 55-second window whenever the +// sandbox is quiet, so PermitWithoutStream is not optional decoration here. +var keepaliveParams = keepalive.ClientParameters{ + Time: 30 * time.Second, + Timeout: 10 * time.Second, + PermitWithoutStream: true, +} + +// LogsClient is the one RPC this package needs off pb.ModalClientClient. +// +// Narrowed so a test can supply a fake without implementing several hundred methods, and so the +// blast radius of talking to the raw proto is one line. +type LogsClient interface { + SandboxGetLogs(context.Context, *pb.SandboxGetLogsRequest, ...grpc.CallOption) (grpc.ServerStreamingClient[pb.TaskLogsBatch], error) +} + +// Credentials are the Modal tokens, and the server to send them to. +type Credentials struct { + TokenID string + TokenSecret string + ServerURL string +} + +// CredentialsFromEnv reads the tokens the SDK would read, minus the profile file. +// +// Env-only on purpose: the SDK's loader also merges ~/.modal.toml, and it is unexported, so there +// is nothing to reuse. In-cluster the tokens arrive as a Secret, which is env either way — what +// diverges is a local run against a toml profile, which has to export the two variables. +func CredentialsFromEnv() (Credentials, error) { + c := Credentials{ + TokenID: os.Getenv("MODAL_TOKEN_ID"), + TokenSecret: os.Getenv("MODAL_TOKEN_SECRET"), + ServerURL: os.Getenv("MODAL_SERVER_URL"), + } + if c.TokenID == "" || c.TokenSecret == "" { + return Credentials{}, fmt.Errorf("MODAL_TOKEN_ID and MODAL_TOKEN_SECRET must be set") + } + if c.ServerURL == "" { + c.ServerURL = defaultServerURL + } + return c, nil +} + +// Dial opens a control-plane connection for reading logs. The caller closes the conn. +// +// Its own connection because the SDK's is unexported (Client.cpClient, with no accessor), and +// cheap to replace for this one RPC: the SDK installs four unary interceptors but exactly ONE +// stream interceptor, and all that one does is attach the static headers below. The rotating-JWT +// machinery is unary-only, so a SandboxGetLogs stream needs none of it. +func Dial(creds Credentials) (*grpc.ClientConn, LogsClient, error) { + target, transport, err := dialTarget(creds.ServerURL) + if err != nil { + return nil, nil, err + } + conn, err := grpc.NewClient(target, + grpc.WithTransportCredentials(transport), + grpc.WithKeepaliveParams(keepaliveParams), + grpc.WithUnaryInterceptor(headerInjectorUnary(creds)), + grpc.WithStreamInterceptor(headerInjectorStream(creds)), + ) + if err != nil { + return nil, nil, fmt.Errorf("dialing modal at %s: %w", creds.ServerURL, err) + } + return conn, pb.NewModalClientClient(conn), nil +} + +func dialTarget(serverURL string) (string, credentials.TransportCredentials, error) { + if after, ok := strings.CutPrefix(serverURL, "https://"); ok { + return after, credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12}), nil + } + // Plaintext is how the SDK reaches a local test server, and the only reason it is allowed. + if after, ok := strings.CutPrefix(serverURL, "http://"); ok { + return after, insecure.NewCredentials(), nil + } + return "", nil, fmt.Errorf("invalid MODAL_SERVER_URL %q: want an http:// or https:// prefix", serverURL) +} + +// authHeaders are the five the SDK sends, reproduced rather than invented: the client type and +// version are what a server-side minimum-version check would read, so identifying ourselves here +// instead risks a rejection we would have no way to diagnose. They pin to the SDK we vendor the +// proto from, and move when it does. +func authHeaders(creds Credentials) []string { + return []string{ + "x-modal-client-type", strconv.Itoa(int(pb.ClientType_CLIENT_TYPE_LIBMODAL_GO)), + "x-modal-client-version", "1.0.0", + "x-modal-libmodal-version", "modal-go/" + sdkVersion, + "x-modal-token-id", creds.TokenID, + "x-modal-token-secret", creds.TokenSecret, + } +} + +func headerInjectorUnary(creds Credentials) grpc.UnaryClientInterceptor { + return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + return invoker(metadata.AppendToOutgoingContext(ctx, authHeaders(creds)...), method, req, reply, cc, opts...) + } +} + +func headerInjectorStream(creds Credentials) grpc.StreamClientInterceptor { + return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + return streamer(metadata.AppendToOutgoingContext(ctx, authHeaders(creds)...), desc, cc, method, opts...) + } +} diff --git a/components/logship/internal/modal/client_test.go b/components/logship/internal/modal/client_test.go new file mode 100644 index 0000000..329f672 --- /dev/null +++ b/components/logship/internal/modal/client_test.go @@ -0,0 +1,82 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modal + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// TestSDKVersionMatchesModulePin is what keeps the version we CLAIM to be honest. The header value +// is a const so nothing has to read build info at runtime, and a const is exactly the kind of +// thing a dependency bump forgets. +// +// Read out of go.mod rather than debug.ReadBuildInfo, which reports no deps at all under go test. +func TestSDKVersionMatchesModulePin(t *testing.T) { + const module = "github.com/modal-labs/modal-client/go" + + gomod, err := os.ReadFile(filepath.Join("..", "..", "go.mod")) + if err != nil { + t.Fatalf("reading go.mod: %v", err) + } + for line := range strings.Lines(string(gomod)) { + fields := strings.Fields(line) + if len(fields) < 2 || fields[0] != module { + continue + } + if want := strings.TrimPrefix(fields[1], "v"); want != sdkVersion { + t.Fatalf("sdkVersion = %q, but go.mod pins %s at %q", sdkVersion, module, want) + } + return + } + t.Fatalf("%s is not required by go.mod", module) +} + +func TestCredentialsFromEnv(t *testing.T) { + t.Run("defaults the server URL", func(t *testing.T) { + t.Setenv("MODAL_TOKEN_ID", "ak-1") + t.Setenv("MODAL_TOKEN_SECRET", "as-1") + t.Setenv("MODAL_SERVER_URL", "") + + creds, err := CredentialsFromEnv() + if err != nil { + t.Fatalf("CredentialsFromEnv: %v", err) + } + if creds.ServerURL != defaultServerURL { + t.Fatalf("ServerURL = %q, want the default", creds.ServerURL) + } + }) + + // A missing token has to fail HERE rather than as an opaque Unauthenticated on the first + // stream, because the tokens are the one thing this cannot fall back to a profile file for. + t.Run("refuses a missing token", func(t *testing.T) { + t.Setenv("MODAL_TOKEN_ID", "ak-1") + t.Setenv("MODAL_TOKEN_SECRET", "") + + if _, err := CredentialsFromEnv(); err == nil { + t.Fatal("CredentialsFromEnv accepted a missing token secret") + } + }) +} + +func TestDialTargetRejectsASchemelessURL(t *testing.T) { + if _, _, err := dialTarget("api.modal.com:443"); err == nil { + t.Fatal("dialTarget accepted a URL with no scheme") + } +} diff --git a/components/logship/internal/modal/fake_server_test.go b/components/logship/internal/modal/fake_server_test.go new file mode 100644 index 0000000..c14bd21 --- /dev/null +++ b/components/logship/internal/modal/fake_server_test.go @@ -0,0 +1,147 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modal + +import ( + "context" + "net" + "testing" + + pb "github.com/modal-labs/modal-client/go/proto/modal_proto" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/test/bufconn" +) + +// A real gRPC server over bufconn rather than a hand-rolled LogsClient, because the generated +// server stub is right there — and a fake client would skip the dial path and the metadata +// headers, which is exactly where a wrong header shows up as an auth failure in production. + +// batch is one scripted server response. Exactly one of lines/markers/fail/hold is usual. +type batch struct { + id string + lines []string + markers int // items carrying a task state and no data + eof bool + // closeWindow ends the stream cleanly after this batch, without eof — what the server's + // 55-second timeout looks like to a client. + closeWindow bool + // fail aborts the stream with this error, once. A retry gets the batches after it. + fail error + // hold keeps the stream open indefinitely, like a sandbox that is still running. + hold bool + timestampNs uint64 +} + +type fakeServer struct { + pb.UnimplementedModalClientServer + history []batch + // opened counts streams, and cursors records the LastEntryId each was opened with, so a test + // can assert what a resume actually asked for. + opened int + cursors []string + fired map[int]bool +} + +func (f *fakeServer) SandboxGetLogs(req *pb.SandboxGetLogsRequest, stream grpc.ServerStreamingServer[pb.TaskLogsBatch]) error { + f.opened++ + f.cursors = append(f.cursors, req.GetLastEntryId()) + + for i := f.startIndex(req.GetLastEntryId()); i < len(f.history); i++ { + b := f.history[i] + if b.fail != nil { + if f.fired[i] { + continue + } + f.fired[i] = true + return b.fail + } + if b.hold { + <-stream.Context().Done() + return stream.Context().Err() + } + if err := stream.Send(toBatch(b, req.GetFileDescriptor())); err != nil { + return err + } + if b.eof || b.closeWindow { + return nil + } + } + // Past the end of the script: the sandbox is finished, which the server says with an eof batch + // carrying no entry ID of its own. + return stream.Send(pb.TaskLogsBatch_builder{Eof: true}.Build()) +} + +// startIndex resolves a cursor to the batch after it. An unrecognized one replays from the +// beginning, with no error, which is what Modal really does. +func (f *fakeServer) startIndex(cursor string) int { + if cursor == CursorStart { + return 0 + } + for i, b := range f.history { + if b.id != "" && b.id == cursor { + return i + 1 + } + } + return 0 +} + +func toBatch(b batch, fd pb.FileDescriptor) *pb.TaskLogsBatch { + items := make([]*pb.TaskLogs, 0, len(b.lines)+b.markers) + for _, line := range b.lines { + items = append(items, pb.TaskLogs_builder{ + Data: line, + TimestampNs: b.timestampNs, + // Left unspecified on purpose: the live server scopes a stream by descriptor and does + // not repeat it per item, so inheriting the request's is the normal path. + FileDescriptor: pb.FileDescriptor_FILE_DESCRIPTOR_UNSPECIFIED, + }.Build()) + } + for range b.markers { + items = append(items, pb.TaskLogs_builder{TaskState: pb.TaskState_TASK_STATE_LOADING_IMAGE}.Build()) + } + return pb.TaskLogsBatch_builder{Items: items, EntryId: b.id, Eof: b.eof}.Build() +} + +func startFakeServer(t *testing.T, history []batch) (LogsClient, *fakeServer) { + t.Helper() + + lis := bufconn.Listen(1024 * 1024) + srv := grpc.NewServer() + fake := &fakeServer{history: history, fired: map[int]bool{}} + pb.RegisterModalClientServer(srv, fake) + go func() { + // Errors here are the listener closing at cleanup, which is not a test failure. + _ = srv.Serve(lis) + }() + + conn, err := grpc.NewClient("passthrough:///bufnet", + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return lis.DialContext(ctx) + }), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithStreamInterceptor(headerInjectorStream(Credentials{TokenID: "ak-test", TokenSecret: "as-test"})), + ) + if err != nil { + t.Fatalf("dialing the fake server: %v", err) + } + t.Cleanup(func() { + _ = conn.Close() + srv.Stop() + }) + return pb.NewModalClientClient(conn), fake +} diff --git a/components/logship/internal/modal/logstream.go b/components/logship/internal/modal/logstream.go new file mode 100644 index 0000000..b6cebcb --- /dev/null +++ b/components/logship/internal/modal/logstream.go @@ -0,0 +1,194 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package modal reads a sandbox's logs with an exact cursor. +// +// The SDK already streams the same RPC, but sb.Stdout bottoms out in a byte pipe: the entry ID, +// the per-item timestamp and the file descriptor are all dropped before a caller sees a byte, and +// the entry ID is what makes a resume exact. Hence the direct RPC — not for the call, but to keep +// what the call already returns. +// +// It deliberately knows nothing about sinks: no epoch-millis conversion, no size limits, no line +// splitting. Those are one sink's rules, and this outlives any one sink. +package modal + +import ( + "context" + "errors" + "fmt" + "io" + "time" + + pb "github.com/modal-labs/modal-client/go/proto/modal_proto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// CursorStart asks for a sandbox's whole retained history. +// +// Any UNRECOGNIZED cursor means the same thing to the server — "1-0" replays everything too, with +// no error — so a corrupted cursor is a silent full replay, not a failure. Nothing here can +// detect that; the duplicate is absorbed downstream. See design.md at the component root. +const CursorStart = "0-0" + +// streamTimeout is how long the server holds one stream open. It is the server's budget, not +// ours: Recv returns io.EOF at the end of it with the sandbox still running and more logs to +// come, which is the whole reason Follow re-opens instead of returning. The SDK's value. +const streamTimeout = 55 * time.Second + +// maxRetries bounds consecutive transient failures. Unlike the SDK's, the budget is restored on +// every batch received — a stream followed for a training run's whole life would otherwise +// exhaust ten isolated blips spread over hours and give up on a healthy sandbox. +const maxRetries = 10 + +// Entry is one log item with the metadata the SDK's byte pipe discards. +type Entry struct { + // Data is a raw chunk, NOT a line: it may hold several newlines, or half of one. Assembling + // lines is the caller's, because whether a line is even the unit depends on the sink. + Data string + // At is the item's timestamp. If the server sent neither timestamp field, it falls back to + // time.Now() so downstream sinks always get a non-zero time. + At time.Time + FD pb.FileDescriptor +} + +// Batch is one server batch, with the cursor that resumes strictly after it. +type Batch struct { + Entries []Entry + Cursor string +} + +// Follow calls fn once per batch of output, from cursor forward, and returns nil when the sandbox +// has no more logs to give. Pass CursorStart, or the empty string, for the whole history. +// +// Reconnection is internal, so the caller sees one continuous sequence across the server's +// 55-second windows and any transient failure. fn is called on the calling goroutine and must not +// block for long: a stalled fn stalls the cursor, and a stalled cursor risks Modal aging out logs +// that were never copied. Returning an error from fn aborts and propagates. +// +// At-least-once, by construction. The cursor advances in memory as batches arrive, so a caller +// that crashes re-reads from wherever its own durable cursor was. +func Follow(ctx context.Context, c LogsClient, sandboxID string, fd pb.FileDescriptor, cursor string, fn func(Batch) error) error { + if cursor == "" { + cursor = CursorStart + } + retries := maxRetries + for { + stream, err := c.SandboxGetLogs(ctx, pb.SandboxGetLogsRequest_builder{ + SandboxId: sandboxID, + FileDescriptor: fd, + Timeout: float32(streamTimeout.Seconds()), + LastEntryId: cursor, + }.Build()) + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + if retryable(err) && retries > 0 { + retries-- + continue + } + return fmt.Errorf("opening log stream for sandbox %s: %w", sandboxID, err) + } + + for { + batch, err := stream.Recv() + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + // io.EOF is the server's window closing, NOT the end of the logs — that is + // batch.Eof below. Re-open from the cursor either way; the only difference is + // whether it costs a retry. + if !errors.Is(err, io.EOF) { + if !retryable(err) || retries == 0 { + return fmt.Errorf("reading log stream for sandbox %s: %w", sandboxID, err) + } + retries-- + } + break + } + retries = maxRetries + + // Guarded, not ordered: the final batch carries eof with an EMPTY entry ID, and + // letting that reach the cursor would make the next resume replay the whole sandbox. + // Checking eof first would work today and break the moment a batch arrives without + // one for any other reason. + if id := batch.GetEntryId(); id != "" { + cursor = id + } + // A batch can be all state markers, so an empty one is normal and still advances the + // cursor above. Only real output is worth waking the caller for. + if entries := entries(batch, fd); len(entries) > 0 { + if err := fn(Batch{Entries: entries, Cursor: cursor}); err != nil { + return err + } + } + if batch.GetEof() { + return nil + } + } + } +} + +// entries keeps the items that are output. want is the descriptor the stream was opened for, used +// only when an item does not name its own. +func entries(batch *pb.TaskLogsBatch, want pb.FileDescriptor) []Entry { + items := batch.GetItems() + out := make([]Entry, 0, len(items)) + for _, item := range items { + // Task state and progress markers ride the same stream carrying no data. They are not + // output, and a sink like CloudWatch rejects an empty message outright. + if item.GetData() == "" { + continue + } + fd := item.GetFileDescriptor() + if fd == pb.FileDescriptor_FILE_DESCRIPTOR_UNSPECIFIED { + fd = want + } + out = append(out, Entry{Data: item.GetData(), At: itemTime(item), FD: fd}) + } + return out +} + +// itemTime prefers timestamp_ns. The float64 seconds field cannot represent every millisecond +// exactly, and a sink with millisecond resolution can therefore land two lines from the same +// millisecond in the wrong order. +func itemTime(item *pb.TaskLogs) time.Time { + if ns := item.GetTimestampNs(); ns > 0 { + return time.Unix(0, int64(ns)) + } + if sec := item.GetTimestamp(); sec > 0 { + return time.Unix(0, int64(sec*float64(time.Second))) + } + return time.Now() +} + +// retryable mirrors the SDK's classification. Canceled is in it there and stays here: a server +// that cancels a log stream is not the same event as our own ctx being cancelled, which Follow +// checks first. +func retryable(err error) bool { + st, ok := status.FromError(err) + if !ok { + return false + } + switch st.Code() { + case codes.DeadlineExceeded, codes.Unavailable, codes.Canceled, codes.Internal, codes.Unknown: + return true + default: + return false + } +} diff --git a/components/logship/internal/modal/logstream_test.go b/components/logship/internal/modal/logstream_test.go new file mode 100644 index 0000000..f7405eb --- /dev/null +++ b/components/logship/internal/modal/logstream_test.go @@ -0,0 +1,225 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modal + +import ( + "context" + "strings" + "testing" + "time" + + pb "github.com/modal-labs/modal-client/go/proto/modal_proto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// The four cursor behaviours below are what a live sandbox actually did (see the component's +// design.md); fakeServer reproduces them so the loop can be tested against them without a +// Modal account. + +func TestFollow_CursorSemantics(t *testing.T) { + history := []batch{ + {id: "100-0", lines: []string{"first\n"}}, + {id: "200-0", lines: []string{"second\n"}}, + {id: "300-0", lines: []string{"third\n"}}, + } + cases := []struct { + name string + cursor string + want []string + }{ + {"whole history from the start cursor", CursorStart, []string{"first\n", "second\n", "third\n"}}, + {"whole history from an empty cursor", "", []string{"first\n", "second\n", "third\n"}}, + {"only what followed the first entry", "100-0", []string{"second\n", "third\n"}}, + {"nothing but eof from the last entry", "300-0", nil}, + // The server does not reject a cursor it does not know, so this replays instead of + // failing. Silent by design on Modal's side, and nothing here can tell the difference. + {"whole history from an unrecognized cursor", "1-0", []string{"first\n", "second\n", "third\n"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + client, _ := startFakeServer(t, history) + got := collect(t, client, tc.cursor) + if strings.Join(got, "") != strings.Join(tc.want, "") { + t.Fatalf("got %q, want %q", got, tc.want) + } + }) + } +} + +func TestFollow_CursorSurvivesTheEmptyEntryIDOnEOF(t *testing.T) { + // The eof batch carries a final line and no entry ID at all. Taking that ID would leave the + // caller resuming from "" — a full replay of everything already shipped. + history := []batch{ + {id: "100-0", lines: []string{"first\n"}}, + {id: "", lines: []string{"last\n"}, eof: true}, + } + client, _ := startFakeServer(t, history) + + var last string + if err := Follow(t.Context(), client, "sb-1", pb.FileDescriptor_FILE_DESCRIPTOR_STDOUT, CursorStart, + func(b Batch) error { + last = b.Cursor + return nil + }); err != nil { + t.Fatalf("Follow: %v", err) + } + if last != "100-0" { + t.Fatalf("cursor %q after the eof batch, want the last real entry ID", last) + } +} + +func TestFollow_ReopensAcrossTheServerWindow(t *testing.T) { + // Two streams' worth of history: the first ends without eof, exactly as a 55-second server + // window does. A loop that mistook that io.EOF for the end would stop after "first". + client, srv := startFakeServer(t, []batch{ + {id: "100-0", lines: []string{"first\n"}, closeWindow: true}, + {id: "200-0", lines: []string{"second\n"}, eof: true}, + }) + if got := collect(t, client, CursorStart); strings.Join(got, "") != "first\nsecond\n" { + t.Fatalf("got %q across the window boundary", got) + } + if srv.opened != 2 { + t.Fatalf("%d streams opened, want 2 — the window close must reopen", srv.opened) + } + if srv.cursors[1] != "100-0" { + t.Fatalf("reopened at %q, want the last delivered entry ID", srv.cursors[1]) + } +} + +func TestFollow_SkipsItemsWithNoData(t *testing.T) { + // State and progress markers ride the same stream. They must not reach a sink, and a batch + // made only of them must still advance the cursor. + client, srv := startFakeServer(t, []batch{ + {id: "100-0", markers: 2}, + {id: "200-0", lines: []string{"real\n"}, closeWindow: true}, + {id: "300-0", eof: true}, + }) + if got := collect(t, client, CursorStart); strings.Join(got, "") != "real\n" { + t.Fatalf("got %q, want only the item carrying data", got) + } + if srv.cursors[1] != "200-0" { + t.Fatalf("reopened at %q, want a marker-only batch to have advanced the cursor", srv.cursors[1]) + } +} + +func TestFollow_RetriesTransientAndResumes(t *testing.T) { + client, srv := startFakeServer(t, []batch{ + {id: "100-0", lines: []string{"first\n"}}, + {fail: status.Error(codes.Unavailable, "server going away")}, + {id: "200-0", lines: []string{"second\n"}, eof: true}, + }) + if got := collect(t, client, CursorStart); strings.Join(got, "") != "first\nsecond\n" { + t.Fatalf("got %q, want the stream to resume after a retryable error", got) + } + if srv.cursors[1] != "100-0" { + t.Fatalf("retried at %q, want no re-delivery of what already arrived", srv.cursors[1]) + } +} + +func TestFollow_StopsOnPermanentError(t *testing.T) { + client, _ := startFakeServer(t, []batch{ + {fail: status.Error(codes.PermissionDenied, "bad token")}, + }) + err := Follow(t.Context(), client, "sb-1", pb.FileDescriptor_FILE_DESCRIPTOR_STDOUT, CursorStart, + func(Batch) error { return nil }) + if err == nil { + t.Fatal("Follow returned nil on a non-retryable error") + } + if !strings.Contains(err.Error(), "sb-1") { + t.Fatalf("error %q does not name the sandbox", err) + } +} + +func TestFollow_KeepsTheItemTimestamp(t *testing.T) { + ns := uint64(1788477379282_000_000) + client, _ := startFakeServer(t, []batch{ + {id: "100-0", lines: []string{"stamped\n"}, timestampNs: ns, eof: true}, + }) + var got []Entry + if err := Follow(t.Context(), client, "sb-1", pb.FileDescriptor_FILE_DESCRIPTOR_STDERR, CursorStart, + func(b Batch) error { + got = append(got, b.Entries...) + return nil + }); err != nil { + t.Fatalf("Follow: %v", err) + } + if len(got) != 1 { + t.Fatalf("%d entries, want 1", len(got)) + } + if want := time.Unix(0, int64(ns)); !got[0].At.Equal(want) { + t.Fatalf("At = %v, want %v", got[0].At, want) + } + // The item did not name a descriptor, so it inherits the one the stream was opened for — + // otherwise every entry would land as UNSPECIFIED and the two streams would merge. + if got[0].FD != pb.FileDescriptor_FILE_DESCRIPTOR_STDERR { + t.Fatalf("FD = %v, want the requested descriptor", got[0].FD) + } +} + +func TestFollow_PropagatesCallbackError(t *testing.T) { + client, _ := startFakeServer(t, []batch{ + {id: "100-0", lines: []string{"first\n"}}, + {id: "200-0", lines: []string{"second\n"}, eof: true}, + }) + sentinel := status.Error(codes.ResourceExhausted, "sink is full") + calls := 0 + err := Follow(t.Context(), client, "sb-1", pb.FileDescriptor_FILE_DESCRIPTOR_STDOUT, CursorStart, + func(Batch) error { + calls++ + return sentinel + }) + if err == nil || !strings.Contains(err.Error(), "sink is full") { + t.Fatalf("err = %v, want the callback's error", err) + } + if calls != 1 { + t.Fatalf("%d callbacks, want the first failure to abort", calls) + } +} + +func TestFollow_HonoursContextCancellation(t *testing.T) { + client, _ := startFakeServer(t, []batch{ + {id: "100-0", lines: []string{"first\n"}}, + // The stream then stays open with no eof, which is what a running sandbox looks like. + // Without it the fake would close and the cancellation would race the final batch. + {hold: true}, + }) + ctx, cancel := context.WithCancel(t.Context()) + err := Follow(ctx, client, "sb-1", pb.FileDescriptor_FILE_DESCRIPTOR_STDOUT, CursorStart, + func(Batch) error { + cancel() + return nil + }) + if !strings.Contains(err.Error(), context.Canceled.Error()) { + t.Fatalf("err = %v, want context.Canceled", err) + } +} + +func collect(t *testing.T, c LogsClient, cursor string) []string { + t.Helper() + var got []string + if err := Follow(t.Context(), c, "sb-1", pb.FileDescriptor_FILE_DESCRIPTOR_STDOUT, cursor, + func(b Batch) error { + for _, e := range b.Entries { + got = append(got, e.Data) + } + return nil + }); err != nil { + t.Fatalf("Follow: %v", err) + } + return got +} diff --git a/components/logship/internal/modal/pool.go b/components/logship/internal/modal/pool.go new file mode 100644 index 0000000..3c1fff3 --- /dev/null +++ b/components/logship/internal/modal/pool.go @@ -0,0 +1,167 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modal + +import ( + "errors" + "sync" + + "google.golang.org/grpc" +) + +// StreamsPerConn is how many log streams share one connection. +// +// The number exists because of how gRPC-go fails here, not because of a measured optimum: HTTP/2 +// caps concurrent streams per connection via SETTINGS_MAX_CONCURRENT_STREAMS, and gRPC-go *queues* +// RPCs past that cap rather than returning an error. On a single connection, 1,000 log streams would +// leave most of them waiting forever with nothing logged and no error surfaced anywhere — the worst +// possible shape for a component whose job is not losing logs. 100 is comfortably under the 100-ish +// that servers typically advertise while keeping the pool small enough that a connection failure +// takes out a bounded slice of the fleet. +const StreamsPerConn = 100 + +// Pool spreads log streams across several connections to Modal. +// +// Least-loaded, and every stream is one long-lived RPC of near-identical cost, so a count of live +// streams IS the load — which is why Client hands back a release. Round-robin will not do here: a +// modulo over a pool that has gained a connection re-partitions the residues without moving the +// streams already handed out, so the first connection keeps its whole historical share and ends up +// several times over StreamsPerConn, which is the silent-queueing case that constant exists to +// prevent. Safe for concurrent use. +type Pool struct { + creds Credentials + + // One plain Mutex: picking a connection writes the count that made the pick correct, so there is + // no read-only path left for an RWMutex to help. + mu sync.Mutex + conns []*grpc.ClientConn + clients []LogsClient + // live counts the streams holding each client, parallel to clients. + live []int +} + +// NewPool dials enough connections for streams. It rounds up, and always dials at least one. +// +// Grow it with Grow as the fleet grows; there is no fleet size to know here. +func NewPool(creds Credentials, streams int) (*Pool, error) { + p := &Pool{creds: creds} + if err := p.Grow(streams); err != nil { + // Close what did open; a half-built pool leaks connections for the process's life. + return nil, errors.Join(err, p.Close()) + } + return p, nil +} + +// Grow dials whatever streams needs beyond what the pool already has, and never shrinks. +// +// It must be called BEFORE the streams it accounts for are opened, which is the whole contract: a +// connection added afterwards cannot take over streams already queued behind HTTP/2's concurrency +// cap, and gRPC-go queues them silently rather than failing, so by the time the symptom is visible +// the stuck streams stay stuck. Growing ahead of demand is what keeps that case from arising — +// there is no cap to guess and no fleet size to configure, only the invariant that no connection is +// handed more than StreamsPerConn. +// +// Not shrinking is deliberate: connections are cheap (grpc.NewClient does not connect until an RPC +// needs it) and closing one under a live stream would end it. +func (p *Pool) Grow(streams int) error { + want := (max(streams, 1) + StreamsPerConn - 1) / StreamsPerConn + + p.mu.Lock() + defer p.mu.Unlock() + for len(p.conns) < want { + conn, client, err := Dial(p.creds) + if err != nil { + // Keep what did open: they are already carrying streams, and the caller's own error + // path decides whether the instance that needed the new one ships. + return err + } + p.conns = append(p.conns, conn) + p.clients = append(p.clients, client) + p.live = append(p.live, 0) + } + return nil +} + +// Client picks the connection carrying the fewest streams, and returns it with the release that gives +// the slot back. +// +// Callers hold the client for the life of their stream — re-picking per RPC would move a re-opened +// stream to a different connection every 55 seconds and make the distribution drift — and must call +// release when that stream ends, or the pool never learns the connection is free again. Release is +// idempotent, so a defer that overlaps an error path cannot double-count a slot into existence. +// +// A pool whose every connection already sits at StreamsPerConn still hands out the least-bad one +// rather than failing. Grow is what reserves capacity ahead of demand; refusing here would abandon an +// instance's logs entirely over an overshoot whose actual cost is some queueing. +func (p *Pool) Client() (LogsClient, func()) { + p.mu.Lock() + defer p.mu.Unlock() + + i := 0 + for j, n := range p.live { + if n < p.live[i] { + i = j + } + } + p.live[i]++ + + var once sync.Once + return p.clients[i], func() { + once.Do(func() { + p.mu.Lock() + defer p.mu.Unlock() + // Close empties live, so a stream ending after it has nothing left to give back. + if i < len(p.live) && p.live[i] > 0 { + p.live[i]-- + } + }) + } +} + +// Live is how many slots are handed out and not yet returned. +// +// It is what reserving capacity has to count against, because a cancelled stream keeps its slot until +// its RPC unwinds: any figure derived from a tracked-instance count runs ahead of this one during a +// teardown. See Provider.Reserve. +func (p *Pool) Live() int { + p.mu.Lock() + defer p.mu.Unlock() + n := 0 + for _, c := range p.live { + n += c + } + return n +} + +// Len is the number of connections, which is what a caller checks a stream count against. +func (p *Pool) Len() int { + p.mu.Lock() + defer p.mu.Unlock() + return len(p.conns) +} + +func (p *Pool) Close() error { + p.mu.Lock() + defer p.mu.Unlock() + + errs := make([]error, 0, len(p.conns)) + for _, c := range p.conns { + errs = append(errs, c.Close()) + } + p.conns, p.clients, p.live = nil, nil, nil + return errors.Join(errs...) +} diff --git a/components/logship/internal/modal/pool_test.go b/components/logship/internal/modal/pool_test.go new file mode 100644 index 0000000..2ea4743 --- /dev/null +++ b/components/logship/internal/modal/pool_test.go @@ -0,0 +1,179 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modal + +import ( + "sync" + "testing" +) + +func TestNewPool_SizesFromTheStreamCount(t *testing.T) { + // grpc.NewClient does not connect, so this exercises the arithmetic without a server. Rounding + // up matters: 1,000 streams over 100 per connection is 10, and 1,001 must not be 10 as well. + for _, tc := range []struct{ streams, want int }{ + {0, 1}, {1, 1}, {100, 1}, {101, 2}, {1000, 10}, {1001, 11}, + } { + p, err := NewPool(Credentials{ServerURL: "http://localhost:1"}, tc.streams) + if err != nil { + t.Fatalf("NewPool(%d): %v", tc.streams, err) + } + if p.Len() != tc.want { + t.Errorf("NewPool(%d).Len() = %d, want %d", tc.streams, p.Len(), tc.want) + } + if err := p.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + } +} + +func TestPool_SpreadsStreamsEvenly(t *testing.T) { + // The point of the pool: 1,000 streams on one connection stall silently, so an uneven hand-out + // would reproduce the bug on a subset. + p, err := NewPool(Credentials{ServerURL: "http://localhost:1"}, 300) + if err != nil { + t.Fatalf("NewPool: %v", err) + } + defer func() { _ = p.Close() }() + + counts := map[LogsClient]int{} + var mu sync.Mutex + var wg sync.WaitGroup + for range 300 { + wg.Add(1) + go func() { + defer wg.Done() + c, _ := p.Client() // not released: all 300 streams are live at once + mu.Lock() + counts[c]++ + mu.Unlock() + }() + } + wg.Wait() + + if len(counts) != 3 { + t.Fatalf("used %d of 3 connections", len(counts)) + } + for c, n := range counts { + if n != 100 { + t.Errorf("connection %p took %d streams, want 100", c, n) + } + } +} + +func TestPool_HoldsTheCapWhileItGrows(t *testing.T) { + // The fleet's real sequence: Reserve widens the pool BEFORE each instance's streams open, two + // streams per instance. Modulo round-robin passed the one-connection case and then drifted, because + // a pool that gains a connection re-partitions the residues without moving the streams already + // handed out — at 200 instances it left 208 streams on the first connection against a cap of 100. + p, err := NewPool(Credentials{ServerURL: "http://localhost:1"}, len(Descriptors)) + if err != nil { + t.Fatalf("NewPool: %v", err) + } + defer func() { _ = p.Close() }() + + const instances = 200 + live := map[LogsClient]int{} + for k := 1; k <= instances; k++ { + if err := p.Grow(k * len(Descriptors)); err != nil { + t.Fatalf("Grow(%d): %v", k*len(Descriptors), err) + } + for range len(Descriptors) { + c, _ := p.Client() // held: every one of these streams is still running + live[c]++ + } + } + + want := instances * len(Descriptors) / StreamsPerConn + if p.Len() != want || len(live) != want { + t.Fatalf("%d connections and %d used, want %d of each", p.Len(), len(live), want) + } + for c, n := range live { + if n > StreamsPerConn { + t.Errorf("connection %p carries %d streams, over the %d cap", c, n, StreamsPerConn) + } + } +} + +func TestReserve_CountsTheStreamsStillDrainingAfterAForget(t *testing.T) { + // Supervisor.Forget untracks an instance before its streams give their slots back, and watch.ensure + // forgets and re-ensures in the same handler — so a replacement reserved from the decremented count + // used to open into a pool sized as if the departing streams had already gone. It bites only with the + // pool exactly on a connection boundary, which is where an even fleet size sits. + p, err := NewPool(Credentials{ServerURL: "http://localhost:1"}, StreamsPerConn) + if err != nil { + t.Fatalf("NewPool: %v", err) + } + defer func() { _ = p.Close() }() + + for range StreamsPerConn { + p.Client() // held: one connection exactly full, nothing released + } + // The fleet that fits in those slots. Unchanged by the churn, since Forget already decremented and + // the replacement adds itself back. + prov := &Provider{pool: p} + if err := prov.Reserve(StreamsPerConn / len(Descriptors)); err != nil { + t.Fatalf("Reserve: %v", err) + } + for range len(Descriptors) { + p.Client() + } + + if p.Len() != 2 { + t.Errorf("Len() = %d, want the reserve to have dialed for the draining streams", p.Len()) + } + for i, n := range p.live { + if n > StreamsPerConn { + t.Errorf("connection %d carries %d streams, over the %d cap", i, n, StreamsPerConn) + } + } + if got := p.Live(); got != StreamsPerConn+len(Descriptors) { + t.Errorf("Live() = %d, want every held slot counted", got) + } +} + +func TestPool_ReleaseFreesTheSlot(t *testing.T) { + // Without a release the pool counts a finished stream forever, and since every retry builds a fresh + // Source, a fleet that never grew would still walk its connections up past the cap. + p, err := NewPool(Credentials{ServerURL: "http://localhost:1"}, 2*StreamsPerConn) + if err != nil { + t.Fatalf("NewPool: %v", err) + } + defer func() { _ = p.Close() }() + + first, release := p.Client() + if second, _ := p.Client(); second == first { + t.Fatal("the second stream took the connection already carrying one") + } + release() + if again, _ := p.Client(); again != first { + t.Fatal("after a release the emptiest connection was not the one picked") + } + + release() + release() + // Read directly: a double release must not invent capacity, and a negative count would make this + // connection win every pick from here on — the exact overload the count exists to prevent. + if p.live[0] != 1 { + t.Fatalf("live[0] = %d after repeated releases, want 1", p.live[0]) + } +} + +func TestNewPool_RejectsABadServerURL(t *testing.T) { + if _, err := NewPool(Credentials{ServerURL: "localhost:443"}, 10); err == nil { + t.Fatal("NewPool accepted a URL with no scheme") + } +} diff --git a/components/logship/internal/modal/provider.go b/components/logship/internal/modal/provider.go new file mode 100644 index 0000000..4ff0fbc --- /dev/null +++ b/components/logship/internal/modal/provider.go @@ -0,0 +1,80 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modal + +import ( + "fmt" + + "github.com/InftyAI/Nebula/components/logship/internal/ship" +) + +// Provider reads Modal sandbox logs. It implements the log provider port; see internal/provider. +// +// One per process, holding the pool every sandbox's streams are spread over — which is why it is a +// long-lived object rather than something built per instance. +type Provider struct { + pool *Pool +} + +// Open reads the tokens from the environment and builds the pool at its smallest useful size, one +// sandbox's worth. Reserve widens it from there. +func Open() (*Provider, error) { + creds, err := CredentialsFromEnv() + if err != nil { + return nil, err + } + pool, err := NewPool(creds, len(Descriptors)) + if err != nil { + return nil, err + } + return &Provider{pool: pool}, nil +} + +func (p *Provider) Streams() []string { return Descriptors } + +// Source picks the connection this stream keeps for its life. See Pool.Client for why it is not +// re-picked per RPC, and Source.Follow for where the slot goes back. +func (p *Provider) Source(instanceID, stream string) (ship.Source, error) { + fd, err := ParseDescriptor(stream) + if err != nil { + // Before Client, so a name that maps to no descriptor cannot leak a slot. + return nil, err + } + client, release := p.pool.Client() + return Source{Client: client, Sandbox: instanceID, FD: fd, release: release}, nil +} + +// Reserve widens the pool for instances sandboxes. See Pool.Grow: the ordering — before the streams +// open — is the whole contract, and growing past what is needed costs an unconnected socket. +// +// Floored at what the pool is already carrying plus this instance's own streams, because an instance +// count is not that: Supervisor.Forget drops an instance before its streams release their slots, so a +// replacement arriving while the pool sits exactly on a connection boundary would open into a pool +// sized as if the departing streams were already gone. A floor and not a lock — concurrent Ensures can +// still overshoot, which Pool.Client tolerates on purpose. +// +// The connection count rides in the error because the caller reports the failure and does not +// otherwise know the shape of what failed. +func (p *Provider) Reserve(instances int) error { + streams := max(instances*len(Descriptors), p.pool.Live()+len(Descriptors)) + if err := p.pool.Grow(streams); err != nil { + return fmt.Errorf("widening the pool past its %d connections: %w", p.pool.Len(), err) + } + return nil +} + +func (p *Provider) Close() error { return p.pool.Close() } diff --git a/components/logship/internal/modal/source.go b/components/logship/internal/modal/source.go new file mode 100644 index 0000000..2876d08 --- /dev/null +++ b/components/logship/internal/modal/source.go @@ -0,0 +1,95 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modal + +import ( + "context" + "fmt" + + pb "github.com/modal-labs/modal-client/go/proto/modal_proto" + + "github.com/InftyAI/Nebula/components/logship/internal/ship" +) + +// Source binds one sandbox and one file descriptor to a ship.Source. The dependency runs this way +// round on purpose: ship declares the port and knows nothing about Modal. +// +// One per stream, because a cursor is only meaningful for one descriptor of one sandbox. +type Source struct { + Client LogsClient + Sandbox string + FD pb.FileDescriptor + + // release returns Client's slot to the pool it came from; nil for a Source built without one. + // Unexported because only Provider.Source can pair it with the Client it accounts for. + release func() +} + +// Follow implements ship.Source. It drops the per-entry FD: the stream was opened for one, so every +// entry in it has the same one, and repeating it per line would be the only thing ship had to know +// about Modal's proto. +// +// This call is the stream's whole life, which is why the pool's slot goes back here rather than on a +// Close the port does not have: a Pipeline is single-use, so each retry builds a fresh Source and +// Follows it exactly once. +func (s Source) Follow(ctx context.Context, cursor string, fn func(ship.Batch) error) error { + if s.release != nil { + defer s.release() + } + return Follow(ctx, s.Client, s.Sandbox, s.FD, cursor, func(b Batch) error { + entries := make([]ship.Entry, 0, len(b.Entries)) + for _, e := range b.Entries { + entries = append(entries, ship.Entry{Data: e.Data, At: e.At}) + } + return fn(ship.Batch{Entries: entries, Cursor: b.Cursor}) + }) +} + +// Descriptor names an FD for a CloudWatch stream name. Anything but the two real descriptors is +// "unknown" rather than an error: a stream that ships under an odd name is recoverable, and a +// sandbox whose logs are dropped over an enum value is not. +func Descriptor(fd pb.FileDescriptor) string { + switch fd { + case pb.FileDescriptor_FILE_DESCRIPTOR_STDOUT: + return "stdout" + case pb.FileDescriptor_FILE_DESCRIPTOR_STDERR: + return "stderr" + default: + return "unknown" + } +} + +// Descriptors are the streams a sandbox has, in the order they are worth reading. The supervisor +// names streams with plain strings so it need not know Modal exists, so this is what it is given. +var Descriptors = []string{ + Descriptor(pb.FileDescriptor_FILE_DESCRIPTOR_STDOUT), + Descriptor(pb.FileDescriptor_FILE_DESCRIPTOR_STDERR), +} + +// ParseDescriptor is Descriptor's inverse, and unlike it this one errors: a name that maps to no +// descriptor would otherwise open a stream on FILE_DESCRIPTOR_UNSPECIFIED and ship nothing, silently. +func ParseDescriptor(name string) (pb.FileDescriptor, error) { + for _, fd := range []pb.FileDescriptor{ + pb.FileDescriptor_FILE_DESCRIPTOR_STDOUT, + pb.FileDescriptor_FILE_DESCRIPTOR_STDERR, + } { + if Descriptor(fd) == name { + return fd, nil + } + } + return pb.FileDescriptor_FILE_DESCRIPTOR_UNSPECIFIED, fmt.Errorf("unknown descriptor %q", name) +} diff --git a/components/logship/internal/modal/source_test.go b/components/logship/internal/modal/source_test.go new file mode 100644 index 0000000..8e1cdb5 --- /dev/null +++ b/components/logship/internal/modal/source_test.go @@ -0,0 +1,61 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modal + +import ( + "testing" + + pb "github.com/modal-labs/modal-client/go/proto/modal_proto" +) + +func TestDescriptorRoundTrips(t *testing.T) { + // The two directions are used at opposite ends of the wiring — a name goes into a stream name and + // comes back out to open an RPC — so a rename that touches only one of them loses a whole stream. + for _, fd := range []pb.FileDescriptor{ + pb.FileDescriptor_FILE_DESCRIPTOR_STDOUT, + pb.FileDescriptor_FILE_DESCRIPTOR_STDERR, + } { + got, err := ParseDescriptor(Descriptor(fd)) + if err != nil { + t.Fatalf("ParseDescriptor(Descriptor(%v)): %v", fd, err) + } + if got != fd { + t.Fatalf("round trip of %v gave %v", fd, got) + } + } +} + +func TestParseDescriptorRejectsAnUnknownName(t *testing.T) { + // Returning UNSPECIFIED with no error would open a stream that ships nothing at all. + if _, err := ParseDescriptor("stdlog"); err == nil { + t.Fatal("ParseDescriptor accepted a name that maps to no descriptor") + } + if _, err := ParseDescriptor(Descriptor(pb.FileDescriptor_FILE_DESCRIPTOR_UNSPECIFIED)); err == nil { + t.Fatal(`ParseDescriptor accepted "unknown", which Descriptor uses as a fallback name`) + } +} + +func TestDescriptorsAreTheStreamsAPipelineIsBuiltFor(t *testing.T) { + if len(Descriptors) != 2 { + t.Fatalf("Descriptors = %v, want the two real descriptors", Descriptors) + } + for _, name := range Descriptors { + if _, err := ParseDescriptor(name); err != nil { + t.Fatalf("Descriptors contains %q, which ParseDescriptor rejects: %v", name, err) + } + } +} diff --git a/components/logship/internal/provider/provider.go b/components/logship/internal/provider/provider.go new file mode 100644 index 0000000..1b98d49 --- /dev/null +++ b/components/logship/internal/provider/provider.go @@ -0,0 +1,168 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package provider is the port logship reads instance logs through: one implementation per Nebula +// provider, Modal being the first. +// +// Which one a stream comes from is decided per Pod, from the provider nodeSelector Nebula placed it +// with, and is deliberately not configured anywhere. A configured provider would have to be kept in +// agreement with the cluster's node pools, and being wrong about it looks exactly like a quiet +// cluster: every Pod skipped, nothing shipped, no error. Routing per Pod also means a cluster split +// across two providers ships from both, with one process and no extra wiring. +package provider + +import ( + "errors" + "fmt" + "sort" + "sync" + + "github.com/InftyAI/Nebula/components/logship/internal/ship" +) + +// The provider names, which are the values Nebula's placement controller writes to a Pod's provider +// nodeSelector. A name is what a backend is registered under, so one that disagrees with the +// controller's makes every Pod on that provider unshippable. +const ( + ProviderModal = "modal" +) + +// Provider reads one backend's instance logs. Implementations live in their own package and are +// registered on a Set; nothing outside this port may name one. +// +// Every method may be called concurrently, and for many instances at once: one process follows the +// whole cluster's streams. +type Provider interface { + // Streams are the streams each of this backend's instances has, in the order they are worth + // reading, named in whatever vocabulary the backend uses. One pipeline runs per name per + // instance, and nothing between here and Source interprets a name — it only has to round-trip. + Streams() []string + + // Source binds one instance's stream so it can be followed from a cursor. + // + // It must fail rather than return a Source that ships nothing: an instance id or stream name the + // backend does not recognise is a wiring mistake, and the symptom of a silent one is a run whose + // logs are simply absent. + Source(instanceID, stream string) (ship.Source, error) + + // Reserve makes room for instances instances BEFORE any of their streams open, for a backend + // whose transport caps concurrent streams per connection. A backend with no such cap does + // nothing here. + // + // instances is an upper bound, not this backend's exact share: a caller that cannot cheaply + // attribute the fleet per backend passes the whole fleet's size, so over-reserving has to be + // cheap and must never be an error. + Reserve(instances int) error + + // Close releases the backend's connections, once, at shutdown. + Close() error +} + +// Factory opens one backend. Called at most once per process and only when a Pod needs that +// backend: opening reads credentials and dials, and a cluster with nothing on a provider must not +// need either to be present. +type Factory func() (Provider, error) + +// Set is this process's providers — what is registered, and what has been opened. +// +// A backend's name lives in its registration rather than on the Provider itself, so the two cannot +// disagree about the value a Pod's nodeSelector has to match. Safe for concurrent use. +type Set struct { + mu sync.Mutex + factory map[string]Factory + open map[string]Provider +} + +func NewSet() *Set { + return &Set{factory: map[string]Factory{}, open: map[string]Provider{}} +} + +// Register adds a backend under the name a Pod's provider nodeSelector carries. +// +// Panics on a duplicate or an empty name, because both are programmer errors in this process's own +// wiring: picking one of two factories at runtime would hide which backend is actually reading. +func (s *Set) Register(name string, open Factory) { + if name == "" || open == nil { + panic("provider: Register needs a name and a factory") + } + + s.mu.Lock() + defer s.mu.Unlock() + if _, dup := s.factory[name]; dup { + panic(fmt.Sprintf("provider: duplicate registration for %q", name)) + } + s.factory[name] = open +} + +// Get returns the backend for name, opening it on first use. +// +// An unregistered name is an error and never a nil Provider: it means the cluster placed a Pod on a +// backend this build cannot read, which is a real state — a provider added to Nebula before it is +// added here — and its only other symptom is logs that never arrive. +// +// A factory that fails is retried on the next call rather than remembered. The cause is nearly always +// a missing credential, so the retry is unlikely to succeed, and it is the repetition that gets the +// misconfiguration into the log where someone sees it. +func (s *Set) Get(name string) (Provider, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if p := s.open[name]; p != nil { + return p, nil + } + open := s.factory[name] + if open == nil { + return nil, fmt.Errorf("no log provider for %q; this build has %v", name, s.names()) + } + p, err := open() + if err != nil { + return nil, fmt.Errorf("opening the %s log provider: %w", name, err) + } + s.open[name] = p + return p, nil +} + +// Names are the registered names, sorted, for an error that says what this build does support. +func (s *Set) Names() []string { + s.mu.Lock() + defer s.mu.Unlock() + return s.names() +} + +// Close closes every backend that was opened, leaving the registrations. Errors are joined: one +// backend failing to close says nothing about the others, and all of them are worth reporting. +func (s *Set) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + + errs := make([]error, 0, len(s.open)) + for name, p := range s.open { + if err := p.Close(); err != nil { + errs = append(errs, fmt.Errorf("closing the %s log provider: %w", name, err)) + } + } + s.open = map[string]Provider{} + return errors.Join(errs...) +} + +func (s *Set) names() []string { + names := make([]string, 0, len(s.factory)) + for name := range s.factory { + names = append(names, name) + } + sort.Strings(names) + return names +} diff --git a/components/logship/internal/provider/provider_test.go b/components/logship/internal/provider/provider_test.go new file mode 100644 index 0000000..ebed642 --- /dev/null +++ b/components/logship/internal/provider/provider_test.go @@ -0,0 +1,128 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provider + +import ( + "fmt" + "strings" + "testing" + + "github.com/InftyAI/Nebula/components/logship/internal/ship" +) + +type stub struct { + opened int + closed int +} + +func (s *stub) Streams() []string { return []string{"stdout"} } +func (s *stub) Source(string, string) (ship.Source, error) { + return nil, fmt.Errorf("not used here") +} +func (s *stub) Reserve(int) error { return nil } +func (s *stub) Close() error { s.closed++; return nil } + +func (s *stub) factory() Factory { + return func() (Provider, error) { + s.opened++ + return s, nil + } +} + +func TestGetOpensOnceAndClosesWhatItOpened(t *testing.T) { + // Opening once per name is what makes a backend's connection pool the whole process's, rather + // than one pool per instance the watch happens to deliver. + p := &stub{} + s := NewSet() + s.Register(ProviderModal, p.factory()) + + for range 3 { + if _, err := s.Get(ProviderModal); err != nil { + t.Fatal(err) + } + } + if p.opened != 1 { + t.Errorf("opened %d times, want 1", p.opened) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + if p.closed != 1 { + t.Errorf("closed %d times, want 1", p.closed) + } +} + +func TestNothingIsOpenedUntilAPodNeedsIt(t *testing.T) { + // The reason this is lazy: a cluster with nothing on a provider must not need its credentials to + // be present, so the process cannot open every registered backend at startup. + p := &stub{} + s := NewSet() + s.Register(ProviderModal, p.factory()) + + if err := s.Close(); err != nil { + t.Fatal(err) + } + if p.opened != 0 || p.closed != 0 { + t.Errorf("opened %d, closed %d; want a registration to touch neither", p.opened, p.closed) + } +} + +func TestGetOnAnUnregisteredProviderSaysWhatIsSupported(t *testing.T) { + // Nebula gaining a provider before logship does is a real state, and the Pods it places are then + // unreadable here. The error is the only thing that tells them apart from an idle cluster. + s := NewSet() + s.Register(ProviderModal, (&stub{}).factory()) + + _, err := s.Get("aws") + if err == nil { + t.Fatal("Get accepted a provider with no backend") + } + if !strings.Contains(err.Error(), ProviderModal) { + t.Errorf("error %q does not name the providers this build has", err) + } +} + +func TestAFailedOpenIsRetried(t *testing.T) { + // Not remembered: the repetition in the log is what surfaces a missing credential, and a cached + // failure would report it once, at a moment nobody was looking. + calls := 0 + s := NewSet() + s.Register(ProviderModal, func() (Provider, error) { + calls++ + return nil, fmt.Errorf("MODAL_TOKEN_ID is not set") + }) + + for range 2 { + if _, err := s.Get(ProviderModal); err == nil { + t.Fatal("Get hid a factory failure") + } + } + if calls != 2 { + t.Errorf("factory called %d times, want one per Get", calls) + } +} + +func TestRegisterRejectsADuplicate(t *testing.T) { + defer func() { + if recover() == nil { + t.Error("a second registration for one name did not panic") + } + }() + s := NewSet() + s.Register(ProviderModal, (&stub{}).factory()) + s.Register(ProviderModal, (&stub{}).factory()) +} diff --git a/components/logship/internal/ship/batch.go b/components/logship/internal/ship/batch.go new file mode 100644 index 0000000..e0a7f1c --- /dev/null +++ b/components/logship/internal/ship/batch.go @@ -0,0 +1,240 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ship + +import ( + "fmt" + "strings" + "time" + "unicode/utf8" +) + +// Event is one message as a Sink accepts it. +type Event struct { + Message string + At time.Time + // Cursor is the source cursor of the line this came from, which is what lets a reader collapse + // the duplicates a replay produces. + Cursor string +} + +// Limits are one sink's per-request caps. A zero field means that limit does not apply. +// +// PerEventOverhead is the part that gets forgotten: CloudWatch charges 26 bytes per event against +// MaxBytes on top of the message, so a sum of message lengths passes every local test and then +// fails under a flood of short lines. It is counted against MaxEventBytes too — AWS documents the +// 26 bytes plainly for the batch and vaguely for the event, and over-reserving costs one short +// split at the boundary while under-reserving costs a rejected request. +type Limits struct { + MaxEvents int + MaxBytes int + MaxEventBytes int + PerEventOverhead int +} + +// Formatter renders a line as the message a sink stores. The whole wire format lives in one of +// these, because it is the one decision that is permanent per stream: already-shipped events cannot +// be reformatted, so changing it puts a discontinuity in the middle of a log group. +// +// The real one is Record.Formatter, which matches the consumer's schema. Every formatter carries the +// cursor, which is not decoration: it is what lets a reader collapse the duplicates a replay +// produces, and it is also why a blank line never becomes an empty message — which every log API +// rejects. +type Formatter func(Line) string + +// FormatCompact renders ` `. For tests and for reading a stream raw under `aws logs +// tail`; it carries no identity, so nothing that queries by tenant can use it. +func FormatCompact(l Line) string { + return l.Cursor + " " + l.Data +} + +// encodeJSONString writes s as a JSON string. encoding/json would allocate a map and an +// intermediate []byte per line; at 1,000 streams this runs on every line of the fleet. +func encodeJSONString(b *strings.Builder, s string) { + b.WriteByte('"') + for i := range len(s) { + switch c := s[i]; { + case c == '"' || c == '\\': + b.WriteByte('\\') + b.WriteByte(c) + case c == '\n': + b.WriteString(`\n`) + case c == '\r': + b.WriteString(`\r`) + case c == '\t': + b.WriteString(`\t`) + case c < 0x20: + // Control bytes are legal in a log line and illegal raw in JSON. \u00XX rather than + // dropping them: a terminal escape sequence in the output is sometimes the evidence. + b.WriteString(`\u00`) + const hex = "0123456789abcdef" + b.WriteByte(hex[c>>4]) + b.WriteByte(hex[c&0xf]) + default: + // Written bytewise, so an invalid UTF-8 byte goes out as it arrived. That makes the record + // technically not JSON (RFC 8259 requires UTF-8), and it does NOT preserve the byte the way + // it looks like it does: the consumer's encoding/json accepts the record and coerces it to + // U+FFFD anyway. Kept because the alternative is escaping to \u00XX, which is 6 bytes for + // every byte and pushes a binary-ish line into the oversized drop at MaxEventBytes. + b.WriteByte(c) + } + } + b.WriteByte('"') +} + +// Batcher groups lines into requests a Sink will accept. +// +// It has no clock: a batch is returned when a limit would be exceeded, and the caller flushes on +// whatever interval it wants. Owning a timer here would mean owning a goroutine, and the shipping +// loop already has one. +// +// Not safe for concurrent use — one batcher per stream, like Assembler. +type Batcher struct { + limits Limits + format Formatter + + pending []Event + bytes int + // last is the highest timestamp emitted so far. PutLogEvents rejects an entire request if one + // event is out of order, so a batch has to be non-decreasing. + last time.Time + clamped int + oversized int +} + +func NewBatcher(limits Limits, format Formatter) *Batcher { + if format == nil { + format = FormatCompact + } + return &Batcher{limits: limits, format: format} +} + +// Add returns the requests these lines filled, in order. Anything short of a limit is held for a +// later Add or for Flush — including a batch sitting exactly on a limit, since "full" is discovered +// by the event that does not fit. +func (b *Batcher) Add(lines ...Line) [][]Event { + var out [][]Event + for _, l := range lines { + for _, e := range b.events(l) { + if full := b.push(e); full != nil { + out = append(out, full) + } + } + } + return out +} + +// Flush returns the pending request, or nil if there is nothing pending. +func (b *Batcher) Flush() []Event { + if len(b.pending) == 0 { + return nil + } + out := b.pending + b.pending, b.bytes = nil, 0 + return out +} + +// Clamped counts lines whose timestamp went backwards and was raised to its predecessor's. Nonzero +// means the durable copy is dated slightly wrong; it is a metric rather than an error, because the +// alternative is a rejected request or a reordered log. +func (b *Batcher) Clamped() int { return b.clamped } + +// Oversized counts lines dropped for exceeding the per-event cap, each replaced by a notice. Nonzero +// means text is missing from the durable copy on purpose — see messages. +func (b *Batcher) Oversized() int { return b.oversized } + +func (b *Batcher) events(l Line) []Event { + at := l.At + if at.Before(b.last) { + at = b.last + b.clamped++ + } + b.last = at + + msgs := b.messages(l) + out := make([]Event, 0, len(msgs)) + for _, m := range msgs { + out = append(out, Event{Message: m, At: at, Cursor: l.Cursor}) + } + return out +} + +// messages formats a line, replacing it with a notice if the result exceeds the per-event cap. +// +// Dropped rather than split, and the loss is deliberate. A split has to cut the line's DATA, and an +// envelope has no decodable prefix — Record.Formatter's adopt is all-or-nothing, so every piece would +// be re-wrapped with the Record's fallbacks, INFO/user. The consumer gates visibility on that category +// and hides `private` from callers without the developer-view role, so splitting a private line +// publishes it. Losing the text is the lesser failure; the notice is what keeps it from being a silent +// one. +func (b *Batcher) messages(l Line) []string { + msg := b.format(l) + // Only the unset cap disables this. A positive MaxEventBytes below PerEventOverhead leaves no room + // for any payload at all, and reading that as "unlimited" would ship an event guaranteed to breach + // the cap — failing open on the one misconfiguration this guard exists to catch. + if b.limits.MaxEventBytes <= 0 { + return []string{msg} + } + limit := b.limits.MaxEventBytes - b.limits.PerEventOverhead + if len(msg) <= limit { + return []string{msg} + } + + b.oversized++ + // The line's own timestamp and cursor, so the gap is locatable in the stream — but no part of the + // line itself, since whatever made it too long is exactly what must not be published unlabelled. + notice := b.format(withData(l, fmt.Sprintf( + "logship dropped a %d-byte line: over the %d-byte event limit", len(l.Data), limit))) + if len(notice) > limit { + // A cap below the record's own envelope, which is a misconfiguration rather than a long line. + // Nothing is shipped at all: emitting an over-cap event is the one thing the cap forbids, and + // the counter is what says so. + return nil + } + return []string{notice} +} + +// runeBoundary rounds n down to a rune boundary. A cut inside a multi-byte rune reaches the sink as +// U+FFFD, which corrupts the durable copy rather than merely splitting it. +func runeBoundary(s string, n int) int { + for n > 0 && n < len(s) && !utf8.RuneStart(s[n]) { + n-- + } + return n +} + +func withData(l Line, data string) Line { + l.Data = data + return l +} + +func (b *Batcher) push(e Event) []Event { + size := len(e.Message) + b.limits.PerEventOverhead + + var full []Event + if len(b.pending) > 0 && b.exceeds(len(b.pending)+1, b.bytes+size) { + full = b.Flush() + } + b.pending = append(b.pending, e) + b.bytes += size + return full +} + +func (b *Batcher) exceeds(events, bytes int) bool { + return (b.limits.MaxEvents > 0 && events > b.limits.MaxEvents) || + (b.limits.MaxBytes > 0 && bytes > b.limits.MaxBytes) +} diff --git a/components/logship/internal/ship/batch_test.go b/components/logship/internal/ship/batch_test.go new file mode 100644 index 0000000..50210da --- /dev/null +++ b/components/logship/internal/ship/batch_test.go @@ -0,0 +1,376 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ship + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +func TestBatcher_CountsPerEventOverhead(t *testing.T) { + // The whole point of the field: by message length these four fit in one request (28 bytes of + // 100), and the request would be rejected at 132. + b := NewBatcher(Limits{MaxBytes: 100, PerEventOverhead: 26}, FormatCompact) + lines := make([]Line, 4) + for i := range lines { + lines[i] = Line{Data: "abc", At: t1, Cursor: "1-0"} // "1-0 abc" = 7 + 26 = 33 bytes + } + + got := b.Add(lines...) + if len(got) != 1 || len(got[0]) != 3 { + t.Fatalf("got %v, want one request of 3 events", sizes(got)) + } + if rest := b.Flush(); len(rest) != 1 { + t.Fatalf("%d events pending, want the fourth", len(rest)) + } +} + +func TestBatcher_CapsTheEventCount(t *testing.T) { + b := NewBatcher(Limits{MaxEvents: 2}, FormatCompact) + lines := make([]Line, 5) + for i := range lines { + lines[i] = Line{Data: "x", At: t1, Cursor: "1-0"} + } + + if got := b.Add(lines...); len(got) != 2 || len(got[0]) != 2 || len(got[1]) != 2 { + t.Fatalf("got %v, want two requests of 2", sizes(got)) + } + if rest := b.Flush(); len(rest) != 1 { + t.Fatalf("%d events pending, want the fifth", len(rest)) + } +} + +func TestBatcher_DropsAnOversizedLine(t *testing.T) { + b := NewBatcher(Limits{MaxEventBytes: 100}, FormatCompact) + line := Line{Data: strings.Repeat("x", 500), At: t1, Cursor: "100-0"} + + b.Add(line) + events := b.Flush() + if len(events) != 1 { + t.Fatalf("%d events, want one notice", len(events)) + } + e := events[0] + if len(e.Message) > 100 { + t.Fatalf("the notice itself is %d bytes, over the cap", len(e.Message)) + } + // The notice stands in for the line, so it has to be findable where the line was. + if e.Cursor != "100-0" || !e.At.Equal(t1) { + t.Fatalf("notice carries %q at %v, want the line's own", e.Cursor, e.At) + } + if !strings.Contains(e.Message, "dropped") || !strings.Contains(e.Message, "500") { + t.Fatalf("notice %q does not say what was dropped", e.Message) + } + if b.Oversized() != 1 { + t.Fatalf("Oversized() = %d, want 1", b.Oversized()) + } +} + +func TestBatcher_DropDoesNotRelabelAnEnvelope(t *testing.T) { + // The whole reason an oversized line is dropped rather than split. A cut envelope has no decodable + // prefix, so adopt would reject every piece and Record.Formatter would wrap each with its own + // fallbacks — INFO/user. The consumer shows category user to everyone and private only to the + // developer-view role, so a split would publish the text of a private line. + format := Record{Pod: "p", Labels: map[string]string{"app": "sandbox"}}.Formatter() + b := NewBatcher(Limits{MaxEventBytes: 400}, format) + const secret = "PRIVATE-TRACEBACK-" + data := `{"level":"ERROR","category":"private","message":"` + strings.Repeat(secret, 100) + `"}` + + b.Add(Line{Data: data, At: t1, Cursor: "100-0"}) + events := b.Flush() + if len(events) != 1 { + t.Fatalf("%d events, want one notice", len(events)) + } + if strings.Contains(events[0].Message, secret) { + t.Fatalf("the dropped line's own text was shipped: %q", events[0].Message) + } + if len(events[0].Message) > 400 { + t.Fatalf("the notice itself is %d bytes, over the cap", len(events[0].Message)) + } + rec := decodeRecord(t, events[0].Message) + if rec.inner.ID != "100-0" { + t.Fatalf("notice carries id %q, want the line's", rec.inner.ID) + } +} + +func TestBatcher_ShipsNothingWhenEvenTheNoticeCannotFit(t *testing.T) { + // A cap below the record envelope is a misconfiguration, and the invariant still holds: whatever + // else happens, no event over the cap is emitted. The counter is the only trace left. + b := NewBatcher(Limits{MaxEventBytes: 20}, FormatCompact) + + b.Add(Line{Data: strings.Repeat("x", 100), At: t1, Cursor: "100-0"}) + if events := b.Flush(); events != nil { + t.Fatalf("shipped %v, want nothing", events) + } + if b.Oversized() != 1 { + t.Fatalf("Oversized() = %d, want 1", b.Oversized()) + } +} + +func TestBatcher_TreatsOnlyAnUnsetCapAsUnlimited(t *testing.T) { + // A cap at or below PerEventOverhead leaves no room for a payload, which is a tighter version of the + // case above and must not come out the other side as "no cap at all". Reading it that way made the + // guard fail open on exactly the misconfiguration it exists to catch — and inconsistently, since one + // byte looser already shipped nothing. + line := Line{Data: strings.Repeat("x", 100), At: t1, Cursor: "100-0"} + + for _, mb := range []int{1, 26, 27} { + b := NewBatcher(Limits{MaxEventBytes: mb, PerEventOverhead: 26}, FormatCompact) + b.Add(line) + if events := b.Flush(); events != nil { + t.Fatalf("MaxEventBytes=%d shipped %v, want nothing", mb, events) + } + if b.Oversized() != 1 { + t.Fatalf("MaxEventBytes=%d: Oversized() = %d, want 1", mb, b.Oversized()) + } + } + + // Unset is still the documented way to disable it — see Limits. + b := NewBatcher(Limits{PerEventOverhead: 26}, FormatCompact) + b.Add(line) + if events := b.Flush(); len(events) != 1 || events[0].Message != "100-0 "+line.Data { + t.Fatalf("got %v, want the line unchanged", events) + } +} + +func TestBatcher_KeepsALineThatFits(t *testing.T) { + b := NewBatcher(Limits{MaxEventBytes: 40}, FormatCompact) + line := Line{Data: strings.Repeat("x", 10), At: t1, Cursor: "1-0"} + + b.Add(line) + events := b.Flush() + if len(events) != 1 || events[0].Message != "1-0 "+line.Data { + t.Fatalf("got %v, want the line unchanged", events) + } + if b.Oversized() != 0 { + t.Fatalf("Oversized() = %d, want 0", b.Oversized()) + } +} + +func TestRecordFormatter_MatchesTheConsumersSchema(t *testing.T) { + // Every field here is read by the consumer's log client: labels by the tenant filter, pod_name by + // the worker-index lookup, and level/category from INSIDE log — a plain string there is treated + // as private, which hides the line from anyone without the developer-view role. + labels := map[string]string{ + "app": "sandbox", + "example.com/org-id": "org-1", + "example.com/team-id": "team-1", + "example.com/experiment-id": "exp-1", + } + format := Record{Pod: "worker-3", Labels: labels}.Formatter() + + rec := decodeRecord(t, format(Line{Data: "training step 1", At: t1, Cursor: "100-0"})) + if rec.Kubernetes.PodName != "worker-3" { + t.Fatalf("pod_name = %q", rec.Kubernetes.PodName) + } + for k, v := range labels { + if rec.Kubernetes.Labels[k] != v { + t.Fatalf("labels[%q] = %q, want %q", k, rec.Kubernetes.Labels[k], v) + } + } + if rec.Time != t1.UTC().Format(time.RFC3339Nano) { + t.Fatalf("time = %q, want %q", rec.Time, t1.UTC().Format(time.RFC3339Nano)) + } + if rec.inner.Message != "training step 1" { + t.Fatalf("message = %q", rec.inner.Message) + } + if rec.inner.Level != LevelInfo || rec.inner.Category != CategoryUser { + t.Fatalf("level/category = %q/%q, want %q/%q", rec.inner.Level, rec.inner.Category, LevelInfo, CategoryUser) + } +} + +func TestRecordFormatter_KeepsTheWorkloadsOwnLevelAndCategory(t *testing.T) { + // The consumer unwraps log exactly once, so an envelope nested inside ours is never read: every + // sandbox line arrived as INFO/user with the real envelope stranded in message as text. A workload + // that logs `system` is then absent from a system query while its user query doubles. + format := Record{Pod: "p"}.Formatter() + data := `{"level":"WARN","category":"system","message":"loading checkpoint"}` + + rec := decodeRecord(t, format(Line{Data: data, At: t1, Cursor: "100-0"})) + if rec.inner.Level != "WARN" || rec.inner.Category != "system" { + t.Fatalf("level/category = %q/%q, want WARN/system", rec.inner.Level, rec.inner.Category) + } + if rec.inner.Message != "loading checkpoint" { + t.Fatalf("message = %q, want the text, not the envelope", rec.inner.Message) + } + if rec.inner.ID != "100-0" { + t.Fatalf("id = %q, want the cursor", rec.inner.ID) + } +} + +func TestRecordFormatter_SuppliesOnlyTheCategoryAnEnvelopeLacks(t *testing.T) { + // Category is the visibility gate — an envelope without one is read as private and hidden from + // every caller lacking the developer-view role. Level needs no such fallback: the consumer + // defaults it to INFO itself, so adding one would only overwrite what the workload meant. + format := Record{Pod: "p"}.Formatter() + + rec := decodeRecord(t, format(Line{Data: `{"level":"DEBUG","message":"x"}`, At: t1, Cursor: "1-0"})) + if rec.inner.Category != CategoryUser { + t.Fatalf("category = %q, want %q", rec.inner.Category, CategoryUser) + } + if rec.inner.Level != "DEBUG" { + t.Fatalf("level = %q, want the workload's", rec.inner.Level) + } +} + +func TestRecordFormatter_WrapsWhatTheConsumerCouldNotRead(t *testing.T) { + // Adopting an object the consumer cannot decode into {level, category, message} strings would be + // worse than wrapping it: its decode fails, the category falls back to private, and the line + // disappears. So the bar is not "valid JSON" but "an envelope that survives that decode". + for _, data := range []string{ + "training step 1", // the ordinary case: not JSON at all + ` {"message":"x"`, // truncated, e.g. a line the sandbox was killed mid-write + `{"message":42}`, // decodes to nothing the consumer can render + `{"level":{},"message":"x"}`, + `{"message":null}`, + `{"level":"WARN"}`, // an object, but no text to show + `[{"message":"x"}]`, + `{"message":"x"} trailing`, + } { + rec := decodeRecord(t, Record{Pod: "p"}.Formatter()(Line{Data: data, At: t1, Cursor: "1-0"})) + if rec.inner.Message != data { + t.Fatalf("%q: message = %q, want the line verbatim", data, rec.inner.Message) + } + if rec.inner.Category != CategoryUser || rec.inner.Level != LevelInfo { + t.Fatalf("%q: level/category = %q/%q, want the fallbacks", data, rec.inner.Level, rec.inner.Category) + } + } +} + +func TestRecordFormatter_CursorOutranksAWorkloadsOwnID(t *testing.T) { + // Duplicate keys are legal and every decoder keeps the last, which is why the cursor is appended + // rather than prepended: a workload logging its own id would otherwise shadow the only thing that + // can resume a stream. + format := Record{Pod: "p"}.Formatter() + + rec := decodeRecord(t, format(Line{Data: `{"id":"theirs","message":"x"}`, At: t1, Cursor: "100-0"})) + if rec.inner.ID != "100-0" { + t.Fatalf("id = %q, want the cursor", rec.inner.ID) + } +} + +func TestRecordFormatter_EscapesWhatWouldBreakAQuery(t *testing.T) { + // Twice over: the text is escaped into log, and log is escaped into the record. A quote in the + // output reaches CloudWatch as `\\\"`, and getting either layer wrong loses the whole line. + want := "say \"hi\"\tC:\\path\x1b[0m" + format := Record{Pod: "p"}.Formatter() + + if got := decodeRecord(t, format(Line{Cursor: "100-0", Data: want})).inner.Message; got != want { + t.Fatalf("round-tripped to %q, want %q", got, want) + } +} + +func TestRecordFormatter_IsByteStableAcrossLabelOrder(t *testing.T) { + // Map iteration order is random, and an unstable encoding would make every size calculation in + // fit() — and every diff of a shipped stream — depend on it. + labels := map[string]string{"app": "a", "b": "2", "c": "3", "d": "4", "e": "5"} + line := Line{Data: "x", At: t1, Cursor: "1-0"} + + want := Record{Pod: "p", Labels: labels}.Formatter()(line) + for range 20 { + if got := (Record{Pod: "p", Labels: labels}).Formatter()(line); got != want { + t.Fatalf("encoding varies with map order:\n%q\n%q", got, want) + } + } +} + +// decodeRecord parses the two nested envelopes a Record produces, failing the test if either layer +// is malformed. +func decodeRecord(t *testing.T, msg string) recordShape { + t.Helper() + var rec recordShape + if err := json.Unmarshal([]byte(msg), &rec); err != nil { + t.Fatalf("record %q is not valid JSON: %v", msg, err) + } + if err := json.Unmarshal([]byte(rec.Log), &rec.inner); err != nil { + t.Fatalf("log field %q is not valid JSON: %v", rec.Log, err) + } + return rec +} + +// recordShape mirrors the consumer's log record, plus the inner envelope its buildLine looks for. +type recordShape struct { + Time string `json:"time"` + Log string `json:"log"` + Kubernetes struct { + Labels map[string]string `json:"labels"` + PodName string `json:"pod_name"` + } `json:"kubernetes"` + + inner struct { + Level string `json:"level"` + Category string `json:"category"` + Message string `json:"message"` + ID string `json:"id"` + } +} + +func TestBatcher_ClampsATimestampThatGoesBackwards(t *testing.T) { + // One out-of-order event rejects the whole request, so this cannot be left to the sink. Clamping + // misdates a line by milliseconds; sorting would reorder the log itself. + b := NewBatcher(Limits{}, FormatCompact) + b.Add( + Line{Data: "later", At: t2, Cursor: "200-0"}, + Line{Data: "earlier", At: t1, Cursor: "100-0"}, + ) + + events := b.Flush() + if len(events) != 2 { + t.Fatalf("%d events, want 2", len(events)) + } + if !events[1].At.Equal(t2) { + t.Fatalf("At = %v, want it raised to %v", events[1].At, t2) + } + if b.Clamped() != 1 { + t.Fatalf("Clamped() = %d, want 1", b.Clamped()) + } +} + +func TestBatcher_FormatKeepsABlankLineNonEmpty(t *testing.T) { + // An empty message is rejected outright by PutLogEvents, and a blank line is real output. The + // prefix is what stops those two facts from colliding. + if got := FormatCompact(Line{Data: "", Cursor: "100-0"}); got == "" { + t.Fatal("a blank line formatted to an empty message") + } +} + +func TestBatcher_ZeroLimitsHoldEverything(t *testing.T) { + b := NewBatcher(Limits{}, FormatCompact) + lines := make([]Line, 1000) + for i := range lines { + lines[i] = Line{Data: strings.Repeat("x", 1000), At: t1, Cursor: "1-0"} + } + + if got := b.Add(lines...); got != nil { + t.Fatalf("got %v, want no request when no limit applies", sizes(got)) + } + if rest := b.Flush(); len(rest) != 1000 { + t.Fatalf("%d events, want all of them", len(rest)) + } + if again := b.Flush(); again != nil { + t.Fatalf("second Flush gave %d events, want nothing", len(again)) + } +} + +func sizes(batches [][]Event) []int { + out := make([]int, 0, len(batches)) + for _, b := range batches { + out = append(out, len(b)) + } + return out +} diff --git a/components/logship/internal/ship/pipeline.go b/components/logship/internal/ship/pipeline.go new file mode 100644 index 0000000..3444f90 --- /dev/null +++ b/components/logship/internal/ship/pipeline.go @@ -0,0 +1,311 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ship + +import ( + "context" + "sync" + "time" +) + +// Defaults sized for the fleet rather than for one stream: what matters is the per-stream bound +// times 1,000 streams, so these are 64 MiB of buffers and 200 requests per second, not 1 GiB and +// 1,000. See design.md's Scale section. +const ( + DefaultInterval = 5 * time.Second + DefaultMaxPendingBytes = 64 << 10 + + // Only throttling is retried, and not for long: a stream that retries forever is a stream whose + // logs age out of Modal while it waits. + maxPutAttempts = 5 + baseBackoff = 200 * time.Millisecond +) + +// Config describes one stream's copy. Source, Sink and Format are required. +type Config struct { + Source Source + Sink Sink + Format Formatter + Limits Limits + + // Cursor is where to resume. Empty means the source's beginning, which is a full replay of + // whatever it still holds. + Cursor string + // CollapseFrames discards a progress bar's superseded frames. See Assembler. + CollapseFrames bool + + // Interval flushes a stream too quiet to fill a batch. Zero means DefaultInterval. + Interval time.Duration + // MaxPendingBytes bounds lines read but not yet shipped. Zero means DefaultMaxPendingBytes; + // negative means unbounded, which trades the memory bound for never dropping. + MaxPendingBytes int + + // Throttled tells "slow down" apart from "this batch is lost". Nil retries nothing. It is a + // function rather than an error check here because the answer belongs to the sink's API, and + // this package must not import an adapter. + Throttled func(error) bool + // Shipped is called with the cursor of the last event of every accepted batch, in order. It is + // the record half of read -> put -> record: without it a restart replays from the beginning, + // which the consumer shows as duplicated lines. + Shipped func(cursor string) + + // Log reports a failure worth an operator's attention, and matches supervise's signature so cmd + // passes the same function to both. Nil is silent, which is fine for tests: the counters still + // hold what happened. + Log func(msg string, keysAndValues ...any) +} + +// Stats is one stream's counters. Dropped and Failed are the two ways a line does not arrive, and +// they are separate because one is our backpressure and the other is the sink refusing. +type Stats struct { + Lines int + Events int + // Dropped counts lines the reader threw away because the buffer was full. + Dropped int + DroppedBytes int + // Failed counts events the sink would not take, after retries. + Failed int + // Clamped counts lines whose timestamp was raised to keep a batch ordered. See Batcher. + Clamped int + // Oversized counts lines dropped for exceeding the per-event cap, each shipped as a notice + // instead. A third way a line does not arrive, and separate for the same reason: this one is our + // own format's cap, not backpressure or the sink. See Batcher.messages. + Oversized int +} + +// Pipeline copies one stream: source -> assembler -> batcher -> sink. +// +// Two goroutines, and it has to be two. The source calls back on its own goroutine and a stalled +// callback stalls the cursor, so shipping cannot happen inline; and a stream quiet enough to need +// the interval flush needs a select the synchronous callback cannot provide. They are joined by a +// byte-bounded queue that drops rather than blocks — a reader waiting on CloudWatch is a reader not +// draining Modal, and Modal's retention is the clock this whole component exists to beat. +type Pipeline struct { + cfg Config + queue *queue + + mu sync.Mutex + stats Stats +} + +func New(cfg Config) *Pipeline { + if cfg.Interval <= 0 { + cfg.Interval = DefaultInterval + } + if cfg.MaxPendingBytes == 0 { + cfg.MaxPendingBytes = DefaultMaxPendingBytes + } + return &Pipeline{cfg: cfg, queue: newQueue(cfg.MaxPendingBytes)} +} + +// Run copies until the stream ends, ctx is cancelled, or the source fails, and returns the source's +// error. It always drains what it has already read first, including the unterminated fragment a +// sandbox that exits mid-line leaves behind — on cancellation those puts will fail with ctx, so a +// caller that wants the tail shipped gives the drain its own deadline rather than the cancelled one. +func (p *Pipeline) Run(ctx context.Context) error { + done := make(chan struct{}) + go func() { + defer close(done) + p.deliver(ctx) + }() + + asm := NewAssembler(p.cfg.CollapseFrames) + err := p.cfg.Source.Follow(ctx, p.cfg.Cursor, func(b Batch) error { + p.push(asm.Add(b)) + return nil + }) + p.push(asm.Flush()) + p.queue.close() + <-done + return err +} + +// Stats reports this stream's counters. Safe to call while Run is in flight. +func (p *Pipeline) Stats() Stats { + p.mu.Lock() + defer p.mu.Unlock() + out := p.stats + out.Dropped, out.DroppedBytes = p.queue.drops() + return out +} + +func (p *Pipeline) push(lines []Line) { + if len(lines) == 0 { + return + } + p.record(func(s *Stats) { s.Lines += len(lines) }) + p.queue.push(lines) +} + +func (p *Pipeline) deliver(ctx context.Context) { + b := NewBatcher(p.cfg.Limits, p.cfg.Format) + tick := time.NewTicker(p.cfg.Interval) + defer tick.Stop() + + for { + select { + case _, open := <-p.queue.wake: + // Taken before the open check, because close happens after the last push: a closed + // queue still has the final lines in it. + for _, full := range b.Add(p.queue.take()...) { + p.put(ctx, full) + } + if !open { + p.put(ctx, b.Flush()) + p.record(func(s *Stats) { s.Clamped, s.Oversized = b.Clamped(), b.Oversized() }) + return + } + case <-tick.C: + p.put(ctx, b.Flush()) + p.record(func(s *Stats) { s.Clamped, s.Oversized = b.Clamped(), b.Oversized() }) + } + } +} + +// put ships one batch, retrying only what the sink says is a rate problem. Anything else is counted +// and dropped: the alternative is a stream that stops advancing, and a cursor that stops advancing +// loses logs permanently rather than partially. +func (p *Pipeline) put(ctx context.Context, events []Event) { + if len(events) == 0 { + return + } + var err error + for attempt := range maxPutAttempts { + err = p.cfg.Sink.Put(ctx, events) + if err == nil { + p.record(func(s *Stats) { s.Events += len(events) }) + if p.cfg.Shipped != nil { + p.cfg.Shipped(events[len(events)-1].Cursor) + } + return + } + if p.cfg.Throttled == nil || !p.cfg.Throttled(err) { + break + } + if !sleep(ctx, baseBackoff< 0 && q.bytes+size > q.max && len(q.lines) > 0 { + q.dropped++ + // Payload only: this counter answers how much log was lost, not how much memory was + // refused. + q.droppedBytes += len(l.Data) + continue + } + q.lines = append(q.lines, l) + q.bytes += size + } + q.mu.Unlock() + + select { + case q.wake <- struct{}{}: + default: + } +} + +func (q *queue) take() []Line { + q.mu.Lock() + defer q.mu.Unlock() + out := q.lines + q.lines, q.bytes = nil, 0 + return out +} + +func (q *queue) drops() (int, int) { + q.mu.Lock() + defer q.mu.Unlock() + return q.dropped, q.droppedBytes +} + +func (q *queue) close() { close(q.wake) } diff --git a/components/logship/internal/ship/pipeline_test.go b/components/logship/internal/ship/pipeline_test.go new file mode 100644 index 0000000..a17b68a --- /dev/null +++ b/components/logship/internal/ship/pipeline_test.go @@ -0,0 +1,390 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ship + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + "time" +) + +func TestPipeline_ShipsEveryLineAndResumesFromTheCursor(t *testing.T) { + src := &fakeSource{batches: []Batch{ + {Entries: []Entry{{Data: "a\nb\n", At: t1}}, Cursor: "1-0"}, + {Entries: []Entry{{Data: "c\n", At: t2}}, Cursor: "2-0"}, + }} + sink := &fakeSink{} + var shipped []string + p := New(Config{ + Source: src, Sink: sink, Format: FormatCompact, + Cursor: "5-0", + Shipped: func(c string) { shipped = append(shipped, c) }, + }) + + if err := p.Run(context.Background()); err != nil { + t.Fatalf("Run: %v", err) + } + if src.cursor != "5-0" { + t.Fatalf("followed from %q, want the configured cursor", src.cursor) + } + want := []string{"1-0 a", "1-0 b", "2-0 c"} + if got := sink.messages(); !equal(got, want) { + t.Fatalf("shipped %q, want %q", got, want) + } + // The record half of read -> put -> record: the highest cursor of the batch, once, after it + // landed. Recording per line would checkpoint output that has not been accepted yet. + if !equal(shipped, []string{"2-0"}) { + t.Fatalf("recorded %q, want [2-0]", shipped) + } + if s := p.Stats(); s.Lines != 3 || s.Events != 3 || s.Dropped != 0 || s.Failed != 0 { + t.Fatalf("Stats = %+v", s) + } +} + +func TestPipeline_FlushesTheUnterminatedTail(t *testing.T) { + // A sandbox that exits mid-line has still printed that text, and it is usually the panic. + src := &fakeSource{batches: []Batch{ + {Entries: []Entry{{Data: "done\nfatal: no newline", At: t1}}, Cursor: "1-0"}, + }} + sink := &fakeSink{} + + if err := New(Config{Source: src, Sink: sink, Format: FormatCompact}).Run(context.Background()); err != nil { + t.Fatalf("Run: %v", err) + } + want := []string{"1-0 done", "1-0 fatal: no newline"} + if got := sink.messages(); !equal(got, want) { + t.Fatalf("shipped %q, want %q", got, want) + } +} + +func TestPipeline_ReturnsTheSourcesErrorAfterShippingWhatItRead(t *testing.T) { + // The lines before the failure are already durable-worthy; losing them because the stream broke + // afterwards would defeat the point. + boom := errors.New("stream broke") + src := &fakeSource{ + batches: []Batch{{Entries: []Entry{{Data: "before\n", At: t1}}, Cursor: "1-0"}}, + err: boom, + } + sink := &fakeSink{} + + if err := New(Config{Source: src, Sink: sink, Format: FormatCompact}).Run(context.Background()); !errors.Is(err, boom) { + t.Fatalf("Run = %v, want %v", err, boom) + } + if got := sink.messages(); !equal(got, []string{"1-0 before"}) { + t.Fatalf("shipped %q, want the line read before the failure", got) + } +} + +func TestPipeline_FlushesAQuietStreamOnTheInterval(t *testing.T) { + // The batch is nowhere near full and the stream stays open, so nothing but the clock will ship + // this. Without it a low-volume workload's logs sit in memory for the length of the run. + src := &holdSource{ + batches: []Batch{{Entries: []Entry{{Data: "one quiet line\n", At: t1}}, Cursor: "1-0"}}, + release: make(chan struct{}), + } + sink := &fakeSink{} + p := New(Config{Source: src, Sink: sink, Format: FormatCompact, Interval: 10 * time.Millisecond}) + + done := make(chan error, 1) + go func() { done <- p.Run(context.Background()) }() + + if !waitFor(func() bool { return len(sink.messages()) == 1 }) { + t.Fatal("nothing shipped while the stream stayed open") + } + close(src.release) + if err := <-done; err != nil { + t.Fatalf("Run: %v", err) + } +} + +func TestPipeline_DropsRatherThanStallingTheReader(t *testing.T) { + // The trade the whole component turns on: a reader blocked on CloudWatch is a reader not + // draining Modal, and Modal is the copy on a retention clock. So the buffer overflows into a + // counter and the reader keeps going. + lines := make([]Entry, 200) + for i := range lines { + lines[i] = Entry{Data: strings.Repeat("x", 100) + "\n", At: t1} + } + sink := &fakeSink{gate: make(chan struct{})} + src := &fakeSource{batches: []Batch{{Entries: lines, Cursor: "1-0"}}} + p := New(Config{Source: src, Sink: sink, Format: FormatCompact, MaxPendingBytes: 500}) + + done := make(chan error, 1) + go func() { done <- p.Run(context.Background()) }() + + // The reader finishes all 200 lines with the sink held shut. + if !waitFor(func() bool { return p.Stats().Lines == 200 }) { + t.Fatalf("reader stalled at %d lines", p.Stats().Lines) + } + close(sink.gate) + if err := <-done; err != nil { + t.Fatalf("Run: %v", err) + } + + s := p.Stats() + if s.Dropped == 0 { + t.Fatal("nothing dropped, so the bound did not apply") + } + if s.DroppedBytes < s.Dropped { + t.Fatalf("Dropped = %d but DroppedBytes = %d", s.Dropped, s.DroppedBytes) + } + if s.Events+s.Dropped != s.Lines { + t.Fatalf("%d shipped + %d dropped != %d read", s.Events, s.Dropped, s.Lines) + } +} + +func TestPipeline_CountsWhatAQueuedLineCostsBesidesItsBytes(t *testing.T) { + // A blank line is real output with no payload, so a payload-only bound does not bound it at all: + // 2,000 of them would sit inside a 500-byte buffer, which is the unbounded line count the byte + // bound replaced. + src := &fakeSource{batches: []Batch{ + {Entries: []Entry{{Data: strings.Repeat("\n", 2000), At: t1}}, Cursor: "1-0"}, + }} + p := New(Config{Source: src, Sink: &fakeSink{}, Format: FormatCompact, MaxPendingBytes: 500}) + + if err := p.Run(context.Background()); err != nil { + t.Fatalf("Run: %v", err) + } + s := p.Stats() + if s.Lines != 2000 { + t.Fatalf("Lines = %d, want one per newline", s.Lines) + } + if s.Dropped == 0 { + t.Fatal("2,000 blank lines fit a 500-byte buffer, so the bound is not a memory bound") + } + if s.Events+s.Dropped != s.Lines { + t.Fatalf("%d shipped + %d dropped != %d read", s.Events, s.Dropped, s.Lines) + } +} + +func TestPipeline_AdmitsALineBiggerThanTheWholeBuffer(t *testing.T) { + // Otherwise a line over the bound is unshippable forever rather than merely awkward, and the + // assembler emits exactly such a line at maxFragment. + src := &fakeSource{batches: []Batch{ + {Entries: []Entry{{Data: strings.Repeat("y", 4096) + "\n", At: t1}}, Cursor: "1-0"}, + }} + sink := &fakeSink{} + p := New(Config{Source: src, Sink: sink, Format: FormatCompact, MaxPendingBytes: 64}) + + if err := p.Run(context.Background()); err != nil { + t.Fatalf("Run: %v", err) + } + if s := p.Stats(); s.Events != 1 || s.Dropped != 0 { + t.Fatalf("Stats = %+v, want the oversized line shipped", s) + } +} + +func TestPipeline_RetriesAThrottleAndOnlyAThrottle(t *testing.T) { + throttle := errors.New("slow down") + src := &fakeSource{batches: []Batch{{Entries: []Entry{{Data: "a\n", At: t1}}, Cursor: "1-0"}}} + sink := &fakeSink{failures: 1, err: throttle} + p := New(Config{ + Source: src, Sink: sink, Format: FormatCompact, + Throttled: func(err error) bool { return errors.Is(err, throttle) }, + }) + + if err := p.Run(context.Background()); err != nil { + t.Fatalf("Run: %v", err) + } + if sink.attempts() != 2 { + t.Fatalf("%d attempts, want the throttled one then the retry", sink.attempts()) + } + if s := p.Stats(); s.Events != 1 || s.Failed != 0 { + t.Fatalf("Stats = %+v, want the retry to have landed", s) + } +} + +func TestPipeline_DropsWhatTheSinkRefusesWithoutRetrying(t *testing.T) { + // A malformed batch retried five times is five times the damage and the same outcome. Counted + // and abandoned, so the cursor keeps moving. + src := &fakeSource{batches: []Batch{{Entries: []Entry{{Data: "a\n", At: t1}}, Cursor: "1-0"}}} + sink := &fakeSink{failures: 10, err: errors.New("invalid")} + var shipped []string + p := New(Config{ + Source: src, Sink: sink, Format: FormatCompact, + Throttled: func(error) bool { return false }, + Shipped: func(c string) { shipped = append(shipped, c) }, + }) + + if err := p.Run(context.Background()); err != nil { + t.Fatalf("Run: %v", err) + } + if sink.attempts() != 1 { + t.Fatalf("%d attempts, want no retry for a non-throttle", sink.attempts()) + } + if s := p.Stats(); s.Failed != 1 || s.Events != 0 { + t.Fatalf("Stats = %+v, want one failed event", s) + } + // The cursor must not advance past output that was never stored, or the loss becomes permanent + // at the next restart instead of recoverable. + if len(shipped) != 0 { + t.Fatalf("recorded %q for a batch that never landed", shipped) + } +} + +func TestPipeline_SaysSoOnceWhenTheSinkRefuses(t *testing.T) { + // The sink is shared by every stream, so a broken one loses the whole fleet's logs with the + // process still up. One line rather than one per batch, or the flood buries what it announces. + src := &fakeSource{batches: []Batch{ + {Entries: []Entry{{Data: "a\nb\nc\n", At: t1}}, Cursor: "1-0"}, + }} + sink := &fakeSink{failures: 10, err: errors.New("invalid")} + var logged int + p := New(Config{ + Source: src, Sink: sink, Format: FormatCompact, + // One event per put, so the dedup is actually exercised rather than hidden by batching. + Limits: Limits{MaxEvents: 1}, + Log: func(string, ...any) { logged++ }, + }) + + if err := p.Run(context.Background()); err != nil { + t.Fatalf("Run: %v", err) + } + if sink.attempts() != 3 { + t.Fatalf("%d puts, want one per line", sink.attempts()) + } + if s := p.Stats(); s.Failed != 3 { + t.Fatalf("Stats = %+v, want every lost line counted", s) + } + if logged != 1 { + t.Fatalf("logged %d times for 3 refused batches, want 1", logged) + } +} + +func TestPipeline_CancellationStopsTheReaderAndTheShipper(t *testing.T) { + src := &holdSource{release: make(chan struct{})} + p := New(Config{Source: src, Sink: &fakeSink{}, Format: FormatCompact}) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- p.Run(ctx) }() + cancel() + + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("Run = %v, want context.Canceled", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Run did not return after cancellation") + } +} + +type fakeSource struct { + batches []Batch + err error + cursor string +} + +func (f *fakeSource) Follow(_ context.Context, cursor string, fn func(Batch) error) error { + f.cursor = cursor + for _, b := range f.batches { + if err := fn(b); err != nil { + return err + } + } + return f.err +} + +// holdSource emits its batches and then stays open, which is what a live sandbox does and what makes +// the interval flush and cancellation observable. +type holdSource struct { + batches []Batch + release chan struct{} +} + +func (h *holdSource) Follow(ctx context.Context, _ string, fn func(Batch) error) error { + for _, b := range h.batches { + if err := fn(b); err != nil { + return err + } + } + select { + case <-h.release: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +type fakeSink struct { + mu sync.Mutex + // failures is how many leading Puts return err; gate, when set, holds the first Put until it is + // closed, which is how the reader gets observed running ahead of the sink. + failures int + err error + gate chan struct{} + + tries int + got []Event +} + +func (f *fakeSink) Put(_ context.Context, events []Event) error { + if f.gate != nil { + <-f.gate + } + f.mu.Lock() + defer f.mu.Unlock() + f.tries++ + if f.failures > 0 { + f.failures-- + return f.err + } + f.got = append(f.got, events...) + return nil +} + +func (f *fakeSink) messages() []string { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]string, 0, len(f.got)) + for _, e := range f.got { + out = append(out, e.Message) + } + return out +} + +func (f *fakeSink) attempts() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.tries +} + +func waitFor(cond func() bool) bool { + for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); { + if cond() { + return true + } + time.Sleep(time.Millisecond) + } + return cond() +} + +func equal(got, want []string) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true +} diff --git a/components/logship/internal/ship/port.go b/components/logship/internal/ship/port.go new file mode 100644 index 0000000..38a3b11 --- /dev/null +++ b/components/logship/internal/ship/port.go @@ -0,0 +1,46 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ship + +import "context" + +// Source follows one stream, delivering batches strictly after cursor and returning when the stream +// ends. An adapter binds whatever identifies the stream — for Modal, a sandbox and a file descriptor +// — so nothing here has to know what a stream is named. +// +// fn's error aborts the follow, and a source must not lose the batch it was called with: the caller +// decides what a rejected batch means, and the only cursor is the one it has accepted. +type Source interface { + Follow(ctx context.Context, cursor string, fn func(Batch) error) error +} + +// Sink appends events somewhere durable. +// +// Append-only, deliberately: the shipper never reads its destination, so there is no Tail and the +// cursor is never recovered from what landed. See design.md — that decision is what makes a +// restart a replay. +// +// Put's events are already sized and ordered for the sink by Batcher; a sink that has limits states +// them as Limits rather than enforcing them again here. +// +// One Sink may be shared by every stream in the process, and the real one is: Put must be safe for +// concurrent use, and must deliver a batch whole rather than event by event, because a sink with one +// destination has no way to tell two interleaved batches apart afterwards. Nothing closes a Sink — +// a stream ending says nothing about a destination the other thousand are still writing to. +type Sink interface { + Put(ctx context.Context, events []Event) error +} diff --git a/components/logship/internal/ship/record.go b/components/logship/internal/ship/record.go new file mode 100644 index 0000000..da82511 --- /dev/null +++ b/components/logship/internal/ship/record.go @@ -0,0 +1,178 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ship + +import ( + "encoding/json" + "maps" + "slices" + "strings" + "time" +) + +// Fallbacks for a line that carries no envelope of its own. INFO because bare sandbox output has no +// level; "user" because the alternative is invisible — see Record.Category. +const ( + LevelInfo = "INFO" + CategoryUser = "user" +) + +// Record is the identity stamped into every event, and the reason it is per-event rather than in the +// stream name: the consumer pages the whole log group with FilterLogEvents and a +// `{$.…kubernetes.labels…}` pattern, never reading a stream name. Which also means a shipped event +// stays attributable after its Pod is deleted, since nothing has to be joined against the cluster. +// +// The shape is Fluent Bit's Kubernetes-filter output, copied rather than invented — but a record now +// travels through that filter instead of past it, so it arrives nested one level down under +// `log_processed` while a node's own logs stay at the top level. Same field names, two depths: the +// consumer needs a clause for each, and neither may be dropped. See emit and design.md. +type Record struct { + // Pod lands in kubernetes.pod_name, which the consumer maps a worker index onto. An empty one + // ships fine and is unreachable through that path. + Pod string + // Labels lands in kubernetes.labels. The consumer's tenant filter reads its org, team and + // experiment IDs from here, and the source name from `app` — a missing one is not an error here + // but a line no query will match. + Labels map[string]string + + // Level and Category go INSIDE the log field, not beside it: the consumer parses log as JSON and + // defaults a plain string to category "private", which is hidden from every caller without the + // developer-view role. Shipping raw stdout bare would ship it invisibly. + // + // Both are fallbacks, applied only to a line that is not already an envelope — see adopt. A + // workload that logs its own level and category keeps them, and overriding them here is what made + // every sandbox line arrive as INFO/user. + Level string + Category string +} + +// Formatter renders lines as this record. +// +// Built once per stream because the identity is constant for one: pod name and labels are encoded +// here and only the timestamp, cursor and text are rebuilt per line. +func (r Record) Formatter() Formatter { + category := nonEmpty(r.Category, CategoryUser) + // Doubly encoded, unavoidably: log is a JSON string whose contents are themselves JSON, so a + // quote in the output reaches CloudWatch as `\\\"`. That is the consumer's existing format. + head := `{"level":` + quoteJSON(nonEmpty(r.Level, LevelInfo)) + + `,"category":` + quoteJSON(category) + `,"id":` + tail := r.metadata() + + return func(l Line) string { + log, ok := adopt(l.Data, l.Cursor, category) + if !ok { + var inner strings.Builder + inner.WriteString(head) + encodeJSONString(&inner, l.Cursor) + inner.WriteString(`,"message":`) + encodeJSONString(&inner, l.Data) + inner.WriteByte('}') + log = inner.String() + } + + var b strings.Builder + b.WriteString(`{"time":"`) + // RFC3339Nano is what the CRI parser leaves in Fluent Bit's time field, and the consumer + // passes it through to its client verbatim. So the nanoseconds the sink has to truncate for + // PutLogEvents survive here for free. + b.WriteString(l.At.UTC().Format(time.RFC3339Nano)) + b.WriteString(`","log":`) + encodeJSONString(&b, log) + b.WriteString(tail) + return b.String() + } +} + +// adopt returns a line that is ALREADY an envelope, with the cursor added, plus a category if it had +// none. Not ok for anything else, which the caller wraps instead. +// +// This is the whole reason a workload's level and category survive: the consumer unwraps `log` exactly +// once, so nesting an envelope inside ours makes ours the only one it reads and buries theirs in +// `message` as text. +// +// The test is deliberately the consumer's own decode rather than "is this JSON". An object it cannot +// decode into {level, category, message} strings, or one with no message at all, falls back to +// category private and renders the raw JSON as the text — so adopting one would hide a line the wrap +// shows. Cost is a parse per line that starts with `{`, which is why the cheap byte test comes first. +func adopt(data, cursor, category string) (string, bool) { + s := strings.TrimSpace(data) + if len(s) == 0 || s[0] != '{' { + return "", false + } + // Pointers to tell an absent category from an empty one, and to reject a null message the way the + // consumer's plain strings would not. Level is read by nothing and still has to be declared: it is + // what makes a non-string level fail this decode, as it would fail the consumer's. + var env struct { + Category *string `json:"category"` + Message *string `json:"message"` + Level *string `json:"level"` + } + if json.Unmarshal([]byte(s), &env) != nil || env.Message == nil { + return "", false + } + + // Spliced in before the closing brace rather than re-marshalled, so the workload's field order, + // number formatting and unknown fields survive untouched. Last also settles a collision: every + // JSON decoder keeps the last of a duplicate key, so a workload logging its own `id` cannot + // shadow the cursor. No level is added — the consumer already defaults that to INFO. + var b strings.Builder + b.WriteString(s[:len(s)-1]) + b.WriteByte(',') + if env.Category == nil || *env.Category == "" { + b.WriteString(`"category":`) + encodeJSONString(&b, category) + b.WriteByte(',') + } + b.WriteString(`"id":`) + encodeJSONString(&b, cursor) + b.WriteByte('}') + return b.String(), true +} + +// metadata encodes the kubernetes block and the closing braces of the whole record. +// +// Labels are sorted so a stream's events are byte-identical run to run, which map order would +// otherwise make random. +func (r Record) metadata() string { + var b strings.Builder + b.WriteString(`,"kubernetes":{"pod_name":`) + encodeJSONString(&b, r.Pod) + b.WriteString(`,"labels":{`) + for i, k := range slices.Sorted(maps.Keys(r.Labels)) { + if i > 0 { + b.WriteByte(',') + } + encodeJSONString(&b, k) + b.WriteByte(':') + encodeJSONString(&b, r.Labels[k]) + } + b.WriteString(`}}}`) + return b.String() +} + +func quoteJSON(s string) string { + var b strings.Builder + encodeJSONString(&b, s) + return b.String() +} + +func nonEmpty(s, fallback string) string { + if s == "" { + return fallback + } + return s +} diff --git a/components/logship/internal/ship/ship.go b/components/logship/internal/ship/ship.go new file mode 100644 index 0000000..eb88d0d --- /dev/null +++ b/components/logship/internal/ship/ship.go @@ -0,0 +1,193 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package ship holds the ports and the mechanism, and imports neither of its adapters: a provider +// satisfies Source, a log service satisfies Sink, and cmd is where the two meet. Everything here +// is scoped to ONE stream — one sandbox, one file descriptor — because that is what makes a cursor +// meaningful and a CloudWatch stream monotonic. +package ship + +import ( + "strings" + "time" +) + +// Entry is one chunk of output as a source produced it. Data is not a line: it may hold several +// newlines, or half of one. See Assembler. +type Entry struct { + Data string + At time.Time +} + +// Batch is a source's batch, with the cursor that resumes strictly after it. +type Batch struct { + Entries []Entry + Cursor string +} + +// Line is one assembled line, with the cursor of the batch it was COMPLETED in. +// +// Completed, not started: a resume from that cursor re-delivers nothing this line needed, whereas a +// resume from where the line began would re-deliver the rest of that batch. Duplicates over losses +// is the whole trade, but there is no reason to pay for a duplicate that is avoidable. +// +// Data can be empty — a blank line is real output. It is the sink's job not to turn that into an +// empty message, which the entry-ID prefix already prevents. +type Line struct { + Data string + At time.Time + Cursor string +} + +// maxFragment bounds one Line, terminated or not. +// +// Not defensive for the unterminated case: a `tqdm` progress bar emits a carriage-returned frame per +// update and no newline until the run ends, so an assembler that simply waited for one would hold a +// whole training run's progress output as a single pending line. A terminated line needs the same cap +// for a different reason, which is that the queue's byte bound assumes it — see fill. +const maxFragment = 64 * 1024 + +// Assembler turns one stream's chunks into lines. +// +// Not safe for concurrent use, and not meant to be: one assembler belongs to one stream, which is +// read by one goroutine. +type Assembler struct { + // collapseFrames discards a line's superseded carriage-returned frames, keeping only the + // newest. It trades the timing of a run's progress for its size — see docs on the ingest cost + // in design.md — and it is also what keeps a progress bar from reaching maxFragment at + // all, since each frame replaces the last instead of extending it. + collapseFrames bool + + buf strings.Builder + // at is the timestamp of the chunk that contributed the pending line's FIRST byte, so a line + // is dated when it was emitted rather than when it happened to finish. + at time.Time + cursor string + // pendingCR holds a chunk's trailing CR, which cannot be classified until the next byte arrives: + // see Add. Kept out of buf so that whichever it turns out to be, nothing has been committed yet. + pendingCR string +} + +func NewAssembler(collapseFrames bool) *Assembler { + return &Assembler{collapseFrames: collapseFrames} +} + +// Add returns the lines this batch completed. A trailing fragment is held for a later batch, or for +// Flush; anything that would exceed maxFragment is split instead, whether a newline terminated it or +// not — see fill. +func (a *Assembler) Add(b Batch) []Line { + a.cursor = b.Cursor + + var out []Line + for _, e := range b.Entries { + // A trailing CR is ambiguous until the next byte decides it: the first half of a CRLF + // terminator, or a progress frame's separator. Holding it back and putting it in front of the + // next chunk is what lets the loops below see it together with the byte that decides — a CR + // resolved against the wrong one either leaves a stray CR in the line or, once frames collapse, + // takes the whole line's text as overwritten. + data := a.pendingCR + e.Data + a.pendingCR = "" + if strings.HasSuffix(data, "\r") { + data, a.pendingCR = data[:len(data)-1], "\r" + } + for { + i := strings.IndexByte(data, '\n') + if i < 0 { + break + } + // A trailing CR here is a Windows line ending, not a progress frame, and dropping it + // before the frame logic is what stops "line\r\n" from collapsing to nothing. + out = a.fill(out, e.At, strings.TrimSuffix(data[:i], "\r")) + out = append(out, a.take()) + data = data[i+1:] + } + out = a.fill(out, e.At, data) + } + return out +} + +// fill writes s into the pending line, taking it whenever it would exceed maxFragment. +// +// Both of Add's paths go through here, because whether a chunk ends in a newline only decides WHICH +// bound a Line the size of that chunk defeats: the queue admits any single line into an empty buffer — +// it has to, or a long line could never ship at all — so it is a Line over maxFragment that turns the +// queue's byte bound into "one largest line". Splitting rather than writing and checking after is what +// keeps a megabyte-long item from ever being one Line. +// +// A CR in a later piece still discards what an earlier one left pending, so collapsing is unaffected +// within the buffer; what it cannot do any more is reach back past a piece already emitted. +func (a *Assembler) fill(out []Line, at time.Time, s string) []Line { + for { + room := maxFragment - a.buf.Len() + // Not `<`: an exact fit is not over the cap, and emitting it would leave Add's newline path + // taking an empty buffer as a blank line. A dump of any power-of-two size lands exactly here. + if len(s) <= room { + break + } + // Rounded down so a cut never lands inside a rune. Zero means the pending fragment left room + // for less than one rune, and taking it is what makes room. + n := runeBoundary(s, room) + if n == 0 && a.buf.Len() == 0 { + // Except when there was no boundary to find in a whole fragment's worth of bytes, where + // taking the pending line makes none either and the loop would spin without advancing. + // Cut at the cap: this is already invalid UTF-8, and a hang is the worse of the two. + n = room + } + a.write(at, s[:n]) + out = append(out, a.take()) + s = s[n:] + } + a.write(at, s) + return out +} + +// Flush returns the pending fragment, if any, as a line with no terminator of its own. For the end +// of a stream: a sandbox that exits mid-line has still printed that text. +// +// A CR left pending is dropped rather than resolved. Nothing follows it to overwrite what it separated, +// so collapsing on it here would erase text that was still on the terminal when the sandbox exited — +// and the other reading, a CRLF whose LF never came, wants it gone too. +func (a *Assembler) Flush() []Line { + a.pendingCR = "" + if a.buf.Len() == 0 { + return nil + } + return []Line{a.take()} +} + +func (a *Assembler) write(at time.Time, s string) { + if a.collapseFrames { + // Everything before the last CR has been overwritten on the terminal this was meant for, + // including whatever is already pending. + if i := strings.LastIndexByte(s, '\r'); i >= 0 { + a.buf.Reset() + s = s[i+1:] + } + } + // Before the write rather than after, and unguarded by s != "": a blank line is taken + // immediately by the caller, and it still has to carry the time it was printed. + if a.buf.Len() == 0 { + a.at = at + } + a.buf.WriteString(s) +} + +func (a *Assembler) take() Line { + line := Line{Data: a.buf.String(), At: a.at, Cursor: a.cursor} + a.buf.Reset() + a.at = time.Time{} + return line +} diff --git a/components/logship/internal/ship/ship_test.go b/components/logship/internal/ship/ship_test.go new file mode 100644 index 0000000..c8d02be --- /dev/null +++ b/components/logship/internal/ship/ship_test.go @@ -0,0 +1,409 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ship + +import ( + "strings" + "testing" + "time" + "unicode/utf8" +) + +var ( + t1 = time.Unix(1788477379, 229_000_000) + t2 = time.Unix(1788477379, 282_000_000) +) + +func TestAssembler_SplitsChunksIntoLines(t *testing.T) { + a := NewAssembler(false) + got := a.Add(Batch{Cursor: "100-0", Entries: []Entry{{Data: "one\ntwo\nthree", At: t1}}}) + + if want := []string{"one", "two"}; !equalData(got, want) { + t.Fatalf("got %v, want %v — the unterminated tail must be held", data(got), want) + } + // Held, not emitted: "three" may still be half a line. + if lines := a.Flush(); len(lines) != 1 || lines[0].Data != "three" { + t.Fatalf("Flush gave %v, want the pending fragment", data(lines)) + } +} + +func TestAssembler_LineSpanningBatchesTakesTheCompletingCursor(t *testing.T) { + // The reason this matters: resuming from the cursor a line STARTED in re-delivers the rest of + // that batch. Resuming from where it completed re-delivers nothing it needed. + a := NewAssembler(false) + if lines := a.Add(Batch{Cursor: "100-0", Entries: []Entry{{Data: "half", At: t1}}}); len(lines) != 0 { + t.Fatalf("got %v, want nothing until the line completes", data(lines)) + } + got := a.Add(Batch{Cursor: "200-0", Entries: []Entry{{Data: " and half\n", At: t2}}}) + + if len(got) != 1 || got[0].Data != "half and half" { + t.Fatalf("got %v, want the joined line", data(got)) + } + if got[0].Cursor != "200-0" { + t.Fatalf("cursor %q, want the batch that completed the line", got[0].Cursor) + } + // Dated from its first byte, so a line is stamped when it was printed rather than when the + // rest of it happened to arrive. + if !got[0].At.Equal(t1) { + t.Fatalf("At = %v, want the chunk that started the line (%v)", got[0].At, t1) + } +} + +func TestAssembler_KeepsBlankLines(t *testing.T) { + // A blank line is real output, and it still has to carry a timestamp: the sink stamps events + // from this, and a zero time would land it in 1970. + a := NewAssembler(false) + got := a.Add(Batch{Cursor: "100-0", Entries: []Entry{{Data: "\n", At: t1}}}) + + if len(got) != 1 || got[0].Data != "" { + t.Fatalf("got %v, want one empty line", data(got)) + } + if !got[0].At.Equal(t1) { + t.Fatalf("At = %v, want %v", got[0].At, t1) + } +} + +func TestAssembler_StripsTheCarriageReturnOfACRLF(t *testing.T) { + for _, collapse := range []bool{false, true} { + a := NewAssembler(collapse) + got := a.Add(Batch{Cursor: "100-0", Entries: []Entry{{Data: "windows\r\n", At: t1}}}) + + // In collapse mode this is the case that would otherwise vanish entirely: the CR looks + // exactly like a progress frame separator unless it is taken off first. + if len(got) != 1 || got[0].Data != "windows" { + t.Fatalf("collapseFrames=%v: got %v, want the line without its CR", collapse, data(got)) + } + } +} + +func TestAssembler_StripsACRLFSplitBetweenChunks(t *testing.T) { + // The two bytes need not arrive together. Resolved without its LF in view, the CR is either left in + // the line or — once frames collapse — read as a frame separator, which discards the line's text and + // ships an empty line in its place. + for _, collapse := range []bool{false, true} { + a := NewAssembler(collapse) + got := a.Add(Batch{Cursor: "100-0", Entries: []Entry{ + {Data: "windows\r", At: t1}, + {Data: "\n", At: t2}, + }}) + + if !equalData(got, []string{"windows"}) { + t.Fatalf("collapseFrames=%v: got %v, want what an unsplit CRLF gives", collapse, data(got)) + } + } +} + +func TestAssembler_HoldsATrailingCRAcrossBatches(t *testing.T) { + // The likelier of the two boundaries, since a batch ends wherever the source's page does — and the + // held CR has to survive between Add calls, not merely between the entries of one. + for _, collapse := range []bool{false, true} { + a := NewAssembler(collapse) + if got := a.Add(Batch{Cursor: "100-0", Entries: []Entry{{Data: "windows\r", At: t1}}}); len(got) != 0 { + t.Fatalf("collapseFrames=%v: %v completed on the CR alone", collapse, data(got)) + } + + got := a.Add(Batch{Cursor: "200-0", Entries: []Entry{{Data: "\nnext\n", At: t2}}}) + if !equalData(got, []string{"windows", "next"}) { + t.Fatalf("collapseFrames=%v: got %v, want both lines", collapse, data(got)) + } + // Completed in the second batch, so that is the cursor a resume must not pass — see Line. + if got[0].Cursor != "200-0" { + t.Fatalf("collapseFrames=%v: cursor = %q, want the completing batch's", collapse, got[0].Cursor) + } + } +} + +func TestAssembler_ATrailingCRIsStillAFrameSeparator(t *testing.T) { + // The other half of holding a CR back: one that turns out NOT to precede an LF has to behave exactly + // as it would have arriving in one piece. Otherwise fixing CRLF would break every progress bar whose + // chunk happens to end on the separator, which is where tqdm's writes land. + entries := []Entry{ + {Data: " 10%|## \r", At: t1}, + {Data: " 50%|##### \r", At: t1}, + {Data: "100%|##########\n", At: t2}, + } + + a := NewAssembler(true) + if got := a.Add(Batch{Cursor: "100-0", Entries: entries}); !equalData(got, []string{"100%|##########"}) { + t.Fatalf("collapsed: got %v, want only the final frame", data(got)) + } + b := NewAssembler(false) + want := " 10%|## \r 50%|##### \r100%|##########" + if got := b.Add(Batch{Cursor: "100-0", Entries: entries}); !equalData(got, []string{want}) { + t.Fatalf("verbatim: got %q, want every frame", data(got)) + } +} + +func TestAssembler_FlushDropsAHeldCR(t *testing.T) { + // Nothing follows to overwrite what the CR separated, so collapsing on it here would erase text that + // was still on the terminal when the sandbox exited. + for _, collapse := range []bool{false, true} { + a := NewAssembler(collapse) + a.Add(Batch{Cursor: "100-0", Entries: []Entry{{Data: "half a line\r", At: t1}}}) + + if got := a.Flush(); !equalData(got, []string{"half a line"}) { + t.Fatalf("collapseFrames=%v: got %v, want the text without its CR", collapse, data(got)) + } + } +} + +func TestAssembler_ProgressFrames(t *testing.T) { + // What tqdm actually emits: a frame per update, carriage-returned, with no newline until the + // bar is done. + frames := Batch{Cursor: "100-0", Entries: []Entry{ + {Data: "\r 10%|## ", At: t1}, + {Data: "\r 50%|##### ", At: t1}, + {Data: "\r100%|##########\n", At: t2}, + }} + + t.Run("collapsed keeps only the last", func(t *testing.T) { + a := NewAssembler(true) + got := a.Add(frames) + if len(got) != 1 || got[0].Data != "100%|##########" { + t.Fatalf("got %v, want only the final frame", data(got)) + } + // The frame that survived is the one that set the timestamp, not the first update. + if !got[0].At.Equal(t2) { + t.Fatalf("At = %v, want the surviving frame's time", got[0].At) + } + }) + + t.Run("kept verbatim otherwise", func(t *testing.T) { + a := NewAssembler(false) + got := a.Add(frames) + if len(got) != 1 || got[0].Data != "\r 10%|## \r 50%|##### \r100%|##########" { + t.Fatalf("got %q, want every frame", data(got)) + } + }) +} + +func TestAssembler_CapsAnUnterminatedLine(t *testing.T) { + // A source that never sends a newline must not be able to grow the pending line without + // bound; a progress bar in the un-collapsed mode is the ordinary way that happens. + a := NewAssembler(false) + chunk := strings.Repeat("x", maxFragment/2) + + if lines := a.Add(Batch{Cursor: "100-0", Entries: []Entry{{Data: chunk, At: t1}}}); len(lines) != 0 { + t.Fatalf("%d lines below the cap, want none", len(lines)) + } + // Held at exactly the cap rather than emitted, because a fragment that fills it is not over it — + // see fill. One more byte takes it, so the bound this test exists for is unaffected. + if got := a.Add(Batch{Cursor: "200-0", Entries: []Entry{{Data: chunk, At: t2}}}); len(got) != 0 { + t.Fatalf("%d lines at exactly the cap, want it held", len(got)) + } + if lines := a.Flush(); len(lines) != 1 || len(lines[0].Data) != maxFragment { + t.Fatalf("Flush gave %v, want one line of %d bytes", data(lines), maxFragment) + } +} + +func TestAssembler_DoesNotFollowAnExactCapWithABlankLine(t *testing.T) { + // Emitting at exactly the cap left the buffer empty, and the newline path takes unconditionally — + // so a line whose length was a multiple of the cap shipped a spurious blank record after it. Not the + // 1-in-65536 accident it looks like: maxFragment is 64 KiB, so every power-of-two-sized dump lands + // on the boundary exactly. + exact := strings.Repeat("x", maxFragment) + + for _, collapse := range []bool{false, true} { + a := NewAssembler(collapse) + got := a.Add(Batch{Cursor: "100-0", Entries: []Entry{{Data: exact + "\n", At: t1}}}) + if !equalData(got, []string{exact}) { + t.Fatalf("collapseFrames=%v: %d lines of %v bytes, want one of %d", + collapse, len(got), lengths(got), maxFragment) + } + + // The likelier arrival, since a batch ends wherever the source's page does. + b := NewAssembler(collapse) + b.Add(Batch{Cursor: "100-0", Entries: []Entry{{Data: exact, At: t1}}}) + if got := b.Add(Batch{Cursor: "200-0", Entries: []Entry{{Data: "\n", At: t2}}}); !equalData(got, []string{exact}) { + t.Fatalf("collapseFrames=%v: a newline in a later batch gave %v bytes, want one line of %d", + collapse, lengths(got), maxFragment) + } + } + + // And no blank between the pieces of a longer multiple, nor after the last one. + a := NewAssembler(false) + got := a.Add(Batch{Cursor: "100-0", Entries: []Entry{{Data: strings.Repeat("x", 3*maxFragment) + "\n", At: t1}}}) + if len(got) != 3 { + t.Fatalf("%d lines for 3x the cap plus its newline, want 3: %v", len(got), lengths(got)) + } + if rest := a.Flush(); rest != nil { + t.Fatalf("Flush gave %v, want nothing", data(rest)) + } +} + +func TestAssembler_SplitsAChunkThatIsItselfOverTheCap(t *testing.T) { + // The cap has to bound the line, not merely flush after it: one Modal item can be megabytes, and a + // Line that size rides the queue's oversized-line escape only to be dropped by the batcher. + a := NewAssembler(false) + got := a.Add(Batch{Cursor: "100-0", Entries: []Entry{ + {Data: strings.Repeat("x", 5*maxFragment/2), At: t1}, + }}) + + if len(got) != 2 { + t.Fatalf("%d lines for 2.5x the cap, want 2", len(got)) + } + for i, l := range got { + if len(l.Data) != maxFragment { + t.Fatalf("line %d is %d bytes, want the cap %d", i, len(l.Data), maxFragment) + } + } + if rest := a.Flush(); len(rest) != 1 || len(rest[0].Data) != maxFragment/2 { + t.Fatalf("Flush gave %v, want the remaining half-cap fragment", data(rest)) + } +} + +func TestAssembler_CapsALineItsOwnNewlineTerminated(t *testing.T) { + // The terminated case used to skip the cap entirely, because the newline path wrote its whole prefix + // before the bounded loop ever saw it. That matters at the queue rather than here: it admits any one + // line into an empty buffer, so a Line over the cap turns a 64 KiB byte bound into "one longest + // line", and 1,000 streams of those is a memory figure nobody can size a Deployment from. + a := NewAssembler(false) + got := a.Add(Batch{Cursor: "100-0", Entries: []Entry{ + {Data: strings.Repeat("x", 5*maxFragment/2) + "\n", At: t1}, + }}) + + if len(got) != 3 { + t.Fatalf("%d lines for 2.5x the cap plus its newline, want 3", len(got)) + } + var total int + for i, l := range got { + if len(l.Data) > maxFragment { + t.Fatalf("line %d is %d bytes, over the cap %d", i, len(l.Data), maxFragment) + } + total += len(l.Data) + } + // Split, not truncated: the newline is the only byte that should be missing. + if total != 5*maxFragment/2 { + t.Fatalf("the pieces hold %d bytes, want the line's %d", total, 5*maxFragment/2) + } + if rest := a.Flush(); rest != nil { + t.Fatalf("Flush gave %v, want nothing — the newline took the last piece", data(rest)) + } +} + +func TestAssembler_AdvancesWhenNoRuneBoundaryExists(t *testing.T) { + // runeBoundary walks down to zero when the window holds no rune start, and taking the pending line + // cannot conjure one — so this used to spin, emitting empty lines until the process died. A stream + // of continuation bytes is invalid UTF-8 either way; the cut is the lesser failure. + a := NewAssembler(false) + flood := strings.Repeat("\x80", 2*maxFragment) + + got := a.Add(Batch{Cursor: "100-0", Entries: []Entry{{Data: flood, At: t1}}}) + got = append(got, a.Flush()...) + + var joined strings.Builder + for i, l := range got { + if len(l.Data) > maxFragment { + t.Fatalf("piece %d is %d bytes, over the cap %d", i, len(l.Data), maxFragment) + } + joined.WriteString(l.Data) + } + // Passed through as it arrived rather than replaced by U+FFFD: the durable copy is the evidence. + if joined.String() != flood { + t.Fatalf("reassembled %d bytes, want the %d that arrived", joined.Len(), len(flood)) + } +} + +func TestAssembler_CutsAFragmentOnRuneBoundaries(t *testing.T) { + // A cut inside a multi-byte rune reaches the sink as U+FFFD, which corrupts the durable copy + // rather than merely splitting it. The leading chunk is swept because whether the cap lands + // mid-rune depends on what is already pending as much as on the data. + body := strings.Repeat("日本語", maxFragment/9+16) // 9 bytes a repeat, so comfortably over the cap + for pad := 0; pad < 9; pad++ { + a := NewAssembler(false) + want := strings.Repeat("a", pad) + body + got := a.Add(Batch{Cursor: "100-0", Entries: []Entry{{Data: want, At: t1}}}) + got = append(got, a.Flush()...) + + var joined strings.Builder + for i, l := range got { + if !utf8.ValidString(l.Data) { + t.Fatalf("pad %d: piece %d is not valid UTF-8", pad, i) + } + joined.WriteString(l.Data) + } + if joined.String() != want { + t.Fatalf("pad %d: reassembled %d bytes, want %d", pad, joined.Len(), len(want)) + } + } +} + +func TestAssembler_ACarriageReturnCannotReachPastAnEmittedPiece(t *testing.T) { + // The one thing splitting at the cap costs, asserted so it is a decision rather than a surprise: + // collapsing still discards what is pending, but a piece already emitted cannot be un-emitted, so + // a CR reaches back maxFragment rather than without limit. Honouring it further is the unbounded + // buffer the cap exists to prevent. + a := NewAssembler(true) + got := a.Add(Batch{Cursor: "100-0", Entries: []Entry{ + {Data: strings.Repeat("x", 2*maxFragment) + "\rdone", At: t1}, + }}) + + if len(got) != 2 { + t.Fatalf("%d lines, want the two full pieces emitted before the CR arrived", len(got)) + } + if rest := a.Flush(); len(rest) != 1 || rest[0].Data != "done" { + t.Fatalf("Flush gave %v, want the CR to have collapsed what was still pending", data(rest)) + } +} + +func TestAssembler_CollapsingKeepsAProgressBarBounded(t *testing.T) { + // The other half of the cap's argument: with frames collapsed, a bar that never terminates + // holds one frame rather than accumulating toward maxFragment at all. + a := NewAssembler(true) + for range 1000 { + if lines := a.Add(Batch{Cursor: "100-0", Entries: []Entry{{Data: "\r" + strings.Repeat("#", 200), At: t1}}}); len(lines) != 0 { + t.Fatalf("got %v, want nothing until the bar terminates", data(lines)) + } + } + if a.buf.Len() != 200 { + t.Fatalf("pending fragment is %d bytes after 1000 frames, want one frame's worth", a.buf.Len()) + } +} + +func TestAssembler_FlushIsIdempotent(t *testing.T) { + a := NewAssembler(false) + a.Add(Batch{Cursor: "100-0", Entries: []Entry{{Data: "tail", At: t1}}}) + if lines := a.Flush(); len(lines) != 1 { + t.Fatalf("%d lines, want the fragment", len(lines)) + } + // A drain can be attempted twice — once at eof, once by the caller shutting down — and the + // second must not re-emit what the first already shipped. + if lines := a.Flush(); lines != nil { + t.Fatalf("second Flush gave %v, want nothing", data(lines)) + } +} + +func data(lines []Line) []string { + out := make([]string, 0, len(lines)) + for _, l := range lines { + out = append(out, l.Data) + } + return out +} + +// lengths is for failures where the byte counts are the whole story and the data is 64 KiB of "x". +func lengths(lines []Line) []int { + out := make([]int, 0, len(lines)) + for _, l := range lines { + out = append(out, len(l.Data)) + } + return out +} + +func equalData(got []Line, want []string) bool { + return strings.Join(data(got), "\x00") == strings.Join(want, "\x00") +} diff --git a/components/logship/internal/supervise/supervisor.go b/components/logship/internal/supervise/supervisor.go new file mode 100644 index 0000000..4d533fe --- /dev/null +++ b/components/logship/internal/supervise/supervisor.go @@ -0,0 +1,363 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package supervise owns the goroutines: it turns a changing set of instances into a running set of +// pipelines. It knows nothing about Kubernetes, Modal or CloudWatch — a watch calls Ensure and +// Forget, and a Builder supplied by cmd decides what a stream actually connects to. +package supervise + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/InftyAI/Nebula/components/logship/internal/ship" +) + +// Defaults for restart pacing. Bounded rather than infinite because every restart resumes from the +// last durable cursor, so an instance that can never be read would otherwise re-ship its history +// on a loop and bill us for it. +const ( + DefaultMaxRestarts = 5 + DefaultMinBackoff = time.Second + DefaultMaxBackoff = 30 * time.Second +) + +// Instance is one external instance whose output is being copied, with everything needed to name +// and label its events. Sourced from the Pod, which carries the provider, the provider's instance id +// and the tenant labels on the same object. +// +// No namespace: the consumer identifies a record by pod name and labels alone, so carrying one +// would only be a field that looks stamped and is not. +type Instance struct { + // Provider is which backend reads this instance, and travels with ID because ID is only + // meaningful to that one — the same field holds a Modal sandbox id or an EC2 instance id. Carried + // and never interpreted here; Config.Streams and Build are what route on it. + Provider string + + ID string + Pod string + Labels map[string]string +} + +// Ref is what an instance is tracked under. An ID alone will not do: it is minted by the provider that +// owns it, so two backends can mint the same string — see Instance.Provider. Keyed by the pair, that +// collision is two tracked instances; keyed by the ID alone, the second Ensure was a silent no-op and +// either Forget stopped the other one's streams. +type Ref struct { + Provider string + ID string +} + +func (i Instance) Ref() Ref { return Ref{Provider: i.Provider, ID: i.ID} } + +// Builder makes the pipeline for one stream of one instance. +// +// Called again on every restart, and it has to be: a Pipeline is single-use — Run closes its queue — +// so a retry needs a fresh one, with whatever cursor the checkpoint holds by then. +type Builder func(inst Instance, stream string) (*ship.Pipeline, error) + +// Config describes the fleet. Streams and Build are required. +type Config struct { + // Streams names one instance's streams, one pipeline each. Two of them for a Modal sandbox: + // stdout and stderr have separate cursors, so they are separate all the way down rather than + // merged and re-split. + // + // A function of the instance rather than one list for the fleet, because the streams belong to + // the provider that reads the instance, and a cluster can be split across two providers whose + // streams are named differently. + Streams func(inst Instance) []string + Build Builder + + MaxRestarts int + MinBackoff time.Duration + MaxBackoff time.Duration + + // Log matches logr.Logger.Info's signature so cmd can pass it straight in. Nil is silent, which + // is fine for tests and wrong in production: abandoning a stream is the one event here that + // nothing else will tell anyone about. + Log func(msg string, keysAndValues ...any) +} + +// Stats is the fleet view. Shipping is summed across every stream this supervisor has ever run, so +// the counters survive a restart of the stream that produced them. +type Stats struct { + // Instances is how many are tracked right now; Started is how many ever were. + Instances int + Started int + // Streams is how many pipelines are live right now. + Streams int + + Restarts int + // Abandoned counts streams that used up their restart budget. Each one is an instance whose + // remaining logs will not be copied, which is the alert worth having. + Abandoned int + // Completed counts streams that reached their source's end, which is the normal way to finish. + Completed int + + Shipping ship.Stats +} + +// Supervisor runs one set of pipelines per instance. +// +// It holds the context it was built with, which is the one place that is the right design rather +// than a smell: its entire job is owning goroutine lifetimes, and the alternative — a context per +// Ensure call — would tie a stream's life to a reconcile that has already returned. +// +// Safe for concurrent use. Ensure and Forget are cheap and non-blocking, because the caller is a +// watch loop. +type Supervisor struct { + cfg Config + ctx context.Context + + mu sync.Mutex + known map[Ref]*tracked + stats Stats + stopped bool + + wg sync.WaitGroup +} + +// tracked is one instance's running state. live holds the current pipeline per stream, so Stats can +// read counters in flight; a retired pipeline's counters are folded into Supervisor.stats first, so +// nothing accumulates per restart. +type tracked struct { + inst Instance + cancel context.CancelFunc + live map[string]*ship.Pipeline +} + +// New builds a supervisor. It panics on a Config missing Streams or Build, which is a wiring mistake +// worth having at startup: the alternative surfaces as the process dying on the first Pod the watch +// delivers, in production, hours later. +func New(ctx context.Context, cfg Config) *Supervisor { + if cfg.Streams == nil || cfg.Build == nil { + panic("supervise: Config.Streams and Config.Build are required") + } + if cfg.MaxRestarts <= 0 { + cfg.MaxRestarts = DefaultMaxRestarts + } + if cfg.MinBackoff <= 0 { + cfg.MinBackoff = DefaultMinBackoff + } + if cfg.MaxBackoff < cfg.MinBackoff { + cfg.MaxBackoff = max(DefaultMaxBackoff, cfg.MinBackoff) + } + return &Supervisor{ctx: ctx, cfg: cfg, known: map[Ref]*tracked{}} +} + +// Ensure starts copying inst, and does nothing if it is already known. +// +// Idempotent on the Ref, deliberately: a watch re-delivers the same Pod on every unrelated update, +// and treating one of those as new would start a second set of pipelines that replay the instance +// from the beginning. An instance whose streams have all finished stays known for the same reason — +// only Forget takes it out. +func (s *Supervisor) Ensure(inst Instance) { + // Said out loud because it is otherwise the quietest way to ship nothing: the instance is tracked, + // Forget works, Stats counts it, and no pipeline ever runs. + streams := s.cfg.Streams(inst) + if len(streams) == 0 { + s.log("instance has no streams to follow", "instance", inst.ID, "provider", inst.Provider) + return + } + + ref := inst.Ref() + + s.mu.Lock() + if s.stopped || ref.ID == "" || s.known[ref] != nil { + s.mu.Unlock() + return + } + ctx, cancel := context.WithCancel(s.ctx) + t := &tracked{inst: inst, cancel: cancel, live: map[string]*ship.Pipeline{}} + s.known[ref] = t + s.stats.Instances++ + s.stats.Started++ + // The whole count, and before the unlock: Shutdown takes this same lock and then waits on the + // group, so a counter raised after the unlock lets it find zero and return while these streams are + // still starting — against a provider its caller is about to close. Nothing may return between here + // and the loop below, or Wait never comes back. + s.wg.Add(len(streams)) + s.mu.Unlock() + + for _, name := range streams { + go func() { + defer s.wg.Done() + s.follow(ctx, t, name) + }() + } +} + +// Forget stops copying an instance and drops it. +// +// Returns immediately; the goroutines wind down on their own. In-flight puts are cut off, which is +// why the drain finalizer exists — it is what keeps a Pod's deletion from reaching here before the +// tail of the log has shipped. +func (s *Supervisor) Forget(ref Ref) { + s.mu.Lock() + t := s.known[ref] + if t != nil { + delete(s.known, ref) + s.stats.Instances-- + } + s.mu.Unlock() + + if t != nil { + t.cancel() + } +} + +// Shutdown cancels every stream and waits. The pipelines will fail their final puts against the +// cancelled context, so a caller that wants the tails shipped drains before calling this. +func (s *Supervisor) Shutdown() { + s.mu.Lock() + s.stopped = true + cancels := make([]context.CancelFunc, 0, len(s.known)) + for ref, t := range s.known { + cancels = append(cancels, t.cancel) + delete(s.known, ref) + } + s.stats.Instances = 0 + s.mu.Unlock() + + for _, cancel := range cancels { + cancel() + } + s.wg.Wait() +} + +// Stats reports the fleet's counters, folding in every live pipeline's own. +func (s *Supervisor) Stats() Stats { + s.mu.Lock() + defer s.mu.Unlock() + + out := s.stats + for _, t := range s.known { + for _, p := range t.live { + out.Streams++ + add(&out.Shipping, p.Stats()) + } + } + return out +} + +// InstanceCount is how many instances are tracked, for a caller that needs the size without the fold. +// +// Separate from Stats because Stats walks every live pipeline and takes each one's lock — the same +// lock a pipeline holds to account for the lines it ships — which is the wrong shape for something +// called once per watch event. +func (s *Supervisor) InstanceCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.stats.Instances +} + +// follow runs one stream, restarting it on failure until the budget is spent. +func (s *Supervisor) follow(ctx context.Context, t *tracked, name string) { + backoff := s.cfg.MinBackoff + for attempt := 0; ; attempt++ { + ended, err := s.runOnce(ctx, t, name) + if ctx.Err() != nil { + return + } + if ended { + // The source reached its end. Restarting would replay the whole instance and then end + // again, forever — this is the one outcome that must not be retried. + s.record(func(st *Stats) { st.Completed++ }) + return + } + if attempt >= s.cfg.MaxRestarts { + // Logged before the counter it belongs to, because Abandoned is the counter an operator + // alerts on: published first, it points at an explanation that has not been written yet. + s.log("abandoning stream after exhausting its restart budget", + "provider", t.inst.Provider, "instance", t.inst.ID, "stream", name, "restarts", attempt, "err", err) + s.record(func(st *Stats) { st.Abandoned++ }) + return + } + s.record(func(st *Stats) { st.Restarts++ }) + if !sleep(ctx, backoff) { + return + } + backoff = min(backoff*2, s.cfg.MaxBackoff) + } +} + +// runOnce builds and runs one pipeline. A build failure counts against the same restart budget as a +// run failure: both mean this stream is not copying, and the cause is usually the same outage. +func (s *Supervisor) runOnce(ctx context.Context, t *tracked, name string) (bool, error) { + p, err := s.cfg.Build(t.inst, name) + if err != nil { + return false, fmt.Errorf("build stream %s/%s: %w", t.inst.ID, name, err) + } + s.setLive(t, name, p) + defer s.retire(t, name, p) + + if err := p.Run(ctx); err != nil { + return false, err + } + return true, nil +} + +func (s *Supervisor) setLive(t *tracked, name string, p *ship.Pipeline) { + s.mu.Lock() + defer s.mu.Unlock() + t.live[name] = p +} + +// retire folds a finished pipeline's counters into the fleet total and drops the pointer, so a +// stream that has restarted five times is still five sets of counters and one live object. +func (s *Supervisor) retire(t *tracked, name string, p *ship.Pipeline) { + s.mu.Lock() + defer s.mu.Unlock() + if t.live[name] == p { + delete(t.live, name) + } + add(&s.stats.Shipping, p.Stats()) +} + +func (s *Supervisor) log(msg string, kv ...any) { + if s.cfg.Log != nil { + s.cfg.Log(msg, kv...) + } +} + +func (s *Supervisor) record(f func(*Stats)) { + s.mu.Lock() + defer s.mu.Unlock() + f(&s.stats) +} + +func add(dst *ship.Stats, src ship.Stats) { + dst.Lines += src.Lines + dst.Events += src.Events + dst.Dropped += src.Dropped + dst.DroppedBytes += src.DroppedBytes + dst.Failed += src.Failed + dst.Clamped += src.Clamped + dst.Oversized += src.Oversized +} + +func sleep(ctx context.Context, d time.Duration) bool { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-t.C: + return true + case <-ctx.Done(): + return false + } +} diff --git a/components/logship/internal/supervise/supervisor_test.go b/components/logship/internal/supervise/supervisor_test.go new file mode 100644 index 0000000..e22fae1 --- /dev/null +++ b/components/logship/internal/supervise/supervisor_test.go @@ -0,0 +1,507 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package supervise + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/InftyAI/Nebula/components/logship/internal/ship" +) + +// only is Config.Streams for a fleet whose instances all have the same streams, which is every fleet +// here: what varies per provider in production is not what these tests are about. +func only(names ...string) func(Instance) []string { + return func(Instance) []string { return names } +} + +func TestEnsure_StartsOnePipelinePerStream(t *testing.T) { + b := &builder{block: true} + s := New(context.Background(), Config{Streams: only("stdout", "stderr"), Build: b.build}) + defer s.Shutdown() + + s.Ensure(Instance{ID: "sb-1", Pod: "p"}) + + waitFor(t, "both streams built", func() bool { return b.count() == 2 }) + if got := b.streams(); got["stdout"] != 1 || got["stderr"] != 1 { + t.Fatalf("built %v, want one of each", got) + } + if st := s.Stats(); st.Instances != 1 || st.Started != 1 || st.Streams != 2 { + t.Fatalf("Stats() = %+v", st) + } +} + +func TestInstanceCount_AgreesWithStats(t *testing.T) { + // The fleet reserves capacity from InstanceCount on every watch event and reports Stats once at + // shutdown. Two counters of the same thing, so the cheap one has to answer what the fold would. + b := &builder{block: true} + s := New(context.Background(), Config{Streams: only("stdout"), Build: b.build}) + defer s.Shutdown() + + s.Ensure(Instance{ID: "sb-1", Pod: "p"}) + s.Ensure(Instance{ID: "sb-2", Pod: "p"}) + waitFor(t, "both instances running", func() bool { return b.running() == 2 }) + if got, want := s.InstanceCount(), s.Stats().Instances; got != want || got != 2 { + t.Fatalf("InstanceCount() = %d, Stats().Instances = %d, want 2", got, want) + } + + s.Forget(Ref{ID: "sb-1"}) + + if got, want := s.InstanceCount(), s.Stats().Instances; got != want || got != 1 { + t.Fatalf("after Forget: InstanceCount() = %d, Stats().Instances = %d, want 1", got, want) + } +} + +func TestEnsure_IsIdempotentOnTheInstanceID(t *testing.T) { + // A watch re-delivers the same Pod on every unrelated update; a second set of pipelines would + // replay the instance from the beginning. + b := &builder{block: true} + s := New(context.Background(), Config{Streams: only("stdout"), Build: b.build}) + defer s.Shutdown() + + for range 5 { + s.Ensure(Instance{ID: "sb-1", Pod: "p"}) + } + + waitFor(t, "one stream built", func() bool { return b.count() == 1 }) + time.Sleep(20 * time.Millisecond) // give a duplicate a chance to show up + if got := b.count(); got != 1 { + t.Fatalf("%d pipelines for one instance, want 1", got) + } + if st := s.Stats(); st.Started != 1 { + t.Fatalf("Started = %d, want 1", st.Started) + } +} + +func TestSupervisor_TellsTwoProvidersApartOnTheSameID(t *testing.T) { + // An id is minted by the provider that owns it, so nothing stops two backends from using the same + // string — see Ref. Keyed by the id alone, the second Ensure was a silent no-op that shipped none of + // that instance's logs, and either Forget cancelled the other one's streams. + one := Instance{Provider: "modal", ID: "sb-1", Pod: "p"} + two := Instance{Provider: "aws", ID: "sb-1", Pod: "q"} + + b := &builder{block: true} + s := New(context.Background(), Config{Streams: only("stdout"), Build: b.build}) + defer s.Shutdown() + + s.Ensure(one) + s.Ensure(two) + + waitFor(t, "both instances running", func() bool { return b.running() == 2 }) + if st := s.Stats(); st.Instances != 2 || st.Started != 2 { + t.Fatalf("Stats() = %+v, want both instances tracked", st) + } + + s.Forget(one.Ref()) + + waitFor(t, "only the forgotten instance stopped", func() bool { return b.runningFor(one.Ref()) == 0 }) + if got := b.runningFor(two.Ref()); got != 1 { + t.Fatalf("%d streams running for the other provider's instance, want 1", got) + } + if st := s.Stats(); st.Instances != 1 { + t.Fatalf("Instances = %d, want the other provider's still tracked", st.Instances) + } +} + +func TestEnsure_IgnoresAnInstanceWithNoID(t *testing.T) { + // The provider half of a Ref is no identity on its own. A Pod whose instance-id annotation has not + // landed yet is not an instance called "" — it is one we cannot track, and the next resync gets it. + b := &builder{block: true} + s := New(context.Background(), Config{Streams: only("stdout"), Build: b.build}) + defer s.Shutdown() + + s.Ensure(Instance{Pod: "p"}) + + time.Sleep(20 * time.Millisecond) + if got := b.count(); got != 0 { + t.Fatalf("%d pipelines for an instance with no id, want 0", got) + } +} + +func TestFollow_DoesNotRestartAStreamThatEnded(t *testing.T) { + // The sandbox finished. Restarting would replay the whole instance and end again, forever. + b := &builder{} + s := New(context.Background(), Config{ + Streams: only("stdout"), Build: b.build, MinBackoff: time.Millisecond, + }) + defer s.Shutdown() + + s.Ensure(Instance{ID: "sb-1"}) + + waitFor(t, "stream completed", func() bool { return s.Stats().Completed == 1 }) + time.Sleep(20 * time.Millisecond) + if got := b.count(); got != 1 { + t.Fatalf("%d pipelines after a clean end, want 1", got) + } + if st := s.Stats(); st.Restarts != 0 { + t.Fatalf("Restarts = %d after a clean end, want 0", st.Restarts) + } +} + +func TestFollow_RestartsAFailedStreamUntilItSucceeds(t *testing.T) { + b := &builder{failures: 2} + s := New(context.Background(), Config{ + Streams: only("stdout"), Build: b.build, MinBackoff: time.Millisecond, + }) + defer s.Shutdown() + + s.Ensure(Instance{ID: "sb-1"}) + + waitFor(t, "stream completed after retrying", func() bool { return s.Stats().Completed == 1 }) + if st := s.Stats(); st.Restarts != 2 || st.Abandoned != 0 { + t.Fatalf("Stats() = %+v, want 2 restarts and nothing abandoned", st) + } + if got := b.count(); got != 3 { + t.Fatalf("%d pipelines built, want 3 — a Pipeline is single-use, so a retry needs a fresh one", got) + } +} + +func TestFollow_GivesUpWhenTheRestartBudgetIsSpent(t *testing.T) { + // Bounded on purpose: every restart resumes from the last durable cursor, so a stream that can + // never be read would otherwise re-ship its history on a loop and bill us for it. + b := &builder{failures: 1000} + var logged int + var mu sync.Mutex + s := New(context.Background(), Config{ + Streams: only("stdout"), Build: b.build, MaxRestarts: 3, MinBackoff: time.Millisecond, + Log: func(string, ...any) { mu.Lock(); logged++; mu.Unlock() }, + }) + defer s.Shutdown() + + s.Ensure(Instance{ID: "sb-1"}) + + // Both halves, because abandoning silently is the failure worth catching and nothing else reports + // it — and waiting on the counter alone would race the log it is published alongside. + waitFor(t, "stream abandoned and said so", func() bool { + if s.Stats().Abandoned != 1 { + return false + } + mu.Lock() + defer mu.Unlock() + return logged > 0 + }) + if st := s.Stats(); st.Restarts != 3 || st.Completed != 0 { + t.Fatalf("Stats() = %+v, want 3 restarts and nothing completed", st) + } + // 4 attempts for 3 restarts: the budget counts retries, not tries. + if got := b.count(); got != 4 { + t.Fatalf("%d attempts, want 4", got) + } +} + +func TestFollow_CountsABuildFailureAgainstTheSameBudget(t *testing.T) { + // Both mean the stream is not copying, and the cause is usually the same outage. + b := &builder{buildErr: errors.New("no credentials")} + s := New(context.Background(), Config{ + Streams: only("stdout"), Build: b.build, MaxRestarts: 2, MinBackoff: time.Millisecond, + }) + defer s.Shutdown() + + s.Ensure(Instance{ID: "sb-1"}) + + waitFor(t, "stream abandoned", func() bool { return s.Stats().Abandoned == 1 }) + if got := b.count(); got != 3 { + t.Fatalf("%d build attempts, want 3", got) + } +} + +func TestForget_CancelsTheStreamsAndUntracksTheInstance(t *testing.T) { + b := &builder{block: true} + s := New(context.Background(), Config{ + Streams: only("stdout", "stderr"), Build: b.build, MinBackoff: time.Millisecond, + }) + defer s.Shutdown() + + s.Ensure(Instance{ID: "sb-1"}) + waitFor(t, "both streams running", func() bool { return b.running() == 2 }) + + s.Forget(Ref{ID: "sb-1"}) + + waitFor(t, "both streams stopped", func() bool { return b.running() == 0 }) + if st := s.Stats(); st.Instances != 0 || st.Streams != 0 { + t.Fatalf("Stats() = %+v, want nothing tracked", st) + } + // A cancelled stream is neither finished nor broken, so it must not spend the abandon budget. + if st := s.Stats(); st.Abandoned != 0 || st.Restarts != 0 { + t.Fatalf("Stats() = %+v, want a cancellation to be neither a restart nor an abandonment", st) + } +} + +func TestForget_IsHarmlessForAnInstanceItNeverKnew(t *testing.T) { + s := New(context.Background(), Config{Streams: only("stdout"), Build: (&builder{}).build}) + defer s.Shutdown() + + s.Forget(Ref{ID: "sb-unknown"}) + + if st := s.Stats(); st.Instances != 0 { + t.Fatalf("Instances = %d after forgetting an unknown id", st.Instances) + } +} + +func TestStats_SurviveTheStreamThatProducedThem(t *testing.T) { + // A restart replaces the Pipeline that holds the counters, so retire has to fold them into the + // fleet total — otherwise a restart resets the numbers a metric is built on. + b := &builder{failures: 1, lines: 3} + s := New(context.Background(), Config{ + Streams: only("stdout"), Build: b.build, MinBackoff: time.Millisecond, + }) + defer s.Shutdown() + + s.Ensure(Instance{ID: "sb-1"}) + + waitFor(t, "stream completed after retrying", func() bool { return s.Stats().Completed == 1 }) + // Both attempts shipped their lines, and the failed one's counters are still counted. + if got := s.Stats().Shipping.Lines; got != 6 { + t.Fatalf("Shipping.Lines = %d across two attempts of 3 lines, want 6", got) + } +} + +func TestStats_CountLiveAndRetiredPipelinesExactlyOnce(t *testing.T) { + // The live pointer is dropped as the counters are folded in, under one lock. Getting that wrong + // double-counts every finished stream. + b := &builder{lines: 2} + s := New(context.Background(), Config{ + Streams: only("stdout", "stderr"), Build: b.build, MinBackoff: time.Millisecond, + }) + defer s.Shutdown() + + s.Ensure(Instance{ID: "sb-1"}) + + waitFor(t, "both streams completed", func() bool { return s.Stats().Completed == 2 }) + if got := s.Stats().Shipping.Lines; got != 4 { + t.Fatalf("Shipping.Lines = %d for two streams of 2 lines, want 4", got) + } + if got := s.Stats().Streams; got != 0 { + t.Fatalf("Streams = %d after both ended, want 0", got) + } +} + +func TestShutdown_StopsEverythingAndWaits(t *testing.T) { + b := &builder{block: true} + s := New(context.Background(), Config{Streams: only("stdout", "stderr"), Build: b.build}) + + s.Ensure(Instance{ID: "sb-1"}) + s.Ensure(Instance{ID: "sb-2"}) + waitFor(t, "four streams running", func() bool { return b.running() == 4 }) + + s.Shutdown() + + // Shutdown waits, so this needs no polling: a goroutine still running here is a leak. + if got := b.running(); got != 0 { + t.Fatalf("%d streams still running after Shutdown returned", got) + } + if st := s.Stats(); st.Instances != 0 { + t.Fatalf("Instances = %d after Shutdown", st.Instances) + } +} + +func TestEnsure_IsRefusedAfterShutdown(t *testing.T) { + // Ensure adds to the WaitGroup that Shutdown already waited on, so admitting one here would both + // leak a goroutine and race the wg. + b := &builder{block: true} + s := New(context.Background(), Config{Streams: only("stdout"), Build: b.build}) + s.Shutdown() + + s.Ensure(Instance{ID: "sb-1"}) + + time.Sleep(20 * time.Millisecond) + if got := b.count(); got != 0 { + t.Fatalf("%d pipelines started after Shutdown, want 0", got) + } +} + +func TestNew_FixesUpAnUnusableBackoffRange(t *testing.T) { + wired := Config{Streams: only("stdout"), Build: (&builder{}).build} + + cfg := wired + cfg.MinBackoff, cfg.MaxBackoff = time.Minute, time.Millisecond + s := New(context.Background(), cfg) + if s.cfg.MaxBackoff < s.cfg.MinBackoff { + t.Fatalf("MaxBackoff %v < MinBackoff %v", s.cfg.MaxBackoff, s.cfg.MinBackoff) + } + d := New(context.Background(), wired) + if d.cfg.MaxRestarts != DefaultMaxRestarts || d.cfg.MinBackoff != DefaultMinBackoff { + t.Fatalf("defaults = %d/%v", d.cfg.MaxRestarts, d.cfg.MinBackoff) + } +} + +func TestNew_RefusesAConfigWithNoStreams(t *testing.T) { + // At startup rather than on the first Pod: an instance whose streams are unknown is one whose logs + // are simply absent, and this is the one moment that can still be a loud failure. + defer func() { + if recover() == nil { + t.Error("New accepted a Config with no Streams") + } + }() + New(context.Background(), Config{Build: (&builder{}).build}) +} + +func TestEnsure_SaysSoWhenAnInstanceHasNoStreams(t *testing.T) { + // The quietest way to ship nothing: tracked, counted, and no pipeline ever built. A provider whose + // stream list came back empty has to be as loud as one that failed outright. + b := &builder{} + var logged int + s := New(context.Background(), Config{ + Streams: func(Instance) []string { return nil }, + Build: b.build, + Log: func(string, ...any) { logged++ }, + }) + defer s.Shutdown() + + s.Ensure(Instance{ID: "sb-1", Provider: "nowhere"}) + + if logged == 0 { + t.Error("an instance with no streams was skipped without a word") + } + if st := s.Stats(); st.Instances != 0 || st.Started != 0 { + t.Errorf("Stats counts an instance nothing is following: %+v", st) + } + if got := b.count(); got != 0 { + t.Errorf("%d pipelines built, want 0", got) + } +} + +// builder is a Config.Build that hands out real Pipelines over fake ports, and records what it was +// asked for. Real pipelines because Builder returns a concrete *ship.Pipeline, which is also what +// makes the Stats folding worth testing here rather than mocking away. +type builder struct { + // failures is how many of the first attempts fail before one succeeds; buildErr fails the build + // itself; block makes a stream run until its context is cancelled; lines is how many lines each + // attempt ships. + failures int + buildErr error + block bool + lines int + + mu sync.Mutex + built int + byStream map[string]int + live int + // liveByRef is live broken down by the instance the stream belongs to, for the tests where which + // instance is still running is the whole question. + liveByRef map[Ref]int +} + +func (b *builder) build(inst Instance, stream string) (*ship.Pipeline, error) { + b.mu.Lock() + b.built++ + attempt := b.built + if b.byStream == nil { + b.byStream = map[string]int{} + } + b.byStream[stream]++ + b.mu.Unlock() + + if b.buildErr != nil { + return nil, b.buildErr + } + return ship.New(ship.Config{ + Source: &fakeSource{owner: b, ref: inst.Ref(), block: b.block, fail: attempt <= b.failures, lines: b.lines}, + Sink: fakeSink{}, + Limits: ship.Limits{MaxEvents: 100, MaxBytes: 1 << 20, MaxEventBytes: 1 << 10}, + Interval: time.Millisecond, + }), nil +} + +func (b *builder) count() int { + b.mu.Lock() + defer b.mu.Unlock() + return b.built +} + +func (b *builder) running() int { + b.mu.Lock() + defer b.mu.Unlock() + return b.live +} + +func (b *builder) streams() map[string]int { + b.mu.Lock() + defer b.mu.Unlock() + out := map[string]int{} + for k, v := range b.byStream { + out[k] = v + } + return out +} + +func (b *builder) runningFor(ref Ref) int { + b.mu.Lock() + defer b.mu.Unlock() + return b.liveByRef[ref] +} + +func (b *builder) enter(ref Ref, delta int) { + b.mu.Lock() + defer b.mu.Unlock() + b.live += delta + if b.liveByRef == nil { + b.liveByRef = map[Ref]int{} + } + b.liveByRef[ref] += delta +} + +type fakeSource struct { + owner *builder + ref Ref + block bool + fail bool + lines int +} + +func (s *fakeSource) Follow(ctx context.Context, cursor string, fn func(ship.Batch) error) error { + s.owner.enter(s.ref, 1) + defer s.owner.enter(s.ref, -1) + + for i := range s.lines { + err := fn(ship.Batch{ + Entries: []ship.Entry{{Data: fmt.Sprintf("line %d\n", i), At: time.Now()}}, + Cursor: fmt.Sprintf("%d-0", i), + }) + if err != nil { + return err + } + } + if s.fail { + return errors.New("stream broke") + } + if s.block { + <-ctx.Done() + return ctx.Err() + } + return nil +} + +type fakeSink struct{} + +func (fakeSink) Put(context.Context, []ship.Event) error { return nil } + +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} diff --git a/components/logship/internal/watch/client.go b/components/logship/internal/watch/client.go new file mode 100644 index 0000000..32d21a0 --- /dev/null +++ b/components/logship/internal/watch/client.go @@ -0,0 +1,43 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watch + +import ( + "fmt" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// NewClient builds a clientset from the ServiceAccount in the Pod, falling back to the ambient +// kubeconfig so the watch can be run against a cluster from a laptop. +// +// The fallback is not a convenience only: this cluster is reached through an SSM tunnel, and being +// able to run the watch locally against it is how the Pod selector gets checked before a deploy. +func NewClient() (*kubernetes.Clientset, error) { + cfg, err := rest.InClusterConfig() + if err != nil { + loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}) + cfg, err = loader.ClientConfig() + if err != nil { + return nil, fmt.Errorf("no in-cluster config and no usable kubeconfig: %w", err) + } + } + return kubernetes.NewForConfig(cfg) +} diff --git a/components/logship/internal/watch/watch.go b/components/logship/internal/watch/watch.go new file mode 100644 index 0000000..d833005 --- /dev/null +++ b/components/logship/internal/watch/watch.go @@ -0,0 +1,251 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package watch turns the cluster's Nebula Pods into the instance set supervise runs, so the process +// finds its own work instead of being told one sandbox on the command line. +// +// It watches Pods rather than the NodeClaims the drain finalizer will read, because one Pod carries +// both halves of what a record needs: the provider's instance id on an annotation, and the tenant +// labels the consumer filters on. Reading them off two objects would mean joining them. +package watch + +import ( + "context" + "fmt" + "maps" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/InftyAI/Nebula/components/logship/internal/supervise" +) + +// The keys the watch reads a Pod's identity from. Hardcoded, and not configurable: Nebula's placement +// controller and virtual kubelet write all three as a set, so anything that could configure them apart +// could only configure them into disagreement. +// +// Nebula's own group, which is NOT the domain the tenant labels use — those belong to the consumer +// and ride along in instanceFor's clone rather than being read by name. A Pod carries both domains at +// once, so a rename here does not touch them and must not. +const ( + EnabledLabel = "nebula.inftyai.com/enabled" + InstanceIDAnnotation = "nebula.inftyai.com/instance-id" + + // ProviderSelector is a spec.nodeSelector key, not a label: it is what routes the Pod to a + // provider's virtual node, and the only thing on the object that names the provider. + ProviderSelector = "nebula.inftyai.com/provider" + + EnabledValue = "true" +) + +// DefaultResync is how often the informer re-delivers the Pods it already has as updates. It is NOT a +// relist: nothing is fetched, so it cannot notice a Pod that disappeared. A dropped event is repaired +// only when the watch itself drops and the reflector re-lists, which is what synthesizes a missed +// delete. +// +// It is load-bearing for the other direction: an instance the fleet refused for want of capacity is +// retried by no other path, since Ensure on an already-tracked instance is a no-op. +const DefaultResync = 10 * time.Minute + +// instanceFor decides whether a Pod is one to ship, and builds its Instance. +// +// Reading the provider off the nodeSelector is not a formality: InstanceIDAnnotation holds *the +// provider's* id, so it is a sandbox id on one Pod and an EC2 instance id on the next, and an id +// handed to the wrong backend fails every attempt and spends the whole restart budget doing it. The +// nodeSelector is also the only place to read it — nothing on the Pod's labels names the provider. +// +// Which providers can actually be read is not decided here. A Pod on one this build has no backend +// for is still an Instance, so the caller can say so; dropping it here would make it look identical +// to a Pod Nebula never placed. +// +// Phase is deliberately not consulted. An instance's stream ends on its own and supervise does not +// retry a stream that ended, so shipping a terminal Pod costs nothing; stopping at the terminal phase +// would instead cut the tail of the log off, which is the part that says why the run ended. +func instanceFor(pod *corev1.Pod) (supervise.Instance, bool) { + if pod.Labels[EnabledLabel] != EnabledValue { + return supervise.Instance{}, false + } + provider := pod.Spec.NodeSelector[ProviderSelector] + if provider == "" { + return supervise.Instance{}, false + } + id := pod.Annotations[InstanceIDAnnotation] + if id == "" { + return supervise.Instance{}, false + } + return supervise.Instance{ + Provider: provider, + ID: id, + Pod: pod.Name, + // Cloned: the informer's object is shared cache state that no handler may retain a piece of, + // and this map outlives the event that delivered it. + Labels: maps.Clone(pod.Labels), + }, true +} + +// fleet is the subset of *supervise.Supervisor the watch drives, extracted so tests can assert on the +// calls without building real pipelines. +type fleet interface { + Ensure(inst supervise.Instance) + Forget(ref supervise.Ref) +} + +// Watcher drives a fleet from the Pods of one cluster. Client and Fleet are required. +type Watcher struct { + Client kubernetes.Interface + Fleet fleet + + // Resync is the informer's relist interval; zero means DefaultResync. + Resync time.Duration + + // Log matches supervise's, so cmd passes the same function to both. Nil is silent. + Log func(msg string, keysAndValues ...any) + + mu sync.Mutex + + // shipped is the instance each Pod was last started for, keyed namespace/name because instance Pod + // names repeat across the one-namespace-per-org layout. + // + // It exists to notice a REPLACED instance: re-provisioning rewrites the annotation in place, so the + // one it displaced gets no delete event of its own, and nothing else would ever forget it — a delete + // only ever names the instance the Pod carries now. + shipped map[string]supervise.Ref +} + +// Run watches until ctx is cancelled, and returns early only if the Pod cache never syncs. +// +// The selector is server-side, so the API server never sends this process a Pod that is not Nebula's. +// That matters at scale, and it also means removing the enabled label arrives here as a delete — +// which is the only way an already-shipping instance stops short of the Pod going away. +func (w *Watcher) Run(ctx context.Context) error { + resync := w.Resync + if resync <= 0 { + resync = DefaultResync + } + factory := informers.NewSharedInformerFactoryWithOptions(w.Client, resync, + informers.WithTweakListOptions(func(o *metav1.ListOptions) { + o.LabelSelector = EnabledLabel + "=" + EnabledValue + })) + informer := factory.Core().V1().Pods().Informer() + + if _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { w.ensure(obj) }, + // Ensure on update too, and not as a refresh: the id is absent at CREATE and written when + // Provision returns one, so for most instances the update IS the event that starts shipping. + UpdateFunc: func(_, obj any) { w.ensure(obj) }, + DeleteFunc: w.forget, + }); err != nil { + return fmt.Errorf("add pod handler: %w", err) + } + + factory.Start(ctx.Done()) + defer factory.Shutdown() + // Waited on so that an API server that never answers is an error here rather than a process that + // sits shipping nothing. The initial list arrives through AddFunc like any other event, so there + // is nothing to read out of the store afterwards. + if !cache.WaitForCacheSync(ctx.Done(), informer.HasSynced) { + // It also returns false on a cancelled ctx, which is a shutdown during startup rather than a + // failure -- and a routine one, since it polls every 100ms and a signal can beat the tick. + if ctx.Err() != nil { + return nil + } + return fmt.Errorf("pod cache did not sync") + } + + <-ctx.Done() + return nil +} + +func (w *Watcher) ensure(obj any) { + pod, ok := obj.(*corev1.Pod) + if !ok { + return + } + inst, ok := instanceFor(pod) + if !ok { + return + } + if old, ok := w.replace(podKey(pod), inst.Ref()); ok { + // This cuts the replaced instance's streams off mid-flight, losing whatever tail had not + // shipped — the same loss the drain finalizer will close. Stopping it anyway: the Pod no longer + // claims that instance, so leaving it running ships two instances' output under one pod name, + // and nothing else would ever take it out of the tracked set. + w.log("this Pod's instance was replaced", "pod", pod.Name, "was", old.ID, "now", inst.ID) + w.Fleet.Forget(old) + } + w.Fleet.Ensure(inst) +} + +// forget stops an instance when its Pod goes. +// +// Both the instance on the object and the one this Pod was started for, because they differ exactly +// when the update that rewrote the annotation is the event that got dropped. Forgetting something +// nothing was started for is a no-op, so the union is free and the alternative leaks. +func (w *Watcher) forget(obj any) { + if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = tombstone.Obj + } + pod, ok := obj.(*corev1.Pod) + if !ok { + return + } + started := w.drop(podKey(pod)) + // Both halves, as instanceFor requires: an id without the provider that minted it is not a Ref this + // fleet could ever have tracked. + on := supervise.Ref{Provider: pod.Spec.NodeSelector[ProviderSelector], ID: pod.Annotations[InstanceIDAnnotation]} + if on.Provider != "" && on.ID != "" && on != started { + w.Fleet.Forget(on) + } + if started.ID != "" { + w.Fleet.Forget(started) + } +} + +// replace records the instance now on this Pod and returns the one it displaced, if there was one. +func (w *Watcher) replace(key string, ref supervise.Ref) (supervise.Ref, bool) { + w.mu.Lock() + defer w.mu.Unlock() + if w.shipped == nil { + w.shipped = map[string]supervise.Ref{} + } + old := w.shipped[key] + w.shipped[key] = ref + // The whole Ref, not the id: a Pod that changed providers displaced the old one just as surely, and + // comparing the pair covers that without a case for it. + return old, old.ID != "" && old != ref +} + +// drop stops tracking the Pod, returning the instance it was started for. +func (w *Watcher) drop(key string) supervise.Ref { + w.mu.Lock() + defer w.mu.Unlock() + started := w.shipped[key] + delete(w.shipped, key) + return started +} + +func podKey(pod *corev1.Pod) string { return pod.Namespace + "/" + pod.Name } + +func (w *Watcher) log(msg string, kv ...any) { + if w.Log != nil { + w.Log(msg, kv...) + } +} diff --git a/components/logship/internal/watch/watch_test.go b/components/logship/internal/watch/watch_test.go new file mode 100644 index 0000000..aa11284 --- /dev/null +++ b/components/logship/internal/watch/watch_test.go @@ -0,0 +1,295 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watch + +import ( + "context" + "sync" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/InftyAI/Nebula/components/logship/internal/supervise" +) + +// sandboxPod is a Nebula sandbox Pod as it looks once the virtual kubelet has written the id back. +// Copied from a real one, including the two label domains: the nebula.inftyai.com markers this package +// reads by name, and the consumer's own tenant labels on a separate domain. +func sandboxPod(name, id string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "org-00000000-0000-4000-8000-000000000001", + Labels: map[string]string{ + EnabledLabel: EnabledValue, + "nebula.inftyai.com/nodepool": "modal", + "nebula.inftyai.com/accelerator-type": "l4", + "app": "sandbox", + "example.com/org-id": "o1", + "example.com/team-id": "t1", + "example.com/experiment-id": "e1", + }, + Annotations: map[string]string{InstanceIDAnnotation: id}, + }, + Spec: corev1.PodSpec{NodeSelector: map[string]string{ProviderSelector: "modal"}}, + } +} + +func TestInstanceFor(t *testing.T) { + t.Run("a provisioned pod ships", func(t *testing.T) { + inst, ok := instanceFor(sandboxPod("exp-1-sandbox-0", "sb-abc123")) + if !ok { + t.Fatal("expected a shippable instance") + } + if inst.ID != "sb-abc123" || inst.Pod != "exp-1-sandbox-0" { + t.Errorf("got %+v, want the instance id and the pod's own name", inst) + } + if inst.Labels["app"] != "sandbox" || inst.Labels["example.com/org-id"] != "o1" { + t.Errorf("labels = %v, want the tenancy the consumer filters on", inst.Labels) + } + }) + + // The one that matters: the annotation holds the PROVIDER's id, so this Pod offers an EC2 instance + // id, and it must reach the fleet labelled as such. Handing it to another provider's reader would + // fail every attempt and burn the restart budget doing it. + t.Run("the provider travels with the id", func(t *testing.T) { + pod := sandboxPod("exp-1-sandbox-0", "i-0abc123def456") + pod.Spec.NodeSelector[ProviderSelector] = "aws" + inst, ok := instanceFor(pod) + if !ok { + t.Fatal("expected a Pod on another provider to still be an instance") + } + if inst.Provider != "aws" { + t.Errorf("Provider = %q, want the nodeSelector's value", inst.Provider) + } + }) + + t.Run("not shippable yet or not ours", func(t *testing.T) { + for name, mangle := range map[string]func(*corev1.Pod){ + "id not written back yet": func(p *corev1.Pod) { delete(p.Annotations, InstanceIDAnnotation) }, + "empty id": func(p *corev1.Pod) { p.Annotations[InstanceIDAnnotation] = "" }, + "not placed yet": func(p *corev1.Pod) { p.Spec.NodeSelector = nil }, + "not opted into nebula": func(p *corev1.Pod) { delete(p.Labels, EnabledLabel) }, + "opted in with a typo": func(p *corev1.Pod) { p.Labels[EnabledLabel] = "True" }, + } { + t.Run(name, func(t *testing.T) { + pod := sandboxPod("exp-1-sandbox-0", "sb-abc123") + mangle(pod) + if _, ok := instanceFor(pod); ok { + t.Error("expected the Pod to be skipped") + } + }) + } + }) + + // The handler must not hold a piece of the informer's object: the cache is shared, and its copies + // are not the watch's to keep. + t.Run("labels are copied out of the cache", func(t *testing.T) { + pod := sandboxPod("exp-1-sandbox-0", "sb-abc123") + inst, _ := instanceFor(pod) + inst.Labels["app"] = "mutated" + if pod.Labels["app"] != "sandbox" { + t.Error("instanceFor aliased the Pod's label map") + } + }) + + // A terminal Pod keeps shipping on purpose: the stream ends by itself, and dropping it here would + // cut off the tail that says why the run ended. + t.Run("a terminal pod still ships", func(t *testing.T) { + pod := sandboxPod("exp-1-sandbox-0", "sb-abc123") + pod.Status.Phase = corev1.PodFailed + if _, ok := instanceFor(pod); !ok { + t.Error("expected a failed Pod's tail to still be shipped") + } + }) +} + +// modalRef is what a sandboxPod's instance is tracked under: the provider travels with the id, so a +// Forget names both — see supervise.Ref. +func modalRef(id string) supervise.Ref { + return supervise.Ref{Provider: "modal", ID: id} +} + +// recorder stands in for the Supervisor: the watch's contract is which calls it makes, not what the +// pipelines then do. +type recorder struct { + mu sync.Mutex + ensured []string + forgot []supervise.Ref +} + +func (r *recorder) Ensure(inst supervise.Instance) { + r.mu.Lock() + defer r.mu.Unlock() + r.ensured = append(r.ensured, inst.ID) +} + +func (r *recorder) Forget(ref supervise.Ref) { + r.mu.Lock() + defer r.mu.Unlock() + r.forgot = append(r.forgot, ref) +} + +func (r *recorder) waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + r.mu.Lock() + ok := cond() + r.mu.Unlock() + if ok { + return + } + time.Sleep(5 * time.Millisecond) + } + r.mu.Lock() + defer r.mu.Unlock() + t.Fatalf("timed out waiting for %s (ensured=%v forgot=%v)", what, r.ensured, r.forgot) +} + +func TestWatcherDrivesTheFleet(t *testing.T) { + // Present before the watch starts, so it arrives through the initial list rather than an event. + existing := sandboxPod("exp-1-sandbox-0", "sb-existing") + client := fake.NewSimpleClientset(existing) + rec := &recorder{} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + w := &Watcher{Client: client, Fleet: rec, Resync: time.Hour} + done := make(chan error, 1) + go func() { done <- w.Run(ctx) }() + + rec.waitFor(t, "the existing pod to be ensured", func() bool { + return len(rec.ensured) == 1 && rec.ensured[0] == "sb-existing" + }) + // The ordinary path: a Pod is created before it has an id, and gains one when Provision returns. + pending := sandboxPod("exp-1-sandbox-1", "") + delete(pending.Annotations, InstanceIDAnnotation) + pods := client.CoreV1().Pods(pending.Namespace) + if _, err := pods.Create(ctx, pending, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + provisioned := sandboxPod("exp-1-sandbox-1", "sb-new") + if _, err := pods.Update(ctx, provisioned, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + rec.waitFor(t, "the update that carries the new id", func() bool { + for _, id := range rec.ensured { + if id == "sb-new" { + return true + } + } + return false + }) + + if err := pods.Delete(ctx, provisioned.Name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + rec.waitFor(t, "the delete to forget it", func() bool { + return len(rec.forgot) == 1 && rec.forgot[0] == modalRef("sb-new") + }) + + cancel() + if err := <-done; err != nil { + t.Errorf("Run returned %v, want nil on cancellation", err) + } +} + +// Re-provisioning rewrites the annotation on the SAME Pod, so the instance it displaced never gets a +// delete event. Nothing else takes it out: an instance is forgotten by id, and every delete names the +// id the Pod carries now. +func TestAReplacedInstanceIsForgotten(t *testing.T) { + pod := sandboxPod("exp-1-sandbox-0", "sb-old") + client := fake.NewSimpleClientset(pod) + rec := &recorder{} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + w := &Watcher{Client: client, Fleet: rec, Resync: time.Hour} + go func() { _ = w.Run(ctx) }() + rec.waitFor(t, "the first instance", func() bool { + return len(rec.ensured) > 0 && rec.ensured[0] == "sb-old" + }) + + pods := client.CoreV1().Pods(pod.Namespace) + if _, err := pods.Update(ctx, sandboxPod("exp-1-sandbox-0", "sb-new"), metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + rec.waitFor(t, "the replaced instance to be forgotten", func() bool { + return len(rec.forgot) == 1 && rec.forgot[0] == modalRef("sb-old") + }) + rec.waitFor(t, "the replacement to ship", func() bool { + for _, id := range rec.ensured { + if id == "sb-new" { + return true + } + } + return false + }) + + // And only the replacement is left to stop: forgetting sb-old twice would be harmless, but it would + // mean the watch still believes the Pod owns it. + if err := pods.Delete(ctx, pod.Name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + rec.waitFor(t, "the delete", func() bool { + return len(rec.forgot) == 2 && rec.forgot[1] == modalRef("sb-new") + }) +} + +// An unchanged id must not be read as a replacement: the informer re-delivers the same Pod on every +// unrelated update and on every resync, and forgetting there would stop a healthy instance and then +// restart it from the beginning of its history. +func TestAnUnchangedIdIsNotAReplacement(t *testing.T) { + w := &Watcher{Fleet: &recorder{}} + first := modalRef("sb-1") + + if old, ok := w.replace("ns/pod", first); ok { + t.Errorf("first sighting displaced %v, want nothing", old) + } + if old, ok := w.replace("ns/pod", first); ok { + t.Errorf("re-delivery displaced %v, want nothing", old) + } + if old, ok := w.replace("ns/pod", modalRef("sb-2")); !ok || old != first { + t.Errorf("replacement displaced %v (ok=%t), want %v", old, ok, first) + } + // Not reachable through a Pod, whose nodeSelector cannot change. Asserted because the comparison is + // on the whole Ref: an unchanged id is not by itself what makes this a re-delivery. + if old, ok := w.replace("ns/pod", supervise.Ref{Provider: "aws", ID: "sb-2"}); !ok || old.Provider != "modal" { + t.Errorf("a second provider on the same id displaced %v (ok=%t), want the modal one", old, ok) + } +} + +// Instance Pod names are unique per org namespace, not per cluster: two orgs running the same +// experiment name would otherwise evict each other's instances. +func TestTwoOrgsCanRunTheSamePodName(t *testing.T) { + w := &Watcher{Fleet: &recorder{}} + a := sandboxPod("exp-1-sandbox-0", "sb-a") + b := sandboxPod("exp-1-sandbox-0", "sb-b") + b.Namespace = "org-00000000-0000-4000-8000-000000000002" + + w.ensure(a) + w.ensure(b) + if got := w.Fleet.(*recorder).forgot; len(got) != 0 { + t.Errorf("forgot %v, want nothing: these are different Pods", got) + } +} diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml index cd43a12..61d4e3f 100644 --- a/config/prometheus/monitor.yaml +++ b/config/prometheus/monitor.yaml @@ -13,6 +13,11 @@ spec: - path: /metrics port: https # Ensure this is the name of the port that exposes HTTPS metrics scheme: https + # Coupled to accrualInterval (30s), not a taste in freshness: a cost series is published at 0 + # and first charged one accrual tick later, and a scrape has to land in between or those + # dollars reach no increase() query. Omitting this inherits the Prometheus global — 30s in + # kube-prometheus-stack, which leaves no margin for jitter. See docs/metrics.md, Known gaps. + interval: 15s bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables diff --git a/docs/metrics.md b/docs/metrics.md index eb69a2e..d11f6b7 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -127,7 +127,7 @@ What the fleet has actually spent. # Dollars spent yesterday, by provider. sum by (provider) (increase(nebula_cost_usd_total[1d])) -# Recent spend, fleet-wide. Keep the range well above BOTH the accrual interval (1m) and the +# Recent spend, fleet-wide. Keep the range well above BOTH the accrual interval (30s) and the # scrape interval: increase() needs two samples inside the range, so [1m] against a 60s scrape # usually returns nothing at all. sum(increase(nebula_cost_usd_total[5m])) @@ -447,10 +447,14 @@ knowing before trusting a dashboard. it. Baselines are what keep a claim from relying on sharing a series with another: `markPhase` re-publishes them on every pass, so a mid-process label set — a new tenant, or a shape nothing was running on at startup — is covered as well as a claim that predates the process. But the mechanism - is worth nothing unless **the scrape interval is shorter than `accrualInterval`**: the gap between a - baseline and the first window charged on it is one tick, and a scrape has to land inside it. The - tick is 30s, so a 15s scrape clears it and a 60s one does not — verify yours before trusting any - `increase()` figure here. + needs **a scrape interval shorter than `accrualInterval`**, and even then it improves the odds + rather than clearing the problem. The gap a scrape has to land in is not one tick: the baseline is + published the moment a claim becomes chargeable and the first charge lands at the *next* accrual + tick, so the gap is anywhere from nothing to a full interval, averaging 15s against a 30s tick. + Roughly three quarters of new series get a usable baseline at a 15s scrape, half at 30s, a quarter + at 60s. So a slower scrape makes this worse but no scrape makes it go away, and what it leaves + behind is an undercount — verify yours before trusting any `increase()` figure here, and see + `config/prometheus/monitor.yaml` for where the interval is set. One case no baseline can help: an **instance born and gone inside one scrape interval**, where the baseline and the charge land in the same scrape regardless. It biases toward *undercounting*, and diff --git a/go.mod b/go.mod index 30a446b..d86fa72 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/InftyAI/Nebula -go 1.24.0 +go 1.25.0 require ( github.com/aws/aws-sdk-go-v2 v1.43.0 diff --git a/internal/controller/nodeclaim_controller.go b/internal/controller/nodeclaim_controller.go index e6dadf4..bb19cfa 100644 --- a/internal/controller/nodeclaim_controller.go +++ b/internal/controller/nodeclaim_controller.go @@ -297,7 +297,7 @@ func (r *NodeClaimReconciler) provider(name string) (provider.Provider, bool) { // still reclaims by asking the provider what exists. func (r *NodeClaimReconciler) desiredPhase(nc *nebulav1alpha1.NodeClaim, pod *corev1.Pod) nebulav1alpha1.NodeClaimPhase { switch { - case isTerminal(pod.Status.Phase): + case util.IsTerminalPodPhase(pod.Status.Phase): return nebulav1alpha1.NodeClaimTerminated case !pod.DeletionTimestamp.IsZero(): return nebulav1alpha1.NodeClaimTerminating @@ -443,11 +443,6 @@ func (r *NodeClaimReconciler) wasBound(nc *nebulav1alpha1.NodeClaim) bool { nc.Status.Phase == nebulav1alpha1.NodeClaimTerminated } -// isTerminal reports whether a Pod phase is an end state that will not progress. -func isTerminal(phase corev1.PodPhase) bool { - return phase == corev1.PodFailed || phase == corev1.PodSucceeded -} - // deleteSelf deletes the claim, which triggers reconcileDelete (the backstop) // via the deletion timestamp + finalizer. Idempotent: a NotFound is treated as // success. diff --git a/internal/controller/pod_placement_helpers.go b/internal/controller/pod_placement_helpers.go index 3614bb1..049ce98 100644 --- a/internal/controller/pod_placement_helpers.go +++ b/internal/controller/pod_placement_helpers.go @@ -411,7 +411,7 @@ func (r *PodPlacementReconciler) reapTerminalPod(ctx context.Context, pod *corev if !pod.DeletionTimestamp.IsZero() { return true, nil // already being deleted; nothing more to place } - if !isTerminal(pod.Status.Phase) { + if !util.IsTerminalPodPhase(pod.Status.Phase) { return false, nil } if !isControllerOwned(pod) { diff --git a/pkg/util/pod.go b/pkg/util/pod.go new file mode 100644 index 0000000..c6fbbfc --- /dev/null +++ b/pkg/util/pod.go @@ -0,0 +1,29 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import corev1 "k8s.io/api/core/v1" + +// IsTerminalPodPhase reports whether a Pod phase is an end state that will not progress. +// +// Shared because the two readers must agree: the control plane reaps and un-gates on a terminal +// Pod, while the virtual node refuses to move one (a provider still listing the instance would +// otherwise walk a Failed Pod back to Running). A predicate that drifted between them would +// resurrect exactly the Pods the other side had written off. +func IsTerminalPodPhase(phase corev1.PodPhase) bool { + return phase == corev1.PodFailed || phase == corev1.PodSucceeded +} diff --git a/pkg/util/pod_test.go b/pkg/util/pod_test.go new file mode 100644 index 0000000..c8f071b --- /dev/null +++ b/pkg/util/pod_test.go @@ -0,0 +1,40 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" +) + +func TestIsTerminalPodPhase(t *testing.T) { + // Pending is the one worth stating: a Pod that has not started yet is not terminal, and + // treating it as one would reap a workload mid-provision. + for phase, want := range map[corev1.PodPhase]bool{ + corev1.PodFailed: true, + corev1.PodSucceeded: true, + corev1.PodRunning: false, + corev1.PodPending: false, + corev1.PodUnknown: false, + "": false, + } { + if got := IsTerminalPodPhase(phase); got != want { + t.Errorf("IsTerminalPodPhase(%q) = %v, want %v", phase, got, want) + } + } +} diff --git a/pkg/vnode/handler.go b/pkg/vnode/handler.go index f231cb9..196bf45 100644 --- a/pkg/vnode/handler.go +++ b/pkg/vnode/handler.go @@ -661,13 +661,17 @@ func (h *Handler) reconcileOnce(ctx context.Context) { h.mu.Lock() emit := make([]*corev1.Pod, 0, len(h.tracked)) tracked := len(h.tracked) - matched := 0 + matched, frozen := 0, 0 for _, tp := range h.tracked { inst, present := byClaim[tp.claimName] before := statusSignature(tp.pod) - if !present { + switch { + // No reverse for a terminal or deleting pod. + case util.IsTerminalPodPhase(tp.pod.Status.Phase) || !tp.pod.DeletionTimestamp.IsZero(): + frozen++ + case !present: applyState(tp.pod, provider.InstanceTerminated, "", h.nowFn()) - } else { + default: matched++ applyState(tp.pod, inst.State, inst.Endpoint, h.nowFn()) // The observed address, for a provider that cannot know it before boot. @@ -688,9 +692,10 @@ func (h *Handler) reconcileOnce(ctx context.Context) { // V(1) so a healthy steady state stays quiet. tracked>0 with matched==0 means the // claim names don't line up with what List returns — the classic "provisioned but - // never Running". + // never Running". frozen is what keeps that reading valid: a terminal or deleting pod is + // deliberately not matched, so without it those would look like the same failure. log.V(1).Info("poll tick", - "listed", len(instances), "tracked", tracked, "matched", matched) + "listed", len(instances), "tracked", tracked, "matched", matched, "frozen", frozen) // Re-emit EVERY tracked pod each tick, which makes status propagation // level-triggered. VK dedups an emit against the last status IT received from us, diff --git a/pkg/vnode/handler_test.go b/pkg/vnode/handler_test.go index 8280a1e..89581a5 100644 --- a/pkg/vnode/handler_test.go +++ b/pkg/vnode/handler_test.go @@ -759,6 +759,70 @@ func TestReconcileOnce_ReportsRunning(t *testing.T) { } } +func TestReconcileOnce_LeavesATerminalPodTerminal(t *testing.T) { + // A provision that failed AFTER the backend created something leaves an instance carrying the + // claim tag while the error dropped its id (Modal's mint runs after the sandbox exists). The + // poll loop matches by claim, so it would find that instance and walk the Pod back out of + // Failed — into Running with no instance id, which is a workload nothing can reach: logs and + // exec both refuse without one, and no connect token was ever persisted. + fp := &fakeProvider{provisionErr: errors.New("mint credential: context deadline exceeded")} + h := NewHandler(fp, nil, nil, openCluster()) + pod := testPod("default", "p1") + + if err := h.CreatePod(context.Background(), pod); err == nil { + t.Fatal("expected CreatePod to return the provision error") + } + // The orphan the failed call left behind, tagged with this pod's claim. + fp.list = []provider.Instance{{ + ID: "inst-1", ClaimName: "default-p1", State: provider.InstanceRunning, Endpoint: "5.6.7.8", + }} + h.reconcileOnce(context.Background()) + + got, err := h.GetPod(context.Background(), "default", "p1") + if err != nil { + t.Fatalf("GetPod: %v", err) + } + if got.Status.Phase != corev1.PodFailed { + t.Fatalf("phase = %q, want Failed to survive the poll tick", got.Status.Phase) + } + if got.Annotations[nebulav1alpha1.InstanceIDAnnotation] != "" { + t.Fatalf("instance id = %q, want none: the poll loop never records one", + got.Annotations[nebulav1alpha1.InstanceIDAnnotation]) + } +} + +func TestReconcileOnce_LeavesADeletingPodAlone(t *testing.T) { + // A pod whose deletion is in flight is on its way to terminal by our own hand, so the + // instance's live state is no longer news: reporting Ready=True again would re-promote a + // workload that is being torn down. + fp := &fakeProvider{provisionID: "inst-1"} + h := NewHandler(fp, nil, nil, openCluster()) + pod := testPod("default", "p1") + if err := h.CreatePod(context.Background(), pod); err != nil { + t.Fatalf("CreatePod: %v", err) + } + + // VK delivers the deletion through UpdatePod before it calls DeletePod. + deleting := pod.DeepCopy() + deleting.DeletionTimestamp = ptrNow(metav1.Now()) + if err := h.UpdatePod(context.Background(), deleting); err != nil { + t.Fatalf("UpdatePod: %v", err) + } + + fp.list = []provider.Instance{{ + ID: "inst-1", ClaimName: "default-p1", State: provider.InstanceRunning, Endpoint: "5.6.7.8", + }} + h.reconcileOnce(context.Background()) + + got, err := h.GetPod(context.Background(), "default", "p1") + if err != nil { + t.Fatalf("GetPod: %v", err) + } + if got.Status.Phase == corev1.PodRunning { + t.Fatal("a deleting pod was promoted to Running by the poll loop") + } +} + func TestReconcileOnce_DNSEndpointNotWrittenToPodIP(t *testing.T) { // AWS reports a public DNS name as the endpoint. PodIP is validated by the API // server as a literal IP, so a DNS name there fails the whole status write; it