diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ac341b952..670b39edb 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -155,6 +155,12 @@ jobs: with: go-version: 1.21.8 + # The CLI embeds the Linux agents, which must have the same version as the CLI + - name: Build Linux agents + shell: bash + run: | + LDFLAGS="-s -w -X github.com/loft-sh/devpod/pkg/version.version=v${{ needs.create-release.outputs.original_package_version }} -X github.com/loft-sh/devpod/pkg/devcontainer/crane.craneSigningKey=${{ secrets.CRANE_PRIVATE_KEY }}" ./hack/build-agents.sh + - name: Build Sidecar CLI if: matrix.settings.host != 'windows-latest' run: | diff --git a/AGENTS.md b/AGENTS.md index 12315d38e..a356b5308 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,6 +29,7 @@ go build ./... go vet ./... ./hack/unit-tests.sh # what CI runs; `go test ./pkg/... ./cmd/...` is the quick version make build # local CLI + desktop build via hack/rebuild.sh +./hack/build-agents.sh # the Linux agents the CLI embeds; run before `go build` of a CLI you want to use ``` Lint, the same way CI does (golangci-lint v1.60.1 cannot be built with the newest Go): @@ -64,15 +65,16 @@ cd src-tauri && cargo +stable check deep link scheme, the Flatpak id and existing installs. - **Dependencies are vendored.** Builds use `-mod=vendor`. After changing imports or `go.mod`, run `GOFLAGS=-mod=mod go mod tidy && go mod vendor` and commit `vendor/`. -- **The same binary runs on both sides.** The CLI injects itself as the agent into the machine/container (or the - agent downloads it from `curityio/devpod` releases). Structs serialized between them (`CLIOptions`, - `AgentWorkspaceInfo`, `ContainerWorkspaceInfo`, etc.) are a wire format between CLI and agent. -- **Dev builds can run a stale agent.** Unless the host is Linux with the container's arch (then the CLI injects - itself), the CLI injects `devpod-linux-` from `devpod-cache/` under Go's `os.TempDir()` (`$TMPDIR` on - macOS) and never refreshes a file that is already there. `hack/build-e2e.sh` and `hack/rebuild.sh` (when building - linux) copy fresh binaries there; if you build some other way, copy them yourself. Also, all dev builds report - version `v0.0.0`, so a container that already has an agent keeps it: recreate the workspace to test agent - changes. +- **The agent is embedded in the CLI.** `hack/build-agents.sh` builds the Linux agents (amd64 and arm64, with the + `devpod_agent` build tag) into `pkg/agent/agentbin/bin/`, and the CLI embeds them; it never downloads an agent. + The agent injects itself from the machine into the container. Without that script, `go build` makes a CLI with + no agents, which can't create workspaces (unless it's built for Linux, where it uses itself). DevPod replaces an + agent of another version, but state an older agent wrote on a machine or in a container (e.g. the serialized + `AgentWorkspaceInfo` and `ContainerWorkspaceInfo`) is still read by the new one. +- **Builds without a version keep a container's agent.** DevPod only replaces an agent whose version differs. + `hack/rebuild.sh` and `hack/build-e2e.sh` give each build its own version (`v0.0.0-dev.`, or `VERSION`), + shared by the CLI and its agents. A plain `hack/build-agents.sh` and `go build` both use `v0.0.0`, so a container + that already has a `v0.0.0` agent keeps it. - **Don't import `k8s.io/client-go/kubernetes`** (or its typed clients, or anything that uses them, such as go-containerregistry's `authn/kubernetes`). Its scheme links every Kubernetes API group and adds over 20 MB to the binary, which also runs as the agent. Use `pkg/kubeclient`, which only registers core/v1 and rbac/v1; add a diff --git a/CHANGELOG.md b/CHANGELOG.md index fbd6354e4..58171eec5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,14 @@ Add new entries to the top of the "Unreleased" section. ### Changed +- The DevPod CLI embeds the Linux agent for amd64 and arm64 ([#19](https://github.com/curityio/devpod/pull/19)), + whatever platform it runs on, and copies it into machines and dev containers itself. It no longer downloads the + agent from GitHub releases, so the agent always has the same version as the CLI, and workspaces can be created + without access to GitHub. The `AGENT_URL` context option, the `DEVPOD_AGENT_URL` environment variable, the + provider's `agent.downloadURL` and the desktop app's "Agent URL" setting are removed. +- When DevPod replaces the agent of an existing dev container that a different DevPod version set up, it warns that + you may want to re-create the workspace with `devpod up --recreate` + ([#19](https://github.com/curityio/devpod/pull/19)). - The `devpod` binary is about 40% smaller ([#21](https://github.com/curityio/devpod/pull/21)): the stripped linux/amd64 build went from 68 MB to 41 MB (20 MB to 14 MB gzipped). The Kubernetes driver and the in-cluster registry keychain now use a small client for the core/v1 and rbac/v1 APIs instead of client-go's clientset, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 00e48a713..0c6f6be6c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,16 +3,12 @@ ## Development Setup 1. Clone the repository locally -2. If you want to change something in DevPod agent code: - 1. Exchange the URL in [DefaultAgentDownloadURL](./pkg/agent/agent.go) with a - custom public repository release you have created. - 2. Build devpod via: `./hack/rebuild.sh` - 3. Upload `test/devpod-linux-amd64` and `test/devpod-linux-arm64` to the public - repository release assets. -3. Build devpod via: `./hack/rebuild.sh` (asking for sudo password) -4. Add docker provider via `devpod provider add docker` -5. Configure docker provider via `devpod use provider docker` -6. Start devpod in vscode with `devpod up examples/simple` +2. Build devpod via: `./hack/rebuild.sh` (asking for sudo password). It first builds the Linux agents with + `./hack/build-agents.sh`, which the CLI embeds and copies into machines and dev containers. Each build gets its + own version (`v0.0.0-dev.`), so DevPod replaces the agent in existing workspaces. +3. Add docker provider via `devpod provider add docker` +4. Configure docker provider via `devpod use provider docker` +5. Start devpod in vscode with `devpod up examples/simple` ## Build from source @@ -21,7 +17,12 @@ Prerequisites CLI: - [Go 1.20](https://go.dev/doc/install) Once installed, run -`CGO_ENABLED=0 go build -ldflags "-s -w" -o devpod-cli` +`./hack/build-agents.sh && CGO_ENABLED=0 go build -ldflags "-s -w" -o devpod-cli` + +This build has version `v0.0.0`, so DevPod won't replace a `v0.0.0` agent that an earlier build put into a +workspace. Without `./hack/build-agents.sh`, the CLI has no agent to copy into machines and dev containers (only a CLI built for +Linux can then use itself as the agent, for its own architecture). Pass the same `-ldflags` to both (as `LDFLAGS` +for the script), so that the agent has the same version as the CLI. Prerequisites GUI: diff --git a/cmd/agent/workspace/logs.go b/cmd/agent/workspace/logs.go index c714898c7..6fb470795 100644 --- a/cmd/agent/workspace/logs.go +++ b/cmd/agent/workspace/logs.go @@ -49,7 +49,7 @@ func (cmd *LogsCmd) Run(ctx context.Context) error { logger := log.Default.ErrorStreamOnly() // create new runner - runner, err := devcontainer.NewRunner(agent.ContainerDevPodHelperLocation, agent.DefaultAgentDownloadURL(), workspaceInfo, logger) + runner, err := devcontainer.NewRunner(agent.ContainerDevPodHelperLocation, workspaceInfo, logger) if err != nil { return fmt.Errorf("create runner: %w", err) } diff --git a/cmd/agent/workspace/up.go b/cmd/agent/workspace/up.go index f08fb85ff..11e789135 100644 --- a/cmd/agent/workspace/up.go +++ b/cmd/agent/workspace/up.go @@ -137,7 +137,7 @@ func (cmd *UpCmd) devPodUp(ctx context.Context, workspaceInfo *provider2.AgentWo } func CreateRunner(workspaceInfo *provider2.AgentWorkspaceInfo, log log.Logger) (devcontainer.Runner, error) { - return devcontainer.NewRunner(agent.ContainerDevPodHelperLocation, agent.DefaultAgentDownloadURL(), workspaceInfo, log) + return devcontainer.NewRunner(agent.ContainerDevPodHelperLocation, workspaceInfo, log) } func InitContentFolder(workspaceInfo *provider2.AgentWorkspaceInfo, log log.Logger) (bool, error) { diff --git a/cmd/build.go b/cmd/build.go index 03a998183..2c5e5f7ad 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -208,8 +208,6 @@ func buildAgentClient(ctx context.Context, workspaceClient client.WorkspaceClien }, workspaceClient.AgentLocal(), workspaceClient.AgentPath(), - workspaceClient.AgentURL(), - true, command, stdinReader, stdoutWriter, diff --git a/cmd/logs.go b/cmd/logs.go index c3c9dffd8..fa30631f0 100644 --- a/cmd/logs.go +++ b/cmd/logs.go @@ -94,8 +94,6 @@ func (cmd *LogsCmd) Run(ctx context.Context, args []string) error { }, client.AgentLocal(), client.AgentPath(), - client.AgentURL(), - true, sshServerCmd, stdinReader, stdoutWriter, diff --git a/cmd/machine/ssh.go b/cmd/machine/ssh.go index 47d641217..733a161ee 100644 --- a/cmd/machine/ssh.go +++ b/cmd/machine/ssh.go @@ -87,8 +87,6 @@ func (cmd *SSHCmd) Run(ctx context.Context, args []string) error { }, machineClient.AgentLocal(), machineClient.AgentPath(), - machineClient.AgentURL(), - true, command, stdin, stdout, diff --git a/cmd/up.go b/cmd/up.go index 17a3c19d0..cb1fe4dc8 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -408,8 +408,6 @@ func (cmd *UpCmd) devPodUpMachine( }, client.AgentLocal(), client.AgentPath(), - client.AgentURL(), - true, sshCmd, sshTunnelStdinReader, sshTunnelStdoutWriter, diff --git a/desktop/src/types.ts b/desktop/src/types.ts index 6245eab5b..8fc5091ec 100644 --- a/desktop/src/types.ts +++ b/desktop/src/types.ts @@ -200,10 +200,7 @@ export type TSupportedIDE = (typeof SUPPORTED_IDES)[number] //#region Context export type TContextOptions = Record // See pkg/config/context.go -export type TContextOptionName = - | "AGENT_URL" - | "SSH_INJECT_DOCKER_CREDENTIALS" - | "SSH_INJECT_GIT_CREDENTIALS" +export type TContextOptionName = "SSH_INJECT_DOCKER_CREDENTIALS" | "SSH_INJECT_GIT_CREDENTIALS" export type TContextOption = Readonly<{ name: TContextOptionName description: string | null | undefined diff --git a/desktop/src/views/Settings/Settings.tsx b/desktop/src/views/Settings/Settings.tsx index 5d429b5a0..d4c8027d6 100644 --- a/desktop/src/views/Settings/Settings.tsx +++ b/desktop/src/views/Settings/Settings.tsx @@ -39,7 +39,6 @@ import { } from "../../lib" import { useWelcomeModal } from "../../useWelcomeModal" import { - useAgentURLOption, useDockerCredentialsForwardingOption, useGitCredentialsForwardingOption, } from "./useContextOptions" @@ -88,7 +87,6 @@ export function Settings() { function GeneralSettings() { const { settings, set } = useChangeSettings() const { modal: welcomeModal, show: showWelcomeModal } = useWelcomeModal() - const { input: agentURLInput, helpText: agentURLHelpText } = useAgentURLOption() const { input: proxyInput, helpText: proxyHelpText } = useProxyOptions() const { badge: installCLIBadge, @@ -127,10 +125,6 @@ function GeneralSettings() { - - {agentURLInput} - - {proxyInput} diff --git a/desktop/src/views/Settings/useContextOptions.tsx b/desktop/src/views/Settings/useContextOptions.tsx index 1ef62ef62..645266fd7 100644 --- a/desktop/src/views/Settings/useContextOptions.tsx +++ b/desktop/src/views/Settings/useContextOptions.tsx @@ -1,12 +1,9 @@ -import { Code, Switch } from "@chakra-ui/react" +import { Switch } from "@chakra-ui/react" import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query" -import { useCallback, useMemo } from "react" +import { useMemo } from "react" import { client } from "../../client" import { QueryKeys } from "../../queryKeys" import { TContextOptionName } from "../../types" -import { ClearableInput } from "./ClearableInput" - -const DEFAULT_DEVPOD_AGENT_URL = "https://github.com/curityio/devpod/releases/latest/download/" export function useContextOptions() { const queryClient = useQueryClient() @@ -32,41 +29,6 @@ export function useContextOptions() { ) } -export function useAgentURLOption() { - const { options, updateOption } = useContextOptions() - - const handleChanged = useCallback( - (newValue: string) => { - const value = newValue.trim() - updateOption({ option: "AGENT_URL", value }) - }, - [updateOption] - ) - - const input = useMemo( - () => ( - - ), - [handleChanged, options?.AGENT_URL.value] - ) - - const helpText = useMemo( - () => ( - <> - Set the Agent URL. If you leave this empty, it will be pulled from{" "} - {DEFAULT_DEVPOD_AGENT_URL} - - ), - [] - ) - - return { input, helpText } -} - export function useDockerCredentialsForwardingOption() { const { options, updateOption } = useContextOptions() diff --git a/e2e/e2e_suite_test.go b/e2e/e2e_suite_test.go index fa1dab17e..19fdfde34 100644 --- a/e2e/e2e_suite_test.go +++ b/e2e/e2e_suite_test.go @@ -1,17 +1,12 @@ package e2e import ( - "os" - "runtime" "testing" - "time" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" - "github.com/loft-sh/devpod/e2e/framework" - // Register tests _ "github.com/loft-sh/devpod/e2e/tests/build" _ "github.com/loft-sh/devpod/e2e/tests/context" @@ -30,17 +25,6 @@ import ( // generated in this directory, and cluster logs will also be saved. // This function is called on each Ginkgo node in parallel mode. func TestRunE2ETests(t *testing.T) { - if runtime.GOOS != "linux" { - go framework.ServeAgent() - - // wait for http server to be up and running - for { - time.Sleep(time.Second) - if os.Getenv("DEVPOD_AGENT_URL") != "" { - break - } - } - } gomega.RegisterFailHandler(ginkgo.Fail) ginkgo.RunSpecs(t, "DevPod e2e suite") } diff --git a/e2e/framework/server_utils.go b/e2e/framework/server_utils.go deleted file mode 100644 index db6e23fc9..000000000 --- a/e2e/framework/server_utils.go +++ /dev/null @@ -1,85 +0,0 @@ -package framework - -import ( - "fmt" - "log" - "net" - "net/http" - "os" - "path/filepath" -) - -// ServeAgent will be a simple http file server that will expose our -// freshly compiled devpod binaries to be downloaded as agents. -// useful for non-linux runners -func ServeAgent() { - // Specify the directory containing the files you want to serve - dir := "bin" - - wd, err := os.Getwd() - if err == nil { - dir = filepath.Join(wd, "bin") - } - - // Create a file server handler for the specified directory - fileServer := http.FileServer(http.Dir(dir)) - - // Register the file server handler to serve files under the /files route - http.Handle("/files/", http.StripPrefix("/files", fileServer)) - - ip := getIP() - - listener, err := net.Listen("tcp", fmt.Sprintf("%v:0", ip)) - if err != nil { - log.Fatal(err) - } - - addr := listener.Addr().String() - err = os.Setenv("DEVPOD_AGENT_URL", "http://"+addr+"/files/") - if err != nil { - log.Fatal(err) - } - - // Start the HTTP server on port 8080 - log.Printf("Server started on %s", addr) - - err = http.Serve(listener, nil) - if err != nil { - log.Fatal(err) - } -} - -func getIP() string { - // Get a list of network interfaces - ifaces, err := net.Interfaces() - if err != nil { - return "0.0.0.0" - } - - // Iterate over each network interface - for _, iface := range ifaces { - addrs, err := iface.Addrs() - if err != nil { - return "0.0.0.0" - } - - for _, addr := range addrs { - switch v := addr.(type) { - case *net.IPAddr: - if v.IP.To4() != nil { - if v.IP.DefaultMask().String() == "ffffff00" || v.IP.DefaultMask().String() == "ff000000" { - return v.IP.String() - } - } - case *net.IPNet: - if v.IP.To4() != nil { - if v.IP.DefaultMask().String() == "ffffff00" || v.IP.DefaultMask().String() == "ff000000" { - return v.IP.String() - } - } - } - } - } - - return "0.0.0.0" -} diff --git a/hack/build-agents.sh b/hack/build-agents.sh new file mode 100755 index 000000000..26893d0aa --- /dev/null +++ b/hack/build-agents.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Builds the Linux agent binaries (amd64 and arm64) that get embedded into the devpod CLI, see pkg/agent/agentbin. +# Run it before building the CLI, with the same LDFLAGS (the version especially), so that the agent matches the CLI. +# Without LDFLAGS, the agents have version v0.0.0, like a plain `go build` of the CLI. + +set -e + +SRC_DIR="${SRCDIR:=.}" +LDFLAGS="${LDFLAGS:=-s -w}" +OUT_DIR="$SRC_DIR/pkg/agent/agentbin/bin" + +for arch in amd64 arm64; do + echo "[INFO] Building agent for linux/$arch" + CGO_ENABLED=0 GOOS=linux GOARCH=$arch go build -tags devpod_agent -ldflags "$LDFLAGS" -o "$OUT_DIR/devpod-linux-$arch" "$SRC_DIR" + gzip -9 -n -f "$OUT_DIR/devpod-linux-$arch" +done +echo "[INFO] Built agents in $OUT_DIR" diff --git a/hack/build-e2e.sh b/hack/build-e2e.sh index 54f63c3a1..1df676f4e 100755 --- a/hack/build-e2e.sh +++ b/hack/build-e2e.sh @@ -11,21 +11,21 @@ then mkdir ./$BUILD_DIR fi -CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o $BUILD_DIR/devpod-linux-amd64 $SRC_DIR -CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags "-s -w" -o $BUILD_DIR/devpod-linux-arm64 $SRC_DIR -CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -ldflags "-s -w" -o $BUILD_DIR/devpod-darwin-arm64 $SRC_DIR -CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags "-s -w" -o $BUILD_DIR/devpod-darwin-amd64 $SRC_DIR +# Each build gets its own version, so that DevPod replaces the agent that an earlier build put into a container. +# The CLI and the agents it embeds must have the same version. +VERSION="${VERSION:-v0.0.0-dev.$(date -u +%Y%m%d%H%M%S)}" +LDFLAGS="-s -w -X github.com/loft-sh/devpod/pkg/version.version=$VERSION" +echo "Building version $VERSION" + +# the CLI embeds the Linux agents +SRCDIR=$SRC_DIR LDFLAGS="$LDFLAGS" $SRC_DIR/hack/build-agents.sh + +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "$LDFLAGS" -o $BUILD_DIR/devpod-linux-amd64 $SRC_DIR +CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags "$LDFLAGS" -o $BUILD_DIR/devpod-linux-arm64 $SRC_DIR +CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -ldflags "$LDFLAGS" -o $BUILD_DIR/devpod-darwin-arm64 $SRC_DIR +CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags "$LDFLAGS" -o $BUILD_DIR/devpod-darwin-amd64 $SRC_DIR chmod +x $BUILD_DIR/devpod-linux-amd64 chmod +x $BUILD_DIR/devpod-linux-arm64 chmod +x $BUILD_DIR/devpod-darwin-arm64 chmod +x $BUILD_DIR/devpod-darwin-amd64 - -# The CLI injects the agent from os.TempDir()/devpod-cache, and dev builds never refresh it, -# so put the new binaries where Go looks: $TMPDIR when set (as on macOS), /tmp otherwise. -CACHE_DIR="${TMPDIR:-/tmp}" -CACHE_DIR="${CACHE_DIR%/}/devpod-cache" -mkdir -p "$CACHE_DIR" -cp $BUILD_DIR/devpod-linux-amd64 "$CACHE_DIR/devpod-linux-amd64" -cp $BUILD_DIR/devpod-linux-arm64 "$CACHE_DIR/devpod-linux-arm64" -echo "Copied the linux agent binaries to $CACHE_DIR" diff --git a/hack/rebuild.sh b/hack/rebuild.sh index ff0cfe6fc..a144eb07a 100755 --- a/hack/rebuild.sh +++ b/hack/rebuild.sh @@ -2,10 +2,6 @@ set -e -if [[ -z "${TMPDIR}" ]]; then - TMPDIR="/tmp" -fi - if [[ -z "${BUILD_PLATFORMS}" ]]; then BUILD_PLATFORMS="linux windows darwin" fi @@ -14,6 +10,15 @@ if [[ -z "${BUILD_ARCHS}" ]]; then BUILD_ARCHS="amd64 arm64" fi +# Each build gets its own version, so that DevPod replaces the agent that an earlier build put into a container. +# The CLI and the agents it embeds must have the same version. +VERSION="${VERSION:-v0.0.0-dev.$(date -u +%Y%m%d%H%M%S)}" +LDFLAGS="-s -w -X github.com/loft-sh/devpod/pkg/version.version=$VERSION" +echo "[INFO] Building version $VERSION" + +# the CLI embeds the Linux agents, so they're always needed +LDFLAGS="$LDFLAGS" ./hack/build-agents.sh + for os in $BUILD_PLATFORMS; do for arch in $BUILD_ARCHS; do # don't build for arm on windows @@ -23,9 +28,9 @@ for os in $BUILD_PLATFORMS; do echo "[INFO] Building for $os/$arch" if [[ $RACE == "yes" ]]; then echo "Building devpod with race detector" - CGO_ENABLED=1 GOOS=$os GOARCH=$arch go build -race -ldflags "-s -w" -o test/devpod-cli-$os-$arch + CGO_ENABLED=1 GOOS=$os GOARCH=$arch go build -race -ldflags "$LDFLAGS" -o test/devpod-cli-$os-$arch else - CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -ldflags "-s -w" -o test/devpod-cli-$os-$arch + CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -ldflags "$LDFLAGS" -o test/devpod-cli-$os-$arch fi done done @@ -34,20 +39,15 @@ echo "[INFO] Built binaries for all platforms in test/ directory" # if not true, install the binary if [[ "${SKIP_INSTALL}" != "true" ]]; then if command -v sudo &> /dev/null; then - go build -o test/devpod && sudo mv test/devpod /usr/local/bin/ + go build -ldflags "$LDFLAGS" -o test/devpod && sudo mv test/devpod /usr/local/bin/ else - go install . + go install -ldflags "$LDFLAGS" . fi echo "[INFO] Installed devpod binary to /usr/local/bin" else echo "[INFO] Skipping install of devpod binary" fi -if [[ $BUILD_PLATFORMS == *"linux"* ]]; then - cp test/devpod-cli-linux-amd64 test/devpod-linux-amd64 - cp test/devpod-cli-linux-arm64 test/devpod-linux-arm64 -fi - if [ -d "desktop/src-tauri/bin" ]; then if [[ $BUILD_PLATFORMS == *"linux"* ]]; then cp test/devpod-cli-linux-amd64 desktop/src-tauri/bin/devpod-cli-x86_64-unknown-linux-gnu @@ -62,11 +62,3 @@ if [ -d "desktop/src-tauri/bin" ]; then fi echo "[INFO] Copied binaries to desktop/src-tauri/bin" fi - -if [[ $BUILD_PLATFORMS == *"linux"* ]]; then - rm -R $TMPDIR/devpod-cache 2>/dev/null || true - mkdir -p $TMPDIR/devpod-cache - cp test/devpod-cli-linux-amd64 $TMPDIR/devpod-cache/devpod-linux-amd64 - cp test/devpod-cli-linux-arm64 $TMPDIR/devpod-cache/devpod-linux-arm64 - echo "[INFO] Copied binaries to $TMPDIR/devpod-cache" -fi diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 89dfee2eb..a5ee04bb6 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -16,7 +16,6 @@ import ( "github.com/loft-sh/devpod/pkg/command" "github.com/loft-sh/devpod/pkg/compress" provider2 "github.com/loft-sh/devpod/pkg/provider" - "github.com/loft-sh/devpod/pkg/version" "github.com/loft-sh/log" perrors "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -30,25 +29,8 @@ const RemoteDevPodHelperLocation = "/tmp/devpod" const ContainerActivityFile = "/tmp/devpod.activity" -const defaultAgentDownloadURL = "https://github.com/curityio/devpod/releases/download/" - -const EnvDevPodAgentURL = "DEVPOD_AGENT_URL" - const WorkspaceBusyFile = "workspace.lock" -func DefaultAgentDownloadURL() string { - devPodAgentURL := os.Getenv(EnvDevPodAgentURL) - if devPodAgentURL != "" { - return strings.TrimSuffix(devPodAgentURL, "/") + "/" - } - - if version.GetVersion() == version.DevVersion { - return "https://github.com/curityio/devpod/releases/latest/download/" - } - - return defaultAgentDownloadURL + version.GetVersion() -} - func DecodeContainerWorkspaceInfo(workspaceInfoRaw string) (*provider2.ContainerWorkspaceInfo, string, error) { decoded, err := compress.Decompress(workspaceInfoRaw) if err != nil { @@ -318,7 +300,7 @@ func Tunnel( // inject agent err := InjectAgent(ctx, func(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { return exec(ctx, "root", command, stdin, stdout, stderr) - }, false, ContainerDevPodHelperLocation, DefaultAgentDownloadURL(), false, log, timeout) + }, ContainerDevPodHelperLocation, log, timeout) if err != nil { return err } diff --git a/pkg/agent/agentbin/agentbin.go b/pkg/agent/agentbin/agentbin.go new file mode 100644 index 000000000..be2682656 --- /dev/null +++ b/pkg/agent/agentbin/agentbin.go @@ -0,0 +1,52 @@ +// Package agentbin holds the Linux agent binaries that DevPod copies into machines and dev containers. +// +// hack/build-agents.sh builds them into bin/ (gzipped) before the CLI is built, with the same version as the CLI, +// so that the agent always matches the CLI that injects it. Builds that skip it, such as a plain +// `go build ./...`, have no agent binaries. The agent binaries themselves are built with the devpod_agent build +// tag and don't embed anything: an agent only ever injects itself. +package agentbin + +import ( + "compress/gzip" + "errors" + "fmt" + "io" + "io/fs" +) + +// ErrNotEmbedded is returned by Open when this binary has no agent binary for the requested architecture. +var ErrNotEmbedded = errors.New("agent binary not embedded") + +// FileName is the name of the agent binary for linux/, as built by hack/build-agents.sh (without the .gz +// suffix). +func FileName(arch string) string { + return "devpod-linux-" + arch +} + +// Open returns the decompressed agent binary for linux/. +func Open(arch string) (io.ReadCloser, error) { + file, err := files.Open("bin/" + FileName(arch) + ".gz") + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("%w for linux/%s", ErrNotEmbedded, arch) + } + return nil, err + } + + reader, err := gzip.NewReader(file) + if err != nil { + _ = file.Close() + return nil, fmt.Errorf("decompress agent binary for linux/%s: %w", arch, err) + } + + return &gzipFile{Reader: reader, file: file}, nil +} + +type gzipFile struct { + *gzip.Reader + file fs.File +} + +func (g *gzipFile) Close() error { + return errors.Join(g.Reader.Close(), g.file.Close()) +} diff --git a/pkg/agent/agentbin/agentbin_test.go b/pkg/agent/agentbin/agentbin_test.go new file mode 100644 index 000000000..0b64f2386 --- /dev/null +++ b/pkg/agent/agentbin/agentbin_test.go @@ -0,0 +1,13 @@ +package agentbin + +import ( + "errors" + "testing" + + "gotest.tools/assert" +) + +func TestOpenNotEmbedded(t *testing.T) { + _, err := Open("riscv64") + assert.Assert(t, errors.Is(err, ErrNotEmbedded)) +} diff --git a/pkg/agent/agentbin/bin/.gitignore b/pkg/agent/agentbin/bin/.gitignore new file mode 100644 index 000000000..848e83ee4 --- /dev/null +++ b/pkg/agent/agentbin/bin/.gitignore @@ -0,0 +1,3 @@ +# The agent binaries that hack/build-agents.sh builds here are not committed. +* +!.gitignore diff --git a/pkg/agent/agentbin/embed.go b/pkg/agent/agentbin/embed.go new file mode 100644 index 000000000..7433ffba7 --- /dev/null +++ b/pkg/agent/agentbin/embed.go @@ -0,0 +1,13 @@ +//go:build !devpod_agent + +package agentbin + +import "embed" + +// IsAgent is true in the agent binaries, which embed no other agents. +const IsAgent = false + +// bin/.gitignore is always there, so this builds even when the agent binaries are not. +// +//go:embed all:bin +var files embed.FS diff --git a/pkg/agent/agentbin/embed_agent.go b/pkg/agent/agentbin/embed_agent.go new file mode 100644 index 000000000..7a763aa13 --- /dev/null +++ b/pkg/agent/agentbin/embed_agent.go @@ -0,0 +1,11 @@ +//go:build devpod_agent + +package agentbin + +import "embed" + +// IsAgent is true in the agent binaries, which embed no other agents. +const IsAgent = true + +// files is empty: the agent binaries embed no other agents. +var files embed.FS diff --git a/pkg/agent/inject.go b/pkg/agent/inject.go index f57bb7307..04395b91b 100644 --- a/pkg/agent/inject.go +++ b/pkg/agent/inject.go @@ -3,14 +3,14 @@ package agent import ( "bytes" "context" + "errors" "fmt" "io" "os" - "path/filepath" "runtime" "time" - devpodhttp "github.com/loft-sh/devpod/pkg/http" + "github.com/loft-sh/devpod/pkg/agent/agentbin" "github.com/loft-sh/devpod/pkg/inject" "github.com/loft-sh/devpod/pkg/shell" "github.com/loft-sh/devpod/pkg/version" @@ -19,39 +19,42 @@ import ( var waitForInstanceConnectionTimeout = time.Minute * 5 +// InjectAgent installs the agent into a dev container, replacing an agent of a different version. As the +// container was then set up by another DevPod version, it warns the user that they may want to re-create the +// workspace. func InjectAgent( ctx context.Context, exec inject.ExecFunc, - local bool, - remoteAgentPath, - downloadURL string, - preferDownload bool, + remoteAgentPath string, log log.Logger, timeout time.Duration, ) error { - return InjectAgentAndExecute( - ctx, - exec, - local, - remoteAgentPath, - downloadURL, - preferDownload, - "", - nil, - nil, - nil, - log, - timeout, - ) + previousVersion, err := injectAgentAndExecute(ctx, exec, remoteAgentPath, "", nil, nil, nil, log, timeout) + if err != nil { + return err + } + + if previousVersion != "" { + if previousVersion == "unknown" { + previousVersion = "an unknown version" + } + log.Warnf( + "The dev container was set up by DevPod %s, but this is DevPod %s. DevPod updated its agent in the container, "+ + "but to avoid issues you may want to re-create the workspace with `devpod up --recreate`.", + previousVersion, version.GetVersion(), + ) + } + + return nil } +// InjectAgentAndExecute installs the agent at remoteAgentPath unless it is already there in this version, and then +// runs command. If local is true, it runs command locally instead. func InjectAgentAndExecute( ctx context.Context, exec inject.ExecFunc, local bool, remoteAgentPath, - downloadURL string, - preferDownload bool, command string, stdin io.Reader, stdout io.Writer, @@ -69,23 +72,37 @@ func InjectAgentAndExecute( return shell.RunEmulatedShell(ctx, command, stdin, stdout, stderr, nil) } + previousVersion, err := injectAgentAndExecute(ctx, exec, remoteAgentPath, command, stdin, stdout, stderr, log, timeout) + if previousVersion != "" { + log.Debugf("Replaced agent version %s at %s", previousVersion, remoteAgentPath) + } + return err +} + +// injectAgentAndExecute returns the version of the agent it replaced, if any. +func injectAgentAndExecute( + ctx context.Context, + exec inject.ExecFunc, + remoteAgentPath, + command string, + stdin io.Reader, + stdout io.Writer, + stderr io.Writer, + log log.Logger, + timeout time.Duration, +) (string, error) { defer log.Debugf("Done InjectAgentAndExecute") if remoteAgentPath == "" { remoteAgentPath = RemoteDevPodHelperLocation } - if downloadURL == "" { - downloadURL = DefaultAgentDownloadURL() - } versionCheck := fmt.Sprintf(`[ "$(%s version 2>/dev/null || echo 'false')" != "%s" ]`, remoteAgentPath, version.GetVersion()) - if version.GetVersion() == version.DevVersion { - preferDownload = false - } // install devpod into the target // do a simple hello world to check if we can get something now := time.Now() lastMessage := time.Now() + previousVersion := "" for { buf := &bytes.Buffer{} if stderr != nil { @@ -95,19 +112,24 @@ func InjectAgentAndExecute( } scriptParams := &inject.Params{ - Command: command, - AgentRemotePath: remoteAgentPath, - DownloadURLs: inject.NewDownloadURLs(downloadURL), - ExistsCheck: versionCheck, - PreferAgentDownload: preferDownload, - ShouldChmodPath: true, + Command: command, + AgentRemotePath: remoteAgentPath, + ExistsCheck: versionCheck, + ShouldChmodPath: true, } + var binaryErr error wasExecuted, err := inject.InjectAndExecute( ctx, exec, - func(arm bool) (io.ReadCloser, error) { - return injectBinary(arm, downloadURL, log) + func(request inject.BinaryRequest) (io.ReadCloser, error) { + if request.PreviousVersion != "" { + previousVersion = request.PreviousVersion + } + + binary, err := agentBinary(request.Arm) + binaryErr = err + return binary, err }, scriptParams, stdin, @@ -117,10 +139,13 @@ func InjectAgentAndExecute( log, ) if err != nil { - if time.Since(now) > waitForInstanceConnectionTimeout { - return fmt.Errorf("timeout waiting for instance connection: %w", err) + if binaryErr != nil { + // retrying won't help + return "", binaryErr + } else if time.Since(now) > waitForInstanceConnectionTimeout { + return "", fmt.Errorf("timeout waiting for instance connection: %w", err) } else if wasExecuted { - return fmt.Errorf("agent error: %s: %w", buf.String(), err) + return "", fmt.Errorf("agent error: %s: %w", buf.String(), err) } if time.Since(lastMessage) > time.Second*5 { @@ -136,100 +161,43 @@ func InjectAgentAndExecute( break } - return nil + return previousVersion, nil } -func injectBinary(arm bool, tryDownloadURL string, log log.Logger) (io.ReadCloser, error) { - // this means we need to +// agentBinary returns the agent binary for linux/amd64 or linux/arm64: the one embedded in this binary or, if +// there is none, this binary itself when it is built for that platform. This is how an agent, which embeds no +// other agents, injects itself into the dev container. +// +// The agent is never downloaded, so that it always matches the CLI that injects it. +func agentBinary(arm bool) (io.ReadCloser, error) { targetArch := "amd64" if arm { targetArch = "arm64" } - // make sure a linux arm64 binary exists locally - var err error - var binaryPath string + binary, err := agentbin.Open(targetArch) + if err == nil { + return binary, nil + } else if !errors.Is(err, agentbin.ErrNotEmbedded) { + return nil, err + } + if runtime.GOOS == "linux" && runtime.GOARCH == targetArch { - binaryPath, err = os.Executable() + self, err := os.Executable() if err != nil { return nil, fmt.Errorf("get executable: %w", err) } - // check if we still exist - _, err = os.Stat(binaryPath) + file, err := os.Open(self) if err != nil { - binaryPath = "" + return nil, fmt.Errorf("open agent binary: %w", err) } - } - // try to look up runner binaries - if binaryPath == "" { - binaryPath = getRunnerBinary(targetArch) + return file, nil } - // download devpod locally - if binaryPath == "" { - binaryPath, err = downloadAgentLocally(tryDownloadURL, targetArch, log) - if err != nil { - return nil, fmt.Errorf("download agent locally: %w", err) - } - } - - // read file - file, err := os.Open(binaryPath) - if err != nil { - return nil, fmt.Errorf("open agent binary: %w", err) - } - - return file, nil -} - -func downloadAgentLocally(tryDownloadURL, targetArch string, log log.Logger) (string, error) { - agentPath := filepath.Join(os.TempDir(), "devpod-cache", "devpod-linux-"+targetArch) - err := os.MkdirAll(filepath.Dir(agentPath), 0755) - if err != nil { - return "", fmt.Errorf("create agent path: %w", err) - } - - stat, statErr := os.Stat(agentPath) - if version.GetVersion() == version.DevVersion && statErr == nil { - return agentPath, nil - } - - fullDownloadURL := tryDownloadURL + "/devpod-linux-" + targetArch - log.Debugf("Attempting to download DevPod agent from: %s", fullDownloadURL) - - resp, err := devpodhttp.GetHTTPClient().Get(fullDownloadURL) - if err != nil { - return "", fmt.Errorf("download devpod: %w", err) - } - defer resp.Body.Close() - - if statErr == nil && stat.Size() == resp.ContentLength { - return agentPath, nil - } - - log.Infof("Download DevPod Agent...") - file, err := os.Create(agentPath) - if err != nil { - return "", fmt.Errorf("create agent binary: %w", err) - } - defer file.Close() - - _, err = io.Copy(file, resp.Body) - if err != nil { - _ = os.Remove(agentPath) - return "", fmt.Errorf("failed to download devpod from URL %s: %w", fullDownloadURL, err) - } - - return agentPath, nil -} - -func getRunnerBinary(targetArch string) string { - binaryPath := filepath.Join(os.TempDir(), "devpod-cache", "devpod-linux-"+targetArch) - _, err := os.Stat(binaryPath) - if err != nil { - return "" + if agentbin.IsAgent { + return nil, fmt.Errorf("the DevPod agent (%s/%s) can only install itself, not an agent for linux/%s", runtime.GOOS, runtime.GOARCH, targetArch) } - return binaryPath + return nil, fmt.Errorf("this DevPod build has no agent for linux/%s: run hack/build-agents.sh before building DevPod", targetArch) } diff --git a/pkg/client/client.go b/pkg/client/client.go index a216d09d6..84a911459 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -38,9 +38,6 @@ type Client interface { // AgentPath returns the agent path AgentPath() string - // AgentURL returns the agent download url - AgentURL() string - // Create creates a new workspace Create(ctx context.Context, options CreateOptions) error diff --git a/pkg/client/clientimplementation/machine_client.go b/pkg/client/clientimplementation/machine_client.go index 3c6ea5416..51f9ce2d5 100644 --- a/pkg/client/clientimplementation/machine_client.go +++ b/pkg/client/clientimplementation/machine_client.go @@ -75,10 +75,6 @@ func (s *machineClient) AgentLocal() bool { return options.ResolveAgentConfig(s.devPodConfig, s.config, nil, s.machine).Local == "true" } -func (s *machineClient) AgentURL() string { - return options.ResolveAgentConfig(s.devPodConfig, s.config, nil, s.machine).DownloadURL -} - func (s *machineClient) Context() string { return s.machine.Context } diff --git a/pkg/client/clientimplementation/workspace_client.go b/pkg/client/clientimplementation/workspace_client.go index 4e3ce4a35..0c9e0c98b 100644 --- a/pkg/client/clientimplementation/workspace_client.go +++ b/pkg/client/clientimplementation/workspace_client.go @@ -91,13 +91,6 @@ func (s *workspaceClient) AgentPath() string { return options.ResolveAgentConfig(s.devPodConfig, s.config, s.workspace, s.machine).Path } -func (s *workspaceClient) AgentURL() string { - s.m.Lock() - defer s.m.Unlock() - - return options.ResolveAgentConfig(s.devPodConfig, s.config, s.workspace, s.machine).DownloadURL -} - func (s *workspaceClient) Context() string { return s.workspace.Context } diff --git a/pkg/config/context.go b/pkg/config/context.go index b2bb87966..0d4bfe3eb 100644 --- a/pkg/config/context.go +++ b/pkg/config/context.go @@ -9,7 +9,6 @@ const ( ContextOptionSSHInjectDockerCredentials = "SSH_INJECT_DOCKER_CREDENTIALS" ContextOptionSSHInjectGitCredentials = "SSH_INJECT_GIT_CREDENTIALS" ContextOptionExitAfterTimeout = "EXIT_AFTER_TIMEOUT" - ContextOptionAgentURL = "AGENT_URL" ContextOptionDotfilesURL = "DOTFILES_URL" ContextOptionDotfilesScript = "DOTFILES_SCRIPT" ContextOptionSSHAgentForwarding = "SSH_AGENT_FORWARDING" @@ -64,10 +63,6 @@ var ContextOptions = []ContextOption{ Default: "true", Enum: []string{"true", "false"}, }, - { - Name: ContextOptionAgentURL, - Description: "Specifies the agent url to use for DevPod", - }, { Name: ContextOptionDotfilesURL, Description: "Specifies the dotfiles repo url to use for DevPod", diff --git a/pkg/devcontainer/run.go b/pkg/devcontainer/run.go index 95e84026a..2decf84a1 100644 --- a/pkg/devcontainer/run.go +++ b/pkg/devcontainer/run.go @@ -45,7 +45,7 @@ type Runner interface { } func NewRunner( - agentPath, agentDownloadURL string, + agentPath string, workspaceConfig *provider2.AgentWorkspaceInfo, log log.Logger, ) (Runner, error) { @@ -59,7 +59,6 @@ func NewRunner( Driver: driver, AgentPath: agentPath, - AgentDownloadURL: agentDownloadURL, LocalWorkspaceFolder: workspaceConfig.ContentFolder, ID: GetRunnerIDFromWorkspace(workspaceConfig.Workspace), WorkspaceConfig: workspaceConfig, @@ -70,9 +69,8 @@ func NewRunner( type runner struct { Driver driver.Driver - WorkspaceConfig *provider2.AgentWorkspaceInfo - AgentPath string - AgentDownloadURL string + WorkspaceConfig *provider2.AgentWorkspaceInfo + AgentPath string LocalWorkspaceFolder string diff --git a/pkg/devcontainer/setup.go b/pkg/devcontainer/setup.go index 82a83afcd..6fed1f121 100644 --- a/pkg/devcontainer/setup.go +++ b/pkg/devcontainer/setup.go @@ -34,7 +34,7 @@ func (r *runner) setupContainer( // inject agent err := agent.InjectAgent(ctx, func(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { return r.Driver.CommandDevContainer(ctx, r.ID, "root", command, stdin, stdout, stderr) - }, false, agent.ContainerDevPodHelperLocation, agent.DefaultAgentDownloadURL(), false, r.Log, timeout) + }, agent.ContainerDevPodHelperLocation, r.Log, timeout) if err != nil { return nil, errors.Wrap(err, "inject agent") } diff --git a/pkg/inject/download_urls.go b/pkg/inject/download_urls.go deleted file mode 100644 index ca3b3b0b2..000000000 --- a/pkg/inject/download_urls.go +++ /dev/null @@ -1,36 +0,0 @@ -package inject - -import ( - "net/url" - "strings" -) - -const AmdUrl = "devpod-linux-amd64" -const ArmUrl = "devpod-linux-arm64" -const BinNamePlaceholder = "${BIN_NAME}" - -type DownloadURLs struct { - Base string - Amd string - Arm string -} - -func NewDownloadURLs(baseUrl string) *DownloadURLs { - var amdUrl, armUrl string - - // replace ${BIN_NAME} with binary name - if strings.Contains(baseUrl, BinNamePlaceholder) { - baseUrl = strings.TrimSuffix(baseUrl, "/") - amdUrl = strings.Replace(baseUrl, BinNamePlaceholder, AmdUrl, 1) - armUrl = strings.Replace(baseUrl, BinNamePlaceholder, ArmUrl, 1) - } else { - amdUrl, _ = url.JoinPath(baseUrl, AmdUrl) - armUrl, _ = url.JoinPath(baseUrl, ArmUrl) - } - - return &DownloadURLs{ - Base: baseUrl, - Amd: amdUrl, - Arm: armUrl, - } -} diff --git a/pkg/inject/inject.go b/pkg/inject/inject.go index 536ca3869..a28789188 100644 --- a/pkg/inject/inject.go +++ b/pkg/inject/inject.go @@ -21,7 +21,17 @@ var Script string type ExecFunc func(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error -type LocalFile func(arm bool) (io.ReadCloser, error) +// BinaryRequest is what the inject script asks for when the agent needs to be installed. +type BinaryRequest struct { + // Arm is true if the target needs the arm64 binary. + Arm bool + + // PreviousVersion is the version of the agent that gets replaced: empty if there was none, "unknown" if it + // didn't report its version. + PreviousVersion string +} + +type LocalFile func(request BinaryRequest) (io.ReadCloser, error) type injectResult struct { wasExecuted bool @@ -45,10 +55,6 @@ func InjectAndExecute( } log.Debugf("execute inject script") - if scriptParams.PreferAgentDownload { - log.Debugf("download agent from %s", scriptParams.DownloadURLs.Base) - } - defer log.Debugf("done injecting") // start script @@ -169,7 +175,7 @@ func inject( log.Debugf("Inject binary") defer log.Debugf("Done injecting binary") - fileReader, err := getFileReader(localFile, lineStr) + fileReader, err := localFile(parseBinaryRequest(lineStr)) if err != nil { return false, err } @@ -204,9 +210,13 @@ func isInjectingOfBinaryNeeded(lineStr string) bool { return strings.HasPrefix(lineStr, "ARM-") } -func getFileReader(localFile LocalFile, lineStr string) (io.ReadCloser, error) { - isArm := strings.TrimPrefix(lineStr, "ARM-") == "true" - return localFile(isArm) +// parseBinaryRequest parses the "ARM- [previous version]" line of the inject script. +func parseBinaryRequest(lineStr string) BinaryRequest { + arm, previousVersion, _ := strings.Cut(strings.TrimPrefix(lineStr, "ARM-"), " ") + return BinaryRequest{ + Arm: arm == "true", + PreviousVersion: strings.TrimSpace(previousVersion), + } } func performMutualHandshake(line string, stdin io.WriteCloser) error { diff --git a/pkg/inject/inject.sh b/pkg/inject/inject.sh index e88725817..061670584 100644 --- a/pkg/inject/inject.sh +++ b/pkg/inject/inject.sh @@ -8,7 +8,6 @@ INSTALL_DIR="{{ .InstallDir }}" INSTALL_FILENAME="{{ .InstallFilename }}" INSTALL_PATH="$INSTALL_DIR/$INSTALL_FILENAME" -PREFER_DOWNLOAD="{{ .PreferDownload }}" CHMOD_PATH="{{ .ChmodPath }}" # start marker @@ -51,8 +50,10 @@ check_noexec() { return 0 } +# Asks for the agent binary, telling which architecture is needed and which agent version it replaces (if any). +# Exits afterwards: the caller runs the command again once the agent is installed. inject() { - echo "ARM-$(is_arm && echo -n 'true' || echo -n 'false')" + echo "ARM-$(is_arm && printf 'true' || printf 'false') $PREVIOUS_VERSION" $sh_c "cat > $INSTALL_PATH.$$" $sh_c "mv $INSTALL_PATH.$$ $INSTALL_PATH" @@ -64,43 +65,15 @@ inject() { exit 0 } -download() { - DOWNLOAD_URL="{{ .DownloadAmd }}" - if is_arm; then - DOWNLOAD_URL="{{ .DownloadArm }}" - fi - iteration=1 - max_iteration=3 - - while :; do - if [ "$iteration" -gt "$max_iteration" ]; then - >&2 echo "error: failed to download devpod" - exit 1 - fi - - cmd_status="" - if command_exists curl; then - $sh_c "curl -fsSL $DOWNLOAD_URL -o $INSTALL_PATH.$$" && break - cmd_status=$? - elif command_exists wget; then - $sh_c "wget -q $DOWNLOAD_URL -O $INSTALL_PATH.$$" && break - cmd_status=$? - else - echo "error: no download tool found, please install curl or wget" - exit 127 +if {{ .ExistsCheck }}; then + PREVIOUS_VERSION="" + if [ -e "$INSTALL_PATH" ]; then + PREVIOUS_VERSION="$("$INSTALL_PATH" version 2>/dev/null | head -n 1)" + if [ -z "$PREVIOUS_VERSION" ]; then + PREVIOUS_VERSION="unknown" fi - >&2 echo "error: failed to download devpod" - >&2 echo " command returned: ${cmd_status}" - >&2 echo "Trying again in 10 seconds..." - iteration=$((iteration+1)) - sleep 10 - done - - $sh_c "mv $INSTALL_PATH.$$ $INSTALL_PATH" -} - + fi -if {{ .ExistsCheck }}; then user="$(id -un || true)" sh_c='sh -c' @@ -132,27 +105,11 @@ if {{ .ExistsCheck }}; then fi $sh_c "rm -f $INSTALL_PATH 2>/dev/null || true" - if [ "$PREFER_DOWNLOAD" = "true" ]; then - download || inject - else - inject || download - fi - - if [ "$CHMOD_PATH" = "true" ]; then - $sh_c "chmod +x $INSTALL_PATH" - fi - - if {{ .ExistsCheck }}; then - >&2 echo Error: failed to install devpod - exit 1 - fi + inject fi # send parent done stream echo "done" -# set download url -export DEVPOD_AGENT_URL={{ .DownloadBase }} - # Execute command {{ .Command }} diff --git a/pkg/inject/inject_test.go b/pkg/inject/inject_test.go new file mode 100644 index 000000000..351d76bd0 --- /dev/null +++ b/pkg/inject/inject_test.go @@ -0,0 +1,92 @@ +package inject + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/loft-sh/log" + "gotest.tools/assert" +) + +func TestParseBinaryRequest(t *testing.T) { + assert.DeepEqual(t, parseBinaryRequest("ARM-true v1.2.3"), BinaryRequest{Arm: true, PreviousVersion: "v1.2.3"}) + assert.DeepEqual(t, parseBinaryRequest("ARM-false "), BinaryRequest{Arm: false}) + assert.DeepEqual(t, parseBinaryRequest("ARM-false"), BinaryRequest{Arm: false}) +} + +func TestInjectReplacesAgentOfOtherVersion(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("inject.sh needs a POSIX shell") + } + + agentPath := filepath.Join(t.TempDir(), "devpod") + assert.NilError(t, os.WriteFile(agentPath, fakeAgent("v1.0.0"), 0755)) + + requests, err := injectFakeAgent(agentPath, "v2.0.0") + assert.NilError(t, err) + assert.Equal(t, len(requests), 1) + assert.Equal(t, requests[0].PreviousVersion, "v1.0.0") + + installed, err := os.ReadFile(agentPath) + assert.NilError(t, err) + assert.DeepEqual(t, installed, fakeAgent("v2.0.0")) + + // now that the version matches, nothing is injected + requests, err = injectFakeAgent(agentPath, "v2.0.0") + assert.NilError(t, err) + assert.Equal(t, len(requests), 0) +} + +func TestInjectInstallsMissingAgent(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("inject.sh needs a POSIX shell") + } + + agentPath := filepath.Join(t.TempDir(), "devpod") + requests, err := injectFakeAgent(agentPath, "v2.0.0") + assert.NilError(t, err) + assert.Equal(t, len(requests), 1) + assert.Equal(t, requests[0].PreviousVersion, "") +} + +// injectFakeAgent runs inject.sh locally to install a fake agent of the given version at agentPath. +func injectFakeAgent(agentPath, version string) ([]BinaryRequest, error) { + var requests []BinaryRequest + _, err := InjectAndExecute( + context.Background(), + func(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + cmd := exec.CommandContext(ctx, "sh", "-c", command) + cmd.Stdin = stdin + cmd.Stdout = stdout + cmd.Stderr = stderr + return cmd.Run() + }, + func(request BinaryRequest) (io.ReadCloser, error) { + requests = append(requests, request) + return io.NopCloser(bytes.NewReader(fakeAgent(version))), nil + }, + &Params{ + AgentRemotePath: agentPath, + ExistsCheck: fmt.Sprintf(`[ "$(%s version 2>/dev/null || echo 'false')" != "%s" ]`, agentPath, version), + ShouldChmodPath: true, + }, + nil, + nil, + nil, + 10*time.Second, + log.Discard, + ) + return requests, err +} + +func fakeAgent(version string) []byte { + return []byte("#!/bin/sh\nprintf '%s' '" + version + "'\n") +} diff --git a/pkg/inject/script.go b/pkg/inject/script.go index 3b0589ca2..2a3e33b1d 100644 --- a/pkg/inject/script.go +++ b/pkg/inject/script.go @@ -14,11 +14,7 @@ func GenerateScript(script string, params *Params) (string, error) { "ExistsCheck": params.ExistsCheck, "InstallDir": params.InstallDir(), "InstallFilename": params.InstallFilename(), - "PreferDownload": params.PreferDownload(), "ChmodPath": params.ChmodPath(), - "DownloadBase": params.DownloadURLs.Base, - "DownloadAmd": params.DownloadURLs.Amd, - "DownloadArm": params.DownloadURLs.Arm, }) if err != nil { return "", err @@ -30,11 +26,9 @@ func GenerateScript(script string, params *Params) (string, error) { type Params struct { Command string AgentRemotePath string - DownloadURLs *DownloadURLs - ExistsCheck string - PreferAgentDownload bool - ShouldChmodPath bool + ExistsCheck string + ShouldChmodPath bool } func (p *Params) InstallDir() string { @@ -45,10 +39,6 @@ func (p *Params) InstallFilename() string { return path.Base(p.AgentRemotePath) } -func (p *Params) PreferDownload() string { - return strconv.FormatBool(p.PreferAgentDownload) -} - func (p *Params) ChmodPath() string { return strconv.FormatBool(p.ShouldChmodPath) } diff --git a/pkg/options/resolve.go b/pkg/options/resolve.go index 1be706578..a3cc9c089 100644 --- a/pkg/options/resolve.go +++ b/pkg/options/resolve.go @@ -4,7 +4,6 @@ import ( "context" "os" "reflect" - "strings" "github.com/loft-sh/devpod/pkg/agent" "github.com/loft-sh/devpod/pkg/binaries" @@ -264,10 +263,6 @@ func ResolveAgentConfig(devConfig *config.Config, provider *provider2.ProviderCo } else if agentConfig.Path == "" { agentConfig.Path = agent.RemoteDevPodHelperLocation } - agentConfig.DownloadURL = resolver.ResolveDefaultValue(agentConfig.DownloadURL, options) - if agentConfig.DownloadURL == "" { - agentConfig.DownloadURL = resolveAgentDownloadURL(devConfig) - } agentConfig.Timeout = resolver.ResolveDefaultValue(agentConfig.Timeout, options) agentConfig.ContainerTimeout = resolver.ResolveDefaultValue(agentConfig.ContainerTimeout, options) agentConfig.InjectGitCredentials = types.StrBool(resolver.ResolveDefaultValue(string(agentConfig.InjectGitCredentials), options)) @@ -278,21 +273,6 @@ func ResolveAgentConfig(devConfig *config.Config, provider *provider2.ProviderCo return agentConfig } -// resolveAgentDownloadURL resolves the agent download URL (env -> context -> default) -func resolveAgentDownloadURL(devConfig *config.Config) string { - devPodAgentURL := os.Getenv(agent.EnvDevPodAgentURL) - if devPodAgentURL != "" { - return strings.TrimSuffix(devPodAgentURL, "/") + "/" - } - - contextAgentOption, ok := devConfig.Current().Options[config.ContextOptionAgentURL] - if ok && contextAgentOption.Value != "" { - return strings.TrimSuffix(contextAgentOption.Value, "/") + "/" - } - - return agent.DefaultAgentDownloadURL() -} - func filterResolvedOptions(resolvedOptions, beforeConfigOptions, providerValues map[string]config.OptionValue, providerOptions map[string]*types.Option, userOptions map[string]string) { for k := range resolvedOptions { // check if user supplied diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 2a2f7f380..48586255a 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -84,9 +84,6 @@ type ProviderAgentConfig struct { // DataPath is the agent path where data is stored DataPath string `json:"dataPath,omitempty"` - // DownloadURL is the base url where to download the agent from - DownloadURL string `json:"downloadURL,omitempty"` - // Timeout is the timeout in minutes to wait until the agent tries // to turn of the server. Timeout string `json:"inactivityTimeout,omitempty"` diff --git a/pkg/tunnel/container.go b/pkg/tunnel/container.go index 01d1ac8a0..2ccedb4a2 100644 --- a/pkg/tunnel/container.go +++ b/pkg/tunnel/container.go @@ -100,8 +100,6 @@ func (c *ContainerTunnel) Run(ctx context.Context, handler Handler, cfg *config. }, c.client.AgentLocal(), c.client.AgentPath(), - c.client.AgentURL(), - true, command, stdinReader, stdoutWriter,