feat: run in Linux containers

AGENTS.md

@@ -41,6 +41,8 @@ fetcher and the stats report. The repo root is also the Claude Code plugin.
 - Every connection goes through the public-only dialer
   (`fetch.DialContext`); never add a client that bypasses it. Headless
   Chrome reaches the network only through `render`'s proxy on that dialer.
+  An upstream proxy (`use_proxy`, `fetch.UseProxy`) is a CONNECT tunnel
+  inside that dialer; never set `Proxy` on a transport.
 - Hooks and the MCP server must never break a session: report failures as
   tool errors, or exit 0 with empty output. Hooks finish in well under
   50 ms.

README.md

@@ -190,6 +190,7 @@ another file. Everything is on by default.
 | `stats` | `REFETCH_STATS` | `true` | `false` records nothing |
 | `mode` | `REFETCH_MODE` | `"nudge"` | `"inline"`: the hook fetches the page itself and puts its first 8,000 characters next to the failed result |
 | `user_agent` | `REFETCH_USER_AGENT` | `refetch/<version>` | The plain client's User-Agent |
+| `use_proxy` | `REFETCH_USE_PROXY` | `false` | `true` sends connections through `HTTPS_PROXY`, except to hosts `NO_PROXY` lists. Off by default because a proxy that decrypts TLS replaces the Chrome-like handshake with its own |
 
 ```toml
 # ~/.config/refetch/config.toml
@@ -198,9 +199,17 @@ stats = false
 ```
 
 Headless Chrome uses `REFETCH_CHROME` if set, then the standard macOS app
-paths, then `google-chrome` or `chromium` on PATH. refetch never downloads a
-browser. As root (in a container, for example), the step is skipped, because
-Chrome's sandbox does not run as root.
+paths, then `google-chrome` or `chromium` on PATH, then Playwright's Chromium
+(in `PLAYWRIGHT_BROWSERS_PATH`, `/ms-playwright`, `/opt/pw-browsers` or
+`~/.cache/ms-playwright`). refetch never downloads a browser. Chrome's
+sandbox does not run as root, so when refetch runs as root (in a container,
+for example), it runs Chrome as the `nobody` user on Linux and skips the
+step elsewhere. Chrome's sandbox needs user namespaces, which Docker's
+default seccomp profile blocks, so in a container the step needs one that
+allows them (`--security-opt seccomp=unconfined` is the blunt way). refetch
+skips a Playwright Chromium that other users can write to, as in
+Playwright's Docker images; `refetch doctor` names it, and setting
+`REFETCH_CHROME` to it uses it anyway.
 
 To turn refetch off entirely, disable the plugin in `/plugin`.
 

SECURITY.md

@@ -9,7 +9,14 @@ injection can choose it. These guarantees are security boundaries:
   own interface addresses. The check runs on the resolved address of every
   connection and redirect, so DNS rebinding cannot get around it. Headless
   Chrome reaches the network only through a local proxy that applies the
-  same check.
+  same check. Behind an upstream proxy (see below), the check runs before
+  each connection instead.
+- **Chrome never runs as root.** When refetch runs as root on Linux, Chrome
+  (its `--version` check included) runs as the `nobody` user, with its
+  sandbox on; elsewhere the step does not run as root at all. A Playwright
+  Chromium that refetch finds by searching is used only if it and the
+  folders above it belong to root or to you and nobody else can write to
+  them.
 - **Only pinned binaries run.** The plugin shim downloads the binary for
   its version from this repo's GitHub release and checks the archive and the
   binary against the sha256 values pinned in the plugin. It checks the
@@ -43,6 +50,12 @@ injection can choose it. These guarantees are security boundaries:
   127.0.0.1. Another process on your machine could use it to reach public
   addresses for those few seconds; it never reaches anything the address
   check refuses.
+- **An upstream proxy, when you turn it on.** With `use_proxy`,
+  connections go through `HTTPS_PROXY`, and the proxy resolves host names
+  itself. refetch refuses non-public IP addresses and names that resolve on
+  this machine to non-public addresses, but a name that only the proxy can
+  resolve is the proxy's to check. Hosts `NO_PROXY` lists are dialled
+  directly, with the full check.
 - **What the address check cannot see.** An internal network on public IP
   space, or a service that trusts your IP address, looks public.
 

cmd/refetch/doctor.go

@@ -42,7 +42,7 @@ func runDoctor(ctx context.Context, stdout io.Writer) error {
 	} else {
 		checks = append(checks, configCheck())
 	}
-	checks = append(checks, dataCheck(ctx, cfg), chromeCheck(ctx, cfg), env.configDirCheck(), env.permissionCheck(), env.hooksCheck())
+	checks = append(checks, networkCheck(cfg, useProxy(cfg)), dataCheck(ctx, cfg), chromeCheck(ctx, cfg), env.configDirCheck(), env.permissionCheck(), env.hooksCheck())
 	for _, c := range checks {
 		mark := "✓"
 		if !c.ok {
@@ -55,12 +55,24 @@ func runDoctor(ctx context.Context, stdout io.Writer) error {
 	return nil
 }
 
+func networkCheck(cfg config.Config, proxy string) check {
+	switch {
+	case !cfg.UseProxy:
+		return check{true, "network", "direct connections"}
+	case proxy == "":
+		return check{false, "network", "use_proxy is on, but no HTTPS_PROXY is set, so connections go direct"}
+	}
+	return check{true, "network", "through the proxy at " + proxy + " (use_proxy)"}
+}
+
 func versionCheck() check {
 	bin := buildVersion()
 	plugin := os.Getenv("REFETCH_PLUGIN_VERSION")
 	switch {
 	case plugin == "":
 		return check{true, "version", "refetch " + bin + " (not run through the plugin)"}
+	case plugin == "0.0.0":
+		return check{true, "version", "refetch " + bin + ", with the plugin files of a development checkout"}
 	case strings.TrimPrefix(bin, "v") != plugin:
 		return check{false, "version", fmt.Sprintf("binary %s, plugin %s: run /reload-plugins, or check REFETCH_BIN", bin, plugin)}
 	}
@@ -181,6 +193,10 @@ func chromeCheck(ctx context.Context, cfg config.Config) check {
 	if p := render.FindChrome(); p != "" {
 		return check{true, "chrome", fmt.Sprintf("%s at %s; headless Chrome step on (timeout %s)", render.Name(ctx, p), p, cfg.RenderTimeout)}
 	}
+	if p := render.UnsafePlaywrightChrome(); p != "" {
+		return check{false, "chrome", "Playwright's Chromium at " + p + " is skipped: other users can write to it or a folder above it.\n" +
+			"If this machine is yours alone (a container, say), set REFETCH_CHROME=" + p}
+	}
 	return check{false, "chrome", "no Chrome or Chromium found, so the headless Chrome step is skipped.\n" +
 		"Install Google Chrome, or set REFETCH_CHROME to a Chrome binary."}
 }

cmd/refetch/hook.go

@@ -68,6 +68,7 @@ func runHook(ctx context.Context, args []string, stdin io.Reader, stdout, stderr
 		debug(err)
 		return
 	}
+	useProxy(cfg)
 	var in hookInput
 	if err := json.UnmarshalRead(stdin, &in); err != nil {
 		debug(err)

cmd/refetch/main.go

@@ -72,6 +72,7 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
 		_, _ = fmt.Fprintln(stderr, "refetch:", err)
 		return 1
 	}
+	useProxy(cfg)
 	switch cmd {
 	case "mcp":
 		err = runMCP(ctx, cfg, stderr)
@@ -91,6 +92,15 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
 	return 0
 }
 
+// useProxy routes connections through HTTPS_PROXY when use_proxy is on,
+// and returns the proxy's address ("" for none).
+func useProxy(cfg config.Config) string {
+	if !cfg.UseProxy {
+		return ""
+	}
+	return fetch.UseProxy()
+}
+
 // buildVersion is the release version, or the module version for go
 // install builds.
 func buildVersion() string {

cmd/refetch/proxy_test.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/samgozman/refetch/config"
+	"github.com/samgozman/refetch/fetch"
+	"github.com/samgozman/refetch/render"
+)
+
+func TestNetworkCheck(t *testing.T) {
+	t.Cleanup(func() { fetch.UseProxy() }) // after the variables below are restored
+	for _, k := range []string{"https_proxy", "HTTP_PROXY", "http_proxy", "NO_PROXY", "no_proxy"} {
+		t.Setenv(k, "")
+	}
+	t.Setenv("HTTPS_PROXY", "http://user:pw@127.0.0.1:3128")
+	cfg := config.Default()
+	if c := networkCheck(cfg, useProxy(cfg)); !c.ok || c.detail != "direct connections" {
+		t.Errorf("HTTPS_PROXY is ignored without use_proxy: %+v", c)
+	}
+	cfg.UseProxy = true
+	if c := networkCheck(cfg, useProxy(cfg)); !c.ok || c.detail != "through the proxy at http://127.0.0.1:3128 (use_proxy)" {
+		t.Errorf("use_proxy: %+v", c)
+	}
+	t.Setenv("HTTPS_PROXY", "")
+	if c := networkCheck(cfg, useProxy(cfg)); c.ok {
+		t.Errorf("use_proxy without HTTPS_PROXY: %+v", c)
+	}
+}
+
+func TestVersionOfADevelopmentCheckout(t *testing.T) {
+	t.Setenv("REFETCH_PLUGIN_VERSION", "0.0.0")
+	if c := versionCheck(); !c.ok {
+		t.Errorf("REFETCH_BIN with a development checkout: %+v", c)
+	}
+}
+
+func TestChromeCheckNamesAnUnsafePlaywright(t *testing.T) {
+	dir := t.TempDir()
+	bin := filepath.Join(dir, "chromium-1200", "chrome-linux64", "chrome")
+	if err := os.MkdirAll(filepath.Dir(bin), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(bin, []byte("#!/bin/sh\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chmod(filepath.Join(dir, "chromium-1200"), 0o777); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("PLAYWRIGHT_BROWSERS_PATH", dir)
+	t.Setenv("REFETCH_CHROME", "")
+	t.Setenv("PATH", "")
+	t.Setenv("HOME", t.TempDir())
+	if render.FindChrome() != "" {
+		t.Skip("a Chrome is installed in a fixed path here")
+	}
+	c := chromeCheck(t.Context(), config.Default())
+	if c.ok || !strings.Contains(c.detail, "set REFETCH_CHROME="+bin) {
+		t.Errorf("chrome check: %+v", c)
+	}
+}

codecov.yml

@@ -0,0 +1,15 @@
+# CI's cover-check fails the build below 85%; Codecov reports against the
+# same line instead of against the last commit.
+coverage:
+  status:
+    project:
+      default:
+        target: 85%
+    # Some new code only runs as root on Linux (Chrome as nobody), which CI
+    # does not test, so a change is held to 70% of its own lines.
+    patch:
+      default:
+        target: 70%
+comment:
+  layout: "condensed_header, diff, files"
+  require_changes: true

config/config.go

@@ -27,6 +27,7 @@ type Config struct {
 	Stats         bool     `toml:"stats"`          // record events.db rows
 	Mode          string   `toml:"mode"`           // nudge or inline (hook response)
 	UserAgent     string   `toml:"user_agent"`     // the plain client's User-Agent; empty = refetch/<version>
+	UseProxy      bool     `toml:"use_proxy"`      // connect through HTTPS_PROXY; a TLS-decrypting one replaces the Chrome handshake
 }
 
 // Duration is a time.Duration written as "10s" in TOML and env vars.
@@ -123,7 +124,7 @@ func (c *Config) applyEnv(getenv func(string) string) error {
 			}
 		}
 	}
-	bools := map[string]*bool{"REFETCH_RESPECT_ROBOTS": &c.RespectRobots, "REFETCH_STATS": &c.Stats}
+	bools := map[string]*bool{"REFETCH_RESPECT_ROBOTS": &c.RespectRobots, "REFETCH_STATS": &c.Stats, "REFETCH_USE_PROXY": &c.UseProxy}
 	for k, p := range bools {
 		if v := getenv(k); v != "" {
 			b, err := strconv.ParseBool(v)

config/config_test.go

@@ -88,3 +88,15 @@ func TestDurationText(t *testing.T) {
 		t.Error("UnmarshalText accepted \"soon\"")
 	}
 }
+
+func TestUseProxyFromEnv(t *testing.T) {
+	t.Setenv("REFETCH_CONFIG", "/dev/null")
+	t.Setenv("REFETCH_USE_PROXY", "")
+	if c, err := Load(); err != nil || c.UseProxy {
+		t.Errorf("use_proxy on by default: %v", err)
+	}
+	t.Setenv("REFETCH_USE_PROXY", "true")
+	if c, err := Load(); err != nil || !c.UseProxy {
+		t.Errorf("REFETCH_USE_PROXY=true: %v", err)
+	}
+}

fetch/clients.go

@@ -29,15 +29,16 @@ var chromeHeaders = fhttp.Header{
 }
 
 // chromeGet fetches with a Chrome 150 TLS and HTTP/2 fingerprint. It shares
-// the public-only dialer and redirect rules, never uses a proxy or HTTP/3
-// (QUIC would bypass the dialer), and keeps no cookies between fetches.
+// the public-only dialer (and through it any proxy UseProxy set) and the
+// redirect rules, never uses HTTP/3 (QUIC would bypass the dialer), and keeps
+// no cookies between fetches.
 func chromeGet(ctx context.Context, rawURL string, timeout time.Duration) (int, []byte, string, error) {
 	ctx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 	opts := []tlsclient.HttpClientOption{
 		tlsclient.WithClientProfile(profiles.Chrome_150),
 		tlsclient.WithTimeoutSeconds(int(timeout.Seconds())),
-		tlsclient.WithDialer(dialer),
+		tlsclient.WithDialContext(DialContext),
 		tlsclient.WithDisableHttp3(),
 		tlsclient.WithCookieJar(tlsclient.NewCookieJar()),
 		tlsclient.WithCustomRedirectFunc(func(req *fhttp.Request, via []*fhttp.Request) error {
@@ -67,7 +68,7 @@ func chromeGet(ctx context.Context, rawURL string, timeout time.Duration) (int,
 }
 
 var plainClient = &http.Client{
-	Transport: &http.Transport{Proxy: nil, DialContext: dialer.DialContext, TLSClientConfig: &tls.Config{InsecureSkipVerify: e2eLocal}}, // InsecureSkipVerify is true only in the e2e build
+	Transport: &http.Transport{Proxy: nil, DialContext: DialContext, TLSClientConfig: &tls.Config{InsecureSkipVerify: e2eLocal}}, // InsecureSkipVerify is true only in the e2e build
 	CheckRedirect: func(req *http.Request, via []*http.Request) error {
 		return checkRedirect(req.URL, len(via))
 	},

fetch/fetch.go

@@ -350,8 +350,16 @@ func (f *Fetcher) allowed(ctx context.Context, rawURL string) bool {
 var dialer = net.Dialer{Timeout: 10 * time.Second, Control: publicOnly}
 
 // DialContext dials the way both clients do: public addresses only, checked
-// after DNS resolution. The headless Chrome step's proxy dials with it.
+// after DNS resolution, or through the proxy UseProxy set. The headless
+// Chrome step's proxy dials with it.
 func DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	proxy, ok, err := proxyFor(address)
+	if err != nil {
+		return nil, err
+	}
+	if ok {
+		return dialProxied(ctx, proxy, address)
+	}
 	return dialer.DialContext(ctx, network, address)
 }
 
@@ -387,13 +395,7 @@ func publicOnly(_, address string, _ syscall.RawConn) error {
 	if err != nil {
 		return err
 	}
-	if !isPublic(ip) {
-		return fmt.Errorf("%s is not a public address", ip)
-	}
-	if (!e2eLocal || !ip.IsLoopback()) && isLocal(ip) {
-		return fmt.Errorf("%s is not a public address: it is this machine's own", ip)
-	}
-	return nil
+	return checkAddr(ip)
 }
 
 // local caches this machine's interface addresses. A public address on an