feat: render: true forces headless Chrome; fix 403 challenges and files

README.md

@@ -141,14 +141,20 @@ show how often each path was taken. Step by step:
    looks like Chrome's (about 1 s, no JavaScript). A plain Go client is only
    its backup for the Chrome-like client's own TLS or protocol errors.
 4. For a challenge or JavaScript-only page, it runs your installed Chrome or
-   Chromium headless, for 15 s at most. A hard 403 or 429 skips this step,
-   and so does a machine without Chrome.
+   Chromium headless, for 15 s at most. A challenge counts even when it comes
+   as a 403, as Cloudflare's does (`cf-mitigated: challenge`). A plain 403 or
+   429 skips this step, and so does a machine without Chrome.
 5. If both steps fail, the result says why and suggests a browser tool if
    you have one.
 
-Claude can pass `render: false` to skip headless Chrome for one call, and
-`save: true` to get a file path and a short preview instead of the whole
-page in context.
+`fetch_page` returns web pages and text. A PDF, an image or another file
+gets a clear error instead of its bytes.
+
+Claude can pass `render: true` to read a page with headless Chrome even when
+the first step got it (for content that scripts fill in), `render: false` to
+skip headless Chrome for one call, and `save: true` to get a file path and a
+short preview instead of the whole page in context. If Chrome cannot read a
+page the first step got, the result is the first step's text, with a note.
 
 ### With context-mode and RTK
 

cmd/refetch/get.go

@@ -18,6 +18,7 @@ func runGet(ctx context.Context, cfg config.Config, args []string, stdout, stder
 	impersonate := fs.String("impersonate", cfg.Impersonate, "always, fallback or off")
 	raw := fs.Bool("raw", false, "print the response body instead of its text")
 	allowRender := fs.Bool("render", true, "run the headless Chrome step for a challenge or JavaScript-only page")
+	forceRender := fs.Bool("force-render", false, "run the headless Chrome step for every page")
 	fs.Usage = func() {
 		_, _ = fmt.Fprintln(stderr, "Usage: refetch get [flags] <url>\n\nFetches one page, prints each step to stderr and the text to stdout.")
 		fs.PrintDefaults()
@@ -35,7 +36,14 @@ func runGet(ctx context.Context, cfg config.Config, args []string, stdout, stder
 	}
 	fr := newFetcher(ctx, cfg, "cli", stderr)
 	defer fr.close()
-	p, err := fr.get(ctx, fs.Arg(0), *allowRender)
+	mode := renderAuto
+	switch {
+	case !*allowRender:
+		mode = renderNever
+	case *forceRender:
+		mode = renderAlways
+	}
+	p, err := fr.get(ctx, fs.Arg(0), mode)
 	for _, a := range p.Attempts {
 		_, _ = fmt.Fprintf(stderr, "%-13s %-15s HTTP %-3d %8d B %6.2fs  %s\n", a.Client, a.Outcome, a.Status, a.Bytes, a.Duration.Seconds(), a.Err)
 	}
@@ -45,6 +53,9 @@ func runGet(ctx context.Context, cfg config.Config, args []string, stdout, stder
 		}
 		return err
 	}
+	if note := renderNote(p); note != "" {
+		_, _ = fmt.Fprintln(stderr, note)
+	}
 	_, _ = fmt.Fprintf(stderr, "won: %s\n\n", clientNames[p.Last.Client])
 	out := string(p.Body)
 	if !*raw {

cmd/refetch/get_test.go

@@ -64,6 +64,14 @@ func TestFetchPage(t *testing.T) {
 		t.Errorf("fetch: %q, %v", text, err)
 	}
 
+	text, err = fetchPage(t.Context(), fr, dir, fetchInput{URL: u, Render: new(true)})
+	if err != nil || !strings.Contains(text, "Chrome not found (install Google Chrome, or set REFETCH_CHROME). This is the HTTP client's text.") || !strings.Contains(text, "Release notes") {
+		t.Errorf("render: true without Chrome: %q, %v", text, err)
+	}
+	if text, err = fetchPage(t.Context(), fr, dir, fetchInput{URL: u, Render: new(false)}); err != nil || strings.Contains(text, "headless") {
+		t.Errorf("render: false: %q, %v", text, err)
+	}
+
 	long := strings.Repeat("word ", 1000)
 	fp.page = okPage("<p>" + long + "</p>")
 	text, err = fetchPage(t.Context(), fr, dir, fetchInput{URL: u, MaxChars: 10})

cmd/refetch/main.go

@@ -166,14 +166,24 @@ func newFetchClient(cfg config.Config) *fetch.Fetcher {
 	}
 }
 
+// renderMode is fetch_page's render argument for one call.
+type renderMode int
+
+const (
+	renderAuto   renderMode = iota // headless Chrome for a challenge or a JavaScript-only page
+	renderNever                    // render: false
+	renderAlways                   // render: true, also for a page the HTTP clients read or were refused
+)
+
 // get fetches one page: the HTTP clients first, then the headless Chrome
-// step when they got a challenge or a JavaScript-only page. allowRender
-// false (fetch_page's render: false) skips that step for this call. All
-// attempts are recorded under one fetch id.
-func (fr *fetcher) get(ctx context.Context, url string, allowRender bool) (fetch.Page, error) {
+// step when they got a challenge or a JavaScript-only page, or always with
+// renderAlways. All attempts are recorded under one fetch id.
+func (fr *fetcher) get(ctx context.Context, url string, mode renderMode) (fetch.Page, error) {
 	p, err := fr.f.Get(ctx, url)
-	if err != nil && (p.Last.Outcome == fetch.OutcomeChallenge || p.Last.Outcome == fetch.OutcomeNeedsJS) {
-		p, err = fr.renderStep(ctx, url, allowRender, p, err)
+	switch o := p.Last.Outcome; {
+	case err != nil && (o == fetch.OutcomeChallenge || o == fetch.OutcomeNeedsJS),
+		mode == renderAlways && (err == nil || o == fetch.OutcomeRefused):
+		p, err = fr.renderStep(ctx, url, mode, p, err)
 	}
 	if fr.st != nil {
 		id := uuid.New().String()
@@ -191,14 +201,15 @@ func (fr *fetcher) get(ctx context.Context, url string, allowRender bool) (fetch
 }
 
 // renderStep runs step 2, or records why it did not run. A skipped step
-// leaves p.Last on the HTTP attempt, whose outcome the hand-off explains.
-func (fr *fetcher) renderStep(ctx context.Context, url string, allowRender bool, p fetch.Page, httpErr error) (fetch.Page, error) {
+// leaves p.Last on the HTTP attempt, whose outcome the hand-off explains;
+// so does a failed one after the HTTP clients read the page.
+func (fr *fetcher) renderStep(ctx context.Context, url string, mode renderMode, p fetch.Page, httpErr error) (fetch.Page, error) {
 	a := fetch.Attempt{URL: url, Host: p.Last.Host, Client: render.Client}
 	var dom []byte
 	switch {
 	case fr.renderOff:
 		a.Outcome = render.OutcomeSkippedOff
-	case !allowRender:
+	case mode == renderNever:
 		a.Outcome = render.OutcomeSkippedByCall
 	default:
 		start := time.Now()
@@ -230,6 +241,9 @@ func (fr *fetcher) renderStep(ctx context.Context, url string, allowRender bool,
 	case render.OutcomeSkippedOff, render.OutcomeSkippedByCall, render.OutcomeNoChrome, render.OutcomeSkippedRoot:
 		return p, httpErr
 	}
+	if httpErr == nil {
+		return p, nil
+	}
 	p.Last = a
 	return p, fmt.Errorf("%w; then %s", httpErr, cmp.Or(a.Err, a.Outcome))
 }

cmd/refetch/mcp.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"cmp"
 	"context"
 	"crypto/rand"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -34,7 +36,7 @@ type fetchInput struct {
 	URL      string `json:"url" jsonschema:"the http or https page to read"`
 	MaxChars int    `json:"max_chars,omitempty" jsonschema:"text limit in characters, 1000 to 100000, default 20000"`
 	Save     bool   `json:"save,omitempty" jsonschema:"write the full text to a file and return its path plus the first 2000 characters"`
-	Render   *bool  `json:"render,omitempty" jsonschema:"false skips the headless Chrome step for this call"`
+	Render   *bool  `json:"render,omitempty" jsonschema:"true reads the page with headless Chrome even when a plain fetch got it (for content that scripts fill in); false skips the headless Chrome step; omit to let refetch decide"`
 }
 
 func runMCP(ctx context.Context, cfg config.Config, stderr io.Writer) error {
@@ -69,7 +71,15 @@ func fetchPage(ctx context.Context, fr *fetcher, dataDir string, in fetchInput)
 		maxChars = defaultMaxChars
 	}
 	maxChars = min(max(maxChars, 1000), 100000)
-	p, err := fr.get(ctx, in.URL, in.Render == nil || *in.Render)
+	mode := renderAuto
+	switch {
+	case in.Render == nil:
+	case *in.Render:
+		mode = renderAlways
+	default:
+		mode = renderNever
+	}
+	p, err := fr.get(ctx, in.URL, mode)
 	if err != nil {
 		// The error can quote what the site sent (a redirect target), so it
 		// is one capped line.
@@ -81,6 +91,9 @@ func fetchPage(ctx context.Context, fr *fetcher, dataDir string, in fetchInput)
 		via = fmt.Sprintf("HTTP %d, %s", p.Last.Status, via)
 	}
 	head := fmt.Sprintf("Source: %s (%s)\n", in.URL, via)
+	if note := renderNote(p); note != "" {
+		head += note + "\n"
+	}
 	if !in.Save {
 		return head + fence(fetch.Clip(text, maxChars)), nil
 	}
@@ -202,9 +215,27 @@ func skipNote(p fetch.Page) string {
 	return ""
 }
 
+// renderNote says why a page read with render: true is not headless
+// Chrome's.
+func renderNote(p fetch.Page) string {
+	if p.Last.Client == render.Client || len(p.Attempts) == 0 {
+		return ""
+	}
+	a := p.Attempts[len(p.Attempts)-1]
+	if a.Client != render.Client {
+		return ""
+	}
+	if note := skipNote(p); note != "" {
+		return note + " This is the HTTP client's text."
+	}
+	return "headless Chrome step failed: " + oneLine(cmp.Or(a.Err, a.Outcome), 200) + ". This is the HTTP client's text."
+}
+
 func errorHint(status int, err error) string {
 	msg := err.Error()
 	switch {
+	case errors.Is(err, fetch.ErrNotText):
+		return "fetch_page reads web pages and text, not PDFs, images or other files. Open the file with a tool that reads its type, or use other sources."
 	case status == 404 || status == 410:
 		return fmt.Sprintf("The page does not exist (HTTP %d). Check the URL.", status)
 	case status >= 500:

cmd/refetch/render_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"errors"
+	"fmt"
 	"strings"
 	"testing"
 
@@ -45,7 +46,11 @@ func TestRenderStep(t *testing.T) {
 	}
 	for _, tc := range cases {
 		fr := &fetcher{r: tc.r, renderOff: tc.off}
-		p, err := fr.renderStep(t.Context(), u, !tc.noCall, start, httpErr)
+		mode := renderAuto
+		if tc.noCall {
+			mode = renderNever
+		}
+		p, err := fr.renderStep(t.Context(), u, mode, start, httpErr)
 		last := p.Attempts[len(p.Attempts)-1]
 		if last.Client != render.Client || last.Outcome != tc.want || (err == nil) != tc.ok {
 			t.Errorf("%s: last %+v, err %v", tc.name, last, err)
@@ -66,3 +71,53 @@ func TestRenderStep(t *testing.T) {
 		}
 	}
 }
+
+func TestRenderModes(t *testing.T) {
+	const u = pageURL
+	rendered := "<html><body><h1>Rendered notes</h1><p>" + strings.Repeat("Filled in by scripts. ", 10) + "</p></body></html>"
+	refused := fetch.Page{Last: fetch.Attempt{URL: u, Client: fetch.ClientChromeLike, Outcome: fetch.OutcomeRefused, Status: 403}}
+	refused.Attempts = []fetch.Attempt{refused.Last}
+	cases := []struct {
+		name    string
+		page    fetch.Page
+		httpErr error
+		mode    renderMode
+		r       *fakeRenderer
+		want    string // text in the result, or in the error
+		chrome  bool   // Chrome's DOM won
+		note    string // renderNote
+	}{
+		{"auto keeps a page the HTTP clients read", okPage(article), nil, renderAuto, &fakeRenderer{dom: rendered}, "Release notes", false, ""},
+		{"always renders a page the HTTP clients read", okPage(article), nil, renderAlways, &fakeRenderer{dom: rendered}, "Rendered notes", true, ""},
+		{"always renders a refused page", refused, errors.New("refused (HTTP 403)"), renderAlways, &fakeRenderer{dom: rendered}, "Rendered notes", true, ""},
+		{"auto leaves a refused page", refused, errors.New("refused (HTTP 403)"), renderAuto, &fakeRenderer{dom: rendered}, "refused", false, ""},
+		{"always falls back to the HTTP page", okPage(article), nil, renderAlways, &fakeRenderer{err: render.ErrTimeout}, "Release notes", false, "headless Chrome step failed: headless Chrome timed out"},
+		{"always without Chrome", okPage(article), nil, renderAlways, &fakeRenderer{err: render.ErrNoChrome}, "Release notes", false, "Chrome not found"},
+	}
+	for _, tc := range cases {
+		fr := &fetcher{f: &fakePages{page: tc.page, err: tc.httpErr}, r: tc.r}
+		p, err := fr.get(t.Context(), u, tc.mode)
+		got := string(p.Body)
+		if err != nil {
+			got = err.Error()
+		}
+		if !strings.Contains(got, tc.want) || (p.Last.Client == render.Client) != tc.chrome {
+			t.Errorf("%s: %q, last %+v", tc.name, got, p.Last)
+		}
+		if note := renderNote(p); tc.note == "" && note != "" || !strings.Contains(note, tc.note) {
+			t.Errorf("%s: note %q", tc.name, note)
+		}
+		if tc.mode == renderAuto && tc.r.calls != 0 {
+			t.Errorf("%s: Chrome ran", tc.name)
+		}
+	}
+}
+
+func TestNotTextHint(t *testing.T) {
+	p := fetch.Page{Last: fetch.Attempt{Client: fetch.ClientChromeLike, Outcome: fetch.OutcomeError, Status: 200}}
+	p.Attempts = []fetch.Attempt{p.Last}
+	err := fmt.Errorf("%w (application/pdf)", fetch.ErrNotText)
+	if h := handOff(p, err); !strings.Contains(h, "not PDFs, images or other files") {
+		t.Errorf("hand-off %q", h)
+	}
+}

e2e/run.py

@@ -1,14 +1,15 @@
 #!/usr/bin/env python3
-"""Compatibility run: the same five claude -p sessions in four setups.
+"""Compatibility run: the same seven claude -p sessions in five setups.
 
 Setups: refetch alone, + RTK default, + RTK posthook, + context-mode, and
 no-chrome (alone, with REFETCH_CHROME pointing nowhere). Each session asks
 Claude to read one page of the local test server (e2e/server). Pass: every
-page ends with its secret word in the answer; in no-chrome the two pages that
-need the headless Chrome step (js, challenge-js) fail with the skip note.
+page ends with its secret word in the answer; in no-chrome the three pages
+that need the headless Chrome step (js, challenge-js, challenge-403) fail with
+the skip note.
 
 Usage: python3 e2e/run.py [--model sonnet] [--setups alone,context-mode] [--out DIR]
-Needs claude, go, and rtk for the RTK setups. Costs about 20 sessions.
+Needs claude, go, and rtk for the RTK setups. Costs about 35 sessions.
 """
 import argparse
 import concurrent.futures as cf
@@ -21,7 +22,7 @@
 import tempfile
 
 ROOT = pathlib.Path(__file__).resolve().parent.parent
-CASES = {"ok": "apple", "403": "banana", "challenge": "damson", "js": "elder", "slow": "fig", "challenge-js": "grape"}
+CASES = {"ok": "apple", "403": "banana", "challenge": "damson", "js": "elder", "slow": "fig", "challenge-js": "grape", "challenge-403": "honeydew"}
 CTX = "mcp__plugin_context-mode_context-mode__"
 ALLOWED = ["WebFetch", "mcp__plugin_refetch_refetch__fetch_page",
            CTX + "ctx_fetch_and_index", CTX + "ctx_index", CTX + "ctx_search", CTX + "ctx_execute_file"]
@@ -114,7 +115,7 @@ def main():
                                            capture_output=True, text=True).stdout)
             print(f"\n## {setup}")
             for case, tools, ok, answer in results:
-                expect = not (setup == "no-chrome" and case in ("js", "challenge-js"))
+                expect = not (setup == "no-chrome" and case in ("js", "challenge-js", "challenge-403"))
                 mark = "ok  " if ok == expect else "FAIL"
                 failed |= ok != expect
                 print(f"  {mark} {case:10} answered={ok!s:5} tools={','.join(tools)}")

e2e/server/main.go

@@ -26,7 +26,7 @@ import (
 )
 
 // Words are the secret words the sessions must report, by path.
-var words = map[string]string{"/ok": "apple", "/403": "banana", "/challenge": "damson", "/js": "elder", "/slow": "fig", "/challenge-js": "grape"}
+var words = map[string]string{"/ok": "apple", "/403": "banana", "/challenge": "damson", "/js": "elder", "/slow": "fig", "/challenge-js": "grape", "/challenge-403": "honeydew"}
 
 const challenge = `<!DOCTYPE html><html><head><title>Just a moment...</title></head><body>` +
 	`<p>Verifying you are human. This may take a few seconds.</p><div id="challenge-stage"></div>` +
@@ -82,11 +82,15 @@ func main() {
 			http.NotFound(w, r)
 		case r.URL.Path == "/js":
 			_, _ = w.Write([]byte(jsShell))
-		case r.URL.Path == "/challenge-js":
+		case r.URL.Path == "/challenge-js" || r.URL.Path == "/challenge-403":
 			if c, err := r.Cookie("cleared"); err == nil && c.Value == "1" {
 				_, _ = w.Write([]byte(real(r.URL.Path, word)))
 				return
 			}
+			if r.URL.Path == "/challenge-403" { // as Cloudflare sends its managed challenge
+				w.Header().Set("Cf-Mitigated", "challenge")
+				w.WriteHeader(http.StatusForbidden)
+			}
 			_, _ = w.Write([]byte(jsChallenge))
 		case chrome || r.URL.Path == "/ok":
 			_, _ = w.Write([]byte(real(r.URL.Path, word)))