feat: fetch_page returns Markdown, links and tables kept

AGENTS.md

@@ -23,7 +23,9 @@ fetcher and the stats report. The repo root is also the Claude Code plugin.
 ## Layout
 
 - `fetch/`: public package; the clients, the public-only dialer, challenge
-  and JavaScript-only detection, HTML to text
+  and JavaScript-only detection, HTML to Markdown (`Markdown`, what
+  `fetch_page` returns) and to plain text (`Text`, which the detection
+  measures)
 - `render/`: the headless Chrome step: Chrome discovery, the local proxy on
   the public-only dialer, `--dump-dom` under `render_timeout`
 - `store/`: SQLite event log (`tool_calls`, `attempts`, `daily` roll-up)

README.md

@@ -119,7 +119,7 @@ flowchart TD
     subgraph FetchPage ["fetch_page, refetch's MCP tool"]
         Client["1. Chrome-like client<br/>Chrome TLS fingerprint, about 1 s"]
         Client -- "challenge or<br/>JavaScript-only page" --> Headless["2. Headless Chrome<br/>your installed Chrome, 15 s at most"]
-        Client -- "page" --> Text["Page text"]
+        Client -- "page" --> Text["Page as Markdown"]
         Headless -- "page" --> Text
         Client -- "HTTP 403/429<br/>or error" --> Handoff["Hand-off: why it failed"]
         Headless -- "still blocked" --> Handoff
@@ -147,13 +147,19 @@ show how often each path was taken. Step by step:
 5. If both steps fail, the result says why and suggests a browser tool if
    you have one.
 
-`fetch_page` returns web pages and text. A PDF, an image or another file
-gets a clear error instead of its bytes.
+`fetch_page` returns a page as Markdown: headings, lists, links, tables and
+code blocks kept, scripts and hidden elements dropped. Links are absolute, and
+search engines' click-through links point at their targets, so Claude can
+follow a search result or a "next page" link. Only http, https and mailto
+links keep their target, and tracking parameters (`utm_*`, `fbclid`, `gclid`)
+are dropped. Pass `format: "text"` for plain text instead. A PDF, an image or
+another file gets a clear error instead of its bytes.
 
 Claude can pass `render: true` to read a page with headless Chrome even when
 the first step got it (for content that scripts fill in), `render: false` to
 skip headless Chrome for one call, and `save: true` to get a file path and a
-short preview instead of the whole page in context. If Chrome cannot read a
+short preview instead of the whole page in context (the file is `.md`, or
+`.txt` with `format: "text"`). If Chrome cannot read a
 page the first step got, the result is the first step's text, with a note.
 
 ### With context-mode and RTK

SECURITY.md

@@ -25,7 +25,10 @@ injection can choose it. These guarantees are security boundaries:
 - **Hooks never block.** refetch's hooks never deny, rewrite or block a tool
   call, and always exit 0.
 - **Page text is marked untrusted.** `fetch_page` puts page text between two
-  marker lines with a random nonce. A failure reason read from another
+  marker lines with a random nonce, so a page cannot fake where it ends, in
+  Markdown code blocks included. Links in the page keep their target only for
+  http, https and mailto; `javascript:` and `data:` links are shown as their
+  text. A failure reason read from another
   tool's result reaches Claude as a fixed category (an HTTP status, a
   timeout), and other error text as one short line.
 - **Local data stays local.** `events.db` (mode 0600) stores URLs without

cmd/refetch/get.go

@@ -17,6 +17,7 @@ func runGet(ctx context.Context, cfg config.Config, args []string, stdout, stder
 	maxChars := fs.Int("max-chars", defaultMaxChars, "text limit in characters (0 = no limit)")
 	impersonate := fs.String("impersonate", cfg.Impersonate, "always, fallback or off")
 	raw := fs.Bool("raw", false, "print the response body instead of its text")
+	format := fs.String("format", formatMarkdown, "markdown or text")
 	allowRender := fs.Bool("render", true, "run the headless Chrome step for a challenge or JavaScript-only page")
 	forceRender := fs.Bool("force-render", false, "run the headless Chrome step for every page")
 	fs.Usage = func() {
@@ -30,6 +31,9 @@ func runGet(ctx context.Context, cfg config.Config, args []string, stdout, stder
 		fs.Usage()
 		return errors.New("get takes exactly one URL")
 	}
+	if *format != formatMarkdown && *format != formatText {
+		return fmt.Errorf("-format is %s or %s", formatMarkdown, formatText)
+	}
 	cfg.Impersonate = *impersonate
 	if err := cfg.Validate(); err != nil {
 		return err
@@ -59,7 +63,7 @@ func runGet(ctx context.Context, cfg config.Config, args []string, stdout, stder
 	_, _ = fmt.Fprintf(stderr, "won: %s\n\n", clientNames[p.Last.Client])
 	out := string(p.Body)
 	if !*raw {
-		out = fetch.Text(p.Body, p.ContentType)
+		out = pageText(p, fs.Arg(0), *format)
 	}
 	if *maxChars > 0 {
 		out = fetch.Clip(out, *maxChars)

cmd/refetch/get_test.go

@@ -72,6 +72,27 @@ func TestFetchPage(t *testing.T) {
 		t.Errorf("render: false: %q, %v", text, err)
 	}
 
+	fp.page = okPage(`<h2>Notes</h2><p>See <a href="/v2">the v2 notes</a>.</p>`)
+	fp.page.URL = "https://news.example.com/blog/notes"
+	text, err = fetchPage(t.Context(), fr, dir, fetchInput{URL: u})
+	if err != nil || !strings.Contains(text, "## Notes\n\nSee [the v2 notes](https://news.example.com/v2).") {
+		t.Errorf("markdown: %q, %v", text, err)
+	}
+	text, err = fetchPage(t.Context(), fr, dir, fetchInput{URL: u, Format: "text"})
+	if err != nil || !strings.Contains(text, "Notes \nSee the v2 notes .") {
+		t.Errorf("text: %q, %v", text, err)
+	}
+	if _, err := fetchPage(t.Context(), fr, dir, fetchInput{URL: u, Format: "html"}); err == nil || !strings.Contains(err.Error(), `format is "markdown" or "text"`) {
+		t.Errorf("an unknown format: %v", err)
+	}
+	for format, ext := range map[string]string{"": ".md", "text": ".txt"} {
+		text, err = fetchPage(t.Context(), fr, dir, fetchInput{URL: u, Save: true, Format: format})
+		if err != nil || !strings.Contains(text, ext+"\nFirst 2000 characters") {
+			t.Errorf("save as %q: %q, %v", format, text, err)
+		}
+	}
+	fp.page = okPage(article)
+
 	long := strings.Repeat("word ", 1000)
 	fp.page = okPage("<p>" + long + "</p>")
 	text, err = fetchPage(t.Context(), fr, dir, fetchInput{URL: u, MaxChars: 10})
@@ -121,6 +142,18 @@ func TestRunGet(t *testing.T) {
 	if code, _, errOut := get("--impersonate", "sometimes", u); code != 1 || !strings.Contains(errOut, "impersonate") {
 		t.Errorf("bad --impersonate: %d %q", code, errOut)
 	}
+	if code, out, _ := get(u); code != 0 || !strings.Contains(out, "# Release notes") {
+		t.Errorf("get prints Markdown: %d %q", code, out)
+	}
+	if code, out, _ := get("--format", "text", u); code != 0 || strings.Contains(out, "# Release notes") || !strings.Contains(out, "Release notes") {
+		t.Errorf("get --format text: %d %q", code, out)
+	}
+	if code, _, errOut := get("--format", "html", u); code != 1 || !strings.Contains(errOut, "-format is markdown or text") {
+		t.Errorf("bad --format: %d %q", code, errOut)
+	}
+	if code, _, errOut := get("--force-render", u); code != 0 || !strings.Contains(errOut, "This is the HTTP client's text.") {
+		t.Errorf("get --force-render without Chrome: %d %q", code, errOut)
+	}
 
 	// A JavaScript-only page with the headless step off: the skip note.
 	fp.page = fetch.Page{Last: fetch.Attempt{URL: u, Client: fetch.ClientChromeLike, Outcome: fetch.OutcomeNeedsJS, Status: 200}}

cmd/refetch/hook_test.go

@@ -245,11 +245,11 @@ func TestFenceAndPrune(t *testing.T) {
 		t.Errorf("fence: %q / %q", a, b)
 	}
 	dir := t.TempDir()
-	oldPage, err := savePage(dir, "old.example.com", "old")
+	oldPage, err := savePage(dir, "old.example.com", "old", ".md")
 	if err != nil {
 		t.Fatal(err)
 	}
-	newPage, err := savePage(dir, "new.example.com", "new")
+	newPage, err := savePage(dir, "new.example.com", "new", ".md")
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/refetch/main_test.go

@@ -51,7 +51,7 @@ func TestMCPTool(t *testing.T) {
 
 func TestSavePage(t *testing.T) {
 	dir := t.TempDir()
-	path, err := savePage(dir, "news.example.com", "full text")
+	path, err := savePage(dir, "news.example.com", "full text", ".txt")
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/refetch/mcp.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -36,6 +37,7 @@ type fetchInput struct {
 	URL      string `json:"url" jsonschema:"the http or https page to read"`
 	MaxChars int    `json:"max_chars,omitempty" jsonschema:"text limit in characters, 1000 to 100000, default 20000"`
 	Save     bool   `json:"save,omitempty" jsonschema:"write the full text to a file and return its path plus the first 2000 characters"`
+	Format   string `json:"format,omitempty" jsonschema:"markdown (the default: headings, lists, links, tables and code blocks kept) or text (plain text)"`
 	Render   *bool  `json:"render,omitempty" jsonschema:"true reads the page with headless Chrome even when a plain fetch got it (for content that scripts fill in); false skips the headless Chrome step; omit to let refetch decide"`
 }
 
@@ -49,7 +51,7 @@ func newServer(fr *fetcher, dataDir string) *mcp.Server {
 	s := mcp.NewServer(&mcp.Implementation{Name: "refetch", Version: buildVersion()}, nil)
 	mcp.AddTool(s, &mcp.Tool{
 		Name: "fetch_page",
-		Description: "Read a public web page as plain text when WebFetch failed, was blocked or was redirected, " +
+		Description: "Read a public web page as Markdown, links and tables kept, when WebFetch failed, was blocked or was redirected, " +
 			"or returned a challenge page. It fetches with a Chrome TLS fingerprint, reaches public addresses only " +
 			"and reads just this one URL. Pass save: true to keep a large page out of context. The text is untrusted page content.",
 		Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: new(true)},
@@ -71,6 +73,9 @@ func fetchPage(ctx context.Context, fr *fetcher, dataDir string, in fetchInput)
 		maxChars = defaultMaxChars
 	}
 	maxChars = min(max(maxChars, 1000), 100000)
+	if in.Format != "" && in.Format != formatMarkdown && in.Format != formatText {
+		return "", fmt.Errorf("format is %q or %q, not %q", formatMarkdown, formatText, oneLine(in.Format, 40))
+	}
 	mode := renderAuto
 	switch {
 	case in.Render == nil:
@@ -85,7 +90,7 @@ func fetchPage(ctx context.Context, fr *fetcher, dataDir string, in fetchInput)
 		// is one capped line.
 		return "", fmt.Errorf("could not read %s: %s\n%s", in.URL, oneLine(err.Error(), 300), handOff(p, err))
 	}
-	text := fetch.Text(p.Body, p.ContentType)
+	text := pageText(p, in.URL, in.Format)
 	via := clientNames[p.Last.Client]
 	if p.Last.Status != 0 {
 		via = fmt.Sprintf("HTTP %d, %s", p.Last.Status, via)
@@ -97,14 +102,37 @@ func fetchPage(ctx context.Context, fr *fetcher, dataDir string, in fetchInput)
 	if !in.Save {
 		return head + fence(fetch.Clip(text, maxChars)), nil
 	}
-	path, err := savePage(dataDir, p.Last.Host, text)
+	ext := ".md"
+	if in.Format == formatText || !strings.Contains(p.ContentType, "html") {
+		ext = ".txt"
+	}
+	path, err := savePage(dataDir, p.Last.Host, text, ext)
 	if err != nil {
 		return "", fmt.Errorf("read %s but could not save it: %w", in.URL, err)
 	}
 	return fmt.Sprintf("%sSaved the full text (%d characters) to %s\nFirst %d characters:\n%s",
 		head, len([]rune(text)), path, savePreview, fence(fetch.Clip(text, savePreview))), nil
 }
 
+// Output formats of fetch_page and refetch get.
+const (
+	formatMarkdown = "markdown"
+	formatText     = "text"
+)
+
+// pageText is the page in the format asked for ("" is Markdown), its links
+// resolved against the address it came from.
+func pageText(p fetch.Page, rawURL, format string) string {
+	if format == formatText {
+		return fetch.Text(p.Body, p.ContentType)
+	}
+	base, err := url.Parse(cmp.Or(p.URL, rawURL))
+	if err != nil {
+		base = nil
+	}
+	return fetch.Markdown(p.Body, p.ContentType, base)
+}
+
 // fence wraps page text between two marker lines with a random nonce, so
 // the page cannot fake where it ends.
 func fence(text string) string {
@@ -136,21 +164,23 @@ const savedPagesAge = 30 * 24 * time.Hour
 // prunePages removes saved pages older than age.
 func prunePages(dataDir string, age time.Duration) {
 	files, _ := filepath.Glob(filepath.Join(dataDir, "pages", "*.txt"))
-	for _, f := range files {
+	md, _ := filepath.Glob(filepath.Join(dataDir, "pages", "*.md"))
+	for _, f := range append(files, md...) {
 		if fi, err := os.Stat(f); err == nil && time.Since(fi.ModTime()) > age {
 			_ = os.Remove(f)
 		}
 	}
 }
 
-// savePage writes text under <data>/pages and returns the file path.
-func savePage(dataDir, host, text string) (string, error) {
+// savePage writes text under <data>/pages, in a file ending in ext, and
+// returns the file path.
+func savePage(dataDir, host, text, ext string) (string, error) {
 	dir := filepath.Join(dataDir, "pages")
 	if err := os.MkdirAll(dir, 0o700); err != nil {
 		return "", err
 	}
 	name := strings.NewReplacer(":", "_", "/", "_").Replace(host)
-	f, err := os.CreateTemp(dir, fmt.Sprintf("%s-%s-*.txt", name, time.Now().Format("20060102-150405")))
+	f, err := os.CreateTemp(dir, fmt.Sprintf("%s-%s-*%s", name, time.Now().Format("20060102-150405"), ext))
 	if err != nil {
 		return "", err
 	}

e2e/run.py

@@ -1,15 +1,16 @@
 #!/usr/bin/env python3
-"""Compatibility run: the same seven claude -p sessions in five setups.
+"""Compatibility run: the same eight claude -p sessions in five setups.
 
 Setups: refetch alone, + RTK default, + RTK posthook, + context-mode, and
 no-chrome (alone, with REFETCH_CHROME pointing nowhere). Each session asks
 Claude to read one page of the local test server (e2e/server). Pass: every
-page ends with its secret word in the answer; in no-chrome the three pages
+page ends with its secret word in the answer (links: the word on the page its
+link leads to, which needs the link's URL in fetch_page's result); in no-chrome the three pages
 that need the headless Chrome step (js, challenge-js, challenge-403) fail with
 the skip note.
 
 Usage: python3 e2e/run.py [--model sonnet] [--setups alone,context-mode] [--out DIR]
-Needs claude, go, and rtk for the RTK setups. Costs about 35 sessions.
+Needs claude, go, and rtk for the RTK setups. Costs about 40 sessions.
 """
 import argparse
 import concurrent.futures as cf
@@ -22,7 +23,9 @@
 import tempfile
 
 ROOT = pathlib.Path(__file__).resolve().parent.parent
-CASES = {"ok": "apple", "403": "banana", "challenge": "damson", "js": "elder", "slow": "fig", "challenge-js": "grape", "challenge-403": "honeydew"}
+CASES = {"ok": "apple", "403": "banana", "challenge": "damson", "js": "elder", "slow": "fig", "challenge-js": "grape", "challenge-403": "honeydew", "links": "kiwi"}
+# Cases whose prompt is not the default one.
+PROMPTS = {"links": "Read {base}/links, open its link to the release notes and tell me the secret word on that page."}
 CTX = "mcp__plugin_context-mode_context-mode__"
 ALLOWED = ["WebFetch", "mcp__plugin_refetch_refetch__fetch_page",
            CTX + "ctx_fetch_and_index", CTX + "ctx_index", CTX + "ctx_search", CTX + "ctx_execute_file"]
@@ -61,7 +64,7 @@ def settings(setup: str, out: pathlib.Path) -> dict:
 def session(setup, case, base, env, cfg, out, model):
     work = out / setup / "work"
     work.mkdir(parents=True, exist_ok=True)
-    prompt = f"Read {base}/{case} and tell me the secret word on that page."
+    prompt = PROMPTS.get(case, "Read {base}/{case} and tell me the secret word on that page.").format(base=base, case=case)
     cmd = ["claude", "-p", prompt, "--model", model, "--setting-sources", "project",
            "--plugin-dir", str(ROOT), "--settings", json.dumps(cfg),
            "--allowedTools", *ALLOWED, "--output-format", "stream-json", "--verbose"]

e2e/server/main.go

@@ -26,7 +26,7 @@ import (
 )
 
 // Words are the secret words the sessions must report, by path.
-var words = map[string]string{"/ok": "apple", "/403": "banana", "/challenge": "damson", "/js": "elder", "/slow": "fig", "/challenge-js": "grape", "/challenge-403": "honeydew"}
+var words = map[string]string{"/ok": "apple", "/403": "banana", "/challenge": "damson", "/js": "elder", "/slow": "fig", "/challenge-js": "grape", "/challenge-403": "honeydew", "/links": "", "/notes/r7k2q": "kiwi"}
 
 const challenge = `<!DOCTYPE html><html><head><title>Just a moment...</title></head><body>` +
 	`<p>Verifying you are human. This may take a few seconds.</p><div id="challenge-stage"></div>` +
@@ -45,6 +45,13 @@ const jsChallenge = `<!DOCTYPE html><html><head><title>Just a moment...</title><
 	`<p>Verifying you are human. This may take a few seconds.</p>` +
 	`<script>document.cookie="cleared=1; path=/"; setTimeout(function(){location.reload()},300)</script></body></html>`
 
+// links is an index that only names its pages, so a session can reach the
+// secret word only by following a link.
+const links = `<!doctype html><html><head><title>Orchard notes</title></head><body><main><h1>Orchard notes</h1>` +
+	`<p>Our notes for each season are on their own pages.</p><ul>` +
+	`<li><a href="/notes/b1x9z">Spring planting</a></li><li><a href="notes/r7k2q">Release notes for the harvest</a></li>` +
+	`<li><a href="/notes/m4p0c">Winter storage</a></li></ul></main></body></html>`
+
 // real is an ordinary article page of a realistic size (a few KB).
 func real(path, word string) string {
 	para := `<p>These field notes describe a small orchard: the trees planted in each row, when each ` +
@@ -80,6 +87,8 @@ func main() {
 		switch {
 		case !ok:
 			http.NotFound(w, r)
+		case r.URL.Path == "/links" && chrome:
+			_, _ = w.Write([]byte(links))
 		case r.URL.Path == "/js":
 			_, _ = w.Write([]byte(jsShell))
 		case r.URL.Path == "/challenge-js" || r.URL.Path == "/challenge-403":
@@ -94,7 +103,7 @@ func main() {
 			_, _ = w.Write([]byte(jsChallenge))
 		case chrome || r.URL.Path == "/ok":
 			_, _ = w.Write([]byte(real(r.URL.Path, word)))
-		case r.URL.Path == "/403":
+		case r.URL.Path == "/403", r.URL.Path == "/links", strings.HasPrefix(r.URL.Path, "/notes/"):
 			w.WriteHeader(http.StatusForbidden)
 			_, _ = w.Write([]byte("<html><head><title>403 Forbidden</title></head><body>Access denied</body></html>"))
 		case r.URL.Path == "/challenge":

fetch/clients.go

@@ -29,11 +29,11 @@ var chromeHeaders = fhttp.Header{
 }
 
 // reply is one HTTP response, cut at MaxBytes; mitigated is Cloudflare's
-// cf-mitigated header.
+// cf-mitigated header, url the address after redirects.
 type reply struct {
-	status                 int
-	body                   []byte
-	contentType, mitigated string
+	status                      int
+	body                        []byte
+	contentType, mitigated, url string
 }
 
 // chromeGet fetches with a Chrome 150 TLS and HTTP/2 fingerprint. It shares
@@ -72,7 +72,7 @@ func chromeGet(ctx context.Context, rawURL string, timeout time.Duration) (reply
 	}
 	defer func() { _ = resp.Body.Close() }()
 	body, err := io.ReadAll(io.LimitReader(resp.Body, MaxBytes))
-	return reply{resp.StatusCode, body, resp.Header.Get("Content-Type"), resp.Header.Get("Cf-Mitigated")}, err
+	return reply{resp.StatusCode, body, resp.Header.Get("Content-Type"), resp.Header.Get("Cf-Mitigated"), resp.Request.URL.String()}, err
 }
 
 var plainClient = &http.Client{
@@ -96,7 +96,7 @@ func plainGet(ctx context.Context, rawURL, userAgent string, timeout time.Durati
 	}
 	defer func() { _ = resp.Body.Close() }()
 	body, err := io.ReadAll(io.LimitReader(resp.Body, MaxBytes))
-	return reply{resp.StatusCode, body, resp.Header.Get("Content-Type"), resp.Header.Get("Cf-Mitigated")}, err
+	return reply{resp.StatusCode, body, resp.Header.Get("Content-Type"), resp.Header.Get("Cf-Mitigated"), resp.Request.URL.String()}, err
 }
 
 func checkRedirect(u *url.URL, hops int) error {

fetch/fetch.go

@@ -68,6 +68,7 @@ type Attempt struct {
 type Page struct {
 	Body        []byte
 	ContentType string
+	URL         string    // where Body came from, after redirects; "" when unknown
 	Last        Attempt   // the attempt that produced Body, or the last failure
 	Attempts    []Attempt // every attempt of this call, in order
 }
@@ -165,7 +166,7 @@ func (f *Fetcher) try(ctx context.Context, p *Page, client, rawURL string) error
 	}
 	f.record(p, &a)
 	if err == nil {
-		p.Body, p.ContentType = body, ct
+		p.Body, p.ContentType, p.URL = body, ct, r.url
 	}
 	return err
 }