diff --git a/src/fetch/README.md b/src/fetch/README.md
index 65457d4501..6b91fe8f7d 100644
--- a/src/fetch/README.md
+++ b/src/fetch/README.md
@@ -33,7 +33,7 @@ The fetch tool will truncate the response, but by using the `start_index` argume
## Installation
-Optionally: Install node.js, this will cause the fetch server to use a different HTML simplifier that is more robust.
+Optionally: Install node.js, this will cause the fetch server to use a different HTML simplifier that is more robust. If `node` is not available, the server falls back to the Python-only simplifier.
### Using uv (recommended)
@@ -193,6 +193,10 @@ environment variables (honoring `NO_PROXY`). Both `http(s)://` and SOCKS5 proxie
installed with the server). A `socks://` proxy URL, as many desktop proxy settings export it, is treated as
`socks5://`. A proxy setting the server cannot use makes each fetch fail with a tool error that says so.
+### Customization - HTML simplifier
+
+By default, the server uses the Node.js readability simplifier when `node` is available on `PATH`, and otherwise falls back to readabilipy's Python-only simplifier. If a host has Node.js installed but the readability path is slow or misconfigured, add `--no-readability-js` to force the Python-only simplifier.
+
## Windows Configuration
If you're experiencing timeout issues on Windows, you may need to set the `PYTHONIOENCODING` environment variable to ensure proper character encoding:
diff --git a/src/fetch/src/mcp_server_fetch/__init__.py b/src/fetch/src/mcp_server_fetch/__init__.py
index aa0b11a3b7..febeb19f3e 100644
--- a/src/fetch/src/mcp_server_fetch/__init__.py
+++ b/src/fetch/src/mcp_server_fetch/__init__.py
@@ -21,6 +21,11 @@ def main():
action="store_true",
help="Allow fetching private, loopback and link-local addresses (refused by default)",
)
+ parser.add_argument(
+ "--no-readability-js",
+ action="store_true",
+ help="Use readabilipy's Python-only HTML simplifier instead of the optional Node.js readability path",
+ )
args = parser.parse_args()
asyncio.run(
@@ -29,6 +34,7 @@ def main():
args.ignore_robots_txt,
args.proxy_url,
args.allow_private_ips,
+ use_readability_js=False if args.no_readability_js else None,
)
)
diff --git a/src/fetch/src/mcp_server_fetch/server.py b/src/fetch/src/mcp_server_fetch/server.py
index 94914f6b65..6490472528 100644
--- a/src/fetch/src/mcp_server_fetch/server.py
+++ b/src/fetch/src/mcp_server_fetch/server.py
@@ -1,6 +1,7 @@
import asyncio
import ipaddress
import os
+import shutil
import socket
from typing import Annotated, Tuple
from importlib.metadata import version
@@ -143,7 +144,7 @@ async def _refuse_private_destination(request: httpx.Request) -> None:
)
-def extract_content_from_html(html: str) -> str:
+def extract_content_from_html(html: str, use_readability_js: bool | None = None) -> str:
"""Extract and convert HTML content to Markdown format.
Args:
@@ -152,8 +153,11 @@ def extract_content_from_html(html: str) -> str:
Returns:
Simplified markdown version of the content
"""
+ if use_readability_js is None:
+ use_readability_js = shutil.which("node") is not None
+
ret = readabilipy.simple_json.simple_json_from_html_string(
- html, use_readability=True
+ html, use_readability=use_readability_js
)
failed = "Page failed to be simplified from HTML"
if not ret["content"]:
@@ -302,6 +306,7 @@ async def fetch_url(
force_raw: bool = False,
proxy_url: str | None = None,
allow_private_ips: bool = False,
+ use_readability_js: bool | None = None,
) -> Tuple[str, str]:
"""
Fetch the URL and return the content in a form ready for the LLM, as well as a prefix string with status information.
@@ -338,7 +343,9 @@ async def fetch_url(
)
if is_page_html and not force_raw:
- return extract_content_from_html(page_raw), ""
+ return extract_content_from_html(
+ page_raw, use_readability_js=use_readability_js
+ ), ""
return (
page_raw,
@@ -384,6 +391,7 @@ async def serve(
ignore_robots_txt: bool = False,
proxy_url: str | None = None,
allow_private_ips: bool = False,
+ use_readability_js: bool | None = None,
) -> None:
"""Run the fetch MCP server.
@@ -458,6 +466,7 @@ async def call_tool(name, arguments: dict) -> list[TextContent]:
force_raw=args.raw,
proxy_url=proxy_url,
allow_private_ips=allow_private_ips,
+ use_readability_js=use_readability_js,
)
original_length = len(content)
if args.start_index >= original_length:
@@ -476,6 +485,7 @@ async def call_tool(name, arguments: dict) -> list[TextContent]:
if actual_content_length == args.max_length and remaining_content > 0:
next_start = args.start_index + actual_content_length
content += f"\n\nContent truncated. Call the fetch tool with a start_index of {next_start} to get more content."
+
return [TextContent(type="text", text=f"{prefix}Contents of {url}:\n{content}")]
@server.get_prompt()
@@ -495,6 +505,7 @@ async def get_prompt(name: str, arguments: dict | None) -> GetPromptResult:
user_agent_manual,
proxy_url=proxy_url,
allow_private_ips=allow_private_ips,
+ use_readability_js=use_readability_js,
)
# TODO: after SDK bug is addressed, don't catch the exception
except McpError as e:
diff --git a/src/fetch/tests/test_entrypoints.py b/src/fetch/tests/test_entrypoints.py
index 12de04a781..fcad7964c4 100644
--- a/src/fetch/tests/test_entrypoints.py
+++ b/src/fetch/tests/test_entrypoints.py
@@ -31,7 +31,7 @@ def run_main(monkeypatch: pytest.MonkeyPatch, *argv: str) -> AsyncMock:
def test_main_defaults(monkeypatch: pytest.MonkeyPatch) -> None:
serve = run_main(monkeypatch)
- serve.assert_awaited_once_with(None, False, None, False)
+ serve.assert_awaited_once_with(None, False, None, False, use_readability_js=None)
def test_main_passes_every_flag(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -43,9 +43,14 @@ def test_main_passes_every_flag(monkeypatch: pytest.MonkeyPatch) -> None:
"--proxy-url",
"http://proxy.example.com:8080",
"--allow-private-ips",
+ "--no-readability-js",
)
serve.assert_awaited_once_with(
- "MyBot/1.0", True, "http://proxy.example.com:8080", True
+ "MyBot/1.0",
+ True,
+ "http://proxy.example.com:8080",
+ True,
+ use_readability_js=False,
)
@@ -62,6 +67,7 @@ def test_main_help_exits_zero(
"--ignore-robots-txt",
"--proxy-url",
"--allow-private-ips",
+ "--no-readability-js",
):
assert flag in out
@@ -90,7 +96,7 @@ def test_package_init_run_as_main_calls_main(monkeypatch: pytest.MonkeyPatch) ->
monkeypatch.setattr(mcp_server_fetch.server, "serve", serve)
monkeypatch.setattr(sys, "argv", ["mcp-server-fetch"])
runpy.run_module("mcp_server_fetch.__init__", run_name="__main__")
- serve.assert_awaited_once_with(None, False, None, False)
+ serve.assert_awaited_once_with(None, False, None, False, use_readability_js=None)
def console_script() -> Path:
diff --git a/src/fetch/tests/test_protocol.py b/src/fetch/tests/test_protocol.py
index 01269c7002..e123c60f72 100644
--- a/src/fetch/tests/test_protocol.py
+++ b/src/fetch/tests/test_protocol.py
@@ -21,6 +21,8 @@
import httpx
import pytest
+import readabilipy.simple_json
+from unittest.mock import patch
from mcp import ClientSession
from mcp.shared.exceptions import McpError
from mcp.types import INVALID_PARAMS, CallToolResult
@@ -467,6 +469,24 @@ async def test_html_without_node_falls_back_to_pure_python(web: FakeWeb) -> None
assert "https://example.com/x" not in text
+async def test_fetch_forwards_python_only_readability_choice(web: FakeWeb) -> None:
+ """serve(use_readability_js=False) reaches the extractor explicitly."""
+ web.add(ROBOTS, plain("", status=404))
+ web.add(PAGE, html(ARTICLE_HTML))
+ with patch(
+ "readabilipy.simple_json.simple_json_from_html_string",
+ wraps=readabilipy.simple_json.simple_json_from_html_string,
+ ) as spy:
+ async with connect(ignore_robots_txt=True, use_readability_js=False) as (
+ session,
+ _,
+ ):
+ result = await call(session, {"url": PAGE})
+ assert wire(result)["isError"] is False
+ spy.assert_called_once()
+ assert spy.call_args.kwargs["use_readability"] is False
+
+
@pytest.mark.usefixtures("python_readability")
async def test_empty_html_without_node_cannot_be_simplified(web: FakeWeb) -> None:
# The pure-Python extractor returns "" for an empty page, which
diff --git a/src/fetch/tests/test_server.py b/src/fetch/tests/test_server.py
index 4d00250e44..bc950cfdf4 100644
--- a/src/fetch/tests/test_server.py
+++ b/src/fetch/tests/test_server.py
@@ -8,6 +8,7 @@
import httpx
import pytest
+from unittest.mock import patch
from mcp_server_fetch.server import (
PROXY_ENV_VARS,
@@ -93,6 +94,35 @@ def test_empty_content_returns_error(self):
result = extract_content_from_html(html)
assert "" in result
+ def test_can_disable_readability_js(self):
+ """Callers can force the Python-only extraction path."""
+ html = "Hello World
"
+
+ with patch(
+ "readabilipy.simple_json.simple_json_from_html_string",
+ return_value={"content": "Hello World
"},
+ ) as mock_extract:
+ result = extract_content_from_html(html, use_readability_js=False)
+
+ assert "Hello World" in result
+ mock_extract.assert_called_once_with(html, use_readability=False)
+
+ def test_auto_disables_readability_js_without_node(self):
+ """Python-only environments do not try the Node.js path."""
+ html = "Hello World
"
+
+ with (
+ patch("shutil.which", return_value=None),
+ patch(
+ "readabilipy.simple_json.simple_json_from_html_string",
+ return_value={"content": "Hello World
"},
+ ) as mock_extract,
+ ):
+ result = extract_content_from_html(html)
+
+ assert "Hello World" in result
+ mock_extract.assert_called_once_with(html, use_readability=False)
+
class TestNormalizeProxyUrl:
"""Tests for normalize_proxy_url and normalize_proxy_env (#767)."""