diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..fe25b54 Binary files /dev/null and b/.DS_Store differ diff --git a/builders/workflow_builder.py b/builders/workflow_builder.py index 940e952..9b3e36b 100644 --- a/builders/workflow_builder.py +++ b/builders/workflow_builder.py @@ -35,6 +35,11 @@ def navigate(self, url: str): self.current_step = main_step return self + def monitor_changes(self, enabled: bool = True): + """Compare each successful run with the previous successful run.""" + self.meta["monitor"] = enabled + return self + def click(self, selector: str): return self._add_action("click", [selector]) @@ -116,4 +121,4 @@ def get_workflow(self): } def get_meta(self): - return self.meta \ No newline at end of file + return self.meta diff --git a/client.py b/client.py index d014383..2a2e079 100644 --- a/client.py +++ b/client.py @@ -68,7 +68,8 @@ async def get_robot(self, robot_id: str): async def create_robot(self, workflow_file: dict): # Normalize both `type` and `robotType` in meta, mirroring the Node SDK behaviour - meta = workflow_file.get("meta") or {} + meta = dict(workflow_file.get("meta") or {}) + monitor = meta.pop("monitor", None) robot_type_value = meta.get("robotType") or meta.get("type") payload = { **workflow_file, @@ -76,6 +77,7 @@ async def create_robot(self, workflow_file: dict): **meta, "type": robot_type_value, "robotType": robot_type_value, + **({"compareRuns": monitor} if monitor is not None else {}), }, } data = await self._handle( @@ -86,8 +88,16 @@ async def create_robot(self, workflow_file: dict): return data async def update_robot(self, robot_id: str, updates: dict): + payload = dict(updates) + if updates.get("meta") is not None: + meta = dict(updates["meta"]) + monitor = meta.pop("monitor", None) + payload["meta"] = { + **meta, + **({"compareRuns": monitor} if monitor is not None else {}), + } data = await self._handle( - self.client.put(f"/robots/{robot_id}", json=updates) + self.client.put(f"/robots/{robot_id}", json=payload) ) if not data: raise MaxunError(f"Failed to update robot {robot_id}") @@ -138,6 +148,23 @@ async def get_run(self, robot_id: str, run_id: str): raise MaxunError(f"Run {run_id} not found", 404) return data + async def get_run_diff( + self, + robot_id: str, + run_id: str, + format: Optional[str] = None, + ): + """Return the detailed monitoring diff for a completed run.""" + data = await self._handle( + self.client.get( + f"/robots/{robot_id}/runs/{run_id}/diff", + params={"format": format} if format else None, + ) + ) + if not data: + raise MaxunError(f"Monitoring diff for run {run_id} was not found", 404) + return data + async def abort_run(self, robot_id: str, run_id: str): await self._handle( self.client.post(f"/robots/{robot_id}/runs/{run_id}/abort") @@ -182,8 +209,12 @@ async def add_webhook(self, robot_id: str, webhook: dict): return data async def extract_with_llm(self, options: dict): + payload = dict(options) + monitor = payload.pop("monitor", None) + if monitor is not None: + payload["compareRuns"] = monitor return await self._handle( - self.client.post("/extract/llm", json=options, timeout=300) + self.client.post("/extract/llm", json=payload, timeout=300) ) async def create_document_extract_robot( diff --git a/extract.py b/extract.py index fb44376..d3b8703 100644 --- a/extract.py +++ b/extract.py @@ -56,6 +56,7 @@ async def extract( llm_api_key: Optional[str] = None, llm_base_url: Optional[str] = None, robot_name: Optional[str] = None, + monitor: Optional[bool] = None, ) -> Robot: """Create an AI-powered extraction robot from a natural language prompt. @@ -77,6 +78,7 @@ async def extract( "url": url, **{key: value for key, value in llm_options.items() if value is not None}, "robotName": robot_name, + **({"monitor": monitor} if monitor is not None else {}), } robot_data = await self.client.extract_with_llm(options) robot = await self.client.get_robot(robot_data["robotId"]) diff --git a/monitoring_test.py b/monitoring_test.py new file mode 100644 index 0000000..b658c3c --- /dev/null +++ b/monitoring_test.py @@ -0,0 +1,96 @@ +import asyncio +import os + +from dotenv import load_dotenv +from maxun import Config, Scrape + +load_dotenv() + + +async def main(): + api_key = os.environ.get("MAXUN_API_KEY") + base_url = os.environ.get( + "MAXUN_BASE_URL", + "http://localhost:8080/api/sdk", + ) + + if not api_key: + raise RuntimeError("MAXUN_API_KEY is required") + + scrape = Scrape( + Config( + api_key=api_key, + base_url=base_url, + ) + ) + + try: + robot = await scrape.create( + name="Python SDK Monitoring Test", + url="https://www.worldometers.info/world-population/", + formats=["text", "markdown", "html"], + monitor=True, + ) + + print(f"Robot created: {robot.id}") + + print("\nRunning baseline capture...") + baseline = await robot.run() + + print( + { + "runId": baseline.get("runId"), + "hasChanges": baseline.get("hasChanges"), + "changedFormats": baseline.get("changedFormats"), + } + ) + + print("\nRunning comparison capture...") + current = await robot.run() + + print( + { + "runId": current.get("runId"), + "hasChanges": current.get("hasChanges"), + "changedFormats": current.get("changedFormats"), + } + ) + + comparison = await robot.get_run_diff(current["runId"]) + + print( + "\nComparison:", + { + "previousRunId": comparison.get("previousRunId"), + "currentRunId": comparison.get("runId"), + "hasChanges": comparison.get("hasChanges"), + "changedFormats": comparison.get("changedFormats"), + }, + ) + + for format_diff in comparison.get("diffs", []): + print(f"\n--- {format_diff['format']} ---") + + printed = 0 + + for change in format_diff.get("changes", []): + if not change.get("added") and not change.get("removed"): + continue + + prefix = "+" if change.get("added") else "-" + value = change.get("value", "") + + # Keep terminal output manageable, especially for HTML. + print(f"{prefix} {value[:1000]}") + printed += 1 + + if printed >= 20: + print("... remaining changes omitted") + break + + finally: + await scrape.client.client.aclose() + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/robot.py b/robot.py index 4cb908f..d7211ee 100644 --- a/robot.py +++ b/robot.py @@ -28,6 +28,14 @@ async def get_runs(self) -> list: async def get_run(self, run_id: str) -> dict: return await self.client.get_run(self.id, run_id) + async def get_run_diff( + self, + run_id: str, + format: Optional[str] = None, + ) -> dict: + """Return the detailed monitoring diff for a completed run.""" + return await self.client.get_run_diff(self.id, run_id, format) + async def get_latest_run(self) -> Optional[dict]: runs = await self.get_runs() if not runs: diff --git a/scrape.py b/scrape.py index 99b1de1..121ae2c 100644 --- a/scrape.py +++ b/scrape.py @@ -22,6 +22,7 @@ async def create( llm_model: Optional[str] = None, llm_api_key: Optional[str] = None, llm_base_url: Optional[str] = None, + monitor: Optional[bool] = None, ) -> Robot: """ Create a scrape robot. @@ -46,6 +47,8 @@ async def create( } if smart_queries: meta["smartQueries"] = smart_queries.strip() + if monitor is not None: + meta["monitor"] = monitor meta.update(build_llm_payload(llm_provider, llm_model, llm_api_key, llm_base_url)) @@ -58,4 +61,4 @@ async def create( return Robot(self.client, robot_data) def _random_string(self, length: int = 9) -> str: - return "".join(random.choices(string.ascii_lowercase + string.digits, k=length)) \ No newline at end of file + return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))