diff --git a/contest/remote/lib/fetcher.py b/contest/remote/lib/fetcher.py index d41ab209..e8cf4727 100644 --- a/contest/remote/lib/fetcher.py +++ b/contest/remote/lib/fetcher.py @@ -4,9 +4,10 @@ import json import os import re -import requests import subprocess import time +import urllib3 +from urllib3.util.retry import Retry class Fetcher: @@ -26,6 +27,9 @@ def __init__(self, cb, cbarg, name, branches_url, results_path, url_path, tree_p self._tree_path = tree_path self._patches_path = patches_path + self._http = urllib3.PoolManager(timeout=30, + retries=Retry(connect=10, backoff_factor=1)) + # Set last date to something old self._last_date = datetime.datetime.now(datetime.UTC) - datetime.timedelta(weeks=1) if first_run == "force": @@ -33,8 +37,8 @@ def __init__(self, cb, cbarg, name, branches_url, results_path, url_path, tree_p pass elif first_run == "continue": try: - r = requests.get(self._branches_url, timeout=30) - branches = json.loads(r.content.decode('utf-8')) + r = self._http.request('GET', self._branches_url) + branches = json.loads(r.data.decode('utf-8')) branch_date = {} for b in branches: branch_date[b["branch"]] = datetime.datetime.fromisoformat(b["date"]) @@ -49,7 +53,7 @@ def __init__(self, cb, cbarg, name, branches_url, results_path, url_path, tree_p self._last_date = max(branch_date[result["branch"]], self._last_date) print("INFO: Last run date:", self._last_date) - except FileNotFoundError: + except (FileNotFoundError, urllib3.exceptions.HTTPError, ValueError): pass elif first_run == "next": # unless there's a crazy race or time error this will skip newest branch @@ -134,13 +138,12 @@ def _find_branch(self, name): def _run_once(self): try: - r = requests.get(self._branches_url, timeout=30) - except requests.exceptions.RequestException as e: + r = self._http.request('GET', self._branches_url) + branches = json.loads(r.data.decode('utf-8')) + except (urllib3.exceptions.HTTPError, ValueError) as e: print(f'WARN: Failed to fetch branches: {e}') return - branches = json.loads(r.content.decode('utf-8')) - to_test = None newest = self._last_date diff --git a/contest/remote/lib/vm.py b/contest/remote/lib/vm.py index e2c1d3df..d754f32b 100644 --- a/contest/remote/lib/vm.py +++ b/contest/remote/lib/vm.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 import unicodedata -import requests import subprocess import fcntl import json @@ -11,6 +10,8 @@ import shutil import signal import time +import urllib3 +from urllib3.util.retry import Retry from .crash import has_crash, extract_crash @@ -68,6 +69,9 @@ def __init__(self, config, vm_name=""): self.cfg_boot_to = int(config.get('vm', 'boot_timeout')) + self._http = urllib3.PoolManager(timeout=30, + retries=Retry(connect=10, backoff_factor=1)) + self.filter_data = None self.has_kmemleak = None self.has_gcov = self.config.getboolean('vm', 'gcov', fallback=False) @@ -384,8 +388,12 @@ def _load_filters(self): url = self.config.get("remote", "filters", fallback=None) if not url: return None - r = requests.get(url) - self.filter_data = json.loads(r.content.decode('utf-8')) + try: + r = self._http.request('GET', url) + self.filter_data = json.loads(r.data.decode('utf-8')) + except (urllib3.exceptions.HTTPError, ValueError) as e: + print(f"WARN{self.print_pfx} failed to fetch filters: {e}") + return None return self.filter_data def extract_crash(self, out_path):