contest: remote: retry on connect errors

contest/remote/lib/fetcher.py

@@ -4,9 +4,10 @@
 import json
 import os
 import re
-import requests
 import subprocess
 import time
+import urllib3
+from urllib3.util.retry import Retry
 
 
 class Fetcher:
@@ -26,15 +27,18 @@ def __init__(self, cb, cbarg, name, branches_url, results_path, url_path, tree_p
         self._tree_path = tree_path
         self._patches_path = patches_path
 
+        self._http = urllib3.PoolManager(timeout=30,
+                                         retries=Retry(connect=10, backoff_factor=1))
+
         # Set last date to something old
         self._last_date = datetime.datetime.now(datetime.UTC) - datetime.timedelta(weeks=1)
         if first_run == "force":
             # leave _last_date very old, this will force run on newest branch
             pass
         elif first_run == "continue":
             try:
-                r = requests.get(self._branches_url, timeout=30)
-                branches = json.loads(r.content.decode('utf-8'))
+                r = self._http.request('GET', self._branches_url)
+                branches = json.loads(r.data.decode('utf-8'))
                 branch_date = {}
                 for b in branches:
                     branch_date[b["branch"]] = datetime.datetime.fromisoformat(b["date"])
@@ -49,7 +53,7 @@ def __init__(self, cb, cbarg, name, branches_url, results_path, url_path, tree_p
 
                     self._last_date = max(branch_date[result["branch"]], self._last_date)
                 print("INFO: Last run date:", self._last_date)
-            except FileNotFoundError:
+            except (FileNotFoundError, urllib3.exceptions.HTTPError, ValueError):
                 pass
         elif first_run == "next":
             # unless there's a crazy race or time error this will skip newest branch
@@ -134,13 +138,12 @@ def _find_branch(self, name):
 
     def _run_once(self):
         try:
-            r = requests.get(self._branches_url, timeout=30)
-        except requests.exceptions.RequestException as e:
+            r = self._http.request('GET', self._branches_url)
+            branches = json.loads(r.data.decode('utf-8'))
+        except (urllib3.exceptions.HTTPError, ValueError) as e:
             print(f'WARN: Failed to fetch branches: {e}')
             return
 
-        branches = json.loads(r.content.decode('utf-8'))
-
         to_test = None
         newest = self._last_date
 

contest/remote/lib/vm.py

@@ -1,7 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 
 import unicodedata
-import requests
 import subprocess
 import fcntl
 import json
@@ -11,6 +10,8 @@
 import shutil
 import signal
 import time
+import urllib3
+from urllib3.util.retry import Retry
 from .crash import has_crash, extract_crash
 
 
@@ -68,6 +69,9 @@ def __init__(self, config, vm_name=""):
 
         self.cfg_boot_to = int(config.get('vm', 'boot_timeout'))
 
+        self._http = urllib3.PoolManager(timeout=30,
+                                         retries=Retry(connect=10, backoff_factor=1))
+
         self.filter_data = None
         self.has_kmemleak = None
         self.has_gcov = self.config.getboolean('vm', 'gcov', fallback=False)
@@ -384,8 +388,12 @@ def _load_filters(self):
             url = self.config.get("remote", "filters", fallback=None)
             if not url:
                 return None
-            r = requests.get(url)
-            self.filter_data = json.loads(r.content.decode('utf-8'))
+            try:
+                r = self._http.request('GET', url)
+                self.filter_data = json.loads(r.data.decode('utf-8'))
+            except (urllib3.exceptions.HTTPError, ValueError) as e:
+                print(f"WARN{self.print_pfx} failed to fetch filters: {e}")
+                return None
         return self.filter_data
 
     def extract_crash(self, out_path):