Retry transient GitHub config fetch failures

src/sentry_config.py

@@ -4,7 +4,6 @@
 import logging
 import os
 from configparser import ConfigParser
-from functools import lru_cache
 
 import requests
 
@@ -15,6 +14,35 @@
 SENTRY_CONFIG_API_URL = (
     "https://api.github.com/repos/{owner}/.sentry/contents/sentry_config.ini"
 )
+GITHUB_API_TIMEOUT_SECONDS = 10
+GITHUB_API_RETRIES = 2
+GITHUB_API_RETRY_STATUS_CODES = {500, 502, 503, 504}
+
+
+def _fetch_github_config(api_url: str, headers: dict[str, str]) -> requests.Response:
+    for attempt in range(GITHUB_API_RETRIES + 1):
+        try:
+            resp = requests.get(
+                api_url,
+                headers=headers,
+                timeout=GITHUB_API_TIMEOUT_SECONDS,
+            )
+        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
+            if attempt == GITHUB_API_RETRIES:
+                raise
+            logger.warning("Retrying transient GitHub config fetch failure")
+            continue
+
+        if resp.status_code not in GITHUB_API_RETRY_STATUS_CODES:
+            return resp
+        if attempt == GITHUB_API_RETRIES:
+            return resp
+        logger.warning(
+            "Retrying GitHub config fetch after HTTP %s",
+            resp.status_code,
+        )
+
+    raise RuntimeError("unreachable")
 
 
 def fetch_dsn_for_github_org(org: str, token: str) -> str:
@@ -27,7 +55,7 @@
         api_url = SENTRY_CONFIG_API_URL.replace("{owner}", org)
 
         # - Get meta about sentry_config.ini file
-        resp = requests.get(api_url, headers=headers)
+        resp = _fetch_github_config(api_url, headers)
         resp.raise_for_status()
         meta = resp.json()
 

tests/test_sentry_config_file.py

@@ -3,6 +3,7 @@
 from unittest import TestCase
 
 import responses
+from requests.exceptions import ConnectionError
 
 from src.sentry_config import fetch_dsn_for_github_org
 from src.sentry_config import SENTRY_CONFIG_API_URL as api_url
@@ -47,6 +48,23 @@ def setUp(self) -> None:
     def test_fetch_parse_sentry_config_file(self) -> None:
         assert fetch_dsn_for_github_org(org, token) == expected_dsn
 
+    @responses.activate
+    def test_fetch_retries_transient_connection_error(self) -> None:
+        responses.replace(
+            responses.GET,
+            self.api_url,
+            body=ConnectionError("Connection reset by peer"),
+        )
+        responses.add(
+            method="GET",
+            url=self.api_url,
+            json=sentry_config_file_meta,
+            status=200,
+        )
+
+        assert fetch_dsn_for_github_org(org, token) == expected_dsn
+        assert len(responses.calls) == 2
+
     def test_fetch_private_repo(self) -> None:
         pass