Retry transient GitHub config fetch failures

src/sentry_config.py

@@ -3,6 +3,7 @@
 import base64
 import logging
 import os
+import time
 from configparser import ConfigParser
 from functools import lru_cache
 
@@ -15,6 +16,30 @@
 SENTRY_CONFIG_API_URL = (
     "https://api.github.com/repos/{owner}/.sentry/contents/sentry_config.ini"
 )
+GITHUB_API_MAX_ATTEMPTS = 3
+GITHUB_API_RETRY_BACKOFF_SECONDS = 0.5
+GITHUB_API_TIMEOUT_SECONDS = 10
+
+
+def _fetch_github_config_metadata(api_url: str, headers: dict[str, str]) -> requests.Response:
+    for attempt in range(1, GITHUB_API_MAX_ATTEMPTS + 1):
+        try:
+            return requests.get(
+                api_url,
+                headers=headers,
+                timeout=GITHUB_API_TIMEOUT_SECONDS,
+            )
+        except (requests.ConnectionError, requests.Timeout):
+            if attempt == GITHUB_API_MAX_ATTEMPTS:
+                raise
+
+            logger.warning(
+                "Transient failure fetching Sentry config from GitHub; retrying",
+                exc_info=True,
+            )
+            time.sleep(GITHUB_API_RETRY_BACKOFF_SECONDS * attempt)
+
+    raise RuntimeError("unreachable")
 
 
 def fetch_dsn_for_github_org(org: str, token: str) -> str:
@@ -27,7 +52,7 @@
         api_url = SENTRY_CONFIG_API_URL.replace("{owner}", org)
 
         # - Get meta about sentry_config.ini file
-        resp = requests.get(api_url, headers=headers)
+        resp = _fetch_github_config_metadata(api_url, headers)
         resp.raise_for_status()
         meta = resp.json()
 
@@ -45,4 +70,4 @@
 
     except Exception as e:
         logger.exception(e)
-        raise e
+        raise

tests/test_sentry_config_file.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 
 from unittest import TestCase
+from unittest.mock import Mock
+from unittest.mock import patch
 
+import requests
 import responses
 
 from src.sentry_config import fetch_dsn_for_github_org
@@ -47,6 +50,34 @@ def setUp(self) -> None:
     def test_fetch_parse_sentry_config_file(self) -> None:
         assert fetch_dsn_for_github_org(org, token) == expected_dsn
 
+    @patch("src.sentry_config.time.sleep")
+    @patch("src.sentry_config.requests.get")
+    def test_fetch_retries_transient_github_failures(
+        self,
+        mock_get: Mock,
+        mock_sleep: Mock,
+    ) -> None:
+        successful_response = Mock()
+        successful_response.json.return_value = sentry_config_file_meta
+        mock_get.side_effect = [
+            requests.ConnectionError("connection reset by peer"),
+            successful_response,
+        ]
+
+        assert fetch_dsn_for_github_org(org, token) == expected_dsn
+
+        expected_headers = {
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"token {token}",
+        }
+        assert mock_get.call_count == 2
+        mock_get.assert_called_with(
+            self.api_url,
+            headers=expected_headers,
+            timeout=10,
+        )
+        mock_sleep.assert_called_once_with(0.5)
+
     def test_fetch_private_repo(self) -> None:
         pass