Handle transient GitHub API connection resets

src/github_app.py

@@ -4,12 +4,17 @@
 from __future__ import annotations
 
 import contextlib
+import logging
 import time
 from typing import Generator
 
 import jwt
 import requests
 
+from .github_http import github_request
+
+logger = logging.getLogger(__name__)
+
 
 class GithubAppToken:
     def __init__(self, private_key, app_id) -> None:
@@ -19,20 +24,29 @@ def __init__(self, private_key, app_id) -> None:
     # configured by the GitHub App and expire after one hour.
     @contextlib.contextmanager
     def get_token(self, installation_id: int) -> Generator[str, None, None]:
-        req = requests.post(
+        req = github_request(
+            "post",
             url=f"https://api.github.com/app/installations/{installation_id}/access_tokens",
             headers=self.headers,
+            max_attempts=1,
         )
         req.raise_for_status()
         resp = req.json()
         try:
             # This token expires in an hour
             yield resp["token"]
         finally:
-            requests.delete(
-                "https://api.github.com/installation/token",
-                headers={"Authorization": f"token {resp['token']}"},
-            )
+            try:
+                github_request(
+                    "delete",
+                    "https://api.github.com/installation/token",
+                    headers={"Authorization": f"token {resp['token']}"},
+                )
+            except requests.RequestException:
+                logger.warning(
+                    "Failed to revoke GitHub installation token; it will expire automatically.",
+                    exc_info=True,
+                )
 
     def get_jwt_token(self, private_key, app_id):
         payload = {

src/github_http.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import requests
+
+GITHUB_API_TIMEOUT = 10
+GITHUB_API_MAX_ATTEMPTS = 2
+
+logger = logging.getLogger(__name__)
+
+
+def github_request(
+    method: str,
+    url: str,
+    *,
+    max_attempts: int = GITHUB_API_MAX_ATTEMPTS,
+    **kwargs: Any,
+) -> requests.Response:
+    kwargs.setdefault("timeout", GITHUB_API_TIMEOUT)
+
+    for attempt in range(1, max_attempts + 1):
+        try:
+            return requests.request(method, url, **kwargs)
+        except (requests.ConnectionError, requests.Timeout):
+            if attempt == max_attempts:
+                raise
+            logger.warning(
+                "Retrying GitHub API request after a transient network error.",
+                exc_info=True,
+            )
+
+    raise RuntimeError("Unreachable GitHub request retry state.")

src/github_sdk.py

@@ -11,6 +11,8 @@
 from sentry_sdk.envelope import Envelope
 from sentry_sdk.utils import format_timestamp
 
+from .github_http import github_request
+
 
 class GithubSentryError(Exception):
     pass
@@ -42,7 +44,7 @@ def __init__(self, token, dsn, dry_run=False) -> None:
     def _fetch_github(self, url):
         headers = {"Authorization": f"token {self.token}"}
 
-        req = requests.get(url, headers=headers)
+        req = github_request("get", url, headers=headers)
         req.raise_for_status()
         return req
 

src/sentry_config.py

@@ -6,7 +6,7 @@
 from configparser import ConfigParser
 from functools import lru_cache
 
-import requests
+from .github_http import github_request
 
 LOGGING_LEVEL = os.environ.get("LOGGING_LEVEL", logging.INFO)
 logger = logging.getLogger(__name__)
@@ -27,7 +27,7 @@ def fetch_dsn_for_github_org(org: str, token: str) -> str:
         api_url = SENTRY_CONFIG_API_URL.replace("{owner}", org)
 
         # - Get meta about sentry_config.ini file
-        resp = requests.get(api_url, headers=headers)
+        resp = github_request("get", api_url, headers=headers)
         resp.raise_for_status()
         meta = resp.json()
 

tests/test_github_app.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from unittest.mock import patch
+
+import pytest
+import requests
+import responses
+
+from src.github_app import GithubAppToken
+
+CREATE_TOKEN_URL = "https://api.github.com/app/installations/123/access_tokens"
+REVOKE_TOKEN_URL = "https://api.github.com/installation/token"
+
+
+def make_github_app_token():
+    with patch.object(
+        GithubAppToken,
+        "get_authentication_header",
+        return_value={"Authorization": "Bearer jwt"},
+    ):
+        return GithubAppToken(private_key="irrelevant", app_id=1)
+
+
+@responses.activate
+def test_token_revocation_failure_does_not_mask_handler_error():
+    responses.post(CREATE_TOKEN_URL, json={"token": "installation-token"}, status=201)
+    responses.delete(REVOKE_TOKEN_URL, body=requests.ConnectionError("reset"))
+    responses.delete(REVOKE_TOKEN_URL, body=requests.ConnectionError("reset"))
+
+    with pytest.raises(RuntimeError, match="handler failed"):
+        with make_github_app_token().get_token(123) as token:
+            assert token == "installation-token"
+            raise RuntimeError("handler failed")
+
+    assert len(responses.calls) == 3
+    assert responses.calls[1].request.headers["Authorization"] == (
+        "token installation-token"
+    )

tests/test_github_sdk.py

@@ -59,6 +59,19 @@ def test_ensure_raise_error_on_github_api_failure():
     )
 
 
+@responses.activate
+def test_fetch_github_retries_transient_connection_errors():
+    url = "https://api.github.com/repos/example/project/actions/runs/1"
+    responses.get(url, body=requests.ConnectionError("reset"))
+    responses.get(url, json={"ok": True})
+
+    client = GithubClient(dsn=DSN, token=TOKEN)
+    resp = client._fetch_github(url)
+
+    assert resp.json() == {"ok": True}
+    assert len(responses.calls) == 2
+
+
 @freeze_time()
 @responses.activate
 @patch("src.github_sdk.get_uuid")