Retry transient GitHub API fetch failures

src/github_sdk.py

@@ -4,13 +4,23 @@
 import hashlib
 import io
 import logging
+import time
 import uuid
 from datetime import datetime
 
 import requests
 from sentry_sdk.envelope import Envelope
 from sentry_sdk.utils import format_timestamp
 
+GITHUB_REQUEST_RETRIES = 3
+GITHUB_REQUEST_TIMEOUT = 10
+GITHUB_RETRY_BACKOFF_SECONDS = 0.25
+GITHUB_TRANSIENT_ERRORS = (
+    requests.exceptions.ConnectionError,
+    requests.exceptions.SSLError,
+    requests.exceptions.Timeout,
+)
+
 
 class GithubSentryError(Exception):
     pass
@@ -42,9 +52,19 @@
     def _fetch_github(self, url):
         headers = {"Authorization": f"token {self.token}"}
 
-        req = requests.get(url, headers=headers)
-        req.raise_for_status()
-        return req
+        for attempt in range(GITHUB_REQUEST_RETRIES):
+            try:
+                req = requests.get(
+                    url,
+                    headers=headers,
+                    timeout=GITHUB_REQUEST_TIMEOUT,
+                )
+                req.raise_for_status()
+                return req
+            except GITHUB_TRANSIENT_ERRORS:
+                if attempt == GITHUB_REQUEST_RETRIES - 1:
+                    raise
+                time.sleep(GITHUB_RETRY_BACKOFF_SECONDS * 2**attempt)
 
     def _get_extra_metadata(self, job):
         # XXX: This is the slowest call

tests/test_github_sdk.py

@@ -2,7 +2,7 @@
 
 import sys
 from datetime import datetime
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 import pytest
 import requests
@@ -11,7 +11,7 @@
 from requests import HTTPError
 from sentry_sdk.utils import format_timestamp
 
-from src.github_sdk import GithubClient
+from src.github_sdk import GithubClient, GITHUB_REQUEST_TIMEOUT
 
 DSN = "https://foo@random.ingest.sentry.io/bar"
 TOKEN = "irrelevant"
@@ -59,6 +59,44 @@ def test_ensure_raise_error_on_github_api_failure():
     )
 
 
+@patch("src.github_sdk.time.sleep")
+@patch("src.github_sdk.requests.get")
+def test_retries_transient_github_api_failure(mock_get, mock_sleep):
+    url = "https://api.github.com/repos/getsentry/sentry/actions/runs/123"
+    response = Mock()
+    response.raise_for_status.return_value = None
+    mock_get.side_effect = [
+        requests.exceptions.SSLError("transient TLS failure"),
+        response,
+    ]
+
+    client = GithubClient(dsn=DSN, token=TOKEN)
+
+    assert client._fetch_github(url) is response
+    assert mock_get.call_count == 2
+    mock_get.assert_called_with(
+        url,
+        headers={"Authorization": f"token {TOKEN}"},
+        timeout=GITHUB_REQUEST_TIMEOUT,
+    )
+    mock_sleep.assert_called_once_with(0.25)
+
+
+@patch("src.github_sdk.time.sleep")
+@patch("src.github_sdk.requests.get")
+def test_exhausts_transient_github_api_retries(mock_get, mock_sleep):
+    url = "https://api.github.com/repos/getsentry/sentry/actions/runs/123"
+    mock_get.side_effect = requests.exceptions.SSLError("transient TLS failure")
+
+    client = GithubClient(dsn=DSN, token=TOKEN)
+
+    with pytest.raises(requests.exceptions.SSLError):
+        client._fetch_github(url)
+
+    assert mock_get.call_count == 3
+    assert mock_sleep.call_count == 2
+
+
 @freeze_time()
 @responses.activate
 @patch("src.github_sdk.get_uuid")