Retry transient GitHub metadata fetch failures

src/github_sdk.py

@@ -4,13 +4,23 @@
 import hashlib
 import io
 import logging
+import time
 import uuid
 from datetime import datetime
 
 import requests
 from sentry_sdk.envelope import Envelope
 from sentry_sdk.utils import format_timestamp
 
+GITHUB_API_MAX_ATTEMPTS = 3
+GITHUB_API_RETRY_DELAY_SECONDS = 0.5
+GITHUB_API_TIMEOUT_SECONDS = 10
+GITHUB_API_RETRYABLE_EXCEPTIONS = (
+    requests.exceptions.ConnectionError,
+    requests.exceptions.SSLError,
+    requests.exceptions.Timeout,
+)
+
 
 class GithubSentryError(Exception):
     pass
@@ -42,9 +52,19 @@
     def _fetch_github(self, url):
         headers = {"Authorization": f"token {self.token}"}
 
-        req = requests.get(url, headers=headers)
-        req.raise_for_status()
-        return req
+        for attempt in range(GITHUB_API_MAX_ATTEMPTS):
+            try:
+                req = requests.get(
+                    url,
+                    headers=headers,
+                    timeout=GITHUB_API_TIMEOUT_SECONDS,
+                )
+                req.raise_for_status()
+                return req
+            except GITHUB_API_RETRYABLE_EXCEPTIONS:
+                if attempt == GITHUB_API_MAX_ATTEMPTS - 1:
+                    raise
+                time.sleep(GITHUB_API_RETRY_DELAY_SECONDS * (attempt + 1))
 
     def _get_extra_metadata(self, job):
         # XXX: This is the slowest call

tests/test_github_sdk.py

@@ -2,6 +2,7 @@
 
 import sys
 from datetime import datetime
+from unittest.mock import Mock
 from unittest.mock import patch
 
 import pytest
@@ -59,6 +60,50 @@ def test_ensure_raise_error_on_github_api_failure():
     )
 
 
+def test_fetch_github_retries_transient_ssl_errors(monkeypatch):
+    url = "https://api.github.com/repos/example/repo/actions/runs/1"
+    response = Mock()
+    response.raise_for_status.return_value = None
+    request = Mock(
+        side_effect=[
+            requests.exceptions.SSLError("transient tls failure"),
+            response,
+        ],
+    )
+    sleep = Mock()
+    monkeypatch.setattr("src.github_sdk.requests.get", request)
+    monkeypatch.setattr("src.github_sdk.time.sleep", sleep)
+
+    client = GithubClient(dsn=DSN, token=TOKEN)
+
+    assert client._fetch_github(url) is response
+    assert request.call_count == 2
+    request.assert_called_with(
+        url,
+        headers={"Authorization": f"token {TOKEN}"},
+        timeout=10,
+    )
+    sleep.assert_called_once_with(0.5)
+
+
+def test_fetch_github_reraises_transient_errors_after_retries(monkeypatch):
+    url = "https://api.github.com/repos/example/repo/actions/runs/1"
+    request = Mock(side_effect=requests.exceptions.SSLError("transient tls failure"))
+    sleep = Mock()
+    monkeypatch.setattr("src.github_sdk.requests.get", request)
+    monkeypatch.setattr("src.github_sdk.time.sleep", sleep)
+
+    client = GithubClient(dsn=DSN, token=TOKEN)
+
+    with pytest.raises(requests.exceptions.SSLError):
+        client._fetch_github(url)
+
+    assert request.call_count == 3
+    assert sleep.call_count == 2
+    sleep.assert_any_call(0.5)
+    sleep.assert_any_call(1.0)
+
+
 @freeze_time()
 @responses.activate
 @patch("src.github_sdk.get_uuid")