Handle transient Sentry ingest envelope failures in webhook flow

src/github_sdk.py

@@ -16,6 +16,10 @@ class GithubSentryError(Exception):
     pass
 
 
+class SentryEnvelopeSendError(Exception):
+    """Raised when posting a trace envelope to Sentry ingest fails."""
+
+
 def get_uuid():
     return uuid.uuid4().hex
 
@@ -145,7 +149,12 @@ def send_trace(self, job):
             return
         trace = self._generate_trace(job)
         if trace:
-            return self._send_envelope(trace)
+            try:
+                return self._send_envelope(trace)
+            except requests.RequestException as err:
+                raise SentryEnvelopeSendError(
+                    "Failed to send trace envelope to Sentry ingest"
+                ) from err
 
 
 def _base_transaction(job):

src/web_app_handler.py

@@ -8,6 +8,7 @@
 
 from .github_app import GithubAppToken
 from .github_sdk import GithubClient
+from .github_sdk import SentryEnvelopeSendError
 from src.sentry_config import fetch_dsn_for_github_org
 
 LOGGING_LEVEL = os.environ.get("LOGGING_LEVEL", logging.INFO)
@@ -49,16 +50,30 @@ def handle_event(self, data, headers):
                         dsn=dsn,
                         dry_run=self.dry_run,
                     )
-                    client.send_trace(data["workflow_job"])
+                    try:
+                        client.send_trace(data["workflow_job"])
+                    except SentryEnvelopeSendError:
+                        logger.exception(
+                            "Unable to send trace envelope to Sentry for org '%s'. "
+                            "Ignoring transient ingest failure and keeping webhook successful.",
+                            org,
+                        )
             else:
                 # Once the Sentry org has a .sentry repo we can remove the DSN from the deployment
-                dsn = fetch_dsn_for_github_org(org, token)
+                dsn = fetch_dsn_for_github_org(org, self.config.gh.token)
                 client = GithubClient(
                     token=self.config.gh.token,
                     dsn=dsn,
                     dry_run=self.dry_run,
                 )
-                client.send_trace(data["workflow_job"])
+                try:
+                    client.send_trace(data["workflow_job"])
+                except SentryEnvelopeSendError:
+                    logger.exception(
+                        "Unable to send trace envelope to Sentry for org '%s'. "
+                        "Ignoring transient ingest failure and keeping webhook successful.",
+                        org,
+                    )
 
         return reason, http_code
 

tests/test_github_sdk.py

@@ -12,6 +12,7 @@
 from sentry_sdk.utils import format_timestamp
 
 from src.github_sdk import GithubClient
+from src.github_sdk import SentryEnvelopeSendError
 
 DSN = "https://foo@random.ingest.sentry.io/bar"
 TOKEN = "irrelevant"
@@ -158,3 +159,29 @@ def test_send_trace(
     #     resp.request.body
     #     == b"\x1f\x8b\x08\x00\xf1\x16}b\x02\xff\xb5TM\x8f\xd30\x10\xbd\xef\xaf\x88|\x02\xa9m\x1c\xc7\x89\x93H\x08\xd0\x8a;\x12\x9c@\xa8\x9a\xd8\xe3&\xbb\xf9\"vX\xaa\xaa\xff\x1d{\xdb\xee\x97\x96n\xcb\x8aS\xd3\x99\xf1\xf8\xbd7o\xbc\xd9^l\x88]\x0fH\nbG\xe8\x0cH[\xf7\x1d\x99\x11\xd9w\x16;\xbb\xdc'a\x18\x9aZ\x82O\x86W\xe6\xb6\xa2\xc1ne+RD\x19\xa7\xbe\r\xfe\xf2\xf5\xb5r\xd5\t \x139H\x95g\x8c\xcbRC\x96P\xceh\x14K]\xea\xac\x14\xee\xf4\xb3\x97>\xfcW\x10=\xdebP\x81EcM\xf0\x86\xbe=\xe0\xfam\r)6\xbe\\\xa2\xff0\x03t\xbb\x9b\x81\xd3He\xb1\x14()\xcf\x13\xbdk*q\x97\xcd\xa2\x92\x96\x08)\xa0\xd2<\x8b\x04\x08\xaeb\xa6\x04Ms\x9a)\xcd\x1e\xe1r\xadgD\x81\x05\x7f\xc3U_\xbahe\xed`\x8a0\\\xd5\xb6\x9a\xca\x85\xec\xdbp\x85\xd68\xde\xe3:\xdc\xff\x8cSg\xc2$KD\xc2\x04O\xe8{Y\xa1\xbc^\x9a\xa9\xb6\xb8\xd4\xbd\x9c\xcc;;N\xbe\xf50\x9e\xd8q\x98\x9a&\x8c\xe3\x98\x0b\xb2\x9d\x91~\xf8\x9b4\n\x8d\x1c\xeb\xe1\x98z\xc6\x82\x9d\x9cv\xa4\xbf&[\xd7l28zz\x1d\xb4\x9e\xf5\xc7\xaaE\x15|\xb2\xa8\xd7\xae\x18[\xa8\x1b\xaf\xa9\x8f.\xd0G#\xf6a\xe5\xa3\x1e\xa8\x07\xe3\xfa\x8d\xce#u\xeb\xee\x80\xd6#c\x94\xb19\xe5s\x9a~\x8d\xf2\"aE$\xbeyY\x9f/a\xb4\xa0I\x11\xefJ`e\xf6R/\xefp\x9aIJ4\xc6\xa5K\xe7\rY\x1d\xe0\x84\xa0\xd4\xdc\xa1\xae\xbb\xd5\xbc\x81\xb5c\xe1\xad\xd1\xb6\xb5\xf5\xd4U.R\x16e\x90s\x95\x01\x95\x8aQ\xe7\x88(B\x10\xaa\xc44\x93\x89\x88Qs\xe5\xce\x8c8\xf4\xee\xc4S\xcd}f\xea\x96`-\xb6\x83k\x19\xcd\xc8M?^\xeb\xa6\xbf\xf1\x08\x1c\xa6\xc1:8\xb8X\xb7\x8d\x1f\xa5\x9b\xd0r\xc4\x9f\x93\xe3H\x8a\xdbQyq\x9c+\x1d\x87\xef\x9b\xdd\xcc\xbe\xa0\r\xa6!\xf0N\x9a\x1d\x04\x7f\x14\x1b`\xf4\x1bt\xd4\xcc\xf7I*X\xaaK\x0e,\xe6\x11\x17B\x92\xd3\xa6\x91.(\xa5G&\xb2+c\xf4\xae\xec\x8c\xed\xd9\xce\xf6T?;=\x82U%\xc7E\xdd?\xf0\xf3n\xb3\xe7\x95m\x9b\xb9\xed\xe7u\x0b+,\x1a\xf0\x06\xbd\x97\xe4\x9f\xce\x9e'\x1d\x08!RT\x11S\x00\x9c\xe7\xe5i\xd2=\xd0\xe4XY\x92\xbdN\xba\xde\xd8\xe0\xd2\xbf\x19\xfdd\x83;7\x1e\xc4y>{\x1e\xfd\x14\x9da\xb4\x8e\x05\xd39\xcf3x\x89\xfeaI_\xa0\xbf/K^E\xff\xb2o\x87\xc6=5\x8f\xd7\xe4I\xf4<\xba\x8e\xa2\xdbWT\x98\xe8\x88\xbb\xa7\xe1D\xba\xc9\xff\xa4\xfbc{\xf1\x07Hk>,{\x07\x00\x00"
     # )
+
+
+@responses.activate
+def test_send_trace_wraps_envelope_send_failures(
+    jobA_job,
+    jobA_runs,
+    jobA_workflow,
+):
+    responses.get(
+        "https://api.github.com/repos/getsentry/sentry/actions/runs/2104746951",
+        json=jobA_runs,
+    )
+    responses.get(
+        "https://api.github.com/repos/getsentry/sentry/actions/workflows/1174556",
+        json=jobA_workflow,
+    )
+
+    client = GithubClient(dsn=DSN, token=TOKEN)
+    with patch(
+        "src.github_sdk.requests.post",
+        side_effect=requests.exceptions.SSLError("ssl eof"),
+    ):
+        with pytest.raises(SentryEnvelopeSendError) as excinfo:
+            client.send_trace(jobA_job)
+
+    assert str(excinfo.value) == "Failed to send trace envelope to Sentry ingest"

tests/test_web_app_handler.py

@@ -5,6 +5,7 @@
 
 import pytest
 
+from src.github_sdk import SentryEnvelopeSendError
 from src.web_app_handler import WebAppHandler
 
 valid_signature = "d9259f51d3b64e7fe0cbe09d9b08b8ee763170d3521fecc35fd8b453be8cf6a5"
@@ -108,3 +109,28 @@ def test_handle_event_with_secret(monkeypatch, webhook_event):
     )
     assert reason == "OK"
     assert http_code == 200
+
+
+def test_handle_event_ignores_sentry_envelope_send_error(monkeypatch, webhook_event):
+    monkeypatch.setenv("GH_WEBHOOK_SECRET", "fake_secret")
+    monkeypatch.setenv("GH_TOKEN", "irrelevant")
+    monkeypatch.delenv("GH_APP_ID", raising=False)
+
+    payload = json.loads(json.dumps(webhook_event["payload"]))
+    payload["installation"] = {"id": 1}
+
+    handler = WebAppHandler()
+    with (
+        mock.patch("src.web_app_handler.fetch_dsn_for_github_org", return_value="https://foo@random.ingest.sentry.io/bar"),
+        mock.patch(
+            "src.web_app_handler.GithubClient.send_trace",
+            side_effect=SentryEnvelopeSendError("Failed to send trace envelope to Sentry ingest"),
+        ),
+    ):
+        reason, http_code = handler.handle_event(
+            data=payload,
+            headers=webhook_event["headers"],
+        )
+
+    assert reason == "OK"
+    assert http_code == 200