Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 32 additions & 7 deletions src/github_sdk.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
from sentry_sdk.envelope import Envelope
from sentry_sdk.utils import format_timestamp

ENVELOPE_UPLOAD_MAX_ATTEMPTS = 3
ENVELOPE_UPLOAD_TIMEOUT_SECONDS = 10


class GithubSentryError(Exception):
pass
Expand Down Expand Up @@ -128,13 +131,35 @@ def _send_envelope(self, trace):
with gzip.GzipFile(fileobj=body, mode="w") as f:
envelope.serialize_into(f)

req = requests.post(
self.sentry_project_url,
data=body.getvalue(),
headers=headers,
)
req.raise_for_status()
return req
for attempt in range(1, ENVELOPE_UPLOAD_MAX_ATTEMPTS + 1):
try:
req = requests.post(
self.sentry_project_url,
data=body.getvalue(),
headers=headers,
timeout=ENVELOPE_UPLOAD_TIMEOUT_SECONDS,
)
req.raise_for_status()
return req
except (
requests.exceptions.ConnectionError,
requests.exceptions.SSLError,
requests.exceptions.Timeout,
) as err:
if attempt == ENVELOPE_UPLOAD_MAX_ATTEMPTS:
logging.warning(
"Failed sending Sentry envelope after %s attempts: %s",
ENVELOPE_UPLOAD_MAX_ATTEMPTS,
err,
)
return
logging.info(
"Retrying Sentry envelope upload after transient network error "
"(attempt %s/%s): %s",
attempt,
ENVELOPE_UPLOAD_MAX_ATTEMPTS,
err,
)

def send_trace(self, job):
# This can happen when the workflow is skipped and there are no steps
Expand Down
54 changes: 54 additions & 0 deletions tests/test_github_sdk.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
from __future__ import annotations

import logging
import sys
from datetime import datetime
from unittest.mock import Mock
from unittest.mock import patch

import pytest
Expand All @@ -11,6 +13,8 @@
from requests import HTTPError
from sentry_sdk.utils import format_timestamp

from src.github_sdk import ENVELOPE_UPLOAD_MAX_ATTEMPTS
from src.github_sdk import ENVELOPE_UPLOAD_TIMEOUT_SECONDS
from src.github_sdk import GithubClient

DSN = "https://foo@random.ingest.sentry.io/bar"
Expand Down Expand Up @@ -158,3 +162,53 @@ def test_send_trace(
# resp.request.body
# == b"\x1f\x8b\x08\x00\xf1\x16}b\x02\xff\xb5TM\x8f\xd30\x10\xbd\xef\xaf\x88|\x02\xa9m\x1c\xc7\x89\x93H\x08\xd0\x8a;\x12\x9c@\xa8\x9a\xd8\xe3&\xbb\xf9\"vX\xaa\xaa\xff\x1d{\xdb\xee\x97\x96n\xcb\x8aS\xd3\x99\xf1\xf8\xbd7o\xbc\xd9^l\x88]\x0fH\nbG\xe8\x0cH[\xf7\x1d\x99\x11\xd9w\x16;\xbb\xdc'a\x18\x9aZ\x82O\x86W\xe6\xb6\xa2\xc1ne+RD\x19\xa7\xbe\r\xfe\xf2\xf5\xb5r\xd5\t \x139H\x95g\x8c\xcbRC\x96P\xceh\x14K]\xea\xac\x14\xee\xf4\xb3\x97>\xfcW\x10=\xdebP\x81EcM\xf0\x86\xbe=\xe0\xfam\r)6\xbe\\\xa2\xff0\x03t\xbb\x9b\x81\xd3He\xb1\x14()\xcf\x13\xbdk*q\x97\xcd\xa2\x92\x96\x08)\xa0\xd2<\x8b\x04\x08\xaeb\xa6\x04Ms\x9a)\xcd\x1e\xe1r\xadgD\x81\x05\x7f\xc3U_\xbahe\xed`\x8a0\\\xd5\xb6\x9a\xca\x85\xec\xdbp\x85\xd68\xde\xe3:\xdc\xff\x8cSg\xc2$KD\xc2\x04O\xe8{Y\xa1\xbc^\x9a\xa9\xb6\xb8\xd4\xbd\x9c\xcc;;N\xbe\xf50\x9e\xd8q\x98\x9a&\x8c\xe3\x98\x0b\xb2\x9d\x91~\xf8\x9b4\n\x8d\x1c\xeb\xe1\x98z\xc6\x82\x9d\x9cv\xa4\xbf&[\xd7l28zz\x1d\xb4\x9e\xf5\xc7\xaaE\x15|\xb2\xa8\xd7\xae\x18[\xa8\x1b\xaf\xa9\x8f.\xd0G#\xf6a\xe5\xa3\x1e\xa8\x07\xe3\xfa\x8d\xce#u\xeb\xee\x80\xd6#c\x94\xb19\xe5s\x9a~\x8d\xf2\"aE$\xbeyY\x9f/a\xb4\xa0I\x11\xefJ`e\xf6R/\xefp\x9aIJ4\xc6\xa5K\xe7\rY\x1d\xe0\x84\xa0\xd4\xdc\xa1\xae\xbb\xd5\xbc\x81\xb5c\xe1\xad\xd1\xb6\xb5\xf5\xd4U.R\x16e\x90s\x95\x01\x95\x8aQ\xe7\x88(B\x10\xaa\xc44\x93\x89\x88Qs\xe5\xce\x8c8\xf4\xee\xc4S\xcd}f\xea\x96`-\xb6\x83k\x19\xcd\xc8M?^\xeb\xa6\xbf\xf1\x08\x1c\xa6\xc1:8\xb8X\xb7\x8d\x1f\xa5\x9b\xd0r\xc4\x9f\x93\xe3H\x8a\xdbQyq\x9c+\x1d\x87\xef\x9b\xdd\xcc\xbe\xa0\r\xa6!\xf0N\x9a\x1d\x04\x7f\x14\x1b`\xf4\x1bt\xd4\xcc\xf7I*X\xaaK\x0e,\xe6\x11\x17B\x92\xd3\xa6\x91.(\xa5G&\xb2+c\xf4\xae\xec\x8c\xed\xd9\xce\xf6T?;=\x82U%\xc7E\xdd?\xf0\xf3n\xb3\xe7\x95m\x9b\xb9\xed\xe7u\x0b+,\x1a\xf0\x06\xbd\x97\xe4\x9f\xce\x9e'\x1d\x08!RT\x11S\x00\x9c\xe7\xe5i\xd2=\xd0\xe4XY\x92\xbdN\xba\xde\xd8\xe0\xd2\xbf\x19\xfdd\x83;7\x1e\xc4y>{\x1e\xfd\x14\x9da\xb4\x8e\x05\xd39\xcf3x\x89\xfeaI_\xa0\xbf/K^E\xff\xb2o\x87\xc6=5\x8f\xd7\xe4I\xf4<\xba\x8e\xa2\xdbWT\x98\xe8\x88\xbb\xa7\xe1D\xba\xc9\xff\xa4\xfbc{\xf1\x07Hk>,{\x07\x00\x00"
# )


@responses.activate
@patch("src.github_sdk.requests.post")
def test_send_trace_retries_transient_upload_errors(
mock_post,
jobA_job,
jobA_runs,
jobA_workflow,
):
responses.get(jobA_job["run_url"], json=jobA_runs)
responses.get(jobA_runs["workflow_url"], json=jobA_workflow)

success_response = Mock()
success_response.raise_for_status.return_value = None
mock_post.side_effect = [
requests.exceptions.SSLError("transient handshake failure"),
success_response,
]

client = GithubClient(dsn=DSN, token=TOKEN)
response = client.send_trace(jobA_job)

assert response is success_response
assert mock_post.call_count == 2
for call in mock_post.call_args_list:
assert call.kwargs["timeout"] == ENVELOPE_UPLOAD_TIMEOUT_SECONDS


@responses.activate
@patch("src.github_sdk.requests.post")
def test_send_trace_returns_none_when_retries_exhausted(
mock_post,
caplog,
jobA_job,
jobA_runs,
jobA_workflow,
):
responses.get(jobA_job["run_url"], json=jobA_runs)
responses.get(jobA_runs["workflow_url"], json=jobA_workflow)

mock_post.side_effect = requests.exceptions.SSLError("still failing")
client = GithubClient(dsn=DSN, token=TOKEN)

with caplog.at_level(logging.WARNING):
response = client.send_trace(jobA_job)

assert response is None
assert mock_post.call_count == ENVELOPE_UPLOAD_MAX_ATTEMPTS
assert "Failed sending Sentry envelope after" in caplog.text
Loading