Microsoft: handle sporadic underlying connection was closed, cache webhook_notifications_enabled

inbox/events/microsoft/events_provider.py

@@ -73,6 +73,7 @@ def __init__(self, account_id: int, namespace_id: int):
         self.client = MicrosoftGraphClient(
             lambda: self._get_access_token(scopes=MICROSOFT_CALENDAR_SCOPES)
         )
+        self._webhook_notifications_enabled: Optional[bool] = None
 
     def sync_calendars(self) -> CalendarSyncResponse:
         """
@@ -205,8 +206,8 @@ def webhook_notifications_enabled(self, account: Account) -> bool:
         Return True if webhook notifications are enabled for a given account.
 
         This works by creating a dummy subscription and then immediately deleting
-        it. We found that in practice subscriptions don't work for
-        some accounts. There are some theories on the internet
+        it, subsequent calls use cached value. We found that in practice subscriptions
+        don't work for some accounts. There are some theories on the internet
         why it does not work i.e.: Office365 administrator applying a restrictive
         policy or some weird setup when the calendars might still be on on-premise
         servers but everything else in Azure. Microsoft does not give a definite answer,
@@ -216,6 +217,12 @@ def webhook_notifications_enabled(self, account: Account) -> bool:
         * https://learn.microsoft.com/en-us/answers/questions/417261/error-on-adding-subscription-on-events-using-ms-gr.html
         * https://stackoverflow.com/questions/65030751/ms-graph-adding-subscription-returns-extensionerror-and-serviceunavailable
         """
+
+        # First check if we already have cached value since this function is called
+        # repeatedly and there is no need to do extra HTTP request every time.
+        if self._webhook_notifications_enabled is not None:
+            return self._webhook_notifications_enabled
+
         try:
             dummy_subscription = self.client.subscribe_to_calendar_changes(
                 webhook_url=CALENDAR_LIST_WEBHOOK_URL.format(account.public_id),
@@ -227,13 +234,15 @@ def webhook_notifications_enabled(self, account: Account) -> bool:
                 message == "ExtensionError"
                 and "is currently on backend 'Unknown'" in description
             ):
+                self._webhook_notifications_enabled = False
                 return False
 
             raise
 
         subscription_id = cast(MsGraphSubscription, dummy_subscription)["id"]
         self.client.unsubscribe(subscription_id)
 
+        self._webhook_notifications_enabled = True
         return True
 
     def watch_calendar_list(self, account: Account) -> Optional[datetime.datetime]:

inbox/events/microsoft/graph_client.py

@@ -348,7 +348,23 @@ def subscribe(
             "clientState": secret,
         }
 
-        return self.request("POST", "/subscriptions", json=json)
+        max_retries = 5
+        for _ in range(max_retries):
+            try:
+                return self.request("POST", "/subscriptions", json=json)
+            except MicrosoftGraphClientException as e:
+                last_exception = e
+                message, description = e.args
+                if message == "InvalidRequest" and description.startswith(
+                    "The underlying connection was closed"
+                ):
+                    time.sleep(5)
+                    continue
+
+                raise
+
+        last_exception.args = (*last_exception.args, "Max retries reached")
+        raise last_exception
 
     def unsubscribe(self, subscription_id: str) -> Dict[str, Any]:
         """

tests/events/microsoft/test_graph_client.py

@@ -276,6 +276,55 @@ def test_iter_event_instances(client):
     }
 
 
+@responses.activate(registry=OrderedRegistry)
+def test_subscribe_connection_closed_retries(client):
+    responses.post(
+        BASE_URL + "/subscriptions",
+        json={
+            "error": {
+                "code": "InvalidRequest",
+                "message": "The underlying connection was closed: A connection that was expected to be kept alive was closed by the server.",
+            }
+        },
+        status=400,
+    )
+    responses.post(
+        BASE_URL + "/subscriptions", json={}, status=200,
+    )
+
+    with unittest.mock.patch("time.sleep"):
+        client.subscribe(
+            resource_url="/me/calendars",
+            change_types=[],
+            webhook_url="https://example.com",
+            secret="s3cret",
+        )
+
+
+@responses.activate
+def test_subscribe_connection_closed_max_retries(client):
+    responses.post(
+        BASE_URL + "/subscriptions",
+        json={
+            "error": {
+                "code": "InvalidRequest",
+                "message": "The underlying connection was closed: A connection that was expected to be kept alive was closed by the server.",
+            }
+        },
+        status=400,
+    )
+
+    with unittest.mock.patch("time.sleep"), pytest.raises(
+        MicrosoftGraphClientException, match="Max retries reached"
+    ):
+        client.subscribe(
+            resource_url="/me/calendars",
+            change_types=[],
+            webhook_url="https://example.com",
+            secret="s3cret",
+        )
+
+
 @responses.activate
 def test_subscribe_to_calendar_changes(client):
     def request_callback(request):