Keep track when we try and fail to process a pulled event

changelog.d/13589.feature

@@ -0,0 +1 @@
+Keep track when we attempt to backfill an event but fail so we can intelligently back-off in the future.

synapse/handlers/federation_event.py

@@ -862,6 +862,10 @@ async def _process_pulled_event(
             self._sanity_check_event(event)
         except SynapseError as err:
             logger.warning("Event %s failed sanity check: %s", event_id, err)
+            if backfilled:
+                await self._store.record_event_failed_backfill_attempt(
+                    event.room_id, event_id
+                )
             return
 
         try:
@@ -897,6 +901,11 @@ async def _process_pulled_event(
                     backfilled=backfilled,
                 )
         except FederationError as e:
+            if backfilled:
+                await self._store.record_event_failed_backfill_attempt(
+                    event.room_id, event_id
+                )
+
             if e.code == 403:
                 logger.warning("Pulled event %s failed history check.", event_id)
             else:

synapse/storage/databases/main/event_federation.py

@@ -1292,6 +1292,95 @@ def _get_backfill_events(
 
         return event_id_results
 
+    @trace
+    async def record_event_failed_backfill_attempt(
+        self, room_id: str, event_id: str
+    ) -> None:
+        """
+        Record when we fail to backfill an event.
+
+        This information allows us to be more intelligent when we decide to
+        retry (we don't need to fail over and over) and we can process that
+        event in the background so we don't block on it each time.
+
+        Args:
+            room_id: The room where the event failed to backfill from
+            event_id: The event that failed to be backfilled
+        """
+        if self.database_engine.can_native_upsert:
+            await self.db_pool.runInteraction(
+                "record_event_failed_backfill_attempt",
+                self._record_event_failed_backfill_attempt_upsert_native_txn,
+                room_id,
+                event_id,
+                db_autocommit=True,  # Safe as its a single upsert
+            )
+        else:
+            await self.db_pool.runInteraction(
+                "record_event_failed_backfill_attempt",
+                self._record_event_failed_backfill_attempt_upsert_emulated_txn,
+                room_id,
+                event_id,
+            )
+
+    def _record_event_failed_backfill_attempt_upsert_native_txn(
+        self,
+        txn: LoggingTransaction,
+        room_id: str,
+        event_id: str,
+    ) -> None:
+        assert self.database_engine.can_native_upsert
+
+        sql = """
+            INSERT INTO event_failed_backfill_attempts (
+                room_id, event_id, num_attempts, last_attempt_ts
+            )
+                VALUES (?, ?, ?, ?)
+            ON CONFLICT (room_id, event_id) DO UPDATE SET
+                num_attempts=event_failed_backfill_attempts.num_attempts + 1,
+                last_attempt_ts=EXCLUDED.last_attempt_ts;
+        """
+
+        txn.execute(sql, (room_id, event_id, 1, self._clock.time_msec()))
+
+    def _record_event_failed_backfill_attempt_upsert_emulated_txn(
+        self,
+        txn: LoggingTransaction,
+        room_id: str,
+        event_id: str,
+    ) -> None:
+        self.database_engine.lock_table(txn, "event_failed_backfill_attempts")
+
+        prev_row = self.db_pool.simple_select_one_txn(
+            txn,
+            table="event_failed_backfill_attempts",
+            keyvalues={"room_id": room_id, "event_id": event_id},
+            retcols=("num_attempts"),
+            allow_none=True,
+        )
+
+        if not prev_row:
+            self.db_pool.simple_insert_txn(
+                txn,
+                table="event_failed_backfill_attempts",
+                values={
+                    "room_id": room_id,
+                    "event_id": event_id,
+                    "num_attempts": 1,
+                    "last_attempt_ts": self._clock.time_msec(),
+                },
+            )
+        else:
+            self.db_pool.simple_update_one_txn(
+                txn,
+                table="event_failed_backfill_attempts",
+                keyvalues={"room_id": room_id, "event_id": event_id},
+                updatevalues={
+                    "num_attempts": prev_row["num_attempts"] + 1,
+                    "last_attempt_ts": self._clock.time_msec(),
+                },
+            )
+
     async def get_missing_events(
         self,
         room_id: str,

synapse/storage/databases/main/events.py

@@ -2435,17 +2435,31 @@ def _update_backward_extremeties(
             "DELETE FROM event_backward_extremities"
             " WHERE event_id = ? AND room_id = ?"
         )
+        backward_extremity_tuples_to_remove = [
+            (ev.event_id, ev.room_id)
+            for ev in events
+            if not ev.internal_metadata.is_outlier()
+            # If we encountered an event with no prev_events, then we might
+            # as well remove it now because it won't ever have anything else
+            # to backfill from.
+            or len(ev.prev_event_ids()) == 0
+        ]
         txn.execute_batch(
             query,
-            [
-                (ev.event_id, ev.room_id)
-                for ev in events
-                if not ev.internal_metadata.is_outlier()
-                # If we encountered an event with no prev_events, then we might
-                # as well remove it now because it won't ever have anything else
-                # to backfill from.
-                or len(ev.prev_event_ids()) == 0
-            ],
+            backward_extremity_tuples_to_remove,
+        )
+
+        # Clear out the failed backfill attempts after we successfully pulled
+        # the event. Since we no longer need these events as backward
+        # extremities, it also means that they won't be backfilled from again so
+        # we no longer need to store the backfill attempts around it.
+        query = """
+            DELETE FROM event_failed_backfill_attempts
+            WHERE event_id = ? and room_id = ?
+        """
+        txn.execute_batch(
+            query,
+            backward_extremity_tuples_to_remove,
         )
 
 

synapse/storage/schema/main/delta/72/04event_failed_backfill_attempts.sql

@@ -0,0 +1,27 @@
+/* Copyright 2022 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+-- Add a table that keeps track of when we failed to backfill an event. This
+-- allows us to be more intelligent when we decide to retry (we don't need to
+-- fail over and over) and we can process that event in the background so we
+-- don't block on it each time.
+CREATE TABLE IF NOT EXISTS event_failed_backfill_attempts(
+    room_id TEXT NOT NULL,
+    event_id TEXT NOT NULL,
+    num_attempts INT NOT NULL,
+    last_attempt_ts BIGINT NOT NULL,
+    PRIMARY KEY (room_id, event_id)
+);