|
41 | 41 | _FETCHONE_BUFFER_INITIAL_ROWS = 1 |
42 | 42 | _FETCHONE_BUFFER_GROWTH_FACTOR = 10 |
43 | 43 |
|
44 | | -# After a fetchone() batch conversion fails, the batch is retried one row at a time. |
45 | | -# If this many leading rows all fail too, with the same error type and message as |
46 | | -# the batch, the failure is treated as systematic: the rest of the batch is not |
47 | | -# retried row by row (each would cost a separate conversion) and those rows re-raise |
48 | | -# the original batch error instead. Leading rows failing for other reasons do not |
49 | | -# trigger the shortcut, so later good rows are still converted. |
50 | | -_FETCHONE_FALLBACK_PROBE_ROWS = 5 |
51 | | - |
52 | | - |
53 | 44 | class _FailedRow: |
54 | 45 | """Placeholder in the fetchone() row buffer for a row that failed to convert.""" |
55 | 46 |
|
56 | 47 | __slots__ = ("error", "traceback") |
57 | 48 |
|
58 | 49 | def __init__(self, error: Exception): |
59 | 50 | self.error = error |
60 | | - # Snapshot the conversion traceback. Systematic failures park the same |
61 | | - # exception instance in many rows; restoring this before each raise keeps |
| 51 | + # Snapshot the conversion traceback. A converter may raise the same |
| 52 | + # exception instance for many rows; restoring this before each raise keeps |
62 | 53 | # its __traceback__ from growing (and pinning frames) on every fetchone(). |
63 | 54 | self.traceback = error.__traceback__ |
64 | 55 |
|
@@ -250,60 +241,50 @@ def _convert_rows_individually( |
250 | 241 | batch_error: Exception, |
251 | 242 | log_level: int = logging.WARNING, |
252 | 243 | ) -> Optional[List[Any]]: |
253 | | - """Convert ``raw`` row by row after a batch conversion failed. |
254 | | -
|
255 | | - Rows that fail are returned as ``_FailedRow`` placeholders so the error is |
256 | | - raised only when that row is handed out. If the first |
257 | | - ``_FETCHONE_FALLBACK_PROBE_ROWS`` rows all fail with the same error as the |
258 | | - batch (same type and message), the failure is treated as systematic and the |
259 | | - remaining rows are parked with ``batch_error`` instead of being converted one |
260 | | - by one. Otherwise every row is tried individually. Returns None if ``raw`` |
261 | | - has no rows to split, in which case the caller re-raises the batch error. |
| 244 | + """Convert ``raw`` in smaller pieces after a batch conversion failed. |
| 245 | +
|
| 246 | + The failed batch is bisected: each failing range is split in half and the |
| 247 | + halves are converted separately, down to single rows. Only rows that fail |
| 248 | + on their own are returned as ``_FailedRow`` placeholders (raised when that |
| 249 | + row is handed out); every row that converts is kept, so a failure is never |
| 250 | + attributed to a row that was not itself tried. Cost is about |
| 251 | + ``2 * k * log2(n)`` conversions for ``k`` bad rows, and at most ``2n`` |
| 252 | + when every row fails. Returns None if ``raw`` has no rows to split, in |
| 253 | + which case the caller re-raises the batch error. |
262 | 254 | """ |
263 | 255 | n = ResultSet._raw_row_count(raw) |
264 | 256 | if not n: |
265 | 257 | return None |
266 | 258 |
|
267 | | - def same_as_batch(r: Any) -> bool: |
268 | | - return ( |
269 | | - isinstance(r, _FailedRow) |
270 | | - and type(r.error) is type(batch_error) |
271 | | - and str(r.error) == str(batch_error) |
272 | | - ) |
273 | | - |
274 | 259 | rows: List[Any] = [] |
275 | | - systematic = False |
276 | | - for i in range(n): |
277 | | - # Only skip the remaining per-row retries when every probe row failed |
278 | | - # with the same error as the batch; leading rows that fail for their |
279 | | - # own reasons must not cause later good rows to be dropped. |
280 | | - if i == _FETCHONE_FALLBACK_PROBE_ROWS and all( |
281 | | - same_as_batch(r) for r in rows |
282 | | - ): |
283 | | - systematic = True |
284 | | - rows.extend(_FailedRow(batch_error) for _ in range(n - i)) |
285 | | - break |
286 | | - one = raw[i : i + 1] if isinstance(raw, list) else raw.slice(i, 1) |
287 | | - try: |
288 | | - rows.extend(convert(one)) |
289 | | - except Exception as e: |
290 | | - rows.append(_FailedRow(e)) |
| 260 | + |
| 261 | + def convert_range(lo: int, hi: int, error: Optional[Exception]) -> None: |
| 262 | + # ``error`` is the failure already seen for [lo, hi), if it was tried. |
| 263 | + if error is None: |
| 264 | + piece = raw[lo:hi] if isinstance(raw, list) else raw.slice(lo, hi - lo) |
| 265 | + try: |
| 266 | + rows.extend(convert(piece)) |
| 267 | + return |
| 268 | + except Exception as e: |
| 269 | + error = e |
| 270 | + if hi - lo == 1: |
| 271 | + rows.append(_FailedRow(error)) |
| 272 | + return |
| 273 | + mid = (lo + hi) // 2 |
| 274 | + convert_range(lo, mid, None) |
| 275 | + convert_range(mid, hi, None) |
| 276 | + |
| 277 | + convert_range(0, n, batch_error) |
291 | 278 | failed = sum(isinstance(r, _FailedRow) for r in rows) |
292 | 279 | # One summary line, no traceback: the error itself still reaches the caller |
293 | 280 | # when the failing row is fetched. The full traceback is only logged at DEBUG. |
294 | 281 | # ``log_level`` lets the caller log this at WARNING only once per result set. |
295 | 282 | logger.log( |
296 | 283 | log_level, |
297 | | - "fetchone(): failed to convert %d of %d row(s) in a batch%s; each failing " |
| 284 | + "fetchone(): failed to convert %d of %d row(s) in a batch; each failing " |
298 | 285 | "row raises when fetched (%s: %s)", |
299 | 286 | failed, |
300 | 287 | n, |
301 | | - ( |
302 | | - " (first %d rows all failed, rest not retried)" |
303 | | - % _FETCHONE_FALLBACK_PROBE_ROWS |
304 | | - if systematic |
305 | | - else "" |
306 | | - ), |
307 | 288 | type(batch_error).__name__, |
308 | 289 | batch_error, |
309 | 290 | ) |
@@ -379,7 +360,7 @@ def _fetchone_buffered( |
379 | 360 | rows = convert(raw) |
380 | 361 | except Exception as batch_error: |
381 | 362 | # The queue has already moved past this batch, so it can't be |
382 | | - # re-read. Fall back to converting it one row at a time so that, |
| 363 | + # re-read. Fall back to bisecting it down to single rows so that, |
383 | 364 | # as before batching, only the row(s) that fail to convert are |
384 | 365 | # lost: each failing row is parked as a _FailedRow and re-raises |
385 | 366 | # its own error when it is reached. |
|
0 commit comments