|
34 | 34 | # return one row. |
35 | 35 | _FETCHONE_BUFFER_MAX_ROWS = 1000 |
36 | 36 |
|
| 37 | +# After a fetchone() batch conversion fails, the batch is retried one row at a time. |
| 38 | +# If this many leading rows all fail too, the failure is treated as systematic: the |
| 39 | +# rest of the batch is not retried row by row (each would cost a separate |
| 40 | +# conversion) and those rows re-raise the original batch error instead. |
| 41 | +_FETCHONE_FALLBACK_PROBE_ROWS = 5 |
| 42 | + |
37 | 43 |
|
38 | 44 | class _FailedRow: |
39 | 45 | """Placeholder in the fetchone() row buffer for a row that failed to convert.""" |
@@ -207,30 +213,44 @@ def _raw_row_count(raw: Any) -> Optional[int]: |
207 | 213 |
|
208 | 214 | @staticmethod |
209 | 215 | def _convert_rows_individually( |
210 | | - raw: Any, convert: Callable[[Any], List[Row]] |
| 216 | + raw: Any, convert: Callable[[Any], List[Row]], batch_error: Exception |
211 | 217 | ) -> Optional[List[Any]]: |
212 | 218 | """Convert ``raw`` row by row after a batch conversion failed. |
213 | 219 |
|
214 | 220 | Rows that fail are returned as ``_FailedRow`` placeholders so the error is |
215 | | - raised only when that row is handed out. Returns None if ``raw`` has no rows |
216 | | - to split, in which case the caller re-raises the batch error. |
| 221 | + raised only when that row is handed out. If the first |
| 222 | + ``_FETCHONE_FALLBACK_PROBE_ROWS`` rows all fail, the failure is treated as |
| 223 | + systematic and the remaining rows are parked with ``batch_error`` instead of |
| 224 | + being converted one by one. Returns None if ``raw`` has no rows to split, in |
| 225 | + which case the caller re-raises the batch error. |
217 | 226 | """ |
218 | 227 | n = ResultSet._raw_row_count(raw) |
219 | 228 | if not n: |
220 | 229 | return None |
221 | 230 | rows: List[Any] = [] |
| 231 | + systematic = False |
222 | 232 | for i in range(n): |
| 233 | + if i == _FETCHONE_FALLBACK_PROBE_ROWS and all( |
| 234 | + isinstance(r, _FailedRow) for r in rows |
| 235 | + ): |
| 236 | + systematic = True |
| 237 | + rows.extend(_FailedRow(batch_error) for _ in range(n - i)) |
| 238 | + break |
223 | 239 | one = raw[i : i + 1] if isinstance(raw, list) else raw.slice(i, 1) |
224 | 240 | try: |
225 | 241 | rows.extend(convert(one)) |
226 | 242 | except Exception as e: |
227 | 243 | rows.append(_FailedRow(e)) |
228 | 244 | failed = sum(isinstance(r, _FailedRow) for r in rows) |
229 | 245 | logger.warning( |
230 | | - "fetchone(): failed to convert %d of %d row(s) in a batch; each failing " |
| 246 | + "fetchone(): failed to convert %d of %d row(s) in a batch%s; each failing " |
231 | 247 | "row raises when fetched", |
232 | 248 | failed, |
233 | 249 | n, |
| 250 | + " (first %d rows all failed, rest not retried)" % _FETCHONE_FALLBACK_PROBE_ROWS |
| 251 | + if systematic |
| 252 | + else "", |
| 253 | + exc_info=batch_error, |
234 | 254 | ) |
235 | 255 | return rows |
236 | 256 |
|
@@ -270,13 +290,13 @@ def _fetchone_buffered( |
270 | 290 | raw = fetch_raw(max(min(self.arraysize, _FETCHONE_BUFFER_MAX_ROWS), 1)) |
271 | 291 | try: |
272 | 292 | rows = convert(raw) |
273 | | - except Exception: |
| 293 | + except Exception as batch_error: |
274 | 294 | # The queue has already moved past this batch, so it can't be |
275 | 295 | # re-read. Fall back to converting it one row at a time so that, |
276 | 296 | # as before batching, only the row(s) that fail to convert are |
277 | 297 | # lost: each failing row is parked as a _FailedRow and re-raises |
278 | 298 | # its own error when it is reached. |
279 | | - fallback = self._convert_rows_individually(raw, convert) |
| 299 | + fallback = self._convert_rows_individually(raw, convert, batch_error) |
280 | 300 | if fallback is None: |
281 | 301 | raise |
282 | 302 | rows = fallback |
|
0 commit comments