Skip to content

Commit bc2e0f2

Browse files
vuanhphungIsaac
andcommitted
test(kernel): run the pandas-parity tests through the batch path
Row fetches now go through KernelResultSet._rows, so its _convert_arrow_table override had no callers left; remove it and point the parity, out-of-range and fallback-logging tests at _rows. Signed-off-by: Vu Anh Phung <vu.phung@databricks.com> Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 5668a0e commit bc2e0f2

2 files changed

Lines changed: 6 additions & 19 deletions

File tree

‎src/databricks/sql/backend/kernel/result_set.py‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -362,14 +362,7 @@ def _rows(self, batches: List[pyarrow.RecordBatch]) -> List[Row]:
362362
rows = _rows_without_pandas(batches, self._result_schema)
363363
if rows is not None:
364364
return rows
365-
return super()._convert_arrow_table(self._to_table(batches))
366-
367-
def _convert_arrow_table(self, table):
368-
if self.connection.disable_pandas is not True:
369-
rows = _rows_without_pandas(table.to_batches(), self._result_schema)
370-
if rows is not None:
371-
return rows
372-
return super()._convert_arrow_table(table)
365+
return self._convert_arrow_table(self._to_table(batches))
373366

374367
def fetchone(self) -> Optional[Row]:
375368
self._ensure_buffered(1)

‎tests/unit/test_kernel_result_set.py‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -380,7 +380,7 @@ def _convert(table):
380380
with patch.object(
381381
ResultSet, "_convert_arrow_table", autospec=True, side_effect=pandas_path
382382
) as spy:
383-
rows = rs._convert_arrow_table(table)
383+
rows = rs._rows(table.to_batches())
384384
return rows, pandas_path(rs, table), spy.called
385385

386386

@@ -475,7 +475,7 @@ def test_out_of_range_values_convert_as_on_the_pandas_path(column, expected):
475475
table = pa.table({"c": column})
476476
rs = _make_rs(_FakeKernelHandle(table.schema, []))
477477
for convert in (
478-
rs._convert_arrow_table,
478+
lambda t: rs._rows(t.to_batches()),
479479
lambda t: ResultSet._convert_arrow_table(rs, t),
480480
):
481481
if isinstance(expected, list):
@@ -490,7 +490,7 @@ def test_conversion_errors_are_logged_before_falling_back(caplog):
490490
rs = _make_rs(_FakeKernelHandle(table.schema, []))
491491
with caplog.at_level(logging.DEBUG, logger=kernel_result_set.__name__):
492492
with pytest.raises(ValueError):
493-
rs._convert_arrow_table(table)
493+
rs._rows(table.to_batches())
494494
assert "to_pylist() failed for a date32[day] column" in caplog.text
495495

496496

@@ -646,14 +646,8 @@ def test_the_cost_budget_is_computed_once_per_schema_not_per_fetch():
646646
),
647647
schema_cache=cache,
648648
)
649-
assert [
650-
r[0] for r in [rs.fetchone()] + rs.fetchmany(2) + rs.fetchall()
651-
] == [
652-
1,
653-
2,
654-
3,
655-
4,
656-
]
649+
rows = [rs.fetchone()] + rs.fetchmany(2) + rs.fetchall()
650+
assert [row[0] for row in rows] == [1, 2, 3, 4]
657651
assert budget.call_count == 1
658652
_make_rs(
659653
_FakeKernelHandle(pa.schema([("m", pa.int64())]), []), schema_cache=cache

0 commit comments

Comments
 (0)