Skip to content

Commit 5cd7941

Browse files
vuanhphungIsaac
andcommitted
perf(kernel): cap the pandas-free conversion by cost, not rows
to_pylist() costs more per value than pandas, so with only a 128-row cap wide or string/decimal/date-heavy results were slower than the pandas path (16 lineitem columns x 128 rows 0.92-1.15x, 64 strings x 128 rows up to 2.2x, 32 dates x 128 rows up to 4.2x). A result now skips pandas only while rows times the sum of per-type weights stays within 1024, which kept every admitted shape at most 0.67x of the pandas CPU on pandas 1.3-3.1. date64, time32 and time64 now take the pandas path, since to_pylist() rounds pre-1970 date64 values down and wraps out-of-range times where pandas raises. A to_pylist() error (e.g. a date past 9999-12-31) also falls back to pandas, so errors match the pandas path. Signed-off-by: Vu Anh Phung <vu.phung@databricks.com> Co-authored-by: Isaac <no-reply@databricks.com>
1 parent d965a66 commit 5cd7941

3 files changed

Lines changed: 166 additions & 80 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
- Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads.
55
- Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629))
66
- Speed up Thrift response handling: read replies in 1 MiB chunks and decode them with Thrift's C-accelerated protocol, and stop stringifying every request/response for the default no-op debug logger. Wide result schemas, metadata calls with many rows, and inline Arrow results benefit most.
7-
- Speed up fetching small results on the kernel backend (`use_kernel=True`): results of up to 128 rows are converted to rows without building a pandas DataFrame when that gives the same values, which cuts client CPU for a one-row query by about 40%.
7+
- Speed up fetching small results on the kernel backend (`use_kernel=True`): they are converted to rows without building a pandas DataFrame when that is cheaper and gives the same values, which cut client CPU per query by 47% in a local benchmark of 200 concurrent sessions running one-row queries against a mock server.
88

99
# 4.9.0 (2026-11-04)
1010
- Upgrade Databricks SQL Kernel to 1.4.0, which keeps Lakehouse Real-Time session affinity and flushes shared telemetry only after the final session closes.

‎src/databricks/sql/backend/kernel/result_set.py‎

Lines changed: 85 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232

3333
import logging
3434
from collections import deque
35-
from typing import Any, Deque, List, Optional, TYPE_CHECKING, cast
35+
from typing import Any, Deque, List, Optional, Tuple, TYPE_CHECKING, cast
3636

3737
import pyarrow
3838

@@ -48,71 +48,102 @@
4848

4949
logger = logging.getLogger(__name__)
5050

51-
# Results with at most this many rows are converted to rows with to_pylist()
52-
# instead of through a pandas DataFrame, whose fixed cost (hundreds of
53-
# microseconds) dominates small results.
51+
# Small results are converted with to_pylist() instead of through a pandas
52+
# DataFrame, whose fixed cost (100-300 µs) dominates them. to_pylist() costs
53+
# more per value, so a result skips pandas only while its row count times the
54+
# sum of its column weights (per-value cost relative to an integer) is within
55+
# _MAX_COST_WITHOUT_PANDAS, which kept it at least 25% cheaper than pandas for
56+
# every type and width measured (pyarrow 23, pandas 1.3-3.1).
5457
_MAX_ROWS_WITHOUT_PANDAS = 128
55-
# pyarrow converts tz-aware timestamps to Python values far more slowly than
56-
# pandas does, so results with one skip pandas only when tiny.
57-
_MAX_ROWS_WITHOUT_PANDAS_TZ = 4
58+
_MAX_COST_WITHOUT_PANDAS = 1024
5859

5960
# Types for which to_pylist() returns the same values as the pandas path in
60-
# ResultSet._convert_arrow_table. Floats only qualify without NaN, which pandas
61-
# turns into None or keeps depending on its version; timestamps and time64 only
62-
# when not in nanoseconds.
61+
# ResultSet._convert_arrow_table, with their weights. Floats only qualify
62+
# without NaN, which pandas turns into None or keeps depending on its version;
63+
# timestamps only when not in nanoseconds.
6364
_FLOAT_TYPE_IDS = frozenset(t.id for t in (pyarrow.float32(), pyarrow.float64()))
64-
_UNIT_TYPE_IDS = frozenset(
65-
t.id for t in (pyarrow.timestamp("us"), pyarrow.time64("us"))
66-
)
67-
_SAME_AS_PANDAS_TYPE_IDS = _FLOAT_TYPE_IDS | frozenset(
68-
t.id
69-
for t in (
70-
pyarrow.int8(),
71-
pyarrow.int16(),
72-
pyarrow.int32(),
73-
pyarrow.int64(),
74-
pyarrow.uint8(),
75-
pyarrow.uint16(),
76-
pyarrow.uint32(),
77-
pyarrow.uint64(),
78-
pyarrow.bool_(),
79-
pyarrow.decimal128(1),
80-
pyarrow.decimal256(1),
81-
pyarrow.string(),
82-
pyarrow.large_string(),
83-
pyarrow.binary(),
84-
pyarrow.large_binary(),
85-
pyarrow.binary(1),
86-
pyarrow.date32(),
87-
pyarrow.date64(),
88-
pyarrow.time32("s"),
89-
pyarrow.null(),
90-
)
91-
)
92-
93-
94-
def _rows_without_pandas(table: pyarrow.Table, row_factory: Row) -> Optional[List[Row]]:
65+
_TIMESTAMP_TYPE_ID = pyarrow.timestamp("us").id
66+
_VALUE_WEIGHTS = {
67+
**{
68+
t.id: 1
69+
for t in (
70+
pyarrow.int8(),
71+
pyarrow.int16(),
72+
pyarrow.int32(),
73+
pyarrow.int64(),
74+
pyarrow.uint8(),
75+
pyarrow.uint16(),
76+
pyarrow.uint32(),
77+
pyarrow.uint64(),
78+
pyarrow.bool_(),
79+
pyarrow.float32(),
80+
pyarrow.float64(),
81+
pyarrow.null(),
82+
)
83+
},
84+
**{t.id: 3 for t in (pyarrow.string(), pyarrow.large_string())},
85+
**{
86+
t.id: 6
87+
for t in (
88+
pyarrow.binary(),
89+
pyarrow.large_binary(),
90+
pyarrow.binary(1),
91+
pyarrow.decimal128(1),
92+
pyarrow.decimal256(1),
93+
)
94+
},
95+
pyarrow.date32().id: 12,
96+
_TIMESTAMP_TYPE_ID: 12,
97+
}
98+
# pyarrow looks the time zone up again for every tz-aware value, which costs
99+
# tens of microseconds when pytz isn't installed.
100+
_TZ_TIMESTAMP_WEIGHT = 400
101+
102+
103+
def _max_rows_without_pandas(types: List[pyarrow.DataType]) -> int:
104+
"""The most rows of a result with these column types that are converted
105+
with to_pylist(); -1 if such results always go through pandas."""
106+
total = 0
107+
for arrow_type in types:
108+
weight = _VALUE_WEIGHTS.get(arrow_type.id)
109+
if weight is None:
110+
return -1
111+
if arrow_type.id == _TIMESTAMP_TYPE_ID:
112+
if arrow_type.unit == "ns":
113+
return -1
114+
if arrow_type.tz is not None:
115+
weight = _TZ_TIMESTAMP_WEIGHT
116+
total += weight
117+
if total == 0:
118+
return -1
119+
return min(_MAX_ROWS_WITHOUT_PANDAS, _MAX_COST_WITHOUT_PANDAS // total)
120+
121+
122+
def _rows_without_pandas(
123+
table: pyarrow.Table, description: List[Tuple]
124+
) -> Optional[List[Row]]:
95125
"""The rows of ``table`` built from ``to_pylist()``, or None if the table
96-
must go through pandas (too many rows, or a column whose values could
126+
must go through pandas (too costly, or a column whose values could
97127
differ)."""
98128
num_rows = table.num_rows
99-
if num_rows > _MAX_ROWS_WITHOUT_PANDAS or table.num_columns == 0:
129+
# No weight is below 1, so this rules out larger results without
130+
# looking at every column type.
131+
if num_rows * table.num_columns > _MAX_COST_WITHOUT_PANDAS:
132+
return None
133+
if num_rows > _max_rows_without_pandas(table.schema.types):
100134
return None
101-
for arrow_type in table.schema.types:
102-
if arrow_type.id in _UNIT_TYPE_IDS:
103-
if arrow_type.unit == "ns":
104-
return None
105-
tz = getattr(arrow_type, "tz", None)
106-
if tz is not None and num_rows > _MAX_ROWS_WITHOUT_PANDAS_TZ:
107-
return None
108-
elif arrow_type.id not in _SAME_AS_PANDAS_TYPE_IDS:
109-
return None
110135
columns = []
111136
for column in table.itercolumns():
112-
values = column.to_pylist()
137+
try:
138+
values = column.to_pylist()
139+
except Exception:
140+
# E.g. a date past 9999-12-31: pandas then raises its own error,
141+
# as it does without this fast path.
142+
return None
113143
if column.type.id in _FLOAT_TYPE_IDS and any(v != v for v in values):
114144
return None
115145
columns.append(values)
146+
row_factory = Row(*[c[0] for c in description])
116147
return [row_factory(*values) for values in zip(*columns)]
117148

118149

@@ -271,8 +302,7 @@ def fetchmany_arrow(self, size: int) -> pyarrow.Table:
271302

272303
def _convert_arrow_table(self, table):
273304
if self.connection.disable_pandas is not True:
274-
row_factory = Row(*[c[0] for c in self.description])
275-
rows = _rows_without_pandas(table, row_factory)
305+
rows = _rows_without_pandas(table, self.description)
276306
if rows is not None:
277307
return rows
278308
return super()._convert_arrow_table(table)

‎tests/unit/test_kernel_result_set.py‎

Lines changed: 80 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,8 @@ def _columns_skipping_pandas():
262262
"i32": pa.array([1, None, -(2**31), 2**31 - 1], pa.int32()),
263263
"i64": pa.array([1, None, -(2**63), 2**63 - 1], pa.int64()),
264264
"u8": pa.array([1, None, 0, 255], pa.uint8()),
265+
"u16": pa.array([1, None, 0, 2**16 - 1], pa.uint16()),
266+
"u32": pa.array([1, None, 0, 2**32 - 1], pa.uint32()),
265267
"u64": pa.array([1, None, 0, 2**64 - 1], pa.uint64()),
266268
"bool": pa.array([True, None, False, True]),
267269
"f32": pa.array([0.1, None, float("inf"), -0.0], pa.float32()),
@@ -287,10 +289,6 @@ def _columns_skipping_pandas():
287289
datetime.date(9999, 12, 31),
288290
]
289291
),
290-
"date64": pa.array(
291-
[datetime.date(2024, 2, 29), None, datetime.date(1970, 1, 1), None],
292-
pa.date64(),
293-
),
294292
"ts": pa.array(
295293
[
296294
datetime.datetime(2024, 1, 1, 1, 2, 3, 4),
@@ -300,6 +298,7 @@ def _columns_skipping_pandas():
300298
],
301299
pa.timestamp("us"),
302300
),
301+
"ts_s": pa.array([0, None, -1, 2], pa.timestamp("s")),
303302
"ts_ms": pa.array([0, None, 1, 2], pa.timestamp("ms")),
304303
"ts_utc": pa.array(
305304
[
@@ -316,18 +315,27 @@ def _columns_skipping_pandas():
316315
pa.timestamp("us", tz="America/Los_Angeles"),
317316
),
318317
"ts_offset": pa.array([0, None, 1, 2], pa.timestamp("us", tz="+05:30")),
319-
"time": pa.array([0, None, 1, 86399999999], pa.time64("us")),
320-
"time32": pa.array([0, None, 1, 2], pa.time32("ms")),
321318
"null": pa.nulls(4),
322319
}
323320

324321

322+
def test_every_type_skipping_pandas_has_a_column_above():
323+
covered = {column.type.id for column in _columns_skipping_pandas().values()}
324+
assert covered == set(kernel_result_set._VALUE_WEIGHTS)
325+
326+
325327
def _columns_needing_pandas():
326328
return {
327329
# pandas maps NaN to None or keeps it, depending on its version.
328330
"f64_nan": pa.array([float("nan"), None, 1.0, 2.0]),
329331
"f32_nan": pa.array([1.0, float("nan"), None, 2.0], pa.float32()),
330332
"ts_ns": pa.array([1, None, 1000, 2], pa.timestamp("ns")),
333+
# Out-of-range values convert differently; see
334+
# test_out_of_range_values_convert_as_on_the_pandas_path.
335+
"date64": pa.array([0, None, 86_400_000, -1], pa.date64()),
336+
"time32_s": pa.array([0, None, 1, 2], pa.time32("s")),
337+
"time32_ms": pa.array([0, None, 1, 2], pa.time32("ms")),
338+
"time64": pa.array([0, None, 1, 86_399_999_999], pa.time64("us")),
331339
"time_ns": pa.array([1000, None, 2000, 0], pa.time64("ns")),
332340
"duration": pa.array([0, None, 1, 2], pa.duration("us")),
333341
"interval": pa.array(
@@ -391,10 +399,12 @@ def test_small_results_skip_pandas_with_the_same_values(name):
391399
names=["k", name, "k"],
392400
),
393401
]
394-
for table in tables:
395-
rows, expected, used_pandas = _convert(table)
396-
assert not used_pandas
397-
_assert_same_rows(rows, expected)
402+
# Values only; the cost budget is tested separately.
403+
with patch.object(kernel_result_set, "_MAX_COST_WITHOUT_PANDAS", 10**9):
404+
for table in tables:
405+
rows, expected, used_pandas = _convert(table)
406+
assert not used_pandas
407+
_assert_same_rows(rows, expected)
398408

399409

400410
@pytest.mark.parametrize("name", sorted(_columns_needing_pandas()))
@@ -407,25 +417,70 @@ def test_results_whose_values_could_differ_use_pandas(name):
407417
assert len(rows) == len(expected) == 4
408418

409419

410-
def test_large_results_use_pandas():
411-
limit = kernel_result_set._MAX_ROWS_WITHOUT_PANDAS
420+
@pytest.mark.parametrize(
421+
"types",
422+
[
423+
[pa.int64()],
424+
[pa.float64()] * 16,
425+
[pa.string()] * 3,
426+
[pa.decimal128(38, 4), pa.int32(), pa.string(), pa.date32()],
427+
[pa.timestamp("us", "Etc/UTC")],
428+
[pa.timestamp("us"), pa.binary()] * 50,
429+
],
430+
ids=["int", "16_doubles", "3_strings", "mixed", "tz_timestamp", "wide"],
431+
)
432+
def test_results_over_the_cost_budget_use_pandas(types):
433+
limit = kernel_result_set._max_rows_without_pandas(types)
434+
assert 0 < limit <= kernel_result_set._MAX_ROWS_WITHOUT_PANDAS
412435
for num_rows, expect_pandas in ((limit, False), (limit + 1, True)):
413-
rows, expected, used_pandas = _convert(pa.table({"n": list(range(num_rows))}))
436+
table = pa.table(
437+
[pa.nulls(num_rows, t) for t in types],
438+
names=[f"c{i}" for i in range(len(types))],
439+
)
440+
rows, expected, used_pandas = _convert(table)
414441
assert used_pandas is expect_pandas
415442
_assert_same_rows(rows, expected)
416443

417444

418-
def test_tz_aware_timestamps_skip_pandas_only_in_tiny_results():
419-
limit = kernel_result_set._MAX_ROWS_WITHOUT_PANDAS_TZ
420-
for tz, num_rows, expect_pandas in (
421-
("Etc/UTC", limit, False),
422-
("Etc/UTC", limit + 1, True),
423-
(None, limit + 1, False),
445+
def test_cost_budget_scales_with_width_and_type():
446+
max_rows = kernel_result_set._max_rows_without_pandas
447+
assert max_rows([pa.int64()]) == kernel_result_set._MAX_ROWS_WITHOUT_PANDAS
448+
assert max_rows([pa.int64()] * 64) < max_rows([pa.int64()] * 8)
449+
assert max_rows([pa.date32()] * 8) < max_rows([pa.int64()] * 8)
450+
assert max_rows([pa.timestamp("us", "UTC")]) < max_rows([pa.timestamp("us")])
451+
# Too costly for even one row; only empty results skip pandas.
452+
assert max_rows([pa.timestamp("us", "UTC")] * 3) == 0
453+
assert max_rows([]) == -1
454+
assert max_rows([pa.int64(), pa.list_(pa.int64())]) == -1
455+
assert max_rows([pa.timestamp("ns")]) == -1
456+
457+
458+
@pytest.mark.parametrize(
459+
"column, expected",
460+
[
461+
# Before 1970 and not whole days: to_pylist() rounds down.
462+
(pa.array([-1], pa.date64()), [datetime.date(1970, 1, 1)]),
463+
# Outside 0-24h: to_pylist() wraps around.
464+
(pa.array([86_400], pa.time32("s")), ValueError),
465+
(pa.array([-1], pa.time64("us")), ValueError),
466+
# Past datetime's range: to_pylist() raises OverflowError instead.
467+
(pa.array([2_932_897], pa.date32()), ValueError),
468+
(pa.array([253_402_300_800_000_000], pa.timestamp("us", "UTC")), ValueError),
469+
],
470+
ids=["date64", "time32", "time64", "date32", "timestamp"],
471+
)
472+
def test_out_of_range_values_convert_as_on_the_pandas_path(column, expected):
473+
table = pa.table({"c": column})
474+
rs = _make_rs(_FakeKernelHandle(table.schema, []))
475+
for convert in (
476+
rs._convert_arrow_table,
477+
lambda t: ResultSet._convert_arrow_table(rs, t),
424478
):
425-
column = pa.array(range(num_rows), pa.timestamp("us", tz=tz))
426-
rows, expected, used_pandas = _convert(pa.table({"t": column}))
427-
assert used_pandas is expect_pandas
428-
_assert_same_rows(rows, expected)
479+
if isinstance(expected, list):
480+
assert [row[0] for row in convert(table)] == expected
481+
else:
482+
with pytest.raises(expected):
483+
convert(table)
429484

430485

431486
def test_empty_results_skip_pandas():
@@ -435,7 +490,8 @@ def test_empty_results_skip_pandas():
435490

436491

437492
def test_fetch_methods_return_the_pandas_path_values():
438-
schema = pa.schema([("d", pa.decimal128(38, 4)), ("t", pa.timestamp("us", "UTC"))])
493+
schema = pa.schema([("d", pa.decimal128(38, 4)), ("t", pa.timestamp("us"))])
494+
assert kernel_result_set._max_rows_without_pandas(schema.types) >= 3
439495
batch = pa.RecordBatch.from_arrays(
440496
[
441497
pa.array(

0 commit comments

Comments
 (0)