Skip to content

Commit bec22d8

Browse files
vuanhphungIsaac
andcommitted
fix: stop a connection from keeping every cursor it created
Connection._cursors held every cursor ever created, so a long-lived connection grew by one closed cursor per query and close() walked all of them. Cursor.close() now removes its entry; Connection.close() still closes every cursor that is open. Signed-off-by: Vu Anh Phung <vu.phung@databricks.com> Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 0c1b9e8 commit bec22d8

3 files changed

Lines changed: 40 additions & 3 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
- Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads.
55
- Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629))
66
- Speed up Thrift response handling: read replies in 1 MiB chunks and decode them with Thrift's C-accelerated protocol, and stop stringifying every request/response for the default no-op debug logger. Wide result schemas, metadata calls with many rows, and inline Arrow results benefit most.
7+
- Fix a connection keeping every cursor it ever created, so memory no longer grows with each query on a long-lived connection and `close()` no longer slows down with the number of queries run.
78

89
# 4.9.0 (2026-11-04)
910
- Upgrade Databricks SQL Kernel to 1.4.0, which keeps Lakehouse Real-Time session affinity and flushes shared telemetry only after the final session closes.

‎src/databricks/sql/client.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -407,7 +407,9 @@ def read(self) -> Optional[OAuthToken]:
407407
self.disable_pandas = kwargs.get("_disable_pandas", False)
408408
self.lz4_compression = kwargs.get("enable_query_result_lz4_compression", True)
409409
self.use_cloud_fetch = kwargs.get("use_cloud_fetch", True)
410-
self._cursors = [] # type: List[Cursor]
410+
# Open cursors, in creation order. Cursor.close() removes its entry, so a
411+
# long-lived connection does not keep every cursor it ever created.
412+
self._cursors: Dict[Cursor, None] = {}
411413
self.telemetry_batch_size = kwargs.get(
412414
"telemetry_batch_size", TelemetryClientFactory.DEFAULT_BATCH_SIZE
413415
)
@@ -755,7 +757,7 @@ def cursor(
755757
result_buffer_size_bytes=buffer_size_bytes,
756758
row_limit=row_limit,
757759
)
758-
self._cursors.append(cursor)
760+
self._cursors[cursor] = None
759761
return cursor
760762

761763
def close(self) -> None:
@@ -764,7 +766,7 @@ def close(self) -> None:
764766

765767
def _close(self, close_cursors=True) -> None:
766768
if close_cursors:
767-
for cursor in self._cursors:
769+
for cursor in list(self._cursors):
768770
cursor.close()
769771

770772
try:
@@ -1978,6 +1980,7 @@ def cancel(self) -> None:
19781980

19791981
def close(self) -> None:
19801982
"""Close cursor"""
1983+
self.connection._cursors.pop(self, None)
19811984
self.open = False
19821985
self.active_command_id = None
19831986
if self.active_result_set:

‎tests/unit/test_client.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import itertools
77
from decimal import Decimal
88
from datetime import datetime, date
9+
from types import SimpleNamespace
910
from uuid import UUID
1011

1112
from databricks.sql.thrift_api.TCLIService.ttypes import (
@@ -171,6 +172,38 @@ def test_cant_open_cursor_on_closed_connection(self, mock_client_class):
171172
connection.cursor()
172173
self.assertIn("closed", str(cm.exception))
173174

175+
def test_connection_tracks_only_open_cursors(self):
176+
thrift = patch("%s.session.ThriftDatabricksClient" % self.PACKAGE_NAME)
177+
# session.py imports the kernel client lazily; a fake module keeps the
178+
# test independent of the kernel wheel.
179+
kernel = patch.dict(
180+
sys.modules,
181+
{
182+
"%s.backend.kernel.client"
183+
% self.PACKAGE_NAME: SimpleNamespace(KernelDatabricksClient=MagicMock())
184+
},
185+
)
186+
for backend_patch, extra in ((thrift, {}), (kernel, {"use_kernel": True})):
187+
with self.subTest(**extra), backend_patch:
188+
connection = databricks.sql.connect(
189+
**self.DUMMY_CONNECTION_ARGS, **extra
190+
)
191+
for _ in range(3):
192+
with connection.cursor():
193+
pass
194+
open_cursors = [connection.cursor(), connection.cursor()]
195+
result_set = Mock()
196+
open_cursors[0].active_result_set = result_set
197+
closed_twice = connection.cursor()
198+
closed_twice.close()
199+
closed_twice.close()
200+
self.assertEqual(list(connection._cursors), open_cursors)
201+
202+
connection.close()
203+
self.assertFalse(any(cursor.open for cursor in open_cursors))
204+
result_set.close.assert_called_once()
205+
self.assertEqual(list(connection._cursors), [])
206+
174207
@patch("%s.session.ThriftDatabricksClient" % PACKAGE_NAME)
175208
@patch("%s.client.Cursor" % PACKAGE_NAME)
176209
def test_arraysize_buffer_size_passthrough(

0 commit comments

Comments
 (0)