Skip to content

Commit 508709b

Browse files
vuanhphungIsaac
andcommitted
perf(kernel): skip the Python telemetry client on use_kernel connections
The kernel exports telemetry for use_kernel connections, but every kernel connect still registered a no-op client with TelemetryClientFactory under its global lock (and the first one started the flush thread). In a 200-thread cold burst that lock convoys behind the GIL: initialize_telemetry_client took p50 692 ms / max 1.7 s, and connect p50 was ~3.0 s vs ~1.7 s without it. Kernel connections now use a NoopTelemetryClient directly and skip the factory on close. This also stops a kernel connection opened first from leaving a no-op client registered for the host, which silently disabled Python telemetry for later Thrift connections to that host. Co-authored-by: Isaac <no-reply@databricks.com> Signed-off-by: Vu Anh Phung <vu.phung@databricks.com>
1 parent 0c1b9e8 commit 508709b

4 files changed

Lines changed: 82 additions & 8 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
- Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads.
55
- Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629))
66
- Speed up Thrift response handling: read replies in 1 MiB chunks and decode them with Thrift's C-accelerated protocol, and stop stringifying every request/response for the default no-op debug logger. Wide result schemas, metadata calls with many rows, and inline Arrow results benefit most.
7+
- Speed up concurrent `use_kernel=True` connects by skipping the Python telemetry client setup, which the kernel does not use; this also stops a kernel connection from disabling Python telemetry for later Thrift connections to the same host.
78

89
# 4.9.0 (2026-11-04)
910
- Upgrade Databricks SQL Kernel to 1.4.0, which keeps Lakehouse Real-Time session affinity and flushes shared telemetry only after the final session closes.

‎src/databricks/sql/client.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@
8888
TSparkParameter,
8989
)
9090
from databricks.sql.telemetry.telemetry_client import (
91+
NoopTelemetryClient,
9192
TelemetryHelper,
9293
TelemetryClientFactory,
9394
)
@@ -473,6 +474,13 @@ def read(self) -> Optional[OAuthToken]:
473474
self.enable_telemetry = kwargs.get("enable_telemetry", True)
474475
self.telemetry_enabled = TelemetryHelper.is_telemetry_enabled(self)
475476

477+
if self._kernel_owns_telemetry():
478+
# The kernel exports this connection's telemetry. Registering with the
479+
# Python factory would only serialize concurrent connects on its lock
480+
# and start an idle flush thread.
481+
self._telemetry_client = NoopTelemetryClient()
482+
return
483+
476484
TelemetryClientFactory.initialize_telemetry_client(
477485
telemetry_enabled=self.telemetry_enabled,
478486
session_id_hex=self.get_session_id_hex(),
@@ -543,6 +551,9 @@ def read(self) -> Optional[OAuthToken]:
543551
session_id=self.get_session_id_hex(),
544552
)
545553

554+
def _kernel_owns_telemetry(self) -> bool:
555+
return bool(getattr(self.session, "use_kernel", False))
556+
546557
def _open_session_with_reyden_fallback(
547558
self,
548559
server_hostname: str,
@@ -772,7 +783,8 @@ def _close(self, close_cursors=True) -> None:
772783
except Exception as e:
773784
logger.error(f"Attempt to close session raised a local exception: {e}")
774785

775-
TelemetryClientFactory.close(host_url=self.session.host)
786+
if not self._kernel_owns_telemetry():
787+
TelemetryClientFactory.close(host_url=self.session.host)
776788

777789
# Close HTTP client that was created by this connection
778790
if self.http_client:

‎tests/unit/test_session.py‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
OperationalError,
1717
)
1818
from databricks.sql.session import Session
19+
from databricks.sql.telemetry.telemetry_client import NoopTelemetryClient
1920

2021
import databricks.sql
2122

@@ -974,13 +975,9 @@ def test_kernel_http_client_lifecycle(self, mock_thrift, route):
974975
)
975976
assert conn.session.http_client is conn.http_client
976977
assert kernel.call_args.kwargs["http_client"] is conn.http_client
977-
params = telemetry.return_value.export_initial_telemetry_log.call_args.kwargs[
978-
"driver_connection_params"
979-
]
980-
assert params.use_proxy is (not explicit)
981-
assert params.use_system_proxy is (not explicit)
982-
if explicit:
983-
assert params.proxy_host_info is None
978+
# The kernel owns telemetry: no Python telemetry client is used.
979+
telemetry.assert_not_called()
980+
assert isinstance(conn._telemetry_client, NoopTelemetryClient)
984981
with conn.cursor() as cursor:
985982
cursor.execute("SELECT 1")
986983
assert cursor.fetchall() == [(1,)]

‎tests/unit/test_telemetry.py‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -530,6 +530,70 @@ def test_connection_failure_does_not_send_telemetry_for_kernel(
530530

531531
mock_export_failure_log.assert_not_called()
532532

533+
@staticmethod
534+
def _kernel_connection(mock_session):
535+
session = mock_session.return_value
536+
session.use_kernel = True
537+
session.host = "test-host"
538+
return sql.connect(
539+
server_hostname="test-host", http_path="/test-path", use_kernel=True
540+
)
541+
542+
@staticmethod
543+
def _init_python_client(host_url):
544+
with patch(
545+
"databricks.sql.common.unified_http_client.UnifiedHttpClient._setup_pool_managers"
546+
):
547+
TelemetryClientFactory.initialize_telemetry_client(
548+
telemetry_enabled=True,
549+
session_id_hex="thrift-session",
550+
auth_provider=AccessTokenAuthProvider("token"),
551+
host_url=host_url,
552+
batch_size=TelemetryClientFactory.DEFAULT_BATCH_SIZE,
553+
client_context=MagicMock(),
554+
)
555+
556+
@patch("databricks.sql.client.Session")
557+
def test_kernel_connection_skips_python_telemetry_pipeline(self, mock_session):
558+
"""The kernel owns telemetry for use_kernel connections, so connect and
559+
close must not start the Python factory (lock, flush thread, executor)."""
560+
conn = self._kernel_connection(mock_session)
561+
562+
assert isinstance(conn._telemetry_client, NoopTelemetryClient)
563+
assert TelemetryClientFactory._clients == {}
564+
assert TelemetryClientFactory._initialized is False
565+
566+
conn.close()
567+
assert TelemetryClientFactory._clients == {}
568+
assert TelemetryClientFactory._initialized is False
569+
570+
@patch("databricks.sql.client.Session")
571+
def test_kernel_connection_leaves_shared_host_client_alone(self, mock_session):
572+
"""A kernel connection must not take or drop a reference on the Python
573+
client that a Thrift connection to the same host is using."""
574+
self._init_python_client("test-host")
575+
holder = TelemetryClientFactory._clients["test-host"]
576+
577+
conn = self._kernel_connection(mock_session)
578+
assert holder.refcount == 1
579+
conn.close()
580+
581+
assert holder.refcount == 1
582+
assert TelemetryClientFactory._clients["test-host"] is holder
583+
584+
@patch("databricks.sql.client.Session")
585+
def test_thrift_telemetry_after_kernel_connection_is_not_noop(self, mock_session):
586+
"""A kernel connection opened first must not leave a no-op client
587+
registered for the host that later Thrift connections would reuse."""
588+
conn = self._kernel_connection(mock_session)
589+
self._init_python_client("test-host")
590+
591+
assert isinstance(
592+
TelemetryClientFactory.get_telemetry_client("test-host"), TelemetryClient
593+
)
594+
conn.close()
595+
TelemetryClientFactory.close(host_url="test-host")
596+
533597

534598
@patch("databricks.sql.client.Session")
535599
class TestTelemetryFeatureFlag:

0 commit comments

Comments
 (0)