Skip to content

Commit 104f068

Browse files
vuanhphungIsaac
andcommitted
perf(telemetry): serialize telemetry events without dataclasses.asdict
Every statement exports a telemetry event, and asdict's recursive deep copies made to_json about 64 us per event; the thread that fills a batch also serializes all 100 events inline. A field-name walk that drops None fields gives byte-identical JSON in about 19 us. Tests compare the output byte for byte with the previous asdict-based serializer over every event the client exports, the flush payload, all model types, and seeded random events with non-ASCII strings and floats. Signed-off-by: Vu Anh Phung <vu.phung@databricks.com> Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 8d6db43 commit 104f068

3 files changed

Lines changed: 361 additions & 8 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Release History
22

33
# Unreleased
4+
- Serialize telemetry events about 3x faster with byte-identical JSON, which lowers per-statement CPU and raises small-query throughput for multi-threaded clients by about 5-7% when telemetry is enabled.
45
- Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629))
56
- Speed up Thrift response handling: read replies in 1 MiB chunks and decode them with Thrift's C-accelerated protocol, and stop stringifying every request/response for the default no-op debug logger. Wide result schemas, metadata calls with many rows, and inline Arrow results benefit most.
67

‎src/databricks/sql/telemetry/utils.py‎

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,40 @@
11
import json
22
from enum import Enum
3-
from dataclasses import asdict, is_dataclass
3+
from dataclasses import fields, is_dataclass
44
from abc import ABC, abstractmethod
55
import logging
6+
from typing import Any, Dict, Tuple
67

78
logger = logging.getLogger(__name__)
89

10+
_LEAF_TYPES = frozenset((str, int, float, bool))
11+
_FIELD_NAMES: Dict[type, Tuple[str, ...]] = {}
12+
13+
14+
def _to_json_dict(obj: Any) -> Any:
15+
"""``dataclasses.asdict`` minus None fields, without asdict's deep copies.
16+
17+
Telemetry serializes one event per statement, and asdict was most of that cost.
18+
"""
19+
cls = type(obj)
20+
names = _FIELD_NAMES.get(cls)
21+
if names is None:
22+
if not is_dataclass(cls):
23+
if isinstance(obj, Enum):
24+
return obj.value
25+
if isinstance(obj, (list, tuple)):
26+
return [_to_json_dict(v) for v in obj]
27+
if isinstance(obj, dict):
28+
return {k: _to_json_dict(v) for k, v in obj.items()}
29+
return obj
30+
names = _FIELD_NAMES[cls] = tuple(f.name for f in fields(cls))
31+
result: Dict[str, Any] = {}
32+
for name in names:
33+
value = getattr(obj, name)
34+
if value is not None:
35+
result[name] = value if type(value) in _LEAF_TYPES else _to_json_dict(value)
36+
return result
37+
938

1039
class BaseTelemetryClient(ABC):
1140
"""
@@ -47,13 +76,7 @@ def to_json(self) -> str:
4776
f"{self.__class__.__name__} must be a dataclass to use JsonSerializableMixin"
4877
)
4978

50-
return json.dumps(
51-
asdict(
52-
self,
53-
dict_factory=lambda data: {k: v for k, v in data if v is not None},
54-
),
55-
cls=EnumEncoder,
56-
)
79+
return _ENUM_ENCODER.encode(_to_json_dict(self))
5780

5881

5982
class EnumEncoder(json.JSONEncoder):
@@ -67,3 +90,7 @@ def default(self, obj):
6790
if isinstance(obj, Enum):
6891
return obj.value
6992
return super().default(obj)
93+
94+
95+
# Same as json.dumps(..., cls=EnumEncoder) without building an encoder per call.
96+
_ENUM_ENCODER = EnumEncoder()
Lines changed: 325 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,325 @@
1+
"""Telemetry JSON must be byte-for-byte what the asdict-based serializer produced."""
2+
3+
import itertools
4+
import json
5+
import random
6+
from dataclasses import asdict
7+
from enum import Enum, IntEnum
8+
from unittest.mock import MagicMock, patch
9+
10+
import pytest
11+
12+
from databricks.sql.auth.authenticators import AccessTokenAuthProvider
13+
from databricks.sql.telemetry.models.endpoint_models import (
14+
TelemetryRequest,
15+
TelemetryResponse,
16+
)
17+
from databricks.sql.telemetry.models.enums import (
18+
AuthFlow,
19+
AuthMech,
20+
DatabricksClientType,
21+
DriverVolumeOperationType,
22+
ExecutionResultFormat,
23+
StatementType,
24+
)
25+
from databricks.sql.telemetry.models.event import (
26+
ChunkDetails,
27+
DriverConnectionParameters,
28+
DriverErrorInfo,
29+
DriverSystemConfiguration,
30+
DriverVolumeOperation,
31+
HostDetails,
32+
OperationDetail,
33+
ResultLatency,
34+
SqlExecutionEvent,
35+
TelemetryEvent,
36+
)
37+
from databricks.sql.telemetry.models.frontend_logs import (
38+
FrontendLogContext,
39+
FrontendLogEntry,
40+
TelemetryClientContext,
41+
TelemetryFrontendLog,
42+
)
43+
from databricks.sql.telemetry.telemetry_client import (
44+
TelemetryClient,
45+
TelemetryClientFactory,
46+
TelemetryHelper,
47+
)
48+
from databricks.sql.telemetry.utils import EnumEncoder
49+
50+
51+
def main_to_json(obj):
52+
"""JsonSerializableMixin.to_json as it was before this change, kept as the oracle."""
53+
return json.dumps(
54+
asdict(obj, dict_factory=lambda data: {k: v for k, v in data if v is not None}),
55+
cls=EnumEncoder,
56+
)
57+
58+
59+
class _StrEnum(str, Enum):
60+
A = "str-mixin"
61+
62+
63+
class _IntEnum(IntEnum):
64+
ONE = 1
65+
66+
67+
TEXTS = [
68+
"",
69+
"plain",
70+
"é ü ß ñ",
71+
"日本語のテキスト",
72+
"emoji \U0001f600 \U0001f9ea",
73+
'quote " backslash \\ slash /',
74+
"control \n\t\r\x00\x1f\x7f",
75+
"lone surrogate \ud800",
76+
]
77+
FLOATS = [0.0, -0.0, 0.1, 1.5, 2.5e-5, 5e-324, 1e300, float("inf"), float("-inf")]
78+
FLOATS += [float("nan")]
79+
INTS = [0, -1, 7, 2**31, 2**63 + 1]
80+
SCALARS = TEXTS + FLOATS + INTS + [True, False]
81+
82+
83+
def _params(**overrides):
84+
values = dict(
85+
http_path="/sql/1.0/warehouses/abc",
86+
mode=DatabricksClientType.THRIFT,
87+
host_info=HostDetails(host_url="adb-1.azuredatabricks.net", port=443),
88+
)
89+
values.update(overrides)
90+
return DriverConnectionParameters(**values)
91+
92+
93+
FULL_PARAMS = _params(
94+
mode=DatabricksClientType.SEA,
95+
auth_mech=AuthMech.OAUTH,
96+
auth_flow=AuthFlow.CLIENT_CREDENTIALS,
97+
socket_timeout=900.5,
98+
azure_workspace_resource_id="/subscriptions/é/resourceGroups/日本",
99+
azure_tenant_id="",
100+
use_proxy=True,
101+
use_system_proxy=False,
102+
proxy_host_info=HostDetails(host_url="proxy.example.com", port=8080),
103+
use_cf_proxy=False,
104+
cf_proxy_host_info=HostDetails(host_url="cf.example.com", port=0),
105+
non_proxy_hosts=["localhost", None, "127.0.0.1"],
106+
allow_self_signed_support=False,
107+
use_system_trust_store=True,
108+
enable_arrow=True,
109+
enable_direct_results=True,
110+
enable_sea_hybrid_results=False,
111+
http_connection_pool_size=10,
112+
rows_fetched_per_block=100000,
113+
async_poll_interval_millis=2000,
114+
support_many_parameters=True,
115+
enable_complex_datatype_support=False,
116+
allowed_volume_ingestion_paths=["/tmp/a", "/Volumes/é"],
117+
query_tags='{"team": "data-ēng"}',
118+
)
119+
120+
FULL_SQL_EVENT = SqlExecutionEvent(
121+
statement_type=StatementType.METADATA,
122+
is_compressed=False,
123+
execution_result=ExecutionResultFormat.EXTERNAL_LINKS,
124+
retry_count=0,
125+
chunk_id=3,
126+
chunk_details=ChunkDetails(12, 340, 5, 4, 0),
127+
result_latency=ResultLatency(10, None),
128+
operation_detail=OperationDetail(0, 1, "EXECUTE_STATEMENT", False),
129+
)
130+
131+
132+
def _model_cases():
133+
sys_config = TelemetryHelper.get_driver_system_configuration()
134+
event = TelemetryEvent(
135+
system_configuration=sys_config,
136+
driver_connection_params=FULL_PARAMS,
137+
session_id="01f0-session",
138+
sql_statement_id="01f0-statement",
139+
auth_type="pat ☃",
140+
vol_operation=DriverVolumeOperation(DriverVolumeOperationType.PUT, "/V/ü"),
141+
sql_operation=FULL_SQL_EVENT,
142+
error_info=DriverErrorInfo("Error", 'Traceback\n "quoted" é ☃'),
143+
operation_latency_ms=0,
144+
)
145+
log = TelemetryFrontendLog(
146+
frontend_log_event_id="5f0c-event",
147+
context=FrontendLogContext(
148+
client_context=TelemetryClientContext(
149+
timestamp_millis=1794470000000, user_agent="PyDatabricksSqlConnector/é"
150+
)
151+
),
152+
entry=FrontendLogEntry(sql_driver_log=event),
153+
workspace_id=6436897454825492,
154+
)
155+
cases = {
156+
"host": HostDetails(host_url="h", port=1),
157+
"params_minimal": _params(),
158+
"params_full": FULL_PARAMS,
159+
"system_config": sys_config,
160+
"system_config_exotic": DriverSystemConfiguration(
161+
*(TEXTS * 2)[1:10], client_app_name=None, locale_name="ja_JP"
162+
),
163+
"error_info": DriverErrorInfo("", ""),
164+
"chunk_details_empty": ChunkDetails(),
165+
"result_latency_empty": ResultLatency(),
166+
"operation_detail_empty": OperationDetail(),
167+
"sql_event_full": FULL_SQL_EVENT,
168+
"telemetry_event_full": event,
169+
"telemetry_event_minimal": TelemetryEvent(
170+
system_configuration=sys_config, driver_connection_params=_params()
171+
),
172+
"frontend_log": log,
173+
"request": TelemetryRequest(
174+
uploadTime=1, items=["x", "ü"], protoLogs=[main_to_json(log)] * 3
175+
),
176+
"request_no_logs": TelemetryRequest(uploadTime=0, items=[], protoLogs=None),
177+
"response": TelemetryResponse(
178+
errors=["é"], numSuccess=1, numProtoSuccess=2, numRealtimeSuccess=0
179+
),
180+
"containers": _params(
181+
non_proxy_hosts=[
182+
HostDetails(host_url="n", port=2),
183+
AuthMech.PAT,
184+
{"k": AuthFlow.TOKEN_PASSTHROUGH, "none": None, 1: "int key"},
185+
(1, 2.5, None),
186+
[[], {}],
187+
_StrEnum.A,
188+
_IntEnum.ONE,
189+
]
190+
),
191+
}
192+
for vol_type in DriverVolumeOperationType:
193+
cases[f"volume_{vol_type.value}"] = DriverVolumeOperation(vol_type, "/V")
194+
for i, value in enumerate(SCALARS):
195+
cases[f"scalar_{i}"] = _params(
196+
socket_timeout=value, query_tags=value, http_path=value
197+
)
198+
return cases
199+
200+
201+
MODEL_CASES = _model_cases()
202+
203+
204+
@pytest.mark.parametrize("name", list(MODEL_CASES))
205+
def test_models_match_main_serializer(name):
206+
obj = MODEL_CASES[name]
207+
assert obj.to_json() == main_to_json(obj)
208+
209+
210+
def test_unserializable_values_fail_like_main():
211+
for bad in ({1, 2}, object(), {AuthMech.PAT: "enum key"}):
212+
obj = _params(non_proxy_hosts=[bad])
213+
with pytest.raises(TypeError):
214+
main_to_json(obj)
215+
with pytest.raises(TypeError):
216+
obj.to_json()
217+
218+
219+
@pytest.fixture
220+
def client():
221+
executor = MagicMock()
222+
with patch(
223+
"databricks.sql.common.unified_http_client.UnifiedHttpClient._setup_pool_managers"
224+
):
225+
return TelemetryClient(
226+
telemetry_enabled=True,
227+
session_id_hex="01f0-session",
228+
auth_provider=AccessTokenAuthProvider("token"),
229+
host_url="adb-1.azuredatabricks.net",
230+
executor=executor,
231+
batch_size=TelemetryClientFactory.DEFAULT_BATCH_SIZE,
232+
client_context=MagicMock(),
233+
)
234+
235+
236+
def test_client_events_and_flush_payload_match_main_serializer(client):
237+
events = []
238+
with patch.object(client, "_export_event", side_effect=events.append):
239+
client.export_initial_telemetry_log(_params(), "PyDatabricksSqlConnector/4.9")
240+
client.export_initial_telemetry_log(FULL_PARAMS, "agent é ☃", "01f0-other")
241+
client.export_failure_log("OperationalError", "Traceback\n é \U0001f600")
242+
for statement_type, result_format in itertools.product(
243+
StatementType, ExecutionResultFormat
244+
):
245+
client.export_latency_log(
246+
latency_ms=52,
247+
sql_execution_event=SqlExecutionEvent(
248+
statement_type=statement_type,
249+
is_compressed=True,
250+
execution_result=result_format,
251+
retry_count=1,
252+
chunk_id=None,
253+
),
254+
sql_statement_id="01f0-statement",
255+
)
256+
client.export_latency_log(0, FULL_SQL_EVENT, None, session_id="01f0-x")
257+
assert len(events) == 4 + len(StatementType) * len(ExecutionResultFormat)
258+
for event in events:
259+
assert event.to_json() == main_to_json(event)
260+
261+
with patch(
262+
"databricks.sql.telemetry.telemetry_client.time.time", return_value=1794.47
263+
):
264+
client._send_telemetry(events)
265+
sent = client._executor.submit.call_args.kwargs["data"]
266+
expected = TelemetryRequest(
267+
uploadTime=1794470,
268+
items=[],
269+
protoLogs=[main_to_json(event) for event in events],
270+
)
271+
assert sent == main_to_json(expected)
272+
273+
274+
def test_randomized_events_match_main_serializer():
275+
rnd = random.Random(1037)
276+
277+
def maybe(value):
278+
return None if rnd.random() < 0.3 else value
279+
280+
def scalar():
281+
return rnd.choice(SCALARS)
282+
283+
for _ in range(300):
284+
params = _params(
285+
mode=rnd.choice(list(DatabricksClientType)),
286+
host_info=HostDetails(host_url=rnd.choice(TEXTS), port=rnd.choice(INTS)),
287+
auth_mech=maybe(rnd.choice(list(AuthMech))),
288+
auth_flow=maybe(rnd.choice(list(AuthFlow))),
289+
socket_timeout=maybe(scalar()),
290+
proxy_host_info=maybe(HostDetails(host_url=scalar(), port=scalar())),
291+
non_proxy_hosts=maybe([maybe(scalar()) for _ in range(rnd.randint(0, 3))]),
292+
use_proxy=maybe(rnd.random() < 0.5),
293+
query_tags=maybe(scalar()),
294+
)
295+
sql_event = SqlExecutionEvent(
296+
statement_type=rnd.choice(list(StatementType)),
297+
is_compressed=rnd.random() < 0.5,
298+
execution_result=rnd.choice(list(ExecutionResultFormat)),
299+
retry_count=maybe(scalar()),
300+
chunk_id=maybe(scalar()),
301+
chunk_details=maybe(ChunkDetails(*(maybe(scalar()) for _ in range(5)))),
302+
result_latency=maybe(ResultLatency(maybe(scalar()), maybe(scalar()))),
303+
operation_detail=maybe(
304+
OperationDetail(maybe(scalar()), None, maybe(scalar()), maybe(True))
305+
),
306+
)
307+
log = TelemetryFrontendLog(
308+
frontend_log_event_id=scalar(),
309+
context=FrontendLogContext(
310+
client_context=TelemetryClientContext(scalar(), scalar())
311+
),
312+
entry=FrontendLogEntry(
313+
sql_driver_log=TelemetryEvent(
314+
system_configuration=TelemetryHelper.get_driver_system_configuration(),
315+
driver_connection_params=params,
316+
session_id=maybe(scalar()),
317+
sql_statement_id=maybe(scalar()),
318+
sql_operation=maybe(sql_event),
319+
error_info=maybe(DriverErrorInfo(scalar(), scalar())),
320+
operation_latency_ms=maybe(scalar()),
321+
)
322+
),
323+
workspace_id=maybe(scalar()),
324+
)
325+
assert log.to_json() == main_to_json(log)

0 commit comments

Comments
 (0)