Skip to content

Commit f4839f1

Browse files
vuanhphungIsaac
andcommitted
fix: keep the names modules no longer import at load importable
authenticators, auth, telemetry_client, session and the cloudfetch downloader stopped importing OAuthManager, ClientCredentialsTokenSource, RefreshableTokenSource, TokenFederationProvider, UnifiedHttpClient and the telemetry push clients at module load. Downstream imports or patch()es of those module attributes would then fail. Each module now resolves them on first access (PEP 562), as client.py already does, and the places that use them read the module attribute, so a patch there still takes effect. Co-authored-by: Isaac <no-reply@databricks.com> Signed-off-by: Vu Anh Phung <vu.phung@databricks.com>
1 parent 8320850 commit f4839f1

6 files changed

Lines changed: 181 additions & 21 deletions

File tree

‎src/databricks/sql/auth/auth.py‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from typing import Optional, List
1+
from typing import Any, Optional, List, TYPE_CHECKING
22

33
from databricks.sql.auth.authenticators import (
44
AuthProvider,
@@ -9,6 +9,9 @@
99
)
1010
from databricks.sql.auth.common import AuthType, ClientContext
1111

12+
if TYPE_CHECKING:
13+
from databricks.sql.auth.token_federation import TokenFederationProvider
14+
1215

1316
def get_auth_provider(cfg: ClientContext, http_client):
1417
# Determine the base auth provider
@@ -66,9 +69,9 @@ def get_auth_provider(cfg: ClientContext, http_client):
6669

6770
# Always wrap with token federation (falls back gracefully if not needed)
6871
if base_provider:
69-
from databricks.sql.auth.token_federation import TokenFederationProvider
72+
import databricks.sql.auth.auth as _module
7073

71-
return TokenFederationProvider(
74+
return _module.TokenFederationProvider(
7275
hostname=cfg.hostname,
7376
external_provider=base_provider,
7477
http_client=http_client,
@@ -135,3 +138,13 @@ def get_python_sql_connector_auth_provider(hostname: str, http_client, **kwargs)
135138
identity_federation_client_id=kwargs.get("identity_federation_client_id"),
136139
)
137140
return get_auth_provider(cfg, http_client)
141+
142+
143+
def __getattr__(name: str) -> Any:
144+
# Imported on first use rather than at load, so connect() skips them;
145+
# still importable, and patchable here, as module attributes.
146+
if name in ("TokenFederationProvider",):
147+
import databricks.sql.auth.token_federation as _source
148+
149+
return getattr(_source, name)
150+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

‎src/databricks/sql/auth/authenticators.py‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
import abc
22
import logging
3-
from typing import Callable, Dict, List, TYPE_CHECKING
3+
from typing import Any, Callable, Dict, List, TYPE_CHECKING
44
from databricks.sql.common.http import HttpHeader
55
from databricks.sql.auth.endpoint import get_oauth_endpoints
66

77
if TYPE_CHECKING:
8-
from databricks.sql.auth.oauth import RefreshableTokenSource
8+
from databricks.sql.auth.oauth import (
9+
ClientCredentialsTokenSource,
10+
OAuthManager,
11+
RefreshableTokenSource,
12+
)
913
from databricks.sql.auth.common import (
1014
AuthType,
1115
get_effective_azure_login_app_id,
@@ -72,9 +76,9 @@ def __init__(
7276
# Convert to the corresponding scopes in the corresponding IdP
7377
cloud_scopes = idp_endpoint.get_scopes_mapping(scopes)
7478

75-
from databricks.sql.auth.oauth import OAuthManager
79+
import databricks.sql.auth.authenticators as _module
7680

77-
self.oauth_manager = OAuthManager(
81+
self.oauth_manager = _module.OAuthManager(
7882
port_range=redirect_port_range,
7983
client_id=client_id,
8084
idp_endpoint=idp_endpoint,
@@ -205,9 +209,9 @@ def auth_type(self) -> str:
205209
return AuthType.AZURE_SP_M2M.value
206210

207211
def get_token_source(self, resource: str) -> "RefreshableTokenSource":
208-
from databricks.sql.auth.oauth import ClientCredentialsTokenSource
212+
import databricks.sql.auth.authenticators as _module
209213

210-
return ClientCredentialsTokenSource(
214+
return _module.ClientCredentialsTokenSource(
211215
token_url=f"{self.AZURE_AAD_ENDPOINT}/{self.azure_tenant_id}/{self.AZURE_TOKEN_ENDPOINT}",
212216
client_id=self.azure_client_id,
213217
client_secret=self.azure_client_secret,
@@ -238,3 +242,17 @@ def header_factory() -> Dict[str, str]:
238242
return headers
239243

240244
return header_factory
245+
246+
247+
def __getattr__(name: str) -> Any:
248+
# Imported on first use rather than at load, so connect() skips them;
249+
# still importable, and patchable here, as module attributes.
250+
if name in (
251+
"OAuthManager",
252+
"RefreshableTokenSource",
253+
"ClientCredentialsTokenSource",
254+
):
255+
import databricks.sql.auth.oauth as _source
256+
257+
return getattr(_source, name)
258+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

‎src/databricks/sql/cloudfetch/downloader.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import logging
44
from dataclasses import dataclass
5-
from typing import Optional, TYPE_CHECKING
5+
from typing import Any, Optional, TYPE_CHECKING
66

77
import lz4.frame
88
import time
@@ -23,6 +23,16 @@
2323
logger = logging.getLogger(__name__)
2424

2525

26+
def __getattr__(name: str) -> Any:
27+
# Imported on first use rather than at load, so connect() skips them;
28+
# still importable, and patchable here, as module attributes.
29+
if name in ("UnifiedHttpClient",):
30+
import databricks.sql.common.unified_http_client as _source
31+
32+
return getattr(_source, name)
33+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
34+
35+
2636
@dataclass
2737
class DownloadedFile:
2838
"""

‎src/databricks/sql/session.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ def __getattr__(name: str) -> Any:
4444
from databricks.sql.backend.sea.backend import SeaDatabricksClient
4545

4646
return SeaDatabricksClient
47+
if name == "UnifiedHttpClient":
48+
# Same pattern, so connect() doesn't import urllib3.
49+
from databricks.sql.common.unified_http_client import UnifiedHttpClient
50+
51+
return UnifiedHttpClient
4752
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
4853

4954

‎src/databricks/sql/telemetry/telemetry_client.py‎

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import importlib
12
import threading
23
import time
34
import logging
@@ -46,11 +47,34 @@
4647

4748
if TYPE_CHECKING:
4849
from databricks.sql.client import Connection
49-
from databricks.sql.telemetry.telemetry_push_client import ITelemetryPushClient
50+
from databricks.sql.common.unified_http_client import UnifiedHttpClient
51+
from databricks.sql.telemetry.telemetry_push_client import (
52+
CircuitBreakerTelemetryPushClient,
53+
ITelemetryPushClient,
54+
TelemetryPushClient,
55+
)
5056

5157
logger = logging.getLogger(__name__)
5258

5359

60+
_LAZY_NAMES = {
61+
"UnifiedHttpClient": "databricks.sql.common.unified_http_client",
62+
"TelemetryPushClient": "databricks.sql.telemetry.telemetry_push_client",
63+
"CircuitBreakerTelemetryPushClient": "databricks.sql.telemetry.telemetry_push_client",
64+
"ITelemetryPushClient": "databricks.sql.telemetry.telemetry_push_client",
65+
"TokenFederationProvider": "databricks.sql.auth.token_federation",
66+
}
67+
68+
69+
def __getattr__(name: str) -> Any:
70+
# Imported on first use rather than at load, so connect() skips them;
71+
# still importable, and patchable here, as module attributes.
72+
source = _LAZY_NAMES.get(name)
73+
if source is None:
74+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
75+
return getattr(importlib.import_module(source), name)
76+
77+
5478
class TelemetryHelper:
5579
"""Helper class for getting telemetry related information."""
5680

@@ -208,30 +232,26 @@ def __init__(
208232
self._host_url = host_url
209233
self._executor = executor
210234

211-
# Imported here, not at module load, so connect() paths that never
235+
# Resolved here, not at module load, so connect() paths that never
212236
# export telemetry don't import urllib3 and pybreaker.
213-
from databricks.sql.common.unified_http_client import UnifiedHttpClient
214-
from databricks.sql.telemetry.telemetry_push_client import (
215-
CircuitBreakerTelemetryPushClient,
216-
TelemetryPushClient,
217-
)
237+
import databricks.sql.telemetry.telemetry_client as _module
218238

219239
# Create own HTTP client from client context
220-
self._http_client = UnifiedHttpClient(client_context)
240+
self._http_client = _module.UnifiedHttpClient(client_context)
221241

222242
# Create telemetry push client based on circuit breaker enabled flag
223243
if client_context.telemetry_circuit_breaker_enabled:
224244
# Create circuit breaker telemetry push client
225245
# (circuit breakers created on-demand)
226246
self._telemetry_push_client: "ITelemetryPushClient" = (
227-
CircuitBreakerTelemetryPushClient(
228-
TelemetryPushClient(self._http_client),
247+
_module.CircuitBreakerTelemetryPushClient(
248+
_module.TelemetryPushClient(self._http_client),
229249
host_url,
230250
)
231251
)
232252
else:
233253
# Circuit breaker disabled - use direct telemetry push client
234-
self._telemetry_push_client = TelemetryPushClient(self._http_client)
254+
self._telemetry_push_client = _module.TelemetryPushClient(self._http_client)
235255

236256
def _export_event(self, event):
237257
"""Add an event to the batch queue and flush if batch is full"""

‎tests/unit/test_lazy_http_import.py‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,100 @@ def test_oauth_provider_still_builds_its_manager():
8686
manager.assert_called_once()
8787

8888

89+
# Names these modules used to import at load, now resolved on first access.
90+
MOVED_NAMES = [
91+
("databricks.sql.auth.authenticators", "OAuthManager", "databricks.sql.auth.oauth"),
92+
(
93+
"databricks.sql.auth.authenticators",
94+
"RefreshableTokenSource",
95+
"databricks.sql.auth.oauth",
96+
),
97+
(
98+
"databricks.sql.auth.authenticators",
99+
"ClientCredentialsTokenSource",
100+
"databricks.sql.auth.oauth",
101+
),
102+
(
103+
"databricks.sql.auth.auth",
104+
"TokenFederationProvider",
105+
"databricks.sql.auth.token_federation",
106+
),
107+
(
108+
"databricks.sql.telemetry.telemetry_client",
109+
"UnifiedHttpClient",
110+
"databricks.sql.common.unified_http_client",
111+
),
112+
(
113+
"databricks.sql.telemetry.telemetry_client",
114+
"TelemetryPushClient",
115+
"databricks.sql.telemetry.telemetry_push_client",
116+
),
117+
(
118+
"databricks.sql.telemetry.telemetry_client",
119+
"CircuitBreakerTelemetryPushClient",
120+
"databricks.sql.telemetry.telemetry_push_client",
121+
),
122+
(
123+
"databricks.sql.telemetry.telemetry_client",
124+
"TokenFederationProvider",
125+
"databricks.sql.auth.token_federation",
126+
),
127+
(
128+
"databricks.sql.session",
129+
"UnifiedHttpClient",
130+
"databricks.sql.common.unified_http_client",
131+
),
132+
(
133+
"databricks.sql.cloudfetch.downloader",
134+
"UnifiedHttpClient",
135+
"databricks.sql.common.unified_http_client",
136+
),
137+
]
138+
139+
140+
@pytest.mark.parametrize("module_name, name, source", MOVED_NAMES)
141+
def test_moved_names_stay_importable(module_name, name, source):
142+
import importlib
143+
144+
module = importlib.import_module(module_name)
145+
assert getattr(module, name) is getattr(importlib.import_module(source), name)
146+
147+
148+
def test_patching_a_moved_name_reaches_where_it_is_used():
149+
from unittest.mock import MagicMock, patch
150+
151+
from databricks.sql.auth.authenticators import DatabricksOAuthProvider
152+
from databricks.sql.telemetry.telemetry_client import TelemetryClient
153+
154+
with patch(
155+
"databricks.sql.auth.authenticators.OAuthManager"
156+
) as manager, patch.object(DatabricksOAuthProvider, "_initial_get_token"):
157+
DatabricksOAuthProvider(
158+
hostname="example.cloud.databricks.com",
159+
oauth_persistence=None,
160+
redirect_port_range=[8020],
161+
client_id="client",
162+
scopes=["sql"],
163+
http_client=MagicMock(),
164+
)
165+
manager.assert_called_once()
166+
167+
context = MagicMock(telemetry_circuit_breaker_enabled=False)
168+
with patch(
169+
"databricks.sql.telemetry.telemetry_client.UnifiedHttpClient"
170+
) as http_client:
171+
TelemetryClient(
172+
telemetry_enabled=True,
173+
session_id_hex="01ef",
174+
auth_provider=None,
175+
host_url="example.cloud.databricks.com",
176+
executor=MagicMock(),
177+
batch_size=10,
178+
client_context=context,
179+
)
180+
http_client.assert_called_once_with(context)
181+
182+
89183
def test_token_federation_check_does_not_import_it():
90184
script = """
91185
import sys

0 commit comments

Comments
 (0)