perf: don't import the Python HTTP and OAuth client stack in connect()

CHANGELOG.md

@@ -4,6 +4,7 @@
 - Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads.
 - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629))
 - Speed up Thrift response handling: read replies in 1 MiB chunks and decode them with Thrift's C-accelerated protocol, and stop stringifying every request/response for the default no-op debug logger. Wide result schemas, metadata calls with many rows, and inline Arrow results benefit most.
+- Speed up the first `connect()` in a process by not importing `requests`, `urllib3`, the OAuth client libraries and the telemetry HTTP client until something uses them. Kernel connections never import them.
 
 # 4.9.0 (2026-11-04)
 - Upgrade Databricks SQL Kernel to 1.4.0, which keeps Lakehouse Real-Time session affinity and flushes shared telemetry only after the final session closes.

src/databricks/sql/auth/auth.py

@@ -1,4 +1,5 @@
-from typing import Optional, List
+import importlib
+from typing import Any, Optional, List, TYPE_CHECKING
 
 from databricks.sql.auth.authenticators import (
     AuthProvider,
@@ -8,7 +9,9 @@
     AzureServicePrincipalCredentialProvider,
 )
 from databricks.sql.auth.common import AuthType, ClientContext
-from databricks.sql.auth.token_federation import TokenFederationProvider
+
+if TYPE_CHECKING:
+    from databricks.sql.auth.token_federation import TokenFederationProvider
 
 
 def get_auth_provider(cfg: ClientContext, http_client):
@@ -67,7 +70,9 @@ def get_auth_provider(cfg: ClientContext, http_client):
 
     # Always wrap with token federation (falls back gracefully if not needed)
     if base_provider:
-        return TokenFederationProvider(
+        import databricks.sql.auth.auth as _module
+
+        return _module.TokenFederationProvider(
             hostname=cfg.hostname,
             external_provider=base_provider,
             http_client=http_client,
@@ -134,3 +139,17 @@ def get_python_sql_connector_auth_provider(hostname: str, http_client, **kwargs)
         identity_federation_client_id=kwargs.get("identity_federation_client_id"),
     )
     return get_auth_provider(cfg, http_client)
+
+
+_LAZY_NAMES = {
+    "TokenFederationProvider": "databricks.sql.auth.token_federation",
+}
+
+
+def __getattr__(name: str) -> Any:
+    # Imported on first use rather than at load, so connect() skips them;
+    # still importable, and patchable here, as module attributes.
+    source = _LAZY_NAMES.get(name)
+    if source is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    return getattr(importlib.import_module(source), name)

src/databricks/sql/auth/authenticators.py

@@ -1,13 +1,16 @@
+import importlib
 import abc
 import logging
-from typing import Callable, Dict, List
+from typing import Any, Callable, Dict, List, TYPE_CHECKING
 from databricks.sql.common.http import HttpHeader
-from databricks.sql.auth.oauth import (
-    OAuthManager,
-    RefreshableTokenSource,
-    ClientCredentialsTokenSource,
-)
 from databricks.sql.auth.endpoint import get_oauth_endpoints
+
+if TYPE_CHECKING:
+    from databricks.sql.auth.oauth import (
+        ClientCredentialsTokenSource,
+        OAuthManager,
+        RefreshableTokenSource,
+    )
 from databricks.sql.auth.common import (
     AuthType,
     get_effective_azure_login_app_id,
@@ -74,7 +77,9 @@ def __init__(
             # Convert to the corresponding scopes in the corresponding IdP
             cloud_scopes = idp_endpoint.get_scopes_mapping(scopes)
 
-            self.oauth_manager = OAuthManager(
+            import databricks.sql.auth.authenticators as _module
+
+            self.oauth_manager = _module.OAuthManager(
                 port_range=redirect_port_range,
                 client_id=client_id,
                 idp_endpoint=idp_endpoint,
@@ -204,8 +209,10 @@ def __init__(
     def auth_type(self) -> str:
         return AuthType.AZURE_SP_M2M.value
 
-    def get_token_source(self, resource: str) -> RefreshableTokenSource:
-        return ClientCredentialsTokenSource(
+    def get_token_source(self, resource: str) -> "RefreshableTokenSource":
+        import databricks.sql.auth.authenticators as _module
+
+        return _module.ClientCredentialsTokenSource(
             token_url=f"{self.AZURE_AAD_ENDPOINT}/{self.azure_tenant_id}/{self.AZURE_TOKEN_ENDPOINT}",
             client_id=self.azure_client_id,
             client_secret=self.azure_client_secret,
@@ -236,3 +243,19 @@ def header_factory() -> Dict[str, str]:
             return headers
 
         return header_factory
+
+
+_LAZY_NAMES = {
+    "OAuthManager": "databricks.sql.auth.oauth",
+    "RefreshableTokenSource": "databricks.sql.auth.oauth",
+    "ClientCredentialsTokenSource": "databricks.sql.auth.oauth",
+}
+
+
+def __getattr__(name: str) -> Any:
+    # Imported on first use rather than at load, so connect() skips them;
+    # still importable, and patchable here, as module attributes.
+    source = _LAZY_NAMES.get(name)
+    if source is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    return getattr(importlib.import_module(source), name)

src/databricks/sql/auth/common.py

@@ -1,13 +1,23 @@
 from enum import Enum
 import logging
+import sys
 from typing import Optional, List
 from urllib.parse import urlparse
-from databricks.sql.auth.retry import DatabricksRetryPolicy
 from databricks.sql.common.http import HttpMethod
 
 logger = logging.getLogger(__name__)
 
 
+def is_token_federation_provider(auth_provider) -> bool:
+    """``isinstance(auth_provider, TokenFederationProvider)`` without importing
+    token federation (and jwt) just to learn that it isn't in use: an instance
+    can only exist once its module has been imported."""
+    module = sys.modules.get("databricks.sql.auth.token_federation")
+    return module is not None and isinstance(
+        auth_provider, module.TokenFederationProvider
+    )
+
+
 class AuthType(Enum):
     DATABRICKS_OAUTH = "databricks-oauth"
     AZURE_OAUTH = "azure-oauth"

src/databricks/sql/backend/kernel/auth_bridge.py

@@ -80,7 +80,7 @@
     PYSQL_OAUTH_SCOPES,
 )
 from databricks.sql.auth.authenticators import AccessTokenAuthProvider, AuthProvider
-from databricks.sql.auth.token_federation import TokenFederationProvider
+from databricks.sql.auth.common import is_token_federation_provider
 from databricks.sql.exc import NotSupportedError, ProgrammingError
 
 logger = logging.getLogger(__name__)
@@ -115,8 +115,8 @@ def _is_pat(auth_provider: Optional[AuthProvider]) -> bool:
     """
     if isinstance(auth_provider, AccessTokenAuthProvider):
         return True
-    if isinstance(auth_provider, TokenFederationProvider) and isinstance(
-        auth_provider.external_provider, AccessTokenAuthProvider
+    if is_token_federation_provider(auth_provider) and isinstance(
+        getattr(auth_provider, "external_provider", None), AccessTokenAuthProvider
     ):
         return True
     return False

src/databricks/sql/client.py

@@ -72,10 +72,11 @@
 from databricks.sql.backend.types import CommandId, BackendType, CommandState, SessionId
 
 from databricks.sql.auth.common import AuthType, ClientContext
-from databricks.sql.common.unified_http_client import UnifiedHttpClient
 from databricks.sql.common.http import HttpMethod
 
 if TYPE_CHECKING:
+    from databricks.sql.common.unified_http_client import UnifiedHttpClient
+
     # Type-annotation-only imports (deferred by ``from __future__ import
     # annotations``). ``get_protocol_version`` and ``_prepare_native_parameters``
     # are typed with these Thrift-generated types, but the Thrift backend and
@@ -141,6 +142,11 @@ def __getattr__(name: str) -> Any:
         from databricks.sql.thrift_api.TCLIService import ttypes
 
         return getattr(ttypes, name)
+    if name == "UnifiedHttpClient":
+        # Same pattern, so kernel connections don't import urllib3.
+        from databricks.sql.common.unified_http_client import UnifiedHttpClient
+
+        return UnifiedHttpClient
     raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
 
 
@@ -413,11 +419,14 @@ def read(self) -> Optional[OAuthToken]:
         )
 
         client_context = build_client_context(server_hostname, __version__, **kwargs)
-        # The kernel owns its HTTP transport.
+        # The kernel owns its HTTP transport. Resolved through the module so
+        # the class (and urllib3) load only when a connection needs it.
+        import databricks.sql.client as _client_module
+
         self.http_client: Optional[UnifiedHttpClient] = (
             None
             if kwargs.get("use_kernel", False)
-            else UnifiedHttpClient(client_context)
+            else _client_module.UnifiedHttpClient(client_context)
         )
 
         try:

src/databricks/sql/cloudfetch/downloader.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
+import importlib
 import logging
 from dataclasses import dataclass
-from typing import Optional, TYPE_CHECKING
+from typing import Any, Optional, TYPE_CHECKING
 
 import lz4.frame
 import time
@@ -11,7 +12,6 @@
 from databricks.sql.types import SSLOptions
 from databricks.sql.telemetry.latency_logger import log_latency
 from databricks.sql.telemetry.models.event import StatementType
-from databricks.sql.common.unified_http_client import UnifiedHttpClient
 
 if TYPE_CHECKING:
     # Imported for type annotations only. ``from __future__ import annotations``
@@ -24,6 +24,20 @@
 logger = logging.getLogger(__name__)
 
 
+_LAZY_NAMES = {
+    "UnifiedHttpClient": "databricks.sql.common.unified_http_client",
+}
+
+
+def __getattr__(name: str) -> Any:
+    # Imported on first use rather than at load, so connect() skips them;
+    # still importable, and patchable here, as module attributes.
+    source = _LAZY_NAMES.get(name)
+    if source is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    return getattr(importlib.import_module(source), name)
+
+
 @dataclass
 class DownloadedFile:
     """

src/databricks/sql/common/http.py

@@ -1,14 +1,6 @@
-import requests
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 from enum import Enum
-import threading
 from dataclasses import dataclass
-from contextlib import contextmanager
-from typing import Generator, Optional
 import logging
-from requests.adapters import HTTPAdapter
-from databricks.sql.auth.retry import DatabricksRetryPolicy, CommandType
 
 logger = logging.getLogger(__name__)
 

src/databricks/sql/session.py

@@ -12,7 +12,6 @@
 from databricks.sql import USER_AGENT_NAME
 from databricks.sql.backend.databricks_client import DatabricksClient
 from databricks.sql.backend.types import SessionId, BackendType
-from databricks.sql.common.unified_http_client import UnifiedHttpClient
 from databricks.sql.common.agent import detect as detect_agent
 from databricks.sql.common.feature_flag import (
     FeatureFlagsContext,
@@ -23,6 +22,7 @@
 if TYPE_CHECKING:
     from databricks.sql.backend.thrift_backend import ThriftDatabricksClient
     from databricks.sql.backend.sea.backend import SeaDatabricksClient
+    from databricks.sql.common.unified_http_client import UnifiedHttpClient
 
 logger = logging.getLogger(__name__)
 
@@ -44,6 +44,11 @@ def __getattr__(name: str) -> Any:
         from databricks.sql.backend.sea.backend import SeaDatabricksClient
 
         return SeaDatabricksClient
+    if name == "UnifiedHttpClient":
+        # Same pattern, so connect() doesn't import urllib3.
+        from databricks.sql.common.unified_http_client import UnifiedHttpClient
+
+        return UnifiedHttpClient
     raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
 
 
@@ -52,7 +57,7 @@ def __init__(
         self,
         server_hostname: str,
         http_path: str,
-        http_client: Optional[UnifiedHttpClient],
+        http_client: Optional["UnifiedHttpClient"],
         http_headers: Optional[List[Tuple[str, str]]] = None,
         session_configuration: Optional[Dict[str, Any]] = None,
         catalog: Optional[str] = None,