Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion cloud_pipelines_backend/api_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from .instrumentation import contextual_logging

if typing.TYPE_CHECKING:
from . import secret_store
from .launchers import interfaces as launcher_interfaces


Expand All @@ -40,6 +41,7 @@ def setup_routes(
container_launcher_for_log_streaming: "launcher_interfaces.ContainerTaskLauncher[launcher_interfaces.LaunchedContainer] | None" = None,
default_component_library_owner_username: str = "admin",
do_skip_backfill: bool = False,
secret_store: "secret_store.SecretStore | None" = None,
):
def get_session():
with orm.Session(autocommit=False, autoflush=False, bind=db_engine) as session:
Expand Down Expand Up @@ -76,6 +78,7 @@ async def lifespan(app: fastapi.FastAPI):
user_details_getter=user_details_getter,
get_launcher=get_launcher,
lifespan=lifespan,
secret_store=secret_store,
)


Expand All @@ -93,6 +96,7 @@ def _setup_routes_internal(
| typing.Callable[..., abc.Iterator[typing.Any]]
| None
) = None,
secret_store: "secret_store.SecretStore | None" = None,
):
# We request `app: fastapi.FastAPI` instead of just returning the router
# because we want to add exception handler which is only supported for `FastAPI`.
Expand Down Expand Up @@ -462,7 +466,7 @@ def get_current_user(
)

### Secrets routes
secrets_service = api_server_sql.SecretsApiService()
secrets_service = api_server_sql.SecretsApiService(secret_store=secret_store)

router.get("/api/secrets/", tags=["secrets"], **default_config)(
inject_session_dependency(
Expand Down
59 changes: 57 additions & 2 deletions cloud_pipelines_backend/api_server_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from .access_control import checks as access_control_checks
from .access_control import runs as access_control_runs
from . import filter_query_sql
from . import secret_store

if typing.TYPE_CHECKING:
from cloud_pipelines.orchestration.storage_providers import (
Expand Down Expand Up @@ -1251,13 +1252,25 @@ def from_db(cls, secret_row: bts.Secret) -> "SecretInfoResponse":
description=secret_row.description,
)

@classmethod
def from_store(cls, secret: secret_store.StoredSecret) -> "SecretInfoResponse":
return SecretInfoResponse(
secret_name=secret.secret_name,
created_at=secret.created_at,
updated_at=secret.updated_at,
description=secret.description,
)


@dataclasses.dataclass(kw_only=True)
class ListSecretsResponse:
secrets: list[SecretInfoResponse]


class SecretsApiService:
def __init__(self, *, secret_store: secret_store.SecretStore | None = None):
self._secret_store = secret_store

def create_secret(
self,
*,
Expand Down Expand Up @@ -1315,7 +1328,31 @@ def _create_or_update_secret(
raise_if_exists: bool = False,
) -> SecretInfoResponse:
current_time = _get_current_time()
secret = session.get(bts.Secret, (user_id, secret_name))
# With a store, lock the row so that a backfill moving it waits, or has already finished.
secret = session.get(
bts.Secret,
(user_id, secret_name),
with_for_update=self._secret_store is not None,
)
# Not in the table (new, or already moved): the store keeps it. A row is updated where it is.
if not secret and self._secret_store:
store_method = (
self._secret_store.create
if raise_if_exists
else self._secret_store.update
)
saved = store_method(
secret=secret_store.StoredSecret(
user_id=user_id,
secret_name=secret_name,
value=secret_value,
# None keeps the stored description, as an empty one leaves the row's alone below.
description=description or None,
created_at=current_time,
updated_at=current_time,
),
)
return SecretInfoResponse.from_store(saved)
if secret:
if raise_if_exists:
raise errors.ItemAlreadyExistsError(
Expand Down Expand Up @@ -1351,7 +1388,20 @@ def delete_secret(
user_id: str,
secret_name: str,
) -> None:
secret = session.get(bts.Secret, (user_id, secret_name))
secret = session.get(
bts.Secret,
(user_id, secret_name),
with_for_update=self._secret_store is not None,
)
if self._secret_store:
# Also deletes a copy left by a backfill that stopped before deleting the row.
try:
self._secret_store.delete(user_id=user_id, secret_name=secret_name)
except errors.ItemNotFoundError:
if not secret:
raise
if not secret:
return
if not secret:
raise errors.ItemNotFoundError(
f"Secret with name '{secret_name}' does not exist."
Expand All @@ -1365,6 +1415,11 @@ def list_secrets(
session: orm.Session,
user_id: str,
) -> ListSecretsResponse:
if self._secret_store:
stored = self._secret_store.list_secrets(user_id=user_id)
return ListSecretsResponse(
secrets=[SecretInfoResponse.from_store(secret) for secret in stored]
)
secrets = session.scalars(
sql.select(bts.Secret).where(bts.Secret.user_id == user_id)
).all()
Expand Down
47 changes: 47 additions & 0 deletions cloud_pipelines_backend/secret_store.py

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe, secret_interfaces.py? I already see problematic cases in the code: secret_store: secret_store.SecretStore

Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import dataclasses
import datetime
from typing import Any, Protocol


@dataclasses.dataclass(frozen=True, kw_only=True)
class StoredSecret:
"""A secret as a `SecretStore` keeps it; `value` is None when not loaded, and never shown by repr, str or logs."""

user_id: str
secret_name: str
value: str | None = dataclasses.field(default=None, repr=False)
description: str | None = None
created_at: datetime.datetime
updated_at: datetime.datetime
extra_data: dict[str, Any] | None = None


class SecretStore(Protocol):
"""Keeps secrets outside the database, for example in a cloud secret manager.

With a store, new secrets go to the store; a secret still in the `secret` table stays there until moved.
"""

def create(self, *, secret: StoredSecret) -> StoredSecret:
"""Saves a new secret and returns it without its value; raises `errors.ItemAlreadyExistsError` if it exists."""
...

def update(self, *, secret: StoredSecret) -> StoredSecret:
"""Saves a new value for an existing secret and returns it without its value.

Keeps the stored `created_at` and name, and the stored description when `description` is None.
Raises `errors.ItemNotFoundError` when there is no such secret.
"""
...

def get(self, *, user_id: str, secret_name: str) -> str | None:
"""Returns the secret's latest value, or None when there is no such secret."""
...

def list_secrets(self, *, user_id: str) -> list[StoredSecret]:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Naming consistency: This is the only method with _secrets suffix...
We could either call this one list or add suffixes to other ones.
I think list is fine. I've seen it used in other libraries. Yes, we'll have to be careful about using he builtin list in signatures, but only for this class. Which does not have any list. So, it should be fine.

"""Returns the user's secrets, without values."""
...

def delete(self, *, user_id: str, secret_name: str) -> None:
"""Deletes the secret and all its values; raises `errors.ItemNotFoundError` when there is none."""
...
Loading
Loading