Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions cloud_pipelines_backend/api_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -623,6 +623,22 @@
execution_node = session.get(backend_types_sql.ExecutionNode, id)
execution_node.container_execution_status = status

@router.post(
"/api/admin/secrets/backfill_storage",
tags=["admin"],
dependencies=[ensure_admin_user_dependency],
**default_config,
)
def admin_backfill_secret_storage(
session: typing.Annotated[orm.Session, fastapi.Depends(get_session)],
limit: typing.Annotated[int, fastapi.Query(ge=1, le=1000)] = 50,
user_id: str | None = None,
secret_name: str | None = None,
) -> dict[str, typing.Any]:
return secrets_service.backfill_secret_storage(
session=session, limit=limit, user_id=user_id, secret_name=secret_name
)
Comment on lines +638 to +640

@router.get("/api/admin/sql_engine_connection_pool_status")
async def get_sql_engine_connection_pool_status(
session: typing.Annotated[orm.Session, fastapi.Depends(get_session)],
Expand Down
81 changes: 81 additions & 0 deletions cloud_pipelines_backend/api_server_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -1381,6 +1381,87 @@ def _create_or_update_secret(
session.commit()
return response

def backfill_secret_storage(
self,
*,
session: orm.Session,
limit: int,
user_id: str | None = None,
secret_name: str | None = None,
) -> dict[str, typing.Any]:
"""Moves up to `limit` rows from the `secret` table into the store, committing each, and reports what is left.

`user_id` and `secret_name`, when given, move only matching rows. A row the store refuses as invalid (for
example, a value too large) is skipped and listed by name; any other error stops the run, which can then
be repeated.
"""
if not self._secret_store:
raise errors.ApiValidationError("No secret store is configured.")
rows = session.execute(
sql.select(bts.Secret.user_id, bts.Secret.secret_name)
.filter_by(
**({"user_id": user_id} if user_id is not None else {}),
**({"secret_name": secret_name} if secret_name is not None else {}),
)
.order_by(bts.Secret.user_id, bts.Secret.secret_name)
).all()
moved = 0
skipped = []
stopped_on = None
for row_user_id, row_secret_name in rows:
if moved == limit:
break
try:
self._move_row(
session=session, user_id=row_user_id, secret_name=row_secret_name
)
except errors.ApiValidationError:
# Releases the row lock `_move_row` took, so the skipped row is usable during the run.
session.rollback()
skipped.append({"user_id": row_user_id, "secret_name": row_secret_name})
continue
except Exception as e:
session.rollback()
stopped_on = {
"user_id": row_user_id,
"secret_name": row_secret_name,
"error": f"{type(e).__name__}: {e}",
}
break
moved += 1
return {
"moved": moved,
"skipped": skipped,
"left_in_table": session.scalar(
sql.select(sql.func.count()).select_from(bts.Secret)
),
"stopped_on": stopped_on,
}

def _move_row(
self, *, session: orm.Session, user_id: str, secret_name: str
) -> None:
"""Copies one row into the store and deletes it, in one commit; a PUT or DELETE of it waits on the lock."""
row = session.get(bts.Secret, (user_id, secret_name), with_for_update=True)
if not row:
return
secret = secret_store.StoredSecret(
user_id=row.user_id,
secret_name=row.secret_name,
value=row.secret_value,
description=row.description,
created_at=row.created_at,
updated_at=row.updated_at,
extra_data=row.extra_data,
)
try:
self._secret_store.create(secret=secret)
except errors.ItemAlreadyExistsError:
# An earlier run stopped after copying the row; the row is still the truth, so it replaces the copy.
self._secret_store.update(secret=secret)
session.delete(row)
session.commit()

def delete_secret(
self,
*,
Expand Down
162 changes: 159 additions & 3 deletions tests/test_secret_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,23 @@
class _FakeStore(secret_store.SecretStore):
"""An in-memory `SecretStore` that follows its contract, keyed by user and name."""

def __init__(self, *, fail_get: bool = False) -> None:
def __init__(
self,
*,
fail_create: bool = False,
fail_get: bool = False,
invalid_names: frozenset[str] = frozenset(),
) -> None:
self.secrets: dict[tuple[str, str], secret_store.StoredSecret] = {}
self._fail_create = fail_create
self._fail_get = fail_get
self._invalid_names = invalid_names

def create(self, *, secret: secret_store.StoredSecret) -> secret_store.StoredSecret:
if self._fail_create:
raise RuntimeError("store unavailable")
if secret.secret_name in self._invalid_names:
raise errors.ApiValidationError("value too large")
key = (secret.user_id, secret.secret_name)
if key in self.secrets:
raise errors.ItemAlreadyExistsError("exists")
Expand Down Expand Up @@ -278,14 +290,16 @@ def test_stored_secret_never_shows_its_value() -> None:
assert "hunter2" not in f"{secret}"


def _client(*, engine: sqlalchemy.Engine, store: _FakeStore | None) -> TestClient:
def _client(
*, engine: sqlalchemy.Engine, store: _FakeStore | None, admin: bool = False
) -> TestClient:
"""Returns an API client for alice, with the given store passed to setup_routes."""
app = fastapi.FastAPI()
api_router.setup_routes(
app=app,
db_engine=engine,
user_details_getter=lambda: api_router.UserDetails(
name="alice", permissions={"read": True, "write": True, "admin": False}
name="alice", permissions={"read": True, "write": True, "admin": admin}
),
secret_store=store,
)
Expand Down Expand Up @@ -418,3 +432,145 @@ def test_launch_fails_when_the_store_cannot_read(tmp_path: pathlib.Path) -> None
launch, execution = _launch_token(engine=engine, store=_FakeStore(fail_get=True))
launch.assert_not_called()
assert "store unavailable" in _system_error(execution=execution)


def _backfill(
*, engine: sqlalchemy.Engine, store: _FakeStore | None, **params: str | int
) -> dict:
"""Calls the backfill route as an admin and returns its JSON."""
response = _client(engine=engine, store=store, admin=True).post(
"/api/admin/secrets/backfill_storage", params=params
)
assert response.status_code == 200, response.text
return response.json()


def _seed(*, engine: sqlalchemy.Engine, names: list[tuple[str, str]]) -> None:
for user_id, secret_name in names:
_create(engine=engine, store=None, user_id=user_id, secret_name=secret_name)


def test_backfill_without_a_store_is_a_422(tmp_path: pathlib.Path) -> None:
response = _client(engine=_engine(tmp_path=tmp_path), store=None, admin=True).post(
"/api/admin/secrets/backfill_storage"
)
assert response.status_code == 422, response.text


def test_backfill_moves_up_to_the_limit(tmp_path: pathlib.Path) -> None:
engine = _engine(tmp_path=tmp_path)
_seed(engine=engine, names=[("alice", "A"), ("alice", "B"), ("bob", "C")])
store = _FakeStore()
result = _backfill(engine=engine, store=store, limit=2)
assert result == {"moved": 2, "skipped": [], "left_in_table": 1, "stopped_on": None}
assert sorted(store.secrets) == [("alice", "A"), ("alice", "B")]
assert _backfill(engine=engine, store=store)["left_in_table"] == 0


def test_backfill_filters_by_user_and_name(tmp_path: pathlib.Path) -> None:
engine = _engine(tmp_path=tmp_path)
_seed(engine=engine, names=[("alice", "A"), ("alice", "B"), ("bob", "A")])
store = _FakeStore()
assert _backfill(engine=engine, store=store, user_id="bob")["moved"] == 1
assert _backfill(engine=engine, store=store, secret_name="B")["moved"] == 1
assert sorted(store.secrets) == [("alice", "B"), ("bob", "A")]


def test_backfill_skips_an_invalid_row_without_counting_it(
tmp_path: pathlib.Path,
) -> None:
engine = _engine(tmp_path=tmp_path)
_seed(engine=engine, names=[("alice", "A"), ("alice", "BIG"), ("alice", "C")])
store = _FakeStore(invalid_names=frozenset({"BIG"}))
result = _backfill(engine=engine, store=store, limit=2)
assert result == {
"moved": 2,
"skipped": [{"user_id": "alice", "secret_name": "BIG"}],
"left_in_table": 1,
"stopped_on": None,
}
assert [row.secret_name for row in _rows(engine=engine)] == ["BIG"]


def test_backfill_stops_on_an_error_and_can_be_repeated(
tmp_path: pathlib.Path,
) -> None:
engine = _engine(tmp_path=tmp_path)
_seed(engine=engine, names=[("alice", "A"), ("alice", "B")])
result = _backfill(engine=engine, store=_FakeStore(fail_create=True))
assert result == {
"moved": 0,
"skipped": [],
"left_in_table": 2,
"stopped_on": {
"user_id": "alice",
"secret_name": "A",
"error": "RuntimeError: store unavailable",
},
}
assert _backfill(engine=engine, store=_FakeStore())["left_in_table"] == 0


def test_backfill_needs_an_admin(tmp_path: pathlib.Path) -> None:
response = _client(engine=_engine(tmp_path=tmp_path), store=_FakeStore()).post(
"/api/admin/secrets/backfill_storage"
)
assert response.status_code == 403, response.text


def test_backfill_carries_every_column(tmp_path: pathlib.Path) -> None:
engine = _engine(tmp_path=tmp_path)
now = api_server_sql._get_current_time()
with orm.Session(bind=engine) as session:
session.add(
bts.Secret(
user_id="Alice",
secret_name="Token",
secret_value="v1",
description="d",
extra_data={"k": [1]},
created_at=now,
updated_at=now,
)
)
session.commit()
store = _FakeStore()
assert _backfill(engine=engine, store=store, user_id="Alice")["moved"] == 1
assert store.secrets[("Alice", "Token")] == secret_store.StoredSecret(
user_id="Alice",
secret_name="Token",
value="v1",
description="d",
extra_data={"k": [1]},
created_at=now,
updated_at=now,
)


def test_backfill_replaces_a_copy_left_by_a_stopped_run(
tmp_path: pathlib.Path,
) -> None:
engine = _engine(tmp_path=tmp_path)
_seed(engine=engine, names=[("alice", "TOKEN")])
_update(engine=engine, store=None, value="newer")
store = _FakeStore()
store.create(secret=_stored(value="stale copy"))
assert _backfill(engine=engine, store=store)["moved"] == 1
assert store.get(user_id="alice", secret_name="TOKEN") == "newer"
assert _rows(engine=engine) == []


def test_backfill_locks_the_row_it_moves(tmp_path: pathlib.Path) -> None:
"""A PUT or DELETE of the same row then waits for the move to commit."""
engine = _engine(tmp_path=tmp_path)
_seed(engine=engine, names=[("alice", "TOKEN")])
locked = []
with orm.Session(bind=engine) as session:
sqlalchemy.event.listen(
session,
"do_orm_execute",
lambda state: state.statement._for_update_arg is not None
and locked.append(True),
)
_service(store=_FakeStore()).backfill_secret_storage(session=session, limit=10)
assert locked == [True]
Loading