Develop to Stage

.env.example

@@ -2,6 +2,9 @@ JWT_SECRET=your-secret-key
 OSM_DATABASE_URL=postgresql+asyncpg://user:pass@localhost:5432/osm
 TASK_DATABASE_URL=postgresql+asyncpg://user:pass@localhost:5432/tasking_manager
 TDEI_BACKEND_URL=https://portal-api-dev.tdei.us/api/v1/
+# TDEI's public API host (datasets lookup when creating from a dataset). Must
+# point at the same environment as TDEI_BACKEND_URL. Required: no default.
+TDEI_API_BACKEND_URL=https://api-dev.tdei.us/api/v1/
 TDEI_OIDC_REALM=tdei
 TDEI_OIDC_URL=https://account-dev.tdei.us/
 # External TDEI permission changes may take up to this long to become visible.

.github/workflows/docker-image.yml

@@ -33,5 +33,7 @@ jobs:
             context: .
             push: true
             platforms: linux/amd64
+            build-args: |
+              CODE_VERSION=${{ github.sha }}
             tags: ${{ vars.DOCKERHUB_REGISTRY }}/workspaces-backend-v2:dev
 

.github/workflows/push-docker-image.yml

@@ -1,4 +1,4 @@
-name: Push Docker Image to Azure Container Registry
+name: Push Docker Image to Azure Container Registry and Deploy to Azure
 
 on:
   workflow_dispatch:
@@ -59,6 +59,31 @@ jobs:
             context: .
             push: true
             platforms: linux/amd64
+            build-args: |
+              CODE_VERSION=${{ github.sha }}
             tags: ${{ vars.DOCKERHUB_REGISTRY }}/workspaces-backend-v2:${{ needs.set-github-environment.outputs.environment }},
                   ${{ vars.DOCKERHUB_REGISTRY }}/workspaces-backend-v2:${{github.sha}}
-
+  deploy:
+    runs-on: ubuntu-latest
+    needs: [build,set-github-environment]
+    name: "Deploy to Azure"
+    steps:
+      - name: Display environment
+        run: echo "Deploying to ${{ needs.set-github-environment.outputs.environment }}"
+      - name: Login to Azure
+        uses: azure/login@v2.0.0
+        with:
+          creds: ${{ secrets.TDEI_CORE_AZURE_CREDS }}
+      - name: Deploy to Azure
+        uses: azure/webapps-deploy@v3
+        with:
+          app-name: ws-backend-${{ needs.set-github-environment.outputs.environment }}
+          sitecontainers-config: |
+            [
+              {
+                "name": "main",
+                "image": "${{ vars.DOCKERHUB_REGISTRY }}/workspaces-backend-v2:${{ needs.set-github-environment.outputs.environment }}",
+                "targetport": "8000",
+                "isMain": true
+              }
+            ]

CLAUDE.md

@@ -118,6 +118,43 @@ enforced at this layer; if required, it must be enforced downstream
 (`workspaces-openstreetmap-website/`, `workspaces-cgimap/`) — that has not been
 audited here.
 
+**Enforced here, TDEI-role-gated: Create Workspace.** Creation isn't in the
+matrix above because it happens before any workspace role exists. It is gated
+on the caller's TDEI roles *in the target project group*
+(`UserInfo.canCreateWorkspace`, via `_CREATABLE_WORKSPACE_TYPES` in
+`api/core/security.py`), and enforced in `WorkspaceRepository._authorize_create`
+(`api/src/workspaces/repository.py`) → 403:
+
+| TDEI role in the target PG | May create |
+|---|---|
+| `poc` | any workspace type |
+| `osw_data_generator` | OSW only |
+| `flex_data_generator` | Flex only |
+| `pathways_data_generator` | Pathways only |
+| `member` (alone) | nothing |
+
+* Roles are a list, so a `member` who also holds a qualifying role can create.
+* **`tdei-admin` gets no extra rights.** An admin can't be a POC, only a
+  member, so the member rule applies. It is not in `_CREATABLE_WORKSPACE_TYPES`.
+* **Creating from a TDEI dataset** (`tdeiRecordId` set) also looks the dataset
+  up on TDEI (`fetch_tdei_dataset`, against `TDEI_API_BACKEND_URL`, a required
+  setting with no default) and checks:
+  * the dataset exists and the caller can see it, or 404;
+  * its `data_type` matches the workspace type, or 403. Otherwise an OSW
+    generator could open a Flex/Pathways dataset as an OSW workspace;
+  * a **released** dataset (`status == "Publish"`) may come from any project
+    group, but an **unreleased** one only from the project group the workspace
+    is created in, or 403
+    (`UserInfo.canCreateWorkspaceFromDataset`).
+* Dataset and project-group ids are compared as parsed UUIDs (`same_uuid`), so
+  case and hyphens don't matter.
+* If TDEI is unreachable, rejects the token, or returns an unusable body, the
+  result is **502**, not 401. `validate_token` has already accepted the token,
+  so a TDEI rejection here is a config mismatch, not an expired session.
+* `POST /workspaces/check-create` runs the same `_authorize_create` as a
+  pre-flight and returns `{canCreate, reason}` instead of a 403/404. A TDEI
+  failure still propagates as 502, because "TDEI is down" isn't a denial.
+
 ## The OSM proxy layer: routing, auth, and user provisioning
 
 This service is a reverse proxy in front of `osm-web`, which in turn fronts the

Dockerfile

@@ -20,5 +20,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 # set the environment variable for uvicorn to run the FastAPI app
 ENV UV_PROJECT_ENVIRONMENT=.uvenv
 
+# The commit this image was built from, reported to Sentry as the release.
+# Set by the deploy workflow; declared last so a new commit doesn't
+# invalidate the dependency layers above.
+ARG CODE_VERSION=unknown
+ENV CODE_VERSION=$CODE_VERSION
+
 # Run with uvicorn
 CMD ["uv", "run", "uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000"]

api/core/config.py

@@ -1,9 +1,11 @@
 import json
 
+from pydantic import Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
 # Test outline:
+# @test: Test that startup fails when TDEI_API_BACKEND_URL is unset or empty
 # @test: Test that environment variables of the same name as the members of this class are correctly loaded into the members of this class.
 # @test: Test that any environment variables that are not set in the environment are correctly loaded into the members of this class with their default values.
 # @test: Test that strings and number values, empty strings and URLs are all correctly loaded as exemplified by the default values
@@ -28,6 +30,11 @@ class Settings(BaseSettings):
     )
 
     TDEI_BACKEND_URL: str = "https://portal-api-dev.tdei.us/api/v1/"
+    # TDEI's public API gateway (api[-dev|-stage].tdei.us), used to look up
+    # datasets when creating a workspace from one. Required, with no default:
+    # a dev default would quietly send prod tokens to dev. Must point at the
+    # same environment as TDEI_BACKEND_URL.
+    TDEI_API_BACKEND_URL: str = Field(min_length=1)
     TDEI_OIDC_URL: str = "https://account-dev.tdei.us/"
     TDEI_OIDC_REALM: str = "tdei"
 
@@ -113,4 +120,6 @@ def cors_origins_list(self) -> list[str]:
     )
 
 
-settings = Settings()
+# Required fields (TDEI_API_BACKEND_URL) come from the environment, which
+# Pyright can't see.
+settings = Settings()  # pyright: ignore[reportCallIssue]

api/core/security.py

@@ -21,6 +21,7 @@
 from api.core.jwt import validate_and_decode_token
 from api.core.logging import get_logger
 from api.src.users.schemas import WorkspaceUserRoleType
+from api.src.workspaces.schemas import WorkspaceType
 
 # Test outline:
 # @test: Test that the permissions structure here matches what is described in CLAUDE.md
@@ -108,21 +109,29 @@ def _log_rejected_token(token: str, reason: str) -> None:
 
 # Shared HTTP client for TDEI backend calls. Initialized by main.py lifespan.
 _tdei_client: httpx.AsyncClient | None = None
+_tdei_api_client: httpx.AsyncClient | None = None
 
 
 def init_tdei_client() -> None:
-    global _tdei_client
+    global _tdei_client, _tdei_api_client
     _tdei_client = httpx.AsyncClient(
         base_url=settings.TDEI_BACKEND_URL,
         timeout=httpx.Timeout(connect=10, read=30, write=30, pool=10),
     )
+    _tdei_api_client = httpx.AsyncClient(
+        base_url=settings.TDEI_API_BACKEND_URL,
+        timeout=httpx.Timeout(connect=10, read=30, write=30, pool=10),
+    )
 
 
 async def close_tdei_client() -> None:
-    global _tdei_client
+    global _tdei_client, _tdei_api_client
     if _tdei_client is not None:
         await _tdei_client.aclose()
         _tdei_client = None
+    if _tdei_api_client is not None:
+        await _tdei_api_client.aclose()
+        _tdei_api_client = None
 
 
 class TdeiProjectGroupUser:
@@ -223,6 +232,127 @@ async def fetch_project_group_users(
     return out
 
 
+def same_uuid(a: object, b: object) -> bool:
+    """True when both values parse to the same UUID.
+
+    TDEI ids are UUIDs, but nothing guarantees the canonical lowercase,
+    hyphenated form we get from ``str(UUID)``. Comparing parsed values avoids
+    a missed match on case or hyphens. Values that don't parse as UUIDs fall
+    back to exact string comparison.
+    """
+    try:
+        return UUID(str(a)) == UUID(str(b))
+    except ValueError:
+        return str(a) == str(b)
+
+
+class TdeiDataset:
+    """The fields of a TDEI dataset that workspace creation checks, as
+    returned by ``GET /datasets?tdei_dataset_id=...``.
+    """
+
+    # TDEI marks a released dataset "Publish"; unreleased ones are "Pre-Release".
+    RELEASED_STATUS = "Publish"
+
+    def __init__(
+        self,
+        *,
+        tdei_dataset_id: str,
+        project_group_id: str,
+        status: str,
+        data_type: str,
+    ) -> None:
+        self.tdei_dataset_id = tdei_dataset_id
+        self.project_group_id = project_group_id
+        self.status = status
+        # "osw", "flex" or "pathways" -- the WorkspaceType values
+        self.data_type = data_type
+
+    @property
+    def is_released(self) -> bool:
+        return self.status == self.RELEASED_STATUS
+
+    def matches_type(self, workspace_type: WorkspaceType) -> bool:
+        return self.data_type.strip().lower() == workspace_type.value
+
+
+# @test: Test that fetch_tdei_dataset returns the dataset's project group, status and data type, None when TDEI returns no match, and 502 when TDEI is unreachable, rejects the token, answers non-200, or returns a non-JSON or non-list body
+# @test: Test that fetch_tdei_dataset matches the dataset id regardless of case or hyphens
+async def fetch_tdei_dataset(
+    dataset_id: str,
+    bearer_token: str,
+) -> TdeiDataset | None:
+    """Look up one dataset via ``GET /datasets?tdei_dataset_id=...`` on TDEI.
+
+    Returns None when TDEI has no dataset with that id visible to the caller.
+    Raises 502 if TDEI is unreachable or answers with anything but a JSON
+    list -- including a 401: validate_token has already accepted the caller's
+    token, so a rejection here is a TDEI-side mismatch (e.g. a wrong
+    TDEI_API_BACKEND_URL), not an expired session the client should act on.
+    """
+    if _tdei_api_client is None:  # pragma: no cover — lifespan should init this
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail="TDEI API client is not initialised",
+        )
+
+    try:
+        response = await _tdei_api_client.get(
+            "datasets",
+            headers={"Authorization": f"Bearer {bearer_token}"},
+            # status=All includes unreleased datasets, which TDEI omits by default
+            params={"tdei_dataset_id": dataset_id, "status": "All"},
+        )
+    except httpx.RequestError:
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail="Could not reach TDEI backend to fetch dataset details",
+        ) from None
+
+    if response.status_code == 401:
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail="TDEI rejected the bearer token when fetching dataset details",
+        )
+    if response.status_code != 200:
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail=(
+                f"TDEI returned {response.status_code} when fetching "
+                f"dataset {dataset_id}"
+            ),
+        )
+
+    try:
+        rows = response.json()
+    except Exception:
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail="TDEI returned a non-JSON body",
+        ) from None
+
+    if not isinstance(rows, list):
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail="TDEI returned an unexpected response shape for datasets",
+        )
+
+    for row in rows:
+        if not isinstance(row, dict) or not same_uuid(
+            row.get("tdei_dataset_id"), dataset_id
+        ):
+            continue
+        return TdeiDataset(
+            tdei_dataset_id=dataset_id,
+            project_group_id=str(
+                (row.get("project_group") or {}).get("tdei_project_group_id", "")
+            ),
+            status=str(row.get("status", "")),
+            data_type=str(row.get("data_type", "")),
+        )
+    return None
+
+
 def evict_user_from_cache(auth_uid: UUID) -> None:
     """
     Evict a user's cached UserInfo object so that their next request re-fetches
@@ -422,6 +552,16 @@ class TdeiProjectGroupRole(StrEnum):
     PATHWAYS_GENERATOR = "pathways_data_generator"
 
 
+# Workspace types each TDEI project group role may create. MEMBER is absent:
+# members can edit workspace data but not create workspaces.
+_CREATABLE_WORKSPACE_TYPES: dict[TdeiProjectGroupRole, frozenset[WorkspaceType]] = {
+    TdeiProjectGroupRole.POINT_OF_CONTACT: frozenset(WorkspaceType),
+    TdeiProjectGroupRole.OSW_GENERATOR: frozenset({WorkspaceType.OSW}),
+    TdeiProjectGroupRole.FLEX_GENERATOR: frozenset({WorkspaceType.FLEX}),
+    TdeiProjectGroupRole.PATHWAYS_GENERATOR: frozenset({WorkspaceType.PATHWAYS}),
+}
+
+
 class UserInfoPGMembership:
     project_group_name: str
     project_group_id: str
@@ -464,6 +604,40 @@ def getProjectGroupIds(self, withRole="any") -> list[str]:
                 pgids.append(pg.project_group_id)
         return pgids
 
+    # Who can create a workspace of a given type in a PG: POC can create any
+    # type; a data generator only its own type (OSW generator -> OSW, Flex
+    # generator -> Flex, Pathways generator -> Pathways); a plain member
+    # cannot. Roles are a list, so a member who also holds a qualifying role
+    # can create.
+    # @test: Test that canCreateWorkspace is False for a PG the user only has "member" in, True when "member" is combined with a role that covers the workspace type, and False for a PG the user does not belong to
+    # @test: Test that POC can create every workspace type, and each data generator role only its own type (OSW -> OSW, Flex -> Flex, Pathways -> Pathways)
+    def canCreateWorkspace(
+        self, project_group_id: str, workspace_type: WorkspaceType
+    ) -> bool:
+        for pg in self.projectGroups:
+            if not same_uuid(pg.project_group_id, project_group_id):
+                continue
+            for role in pg.tdeiRoles:
+                if workspace_type in _CREATABLE_WORKSPACE_TYPES.get(role, ()):
+                    return True
+        return False
+
+    # Creating from a TDEI dataset additionally depends on the dataset: a
+    # released dataset may come from any PG (even one the user has no role
+    # in), an unreleased one only from the PG the workspace is created in.
+    # @test: Test that canCreateWorkspaceFromDataset allows a released dataset from any PG, allows an unreleased dataset only from the target PG, and is False whenever canCreateWorkspace is False
+    def canCreateWorkspaceFromDataset(
+        self,
+        project_group_id: str,
+        workspace_type: WorkspaceType,
+        dataset: TdeiDataset,
+    ) -> bool:
+        if not self.canCreateWorkspace(project_group_id, workspace_type):
+            return False
+        return dataset.is_released or same_uuid(
+            dataset.project_group_id, project_group_id
+        )
+
     # we get admin/lead rights in two ways:
     # 1. from OSM DB explicitly
     # 2. from TDEI if user has "point_of_contact" role in any PG that owns the workspace
@@ -520,7 +694,7 @@ def get_task_db_session(
     return session
 
 
-# @test:
+# @test: Test that TDEI project group role names are trimmed and lowercased, so "POC" or " poc " grant the same rights as "poc"
 
 
 async def validate_token(
@@ -868,7 +1042,14 @@ async def _validate_token_uncached(
             UserInfoPGMembership(
                 project_group_id=i["tdei_project_group_id"],
                 project_group_name=i["project_group_name"],
-                tdeiRoles=i["roles"],
+                # Normalized so "POC" or " poc" still match TdeiProjectGroupRole,
+                # as the frontend's hasCreationRole does. Roles we don't model
+                # (e.g. tdei-admin) grant nothing here, so they're dropped.
+                tdeiRoles=[
+                    TdeiProjectGroupRole(role)
+                    for r in i.get("roles") or []
+                    if (role := str(r).strip().lower()) in TdeiProjectGroupRole
+                ],
             )
         )