🤖 feat: pool Kind phases P6 to P8, offline stubs for every phase, and deleting a released used member deletes its workspace (#153 PR 13)

.github/workflows/ci.yaml

@@ -683,9 +683,9 @@ jobs:
           kubectl -n coder delete secret coder-tls
           kubectl -n coder delete configmap coder-ca
 
-      # CoderWorkspacePool phases P1-P5 (#153). It runs before the lifecycle driver, which deletes namespace coder at
+      # CoderWorkspacePool phases P1-P8 (#153). It runs before the lifecycle driver, which deletes namespace coder at
       # its end, imports its own template e2e-pool-agent from the agent fixture, and removes its objects when it passes.
-      - name: Workspace pool (create, scale, claim, operator restart)
+      - name: Workspace pool (create, scale, claim, operator restart, use, deletes)
         if: env.E2E_FULL == 'true'
         env:
           E2E_WORKDIR: ${{ runner.temp }}/workspace-pool

api/v1alpha1/coderworkspacepool_types.go

@@ -277,8 +277,8 @@ type CoderWorkspacePoolStatus struct {
 	// +optional
 	BoundReplicas int32 `json:"boundReplicas,omitempty"`
 	// UsedReplicas is the number of members that someone may have used
-	// without a claim. They no longer count toward the pool, and the
-	// controller never deletes their workspaces.
+	// without a claim. They no longer count toward the pool, and the pool
+	// never deletes them.
 	// +optional
 	UsedReplicas int32 `json:"usedReplicas,omitempty"`
 	// PendingClaims is the number of claims that wait for a workspace.

config/crd/bases/coder.com_coderworkspacepools.yaml

@@ -452,8 +452,8 @@ spec:
               usedReplicas:
                 description: |-
                   UsedReplicas is the number of members that someone may have used
-                  without a claim. They no longer count toward the pool, and the
-                  controller never deletes their workspaces.
+                  without a claim. They no longer count toward the pool, and the pool
+                  never deletes them.
                 format: int32
                 type: integer
             type: object

dist/install.yaml

@@ -7697,8 +7697,8 @@ spec:
               usedReplicas:
                 description: |-
                   UsedReplicas is the number of members that someone may have used
-                  without a claim. They no longer count toward the pool, and the
-                  controller never deletes their workspaces.
+                  without a claim. They no longer count toward the pool, and the pool
+                  never deletes them.
                 format: int32
                 type: integer
             type: object

docs/how-to/workspace-pools.md

@@ -60,7 +60,7 @@ spec:
 kubectl -n coder get coderworkspacepools,coderworkspacepoolmembers
 ```
 
-Each member owns one workspace named `kwp-` plus 28 hex characters of the member's UID. Do not rename these workspaces, and do not use an unclaimed one directly. The pool releases a workspace that someone used without a claim, and never deletes it.
+Each member owns one workspace named `kwp-` plus 28 hex characters of the member's UID. Do not rename these workspaces, and do not use an unclaimed one directly. The pool releases a workspace that someone used without a claim, and never deletes it on its own. Opening an unclaimed workspace's page in the Coder dashboard counts as use too: the page asks the workspace agent for its containers, and in Coder v2.37.2 that request moves `last_used_at` ([#444](https://github.com/coder/coder-k8s/issues/444)). The workspaces list does not.
 
 `replicas` has a scale subresource, so `kubectl scale`, HPA, and KEDA can change it:
 
@@ -181,7 +181,7 @@ Checks that fall due together wait in line, so with a 1 s delay some checks wait
 
 - Delete a claim to delete its workspace.
 - Delete a pool to delete its unclaimed workspaces. Claimed workspaces stay with their claims, and the claims stay `Bound`.
-- A member whose workspace someone used without a claim (`Used=True`) leaves the pool. The operator never deletes its workspace, because a person can still have work in it. A cluster admin can delete the member object: the workspace then stays in Coder. Delete it there once nobody needs it.
+- A member whose workspace someone used without a claim (`Used=True`) leaves the pool: the pool removes itself as the member's owner, so neither a scale-down nor deleting the pool deletes it. A cluster admin who deletes such a member deletes its workspace in Coder, as deleting a claim does. To keep the work, copy it out of the workspace first. Deleting the namespace keeps a used member's workspace in Coder. Find it as the next item says. A used member that still lists the pool as its controller is not released yet, and deleting it keeps the workspace in Coder.
 - When the `CoderControlPlane` is deleted, members release their finalizers without a Coder call. Find leftover workspaces in Coder with the filter `owner:<pool owner>` and delete those whose names start with `kwp-`.
 
 With TLS on the control plane and without `spec.tls.operatorAccess`, pools send the operator token to Coder over plain HTTP inside the cluster. See [Where the operator token goes](deploy-controller.md#where-the-operator-token-goes).

docs/reference/api/coderworkspacepool.md

@@ -34,7 +34,7 @@
 | `readyReplicas` | integer | ReadyReplicas is the number of those members whose workspace is ready. |
 | `updatedReplicas` | integer | UpdatedReplicas is the number of those members that use the resolved inputs. |
 | `boundReplicas` | integer | BoundReplicas is the number of members that claims hold. |
-| `usedReplicas` | integer | UsedReplicas is the number of members that someone may have used without a claim. They no longer count toward the pool, and the controller never deletes their workspaces. |
+| `usedReplicas` | integer | UsedReplicas is the number of members that someone may have used without a claim. They no longer count toward the pool, and the pool never deletes them. |
 | `pendingClaims` | integer | PendingClaims is the number of claims that wait for a workspace. |
 | `selector` | string | Selector is the label selector of the scale subresource, in string form: the label coder.com/workspace-pool-uid equal to the pool's UID. It matches no pods unless a template sets that label, so autoscalers must use External or Object metrics with target type AverageValue. |
 | `resolved` | [CoderWorkspacePoolResolved](#coderworkspacepoolresolved) | Resolved holds the inputs that the controller resolved in Coder. |

hack/e2e-workspace-pool.sh

@@ -1,8 +1,9 @@
 #!/usr/bin/env bash
-# CoderWorkspacePool E2E driver (#153 plan 10.4, phases P1 to P5). The Kind job runs it before the lifecycle driver,
-# which deletes namespace coder at its end. It imports its own template from the agent fixture. Once every phase
-# passed, it deletes its claim and pool, waits until their workspaces are gone, and restores the control plane's
-# spec.workspacePools. Every wait is bounded. Offline tests with stubbed tools: hack/e2e-workspace-pool_test.sh.
+# CoderWorkspacePool E2E driver (#153 plan 10.4, phases P1 to P8). The Kind job runs it before the lifecycle driver,
+# which deletes namespace coder at its end. It imports its own template from the agent fixture. P8 deletes its claim,
+# pool, and members and checks that their workspaces are deleted in Coder. Once every phase passed, it restores the
+# control plane's spec.workspacePools. Every wait is bounded. Offline tests with stubbed tools:
+# hack/e2e-workspace-pool_test.sh.
 set -euo pipefail
 umask 077 # $WORK holds the Coder token header and request bodies
 shopt -s inherit_errexit
@@ -21,6 +22,8 @@ POOL=${E2E_POOL:-e2e-pool}
 CLAIM=${E2E_CLAIM:-e2e-claim}
 TEMPLATE_APPLY_TIMEOUT=${E2E_TEMPLATE_APPLY_TIMEOUT:-600s}
 CLAIM_READY_SECONDS=${E2E_CLAIM_READY_SECONDS:-10}
+USED_SECONDS=${E2E_USED_SECONDS:-180} # Coder flushes usage once a minute, and a member reads Coder once a minute
+DRY_RUN_SETTLE_SECONDS=${E2E_DRY_RUN_SETTLE_SECONDS:-10}
 TIMEOUT=${E2E_TIMEOUT_SECONDS:-300}
 POLL=${E2E_POLL_SECONDS:-2}
 PORT=${E2E_CODER_LOCAL_PORT:-13001}
@@ -41,7 +44,8 @@ cleanup() {
   rm -f "$WORK/coder.hdr" "$WORK/coder-body.json" "$WORK/owner.json" "$WORK/owner-login.json" "$WORK/owner.hdr"
   printf '%s\n' "=== RECEIPT ($RESULT) ===" "owner=$OWNER/${OWNER_ID:-unset}" "pool_uid=${POOL_UID:-unset}" \
     "claim_ready_ms=${CLAIM_MS:-unset}" "bind_deadline=${BIND_DEADLINE:-unset}" "restart_member=${RESTART_MEMBER:-unset}" \
-    "live_workspaces_after_restart=${LIVE:-unset}" "member_workspace_rows=${ROWS:-unset}" "phase_seconds=${PHASES[*]:-unset}" |
+    "live_workspaces_after_restart=${LIVE:-unset}" "member_workspace_rows=${ROWS:-unset}" "used_member=${USED:-unset}" \
+    "used_seconds=${USED_S:-unset}" "scaled_down_workspaces=${SCALED_DOWN:-unset}" "phase_seconds=${PHASES[*]:-unset}" |
     tee "$WORK/receipt.txt"
 }
 trap cleanup EXIT
@@ -271,13 +275,84 @@ w=$(coder_api GET "/api/v2/workspaces/$BOUND_WS") || fail "cannot read the bound
 [[ $(deadline_of "$w") == "$BIND_DEADLINE" ]] || fail "the bound deadline changed after the bind extension"
 phase P5
 
-log "=== cleanup: claim, pool, and their workspaces; restore spec.workspacePools"
+log "=== P6: usage on an unclaimed workspace makes its member Used, and the pool releases it"
+members_json
+USED=$(jq -er '[.items[] | select(.spec.claimRef == null and .status.phase == "Ready")] | .[0].metadata.name' "$WORK/members.json") ||
+  fail "no unclaimed ready member to use"
+USED_WS=$(jq -er --arg m "$USED" '.items[] | select(.metadata.name == $m) | .status.workspaceID' "$WORK/members.json")
+coder_api POST "/api/v2/workspaces/$USED_WS/usage" >/dev/null || fail "cannot post usage for workspace $USED_WS"
+released() {
+  k -n "$NS" get coderworkspacepoolmember "$USED" -o json >"$WORK/used.json" || return 1
+  jq -e '(.status.conditions // [] | any(.type == "Used" and .status == "True")) and
+    ([.metadata.ownerReferences[]? | select(.controller == true)] | length == 0)' "$WORK/used.json" >/dev/null
+}
+USED_T0=$SECONDS
+TIMEOUT=$USED_SECONDS wait_until "member $USED Used=True and released by the pool" released
+USED_S=$((SECONDS - USED_T0))
+log "member $USED Used=True and released ${USED_S}s after the usage request"
+wait_until "4 unclaimed ready members and 1 bound after the release" pool_ready 4 1
+jq -e '.status.usedReplicas == 1' "$WORK/pool.json" >/dev/null || fail "pool usedReplicas is not 1: $(jq -c .status "$WORK/pool.json")"
+members_json
+phase P6
+
+log "=== P7: scale to 1 deletes the extra unclaimed workspaces in Coder; the bound and the used one stay"
+UNCLAIMED=$(jq -c --arg m "$USED" '[.items[] | select(.spec.claimRef == null and .metadata.name != $m) | .status.workspaceID]' \
+  "$WORK/members.json")
+k -n "$NS" scale coderworkspacepool "$POOL" --replicas=1 >/dev/null
+wait_until "1 unclaimed ready member and 1 bound" pool_ready 1 1
+three_members() { members_json && jq -e '.items | length == 3' "$WORK/members.json" >/dev/null; }
+wait_until "3 members: 1 unclaimed, 1 bound, 1 used" three_members
+KEPT_WS=$(jq -er --arg m "$USED" '[.items[] | select(.spec.claimRef == null and .metadata.name != $m)] |
+  if length == 1 then .[0].status.workspaceID else error end' "$WORK/members.json") || fail "want 1 unclaimed member after the scale to 1"
+deleted_in_coder() { # <workspace ID>: an include_deleted read shows a succeeded delete build
+  coder_api GET "/api/v2/workspaces/$1?include_deleted=true" |
+    jq -e '.latest_build.transition == "delete" and .latest_build.job.status == "succeeded"' >/dev/null
+}
+live_in_coder() { coder_api GET "/api/v2/workspaces/$1" | jq -e '.latest_build.transition == "start"' >/dev/null; }
+SCALED_DOWN=0
+for ws in $(jq -r --arg kept "$KEPT_WS" '.[] | select(. != $kept)' <<<"$UNCLAIMED"); do
+  wait_until "workspace $ws deleted in Coder" deleted_in_coder "$ws"
+  SCALED_DOWN=$((SCALED_DOWN + 1))
+done
+((SCALED_DOWN == 3)) || fail "the scale to 1 deleted $SCALED_DOWN workspaces, want 3"
+for ws in "$KEPT_WS" "$BOUND_WS" "$USED_WS"; do live_in_coder "$ws" || fail "workspace $ws is not live after the scale to 1"; done
+phase P7
+
+log "=== P8: claim, pool, and used member deletes; a server dry run creates nothing"
+absent() { local out; ! out=$(k -n "$NS" get "$@" -o name 2>&1) && grep -q '(NotFound)' <<<"$out"; } # <kind> <name>
+member_gone() { absent coderworkspacepoolmember "$1"; }
 k -n "$NS" delete coderworkspaceclaim "$CLAIM" --wait=false >/dev/null
+wait_until "member $BOUND of the deleted claim gone" member_gone "$BOUND"
+deleted_in_coder "$BOUND_WS" || fail "the claim's workspace $BOUND_WS is not deleted in Coder"
+log "claim delete: member $BOUND gone, workspace deleted in Coder"
+KEPT=$(jq -er --arg w "$KEPT_WS" '.items[] | select(.status.workspaceID == $w) | .metadata.name' "$WORK/members.json")
 k -n "$NS" delete coderworkspacepool "$POOL" --wait=false >/dev/null
-members_gone() { members_json && jq -e '.items | length == 0' "$WORK/members.json" >/dev/null; }
-wait_until "every member of $POOL gone" members_gone
-no_live() { owner_workspaces && jq -e '.workspaces | length == 0' "$WORK/owner-workspaces.json" >/dev/null; }
-wait_until "no live workspace of $OWNER" no_live
+wait_until "unclaimed member $KEPT of the deleted pool gone" member_gone "$KEPT"
+deleted_in_coder "$KEPT_WS" || fail "the unclaimed workspace $KEPT_WS is not deleted in Coder"
+k -n "$NS" get coderworkspacepoolmember "$USED" -o json >"$WORK/used.json" || fail "the used member $USED went with its pool"
+jq -e '.metadata.deletionTimestamp == null' "$WORK/used.json" >/dev/null || fail "the pool delete deletes the used member $USED"
+live_in_coder "$USED_WS" || fail "the used workspace $USED_WS is not live after the pool delete"
+log "pool delete: member $KEPT gone, workspace deleted in Coder; used member $USED and its workspace stay"
+k -n "$NS" delete coderworkspacepoolmember "$USED" --wait=false >/dev/null
+wait_until "the used member $USED gone" member_gone "$USED"
+deleted_in_coder "$USED_WS" || fail "deleting the used member $USED did not delete workspace $USED_WS"
+log "used member delete: member $USED gone, workspace deleted in Coder"
+k apply --dry-run=server -f "$WORK/pool-manifest.json" >"$WORK/dry-run.out" 2>&1 ||
+  fail "server dry run of the pool failed: $(cat "$WORK/dry-run.out")"
+grep -q 'server dry run' "$WORK/dry-run.out" || fail "kubectl did not report a server dry run: $(cat "$WORK/dry-run.out")"
+nothing_made() { # the pool, a member of it, or a live workspace of the owner after the dry run fails at once
+  absent coderworkspacepool "$POOL" || fail "the server dry run created pool $POOL"
+  k -n "$NS" get coderworkspacepoolmembers -o json | jq -e --arg p "$POOL" '[.items[] | select(.spec.poolName == $p)] |
+    length == 0' >/dev/null || fail "a member of $POOL exists after the server dry run"
+  owner_workspaces && jq -e '.workspaces | length == 0' "$WORK/owner-workspaces.json" >/dev/null ||
+    fail "the owner has live workspaces after the server dry run"
+  ((SECONDS - DRY_T0 >= DRY_RUN_SETTLE_SECONDS))
+}
+DRY_T0=$SECONDS
+wait_until "no pool, member, or workspace ${DRY_RUN_SETTLE_SECONDS}s after the server dry run" nothing_made
+phase P8
+
+log "=== cleanup: restore spec.workspacePools"
 k -n "$NS" patch codercontrolplane "$CP" --type=merge -p "$(jq -nc --argjson orig "$ORIGINAL_POOLS" '{spec: {workspacePools: $orig}}')" \
   >/dev/null || fail "cannot restore spec.workspacePools of $CP"
 kubectl -n "$OP_NS" wait --for=condition=Available deploy/coder-k8s --timeout=120s >/dev/null