🤖 feat: replace old unclaimed pool workspaces and reap idle claimed ones, both opt-in (#153 PR 15a)

api/v1alpha1/coderworkspaceclaim_types.go

@@ -10,7 +10,8 @@ const (
 	// CoderWorkspaceClaimPhaseBound means a member holds the claim.
 	CoderWorkspaceClaimPhaseBound = "Bound"
 	// CoderWorkspaceClaimPhaseLost means someone deleted the claim's
-	// workspace. It is final: a claim never binds again.
+	// workspace, or the pool deleted it after its idle timeout. It is final:
+	// a claim never binds again.
 	CoderWorkspaceClaimPhaseLost = "Lost"
 
 	// CoderWorkspaceClaimConditionReady is True when the claim is bound and

api/v1alpha1/coderworkspacepool_types.go

@@ -123,6 +123,15 @@ type CoderWorkspacePoolClaims struct {
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="claims.mode is immutable"
 	// +optional
 	Mode string `json:"mode,omitempty"`
+	// IdleTimeoutSeconds deletes a claimed workspace, with its data, once
+	// both the bind and the workspace's last use in Coder (last_used_at) are
+	// this long ago. The claim then becomes Lost with reason IdleTimeout.
+	// Leave it out to keep claimed workspaces until their claims are deleted.
+	// Unclaimed workspaces are idle by design and never count.
+	// +kubebuilder:validation:Minimum=600
+	// +kubebuilder:validation:Maximum=2592000
+	// +optional
+	IdleTimeoutSeconds *int32 `json:"idleTimeoutSeconds,omitempty"`
 }
 
 // CoderWorkspacePoolSpec defines a pool of ready Coder workspaces from one
@@ -198,6 +207,14 @@ type CoderWorkspacePoolSpec struct {
 	// +kubebuilder:validation:Maximum=7200
 	// +optional
 	ReadyTimeoutSeconds *int32 `json:"readyTimeoutSeconds,omitempty"`
+	// MaxUnclaimedAgeSeconds replaces an unclaimed workspace that has been
+	// ready this long, new before old: the old workspace stays claimable
+	// until its replacement is ready. Leave it out to keep unclaimed
+	// workspaces until a claim, a scale-down, or an input change.
+	// +kubebuilder:validation:Minimum=3600
+	// +kubebuilder:validation:Maximum=2592000
+	// +optional
+	MaxUnclaimedAgeSeconds *int32 `json:"maxUnclaimedAgeSeconds,omitempty"`
 	// Claims configures how the pool hands out workspaces.
 	// +kubebuilder:default={}
 	// +optional

config/crd/bases/coder.com_coderworkspacepools.yaml

@@ -67,6 +67,17 @@ spec:
                 default: {}
                 description: Claims configures how the pool hands out workspaces.
                 properties:
+                  idleTimeoutSeconds:
+                    description: |-
+                      IdleTimeoutSeconds deletes a claimed workspace, with its data, once
+                      both the bind and the workspace's last use in Coder (last_used_at) are
+                      this long ago. The claim then becomes Lost with reason IdleTimeout.
+                      Leave it out to keep claimed workspaces until their claims are deleted.
+                      Unclaimed workspaces are idle by design and never count.
+                    format: int32
+                    maximum: 2592000
+                    minimum: 600
+                    type: integer
                   mode:
                     default: Reserve
                     description: |-
@@ -105,6 +116,16 @@ spec:
                 maximum: 20
                 minimum: 1
                 type: integer
+              maxUnclaimedAgeSeconds:
+                description: |-
+                  MaxUnclaimedAgeSeconds replaces an unclaimed workspace that has been
+                  ready this long, new before old: the old workspace stays claimable
+                  until its replacement is ready. Leave it out to keep unclaimed
+                  workspaces until a claim, a scale-down, or an input change.
+                format: int32
+                maximum: 2592000
+                minimum: 3600
+                type: integer
               ownerUserID:
                 description: |-
                   OwnerUserID is the UUID of the Coder user who owns every workspace of

dist/install.yaml

@@ -7312,6 +7312,17 @@ spec:
                 default: {}
                 description: Claims configures how the pool hands out workspaces.
                 properties:
+                  idleTimeoutSeconds:
+                    description: |-
+                      IdleTimeoutSeconds deletes a claimed workspace, with its data, once
+                      both the bind and the workspace's last use in Coder (last_used_at) are
+                      this long ago. The claim then becomes Lost with reason IdleTimeout.
+                      Leave it out to keep claimed workspaces until their claims are deleted.
+                      Unclaimed workspaces are idle by design and never count.
+                    format: int32
+                    maximum: 2592000
+                    minimum: 600
+                    type: integer
                   mode:
                     default: Reserve
                     description: |-
@@ -7350,6 +7361,16 @@ spec:
                 maximum: 20
                 minimum: 1
                 type: integer
+              maxUnclaimedAgeSeconds:
+                description: |-
+                  MaxUnclaimedAgeSeconds replaces an unclaimed workspace that has been
+                  ready this long, new before old: the old workspace stays claimable
+                  until its replacement is ready. Leave it out to keep unclaimed
+                  workspaces until a claim, a scale-down, or an input change.
+                format: int32
+                maximum: 2592000
+                minimum: 3600
+                type: integer
               ownerUserID:
                 description: |-
                   OwnerUserID is the UUID of the Coder user who owns every workspace of

docs/how-to/workspace-pools.md

@@ -62,6 +62,8 @@ kubectl -n coder get coderworkspacepools,coderworkspacepoolmembers
 
 Each member owns one workspace named `kwp-` plus 28 hex characters of the member's UID. Do not rename these workspaces, and do not use an unclaimed one directly. The pool releases a workspace that someone used without a claim, and never deletes it on its own. Opening an unclaimed workspace's page in the Coder dashboard counts as use too: the page asks the workspace agent for its containers, and in Coder v2.37.2 that request moves `last_used_at` ([#444](https://github.com/coder/coder-k8s/issues/444)). The workspaces list does not.
 
+An unclaimed workspace stays until a claim takes it, a scale-down removes it, or the pool's inputs change. With `version.active: true`, a new active version of the template is such a change. To also replace workspaces that have been ready for a long time, set `maxUnclaimedAgeSeconds` (at least 3600). The pool replaces outdated workspaces new before old: it builds the replacement first, and the old workspace stays claimable until the replacement is ready. So the replacement needs room under `maxWorkspaces`.
+
 `replicas` has a scale subresource, so `kubectl scale`, HPA, and KEDA can change it:
 
 ```bash
@@ -82,6 +84,11 @@ spec:
 
 A claim with a ready workspace in its pool is `Bound` and `Ready` at once. In the Kind end-to-end test, that took about 1 s. `status.ownerName` and `status.workspaceName` name the workspace. When the workspace has an autostop deadline, the operator moves it once at the bind, to the template's default TTL from now, at least 30 minutes. A workspace's max deadline (from a Premium autostop requirement) can shorten or prevent that extension, and so can a refusal by Coder: the operator then keeps the deadline and does not try again. After that, Coder's normal autostop applies. One claim is one workspace, and a claim never binds a second time. Delete the claim when you are done: that deletes its workspace.
 
+To delete claimed workspaces that nobody uses, set `claims.idleTimeoutSeconds` on the pool (at least 600). Without it, the operator never deletes a claimed workspace on its own.
+
+!!! warning "Idle reaping deletes the workspace and its data"
+    The pool deletes a claimed workspace once both the bind and the workspace's `last_used_at` in Coder are older than `claims.idleTimeoutSeconds`. A stopped workspace counts as idle. The claim then becomes `Lost` with reason `IdleTimeout`, and it never binds again. Right before the delete build, the operator reads the workspace once more. If that read shows a use after the pool's check, or no `last_used_at`, the operator keeps the workspace in Coder and only releases its member. The claim message covers both outcomes: the pool deletes the workspace unless its last check sees a use, and a kept workspace stays in Coder under its `kwp-` name, owned by the pool owner. The message names both, so `coder ssh <owner>/<workspace>` still reaches a kept workspace. To delete kept workspaces later, see [Cleanup](#cleanup). Coder writes `last_used_at` about once a minute, so a session that starts less than a minute before the delete build can still be lost.
+
 ## 5. Grant access
 
 The operator's own RBAC comes with the install. People who use pools need, in their namespace:
@@ -150,7 +157,7 @@ KEDA's operator creates the HPA for a `ScaledObject`, and its own role can updat
 
 ## Cost
 
-Each unclaimed workspace runs all the time, so a pool of `replicas: N` costs N running workspaces plus the claimed ones. A claimed workspace keeps a normal autostop deadline after the bind, so Coder stops it when nobody uses it. A template with a default TTL of 0 has no deadline: its claimed workspaces run until someone stops or deletes them.
+Each unclaimed workspace runs all the time, so a pool of `replicas: N` costs N running workspaces plus the claimed ones. A claimed workspace keeps a normal autostop deadline after the bind, so Coder stops it when nobody uses it. A template with a default TTL of 0 has no deadline: its claimed workspaces run until someone stops or deletes them. For such a template, `claims.idleTimeoutSeconds` is the only automatic limit.
 
 Further limits: `maxInFlight` (default 5) caps the members of a pool that build at the same time, and a pool stops creating with `Stalled` and `MemberFailureLimit` after 5 failed members in a row. To cap the number of pools and claims in a namespace, use a `ResourceQuota` on `count/coderworkspacepools.coder.com` and `count/coderworkspaceclaims.coder.com`.
 
@@ -164,6 +171,7 @@ The operator sends every pool request with its operator token. Measured in Kind
 | A member that builds | 24 | The same two requests every 5 s, until the member is ready |
 | A pool that follows the active version | 6 | The owner, its organization membership, the organization (twice), the template, and the version, once a minute |
 | A pool of 10 ready members | 26 | 10 × 2 + 6 |
+| A claimed workspace with `claims.idleTimeoutSeconds`, after the timeout since the bind | 2 per pool check | `GET /api/v2/workspaces/<id>` and `GET /api/v2/users/me` each time the pool checks it, until it is idle or the claim is gone |
 
 A pool sends no request to `/api/v2/files`, so Coder's limit of 12 file requests per minute never applies. Coder's general limit of 512 requests per minute counts each user and each path. Each member reads its workspace on its own path, but every member also reads `/api/v2/users/me`, one path for all members of the operator. From the rates above, about 500 ready members, or about 40 members that build, reach that limit (not tested at that size). Coder then answers 429, and the operator backs off and retries.
 

docs/reference/api/coderworkspacepool.md

@@ -23,6 +23,7 @@
 | `replicas` | integer | Replicas is the number of unclaimed ready workspaces to keep. Waiting claims get a workspace built on demand on top of this number. The scale subresource writes this field, so kubectl scale, HPA, and KEDA can resize the pool within the same bounds. |
 | `maxInFlight` | integer | MaxInFlight is the number of members of this pool that build at the same time. |
 | `readyTimeoutSeconds` | integer | ReadyTimeoutSeconds bounds the time from a member's creation until its workspace is ready. A member that is not ready in time fails, and the controller replaces it. |
+| `maxUnclaimedAgeSeconds` | integer | MaxUnclaimedAgeSeconds replaces an unclaimed workspace that has been ready this long, new before old: the old workspace stays claimable until its replacement is ready. Leave it out to keep unclaimed workspaces until a claim, a scale-down, or an input change. |
 | `claims` | [CoderWorkspacePoolClaims](#coderworkspacepoolclaims) | Claims configures how the pool hands out workspaces. |
 
 ## Status
@@ -63,6 +64,7 @@ CoderWorkspacePoolClaims configures how the pool hands out workspaces.
 | Field | Type | Description |
 | --- | --- | --- |
 | `mode` | string | Mode is how a claim gets its workspace. Reserve binds a ready workspace to the claim, and the workspace stays with the pool owner. |
+| `idleTimeoutSeconds` | integer | IdleTimeoutSeconds deletes a claimed workspace, with its data, once both the bind and the workspace's last use in Coder (last_used_at) are this long ago. The claim then becomes Lost with reason IdleTimeout. Leave it out to keep claimed workspaces until their claims are deleted. Unclaimed workspaces are idle by design and never count. |
 
 ### CoderWorkspacePoolFailure
 

internal/controller/coderworkspacepool_api_test.go

@@ -81,6 +81,11 @@ func TestCoderWorkspacePoolAPIValidation(t *testing.T) {
 		{name: "too many in flight", mutate: func(s *coderv1alpha1.CoderWorkspacePoolSpec) { s.MaxInFlight = ptrTo(int32(21)) }, wantErr: "spec.maxInFlight"},
 		{name: "short ready timeout", mutate: func(s *coderv1alpha1.CoderWorkspacePoolSpec) { s.ReadyTimeoutSeconds = ptrTo(int32(59)) }, wantErr: "spec.readyTimeoutSeconds"},
 		{name: "long ready timeout", mutate: func(s *coderv1alpha1.CoderWorkspacePoolSpec) { s.ReadyTimeoutSeconds = ptrTo(int32(7201)) }, wantErr: "spec.readyTimeoutSeconds"},
+		{name: "refresh and idle bounds", mutate: func(s *coderv1alpha1.CoderWorkspacePoolSpec) {
+			s.MaxUnclaimedAgeSeconds, s.Claims.IdleTimeoutSeconds = ptrTo(int32(3600)), ptrTo(int32(2592000))
+		}},
+		{name: "young max unclaimed age", mutate: func(s *coderv1alpha1.CoderWorkspacePoolSpec) { s.MaxUnclaimedAgeSeconds = ptrTo(int32(3599)) }, wantErr: "spec.maxUnclaimedAgeSeconds"},
+		{name: "short idle timeout", mutate: func(s *coderv1alpha1.CoderWorkspacePoolSpec) { s.Claims.IdleTimeoutSeconds = ptrTo(int32(599)) }, wantErr: "spec.claims.idleTimeoutSeconds"},
 		{name: "long preset name", mutate: func(s *coderv1alpha1.CoderWorkspacePoolSpec) { s.PresetName = strings.Repeat("p", 65) }, wantErr: "spec.presetName"},
 		{name: "too many parameters", mutate: func(s *coderv1alpha1.CoderWorkspacePoolSpec) { s.Parameters = tooManyParameters }, wantErr: "spec.parameters"},
 		{name: "duplicate parameter", mutate: func(s *coderv1alpha1.CoderWorkspacePoolSpec) {

internal/controller/coderworkspacepool_claim.go

@@ -60,7 +60,7 @@ func poolWant(pool *coderv1alpha1.CoderWorkspacePool, claims []coderv1alpha1.Cod
 // reconcileOrphanClaims writes the status of the claims of a pool that is
 // gone or being deleted: a claim that a member holds stays as its member
 // says, so a bound claim stays Bound, and the others wait with
-// PoolNotFound.
+// PoolNotFound. It also finishes the idle reaps that the pool committed.
 func (r *CoderWorkspacePoolReconciler) reconcileOrphanClaims(ctx context.Context, namespace, poolName string) error {
 	claims, err := r.poolClaims(ctx, namespace, poolName)
 	if err != nil || len(claims) == 0 {
@@ -70,6 +70,9 @@ func (r *CoderWorkspacePoolReconciler) reconcileOrphanClaims(ctx context.Context
 	if err := r.APIReader.List(ctx, all, client.InNamespace(namespace)); err != nil {
 		return fmt.Errorf("list pool members in namespace %s: %w", namespace, err)
 	}
+	if err := r.finishIdleReaps(ctx, claims, all.Items, r.Clock.Now()); err != nil {
+		return err
+	}
 	return r.writeClaimStatuses(ctx, poolName, true, claims, all.Items, "", "")
 }
 
@@ -143,7 +146,7 @@ func (r *CoderWorkspacePoolReconciler) refreshWaitingClaims(
 // otherwise. It also sets the pool label again, in case it drifted.
 func (r *CoderWorkspacePoolReconciler) bindClaims(
 	ctx context.Context, pool *coderv1alpha1.CoderWorkspacePool, claims []coderv1alpha1.CoderWorkspaceClaim,
-	all []coderv1alpha1.CoderWorkspacePoolMember, counts *poolMemberCounts, ownerName string,
+	all []coderv1alpha1.CoderWorkspacePoolMember, counts *poolMemberCounts, ownerName string, now time.Time,
 ) (skipped bool, err error) {
 	if ownerName == "" {
 		return false, fmt.Errorf("assertion failed: pool %s/%s binds claims without the owner's Coder username", pool.Namespace, pool.Name)
@@ -163,7 +166,7 @@ func (r *CoderWorkspacePoolReconciler) bindClaims(
 		return cmp.Or(m.Status.ReadyTime, &m.CreationTimestamp).Time
 	}
 	slices.SortStableFunc(free, func(a, b *coderv1alpha1.CoderWorkspacePoolMember) int {
-		aOld, bOld := poolMemberInputMismatch(a, pool) != "", poolMemberInputMismatch(b, pool) != ""
+		aOld, bOld := !poolMemberCurrent(a, pool, now), !poolMemberCurrent(b, pool, now)
 		if aOld != bOld {
 			if aOld {
 				return 1
@@ -206,7 +209,7 @@ func (r *CoderWorkspacePoolReconciler) bindClaims(
 				j++
 				continue
 			}
-			unused, err := poolMemberUnusedNow(ctx, sdk, free[j], r.Clock.Now())
+			unused, err := poolMemberUnusedNow(ctx, sdk, free[j], now)
 			if err != nil {
 				return false, err
 			}
@@ -229,7 +232,7 @@ func (r *CoderWorkspacePoolReconciler) bindClaims(
 				return false, fmt.Errorf("reserve pool member %s for workspace claim %s/%s: %w", m.Name, claim.Namespace, claim.Name, err)
 			}
 		}
-		current := poolMemberInputMismatch(m, pool) == ""
+		current := poolMemberCurrent(m, pool, now)
 		bound := m.DeepCopy()
 		bound.Spec.ClaimRef = &coderv1alpha1.CoderWorkspaceClaimReference{Name: claim.Name, UID: string(claim.UID)}
 		bound.OwnerReferences = slices.DeleteFunc(bound.OwnerReferences, func(o metav1.OwnerReference) bool { return o.UID == pool.UID })