Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 16 additions & 5 deletions cmd/pipelines/test.go
Original file line number Diff line number Diff line change
Expand Up @@ -459,7 +459,7 @@ func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipel
for {
response, err := w.Pipelines.GetUpdateByPipelineIdAndUpdateId(ctx, pipelineID, updateID)
if err != nil {
return "", err
return "", waitReadError(ctx, err)
}
if response.Update == nil {
return "", fmt.Errorf("no update found with id %s for pipeline %s", updateID, pipelineID)
Expand All @@ -470,7 +470,7 @@ func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipel
// Reading from the last timestamp returns its events again; the run ignores duplicates.
events, err := readTestEvents(ctx, w, pipelineID, updateID, lastTimestamp)
if err != nil {
return "", err
return "", waitReadError(ctx, err)
}
if len(events) > 0 {
lastTimestamp = events[len(events)-1].Timestamp
Expand All @@ -482,7 +482,7 @@ func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipel
// Events can become readable after later ones, so read all of them once more.
events, err := readTestEvents(ctx, w, pipelineID, updateID, "")
if err != nil {
return "", err
return "", waitReadError(ctx, err)
}
addTestEvents(run, printer, events)
return state, nil
Expand All @@ -497,6 +497,18 @@ func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipel
}
}

// waitReadError adds the cause of ctx, such as errTestWaitTimeout, to a read error that ctx's
// cancellation caused. The SDK rate limiter fails a request on a canceled ctx before sending it,
// with the bare context error that hides the cause.
func waitReadError(ctx context.Context, err error) error {
if errors.Is(err, context.Canceled) {
if cause := context.Cause(ctx); cause != nil && !errors.Is(err, cause) {
return fmt.Errorf("%w: %w", cause, err)
}
}
return err
}

// addTestEvents adds events to the run and prints each case that gets its final outcome.
func addTestEvents(run *testRun, printer *testProgressPrinter, events []testPipelineEvent) {
for _, e := range events {
Expand Down Expand Up @@ -756,8 +768,7 @@ of the deployed snapshot, which changes with each deployment.`,
defer stopReads()
state, err := waitForTestUpdate(readCtx, target.w, target.pipelineID, updateID, run, printer)
if err != nil {
// A read that the timeout interrupts fails with the bare context error, so the cause tells.
isTimeout := errors.Is(context.Cause(readCtx), errTestWaitTimeout)
isTimeout := errors.Is(err, errTestWaitTimeout) || errors.Is(context.Cause(readCtx), errTestWaitTimeout)
resumeCmd := target.resumeCommand(cmd, updateID)
if !isTimeout && waitCtx.Err() == nil {
// The SDK client already retries transient errors, so the read failed for good.
Expand Down
29 changes: 29 additions & 0 deletions cmd/pipelines/test_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"os"
Expand Down Expand Up @@ -341,6 +342,34 @@ func TestWaitForTestUpdateTimeout(t *testing.T) {
assert.ErrorIs(t, err, errTestWaitTimeout)
}

// A timeout that passes between reads fails the next read in the SDK rate limiter, with a bare
// context error; the wait still reports the timeout.
func TestWaitForTestUpdateTimeoutBeforeRead(t *testing.T) {
server := testserver.New(t)
w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"})
require.NoError(t, err)
ctx, cancel := context.WithCancelCause(t.Context())
cancel(errTestWaitTimeout)
printer := &testProgressPrinter{out: &bytes.Buffer{}, shortID: func(id string) string { return id }}
_, err = waitForTestUpdate(ctx, w, "p1", "u1", newTestRun(), printer)
assert.ErrorIs(t, err, errTestWaitTimeout)
}

func TestWaitReadError(t *testing.T) {
ctx, cancel := context.WithCancelCause(t.Context())
readErr := errors.New("403 forbidden")
// A real error is kept even if ctx is canceled by now.
cancel(errTestWaitTimeout)
assert.Equal(t, readErr, waitReadError(ctx, readErr))

// A context error gets the cause, and keeps the read's own message.
canceled := fmt.Errorf("failed to fetch pipeline events: failed in rate limiter: %w", context.Canceled)
err := waitReadError(ctx, canceled)
assert.ErrorIs(t, err, errTestWaitTimeout)
assert.ErrorIs(t, err, context.Canceled)
assert.Contains(t, err.Error(), "failed to fetch pipeline events")
}

// The timeout also interrupts a read in flight, so an update that ends after it is not reported.
func TestWaitForTestUpdateTimeoutInterruptsRead(t *testing.T) {
server := testserver.New(t)
Expand Down
Loading