Repository navigation
Expand file tree
/
Copy pathdebugrecord_test.go
More file actions
161 lines (150 loc) · 5.11 KB
/
Copy pathdebugrecord_test.go
File metadata and controls
161 lines (150 loc) · 5.11 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
package provider
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
lanes "github.com/Agent-Field/codeaf/internal/lane"
"github.com/Agent-Field/codeaf/internal/trace"
)
// recordingRun turns the debug record on for ONE run and points the state
// root at a disposable folder, so a test cannot write into whoever ran it
// and cannot leave the process-wide switch on for the tests that follow.
func recordingRun(t *testing.T) context.Context {
t.Helper()
t.Setenv("CODEAF_HOME", t.TempDir())
ctx := trace.Begin(context.Background())
if trace.EnableRun(ctx) == "" {
t.Fatal("EnableRun did not take this run")
}
return ctx
}
func TestACompletedCallWritesItsBodiesIntoTheDebugRecord(t *testing.T) {
ctx := recordingRun(t)
client, _ := newTestClient(t, Config{
SupportsParameter: func(string, string) (bool, bool) { return true, true },
})
if _, err := client.CompleteWithMessages(ctx, userMessages("hello from the record")); err != nil {
t.Fatal(err)
}
folder := filepath.Join(trace.Dir(trace.RunFrom(ctx)), trace.CallsDirName)
entries, err := os.ReadDir(folder)
if err != nil {
t.Fatalf("the debug record wrote no calls folder: %v", err)
}
if len(entries) != 1 {
t.Fatalf("one completion should leave one body file, got %d", len(entries))
}
raw, err := os.ReadFile(filepath.Join(folder, entries[0].Name()))
if err != nil {
t.Fatal(err)
}
var document map[string]any
if err := json.Unmarshal(raw, &document); err != nil {
t.Fatalf("the body file is not JSON: %v\n%s", err, raw)
}
if document["kind"] != "call" {
t.Fatalf("kind = %v, want call", document["kind"])
}
if document["model"] != "sim/model" {
t.Errorf("model = %v, want sim/model", document["model"])
}
if document["call"] == "" || document["call"] == nil {
t.Error("the body file named no call id")
}
if !strings.Contains(string(raw), "hello from the record") {
t.Errorf("the request body did not keep what went out:\n%s", raw)
}
if !strings.Contains(string(raw), "ok") {
t.Errorf("the response body did not keep what came back:\n%s", raw)
}
}
func TestACallWithTheRecordOffWritesNoBodyFile(t *testing.T) {
t.Setenv("CODEAF_HOME", t.TempDir())
ctx := trace.Begin(context.Background())
client, _ := newTestClient(t, Config{})
if _, err := client.CompleteWithMessages(ctx, userMessages("hello")); err != nil {
t.Fatal(err)
}
folder := filepath.Join(trace.Dir(trace.RunFrom(ctx)), trace.CallsDirName)
if _, err := os.Stat(folder); !os.IsNotExist(err) {
t.Fatalf("the record was off and still wrote %s (%v)", folder, err)
}
}
func TestALaneChoiceIsADecisionInTheDebugRecord(t *testing.T) {
ctx := recordingRun(t)
recordLaneChoice(ctx, "sim/model", lanes.Choice{
Order: []string{"together", "fireworks"},
Ignore: []string{"friendli"},
Why: "together has been the fastest this week",
})
event := onlyEvent(t, ctx)
if event["kind"] != "decision" || event["decision"] != "lane" {
t.Fatalf("event = %v, want a lane decision", event)
}
if event["choice"] != "together" {
t.Errorf("choice = %v, want together", event["choice"])
}
if event["subject"] != "sim/model" {
t.Errorf("subject = %v, want the model", event["subject"])
}
alts, _ := event["alternatives"].([]any)
if len(alts) != 2 || alts[0] != "fireworks" || alts[1] != "not friendli" {
t.Errorf("alternatives = %v, want fireworks and not friendli", alts)
}
if event["reason"] != "together has been the fastest this week" {
t.Errorf("reason = %v", event["reason"])
}
}
func TestAFiredHedgeAndARefusedHedgeAreBothDecisions(t *testing.T) {
ctx := recordingRun(t)
race := &hedgeRace{base: ctx, model: "sim/model"}
race.recordHedge("together", "the first machine went quiet", true)
race.recordHedge("fireworks", "budget", false)
events := allEvents(t, ctx)
if len(events) != 2 {
t.Fatalf("got %d events, want a fired hedge and a refused one", len(events))
}
if events[0]["kind"] != "decision" || events[0]["decision"] != "hedge" {
t.Fatalf("first event = %v, want a hedge decision", events[0])
}
if events[0]["choice"] != "together" {
t.Errorf("fired choice = %v, want together", events[0]["choice"])
}
if events[1]["choice"] != "no second machine" {
t.Errorf("refused choice = %v, want no second machine", events[1]["choice"])
}
if events[1]["reason"] != "budget" {
t.Errorf("refused reason = %v, want budget", events[1]["reason"])
}
}
func onlyEvent(t *testing.T, ctx context.Context) map[string]any {
t.Helper()
events := allEvents(t, ctx)
if len(events) != 1 {
t.Fatalf("got %d events, want one", len(events))
}
return events[0]
}
func allEvents(t *testing.T, ctx context.Context) []map[string]any {
t.Helper()
path := filepath.Join(trace.Dir(trace.RunFrom(ctx)), trace.EventsFileName)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("the debug record wrote no events file: %v", err)
}
var events []map[string]any
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
if line == "" {
continue
}
var event map[string]any
if err := json.Unmarshal([]byte(line), &event); err != nil {
t.Fatalf("an events line is not JSON: %v\n%s", err, line)
}
events = append(events, event)
}
return events
}