Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions agents/anyplot/telemetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -384,13 +384,15 @@ def _library(value: str) -> str:
"downloaded_code",
"downloaded_page",
"opened_preview",
"opened_data_view",
"downloaded_data",
"saved_session",
"feedback",
]
"""What the user did with a result; `downloaded_page` is a JavaScript plot's interactive `plot.html` (JS-4b), and
`opened_preview` that page shown in the card (JS-4b-2), a look and no keep. The BFF (`api/routers/agent.py`) and the
page (`app/src/lib/agent.ts`) carry the same set."""
`opened_preview` that page shown in the card (JS-4b-2), a look and no keep. `opened_data_view` is the data panel's
full view of every row (`.view_all()`, version 0: the session's data, not a result), a look and no keep either. The
BFF (`api/routers/agent.py`) and the page (`app/src/lib/agent.ts`) carry the same set."""
OUTCOME_ACTIONS: tuple[str, ...] = get_args(OutcomeAction)
Reaction = Literal["thumbs_up", "thumbs_down", "bug", "idea"]
"""The feedback reactions (`core/database/models.py` `FEEDBACK_REACTIONS`; pinned by a test)."""
Expand Down
26 changes: 26 additions & 0 deletions agents/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
| `POST /v1/sessions/{sid}/dataset/undo` | | `{preview, profile, bindings, warnings, context_lines, roles, suggestion, version}`: the dataset version before the root's last `prepare_data` or `generate_data` is the current one again, its bindings carried over (`dataset_versions.py`); unconfirmed, like any version the user has not confirmed | `409 run_active`, `422 nothing_to_undo` |
| `POST /v1/sessions/{sid}/dataset/to_long` | | The undo route's shape plus `changes`: the page's `.to_long()` applies the wide-table suggestion, recomputed on the server, as the next dataset version with the new columns bound to the roles they were made for; no model call | `409 run_active`, `422 no_suggestion` |
| `POST /v1/sessions/{sid}/dataset/copy` | `{from_session}` | The paste route's dataset shape (version 1) plus `confirmed: true`, `complete`, `missing_roles` and `copied_from` (the source's version number): the admin fan-out's copy of the confirmed current data version of `from_session`, a session of the same user and spec, into this fan-out session, byte for byte, with its bindings, their confirmation and the session language; nothing parsed or judged, no model call, no paste counted | `404 session_expired` (either session, also another user's), `409 run_active` (either session), `409 has_dataset` (the target has data), `422 no_dataset`, `422 unconfirmed`, `422 invalid` (the same session, a target that is no fan-out session, another spec, refused bindings), `429 rate_limited` / `429 daily_limit` (`COPIES_PER_MINUTE` / `_PER_DAY`; never for an admin), `503 capacity` |
| `GET /v1/sessions/{sid}/dataset/data.csv` | | the current dataset version's canonical CSV as stored (at most 4 MiB, `parse.MAX_CANONICAL_BYTES`), for the data panel's full view; no model call, no state write | `422 no_dataset` |
| `POST /v1/sessions/{sid}/messages?theme=` | `{text}` or `{action}` (`create_plot`, `summarize_data`); `theme` (`light`, `dark`, optional) is the site theme the page shows, which a plot run of the turn renders unless the user asked for a theme (`pipeline.run_theme`) | SSE `anyplot/1`; the `done` event carries `dataset` (with `roles` and `changes`) after a turn whose `prepare_data` or `generate_data` changed the data, version 1 when `generate_data` made the session's first data, and `language` (`{language, source}`) after every run, and `suggestion` (the `.to_long()` offer in the session language, or null) after every run on a session with data | `413 too_long`, `409 run_active`, `422` for a `theme` other than `light` or `dark` |
| `POST /v1/sessions/{sid}/cancel` | | `204` | |
| `POST /v1/sessions/{sid}/outcome` | `{action, version?, reaction?}` (closed sets: `telemetry.OutcomeAction`, `telemetry.Reaction`, the latter with `feedback` only) | `204`: the content-free `outcome` telemetry line (`anyplot/telemetry.py`) | `422`, `429 rate_limited` / `429 daily_limit` (`OUTCOMES_PER_MINUTE` / `_PER_DAY`) |
Expand Down Expand Up @@ -1940,6 +1941,31 @@ async def copy_dataset(
)


@app.get("/v1/sessions/{sid}/dataset/data.csv", dependencies=v1_dependencies)
async def dataset_csv(sid: SessionId, user: User, runtime: RuntimeDep) -> Response:
"""The current dataset version's canonical `data.csv` as stored: the data panel's full view (`.view_all()`).

The bytes the next plot run reads (`StoredDataset.csv`, at most `parse.MAX_CANONICAL_BYTES`), never formatted
again, so the page shows every value as the data holds it. A version's `GET .../artifacts/data.csv` is the data
a plot was made from; this is the data the panel shows now, also in step 1 before any plot. A read with no model
call and no state write, so no `_active` check: it neither races nor blocks a turn, and when a turn stores a
new version meanwhile, the page fetches again. `422 no_dataset` without data, as on the other dataset routes.
"""
services = get_services()
session = await runtime.session(user, sid)
view = read_session(session.state)
if view is None:
raise AgentsError(404, "session_expired")
stored = services.datasets.get(view.dataset_id, sid) if view.dataset_id else None
if stored is None:
raise AgentsError(422, "no_dataset")
return Response(
stored.csv.encode("utf-8"),
media_type=ARTIFACT_TYPES["data.csv"],
headers={**_NO_STORE, "X-Content-Type-Options": "nosniff"},
)


def _suggestion(view: SessionView, profile: DatasetProfile, language: str | None = None) -> dict[str, Any] | None:
"""The wide-table suggestion for a dataset answer (`data/shape.py`), or None.

Expand Down
45 changes: 44 additions & 1 deletion api/routers/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,11 @@
_BUNDLE_GZIP_LEVEL = 6
"""zlib's default level: base64 of PNGs shrinks by about a quarter at level 6 and no further at
the API's middleware level 9, which costs about a fifth more time."""
MAX_DATASET_CSV_BYTES = 4 * 1024 * 1024
"""The cap of the current data version's `data.csv` the BFF relays: the canonical CSV a parse may produce
(`agents/anyplot/data/parse.py` `MAX_CANONICAL_BYTES`, copied because the API image does not ship the agents
package; pinned equal by a test). The agents service never stores a larger one; above it the BFF answers
`502 too_large`."""

# Local dev servers on any port. api/main.py passes this same constant to
# CORSMiddleware as `allow_origin_regex`, so the CSRF Origin check and CORS
Expand Down Expand Up @@ -621,6 +626,7 @@ class RenderThemeBody(_StrictBody):
"downloaded_code",
"downloaded_page",
"opened_preview",
"opened_data_view",
"downloaded_data",
"saved_session",
"feedback",
Expand Down Expand Up @@ -951,7 +957,13 @@ def _prepare_bundle(content: bytes, compress: bool) -> bytes | None:
return None
except ValueError, RecursionError:
return None
return gzip.compress(content, compresslevel=_BUNDLE_GZIP_LEVEL, mtime=0) if compress else content
return _gzipped(content) if compress else content


def _gzipped(content: bytes) -> bytes:
"""`content` gzip-compressed. The callers run it in a worker thread: the API's `GZipMiddleware` would compress
a buffered answer of megabytes on the event loop, and it leaves one that has a `Content-Encoding` as it is."""
return gzip.compress(content, compresslevel=_BUNDLE_GZIP_LEVEL, mtime=0)


def _bundle_filename(sid: str) -> str:
Expand Down Expand Up @@ -1339,6 +1351,37 @@ async def copy_dataset(sid: SessionId, body: CopyDatasetBody, ctx: Ctx, client:
return await _call_upstream(client, ctx, "POST", f"/sessions/{sid}/dataset/copy", json_body=payload)


@router.get("/sessions/{sid}/dataset/data.csv")
async def get_dataset_csv(sid: SessionId, request: Request, ctx: Ctx, client: Client) -> Response:
"""The current data version's canonical `data.csv`: the data panel's full view of every row (`.view_all()`).

Read-only, the bytes as the agents service stores them. The BFF reads them whole up to `MAX_DATASET_CSV_BYTES`
(`502 too_large` above), compresses them in a worker thread when the browser takes gzip, as for the bundle, and
answers them with its own media type. `422 no_dataset` before any data, `404 session_expired`, and `502
upstream` when the service cannot be reached; an agents service without the route answers `404 not_found`.
"""
path = f"/sessions/{sid}/dataset/data.csv"
try:
response = await _open_stream(client, ctx, "GET", path)
try:
body = await _read_capped(response, MAX_DATASET_CSV_BYTES)
finally:
await response.aclose()
except _UNREACHABLE as exc:
_log_unreachable(ctx, "GET", path, exc)
raise AgentHTTPError(502, "upstream", ctx.request_id) from exc
if body is None:
logger.warning("agent data.csv over %d bytes refused (ref %s)", MAX_DATASET_CSV_BYTES, ctx.request_id)
raise AgentHTTPError(502, "too_large", ctx.request_id)
headers = {"Cache-Control": _NO_STORE, "X-Content-Type-Options": "nosniff", "X-Request-Id": ctx.request_id}
# The middleware's own test for whether the client takes gzip.
if "gzip" in request.headers.get("Accept-Encoding", ""):
body = await run_in_threadpool(_gzipped, body)
headers["Content-Encoding"] = "gzip"
headers["Vary"] = "Accept-Encoding"
return Response(body, media_type=ARTIFACT_MEDIA_TYPES["data.csv"], headers=headers)


@router.post("/sessions/{sid}/messages", response_class=EventSourceResponse)
async def post_message(upstream: UpstreamStream = Depends(open_message_stream)) -> AsyncIterator[ServerSentEvent]:
"""Relay one chat turn as SSE `anyplot/1`; always ends with a `done` event.
Expand Down
1 change: 1 addition & 0 deletions app/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
"@emotion/styled": "^11.14.1",
"@mui/icons-material": "^9.5.0",
"@mui/material": "^9.5.0",
"@mui/x-data-grid": "^9.17.0",
"force-graph": "^1.51.4",
"fuse.js": "^7.5.0",
"react": "^19.3.1",
Expand Down
80 changes: 80 additions & 0 deletions app/scripts/agent-bff-mock.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,12 @@
* refusals (`404 session_expired`, `409 has_dataset`, `422 no_dataset`, `422 unconfirmed`, `422 invalid`).
* MOCK_NO_COPY=1 answers it `404 not_found`, as an agents service deployed before the route does, so the page's
* fallback (the paste posted again, or `copy_unavailable`) can be driven.
*
* The data panel's full view (`.view_all()`): `GET .../dataset/data.csv` answers the current version's CSV
* (`422 no_dataset` without data). A chat message naming a size such as "10000 rows x 30 columns" (also `×` or
* `by`, at most 20,000 rows and 50 columns) is a table the assistant computed, like `generate_data`: a fixed
* pseudo-random mix of an id, a date, a text column with a comma, a quote and a space run, and number columns with
* a gap every few rows, past the paste's 1 MB, so the view can be driven at size.
*/

import { execFileSync } from 'node:child_process';
Expand Down Expand Up @@ -1065,6 +1071,12 @@ async function streamTurn(req, res, session, body, ref, siteTheme = null) {
send('message', { text: summarizeData(session) });
return;
}
const size = wideTableSize(lower);
if (size) {
await sleep(STEP_MS * 2);
send('message', { text: generateWideTable(session, size) });
return;
}
if (asksForFormula(lower)) {
await sleep(STEP_MS * 2);
send('message', { text: generateData(session, lower) });
Expand Down Expand Up @@ -1687,6 +1699,61 @@ function generateData(session, lower) {
return `I generated ${points} rows from y = ${formula} for ${span}${undo}. Check the preview and confirm the data.`;
}

/** `{rows, columns}` of a message such as "10000 rows x 30 columns", or null; capped like the service's tables. */
function wideTableSize(lower) {
const match = lower.match(/(\d[\d,]*)\s*rows?\s*(?:x|×|by)\s*(\d+)\s*col/);
if (!match) return null;
const rows = Number(match[1].replace(/,/g, ''));
const columns = Number(match[2]);
if (!rows || columns < 3) return null;
return { rows: Math.min(rows, 20000), columns: Math.min(columns, 50) };
}

/**
* A computed table of the asked size for the full view, stored like `generate_data`'s: an id, a date, a text column
* whose values hold a comma, a quote and a run of spaces, then integer and number columns with a gap every 97th
* cell. A fixed linear congruential sequence makes every run the same table.
*/
function generateWideTable(session, size) {
let seed = 20261101;
const next = () => {
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
return seed / 4294967296;
};
const stations = ['Nord', 'Süd, alt', 'Ost "neu"', 'West 2', 'Mitte', 'Hafen'];
const header = ['id', 'date', 'station'];
for (let c = 3; c < size.columns; c += 1)
header.push(c % 2 ? `reading_${c - 2}` : `count_${c - 2}`);
const start = Date.UTC(2020, 0, 1);
const rows = Array.from({ length: size.rows }, (_, r) => {
const row = [
String(r + 1),
new Date(start + r * 86400000).toISOString().slice(0, 10),
stations[Math.floor(next() * stations.length)],
];
for (let c = 3; c < size.columns; c += 1) {
if ((r * size.columns + c) % 97 === 0) row.push('');
else if (c % 2) row.push((next() * 2000 - 500).toFixed(2));
else row.push(String(Math.floor(next() * 1000)));
}
return row;
});
const parsed = parseDataset(toCsv(header, rows));
const changes = [`generated ${size.rows} rows × ${size.columns} columns of test data`];
if (!session.dataset) {
session.dataset = parsed;
session.bindings = defaultBindings(session, parsed.profile);
session.complete = checkBindings(session, session.bindings).complete;
session.confirmed = false;
session.history = [];
session.version = 1;
session.changed = { ...datasetPayload(session), roles: SPECS[session.specId].roles, changes };
} else {
storeVersion(session, parsed, changes);
}
return `I generated ${size.rows.toLocaleString('en-US')} rows and ${size.columns} columns. Check the preview, or .view_all() to see every row.`;
}

/** The step-1 summary the root writes after `summarize_data`, from the profile alone. */
function summarizeData(session) {
if (!session.dataset) return 'Paste your data first; then I can summarize it.';
Expand Down Expand Up @@ -2046,6 +2113,18 @@ async function route(req, res) {
ref
);
}
if (method === 'GET' && rest === '/dataset/data.csv') {
// Read-only, so a turn in flight does not block it, as in the service.
if (!session.dataset) return fail(res, 422, 'no_dataset', ref);
res.writeHead(200, {
'Content-Type': 'text/csv; charset=utf-8',
'Cache-Control': 'private, no-store',
'X-Content-Type-Options': 'nosniff',
'X-Request-Id': ref,
});
res.end(csvText(session));
return;
}
if (method === 'POST' && rest === '/dataset/undo') {
if (session.active) return fail(res, 409, 'run_active', ref);
const previous = session.history.pop();
Expand Down Expand Up @@ -2172,6 +2251,7 @@ async function route(req, res) {
'downloaded_code',
'downloaded_page',
'opened_preview',
'opened_data_view',
'downloaded_data',
];
const valid =
Expand Down
18 changes: 18 additions & 0 deletions app/src/hooks/useAgentSession.save.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ function mockFetch() {
});
}
if (call.method === 'GET' && call.path === '/sessions/S1abcdefgh/bundle') return bundle(call);
if (call.method === 'GET' && call.path === '/sessions/S1abcdefgh/dataset/data.csv') {
return new Response('a,b\n1.50,2\n');
}
if (call.method === 'POST' && call.path === '/sessions/S1abcdefgh/dataset') {
return dataset(call);
}
Expand Down Expand Up @@ -107,6 +110,21 @@ afterEach(() => {
vi.restoreAllMocks();
});

describe('useAgentSession: .view_all()', () => {
it('reads the current version of the session as its text', async () => {
const { result } = await openSession();
await expect(result.current.fetchDataCsv()).resolves.toBe('a,b\n1.50,2\n');
});

it('refuses before a session exists, without a request', async () => {
mockFetch();
const { result } = renderHook(() => useAgentSession(options));
await expect(result.current.fetchDataCsv()).rejects.toMatchObject({ code: 'no_session' });
expect(calls.map(call => call.path)).not.toContain('/sessions/S1abcdefgh/dataset/data.csv');
await waitFor(() => expect(result.current.state.phase).toBe('ready'));
});
});

describe('useAgentSession: .save()', () => {
it('downloads the bundle with the data under the BFF file name and records it', async () => {
const { result } = await openSession();
Expand Down
11 changes: 11 additions & 0 deletions app/src/hooks/useAgentSession.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1552,6 +1552,16 @@ export function useAgentSession({
[token]
);

// `.view_all()`: the current data version's canonical CSV, read-only; the view keeps it, the session state does not.
const fetchDataCsv = useCallback(
async (signal?: AbortSignal): Promise<string> => {
const sid = stateRef.current.sessionId;
if (!sid) throw new AgentApiError(0, 'no_session');
return agentApi.datasetCsv(token, sid, signal);
},
[token]
);

// A dropped binary file read on the server; the data card keeps its own state for it (nothing is stored).
const convertFile = useCallback(
async (dataBase64: string, sheet?: string): Promise<ConvertAnswer> => {
Expand Down Expand Up @@ -2198,6 +2208,7 @@ export function useAgentSession({
fetchArtifact,
reportOutcome,
convertFile,
fetchDataCsv,
saveSession,
setLanguage,
sendFeedback,
Expand Down
32 changes: 32 additions & 0 deletions app/src/lib/agent.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -862,6 +862,38 @@ describe('agentApi.reportOutcome', () => {
});
});

describe('agentApi.datasetCsv', () => {
afterEach(() => {
vi.unstubAllGlobals();
});

it('reads the current version as text with a GET, never parsing it', async () => {
const csv = 'city,value\n"Köln, Nord",1.50\nBonn,\n';
const fetchMock = vi.fn(async () => new Response(csv, { status: 200 }));
vi.stubGlobal('fetch', fetchMock);
const controller = new AbortController();

await expect(agentApi.datasetCsv('', 'S/1', controller.signal)).resolves.toBe(csv);

const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit];
expect(url).toMatch(/\/debug\/agent\/sessions\/S%2F1\/dataset\/data\.csv$/);
expect(init.method ?? 'GET').toBe('GET');
expect(init.signal).toBe(controller.signal);
});

it('throws the route code, such as no_dataset before any data', async () => {
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response('{"detail":"no_dataset","ref":"r1"}', { status: 422 }))
);
await expect(agentApi.datasetCsv('', 'S1')).rejects.toMatchObject({
status: 422,
code: 'no_dataset',
ref: 'r1',
});
});
});

describe('helpers', () => {
it('buckets sizes without exposing them', () => {
expect(sizeBucket(10)).toBe('lt_1kb');
Expand Down
11 changes: 11 additions & 0 deletions app/src/lib/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,8 @@ export const OUTCOME_ACTIONS = [
'downloaded_page',
// The same page shown in the card (since JS-4b-2); a look, not a keep.
'opened_preview',
// The data panel's view of every row (`.view_all()`, version 0); a look at the data, not a keep.
'opened_data_view',
'downloaded_data',
'saved_session',
'feedback',
Expand Down Expand Up @@ -1061,6 +1063,15 @@ export const agentApi = {
token,
post({ from_session: fromSession })
),
/**
* `.view_all()`: the current data version's canonical `data.csv` as the server
* stores it, at most 4 MiB, for the data panel's full view. Read-only; `422
* no_dataset` before any data.
*/
datasetCsv: async (token: string, sid: string, signal?: AbortSignal): Promise<string> => {
const response = await agentFetch(`${sessionPath(sid)}/dataset/data.csv`, token, { signal });
return response.text();
},

/**
* `.to_long()`: the service applies its own wide-table suggestion, without a
Expand Down
3 changes: 3 additions & 0 deletions app/src/pages/AgentChatPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -825,6 +825,9 @@ function AgentChat({
decisionRef={decisionRef}
// Not while the fan-out's panel is open: its sessions copy this session's confirmed data.
locked={!!fanOut}
onLoadData={session.fetchDataCsv}
// A look at the session's data, not at a result: version 0.
onDataViewShown={() => session.reportOutcome('opened_data_view', 0)}
/>

{state.items.length === 0 && !state.run && (
Expand Down
Loading
Loading