Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
444f1f4
feat: column reach keeps how each reached column comes from the trace…
hkaltai Nov 13, 2026
e3f24db
feat: dbt_docs_coverage reports which output columns of a model have …
hkaltai Nov 13, 2026
b7cfc04
feat: foundation for the dbt Power User tools
hkaltai Nov 13, 2026
1eb722e
Merge remote-tracking branch 'origin/main' into feat/pu-mods-foundation
hkaltai Nov 13, 2026
99c3bf6
Merge branch 'feat/pu-mods-foundation' into feat/docs-coverage-and-re…
hkaltai Nov 13, 2026
69b26cd
fix: column reach rates a column read through a schema-qualified rela…
hkaltai Nov 13, 2026
21d0494
fix: harden the dbt Power User tool foundation
hkaltai Nov 13, 2026
978c8ce
style: format column reach and the impact analysis test
hkaltai Nov 13, 2026
2a415fe
fix: docs coverage keeps quoted names that differ by case apart, repo…
hkaltai Nov 13, 2026
36eca67
Merge branch 'feat/pu-mods-foundation' into feat/docs-coverage-and-re…
hkaltai Nov 13, 2026
c8e59ae
Merge remote-tracking branch 'origin/main' into feat/docs-coverage-an…
hkaltai Nov 13, 2026
a05cec9
test: the placeholder check leaves out dbt_docs_coverage, which is im…
hkaltai Nov 13, 2026
0acf970
fix: dbt_docs_coverage failures carry an error so the CLI and MCP rep…
hkaltai Nov 13, 2026
d8c3469
fix: send falsy API bodies, follow cancellation in apiRequest, check …
hkaltai Nov 15, 2026
dace1c2
Merge remote-tracking branch 'origin/main' into feat/pu-mods-foundation
hkaltai Nov 17, 2026
22f12e7
Merge remote-tracking branch 'origin/main' into feat/docs-coverage-an…
hkaltai Nov 17, 2026
18a3fc4
fix: keep the stored API key out of API errors, bound the dbt config …
hkaltai Nov 17, 2026
6024ac9
fix: redact the API key when a refused header races the caller's abor…
hkaltai Nov 17, 2026
f5159c5
docs: say which tools use the dbt adapter helper
hkaltai Nov 17, 2026
f57de3d
docs: rewrap and correct the adapter helper comments
hkaltai Nov 17, 2026
7c3679d
Merge remote-tracking branch 'origin/feat/pu-mods-foundation' into fe…
hkaltai Nov 17, 2026
cd82c98
dbt_docs_coverage: an unreadable select list falls back to the manife…
hkaltai Nov 17, 2026
e84b6ec
test(learn): the publisher mocks carry publishErrorCode, which the CL…
hkaltai Nov 17, 2026
aaff414
fix: never echo a send failure's text, so a trimmed key can't leak
hkaltai Nov 17, 2026
3a98dff
fix: a rejection with no reason is a send failure, not an abort
hkaltai Nov 17, 2026
0a7b0cb
Merge remote-tracking branch 'origin/feat/pu-mods-foundation' into fe…
hkaltai Nov 17, 2026
ca6b577
Merge remote-tracking branch 'origin/main' into feat/docs-coverage-an…
hkaltai Nov 17, 2026
f648aee
docs: the docsCoverage contract describes the manifest-columns fallback
hkaltai Nov 17, 2026
67f4dae
fix: a coded or pre-send TypeError is not a header refusal; bind the …
hkaltai Nov 17, 2026
caa4c7c
fix: send API requests only to an http(s) URL
hkaltai Nov 17, 2026
cebec12
Merge remote-tracking branch 'origin/main' into feat/pu-mods-foundation
hkaltai Nov 17, 2026
b9bd43d
fix: case-colliding outputs get no rating, an exact blank YAML entry …
hkaltai Nov 17, 2026
adaa5e2
Merge remote-tracking branch 'origin/feat/pu-mods-foundation' into fe…
hkaltai Nov 17, 2026
316d550
fix: retry schema merges YAML and compiled columns; case-collision te…
hkaltai Nov 17, 2026
4f7b76d
Merge origin/main: take main's foundation files, keep only the docs-c…
hkaltai Nov 17, 2026
ec46479
test: restore the inherited telemetry setting after the docs coverage…
hkaltai Nov 17, 2026
fd698e9
Merge remote-tracking branch 'origin/main' into feat/docs-coverage-an…
hkaltai Nov 18, 2026
e2a9f93
fix: column_reach rates only a model's real output names
hkaltai Nov 18, 2026
c1c7338
fix: a documented engine-style output name gets no rating; sharper te…
hkaltai Nov 18, 2026
813466f
refactor: column_reach reuses engineMade; pin the SQL-written engine-…
hkaltai Nov 18, 2026
9834f0c
test: column_reach engine-style name mentioned only in a comment or l…
hkaltai Nov 18, 2026
dc28a3e
test: pin that the comment-only engine-style name is carried
hkaltai Nov 18, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 137 additions & 7 deletions packages/opencode/src/altimate/native/dbt/column-reach.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,24 @@
// lineage engine cannot settle is reported as such: models it could not trace (no compiled SQL, or
// SQL it could not parse), models that name the column only outside their select list (where the
// engine reports no edge), and models that read it through a table name several parents share.
import { loadRawManifest, findModel } from "./helpers"
import * as core from "@altimateai/altimate-core"
import { loadRawManifest, findModel, detectDialect } from "./helpers"
import { dbtLineage } from "./lineage"
import { missingColumn, projectLineage, selectsStar, stripSql } from "./project-lineage"
import { engineMade, missingColumn, projectLineage, selectsStar, stripSql } from "./project-lineage"

export interface ColumnReachModel {
name: string
unique_id: string
/** Its output columns that come from the traced column. */
columns: string[]
/**
* How each of those columns (keyed as in `columns`) comes from the traced column, as the lineage
* engine classifies it: Original (carried as is), Alias (renamed), Transformation, Not sure or
* Non select. It covers the whole path from the traced column: a column is Original or Alias only
* when every hop is, so a column read from a calculation is never Original. A column whose
* edges carry no classification has no entry.
*/
lens_types?: Record<string, string>
}

export interface ColumnReachUnsure {
Expand Down Expand Up @@ -58,6 +67,16 @@ export interface ColumnReach {
const MAX_MODELS = 300
const TRACED = new Set(["model", "snapshot"])

// How far a classification is from "the value arrives unchanged", least to most.
const LENS_ORDER = ["Original", "Alias", "Non select", "Not sure", "Transformation"]

/** The classification of a path made of two: the one further from unchanged, unknown if either is. */
function joinLens(a: string | undefined, b: string | undefined): string | undefined {
if (a === undefined || b === undefined) return undefined
const rank = (lens: string) => (LENS_ORDER.includes(lens) ? LENS_ORDER.indexOf(lens) : LENS_ORDER.length)
return rank(a) >= rank(b) ? a : b
}

/** "db"."schema"."orders"."amount" → { parts: ["db","schema","orders"], column: "amount" }, lower-cased. */
function splitSource(source: unknown): { parts: string[]; column: string; exact: string } | null {
const text =
Expand All @@ -66,8 +85,7 @@ function splitSource(source: unknown): { parts: string[]; column: string; exact:
// Split on dots outside quotes: "a"."net.amount" is table a, column net.amount.
// A doubled closing delimiter inside a quoted name is one character ("a""b" is a"b, [a]]b] is a]b).
const raw = [...text.matchAll(/"((?:[^"]|"")*)"|`((?:[^`]|``)*)`|\[((?:[^\]]|\]\])*)\]|([^."`\[\]]+)/g)].map(
(m) =>
m[1]?.replaceAll('""', '"') ?? m[2]?.replaceAll("``", "`") ?? m[3]?.replaceAll("]]", "]") ?? m[4] ?? "",
(m) => m[1]?.replaceAll('""', '"') ?? m[2]?.replaceAll("``", "`") ?? m[3]?.replaceAll("]]", "]") ?? m[4] ?? "",
)
if (raw.length < 2) return null
// `exact` keeps the column's case, for parents that document two columns differing only by case.
Expand All @@ -94,6 +112,68 @@ function relationKeys(node: any): string[] {
return [...keys]
}

type LineageEdge = { source: unknown; target: string; lineage_type?: string; lens_type?: string }

/**
* The engine rates every edge "Not sure" when the table it reads is written with its schema
* ("db"."schema"."orders", as dbt compiles a ref) and its schema lists that table by bare name only.
* The model's SQL is traced again with each parent listed under every way its relation is written,
* and an edge both runs report (matched by output column, table name and source column, as the
* first run drops the qualification) takes the second run's rating. Anything else keeps its own.
*/
function rated(
edges: LineageEdge[],
sql: string,
dialect: string,
parents: string[],
lookup: (id: string) => any,
columnsOf: (id: string) => { name: string; type?: string }[],
): LineageEdge[] {
if (!sql || parents.length === 0 || !edges.some((edge) => edge.lens_type === "Not sure")) return edges
try {
const tables: Record<string, { columns: { name: string; type: string }[] }> = {}
for (const parent of parents) {
const node = lookup(parent)
// The columns the first trace reads the parent against: the YAML's, plus any the compiled SQL gives it that the YAML omits.
const declared = Object.entries<any>(node?.columns ?? {}).map(([key, col]) => ({
name: String(col?.name ?? key),
type: String(col?.data_type ?? col?.type ?? ""),
}))
const known = new Set(declared.map((col) => col.name.toLowerCase()))
const columns = [
Comment thread
hkaltai marked this conversation as resolved.
Resolved
...declared,
...columnsOf(parent)
Comment thread
hkaltai marked this conversation as resolved.
Resolved
.filter((col) => !known.has(col.name.toLowerCase()))
.map((col) => ({ name: col.name, type: String(col.type ?? "") })),
]
if (columns.length === 0) continue

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor: the second trace leaves out every parent with no YAML columns, so its edges stay Not sure.

rated() builds its schema from node.columns (the YAML), not from the columns the first trace already knows (project.columns(id)). An undocumented intermediate model is common, and every column read through it keeps Not sure:

  • jaffle-shop with model columns stripped: 11 of 11 ratings are Not sure.
  • The 2,000-model manifest with columns stripped below the first layer: 1,488 Not sure, against 768 when the columns are listed.

A qualification whose case differs from the manifest (SHOP.MAIN."fct_orders" on duckdb) also stays Not sure.

This is conservative: nothing is copied wrongly. But the claim "rated by what the SQL does, not Not sure" holds only for documented parents. Using project.columns(parent) would close most of the gap.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in adaa5e2: rated now takes the parent's columns from project.columns(parent) when its YAML declares none, so an undocumented intermediate model is retraced. Test added (a parent that lists no YAML columns...). The case-differing qualification (SHOP.MAIN."fct_orders") is not handled: it stays Not sure, which is conservative; recorded as a follow-up.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Round 2: resolved. Verified at ec464792ca:

  • jaffle-shop with model columns stripped now rates the same as with columns.
  • On a synthetic 2,000-model manifest with columns stripped below the first layer, the Not sure share is 34% (37% with columns). In round 1 it was 66% against 34%.
  • Dropping the compiled columns from the retry schema fails the new test.

Probes for the merged schema found no case where a parent's columns attached to the wrong relation (same table name in two schemas, a parent and a source with one name, a non-parent relation). The case-different qualification (SHOP.MAIN.) stays Not sure. That is conservative, and fine as a follow-up.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for verifying. The case-differing qualification (SHOP.MAIN."fct_orders") stays Not sure, which is conservative (nothing is copied wrongly); left as a follow-up.

for (const key of relationKeys(node)) tables[key] = { columns }
Comment thread
hkaltai marked this conversation as resolved.
Resolved
}
if (Object.keys(tables).length === 0) return edges
const schema = core.Schema.fromJson(JSON.stringify({ tables, version: "1" }))
const again: LineageEdge[] =
JSON.parse(JSON.stringify(core.columnLineage(sql, dialect, schema))).column_lineage ?? []
const keyOf = (edge: LineageEdge) => {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor (test gap): the second-trace matching and the several-edges join are not pinned by any test.

These mutations survive all of column-reach.test.ts, impact-analysis.test.ts and dbt-docs-coverage.test.ts:

  • dropping the table name from keyOf (key on target and column only);
  • last-wins instead of joinLens in the second run's lenses map;
  • last-wins instead of joinLens for several edges into one output (line 445);
  • unknown lens strings ranking lowest instead of highest.

These are caught: joinLens ignoring the parent hop, joinLens taking the nearer rating, skipping the second trace, and dropping the target from the key.

A fixture with two relations in the SQL that share a column name but rate differently, plus a mocked engine answer with two differently rated edges into one output, would pin both. Test-only.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Partly fixed in adaa5e2: added a test with two qualified relations sharing a column name but rated differently (pins the table name in keyOf). The remaining mutations (joinLens vs last-wins for the second run and for several edges into one output, unknown lens ranking) are not pinned yet; they are conservative-direction and recorded as a follow-up test task.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Round 2: not resolved, test-only, does not block. At ec464792ca the same four mutations survive: keyOf without the table name, last-wins in the second run, last-wins for several edges into one output, and an unknown lens ranking lowest.

The new "two qualified relations" test does not pin the table name: kept and scaled are different outputs, so their keys differ without it.

Details and a suggested fixture are in the new inline comment on line 160. Review: #1733 (review)

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Acknowledged, test-only and non-blocking. The remaining mutations (last-wins vs joinLens, unknown lens ranking, keyOf table name) fail in the conservative direction (a rating is never made more optimistic). Tracked together with the new comment on line 160.

const source = splitSource(edge.source)
return source ? `${String(edge.target).toLowerCase()}|${source.parts.at(-1)}.${source.column}` : null
Comment thread
hkaltai marked this conversation as resolved.
Resolved
}
const lenses = new Map<string, string>()

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor (test-only): the second-trace and several-edges joins are still unpinned.

These mutations still pass column-reach, impact-analysis and dbt-docs-coverage tests at ec464792ca:

  • keyOf without the table name. The new "two qualified relations" test does not pin it, because kept and scaled already differ by output name, so their keys differ without the table name.
  • Last-wins instead of joinLens in this lenses map.
  • Last-wins instead of joinLens for several edges into one output (line 461).
  • An unknown lens string ranking lowest.

These are caught: removing the case-collision clearing, skipping the second trace, dropping the compiled columns from the retry schema, and folding case when an exact YAML entry exists.

To pin the first: two relations whose columns share a name, read into outputs with the same name in two branches of a union all, rated differently. Test-only, not blocking.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Acknowledged, test-only and not blocking. The surviving mutations all fail in the conservative direction; adding the suggested union-all fixture is recorded as a follow-up test task.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Round 3 (dc28a3e558): not resolved; test-only follow-up. M4, M6, M7 and M8 still survive. One more survives: dropping the realNames membership check and keeping only the engineMade check. Not blocking.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks. Acknowledged as a test-only follow-up, now including the realNames membership mutation. Not changing the branch, to keep your approval.

for (const edge of again) {
const key = keyOf(edge)
// Edges into one output from the same column keep the one furthest from unchanged.
if (key && typeof edge.lens_type === "string")
lenses.set(key, joinLens(lenses.get(key), edge.lens_type) ?? edge.lens_type)
}
return edges.map((edge) => {
const key = edge.lens_type === "Not sure" ? keyOf(edge) : null
const lens = key ? lenses.get(key) : undefined
return lens ? { ...edge, lens_type: lens } : edge
})
} catch {
return edges
}
}

/**
* The parents an edge's source table names, matched at the qualification the edge carries: a
* qualified name that matches no parent names none (it is some other relation), and a bare name
Expand Down Expand Up @@ -274,6 +354,8 @@ export function columnReach(manifestPath: string, model: string, column: string,
if (unsettled) spelled = []
else if (!exact && matches.length > 0) spelled = matches
const trackedExact = new Map<string, Set<string>>([[targetId, new Set(spelled)]])
// How each tracked column comes from the traced column, per node (the traced column itself is Original).
const lensOf = new Map<string, Map<string, string | undefined>>([[targetId, new Map([[column0, "Original"]])]])
const carries = (parent: string, source: { column: string; exact: string }) =>
!!tracked.get(parent)?.has(source.column) &&
(!caseCollides(lookup(parent), source.column) || !!trackedExact.get(parent)?.has(source.exact))
Expand Down Expand Up @@ -307,7 +389,7 @@ export function columnReach(manifestPath: string, model: string, column: string,
unsure.add(id)
continue
}
const edges: { source: unknown; target: string; lineage_type?: string }[] = lineage?.column_lineage?.length
const listed: LineageEdge[] = lineage?.column_lineage?.length
? lineage.column_lineage
: Object.entries<any>(lineage?.column_dict ?? {}).flatMap(([out, sources]) =>
(Array.isArray(sources) ? sources : [sources]).map((source) => ({
Expand All @@ -323,6 +405,14 @@ export function columnReach(manifestPath: string, model: string, column: string,
// cannot say which one is read; a `select *` from a parent whose schema does not list the
// column cannot be expanded to it. Both are possible reads with unknown outputs.
const sqlText: string = nodes[id]?.compiled_code || nodes[id]?.compiled_sql || ""
const edges = rated(
listed,
sqlText,
dialect || detectDialect(manifest),
allParents,
lookup,
(id) => project.columns(id),
)
const bareName = (p: string) => {
const node = lookup(p)
return String(node?.alias || node?.identifier || node?.name || p).toLowerCase()
Expand All @@ -346,6 +436,7 @@ export function columnReach(manifestPath: string, model: string, column: string,

const carried = new Set<string>()
const carriedExact = new Set<string>()
const carriedLens = new Map<string, string | undefined>()
let reads = false
let ambiguous = false
for (const edge of edges) {
Expand All @@ -360,10 +451,35 @@ export function columnReach(manifestPath: string, model: string, column: string,
}
reads = true
if (edge.lineage_type !== "indirect" && typeof edge.target === "string") {
carried.add(edge.target.toLowerCase())
const out = edge.target.toLowerCase()

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Major: outputs that differ only by case share one rating key, so an Alias can land on the wrong column.

out = edge.target.toLowerCase() merges distinct quoted outputs. Then carriedLens and lens_types publish a copyable rating under a key that names both.

Repro (duckdb, real engine). Trace src_o.revenue, where the child's SQL is

select revenue as "Foo", gross_revenue as "foo" from "shop"."main"."src_o"

The result is columns: ["foo"] and lens_types: { foo: "Alias" }. The reached column is "Foo", and "foo" is gross_revenue. The plugin's candidatesFrom offers child.foo as a target for revenue's description.

Lower-cased columns predates this PR, but a copyable rating on that key is new. Either key lens_types by the exact target, or leave out Original/Alias for an output whose lower-cased name another output shares.

Raised by Codex, reproduced by the lead; severity is Codex's.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in adaa5e2: column-reach.ts now publishes no rating for an output whose lower-cased name another traced output shares (checked over all edge targets, not only the carrying ones), so "Foo"/"foo" can't be confused. Regression test: outputs that differ only by case publish no rating.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Round 2: partly resolved. At ec464792ca this repro gets no rating on DuckDB and Postgres: columns: ["foo"] and no lens_types. A test pins it, and removing the clearing makes that test fail.

Still open:

  • A case-colliding sibling with no source edge (0 as "foo", null, count(*), current_date) still gives {"foo": "Alias"}.
  • On Snowflake this exact repro still gives Alias, because the engine upper-cases both targets.

Under this round's bar these are edge inputs, so the round-1 major is now a minor. It is tracked in the new inline comment on line 476, with a suggested fix (caseCollides(nodes[id], out)). Review: #1733 (review)

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Round 2 follow-up, thanks. The round-1 repro ("Foo"/"foo" both with source edges) is fixed and pinned at column-reach.ts:466-476. The remaining variants (a sibling with no source edge, Snowflake upper-casing both targets) are edge inputs: quoted output names that differ only by case. Declining as a documented limitation of the heuristic; the tracking comment on line 476 has the same answer.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor (edge input; best first follow-up): an unaliased calculation gets an engine-invented target rated Original.

select revenue + 1 from "shop"."main"."fct_orders"

Tracing revenue gives columns: ["_0", "_col_0"], lens_types: {"_0": "Original", "_col_0": "Not sure"}. This happens on duckdb, snowflake, postgres and redshift; bigquery gives only _col_0: Not sure. select order_id, revenue * 2, … gives _1: Original. This contradicts the contract's "a column read from a calculation is never Original".

_0 is the engine's name. The warehouse names the column (revenue + 1), ?column? or "REVENUE + 1". So the plugin's offer names a column that does not exist, and Apply would add a stray YAML entry. It can only overwrite a real description when the model also has a real column named _0 (e.g. select revenue + 1, gross_revenue as "_0" gives _0: Original on the gross_revenue column).

Suggested fix: publish a rating only for a target that is one of the model's real output names.

Raised by Codex; the lead widened it to the bare case. Codex and the lead agree it is an edge input under this round's bar.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in e2a9f93: column-reach.ts now publishes a rating only for a target that is one of the model's real output names (project.columns(id), which drops the engine's _N/_col_N names unless the SQL writes them). Regression test: "an unaliased calculation's engine-made output name gets no rating" in column-reach.test.ts (fails without the filter: lens_types had _0: Original).

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Round 3 (dc28a3e558): resolved. select revenue + 1 … and a Postgres revenue::numeric now reach _N/_col_N with no rating. That holds when YAML documents _0, and when _0 appears only in a comment or literal. select revenue as _0 keeps _0: Alias. Mutations that remove the filter or the engineMade check, or that use unstripped SQL, are all caught. Side effect (new inline minor, conservative): an undocumented bare-relation select *, and an inherited real _0 read through select *, now lose their Not sure rating.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for verifying. The side effects you listed in the two new inline comments (lines 478 and 482) are acknowledged there as conservative follow-ups.

// Only one candidate gets here: more than one is ambiguous and was skipped above.
const lens = joinLens(
lensOf.get(candidates[0]!)?.get(source.column),
typeof edge.lens_type === "string" ? edge.lens_type : undefined,
)
// Several edges into one output: it is as far from unchanged as the furthest of them.
carriedLens.set(out, carried.has(out) ? joinLens(carriedLens.get(out), lens) : lens)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
carried.add(out)
carriedExact.add(edge.target)
}
}
// Outputs whose names differ only by case are different columns, but the result keys them in lower
// case: a rating there could name either one, so none is published.
// Every output the engine traced counts, not only those carrying the column: a sibling output
// that shares the lower-cased name is the one a rating could be mistaken for.
const exactByLower = new Map<string, Set<string>>()

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor (edge input, DuckDB only): duplicate output names collapse into one key, so Original can land on the wrong column.

select gross_revenue as revenue, revenue from "shop"."main"."fct_orders"

Tracing revenue gives {"revenue": "Original"}. Checked with real DuckDB: both create table and create view build this model with columns revenue (which holds gross_revenue) and revenue_1 (which holds the traced column). The rating names the wrong one. Snowflake, Postgres and BigQuery refuse duplicate output names, so it is DuckDB only.

Suggested fix: when two edges share one exact target, publish no rating for it.

Raised by Codex; reproduced.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Declining: DuckDB-only edge input (a repeated output name; Snowflake, Postgres and BigQuery refuse to build such a model). Documented limitation, no change.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Round 3 (dc28a3e558): not resolved; accepted as an edge-input follow-up. Still revenue: Original for DuckDB gross_revenue as revenue, revenue. Duplicate output names build only on DuckDB, so this is not blocking.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks. Acknowledged as a DuckDB-only edge-input follow-up; no change in this PR.

for (const edge of edges) {
if (typeof edge.target !== "string") continue
const lower = edge.target.toLowerCase()
exactByLower.set(lower, (exactByLower.get(lower) ?? new Set<string>()).add(edge.target))
}
for (const [out, exact] of exactByLower) if (exact.size > 1) carriedLens.set(out, undefined)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor (edge input; the rest of round-1's major): the collision guard only sees outputs that have a lineage edge, so some case-colliding outputs still get a rating.

exactByLower is built from edge targets. An output with no source edge never enters it. With postgres or duckdb:

select revenue as "Foo", 0 as "foo" from shop.main.fct_orders

Tracing revenue gives lens_types: {"foo": "Alias"}. The same happens with 'x', null, count(*) or current_date as the sibling. On Snowflake the round-1 repro (revenue as "Foo", gross_revenue as "foo") still gives {"foo": "Alias"}, because the engine upper-cases both targets, so the set holds one spelling.

The plugin offers c.foo as a copy target. If the model's YAML lists both Foo and foo, the description can land on the wrong one.

Suggested fix: also clear the rating when caseCollides(nodes[id], out) is true, or when the model's output names (project.columns(id) or extractOutputColumns) hold two spellings of out.

Not blocking under this round's bar: quoted output names that differ only by case are rare. Raised by Codex and by the lead independently; reproduced.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Declining as an edge input (quoted output names differing only by case, where the sibling has no source edge, or Snowflake upper-casing both targets). Nothing is copied unless the YAML also lists both spellings. Documented limitation of the heuristic output-name contract; the engine-made-name case on line 454 is fixed in e2a9f93.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Round 3 (dc28a3e558): not resolved; accepted as an edge-input follow-up. select revenue as "Foo", 0 as "foo" still gives foo: Alias. It needs quoted outputs that differ only by case, and a YAML listing both spellings, before the plugin can copy onto the wrong one. Not blocking under this round's bar.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the round-3 check. Agreed: accepted as an edge-input follow-up (quoted outputs differing only by case beside a sibling with no source edge). No change in this PR.

// A name the engine invents for an unaliased calculation (`_0`) is no column of the model: it gets no rating.
const realNames = new Set(project.columns(id).map((col) => col.name))
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor (edge input, conservative): an undocumented select * over a bare relation name loses its rating, and the loss runs downstream.

Parent p (columns order_id, revenue), child with no YAML columns and SQL select * from p, grandchild select revenue as r2 from child. Tracing p.revenue:

  • round 2 (ec464792ca): child revenue: Not sure, grandchild r2: Not sure;
  • head: both reached with no lens_types.

project.columns(child) is empty here (no YAML, and the star is not expanded), so realNames drops every output. Same on DuckDB, Postgres and Snowflake. With the qualified relation dbt compiles (select * from "shop"."main"."p") the rating is kept.

Not blocking: only Not sure is ever lost (a star is always Not sure, and joinLens keeps the furthest rating), so nothing wrong is copied. A possible follow-up: apply the realNames check only to engine-style names, since that check alone catches every case the new tests pin.

Raised by Codex; reproduced by the lead.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the repro. Agreed it is conservative (only Not sure is lost, nothing wrong is copied). Acknowledged; restricting the realNames check to engine-style names (column-reach.ts:478) is the right follow-up, left out here to keep the approved head.

// A documented `_0` is the engine's placeholder all the same unless the SQL writes that name itself.
const strippedSql = stripSql(sqlText)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[SUGGESTION]: The stripSql behaviour this line adds has no test

Running engineMade on stripSql(sqlText) is the point of this change: a documented _0 whose only occurrence is in a comment (-- _0) or a string literal ('_0') no longer counts as written and loses its rating. Nothing pins that. The drop test (stg_doc) has no occurrence of _0 in its SQL at all, and the new keep test writes _0 as a real alias. Reverting this line to engineMade(out, sqlText) still passes all three tests, so the literal/comment case is unpinned.

A sibling test with 'select revenue + 1 /* _0 */ from "shop"."main"."fct_orders"' (or '_0' in a literal) and YAML _0 expecting no rating would fail on the unstripped text and pass now.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Real gap, fixed in 9834f0c: added "an engine-style name that only a comment or literal mentions gets no rating" (/* _0 */ and '_0', YAML _0, expects no rating). Mutation check: with engineMade(out, sqlText) (unstripped) the new test fails; with the stripped text it passes.

for (const out of carriedLens.keys())
if (!realNames.has(out) || engineMade(out, strippedSql)) carriedLens.set(out, undefined)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor (edge input, conservative): a real _0 column inherited through select * loses its rating.

Parent SQL select order_id, revenue as _0 from raw.o, YAML _0. Child select * from "shop"."main"."p". Tracing p._0: round 2 gives _0: Not sure; head gives no rating, because the child's SQL never writes _0, so engineMade treats it as the engine's placeholder. select _0 from … keeps _0: Original.

Not blocking: an unusual column name, and the loss is in the safe direction. project.columns already treats a name a parent column passes on as real (passed); the same rule here would fix it.

Raised by Codex; reproduced by the lead.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks. Acknowledged: a real _0 inherited through select * loses a Not sure rating at column-reach.ts:482, in the safe direction. Reusing the passed rule from project-lineage.ts as you suggest is a follow-up; no change here.

// The engine has no schema for the tables inside an inlined ephemeral parent, so it reports no
// edge through it: unknown, not unread, even when another parent's edges are traced.
const viaEphemeral = parents.some((p) => lookup(p)?.config?.materialized === "ephemeral")
Expand All @@ -376,10 +492,24 @@ export function columnReach(manifestPath: string, model: string, column: string,
if (reads) {
tracked.set(id, carried)
trackedExact.set(id, carriedExact)
lensOf.set(id, carriedLens)
// Outputs fed by an unsettled or ephemeral parent, or read through an ambiguous name, stay unsettled.
// A NATURAL JOIN beside the column may join on it, or carry it on in another output.
if (ambiguous || fromUnsure || viaEphemeral || partial || opaque) unsure.add(id)
reached.push({ name: nodes[id]?.name ?? id, unique_id: id, columns: [...carried].sort() })
const columns = [...carried].sort()
// Built with fromEntries: assigning to a column named __proto__ would set the prototype instead.
const lens_types: Record<string, string> = Object.fromEntries(
columns.flatMap((col) => {
const lens = carriedLens.get(col)
return lens === undefined ? [] : [[col, lens] as const]
}),
)
reached.push({
name: nodes[id]?.name ?? id,
unique_id: id,
columns,
...(Object.keys(lens_types).length > 0 && { lens_types }),
})
continue
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ const COLUMNS_CALL = /(?<![\w$])columns\s*\(/i
* The engine's own name for an unaliased expression in a SELECT list (`_1`, `_col_1`): not the warehouse's
* name for it. A name the SQL itself writes (`amount as _1`) is a real alias.
*/
function engineMade(name: string, stripped: string): boolean {
export function engineMade(name: string, stripped: string): boolean {
return ENGINE_NAME.test(name) && !new RegExp(`(?<![\\w$])${name}(?![\\w$])`, "i").test(stripped)
}

Expand Down
Loading
Loading