Repository navigation
feat: dbt_docs_coverage and how each reached column comes from the traced one #1733
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
444f1f4
e3f24db
b7cfc04
1eb722e
99c3bf6
69b26cd
21d0494
978c8ce
2a415fe
36eca67
c8e59ae
a05cec9
0acf970
d8c3469
dace1c2
22f12e7
18a3fc4
6024ac9
f5159c5
f57de3d
7c3679d
cd82c98
e84b6ec
aaff414
3a98dff
0a7b0cb
ca6b577
f648aee
67f4dae
caa4c7c
cebec12
b9bd43d
adaa5e2
316d550
4f7b76d
ec46479
fd698e9
e2a9f93
c1c7338
813466f
9834f0c
dc28a3e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,15 +6,24 @@ | |
| // lineage engine cannot settle is reported as such: models it could not trace (no compiled SQL, or | ||
| // SQL it could not parse), models that name the column only outside their select list (where the | ||
| // engine reports no edge), and models that read it through a table name several parents share. | ||
| import { loadRawManifest, findModel } from "./helpers" | ||
| import * as core from "@altimateai/altimate-core" | ||
| import { loadRawManifest, findModel, detectDialect } from "./helpers" | ||
| import { dbtLineage } from "./lineage" | ||
| import { missingColumn, projectLineage, selectsStar, stripSql } from "./project-lineage" | ||
| import { engineMade, missingColumn, projectLineage, selectsStar, stripSql } from "./project-lineage" | ||
|
|
||
| export interface ColumnReachModel { | ||
| name: string | ||
| unique_id: string | ||
| /** Its output columns that come from the traced column. */ | ||
| columns: string[] | ||
| /** | ||
| * How each of those columns (keyed as in `columns`) comes from the traced column, as the lineage | ||
| * engine classifies it: Original (carried as is), Alias (renamed), Transformation, Not sure or | ||
| * Non select. It covers the whole path from the traced column: a column is Original or Alias only | ||
| * when every hop is, so a column read from a calculation is never Original. A column whose | ||
| * edges carry no classification has no entry. | ||
| */ | ||
| lens_types?: Record<string, string> | ||
| } | ||
|
|
||
| export interface ColumnReachUnsure { | ||
|
|
@@ -58,6 +67,16 @@ export interface ColumnReach { | |
| const MAX_MODELS = 300 | ||
| const TRACED = new Set(["model", "snapshot"]) | ||
|
|
||
| // How far a classification is from "the value arrives unchanged", least to most. | ||
| const LENS_ORDER = ["Original", "Alias", "Non select", "Not sure", "Transformation"] | ||
|
|
||
| /** The classification of a path made of two: the one further from unchanged, unknown if either is. */ | ||
| function joinLens(a: string | undefined, b: string | undefined): string | undefined { | ||
| if (a === undefined || b === undefined) return undefined | ||
| const rank = (lens: string) => (LENS_ORDER.includes(lens) ? LENS_ORDER.indexOf(lens) : LENS_ORDER.length) | ||
| return rank(a) >= rank(b) ? a : b | ||
| } | ||
|
|
||
| /** "db"."schema"."orders"."amount" → { parts: ["db","schema","orders"], column: "amount" }, lower-cased. */ | ||
| function splitSource(source: unknown): { parts: string[]; column: string; exact: string } | null { | ||
| const text = | ||
|
|
@@ -66,8 +85,7 @@ function splitSource(source: unknown): { parts: string[]; column: string; exact: | |
| // Split on dots outside quotes: "a"."net.amount" is table a, column net.amount. | ||
| // A doubled closing delimiter inside a quoted name is one character ("a""b" is a"b, [a]]b] is a]b). | ||
| const raw = [...text.matchAll(/"((?:[^"]|"")*)"|`((?:[^`]|``)*)`|\[((?:[^\]]|\]\])*)\]|([^."`\[\]]+)/g)].map( | ||
| (m) => | ||
| m[1]?.replaceAll('""', '"') ?? m[2]?.replaceAll("``", "`") ?? m[3]?.replaceAll("]]", "]") ?? m[4] ?? "", | ||
| (m) => m[1]?.replaceAll('""', '"') ?? m[2]?.replaceAll("``", "`") ?? m[3]?.replaceAll("]]", "]") ?? m[4] ?? "", | ||
| ) | ||
| if (raw.length < 2) return null | ||
| // `exact` keeps the column's case, for parents that document two columns differing only by case. | ||
|
|
@@ -94,6 +112,68 @@ function relationKeys(node: any): string[] { | |
| return [...keys] | ||
| } | ||
|
|
||
| type LineageEdge = { source: unknown; target: string; lineage_type?: string; lens_type?: string } | ||
|
|
||
| /** | ||
| * The engine rates every edge "Not sure" when the table it reads is written with its schema | ||
| * ("db"."schema"."orders", as dbt compiles a ref) and its schema lists that table by bare name only. | ||
| * The model's SQL is traced again with each parent listed under every way its relation is written, | ||
| * and an edge both runs report (matched by output column, table name and source column, as the | ||
| * first run drops the qualification) takes the second run's rating. Anything else keeps its own. | ||
| */ | ||
| function rated( | ||
| edges: LineageEdge[], | ||
| sql: string, | ||
| dialect: string, | ||
| parents: string[], | ||
| lookup: (id: string) => any, | ||
| columnsOf: (id: string) => { name: string; type?: string }[], | ||
| ): LineageEdge[] { | ||
| if (!sql || parents.length === 0 || !edges.some((edge) => edge.lens_type === "Not sure")) return edges | ||
| try { | ||
| const tables: Record<string, { columns: { name: string; type: string }[] }> = {} | ||
| for (const parent of parents) { | ||
| const node = lookup(parent) | ||
| // The columns the first trace reads the parent against: the YAML's, plus any the compiled SQL gives it that the YAML omits. | ||
| const declared = Object.entries<any>(node?.columns ?? {}).map(([key, col]) => ({ | ||
| name: String(col?.name ?? key), | ||
| type: String(col?.data_type ?? col?.type ?? ""), | ||
| })) | ||
| const known = new Set(declared.map((col) => col.name.toLowerCase())) | ||
| const columns = [ | ||
| ...declared, | ||
| ...columnsOf(parent) | ||
|
hkaltai marked this conversation as resolved.
Resolved
|
||
| .filter((col) => !known.has(col.name.toLowerCase())) | ||
| .map((col) => ({ name: col.name, type: String(col.type ?? "") })), | ||
| ] | ||
| if (columns.length === 0) continue | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Minor: the second trace leaves out every parent with no YAML
A qualification whose case differs from the manifest ( This is conservative: nothing is copied wrongly. But the claim "rated by what the SQL does, not Not sure" holds only for documented parents. Using
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in adaa5e2:
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Round 2: resolved. Verified at
Probes for the merged schema found no case where a parent's columns attached to the wrong relation (same table name in two schemas, a parent and a source with one name, a non-parent relation). The case-different qualification (
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for verifying. The case-differing qualification ( |
||
| for (const key of relationKeys(node)) tables[key] = { columns } | ||
|
hkaltai marked this conversation as resolved.
Resolved
|
||
| } | ||
| if (Object.keys(tables).length === 0) return edges | ||
| const schema = core.Schema.fromJson(JSON.stringify({ tables, version: "1" })) | ||
| const again: LineageEdge[] = | ||
| JSON.parse(JSON.stringify(core.columnLineage(sql, dialect, schema))).column_lineage ?? [] | ||
| const keyOf = (edge: LineageEdge) => { | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Minor (test gap): the second-trace matching and the several-edges join are not pinned by any test. These mutations survive all of
These are caught: A fixture with two relations in the SQL that share a column name but rate differently, plus a mocked engine answer with two differently rated edges into one output, would pin both. Test-only.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Partly fixed in adaa5e2: added a test with two qualified relations sharing a column name but rated differently (pins the table name in
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Round 2: not resolved, test-only, does not block. At The new "two qualified relations" test does not pin the table name: Details and a suggested fixture are in the new inline comment on line 160. Review: #1733 (review)
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Acknowledged, test-only and non-blocking. The remaining mutations (last-wins vs |
||
| const source = splitSource(edge.source) | ||
| return source ? `${String(edge.target).toLowerCase()}|${source.parts.at(-1)}.${source.column}` : null | ||
|
hkaltai marked this conversation as resolved.
Resolved
|
||
| } | ||
| const lenses = new Map<string, string>() | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Minor (test-only): the second-trace and several-edges joins are still unpinned. These mutations still pass
These are caught: removing the case-collision clearing, skipping the second trace, dropping the compiled columns from the retry schema, and folding case when an exact YAML entry exists. To pin the first: two relations whose columns share a name, read into outputs with the same name in two branches of a
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Acknowledged, test-only and not blocking. The surviving mutations all fail in the conservative direction; adding the suggested union-all fixture is recorded as a follow-up test task.
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Round 3 (
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks. Acknowledged as a test-only follow-up, now including the |
||
| for (const edge of again) { | ||
| const key = keyOf(edge) | ||
| // Edges into one output from the same column keep the one furthest from unchanged. | ||
| if (key && typeof edge.lens_type === "string") | ||
| lenses.set(key, joinLens(lenses.get(key), edge.lens_type) ?? edge.lens_type) | ||
| } | ||
| return edges.map((edge) => { | ||
| const key = edge.lens_type === "Not sure" ? keyOf(edge) : null | ||
| const lens = key ? lenses.get(key) : undefined | ||
| return lens ? { ...edge, lens_type: lens } : edge | ||
| }) | ||
| } catch { | ||
| return edges | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * The parents an edge's source table names, matched at the qualification the edge carries: a | ||
| * qualified name that matches no parent names none (it is some other relation), and a bare name | ||
|
|
@@ -274,6 +354,8 @@ export function columnReach(manifestPath: string, model: string, column: string, | |
| if (unsettled) spelled = [] | ||
| else if (!exact && matches.length > 0) spelled = matches | ||
| const trackedExact = new Map<string, Set<string>>([[targetId, new Set(spelled)]]) | ||
| // How each tracked column comes from the traced column, per node (the traced column itself is Original). | ||
| const lensOf = new Map<string, Map<string, string | undefined>>([[targetId, new Map([[column0, "Original"]])]]) | ||
| const carries = (parent: string, source: { column: string; exact: string }) => | ||
| !!tracked.get(parent)?.has(source.column) && | ||
| (!caseCollides(lookup(parent), source.column) || !!trackedExact.get(parent)?.has(source.exact)) | ||
|
|
@@ -307,7 +389,7 @@ export function columnReach(manifestPath: string, model: string, column: string, | |
| unsure.add(id) | ||
| continue | ||
| } | ||
| const edges: { source: unknown; target: string; lineage_type?: string }[] = lineage?.column_lineage?.length | ||
| const listed: LineageEdge[] = lineage?.column_lineage?.length | ||
| ? lineage.column_lineage | ||
| : Object.entries<any>(lineage?.column_dict ?? {}).flatMap(([out, sources]) => | ||
| (Array.isArray(sources) ? sources : [sources]).map((source) => ({ | ||
|
|
@@ -323,6 +405,14 @@ export function columnReach(manifestPath: string, model: string, column: string, | |
| // cannot say which one is read; a `select *` from a parent whose schema does not list the | ||
| // column cannot be expanded to it. Both are possible reads with unknown outputs. | ||
| const sqlText: string = nodes[id]?.compiled_code || nodes[id]?.compiled_sql || "" | ||
| const edges = rated( | ||
| listed, | ||
| sqlText, | ||
| dialect || detectDialect(manifest), | ||
| allParents, | ||
| lookup, | ||
| (id) => project.columns(id), | ||
| ) | ||
| const bareName = (p: string) => { | ||
| const node = lookup(p) | ||
| return String(node?.alias || node?.identifier || node?.name || p).toLowerCase() | ||
|
|
@@ -346,6 +436,7 @@ export function columnReach(manifestPath: string, model: string, column: string, | |
|
|
||
| const carried = new Set<string>() | ||
| const carriedExact = new Set<string>() | ||
| const carriedLens = new Map<string, string | undefined>() | ||
| let reads = false | ||
| let ambiguous = false | ||
| for (const edge of edges) { | ||
|
|
@@ -360,10 +451,35 @@ export function columnReach(manifestPath: string, model: string, column: string, | |
| } | ||
| reads = true | ||
| if (edge.lineage_type !== "indirect" && typeof edge.target === "string") { | ||
| carried.add(edge.target.toLowerCase()) | ||
| const out = edge.target.toLowerCase() | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Major: outputs that differ only by case share one rating key, so an
Repro (duckdb, real engine). Trace select revenue as "Foo", gross_revenue as "foo" from "shop"."main"."src_o"The result is Lower-cased Raised by Codex, reproduced by the lead; severity is Codex's.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in adaa5e2:
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Round 2: partly resolved. At Still open:
Under this round's bar these are edge inputs, so the round-1 major is now a minor. It is tracked in the new inline comment on line 476, with a suggested fix (
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Round 2 follow-up, thanks. The round-1 repro (
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Minor (edge input; best first follow-up): an unaliased calculation gets an engine-invented target rated select revenue + 1 from "shop"."main"."fct_orders"Tracing
Suggested fix: publish a rating only for a target that is one of the model's real output names. Raised by Codex; the lead widened it to the bare case. Codex and the lead agree it is an edge input under this round's bar.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in e2a9f93: column-reach.ts now publishes a rating only for a target that is one of the model's real output names (
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Round 3 (
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for verifying. The side effects you listed in the two new inline comments (lines 478 and 482) are acknowledged there as conservative follow-ups. |
||
| // Only one candidate gets here: more than one is ambiguous and was skipped above. | ||
| const lens = joinLens( | ||
| lensOf.get(candidates[0]!)?.get(source.column), | ||
| typeof edge.lens_type === "string" ? edge.lens_type : undefined, | ||
| ) | ||
| // Several edges into one output: it is as far from unchanged as the furthest of them. | ||
| carriedLens.set(out, carried.has(out) ? joinLens(carriedLens.get(out), lens) : lens) | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
| carried.add(out) | ||
| carriedExact.add(edge.target) | ||
| } | ||
| } | ||
| // Outputs whose names differ only by case are different columns, but the result keys them in lower | ||
| // case: a rating there could name either one, so none is published. | ||
| // Every output the engine traced counts, not only those carrying the column: a sibling output | ||
| // that shares the lower-cased name is the one a rating could be mistaken for. | ||
| const exactByLower = new Map<string, Set<string>>() | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Minor (edge input, DuckDB only): duplicate output names collapse into one key, so select gross_revenue as revenue, revenue from "shop"."main"."fct_orders"Tracing Suggested fix: when two edges share one exact target, publish no rating for it. Raised by Codex; reproduced.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Declining: DuckDB-only edge input (a repeated output name; Snowflake, Postgres and BigQuery refuse to build such a model). Documented limitation, no change.
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Round 3 (
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks. Acknowledged as a DuckDB-only edge-input follow-up; no change in this PR. |
||
| for (const edge of edges) { | ||
| if (typeof edge.target !== "string") continue | ||
| const lower = edge.target.toLowerCase() | ||
| exactByLower.set(lower, (exactByLower.get(lower) ?? new Set<string>()).add(edge.target)) | ||
| } | ||
| for (const [out, exact] of exactByLower) if (exact.size > 1) carriedLens.set(out, undefined) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Minor (edge input; the rest of round-1's major): the collision guard only sees outputs that have a lineage edge, so some case-colliding outputs still get a rating.
select revenue as "Foo", 0 as "foo" from shop.main.fct_ordersTracing The plugin offers Suggested fix: also clear the rating when Not blocking under this round's bar: quoted output names that differ only by case are rare. Raised by Codex and by the lead independently; reproduced.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Declining as an edge input (quoted output names differing only by case, where the sibling has no source edge, or Snowflake upper-casing both targets). Nothing is copied unless the YAML also lists both spellings. Documented limitation of the heuristic output-name contract; the engine-made-name case on line 454 is fixed in e2a9f93.
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Round 3 (
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for the round-3 check. Agreed: accepted as an edge-input follow-up (quoted outputs differing only by case beside a sibling with no source edge). No change in this PR. |
||
| // A name the engine invents for an unaliased calculation (`_0`) is no column of the model: it gets no rating. | ||
| const realNames = new Set(project.columns(id).map((col) => col.name)) | ||
|
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Minor (edge input, conservative): an undocumented Parent
Not blocking: only Raised by Codex; reproduced by the lead.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for the repro. Agreed it is conservative (only |
||
| // A documented `_0` is the engine's placeholder all the same unless the SQL writes that name itself. | ||
| const strippedSql = stripSql(sqlText) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [SUGGESTION]: The Running A sibling test with
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Real gap, fixed in 9834f0c: added "an engine-style name that only a comment or literal mentions gets no rating" ( |
||
| for (const out of carriedLens.keys()) | ||
| if (!realNames.has(out) || engineMade(out, strippedSql)) carriedLens.set(out, undefined) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Minor (edge input, conservative): a real Parent SQL Not blocking: an unusual column name, and the loss is in the safe direction. Raised by Codex; reproduced by the lead.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks. Acknowledged: a real |
||
| // The engine has no schema for the tables inside an inlined ephemeral parent, so it reports no | ||
| // edge through it: unknown, not unread, even when another parent's edges are traced. | ||
| const viaEphemeral = parents.some((p) => lookup(p)?.config?.materialized === "ephemeral") | ||
|
|
@@ -376,10 +492,24 @@ export function columnReach(manifestPath: string, model: string, column: string, | |
| if (reads) { | ||
| tracked.set(id, carried) | ||
| trackedExact.set(id, carriedExact) | ||
| lensOf.set(id, carriedLens) | ||
| // Outputs fed by an unsettled or ephemeral parent, or read through an ambiguous name, stay unsettled. | ||
| // A NATURAL JOIN beside the column may join on it, or carry it on in another output. | ||
| if (ambiguous || fromUnsure || viaEphemeral || partial || opaque) unsure.add(id) | ||
| reached.push({ name: nodes[id]?.name ?? id, unique_id: id, columns: [...carried].sort() }) | ||
| const columns = [...carried].sort() | ||
| // Built with fromEntries: assigning to a column named __proto__ would set the prototype instead. | ||
| const lens_types: Record<string, string> = Object.fromEntries( | ||
| columns.flatMap((col) => { | ||
| const lens = carriedLens.get(col) | ||
| return lens === undefined ? [] : [[col, lens] as const] | ||
| }), | ||
| ) | ||
| reached.push({ | ||
| name: nodes[id]?.name ?? id, | ||
| unique_id: id, | ||
| columns, | ||
| ...(Object.keys(lens_types).length > 0 && { lens_types }), | ||
| }) | ||
| continue | ||
| } | ||
|
|
||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.