Skip to content
Merged
146 changes: 132 additions & 14 deletions packages/opencode/src/altimate/native/dbt/project-lineage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,7 @@ export function selectItems(stripped: string): string[] {
if (start < 0) {
if (boundary && word === "select") {
start = i + 6
// A whole word only: `allocation` and `distinct_users` are column names, not modifiers.
const lead = /^\s+(?:distinct\s+on\s*\([^)]*\)|distinct|all|top\s+\d+)(?![\w$])/i.exec(text.slice(start))
if (lead) start += lead[0].length
start += modifierLength(text, start)
from = start
i = start - 1
}
Expand Down Expand Up @@ -179,20 +177,140 @@ export function itemName(item: string): string | null {
return ENDS_EXPRESSION.test(plain[2]!) ? null : unquoteName(plain[2]!)
}

/** The column a SELECT item casts, unaliased (`amount::numeric`, `cast(t.amount as numeric)`); null otherwise. */
function castColumn(item: string): string | null {
const ref = `(?:(?:${QUOTED}|[\\w$]+)\\.)*(${QUOTED}|[\\w$]+)`
// A type name: words (`double precision`, `pg_catalog.int4`), a size (`numeric(10, 2)`), array marks (`int[]`,
// `int[3]`), a quoted name, a further `::` cast.
const type = `((?:[\\w\\s(),:.]|\\[\\d*\\]|"(?:[^"]|"")*")+)`
const m = new RegExp(`^${ref}\\s*::\\s*${type}$`, "i").exec(item) ?? new RegExp(`^cast\\s*\\(\\s*${ref}\\s+as\\s+${type}\\)$`, "i").exec(item)
if (!m || NOT_TYPE.test(m[2]!.replace(new RegExp(QUOTED, "g"), " "))) return null
return unquoteName(m[1]!)
/**
* The length of the modifier a SELECT list starts with at `at` in quote-blanked text: `distinct on (…)` up to its
* matching `)`, `top N`, or the whole word `distinct` or `all` (`allocation` and `distinct_users` are names); 0 when
* there is none.
*/
function modifierLength(text: string, at: number): number {
const rest = text.slice(at)
const on = /^\s+distinct\s+on\s*\(/i.exec(rest)
if (on) {
const end = closingParen(rest, on[0].length - 1)
return end < 0 ? 0 : end + 1
}
// The same name characters as the cast reader's, so `allé` is a name too.
return /^\s+(?:top\s+\d+|(?:distinct|all)(?![\p{L}\p{N}_$]))/iu.exec(rest)?.[0].length ?? 0
}

/** Words that make the rest of an item more than a type (`amount::numeric is not null`, `amount::int as x`). */
/** The index of the `)` that closes the `(` at `open` in quote-blanked text, -1 when it isn't closed. */
function closingParen(text: string, open: number): number {
let depth = 0
for (let i = open; i < text.length; i++) {
if (text[i] === "(") depth++
else if (text[i] === ")" && --depth === 0) return i
}
return -1
}

/** A token of a SELECT item for the cast reader. `other` is any character the reader never accepts (`+`, `-`, `|`, …). */
type CastToken = { kind: "name" | "quoted" | "number" | "string" | "punct" | "other"; text: string }

const CAST_TOKEN =
/\s+|("(?:[^"]|"")*"|`(?:[^`]|``)*`)|('(?:[^']|'')*')|(\d+(?:\.\d*)?(?:e[+-]?\d+)?|\.\d+(?:e[+-]?\d+)?)|([\p{L}_$][\p{L}\p{N}_$]*)|(::|[().,[\]])|([^])/iuy

/** The tokens of a literal- and comment-stripped SELECT item, whitespace dropped. */
function castTokens(item: string): CastToken[] {
const out: CastToken[] = []
CAST_TOKEN.lastIndex = 0
for (let m = CAST_TOKEN.exec(item); m; m = CAST_TOKEN.exec(item)) {
const kind = m[1] ? "quoted" : m[2] ? "string" : m[3] ? "number" : m[4] ? "name" : m[5] ? "punct" : m[6] ? "other" : undefined
if (kind) out.push({ kind, text: m[0] })
}
return out
}

/** Words a type name never holds: an item with one after `::` is more than a cast (`amount::numeric is not null`). */
const NOT_TYPE =
/(?<![\w$])(?:as|is|not|isnull|notnull|null|true|false|between|in|like|ilike|similar|and|or|div|mod|xor|collate|at|over|filter|case|when|then|else|end|escape)(?![\w$])/i
/^(?:as|is|not|isnull|notnull|null|true|false|between|in|like|ilike|similar|and|or|div|mod|xor|collate|at|over|filter|case|when|then|else|end|escape)$/i

/**
* The column a SELECT item casts, unaliased; null otherwise. It reads the item as tokens, in this grammar, and the
* whole item must match:
*
* item = operand { "::" type } (at least one cast or one pair of parentheses)
* operand = column | "(" item ")" | "cast" "(" item "as" type ")"
* column = name { "." name } (a bare `null`, `true`, `false` or `end` is a literal)
* type = word { word | "." | "(" mod { "," mod } ")" | "[" [ number ] "]" } (mod = number | name)
*
* So `amount::numeric`, `(amount)::numeric`, `cast((amount)as numeric)`, `cast(amount as numeric)::int` and `q.end::int`
* name a column; `amount::numeric + 1`, `coalesce(amount)::numeric`, `(1.5)::numeric` and `(null)::int` don't.
*/
export function castColumn(item: string): string | null {
const t = castTokens(item)
let at = 0
let shaped = false
const punct = (k: number, text: string) => t[k]?.kind === "punct" && t[k]!.text === text
const word = (k: number, text: string) => t[k]?.kind === "name" && t[k]!.text.toLowerCase() === text
const named = (k: number) => t[k]?.kind === "quoted" || t[k]?.kind === "name"
// A type modifier: a number (`numeric(10, 2)`) or a name (`varchar(max)`, `geometry(Point, 4326)`).
const modifier = (k: number) => t[k]?.kind === "number" || named(k)

const type = (): boolean => {
let words = 0
for (;;) {
const k = t[at]
if (k && (k.kind === "quoted" || (k.kind === "name" && !NOT_TYPE.test(k.text)))) at++, words++
else if (words && punct(at, ".")) at++
else if (words && punct(at, "(") && modifier(at + 1)) {
at += 2
while (punct(at, ",") && modifier(at + 1)) at += 2
if (!punct(at, ")")) return false
at++
} else if (words && punct(at, "[")) {
at += t[at + 1]?.kind === "number" ? 2 : 1
if (!punct(at, "]")) return false
at++
} else return words > 0
}
}
const column = (): string | null => {
if (!named(at)) return null
const start = at++
while (punct(at, ".") && named(at + 1)) at += 2
const last = t[at - 1]!
// Only the token's own quote is escaped by doubling: `"a``b"` is a``b.
if (last.kind === "quoted") {
const q = last.text[0]!
return last.text.slice(1, -1).replaceAll(q + q, q).toLowerCase()
}
return at - start === 1 && ENDS_EXPRESSION.test(last.text) ? null : last.text.toLowerCase()
}
const operand = (): string | null => {
if (punct(at, "(")) {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor (Codex; lead reproduced). A Postgres escape string with \' can shift the SELECT items, so a real cast gets another column's name. This doesn't block: it needs constructed input, and main already does the same with the plain form.

Repro on Postgres (f = select 1 as id, 2 as b, 3 as c):

-- r
select id, E'it\'s, (b)::int, example', c::int, b as b from shop.main.f
-- child
select * from shop.main.r
  • stripSql ends the literal at \', so selectItems returns id, E''s, (b)::int, example', … The engine's real c::int edge (_col_2) is then read against the fake item (b)::int.
  • Head: columnAnswer(f, c) gives f.c → r.b → child.b, with nothing in may_carry. Base: r._col_2, with child in may_carry.
  • Main does the same with the unparenthesised form E'it\'s, b::int, example', c::int, …, which also gives r.b, child.b at bdd275f. So the cause is older than this PR. The new parenthesised operand only widens it.

Why it isn't a block: the string has to be an E'…' literal with a backslash-escaped quote, and its text has to contain a comma and then a cast of a column. That is constructed SQL, not something ordinary models write. Codex agrees it is minor.

A fix I checked locally, fine for a follow-up: in stripSql, add (?<![\w$])[eE]'(?:[^'\\]|\\[\s\S]|'')*' before the plain '…' alternative, and empty it to '' like the others. With it, the repro gives f.c → r.c → child.c (the plain form too), and column-answer.test.ts passes 84/84.

at++
const name = chain()
if (name === null || !punct(at, ")")) return null
at++
shaped = true
return name
}
if (word(at, "cast") && punct(at + 1, "(")) {
at += 2
const name = chain()
if (name === null || !word(at, "as")) return null
at++
if (!type() || !punct(at, ")")) return null
at++
shaped = true
return name
}
return column()
}
const chain = (): string | null => {
const name = operand()
if (name === null) return null
while (punct(at, "::")) {
at++
if (!type()) return null
shaped = true
}
return name
}

const name = chain()
return name !== null && shaped && at === t.length ? name : null
}

/**
* The alias templates `COLUMNS(...) AS "…"` items give (DuckDB's `"col_\0"`, which hold a `\N` marker), lower-cased.
Expand Down
96 changes: 93 additions & 3 deletions packages/opencode/test/altimate/column-answer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { join } from "node:path"
import { ENGINE, MAX_ANSWER_COLUMNS, columnAnswer, describeAnswer, type ColumnAnswer } from "../../src/altimate/native/dbt/column-answer"
import { columnReach, namesColumn } from "../../src/altimate/native/dbt/column-reach"
import { dbtLineage } from "../../src/altimate/native/dbt/lineage"
import { clearLineageCaches, hasNestedWith, howOf, itemName, quotingIsClear, relationUse, shapeFlags, stripAggFilter, stripSql } from "../../src/altimate/native/dbt/project-lineage"
import { castColumn, clearLineageCaches, hasNestedWith, howOf, itemName, quotingIsClear, relationUse, shapeFlags, stripAggFilter, stripSql } from "../../src/altimate/native/dbt/project-lineage"

const dirs: string[] = []
afterEach(() => {
Expand Down Expand Up @@ -1303,7 +1303,7 @@ describe("both tools: names at the edges", () => {
})

test("a column name that starts with all or distinct isn't read as a SELECT modifier", () => {
for (const name of ["allocation", "distinct_users", "all_time"]) {
for (const name of ["allocation", "distinct_users", "all_time", "all$x"]) {
const path = write(nodes("postgres", [
model("f", [], `select 2.5 as ${name}`, [name]),
model("r", ["f"], `with q as (select * from ${T("f")}) select ${name}::numeric from q`),
Expand All @@ -1316,8 +1316,90 @@ describe("both tools: names at the edges", () => {
}
})

test("a cast after a SELECT modifier is named after its column, so a settled model has it", () => {
// Most throw "r has no column amount" on main, where the modifier or the parentheses hid the cast.
// `distinct on (id) id, …` and `top 5 amount…` already answer there; they stay as controls.
for (const dialect of ["postgres", "redshift"]) {
const lists = ["distinct(amount)::numeric, id", "all(amount)::numeric, id", "distinct ((amount))::numeric, id", "id, (amount)::numeric",
"id, cast((amount) as numeric)"]
// DISTINCT ON is Postgres only; TOP is Redshift's.
if (dialect === "postgres")
lists.push("distinct on(id)amount::numeric, id", "distinct on (coalesce(id, 0)) id, amount::numeric", "distinct on (id) id, amount::numeric")
if (dialect === "redshift") lists.push("top 5amount::numeric, id", "top 5 amount::numeric, id")
for (const list of lists) {
const path = write(nodes(dialect, [
model("f", [], "select 1 as id, 2.5 as amount", ["id", "amount"]),
model("r", ["f"], `with q as (select * from ${T("f")}) select ${list} from q`),
model("child", ["r"], `select * from ${T("r")}`),
]))
expect(() => columnReach(path, "r", "amount")).not.toThrow()
expect(carried(columnAnswer(path, "r", "amount"))).toEqual(["child.amount"])
}
}
})

test("a qualified column named like a literal keeps its name through a cast", () => {
for (const dialect of ["postgres", "redshift"]) {
for (const [name, cast] of [["end", "q.end::int"], ["end", "cast(q.end as int)"], ["end", "(q.end)::int"], ["null", "q.null::int"], ["true", "q.true::int"]]) {
const path = write(nodes(dialect, [
model("f", [], `select 1 as id, 2 as "${name}"`, ["id", name!]),
model("r", ["f"], `with q as (select * from ${T("f")}) select id, ${cast} from q`),
model("child", ["r"], `select * from ${T("r")}`),
]))
expect(() => columnReach(path, "r", name!)).not.toThrow()
expect(carried(columnAnswer(path, "r", name!))).toEqual([`child.${name}`])
}
}
})

// Every cast form the reviews of this reader found, with the name Postgres gives it (null: not a cast of a column, so
// the engine's `_col_N` stays). Items are as `selectItems` hands them over: string literals already stripped to `''`.
const CAST_FORMS: [string, string | null][] = [
// Whole-item casts of a column.
["amount::numeric", "amount"], ["cast(amount as numeric)", "amount"], ["t.amount::numeric", "amount"],
["cast(t.amount as numeric)", "amount"], ["amount::int[]", "amount"], ["cast(amount as int[])", "amount"],
["amount::int[3]", "amount"], ['amount::"numeric"', "amount"], ["amount::pg_catalog.numeric", "amount"],
["amount::double precision", "amount"], ["amount::numeric(10, 2)", "amount"], ["cast(amount as numeric(10, 2))", "amount"],
["amount::timestamp(3) with time zone", "amount"], ["amount::geometry(Point,4326)", "amount"],
["cast(amount as geometry(Point, 4326))", "amount"], ["amount::varchar(max)", "amount"],
// Chains and parentheses.
["amount::numeric::int", "amount"], ["cast(amount as numeric)::int", "amount"], ["cast(cast(amount as numeric) as int)", "amount"],
["(amount::numeric)::int", "amount"], ["(amount)::numeric", "amount"], ["((amount))::numeric", "amount"],
["cast((amount) as numeric)", "amount"], ["(amount)", "amount"],
// No space beside `as`, and quoted names.
["cast((amount)as numeric)", "amount"], ['cast("amount"as numeric)', "amount"], ['cast(amount as"numeric")', "amount"],
['"Amount"::int', "amount"], ['("a""b")::numeric', 'a"b'], ['("null")::int', "null"], ['"a``b"::int', "a``b"], ['`a""b`::int', 'a""b'],
// Qualified names that read like literals or keywords.
["q.end::int", "end"], ["cast(q.end as int)", "end"], ["(q.end)::int", "end"], ["q.null::int", "null"], ["q.true::int", "true"],
["cast(t.has as int)", "has"], ["allé::numeric", "allé"],
// Not a cast of a column.
["amount", null], ["t.amount", null], ["(null)::int", null], ["(true)::boolean", null], ["(1)::int", null], ["1::int", null],
["cast(1.5 as numeric)", null], ["(1.0)::numeric", null], ["''::int", null], ["coalesce(amount)::numeric", null],
["-(amount)::numeric", null], ["(amount)::numeric * 2", null], ["(amount + 1)::int", null], ["amount::numeric + 1", null],
["cast(amount as int) * 2", null], ["amount::text || ''", null], ["cast(amount as int) between 1 and 2", null],
["amount::numeric is not null", null], ["amount::int in (1, 2)", null], ["amount::numeric as _col_1", null],
["amount::timestamp at time zone ''", null], ["amount::text collate \"C\"", null], ["arr[1]::int", null], ["cast(amount)", null],
["cast(amount as)", null], ["amount::", null], ["(amount", null],
]

test("castColumn reads every cast form found so far", () => {
const wrong = CAST_FORMS.filter(([item, name]) => castColumn(item) !== name).map(([item, name]) => `${item} → ${castColumn(item)} (want ${name})`)
expect(wrong).toEqual([])
})

test("mixed and nested cast chains of a column are named after it", () => {
for (const cast of ["cast(amount as numeric)::int", "cast(cast(amount as numeric) as int)", "(amount::numeric)::int"]) {
const path = write(nodes("postgres", [
model("orders", [], "select 1 as order_id, 2.5 as amount", ["order_id", "amount"]),
model("r", ["orders"], `select order_id, ${cast} from ${T("orders")}`),
]))
expect(carried(columnAnswer(path, "orders", "amount"))).toEqual(["r.amount"])
}
})

test("an expression that only starts with a cast isn't named after the column, and a _col_N the SQL writes stays", () => {
for (const item of ["amount::numeric is not null", "amount::int in (1, 2)", "cast(amount as int) between 1 and 2", "amount::numeric as _col_1"]) {
for (const item of ["amount::numeric is not null", "amount::int in (1, 2)", "cast(amount as int) between 1 and 2", "amount::numeric as _col_1",
"(amount + 1)::int", "amount::numeric + 1", "cast(amount as int) * 2"]) {
const path = write(nodes("postgres", [
model("orders", [], "select 1 as order_id, 2.5 as amount", ["order_id", "amount"]),
model("r", ["orders"], `select order_id, ${item} from ${T("orders")}`),
Expand Down Expand Up @@ -1352,6 +1434,14 @@ describe("both tools: names at the edges", () => {
]))
const named = dbtLineage({ manifest_path: clash, model: "child" })
expect(Object.keys((named.raw_lineage as any).column_dict ?? {}).sort()).toEqual(["id", "id_1", "id_1_1"])
// Names repeat regardless of case: `Val, val` is `val, val_1`.
const cased = write(nodes("duckdb", [
model("f", [], "select 1 as id, 2.5 as amount", ["id", "amount"]),
model("r", ["f"], `select id, amount as Val, id as val from ${T("f")}`),
model("child", ["r"], `select * from ${T("r")}`),
]))
const folded = dbtLineage({ manifest_path: cased, model: "child" })
expect(Object.keys((folded.raw_lineage as any).column_dict ?? {}).sort()).toEqual(["id", "val", "val_1"])
})

test("the text line says how many may-carry models are only not traced, without suggesting no effect", () => {
Expand Down
Loading