Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
b7cfc04
feat: foundation for the dbt Power User tools
hkaltai Nov 13, 2026
1eb722e
Merge remote-tracking branch 'origin/main' into feat/pu-mods-foundation
hkaltai Nov 13, 2026
6f445a6
feat: dbt_cte_profile lists a model's CTEs and times one CTE with a c…
hkaltai Nov 13, 2026
21d0494
fix: harden the dbt Power User tool foundation
hkaltai Nov 13, 2026
86e8746
fix: dbt_cte_profile reads a model's own CTEs only, names each one by…
hkaltai Nov 13, 2026
541d1be
fix: dbt_cte_profile ignores Jinja blocks that hold SQL strings, keep…
hkaltai Nov 13, 2026
ba6083f
Merge branch 'feat/pu-mods-foundation' into feat/dbt-cte-profile-tool
hkaltai Nov 13, 2026
85abfe2
fix: dbt_cte_profile builds its count query from the model's own text…
hkaltai Nov 13, 2026
c04d485
fix: dbt_cte_profile refuses a CTE inside a Jinja block tag, and read…
hkaltai Nov 14, 2026
180ee30
fix: dbt_cte_profile reads a bracket name with ]], skips a captured s…
hkaltai Nov 15, 2026
d8c3469
fix: send falsy API bodies, follow cancellation in apiRequest, check …
hkaltai Nov 15, 2026
dace1c2
Merge remote-tracking branch 'origin/main' into feat/pu-mods-foundation
hkaltai Nov 17, 2026
56f1006
fix: dbt_cte_profile undoes ]] in a bracketed CTE name when matching it
hkaltai Nov 17, 2026
1ebaa3b
Merge origin/main into feat/dbt-cte-profile-tool
hkaltai Nov 17, 2026
18a3fc4
fix: keep the stored API key out of API errors, bound the dbt config …
hkaltai Nov 17, 2026
6024ac9
fix: redact the API key when a refused header races the caller's abor…
hkaltai Nov 17, 2026
f5159c5
docs: say which tools use the dbt adapter helper
hkaltai Nov 17, 2026
f57de3d
docs: rewrap and correct the adapter helper comments
hkaltai Nov 17, 2026
f8ccb8b
Merge remote-tracking branch 'origin/feat/pu-mods-foundation' into fe…
hkaltai Nov 17, 2026
e0c5a20
fix: dbt_cte_profile skips captured Jinja blocks inside a CTE body an…
hkaltai Nov 17, 2026
6d82438
fix: dbt_cte_profile runs against the model's own name, numbers posit…
hkaltai Nov 17, 2026
493307a
fix: dbt_cte_profile flags a WITH clause whose first CTE is Jinja-bui…
hkaltai Nov 17, 2026
e591307
fix: dbt_cte_profile's incomplete-list note covers a prefixed or recu…
hkaltai Nov 17, 2026
921ac71
fix: dbt_cte_profile refuses a CTE chain that reads is_incremental() …
hkaltai Nov 17, 2026
8794c16
fix: dbt_cte_profile's is_incremental()/this refusal reads only Jinja…
hkaltai Nov 17, 2026
aaff414
fix: never echo a send failure's text, so a trimmed key can't leak
hkaltai Nov 17, 2026
3a98dff
fix: a rejection with no reason is a send failure, not an abort
hkaltai Nov 17, 2026
2dff02e
Merge remote-tracking branch 'origin/feat/pu-mods-foundation' into fe…
hkaltai Nov 17, 2026
fd04565
Merge remote-tracking branch 'origin/main' into feat/dbt-cte-profile-…
hkaltai Nov 17, 2026
188d0ba
fix: dbt_cte_profile's is_incremental()/this check steps over quoted …
hkaltai Nov 17, 2026
8676ac5
test: the this-in-a-hook refusal case asserts the refusal message
hkaltai Nov 17, 2026
01781b1
docs: dbt_cte_profile's description says macros a CTE calls are not o…
hkaltai Nov 17, 2026
dab9354
docs: dbt_cte_profile's description names where the is_incremental()/…
hkaltai Nov 17, 2026
4cd3aac
Merge origin/main into feat/dbt-cte-profile-tool
hkaltai Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
382 changes: 382 additions & 0 deletions packages/opencode/src/altimate/tools/cte-detect.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,382 @@
/**
* CTE detection and count-query building for the CTE profiler.
*
* Works on the raw model text (Jinja included) with string offsets. The scanner
* skips SQL line/block comments, Jinja comments and string literals, and
* understands quoted CTE names and CTE column lists.
*/

export interface CteInfo {
/** Identifier as written, including a trailing column list such as `my_cte (id, name)`. */
name: string
/** Offset where the name starts. */
nameStart: number
/** Offset just after the name (and its column list). */
nameEnd: number
/** Offset of the first character after the CTE body's opening parenthesis. */
queryStart: number
/** Offset of the CTE body's closing parenthesis. */
queryEnd: number
/** Order of the CTE within its WITH clause. */
index: number
/** Offset of the WITH keyword that opens this CTE's clause. */
withClauseStart: number
}

/** Position just past the string literal, quoted identifier or comment (`--`, block, Jinja) that starts at `pos`; `pos` itself when none does. An unterminated one runs to the end of the text. */
function skipLiteral(text: string, pos: number): number {
const char = text[pos]
const next = text[pos + 1]
if (char === "'" || char === '"' || char === "`") {
Comment thread
hkaltai marked this conversation as resolved.
Resolved
let i = pos + 1
while (i < text.length) {
if (text[i] === char) {
// A doubled quote is an escaped quote.
if (text[i + 1] === char) i += 2
else return i + 1
} else i++
}
return text.length
}
if (char === "$" && !/[\p{L}\p{N}_$]/u.test(text[pos - 1] ?? " ")) {
// A dollar-quoted string (`$$ … $$`, `$tag$ … $tag$`) runs to its matching tag.
const tag = /^\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/.exec(text.substring(pos, pos + 70))?.[0]
if (tag) {
const end = text.indexOf(tag, pos + tag.length)
return end === -1 ? text.length : end + tag.length
}
}
const close = char === "-" && next === "-" ? "\n" : char === "/" && next === "*" ? "*/" : char === "{" && next === "#" ? "#}" : null
Comment thread
hkaltai marked this conversation as resolved.
Resolved
if (close === null) return pos
const end = text.indexOf(close, pos + 2)
if (end === -1) return text.length
return close === "\n" ? end : end + close.length
}

/** Position of the first character at or after `pos` that is not whitespace, a comment, or (when `jinja`) a Jinja `{% %}` tag. */
function skipTrivia(text: string, pos: number, jinja: boolean): number {
while (pos < text.length) {
if (/\s/.test(text[pos])) {
pos++
continue
}
const end = skipLiteral(text, pos)
if (end !== pos && !/['"`]/.test(text[pos])) {
pos = end
continue
}
if (jinja && text[pos] === "{" && text[pos + 1] === "%") {
pos = skipJinjaBlock(text, pos)
continue
}
return pos
}
return pos
}

/** Offset of the `)` that closes the `(` at `openParenPos`, or -1 when it is never closed. */
function findMatchingClosingParen(text: string, openParenPos: number): number {
let depth = 1
let pos = openParenPos + 1
while (pos < text.length) {
const end = skipLiteral(text, pos)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if (end !== pos) {
pos = end
continue
}
// A captured block (`{% set x %}…{% endset %}`) is not the CTE's SQL: a `)` written there does not close it.
if (text[pos] === "{" && text[pos + 1] === "%") {
pos = skipJinjaBlock(text, pos)
continue
}
if (text[pos] === "(") depth++
else if (text[pos] === ")" && --depth === 0) return pos
pos++
}
return -1
}

interface JinjaTag {
/** Tag name (`if`, `set`, `macro`…), without the `end` prefix. */
name: string
/** The tag closes a block (`{% endif %}`). */
closes: boolean
/** The tag opens a block: not an `end` tag, `else`/`elif`, or an inline `{% set x = … %}`. */
opens: boolean
/** Offset just past the tag. */
end: number
}

const BLOCK_TAGS = new Set(["if", "for", "macro", "call", "filter", "with", "set", "raw", "block"])

/** The block tag of the `{% … %}` that spans `at` to `close` (the offset of its `%}`); undefined for any other tag. */
function parseTag(text: string, at: number, close: number): JinjaTag | undefined {
const head = /^\{%[-+]?\s*(end)?(\w+)/.exec(text.substring(at, Math.min(close + 2, at + 200)))
if (!head || !BLOCK_TAGS.has(head[2])) return undefined
const closes = head[1] !== undefined
// `{% set x = … %}` is an inline set; `{% set x %}` and `{% set x | filter %}` open a block.
const inline = head[2] === "set" && /^\{%[-+]?\s*set\s+[\w.,\s()]+=(?!=)/.test(text.substring(at, close + 2))
return { name: head[2], closes, opens: !closes && !inline, end: close + 2 }
}

/**
* The Jinja statement tags of `text` from `from`, in order, in one linear pass. Comments (`{# #}`) and the
* inside of `{% raw %}` are not tags; an unterminated comment or tag ends the text.
*/
function* jinjaTags(text: string, from: number): Generator<JinjaTag> {
let pos = from
for (;;) {
const at = text.indexOf("{", pos)
if (at === -1 || at + 1 >= text.length) return
const kind = text[at + 1]
if (kind !== "#" && kind !== "%") {
pos = at + 1
continue
}
const close = text.indexOf(kind === "#" ? "#}" : "%}", at + 2)
if (close === -1) return
pos = close + 2
if (kind === "#") continue
const tag = parseTag(text, at, close)
if (!tag) continue
if (tag.name === "raw" && tag.opens) {
// Nothing inside `{% raw %}` is Jinja: the block runs to its end tag.
const end = /\{%[-+]?\s*endraw\b/g
end.lastIndex = pos
const found = end.exec(text)
if (!found) {
yield { name: "raw", closes: false, opens: true, end: text.length }
return
}
const endClose = text.indexOf("%}", found.index)
pos = endClose === -1 ? text.length : endClose + 2
yield { name: "raw", closes: false, opens: false, end: pos }
continue
}
yield tag
}
}

/**
* Position just past the Jinja tag at `pos`, and past its whole block when it opens one whose text
* is not the model's own SQL (`{% set x %}…{% endset %}`, `{% call %}`, `{% macro %}`): a query
* written there is a string for dbt to run, not a CTE of the model.
*/
function skipJinjaBlock(text: string, pos: number): number {
const close = text.indexOf("%}", pos + 2)
if (close === -1) return text.length
const first = parseTag(text, pos, close)
if (!first || !first.opens || (first.name !== "set" && first.name !== "call" && first.name !== "macro")) return close + 2
// Blocks of the same kind nest: the block ends at the end tag that matches this one.
let depth = 1
for (const tag of jinjaTags(text, close + 2)) {
if (tag.name !== first.name) continue
if (tag.closes) depth--
else if (tag.opens) depth++
if (depth === 0) return tag.end
}
return text.length
}

/** Every WITH keyword outside comments, strings and parentheses: a WITH inside a CTE body or a subquery belongs to that query, not to the model. */
function findTopLevelWithKeywords(text: string): number[] {
const positions: number[] = []
const keyword = /with\b/iy
let depth = 0
let pos = 0
while (pos < text.length) {
const end = skipLiteral(text, pos)
if (end !== pos) {
pos = end
continue
}
const char = text[pos]
if (char === "{" && text[pos + 1] === "%") {
pos = skipJinjaBlock(text, pos)
continue
}
if (char === "(") depth++
else if (char === ")") depth = Math.max(0, depth - 1)
else if (depth === 0 && (char === "w" || char === "W") && !/[A-Za-z0-9_$]/.test(text[pos - 1] ?? " ")) {
keyword.lastIndex = pos
if (keyword.test(text)) {
positions.push(pos)
pos += 4
continue
}
}
pos++
}
return positions
}

/** Offset just past the identifier (`a`, `"a""b"`, `` `a` ``, `[a]`, `db.schema.a`) that starts at `pos`; -1 when none does. */
function readIdentifier(text: string, pos: number): number {
let end = pos
for (;;) {
const char = text[end]
if (char === '"' || char === "`") {
const close = skipLiteral(text, end)
if (close === end || text[close - 1] !== char || close - end < 2) return -1
end = close
} else if (char === "[") {
const bracketed = /\[(?:[^\]\n]|\]\]){1,256}\]/y
Comment thread
hkaltai marked this conversation as resolved.
Resolved
bracketed.lastIndex = end
if (!bracketed.test(text)) return -1
end = bracketed.lastIndex
} else {
const bare = /[\p{L}_][\p{L}\p{N}_$]*/uy
bare.lastIndex = end
if (!bare.test(text)) return -1
end = bare.lastIndex
}
if (text[end] !== ".") return end
end++
}
}

/** Offset of the `(` that opens a CTE body after `AS [NOT] [MATERIALIZED]` starting at `pos`; -1 when the text there is not such a header. */
function skipHeader(text: string, pos: number): number {
let at = pos
const word = (w: string) => {
const re = new RegExp(`${w}\\b`, "iy")
re.lastIndex = at
if (!re.test(text)) return false
at = skipTrivia(text, at + w.length, true)
return true
}
if (!word("as")) return -1
if (word("not")) {
if (!word("materialized")) return -1
} else word("materialized")
return text[at] === "(" ? at : -1
}

/** The CTEs of the WITH clause that opens at `withPos`: each `name [(columns)] AS (body)`, separated by commas, up to the first thing that is not one. */
function parseWithClause(text: string, withPos: number, ctes: CteInfo[]): void {
let pos = skipTrivia(text, withPos + 4, true)
const keyword = /recursive\b/iy
keyword.lastIndex = pos
if (keyword.test(text)) pos = skipTrivia(text, pos + "recursive".length, true)

for (let index = 0; ; index++) {
const nameStart = pos
const identifierEnd = readIdentifier(text, nameStart)
if (identifierEnd === -1) return
let nameEnd = identifierEnd
let name = text.substring(nameStart, identifierEnd)
pos = skipTrivia(text, identifierEnd, false)
if (text[pos] === "(") {
const close = findMatchingClosingParen(text, pos)
if (close === -1) return
// The name as written plus its column list; a comment between them is left out.
name += ` ${text.substring(pos, close + 1)}`
nameEnd = close + 1
pos = skipTrivia(text, nameEnd, false)
}
// `AS [NOT] [MATERIALIZED] (`, with comments or Jinja tags allowed between the words.
const open = skipHeader(text, pos)
if (open === -1) return
const close = findMatchingClosingParen(text, open)
if (close === -1) return
ctes.push({
name,
nameStart,
nameEnd,
queryStart: open + 1,
queryEnd: close,
index,
withClauseStart: withPos,
})
pos = skipTrivia(text, close + 1, true)
if (text[pos] !== ",") return
pos = skipTrivia(text, pos + 1, true)
}
}

/** Find every CTE of the model's own WITH clauses, in source order. A WITH inside a CTE body or a subquery is not listed: its CTEs cannot be counted without the query around them. */
export function detectCtes(text: string): CteInfo[] {
const ctes: CteInfo[] = []
for (const withPos of findTopLevelWithKeywords(text)) parseWithClause(text, withPos, ctes)
return ctes
}

/**
* Offset from which the CTEs could not be read, or undefined when every CTE was read: the
* comma after the last CTE read of a WITH clause that goes on, or the start of a clause where none was read
* (its first name is built by a Jinja expression or loop). The list is short from there on.
*/
export function unreadCteAfter(text: string, ctes: CteInfo[]): number | undefined {
const found: number[] = []
for (const withPos of findTopLevelWithKeywords(text)) {
Comment thread
hkaltai marked this conversation as resolved.
Resolved
const own = ctes.filter((c) => c.withClauseStart === withPos)
if (own.length === 0) {
// `START WITH` (a hierarchical query) opens no CTE list.
if (/\bstart\s*$/i.test(text.substring(Math.max(0, withPos - 24), withPos))) continue
Comment thread
hkaltai marked this conversation as resolved.
Resolved
let first = skipTrivia(text, withPos + 4, true)
if (/recursive\b/iy.test(text.substring(first, first + 10))) first = skipTrivia(text, first + "recursive".length, true)
const unreadName = /^[\p{L}\p{N}_$]*\{\{/u.exec(text.substring(first, first + 130))
Comment thread
hkaltai marked this conversation as resolved.
Resolved
Comment thread
hkaltai marked this conversation as resolved.
Resolved
if (unreadName) found.push(first)
continue
}
const last = own[own.length - 1]
const next = skipTrivia(text, last.queryEnd + 1, true)
if (text[next] === ",") found.push(next)
}
return found.length ? Math.min(...found) : undefined
}

/** Drop a trailing column list: `my_cte (id, name)` -> `my_cte`. */
export function stripColumnList(rawName: string): string {
const end = readIdentifier(rawName, 0)
return end === -1 || !rawName.substring(end).trimStart().startsWith("(") ? rawName.trim() : rawName.substring(0, end)
}

export function quoteSqlIdentifier(identifier: string): string {
if (identifier.match(/^["'`[]/) || identifier.includes(".")) return identifier
if (!identifier.match(/^[\p{L}_][\p{L}\p{N}_$]*$/u)) return `"${identifier}"`
return identifier
}

/** Name of the first Jinja block (`if`, `for`, `macro`, `call`, `filter`, `with`, `raw`, or `set` as a block) opened in `text` and never closed there; undefined when every block is closed. Tags inside Jinja comments and `{% raw %}` do not count. */
export function unclosedJinjaBlock(text: string): string | undefined {
const open: string[] = []
const counts = new Map<string, number>()
for (const tag of jinjaTags(text, 0)) {
if (tag.closes) {
// An end tag closes the nearest open block of its kind, and any block opened inside it.
if (!counts.get(tag.name)) continue
for (let top = open.pop(); top !== undefined; top = open.pop()) {
counts.set(top, counts.get(top)! - 1)
if (top === tag.name) break
}
} else if (tag.opens) {
open.push(tag.name)
counts.set(tag.name, (counts.get(tag.name) ?? 0) + 1)
}
}
return open[0]
}

/**
* `SELECT COUNT(*)` over one CTE: the model's text up to the end of that CTE (dbt configs, variables,
* Jinja between CTEs and every earlier CTE of the WITH clause, as written), so the query compiles on its own.
* Undefined for a CTE of a second WITH clause in the same file, which has no standalone query.
*/
export function buildCountQuery(text: string, ctes: CteInfo[], target: CteInfo): string | undefined {
// The text before a later WITH holds the earlier clause's whole query: nothing valid stands alone there.
if (ctes.some((c) => c.withClauseStart < target.withClauseStart)) return undefined
if (target.queryStart <= target.nameStart) return undefined
const head = text.substring(0, target.queryEnd + 1).trim()
return `${head}\nSELECT COUNT(*) AS _profile_count FROM ${quoteSqlIdentifier(stripColumnList(target.name))}`
}

/**
* Read the single COUNT(*) value by position: Snowflake and Oracle upper-case
* the alias, so a lookup by name would report 0 rows.
*/
export function extractRowCount(data: Record<string, unknown>[] | undefined): number {
if (!data || data.length === 0) return 0
const first = Object.values(data[0])[0]
return typeof first === "number" ? first : Number(first) || 0
}
Loading
Loading