fix(plugin-oracle): keep NCHAR text on a non-Unicode database charset

CHANGELOG.md

@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Cloudflare Tunnel offered for BigQuery, DuckDB, DynamoDB and other drivers that cannot use it, until their plugin was installed.
 - Oracle NCHAR, NVARCHAR2 and NCLOB columns failing to load on databases with the UTF8 national character set.
+- Oracle grid saves storing NCHAR, NVARCHAR2 and NCLOB text as ¿ or editing the wrong row on a non-Unicode database.
 
 ## [0.78.0] - 2026-10-07
 

Plugins/OracleDriverPlugin/OracleNationalText.swift

@@ -0,0 +1,69 @@
+//
+//  OracleNationalText.swift
+//  OracleDriverPlugin
+//
+
+import Foundation
+
+/// Text for an NCHAR, NVARCHAR2 or NCLOB column, written so it reaches the column whatever the database character set.
+///
+/// A string literal and a CHAR or VARCHAR bind pass through the database character set before the national one, so on
+/// a WE8MSWIN1252 database Lao text was stored as `¿¿¿` with no error, and a row match on it found the wrong row
+/// (measured on 23ai). `UNISTR` carries each character as ASCII or a `\XXXX` UTF-16 escape, which the server turns
+/// straight into the national character set. Text that is all ASCII is in every database character set and keeps its
+/// plain literal.
+internal enum OracleNationalText {
+    /// The most a UNISTR argument may hold. It is a string literal: 800 escapes parse, 801 fail with ORA-01704.
+    static let maxChunkBytes = 4_000
+
+    /// The most a UNISTR result may hold: it is an NVARCHAR2, and past 4,000 bytes an AL16UTF16 database cut it short
+    /// with no error (3,995 `a` and an `é` read back 2,000 characters, measured on 23ai). Each character is counted at
+    /// its widest under either national character set: 2 bytes for ASCII, 3 for every other UTF-16 unit.
+    static let maxChunkNationalBytes = 4_000
+
+    /// The text as `UNISTR` calls joined with `||`, each within both limits above and wrapped in `TO_NCLOB` for an
+    /// NCLOB so the joined value is not held to 4,000 bytes (ORA-01489), or nil when the text is all ASCII. NUL
+    /// characters are dropped, as the plain literal drops them.
+    static func sql(for text: String, asLOB: Bool) -> String? {
+        let scalars = text.unicodeScalars.filter { $0 != "\0" }
+        guard scalars.contains(where: { !$0.isASCII }) else { return nil }
+        var chunks: [String] = []
+        var chunk = ""
+        var chunkBytes = 0
+        var chunkNationalBytes = 0
+        for scalar in scalars {
+            let piece = escaped(scalar)
+            let nationalBytes = scalar.isASCII ? 2 : 3 * String(scalar).utf16.count
+            if chunkBytes + piece.utf8.count > maxChunkBytes
+                || chunkNationalBytes + nationalBytes > maxChunkNationalBytes {
+                chunks.append(chunk)
+                chunk = ""
+                chunkBytes = 0
+                chunkNationalBytes = 0
+            }
+            chunk += piece
+            chunkBytes += piece.utf8.count
+            chunkNationalBytes += nationalBytes
+        }
+        chunks.append(chunk)
+        return chunks
+            .map { asLOB ? "TO_NCLOB(UNISTR('\($0)'))" : "UNISTR('\($0)')" }
+            .joined(separator: " || ")
+    }
+
+    /// One character as UNISTR takes it: printable ASCII as itself, a quote doubled, a backslash as `\\`, and anything
+    /// else as one `\XXXX` per UTF-16 unit. Both halves of a surrogate pair come back together, so a chunk never splits
+    /// a character.
+    private static func escaped(_ scalar: Unicode.Scalar) -> String {
+        switch scalar {
+        case "'":
+            return "''"
+        case "\\":
+            return "\\\\"
+        case " "..."~":
+            return String(scalar)
+        default:
+            return String(scalar).utf16.map { String(format: "\\%04X", $0) }.joined()
+        }
+    }
+}

Plugins/OracleDriverPlugin/OracleRowWriter.swift

@@ -15,8 +15,9 @@ import TableProPluginKit
 /// Every value is written for the type the host reports for its column, so nothing depends on the session's NLS
 /// settings. Measured on 23ai: a DATE or TIMESTAMP value is converted with an explicit mask, because the same text
 /// assigned or compared bare fails with ORA-01861 under the default `DD-MON-RR`; a number is written as a literal,
-/// because a bound `123.45` fails with ORA-01722 once `NLS_NUMERIC_CHARACTERS` is `,.`; and NULL is the literal,
-/// because a NULL bind into an object-type column fails with ORA-00932.
+/// because a bound `123.45` fails with ORA-01722 once `NLS_NUMERIC_CHARACTERS` is `,.`; NULL is the literal,
+/// because a NULL bind into an object-type column fails with ORA-00932; and text for an NCHAR, NVARCHAR2 or NCLOB
+/// column is `UNISTR`, because a literal or bind passes through the database character set (``OracleNationalText``).
 internal struct OracleRowWriter {
     /// What the grid stages for a column the user leaves to the server's default. It is a marker, never a value.
     static let defaultMarker = PluginCellValue.text("__DEFAULT__")
@@ -292,6 +293,8 @@ internal enum OracleValueKind: Equatable {
     case binaryFloat
     case binaryDouble
     case bfile
+    case nationalText
+    case nationalLOB
     case other
 
     init(typeName: String) {
@@ -306,14 +309,18 @@ internal enum OracleValueKind: Equatable {
         case "BINARY_FLOAT": self = .binaryFloat
         case "BINARY_DOUBLE": self = .binaryDouble
         case "BFILE": self = .bfile
+        case "NVARCHAR2", "NCHAR": self = .nationalText
+        case "NCLOB": self = .nationalLOB
         default: self = .other
         }
     }
 
     var isNumeric: Bool {
         switch self {
         case .number, .binaryFloat, .binaryDouble: return true
-        case .date, .timestamp, .timestampWithTimeZone, .timestampWithLocalTimeZone, .bfile, .other: return false
+        case .date, .timestamp, .timestampWithTimeZone, .timestampWithLocalTimeZone, .bfile, .nationalText,
+             .nationalLOB, .other:
+            return false
         }
     }
 
@@ -331,7 +338,7 @@ internal enum OracleValueKind: Equatable {
             return "TO_TIMESTAMP(\(operand), '\(OracleRowWriter.timestampMask)')"
         case .timestampWithTimeZone, .timestampWithLocalTimeZone:
             return "TO_TIMESTAMP_TZ(\(operand), '\(OracleRowWriter.timestampWithTimeZoneMask)')"
-        case .number, .binaryFloat, .binaryDouble, .bfile, .other:
+        case .number, .binaryFloat, .binaryDouble, .bfile, .nationalText, .nationalLOB, .other:
             return operand
         }
     }
@@ -352,6 +359,10 @@ internal enum OracleValueKind: Equatable {
         case .date, .timestamp, .timestampWithTimeZone, .timestampWithLocalTimeZone:
             let keyword = text.trimmingCharacters(in: .whitespaces).uppercased()
             return Self.temporalFunctions.contains(keyword) ? keyword : nil
+        case .nationalText:
+            return OracleNationalText.sql(for: text, asLOB: false)
+        case .nationalLOB:
+            return OracleNationalText.sql(for: text, asLOB: true)
         case .bfile, .other:
             return nil
         }

TableProTests/Plugins/OracleNationalTextTests.swift

@@ -0,0 +1,152 @@
+//
+//  OracleNationalTextTests.swift
+//  TableProTests
+//
+//  Text for NCHAR, NVARCHAR2 and NCLOB columns. Every statement shape below was run against Oracle 23ai with a
+//  WE8MSWIN1252 database character set (national AL16UTF16), an AL32UTF8 one with a UTF8 national character set, and
+//  AL32UTF8 with AL16UTF16: each stored and matched the text, where a plain literal stored ¿¿¿ on the first.
+//
+
+import Foundation
+@testable import TablePro
+import TableProPluginKit
+import Testing
+
+struct OracleNationalTextTests {
+    @Test("ASCII text keeps its plain literal")
+    func asciiTextIsLeftAlone() {
+        #expect(OracleNationalText.sql(for: "abc 123 'q' \\", asLOB: false) == nil)
+        #expect(OracleNationalText.sql(for: "", asLOB: true) == nil)
+    }
+
+    @Test("Other characters are UTF-16 escapes, a quote is doubled and a backslash is escaped")
+    func escapes() {
+        #expect(OracleNationalText.sql(for: "ລາວ", asLOB: false) == "UNISTR('\\0EA5\\0EB2\\0EA7')")
+        #expect(
+            OracleNationalText.sql(for: "é a\\b 'q'", asLOB: false) == "UNISTR('\\00E9 a\\\\b ''q''')"
+        )
+        #expect(OracleNationalText.sql(for: "😀\n", asLOB: false) == "UNISTR('\\D83D\\DE00\\000A')")
+    }
+
+    @Test("NUL is dropped, as the plain literal drops it")
+    func nulIsDropped() {
+        #expect(OracleNationalText.sql(for: "é\0", asLOB: false) == "UNISTR('\\00E9')")
+    }
+
+    @Test("Long text is split into calls of at most 4,000 bytes, never inside a surrogate pair")
+    func chunking() throws {
+        let text = String(repeating: "ດ", count: 799) + "😀" + "ດ"
+        let sql = try #require(OracleNationalText.sql(for: text, asLOB: false))
+        let calls = sql.components(separatedBy: " || ")
+        #expect(calls.count == 2)
+        #expect(calls[0] == "UNISTR('" + String(repeating: "\\0E94", count: 799) + "')")
+        #expect(calls[1] == "UNISTR('\\D83D\\DE00\\0E94')")
+        for call in calls {
+            #expect(call.utf8.count - "UNISTR('')".utf8.count <= OracleNationalText.maxChunkBytes)
+        }
+    }
+
+    /// One UNISTR of 3,995 `a` and an `é` is 3,996 characters, 7,992 bytes in AL16UTF16, and an AL16UTF16 database kept
+    /// 2,000 of them with no error.
+    @Test("Mostly ASCII text is split by its size in the national character set, not only by its escapes")
+    func chunkingBoundsTheDecodedSize() throws {
+        let text = String(repeating: "a", count: 3_995) + "é"
+        let sql = try #require(OracleNationalText.sql(for: text, asLOB: true))
+        let calls = sql.components(separatedBy: " || ")
+        #expect(calls.count == 2)
+        #expect(calls[0] == "TO_NCLOB(UNISTR('" + String(repeating: "a", count: 2_000) + "'))")
+        #expect(calls[1] == "TO_NCLOB(UNISTR('" + String(repeating: "a", count: 1_995) + "\\00E9'))")
+    }
+
+    @Test("Each call is wrapped in TO_NCLOB for an NCLOB, so the joined value has no 4,000-byte limit")
+    func lobChunks() throws {
+        let sql = try #require(OracleNationalText.sql(for: String(repeating: "ດ", count: 801), asLOB: true))
+        #expect(sql.hasPrefix("TO_NCLOB(UNISTR('\\0E94"))
+        #expect(sql.components(separatedBy: " || ").allSatisfy { $0.hasPrefix("TO_NCLOB(UNISTR('") && $0.hasSuffix("'))") })
+        #expect(sql.components(separatedBy: " || ").count == 2)
+    }
+}
+
+struct OracleRowWriterNationalTextTests {
+    private static let columns = ["ID", "NV", "NC", "NCL", "V"]
+    private static let typeNames = [
+        "ID": "NVARCHAR2(20)", "NV": "nvarchar2", "NC": "NCHAR(5)", "NCL": "nclob", "V": "VARCHAR2(20)"
+    ]
+    private static let original: [PluginCellValue] = [.text("ກ"), .text("ລາວ"), .text("ລາວ  "), .null, .text("ລາວ")]
+
+    private func writer(keys: [String]) -> OracleRowWriter {
+        var context = PluginRowWriteContext()
+        context.columnTypeNames = Self.typeNames
+        var writer = OracleRowWriter(qualifiedTable: "\"T1\"", columns: Self.columns, primaryKeyColumns: keys)
+        writer.context = context
+        return writer
+    }
+
+    private func update(_ column: String, to value: PluginCellValue) -> PluginRowChange {
+        let index = Self.columns.firstIndex(of: column) ?? 0
+        return PluginRowChange(
+            rowIndex: 0,
+            type: .update,
+            cellChanges: [(columnIndex: index, columnName: column, oldValue: Self.original[index], newValue: value)],
+            originalRow: Self.original
+        )
+    }
+
+    private func write(_ writer: OracleRowWriter, _ change: PluginRowChange) throws -> PluginRowWrite {
+        let writes = try writer.rowWrites(
+            for: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: []
+        )
+        return try #require(writes.first)
+    }
+
+    @Test("A national column is set and matched by its key through UNISTR")
+    func keyedUpdate() throws {
+        let write = try write(writer(keys: ["ID"]), update("NV", to: .text("ລາວ😀")))
+        #expect(write.statement == "UPDATE \"T1\" SET \"NV\" = UNISTR('\\0EA5\\0EB2\\0EA7\\D83D\\DE00') "
+            + "WHERE \"ID\" = UNISTR('\\0E81')")
+        #expect(write.parameters.isEmpty)
+    }
+
+    @Test("A keyless match writes every national value through UNISTR, NCHAR as the padded value it read")
+    func keylessMatch() throws {
+        let write = try write(writer(keys: []), update("V", to: .text("x")))
+        #expect(write.statement == "UPDATE \"T1\" SET \"V\" = 'x' WHERE \"ID\" = UNISTR('\\0E81') "
+            + "AND \"NV\" = UNISTR('\\0EA5\\0EB2\\0EA7') AND \"NC\" = UNISTR('\\0EA5\\0EB2\\0EA7  ') "
+            + "AND \"NCL\" IS NULL AND \"V\" = 'ລາວ' AND ROWNUM = 1")
+    }
+
+    @Test("An NCLOB value is written through TO_NCLOB, however long, instead of a bind")
+    func nclobIsNotBound() throws {
+        let long = String(repeating: "ດ", count: 3_000)
+        let write = try write(writer(keys: ["ID"]), update("NCL", to: .text(long)))
+        #expect(write.statement.hasPrefix("UPDATE \"T1\" SET \"NCL\" = TO_NCLOB(UNISTR('\\0E94"))
+        #expect(write.parameters.isEmpty)
+    }
+
+    @Test("ASCII text in a national column keeps the plain literal, numeric-looking text quoted")
+    func asciiStaysPlain() throws {
+        let write = try write(writer(keys: ["ID"]), update("NV", to: .text("00123")))
+        #expect(write.statement == "UPDATE \"T1\" SET \"NV\" = '00123' WHERE \"ID\" = UNISTR('\\0E81')")
+    }
+
+    @Test("ASCII text over the literal limit in an NCLOB is still bound")
+    func longAsciiLOBIsBound() throws {
+        let long = String(repeating: "a", count: 5_000)
+        let write = try write(writer(keys: ["ID"]), update("NCL", to: .text(long)))
+        #expect(write.statement.hasPrefix("UPDATE \"T1\" SET \"NCL\" = ? WHERE"))
+        #expect(write.parameters == [.text(long)])
+    }
+
+    @Test("An insert writes national values through UNISTR and other text as before")
+    func insert() throws {
+        let writes = try writer(keys: ["ID"]).rowWrites(
+            for: [PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil)],
+            insertedRowData: [0: [.text("1"), .text("é"), .null, .text("ລ"), .text("é")]],
+            deletedRowIndices: [],
+            insertedRowIndices: [0]
+        )
+        let write = try #require(writes.first)
+        #expect(write.statement == "INSERT INTO \"T1\" (\"ID\", \"NV\", \"NC\", \"NCL\", \"V\") "
+            + "VALUES ('1', UNISTR('\\00E9'), NULL, TO_NCLOB(UNISTR('\\0EA5')), 'é')")
+    }
+}

docs/databases/oracle.mdx

@@ -122,7 +122,7 @@ If the connection drops while a transaction is open, the next write fails with "
 
 The Structure tab and the table DDL show each column's type as it was declared: `NVARCHAR2(100)`, `VARCHAR2(50 CHAR)`, `NUMBER(*,2)`, `TIMESTAMP(6) WITH TIME ZONE`, `REF "HR"."ADDRESS_T"`. `VARCHAR2` and `CHAR` always name their unit, `BYTE` or `CHAR`, so a copied definition means the same column whatever `NLS_LENGTH_SEMANTICS` the target session uses.
 
-NCHAR, NVARCHAR2 and NCLOB read the same under either national character set Oracle allows, `AL16UTF16` or `UTF8`.
+NCHAR, NVARCHAR2 and NCLOB read the same under either national character set Oracle allows, `AL16UTF16` or `UTF8`. A grid save writes text that is not ASCII into them as `UNISTR('\0EA5...')`, so it keeps characters the database character set lacks.
 
 | Oracle type | Display |
 |---|---|
@@ -165,6 +165,7 @@ The SSL mode offers no certificate rows there, so **Verify Identity** checks the
 - BFILE columns show the directory and file name, never the file's contents.
 - SQL*Plus commands such as `SET SERVEROUTPUT ON`, `SHOW ERRORS` and `EXEC` are not SQL and fail when sent. Delete them, and write `EXEC p` as `BEGIN p; END;`. Output needs no `SET SERVEROUTPUT ON`.
 - No Users & Roles pane. Manage accounts with `CREATE USER` and `GRANT` in the editor.
+- On a database whose character set is not Unicode, filters, imports, Copy To and copied SQL still send NCHAR, NVARCHAR2 and NCLOB text through that character set, so a character it lacks becomes `¿`. Write such values with `UNISTR` in the editor.
 
 ## Troubleshooting
 

project.yml

@@ -496,6 +496,7 @@ targets:
       - Plugins/OracleDriverPlugin/OracleBrowseSQL.swift
       - Plugins/OracleDriverPlugin/OracleColumnStatements.swift
       - Plugins/OracleDriverPlugin/OracleIndexStatements.swift
+      - Plugins/OracleDriverPlugin/OracleNationalText.swift
       - Plugins/OracleDriverPlugin/OraclePartitionMapping.swift
       - Plugins/OracleDriverPlugin/OracleRowWriter.swift
       - Plugins/OracleDriverPlugin/OracleTableDDL.swift