Skip to content

Commit fa6356c

Browse files
committed
Strip zero-width and BOM characters during diff normalization, search-replace, and truncation matching
- [Pasted text]  - is not the same as ​- - is something like this [Changes] needed in other processors? └── apps └── editor └── src └── utils └── changes-integration ├── diff-processor │ ├── diff-processor.spec.ts │ ├── diff-processor.ts │ ├── index.ts │ ├── test-cases │ │ ├── deletion-only │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ ├── empty-file │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ ├── empty-file-2 │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ ├── failing │ │ │ ├── diff.txt │ │ │ └── original.txt │ │ ├── failing-2 │ │ │ ├── diff.txt │ │ │ └── original.txt │ │ ├── generic-1 │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ ├── generic-2 │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ ├── generic-3 │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ ├── generic-4 │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ ├── generic-5 │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ ├── generic-6 │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ ├── generic-7 │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ ├── generic-8 │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ ├── generic-9 │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ ├── insert-start-end │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ ├── loose-gap-match │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ ├── multiple-hunks │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ ├── redundant-new-empty-line │ │ │ ├── diff.txt │ │ │ ├── expected.txt │ │ │ └── original.txt │ │ └── same-line-replace │ │ ├── diff.txt │ │ ├── expected.txt │ │ └── original.txt │ ├── types.ts │ └── utils │ ├── apply-blocks.ts │ ├── empty-file-handler.ts │ ├── match-blocks.ts │ ├── normalize.ts │ └── parse-blocks.ts ├── search-replace-processor │ ├── apply-search-replace-to-content │ │ ├── apply-search-replace-to-content.ts │ │ └── index.ts │ ├── index.ts │ ├── parse-search-replace-segments.ts │ └── types.ts └── truncations-processor ├── index.ts ├── test-cases │ ├── basic │ │ ├── expected.txt │ │ ├── original.txt │ │ └── truncated.txt │ ├── edge-truncations │ │ ├── expected.txt │ │ ├── original.txt │ │ └── truncated.txt │ ├── multiple-truncations │ │ ├── expected.txt │ │ ├── original.txt │ │ └── truncated.txt │ └── update-after-truncation │ ├── expected.txt │ ├── original.txt │ └── truncated.txt ├── truncations-processor.spec.ts ├── truncations-processor.ts └── utils ├── find-line-sequence.ts └── is-truncation-line.ts
1 parent 9659579 commit fa6356c

4 files changed

Lines changed: 26 additions & 13 deletions

File tree

‎apps/editor/src/utils/changes-integration/diff-processor/utils/normalize.ts‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,11 @@ export const normalize_original_code = (params: {
1919
.replace(/\r/g, '')
2020
.replace(/\n/g, '')
2121

22-
let line_normalized_processed = line_normalized.replace(/\s+/g, '')
22+
const clean_invisibles = line_normalized.replace(/[\u200B-\u200F\uFEFF]/g, '')
23+
let line_normalized_processed = clean_invisibles.replace(/\s+/g, '')
2324

2425
if (params.use_strict_whitespace) {
25-
line_normalized_processed = line_normalized
26+
line_normalized_processed = clean_invisibles
2627
}
2728

2829
original_code_lines_normalized.push({
@@ -63,11 +64,13 @@ export const normalize_patch_lines = (params: {
6364
line = '-~nnn'
6465
}
6566

67+
const cleaned_line = line.replace(/[\u200B-\u200F\uFEFF]/g, '')
68+
6669
patch_lines_original.push(
67-
line.startsWith(' ') ? line.substring(1, line.length) : line
70+
cleaned_line.startsWith(' ') ? cleaned_line.substring(1, cleaned_line.length) : cleaned_line
6871
)
6972

70-
let line_normalized = line
73+
let line_normalized = cleaned_line
7174
.replace(/\r\n/g, '')
7275
.replace(/\r/g, '')
7376
.replace(/\n/g, '')

‎apps/editor/src/utils/changes-integration/search-replace-processor/apply-search-replace-to-content/apply-search-replace-to-content.ts‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,20 +30,25 @@ export const apply_search_replace_to_content = (params: {
3030
: []
3131

3232
const clean_lines = (lines: string[]) =>
33-
lines.filter((l) => l.trim() !== '')
33+
lines.filter((l) => l.replace(/[\u200B-\u200F\uFEFF]/g, '').trim() !== '')
3434

3535
const clean_context_before = clean_lines(context_before)
3636
const clean_original = clean_lines(segment.original_lines)
3737
const clean_context_after = clean_lines(context_after)
3838

39-
const SEPARATOR = '(?:[ \\t]*\\r?\\n){1,5}'
39+
// Allow invisible characters alongside tabs and spaces
40+
const SEPARATOR = '(?:[ \\t\\u200B-\\u200F\\uFEFF]*\\r?\\n){1,5}'
4041

4142
const build_block_pattern = (lines: string[]) => {
4243
return lines
4344
.map((line) => {
44-
const trimmed = line.trim()
45-
const escaped = escape_reg_exp(trimmed)
46-
return `[ \\t]*${escaped}`
45+
const cleaned = line.replace(/[\u200B-\u200F\uFEFF]/g, '').trim()
46+
// Spread operator iterates code points correctly (safeguards emojis/surrogate pairs).
47+
// Allow optional zero-width characters anywhere inside the matched line.
48+
const flexible = [...cleaned]
49+
.map(escape_reg_exp)
50+
.join('[\\u200B-\\u200F\\uFEFF]*')
51+
return `[ \\t\\u200B-\\u200F\\uFEFF]*${flexible}`
4752
})
4853
.join(SEPARATOR)
4954
}
@@ -91,7 +96,9 @@ export const apply_search_replace_to_content = (params: {
9196
const matched_after =
9297
clean_context_after.length > 0 ? match[group_index++] : ''
9398

94-
const updated_text = segment.updated_lines.join(line_ending)
99+
const updated_text = segment.updated_lines
100+
.map((line) => line.replace(/[\u200B-\u200F\uFEFF]/g, ''))
101+
.join(line_ending)
95102

96103
const replacement_parts: string[] = []
97104
if (matched_before) replacement_parts.push(matched_before)

‎apps/editor/src/utils/changes-integration/truncations-processor/truncations-processor.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ export const process_truncations = (
55
new_text: string,
66
original_text: string
77
): string => {
8-
const new_lines = new_text.split(/\r?\n/)
8+
const cleaned_new_text = new_text.replace(/[\u200B-\u200F\uFEFF]/g, '')
9+
const new_lines = cleaned_new_text.split(/\r?\n/)
910
const original_lines = original_text.split(/\r?\n/)
1011

1112
const blocks: { type: 'code' | 'truncation'; lines: string[] }[] = []

‎apps/editor/src/utils/changes-integration/truncations-processor/utils/find-line-sequence.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ export const find_line_sequence = (params: {
1313
) {
1414
let match = true
1515
for (let j = 0; j < params.sequence.length; j++) {
16-
if (params.lines[i + j] !== params.sequence[j]) {
16+
const orig_clean = params.lines[i + j].replace(/[\u200B-\u200F\uFEFF]/g, '')
17+
if (orig_clean !== params.sequence[j]) {
1718
match = false
1819
break
1920
}
@@ -28,7 +29,8 @@ export const find_line_sequence = (params: {
2829
) {
2930
let match = true
3031
for (let j = 0; j < params.sequence.length; j++) {
31-
if (params.lines[i + j].trim() !== params.sequence[j].trim()) {
32+
const orig_clean = params.lines[i + j].replace(/[\u200B-\u200F\uFEFF]/g, '')
33+
if (orig_clean.trim() !== params.sequence[j].trim()) {
3234
match = false
3335
break
3436
}

0 commit comments

Comments
 (0)