Skip to content

Commit 5c8f6ea

Browse files
committed
Refactor binary file detection to use a centralized utility function
1 parent 1d209d0 commit 5c8f6ea

4 files changed

Lines changed: 71 additions & 13 deletions

File tree

‎apps/editor/src/context/context-initialization.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import {
1717
type DuplicateWorkspaceContext
1818
} from '../constants/state-keys'
1919
import { ContextProvider } from './providers/context/context-provider'
20+
import { is_binary_file } from '../utils/is-binary'
2021

2122
export const token_count_emitter = new EventEmitter()
2223

@@ -267,11 +268,11 @@ export const context_initialization = async (
267268
display_path = vscode.workspace.asRelativePath(file_path)
268269
}
269270

270-
if (content_uint8_array.includes(0)) {
271+
if (is_binary_file(file_path, content_uint8_array)) {
271272
context_text += `<file path="${display_path.replace(
272273
/\\/g,
273274
'/'
274-
)}">\nBinary file\n</file>\n`
275+
)}">Binary file</file>\n`
275276
continue
276277
}
277278

‎apps/editor/src/context/providers/workspace/modules/token-calculator.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { dictionary } from '@shared/constants/dictionary'
55
import { Logger } from '@shared/utils/logger'
66
import type { IWorkspaceProvider } from '../workspace-provider'
77
import { shrink_file } from '@/context/utils/shrink-file'
8+
import { is_binary_file } from '@/utils/is-binary'
89

910
type TokenData = [number, number, number]
1011

@@ -306,7 +307,7 @@ export class TokenCalculator implements vscode.Disposable {
306307

307308
try {
308309
const buffer = await fs.promises.readFile(file_path)
309-
const is_binary = buffer.includes(0)
310+
const is_binary = is_binary_file(file_path, buffer)
310311

311312
let content = is_binary ? 'Binary file' : buffer.toString('utf8')
312313

‎apps/editor/src/utils/files-collector.ts‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { WorkspaceProvider } from '../context/providers/workspace/workspace-prov
44
import { natural_sort } from '@/utils/natural-sort'
55
import { OpenEditorsProvider } from '@/context/providers/open-editors/open-editors-provider'
66
import { shrink_file } from '../context/utils/shrink-file/shrink-file'
7+
import { is_binary_file } from './is-binary'
78

89
export class FilesCollector {
910
private workspace_provider: WorkspaceProvider
@@ -64,18 +65,23 @@ export class FilesCollector {
6465

6566
if (stats.isDirectory()) continue
6667

67-
let content = fs.readFileSync(file_path, 'utf8')
68+
const buffer = fs.readFileSync(file_path)
69+
const is_binary = is_binary_file(file_path, buffer)
6870

69-
const range = this.workspace_provider.get_range(file_path)
70-
if (range) {
71-
content = this.workspace_provider.apply_range_to_content(
72-
content,
73-
range
74-
)
75-
}
71+
let content = is_binary ? 'Binary file' : buffer.toString('utf8')
72+
73+
if (!is_binary) {
74+
const range = this.workspace_provider.get_range(file_path)
75+
if (range) {
76+
content = this.workspace_provider.apply_range_to_content(
77+
content,
78+
range
79+
)
80+
}
7681

77-
if (params?.shrink) {
78-
content = shrink_file(content, path.extname(file_path))
82+
if (params?.shrink) {
83+
content = shrink_file(content, path.extname(file_path))
84+
}
7985
}
8086

8187
const workspace_root = this._get_workspace_root_for_file(file_path)

‎apps/editor/src/utils/is-binary.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import * as path from 'path'
2+
3+
const BINARY_EXTENSIONS = new Set([
4+
// Images
5+
'.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.ico', '.tiff', '.svgz',
6+
// Documents
7+
'.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.pages', '.numbers', '.key',
8+
// Archives
9+
'.zip', '.tar', '.gz', '.rar', '.7z', '.bz2', '.xz', '.iso',
10+
// Audio/Video
11+
'.mp3', '.mp4', '.wav', '.avi', '.mkv', '.mov', '.webm', '.ogg', '.flac',
12+
// Fonts
13+
'.ttf', '.otf', '.woff', '.woff2', '.eot',
14+
// Executables/Compiled
15+
'.exe', '.dll', '.so', '.dylib', '.bin', '.dat', '.class', '.pyc', '.o', '.a', '.lib',
16+
// Databases and others
17+
'.sqlite', '.sqlite3', '.db'
18+
])
19+
20+
export const is_binary_file = (file_path: string, buffer?: Uint8Array): boolean => {
21+
const base = path.basename(file_path).toLowerCase()
22+
if (base == '.ds_store') return true
23+
24+
const ext = path.extname(file_path).toLowerCase()
25+
if (BINARY_EXTENSIONS.has(ext)) {
26+
return true
27+
}
28+
29+
if (!buffer) return false
30+
31+
const len = Math.min(buffer.length, 8000)
32+
let non_printable = 0
33+
34+
for (let i = 0; i < len; i++) {
35+
const byte = buffer[i]
36+
if (byte == 0) {
37+
return true
38+
}
39+
40+
if ((byte < 32 && byte != 9 && byte != 10 && byte != 13) || byte == 127) {
41+
non_printable++
42+
}
43+
}
44+
45+
if (len > 0 && non_printable / len > 0.1) {
46+
return true
47+
}
48+
49+
return false
50+
}

0 commit comments

Comments
 (0)