@@ -2,7 +2,6 @@ import * as fs from 'fs'
22import * as path from 'path'
33import * as vscode from 'vscode'
44import { Logger } from '@shared/utils/logger'
5- import { t } from '@/i18n'
65import type { IWorkspaceProvider } from '../workspace-provider'
76import { is_binary_file } from '@/utils/is-binary'
87import { normalize_path } from '@/utils/normalize-path'
@@ -12,191 +11,9 @@ export class TokenCalculator implements vscode.Disposable {
1211 private _directory_token_counts : Map < string , number > = new Map ( )
1312 private _directory_file_counts : Map < string , number > = new Map ( )
1413 private _directory_selected_token_counts : Map < string , number > = new Map ( )
15- private _notified_roots : Set < string > = new Set ( )
16-
17- private _progress_stack : {
18- progress ?: vscode . Progress < { message ?: string ; increment ?: number } >
19- total_files : number
20- processed_files : number
21- } [ ] = [ ]
2214
2315 constructor ( private _provider : IWorkspaceProvider ) { }
2416
25- private _report_progress ( ) {
26- for ( const state of this . _progress_stack ) {
27- if ( state . total_files > 0 ) {
28- state . processed_files ++
29- const increment = ( 1 / state . total_files ) * 100
30- if ( state . progress ) {
31- state . progress . report ( { increment } )
32- }
33- }
34- }
35- }
36-
37- private async _count_uncached_files_in_dir (
38- dir_path : string
39- ) : Promise < number > {
40- if ( this . _directory_token_counts . has ( dir_path ) ) {
41- return 0
42- }
43-
44- let count = 0
45- try {
46- const workspace_root =
47- this . _provider . get_workspace_root_for_file ( dir_path )
48- if ( ! workspace_root ) return 0
49-
50- const relative_dir_path = path . relative ( workspace_root , dir_path )
51- if (
52- this . _provider . is_excluded (
53- relative_dir_path ? relative_dir_path + '/' : relative_dir_path
54- )
55- ) {
56- return 0
57- }
58-
59- const entries = await fs . promises . readdir ( dir_path , {
60- withFileTypes : true
61- } )
62- for ( const entry of entries ) {
63- if ( entry . isSymbolicLink ( ) ) continue
64-
65- const full_path = path . join ( dir_path , entry . name )
66-
67- const file_workspace_root =
68- this . _provider . get_workspace_root_for_file ( full_path )
69- if ( file_workspace_root && file_workspace_root !== workspace_root ) {
70- continue
71- }
72-
73- const relative_path = path . relative ( workspace_root , full_path )
74- if (
75- this . _provider . is_excluded (
76- entry . isDirectory ( ) ? relative_path + '/' : relative_path
77- )
78- ) {
79- continue
80- }
81-
82- if ( this . _provider . is_ignored_by_patterns ( full_path ) ) {
83- continue
84- }
85-
86- if ( entry . isDirectory ( ) ) {
87- count += await this . _count_uncached_files_in_dir ( full_path )
88- } else if ( entry . isFile ( ) ) {
89- if ( ! this . _file_token_counts . has ( full_path ) ) {
90- count ++
91- }
92- }
93- }
94- } catch { }
95- return count
96- }
97-
98- async with_token_counting_notification < T > ( params : {
99- dir_path ?: string
100- roots ?: string [ ]
101- task : ( ) => Promise < T >
102- } ) : Promise < T > {
103- let total_uncached = 0
104- let should_notify = false
105-
106- if ( params . dir_path ) {
107- total_uncached = await this . _count_uncached_files_in_dir ( params . dir_path )
108- const workspace_roots = this . _provider . get_workspace_roots ( )
109- if ( workspace_roots . includes ( params . dir_path ) ) {
110- if ( ! this . _notified_roots . has ( params . dir_path ) ) {
111- should_notify = true
112- this . _notified_roots . add ( params . dir_path )
113- }
114- }
115- } else if ( params . roots ) {
116- for ( const root of params . roots ) {
117- total_uncached += await this . _count_uncached_files_in_dir ( root )
118- if ( ! this . _notified_roots . has ( root ) ) {
119- should_notify = true
120- this . _notified_roots . add ( root )
121- }
122- }
123- }
124-
125- if ( total_uncached === 0 || ! should_notify ) {
126- return params . task ( )
127- }
128-
129- const progress_state : {
130- progress ?: vscode . Progress < { message ?: string ; increment ?: number } >
131- total_files : number
132- processed_files : number
133- } = {
134- total_files : total_uncached ,
135- processed_files : 0
136- }
137- this . _progress_stack . push ( progress_state )
138-
139- const task_promise = params . task ( )
140-
141- let timeout_handle : NodeJS . Timeout
142- const timeout_promise = new Promise < void > ( ( resolve ) => {
143- timeout_handle = setTimeout ( ( ) => resolve ( ) , 1000 )
144- } )
145-
146- let is_completed = false
147- task_promise . finally ( ( ) => {
148- is_completed = true
149- clearTimeout ( timeout_handle )
150- } )
151-
152- try {
153- await Promise . race ( [ task_promise , timeout_promise ] )
154- } catch ( error ) {
155- const index = this . _progress_stack . indexOf ( progress_state )
156- if ( index !== - 1 ) {
157- this . _progress_stack . splice ( index , 1 )
158- }
159- throw error
160- }
161-
162- if ( ! is_completed ) {
163- return vscode . window . withProgress (
164- {
165- location : vscode . ProgressLocation . Notification ,
166- title : t ( 'context.crunching-token-counts' ) ,
167- cancellable : false
168- } ,
169- async ( progress ) => {
170- progress_state . progress = progress
171- if (
172- progress_state . total_files > 0 &&
173- progress_state . processed_files > 0
174- ) {
175- progress . report ( {
176- increment :
177- ( progress_state . processed_files / progress_state . total_files ) *
178- 100
179- } )
180- }
181- try {
182- return await task_promise
183- } finally {
184- const index = this . _progress_stack . indexOf ( progress_state )
185- if ( index !== - 1 ) {
186- this . _progress_stack . splice ( index , 1 )
187- }
188- }
189- }
190- )
191- } else {
192- const index = this . _progress_stack . indexOf ( progress_state )
193- if ( index !== - 1 ) {
194- this . _progress_stack . splice ( index , 1 )
195- }
196- return task_promise
197- }
198- }
199-
20017 invalidate_token_counts_for_file ( changed_file_path : string ) {
20118 const workspace_root =
20219 this . _provider . get_workspace_root_for_file ( changed_file_path )
@@ -281,7 +98,6 @@ export class TokenCalculator implements vscode.Disposable {
28198
28299 this . _file_token_counts . set ( file_path , estimated_tokens )
283100
284- this . _report_progress ( )
285101 return estimated_tokens
286102 }
287103
@@ -321,15 +137,13 @@ export class TokenCalculator implements vscode.Disposable {
321137
322138 this . _file_token_counts . set ( file_path , token_count )
323139
324- this . _report_progress ( )
325140 return token_count
326141 } catch ( error ) {
327142 Logger . error ( {
328143 function_name : 'calculate_file_tokens' ,
329144 message : `Error calculating tokens for ${ file_path } ` ,
330145 data : error
331146 } )
332- this . _report_progress ( )
333147 return 0
334148 }
335149 }
0 commit comments