@@ -84,36 +84,32 @@ export const make_api_request = async (params: {
8484 let last_on_chunk_time = 0
8585 let last_total_tokens = 0
8686 let total_tokens = 0
87- let first_chunk_reported = false
8887
8988 const handle_chunk_metrics = ( chunk : string ) => {
9089 if ( ! params . on_chunk || ! chunk ) return
9190
92- total_tokens += Math . ceil ( chunk . length / 4 )
91+ const chunk_tokens = Math . ceil ( chunk . length / 4 )
92+ total_tokens += chunk_tokens
93+
94+ const current_time = Date . now ( )
95+
9396 if ( stream_start_time === 0 ) {
94- stream_start_time = Date . now ( )
95- last_on_chunk_time = stream_start_time
96- // Don't calculate TPS on the first chunk, as elapsed time is 0
97+ stream_start_time = current_time
98+ last_on_chunk_time = current_time
99+ last_total_tokens = total_tokens
97100 return
98101 }
99102
100- const current_time = Date . now ( )
101103 const elapsed_since_last = current_time - last_on_chunk_time
104+
102105 if ( elapsed_since_last >= 1000 ) {
103- // Skip the very first report to avoid inflated TPS
104- if ( ! first_chunk_reported ) {
105- first_chunk_reported = true
106- last_on_chunk_time = current_time
107- last_total_tokens = total_tokens
108- return
109- }
110-
111106 const tokens_since_last = total_tokens - last_total_tokens
112107 const seconds_since_last = elapsed_since_last / 1000
113108 const tokens_per_second =
114109 seconds_since_last > 0
115110 ? Math . round ( tokens_since_last / seconds_since_last )
116111 : 0
112+
117113 params . on_chunk ( tokens_per_second , total_tokens )
118114 last_on_chunk_time = current_time
119115 last_total_tokens = total_tokens
@@ -362,12 +358,10 @@ export const make_api_request = async (params: {
362358 message : 'Stream error' ,
363359 data : error
364360 } )
365- // Convert stream errors to a retryable error type
366361 reject ( new StreamAbortError ( error . message ) )
367362 }
368363 } )
369364
370- // Handle abort events specifically
371365 response . data . on ( 'aborted' , ( ) => {
372366 if ( ! stream_closed ) {
373367 Logger . warn ( {
@@ -388,7 +382,6 @@ export const make_api_request = async (params: {
388382 throw params . cancellation_token . reason
389383 }
390384
391- // Check if it's a retryable error
392385 const is_retryable_error =
393386 ( axios . isAxiosError ( error ) &&
394387 ( error . response ?. status == 503 ||
0 commit comments