diff --git a/packages/onebot/src/api-handler.ts b/packages/onebot/src/api-handler.ts index 82c2aab3..4c61b702 100644 --- a/packages/onebot/src/api-handler.ts +++ b/packages/onebot/src/api-handler.ts @@ -1,5 +1,5 @@ import { renderParamsVerbose, summarizeParams } from '@snowluma/common/log-summary'; -import { createLogger, getLogLevel, nextRequestId, runWithRequestId, type Logger } from '@snowluma/common/logger'; +import { createLogger, runWithTraceRequest, type Logger } from '@snowluma/common/logger'; import type { BridgeInterface } from '@snowluma/core/bridge-interface'; import { MessageElementValidationError } from '@snowluma/protocol/element-manifest'; import { @@ -13,7 +13,12 @@ import type { ForwardPreviewMeta } from './modules/message-actions'; import type { ReadSessionTargets } from './message-store'; import type { JsonObject, JsonValue, MessageMeta } from './types'; import { RETCODE, failedResponse, okResponse } from './types'; -import { type StreamSink, wrapStreamFrame, wrapStreamTerminal } from './streaming'; +import { + StreamTransportClosedError, + type StreamSink, + wrapStreamFrame, + wrapStreamTerminal, +} from './streaming'; const moduleLog = createLogger('Bridge.Action'); function summarizeActionParams(action: string, params: JsonObject): string { @@ -251,46 +256,91 @@ export class ApiHandler { } async handle(action: string, params: JsonObject, sink?: StreamSink): Promise { + return runWithTraceRequest(() => this.handleInContext(action, params, sink)); + } + + traceQuiescedAction( + action: string, + params: JsonObject, + response: import('./types').ApiResponse, + ): void { + if (this.acceptingActions) return; + runWithTraceRequest(() => { + const startedAt = Date.now(); + this.traceActionInput(action, params); + this.traceActionTerminal(action, response, startedAt, 'failed', 'quiesced'); + }); + } + + traceQuiescedStreamRequest(rawRequest: string): void { + const request = parseStreamRequest(rawRequest); + if (!request) return; + this.traceQuiescedAction( + request.action, + request.params, + failedResponse(RETCODE.ACTION_FAILED, 'OneBot instance is shutting down'), + ); + } + + private async handleInContext( + action: string, + params: JsonObject, + sink?: StreamSink, + ): Promise { + const startedAt = Date.now(); + this.traceActionInput(action, params); + if (!this.acceptingActions) { const response = failedResponse(RETCODE.ACTION_FAILED, 'OneBot instance is shutting down'); this.log.warn('rejected Action %s after instance quiesce', action); + this.traceActionTerminal(action, response, startedAt, 'failed', 'quiesced'); this.notifyObservers(action, params, response, 0); return response; } + const registered = this.handlers.get(action); if (!registered) { + const response = failedResponse(RETCODE.UNKNOWN_ACTION, 'unknown action'); this.log.debug('unknown action %s', action); - return failedResponse(RETCODE.UNKNOWN_ACTION, 'unknown action'); + this.traceActionTerminal(action, response, startedAt, 'failed', 'unknown_action'); + return response; } - // Correlate the whole request — entry, every outbound packet it triggers, - // and the exit — under one `[req#N]` tag via AsyncLocalStorage. Only pay - // the wrap + id allocation when trace is actually live, so the default - // path stays allocation-free. - if (getLogLevel() !== 'trace') { - return this.runAction(action, registered.handler, params, sink); - } - return runWithRequestId(nextRequestId(), () => this.runAction(action, registered.handler, params, sink)); + return this.runAction(action, registered.handler, params, sink, startedAt); } private async runAction( action: string, handler: ActionHandler, params: JsonObject, - sink?: StreamSink, + sink: StreamSink | undefined, + startedAt: number, ): Promise { // Terse breadcrumb to the log file (debug, which is written to disk under the default // file level): lets the operator grep "what did the bot get asked to do" in post-mortems. this.log.debug('%s params=%s', action, summarizeActionParams(action, params)); - // Full request shape, memory-only (trace). Lazy producer → the deep render - // only runs when trace is live. - this.log.trace(() => [`${action} ⇐ %s`, renderParamsVerbose(params)]); - const startedAt = Date.now(); let response: import('./types').ApiResponse; + let outcome: 'ok' | 'failed' | 'cancelled'; + let reason: 'response_returned' | 'handler_threw' | 'transport_closed'; try { - response = await handler(params, sink); - this.log.trace(() => [`${action} ⇒ ${response.status} (${Date.now() - startedAt}ms)`]); + response = await handler( + params, + sink + ? { + send: async (frame) => { + this.log.trace(() => [ + 'action_stream_frame action=%s frame=%s', + action, + renderParamsVerbose(frame), + ]); + await sink.send(frame); + }, + } + : undefined, + ); + outcome = response.status === 'ok' ? 'ok' : 'failed'; + reason = 'response_returned'; } catch (error) { // Single error seam: typed message-contract failures are caller errors // and therefore map to BAD_REQUEST. Bridge/OIDB business failures and @@ -309,11 +359,42 @@ export class ApiHandler { ? RETCODE.BAD_REQUEST : RETCODE.ACTION_FAILED; response = failedResponse(retcode, message); + outcome = error instanceof StreamTransportClosedError ? 'cancelled' : 'failed'; + reason = error instanceof StreamTransportClosedError ? 'transport_closed' : 'handler_threw'; } + this.traceActionTerminal(action, response, startedAt, outcome, reason); this.notifyObservers(action, params, response, Date.now() - startedAt); return response; } + private traceActionInput( + action: string, + params: JsonObject, + ): void { + this.log.trace(() => [ + 'action_input action=%s params=%s', + action, + renderParamsVerbose(params), + ]); + } + + private traceActionTerminal( + action: string, + response: import('./types').ApiResponse, + startedAt: number, + outcome: 'ok' | 'failed' | 'cancelled', + reason: 'response_returned' | 'handler_threw' | 'transport_closed' | 'unknown_action' | 'quiesced', + ): void { + this.log.trace(() => [ + 'action_terminal action=%s outcome=%s reason=%s ms=%d response=%s', + action, + outcome, + reason, + Date.now() - startedAt, + renderParamsVerbose(response), + ]); + } + private notifyObservers( action: string, params: JsonObject, @@ -345,21 +426,9 @@ export class ApiHandler { ); if (!rawRequest.trim()) { await bad(); return; } - let action: string; - let params: JsonObject; - let echo: JsonValue | undefined; - try { - const parsed = JSON.parse(rawRequest) as unknown; - if (!isJsonObject(parsed)) { await bad(); return; } - const a = asString(parsed.action); - if (!a) { await bad(); return; } - action = a; - params = isJsonObject(parsed.params) ? parsed.params : {}; - echo = parsed.echo !== undefined ? toJsonValue(parsed.echo) : undefined; - } catch { - await bad(); - return; - } + const request = parseStreamRequest(rawRequest); + if (!request) { await bad(); return; } + const { action, params, echo } = request; if (!this.isStreamAction(action)) { const response = await this.handle(action, params); @@ -370,7 +439,7 @@ export class ApiHandler { const sink: StreamSink = { send: async (frame) => { - if (isAlive && !isAlive()) throw new Error('stream transport closed'); + if (isAlive && !isAlive()) throw new StreamTransportClosedError(); await emit(JSON.stringify(wrapStreamFrame(frame, echo))); }, }; @@ -379,6 +448,29 @@ export class ApiHandler { } } +interface StreamRequest { + action: string; + params: JsonObject; + echo: JsonValue | undefined; +} + +function parseStreamRequest(rawRequest: string): StreamRequest | null { + if (!rawRequest.trim()) return null; + try { + const parsed = JSON.parse(rawRequest) as unknown; + if (!isJsonObject(parsed)) return null; + const action = asString(parsed.action); + if (!action) return null; + return { + action, + params: isJsonObject(parsed.params) ? parsed.params : {}, + echo: parsed.echo !== undefined ? toJsonValue(parsed.echo) : undefined, + }; + } catch { + return null; + } +} + function isJsonObject(value: unknown): value is JsonObject { return typeof value === 'object' && value !== null && !Array.isArray(value); } diff --git a/packages/onebot/src/event-pipeline.ts b/packages/onebot/src/event-pipeline.ts index b1f6f80d..1dd18296 100644 --- a/packages/onebot/src/event-pipeline.ts +++ b/packages/onebot/src/event-pipeline.ts @@ -1,4 +1,4 @@ -import { createLogger, getLogLevel, nextRequestId, runWithRequestId, type Logger } from '@snowluma/common/logger'; +import { createLogger, runWithTraceRequest, type Logger } from '@snowluma/common/logger'; import { renderParamsVerbose } from '@snowluma/common/log-summary'; import type { QQEventVariant } from '@snowluma/protocol/events'; import { convertEvent } from './event-converter'; @@ -34,39 +34,55 @@ export function registerEventPipeline(ctx: OneBotInstanceContext): EventPipeline let accepting = true; const track = ( - kind: QQEventVariant['kind'], - start: () => void | Promise, + event: QQEventVariant, + start: (state: EventTraceState) => void | Promise, ): Promise => { if (!accepting) return Promise.resolve(); - let operation: Promise; - try { - operation = Promise.resolve(start()); - } catch (error) { - operation = Promise.reject(error); - } - const tracked = operation.then( - () => undefined, - (error) => { - log.error( - 'event pipeline handler failed kind=%s: %s', - kind, - error instanceof Error ? (error.stack ?? error.message) : String(error), - ); - }, - ); - inFlight.add(tracked); - void tracked.then(() => { inFlight.delete(tracked); }); - return tracked; + return runWithTraceRequest(() => { + const state: EventTraceState = { + startedAt: Date.now(), + handedOff: false, + terminalEmitted: false, + }; + log.trace(() => [ + 'event_input kind=%s event=%s', + event.kind, + renderParamsVerbose(event), + ]); + + let operation: Promise; + try { + operation = Promise.resolve(start(state)); + } catch (error) { + operation = Promise.reject(error); + } + const tracked = operation.then( + () => undefined, + (error) => { + if (!state.handedOff && !state.terminalEmitted) { + traceEventTerminal(log, event.kind, state, 'failed', 'pipeline_threw', error); + } + log.error( + 'event pipeline handler failed kind=%s: %s', + event.kind, + error instanceof Error ? (error.stack ?? error.message) : String(error), + ); + }, + ); + inFlight.add(tracked); + void tracked.then(() => { inFlight.delete(tracked); }); + return tracked; + }); }; disposers.push( - ctx.bridge.events.on('group_message', (event) => track(event.kind, async () => { + ctx.bridge.events.on('group_message', (event) => track(event, async (state) => { cacheGroupMessageMeta(ctx, event); - await convertAndDispatch(ctx, log, event); + await convertAndDispatch(ctx, log, event, state); })), ); disposers.push( - ctx.bridge.events.on('friend_message', (event) => track(event.kind, async () => { + ctx.bridge.events.on('friend_message', (event) => track(event, async (state) => { cachePrivateMessageMeta( ctx, event.peerUin ?? event.senderUin, @@ -78,11 +94,11 @@ export function registerEventPipeline(ctx: OneBotInstanceContext): EventPipeline event.sequenceAuthoritative !== false, event.senderUin === ctx.selfId ? 'outgoing' : 'incoming', ); - await convertAndDispatch(ctx, log, event); + await convertAndDispatch(ctx, log, event, state); })), ); disposers.push( - ctx.bridge.events.on('temp_message', (event) => track(event.kind, async () => { + ctx.bridge.events.on('temp_message', (event) => track(event, async (state) => { cachePrivateMessageMeta( ctx, event.senderUin, @@ -97,12 +113,12 @@ export function registerEventPipeline(ctx: OneBotInstanceContext): EventPipeline // Record this group temp session so a later reply is limited to sessions // the peer opened. ctx.tempSessions.record(event.senderUin, event.groupId); - await convertAndDispatch(ctx, log, event); + await convertAndDispatch(ctx, log, event, state); })), ); for (const kind of NOTICE_KINDS) { disposers.push( - ctx.bridge.events.on(kind, (event) => track(event.kind, async () => { + ctx.bridge.events.on(kind, (event) => track(event, async (state) => { if (event.kind === 'group_msg_emoji_like') { cacheReaction(ctx, event); } @@ -125,15 +141,16 @@ export function registerEventPipeline(ctx: OneBotInstanceContext): EventPipeline ); } } - await convertAndDispatch(ctx, log, event, messageIdOverride); + await convertAndDispatch(ctx, log, event, state, messageIdOverride); })), ); } // Internal-only: voice-to-text result push. Not converted to a OneBot event — // it just unblocks the fetch_ptt_text call waiting on this msgId. disposers.push( - ctx.bridge.events.on('ptt_trans_result', (event) => track(event.kind, () => { + ctx.bridge.events.on('ptt_trans_result', (event) => track(event, (state) => { deliverPttTransText(pttTransKey(event.selfUin, event.msgId), event.text); + traceEventTerminal(log, event.kind, state, 'internal', 'waiter_notified'); })), ); @@ -188,44 +205,53 @@ const NOTICE_KINDS = [ 'group_msg_emoji_like', ] as const satisfies readonly QQEventVariant['kind'][]; +interface EventTraceState { + startedAt: number; + handedOff: boolean; + terminalEmitted: boolean; +} + +type PipelineEventOutcome = 'dropped' | 'failed' | 'internal'; +type PipelineEventReason = + | 'converter_returned_null' + | 'converter_threw' + | 'pipeline_threw' + | 'recalled_before_backfill' + | 'recalled_after_backfill' + | 'waiter_notified'; + async function convertAndDispatch( ctx: OneBotInstanceContext, log: Logger, event: QQEventVariant, + state: EventTraceState, messageIdOverride?: number, ): Promise { - // Inbound choke point — the receive-side mirror of the outbound api-handler. - // Correlate the whole receive chain (decode → convert, incl. any rkey-fetch - // packets the conversion triggers, → dispatch) under one [req#N]. Only pay - // the AsyncLocalStorage wrap + id when trace is actually live. - if (getLogLevel() !== 'trace') { - await runConvertAndDispatch(ctx, log, event, messageIdOverride); - return; + let converted; + try { + converted = await convertEvent(ctx.converterCtx, event); + } catch (error) { + traceEventTerminal(log, event.kind, state, 'failed', 'converter_threw', error); + throw error; } - await runWithRequestId(nextRequestId(), () => ( - runConvertAndDispatch(ctx, log, event, messageIdOverride) - )); -} -async function runConvertAndDispatch( - ctx: OneBotInstanceContext, - log: Logger, - event: QQEventVariant, - messageIdOverride?: number, -): Promise { - // Raw inbound event, memory-only (trace). Lazy → the deep render runs only - // when trace is live. - log.trace(() => [`recv ${event.kind} ⇐ %s`, renderParamsVerbose(event)]); - const startedAt = Date.now(); - const converted = await convertEvent(ctx.converterCtx, event); if (!converted) { - log.trace(() => [`recv ${event.kind} ⇒ dropped (${Date.now() - startedAt}ms)`]); + traceEventTerminal(log, event.kind, state, 'dropped', 'converter_returned_null'); return; } if (messageIdOverride !== undefined && event.kind === 'friend_recall') { converted.message_id = messageIdOverride; } - if (event.kind === 'friend_message' && isFriendMessageRecalled(ctx, log, event)) return; + log.trace(() => [ + 'event_converted kind=%s event=%s', + event.kind, + renderParamsVerbose(converted), + ]); + + if (event.kind === 'friend_message' && isFriendMessageRecalled(ctx, log, event)) { + traceEventTerminal(log, event.kind, state, 'dropped', 'recalled_before_backfill'); + return; + } // If this message quotes one we don't have, fetch + persist it first (gated + // throttled) so a consumer's get_msg on the quote resolves. No-op for the // common case (no reply, or the quoted message is already stored). Never let a @@ -240,12 +266,43 @@ async function runConvertAndDispatch( event.kind, error instanceof Error ? (error.stack ?? error.message) : String(error), ); + log.trace(() => [ + 'event_branch kind=%s reason=reply_backfill_failed error=%s', + event.kind, + error instanceof Error ? (error.stack ?? error.message) : String(error), + ]); } // Backfill can await several network/media operations. Re-check immediately // before dispatch so a recall that arrived during that gap wins the race. - if (event.kind === 'friend_message' && isFriendMessageRecalled(ctx, log, event)) return; - ctx.dispatchEvent(converted, 'bridge'); - log.trace(() => [`recv ${event.kind} ⇒ ${String(converted.post_type ?? '?')} (${Date.now() - startedAt}ms)`]); + if (event.kind === 'friend_message' && isFriendMessageRecalled(ctx, log, event)) { + traceEventTerminal(log, event.kind, state, 'dropped', 'recalled_after_backfill'); + return; + } + + ctx.dispatchEvent(converted, 'bridge', state.startedAt); + state.handedOff = true; +} + +function traceEventTerminal( + log: Logger, + kind: QQEventVariant['kind'], + state: EventTraceState, + outcome: PipelineEventOutcome, + reason: PipelineEventReason, + error?: unknown, +): void { + if (state.terminalEmitted) return; + state.terminalEmitted = true; + log.trace(() => [ + 'event_terminal kind=%s outcome=%s reason=%s ms=%d%s', + kind, + outcome, + reason, + Date.now() - state.startedAt, + error === undefined + ? '' + : ` error=${error instanceof Error ? (error.stack ?? error.message) : String(error)}`, + ]); } function cacheGroupMessageMeta(ctx: OneBotInstanceContext, event: Extract): void { diff --git a/packages/onebot/src/instance-context.ts b/packages/onebot/src/instance-context.ts index d4fa7963..46fe6e46 100644 --- a/packages/onebot/src/instance-context.ts +++ b/packages/onebot/src/instance-context.ts @@ -53,7 +53,11 @@ export interface OneBotInstanceContext { config: OneBotConfig; musicSignUrl?: string; cacheMessageMeta(messageId: number, meta: MessageMeta): void; - dispatchEvent(event: JsonObject, source?: 'bridge' | 'send'): void; + dispatchEvent( + event: JsonObject, + source?: 'bridge' | 'send', + startedAt?: number, + ): void; } export function buildApiContext(ref: OneBotInstanceContext): ApiActionContext { diff --git a/packages/onebot/src/instance.ts b/packages/onebot/src/instance.ts index 6b733bbc..8b7061b8 100644 --- a/packages/onebot/src/instance.ts +++ b/packages/onebot/src/instance.ts @@ -171,7 +171,7 @@ export class OneBotInstance { config, musicSignUrl: globalSettings.musicSignUrl, cacheMessageMeta: (messageId, meta) => this.cacheMessageMeta(messageId, meta), - dispatchEvent: (event, source = 'bridge') => this.dispatchEvent(event, source), + dispatchEvent: (event, source = 'bridge', startedAt) => this.dispatchEvent(event, source, startedAt), }; this.ctx = ctx; @@ -343,7 +343,11 @@ export class OneBotInstance { return this.pids.size === 0; } - private dispatchEvent(event: JsonObject, source: 'bridge' | 'send' = 'bridge'): void { + private dispatchEvent( + event: JsonObject, + source: 'bridge' | 'send' = 'bridge', + startedAt = Date.now(), + ): void { if (source === 'bridge') { this.cacheMessageEvent(event); if (this.consumePendingSelfSentEcho(event)) { @@ -352,6 +356,7 @@ export class OneBotInstance { toInt(event.message_id), toInt(event.message_seq), ); + this.traceEventHandoffTerminal(event, startedAt, 'suppressed', 'duplicate_self_echo'); return; } } else { @@ -364,12 +369,31 @@ export class OneBotInstance { if (source === 'bridge') this.logReceivedMessage(event); // Built-in `#sl` only consumes events observed from QQ. Action-originated // messages must not recursively trigger the command handler. - if (source === 'bridge' && this.handleStatusCommand(event)) return; + if (source === 'bridge' && this.handleStatusCommand(event)) { + this.traceEventHandoffTerminal(event, startedAt, 'suppressed', 'status_command'); + return; + } + this.traceEventHandoffTerminal(event, startedAt, 'reporting_started'); void this.networkManager.emitEvent(event).catch((err) => { this.log.warn('emitEvent failed: %s', err instanceof Error ? (err.stack ?? err.message) : String(err)); }); } + private traceEventHandoffTerminal( + event: JsonObject, + startedAt: number, + outcome: 'suppressed' | 'reporting_started', + reason?: 'duplicate_self_echo' | 'status_command', + ): void { + this.log.trace(() => [ + 'event_terminal post_type=%s outcome=%s%s ms=%d', + String(event.post_type ?? '?'), + outcome, + reason === undefined ? '' : ` reason=${reason}`, + Date.now() - startedAt, + ]); + } + private rememberPendingSelfSentEcho(event: JsonObject): void { const key = selfSentEventKey(event); if (key === null) return; diff --git a/packages/onebot/src/network/http-post-adapter.ts b/packages/onebot/src/network/http-post-adapter.ts index 3c8669b3..2aec1fd9 100644 --- a/packages/onebot/src/network/http-post-adapter.ts +++ b/packages/onebot/src/network/http-post-adapter.ts @@ -80,29 +80,57 @@ export class HttpPostAdapter extends IOneBotNetworkAdapter { } const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const startedAt = Date.now(); + let response: Response; try { - const response = await fetch(config.url, { + response = await fetch(config.url, { method: 'POST', headers, body: payload, signal: AbortSignal.timeout(timeoutMs), }); - if (response.ok) { - this.lastDelivery = { at: Date.now(), ok: true }; - const contentType = response.headers.get('content-type') ?? ''; - if (contentType.includes('application/json')) { - const body = await response.text(); - if (body.trim()) { - await this.handleQuickOperation(event, body); - } - } - } else { - this.lastDelivery = { at: Date.now(), ok: false }; - this.log.warn('[%s] POST %s returned %d', this.name, config.url, response.status); - } } catch (error) { this.lastDelivery = { at: Date.now(), ok: false }; this.log.warn('[%s] POST %s failed: %s', this.name, config.url, error instanceof Error ? error.message : String(error)); + this.log.trace(() => [ + 'http_report_terminal target=%s outcome=transport_failed ms=%d error=%s', + this.name, + Date.now() - startedAt, + error instanceof Error ? (error.stack ?? error.message) : String(error), + ]); + return; + } + + if (!response.ok) { + this.lastDelivery = { at: Date.now(), ok: false }; + this.log.warn('[%s] POST %s returned %d', this.name, config.url, response.status); + this.log.trace( + 'http_report_terminal target=%s outcome=rejected status=%d ms=%d', + this.name, + response.status, + Date.now() - startedAt, + ); + return; + } + + this.lastDelivery = { at: Date.now(), ok: true }; + this.log.trace( + 'http_report_terminal target=%s outcome=accepted status=%d ms=%d', + this.name, + response.status, + Date.now() - startedAt, + ); + const contentType = response.headers.get('content-type') ?? ''; + if (!contentType.includes('application/json')) return; + try { + const body = await response.text(); + if (body.trim()) await this.handleQuickOperation(event, body); + } catch (error) { + this.log.warn( + '[%s] quick operation response read failed: %s', + this.name, + error instanceof Error ? error.message : String(error), + ); } } diff --git a/packages/onebot/src/network/http-server-adapter.ts b/packages/onebot/src/network/http-server-adapter.ts index 548bc8f2..c9271643 100644 --- a/packages/onebot/src/network/http-server-adapter.ts +++ b/packages/onebot/src/network/http-server-adapter.ts @@ -9,7 +9,12 @@ import { import type { ApiHandler } from '../api-handler'; import type { DispatchPayload } from '../event-filter'; import type { ApiResponse, HttpServerNetwork, JsonObject, JsonValue } from '../types'; -import { type StreamSink, wrapStreamFrame, wrapStreamTerminal } from '../streaming'; +import { + StreamTransportClosedError, + type StreamSink, + wrapStreamFrame, + wrapStreamTerminal, +} from '../streaming'; import { IOneBotNetworkAdapter, type AdapterStatus, type NetworkAdapterContext } from './adapter'; import { isAuthorized, normalizePath } from './utils'; import { WsServerConnections } from './ws-server-connections'; @@ -233,6 +238,14 @@ export class HttpServerAdapter extends IOneBotNetworkAdapter } private async handleRequest(req: IncomingMessage, res: ServerResponse): Promise { + const apiQuiescedAtIngress = this.ctx.api.isAcceptingActions === false; + if (apiQuiescedAtIngress) { + writeJson(res, 503, serverClosingResponse()); + } + const respond = (statusCode: number, data: unknown): void => { + if (apiQuiescedAtIngress) return; + writeJson(res, statusCode, data); + }; const expectedPath = normalizePath(this.config.path ?? '/'); const accessToken = this.config.accessToken ?? ''; const parsedUrl = new URL(req.url ?? '/', 'http://127.0.0.1'); @@ -245,17 +258,17 @@ export class HttpServerAdapter extends IOneBotNetworkAdapter } else if (incomingPath.startsWith(ep)) { action = incomingPath.substring(ep.length); } else { - writeJson(res, 404, { status: 'failed', retcode: 1404, data: null, wording: 'not found' }); + respond(404, { status: 'failed', retcode: 1404, data: null, wording: 'not found' }); return; } if (!isAuthorized(req, accessToken)) { - writeJson(res, 401, { status: 'failed', retcode: 1401, data: null, wording: 'unauthorized' }); + respond(401, { status: 'failed', retcode: 1401, data: null, wording: 'unauthorized' }); return; } if (req.method === 'GET' && !action) { - writeJson(res, 200, { status: 'ok', retcode: 0, data: { online: true } }); + respond(200, { status: 'ok', retcode: 0, data: { online: true } }); return; } @@ -306,7 +319,7 @@ export class HttpServerAdapter extends IOneBotNetworkAdapter } } catch { if (contentType.includes('application/json')) { - writeJson(res, 400, { status: 'failed', retcode: 1400, data: null, wording: 'bad request: invalid json' }); + respond(400, { status: 'failed', retcode: 1400, data: null, wording: 'bad request: invalid json' }); return; } // 无 content-type,JSON 失败则 fallback 到 urlencoded @@ -328,17 +341,28 @@ export class HttpServerAdapter extends IOneBotNetworkAdapter } } } else { - writeJson(res, 400, { status: 'failed', retcode: 1400, data: null, wording: `bad request: unsupported content-type: ${contentType}` }); + respond(400, { status: 'failed', retcode: 1400, data: null, wording: `bad request: unsupported content-type: ${contentType}` }); return; } } } else { - writeJson(res, 405, { status: 'failed', retcode: 1400, data: null, wording: 'method not allowed' }); + respond(405, { status: 'failed', retcode: 1400, data: null, wording: 'method not allowed' }); return; } if (!action) { - writeJson(res, 400, { status: 'failed', retcode: 1400, data: null, wording: 'bad request: missing action' }); + respond(400, { status: 'failed', retcode: 1400, data: null, wording: 'bad request: missing action' }); + return; + } + + if (apiQuiescedAtIngress) { + const response = serverClosingResponse(); + this.ctx.api.traceQuiescedAction( + action, + params as JsonObject, + response, + ); + respond(503, response); return; } @@ -353,16 +377,26 @@ export class HttpServerAdapter extends IOneBotNetworkAdapter if (echo !== undefined) { response.echo = echo as JsonValue; } - writeJson(res, 200, response); + respond(200, response); } catch (error) { const wording = error instanceof Error ? error.message : 'internal error'; - writeJson(res, 500, { status: 'failed', retcode: 1200, data: null, wording }); + respond(500, { status: 'failed', retcode: 1200, data: null, wording }); } } private trackInboundAction(req: IncomingMessage, res: ServerResponse): void { - if (!this.acceptingActions || this.ctx.api.isAcceptingActions === false) { - writeJson(res, 503, { status: 'failed', retcode: 1200, data: null, wording: 'server closing' }); + if (!this.acceptingActions) { + writeJson(res, 503, serverClosingResponse()); + return; + } + if (this.ctx.api.isAcceptingActions === false) { + void this.handleRequest(req, res).catch((error) => { + this.log.warn( + '[%s] quiesced inbound action trace failed: %s', + this.name, + error instanceof Error ? (error.stack ?? error.message) : String(error), + ); + }); return; } let action: Promise; @@ -384,6 +418,15 @@ export class HttpServerAdapter extends IOneBotNetworkAdapter } } +function serverClosingResponse(): ApiResponse { + return { + status: 'failed', + retcode: 1200, + data: null, + wording: 'server closing', + }; +} + function isAlreadyClosedError(error: Error): boolean { return (error as NodeJS.ErrnoException).code === 'ERR_SERVER_NOT_RUNNING'; } @@ -431,13 +474,21 @@ async function streamHttpResponse( ): Promise { res.statusCode = 200; const writeFrame = (frame: ApiResponse): Promise => - new Promise((resolve) => { - if (res.writableEnded || res.destroyed) { resolve(); return; } - res.write(JSON.stringify(frame) + '\r\n\r\n', () => resolve()); + new Promise((resolve, reject) => { + if (res.writableEnded || res.destroyed) { + reject(new StreamTransportClosedError('stream client disconnected')); + return; + } + res.write(JSON.stringify(frame) + '\r\n\r\n', (error) => { + if (error) { + reject(new StreamTransportClosedError('stream client disconnected')); + return; + } + resolve(); + }); }); const sink: StreamSink = { send: async (frame) => { - if (res.writableEnded || res.destroyed) throw new Error('stream client disconnected'); await writeFrame(wrapStreamFrame(frame, echo)); }, }; diff --git a/packages/onebot/src/network/network-manager.ts b/packages/onebot/src/network/network-manager.ts index a430d0e5..38f48f8f 100644 --- a/packages/onebot/src/network/network-manager.ts +++ b/packages/onebot/src/network/network-manager.ts @@ -250,20 +250,34 @@ export class OneBotNetworkManager { try { cb(event); } catch (error) { log.warn('debug subscriber error: %s', errMessage(error)); } } } - if (!this.hasActiveAdapters()) return; - const payload = buildDispatchPayload(event); - const tasks: Promise[] = []; - for (const { adapter } of this.adapters.values()) { - if (!adapter.isActive) continue; - tasks.push( - Promise.resolve() - .then(() => adapter.onEvent(event, payload)) - .catch((error) => { - log.warn('adapter [%s] onEvent error: %s', adapter.name, errMessage(error)); - }), - ); + + const active = [...this.adapters.values()] + .map(({ adapter }) => adapter) + .filter((adapter) => adapter.isActive); + if (active.length === 0) { + log.trace('report_terminal outcome=no_active_targets'); + return; } + log.trace(() => [ + 'report_targets targets=%s', + JSON.stringify(active.map((adapter) => adapter.name)), + ]); + + const payload = buildDispatchPayload(event); + const tasks = active.map((adapter) => ( + Promise.resolve() + .then(() => adapter.onEvent(event, payload)) + .catch((error) => { + log.warn('adapter [%s] onEvent error: %s', adapter.name, errMessage(error)); + log.trace(() => [ + 'report_target_failed target=%s error=%s', + adapter.name, + error instanceof Error ? (error.stack ?? error.message) : String(error), + ]); + }) + )); await Promise.allSettled(tasks); + log.trace('report_terminal outcome=settled'); } private async reconcileNow(desired: DesiredNetworkAdapter[]): Promise { diff --git a/packages/onebot/src/network/utils.ts b/packages/onebot/src/network/utils.ts index d4e3d13b..031c8f5a 100644 --- a/packages/onebot/src/network/utils.ts +++ b/packages/onebot/src/network/utils.ts @@ -1,3 +1,4 @@ +import { StreamTransportClosedError } from '../streaming'; import type { WebSocket } from '@snowluma/websocket'; import type { IncomingMessage } from 'http'; @@ -34,13 +35,25 @@ export function safeSend(socket: WebSocket, payload: string, onError?: (err: Err /** Backpressure-aware send: resolves only once `ws` has flushed `payload` to * the socket (the `send` callback fires post-write), so a streaming producer * that awaits this won't outrun a slow client and balloon the send buffer. - * Never rejects — a closed socket or send error resolves quietly (the stream - * sink handles liveness/abort separately so a dead client can't crash the - * per-message handler with an unhandled rejection). */ + * A closed socket or failed write rejects with the shared transport sentinel, + * allowing the action boundary to classify an in-progress stream cancellation. */ export function safeSendAsync(socket: WebSocket, payload: string): Promise { - return new Promise((resolve) => { - if (socket.readyState !== 1 /* WebSocket.OPEN */) { resolve(); return; } - socket.send(payload, () => resolve()); + return new Promise((resolve, reject) => { + if (socket.readyState !== 1 /* WebSocket.OPEN */) { + reject(new StreamTransportClosedError('stream client disconnected')); + return; + } + try { + socket.send(payload, (error?: Error | null) => { + if (error) { + reject(new StreamTransportClosedError('stream client disconnected')); + return; + } + resolve(); + }); + } catch { + reject(new StreamTransportClosedError('stream client disconnected')); + } }); } diff --git a/packages/onebot/src/network/ws-client-adapter.ts b/packages/onebot/src/network/ws-client-adapter.ts index baa73f17..7bad4279 100644 --- a/packages/onebot/src/network/ws-client-adapter.ts +++ b/packages/onebot/src/network/ws-client-adapter.ts @@ -149,6 +149,14 @@ export class WsClientAdapter extends IOneBotNetworkAdapter { this.log.warn('[%s] rejected inbound action from stale socket', this.name); return; } + if (this.ctx.api.isAcceptingActions === false) { + this.log.warn('[%s] rejected inbound action after instance quiesce', this.name); + if (this.role === 'Api' || this.role === 'Universal') { + const text = rawDataToString(raw); + if (text) this.ctx.api.traceQuiescedStreamRequest(text); + } + return; + } this.trackInboundAction(() => this.handleApiMessage(socket, raw)); }); @@ -210,7 +218,10 @@ export class WsClientAdapter extends IOneBotNetworkAdapter { } private trackInboundAction(start: () => Promise): void { - if (!this.acceptingActions || this.ctx.api.isAcceptingActions === false) { + if ( + !this.acceptingActions + || this.ctx.api.isAcceptingActions === false + ) { this.log.warn('[%s] rejected inbound action while adapter is closing', this.name); return; } diff --git a/packages/onebot/src/network/ws-server-connections.ts b/packages/onebot/src/network/ws-server-connections.ts index f9129721..d07d04af 100644 --- a/packages/onebot/src/network/ws-server-connections.ts +++ b/packages/onebot/src/network/ws-server-connections.ts @@ -146,6 +146,14 @@ export class WsServerConnections { // handshake necessarily reaches the peer. Ignore any frame arriving in // that interval so old credentials cannot start another Action. if (!this.connections.has(socket)) return; + if (this.ctx.api.isAcceptingActions === false) { + this.log.warn('[%s] rejected inbound action after instance quiesce', this.name); + if (role === 'Api' || role === 'Universal') { + const text = rawDataToString(raw); + if (text) this.ctx.api.traceQuiescedStreamRequest(text); + } + return; + } this.trackInboundAction(() => this.handleApiMessage(socket, role, raw)); }); socket.on('close', () => { @@ -203,7 +211,10 @@ export class WsServerConnections { } trackInboundAction(start: () => Promise): void { - if (!this.acceptingActions || this.ctx.api.isAcceptingActions === false) { + if ( + !this.acceptingActions + || this.ctx.api.isAcceptingActions === false + ) { this.log.warn('[%s] rejected inbound action while adapter is closing', this.name); return; } diff --git a/packages/onebot/src/streaming.ts b/packages/onebot/src/streaming.ts index 80d03aa3..11731221 100644 --- a/packages/onebot/src/streaming.ts +++ b/packages/onebot/src/streaming.ts @@ -39,6 +39,13 @@ export interface StreamSink { send(frame: JsonValue): Promise; } +export class StreamTransportClosedError extends Error { + constructor(message = 'stream transport closed') { + super(message); + this.name = 'StreamTransportClosedError'; + } +} + /** Fallback sink for when a stream action is invoked over a transport that * can't stream (or by a plain client): intermediate frames are dropped, the * terminal response is still returned normally. */ diff --git a/packages/onebot/tests/api-handler-trace.test.ts b/packages/onebot/tests/api-handler-trace.test.ts new file mode 100644 index 00000000..ba28b133 --- /dev/null +++ b/packages/onebot/tests/api-handler-trace.test.ts @@ -0,0 +1,239 @@ +import { + afterEach, + beforeEach, + describe, + expect, + it, + vi, +} from 'vitest'; +import { defineStreamAction } from '../src/action-kit'; +import { + currentRequestId, + getLogLevel, + nextRequestId, + runWithRequestId, + setLogLevel, + subscribeLogs, + type LogEntry, +} from '@snowluma/common/logger'; +import { StreamTransportClosedError, StreamStatus } from '../src/streaming'; +import type { JsonObject } from '../src/types'; +import { createCompiledTestHandler, testAction } from './helpers/compiled-action-handler'; + +function emptyContext(): any { + return {}; +} + +function actionEntries(entries: LogEntry[], action: string): LogEntry[] { + return entries.filter((entry) => ( + entry.scope === 'Bridge.Action' + && entry.level === 'trace' + && entry.message.includes(`action=${action}`) + )); +} + +describe('ApiHandler TRACE lifecycle', () => { + let previousLevel: string; + let entries: LogEntry[]; + let unsubscribe: () => void; + + beforeEach(() => { + previousLevel = getLogLevel(); + setLogLevel('trace'); + entries = []; + unsubscribe = subscribeLogs((entry) => entries.push(entry)); + }); + + afterEach(() => { + unsubscribe(); + setLogLevel(previousLevel); + }); + + it('records complete input and returned response with one terminal', async () => { + const secret = 'trace-secret-' + 'x'.repeat(200); + const handler = createCompiledTestHandler(emptyContext(), [ + testAction('trace_ok', async () => ({ + status: 'ok', + retcode: 0, + data: { nested: { secret, values: [1, 2, 3] } }, + })), + ]); + + await handler.handle('trace_ok', { nested: { access_token: secret } }); + + const lifecycle = actionEntries(entries, 'trace_ok'); + expect(lifecycle.filter((entry) => entry.message.startsWith('action_input'))).toHaveLength(1); + expect(lifecycle.filter((entry) => entry.message.startsWith('action_terminal'))).toHaveLength(1); + expect(lifecycle[0]?.message).toContain(secret); + expect(lifecycle.at(-1)?.message).toContain(secret); + expect(lifecycle.at(-1)?.message).toContain('outcome=ok'); + expect(lifecycle.at(-1)?.message).toContain('reason=response_returned'); + expect(new Set(lifecycle.map((entry) => entry.req)).size).toBe(1); + expect(lifecycle[0]?.req).toBeTypeOf('number'); + }); + + it('distinguishes returned failure, handler throw, unknown action, and quiesce', async () => { + const handler = createCompiledTestHandler(emptyContext(), [ + testAction('returned_failure', async () => ({ + status: 'failed', retcode: 1400, data: null, wording: 'invalid input', + })), + testAction('thrown_failure', async () => { throw new Error('exploded'); }), + ]); + + await handler.handle('returned_failure', {}); + await handler.handle('thrown_failure', {}); + await handler.handle('unknown_trace_action', {}); + handler.quiesce(); + await handler.handle('returned_failure', {}); + + const returned = actionEntries(entries, 'returned_failure') + .filter((entry) => entry.message.startsWith('action_terminal')); + expect(returned[0]?.message).toContain('outcome=failed reason=response_returned'); + expect(returned[0]?.message).toContain('wording:"invalid input"'); + expect(returned[1]?.message).toContain('outcome=failed reason=quiesced'); + + const thrown = actionEntries(entries, 'thrown_failure') + .filter((entry) => entry.message.startsWith('action_terminal')); + expect(thrown).toHaveLength(1); + expect(thrown[0]?.message).toContain('outcome=failed reason=handler_threw'); + expect(thrown[0]?.message).toContain('wording:"exploded"'); + + const unknown = actionEntries(entries, 'unknown_trace_action') + .filter((entry) => entry.message.startsWith('action_terminal')); + expect(unknown).toHaveLength(1); + expect(unknown[0]?.message).toContain('outcome=failed reason=unknown_action'); + }); + + it('records parsed transport rejections without executing actions or observers', () => { + const run = vi.fn(async () => ({ status: 'ok' as const, retcode: 0, data: null })); + const observer = vi.fn(); + const handler = createCompiledTestHandler(emptyContext(), [ + testAction('transport_quiesced', run), + ]); + handler.setObserver(observer); + handler.quiesce(); + + handler.traceQuiescedStreamRequest(JSON.stringify({ + action: 'transport_quiesced', + params: { access_token: 'complete-secret' }, + echo: 'not-part-of-params', + })); + handler.traceQuiescedStreamRequest('not json'); + handler.traceQuiescedStreamRequest(JSON.stringify({ params: {} })); + + const lifecycle = actionEntries(entries, 'transport_quiesced'); + expect(lifecycle).toHaveLength(2); + expect(lifecycle[0]?.message).toContain('action_input'); + expect(lifecycle[0]?.message).toContain('complete-secret'); + expect(lifecycle[0]?.message).not.toContain('not-part-of-params'); + expect(lifecycle[1]?.message).toContain('outcome=failed reason=quiesced'); + expect(new Set(lifecycle.map((entry) => entry.req)).size).toBe(1); + expect(run).not.toHaveBeenCalled(); + expect(observer).not.toHaveBeenCalled(); + }); + + it('ignores transport-rejection tracing while actions are still accepted', () => { + const handler = createCompiledTestHandler(emptyContext(), [ + testAction('still_accepting', async () => ({ status: 'ok', retcode: 0, data: null })), + ]); + + handler.traceQuiescedStreamRequest(JSON.stringify({ + action: 'still_accepting', + params: {}, + })); + + expect(actionEntries(entries, 'still_accepting')).toHaveLength(0); + }); + + it('records successful stream frames and classifies transport closure only', async () => { + const stream = defineStreamAction({ + name: 'trace_stream', + params: {}, + run: async (_params, _ctx, _raw, sink) => { + await sink.send({ type: StreamStatus.Stream, chunk: 'complete-frame' }); + return { + status: 'ok', + retcode: 0, + data: { type: StreamStatus.Response }, + }; + }, + }); + const handler = createCompiledTestHandler(emptyContext(), [stream]); + + await handler.handle('trace_stream', {}, { send: async () => {} }); + const frames = actionEntries(entries, 'trace_stream') + .filter((entry) => entry.message.startsWith('action_stream_frame')); + expect(frames).toHaveLength(1); + expect(frames[0]?.message).toContain('complete-frame'); + + entries = []; + await handler.handle('trace_stream', {}, { + send: async () => { throw new StreamTransportClosedError(); }, + }); + const cancelled = actionEntries(entries, 'trace_stream') + .filter((entry) => entry.message.startsWith('action_terminal')); + expect(cancelled).toHaveLength(1); + expect(cancelled[0]?.message).toContain('outcome=cancelled reason=transport_closed'); + expect(actionEntries(entries, 'trace_stream') + .filter((entry) => entry.message.startsWith('action_stream_frame'))).toHaveLength(1); + + entries = []; + await handler.handle('trace_stream', {}, { + send: async () => { throw new Error('arbitrary sink failure'); }, + }); + const failed = actionEntries(entries, 'trace_stream') + .filter((entry) => entry.message.startsWith('action_terminal')); + expect(failed).toHaveLength(1); + expect(failed[0]?.message).toContain('outcome=failed reason=handler_threw'); + }); + + it('reuses an ambient request and gives separate top-level actions separate ids', async () => { + let handler: ReturnType; + handler = createCompiledTestHandler(emptyContext(), [ + testAction('inner_trace', async () => ({ status: 'ok', retcode: 0, data: null })), + testAction('outer_trace', async () => { + await handler.handle('inner_trace', {}); + return { status: 'ok', retcode: 0, data: null }; + }), + ]); + + await handler.handle('outer_trace', {}); + const nested = [ + ...actionEntries(entries, 'outer_trace'), + ...actionEntries(entries, 'inner_trace'), + ]; + expect(new Set(nested.map((entry) => entry.req)).size).toBe(1); + + entries = []; + await handler.handle('inner_trace', {}); + await handler.handle('inner_trace', {}); + const topLevelInputs = actionEntries(entries, 'inner_trace') + .filter((entry) => entry.message.startsWith('action_input')); + expect(topLevelInputs).toHaveLength(2); + expect(topLevelInputs[0]?.req).not.toBe(topLevelInputs[1]?.req); + + entries = []; + await runWithRequestId(424242, () => handler.handle('inner_trace', {})); + expect(actionEntries(entries, 'inner_trace').every((entry) => entry.req === 424242)).toBe(true); + }); + + it('does not allocate a request or render the complete object while TRACE is disabled', async () => { + setLogLevel('info'); + const nested: JsonObject = {}; + Object.defineProperty(nested, 'danger', { + enumerable: true, + get: () => { throw new Error('verbose renderer ran'); }, + }); + const handler = createCompiledTestHandler(emptyContext(), [ + testAction('trace_off', async () => ({ status: 'ok', retcode: 0, data: null })), + ]); + const before = nextRequestId(); + + await expect(handler.handle('trace_off', { nested })).resolves.toMatchObject({ status: 'ok' }); + + const after = nextRequestId(); + expect(after).toBe(before + 1); + expect(currentRequestId()).toBeUndefined(); + expect(actionEntries(entries, 'trace_off')).toHaveLength(0); + }); +}); diff --git a/packages/onebot/tests/event-pipeline.test.ts b/packages/onebot/tests/event-pipeline.test.ts index c3ee5f50..38b8eb2e 100644 --- a/packages/onebot/tests/event-pipeline.test.ts +++ b/packages/onebot/tests/event-pipeline.test.ts @@ -1,5 +1,13 @@ import { describe, expect, it, vi } from 'vitest'; -import { getRecentLogs } from '@snowluma/common/logger'; +import { + getLogLevel, + getRecentLogs, + nextRequestId, + runWithRequestId, + setLogLevel, + subscribeLogs, + type LogEntry, +} from '@snowluma/common/logger'; import { BridgeEventBus } from '@snowluma/protocol/event-bus'; // Wrap convertEvent in a vi.fn that delegates to the real impl by @@ -100,6 +108,11 @@ function makeContext(extra: Partial = {}): { bus: BridgeEventBus; metaCalls: Array<{ id: number; meta: MessageMeta }>; dispatchCalls: JsonObject[]; + dispatchArgs: Array<{ + event: JsonObject; + source: 'bridge' | 'send' | undefined; + startedAt: number | undefined; + }>; } { const bus = new BridgeEventBus(); const fakeBridge: FakeBridge = { events: bus }; @@ -112,6 +125,11 @@ function makeContext(extra: Partial = {}): { }; const metaCalls: Array<{ id: number; meta: MessageMeta }> = []; const dispatchCalls: JsonObject[] = []; + const dispatchArgs: Array<{ + event: JsonObject; + source: 'bridge' | 'send' | undefined; + startedAt: number | undefined; + }> = []; const ctx: OneBotInstanceContext = { uin: SELF_UIN, @@ -131,11 +149,14 @@ function makeContext(extra: Partial = {}): { converterCtx, config: { networks: { httpServers: [], httpClients: [], wsServers: [], wsClients: [] } } as never, cacheMessageMeta: (id, meta) => { metaCalls.push({ id, meta }); }, - dispatchEvent: (event) => { dispatchCalls.push(event); }, + dispatchEvent: (event, source, startedAt) => { + dispatchCalls.push(event); + dispatchArgs.push({ event, source, startedAt }); + }, ...extra, }; - return { ctx, bus, metaCalls, dispatchCalls }; + return { ctx, bus, metaCalls, dispatchCalls, dispatchArgs }; } describe('registerEventPipeline', () => { @@ -382,4 +403,137 @@ describe('registerEventPipeline', () => { await bus.emit(makeFriendMessage()); expect(dispatchCalls).toHaveLength(0); }); + + it('traces complete input and conversion under one request before handoff', async () => { + const previousLevel = getLogLevel(); + const entries: LogEntry[] = []; + const unsubscribe = subscribeLogs((entry) => entries.push(entry)); + setLogLevel('trace'); + try { + const marker = 'complete-event-' + 'x'.repeat(200); + const { ctx, bus, dispatchArgs } = makeContext(); + registerEventPipeline(ctx); + + await bus.emit({ + ...makeFriendMessage(), + senderNick: marker, + elements: [{ type: 'text', text: marker }], + }); + + const lifecycle = entries.filter((entry) => ( + entry.scope === 'Event' + && entry.level === 'trace' + && (entry.message.startsWith('event_input kind=friend_message') + || entry.message.startsWith('event_converted kind=friend_message')) + )); + expect(lifecycle).toHaveLength(2); + expect(lifecycle[0]?.message).toContain(marker); + expect(lifecycle[1]?.message).toContain(marker); + expect(new Set(lifecycle.map((entry) => entry.req)).size).toBe(1); + expect(lifecycle[0]?.req).toBeTypeOf('number'); + expect(dispatchArgs[0]).toMatchObject({ source: 'bridge' }); + expect(dispatchArgs[0]?.startedAt).toBeTypeOf('number'); + expect(entries.some((entry) => ( + entry.scope === 'Event' + && entry.message.startsWith('event_terminal kind=friend_message') + ))).toBe(false); + } finally { + unsubscribe(); + setLogLevel(previousLevel); + } + }); + + it('classifies converter null, converter throw, recall, backfill failure, and internal consumption', async () => { + const previousLevel = getLogLevel(); + const entries: LogEntry[] = []; + const unsubscribe = subscribeLogs((entry) => entries.push(entry)); + setLogLevel('trace'); + try { + vi.mocked(convertEvent).mockResolvedValueOnce(null); + const nullCase = makeContext(); + registerEventPipeline(nullCase.ctx); + await nullCase.bus.emit(makeFriendMessage()); + + vi.mocked(convertEvent).mockRejectedValueOnce(new Error('converter exploded')); + const throwCase = makeContext(); + registerEventPipeline(throwCase.ctx); + await throwCase.bus.emit(makeFriendMessage()); + + const recalledCase = makeContext({ + messageStore: { isPrivateMessageRecalled: () => true } as never, + }); + registerEventPipeline(recalledCase.ctx); + await recalledCase.bus.emit(makeFriendMessage()); + + vi.mocked(backfillReplyTarget).mockRejectedValueOnce(new Error('backfill exploded')); + const branchCase = makeContext(); + registerEventPipeline(branchCase.ctx); + await branchCase.bus.emit(makeFriendMessage()); + + const internalCase = makeContext(); + registerEventPipeline(internalCase.ctx); + await internalCase.bus.emit({ + kind: 'ptt_trans_result', + time: 1700000000, + selfUin: SELF_ID, + msgId: 99, + text: 'done', + }); + + const terminals = entries + .filter((entry) => entry.scope === 'Event' && entry.message.startsWith('event_terminal')) + .map((entry) => entry.message); + expect(terminals).toEqual(expect.arrayContaining([ + expect.stringContaining('outcome=dropped reason=converter_returned_null'), + expect.stringContaining('outcome=failed reason=converter_threw'), + expect.stringContaining('outcome=dropped reason=recalled_before_backfill'), + expect.stringContaining('outcome=internal reason=waiter_notified'), + ])); + expect(entries.some((entry) => ( + entry.scope === 'Event' + && entry.message.includes('event_branch kind=friend_message reason=reply_backfill_failed') + && entry.message.includes('backfill exploded') + ))).toBe(true); + expect(branchCase.dispatchCalls).toHaveLength(1); + expect(terminals.filter((message) => message.includes('converter exploded'))).toHaveLength(1); + } finally { + unsubscribe(); + setLogLevel(previousLevel); + } + }); + + it('reuses an ambient request, separates top-level events, and allocates none while TRACE is disabled', async () => { + const previousLevel = getLogLevel(); + const entries: LogEntry[] = []; + const unsubscribe = subscribeLogs((entry) => entries.push(entry)); + try { + setLogLevel('trace'); + const { ctx, bus } = makeContext(); + registerEventPipeline(ctx); + await runWithRequestId(515151, () => bus.emit(makeFriendMessage())); + expect(entries.filter((entry) => ( + entry.scope === 'Event' + && entry.level === 'trace' + && entry.message.includes('kind=friend_message') + )).every((entry) => entry.req === 515151)).toBe(true); + + entries.length = 0; + await bus.emit(makeFriendMessage()); + await bus.emit(makeFriendMessage()); + const inputs = entries.filter((entry) => ( + entry.scope === 'Event' + && entry.message.startsWith('event_input kind=friend_message') + )); + expect(inputs).toHaveLength(2); + expect(inputs[0]?.req).not.toBe(inputs[1]?.req); + + setLogLevel('info'); + const before = nextRequestId(); + await bus.emit(makeFriendMessage()); + expect(nextRequestId()).toBe(before + 1); + } finally { + unsubscribe(); + setLogLevel(previousLevel); + } + }); }); diff --git a/packages/onebot/tests/http-post-lifecycle.test.ts b/packages/onebot/tests/http-post-lifecycle.test.ts index 7afb4aa1..37620b62 100644 --- a/packages/onebot/tests/http-post-lifecycle.test.ts +++ b/packages/onebot/tests/http-post-lifecycle.test.ts @@ -1,4 +1,11 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + getLogLevel, + runWithRequestId, + setLogLevel, + subscribeLogs, + type LogEntry, +} from '@snowluma/common/logger'; import { createHmac } from 'crypto'; import { buildDispatchPayload } from '../src/event-filter'; import { HttpPostAdapter } from '../src/network/http-post-adapter'; @@ -42,6 +49,44 @@ describe('HttpPostAdapter event ownership', () => { expect(completed).toBe(true); }); + it.each([ + ['accepted', () => Promise.resolve(new Response(null, { status: 204 })), 204], + ['rejected', () => Promise.resolve(new Response(null, { status: 503 })), 503], + ['transport_failed', () => Promise.reject(new Error('network down')), undefined], + ] as const)('traces a truthful %s terminal', async (outcome, fetchImpl, status) => { + const previousLevel = getLogLevel(); + const entries: LogEntry[] = []; + const unsubscribe = subscribeLogs((entry) => entries.push(entry)); + setLogLevel('trace'); + vi.stubGlobal('fetch', vi.fn(fetchImpl)); + try { + const config: HttpClientNetwork = { + name: 'post', + url: 'http://127.0.0.1:5700/events', + messageFormat: 'array', + reportSelfMessage: false, + }; + const adapter = new HttpPostAdapter('post', config, CTX); + adapter.open(); + const event: JsonObject = { post_type: 'notice', notice_type: 'trace' }; + + await runWithRequestId(7100, () => adapter.onEvent(event, buildDispatchPayload(event))); + + const terminals = entries.filter((entry) => ( + entry.scope === 'OneBot.POST' + && entry.level === 'trace' + && entry.req === 7100 + && entry.message.startsWith('http_report_terminal') + )); + expect(terminals).toHaveLength(1); + expect(terminals[0]?.message).toContain(`target=post outcome=${outcome}`); + if (status !== undefined) expect(terminals[0]?.message).toContain(`status=${status}`); + } finally { + unsubscribe(); + setLogLevel(previousLevel); + } + }); + it('uses one coherent config epoch when reload races HMAC computation', async () => { const fetchMock = vi.fn(async () => new Response(null, { status: 204 })); vi.stubGlobal('fetch', fetchMock); diff --git a/packages/onebot/tests/http-server-action-lifecycle.test.ts b/packages/onebot/tests/http-server-action-lifecycle.test.ts index 70adefc9..cb8b8d4e 100644 --- a/packages/onebot/tests/http-server-action-lifecycle.test.ts +++ b/packages/onebot/tests/http-server-action-lifecycle.test.ts @@ -2,17 +2,187 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import http from 'http'; import type { AddressInfo } from 'net'; import { HttpServerAdapter } from '../src/network/http-server-adapter'; +import { StreamTransportClosedError, type StreamSink } from '../src/streaming'; import type { NetworkAdapterContext } from '../src/network/adapter'; -import type { HttpServerNetwork } from '../src/types'; +import type { HttpServerNetwork, JsonObject } from '../src/types'; describe('HttpServerAdapter inbound Action ownership', () => { let adapter: HttpServerAdapter | null = null; + function request( + port: number, + path: string, + ): Promise<{ status: number; body: JsonObject }> { + return new Promise((resolve, reject) => { + const outgoing = http.request({ + host: '127.0.0.1', + port, + path, + method: 'GET', + }, (response) => { + const chunks: Buffer[] = []; + response.on('data', (chunk: Buffer) => chunks.push(chunk)); + response.once('end', () => { + resolve({ + status: response.statusCode ?? 0, + body: JSON.parse(Buffer.concat(chunks).toString('utf8')) as JsonObject, + }); + }); + }); + outgoing.once('error', reject); + outgoing.end(); + }); + } + afterEach(async () => { await adapter?.close(); adapter = null; }); + it('records a parsed HTTP action rejected by API quiesce while preserving 503', async () => { + const traceQuiescedAction = vi.fn(); + const apiHandle = vi.fn(); + const ctx: NetworkAdapterContext = { + uin: '10001', + api: { + isAcceptingActions: false, + isStreamAction: vi.fn(() => false), + handle: apiHandle, + traceQuiescedAction, + } as never, + buildLifecycleEvent: () => ({}), + buildHeartbeatEvent: () => ({}), + }; + const config: HttpServerNetwork = { + name: 'http-quiesced-action', + host: '127.0.0.1', + port: 0, + path: '/', + messageFormat: 'array', + reportSelfMessage: false, + }; + adapter = new HttpServerAdapter(config.name, config, ctx); + await adapter.open(); + const server = (adapter as unknown as { server: http.Server }).server; + const port = (server.address() as AddressInfo).port; + + const actionResponse = await request(port, '/get_status?detail=%22complete%22'); + const healthResponse = await request(port, '/'); + + expect(actionResponse).toMatchObject({ + status: 503, + body: { + status: 'failed', + retcode: 1200, + data: null, + wording: 'server closing', + }, + }); + expect(healthResponse.status).toBe(503); + expect(traceQuiescedAction).toHaveBeenCalledOnce(); + expect(traceQuiescedAction).toHaveBeenCalledWith( + 'get_status', + { detail: 'complete' }, + actionResponse.body, + ); + expect(apiHandle).not.toHaveBeenCalled(); + }); + + it('does not add an API-quiesced HTTP request to the action drain', async () => { + let releaseTrace!: () => void; + const traceGate = new Promise((resolve) => { releaseTrace = resolve; }); + const ctx: NetworkAdapterContext = { + uin: '10001', + api: { isAcceptingActions: false } as never, + buildLifecycleEvent: () => ({}), + buildHeartbeatEvent: () => ({}), + }; + const config: HttpServerNetwork = { + name: 'http-quiesced-drain', + host: '127.0.0.1', + port: 0, + path: '/', + messageFormat: 'array', + reportSelfMessage: false, + }; + adapter = new HttpServerAdapter(config.name, config, ctx); + const internals = adapter as unknown as { + acceptingActions: boolean; + handleRequest: () => Promise; + trackInboundAction: (req: http.IncomingMessage, res: http.ServerResponse) => void; + inFlightActions: Set>; + }; + internals.acceptingActions = true; + internals.handleRequest = vi.fn(() => traceGate); + + internals.trackInboundAction({} as http.IncomingMessage, {} as http.ServerResponse); + + expect(internals.handleRequest).toHaveBeenCalledOnce(); + expect(internals.inFlightActions.size).toBe(0); + releaseTrace(); + await traceGate; + adapter = null; + }); + + it('classifies a disconnected streaming HTTP client as transport closure', async () => { + let actionStarted!: () => void; + const started = new Promise((resolve) => { actionStarted = resolve; }); + let releaseFrame!: () => void; + const frameGate = new Promise((resolve) => { releaseFrame = resolve; }); + let streamError: unknown; + const apiHandle = vi.fn(async (_action: string, _params: unknown, sink?: StreamSink) => { + actionStarted(); + await frameGate; + try { + await sink!.send({ type: 'stream', chunk: 'after-disconnect' }); + } catch (error) { + streamError = error; + } + return { status: 'ok' as const, retcode: 0, data: null }; + }); + const ctx: NetworkAdapterContext = { + uin: '10001', + api: { handle: apiHandle, isStreamAction: () => true } as never, + buildLifecycleEvent: () => ({}), + buildHeartbeatEvent: () => ({}), + }; + const config: HttpServerNetwork = { + name: 'http-stream-disconnect', + host: '127.0.0.1', + port: 0, + path: '/', + messageFormat: 'array', + reportSelfMessage: false, + }; + adapter = new HttpServerAdapter(config.name, config, ctx); + await adapter.open(); + const server = (adapter as unknown as { server: http.Server }).server; + const port = (server.address() as AddressInfo).port; + const serverSocketClosed = new Promise((resolve) => { + server.once('connection', (socket) => { + socket.once('close', () => resolve()); + }); + }); + + const request = http.request({ + host: '127.0.0.1', + port, + path: '/stream_action', + method: 'GET', + }); + request.on('error', () => { /* expected after destroy */ }); + request.end(); + await started; + request.destroy(); + await serverSocketClosed; + + releaseFrame(); + await vi.waitFor(() => { + expect(apiHandle).toHaveResolved(); + }); + expect(streamError).toBeInstanceOf(StreamTransportClosedError); + }); + it('waits for a detached Action after the HTTP client disconnects', async () => { let releaseAction!: () => void; const actionGate = new Promise((resolve) => { releaseAction = resolve; }); diff --git a/packages/onebot/tests/http-server-websocket.test.ts b/packages/onebot/tests/http-server-websocket.test.ts index f3bee473..25724cc5 100644 --- a/packages/onebot/tests/http-server-websocket.test.ts +++ b/packages/onebot/tests/http-server-websocket.test.ts @@ -130,6 +130,48 @@ describe('HttpServerAdapter — optional WebSocket transport', () => { }); }); + it('records and drops a valid WebSocket action after API quiesce', async () => { + const ctx = context(); + const api = ctx.api as unknown as { + isAcceptingActions: boolean; + processStreamRequest: ReturnType; + traceQuiescedStreamRequest: ReturnType; + }; + api.traceQuiescedStreamRequest = vi.fn(); + const config = { + name: 'quiesced-websocket', + host: '127.0.0.1', + port: 0, + path: '/', + enabled: true, + enableWebSocket: true, + messageFormat: 'array', + reportSelfMessage: false, + } satisfies HttpServerNetwork; + adapter = new HttpServerAdapter(config.name, config, ctx); + await adapter.open(); + const address = (adapter as unknown as { server: http.Server }).server.address() as AddressInfo; + socket = new WebSocket(`ws://127.0.0.1:${String(address.port)}/api`); + await waitForOpen(socket); + const received = vi.fn(); + socket.on('message', received); + api.isAcceptingActions = false; + const raw = JSON.stringify({ + action: 'get_status', + params: { detail: 'complete' }, + echo: 'dropped', + }); + + socket.send(raw); + + await vi.waitFor(() => { + expect(api.traceQuiescedStreamRequest).toHaveBeenCalledWith(raw); + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(api.processStreamRequest).not.toHaveBeenCalled(); + expect(received).not.toHaveBeenCalled(); + }); + it('disconnects authenticated sockets when the shared token changes without rebinding HTTP', async () => { const config = { name: 'rotating-token', diff --git a/packages/onebot/tests/network-manager.test.ts b/packages/onebot/tests/network-manager.test.ts index 24843390..53246ace 100644 --- a/packages/onebot/tests/network-manager.test.ts +++ b/packages/onebot/tests/network-manager.test.ts @@ -1,4 +1,11 @@ import { describe, expect, it } from 'vitest'; +import { + getLogLevel, + runWithRequestId, + setLogLevel, + subscribeLogs, + type LogEntry, +} from '@snowluma/common/logger'; import { IOneBotNetworkAdapter, NetworkReloadType, OneBotNetworkManager, type AdapterStatus } from '../src/network'; import type { NetworkAdapterContext } from '../src/network'; import type { JsonObject, NetworkBase } from '../src/types'; @@ -139,6 +146,46 @@ describe('OneBotNetworkManager', () => { expect(bad.events).toHaveLength(1); }); + it('traces active targets, isolated failures, settlement, and no-target conclusion', async () => { + const previousLevel = getLogLevel(); + const entries: LogEntry[] = []; + const unsubscribe = subscribeLogs((entry) => entries.push(entry)); + setLogLevel('trace'); + try { + const mgr = new OneBotNetworkManager(); + const ok = makeAdapter('ok-target'); + const bad = makeAdapter('bad-target', { failOnEvent: true }); + mgr.register(ok); + mgr.register(bad); + await mgr.openAll(); + + await runWithRequestId(7001, () => mgr.emitEvent(SAMPLE_EVENT)); + const report = entries.filter((entry) => ( + entry.scope === 'OneBot.Network' + && entry.level === 'trace' + && entry.req === 7001 + )); + expect(report.find((entry) => entry.message.startsWith('report_targets'))?.message) + .toContain('["ok-target","bad-target"]'); + expect(report.find((entry) => entry.message.startsWith('report_target_failed'))?.message) + .toContain('target=bad-target'); + expect(report.filter((entry) => entry.message === 'report_terminal outcome=settled')) + .toHaveLength(1); + + entries.length = 0; + const empty = new OneBotNetworkManager(); + await runWithRequestId(7002, () => empty.emitEvent(SAMPLE_EVENT)); + expect(entries.filter((entry) => ( + entry.scope === 'OneBot.Network' + && entry.req === 7002 + && entry.message === 'report_terminal outcome=no_active_targets' + ))).toHaveLength(1); + } finally { + unsubscribe(); + setLogLevel(previousLevel); + } + }); + it('rejects duplicate registration instead of detaching the old close', async () => { const mgr = new OneBotNetworkManager(); const first = makeAdapter('dup'); diff --git a/packages/onebot/tests/network-utils.test.ts b/packages/onebot/tests/network-utils.test.ts new file mode 100644 index 00000000..5e507d54 --- /dev/null +++ b/packages/onebot/tests/network-utils.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { WebSocket } from '@snowluma/websocket'; +import { StreamTransportClosedError } from '../src/streaming'; +import { safeSendAsync } from '../src/network/utils'; + +describe('safeSendAsync', () => { + it('resolves after a successful WebSocket write callback', async () => { + const socket = { + readyState: 1, + send: vi.fn((_payload: string, callback: (error?: Error | null) => void) => { + callback(null); + }), + } as unknown as WebSocket; + + await expect(safeSendAsync(socket, 'frame')).resolves.toBeUndefined(); + expect(socket.send).toHaveBeenCalledWith('frame', expect.any(Function)); + }); + + it('rejects when the socket is not open', async () => { + const socket = { + readyState: 3, + send: vi.fn(), + } as unknown as WebSocket; + + await expect(safeSendAsync(socket, 'frame')) + .rejects.toBeInstanceOf(StreamTransportClosedError); + expect(socket.send).not.toHaveBeenCalled(); + }); + + it('rejects a synchronous WebSocket send failure', async () => { + const socket = { + readyState: 1, + send: vi.fn(() => { throw new Error('socket gone'); }), + } as unknown as WebSocket; + + await expect(safeSendAsync(socket, 'frame')) + .rejects.toBeInstanceOf(StreamTransportClosedError); + }); + + it('rejects an asynchronous WebSocket write failure', async () => { + const socket = { + readyState: 1, + send: vi.fn((_payload: string, callback: (error?: Error | null) => void) => { + callback(new Error('write failed')); + }), + } as unknown as WebSocket; + + await expect(safeSendAsync(socket, 'frame')) + .rejects.toBeInstanceOf(StreamTransportClosedError); + }); +}); diff --git a/packages/onebot/tests/self-sent-events.test.ts b/packages/onebot/tests/self-sent-events.test.ts index 57d20739..0ec9318c 100644 --- a/packages/onebot/tests/self-sent-events.test.ts +++ b/packages/onebot/tests/self-sent-events.test.ts @@ -1,4 +1,12 @@ import { describe, expect, it, vi } from 'vitest'; +import { + createLogger, + getLogLevel, + runWithRequestId, + setLogLevel, + subscribeLogs, + type LogEntry, +} from '@snowluma/common/logger'; import type { BridgeInterface } from '../../core/src/bridge/bridge-interface'; import { OneBotInstance } from '../src/instance'; import { buildApiContext, type OneBotInstanceContext } from '../src/instance-context'; @@ -97,7 +105,10 @@ interface InstanceHarness { stored: Map; } -function makeInstanceHarness(): InstanceHarness { +function makeInstanceHarness(options: { + statusCommand?: { enabled: boolean; swallow: boolean; cooldownSeconds: number; trigger: string }; + logger?: ReturnType; +} = {}): InstanceHarness { const stored = new Map(); const emitEvent = vi.fn(async (_event: JsonObject) => {}); const instance = Object.create(OneBotInstance.prototype) as OneBotInstance; @@ -105,10 +116,11 @@ function makeInstanceHarness(): InstanceHarness { bridge: { identity: {} }, ctx: { config: { - statusCommand: { enabled: false, swallow: false, cooldownSeconds: 5, trigger: '#sl' }, + statusCommand: options.statusCommand + ?? { enabled: false, swallow: false, cooldownSeconds: 5, trigger: '#sl' }, }, }, - log: { success: vi.fn(), trace: vi.fn(), warn: vi.fn() }, + log: options.logger ?? { success: vi.fn(), trace: vi.fn(), warn: vi.fn() }, messageStore: { findEvent: (messageId: number) => stored.get(messageId) ?? null, storeEvent: (messageId: number, _isGroup: boolean, _sessionId: number, _sequence: number, _eventName: string, event: JsonObject) => { @@ -116,7 +128,9 @@ function makeInstanceHarness(): InstanceHarness { }, }, networkManager: { emitEvent }, - statusCommandCooldown: new Map(), + acceptingActions: true, + uin: String(SELF_ID), + statusCommandCooldown: new Map([['p:20002', Date.now()]]), pendingSelfSentEchoes: new Map(), }); const dispatch = (instance as unknown as { @@ -394,6 +408,56 @@ describe('OneBot self-sent events', () => { expect(stored.get(987654)?.raw_message).toBe('canonical echo'); }); + it('records one truthful handoff terminal for reporting, duplicate suppression, and status suppression', () => { + const previousLevel = getLogLevel(); + const entries: LogEntry[] = []; + const unsubscribe = subscribeLogs((entry) => entries.push(entry)); + setLogLevel('trace'); + try { + const logger = createLogger('Event').child({ uin: SELF_ID }); + const reporting = makeInstanceHarness({ logger }); + runWithRequestId(6001, () => reporting.dispatch(privateSentEvent(), 'send')); + expect(reporting.emitEvent).toHaveBeenCalledOnce(); + + const duplicate = makeInstanceHarness({ logger }); + runWithRequestId(6002, () => { + duplicate.dispatch(privateSentEvent(), 'send'); + duplicate.dispatch(privateSentEvent({ raw_message: 'canonical' }), 'bridge'); + }); + expect(duplicate.emitEvent).toHaveBeenCalledOnce(); + + const status = makeInstanceHarness({ + logger, + statusCommand: { enabled: true, swallow: true, cooldownSeconds: 5, trigger: '#sl' }, + }); + runWithRequestId(6003, () => status.dispatch(privateSentEvent({ + post_type: 'message', + user_id: PEER_ID, + target_id: undefined, + raw_message: '#sl', + message: [{ type: 'text', data: { text: '#sl' } }], + }), 'bridge')); + expect(status.emitEvent).not.toHaveBeenCalled(); + + const terminals = entries.filter((entry) => ( + entry.scope === 'Event' + && entry.level === 'trace' + && entry.message.startsWith('event_terminal') + )); + expect(terminals.filter((entry) => entry.req === 6001)).toHaveLength(1); + expect(terminals.find((entry) => entry.req === 6001)?.message).toContain('outcome=reporting_started'); + expect(terminals.filter((entry) => entry.req === 6002)).toHaveLength(2); + expect(terminals.find((entry) => ( + entry.req === 6002 && entry.message.includes('reason=duplicate_self_echo') + ))?.message).toContain('outcome=suppressed'); + expect(terminals.filter((entry) => entry.req === 6003)).toHaveLength(1); + expect(terminals[terminals.length - 1]?.message).toContain('outcome=suppressed reason=status_command'); + } finally { + unsubscribe(); + setLogLevel(previousLevel); + } + }); + it('does not dispatch an event for a receipt without a reliable sequence', async () => { const { ctx, dispatchEvent } = makeContext({ ...RECEIPT, sequence: 0 }); const api = buildApiContext(ctx); diff --git a/packages/onebot/tests/ws-client-stale-close.test.ts b/packages/onebot/tests/ws-client-stale-close.test.ts index 2ed4d79d..eeecf9e8 100644 --- a/packages/onebot/tests/ws-client-stale-close.test.ts +++ b/packages/onebot/tests/ws-client-stale-close.test.ts @@ -126,6 +126,32 @@ describe('WsClientAdapter — stale-socket close guard (issue #97)', () => { expect(closed).toBe(true); }); + it('records and drops a valid action after API quiesce', async () => { + const processStreamRequest = vi.fn(async () => {}); + const traceQuiescedStreamRequest = vi.fn(); + const quiescedCtx: NetworkAdapterContext = { + ...ctx(), + api: { + isAcceptingActions: false, + processStreamRequest, + traceQuiescedStreamRequest, + } as never, + }; + const adapter = new WsClientAdapter('ws', cfg(), quiescedCtx); + adapter.open(); + const socket = instances[0]; + const raw = Buffer.from(JSON.stringify({ + action: 'get_status', + params: { detail: 'complete' }, + })); + + socket.emit('message', raw); + + expect(traceQuiescedStreamRequest).toHaveBeenCalledWith(raw.toString('utf8')); + expect(processStreamRequest).not.toHaveBeenCalled(); + await adapter.close(); + }); + it('ignores the close event from a socket replaced by a hot reload', async () => { vi.useFakeTimers(); try {