fix(onebot): bring an account back when shutdown stalls
If the previous OneBot session does not finish closing, the account now comes back online instead of staying disconnected until QQ is restarted. Fixes #509
This commit is contained in:
@@ -273,6 +273,20 @@ export class OneBotInstance {
|
||||
this.ctx.musicSignUrl = globalSettings.musicSignUrl;
|
||||
}
|
||||
|
||||
/** Drop listeners and sockets without waiting for shutdown to finish.
|
||||
* Used when dispose() does not settle and a replacement session must bind. */
|
||||
forceRelease(): void {
|
||||
try {
|
||||
this.quiesce();
|
||||
} catch (error) {
|
||||
this.log.warn(
|
||||
'quiesce during forced release failed: %s',
|
||||
error instanceof Error ? error.message : String(error),
|
||||
);
|
||||
}
|
||||
this.networkManager.forceRelease();
|
||||
}
|
||||
|
||||
/** Synchronously stop every ingress seam for this instance generation.
|
||||
*
|
||||
* This operation is sticky and idempotent. It is intentionally separate
|
||||
|
||||
@@ -43,7 +43,13 @@ export interface DatabaseMigrationTask {
|
||||
cancel(): void;
|
||||
}
|
||||
|
||||
/** A peer that never finishes a close handshake must not keep the next
|
||||
* account session offline. Ten seconds covers a normal drain. */
|
||||
const SESSION_RELEASE_DEADLINE_MS = 10_000;
|
||||
|
||||
export interface OneBotManagerOptions {
|
||||
/** How long a retiring instance may block the replacement session. */
|
||||
sessionReleaseDeadlineMs?: number;
|
||||
createDatabaseMigrationTask?: (uin: string) => DatabaseMigrationTask;
|
||||
createInstance?: (
|
||||
uin: string,
|
||||
@@ -91,8 +97,12 @@ export class OneBotManager {
|
||||
private readonly createInstance: NonNullable<OneBotManagerOptions['createInstance']>;
|
||||
private disposePromise: Promise<void> | null = null;
|
||||
private disposed = false;
|
||||
private readonly sessionReleaseDeadlineMs: number;
|
||||
|
||||
constructor(options: OneBotManagerOptions = {}) {
|
||||
this.sessionReleaseDeadlineMs = options.sessionReleaseDeadlineMs && options.sessionReleaseDeadlineMs > 0
|
||||
? options.sessionReleaseDeadlineMs
|
||||
: SESSION_RELEASE_DEADLINE_MS;
|
||||
this.createDatabaseMigrationTask = options.createDatabaseMigrationTask
|
||||
?? createMessageStoreMigrationTask;
|
||||
this.createInstance = options.createInstance
|
||||
@@ -447,20 +457,58 @@ export class OneBotManager {
|
||||
return result;
|
||||
}
|
||||
|
||||
private async releaseRetiringInstance(
|
||||
uin: string,
|
||||
pending: { bridge: BridgeInterface; cancelled: boolean },
|
||||
instance: OneBotInstance,
|
||||
): Promise<void> {
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
const timeout = new Promise<'timeout'>((resolve) => {
|
||||
timer = setTimeout(() => resolve('timeout'), this.sessionReleaseDeadlineMs);
|
||||
timer.unref?.();
|
||||
});
|
||||
// Map rejection here so a dispose() that fails after the deadline still
|
||||
// has a handler. The race may already have moved on.
|
||||
const releasing = instance.dispose().then(
|
||||
() => 'released' as const,
|
||||
(error: unknown) => ({ failed: error }),
|
||||
);
|
||||
try {
|
||||
const outcome = await Promise.race([releasing, timeout]);
|
||||
if (outcome === 'timeout') {
|
||||
log.error(
|
||||
'session release timed out; forcing adapters closed and starting the replacement: UIN=%s',
|
||||
uin,
|
||||
);
|
||||
try {
|
||||
instance.forceRelease?.();
|
||||
} catch (error) {
|
||||
log.error(
|
||||
'forced session release failed: UIN=%s: %s',
|
||||
uin,
|
||||
error instanceof Error ? error.message : String(error),
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (typeof outcome === 'object') {
|
||||
// Do not leave this UIN permanently guarded by a failed handoff. A
|
||||
// later session-start observation may retry the still-visible retire.
|
||||
if (this.pendingStarts.get(uin) === pending) this.pendingStarts.delete(uin);
|
||||
throw new InstanceLifecycleError(instance, outcome.failed);
|
||||
}
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
private async finishRetiringBeforeStart(
|
||||
uin: string,
|
||||
pending: { bridge: BridgeInterface; cancelled: boolean },
|
||||
retiring: OneBotInstance[],
|
||||
): Promise<void> {
|
||||
for (const instance of retiring) {
|
||||
try {
|
||||
await instance.dispose();
|
||||
} catch (error) {
|
||||
// Do not leave this UIN permanently guarded by a failed handoff. A
|
||||
// later session-start observation may retry the still-visible retire.
|
||||
if (this.pendingStarts.get(uin) === pending) this.pendingStarts.delete(uin);
|
||||
throw new InstanceLifecycleError(instance, error);
|
||||
}
|
||||
await this.releaseRetiringInstance(uin, pending, instance);
|
||||
this.retirementSucceeded(instance);
|
||||
}
|
||||
if (this.pendingStarts.get(uin) !== pending) return;
|
||||
|
||||
@@ -61,6 +61,11 @@ export abstract class IOneBotNetworkAdapter<C extends NetworkBase> {
|
||||
abstract open(): void | Promise<void>;
|
||||
abstract close(): void | Promise<void>;
|
||||
|
||||
/** Drop sockets and listeners without waiting for a peer handshake.
|
||||
* Graceful `close()` can sit forever on a half-open connection; shutdown
|
||||
* uses this so the next account session can bind the same ports. */
|
||||
forceClose(): void { /* adapters that own a listener override this */ }
|
||||
|
||||
abstract onEvent(event: JsonObject, payload: DispatchPayload): void | Promise<void>;
|
||||
|
||||
/** Report live connection health for the WebUI dashboard. */
|
||||
|
||||
@@ -25,6 +25,7 @@ export class HttpServerAdapter extends IOneBotNetworkAdapter<HttpServerNetwork>
|
||||
private server: Server | null = null;
|
||||
private webSocketServer: WebSocketServer | null = null;
|
||||
private listening = false;
|
||||
private releaseForced = false;
|
||||
private closePromise: Promise<void> | null = null;
|
||||
private acceptingActions = false;
|
||||
private readonly inFlightActions = new Set<Promise<void>>();
|
||||
@@ -113,6 +114,7 @@ export class HttpServerAdapter extends IOneBotNetworkAdapter<HttpServerNetwork>
|
||||
try {
|
||||
await attempt;
|
||||
} catch (error) {
|
||||
if (this.releaseForced) return;
|
||||
if (!httpReleased && server?.listening === true) {
|
||||
// A failed HTTP close callback cannot prove the listener was released.
|
||||
// The upgrade handler is still attached, so the old ingress can accept
|
||||
@@ -128,6 +130,26 @@ export class HttpServerAdapter extends IOneBotNetworkAdapter<HttpServerNetwork>
|
||||
}
|
||||
}
|
||||
|
||||
override forceClose(): void {
|
||||
this.releaseForced = true;
|
||||
this.acceptingActions = false;
|
||||
this.webSocketConnections.stopAccepting();
|
||||
this.isEnabled = false;
|
||||
this.listening = false;
|
||||
this.webSocketConnections.terminateAll();
|
||||
const webSocketServer = this.webSocketServer;
|
||||
this.webSocketServer = null;
|
||||
const server = this.server;
|
||||
this.server = null;
|
||||
if (webSocketServer) {
|
||||
try { webSocketServer.destroy(); } catch { /* already dropping */ }
|
||||
}
|
||||
if (server) {
|
||||
try { server.closeAllConnections(); } catch { /* already closed */ }
|
||||
try { server.close(() => undefined); } catch { /* already closed */ }
|
||||
}
|
||||
}
|
||||
|
||||
override describeStatus(): AdapterStatus {
|
||||
if (!this.isEnabled) return { name: this.name, kind: 'httpServer', status: 'disabled', detail: '未启用' };
|
||||
if (!this.listening) return { name: this.name, kind: 'httpServer', status: 'down', detail: '未监听(端口被占用?)' };
|
||||
|
||||
@@ -228,6 +228,20 @@ export class OneBotNetworkManager {
|
||||
return this.shutdown();
|
||||
}
|
||||
|
||||
/** Best-effort immediate release. Does not wait for the operation queue,
|
||||
* in-flight dispatches, or peer close handshakes. */
|
||||
forceRelease(): void {
|
||||
this.acceptingReconciles = false;
|
||||
this.acceptingEvents = false;
|
||||
for (const { adapter } of this.adapters.values()) {
|
||||
try {
|
||||
adapter.forceClose();
|
||||
} catch (error) {
|
||||
log.error('forced adapter release failed: %s', errMessage(error));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitEvent(event: JsonObject): Promise<void> {
|
||||
if (!this.acceptingEvents) {
|
||||
return Promise.reject(new Error('network manager is shutting down; event dispatch rejected'));
|
||||
|
||||
@@ -60,6 +60,21 @@ export class WsClientAdapter extends IOneBotNetworkAdapter<WsClientNetwork> {
|
||||
}
|
||||
}
|
||||
|
||||
override forceClose(): void {
|
||||
this.explicitlyClosed = true;
|
||||
this.acceptingActions = false;
|
||||
this.isEnabled = false;
|
||||
this.connected = false;
|
||||
this.cancelReconnect();
|
||||
this.heartbeatStop?.();
|
||||
this.heartbeatStop = null;
|
||||
const socket = this.socket;
|
||||
this.socket = null;
|
||||
if (socket) {
|
||||
try { socket.terminate(); } catch { /* already gone */ }
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.explicitlyClosed = true;
|
||||
this.acceptingActions = false;
|
||||
|
||||
@@ -15,6 +15,7 @@ const moduleLog = createLogger('OneBot.WS-Server');
|
||||
export class WsServerAdapter extends IOneBotNetworkAdapter<WsServerNetwork> {
|
||||
private wss: WebSocketServer | null = null;
|
||||
private listening = false;
|
||||
private releaseForced = false;
|
||||
private closePromise: Promise<void> | null = null;
|
||||
private readonly connections: WsServerConnections;
|
||||
|
||||
@@ -66,6 +67,7 @@ export class WsServerAdapter extends IOneBotNetworkAdapter<WsServerNetwork> {
|
||||
await attempt;
|
||||
if (wss && this.wss === wss) this.wss = null;
|
||||
} catch (error) {
|
||||
if (this.releaseForced) return;
|
||||
// A failed close callback leaves release ambiguous. Retain the server
|
||||
// reference and active binding state so a later shutdown can retry.
|
||||
this.isEnabled = wasEnabled;
|
||||
@@ -77,6 +79,19 @@ export class WsServerAdapter extends IOneBotNetworkAdapter<WsServerNetwork> {
|
||||
}
|
||||
}
|
||||
|
||||
override forceClose(): void {
|
||||
this.releaseForced = true;
|
||||
this.isEnabled = false;
|
||||
this.listening = false;
|
||||
this.connections.stopAccepting();
|
||||
this.connections.terminateAll();
|
||||
const wss = this.wss;
|
||||
this.wss = null;
|
||||
if (wss) {
|
||||
try { wss.destroy(); } catch { /* already dropping */ }
|
||||
}
|
||||
}
|
||||
|
||||
override describeStatus(): AdapterStatus {
|
||||
if (!this.isEnabled) return { name: this.name, kind: 'wsServer', status: 'disabled', detail: '未启用' };
|
||||
if (!this.listening) return { name: this.name, kind: 'wsServer', status: 'down', detail: '未监听(端口被占用?)' };
|
||||
|
||||
@@ -212,6 +212,17 @@ export class WsServerConnections {
|
||||
this.connections.clear();
|
||||
}
|
||||
|
||||
/** Destroy sockets immediately. A graceful close frame can leave the
|
||||
* listener waiting on a peer that will never answer. */
|
||||
terminateAll(): void {
|
||||
this.stopAccepting();
|
||||
for (const connection of this.connections.values()) {
|
||||
connection.stopHeartbeat();
|
||||
try { connection.socket.terminate(); } catch { /* already gone */ }
|
||||
}
|
||||
this.connections.clear();
|
||||
}
|
||||
|
||||
private async handleApiMessage(socket: WebSocket, role: WsRole, raw: Buffer | string): Promise<void> {
|
||||
const text = rawDataToString(raw);
|
||||
if (!text) {
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import { once } from 'node:events';
|
||||
import http from 'node:http';
|
||||
import net from 'node:net';
|
||||
import type { AddressInfo } from 'node:net';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import type { NetworkAdapterContext } from '../src/network/adapter';
|
||||
import { HttpServerAdapter } from '../src/network/http-server-adapter';
|
||||
import type { HttpServerNetwork } from '../src/types';
|
||||
|
||||
describe('HttpServerAdapter forced release', () => {
|
||||
it('drops a held connection so the port can be bound again', async () => {
|
||||
const ctx = {
|
||||
uin: '10001',
|
||||
api: { handle: async () => ({ status: 'ok', retcode: 0, data: null }), isStreamAction: () => false },
|
||||
buildLifecycleEvent: () => ({}),
|
||||
buildHeartbeatEvent: () => ({}),
|
||||
} as unknown as NetworkAdapterContext;
|
||||
const config: HttpServerNetwork = {
|
||||
name: 'http-force-close',
|
||||
host: '127.0.0.1',
|
||||
port: 0,
|
||||
path: '/',
|
||||
messageFormat: 'array',
|
||||
reportSelfMessage: false,
|
||||
};
|
||||
const adapter = new HttpServerAdapter(config.name, config, ctx);
|
||||
let socket: net.Socket | null = null;
|
||||
try {
|
||||
await adapter.open();
|
||||
const server = (adapter as unknown as { server: http.Server | null }).server;
|
||||
expect(server).not.toBeNull();
|
||||
const port = (server!.address() as AddressInfo).port;
|
||||
socket = net.connect({ host: '127.0.0.1', port });
|
||||
await once(socket, 'connect');
|
||||
|
||||
adapter.forceClose();
|
||||
|
||||
expect(server!.listening).toBe(false);
|
||||
await expect(listenOn(port)).resolves.toBeUndefined();
|
||||
} finally {
|
||||
socket?.destroy();
|
||||
await adapter.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
function listenOn(port: number): Promise<void> {
|
||||
const server = net.createServer();
|
||||
return new Promise((resolve, reject) => {
|
||||
server.once('error', reject);
|
||||
server.listen(port, '127.0.0.1', () => {
|
||||
server.close(() => resolve());
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -569,6 +569,55 @@ describe('OneBotManager lifecycle failure accounting', () => {
|
||||
expect((manager as unknown as { pendingStarts: Map<string, unknown> }).pendingStarts.has('10001')).toBe(false);
|
||||
});
|
||||
|
||||
it('starts the replacement account when release never finishes', async () => {
|
||||
const originalCwd = process.cwd();
|
||||
const root = mkdtempSync(path.join(tmpdir(), 'snowluma-manager-release-'));
|
||||
process.chdir(root);
|
||||
const forceRelease = vi.fn();
|
||||
const old = {
|
||||
...fakeInstance('10001', () => new Promise(() => undefined)),
|
||||
forceRelease,
|
||||
} as unknown as OneBotInstance;
|
||||
const replacement = {
|
||||
...fakeInstance('10001', async () => undefined),
|
||||
waitUntilNetworkReady: vi.fn(async () => ({ applied: true, statuses: [], errors: [] })),
|
||||
startLoginHistorySync: vi.fn(),
|
||||
startGroupRequestPolling: vi.fn(),
|
||||
} as unknown as OneBotInstance;
|
||||
let created = false;
|
||||
const manager = new OneBotManager({
|
||||
sessionReleaseDeadlineMs: 30,
|
||||
createDatabaseMigrationTask: () => ({
|
||||
beginMigration: vi.fn(),
|
||||
cancel: vi.fn(),
|
||||
start: (next) => { next.onReady(); },
|
||||
}),
|
||||
createInstance: () => {
|
||||
created = true;
|
||||
return replacement;
|
||||
},
|
||||
});
|
||||
const internals = manager as unknown as {
|
||||
retiringInstances: Set<OneBotInstance>;
|
||||
onSessionStarted(uin: string, bridge: never): void;
|
||||
pendingStarts: Map<string, unknown>;
|
||||
};
|
||||
try {
|
||||
internals.retiringInstances.add(old);
|
||||
internals.onSessionStarted('10001', fakeBridge() as never);
|
||||
await vi.waitFor(() => {
|
||||
expect(created).toBe(true);
|
||||
});
|
||||
expect(forceRelease).toHaveBeenCalledOnce();
|
||||
expect(manager.getInstance('10001')).toBe(replacement);
|
||||
expect(internals.pendingStarts.has('10001')).toBe(false);
|
||||
expect(internals.retiringInstances.has(old)).toBe(false);
|
||||
} finally {
|
||||
process.chdir(originalCwd);
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('allows a later same-UIN start observation to retry a failed handoff', async () => {
|
||||
const manager = new OneBotManager();
|
||||
let attempts = 0;
|
||||
|
||||
@@ -282,4 +282,21 @@ export class WebSocketServer extends EventEmitter {
|
||||
setImmediate(cb);
|
||||
}
|
||||
}
|
||||
|
||||
/** Destroy clients and, when this server owns the HTTP listener, that
|
||||
* listener too. An external HTTP server is left to its owner. */
|
||||
destroy(): void {
|
||||
for (const ws of this.clients) {
|
||||
try { ws.terminate(); } catch { /* noop */ }
|
||||
}
|
||||
if (this._externalServer && this._upgradeHandler) {
|
||||
this._externalServer.removeListener('upgrade', this._upgradeHandler);
|
||||
this._upgradeHandler = null;
|
||||
}
|
||||
if (this._server) {
|
||||
try { this._server.closeAllConnections(); } catch { /* noop */ }
|
||||
try { this._server.close(() => undefined); } catch { /* noop */ }
|
||||
this._server = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user