diff --git a/packages/onebot/src/message-store-migration-task.ts b/packages/onebot/src/message-store-migration-task.ts index 7253cd55..92e4488f 100644 --- a/packages/onebot/src/message-store-migration-task.ts +++ b/packages/onebot/src/message-store-migration-task.ts @@ -1,6 +1,7 @@ import { createRequire } from 'node:module'; import path from 'node:path'; import { Worker, type WorkerOptions } from 'node:worker_threads'; +import { createLogger } from '@snowluma/common/logger'; import type { DatabaseMigrationCallbacks, DatabaseMigrationTask, @@ -29,6 +30,7 @@ export function createMessageStoreMigrationTask( uin: string, createWorker: CreateMigrationWorker = (filename, options) => new Worker(filename, options), ): DatabaseMigrationTask { + const log = createLogger('OneBot.MessageMigration').child({ uin: Number(uin) }); let worker: MessageStoreMigrationWorkerHandle | null = null; let cancelled = false; let finished = false; @@ -59,6 +61,15 @@ export function createMessageStoreMigrationTask( if (cancelled) return; if (message.kind === 'ready') { callbacks.onReady(); + } else if (message.kind === 'preparation') { + const progress = message.progress; + if (progress.stage === 'started') { + log.info('preparing message history migration'); + } else if (progress.stage === 'indexed') { + log.info('message history migration preparation finished in %dms', Math.round(progress.elapsedMs)); + } else { + log.info('message history migration: pending=%d checked in %dms', progress.total, Math.round(progress.elapsedMs)); + } } else if (message.kind === 'progress') { callbacks.onProgress(message.status, message.elapsedMs); if (message.status.phase === 'complete') finished = true; diff --git a/packages/onebot/src/message-store-migration-worker.ts b/packages/onebot/src/message-store-migration-worker.ts index a400821e..5e546d62 100644 --- a/packages/onebot/src/message-store-migration-worker.ts +++ b/packages/onebot/src/message-store-migration-worker.ts @@ -3,6 +3,7 @@ import { migrateLegacyMedia } from './media-store-migration'; import { MessageStoreMigrator, prepareMessageStoreDatabase, + type MessageStoreMigrationPreparation, type MessageStoreMigrationStatus, } from './message-store-migration'; @@ -17,6 +18,7 @@ export interface MessageStoreMigrationWorkerData { export type MessageStoreMigrationWorkerMessage = | { kind: 'ready' } + | { kind: 'preparation'; progress: MessageStoreMigrationPreparation } | { kind: 'progress'; status: MessageStoreMigrationStatus; @@ -55,7 +57,9 @@ export async function runMessageStoreMigrationWorker( port.postMessage({ kind: 'ready' } satisfies MessageStoreMigrationWorkerMessage); await startRequested; if (cancelled) return; - migrator = new MessageStoreMigrator(data.dbPath); + migrator = new MessageStoreMigrator(data.dbPath, progress => { + port.postMessage({ kind: 'preparation', progress } satisfies MessageStoreMigrationWorkerMessage); + }); let lastProgressAt: number | null = null; while (!cancelled) { diff --git a/packages/onebot/src/message-store-migration.ts b/packages/onebot/src/message-store-migration.ts index d38b826b..aa81fcf7 100644 --- a/packages/onebot/src/message-store-migration.ts +++ b/packages/onebot/src/message-store-migration.ts @@ -2,6 +2,7 @@ import { DatabaseSync, type StatementSync } from 'node:sqlite'; import { openSqliteDb } from './sqlite-open'; const CLASSIFICATION_VERSION = 1; +const PENDING_INDEX_NAME = `idx_messages_pending_classification_v${CLASSIFICATION_VERSION}`; const MIGRATION_NAME = 'message-classification-v1'; const LEGACY_PRIVATE_SEQUENCE_MIGRATION = 'private-nt-sequence-v1'; @@ -13,6 +14,11 @@ export interface MessageStoreMigrationStatus { total: number; } +export type MessageStoreMigrationPreparation = + | { stage: 'started' } + | { stage: 'indexed'; elapsedMs: number } + | { stage: 'counted'; total: number; elapsedMs: number }; + interface MigrationStateRow { phase: MessageStoreMigrationPhase; processed: number; @@ -39,42 +45,47 @@ export class MessageStoreMigrator { private readonly updateState: StatementSync; private state: MigrationStateRow; - constructor(dbPath: string) { + constructor(dbPath: string, reportPreparation?: (progress: MessageStoreMigrationPreparation) => void) { this.db = openSqliteDb(dbPath.replace(/\.json$/, '.db')); - createMigrationStateTable(this.db); - this.state = this.loadOrCreateState(); - this.selectInitialBatch = this.db.prepare(` - SELECT message_hash, is_group, sequence, sequence_authoritative, client_sequence, private_direction, random, data - FROM messages - WHERE classification_version < ? - ORDER BY message_hash ASC - LIMIT ? - `); - this.selectNextBatch = this.db.prepare(` - SELECT message_hash, is_group, sequence, sequence_authoritative, client_sequence, private_direction, random, data - FROM messages - WHERE classification_version < ? AND message_hash > ? - ORDER BY message_hash ASC - LIMIT ? - `); - this.updateRow = this.db.prepare(` - UPDATE messages - SET sequence_authoritative = ?, private_direction = ?, classification_version = ? - WHERE message_hash = ? - AND classification_version < ? - AND is_group = ? - AND sequence = ? - AND sequence_authoritative = ? - AND client_sequence = ? - AND private_direction = ? - AND random = ? - AND data IS ? - `); - this.updateState = this.db.prepare(` - UPDATE message_store_migration_state - SET phase = ?, processed = ?, total = ?, cursor = ?, updated_at = ? - WHERE name = ? - `); + try { + createMigrationStateTable(this.db); + this.state = this.loadOrCreateState(reportPreparation); + this.selectInitialBatch = this.db.prepare(` + SELECT message_hash, is_group, sequence, sequence_authoritative, client_sequence, private_direction, random, data + FROM messages + WHERE classification_version < ? + ORDER BY message_hash ASC + LIMIT ? + `); + this.selectNextBatch = this.db.prepare(` + SELECT message_hash, is_group, sequence, sequence_authoritative, client_sequence, private_direction, random, data + FROM messages + WHERE classification_version < ? AND message_hash > ? + ORDER BY message_hash ASC + LIMIT ? + `); + this.updateRow = this.db.prepare(` + UPDATE messages + SET sequence_authoritative = ?, private_direction = ?, classification_version = ? + WHERE message_hash = ? + AND classification_version < ? + AND is_group = ? + AND sequence = ? + AND sequence_authoritative = ? + AND client_sequence = ? + AND private_direction = ? + AND random = ? + AND data IS ? + `); + this.updateState = this.db.prepare(` + UPDATE message_store_migration_state + SET phase = ?, processed = ?, total = ?, cursor = ?, updated_at = ? + WHERE name = ? + `); + } catch (error) { + this.db.close(); + throw error; + } } close(): void { @@ -153,20 +164,32 @@ export class MessageStoreMigrator { return this.getStatus(); } - private loadOrCreateState(): MigrationStateRow { + private loadOrCreateState(reportPreparation?: (progress: MessageStoreMigrationPreparation) => void): MigrationStateRow { const existing = this.db.prepare(` SELECT phase, processed, total, cursor FROM message_store_migration_state WHERE name = ? `).get(MIGRATION_NAME) as MigrationStateRow | undefined; + if (existing?.phase === 'complete') return existing; + + reportPreparation?.({ stage: 'started' }); + const preparationStartedAt = performance.now(); + this.db.exec(` + CREATE INDEX IF NOT EXISTS ${PENDING_INDEX_NAME} + ON messages(message_hash, classification_version) + WHERE classification_version < ${CLASSIFICATION_VERSION} + `); + reportPreparation?.({ stage: 'indexed', elapsedMs: Math.max(0, performance.now() - preparationStartedAt) }); if (existing) return existing; + const countStartedAt = performance.now(); const row = this.db.prepare(` SELECT COUNT(*) AS total FROM messages WHERE classification_version < ? `).get(CLASSIFICATION_VERSION) as { total: number }; const total = Number(row.total); + reportPreparation?.({ stage: 'counted', total, elapsedMs: Math.max(0, performance.now() - countStartedAt) }); const now = Math.floor(Date.now() / 1000); const state: MigrationStateRow = { phase: total === 0 ? 'complete' : 'migrating', diff --git a/packages/onebot/tests/message-store-migration-task.test.ts b/packages/onebot/tests/message-store-migration-task.test.ts index 186196ea..d34e1368 100644 --- a/packages/onebot/tests/message-store-migration-task.test.ts +++ b/packages/onebot/tests/message-store-migration-task.test.ts @@ -1,5 +1,6 @@ import { EventEmitter } from 'node:events'; import { describe, expect, it, vi } from 'vitest'; +import { subscribeLogs } from '@snowluma/common/logger'; import type { DatabaseMigrationCallbacks } from '../src/manager'; import { createMessageStoreMigrationTask, @@ -22,6 +23,32 @@ function callbacks(): DatabaseMigrationCallbacks { } describe('message-store migration task lifecycle', () => { + it('records worker preparation through the parent logger with account context', () => { + const worker = fakeWorker(); + const handlers = callbacks(); + const entries: Array<{ scope: string; uin?: number; message: string }> = []; + const unsubscribe = subscribeLogs(entry => entries.push(entry)); + const task = createMessageStoreMigrationTask('10001', () => worker); + try { + task.start(handlers); + worker.emit('message', { kind: 'preparation', progress: { stage: 'started' } }); + worker.emit('message', { kind: 'preparation', progress: { stage: 'indexed', elapsedMs: 12.5 } }); + worker.emit('message', { kind: 'preparation', progress: { stage: 'counted', total: 3, elapsedMs: 2.1 } }); + expect(entries.map(entry => ({ scope: entry.scope, uin: entry.uin, message: entry.message }))).toEqual([ + { scope: 'OneBot.MessageMigration', uin: 10001, message: 'preparing message history migration' }, + { scope: 'OneBot.MessageMigration', uin: 10001, message: 'message history migration preparation finished in 13ms' }, + { scope: 'OneBot.MessageMigration', uin: 10001, message: 'message history migration: pending=3 checked in 2ms' }, + ]); + expect(handlers.onProgress).not.toHaveBeenCalled(); + expect(handlers.onFailed).not.toHaveBeenCalled(); + task.cancel(); + worker.emit('message', { kind: 'preparation', progress: { stage: 'started' } }); + expect(entries).toHaveLength(3); + } finally { + unsubscribe(); + } + }); + it('reports an exit before completion even when its code is zero', () => { const worker = fakeWorker(); const handlers = callbacks(); diff --git a/packages/onebot/tests/message-store-migration-worker.test.ts b/packages/onebot/tests/message-store-migration-worker.test.ts index 9fb00a9e..aec613cc 100644 --- a/packages/onebot/tests/message-store-migration-worker.test.ts +++ b/packages/onebot/tests/message-store-migration-worker.test.ts @@ -9,6 +9,7 @@ import { inspectMessageStoreMigration, MessageStoreMigrator, prepareMessageStoreDatabase, + type MessageStoreMigrationPreparation, } from '../src/message-store-migration'; import { isMessageStoreMigrationWorkerData, @@ -25,20 +26,24 @@ function workerPayload(dbPath: string): MessageStoreMigrationWorkerData { function createControlPort(): { port: MessagePort; messages: MessageStoreMigrationWorkerMessage[]; + preparations: MessageStoreMigrationPreparation[]; send(message: unknown): void; listenerCount(): number; } { const messages: MessageStoreMigrationWorkerMessage[] = []; + const preparations: MessageStoreMigrationPreparation[] = []; const events = new EventEmitter(); return { port: { on: events.on.bind(events), off: events.off.bind(events), postMessage(message: MessageStoreMigrationWorkerMessage) { - messages.push(message); + if (message.kind === 'preparation') preparations.push(message.progress); + else messages.push(message); }, } as unknown as MessagePort, messages, + preparations, send(message: unknown) { events.emit('message', message); }, @@ -174,6 +179,37 @@ describe('runMessageStoreMigrationWorker', () => { ]); }); + it('forwards preparation timings only after start without opening a worker log transport', async () => { + const { subscribeLogs } = await import('@snowluma/common/logger'); + const { getFileTransport } = await import('@snowluma/common/log-file-transport'); + const fileTransport = getFileTransport(); + const write = vi.spyOn(fileTransport, 'write').mockImplementation(() => {}); + const logs: string[] = []; + const unsubscribe = subscribeLogs(entry => logs.push(entry.scope)); + vi.stubEnv('SNOWLUMA_LOG_FILE', '1'); + const dbPath = path.join(tmpDir, 'messages.db'); + seedUnclassified(dbPath, 1); + const control = createControlPort(); + try { + const running = runMessageStoreMigrationWorker(workerPayload(dbPath), control.port); + await Promise.resolve(); + expect(control.preparations).toEqual([]); + control.send('start'); + await running; + expect(control.preparations).toEqual([ + { stage: 'started' }, + { stage: 'indexed', elapsedMs: expect.any(Number) }, + { stage: 'counted', total: 1, elapsedMs: expect.any(Number) }, + ]); + expect(control.preparations.filter(progress => progress.stage !== 'started').every(progress => progress.elapsedMs >= 0)).toBe(true); + expect(logs).toEqual([]); + expect(write).not.toHaveBeenCalled(); + } finally { + unsubscribe(); + vi.unstubAllEnvs(); + } + }); + it('returns after cancel without posting progress', async () => { const dbPath = path.join(tmpDir, 'messages.db'); seedUnclassified(dbPath, 1); @@ -342,7 +378,7 @@ describe('runMessageStoreMigrationWorker', () => { expect(control.messages).toEqual([ { kind: 'ready' }, - { kind: 'failed', message: 'no such table: messages' }, + { kind: 'failed', message: 'no such table: main.messages' }, ]); }); diff --git a/packages/onebot/tests/message-store-migration.test.ts b/packages/onebot/tests/message-store-migration.test.ts new file mode 100644 index 00000000..7479a5ad --- /dev/null +++ b/packages/onebot/tests/message-store-migration.test.ts @@ -0,0 +1,146 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { MessageStoreMigrator, prepareMessageStoreDatabase } from '../src/message-store-migration'; + +const pendingIndex = 'idx_messages_pending_classification_v1'; + +describe('message history migration scans', () => { + let directory: string; + let dbPath: string; + let db: DatabaseSync; + let migrator: MessageStoreMigrator | undefined; + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'snowluma-migration-scans-')); + dbPath = path.join(directory, 'messages.db'); + prepareMessageStoreDatabase(dbPath); + db = new DatabaseSync(dbPath); + const insert = db.prepare(`INSERT INTO messages + (message_hash, is_group, session_id, sequence, client_sequence, event_name, data, media_data, classification_version) + VALUES (?, 0, 42, 1, 1, 'message.private', ?, ?, ?)`); + const data = JSON.stringify({ + post_type: 'message', message_type: 'private', sub_type: 'friend', + message: [{ type: 'text', data: { text: 'x'.repeat(3000) } }], + }); + db.exec('BEGIN'); + for (let hash = -500; hash <= 500; hash++) { + insert.run(hash, data, '{"retained":"snapshot"}', [-400, 0, 400].includes(hash) ? 0 : 1); + } + db.exec('COMMIT'); + }); + + afterEach(() => { + vi.restoreAllMocks(); + migrator?.close(); + migrator = undefined; + db.close(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + function capturedPendingQueries(queries: string[]): string[] { + return [...new Set(queries.filter(sql => /SELECT[\s\S]+FROM messages\s+WHERE classification_version < \?/.test(sql)))]; + } + + function expectPendingQueryPlans(queries: string[]): void { + expect(queries).toHaveLength(3); + const planDb = new DatabaseSync(dbPath); + try { + for (const sql of queries) { + const args = sql.includes('LIMIT') ? (sql.includes('message_hash >') ? [1, -400, 1] : [1, 1]) : [1]; + const details = planDb.prepare(`EXPLAIN QUERY PLAN ${sql}`).all(...args).map(row => String(row.detail)).join('\n'); + expect(details).toContain(pendingIndex); + expect(details).not.toContain('USE TEMP B-TREE'); + if (sql.includes('COUNT(*)')) expect(details).toContain('COVERING INDEX'); + } + } finally { + planDb.close(); + } + } + + it('uses pending rows for the actual count and ordered batch queries without changing history', () => { + const before = db.prepare('SELECT message_hash, data, media_data FROM messages ORDER BY message_hash').all(); + const prepare = vi.spyOn(DatabaseSync.prototype, 'prepare'); + migrator = new MessageStoreMigrator(dbPath); + const queries = capturedPendingQueries(prepare.mock.calls.map(([sql]) => sql)); + expectPendingQueryPlans(queries); + expect(migrator.getStatus()).toEqual({ phase: 'migrating', processed: 0, total: 3 }); + expect(migrator.runBatch(2)).toEqual({ phase: 'migrating', processed: 2, total: 3 }); + expect(migrator.runBatch(2)).toEqual({ phase: 'complete', processed: 3, total: 3 }); + expectPendingQueryPlans(queries); + expect(db.prepare('SELECT COUNT(*) AS n FROM messages WHERE classification_version < 1').get()).toEqual({ n: 0 }); + expect(db.prepare('SELECT message_hash, data, media_data FROM messages ORDER BY message_hash').all()).toEqual(before); + expect(db.prepare('SELECT message_hash, private_direction FROM messages WHERE message_hash IN (-400, 0, 400) ORDER BY message_hash').all()).toEqual([ + { message_hash: -400, private_direction: 0 }, + { message_hash: 0, private_direction: 0 }, + { message_hash: 400, private_direction: 0 }, + ]); + }); + + it('adds the index when resuming a migration started by an older version', () => { + migrator = new MessageStoreMigrator(dbPath); + expect(migrator.runBatch(1)).toEqual({ phase: 'migrating', processed: 1, total: 3 }); + migrator.close(); migrator = undefined; + db.exec(`DROP INDEX IF EXISTS ${pendingIndex}`); + + migrator = new MessageStoreMigrator(dbPath); + expect(db.prepare('SELECT name FROM sqlite_schema WHERE type = ? AND name = ?').get('index', pendingIndex)).toEqual({ name: pendingIndex }); + expect(migrator.getStatus()).toEqual({ phase: 'migrating', processed: 1, total: 3 }); + expect(migrator.runBatch(3)).toEqual({ phase: 'complete', processed: 3, total: 3 }); + }); + + it('does not rebuild an existing pending index when an interrupted migration resumes', () => { + migrator = new MessageStoreMigrator(dbPath); + migrator.runBatch(1); + migrator.close(); migrator = undefined; + const schemaVersion = db.prepare('PRAGMA schema_version').get(); + + migrator = new MessageStoreMigrator(dbPath); + expect(db.prepare('PRAGMA schema_version').get()).toEqual(schemaVersion); + expect(migrator.runBatch(3)).toEqual({ phase: 'complete', processed: 3, total: 3 }); + }); + + it('revisits rows left behind the cursor after a concurrent update prevented classification', () => { + db.exec(`CREATE TRIGGER defer_first BEFORE UPDATE ON messages + WHEN OLD.message_hash = -400 BEGIN SELECT RAISE(IGNORE); END`); + migrator = new MessageStoreMigrator(dbPath); + expect(migrator.runBatch(1)).toEqual({ phase: 'migrating', processed: 0, total: 3 }); + db.exec('DROP TRIGGER defer_first'); + expect(migrator.runBatch(2)).toEqual({ phase: 'migrating', processed: 2, total: 3 }); + expect(migrator.runBatch(2)).toEqual({ phase: 'complete', processed: 3, total: 3 }); + expect(db.prepare('SELECT COUNT(*) AS n FROM messages WHERE classification_version < 1').get()).toEqual({ n: 0 }); + expect(db.prepare('SELECT COUNT(*) AS n FROM messages').get()).toEqual({ n: 1001 }); + }); + + it('skips index creation and pending counts when an older version already completed migration', () => { + migrator = new MessageStoreMigrator(dbPath); + migrator.runBatch(10); + migrator.close(); migrator = undefined; + db.exec(`DROP INDEX IF EXISTS ${pendingIndex}`); + const exec = vi.spyOn(DatabaseSync.prototype, 'exec'); + const prepare = vi.spyOn(DatabaseSync.prototype, 'prepare'); + + migrator = new MessageStoreMigrator(dbPath); + expect(migrator.runBatch(10)).toEqual({ phase: 'complete', processed: 3, total: 3 }); + expect(exec.mock.calls.some(([sql]) => /CREATE\s+INDEX/i.test(sql))).toBe(false); + expect(prepare.mock.calls.some(([sql]) => sql.includes('COUNT(*)'))).toBe(false); + expect(db.prepare('SELECT name FROM sqlite_schema WHERE type = ? AND name = ?').get('index', pendingIndex)).toBeUndefined(); + }); + + it('propagates preparation failure and closes the failed migration connection', () => { + const originalExec = DatabaseSync.prototype.exec; + const failure = new Error('pending index unavailable'); + let failedDb: DatabaseSync | undefined; + vi.spyOn(DatabaseSync.prototype, 'exec').mockImplementation(function (this: DatabaseSync, sql) { + if (sql.includes(pendingIndex)) { failedDb = this; throw failure; } + return originalExec.call(this, sql); + }); + + expect(() => { migrator = new MessageStoreMigrator(dbPath); }).toThrow(failure); + expect(failedDb).toBeDefined(); + expect(() => failedDb!.prepare('SELECT 1')).toThrow(/not open/); + expect(db.prepare('SELECT COUNT(*) AS n FROM messages WHERE classification_version < 1').get()).toEqual({ n: 3 }); + }); +});