diff --git a/packages/core/src/webui/restore.ts b/packages/core/src/webui/restore.ts new file mode 100644 index 00000000..42e24251 --- /dev/null +++ b/packages/core/src/webui/restore.ts @@ -0,0 +1,391 @@ +import { createLogger } from '@snowluma/common/logger'; +import { randomUUID } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { + prepareRestorePlan, + RestorePreflightError, + type RestoreMigration, +} from './backup'; + +const log = createLogger('WebUI.Restore'); + +export type RestorePhase = 'preflight' | 'snapshot' | 'stage' | 'commit' | 'rollback' | 'cleanup'; + +export interface RestoreOperation { + phase: Exclude; + operation: + | 'copy-current' + | 'write-staged' + | 'chmod-staged' + | 'rename-staged' + | 'restore-current' + | 'remove-created' + | 'remove-transaction'; + file?: string; +} + +export interface RestoreBackupOptions { + configDir: string; + restoreCredentials: boolean; + /** Deterministic id for tests/diagnostics; production normally omits it. */ + transactionId?: string; + /** Filesystem-boundary fault hook used by the transaction fault matrix. */ + beforeOperation?: (operation: RestoreOperation) => void; +} + +export interface RestoreBackupResult { + transactionId: string; + restored: string[]; + skipped: string[]; + migrated: RestoreMigration[]; + restartRequiredToApply: boolean; +} + +export class RestoreTransactionError extends Error { + readonly phase: RestorePhase; + readonly transactionId: string; + readonly rollbackSucceeded?: boolean; + readonly snapshotDir?: string; + readonly committed: boolean; + readonly failedFiles: string[]; + + constructor(options: { + phase: RestorePhase; + transactionId: string; + message: string; + cause: unknown; + rollbackSucceeded?: boolean; + snapshotDir?: string; + committed?: boolean; + failedFiles?: string[]; + }) { + super(options.message, { cause: options.cause }); + this.name = 'RestoreTransactionError'; + this.phase = options.phase; + this.transactionId = options.transactionId; + this.rollbackSucceeded = options.rollbackSucceeded; + this.snapshotDir = options.snapshotDir; + this.committed = options.committed ?? false; + this.failedFiles = options.failedFiles ?? []; + } +} + +interface CommittedFile { + name: string; + existed: boolean; +} + +/** + * Restore one validated overlay as a process-level transaction. + * + * This deliberately does not promise recovery from power loss or SIGKILL. + * Within the running process, every failure before commit leaves live files + * untouched. A partial commit is rolled back when possible; incomplete + * rollback retains the transaction directory and reports its recovery path. + */ +export function restoreBackup(parsed: unknown, options: RestoreBackupOptions): RestoreBackupResult { + const transactionId = options.transactionId ?? randomUUID(); + if (!/^[A-Za-z0-9_-]+$/.test(transactionId)) { + throw new Error('restore transactionId must contain only letters, numbers, _ or -'); + } + const configDir = path.resolve(options.configDir); + const readCurrent = (name: string): Buffer | null => { + const file = path.join(configDir, name); + try { + return fs.readFileSync(file); + } catch (error) { + if (isMissingPathError(error)) return null; + throw error; + } + }; + const listCurrentOneBotNames = (): string[] => { + try { + return fs.readdirSync(configDir); + } catch (error) { + if (isMissingPathError(error)) return []; + throw error; + } + }; + + log.info('restore transaction=%s phase=preflight', transactionId); + let prepared: ReturnType; + try { + prepared = prepareRestorePlan(parsed, { + restoreCredentials: options.restoreCredentials, + readCurrent, + listCurrentOneBotNames, + }); + } catch (error) { + const file = error instanceof RestorePreflightError ? error.file : undefined; + log.warn( + 'restore transaction=%s phase=preflight file=%s failed error=%s', + transactionId, + file ?? '-', + errorMessage(error), + ); + throw new RestoreTransactionError({ + phase: 'preflight', + transactionId, + message: `restore preflight failed: ${errorMessage(error)}`, + cause: error, + failedFiles: file ? [file] : [], + }); + } + const names = prepared.restore.map((file) => file.name); + log.info('restore transaction=%s phase=preflight-complete files=%s', transactionId, names.join(',')); + + if (prepared.restore.length === 0) { + return { + transactionId, + restored: [], + skipped: prepared.skipped, + migrated: prepared.migrated, + restartRequiredToApply: false, + }; + } + + const transactionDir = path.join(configDir, `.restore-transaction-${transactionId}`); + const snapshotRoot = path.join(transactionDir, 'snapshot'); + const stagedRoot = path.join(transactionDir, 'staged'); + const rollbackRoot = path.join(transactionDir, 'rollback'); + const existed = new Map(); + const committed: CommittedFile[] = []; + let phase: 'snapshot' | 'stage' | 'commit' = 'snapshot'; + let activeFile: string | undefined; + let transactionCreated = false; + + try { + fs.mkdirSync(configDir, { recursive: true }); + fs.mkdirSync(transactionDir, { mode: 0o700 }); + transactionCreated = true; + fs.chmodSync(transactionDir, 0o700); + + log.info('restore transaction=%s phase=snapshot files=%s', transactionId, names.join(',')); + for (const file of prepared.restore) { + activeFile = file.name; + const live = path.join(configDir, file.name); + let wasPresent: boolean; + try { + fs.lstatSync(live); + wasPresent = true; + } catch (error) { + if (!isMissingPathError(error)) throw error; + wasPresent = false; + } + existed.set(file.name, wasPresent); + if (!wasPresent) continue; + const snapshot = path.join(snapshotRoot, file.name); + fs.mkdirSync(path.dirname(snapshot), { recursive: true }); + invoke(options, { phase: 'snapshot', operation: 'copy-current', file: file.name }); + fs.copyFileSync(live, snapshot); + } + + phase = 'stage'; + log.info('restore transaction=%s phase=stage files=%s', transactionId, names.join(',')); + for (const file of prepared.restore) { + activeFile = file.name; + const staged = path.join(stagedRoot, file.name); + fs.mkdirSync(path.dirname(staged), { recursive: true }); + invoke(options, { phase: 'stage', operation: 'write-staged', file: file.name }); + fs.writeFileSync(staged, file.data, { flag: 'wx', mode: file.mode }); + invoke(options, { phase: 'stage', operation: 'chmod-staged', file: file.name }); + fs.chmodSync(staged, file.mode); + } + + phase = 'commit'; + log.info('restore transaction=%s phase=commit files=%s', transactionId, names.join(',')); + for (const file of prepared.restore) { + activeFile = file.name; + const staged = path.join(stagedRoot, file.name); + const live = path.join(configDir, file.name); + fs.mkdirSync(path.dirname(live), { recursive: true }); + invoke(options, { phase: 'commit', operation: 'rename-staged', file: file.name }); + fs.renameSync(staged, live); + committed.push({ name: file.name, existed: existed.get(file.name) === true }); + } + activeFile = undefined; + } catch (error) { + if (phase !== 'commit') { + const cleanupError = transactionCreated ? removeTransaction(options, transactionDir) : null; + if (cleanupError) { + log.error( + 'restore transaction=%s phase=cleanup failed-after=%s file=%s original-error=%s cleanup-error=%s', + transactionId, + phase, + activeFile ?? '-', + errorMessage(error), + errorMessage(cleanupError), + ); + throw new RestoreTransactionError({ + phase: 'cleanup', + transactionId, + message: `restore ${phase} failed and transaction cleanup also failed`, + cause: new AggregateError([error, cleanupError], `restore ${phase} and cleanup both failed`), + snapshotDir: transactionDir, + failedFiles: activeFile ? [activeFile] : [], + }); + } + log.warn( + 'restore transaction=%s phase=%s file=%s failed error=%s', + transactionId, + phase, + activeFile ?? '-', + errorMessage(error), + ); + throw new RestoreTransactionError({ + phase, + transactionId, + message: `restore ${phase} failed before live commit`, + cause: error, + failedFiles: activeFile ? [activeFile] : [], + }); + } + + const rollbackErrors = rollbackCommitted(options, configDir, snapshotRoot, rollbackRoot, committed); + if (rollbackErrors.length > 0) { + const failedFiles = rollbackErrors.map((entry) => entry.name); + const rollbackDetails = rollbackErrors + .map((entry) => `${entry.name}:${errorMessage(entry.error)}`) + .join(';'); + log.error( + 'restore transaction=%s phase=rollback commit-file=%s commit-error=%s failed=%s snapshot=%s', + transactionId, + activeFile ?? '-', + errorMessage(error), + rollbackDetails, + transactionDir, + ); + throw new RestoreTransactionError({ + phase: 'rollback', + transactionId, + message: 'restore commit failed and rollback was incomplete', + cause: new AggregateError( + [error, ...rollbackErrors.map((entry) => entry.error)], + 'restore commit and rollback both failed', + ), + rollbackSucceeded: false, + snapshotDir: transactionDir, + committed: committed.length > 0, + failedFiles, + }); + } + + const cleanupError = removeTransaction(options, transactionDir); + if (cleanupError) { + log.error( + 'restore transaction=%s phase=cleanup failed-after=rollback commit-file=%s commit-error=%s cleanup-error=%s snapshot=%s', + transactionId, + activeFile ?? '-', + errorMessage(error), + errorMessage(cleanupError), + transactionDir, + ); + throw new RestoreTransactionError({ + phase: 'cleanup', + transactionId, + message: 'restore commit failed, rollback succeeded, but transaction cleanup failed', + cause: new AggregateError([error, cleanupError], 'restore commit and post-rollback cleanup both failed'), + rollbackSucceeded: true, + snapshotDir: transactionDir, + failedFiles: activeFile ? [activeFile] : [], + }); + } + log.warn( + 'restore transaction=%s phase=commit file=%s failed rollback=complete error=%s', + transactionId, + activeFile ?? '-', + errorMessage(error), + ); + throw new RestoreTransactionError({ + phase: 'commit', + transactionId, + message: 'restore commit failed; every committed file was rolled back', + cause: error, + rollbackSucceeded: true, + failedFiles: activeFile ? [activeFile] : [], + }); + } + + const cleanupError = removeTransaction(options, transactionDir); + if (cleanupError) { + log.error( + 'restore transaction=%s phase=cleanup failed-after=commit cleanup-error=%s snapshot=%s', + transactionId, + errorMessage(cleanupError), + transactionDir, + ); + throw new RestoreTransactionError({ + phase: 'cleanup', + transactionId, + message: 'restore committed successfully, but transaction cleanup failed', + cause: cleanupError, + snapshotDir: transactionDir, + committed: true, + }); + } + + log.info('restore transaction=%s phase=complete files=%s', transactionId, names.join(',')); + return { + transactionId, + restored: names, + skipped: prepared.skipped, + migrated: prepared.migrated, + restartRequiredToApply: true, + }; +} + +function rollbackCommitted( + options: RestoreBackupOptions, + configDir: string, + snapshotRoot: string, + rollbackRoot: string, + committed: readonly CommittedFile[], +): Array<{ name: string; error: unknown }> { + const errors: Array<{ name: string; error: unknown }> = []; + for (const file of [...committed].reverse()) { + const live = path.join(configDir, file.name); + try { + if (!file.existed) { + invoke(options, { phase: 'rollback', operation: 'remove-created', file: file.name }); + fs.rmSync(live, { force: true }); + continue; + } + const snapshot = path.join(snapshotRoot, file.name); + const rollback = path.join(rollbackRoot, file.name); + fs.mkdirSync(path.dirname(rollback), { recursive: true }); + invoke(options, { phase: 'rollback', operation: 'restore-current', file: file.name }); + fs.copyFileSync(snapshot, rollback); + fs.chmodSync(rollback, fs.statSync(snapshot).mode & 0o777); + fs.renameSync(rollback, live); + } catch (error) { + errors.push({ name: file.name, error }); + } + } + return errors; +} + +function removeTransaction(options: RestoreBackupOptions, transactionDir: string): unknown | null { + try { + invoke(options, { phase: 'cleanup', operation: 'remove-transaction' }); + fs.rmSync(transactionDir, { recursive: true, force: true }); + return null; + } catch (error) { + return error; + } +} + +function invoke(options: RestoreBackupOptions, operation: RestoreOperation): void { + options.beforeOperation?.(operation); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function isMissingPathError(error: unknown): boolean { + return typeof error === 'object' + && error !== null + && 'code' in error + && error.code === 'ENOENT'; +} diff --git a/packages/core/tests/webui-restore.test.ts b/packages/core/tests/webui-restore.test.ts new file mode 100644 index 00000000..b7659bfd --- /dev/null +++ b/packages/core/tests/webui-restore.test.ts @@ -0,0 +1,366 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { + RestoreTransactionError, + restoreBackup, + type RestoreOperation, +} from '../src/webui/restore'; + +const TS = '2026-06-18T00:00:00.000Z'; + +function backupWith(files: Record) { + return { version: 1, app: 'snowluma', files }; +} + +function runtime(port: number): { encoding: 'utf8'; data: string } { + return { encoding: 'utf8', data: JSON.stringify({ webuiPort: port }) }; +} + +function notifications(seconds: number): { encoding: 'utf8'; data: string } { + return { + encoding: 'utf8', + data: JSON.stringify({ version: 1, debounceSeconds: seconds, channels: [] }), + }; +} + +function validAuth() { + return { + passwordHash: 'ab'.repeat(64), + passwordSalt: 'cd'.repeat(16), + mustChangePassword: false, + generatedAt: TS, + updatedAt: TS, + }; +} + +describe('restoreBackup transaction', () => { + let root: string; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'snowluma-restore-')); + }); + + afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }); + }); + + const livePath = (name: string) => path.join(root, name); + const writeLive = (name: string, data: string, mode = 0o644) => { + fs.mkdirSync(path.dirname(livePath(name)), { recursive: true }); + fs.writeFileSync(livePath(name), data, { mode }); + fs.chmodSync(livePath(name), mode); + }; + const readLive = (name: string) => fs.readFileSync(livePath(name), 'utf8'); + const transactionDirs = () => fs.readdirSync(root).filter((name) => name.startsWith('.restore-transaction-')); + + function run( + files: Record, + beforeOperation?: (operation: RestoreOperation) => void, + ) { + return restoreBackup(backupWith(files), { + configDir: root, + restoreCredentials: true, + transactionId: 'test-transaction', + beforeOperation, + }); + } + + it('commits every prepared file, enforces credential mode, and removes the temporary snapshot', () => { + writeLive('runtime.json', '{"old":true}'); + writeLive('webui.json', JSON.stringify(validAuth()), 0o644); + + const result = run({ + 'runtime.json': runtime(5100), + 'webui.json': { encoding: 'utf8', data: JSON.stringify(validAuth()) }, + }); + + expect(JSON.parse(readLive('runtime.json')).webuiPort).toBe(5100); + expect(JSON.parse(readLive('webui.json')).passwordHash).toBe(validAuth().passwordHash); + expect(fs.statSync(livePath('webui.json')).mode & 0o777).toBe(0o600); + expect(result).toEqual(expect.objectContaining({ + transactionId: 'test-transaction', + restored: ['runtime.json', 'webui.json'], + skipped: [], + restartRequiredToApply: true, + })); + expect(transactionDirs()).toEqual([]); + }); + + it('does not request a restart or create a transaction when nothing is selected', () => { + const result = run({}); + + expect(result).toEqual(expect.objectContaining({ + restored: [], + skipped: [], + migrated: [], + restartRequiredToApply: false, + })); + expect(transactionDirs()).toEqual([]); + }); + + it.each([ + ['snapshot', 'copy-current'], + ['stage', 'write-staged'], + ['stage', 'chmod-staged'], + ] as const)('leaves every live file untouched when %s fails', (phase, operation) => { + const oldRuntime = '{"old":"runtime"}'; + const oldNotifications = '{"old":"notifications"}'; + writeLive('runtime.json', oldRuntime); + writeLive('notifications.json', oldNotifications); + + expect(() => run({ + 'runtime.json': runtime(5101), + 'notifications.json': notifications(45), + }, (point) => { + if (point.phase === phase && point.operation === operation && point.file === 'notifications.json') { + throw new Error(`injected ${phase} failure`); + } + })).toThrow(RestoreTransactionError); + + expect(readLive('runtime.json')).toBe(oldRuntime); + expect(readLive('notifications.json')).toBe(oldNotifications); + expect(transactionDirs()).toEqual([]); + }); + + it('rolls every file back when a later commit rename fails', () => { + const oldRuntime = '{"old":"runtime"}'; + const oldNotifications = '{"old":"notifications"}'; + writeLive('runtime.json', oldRuntime); + writeLive('notifications.json', oldNotifications); + + let thrown: unknown; + try { + run({ + 'runtime.json': runtime(5102), + 'notifications.json': notifications(60), + }, (point) => { + if (point.phase === 'commit' && point.operation === 'rename-staged' && point.file === 'notifications.json') { + throw new Error('injected commit failure'); + } + }); + } catch (error) { + thrown = error; + } + + expect(thrown).toEqual(expect.objectContaining({ + name: 'RestoreTransactionError', + phase: 'commit', + rollbackSucceeded: true, + committed: false, + })); + expect(readLive('runtime.json')).toBe(oldRuntime); + expect(readLive('notifications.json')).toBe(oldNotifications); + expect(transactionDirs()).toEqual([]); + }); + + it('removes files that did not exist before when rolling a partial commit back', () => { + const oldRuntime = '{"old":"runtime"}'; + const oldUi = '{}'; + writeLive('runtime.json', oldRuntime); + writeLive('ui.json', oldUi); + + expect(() => run({ + 'runtime.json': runtime(5103), + 'notifications.json': notifications(75), + 'ui.json': { encoding: 'utf8', data: '{}' }, + }, (point) => { + if (point.phase === 'commit' && point.operation === 'rename-staged' && point.file === 'ui.json') { + throw new Error('injected final commit failure'); + } + })).toThrow(RestoreTransactionError); + + expect(readLive('runtime.json')).toBe(oldRuntime); + expect(fs.existsSync(livePath('notifications.json'))).toBe(false); + expect(readLive('ui.json')).toBe(oldUi); + expect(transactionDirs()).toEqual([]); + }); + + it('retains the snapshot and reports its path only when rollback itself fails', () => { + writeLive('runtime.json', '{"old":"runtime"}'); + writeLive('notifications.json', '{"old":"notifications"}'); + + let thrown: unknown; + try { + run({ + 'runtime.json': runtime(5104), + 'notifications.json': notifications(90), + }, (point) => { + if (point.phase === 'commit' && point.operation === 'rename-staged' && point.file === 'notifications.json') { + throw new Error('injected commit failure'); + } + if (point.phase === 'rollback' && point.operation === 'restore-current' && point.file === 'runtime.json') { + throw new Error('injected rollback failure'); + } + }); + } catch (error) { + thrown = error; + } + + expect(thrown).toEqual(expect.objectContaining({ + name: 'RestoreTransactionError', + phase: 'rollback', + rollbackSucceeded: false, + snapshotDir: expect.stringContaining('.restore-transaction-test-transaction'), + })); + const snapshotDir = (thrown as RestoreTransactionError).snapshotDir!; + expect(fs.existsSync(snapshotDir)).toBe(true); + expect(fs.statSync(snapshotDir).mode & 0o777).toBe(0o700); + expect(fs.existsSync(path.join(snapshotDir, 'snapshot', 'runtime.json'))).toBe(true); + const causes = ((thrown as RestoreTransactionError).cause as AggregateError).errors; + expect(causes.map((error: unknown) => error instanceof Error ? error.message : String(error))).toEqual([ + 'injected commit failure', + 'injected rollback failure', + ]); + }); + + it('performs no filesystem transaction when semantic preflight fails', () => { + const oldRuntime = '{"old":"runtime"}'; + writeLive('runtime.json', oldRuntime); + const operations: RestoreOperation[] = []; + + let thrown: unknown; + try { + run({ + 'runtime.json': { encoding: 'utf8', data: JSON.stringify({ webuiPort: 70_000 }) }, + }, (operation) => operations.push(operation)); + } catch (error) { + thrown = error; + } + + expect(thrown).toEqual(expect.objectContaining({ + name: 'RestoreTransactionError', + phase: 'preflight', + transactionId: 'test-transaction', + failedFiles: ['runtime.json'], + })); + expect((thrown as RestoreTransactionError).cause).toEqual(expect.objectContaining({ + name: 'RestorePreflightError', + file: 'runtime.json', + })); + expect(operations).toEqual([]); + expect(readLive('runtime.json')).toBe(oldRuntime); + expect(transactionDirs()).toEqual([]); + }); + + it('fails preflight when a required current OneBot file cannot be read', () => { + fs.symlinkSync('onebot.json', livePath('onebot.json')); + + let thrown: unknown; + try { + run({ + 'onebot_12345.json': { + encoding: 'utf8', + data: JSON.stringify({ + mode: 'overlay', + httpClients: [{ name: 'remote', url: 'https://example.test/onebot' }], + }), + }, + }); + } catch (error) { + thrown = error; + } + + expect(thrown).toEqual(expect.objectContaining({ + name: 'RestoreTransactionError', + phase: 'preflight', + failedFiles: ['onebot.json'], + })); + expect((thrown as RestoreTransactionError).cause).toEqual(expect.objectContaining({ + name: 'RestorePreflightError', + file: 'onebot.json', + })); + expect(transactionDirs()).toEqual([]); + }); + + it('reports a committed restore distinctly when final cleanup fails', () => { + writeLive('runtime.json', '{"old":"runtime"}'); + + let thrown: unknown; + try { + run({ 'runtime.json': runtime(5105) }, (point) => { + if (point.phase === 'cleanup') throw new Error('injected cleanup failure'); + }); + } catch (error) { + thrown = error; + } + + expect(thrown).toEqual(expect.objectContaining({ + name: 'RestoreTransactionError', + phase: 'cleanup', + committed: true, + snapshotDir: expect.stringContaining('.restore-transaction-test-transaction'), + })); + expect(JSON.parse(readLive('runtime.json')).webuiPort).toBe(5105); + expect(fs.existsSync((thrown as RestoreTransactionError).snapshotDir!)).toBe(true); + }); + + it('never removes an existing transaction directory when id allocation collides', () => { + writeLive('runtime.json', '{"old":"runtime"}'); + const existing = path.join(root, '.restore-transaction-test-transaction'); + fs.mkdirSync(path.join(existing, 'snapshot'), { recursive: true }); + const marker = path.join(existing, 'snapshot', 'manual-recovery-marker'); + fs.writeFileSync(marker, 'keep me'); + + expect(() => run({ 'runtime.json': runtime(5106) })).toThrow(RestoreTransactionError); + + expect(fs.readFileSync(marker, 'utf8')).toBe('keep me'); + expect(readLive('runtime.json')).toBe('{"old":"runtime"}'); + }); + + it('keeps the original stage failure file and both causes when cleanup also fails', () => { + writeLive('runtime.json', '{"old":"runtime"}'); + writeLive('notifications.json', '{"old":"notifications"}'); + + let thrown: unknown; + try { + run({ + 'runtime.json': runtime(5107), + 'notifications.json': notifications(30), + }, (point) => { + if (point.phase === 'stage' && point.operation === 'write-staged' && point.file === 'notifications.json') { + throw new Error('stage-root-cause'); + } + if (point.phase === 'cleanup') throw new Error('cleanup-root-cause'); + }); + } catch (error) { + thrown = error; + } + + expect(thrown).toEqual(expect.objectContaining({ + phase: 'cleanup', + committed: false, + failedFiles: ['notifications.json'], + })); + const causes = ((thrown as RestoreTransactionError).cause as AggregateError).errors; + expect(causes.map((error: unknown) => (error as Error).message)).toEqual(['stage-root-cause', 'cleanup-root-cause']); + }); + + it('keeps the failed commit file and both causes when post-rollback cleanup fails', () => { + writeLive('runtime.json', '{"old":"runtime"}'); + writeLive('notifications.json', '{"old":"notifications"}'); + + let thrown: unknown; + try { + run({ + 'runtime.json': runtime(5108), + 'notifications.json': notifications(30), + }, (point) => { + if (point.phase === 'commit' && point.file === 'notifications.json') throw new Error('commit-root-cause'); + if (point.phase === 'cleanup') throw new Error('cleanup-root-cause'); + }); + } catch (error) { + thrown = error; + } + + expect(thrown).toEqual(expect.objectContaining({ + phase: 'cleanup', + rollbackSucceeded: true, + committed: false, + failedFiles: ['notifications.json'], + })); + const causes = ((thrown as RestoreTransactionError).cause as AggregateError).errors; + expect(causes.map((error: unknown) => (error as Error).message)).toEqual(['commit-root-cause', 'cleanup-root-cause']); + }); +});