feat(video): stream video uploads from disk instead of buffering (phase 4)

Wire the streaming primitives into the video send path. `loadVideo` now stages
the source onto a local disk path (stageSourceToDisk — hardlink/download),
hashes it in one streaming pass (hashFileStreaming), and returns a `fileSource`
instead of a full in-memory buffer; the main video sub-file PUTs through a
`FileChunkSource`, so a multi-GiB video never holds more than one 1 MiB chunk in
RAM. The thumbnail and the fingerprint (noByteFallback) fast path stay
byte-based and unchanged.

`runPuts` now sizes its fast-path / fast-only checks off `fileSource ?? bytes`
and resolves the highway session before opening the file handle (the handle's
owner, uploadHighwayHttp, must be the next call so its finally-close always
runs). A size+mtime guard re-checks the staged file before the PUT so a source
mutated mid-send fails cleanly rather than shipping bytes that disagree with the
already-sent hashes. The staged temp is released in every exit path.

Output is byte-identical to the buffered path (the OIDB fileInfo md5/sha1/size
and the uploaded bytes are unchanged); the old readFileSync-based
stageVideoSource is removed. Tests cover the disk path (fileSource not bytes,
streamed hash == buffered hash), cleanup on upload failure, and empty sources.
This commit is contained in:
DrSmoothl
2026-07-05 03:23:19 +08:00
parent 0e4d958b5f
commit e83eb80f2c
3 changed files with 148 additions and 82 deletions
@@ -10,6 +10,12 @@
// always provides bytes), and the video-specific OIDB fields.
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { promises as fsp } from 'fs';
import fs from 'fs';
import os from 'os';
import path from 'path';
import { pathToFileURL } from 'url';
import { computeHashes } from '@snowluma/protocol/highway/utils';
vi.mock('@snowluma/protocol/highway/pipeline', () => ({
runNtv2Upload: vi.fn(async () => ({ msgInfo: { msgInfoBody: [], extBizInfo: {} } })),
@@ -158,3 +164,69 @@ describe('video-upload', () => {
expect(args[1]).toBeUndefined();
});
});
describe('video-upload — streaming path (real on-disk source)', () => {
beforeEach(() => { vi.mocked(pipeline.runNtv2Upload).mockClear(); });
it('stages + stream-hashes the source and PUTs via fileSource, never buffering bytes', async () => {
const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'sl-vidstream-'));
const file = path.join(dir, 'clip.mp4');
const data = Buffer.alloc(3 * 1024 * 1024 + 77); // multi-1MiB-block + tail
for (let i = 0; i < data.length; i++) data[i] = (i * 37 + 5) & 0xff;
fs.writeFileSync(file, data);
try {
const bridge = { identity: { uin: '10001' } } as any;
await uploadVideoMsgInfo(bridge, true, 12345, { type: 'video', url: pathToFileURL(file).href } as any);
const args = vi.mocked(pipeline.runNtv2Upload).mock.calls.at(-1)![0];
const main = args.uploads[0]!;
// Main file streams from disk — fileSource set, no in-memory bytes.
expect(main.fileSource).toBeDefined();
expect(main.fileSource!.fileSize).toBe(data.length);
expect(main.bytes.length).toBe(0);
// The OIDB fileInfo carries the STREAMED hashes/size — must equal the
// buffered hash of the same bytes (byte-identical guarantee end-to-end).
const fi = (args.uploadInfo[0] as any).fileInfo;
expect(fi.fileSize).toBe(data.length);
expect(fi.fileHash).toBe(computeHashes(data).md5Hex);
expect(fi.fileSha1).toBe(computeHashes(data).sha1Hex);
// The staged temp is a copy/hardlink under the stage dir (NOT the source)
// and is cleaned up after the send.
expect(main.fileSource!.filePath).not.toBe(file);
expect(fs.existsSync(main.fileSource!.filePath)).toBe(false);
} finally {
await fsp.rm(dir, { recursive: true, force: true });
}
});
it('cleans up the staged temp even when the upload throws', async () => {
const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'sl-vidfail-'));
const file = path.join(dir, 'clip.mp4');
fs.writeFileSync(file, Buffer.alloc(2048, 7));
try {
vi.mocked(pipeline.runNtv2Upload).mockRejectedValueOnce(new Error('upload boom'));
await expect(
uploadVideoMsgInfo({ identity: { uin: '1' } } as any, true, 12345, { type: 'video', url: pathToFileURL(file).href } as any),
).rejects.toThrow(/upload boom/);
// The staged temp created in loadVideo must be removed by the finally.
const args = vi.mocked(pipeline.runNtv2Upload).mock.calls.at(-1)![0];
const stagedPath = args.uploads[0]!.fileSource!.filePath;
expect(fs.existsSync(stagedPath)).toBe(false);
} finally {
await fsp.rm(dir, { recursive: true, force: true });
}
});
it('rejects an empty video source (0 bytes)', async () => {
const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'sl-vidempty-'));
const file = path.join(dir, 'empty.mp4');
fs.writeFileSync(file, new Uint8Array(0));
try {
await expect(
uploadVideoMsgInfo({ identity: { uin: '1' } } as any, true, 12345, { type: 'video', url: pathToFileURL(file).href } as any),
).rejects.toThrow(/empty/);
} finally {
await fsp.rm(dir, { recursive: true, force: true });
}
});
});
+25 -6
View File
@@ -13,7 +13,7 @@ import { protobuf_decode, protobuf_encode } from '@snowluma/proton';
import crypto from 'crypto';
import type { BridgeContext } from '../bridge-context';
import { makeOidbEnvelope } from '../bridge-oidb';
import { BufferChunkSource, buildHighwayExtend, fetchHighwaySession, uploadHighwayHttp } from './highway-client';
import { BufferChunkSource, FileChunkSource, buildHighwayExtend, fetchHighwaySession, uploadHighwayHttp } from './highway-client';
const moduleLog = createLogger('Highway');
@@ -37,8 +37,13 @@ export interface MediaSubFileUpload {
cmdId: number;
/** Bytes to upload. Empty when the caller is forwarding from cached
* fingerprints; in that case set fastOnlyError so we throw with a
* typed message when the server actually demands the bytes. */
* typed message when the server actually demands the bytes. Also empty
* when `fileSource` is set (the bytes are streamed from disk instead). */
bytes: Uint8Array;
/** When set, this sub-file streams from a disk file instead of `bytes`
* (which should be empty). runPuts opens a `FileChunkSource`; all size /
* data-presence decisions use `fileSource.fileSize`, not `bytes.length`. */
fileSource?: { filePath: string; fileSize: number };
/** md5 used for the highway request. */
md5: Uint8Array;
/** sha1 — single buffer or per-1MB block array. Passed verbatim to
@@ -191,17 +196,22 @@ export async function runNtv2Upload(params: NtV2UploadParams): Promise<NTV2Uploa
for (const sub of uploads) {
const target = sub.source === 'top' ? upload : upload.subFileInfos?.[sub.source];
const uKey = target?.uKey ?? '';
// Data size / presence comes from fileSource (streamed) when set, else
// from the in-memory bytes — a streamed sub-file has empty `bytes`, so
// keying the checks below on `bytes.length` would wrongly treat it as a
// fast-only / empty sub-file.
const subSize = sub.fileSource ? sub.fileSource.fileSize : sub.bytes.length;
// No uKey: the server fast-pathed this sub-file (or msgInfo is absent
// entirely) — it already holds (or claims to hold) the resource, so
// there are no bytes to push for it.
if (!uKey || !upload.msgInfo) {
if (!uKey && upload.msgInfo && sub.bytes.length > 0) {
if (!uKey && upload.msgInfo && subSize > 0) {
log.debug('%s fast-upload hit for sub=%s (server reusing cached resource)', label, String(sub.source));
}
continue;
}
if (sub.bytes.length === 0) {
if (subSize === 0) {
if (sub.fastOnlyError) throw new Error(sub.fastOnlyError);
continue;
}
@@ -215,9 +225,18 @@ export async function runNtv2Upload(params: NtV2UploadParams): Promise<NTV2Uploa
sub.sha1,
sub.subFileIndex ?? 0,
);
log.debug('%s OIDB requires bytes, PUT %d bytes (sub=%s)', label, sub.bytes.length, String(sub.source));
// Resolve the (lazy, cached) session BEFORE opening the ChunkSource, so
// there is no fallible await between opening the FileChunkSource handle
// and the uploadHighwayHttp call that owns closing it — otherwise a
// session-fetch failure on the first PUT would leak the open handle.
const putSession = await getSession();
// uploadHighwayHttp owns the ChunkSource and closes it exactly once.
const chunkSource = sub.fileSource
? await FileChunkSource.open(sub.fileSource.filePath, sub.fileSource.fileSize)
: new BufferChunkSource(sub.bytes);
log.debug('%s OIDB requires bytes, PUT %d bytes (sub=%s)', label, subSize, String(sub.source));
const t0 = Date.now();
await uploadHighwayHttp(bridge, await getSession(), sub.cmdId, new BufferChunkSource(sub.bytes), sub.md5, extend);
await uploadHighwayHttp(bridge, putSession, sub.cmdId, chunkSource, sub.md5, extend);
log.debug('%s PUT done in %dms', label, Date.now() - t0);
didPut = true;
}
+51 -76
View File
@@ -1,7 +1,6 @@
import crypto from 'crypto';
import fs from 'fs';
import os from 'os';
import path from 'path';
import { promises as fsp } from 'fs';
import { createLogger } from '@snowluma/common/logger';
import type { BridgeContext } from '../bridge-context';
import type { MessageElement } from '../events';
@@ -18,6 +17,8 @@ import {
loadBinarySource,
resolveLocalFilePath,
} from './utils';
import { stageSourceToDisk } from './stage';
import { hashFileStreaming } from './hash-file';
import { Sha1Stream } from './sha1-stream';
const moduleLog = createLogger('Highway.Video');
@@ -61,15 +62,18 @@ const FALLBACK_THUMB = Buffer.from(
);
interface VideoPayload {
/** Video bytes. Empty when forwarding from cached fingerprints. */
/** Main-file bytes. Empty on the normal path (streamed from `fileSource`)
* and when forwarding from cached fingerprints (fastOnly). */
bytes: Uint8Array;
/** Normal path: the staged video streams from this disk file instead of
* buffering `bytes`. Absent for the fingerprint fast path. */
fileSource?: { filePath: string; fileSize: number };
md5: Uint8Array;
sha1: Uint8Array;
sha1Blocks: Uint8Array[];
md5Hex: string;
sha1Hex: string;
fileName: string;
filePath: string;
fileSize: number;
width: number;
height: number;
@@ -81,7 +85,12 @@ interface VideoPayload {
* always present (FALLBACK_THUMB at worst) so its sub-file uploads
* normally regardless. */
fastOnly: boolean;
cleanups: Array<() => void>;
/** Baseline stat of the staged file captured before hashing; re-checked
* before the upload to fail cleanly if the source was mutated mid-send.
* Absent for the fingerprint fast path. */
guard?: { size: number; mtimeMs: number };
/** Release the staged temp (hardlink / download). Best-effort, idempotent. */
cleanup: () => Promise<void>;
}
function makeFallbackThumb(): ThumbPayload {
@@ -107,7 +116,6 @@ function videoPayloadFromFingerprint(element: MessageElement): VideoPayload {
md5Hex: element.md5Hex ?? '',
sha1Hex: element.sha1Hex ?? '',
fileName: element.fileName || `${element.md5Hex ?? 'video'}.mp4`,
filePath: '',
fileSize: element.fileSize ?? 0,
width: element.width ?? 0,
height: element.height ?? 0,
@@ -115,7 +123,7 @@ function videoPayloadFromFingerprint(element: MessageElement): VideoPayload {
videoFormat: element.videoFormat ?? 0,
thumb: makeFallbackThumb(),
fastOnly: true,
cleanups: [],
cleanup: async () => { /* nothing staged */ },
};
}
@@ -154,54 +162,7 @@ export function computeVideoSha1Blocks(bytes: Uint8Array): Uint8Array[] {
return blocks;
}
// ─────────────── source staging + thumb extraction ───────────────
function defaultVideoTempDir(): string {
return path.join(os.tmpdir(), 'snowluma-video');
}
function sourceExtension(fileName: string, source: string): string {
const fromName = path.extname(fileName);
if (fromName) return fromName;
const local = resolveLocalFilePath(source);
const fromSource = local ? path.extname(local) : '';
return fromSource || '.mp4';
}
async function stageVideoSource(element: MessageElement, tempDir: string, cleanups: Array<() => void>): Promise<{
bytes: Uint8Array;
filePath: string;
fileName: string;
}> {
const source = element.url || element.fileId || '';
if (!source) throw new Error('video source is empty');
const local = resolveLocalFilePath(source);
if (local && fs.existsSync(local)) {
const stat = fs.statSync(local);
if (stat.size > MAX_VIDEO_SIZE) {
throw new Error(`video file too large: ${(stat.size / (1024 * 1024)).toFixed(2)} MB > ${MAX_VIDEO_SIZE / (1024 * 1024)} MB`);
}
return {
bytes: new Uint8Array(fs.readFileSync(local)),
filePath: local,
fileName: element.fileName || path.basename(local),
};
}
const loaded = await loadBinarySource(source, 'video', MAX_VIDEO_SIZE);
const fileName = element.fileName || loaded.fileName || '';
const stagedPath = path.join(tempDir, `snowluma-video-in-${crypto.randomUUID()}${sourceExtension(fileName, source)}`);
fs.writeFileSync(stagedPath, Buffer.from(loaded.bytes));
cleanups.push(() => { try { fs.unlinkSync(stagedPath); } catch { /* ignore */ } });
return {
bytes: loaded.bytes,
filePath: stagedPath,
fileName,
};
}
// ─────────────── thumb extraction ───────────────
async function loadThumb(element: MessageElement, videoPath: string): Promise<{
thumb: ThumbPayload;
@@ -270,42 +231,44 @@ async function loadVideo(element: MessageElement): Promise<VideoPayload> {
return videoPayloadFromFingerprint(element);
}
const tempDir = defaultVideoTempDir();
const cleanups: Array<() => void> = [];
fs.mkdirSync(tempDir, { recursive: true });
const source = element.url || element.fileId || '';
if (!source) throw new Error('video source is empty');
// Stage onto a local disk path (hardlink / streamed download) and hash it in
// one streaming pass — the video is never fully buffered in RAM. maxBytes =
// MAX_VIDEO_SIZE; oversize sources throw here and the OneBot layer re-routes
// them to the file pipeline (matches the message-actions size fallback).
const staged = await stageSourceToDisk(source, MAX_VIDEO_SIZE);
try {
const staged = await stageVideoSource(element, tempDir, cleanups);
if (staged.bytes.length === 0) throw new Error('video file is empty');
if (staged.bytes.length > MAX_VIDEO_SIZE) {
throw new Error(`video file too large: ${(staged.bytes.length / (1024 * 1024)).toFixed(2)} MB > ${MAX_VIDEO_SIZE / (1024 * 1024)} MB`);
}
if (staged.fileSize === 0) throw new Error('video file is empty');
const hashes = computeHashes(staged.bytes);
// Baseline stat captured BEFORE hashing; re-checked before the upload so a
// source mutated mid-send fails cleanly (see uploadVideoMsgInfo).
const g = await fsp.stat(staged.filePath);
const hashes = await hashFileStreaming(staged.filePath);
const { thumb, width, height, duration } = await loadThumb(element, staged.filePath);
return {
bytes: staged.bytes,
bytes: new Uint8Array(0),
fileSource: { filePath: staged.filePath, fileSize: staged.fileSize },
md5: hashes.md5,
sha1: hashes.sha1,
sha1Blocks: computeVideoSha1Blocks(staged.bytes),
sha1Blocks: hashes.sha1Blocks,
md5Hex: hashes.md5Hex,
sha1Hex: hashes.sha1Hex,
fileName: staged.fileName || `${hashes.md5Hex}.mp4`,
filePath: staged.filePath,
fileSize: staged.bytes.length,
fileName: (element.fileName || staged.fileName) || `${hashes.md5Hex}.mp4`,
fileSize: staged.fileSize,
width,
height,
duration,
videoFormat: 0,
thumb,
fastOnly: false,
cleanups: [...cleanups],
guard: { size: g.size, mtimeMs: g.mtimeMs },
cleanup: staged.cleanup,
};
} catch (err) {
for (const fn of cleanups.reverse()) {
try { fn(); } catch { /* best-effort cleanup */ }
}
await staged.cleanup();
throw err;
}
}
@@ -338,6 +301,7 @@ export async function uploadVideoMsgInfo(
source: 'top',
cmdId: isGroup ? GROUP_VIDEO_CMD_ID : PRIVATE_VIDEO_CMD_ID,
bytes: video.bytes,
fileSource: video.fileSource, // streamed from disk (undefined on the fingerprint fast path)
md5: video.md5,
sha1: video.sha1Blocks,
subFileIndex: 0,
@@ -354,6 +318,19 @@ export async function uploadVideoMsgInfo(
},
];
// Mutation guard: the streaming path reads the staged file twice (hash pass
// in loadVideo, then the highway PUT). For a hardlinked local source a
// concurrent in-place write would make the uploaded bytes disagree with the
// hashes we already sent. Re-stat before the PUT and fail cleanly if the
// file changed since the baseline (the server's per-chunk md5 check is the
// backstop during the PUT itself).
if (video.guard && video.fileSource) {
const now = await fsp.stat(video.fileSource.filePath);
if (now.size !== video.guard.size || now.mtimeMs !== video.guard.mtimeMs) {
throw new Error('video source changed during send (mutated between hashing and upload)');
}
}
const upload = await runNtv2Upload({
bridge,
isGroup,
@@ -436,8 +413,6 @@ export async function uploadVideoMsgInfo(
log.debug('video upload completed: md5=%s scene=%s', video.md5Hex, isGroup ? 'group' : 'c2c');
return finalizeMediaMsgInfo(upload);
} finally {
for (const fn of video.cleanups) {
try { fn(); } catch { /* best-effort cleanup */ }
}
await video.cleanup();
}
}