feat(video): stream video uploads from disk instead of buffering (phase 4)
Wire the streaming primitives into the video send path. `loadVideo` now stages the source onto a local disk path (stageSourceToDisk — hardlink/download), hashes it in one streaming pass (hashFileStreaming), and returns a `fileSource` instead of a full in-memory buffer; the main video sub-file PUTs through a `FileChunkSource`, so a multi-GiB video never holds more than one 1 MiB chunk in RAM. The thumbnail and the fingerprint (noByteFallback) fast path stay byte-based and unchanged. `runPuts` now sizes its fast-path / fast-only checks off `fileSource ?? bytes` and resolves the highway session before opening the file handle (the handle's owner, uploadHighwayHttp, must be the next call so its finally-close always runs). A size+mtime guard re-checks the staged file before the PUT so a source mutated mid-send fails cleanly rather than shipping bytes that disagree with the already-sent hashes. The staged temp is released in every exit path. Output is byte-identical to the buffered path (the OIDB fileInfo md5/sha1/size and the uploaded bytes are unchanged); the old readFileSync-based stageVideoSource is removed. Tests cover the disk path (fileSource not bytes, streamed hash == buffered hash), cleanup on upload failure, and empty sources.
This commit is contained in:
@@ -10,6 +10,12 @@
|
||||
// always provides bytes), and the video-specific OIDB fields.
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { promises as fsp } from 'fs';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { pathToFileURL } from 'url';
|
||||
import { computeHashes } from '@snowluma/protocol/highway/utils';
|
||||
|
||||
vi.mock('@snowluma/protocol/highway/pipeline', () => ({
|
||||
runNtv2Upload: vi.fn(async () => ({ msgInfo: { msgInfoBody: [], extBizInfo: {} } })),
|
||||
@@ -158,3 +164,69 @@ describe('video-upload', () => {
|
||||
expect(args[1]).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('video-upload — streaming path (real on-disk source)', () => {
|
||||
beforeEach(() => { vi.mocked(pipeline.runNtv2Upload).mockClear(); });
|
||||
|
||||
it('stages + stream-hashes the source and PUTs via fileSource, never buffering bytes', async () => {
|
||||
const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'sl-vidstream-'));
|
||||
const file = path.join(dir, 'clip.mp4');
|
||||
const data = Buffer.alloc(3 * 1024 * 1024 + 77); // multi-1MiB-block + tail
|
||||
for (let i = 0; i < data.length; i++) data[i] = (i * 37 + 5) & 0xff;
|
||||
fs.writeFileSync(file, data);
|
||||
try {
|
||||
const bridge = { identity: { uin: '10001' } } as any;
|
||||
await uploadVideoMsgInfo(bridge, true, 12345, { type: 'video', url: pathToFileURL(file).href } as any);
|
||||
|
||||
const args = vi.mocked(pipeline.runNtv2Upload).mock.calls.at(-1)![0];
|
||||
const main = args.uploads[0]!;
|
||||
// Main file streams from disk — fileSource set, no in-memory bytes.
|
||||
expect(main.fileSource).toBeDefined();
|
||||
expect(main.fileSource!.fileSize).toBe(data.length);
|
||||
expect(main.bytes.length).toBe(0);
|
||||
// The OIDB fileInfo carries the STREAMED hashes/size — must equal the
|
||||
// buffered hash of the same bytes (byte-identical guarantee end-to-end).
|
||||
const fi = (args.uploadInfo[0] as any).fileInfo;
|
||||
expect(fi.fileSize).toBe(data.length);
|
||||
expect(fi.fileHash).toBe(computeHashes(data).md5Hex);
|
||||
expect(fi.fileSha1).toBe(computeHashes(data).sha1Hex);
|
||||
// The staged temp is a copy/hardlink under the stage dir (NOT the source)
|
||||
// and is cleaned up after the send.
|
||||
expect(main.fileSource!.filePath).not.toBe(file);
|
||||
expect(fs.existsSync(main.fileSource!.filePath)).toBe(false);
|
||||
} finally {
|
||||
await fsp.rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('cleans up the staged temp even when the upload throws', async () => {
|
||||
const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'sl-vidfail-'));
|
||||
const file = path.join(dir, 'clip.mp4');
|
||||
fs.writeFileSync(file, Buffer.alloc(2048, 7));
|
||||
try {
|
||||
vi.mocked(pipeline.runNtv2Upload).mockRejectedValueOnce(new Error('upload boom'));
|
||||
await expect(
|
||||
uploadVideoMsgInfo({ identity: { uin: '1' } } as any, true, 12345, { type: 'video', url: pathToFileURL(file).href } as any),
|
||||
).rejects.toThrow(/upload boom/);
|
||||
// The staged temp created in loadVideo must be removed by the finally.
|
||||
const args = vi.mocked(pipeline.runNtv2Upload).mock.calls.at(-1)![0];
|
||||
const stagedPath = args.uploads[0]!.fileSource!.filePath;
|
||||
expect(fs.existsSync(stagedPath)).toBe(false);
|
||||
} finally {
|
||||
await fsp.rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects an empty video source (0 bytes)', async () => {
|
||||
const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'sl-vidempty-'));
|
||||
const file = path.join(dir, 'empty.mp4');
|
||||
fs.writeFileSync(file, new Uint8Array(0));
|
||||
try {
|
||||
await expect(
|
||||
uploadVideoMsgInfo({ identity: { uin: '1' } } as any, true, 12345, { type: 'video', url: pathToFileURL(file).href } as any),
|
||||
).rejects.toThrow(/empty/);
|
||||
} finally {
|
||||
await fsp.rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -13,7 +13,7 @@ import { protobuf_decode, protobuf_encode } from '@snowluma/proton';
|
||||
import crypto from 'crypto';
|
||||
import type { BridgeContext } from '../bridge-context';
|
||||
import { makeOidbEnvelope } from '../bridge-oidb';
|
||||
import { BufferChunkSource, buildHighwayExtend, fetchHighwaySession, uploadHighwayHttp } from './highway-client';
|
||||
import { BufferChunkSource, FileChunkSource, buildHighwayExtend, fetchHighwaySession, uploadHighwayHttp } from './highway-client';
|
||||
|
||||
const moduleLog = createLogger('Highway');
|
||||
|
||||
@@ -37,8 +37,13 @@ export interface MediaSubFileUpload {
|
||||
cmdId: number;
|
||||
/** Bytes to upload. Empty when the caller is forwarding from cached
|
||||
* fingerprints; in that case set fastOnlyError so we throw with a
|
||||
* typed message when the server actually demands the bytes. */
|
||||
* typed message when the server actually demands the bytes. Also empty
|
||||
* when `fileSource` is set (the bytes are streamed from disk instead). */
|
||||
bytes: Uint8Array;
|
||||
/** When set, this sub-file streams from a disk file instead of `bytes`
|
||||
* (which should be empty). runPuts opens a `FileChunkSource`; all size /
|
||||
* data-presence decisions use `fileSource.fileSize`, not `bytes.length`. */
|
||||
fileSource?: { filePath: string; fileSize: number };
|
||||
/** md5 used for the highway request. */
|
||||
md5: Uint8Array;
|
||||
/** sha1 — single buffer or per-1MB block array. Passed verbatim to
|
||||
@@ -191,17 +196,22 @@ export async function runNtv2Upload(params: NtV2UploadParams): Promise<NTV2Uploa
|
||||
for (const sub of uploads) {
|
||||
const target = sub.source === 'top' ? upload : upload.subFileInfos?.[sub.source];
|
||||
const uKey = target?.uKey ?? '';
|
||||
// Data size / presence comes from fileSource (streamed) when set, else
|
||||
// from the in-memory bytes — a streamed sub-file has empty `bytes`, so
|
||||
// keying the checks below on `bytes.length` would wrongly treat it as a
|
||||
// fast-only / empty sub-file.
|
||||
const subSize = sub.fileSource ? sub.fileSource.fileSize : sub.bytes.length;
|
||||
// No uKey: the server fast-pathed this sub-file (or msgInfo is absent
|
||||
// entirely) — it already holds (or claims to hold) the resource, so
|
||||
// there are no bytes to push for it.
|
||||
if (!uKey || !upload.msgInfo) {
|
||||
if (!uKey && upload.msgInfo && sub.bytes.length > 0) {
|
||||
if (!uKey && upload.msgInfo && subSize > 0) {
|
||||
log.debug('%s fast-upload hit for sub=%s (server reusing cached resource)', label, String(sub.source));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (sub.bytes.length === 0) {
|
||||
if (subSize === 0) {
|
||||
if (sub.fastOnlyError) throw new Error(sub.fastOnlyError);
|
||||
continue;
|
||||
}
|
||||
@@ -215,9 +225,18 @@ export async function runNtv2Upload(params: NtV2UploadParams): Promise<NTV2Uploa
|
||||
sub.sha1,
|
||||
sub.subFileIndex ?? 0,
|
||||
);
|
||||
log.debug('%s OIDB requires bytes, PUT %d bytes (sub=%s)', label, sub.bytes.length, String(sub.source));
|
||||
// Resolve the (lazy, cached) session BEFORE opening the ChunkSource, so
|
||||
// there is no fallible await between opening the FileChunkSource handle
|
||||
// and the uploadHighwayHttp call that owns closing it — otherwise a
|
||||
// session-fetch failure on the first PUT would leak the open handle.
|
||||
const putSession = await getSession();
|
||||
// uploadHighwayHttp owns the ChunkSource and closes it exactly once.
|
||||
const chunkSource = sub.fileSource
|
||||
? await FileChunkSource.open(sub.fileSource.filePath, sub.fileSource.fileSize)
|
||||
: new BufferChunkSource(sub.bytes);
|
||||
log.debug('%s OIDB requires bytes, PUT %d bytes (sub=%s)', label, subSize, String(sub.source));
|
||||
const t0 = Date.now();
|
||||
await uploadHighwayHttp(bridge, await getSession(), sub.cmdId, new BufferChunkSource(sub.bytes), sub.md5, extend);
|
||||
await uploadHighwayHttp(bridge, putSession, sub.cmdId, chunkSource, sub.md5, extend);
|
||||
log.debug('%s PUT done in %dms', label, Date.now() - t0);
|
||||
didPut = true;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import crypto from 'crypto';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { promises as fsp } from 'fs';
|
||||
import { createLogger } from '@snowluma/common/logger';
|
||||
import type { BridgeContext } from '../bridge-context';
|
||||
import type { MessageElement } from '../events';
|
||||
@@ -18,6 +17,8 @@ import {
|
||||
loadBinarySource,
|
||||
resolveLocalFilePath,
|
||||
} from './utils';
|
||||
import { stageSourceToDisk } from './stage';
|
||||
import { hashFileStreaming } from './hash-file';
|
||||
import { Sha1Stream } from './sha1-stream';
|
||||
|
||||
const moduleLog = createLogger('Highway.Video');
|
||||
@@ -61,15 +62,18 @@ const FALLBACK_THUMB = Buffer.from(
|
||||
);
|
||||
|
||||
interface VideoPayload {
|
||||
/** Video bytes. Empty when forwarding from cached fingerprints. */
|
||||
/** Main-file bytes. Empty on the normal path (streamed from `fileSource`)
|
||||
* and when forwarding from cached fingerprints (fastOnly). */
|
||||
bytes: Uint8Array;
|
||||
/** Normal path: the staged video streams from this disk file instead of
|
||||
* buffering `bytes`. Absent for the fingerprint fast path. */
|
||||
fileSource?: { filePath: string; fileSize: number };
|
||||
md5: Uint8Array;
|
||||
sha1: Uint8Array;
|
||||
sha1Blocks: Uint8Array[];
|
||||
md5Hex: string;
|
||||
sha1Hex: string;
|
||||
fileName: string;
|
||||
filePath: string;
|
||||
fileSize: number;
|
||||
width: number;
|
||||
height: number;
|
||||
@@ -81,7 +85,12 @@ interface VideoPayload {
|
||||
* always present (FALLBACK_THUMB at worst) so its sub-file uploads
|
||||
* normally regardless. */
|
||||
fastOnly: boolean;
|
||||
cleanups: Array<() => void>;
|
||||
/** Baseline stat of the staged file captured before hashing; re-checked
|
||||
* before the upload to fail cleanly if the source was mutated mid-send.
|
||||
* Absent for the fingerprint fast path. */
|
||||
guard?: { size: number; mtimeMs: number };
|
||||
/** Release the staged temp (hardlink / download). Best-effort, idempotent. */
|
||||
cleanup: () => Promise<void>;
|
||||
}
|
||||
|
||||
function makeFallbackThumb(): ThumbPayload {
|
||||
@@ -107,7 +116,6 @@ function videoPayloadFromFingerprint(element: MessageElement): VideoPayload {
|
||||
md5Hex: element.md5Hex ?? '',
|
||||
sha1Hex: element.sha1Hex ?? '',
|
||||
fileName: element.fileName || `${element.md5Hex ?? 'video'}.mp4`,
|
||||
filePath: '',
|
||||
fileSize: element.fileSize ?? 0,
|
||||
width: element.width ?? 0,
|
||||
height: element.height ?? 0,
|
||||
@@ -115,7 +123,7 @@ function videoPayloadFromFingerprint(element: MessageElement): VideoPayload {
|
||||
videoFormat: element.videoFormat ?? 0,
|
||||
thumb: makeFallbackThumb(),
|
||||
fastOnly: true,
|
||||
cleanups: [],
|
||||
cleanup: async () => { /* nothing staged */ },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -154,54 +162,7 @@ export function computeVideoSha1Blocks(bytes: Uint8Array): Uint8Array[] {
|
||||
return blocks;
|
||||
}
|
||||
|
||||
// ─────────────── source staging + thumb extraction ───────────────
|
||||
|
||||
function defaultVideoTempDir(): string {
|
||||
return path.join(os.tmpdir(), 'snowluma-video');
|
||||
}
|
||||
|
||||
function sourceExtension(fileName: string, source: string): string {
|
||||
const fromName = path.extname(fileName);
|
||||
if (fromName) return fromName;
|
||||
|
||||
const local = resolveLocalFilePath(source);
|
||||
const fromSource = local ? path.extname(local) : '';
|
||||
return fromSource || '.mp4';
|
||||
}
|
||||
|
||||
async function stageVideoSource(element: MessageElement, tempDir: string, cleanups: Array<() => void>): Promise<{
|
||||
bytes: Uint8Array;
|
||||
filePath: string;
|
||||
fileName: string;
|
||||
}> {
|
||||
const source = element.url || element.fileId || '';
|
||||
if (!source) throw new Error('video source is empty');
|
||||
|
||||
const local = resolveLocalFilePath(source);
|
||||
if (local && fs.existsSync(local)) {
|
||||
const stat = fs.statSync(local);
|
||||
if (stat.size > MAX_VIDEO_SIZE) {
|
||||
throw new Error(`video file too large: ${(stat.size / (1024 * 1024)).toFixed(2)} MB > ${MAX_VIDEO_SIZE / (1024 * 1024)} MB`);
|
||||
}
|
||||
return {
|
||||
bytes: new Uint8Array(fs.readFileSync(local)),
|
||||
filePath: local,
|
||||
fileName: element.fileName || path.basename(local),
|
||||
};
|
||||
}
|
||||
|
||||
const loaded = await loadBinarySource(source, 'video', MAX_VIDEO_SIZE);
|
||||
const fileName = element.fileName || loaded.fileName || '';
|
||||
const stagedPath = path.join(tempDir, `snowluma-video-in-${crypto.randomUUID()}${sourceExtension(fileName, source)}`);
|
||||
fs.writeFileSync(stagedPath, Buffer.from(loaded.bytes));
|
||||
cleanups.push(() => { try { fs.unlinkSync(stagedPath); } catch { /* ignore */ } });
|
||||
|
||||
return {
|
||||
bytes: loaded.bytes,
|
||||
filePath: stagedPath,
|
||||
fileName,
|
||||
};
|
||||
}
|
||||
// ─────────────── thumb extraction ───────────────
|
||||
|
||||
async function loadThumb(element: MessageElement, videoPath: string): Promise<{
|
||||
thumb: ThumbPayload;
|
||||
@@ -270,42 +231,44 @@ async function loadVideo(element: MessageElement): Promise<VideoPayload> {
|
||||
return videoPayloadFromFingerprint(element);
|
||||
}
|
||||
|
||||
const tempDir = defaultVideoTempDir();
|
||||
const cleanups: Array<() => void> = [];
|
||||
fs.mkdirSync(tempDir, { recursive: true });
|
||||
const source = element.url || element.fileId || '';
|
||||
if (!source) throw new Error('video source is empty');
|
||||
|
||||
// Stage onto a local disk path (hardlink / streamed download) and hash it in
|
||||
// one streaming pass — the video is never fully buffered in RAM. maxBytes =
|
||||
// MAX_VIDEO_SIZE; oversize sources throw here and the OneBot layer re-routes
|
||||
// them to the file pipeline (matches the message-actions size fallback).
|
||||
const staged = await stageSourceToDisk(source, MAX_VIDEO_SIZE);
|
||||
try {
|
||||
const staged = await stageVideoSource(element, tempDir, cleanups);
|
||||
if (staged.bytes.length === 0) throw new Error('video file is empty');
|
||||
if (staged.bytes.length > MAX_VIDEO_SIZE) {
|
||||
throw new Error(`video file too large: ${(staged.bytes.length / (1024 * 1024)).toFixed(2)} MB > ${MAX_VIDEO_SIZE / (1024 * 1024)} MB`);
|
||||
}
|
||||
if (staged.fileSize === 0) throw new Error('video file is empty');
|
||||
|
||||
const hashes = computeHashes(staged.bytes);
|
||||
// Baseline stat captured BEFORE hashing; re-checked before the upload so a
|
||||
// source mutated mid-send fails cleanly (see uploadVideoMsgInfo).
|
||||
const g = await fsp.stat(staged.filePath);
|
||||
const hashes = await hashFileStreaming(staged.filePath);
|
||||
const { thumb, width, height, duration } = await loadThumb(element, staged.filePath);
|
||||
|
||||
return {
|
||||
bytes: staged.bytes,
|
||||
bytes: new Uint8Array(0),
|
||||
fileSource: { filePath: staged.filePath, fileSize: staged.fileSize },
|
||||
md5: hashes.md5,
|
||||
sha1: hashes.sha1,
|
||||
sha1Blocks: computeVideoSha1Blocks(staged.bytes),
|
||||
sha1Blocks: hashes.sha1Blocks,
|
||||
md5Hex: hashes.md5Hex,
|
||||
sha1Hex: hashes.sha1Hex,
|
||||
fileName: staged.fileName || `${hashes.md5Hex}.mp4`,
|
||||
filePath: staged.filePath,
|
||||
fileSize: staged.bytes.length,
|
||||
fileName: (element.fileName || staged.fileName) || `${hashes.md5Hex}.mp4`,
|
||||
fileSize: staged.fileSize,
|
||||
width,
|
||||
height,
|
||||
duration,
|
||||
videoFormat: 0,
|
||||
thumb,
|
||||
fastOnly: false,
|
||||
cleanups: [...cleanups],
|
||||
guard: { size: g.size, mtimeMs: g.mtimeMs },
|
||||
cleanup: staged.cleanup,
|
||||
};
|
||||
} catch (err) {
|
||||
for (const fn of cleanups.reverse()) {
|
||||
try { fn(); } catch { /* best-effort cleanup */ }
|
||||
}
|
||||
await staged.cleanup();
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -338,6 +301,7 @@ export async function uploadVideoMsgInfo(
|
||||
source: 'top',
|
||||
cmdId: isGroup ? GROUP_VIDEO_CMD_ID : PRIVATE_VIDEO_CMD_ID,
|
||||
bytes: video.bytes,
|
||||
fileSource: video.fileSource, // streamed from disk (undefined on the fingerprint fast path)
|
||||
md5: video.md5,
|
||||
sha1: video.sha1Blocks,
|
||||
subFileIndex: 0,
|
||||
@@ -354,6 +318,19 @@ export async function uploadVideoMsgInfo(
|
||||
},
|
||||
];
|
||||
|
||||
// Mutation guard: the streaming path reads the staged file twice (hash pass
|
||||
// in loadVideo, then the highway PUT). For a hardlinked local source a
|
||||
// concurrent in-place write would make the uploaded bytes disagree with the
|
||||
// hashes we already sent. Re-stat before the PUT and fail cleanly if the
|
||||
// file changed since the baseline (the server's per-chunk md5 check is the
|
||||
// backstop during the PUT itself).
|
||||
if (video.guard && video.fileSource) {
|
||||
const now = await fsp.stat(video.fileSource.filePath);
|
||||
if (now.size !== video.guard.size || now.mtimeMs !== video.guard.mtimeMs) {
|
||||
throw new Error('video source changed during send (mutated between hashing and upload)');
|
||||
}
|
||||
}
|
||||
|
||||
const upload = await runNtv2Upload({
|
||||
bridge,
|
||||
isGroup,
|
||||
@@ -436,8 +413,6 @@ export async function uploadVideoMsgInfo(
|
||||
log.debug('video upload completed: md5=%s scene=%s', video.md5Hex, isGroup ? 'group' : 'c2c');
|
||||
return finalizeMediaMsgInfo(upload);
|
||||
} finally {
|
||||
for (const fn of video.cleanups) {
|
||||
try { fn(); } catch { /* best-effort cleanup */ }
|
||||
}
|
||||
await video.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user