refactor(protocol): reuse tested Sha1Stream in video block-sha1 (drop dup)

video-upload.ts 有一份 ~110 行手写、**无测试**的流式 SHA1(Sha1StreamState),与
sha1-stream.ts 里已充分测试的 Sha1Stream 逐字节相同(transform 数学等价、中间 state
小端)。删掉该重复,computeVideoSha1Blocks 改用 Sha1Stream 做 1MB 块的中间 state、
Node crypto 做整文件 final(与受测的 computeSha1StateV 同一套路;已实测旧 final() ===
Node crypto 整文件 sha1)。

- 新增 video-sha1-blocks.test.ts(4 例):golden 值从**旧实现**抓取,锁定重写 byte-
  identical(两个 1MB 中间 state + overall),并给这个原本无测试的函数永久覆盖。
- 删掉的 tail-update 只影响旧自实现 final,不动任何中间块(tail<64B 不触发 1MB 边界)。

protocol 565 全绿、typecheck 干净;对抗性 Agent 独立复现旧实现确认 golden 逐字节
相同、transform 数学等价无精度问题、无残留引用。
This commit is contained in:
DrSmoothl
2026-07-01 20:14:55 +08:00
parent c822c5dab6
commit ade76d3d2f
2 changed files with 62 additions and 131 deletions
+16 -131
View File
@@ -18,6 +18,7 @@ import {
loadBinarySource,
resolveLocalFilePath,
} from './utils';
import { Sha1Stream } from './sha1-stream';
const moduleLog = createLogger('Highway.Video');
@@ -35,6 +36,7 @@ export const GROUP_VIDEO_THUMB_CMD_ID = 1006;
export const MAX_VIDEO_SIZE = 100 * 1024 * 1024;
const MAX_VIDEO_SIZE_HARD = 1536 * 1024 * 1024;
const SHA1_STREAM_BLOCK_SIZE = 1024 * 1024;
const SHA1_BLOCK_SIZE = 64;
export function getVideoSourceSize(element: MessageElement): number | null {
if (element.fileSize && element.fileSize > 0) return element.fileSize;
@@ -123,143 +125,26 @@ interface ThumbPayload {
// ─────────────── 1MB-block sha1 (Highway main-video extend) ───────────────
// Highway expects sha1 computed over each 1 MB block of the file, plus
// the final overall sha1. This is a streaming implementation that doesn't
// reuse Node's crypto because Node only exposes the final digest.
class Sha1StreamState {
readonly blockSize = 64;
private readonly padding = Buffer.concat([Buffer.from([0x80]), Buffer.alloc(63)]);
private readonly state = new Uint32Array(5);
private readonly count = new Uint32Array(2);
private readonly buffer = Buffer.allocUnsafe(this.blockSize);
private readonly w = new Uint32Array(80);
constructor() {
this.reset();
}
private reset(): void {
this.state[0] = 0x67452301;
this.state[1] = 0xEFCDAB89;
this.state[2] = 0x98BADCFE;
this.state[3] = 0x10325476;
this.state[4] = 0xC3D2E1F0;
this.count[0] = 0;
this.count[1] = 0;
this.buffer.fill(0);
}
private rotateLeft(value: number, offset: number): number {
return ((value << offset) | (value >>> (32 - offset))) >>> 0;
}
private transform(chunk: Uint8Array, offset: number): void {
const view = new DataView(chunk.buffer, chunk.byteOffset + offset, this.blockSize);
for (let i = 0; i < 16; i++) {
this.w[i] = view.getUint32(i * 4, false);
}
for (let i = 16; i < 80; i++) {
this.w[i] = this.rotateLeft(this.w[i - 3] ^ this.w[i - 8] ^ this.w[i - 14] ^ this.w[i - 16], 1);
}
let a = this.state[0];
let b = this.state[1];
let c = this.state[2];
let d = this.state[3];
let e = this.state[4];
for (let i = 0; i < 80; i++) {
let temp: number;
if (i < 20) {
temp = ((b & c) | (~b & d)) + 0x5A827999;
} else if (i < 40) {
temp = (b ^ c ^ d) + 0x6ED9EBA1;
} else if (i < 60) {
temp = ((b & c) | (b & d) | (c & d)) + 0x8F1BBCDC;
} else {
temp = (b ^ c ^ d) + 0xCA62C1D6;
}
temp += (this.rotateLeft(a, 5) + e + this.w[i]) >>> 0;
e = d;
d = c;
c = this.rotateLeft(b, 30);
b = a;
a = temp >>> 0;
}
this.state[0] = (this.state[0] + a) >>> 0;
this.state[1] = (this.state[1] + b) >>> 0;
this.state[2] = (this.state[2] + c) >>> 0;
this.state[3] = (this.state[3] + d) >>> 0;
this.state[4] = (this.state[4] + e) >>> 0;
}
update(data: Uint8Array): void {
let index = (this.count[0] >>> 3) & 0x3F;
const dataLen = data.length;
this.count[0] = (this.count[0] + (dataLen << 3)) >>> 0;
if (this.count[0] < (dataLen << 3)) this.count[1] = (this.count[1] + 1) >>> 0;
this.count[1] = (this.count[1] + (dataLen >>> 29)) >>> 0;
const partLen = this.blockSize - index;
let i = 0;
if (dataLen >= partLen) {
this.buffer.set(data.subarray(0, partLen), index);
this.transform(this.buffer, 0);
for (i = partLen; i + this.blockSize <= dataLen; i += this.blockSize) {
this.transform(data, i);
}
index = 0;
}
if (i < dataLen) {
this.buffer.set(data.subarray(i, dataLen), index);
}
}
hash(bigEndian = true): Uint8Array {
const digest = Buffer.allocUnsafe(20);
for (let i = 0; i < 5; i++) {
if (bigEndian) digest.writeUInt32BE(this.state[i], i * 4);
else digest.writeUInt32LE(this.state[i], i * 4);
}
return new Uint8Array(digest);
}
final(): Uint8Array {
const bits = Buffer.allocUnsafe(8);
bits.writeUInt32BE(this.count[1], 0);
bits.writeUInt32BE(this.count[0], 4);
const index = (this.count[0] >>> 3) & 0x3F;
const padLen = index < 56 ? 56 - index : 120 - index;
this.update(this.padding.subarray(0, padLen));
this.update(bits);
return this.hash(true);
}
}
function computeVideoSha1Blocks(bytes: Uint8Array): Uint8Array[] {
const sha1 = new Sha1StreamState();
// Highway expects the sha1 of each 1 MB block (intermediate un-finalized state,
// little-endian) plus the overall sha1. Reuses the well-tested streaming
// Sha1Stream for the block states and Node crypto for the overall digest (same
// split as computeSha1StateV), replacing a ~110-line hand-rolled, untested
// SHA1 duplicate.
export function computeVideoSha1Blocks(bytes: Uint8Array): Uint8Array[] {
const sha1 = new Sha1Stream();
const blocks: Uint8Array[] = [];
let bytesRead = 0;
let offset = 0;
while (offset + sha1.blockSize <= bytes.length) {
const block = bytes.subarray(offset, offset + sha1.blockSize);
sha1.update(block);
offset += sha1.blockSize;
bytesRead += sha1.blockSize;
while (offset + SHA1_BLOCK_SIZE <= bytes.length) {
sha1.update(bytes.subarray(offset, offset + SHA1_BLOCK_SIZE));
offset += SHA1_BLOCK_SIZE;
bytesRead += SHA1_BLOCK_SIZE;
if (bytesRead % SHA1_STREAM_BLOCK_SIZE === 0) {
blocks.push(sha1.hash(false));
blocks.push(sha1.hash(true)); // little-endian intermediate state
}
}
if (offset < bytes.length) sha1.update(bytes.subarray(offset));
blocks.push(sha1.final());
// Overall SHA1 (finalized) via Node crypto — the reference impl.
blocks.push(new Uint8Array(crypto.createHash('sha1').update(Buffer.from(bytes)).digest()));
return blocks;
}
@@ -0,0 +1,46 @@
import { describe, it, expect } from 'vitest';
import { createHash } from 'node:crypto';
import { computeVideoSha1Blocks } from '../../src/highway/video-upload';
// Deterministic input generator (matches the golden-capture script).
function det(len: number): Uint8Array {
const b = new Uint8Array(len);
for (let i = 0; i < len; i++) b[i] = (i * 31 + 7) & 0xFF;
return b;
}
const hex = (u: Uint8Array): string => Buffer.from(u).toString('hex');
const nodeSha1 = (b: Uint8Array): string => createHash('sha1').update(Buffer.from(b)).digest('hex');
describe('computeVideoSha1Blocks', () => {
// GOLDEN: captured from the previous hand-rolled Sha1StreamState implementation
// before the F3 rewrite (Sha1Stream + Node crypto), so this locks the rewrite
// byte-identical for the block states AND the overall digest.
it('reproduces the pre-refactor golden 1MB-block states + overall sha1', () => {
const input = det(2 * 1024 * 1024 + 12345); // 2 full 1MB blocks + a partial tail
expect(computeVideoSha1Blocks(input).map(hex)).toEqual([
'ab9a20fd6c98fc1f8f0c985cd552ff14c6feacfd', // intermediate un-finalized state @1MB (little-endian)
'2ccc5923b75f46ebd8ccb78e8c413c18025178b9', // intermediate un-finalized state @2MB
'dd9cfd4648ab460260afa820c98bfec1b17655d4', // overall (finalized)
]);
});
it('the final block equals the standard full-file SHA1', () => {
const input = det(2 * 1024 * 1024 + 12345);
const blocks = computeVideoSha1Blocks(input);
expect(hex(blocks[blocks.length - 1]!)).toBe(nodeSha1(input));
});
it('a file shorter than 1MB yields just the overall sha1', () => {
const input = det(1000);
const blocks = computeVideoSha1Blocks(input);
expect(blocks).toHaveLength(1);
expect(hex(blocks[0]!)).toBe(nodeSha1(input));
});
it('a file that is an exact 1MB multiple emits the boundary state then the overall', () => {
const input = det(1024 * 1024);
const blocks = computeVideoSha1Blocks(input);
expect(blocks).toHaveLength(2);
expect(hex(blocks[1]!)).toBe(nodeSha1(input)); // last is the finalized overall
});
});