perf(highway): single-connection pipelined upload — supersede #211 concurrency

The #211 concurrency attempt (c17fff6) was disproven on a live endpoint.
QQ's highway server requires blocks to arrive in offset order and rejects
an out-of-order PUT with error_code=102902 — ~20% failure under a real
account even after deferring the finalize block. Parallel connections
can't guarantee arrival order, so that approach is unshippable.

Replace it with a single persistent connection that PUTs blocks in strict
offset order, reusing the socket across the whole file via HTTP keep-alive:
- In order: the server never sees an out-of-order block, so no 102902.
- One warm connection: the TCP congestion window stays open across the
  file instead of paying slow-start on a fresh connection per 1 MiB block.
  That per-block cold start was the real throughput cap behind #211.

Adaptive to QQ's edge nodes FIN'ing between chunks (the reason #118 went
one-connection-per-chunk): on a transport failure, or a peer half-close
after a response, drop the socket, reconnect, and re-send the SAME block
(re-PUTting an offset is idempotent). Worst case degrades to one
connection per block — no worse than #118, and still strictly in order.

Also fixes a latent bug on the reconnect path, surfaced by review: a
response truncated mid-stream (header arrived, Content-Length unmet, then
FIN) was resolved as a partial frame and then fatally rejected by
unpackHighwayFrame OUTSIDE the retry loop, turning a recoverable mid-body
FIN into a failed upload. A truncated response now rejects as a retryable
transport failure so the block is re-sent. Added an idle read timeout so
a silent peer can't hang the upload forever (the persistent socket widens
that exposure).

Verified live (tempserver, repeated 48 MiB group-file uploads): 1
connection per file, 0 failures across dense repeats, ~1.4x throughput
over the old per-chunk-connection serial path in a low-RTT datacenter —
the win is larger on high-RTT home links where per-block slow-start hurt
most (the original #211 report). Unit tests cover keep-alive reuse,
FIN-after-each-response degradation, mid-stream reconnect+resume, body-
truncation retry, in-order exactly-once coverage, and close-once.
This commit is contained in:
DrSmoothl
2026-07-09 13:55:25 +08:00
parent c17fff6e2f
commit 7ab91c81e9
2 changed files with 358 additions and 360 deletions
+215 -214
View File
@@ -1,23 +1,24 @@
// highway-client.uploadHighwayHttp 的连接生命周期测试。
//
// 回归覆盖:当上传数据 > HIGHWAY_BLOCK_SIZE (1 MiB) 时,旧实现会复用
// 同一个 TCP socket 在 keep-alive 模式下连续 POST,但 QQ highway 的
// 边缘节点经常在第一次响应后立刻 FIN 关闭,导致第二次 POST 报
// `connection closed before response`。用户上报的 1.19 MB 图片
// (onebot retcode=1200)就是这个分支。
// 方案 3(#211):单连接、按 offset 顺序、流水线上传,跨 chunk 复用同一个
// TCP 连接(HTTP keep-alive)。要点:
// - 顺序发送 → 天然保序,避开 QQ highway 对乱序块的 error_code=102902;
// - 复用连接 → TCP 拥塞窗口全程保持,省掉每块新建连接的 slow-start。
// 边缘节点可能在响应后 FIN(#118);此时自适应地丢弃 socket + 重连 + 重发
// 同一块(重发幂等),最坏退化成“每块一连接”,不比旧实现差,且仍然保序。
//
// 修复方案:每个 chunk 独立建一个新的 TCP 连接。本测试通过 mock
// `net.createConnection` 计数建连次数,断言每个 chunk 一连接的不变量。
// 本测试用两种 mock 服务端行为锁定这套自适应逻辑:
// - keepalive:每个 POST 回一个响应,连接保持 → 整个文件 1 个连接;
// - closeafter:每个响应后半关闭(readableEnded/emit close)→ 退化成每块
// 一连接,但仍全部成功。
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { EventEmitter } from 'events';
// highway-client 解析响应只关心 errorCode === 0 —— 一个完全空的
// RespDataHighwayHead protobuf 体(0 字节)会被解为 ,errorCode
// 是 undefined,走 falsy 通过分支。
//
// HTTP body 是 highway frame:
// 0x28 | head_len(BE32) | body_len(BE32) | head | body | 0x29
// RespDataHighwayHead protobuf 体(0 字节)会被解为 errorCode=undefined,
// 走 falsy 通过分支。
// highway frame:0x28 | head_len(BE32) | body_len(BE32) | head | body | 0x29
function buildEmptyHighwayResponseFrame(): Buffer {
const frame = Buffer.alloc(10);
frame[0] = 0x28;
@@ -27,59 +28,113 @@ function buildEmptyHighwayResponseFrame(): Buffer {
return frame;
}
function buildHttpResponse(frameBody: Buffer): Buffer {
const headers = [
'HTTP/1.1 200 OK',
`Content-Length: ${frameBody.length}`,
'Connection: close',
'',
'',
].join('\r\n');
return Buffer.concat([Buffer.from(headers, 'ascii'), frameBody]);
// errorCode 帧:RespDataHighwayHead.field3 (errorCode) = 921。tag(field3,
// varint)=0x18,varint(921)=0x99 0x07。定义即拒绝(不重试)。
function buildErrorCodeFrame(): Buffer {
const head = Buffer.from([0x18, 0x99, 0x07]);
const frame = Buffer.alloc(9 + head.length + 1);
frame[0] = 0x28;
frame.writeUInt32BE(head.length, 1);
frame.writeUInt32BE(0, 5);
head.copy(frame, 9);
frame[9 + head.length] = 0x29;
return frame;
}
// FakeSocket 模拟 net.Socket 的最小子集。highway-client 用到:
// - 'data' / 'error' / 'close' 事件
// - .write(data, cb) cb 会被 socketWrite 等待
// - .destroy()
// - .setTimeout / .once / .removeListener (继承自 EventEmitter)
function httpWrap(frame: Buffer): Buffer {
const headers = ['HTTP/1.1 200 OK', `Content-Length: ${frame.length}`, '', ''].join('\r\n');
return Buffer.concat([Buffer.from(headers, 'ascii'), frame]);
}
type ServerMode = 'keepalive' | 'closeafter';
// FakeSocket 模拟 net.Socket 的最小子集,支持在一个连接上顺序处理多个 POST。
// 累积写入的字节,按 `\r\n\r\n` + Content-Length 切出每个完整 POST,逐个回
// 响应。`mode` 控制响应后是否半关闭;`failOnPost` 用于模拟链路中途断开。
class FakeSocket extends EventEmitter {
destroyed = false;
// 每个 POST = 2 次 write(HTTP header + body)。第二次 write 落地后
// 异步回放响应并立刻 close —— 复现 QQ highway 真实行为:响应一发完
// 就 FIN,旧实现的下一次 POST 必然撞上已关闭的 socket。
private writeCount = 0;
private response: Buffer;
constructor(response: Buffer) {
readableEnded = false;
writable = true;
posts = 0;
private buf = Buffer.alloc(0);
constructor(
private readonly responseFrame: Buffer,
private readonly mode: ServerMode,
private readonly failOnPost = -1,
private readonly truncateOnPost = -1,
) {
super();
this.response = response;
}
setTimeout(_ms: number) { /* no-op */ }
setTimeout(_ms?: number) { /* no-op */ }
write(_data: unknown, cb?: (err?: Error) => void): boolean {
this.writeCount += 1;
if (this.writeCount === 2) {
setImmediate(() => {
this.emit('data', this.response);
this.emit('close');
});
write(data: unknown, cb?: (err?: Error) => void): boolean {
if (this.destroyed || !this.writable) {
const err = new Error('write after socket end');
if (cb) cb(err);
setImmediate(() => this.emit('error', err));
return false;
}
this.buf = Buffer.concat([this.buf, Buffer.from(data as Uint8Array)]);
this.drainPosts();
if (cb) cb();
return true;
}
private drainPosts(): void {
for (;;) {
const s = this.buf.toString('latin1');
const he = s.indexOf('\r\n\r\n');
if (he < 0) return;
const m = s.slice(0, he).match(/content-length:\s*(\d+)/i);
const cl = m ? parseInt(m[1], 10) : 0;
const total = he + 4 + cl;
if (this.buf.length < total) return;
this.buf = this.buf.subarray(total);
this.posts += 1;
const thisPost = this.posts;
setImmediate(() => {
if (this.destroyed) return;
if (thisPost === this.failOnPost) {
this.writable = false;
this.readableEnded = true;
this.emit('error', new Error('ECONNRESET'));
this.emit('close');
return;
}
if (thisPost === this.truncateOnPost) {
// Send the HTTP header + only part of the declared body, then FIN —
// a truncated response. Dropping the last 3 body bytes keeps the
// header intact so Content-Length parses but is never satisfied.
const full = httpWrap(this.responseFrame);
this.emit('data', full.subarray(0, full.length - 3));
this.writable = false;
this.readableEnded = true;
this.emit('close');
return;
}
this.emit('data', httpWrap(this.responseFrame));
if (this.mode === 'closeafter') {
this.readableEnded = true;
this.writable = false;
this.emit('end');
this.emit('close');
}
});
}
}
destroy() {
this.destroyed = true;
this.writable = false;
}
}
// vitest 把 vi.mock 工厂 hoist 到文件顶部,所以工厂里不能引用普通的
// 顶层 const —— 那会触发 "Cannot access X before initialization"。
// 用 vi.hoisted 让这两个值跟着 mock 一起被 hoist。
const { createdSockets, createConnectionMock } = vi.hoisted(() => ({
createdSockets: [] as FakeSocket[],
// vitest 把 vi.mock 工厂 hoist 到文件顶部,所以工厂里不能引用普通的顶层
// const。用 vi.hoisted 让这些值跟着 mock 一起 hoist。
const { createdSockets, socketFactoryRef, createConnectionMock } = vi.hoisted(() => ({
createdSockets: [] as unknown[],
socketFactoryRef: { make: null as null | ((idx: number) => unknown) },
createConnectionMock: vi.fn(),
}));
@@ -88,15 +143,18 @@ vi.mock('net', () => ({
createConnection: createConnectionMock,
}));
import { uploadHighwayHttp, BufferChunkSource, type ChunkSource, type HighwaySession } from '@snowluma/protocol/highway';
import {
uploadHighwayHttp,
BufferChunkSource,
type ChunkSource,
type HighwaySession,
} from '@snowluma/protocol/highway';
import type { BridgeContext } from '@snowluma/protocol/bridge-context';
const HIGHWAY_BLOCK_SIZE = 1024 * 1024;
function makeBridge(): BridgeContext {
return {
identity: { uin: '10001' } as unknown,
} as unknown as BridgeContext;
return { identity: { uin: '10001' } as unknown } as unknown as BridgeContext;
}
function makeSession(): HighwaySession {
@@ -108,198 +166,141 @@ function makeSession(): HighwaySession {
};
}
describe('uploadHighwayHttp connection lifecycle', () => {
beforeEach(() => {
createdSockets.length = 0;
createConnectionMock.mockReset();
createConnectionMock.mockImplementation((_opts: unknown, listener?: () => void) => {
const sock = new FakeSocket(buildHttpResponse(buildEmptyHighwayResponseFrame()));
createdSockets.push(sock);
// net.createConnection 在握手成功后调用 listener。同步触发即可,
// tcpConnect 的 Promise 会立刻 resolve。
if (listener) setImmediate(listener);
return sock as unknown as ReturnType<typeof createConnectionMock>;
});
// 安装一个 socket 工厂:createConnection 每次调用按序号创建一个 FakeSocket。
function installFactory(make: (idx: number) => FakeSocket): void {
createdSockets.length = 0;
socketFactoryRef.make = make as (idx: number) => unknown;
createConnectionMock.mockReset();
createConnectionMock.mockImplementation((_opts: unknown, listener?: () => void) => {
const idx = createdSockets.length;
const sock = socketFactoryRef.make!(idx) as FakeSocket;
createdSockets.push(sock);
if (listener) setImmediate(listener); // 握手成功后异步回调 → tcpConnect resolve
return sock as unknown as ReturnType<typeof createConnectionMock>;
});
}
// 顺序读取记录:offset/length + 关闭时机,用于数据完整性与 close 语义。
class RecordingChunkSource implements ChunkSource {
reads: Array<{ offset: number; length: number }> = [];
inFlight = 0;
closeCount = 0;
closedWhileReading = false;
constructor(readonly size: number) {}
async read(offset: number, length: number): Promise<Uint8Array> {
this.inFlight += 1;
this.reads.push({ offset, length });
await Promise.resolve();
this.inFlight -= 1;
return new Uint8Array(length);
}
async close(): Promise<void> {
if (this.inFlight > 0) this.closedWhileReading = true;
this.closeCount += 1;
}
}
describe('uploadHighwayHttp single-connection pipelined upload (#211)', () => {
afterEach(() => vi.restoreAllMocks());
it('reuses ONE connection for a multi-block payload when keep-alive holds', async () => {
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'));
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks
await uploadHighwayHttp(
makeBridge(), makeSession(), 1004, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0),
);
// The whole file went over a single warm connection — the #211 win.
expect(createConnectionMock).toHaveBeenCalledTimes(1);
});
afterEach(() => {
vi.restoreAllMocks();
});
it('uses a single TCP connection for a sub-block payload', async () => {
// 0.5 MiB → 1 chunk。覆盖 PTT / 小图的常见场景。
const bytes = new Uint8Array(512 * 1024);
it('single sub-block payload uses one connection', async () => {
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'));
const bytes = new Uint8Array(512 * 1024); // 1 block (PTT / small image)
await uploadHighwayHttp(
makeBridge(), makeSession(), 1003, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0),
);
expect(createConnectionMock).toHaveBeenCalledTimes(1);
expect(createdSockets[0]!.destroyed).toBe(true);
});
it('uses a fresh TCP connection per chunk for multi-block payloads', async () => {
// 1.5 MiB → ceil(1.5) = 2 chunks。这正是用户上报的 1.19 MB 图片
// 触发的代码路径:旧实现复用 socket 时第二次 POST 撞上 keep-alive
// 关闭,新实现必须为每个 chunk 各开一个 socket。
const bytes = new Uint8Array(Math.floor(1.5 * HIGHWAY_BLOCK_SIZE));
await uploadHighwayHttp(
makeBridge(), makeSession(), 1003, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0),
);
expect(createConnectionMock).toHaveBeenCalledTimes(2);
expect(createdSockets).toHaveLength(2);
for (const sock of createdSockets) {
expect(sock.destroyed).toBe(true);
}
});
it('opens N connections for N chunks (3 chunks here)', async () => {
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE));
await uploadHighwayHttp(
makeBridge(), makeSession(), 1004, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0),
);
expect(createConnectionMock).toHaveBeenCalledTimes(3);
});
it('a zero-length source opens ZERO connections (empty-input invariant)', async () => {
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'));
await uploadHighwayHttp(
makeBridge(), makeSession(), 1003, new BufferChunkSource(new Uint8Array(0)), new Uint8Array(16), new Uint8Array(0),
);
expect(createConnectionMock).not.toHaveBeenCalled();
});
});
// 并发上传(#211)的正确性回归。重点覆盖用户要求的四个维度:
// 数据安全 —— 每个 offset 恰好读一次、大小正确、覆盖整段 [0, size);
// 原子性 —— 首个永久失败即中止,不再领新 chunk,整体 reject;
// 回退 —— 失败/成功都只 close() 一次,且绝不在有 read 在飞时 close;
// 一致性 —— 并发确实发生(同时在飞的连接 > 1)但不超过 chunk 数。
describe('uploadHighwayHttp concurrent upload safety (#211)', () => {
// 记录每次 read 的 offset/length,跟踪在飞 read 数与 close 时机。read 故意
// 异步(await 一个 microtask),这样 close-while-reading 一旦发生就能被捕获。
class RecordingChunkSource implements ChunkSource {
reads: Array<{ offset: number; length: number }> = [];
inFlight = 0;
maxInFlight = 0;
closeCount = 0;
closedWhileReading = false;
constructor(readonly size: number) {}
async read(offset: number, length: number): Promise<Uint8Array> {
this.inFlight += 1;
this.maxInFlight = Math.max(this.maxInFlight, this.inFlight);
this.reads.push({ offset, length });
await Promise.resolve();
this.inFlight -= 1;
return new Uint8Array(length);
}
async close(): Promise<void> {
if (this.inFlight > 0) this.closedWhileReading = true;
this.closeCount += 1;
}
}
it('degrades to one connection per block when the server FINs after every response (#118 regime)', async () => {
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'closeafter'));
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks
await uploadHighwayHttp(
makeBridge(), makeSession(), 1004, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0),
);
// Server closes after each response → adaptively reconnects per block, but
// the upload still completes. No worse than the old one-conn-per-chunk.
expect(createConnectionMock).toHaveBeenCalledTimes(3);
});
// errorCode 帧:RespDataHighwayHead.field3 (errorCode) = 921。
// 定义即拒绝(不重试),用来制造“瞬时永久失败”而不吃重试 sleep。
// varint(921) = 0x99 0x07,tag(field3,varint) = 0x18。
function buildErrorCodeFrame(): Buffer {
const head = Buffer.from([0x18, 0x99, 0x07]);
const frame = Buffer.alloc(9 + head.length + 1);
frame[0] = 0x28;
frame.writeUInt32BE(head.length, 1);
frame.writeUInt32BE(0, 5);
head.copy(frame, 9);
frame[9 + head.length] = 0x29;
return frame;
}
it('reconnects and re-sends the SAME block on a mid-stream transport error', async () => {
// First connection dies while handling its first POST; a fresh connection
// must re-send that same block (offset unchanged, idempotent) and finish.
installFactory((idx) =>
idx === 0
? new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive', /* failOnPost */ 1)
: new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'),
);
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks
await uploadHighwayHttp(
makeBridge(), makeSession(), 1004, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0),
);
// Connection #0 failed on block 0 → reconnect (#1) resumes and keep-alives
// the rest: exactly two connections total.
expect(createConnectionMock).toHaveBeenCalledTimes(2);
});
// 跟踪同时存活(已连接、未 destroy)的 socket 峰值,用来断言并发确实发生。
let liveSockets = 0;
let maxLiveSockets = 0;
it('reconnects and re-sends the block when a response body is truncated mid-stream', async () => {
// Peer FINs after the HTTP header but before the highway body is complete.
// That truncation must be treated as a retryable transport failure (reject
// in readHttpResponseBody), NOT resolved as a partial frame — otherwise
// unpackHighwayFrame throws OUTSIDE the retry loop and fails the upload.
installFactory((idx) =>
idx === 0
? new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive', -1, /* truncateOnPost */ 1)
: new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'),
);
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks
await uploadHighwayHttp(
makeBridge(), makeSession(), 1004, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0),
);
// conn #0 truncated block 0 → reconnect (#1) re-sends it and finishes.
expect(createConnectionMock).toHaveBeenCalledTimes(2);
});
function installMock(responseFrame: Buffer): void {
createConnectionMock.mockReset();
liveSockets = 0;
maxLiveSockets = 0;
createConnectionMock.mockImplementation((_opts: unknown, listener?: () => void) => {
const sock = new FakeSocket(buildHttpResponse(responseFrame));
liveSockets += 1;
maxLiveSockets = Math.max(maxLiveSockets, liveSockets);
const origDestroy = sock.destroy.bind(sock);
sock.destroy = () => { if (!sock.destroyed) liveSockets -= 1; origDestroy(); };
createdSockets.push(sock);
if (listener) setImmediate(listener);
return sock as unknown as ReturnType<typeof createConnectionMock>;
});
}
beforeEach(() => { createdSockets.length = 0; });
afterEach(() => { vi.restoreAllMocks(); });
it('covers every offset exactly once and closes the source exactly once (data safety + consistency)', async () => {
installMock(buildEmptyHighwayResponseFrame());
const chunks = 8;
const size = chunks * HIGHWAY_BLOCK_SIZE;
it('reads every offset exactly once, in order, and closes the source once (data safety)', async () => {
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'));
const size = 4 * HIGHWAY_BLOCK_SIZE + 4321; // 4 full + 1 ragged
const source = new RecordingChunkSource(size);
await uploadHighwayHttp(
makeBridge(), makeSession(), 1004, source, new Uint8Array(16), new Uint8Array(0),
);
// Exactly-once, correctly-sized, gap-free coverage of [0, size).
const offsets = source.reads.map((r) => r.offset).sort((a, b) => a - b);
expect(offsets).toEqual(Array.from({ length: chunks }, (_, i) => i * HIGHWAY_BLOCK_SIZE));
expect(new Set(offsets).size).toBe(chunks); // no duplicate offset
expect(source.reads.every((r) => r.length === HIGHWAY_BLOCK_SIZE)).toBe(true);
// One connection per chunk; source closed once, never mid-read.
expect(createConnectionMock).toHaveBeenCalledTimes(chunks);
const offsets = source.reads.map((r) => r.offset);
// Strictly increasing, gap-free, exactly-once coverage of [0, size).
expect(offsets).toEqual([0, HIGHWAY_BLOCK_SIZE, 2 * HIGHWAY_BLOCK_SIZE, 3 * HIGHWAY_BLOCK_SIZE, 4 * HIGHWAY_BLOCK_SIZE]);
expect(source.reads[source.reads.length - 1]!.length).toBe(4321); // ragged tail
expect(source.reads.reduce((n, r) => n + r.length, 0)).toBe(size);
expect(source.closeCount).toBe(1);
expect(source.closedWhileReading).toBe(false);
});
it('actually uploads chunks in parallel but never exceeds the chunk count (concurrency)', async () => {
installMock(buildEmptyHighwayResponseFrame());
const chunks = 8;
const source = new RecordingChunkSource(chunks * HIGHWAY_BLOCK_SIZE);
await uploadHighwayHttp(
makeBridge(), makeSession(), 1004, source, new Uint8Array(16), new Uint8Array(0),
);
// Parallelism happened (more than one connection in flight at once) …
expect(maxLiveSockets).toBeGreaterThan(1);
// … but the pool is bounded and never opens more than there are chunks.
expect(maxLiveSockets).toBeLessThanOrEqual(chunks);
});
it('the last (ragged) chunk carries the remainder length', async () => {
installMock(buildEmptyHighwayResponseFrame());
const size = 2 * HIGHWAY_BLOCK_SIZE + 12345; // 2 full + 1 partial
const source = new RecordingChunkSource(size);
await uploadHighwayHttp(
makeBridge(), makeSession(), 1004, source, new Uint8Array(16), new Uint8Array(0),
);
const byOffset = new Map(source.reads.map((r) => [r.offset, r.length]));
expect(byOffset.get(0)).toBe(HIGHWAY_BLOCK_SIZE);
expect(byOffset.get(HIGHWAY_BLOCK_SIZE)).toBe(HIGHWAY_BLOCK_SIZE);
expect(byOffset.get(2 * HIGHWAY_BLOCK_SIZE)).toBe(12345);
const total = source.reads.reduce((n, r) => n + r.length, 0);
expect(total).toBe(size); // every byte accounted for exactly once
});
it('aborts the whole upload on a definitive server reject, closing the source once and never mid-read (atomicity + rollback)', async () => {
// Every chunk gets error_code=921 → definitive reject, no retries. The first
// failure must abort; the source must still be closed exactly once and never
// while a read is in flight.
installMock(buildErrorCodeFrame());
const source = new RecordingChunkSource(8 * HIGHWAY_BLOCK_SIZE);
it('rejects on a definitive server error_code and closes the source once (no retry)', async () => {
installFactory(() => new FakeSocket(buildErrorCodeFrame(), 'keepalive'));
const source = new RecordingChunkSource(3 * HIGHWAY_BLOCK_SIZE);
await expect(uploadHighwayHttp(
makeBridge(), makeSession(), 1004, source, new Uint8Array(16), new Uint8Array(0),
)).rejects.toThrow(/error_code=921/);
expect(source.closeCount).toBe(1);
expect(source.closedWhileReading).toBe(false);
// Atomicity: once aborted we stop claiming new chunks, so not all 8 were read.
expect(source.reads.length).toBeLessThan(8);
// Definitive reject on block 0 → stop immediately, don't PUT later blocks.
expect(source.reads.length).toBeLessThan(3);
});
});
+143 -146
View File
@@ -28,17 +28,13 @@ const HIGHWAY_BLOCK_SIZE = 1024 * 1024;
const HIGHWAY_MAX_CHUNK_ATTEMPTS = 3;
const HIGHWAY_RETRY_BASE_MS = 300;
// Concurrent chunk upload fan-out. A large file's chunks are independent —
// each highway frame is self-contained (its own dataOffset, chunkMd5, fileMd5
// and serviceTicket) and the server reassembles by offset, so chunks can be
// PUT in any order over parallel connections. Uploading them one-at-a-time
// (the old behavior) capped throughput at ~1 chunk / RTT AND paid TCP
// slow-start on every fresh per-chunk connection, so a 1 MiB chunk finished
// before its cold congestion window opened — the upload never reached line
// rate (issue #211). A small pool overlaps the RTT stalls and lets several
// connections ramp their windows at once. Kept modest so we stay a good
// citizen on QQ's highway edge nodes (which rate-limit — see error_code=921).
const HIGHWAY_UPLOAD_CONCURRENCY = 4;
// Idle read timeout for a highway response. tcpConnect() clears the socket's
// own timeout once connected, so a peer that accepts the connection but then
// neither responds nor FINs would hang the whole upload forever. The single
// persistent connection (issue #211) keeps a socket alive across the whole
// file, widening that exposure, so the response reader arms its own idle timer
// and treats a stall as a retryable transport failure.
const HIGHWAY_READ_IDLE_MS = 30000;
const sleepMs = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
@@ -196,7 +192,18 @@ function readHttpResponseBody(socket: net.Socket): Promise<Uint8Array> {
let totalNeeded = 0;
let settled = false;
// Reset on every inbound chunk; fire if the peer goes silent mid-response.
let idleTimer: ReturnType<typeof setTimeout> | null = null;
const armIdle = () => {
if (idleTimer) clearTimeout(idleTimer);
idleTimer = setTimeout(
() => finish(() => reject(new Error('highway response read timeout (peer idle)'))),
HIGHWAY_READ_IDLE_MS,
);
};
const detach = () => {
if (idleTimer) { clearTimeout(idleTimer); idleTimer = null; }
socket.off('data', onData);
socket.off('error', onError);
socket.off('close', onClose);
@@ -208,6 +215,7 @@ function readHttpResponseBody(socket: net.Socket): Promise<Uint8Array> {
fn();
};
const onData = (chunk: Buffer) => {
armIdle();
chunks.push(chunk);
const buf = Buffer.concat(chunks);
if (headerEnd < 0) {
@@ -227,17 +235,30 @@ function readHttpResponseBody(socket: net.Socket): Promise<Uint8Array> {
const onError = (err: Error) => finish(() => reject(err));
const onClose = () => finish(() => {
const buf = Buffer.concat(chunks);
if (headerEnd >= 0) resolve(new Uint8Array(buf.subarray(headerEnd)));
// QQ 的 highway 边缘节点对单连接多 POST 的 keep-alive 支持不稳定:
// 第一次 POST 拿到响应后服务器经常立刻 FIN 关闭,下一次写入的请求体
// 走到一个已被对端 close 的 socket 上,触发这里。诊断信息把已收到的
// 字节数和是否解析到 header 一起带出来,便于区分 “连接刚握手就被关”
// 和 “响应读到一半被截断” 两种情况。
else reject(new Error(
`connection closed before response (received=${buf.length}B, headerSeen=${headerEnd >= 0})`
));
// A fully-received body would already have resolved in onData; reaching
// here means the peer FIN'd before the response completed. Only a
// close-delimited body (no Content-Length) is legitimately ended by the
// FIN — resolve that. A declared-but-unmet Content-Length, or a
// half-received header, is a TRUNCATED response: reject it as a
// retryable transport failure so the caller's attempt loop reconnects
// and re-sends this block, instead of resolving a partial frame that
// unpackHighwayFrame would fatally reject OUTSIDE that loop (turning a
// recoverable mid-body FIN into a failed upload).
//
// QQ 的 highway 边缘节点对单连接多 POST 的 keep-alive 支持不稳定,常在
// 响应后立刻 FIN;诊断信息带上已收字节数、是否见到 header、需要多少,
// 便于区分 “连接刚握手就被关” 和 “响应读到一半被截断”。
if (headerEnd >= 0 && contentLength === 0) {
resolve(new Uint8Array(buf.subarray(headerEnd)));
} else {
reject(new Error(
`connection closed before full response (received=${buf.length}B, ` +
`headerSeen=${headerEnd >= 0}${headerEnd >= 0 ? `, need=${totalNeeded}` : ''})`,
));
}
});
armIdle();
socket.on('data', onData);
socket.on('error', onError);
socket.on('close', onClose);
@@ -318,142 +339,118 @@ export async function uploadHighwayHttp(
const pathStr = `/cgi-bin/httpconn?htcmd=0x6FF0087&uin=${bridge.identity.uin}`;
const totalSize = source.size;
// Upload exactly one chunk. Extracted verbatim from the old sequential loop
// body — the per-chunk wire behavior is unchanged, only the *scheduling*
// across chunks becomes concurrent (see HIGHWAY_UPLOAD_CONCURRENCY).
// Single-connection, in-order, pipelined upload (issue #211).
//
// 每个 chunk 仍然一个独立的 TCP 连接。
// Blocks are PUT in STRICT offset order over ONE persistent connection,
// reused across chunks via HTTP keep-alive. Two properties matter:
// - In order: QQ's highway server reassembles by offset but REJECTS a
// block that arrives after a later one (error_code=102902) — parallel or
// out-of-order PUTs do not work (proven on a real endpoint). Sequential
// send guarantees in-order arrival.
// - One warm connection: reusing the socket keeps the TCP congestion
// window open across the whole file instead of paying slow-start on a
// fresh connection for every 1 MiB block. That per-block cold start was
// the real throughput cap behind #211.
//
// 旧实现整个文件复用同一个 socket,依赖 `Connection: keep-alive` 跑多次
// POST。但 QQ highway 边缘节点(以及链路上的代理/NAT)经常在第一次响应
// 之后立刻 FIN 关闭连接,导致第二个 chunk 写入一个已被对端关闭的 socket,
// `readHttpResponseBody` 立刻收到 `close` 事件并抛 `connection closed
// before response`(#118)。所以每个 chunk 独立建连的不变量必须保留 ——
// 并发只是让多个这样的一次性连接同时在飞,连接之间没有任何共享状态。
// QQ's edge nodes (and proxies/NAT on the path) sometimes FIN between
// chunks — that is what forced the one-connection-per-chunk workaround in
// #118. We handle it adaptively: if reusing the socket fails, or the peer
// half-closed after a response, we drop it, reconnect, and re-send the SAME
// block (re-PUTting an offset is idempotent). Worst case — a server that
// closes after every response — degrades to one connection per block: no
// worse than the #118 behavior, and still strictly in order (so never
// error_code=102902). `connectCount` is logged so a real run reveals which
// regime we hit (1 connection = keep-alive honored; ~chunkCount = closed
// every time).
//
// Data safety: `source.read` returns a fresh buffer per call (FileChunkSource
// allocates; BufferChunkSource returns a read-only subarray), and reads are
// positional (offset-addressed), so concurrent reads at distinct offsets are
// race-free. Each chunk computes its own md5 over its own bytes.
const uploadChunk = async (offset: number, chunkSize: number): Promise<void> => {
const chunk = await source.read(offset, chunkSize);
const chunkMd5 = computeMd5(chunk);
const head = makeHighwayHead(
bridge.identity.uin, commandId, totalSize, offset, chunkSize,
chunkMd5, fileMd5, session.sigSession, extend,
);
const frame = packHighwayFrame(head, chunk);
// Retry the connect + POST for this chunk on transient transport errors
// (peer FIN before response, ECONNRESET, connect refused/timeout). Each
// attempt uses a fresh connection; re-sending the same offset range is
// idempotent on the server.
let responseBody: Uint8Array | undefined;
for (let attempt = 1; ; attempt++) {
let socket: net.Socket;
try {
socket = await tcpConnect(session.host, session.port);
} catch (err) {
if (attempt >= HIGHWAY_MAX_CHUNK_ATTEMPTS) {
throw new Error(
`highway connect failed after ${attempt} attempts ` +
`(cmdId=${commandId} offset=${offset}/${totalSize}): ${String(err)}`,
);
}
await sleepMs(HIGHWAY_RETRY_BASE_MS * attempt);
continue;
}
try {
responseBody = await httpPostFrame(socket, session.host, pathStr, frame);
break;
} catch (err) {
if (attempt >= HIGHWAY_MAX_CHUNK_ATTEMPTS) {
throw new Error(
`highway upload transport failed after ${attempt} attempts ` +
`(cmdId=${commandId} chunk=${chunkSize}/${totalSize} offset=${offset}): ${String(err)}`,
);
}
log.trace('chunk offset=%d attempt %d failed (%s), retrying', offset, attempt, String(err));
await sleepMs(HIGHWAY_RETRY_BASE_MS * attempt);
} finally {
socket.destroy();
}
}
// Unreachable: the retry loop only exits via break (responseBody set) or
// throw — the guard just narrows the type for the compiler.
if (!responseBody) throw new Error('highway upload: missing response');
const { head: respHead } = unpackHighwayFrame(responseBody);
const resp = protobuf_decode<RespDataHighwayHead>(respHead);
if (resp?.errorCode && resp.errorCode !== 0) {
// Surface every diagnostic the highway response carries so
// user reports of `error_code=921` and friends include the
// server-side context (segHead.retCode, chunk size, file md5)
// — without these we can't tell apart a malformed-payload
// reject, a session-ticket mismatch, or a per-account rate-
// limit.
const segRetCode = resp.msgSegHead?.retCode ?? 0;
const fileMd5Hex = Buffer.from(fileMd5).toString('hex');
throw new Error(
`highway upload error_code=${resp.errorCode}` +
` (cmdId=${commandId} chunk=${chunkSize}/${totalSize}` +
` offset=${offset} segRetCode=${segRetCode}` +
` fileMd5=${fileMd5Hex.slice(0, 16)}…)`,
);
}
};
// Concurrent worker pool over the chunk range.
//
// Atomicity / consistency: an offset is claimed synchronously — there is NO
// await between reading `nextOffset` and advancing it — so two workers can
// never claim the same chunk, and every byte range is PUT exactly once. On
// the first permanent chunk failure a worker sets `aborted`; peers stop
// pulling new chunks but drain the one already in flight. `worker()` NEVER
// rejects (it captures its error), so `Promise.all` always resolves only
// after every in-flight read/socket has settled — that is what makes it safe
// to `source.close()` in the finally without racing a live read.
//
// Rollback: a thrown upload leaves the server holding an incomplete set of
// chunks that it never finalizes — identical to the pre-concurrency failure
// mode. The caller's publish/finalize step (which makes the file visible)
// only runs when this resolves, so a partial upload is never published.
let nextOffset = 0;
let uploaded = 0;
let aborted = false;
let firstError: Error | null = null;
const worker = async (): Promise<void> => {
while (!aborted) {
const offset = nextOffset;
if (offset >= totalSize) return;
const chunkSize = Math.min(HIGHWAY_BLOCK_SIZE, totalSize - offset);
nextOffset = offset + chunkSize; // claim — synchronous, race-free
try {
await uploadChunk(offset, chunkSize);
} catch (err) {
aborted = true;
firstError ??= err instanceof Error ? err : new Error(String(err));
return;
}
uploaded += chunkSize;
log.trace('uploaded %d/%d bytes', uploaded, totalSize);
}
};
// `source` may buffer the whole file (BufferChunkSource) or stream it from
// disk (FileChunkSource). We own the source and close it exactly once below,
// only after all workers have settled.
// disk (FileChunkSource); either way one chunk is read/held at a time. We
// own the source and close it exactly once in the `finally` below.
let succeeded = false;
let socket: net.Socket | null = null;
let connectCount = 0;
const dropSocket = (): void => {
if (socket) { socket.destroy(); socket = null; }
};
try {
const chunkCount = Math.max(1, Math.ceil(totalSize / HIGHWAY_BLOCK_SIZE));
const workerCount = Math.min(HIGHWAY_UPLOAD_CONCURRENCY, chunkCount);
await Promise.all(Array.from({ length: workerCount }, () => worker()));
if (firstError) throw firstError;
let offset = 0;
while (offset < totalSize) {
const chunkSize = Math.min(HIGHWAY_BLOCK_SIZE, totalSize - offset);
const chunk = await source.read(offset, chunkSize);
const chunkMd5 = computeMd5(chunk);
const head = makeHighwayHead(
bridge.identity.uin, commandId, totalSize, offset, chunkSize,
chunkMd5, fileMd5, session.sigSession, extend,
);
const frame = packHighwayFrame(head, chunk);
// Send this block, reusing the live connection when there is one. On any
// transport failure (peer FIN between chunks, ECONNRESET, connect
// refused/timeout) drop the socket, reconnect, and retry THIS block —
// re-sending the same offset range is idempotent on the server.
let responseBody: Uint8Array | undefined;
for (let attempt = 1; ; attempt++) {
try {
if (!socket) {
socket = await tcpConnect(session.host, session.port);
connectCount += 1;
}
responseBody = await httpPostFrame(socket, session.host, pathStr, frame);
break;
} catch (err) {
dropSocket();
if (attempt >= HIGHWAY_MAX_CHUNK_ATTEMPTS) {
throw new Error(
`highway upload transport failed after ${attempt} attempts ` +
`(cmdId=${commandId} chunk=${chunkSize}/${totalSize} offset=${offset}): ${String(err)}`,
);
}
log.trace('chunk offset=%d attempt %d failed (%s), reconnecting', offset, attempt, String(err));
await sleepMs(HIGHWAY_RETRY_BASE_MS * attempt);
}
}
// Unreachable: the retry loop only exits via break (responseBody set) or
// throw — the guard just narrows the type for the compiler.
if (!responseBody) throw new Error('highway upload: missing response');
const { head: respHead } = unpackHighwayFrame(responseBody);
const resp = protobuf_decode<RespDataHighwayHead>(respHead);
if (resp?.errorCode && resp.errorCode !== 0) {
// Surface every diagnostic the highway response carries so
// user reports of `error_code=921` and friends include the
// server-side context (segHead.retCode, chunk size, file md5)
// — without these we can't tell apart a malformed-payload
// reject, a session-ticket mismatch, or a per-account rate-
// limit.
const segRetCode = resp.msgSegHead?.retCode ?? 0;
const fileMd5Hex = Buffer.from(fileMd5).toString('hex');
throw new Error(
`highway upload error_code=${resp.errorCode}` +
` (cmdId=${commandId} chunk=${chunkSize}/${totalSize}` +
` offset=${offset} segRetCode=${segRetCode}` +
` fileMd5=${fileMd5Hex.slice(0, 16)}…)`,
);
}
// If the peer half-closed after sending its response, the socket can no
// longer carry the next block — drop it now so the next iteration
// reconnects cleanly instead of writing into a dead socket.
if (socket && (socket.destroyed || socket.readableEnded || !socket.writable)) {
dropSocket();
}
offset += chunkSize;
log.trace('uploaded %d/%d bytes', offset, totalSize);
}
succeeded = true;
if (totalSize > 0) {
log.debug('highway upload done: %d bytes over %d connection(s) (cmdId=%d)',
totalSize, connectCount, commandId);
}
} finally {
// Best-effort close — never mask a primary upload error. A close failure
// is only surfaced when the upload itself succeeded.
dropSocket();
try {
await source.close();
} catch (closeErr) {