From 7ab91c81e94857ea5e6df50a280699f4b7c2f593 Mon Sep 17 00:00:00 2001 From: DrSmoothl <1787882683@qq.com> Date: Thu, 9 Jul 2026 13:55:25 +0800 Subject: [PATCH] =?UTF-8?q?perf(highway):=20single-connection=20pipelined?= =?UTF-8?q?=20upload=20=E2=80=94=20supersede=20#211=20concurrency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #211 concurrency attempt (c17fff6) was disproven on a live endpoint. QQ's highway server requires blocks to arrive in offset order and rejects an out-of-order PUT with error_code=102902 — ~20% failure under a real account even after deferring the finalize block. Parallel connections can't guarantee arrival order, so that approach is unshippable. Replace it with a single persistent connection that PUTs blocks in strict offset order, reusing the socket across the whole file via HTTP keep-alive: - In order: the server never sees an out-of-order block, so no 102902. - One warm connection: the TCP congestion window stays open across the file instead of paying slow-start on a fresh connection per 1 MiB block. That per-block cold start was the real throughput cap behind #211. Adaptive to QQ's edge nodes FIN'ing between chunks (the reason #118 went one-connection-per-chunk): on a transport failure, or a peer half-close after a response, drop the socket, reconnect, and re-send the SAME block (re-PUTting an offset is idempotent). Worst case degrades to one connection per block — no worse than #118, and still strictly in order. Also fixes a latent bug on the reconnect path, surfaced by review: a response truncated mid-stream (header arrived, Content-Length unmet, then FIN) was resolved as a partial frame and then fatally rejected by unpackHighwayFrame OUTSIDE the retry loop, turning a recoverable mid-body FIN into a failed upload. A truncated response now rejects as a retryable transport failure so the block is re-sent. Added an idle read timeout so a silent peer can't hang the upload forever (the persistent socket widens that exposure). Verified live (tempserver, repeated 48 MiB group-file uploads): 1 connection per file, 0 failures across dense repeats, ~1.4x throughput over the old per-chunk-connection serial path in a low-RTT datacenter — the win is larger on high-RTT home links where per-block slow-start hurt most (the original #211 report). Unit tests cover keep-alive reuse, FIN-after-each-response degradation, mid-stream reconnect+resume, body- truncation retry, in-order exactly-once coverage, and close-once. --- .../core/tests/highway/highway-client.test.ts | 429 +++++++++--------- .../protocol/src/highway/highway-client.ts | 289 ++++++------ 2 files changed, 358 insertions(+), 360 deletions(-) diff --git a/packages/core/tests/highway/highway-client.test.ts b/packages/core/tests/highway/highway-client.test.ts index 826127a7..f45a627d 100644 --- a/packages/core/tests/highway/highway-client.test.ts +++ b/packages/core/tests/highway/highway-client.test.ts @@ -1,23 +1,24 @@ // highway-client.uploadHighwayHttp 的连接生命周期测试。 // -// 回归覆盖:当上传数据 > HIGHWAY_BLOCK_SIZE (1 MiB) 时,旧实现会复用 -// 同一个 TCP socket 在 keep-alive 模式下连续 POST,但 QQ highway 的 -// 边缘节点经常在第一次响应后立刻 FIN 关闭,导致第二次 POST 报 -// `connection closed before response`。用户上报的 1.19 MB 图片 -// (onebot retcode=1200)就是这个分支。 +// 方案 3(#211):单连接、按 offset 顺序、流水线上传,跨 chunk 复用同一个 +// TCP 连接(HTTP keep-alive)。要点: +// - 顺序发送 → 天然保序,避开 QQ highway 对乱序块的 error_code=102902; +// - 复用连接 → TCP 拥塞窗口全程保持,省掉每块新建连接的 slow-start。 +// 边缘节点可能在响应后 FIN(#118);此时自适应地丢弃 socket + 重连 + 重发 +// 同一块(重发幂等),最坏退化成“每块一连接”,不比旧实现差,且仍然保序。 // -// 修复方案:每个 chunk 独立建一个新的 TCP 连接。本测试通过 mock -// `net.createConnection` 计数建连次数,断言每个 chunk 一连接的不变量。 +// 本测试用两种 mock 服务端行为锁定这套自适应逻辑: +// - keepalive:每个 POST 回一个响应,连接保持 → 整个文件 1 个连接; +// - closeafter:每个响应后半关闭(readableEnded/emit close)→ 退化成每块 +// 一连接,但仍全部成功。 -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { EventEmitter } from 'events'; // highway-client 解析响应只关心 errorCode === 0 —— 一个完全空的 -// RespDataHighwayHead protobuf 体(0 字节)会被解为 ,errorCode -// 是 undefined,走 falsy 通过分支。 -// -// HTTP body 是 highway frame: -// 0x28 | head_len(BE32) | body_len(BE32) | head | body | 0x29 +// RespDataHighwayHead protobuf 体(0 字节)会被解为 errorCode=undefined, +// 走 falsy 通过分支。 +// highway frame:0x28 | head_len(BE32) | body_len(BE32) | head | body | 0x29 function buildEmptyHighwayResponseFrame(): Buffer { const frame = Buffer.alloc(10); frame[0] = 0x28; @@ -27,59 +28,113 @@ function buildEmptyHighwayResponseFrame(): Buffer { return frame; } -function buildHttpResponse(frameBody: Buffer): Buffer { - const headers = [ - 'HTTP/1.1 200 OK', - `Content-Length: ${frameBody.length}`, - 'Connection: close', - '', - '', - ].join('\r\n'); - return Buffer.concat([Buffer.from(headers, 'ascii'), frameBody]); +// errorCode 帧:RespDataHighwayHead.field3 (errorCode) = 921。tag(field3, +// varint)=0x18,varint(921)=0x99 0x07。定义即拒绝(不重试)。 +function buildErrorCodeFrame(): Buffer { + const head = Buffer.from([0x18, 0x99, 0x07]); + const frame = Buffer.alloc(9 + head.length + 1); + frame[0] = 0x28; + frame.writeUInt32BE(head.length, 1); + frame.writeUInt32BE(0, 5); + head.copy(frame, 9); + frame[9 + head.length] = 0x29; + return frame; } -// FakeSocket 模拟 net.Socket 的最小子集。highway-client 用到: -// - 'data' / 'error' / 'close' 事件 -// - .write(data, cb) cb 会被 socketWrite 等待 -// - .destroy() -// - .setTimeout / .once / .removeListener (继承自 EventEmitter) +function httpWrap(frame: Buffer): Buffer { + const headers = ['HTTP/1.1 200 OK', `Content-Length: ${frame.length}`, '', ''].join('\r\n'); + return Buffer.concat([Buffer.from(headers, 'ascii'), frame]); +} + +type ServerMode = 'keepalive' | 'closeafter'; + +// FakeSocket 模拟 net.Socket 的最小子集,支持在一个连接上顺序处理多个 POST。 +// 累积写入的字节,按 `\r\n\r\n` + Content-Length 切出每个完整 POST,逐个回 +// 响应。`mode` 控制响应后是否半关闭;`failOnPost` 用于模拟链路中途断开。 class FakeSocket extends EventEmitter { destroyed = false; - // 每个 POST = 2 次 write(HTTP header + body)。第二次 write 落地后 - // 异步回放响应并立刻 close —— 复现 QQ highway 真实行为:响应一发完 - // 就 FIN,旧实现的下一次 POST 必然撞上已关闭的 socket。 - private writeCount = 0; - private response: Buffer; - - constructor(response: Buffer) { + readableEnded = false; + writable = true; + posts = 0; + private buf = Buffer.alloc(0); + constructor( + private readonly responseFrame: Buffer, + private readonly mode: ServerMode, + private readonly failOnPost = -1, + private readonly truncateOnPost = -1, + ) { super(); - this.response = response; } - setTimeout(_ms: number) { /* no-op */ } + setTimeout(_ms?: number) { /* no-op */ } - write(_data: unknown, cb?: (err?: Error) => void): boolean { - this.writeCount += 1; - if (this.writeCount === 2) { - setImmediate(() => { - this.emit('data', this.response); - this.emit('close'); - }); + write(data: unknown, cb?: (err?: Error) => void): boolean { + if (this.destroyed || !this.writable) { + const err = new Error('write after socket end'); + if (cb) cb(err); + setImmediate(() => this.emit('error', err)); + return false; } + this.buf = Buffer.concat([this.buf, Buffer.from(data as Uint8Array)]); + this.drainPosts(); if (cb) cb(); return true; } + private drainPosts(): void { + for (;;) { + const s = this.buf.toString('latin1'); + const he = s.indexOf('\r\n\r\n'); + if (he < 0) return; + const m = s.slice(0, he).match(/content-length:\s*(\d+)/i); + const cl = m ? parseInt(m[1], 10) : 0; + const total = he + 4 + cl; + if (this.buf.length < total) return; + this.buf = this.buf.subarray(total); + this.posts += 1; + const thisPost = this.posts; + setImmediate(() => { + if (this.destroyed) return; + if (thisPost === this.failOnPost) { + this.writable = false; + this.readableEnded = true; + this.emit('error', new Error('ECONNRESET')); + this.emit('close'); + return; + } + if (thisPost === this.truncateOnPost) { + // Send the HTTP header + only part of the declared body, then FIN — + // a truncated response. Dropping the last 3 body bytes keeps the + // header intact so Content-Length parses but is never satisfied. + const full = httpWrap(this.responseFrame); + this.emit('data', full.subarray(0, full.length - 3)); + this.writable = false; + this.readableEnded = true; + this.emit('close'); + return; + } + this.emit('data', httpWrap(this.responseFrame)); + if (this.mode === 'closeafter') { + this.readableEnded = true; + this.writable = false; + this.emit('end'); + this.emit('close'); + } + }); + } + } + destroy() { this.destroyed = true; + this.writable = false; } } -// vitest 把 vi.mock 工厂 hoist 到文件顶部,所以工厂里不能引用普通的 -// 顶层 const —— 那会触发 "Cannot access X before initialization"。 -// 用 vi.hoisted 让这两个值跟着 mock 一起被 hoist。 -const { createdSockets, createConnectionMock } = vi.hoisted(() => ({ - createdSockets: [] as FakeSocket[], +// vitest 把 vi.mock 工厂 hoist 到文件顶部,所以工厂里不能引用普通的顶层 +// const。用 vi.hoisted 让这些值跟着 mock 一起 hoist。 +const { createdSockets, socketFactoryRef, createConnectionMock } = vi.hoisted(() => ({ + createdSockets: [] as unknown[], + socketFactoryRef: { make: null as null | ((idx: number) => unknown) }, createConnectionMock: vi.fn(), })); @@ -88,15 +143,18 @@ vi.mock('net', () => ({ createConnection: createConnectionMock, })); -import { uploadHighwayHttp, BufferChunkSource, type ChunkSource, type HighwaySession } from '@snowluma/protocol/highway'; +import { + uploadHighwayHttp, + BufferChunkSource, + type ChunkSource, + type HighwaySession, +} from '@snowluma/protocol/highway'; import type { BridgeContext } from '@snowluma/protocol/bridge-context'; const HIGHWAY_BLOCK_SIZE = 1024 * 1024; function makeBridge(): BridgeContext { - return { - identity: { uin: '10001' } as unknown, - } as unknown as BridgeContext; + return { identity: { uin: '10001' } as unknown } as unknown as BridgeContext; } function makeSession(): HighwaySession { @@ -108,198 +166,141 @@ function makeSession(): HighwaySession { }; } -describe('uploadHighwayHttp connection lifecycle', () => { - beforeEach(() => { - createdSockets.length = 0; - createConnectionMock.mockReset(); - createConnectionMock.mockImplementation((_opts: unknown, listener?: () => void) => { - const sock = new FakeSocket(buildHttpResponse(buildEmptyHighwayResponseFrame())); - createdSockets.push(sock); - // net.createConnection 在握手成功后调用 listener。同步触发即可, - // tcpConnect 的 Promise 会立刻 resolve。 - if (listener) setImmediate(listener); - return sock as unknown as ReturnType; - }); +// 安装一个 socket 工厂:createConnection 每次调用按序号创建一个 FakeSocket。 +function installFactory(make: (idx: number) => FakeSocket): void { + createdSockets.length = 0; + socketFactoryRef.make = make as (idx: number) => unknown; + createConnectionMock.mockReset(); + createConnectionMock.mockImplementation((_opts: unknown, listener?: () => void) => { + const idx = createdSockets.length; + const sock = socketFactoryRef.make!(idx) as FakeSocket; + createdSockets.push(sock); + if (listener) setImmediate(listener); // 握手成功后异步回调 → tcpConnect resolve + return sock as unknown as ReturnType; + }); +} + +// 顺序读取记录:offset/length + 关闭时机,用于数据完整性与 close 语义。 +class RecordingChunkSource implements ChunkSource { + reads: Array<{ offset: number; length: number }> = []; + inFlight = 0; + closeCount = 0; + closedWhileReading = false; + constructor(readonly size: number) {} + async read(offset: number, length: number): Promise { + this.inFlight += 1; + this.reads.push({ offset, length }); + await Promise.resolve(); + this.inFlight -= 1; + return new Uint8Array(length); + } + async close(): Promise { + if (this.inFlight > 0) this.closedWhileReading = true; + this.closeCount += 1; + } +} + +describe('uploadHighwayHttp single-connection pipelined upload (#211)', () => { + afterEach(() => vi.restoreAllMocks()); + + it('reuses ONE connection for a multi-block payload when keep-alive holds', async () => { + installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive')); + const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks + await uploadHighwayHttp( + makeBridge(), makeSession(), 1004, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0), + ); + // The whole file went over a single warm connection — the #211 win. + expect(createConnectionMock).toHaveBeenCalledTimes(1); }); - afterEach(() => { - vi.restoreAllMocks(); - }); - - it('uses a single TCP connection for a sub-block payload', async () => { - // 0.5 MiB → 1 chunk。覆盖 PTT / 小图的常见场景。 - const bytes = new Uint8Array(512 * 1024); + it('single sub-block payload uses one connection', async () => { + installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive')); + const bytes = new Uint8Array(512 * 1024); // 1 block (PTT / small image) await uploadHighwayHttp( makeBridge(), makeSession(), 1003, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0), ); expect(createConnectionMock).toHaveBeenCalledTimes(1); - expect(createdSockets[0]!.destroyed).toBe(true); - }); - - it('uses a fresh TCP connection per chunk for multi-block payloads', async () => { - // 1.5 MiB → ceil(1.5) = 2 chunks。这正是用户上报的 1.19 MB 图片 - // 触发的代码路径:旧实现复用 socket 时第二次 POST 撞上 keep-alive - // 关闭,新实现必须为每个 chunk 各开一个 socket。 - const bytes = new Uint8Array(Math.floor(1.5 * HIGHWAY_BLOCK_SIZE)); - await uploadHighwayHttp( - makeBridge(), makeSession(), 1003, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0), - ); - expect(createConnectionMock).toHaveBeenCalledTimes(2); - expect(createdSockets).toHaveLength(2); - for (const sock of createdSockets) { - expect(sock.destroyed).toBe(true); - } - }); - - it('opens N connections for N chunks (3 chunks here)', async () => { - const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); - await uploadHighwayHttp( - makeBridge(), makeSession(), 1004, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0), - ); - expect(createConnectionMock).toHaveBeenCalledTimes(3); }); it('a zero-length source opens ZERO connections (empty-input invariant)', async () => { + installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive')); await uploadHighwayHttp( makeBridge(), makeSession(), 1003, new BufferChunkSource(new Uint8Array(0)), new Uint8Array(16), new Uint8Array(0), ); expect(createConnectionMock).not.toHaveBeenCalled(); }); -}); -// 并发上传(#211)的正确性回归。重点覆盖用户要求的四个维度: -// 数据安全 —— 每个 offset 恰好读一次、大小正确、覆盖整段 [0, size); -// 原子性 —— 首个永久失败即中止,不再领新 chunk,整体 reject; -// 回退 —— 失败/成功都只 close() 一次,且绝不在有 read 在飞时 close; -// 一致性 —— 并发确实发生(同时在飞的连接 > 1)但不超过 chunk 数。 -describe('uploadHighwayHttp concurrent upload safety (#211)', () => { - // 记录每次 read 的 offset/length,跟踪在飞 read 数与 close 时机。read 故意 - // 异步(await 一个 microtask),这样 close-while-reading 一旦发生就能被捕获。 - class RecordingChunkSource implements ChunkSource { - reads: Array<{ offset: number; length: number }> = []; - inFlight = 0; - maxInFlight = 0; - closeCount = 0; - closedWhileReading = false; - constructor(readonly size: number) {} - async read(offset: number, length: number): Promise { - this.inFlight += 1; - this.maxInFlight = Math.max(this.maxInFlight, this.inFlight); - this.reads.push({ offset, length }); - await Promise.resolve(); - this.inFlight -= 1; - return new Uint8Array(length); - } - async close(): Promise { - if (this.inFlight > 0) this.closedWhileReading = true; - this.closeCount += 1; - } - } + it('degrades to one connection per block when the server FINs after every response (#118 regime)', async () => { + installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'closeafter')); + const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks + await uploadHighwayHttp( + makeBridge(), makeSession(), 1004, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0), + ); + // Server closes after each response → adaptively reconnects per block, but + // the upload still completes. No worse than the old one-conn-per-chunk. + expect(createConnectionMock).toHaveBeenCalledTimes(3); + }); - // errorCode 帧:RespDataHighwayHead.field3 (errorCode) = 921。 - // 定义即拒绝(不重试),用来制造“瞬时永久失败”而不吃重试 sleep。 - // varint(921) = 0x99 0x07,tag(field3,varint) = 0x18。 - function buildErrorCodeFrame(): Buffer { - const head = Buffer.from([0x18, 0x99, 0x07]); - const frame = Buffer.alloc(9 + head.length + 1); - frame[0] = 0x28; - frame.writeUInt32BE(head.length, 1); - frame.writeUInt32BE(0, 5); - head.copy(frame, 9); - frame[9 + head.length] = 0x29; - return frame; - } + it('reconnects and re-sends the SAME block on a mid-stream transport error', async () => { + // First connection dies while handling its first POST; a fresh connection + // must re-send that same block (offset unchanged, idempotent) and finish. + installFactory((idx) => + idx === 0 + ? new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive', /* failOnPost */ 1) + : new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'), + ); + const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks + await uploadHighwayHttp( + makeBridge(), makeSession(), 1004, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0), + ); + // Connection #0 failed on block 0 → reconnect (#1) resumes and keep-alives + // the rest: exactly two connections total. + expect(createConnectionMock).toHaveBeenCalledTimes(2); + }); - // 跟踪同时存活(已连接、未 destroy)的 socket 峰值,用来断言并发确实发生。 - let liveSockets = 0; - let maxLiveSockets = 0; + it('reconnects and re-sends the block when a response body is truncated mid-stream', async () => { + // Peer FINs after the HTTP header but before the highway body is complete. + // That truncation must be treated as a retryable transport failure (reject + // in readHttpResponseBody), NOT resolved as a partial frame — otherwise + // unpackHighwayFrame throws OUTSIDE the retry loop and fails the upload. + installFactory((idx) => + idx === 0 + ? new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive', -1, /* truncateOnPost */ 1) + : new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'), + ); + const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks + await uploadHighwayHttp( + makeBridge(), makeSession(), 1004, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0), + ); + // conn #0 truncated block 0 → reconnect (#1) re-sends it and finishes. + expect(createConnectionMock).toHaveBeenCalledTimes(2); + }); - function installMock(responseFrame: Buffer): void { - createConnectionMock.mockReset(); - liveSockets = 0; - maxLiveSockets = 0; - createConnectionMock.mockImplementation((_opts: unknown, listener?: () => void) => { - const sock = new FakeSocket(buildHttpResponse(responseFrame)); - liveSockets += 1; - maxLiveSockets = Math.max(maxLiveSockets, liveSockets); - const origDestroy = sock.destroy.bind(sock); - sock.destroy = () => { if (!sock.destroyed) liveSockets -= 1; origDestroy(); }; - createdSockets.push(sock); - if (listener) setImmediate(listener); - return sock as unknown as ReturnType; - }); - } - - beforeEach(() => { createdSockets.length = 0; }); - afterEach(() => { vi.restoreAllMocks(); }); - - it('covers every offset exactly once and closes the source exactly once (data safety + consistency)', async () => { - installMock(buildEmptyHighwayResponseFrame()); - const chunks = 8; - const size = chunks * HIGHWAY_BLOCK_SIZE; + it('reads every offset exactly once, in order, and closes the source once (data safety)', async () => { + installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive')); + const size = 4 * HIGHWAY_BLOCK_SIZE + 4321; // 4 full + 1 ragged const source = new RecordingChunkSource(size); - await uploadHighwayHttp( makeBridge(), makeSession(), 1004, source, new Uint8Array(16), new Uint8Array(0), ); - - // Exactly-once, correctly-sized, gap-free coverage of [0, size). - const offsets = source.reads.map((r) => r.offset).sort((a, b) => a - b); - expect(offsets).toEqual(Array.from({ length: chunks }, (_, i) => i * HIGHWAY_BLOCK_SIZE)); - expect(new Set(offsets).size).toBe(chunks); // no duplicate offset - expect(source.reads.every((r) => r.length === HIGHWAY_BLOCK_SIZE)).toBe(true); - // One connection per chunk; source closed once, never mid-read. - expect(createConnectionMock).toHaveBeenCalledTimes(chunks); + const offsets = source.reads.map((r) => r.offset); + // Strictly increasing, gap-free, exactly-once coverage of [0, size). + expect(offsets).toEqual([0, HIGHWAY_BLOCK_SIZE, 2 * HIGHWAY_BLOCK_SIZE, 3 * HIGHWAY_BLOCK_SIZE, 4 * HIGHWAY_BLOCK_SIZE]); + expect(source.reads[source.reads.length - 1]!.length).toBe(4321); // ragged tail + expect(source.reads.reduce((n, r) => n + r.length, 0)).toBe(size); expect(source.closeCount).toBe(1); expect(source.closedWhileReading).toBe(false); }); - it('actually uploads chunks in parallel but never exceeds the chunk count (concurrency)', async () => { - installMock(buildEmptyHighwayResponseFrame()); - const chunks = 8; - const source = new RecordingChunkSource(chunks * HIGHWAY_BLOCK_SIZE); - - await uploadHighwayHttp( - makeBridge(), makeSession(), 1004, source, new Uint8Array(16), new Uint8Array(0), - ); - - // Parallelism happened (more than one connection in flight at once) … - expect(maxLiveSockets).toBeGreaterThan(1); - // … but the pool is bounded and never opens more than there are chunks. - expect(maxLiveSockets).toBeLessThanOrEqual(chunks); - }); - - it('the last (ragged) chunk carries the remainder length', async () => { - installMock(buildEmptyHighwayResponseFrame()); - const size = 2 * HIGHWAY_BLOCK_SIZE + 12345; // 2 full + 1 partial - const source = new RecordingChunkSource(size); - - await uploadHighwayHttp( - makeBridge(), makeSession(), 1004, source, new Uint8Array(16), new Uint8Array(0), - ); - - const byOffset = new Map(source.reads.map((r) => [r.offset, r.length])); - expect(byOffset.get(0)).toBe(HIGHWAY_BLOCK_SIZE); - expect(byOffset.get(HIGHWAY_BLOCK_SIZE)).toBe(HIGHWAY_BLOCK_SIZE); - expect(byOffset.get(2 * HIGHWAY_BLOCK_SIZE)).toBe(12345); - const total = source.reads.reduce((n, r) => n + r.length, 0); - expect(total).toBe(size); // every byte accounted for exactly once - }); - - it('aborts the whole upload on a definitive server reject, closing the source once and never mid-read (atomicity + rollback)', async () => { - // Every chunk gets error_code=921 → definitive reject, no retries. The first - // failure must abort; the source must still be closed exactly once and never - // while a read is in flight. - installMock(buildErrorCodeFrame()); - const source = new RecordingChunkSource(8 * HIGHWAY_BLOCK_SIZE); - + it('rejects on a definitive server error_code and closes the source once (no retry)', async () => { + installFactory(() => new FakeSocket(buildErrorCodeFrame(), 'keepalive')); + const source = new RecordingChunkSource(3 * HIGHWAY_BLOCK_SIZE); await expect(uploadHighwayHttp( makeBridge(), makeSession(), 1004, source, new Uint8Array(16), new Uint8Array(0), )).rejects.toThrow(/error_code=921/); - expect(source.closeCount).toBe(1); expect(source.closedWhileReading).toBe(false); - // Atomicity: once aborted we stop claiming new chunks, so not all 8 were read. - expect(source.reads.length).toBeLessThan(8); + // Definitive reject on block 0 → stop immediately, don't PUT later blocks. + expect(source.reads.length).toBeLessThan(3); }); }); diff --git a/packages/protocol/src/highway/highway-client.ts b/packages/protocol/src/highway/highway-client.ts index 82f8bb72..422c52e5 100644 --- a/packages/protocol/src/highway/highway-client.ts +++ b/packages/protocol/src/highway/highway-client.ts @@ -28,17 +28,13 @@ const HIGHWAY_BLOCK_SIZE = 1024 * 1024; const HIGHWAY_MAX_CHUNK_ATTEMPTS = 3; const HIGHWAY_RETRY_BASE_MS = 300; -// Concurrent chunk upload fan-out. A large file's chunks are independent — -// each highway frame is self-contained (its own dataOffset, chunkMd5, fileMd5 -// and serviceTicket) and the server reassembles by offset, so chunks can be -// PUT in any order over parallel connections. Uploading them one-at-a-time -// (the old behavior) capped throughput at ~1 chunk / RTT AND paid TCP -// slow-start on every fresh per-chunk connection, so a 1 MiB chunk finished -// before its cold congestion window opened — the upload never reached line -// rate (issue #211). A small pool overlaps the RTT stalls and lets several -// connections ramp their windows at once. Kept modest so we stay a good -// citizen on QQ's highway edge nodes (which rate-limit — see error_code=921). -const HIGHWAY_UPLOAD_CONCURRENCY = 4; +// Idle read timeout for a highway response. tcpConnect() clears the socket's +// own timeout once connected, so a peer that accepts the connection but then +// neither responds nor FINs would hang the whole upload forever. The single +// persistent connection (issue #211) keeps a socket alive across the whole +// file, widening that exposure, so the response reader arms its own idle timer +// and treats a stall as a retryable transport failure. +const HIGHWAY_READ_IDLE_MS = 30000; const sleepMs = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); @@ -196,7 +192,18 @@ function readHttpResponseBody(socket: net.Socket): Promise { let totalNeeded = 0; let settled = false; + // Reset on every inbound chunk; fire if the peer goes silent mid-response. + let idleTimer: ReturnType | null = null; + const armIdle = () => { + if (idleTimer) clearTimeout(idleTimer); + idleTimer = setTimeout( + () => finish(() => reject(new Error('highway response read timeout (peer idle)'))), + HIGHWAY_READ_IDLE_MS, + ); + }; + const detach = () => { + if (idleTimer) { clearTimeout(idleTimer); idleTimer = null; } socket.off('data', onData); socket.off('error', onError); socket.off('close', onClose); @@ -208,6 +215,7 @@ function readHttpResponseBody(socket: net.Socket): Promise { fn(); }; const onData = (chunk: Buffer) => { + armIdle(); chunks.push(chunk); const buf = Buffer.concat(chunks); if (headerEnd < 0) { @@ -227,17 +235,30 @@ function readHttpResponseBody(socket: net.Socket): Promise { const onError = (err: Error) => finish(() => reject(err)); const onClose = () => finish(() => { const buf = Buffer.concat(chunks); - if (headerEnd >= 0) resolve(new Uint8Array(buf.subarray(headerEnd))); - // QQ 的 highway 边缘节点对单连接多 POST 的 keep-alive 支持不稳定: - // 第一次 POST 拿到响应后服务器经常立刻 FIN 关闭,下一次写入的请求体 - // 走到一个已被对端 close 的 socket 上,触发这里。诊断信息把已收到的 - // 字节数和是否解析到 header 一起带出来,便于区分 “连接刚握手就被关” - // 和 “响应读到一半被截断” 两种情况。 - else reject(new Error( - `connection closed before response (received=${buf.length}B, headerSeen=${headerEnd >= 0})` - )); + // A fully-received body would already have resolved in onData; reaching + // here means the peer FIN'd before the response completed. Only a + // close-delimited body (no Content-Length) is legitimately ended by the + // FIN — resolve that. A declared-but-unmet Content-Length, or a + // half-received header, is a TRUNCATED response: reject it as a + // retryable transport failure so the caller's attempt loop reconnects + // and re-sends this block, instead of resolving a partial frame that + // unpackHighwayFrame would fatally reject OUTSIDE that loop (turning a + // recoverable mid-body FIN into a failed upload). + // + // QQ 的 highway 边缘节点对单连接多 POST 的 keep-alive 支持不稳定,常在 + // 响应后立刻 FIN;诊断信息带上已收字节数、是否见到 header、需要多少, + // 便于区分 “连接刚握手就被关” 和 “响应读到一半被截断”。 + if (headerEnd >= 0 && contentLength === 0) { + resolve(new Uint8Array(buf.subarray(headerEnd))); + } else { + reject(new Error( + `connection closed before full response (received=${buf.length}B, ` + + `headerSeen=${headerEnd >= 0}${headerEnd >= 0 ? `, need=${totalNeeded}` : ''})`, + )); + } }); + armIdle(); socket.on('data', onData); socket.on('error', onError); socket.on('close', onClose); @@ -318,142 +339,118 @@ export async function uploadHighwayHttp( const pathStr = `/cgi-bin/httpconn?htcmd=0x6FF0087&uin=${bridge.identity.uin}`; const totalSize = source.size; - // Upload exactly one chunk. Extracted verbatim from the old sequential loop - // body — the per-chunk wire behavior is unchanged, only the *scheduling* - // across chunks becomes concurrent (see HIGHWAY_UPLOAD_CONCURRENCY). + // Single-connection, in-order, pipelined upload (issue #211). // - // 每个 chunk 仍然一个独立的 TCP 连接。 + // Blocks are PUT in STRICT offset order over ONE persistent connection, + // reused across chunks via HTTP keep-alive. Two properties matter: + // - In order: QQ's highway server reassembles by offset but REJECTS a + // block that arrives after a later one (error_code=102902) — parallel or + // out-of-order PUTs do not work (proven on a real endpoint). Sequential + // send guarantees in-order arrival. + // - One warm connection: reusing the socket keeps the TCP congestion + // window open across the whole file instead of paying slow-start on a + // fresh connection for every 1 MiB block. That per-block cold start was + // the real throughput cap behind #211. // - // 旧实现整个文件复用同一个 socket,依赖 `Connection: keep-alive` 跑多次 - // POST。但 QQ highway 边缘节点(以及链路上的代理/NAT)经常在第一次响应 - // 之后立刻 FIN 关闭连接,导致第二个 chunk 写入一个已被对端关闭的 socket, - // `readHttpResponseBody` 立刻收到 `close` 事件并抛 `connection closed - // before response`(#118)。所以每个 chunk 独立建连的不变量必须保留 —— - // 并发只是让多个这样的一次性连接同时在飞,连接之间没有任何共享状态。 + // QQ's edge nodes (and proxies/NAT on the path) sometimes FIN between + // chunks — that is what forced the one-connection-per-chunk workaround in + // #118. We handle it adaptively: if reusing the socket fails, or the peer + // half-closed after a response, we drop it, reconnect, and re-send the SAME + // block (re-PUTting an offset is idempotent). Worst case — a server that + // closes after every response — degrades to one connection per block: no + // worse than the #118 behavior, and still strictly in order (so never + // error_code=102902). `connectCount` is logged so a real run reveals which + // regime we hit (1 connection = keep-alive honored; ~chunkCount = closed + // every time). // - // Data safety: `source.read` returns a fresh buffer per call (FileChunkSource - // allocates; BufferChunkSource returns a read-only subarray), and reads are - // positional (offset-addressed), so concurrent reads at distinct offsets are - // race-free. Each chunk computes its own md5 over its own bytes. - const uploadChunk = async (offset: number, chunkSize: number): Promise => { - const chunk = await source.read(offset, chunkSize); - const chunkMd5 = computeMd5(chunk); - const head = makeHighwayHead( - bridge.identity.uin, commandId, totalSize, offset, chunkSize, - chunkMd5, fileMd5, session.sigSession, extend, - ); - const frame = packHighwayFrame(head, chunk); - - // Retry the connect + POST for this chunk on transient transport errors - // (peer FIN before response, ECONNRESET, connect refused/timeout). Each - // attempt uses a fresh connection; re-sending the same offset range is - // idempotent on the server. - let responseBody: Uint8Array | undefined; - for (let attempt = 1; ; attempt++) { - let socket: net.Socket; - try { - socket = await tcpConnect(session.host, session.port); - } catch (err) { - if (attempt >= HIGHWAY_MAX_CHUNK_ATTEMPTS) { - throw new Error( - `highway connect failed after ${attempt} attempts ` + - `(cmdId=${commandId} offset=${offset}/${totalSize}): ${String(err)}`, - ); - } - await sleepMs(HIGHWAY_RETRY_BASE_MS * attempt); - continue; - } - try { - responseBody = await httpPostFrame(socket, session.host, pathStr, frame); - break; - } catch (err) { - if (attempt >= HIGHWAY_MAX_CHUNK_ATTEMPTS) { - throw new Error( - `highway upload transport failed after ${attempt} attempts ` + - `(cmdId=${commandId} chunk=${chunkSize}/${totalSize} offset=${offset}): ${String(err)}`, - ); - } - log.trace('chunk offset=%d attempt %d failed (%s), retrying', offset, attempt, String(err)); - await sleepMs(HIGHWAY_RETRY_BASE_MS * attempt); - } finally { - socket.destroy(); - } - } - - // Unreachable: the retry loop only exits via break (responseBody set) or - // throw — the guard just narrows the type for the compiler. - if (!responseBody) throw new Error('highway upload: missing response'); - const { head: respHead } = unpackHighwayFrame(responseBody); - const resp = protobuf_decode(respHead); - if (resp?.errorCode && resp.errorCode !== 0) { - // Surface every diagnostic the highway response carries so - // user reports of `error_code=921` and friends include the - // server-side context (segHead.retCode, chunk size, file md5) - // — without these we can't tell apart a malformed-payload - // reject, a session-ticket mismatch, or a per-account rate- - // limit. - const segRetCode = resp.msgSegHead?.retCode ?? 0; - const fileMd5Hex = Buffer.from(fileMd5).toString('hex'); - throw new Error( - `highway upload error_code=${resp.errorCode}` + - ` (cmdId=${commandId} chunk=${chunkSize}/${totalSize}` + - ` offset=${offset} segRetCode=${segRetCode}` + - ` fileMd5=${fileMd5Hex.slice(0, 16)}…)`, - ); - } - }; - - // Concurrent worker pool over the chunk range. - // - // Atomicity / consistency: an offset is claimed synchronously — there is NO - // await between reading `nextOffset` and advancing it — so two workers can - // never claim the same chunk, and every byte range is PUT exactly once. On - // the first permanent chunk failure a worker sets `aborted`; peers stop - // pulling new chunks but drain the one already in flight. `worker()` NEVER - // rejects (it captures its error), so `Promise.all` always resolves only - // after every in-flight read/socket has settled — that is what makes it safe - // to `source.close()` in the finally without racing a live read. - // - // Rollback: a thrown upload leaves the server holding an incomplete set of - // chunks that it never finalizes — identical to the pre-concurrency failure - // mode. The caller's publish/finalize step (which makes the file visible) - // only runs when this resolves, so a partial upload is never published. - let nextOffset = 0; - let uploaded = 0; - let aborted = false; - let firstError: Error | null = null; - - const worker = async (): Promise => { - while (!aborted) { - const offset = nextOffset; - if (offset >= totalSize) return; - const chunkSize = Math.min(HIGHWAY_BLOCK_SIZE, totalSize - offset); - nextOffset = offset + chunkSize; // claim — synchronous, race-free - try { - await uploadChunk(offset, chunkSize); - } catch (err) { - aborted = true; - firstError ??= err instanceof Error ? err : new Error(String(err)); - return; - } - uploaded += chunkSize; - log.trace('uploaded %d/%d bytes', uploaded, totalSize); - } - }; - // `source` may buffer the whole file (BufferChunkSource) or stream it from - // disk (FileChunkSource). We own the source and close it exactly once below, - // only after all workers have settled. + // disk (FileChunkSource); either way one chunk is read/held at a time. We + // own the source and close it exactly once in the `finally` below. let succeeded = false; + let socket: net.Socket | null = null; + let connectCount = 0; + const dropSocket = (): void => { + if (socket) { socket.destroy(); socket = null; } + }; try { - const chunkCount = Math.max(1, Math.ceil(totalSize / HIGHWAY_BLOCK_SIZE)); - const workerCount = Math.min(HIGHWAY_UPLOAD_CONCURRENCY, chunkCount); - await Promise.all(Array.from({ length: workerCount }, () => worker())); - if (firstError) throw firstError; + let offset = 0; + while (offset < totalSize) { + const chunkSize = Math.min(HIGHWAY_BLOCK_SIZE, totalSize - offset); + const chunk = await source.read(offset, chunkSize); + const chunkMd5 = computeMd5(chunk); + const head = makeHighwayHead( + bridge.identity.uin, commandId, totalSize, offset, chunkSize, + chunkMd5, fileMd5, session.sigSession, extend, + ); + const frame = packHighwayFrame(head, chunk); + + // Send this block, reusing the live connection when there is one. On any + // transport failure (peer FIN between chunks, ECONNRESET, connect + // refused/timeout) drop the socket, reconnect, and retry THIS block — + // re-sending the same offset range is idempotent on the server. + let responseBody: Uint8Array | undefined; + for (let attempt = 1; ; attempt++) { + try { + if (!socket) { + socket = await tcpConnect(session.host, session.port); + connectCount += 1; + } + responseBody = await httpPostFrame(socket, session.host, pathStr, frame); + break; + } catch (err) { + dropSocket(); + if (attempt >= HIGHWAY_MAX_CHUNK_ATTEMPTS) { + throw new Error( + `highway upload transport failed after ${attempt} attempts ` + + `(cmdId=${commandId} chunk=${chunkSize}/${totalSize} offset=${offset}): ${String(err)}`, + ); + } + log.trace('chunk offset=%d attempt %d failed (%s), reconnecting', offset, attempt, String(err)); + await sleepMs(HIGHWAY_RETRY_BASE_MS * attempt); + } + } + + // Unreachable: the retry loop only exits via break (responseBody set) or + // throw — the guard just narrows the type for the compiler. + if (!responseBody) throw new Error('highway upload: missing response'); + const { head: respHead } = unpackHighwayFrame(responseBody); + const resp = protobuf_decode(respHead); + if (resp?.errorCode && resp.errorCode !== 0) { + // Surface every diagnostic the highway response carries so + // user reports of `error_code=921` and friends include the + // server-side context (segHead.retCode, chunk size, file md5) + // — without these we can't tell apart a malformed-payload + // reject, a session-ticket mismatch, or a per-account rate- + // limit. + const segRetCode = resp.msgSegHead?.retCode ?? 0; + const fileMd5Hex = Buffer.from(fileMd5).toString('hex'); + throw new Error( + `highway upload error_code=${resp.errorCode}` + + ` (cmdId=${commandId} chunk=${chunkSize}/${totalSize}` + + ` offset=${offset} segRetCode=${segRetCode}` + + ` fileMd5=${fileMd5Hex.slice(0, 16)}…)`, + ); + } + + // If the peer half-closed after sending its response, the socket can no + // longer carry the next block — drop it now so the next iteration + // reconnects cleanly instead of writing into a dead socket. + if (socket && (socket.destroyed || socket.readableEnded || !socket.writable)) { + dropSocket(); + } + + offset += chunkSize; + log.trace('uploaded %d/%d bytes', offset, totalSize); + } succeeded = true; + if (totalSize > 0) { + log.debug('highway upload done: %d bytes over %d connection(s) (cmdId=%d)', + totalSize, connectCount, commandId); + } } finally { // Best-effort close — never mask a primary upload error. A close failure // is only surfaced when the upload itself succeeded. + dropSocket(); try { await source.close(); } catch (closeErr) {