perf(highway): single-connection pipelined upload — supersede #211 concurrency
The #211 concurrency attempt (c17fff6) was disproven on a live endpoint.
QQ's highway server requires blocks to arrive in offset order and rejects
an out-of-order PUT with error_code=102902 — ~20% failure under a real
account even after deferring the finalize block. Parallel connections
can't guarantee arrival order, so that approach is unshippable.
Replace it with a single persistent connection that PUTs blocks in strict
offset order, reusing the socket across the whole file via HTTP keep-alive:
- In order: the server never sees an out-of-order block, so no 102902.
- One warm connection: the TCP congestion window stays open across the
file instead of paying slow-start on a fresh connection per 1 MiB block.
That per-block cold start was the real throughput cap behind #211.
Adaptive to QQ's edge nodes FIN'ing between chunks (the reason #118 went
one-connection-per-chunk): on a transport failure, or a peer half-close
after a response, drop the socket, reconnect, and re-send the SAME block
(re-PUTting an offset is idempotent). Worst case degrades to one
connection per block — no worse than #118, and still strictly in order.
Also fixes a latent bug on the reconnect path, surfaced by review: a
response truncated mid-stream (header arrived, Content-Length unmet, then
FIN) was resolved as a partial frame and then fatally rejected by
unpackHighwayFrame OUTSIDE the retry loop, turning a recoverable mid-body
FIN into a failed upload. A truncated response now rejects as a retryable
transport failure so the block is re-sent. Added an idle read timeout so
a silent peer can't hang the upload forever (the persistent socket widens
that exposure).
Verified live (tempserver, repeated 48 MiB group-file uploads): 1
connection per file, 0 failures across dense repeats, ~1.4x throughput
over the old per-chunk-connection serial path in a low-RTT datacenter —
the win is larger on high-RTT home links where per-block slow-start hurt
most (the original #211 report). Unit tests cover keep-alive reuse,
FIN-after-each-response degradation, mid-stream reconnect+resume, body-
truncation retry, in-order exactly-once coverage, and close-once.
This commit is contained in:
@@ -1,23 +1,24 @@
|
||||
// highway-client.uploadHighwayHttp 的连接生命周期测试。
|
||||
//
|
||||
// 回归覆盖:当上传数据 > HIGHWAY_BLOCK_SIZE (1 MiB) 时,旧实现会复用
|
||||
// 同一个 TCP socket 在 keep-alive 模式下连续 POST,但 QQ highway 的
|
||||
// 边缘节点经常在第一次响应后立刻 FIN 关闭,导致第二次 POST 报
|
||||
// `connection closed before response`。用户上报的 1.19 MB 图片
|
||||
// (onebot retcode=1200)就是这个分支。
|
||||
// 方案 3(#211):单连接、按 offset 顺序、流水线上传,跨 chunk 复用同一个
|
||||
// TCP 连接(HTTP keep-alive)。要点:
|
||||
// - 顺序发送 → 天然保序,避开 QQ highway 对乱序块的 error_code=102902;
|
||||
// - 复用连接 → TCP 拥塞窗口全程保持,省掉每块新建连接的 slow-start。
|
||||
// 边缘节点可能在响应后 FIN(#118);此时自适应地丢弃 socket + 重连 + 重发
|
||||
// 同一块(重发幂等),最坏退化成“每块一连接”,不比旧实现差,且仍然保序。
|
||||
//
|
||||
// 修复方案:每个 chunk 独立建一个新的 TCP 连接。本测试通过 mock
|
||||
// `net.createConnection` 计数建连次数,断言每个 chunk 一连接的不变量。
|
||||
// 本测试用两种 mock 服务端行为锁定这套自适应逻辑:
|
||||
// - keepalive:每个 POST 回一个响应,连接保持 → 整个文件 1 个连接;
|
||||
// - closeafter:每个响应后半关闭(readableEnded/emit close)→ 退化成每块
|
||||
// 一连接,但仍全部成功。
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { EventEmitter } from 'events';
|
||||
|
||||
// highway-client 解析响应只关心 errorCode === 0 —— 一个完全空的
|
||||
// RespDataHighwayHead protobuf 体(0 字节)会被解为 ,errorCode
|
||||
// 是 undefined,走 falsy 通过分支。
|
||||
//
|
||||
// HTTP body 是 highway frame:
|
||||
// 0x28 | head_len(BE32) | body_len(BE32) | head | body | 0x29
|
||||
// RespDataHighwayHead protobuf 体(0 字节)会被解为 errorCode=undefined,
|
||||
// 走 falsy 通过分支。
|
||||
// highway frame:0x28 | head_len(BE32) | body_len(BE32) | head | body | 0x29
|
||||
function buildEmptyHighwayResponseFrame(): Buffer {
|
||||
const frame = Buffer.alloc(10);
|
||||
frame[0] = 0x28;
|
||||
@@ -27,59 +28,113 @@ function buildEmptyHighwayResponseFrame(): Buffer {
|
||||
return frame;
|
||||
}
|
||||
|
||||
function buildHttpResponse(frameBody: Buffer): Buffer {
|
||||
const headers = [
|
||||
'HTTP/1.1 200 OK',
|
||||
`Content-Length: ${frameBody.length}`,
|
||||
'Connection: close',
|
||||
'',
|
||||
'',
|
||||
].join('\r\n');
|
||||
return Buffer.concat([Buffer.from(headers, 'ascii'), frameBody]);
|
||||
// errorCode 帧:RespDataHighwayHead.field3 (errorCode) = 921。tag(field3,
|
||||
// varint)=0x18,varint(921)=0x99 0x07。定义即拒绝(不重试)。
|
||||
function buildErrorCodeFrame(): Buffer {
|
||||
const head = Buffer.from([0x18, 0x99, 0x07]);
|
||||
const frame = Buffer.alloc(9 + head.length + 1);
|
||||
frame[0] = 0x28;
|
||||
frame.writeUInt32BE(head.length, 1);
|
||||
frame.writeUInt32BE(0, 5);
|
||||
head.copy(frame, 9);
|
||||
frame[9 + head.length] = 0x29;
|
||||
return frame;
|
||||
}
|
||||
|
||||
// FakeSocket 模拟 net.Socket 的最小子集。highway-client 用到:
|
||||
// - 'data' / 'error' / 'close' 事件
|
||||
// - .write(data, cb) cb 会被 socketWrite 等待
|
||||
// - .destroy()
|
||||
// - .setTimeout / .once / .removeListener (继承自 EventEmitter)
|
||||
function httpWrap(frame: Buffer): Buffer {
|
||||
const headers = ['HTTP/1.1 200 OK', `Content-Length: ${frame.length}`, '', ''].join('\r\n');
|
||||
return Buffer.concat([Buffer.from(headers, 'ascii'), frame]);
|
||||
}
|
||||
|
||||
type ServerMode = 'keepalive' | 'closeafter';
|
||||
|
||||
// FakeSocket 模拟 net.Socket 的最小子集,支持在一个连接上顺序处理多个 POST。
|
||||
// 累积写入的字节,按 `\r\n\r\n` + Content-Length 切出每个完整 POST,逐个回
|
||||
// 响应。`mode` 控制响应后是否半关闭;`failOnPost` 用于模拟链路中途断开。
|
||||
class FakeSocket extends EventEmitter {
|
||||
destroyed = false;
|
||||
// 每个 POST = 2 次 write(HTTP header + body)。第二次 write 落地后
|
||||
// 异步回放响应并立刻 close —— 复现 QQ highway 真实行为:响应一发完
|
||||
// 就 FIN,旧实现的下一次 POST 必然撞上已关闭的 socket。
|
||||
private writeCount = 0;
|
||||
private response: Buffer;
|
||||
|
||||
constructor(response: Buffer) {
|
||||
readableEnded = false;
|
||||
writable = true;
|
||||
posts = 0;
|
||||
private buf = Buffer.alloc(0);
|
||||
constructor(
|
||||
private readonly responseFrame: Buffer,
|
||||
private readonly mode: ServerMode,
|
||||
private readonly failOnPost = -1,
|
||||
private readonly truncateOnPost = -1,
|
||||
) {
|
||||
super();
|
||||
this.response = response;
|
||||
}
|
||||
|
||||
setTimeout(_ms: number) { /* no-op */ }
|
||||
setTimeout(_ms?: number) { /* no-op */ }
|
||||
|
||||
write(_data: unknown, cb?: (err?: Error) => void): boolean {
|
||||
this.writeCount += 1;
|
||||
if (this.writeCount === 2) {
|
||||
setImmediate(() => {
|
||||
this.emit('data', this.response);
|
||||
this.emit('close');
|
||||
});
|
||||
write(data: unknown, cb?: (err?: Error) => void): boolean {
|
||||
if (this.destroyed || !this.writable) {
|
||||
const err = new Error('write after socket end');
|
||||
if (cb) cb(err);
|
||||
setImmediate(() => this.emit('error', err));
|
||||
return false;
|
||||
}
|
||||
this.buf = Buffer.concat([this.buf, Buffer.from(data as Uint8Array)]);
|
||||
this.drainPosts();
|
||||
if (cb) cb();
|
||||
return true;
|
||||
}
|
||||
|
||||
private drainPosts(): void {
|
||||
for (;;) {
|
||||
const s = this.buf.toString('latin1');
|
||||
const he = s.indexOf('\r\n\r\n');
|
||||
if (he < 0) return;
|
||||
const m = s.slice(0, he).match(/content-length:\s*(\d+)/i);
|
||||
const cl = m ? parseInt(m[1], 10) : 0;
|
||||
const total = he + 4 + cl;
|
||||
if (this.buf.length < total) return;
|
||||
this.buf = this.buf.subarray(total);
|
||||
this.posts += 1;
|
||||
const thisPost = this.posts;
|
||||
setImmediate(() => {
|
||||
if (this.destroyed) return;
|
||||
if (thisPost === this.failOnPost) {
|
||||
this.writable = false;
|
||||
this.readableEnded = true;
|
||||
this.emit('error', new Error('ECONNRESET'));
|
||||
this.emit('close');
|
||||
return;
|
||||
}
|
||||
if (thisPost === this.truncateOnPost) {
|
||||
// Send the HTTP header + only part of the declared body, then FIN —
|
||||
// a truncated response. Dropping the last 3 body bytes keeps the
|
||||
// header intact so Content-Length parses but is never satisfied.
|
||||
const full = httpWrap(this.responseFrame);
|
||||
this.emit('data', full.subarray(0, full.length - 3));
|
||||
this.writable = false;
|
||||
this.readableEnded = true;
|
||||
this.emit('close');
|
||||
return;
|
||||
}
|
||||
this.emit('data', httpWrap(this.responseFrame));
|
||||
if (this.mode === 'closeafter') {
|
||||
this.readableEnded = true;
|
||||
this.writable = false;
|
||||
this.emit('end');
|
||||
this.emit('close');
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
destroy() {
|
||||
this.destroyed = true;
|
||||
this.writable = false;
|
||||
}
|
||||
}
|
||||
|
||||
// vitest 把 vi.mock 工厂 hoist 到文件顶部,所以工厂里不能引用普通的
|
||||
// 顶层 const —— 那会触发 "Cannot access X before initialization"。
|
||||
// 用 vi.hoisted 让这两个值跟着 mock 一起被 hoist。
|
||||
const { createdSockets, createConnectionMock } = vi.hoisted(() => ({
|
||||
createdSockets: [] as FakeSocket[],
|
||||
// vitest 把 vi.mock 工厂 hoist 到文件顶部,所以工厂里不能引用普通的顶层
|
||||
// const。用 vi.hoisted 让这些值跟着 mock 一起 hoist。
|
||||
const { createdSockets, socketFactoryRef, createConnectionMock } = vi.hoisted(() => ({
|
||||
createdSockets: [] as unknown[],
|
||||
socketFactoryRef: { make: null as null | ((idx: number) => unknown) },
|
||||
createConnectionMock: vi.fn(),
|
||||
}));
|
||||
|
||||
@@ -88,15 +143,18 @@ vi.mock('net', () => ({
|
||||
createConnection: createConnectionMock,
|
||||
}));
|
||||
|
||||
import { uploadHighwayHttp, BufferChunkSource, type ChunkSource, type HighwaySession } from '@snowluma/protocol/highway';
|
||||
import {
|
||||
uploadHighwayHttp,
|
||||
BufferChunkSource,
|
||||
type ChunkSource,
|
||||
type HighwaySession,
|
||||
} from '@snowluma/protocol/highway';
|
||||
import type { BridgeContext } from '@snowluma/protocol/bridge-context';
|
||||
|
||||
const HIGHWAY_BLOCK_SIZE = 1024 * 1024;
|
||||
|
||||
function makeBridge(): BridgeContext {
|
||||
return {
|
||||
identity: { uin: '10001' } as unknown,
|
||||
} as unknown as BridgeContext;
|
||||
return { identity: { uin: '10001' } as unknown } as unknown as BridgeContext;
|
||||
}
|
||||
|
||||
function makeSession(): HighwaySession {
|
||||
@@ -108,198 +166,141 @@ function makeSession(): HighwaySession {
|
||||
};
|
||||
}
|
||||
|
||||
describe('uploadHighwayHttp connection lifecycle', () => {
|
||||
beforeEach(() => {
|
||||
createdSockets.length = 0;
|
||||
createConnectionMock.mockReset();
|
||||
createConnectionMock.mockImplementation((_opts: unknown, listener?: () => void) => {
|
||||
const sock = new FakeSocket(buildHttpResponse(buildEmptyHighwayResponseFrame()));
|
||||
createdSockets.push(sock);
|
||||
// net.createConnection 在握手成功后调用 listener。同步触发即可,
|
||||
// tcpConnect 的 Promise 会立刻 resolve。
|
||||
if (listener) setImmediate(listener);
|
||||
return sock as unknown as ReturnType<typeof createConnectionMock>;
|
||||
});
|
||||
// 安装一个 socket 工厂:createConnection 每次调用按序号创建一个 FakeSocket。
|
||||
function installFactory(make: (idx: number) => FakeSocket): void {
|
||||
createdSockets.length = 0;
|
||||
socketFactoryRef.make = make as (idx: number) => unknown;
|
||||
createConnectionMock.mockReset();
|
||||
createConnectionMock.mockImplementation((_opts: unknown, listener?: () => void) => {
|
||||
const idx = createdSockets.length;
|
||||
const sock = socketFactoryRef.make!(idx) as FakeSocket;
|
||||
createdSockets.push(sock);
|
||||
if (listener) setImmediate(listener); // 握手成功后异步回调 → tcpConnect resolve
|
||||
return sock as unknown as ReturnType<typeof createConnectionMock>;
|
||||
});
|
||||
}
|
||||
|
||||
// 顺序读取记录:offset/length + 关闭时机,用于数据完整性与 close 语义。
|
||||
class RecordingChunkSource implements ChunkSource {
|
||||
reads: Array<{ offset: number; length: number }> = [];
|
||||
inFlight = 0;
|
||||
closeCount = 0;
|
||||
closedWhileReading = false;
|
||||
constructor(readonly size: number) {}
|
||||
async read(offset: number, length: number): Promise<Uint8Array> {
|
||||
this.inFlight += 1;
|
||||
this.reads.push({ offset, length });
|
||||
await Promise.resolve();
|
||||
this.inFlight -= 1;
|
||||
return new Uint8Array(length);
|
||||
}
|
||||
async close(): Promise<void> {
|
||||
if (this.inFlight > 0) this.closedWhileReading = true;
|
||||
this.closeCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
describe('uploadHighwayHttp single-connection pipelined upload (#211)', () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it('reuses ONE connection for a multi-block payload when keep-alive holds', async () => {
|
||||
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'));
|
||||
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks
|
||||
await uploadHighwayHttp(
|
||||
makeBridge(), makeSession(), 1004, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0),
|
||||
);
|
||||
// The whole file went over a single warm connection — the #211 win.
|
||||
expect(createConnectionMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('uses a single TCP connection for a sub-block payload', async () => {
|
||||
// 0.5 MiB → 1 chunk。覆盖 PTT / 小图的常见场景。
|
||||
const bytes = new Uint8Array(512 * 1024);
|
||||
it('single sub-block payload uses one connection', async () => {
|
||||
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'));
|
||||
const bytes = new Uint8Array(512 * 1024); // 1 block (PTT / small image)
|
||||
await uploadHighwayHttp(
|
||||
makeBridge(), makeSession(), 1003, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0),
|
||||
);
|
||||
expect(createConnectionMock).toHaveBeenCalledTimes(1);
|
||||
expect(createdSockets[0]!.destroyed).toBe(true);
|
||||
});
|
||||
|
||||
it('uses a fresh TCP connection per chunk for multi-block payloads', async () => {
|
||||
// 1.5 MiB → ceil(1.5) = 2 chunks。这正是用户上报的 1.19 MB 图片
|
||||
// 触发的代码路径:旧实现复用 socket 时第二次 POST 撞上 keep-alive
|
||||
// 关闭,新实现必须为每个 chunk 各开一个 socket。
|
||||
const bytes = new Uint8Array(Math.floor(1.5 * HIGHWAY_BLOCK_SIZE));
|
||||
await uploadHighwayHttp(
|
||||
makeBridge(), makeSession(), 1003, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0),
|
||||
);
|
||||
expect(createConnectionMock).toHaveBeenCalledTimes(2);
|
||||
expect(createdSockets).toHaveLength(2);
|
||||
for (const sock of createdSockets) {
|
||||
expect(sock.destroyed).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('opens N connections for N chunks (3 chunks here)', async () => {
|
||||
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE));
|
||||
await uploadHighwayHttp(
|
||||
makeBridge(), makeSession(), 1004, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0),
|
||||
);
|
||||
expect(createConnectionMock).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('a zero-length source opens ZERO connections (empty-input invariant)', async () => {
|
||||
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'));
|
||||
await uploadHighwayHttp(
|
||||
makeBridge(), makeSession(), 1003, new BufferChunkSource(new Uint8Array(0)), new Uint8Array(16), new Uint8Array(0),
|
||||
);
|
||||
expect(createConnectionMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// 并发上传(#211)的正确性回归。重点覆盖用户要求的四个维度:
|
||||
// 数据安全 —— 每个 offset 恰好读一次、大小正确、覆盖整段 [0, size);
|
||||
// 原子性 —— 首个永久失败即中止,不再领新 chunk,整体 reject;
|
||||
// 回退 —— 失败/成功都只 close() 一次,且绝不在有 read 在飞时 close;
|
||||
// 一致性 —— 并发确实发生(同时在飞的连接 > 1)但不超过 chunk 数。
|
||||
describe('uploadHighwayHttp concurrent upload safety (#211)', () => {
|
||||
// 记录每次 read 的 offset/length,跟踪在飞 read 数与 close 时机。read 故意
|
||||
// 异步(await 一个 microtask),这样 close-while-reading 一旦发生就能被捕获。
|
||||
class RecordingChunkSource implements ChunkSource {
|
||||
reads: Array<{ offset: number; length: number }> = [];
|
||||
inFlight = 0;
|
||||
maxInFlight = 0;
|
||||
closeCount = 0;
|
||||
closedWhileReading = false;
|
||||
constructor(readonly size: number) {}
|
||||
async read(offset: number, length: number): Promise<Uint8Array> {
|
||||
this.inFlight += 1;
|
||||
this.maxInFlight = Math.max(this.maxInFlight, this.inFlight);
|
||||
this.reads.push({ offset, length });
|
||||
await Promise.resolve();
|
||||
this.inFlight -= 1;
|
||||
return new Uint8Array(length);
|
||||
}
|
||||
async close(): Promise<void> {
|
||||
if (this.inFlight > 0) this.closedWhileReading = true;
|
||||
this.closeCount += 1;
|
||||
}
|
||||
}
|
||||
it('degrades to one connection per block when the server FINs after every response (#118 regime)', async () => {
|
||||
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'closeafter'));
|
||||
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks
|
||||
await uploadHighwayHttp(
|
||||
makeBridge(), makeSession(), 1004, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0),
|
||||
);
|
||||
// Server closes after each response → adaptively reconnects per block, but
|
||||
// the upload still completes. No worse than the old one-conn-per-chunk.
|
||||
expect(createConnectionMock).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
// errorCode 帧:RespDataHighwayHead.field3 (errorCode) = 921。
|
||||
// 定义即拒绝(不重试),用来制造“瞬时永久失败”而不吃重试 sleep。
|
||||
// varint(921) = 0x99 0x07,tag(field3,varint) = 0x18。
|
||||
function buildErrorCodeFrame(): Buffer {
|
||||
const head = Buffer.from([0x18, 0x99, 0x07]);
|
||||
const frame = Buffer.alloc(9 + head.length + 1);
|
||||
frame[0] = 0x28;
|
||||
frame.writeUInt32BE(head.length, 1);
|
||||
frame.writeUInt32BE(0, 5);
|
||||
head.copy(frame, 9);
|
||||
frame[9 + head.length] = 0x29;
|
||||
return frame;
|
||||
}
|
||||
it('reconnects and re-sends the SAME block on a mid-stream transport error', async () => {
|
||||
// First connection dies while handling its first POST; a fresh connection
|
||||
// must re-send that same block (offset unchanged, idempotent) and finish.
|
||||
installFactory((idx) =>
|
||||
idx === 0
|
||||
? new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive', /* failOnPost */ 1)
|
||||
: new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'),
|
||||
);
|
||||
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks
|
||||
await uploadHighwayHttp(
|
||||
makeBridge(), makeSession(), 1004, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0),
|
||||
);
|
||||
// Connection #0 failed on block 0 → reconnect (#1) resumes and keep-alives
|
||||
// the rest: exactly two connections total.
|
||||
expect(createConnectionMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
// 跟踪同时存活(已连接、未 destroy)的 socket 峰值,用来断言并发确实发生。
|
||||
let liveSockets = 0;
|
||||
let maxLiveSockets = 0;
|
||||
it('reconnects and re-sends the block when a response body is truncated mid-stream', async () => {
|
||||
// Peer FINs after the HTTP header but before the highway body is complete.
|
||||
// That truncation must be treated as a retryable transport failure (reject
|
||||
// in readHttpResponseBody), NOT resolved as a partial frame — otherwise
|
||||
// unpackHighwayFrame throws OUTSIDE the retry loop and fails the upload.
|
||||
installFactory((idx) =>
|
||||
idx === 0
|
||||
? new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive', -1, /* truncateOnPost */ 1)
|
||||
: new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'),
|
||||
);
|
||||
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks
|
||||
await uploadHighwayHttp(
|
||||
makeBridge(), makeSession(), 1004, new BufferChunkSource(bytes), new Uint8Array(16), new Uint8Array(0),
|
||||
);
|
||||
// conn #0 truncated block 0 → reconnect (#1) re-sends it and finishes.
|
||||
expect(createConnectionMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
function installMock(responseFrame: Buffer): void {
|
||||
createConnectionMock.mockReset();
|
||||
liveSockets = 0;
|
||||
maxLiveSockets = 0;
|
||||
createConnectionMock.mockImplementation((_opts: unknown, listener?: () => void) => {
|
||||
const sock = new FakeSocket(buildHttpResponse(responseFrame));
|
||||
liveSockets += 1;
|
||||
maxLiveSockets = Math.max(maxLiveSockets, liveSockets);
|
||||
const origDestroy = sock.destroy.bind(sock);
|
||||
sock.destroy = () => { if (!sock.destroyed) liveSockets -= 1; origDestroy(); };
|
||||
createdSockets.push(sock);
|
||||
if (listener) setImmediate(listener);
|
||||
return sock as unknown as ReturnType<typeof createConnectionMock>;
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(() => { createdSockets.length = 0; });
|
||||
afterEach(() => { vi.restoreAllMocks(); });
|
||||
|
||||
it('covers every offset exactly once and closes the source exactly once (data safety + consistency)', async () => {
|
||||
installMock(buildEmptyHighwayResponseFrame());
|
||||
const chunks = 8;
|
||||
const size = chunks * HIGHWAY_BLOCK_SIZE;
|
||||
it('reads every offset exactly once, in order, and closes the source once (data safety)', async () => {
|
||||
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'));
|
||||
const size = 4 * HIGHWAY_BLOCK_SIZE + 4321; // 4 full + 1 ragged
|
||||
const source = new RecordingChunkSource(size);
|
||||
|
||||
await uploadHighwayHttp(
|
||||
makeBridge(), makeSession(), 1004, source, new Uint8Array(16), new Uint8Array(0),
|
||||
);
|
||||
|
||||
// Exactly-once, correctly-sized, gap-free coverage of [0, size).
|
||||
const offsets = source.reads.map((r) => r.offset).sort((a, b) => a - b);
|
||||
expect(offsets).toEqual(Array.from({ length: chunks }, (_, i) => i * HIGHWAY_BLOCK_SIZE));
|
||||
expect(new Set(offsets).size).toBe(chunks); // no duplicate offset
|
||||
expect(source.reads.every((r) => r.length === HIGHWAY_BLOCK_SIZE)).toBe(true);
|
||||
// One connection per chunk; source closed once, never mid-read.
|
||||
expect(createConnectionMock).toHaveBeenCalledTimes(chunks);
|
||||
const offsets = source.reads.map((r) => r.offset);
|
||||
// Strictly increasing, gap-free, exactly-once coverage of [0, size).
|
||||
expect(offsets).toEqual([0, HIGHWAY_BLOCK_SIZE, 2 * HIGHWAY_BLOCK_SIZE, 3 * HIGHWAY_BLOCK_SIZE, 4 * HIGHWAY_BLOCK_SIZE]);
|
||||
expect(source.reads[source.reads.length - 1]!.length).toBe(4321); // ragged tail
|
||||
expect(source.reads.reduce((n, r) => n + r.length, 0)).toBe(size);
|
||||
expect(source.closeCount).toBe(1);
|
||||
expect(source.closedWhileReading).toBe(false);
|
||||
});
|
||||
|
||||
it('actually uploads chunks in parallel but never exceeds the chunk count (concurrency)', async () => {
|
||||
installMock(buildEmptyHighwayResponseFrame());
|
||||
const chunks = 8;
|
||||
const source = new RecordingChunkSource(chunks * HIGHWAY_BLOCK_SIZE);
|
||||
|
||||
await uploadHighwayHttp(
|
||||
makeBridge(), makeSession(), 1004, source, new Uint8Array(16), new Uint8Array(0),
|
||||
);
|
||||
|
||||
// Parallelism happened (more than one connection in flight at once) …
|
||||
expect(maxLiveSockets).toBeGreaterThan(1);
|
||||
// … but the pool is bounded and never opens more than there are chunks.
|
||||
expect(maxLiveSockets).toBeLessThanOrEqual(chunks);
|
||||
});
|
||||
|
||||
it('the last (ragged) chunk carries the remainder length', async () => {
|
||||
installMock(buildEmptyHighwayResponseFrame());
|
||||
const size = 2 * HIGHWAY_BLOCK_SIZE + 12345; // 2 full + 1 partial
|
||||
const source = new RecordingChunkSource(size);
|
||||
|
||||
await uploadHighwayHttp(
|
||||
makeBridge(), makeSession(), 1004, source, new Uint8Array(16), new Uint8Array(0),
|
||||
);
|
||||
|
||||
const byOffset = new Map(source.reads.map((r) => [r.offset, r.length]));
|
||||
expect(byOffset.get(0)).toBe(HIGHWAY_BLOCK_SIZE);
|
||||
expect(byOffset.get(HIGHWAY_BLOCK_SIZE)).toBe(HIGHWAY_BLOCK_SIZE);
|
||||
expect(byOffset.get(2 * HIGHWAY_BLOCK_SIZE)).toBe(12345);
|
||||
const total = source.reads.reduce((n, r) => n + r.length, 0);
|
||||
expect(total).toBe(size); // every byte accounted for exactly once
|
||||
});
|
||||
|
||||
it('aborts the whole upload on a definitive server reject, closing the source once and never mid-read (atomicity + rollback)', async () => {
|
||||
// Every chunk gets error_code=921 → definitive reject, no retries. The first
|
||||
// failure must abort; the source must still be closed exactly once and never
|
||||
// while a read is in flight.
|
||||
installMock(buildErrorCodeFrame());
|
||||
const source = new RecordingChunkSource(8 * HIGHWAY_BLOCK_SIZE);
|
||||
|
||||
it('rejects on a definitive server error_code and closes the source once (no retry)', async () => {
|
||||
installFactory(() => new FakeSocket(buildErrorCodeFrame(), 'keepalive'));
|
||||
const source = new RecordingChunkSource(3 * HIGHWAY_BLOCK_SIZE);
|
||||
await expect(uploadHighwayHttp(
|
||||
makeBridge(), makeSession(), 1004, source, new Uint8Array(16), new Uint8Array(0),
|
||||
)).rejects.toThrow(/error_code=921/);
|
||||
|
||||
expect(source.closeCount).toBe(1);
|
||||
expect(source.closedWhileReading).toBe(false);
|
||||
// Atomicity: once aborted we stop claiming new chunks, so not all 8 were read.
|
||||
expect(source.reads.length).toBeLessThan(8);
|
||||
// Definitive reject on block 0 → stop immediately, don't PUT later blocks.
|
||||
expect(source.reads.length).toBeLessThan(3);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -28,17 +28,13 @@ const HIGHWAY_BLOCK_SIZE = 1024 * 1024;
|
||||
const HIGHWAY_MAX_CHUNK_ATTEMPTS = 3;
|
||||
const HIGHWAY_RETRY_BASE_MS = 300;
|
||||
|
||||
// Concurrent chunk upload fan-out. A large file's chunks are independent —
|
||||
// each highway frame is self-contained (its own dataOffset, chunkMd5, fileMd5
|
||||
// and serviceTicket) and the server reassembles by offset, so chunks can be
|
||||
// PUT in any order over parallel connections. Uploading them one-at-a-time
|
||||
// (the old behavior) capped throughput at ~1 chunk / RTT AND paid TCP
|
||||
// slow-start on every fresh per-chunk connection, so a 1 MiB chunk finished
|
||||
// before its cold congestion window opened — the upload never reached line
|
||||
// rate (issue #211). A small pool overlaps the RTT stalls and lets several
|
||||
// connections ramp their windows at once. Kept modest so we stay a good
|
||||
// citizen on QQ's highway edge nodes (which rate-limit — see error_code=921).
|
||||
const HIGHWAY_UPLOAD_CONCURRENCY = 4;
|
||||
// Idle read timeout for a highway response. tcpConnect() clears the socket's
|
||||
// own timeout once connected, so a peer that accepts the connection but then
|
||||
// neither responds nor FINs would hang the whole upload forever. The single
|
||||
// persistent connection (issue #211) keeps a socket alive across the whole
|
||||
// file, widening that exposure, so the response reader arms its own idle timer
|
||||
// and treats a stall as a retryable transport failure.
|
||||
const HIGHWAY_READ_IDLE_MS = 30000;
|
||||
|
||||
const sleepMs = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
@@ -196,7 +192,18 @@ function readHttpResponseBody(socket: net.Socket): Promise<Uint8Array> {
|
||||
let totalNeeded = 0;
|
||||
let settled = false;
|
||||
|
||||
// Reset on every inbound chunk; fire if the peer goes silent mid-response.
|
||||
let idleTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
const armIdle = () => {
|
||||
if (idleTimer) clearTimeout(idleTimer);
|
||||
idleTimer = setTimeout(
|
||||
() => finish(() => reject(new Error('highway response read timeout (peer idle)'))),
|
||||
HIGHWAY_READ_IDLE_MS,
|
||||
);
|
||||
};
|
||||
|
||||
const detach = () => {
|
||||
if (idleTimer) { clearTimeout(idleTimer); idleTimer = null; }
|
||||
socket.off('data', onData);
|
||||
socket.off('error', onError);
|
||||
socket.off('close', onClose);
|
||||
@@ -208,6 +215,7 @@ function readHttpResponseBody(socket: net.Socket): Promise<Uint8Array> {
|
||||
fn();
|
||||
};
|
||||
const onData = (chunk: Buffer) => {
|
||||
armIdle();
|
||||
chunks.push(chunk);
|
||||
const buf = Buffer.concat(chunks);
|
||||
if (headerEnd < 0) {
|
||||
@@ -227,17 +235,30 @@ function readHttpResponseBody(socket: net.Socket): Promise<Uint8Array> {
|
||||
const onError = (err: Error) => finish(() => reject(err));
|
||||
const onClose = () => finish(() => {
|
||||
const buf = Buffer.concat(chunks);
|
||||
if (headerEnd >= 0) resolve(new Uint8Array(buf.subarray(headerEnd)));
|
||||
// QQ 的 highway 边缘节点对单连接多 POST 的 keep-alive 支持不稳定:
|
||||
// 第一次 POST 拿到响应后服务器经常立刻 FIN 关闭,下一次写入的请求体
|
||||
// 走到一个已被对端 close 的 socket 上,触发这里。诊断信息把已收到的
|
||||
// 字节数和是否解析到 header 一起带出来,便于区分 “连接刚握手就被关”
|
||||
// 和 “响应读到一半被截断” 两种情况。
|
||||
else reject(new Error(
|
||||
`connection closed before response (received=${buf.length}B, headerSeen=${headerEnd >= 0})`
|
||||
));
|
||||
// A fully-received body would already have resolved in onData; reaching
|
||||
// here means the peer FIN'd before the response completed. Only a
|
||||
// close-delimited body (no Content-Length) is legitimately ended by the
|
||||
// FIN — resolve that. A declared-but-unmet Content-Length, or a
|
||||
// half-received header, is a TRUNCATED response: reject it as a
|
||||
// retryable transport failure so the caller's attempt loop reconnects
|
||||
// and re-sends this block, instead of resolving a partial frame that
|
||||
// unpackHighwayFrame would fatally reject OUTSIDE that loop (turning a
|
||||
// recoverable mid-body FIN into a failed upload).
|
||||
//
|
||||
// QQ 的 highway 边缘节点对单连接多 POST 的 keep-alive 支持不稳定,常在
|
||||
// 响应后立刻 FIN;诊断信息带上已收字节数、是否见到 header、需要多少,
|
||||
// 便于区分 “连接刚握手就被关” 和 “响应读到一半被截断”。
|
||||
if (headerEnd >= 0 && contentLength === 0) {
|
||||
resolve(new Uint8Array(buf.subarray(headerEnd)));
|
||||
} else {
|
||||
reject(new Error(
|
||||
`connection closed before full response (received=${buf.length}B, ` +
|
||||
`headerSeen=${headerEnd >= 0}${headerEnd >= 0 ? `, need=${totalNeeded}` : ''})`,
|
||||
));
|
||||
}
|
||||
});
|
||||
|
||||
armIdle();
|
||||
socket.on('data', onData);
|
||||
socket.on('error', onError);
|
||||
socket.on('close', onClose);
|
||||
@@ -318,142 +339,118 @@ export async function uploadHighwayHttp(
|
||||
const pathStr = `/cgi-bin/httpconn?htcmd=0x6FF0087&uin=${bridge.identity.uin}`;
|
||||
const totalSize = source.size;
|
||||
|
||||
// Upload exactly one chunk. Extracted verbatim from the old sequential loop
|
||||
// body — the per-chunk wire behavior is unchanged, only the *scheduling*
|
||||
// across chunks becomes concurrent (see HIGHWAY_UPLOAD_CONCURRENCY).
|
||||
// Single-connection, in-order, pipelined upload (issue #211).
|
||||
//
|
||||
// 每个 chunk 仍然一个独立的 TCP 连接。
|
||||
// Blocks are PUT in STRICT offset order over ONE persistent connection,
|
||||
// reused across chunks via HTTP keep-alive. Two properties matter:
|
||||
// - In order: QQ's highway server reassembles by offset but REJECTS a
|
||||
// block that arrives after a later one (error_code=102902) — parallel or
|
||||
// out-of-order PUTs do not work (proven on a real endpoint). Sequential
|
||||
// send guarantees in-order arrival.
|
||||
// - One warm connection: reusing the socket keeps the TCP congestion
|
||||
// window open across the whole file instead of paying slow-start on a
|
||||
// fresh connection for every 1 MiB block. That per-block cold start was
|
||||
// the real throughput cap behind #211.
|
||||
//
|
||||
// 旧实现整个文件复用同一个 socket,依赖 `Connection: keep-alive` 跑多次
|
||||
// POST。但 QQ highway 边缘节点(以及链路上的代理/NAT)经常在第一次响应
|
||||
// 之后立刻 FIN 关闭连接,导致第二个 chunk 写入一个已被对端关闭的 socket,
|
||||
// `readHttpResponseBody` 立刻收到 `close` 事件并抛 `connection closed
|
||||
// before response`(#118)。所以每个 chunk 独立建连的不变量必须保留 ——
|
||||
// 并发只是让多个这样的一次性连接同时在飞,连接之间没有任何共享状态。
|
||||
// QQ's edge nodes (and proxies/NAT on the path) sometimes FIN between
|
||||
// chunks — that is what forced the one-connection-per-chunk workaround in
|
||||
// #118. We handle it adaptively: if reusing the socket fails, or the peer
|
||||
// half-closed after a response, we drop it, reconnect, and re-send the SAME
|
||||
// block (re-PUTting an offset is idempotent). Worst case — a server that
|
||||
// closes after every response — degrades to one connection per block: no
|
||||
// worse than the #118 behavior, and still strictly in order (so never
|
||||
// error_code=102902). `connectCount` is logged so a real run reveals which
|
||||
// regime we hit (1 connection = keep-alive honored; ~chunkCount = closed
|
||||
// every time).
|
||||
//
|
||||
// Data safety: `source.read` returns a fresh buffer per call (FileChunkSource
|
||||
// allocates; BufferChunkSource returns a read-only subarray), and reads are
|
||||
// positional (offset-addressed), so concurrent reads at distinct offsets are
|
||||
// race-free. Each chunk computes its own md5 over its own bytes.
|
||||
const uploadChunk = async (offset: number, chunkSize: number): Promise<void> => {
|
||||
const chunk = await source.read(offset, chunkSize);
|
||||
const chunkMd5 = computeMd5(chunk);
|
||||
const head = makeHighwayHead(
|
||||
bridge.identity.uin, commandId, totalSize, offset, chunkSize,
|
||||
chunkMd5, fileMd5, session.sigSession, extend,
|
||||
);
|
||||
const frame = packHighwayFrame(head, chunk);
|
||||
|
||||
// Retry the connect + POST for this chunk on transient transport errors
|
||||
// (peer FIN before response, ECONNRESET, connect refused/timeout). Each
|
||||
// attempt uses a fresh connection; re-sending the same offset range is
|
||||
// idempotent on the server.
|
||||
let responseBody: Uint8Array | undefined;
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
let socket: net.Socket;
|
||||
try {
|
||||
socket = await tcpConnect(session.host, session.port);
|
||||
} catch (err) {
|
||||
if (attempt >= HIGHWAY_MAX_CHUNK_ATTEMPTS) {
|
||||
throw new Error(
|
||||
`highway connect failed after ${attempt} attempts ` +
|
||||
`(cmdId=${commandId} offset=${offset}/${totalSize}): ${String(err)}`,
|
||||
);
|
||||
}
|
||||
await sleepMs(HIGHWAY_RETRY_BASE_MS * attempt);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
responseBody = await httpPostFrame(socket, session.host, pathStr, frame);
|
||||
break;
|
||||
} catch (err) {
|
||||
if (attempt >= HIGHWAY_MAX_CHUNK_ATTEMPTS) {
|
||||
throw new Error(
|
||||
`highway upload transport failed after ${attempt} attempts ` +
|
||||
`(cmdId=${commandId} chunk=${chunkSize}/${totalSize} offset=${offset}): ${String(err)}`,
|
||||
);
|
||||
}
|
||||
log.trace('chunk offset=%d attempt %d failed (%s), retrying', offset, attempt, String(err));
|
||||
await sleepMs(HIGHWAY_RETRY_BASE_MS * attempt);
|
||||
} finally {
|
||||
socket.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
// Unreachable: the retry loop only exits via break (responseBody set) or
|
||||
// throw — the guard just narrows the type for the compiler.
|
||||
if (!responseBody) throw new Error('highway upload: missing response');
|
||||
const { head: respHead } = unpackHighwayFrame(responseBody);
|
||||
const resp = protobuf_decode<RespDataHighwayHead>(respHead);
|
||||
if (resp?.errorCode && resp.errorCode !== 0) {
|
||||
// Surface every diagnostic the highway response carries so
|
||||
// user reports of `error_code=921` and friends include the
|
||||
// server-side context (segHead.retCode, chunk size, file md5)
|
||||
// — without these we can't tell apart a malformed-payload
|
||||
// reject, a session-ticket mismatch, or a per-account rate-
|
||||
// limit.
|
||||
const segRetCode = resp.msgSegHead?.retCode ?? 0;
|
||||
const fileMd5Hex = Buffer.from(fileMd5).toString('hex');
|
||||
throw new Error(
|
||||
`highway upload error_code=${resp.errorCode}` +
|
||||
` (cmdId=${commandId} chunk=${chunkSize}/${totalSize}` +
|
||||
` offset=${offset} segRetCode=${segRetCode}` +
|
||||
` fileMd5=${fileMd5Hex.slice(0, 16)}…)`,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
// Concurrent worker pool over the chunk range.
|
||||
//
|
||||
// Atomicity / consistency: an offset is claimed synchronously — there is NO
|
||||
// await between reading `nextOffset` and advancing it — so two workers can
|
||||
// never claim the same chunk, and every byte range is PUT exactly once. On
|
||||
// the first permanent chunk failure a worker sets `aborted`; peers stop
|
||||
// pulling new chunks but drain the one already in flight. `worker()` NEVER
|
||||
// rejects (it captures its error), so `Promise.all` always resolves only
|
||||
// after every in-flight read/socket has settled — that is what makes it safe
|
||||
// to `source.close()` in the finally without racing a live read.
|
||||
//
|
||||
// Rollback: a thrown upload leaves the server holding an incomplete set of
|
||||
// chunks that it never finalizes — identical to the pre-concurrency failure
|
||||
// mode. The caller's publish/finalize step (which makes the file visible)
|
||||
// only runs when this resolves, so a partial upload is never published.
|
||||
let nextOffset = 0;
|
||||
let uploaded = 0;
|
||||
let aborted = false;
|
||||
let firstError: Error | null = null;
|
||||
|
||||
const worker = async (): Promise<void> => {
|
||||
while (!aborted) {
|
||||
const offset = nextOffset;
|
||||
if (offset >= totalSize) return;
|
||||
const chunkSize = Math.min(HIGHWAY_BLOCK_SIZE, totalSize - offset);
|
||||
nextOffset = offset + chunkSize; // claim — synchronous, race-free
|
||||
try {
|
||||
await uploadChunk(offset, chunkSize);
|
||||
} catch (err) {
|
||||
aborted = true;
|
||||
firstError ??= err instanceof Error ? err : new Error(String(err));
|
||||
return;
|
||||
}
|
||||
uploaded += chunkSize;
|
||||
log.trace('uploaded %d/%d bytes', uploaded, totalSize);
|
||||
}
|
||||
};
|
||||
|
||||
// `source` may buffer the whole file (BufferChunkSource) or stream it from
|
||||
// disk (FileChunkSource). We own the source and close it exactly once below,
|
||||
// only after all workers have settled.
|
||||
// disk (FileChunkSource); either way one chunk is read/held at a time. We
|
||||
// own the source and close it exactly once in the `finally` below.
|
||||
let succeeded = false;
|
||||
let socket: net.Socket | null = null;
|
||||
let connectCount = 0;
|
||||
const dropSocket = (): void => {
|
||||
if (socket) { socket.destroy(); socket = null; }
|
||||
};
|
||||
try {
|
||||
const chunkCount = Math.max(1, Math.ceil(totalSize / HIGHWAY_BLOCK_SIZE));
|
||||
const workerCount = Math.min(HIGHWAY_UPLOAD_CONCURRENCY, chunkCount);
|
||||
await Promise.all(Array.from({ length: workerCount }, () => worker()));
|
||||
if (firstError) throw firstError;
|
||||
let offset = 0;
|
||||
while (offset < totalSize) {
|
||||
const chunkSize = Math.min(HIGHWAY_BLOCK_SIZE, totalSize - offset);
|
||||
const chunk = await source.read(offset, chunkSize);
|
||||
const chunkMd5 = computeMd5(chunk);
|
||||
const head = makeHighwayHead(
|
||||
bridge.identity.uin, commandId, totalSize, offset, chunkSize,
|
||||
chunkMd5, fileMd5, session.sigSession, extend,
|
||||
);
|
||||
const frame = packHighwayFrame(head, chunk);
|
||||
|
||||
// Send this block, reusing the live connection when there is one. On any
|
||||
// transport failure (peer FIN between chunks, ECONNRESET, connect
|
||||
// refused/timeout) drop the socket, reconnect, and retry THIS block —
|
||||
// re-sending the same offset range is idempotent on the server.
|
||||
let responseBody: Uint8Array | undefined;
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
if (!socket) {
|
||||
socket = await tcpConnect(session.host, session.port);
|
||||
connectCount += 1;
|
||||
}
|
||||
responseBody = await httpPostFrame(socket, session.host, pathStr, frame);
|
||||
break;
|
||||
} catch (err) {
|
||||
dropSocket();
|
||||
if (attempt >= HIGHWAY_MAX_CHUNK_ATTEMPTS) {
|
||||
throw new Error(
|
||||
`highway upload transport failed after ${attempt} attempts ` +
|
||||
`(cmdId=${commandId} chunk=${chunkSize}/${totalSize} offset=${offset}): ${String(err)}`,
|
||||
);
|
||||
}
|
||||
log.trace('chunk offset=%d attempt %d failed (%s), reconnecting', offset, attempt, String(err));
|
||||
await sleepMs(HIGHWAY_RETRY_BASE_MS * attempt);
|
||||
}
|
||||
}
|
||||
|
||||
// Unreachable: the retry loop only exits via break (responseBody set) or
|
||||
// throw — the guard just narrows the type for the compiler.
|
||||
if (!responseBody) throw new Error('highway upload: missing response');
|
||||
const { head: respHead } = unpackHighwayFrame(responseBody);
|
||||
const resp = protobuf_decode<RespDataHighwayHead>(respHead);
|
||||
if (resp?.errorCode && resp.errorCode !== 0) {
|
||||
// Surface every diagnostic the highway response carries so
|
||||
// user reports of `error_code=921` and friends include the
|
||||
// server-side context (segHead.retCode, chunk size, file md5)
|
||||
// — without these we can't tell apart a malformed-payload
|
||||
// reject, a session-ticket mismatch, or a per-account rate-
|
||||
// limit.
|
||||
const segRetCode = resp.msgSegHead?.retCode ?? 0;
|
||||
const fileMd5Hex = Buffer.from(fileMd5).toString('hex');
|
||||
throw new Error(
|
||||
`highway upload error_code=${resp.errorCode}` +
|
||||
` (cmdId=${commandId} chunk=${chunkSize}/${totalSize}` +
|
||||
` offset=${offset} segRetCode=${segRetCode}` +
|
||||
` fileMd5=${fileMd5Hex.slice(0, 16)}…)`,
|
||||
);
|
||||
}
|
||||
|
||||
// If the peer half-closed after sending its response, the socket can no
|
||||
// longer carry the next block — drop it now so the next iteration
|
||||
// reconnects cleanly instead of writing into a dead socket.
|
||||
if (socket && (socket.destroyed || socket.readableEnded || !socket.writable)) {
|
||||
dropSocket();
|
||||
}
|
||||
|
||||
offset += chunkSize;
|
||||
log.trace('uploaded %d/%d bytes', offset, totalSize);
|
||||
}
|
||||
succeeded = true;
|
||||
if (totalSize > 0) {
|
||||
log.debug('highway upload done: %d bytes over %d connection(s) (cmdId=%d)',
|
||||
totalSize, connectCount, commandId);
|
||||
}
|
||||
} finally {
|
||||
// Best-effort close — never mask a primary upload error. A close failure
|
||||
// is only surfaced when the upload itself succeeded.
|
||||
dropSocket();
|
||||
try {
|
||||
await source.close();
|
||||
} catch (closeErr) {
|
||||
|
||||
Reference in New Issue
Block a user