refactor(common,core): extract mapWithConcurrency util; use in forward enrich
阶段 2:把「限量+并发分批」的手写循环收进一个 common util。此形状已被复制进 私有 SLE(friend.ts),是已发生的复制漂移;抽出后堵住它、并给未来的富化点一个 现成原语(cap + 并发批 + best-effort 是防 Tencent 风控封号的标准配方)。 - 新增 common/concurrency.ts:mapWithConcurrency(items, concurrency, fn),批处理 语义(每批 Promise.all 等齐再下一批,非 rolling pool,保持 forward 现有行为), 结果按输入顺序返回。7 个单测覆盖顺序/绝对index/峰值≤limit+跨批serialize/ 不整除/单批/空/concurrency<1。 - 迁移 forward.ts residue 昵称富化的并发批循环;cap(slice MAX)/dedup(Set)/ best-effort(fn 内 try/catch) 保留在 callsite。行为严格不变。 对抗性 Agent 逐条对比 HEAD 确认等价。common 50 / core 467 全绿、typecheck 干净。
This commit is contained in:
@@ -24,6 +24,10 @@
|
||||
"types": "./src/single-flight-cache.ts",
|
||||
"default": "./src/single-flight-cache.ts"
|
||||
},
|
||||
"./concurrency": {
|
||||
"types": "./src/concurrency.ts",
|
||||
"default": "./src/concurrency.ts"
|
||||
},
|
||||
"./hex": {
|
||||
"types": "./src/hex.ts",
|
||||
"default": "./src/hex.ts"
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* Map `items` through `fn` in sequential batches of at most `concurrency`
|
||||
* items, awaiting each batch (`Promise.all`) before starting the next.
|
||||
* Results come back in input order.
|
||||
*
|
||||
* Batched, not a rolling pool: every item in a batch must settle before the
|
||||
* next batch starts. This caps peak parallelism — used to keep rate-limited
|
||||
* fan-out (e.g. the per-uin OIDB profile fetches during forward-node
|
||||
* enrichment) from bursting past a safe width and tripping Tencent
|
||||
* risk-control. Pair it with a cap on `items` (slice) and per-item error
|
||||
* handling inside `fn` for a fully bounded, best-effort enrichment — the
|
||||
* shape that was previously hand-rolled at each such call site.
|
||||
*/
|
||||
export async function mapWithConcurrency<T, R>(
|
||||
items: readonly T[],
|
||||
concurrency: number,
|
||||
fn: (item: T, index: number) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const limit = Math.max(1, Math.trunc(concurrency));
|
||||
const results = new Array<R>(items.length);
|
||||
for (let i = 0; i < items.length; i += limit) {
|
||||
const batch = items.slice(i, i + limit);
|
||||
const settled = await Promise.all(batch.map((item, j) => fn(item, i + j)));
|
||||
for (let j = 0; j < settled.length; j++) results[i + j] = settled[j]!;
|
||||
}
|
||||
return results;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { mapWithConcurrency } from '../src/concurrency';
|
||||
|
||||
const tick = () => new Promise<void>((r) => setTimeout(r, 0));
|
||||
|
||||
describe('mapWithConcurrency', () => {
|
||||
it('returns results in input order', async () => {
|
||||
const out = await mapWithConcurrency([1, 2, 3, 4, 5], 2, async (n) => n * 10);
|
||||
expect(out).toEqual([10, 20, 30, 40, 50]);
|
||||
});
|
||||
|
||||
it('passes the absolute index (across batch boundaries)', async () => {
|
||||
const seen: Array<[number, number]> = [];
|
||||
await mapWithConcurrency(['a', 'b', 'c', 'd', 'e'], 2, async (item, i) => {
|
||||
seen.push([i, item.charCodeAt(0)]);
|
||||
return i;
|
||||
});
|
||||
expect(seen.map(([i]) => i).sort((x, y) => x - y)).toEqual([0, 1, 2, 3, 4]);
|
||||
});
|
||||
|
||||
it('never runs more than `concurrency` at once (batched), and serializes across batches', async () => {
|
||||
let inFlight = 0;
|
||||
let peak = 0;
|
||||
const order: number[] = [];
|
||||
await mapWithConcurrency([0, 1, 2, 3, 4, 5, 6], 3, async (n) => {
|
||||
inFlight++;
|
||||
peak = Math.max(peak, inFlight);
|
||||
await tick();
|
||||
order.push(n);
|
||||
inFlight--;
|
||||
return n;
|
||||
});
|
||||
expect(peak).toBeLessThanOrEqual(3);
|
||||
// batched: first 3 (0,1,2) settle before the next batch (3,4,5) starts.
|
||||
expect(order.slice(0, 3).sort((a, b) => a - b)).toEqual([0, 1, 2]);
|
||||
expect(order.slice(3, 6).sort((a, b) => a - b)).toEqual([3, 4, 5]);
|
||||
expect(order[6]).toBe(6);
|
||||
});
|
||||
|
||||
it('handles a length not divisible by concurrency (short final batch)', async () => {
|
||||
const out = await mapWithConcurrency([1, 2, 3, 4, 5], 3, async (n) => n);
|
||||
expect(out).toEqual([1, 2, 3, 4, 5]);
|
||||
});
|
||||
|
||||
it('runs everything in one batch when concurrency >= length', async () => {
|
||||
let peak = 0, inFlight = 0;
|
||||
const out = await mapWithConcurrency([1, 2, 3], 10, async (n) => {
|
||||
inFlight++; peak = Math.max(peak, inFlight); await tick(); inFlight--; return n;
|
||||
});
|
||||
expect(out).toEqual([1, 2, 3]);
|
||||
expect(peak).toBe(3);
|
||||
});
|
||||
|
||||
it('returns [] for an empty input without calling fn', async () => {
|
||||
let called = 0;
|
||||
const out = await mapWithConcurrency([], 4, async (n) => { called++; return n; });
|
||||
expect(out).toEqual([]);
|
||||
expect(called).toBe(0);
|
||||
});
|
||||
|
||||
it('treats concurrency < 1 as 1 (fully serial)', async () => {
|
||||
let peak = 0, inFlight = 0;
|
||||
await mapWithConcurrency([1, 2, 3], 0, async (n) => {
|
||||
inFlight++; peak = Math.max(peak, inFlight); await tick(); inFlight--; return n;
|
||||
});
|
||||
expect(peak).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { PacketInfo } from '@snowluma/common/protocol-types';
|
||||
import { mapWithConcurrency } from '@snowluma/common/concurrency';
|
||||
import type {
|
||||
LongMsgResult,
|
||||
RecvLongMsgReq,
|
||||
@@ -676,14 +677,12 @@ export class ForwardApi {
|
||||
.slice(0, FORWARD_PROFILE_MAX);
|
||||
if (uins.length > 0) {
|
||||
const byUin = new Map<number, string>();
|
||||
for (let i = 0; i < uins.length; i += FORWARD_PROFILE_CONCURRENCY) {
|
||||
const batch = uins.slice(i, i + FORWARD_PROFILE_CONCURRENCY);
|
||||
const results = await Promise.all(batch.map(async (uin): Promise<readonly [number, string]> => {
|
||||
const results = await mapWithConcurrency(uins, FORWARD_PROFILE_CONCURRENCY,
|
||||
async (uin): Promise<readonly [number, string]> => {
|
||||
try { return [uin, (await this.ctx.apis.contacts.fetchUserProfile(uin)).nickname] as const; }
|
||||
catch { return [uin, ''] as const; }
|
||||
}));
|
||||
for (const [uin, nick] of results) byUin.set(uin, nick);
|
||||
}
|
||||
});
|
||||
for (const [uin, nick] of results) byUin.set(uin, nick);
|
||||
for (const n of residue) {
|
||||
if (!this.needsName(n)) continue;
|
||||
const nick = byUin.get(n.userUin);
|
||||
|
||||
Reference in New Issue
Block a user