fix(protocol): scope system dedup to native routes

Derive QQ NT's peerUid_chatType_msg_seq_random identity from its reverse-engineered static routing table before event dispatch. Unknown routes and incomplete identities fail open, while C2C and group peers use their native namespaces.

Refines #266
This commit is contained in:
DrSmoothl
2026-07-29 01:25:54 +08:00
parent 5eb5aaf0a3
commit 5d1dbecf2e
3 changed files with 235 additions and 103 deletions
+31 -29
View File
@@ -21,9 +21,16 @@ import { decodeGroupMessage } from './decoders/group-message';
import { decodeTempMessage } from './decoders/temp-message';
import { PkgType } from './enums';
import { MsgPushRegistry } from './registry';
import { SysMsgDedup } from './sysmsg-dedup';
import {
deriveSysMsgDedupIdentity,
SysMsgDedup,
} from './sysmsg-dedup';
export { SysMsgDedup } from './sysmsg-dedup';
export {
deriveSysMsgDedupIdentity,
SysMsgDedup,
} from './sysmsg-dedup';
export type { SysMsgDedupIdentity } from './sysmsg-dedup';
export { SSO_GET_GROUP_MSG_CMD, fetchGroupMessageRange } from './fetch-group-history';
export {
@@ -63,13 +70,6 @@ const MESSAGE_KINDS = new Set<QQEventVariant['kind']>([
'friend_message', 'group_message', 'temp_message',
]);
// Replacement snapshots describe the complete current state. They are
// idempotent, but not duplicate events: a later snapshot may legitimately
// reuse the enclosing system-message identity while carrying new state.
const STATE_SNAPSHOT_KINDS = new Set<QQEventVariant['kind']>([
'online_devices_changed',
]);
/**
* Summarise a body that decoded to zero elements despite carrying content —
* each element's field names, every `commonElem`'s serviceType/businessType +
@@ -118,6 +118,28 @@ export function parseMsgPush(
): QQEventVariant[] {
const ctx = buildContext(pkt, identity);
if (!ctx) return [];
// #137/#266: mirror QQ NT `sys_msg_mgr.cc::ProcessRecvSysMsg` before
// dispatch. Its static routing table is the scope boundary: listed routes
// use the native global key, while unlisted routes fail open. Running this
// before registry decode also covers native system packets whose outer type
// SnowLuma otherwise interprets as a chat message or does not decode at all.
if (dedup) {
const dedupIdentity = deriveSysMsgDedupIdentity(ctx);
if (dedupIdentity && dedup.seenDuplicate(dedupIdentity)) {
log.debug(
'dropped duplicate system push (peer=%s chatType=%d seq=%d msgType=%d/%d msgId=%d)',
dedupIdentity.peerUid,
dedupIdentity.chatType,
ctx.head.sequence,
ctx.head.msgType,
ctx.head.subType,
ctx.head.msgId,
);
return [];
}
}
const events = registry.decode(ctx);
const out = events.filter((ev) => {
if (!MESSAGE_KINDS.has(ev.kind)) return true;
@@ -151,25 +173,5 @@ export function parseMsgPush(
return false;
});
// #137: mirror QQ NT `sys_msg_mgr.cc::ProcessRecvSysMsg` global-key dedup.
// The server pushes some system notices twice (e.g. inviting an official
// robot → two `group_member_increase`); the kernel drops the duplicate by
// (peer, seq, random) before any listener sees it, but we read the raw
// OlPush, so we replicate the drop. Scoped to system pushes only — chat
// messages have their own NT dedup path and the forward re-parse re-runs
// this without a tracker. A push that decodes to a message kind, or to
// nothing, is never deduped here.
if (
dedup
&& out.length > 0
&& out.every((ev) => !MESSAGE_KINDS.has(ev.kind))
&& out.every((ev) => !STATE_SNAPSHOT_KINDS.has(ev.kind))
) {
if (dedup.seenDuplicate(ctx.head, ctx.fromUin)) {
log.debug('dropped duplicate system push (kinds=%s seq=%d from=%d msgType=%d msgId=%d)',
out.map((ev) => ev.kind).join(','), ctx.head.sequence, ctx.fromUin, ctx.head.msgType, ctx.head.msgId);
return [];
}
}
return out;
}
+58 -37
View File
@@ -1,42 +1,66 @@
import type { MsgPushHead } from './context';
import type { MsgPushContext } from './context';
export type SysMsgChatType = 1 | 2;
export interface SysMsgDedupIdentity {
readonly peerUid: string;
readonly chatType: SysMsgChatType;
readonly sequence: number;
readonly random: number;
}
function getNativeSysMsgChatType(msgType: number, subType: number): SysMsgChatType | null {
if (msgType === 528 && subType === 290) return 1;
if (msgType === 166 && (subType === 75 || subType === 129 || subType === 131 || subType === 133 || subType === 135)) return 1;
if (msgType === 167 && subType === 133) return 1;
if (msgType === 732 && (subType === 12 || subType === 16 || subType === 20)) return 2;
if ((msgType === 33 || msgType === 34 || msgType === 85) && subType === 0) return 2;
return null;
}
/**
* Mirrors QQ NT's system-message dedup so SnowLuma stops double-reporting the
* duplicate sys pushes the server emits for some events (#137: inviting an
* official robot pushes the `group_member_increase` notice twice → two
* `notice.group_increase`; a normal member is pushed once).
* Derive the identity used by QQ NT's system-message dedup. Only the static
* `(msgType, subType)` routes proven in `sys_msg_mgr.cc` participate; unknown
* routes fail open rather than risking suppression of unrelated events.
*/
export function deriveSysMsgDedupIdentity(
ctx: Pick<MsgPushContext, 'head' | 'fromUin' | 'fromUid'>,
): SysMsgDedupIdentity | null {
const { head } = ctx;
const chatType = getNativeSysMsgChatType(head.msgType, head.subType);
// Decoded zero is also the protobuf default for an absent field. Without a
// complete server identity, prefer a duplicate event over suppressing two
// unrelated pushes that collapse to the same default-valued key.
if (chatType === null || head.sequence === 0 || head.msgId === 0) return null;
const peerUid = chatType === 1 ? ctx.fromUid : String(ctx.fromUin);
if (!peerUid || (chatType === 2 && ctx.fromUin === 0)) return null;
return {
peerUid,
chatType,
sequence: head.sequence,
random: head.msgId,
};
}
/**
* Mirrors QQ NT's system-message dedup so SnowLuma stops double-reporting raw
* OlPush system events that the QQ kernel would discard before notifying its
* listeners.
*
* RE of `wrapper.linux.node` — `sys_msg_mgr.cc::ProcessRecvSysMsg`
* (`sub_37BADE0`): every received system message is keyed and looked up in a
* per-account set; an already-seen key is logged ("on recv sys msg, ignore dup
* msg … global_key={}", sys_msg_mgr.cc:467) and the whole message is dropped
* before the UI / JS listeners ever see it. The key (`sub_37BC4E0`) is
* macOS QQ 6.9.98 `sys_msg_mgr.cc::ProcessRecvSysMsg` (`sub_E4B94A`) builds:
*
* global_key = `{peerUid}_{chatType}_{msg_seq}_{random}`
*
* whose per-message discriminators are `msg_seq` (attr 40003 = contentHead
* field 5) and `random` (attr 40002 = contentHead field 4) — exactly
* {@link MsgPushHead.sequence} and {@link MsgPushHead.msgId} here
* (`msg_header_codec_helper.cc` sub_37C8AD0). Kernel-based bots (NapCat /
* LLOneBot) receive events *after* this dedup; SnowLuma reads the raw OlPush
* *before* it, so we replicate it here.
*
* We dedup at the push level (drop the whole duplicate, like NT) using
* `sequence` + `msgId` and `fromUin` for peer scoping (a stand-in for peerUid —
* `msg_seq` can be per-conversation, so the peer keeps two groups' pushes from
* colliding; `random`/`msgId` makes the collision astronomically unlikely
* anyway). `msgType` and `subType` are deliberately absent: they are not part
* of QQ NT's global key, and one server system message may arrive through
* different outer package types (#266). A push with no server identity
* (`sequence` or `msgId` 0) is never deduped — without a real per-message id we
* cannot distinguish a true duplicate from two distinct events, and dropping
* then would be a regression.
* `sub_E523C8` maps content-head fields 5 and 4 to `msg_seq` and `random`.
* `sub_E4B30D` supplies chatType from a static `(msgType, subType)` table, and
* `sub_E4B62A` selects response-head `fromUid` for C2C or decimal `fromUin` for
* groups. {@link deriveSysMsgDedupIdentity} mirrors those routes exactly.
*/
export class SysMsgDedup {
private readonly seen = new Set<string>();
// Fixed-capacity ring of keys for O(1) bounded eviction (oldest-out). System
// pushes are low-frequency, so a duplicate always lands within a few entries
// of its original — eviction only matters far beyond the dup window.
private readonly ring: (string | undefined)[];
private cursor = 0;
@@ -44,13 +68,10 @@ export class SysMsgDedup {
this.ring = new Array<string | undefined>(capacity);
}
/**
* Returns true if a system push with this identity was already seen (caller
* should drop it); otherwise records it and returns false.
*/
seenDuplicate(head: Pick<MsgPushHead, 'sequence' | 'msgId'>, fromUin: number): boolean {
if (head.sequence === 0 || head.msgId === 0) return false;
const key = `${fromUin}:${head.sequence}:${head.msgId}`;
/** Returns true when this native system-message identity was already seen. */
seenDuplicate(identity: SysMsgDedupIdentity): boolean {
if (identity.sequence === 0 || identity.random === 0 || !identity.peerUid) return false;
const key = `${identity.peerUid}_${identity.chatType}_${identity.sequence}_${identity.random}`;
if (this.seen.has(key)) return true;
const evicted = this.ring[this.cursor];
if (evicted !== undefined) this.seen.delete(evicted);
@@ -5,8 +5,9 @@
// global key whose per-message discriminators are msg_seq (contentHead field 5
// = head.sequence) and random (contentHead field 4 = head.msgId); kernel-based
// bots see events only after that dedup. SnowLuma reads the raw OlPush, so
// parseMsgPush must replicate the drop for system pushes — but only when given
// a dedup tracker (the live path), and never for chat messages.
// parseMsgPush must replicate the drop for routes in QQ NT's static system-
// message table, but only when given a dedup tracker (the live path). Ordinary
// chat routes remain outside that table even when their outer msgType matches.
import { describe, expect, it } from 'vitest';
import type { PacketInfo } from '@snowluma/common/protocol-types';
@@ -14,7 +15,12 @@ import { protobuf_encode } from '@snowluma/proton';
import type { PushMsg, PushMsgBody } from '@snowluma/proto-defs/message';
import type { GroupChange, OnlineDeviceNotify } from '@snowluma/proto-defs/notify';
import type { IdentityService } from '../../src/identity-service';
import { parseMsgPush, SysMsgDedup } from '../../src/msg-push';
import {
deriveSysMsgDedupIdentity,
parseMsgPush,
SysMsgDedup,
type SysMsgDedupIdentity,
} from '../../src/msg-push';
const identity = { findUinByUid: () => null, findFriend: () => undefined } as unknown as IdentityService;
@@ -40,6 +46,14 @@ function memberIncreasePush(opts: { groupId: number; memberUin: number; sequence
});
}
function c2cPacket(msgType: number, subType: number): PacketInfo {
return pushPacket({
responseHead: { fromUin: 10001, fromUid: 'u_x' },
contentHead: { msgType, subType, sequence: 900, timestamp: 1781540572, msgId: 7 },
body: { richText: { elems: [{ text: { str: 'hi' } }] } },
});
}
function onlineDevicesPush(deviceName: string): PacketInfo {
return pushPacket({
responseHead: { fromUin: 2000000001, fromUid: '' },
@@ -52,47 +66,135 @@ function onlineDevicesPush(deviceName: string): PacketInfo {
});
}
function dedupIdentity(overrides: Partial<SysMsgDedupIdentity> = {}): SysMsgDedupIdentity {
return {
peerUid: '700',
chatType: 2,
sequence: 500,
random: 12345,
...overrides,
};
}
function deriveIdentity(opts: {
msgType: number;
subType: number;
sequence?: number;
msgId?: number;
fromUin?: number;
fromUid?: string;
}) {
return deriveSysMsgDedupIdentity({
head: {
msgType: opts.msgType,
subType: opts.subType,
c2cCmd: 0,
sequence: opts.sequence ?? 500,
ntMsgSeq: 0,
timestamp: 1781540572,
msgId: opts.msgId ?? 12345,
},
fromUin: opts.fromUin ?? 700,
fromUid: opts.fromUid ?? 'u_peer',
});
}
describe('deriveSysMsgDedupIdentity', () => {
it.each([
[528, 290],
[166, 75],
[166, 129],
[166, 131],
[166, 133],
[166, 135],
[167, 133],
])('maps native C2C route %d/%d to fromUid and chatType 1', (msgType, subType) => {
expect(deriveIdentity({ msgType, subType, fromUin: 700, fromUid: 'u_peer' })).toEqual({
peerUid: 'u_peer',
chatType: 1,
sequence: 500,
random: 12345,
});
});
it.each([
[732, 12],
[732, 16],
[732, 20],
[33, 0],
[34, 0],
[85, 0],
])('maps native group route %d/%d to decimal fromUin and chatType 2', (msgType, subType) => {
expect(deriveIdentity({ msgType, subType, fromUin: 700, fromUid: 'ignored' })).toEqual({
peerUid: '700',
chatType: 2,
sequence: 500,
random: 12345,
});
});
it('[#266] fails open for an outer package pair absent from QQ NT’s static table', () => {
expect(deriveIdentity({ msgType: 528, subType: 99 })).toBeNull();
});
it('fails open when the native sequence, random, or route-specific peer is missing', () => {
expect(deriveIdentity({ msgType: 33, subType: 0, sequence: 0 })).toBeNull();
expect(deriveIdentity({ msgType: 33, subType: 0, msgId: 0 })).toBeNull();
expect(deriveIdentity({ msgType: 33, subType: 0, fromUin: 0 })).toBeNull();
expect(deriveIdentity({ msgType: 528, subType: 290, fromUid: '' })).toBeNull();
});
});
describe('SysMsgDedup', () => {
it('flags the second push with the same (peer, seq, msgId)', () => {
it('flags the second push with the same native global key', () => {
const d = new SysMsgDedup();
const head = { msgType: 33, subType: 0, sequence: 500, msgId: 12345 };
expect(d.seenDuplicate(head, 700)).toBe(false);
expect(d.seenDuplicate(head, 700)).toBe(true);
expect(d.seenDuplicate(dedupIdentity())).toBe(false);
expect(d.seenDuplicate(dedupIdentity())).toBe(true);
});
it('treats a different seq, msgId, or peer as a distinct push', () => {
it('treats a different sequence, random, peer, or chat type as distinct', () => {
const d = new SysMsgDedup();
const base = { msgType: 33, subType: 0, sequence: 500, msgId: 12345 };
expect(d.seenDuplicate(base, 700)).toBe(false);
expect(d.seenDuplicate({ ...base, sequence: 501 }, 700)).toBe(false);
expect(d.seenDuplicate({ ...base, msgId: 99999 }, 700)).toBe(false);
expect(d.seenDuplicate(base, 701)).toBe(false);
expect(d.seenDuplicate(dedupIdentity())).toBe(false);
expect(d.seenDuplicate(dedupIdentity({ sequence: 501 }))).toBe(false);
expect(d.seenDuplicate(dedupIdentity({ random: 99999 }))).toBe(false);
expect(d.seenDuplicate(dedupIdentity({ peerUid: '701' }))).toBe(false);
expect(d.seenDuplicate(dedupIdentity({ chatType: 1 }))).toBe(false);
});
it('[#266] follows QQ NT and dedups one identity across outer package types', () => {
it('[#266] dedups different listed outer routes that derive the same native identity', () => {
const d = new SysMsgDedup();
const first = { msgType: 33, subType: 0, sequence: 500, msgId: 12345 };
const alternateWrapper = { msgType: 528, subType: 99, sequence: 500, msgId: 12345 };
expect(d.seenDuplicate(first, 700)).toBe(false);
expect(d.seenDuplicate(alternateWrapper, 700)).toBe(true);
const first = deriveIdentity({ msgType: 166, subType: 133 });
const alternateRoute = deriveIdentity({ msgType: 167, subType: 133 });
expect(first).not.toBeNull();
expect(alternateRoute).toEqual(first);
expect(d.seenDuplicate(first!)).toBe(false);
expect(d.seenDuplicate(alternateRoute!)).toBe(true);
});
it('never dedups a push with no server identity (seq or msgId 0)', () => {
it('[#266] does not record an unlisted outer route', () => {
const d = new SysMsgDedup();
expect(d.seenDuplicate({ msgType: 33, subType: 0, sequence: 0, msgId: 0 }, 700)).toBe(false);
expect(d.seenDuplicate({ msgType: 33, subType: 0, sequence: 0, msgId: 0 }, 700)).toBe(false);
expect(d.seenDuplicate({ msgType: 33, subType: 0, sequence: 5, msgId: 0 }, 700)).toBe(false);
expect(d.seenDuplicate({ msgType: 33, subType: 0, sequence: 0, msgId: 5 }, 700)).toBe(false);
const unlisted = deriveIdentity({ msgType: 528, subType: 99 });
const listed = deriveIdentity({ msgType: 528, subType: 290 });
expect(unlisted).toBeNull();
expect(listed).not.toBeNull();
expect(d.seenDuplicate(listed!)).toBe(false);
});
it('never dedups a manually supplied identity without a server discriminator or peer', () => {
const d = new SysMsgDedup();
expect(d.seenDuplicate(dedupIdentity({ sequence: 0 }))).toBe(false);
expect(d.seenDuplicate(dedupIdentity({ sequence: 0 }))).toBe(false);
expect(d.seenDuplicate(dedupIdentity({ random: 0 }))).toBe(false);
expect(d.seenDuplicate(dedupIdentity({ peerUid: '' }))).toBe(false);
});
it('evicts the oldest key once capacity is exceeded (bounded memory)', () => {
const d = new SysMsgDedup(2);
const k = (sequence: number) => ({ msgType: 33, subType: 0, sequence, msgId: 1 });
expect(d.seenDuplicate(k(1), 1)).toBe(false); // [1]
expect(d.seenDuplicate(k(2), 1)).toBe(false); // [1,2]
expect(d.seenDuplicate(k(3), 1)).toBe(false); // [2,3] — evicts 1
expect(d.seenDuplicate(k(1), 1)).toBe(false); // 1 was evicted → seen as new again
expect(d.seenDuplicate(k(3), 1)).toBe(true); // 3 still tracked
expect(d.seenDuplicate(dedupIdentity({ sequence: 1, random: 1 }))).toBe(false);
expect(d.seenDuplicate(dedupIdentity({ sequence: 2, random: 1 }))).toBe(false);
expect(d.seenDuplicate(dedupIdentity({ sequence: 3, random: 1 }))).toBe(false);
expect(d.seenDuplicate(dedupIdentity({ sequence: 1, random: 1 }))).toBe(false);
expect(d.seenDuplicate(dedupIdentity({ sequence: 3, random: 1 }))).toBe(true);
});
});
@@ -124,15 +226,22 @@ describe('parseMsgPush — system-push dedup (#137)', () => {
expect(b[0]).toMatchObject({ userUin: 222 });
});
it('does not dedup chat messages, even with an identical (seq, msgId)', () => {
it('[#266] dedups a listed C2C route before its outer type is decoded as a chat message', () => {
const dedup = new SysMsgDedup();
const chat = (): PacketInfo => pushPacket({
responseHead: { fromUin: 10001, fromUid: 'u_x' },
contentHead: { msgType: 166, subType: 0, sequence: 900, timestamp: 1781540572, msgId: 7 },
body: { richText: { elems: [{ text: { str: 'hi' } }] } },
});
expect(parseMsgPush(chat(), identity, dedup)).toHaveLength(1);
expect(parseMsgPush(chat(), identity, dedup)).toHaveLength(1);
expect(parseMsgPush(c2cPacket(166, 133), identity, dedup)).toHaveLength(1);
expect(parseMsgPush(c2cPacket(166, 133), identity, dedup)).toEqual([]);
});
it('[#266] records a listed undecoded route before a matching listed route is dispatched', () => {
const dedup = new SysMsgDedup();
expect(parseMsgPush(c2cPacket(167, 133), identity, dedup)).toEqual([]);
expect(parseMsgPush(c2cPacket(166, 133), identity, dedup)).toEqual([]);
});
it('does not dedup ordinary chat messages outside the native system-message table', () => {
const dedup = new SysMsgDedup();
expect(parseMsgPush(c2cPacket(166, 0), identity, dedup)).toHaveLength(1);
expect(parseMsgPush(c2cPacket(166, 0), identity, dedup)).toHaveLength(1);
});
it('does not dedup replacement state snapshots with the same server identity', () => {