fix: 保留消息投递确定结果

This commit is contained in:
sunlei 2026-07-24 13:52:38 +08:00
parent d74545d17b
commit d64d5aad49
2 changed files with 145 additions and 23 deletions

View File

@ -35,6 +35,16 @@ type ClaimToken = {
leaseUntil: KtDateTime; leaseUntil: KtDateTime;
}; };
type OwnerTransition = Pick<
QqbotMessageDelivery,
| 'lastErrorCode'
| 'lastErrorMessage'
| 'nextAttemptAt'
| 'processingLeaseUntil'
| 'sendLogId'
| 'status'
>;
type PreparedDelivery = type PreparedDelivery =
| { kind: 'send'; delivery: QqbotMessageDelivery } | { kind: 'send'; delivery: QqbotMessageDelivery }
| { kind: 'stale' } | { kind: 'stale' }
@ -395,9 +405,7 @@ export class SystemMessageDeliveryRunnerService {
sendLogId: null | string, sendLogId: null | string,
nextAttemptAt: KtDateTime | null = null, nextAttemptAt: KtDateTime | null = null,
): Promise<void> { ): Promise<void> {
const result = await this.dataSource await this.persistOwnerTransition(token, {
.getRepository(QqbotMessageDelivery)
.update(this.ownerWhere(token), {
lastErrorCode: code, lastErrorCode: code,
lastErrorMessage: message, lastErrorMessage: message,
nextAttemptAt, nextAttemptAt,
@ -405,7 +413,6 @@ export class SystemMessageDeliveryRunnerService {
sendLogId: sendLogId ?? token.delivery.sendLogId, sendLogId: sendLogId ?? token.delivery.sendLogId,
status, status,
}); });
if (result.affected !== 1) return;
} }
/** Retries until the event-derived expiration boundary and otherwise writes a final failure. */ /** Retries until the event-derived expiration boundary and otherwise writes a final failure. */
@ -423,9 +430,7 @@ export class SystemMessageDeliveryRunnerService {
await this.finish(token, 'failed', code, message, sendLogId); await this.finish(token, 'failed', code, message, sendLogId);
return; return;
} }
const result = await this.dataSource await this.persistOwnerTransition(token, {
.getRepository(QqbotMessageDelivery)
.update(this.ownerWhere(token), {
lastErrorCode: code, lastErrorCode: code,
lastErrorMessage: message, lastErrorMessage: message,
nextAttemptAt: next, nextAttemptAt: next,
@ -433,7 +438,31 @@ export class SystemMessageDeliveryRunnerService {
sendLogId: sendLogId ?? token.delivery.sendLogId, sendLogId: sendLogId ?? token.delivery.sendLogId,
status: 'retry', status: 'retry',
}); });
if (result.affected !== 1) return; }
/**
* Persists one authoritative owner transition with one identical bounded retry.
* Repeated failure leaves the processing lease for durable recovery instead of
* replacing the known business outcome with a generic classification.
* @param token - The exact attempt-and-lease owner fence.
* @param values - The complete authoritative transition payload to repeat.
* @returns A promise that settles after success, stale ownership, or retry exhaustion.
*/
private async persistOwnerTransition(
token: ClaimToken,
values: OwnerTransition,
): Promise<void> {
const deliveries = this.dataSource.getRepository(QqbotMessageDelivery);
const owner = this.ownerWhere(token);
try {
await deliveries.update(owner, values);
} catch {
try {
await deliveries.update(owner, values);
} catch {
// A later scan recovers the lease if neither ambiguous write committed.
}
}
} }
/** Builds the exact attempt-and-lease owner fence used by every post-claim mutation. */ /** Builds the exact attempt-and-lease owner fence used by every post-claim mutation. */

View File

@ -272,6 +272,10 @@ function setup(seed: Partial<Store> = {}) {
const queries: QueryRecord[] = []; const queries: QueryRecord[] = [];
const senderDepths: number[] = []; const senderDepths: number[] = [];
const preparationClaimLocks: boolean[] = []; const preparationClaimLocks: boolean[] = [];
const externalUpdates: Array<{
values: Record<string, unknown>;
where: Record<string, unknown>;
}> = [];
const activeLocks = new Set<string>(); const activeLocks = new Set<string>();
let transactionDepth = 0; let transactionDepth = 0;
let externalUpdateFailures = 0; let externalUpdateFailures = 0;
@ -481,6 +485,7 @@ function setup(seed: Partial<Store> = {}) {
values: Record<string, unknown>, values: Record<string, unknown>,
) => { ) => {
if (!context) { if (!context) {
externalUpdates.push({ values, where });
beforeExternalUpdate?.(state, where); beforeExternalUpdate?.(state, where);
if (externalUpdateFailures > 0) { if (externalUpdateFailures > 0) {
externalUpdateFailures -= 1; externalUpdateFailures -= 1;
@ -565,6 +570,7 @@ function setup(seed: Partial<Store> = {}) {
) => { ) => {
beforePreparationDeliveryRead = callback; beforePreparationDeliveryRead = callback;
}, },
externalUpdates,
failExternalUpdates: (count: number) => { failExternalUpdates: (count: number) => {
externalUpdateFailures = count; externalUpdateFailures = count;
}, },
@ -1087,6 +1093,79 @@ describe('System message delivery runner direct preflight contracts', () => {
}, },
); );
it('16 retries the exact permanent outcome after one final persistence failure', async () => {
const harness = setup();
harness.sender.sendStrictPlainText.mockRejectedValueOnce(
new QqbotSendAttemptError({
code: 'onebot_rejected',
message: 'unsafe raw permanent rejection',
retryable: false,
sendLogId: 'latest-log',
}),
);
harness.failExternalUpdates(1);
await harness.runner.runOnce(NOW);
expect(harness.externalUpdates).toHaveLength(2);
expect(harness.externalUpdates[1]).toEqual(harness.externalUpdates[0]);
expect(harness.getState().deliveries[0]).toMatchObject({
lastErrorCode: 'onebot_rejected',
lastErrorMessage: 'OneBot rejected the send action',
nextAttemptAt: null,
processingLeaseUntil: null,
sendLogId: 'latest-log',
status: 'failed',
});
await harness.runner.runOnce(new Date(NOW.getTime() + 10_000));
expect(harness.sender.sendStrictPlainText).toHaveBeenCalledTimes(1);
});
it('6 retries the exact successful outcome after one final persistence failure', async () => {
const harness = setup();
harness.failExternalUpdates(1);
await harness.runner.runOnce(NOW);
expect(harness.externalUpdates).toHaveLength(2);
expect(harness.externalUpdates[1]).toEqual(harness.externalUpdates[0]);
expect(harness.getState().deliveries[0]).toMatchObject({
lastErrorCode: null,
lastErrorMessage: null,
nextAttemptAt: null,
processingLeaseUntil: null,
sendLogId: 'send-log-1',
status: 'success',
});
});
it('17 retries the exact typed retry outcome after one final persistence failure', async () => {
const harness = setup();
harness.sender.sendStrictPlainText.mockRejectedValueOnce(
new QqbotSendAttemptError({
code: 'onebot_timeout',
message: 'unsafe raw timeout',
retryable: true,
sendLogId: 'retry-log',
}),
);
harness.failExternalUpdates(1);
await harness.runner.runOnce(NOW);
expect(harness.externalUpdates).toHaveLength(2);
expect(harness.externalUpdates[1]).toEqual(harness.externalUpdates[0]);
expect(harness.getState().deliveries[0]).toMatchObject({
lastErrorCode: 'onebot_timeout',
lastErrorMessage: 'OneBot send timed out',
nextAttemptAt: new KtDateTime(NOW.getTime() + 10_000),
processingLeaseUntil: null,
sendLogId: 'retry-log',
status: 'retry',
});
});
it('17 retries a typed first failure after 10 seconds with exact safe details', async () => { it('17 retries a typed first failure after 10 seconds with exact safe details', async () => {
const harness = setup(); const harness = setup();
harness.sender.sendStrictPlainText.mockRejectedValueOnce( harness.sender.sendStrictPlainText.mockRejectedValueOnce(
@ -1272,9 +1351,23 @@ describe('System message delivery runner direct preflight contracts', () => {
], ],
targets: [target(), target({ id: '702' })], targets: [target(), target({ id: '702' })],
}); });
harness.failExternalUpdates(3); harness.failExternalUpdates(2);
expect(await harness.runner.runOnce(NOW)).toBe(2); expect(await harness.runner.runOnce(NOW)).toBe(2);
expect(harness.sender.sendStrictPlainText).toHaveBeenCalledTimes(2); expect(harness.sender.sendStrictPlainText).toHaveBeenCalledTimes(2);
expect(harness.externalUpdates.slice(0, 2)).toEqual([
harness.externalUpdates[0],
harness.externalUpdates[0],
]);
expect(
harness.getState().deliveries.find((item) => item.id === '801'),
).toMatchObject({
lastErrorCode: 'old_error',
processingLeaseUntil: new KtDateTime(
NOW.getTime() + SYSTEM_MESSAGE_LEASE_MS,
),
sendLogId: 'old-log',
status: 'processing',
});
expect( expect(
harness.getState().deliveries.find((item) => item.id === '802')?.status, harness.getState().deliveries.find((item) => item.id === '802')?.status,
).toBe('success'); ).toBe('success');