feat: 实现耐久消息投递核心
This commit is contained in:
parent
f18c0dc398
commit
d2d9316465
@ -0,0 +1,169 @@
|
|||||||
|
import {
|
||||||
|
Injectable,
|
||||||
|
Logger,
|
||||||
|
type OnModuleDestroy,
|
||||||
|
type OnModuleInit,
|
||||||
|
} from '@nestjs/common';
|
||||||
|
import { DataSource, In } from 'typeorm';
|
||||||
|
import { isIP } from 'node:net';
|
||||||
|
import { KtDateTime } from '@/common';
|
||||||
|
import { QqbotMessageDelivery } from '../../infrastructure/persistence/message-push/qqbot-message-delivery.entity';
|
||||||
|
import { QqbotMessageEvent } from '../../infrastructure/persistence/message-push/qqbot-message-event.entity';
|
||||||
|
import { QqbotMessageSubscription } from '../../infrastructure/persistence/message-push/qqbot-message-subscription.entity';
|
||||||
|
import {
|
||||||
|
SYSTEM_MESSAGE_BATCH_SIZE,
|
||||||
|
SYSTEM_MESSAGE_SCAN_INTERVAL_MS,
|
||||||
|
} from './system-message-runner.constants';
|
||||||
|
import { SystemMessageDeliveryRunnerService } from './system-message-delivery-runner.service';
|
||||||
|
import { SystemMessageFanoutService } from './system-message-fanout.service';
|
||||||
|
|
||||||
|
const NETWORK_STUN_SOURCE = 'network.stun.mapping-port-changed';
|
||||||
|
|
||||||
|
/** Coordinates durable fan-out and delivery recovery without coupling Core to Admin providers. */
|
||||||
|
@Injectable()
|
||||||
|
export class SystemMessageDeliveryCoordinatorService
|
||||||
|
implements OnModuleInit, OnModuleDestroy
|
||||||
|
{
|
||||||
|
private readonly logger = new Logger(
|
||||||
|
SystemMessageDeliveryCoordinatorService.name,
|
||||||
|
);
|
||||||
|
private destroyed = false;
|
||||||
|
private drainRequested = false;
|
||||||
|
private drainPromise: null | Promise<void> = null;
|
||||||
|
private scanInterval?: NodeJS.Timeout;
|
||||||
|
private startupTimer?: NodeJS.Timeout;
|
||||||
|
|
||||||
|
/** Creates the Core-owned coordinator and both bounded persistence runners. */
|
||||||
|
constructor(
|
||||||
|
private readonly dataSource: DataSource,
|
||||||
|
private readonly fanoutRunner: SystemMessageFanoutService,
|
||||||
|
private readonly deliveryRunner: SystemMessageDeliveryRunnerService,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
/** Starts one post-init wake and an unref'ed durable five-second recovery scan. */
|
||||||
|
onModuleInit(): void {
|
||||||
|
if (this.destroyed || this.scanInterval) return;
|
||||||
|
this.startupTimer = setTimeout(() => {
|
||||||
|
this.startupTimer = undefined;
|
||||||
|
this.requestDrain();
|
||||||
|
}, 0);
|
||||||
|
this.startupTimer.unref?.();
|
||||||
|
this.scanInterval = setInterval(
|
||||||
|
() => this.requestDrain(),
|
||||||
|
SYSTEM_MESSAGE_SCAN_INTERVAL_MS,
|
||||||
|
);
|
||||||
|
this.scanInterval.unref?.();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Stops timers and waits for the active bounded pass without allowing a replacement pass. */
|
||||||
|
async onModuleDestroy(): Promise<void> {
|
||||||
|
this.destroyed = true;
|
||||||
|
this.drainRequested = false;
|
||||||
|
if (this.startupTimer) clearTimeout(this.startupTimer);
|
||||||
|
if (this.scanInterval) clearInterval(this.scanInterval);
|
||||||
|
this.startupTimer = undefined;
|
||||||
|
this.scanInterval = undefined;
|
||||||
|
await this.drainPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Coalesces one or many post-commit wakeups into at most one active drain loop. */
|
||||||
|
requestDrain(): void {
|
||||||
|
if (this.destroyed) return;
|
||||||
|
this.drainRequested = true;
|
||||||
|
if (this.drainPromise) return;
|
||||||
|
this.drainPromise = this.drainLoop()
|
||||||
|
.catch((error: unknown) =>
|
||||||
|
this.logger.warn(
|
||||||
|
'System message drain failed',
|
||||||
|
error instanceof Error ? error.message : undefined,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.finally(() => {
|
||||||
|
this.drainPromise = null;
|
||||||
|
if (!this.destroyed && this.drainRequested) this.requestDrain();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Wakes only address-relevant DDNS-blocked rows after the DDNS transition has committed. */
|
||||||
|
async notifyDdnsSynced(input: {
|
||||||
|
appliedAddress: string;
|
||||||
|
ddnsRecordId: string;
|
||||||
|
}): Promise<void> {
|
||||||
|
if (
|
||||||
|
this.destroyed ||
|
||||||
|
isIP(input.appliedAddress) !== 4 ||
|
||||||
|
!input.ddnsRecordId
|
||||||
|
)
|
||||||
|
return;
|
||||||
|
const advanced = await this.dataSource.transaction(async (manager) => {
|
||||||
|
const subscriptions = await manager
|
||||||
|
.getRepository(QqbotMessageSubscription)
|
||||||
|
.find({
|
||||||
|
where: {
|
||||||
|
enabled: true,
|
||||||
|
isDeleted: false,
|
||||||
|
sourceKey: NETWORK_STUN_SOURCE,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const subscriptionIds = subscriptions
|
||||||
|
.filter(
|
||||||
|
(subscription) =>
|
||||||
|
typeof subscription.sourceConfig?.ddnsRecordId === 'string' &&
|
||||||
|
subscription.sourceConfig.ddnsRecordId === input.ddnsRecordId,
|
||||||
|
)
|
||||||
|
.map((subscription) => subscription.id);
|
||||||
|
if (!subscriptionIds.length) return 0;
|
||||||
|
const events = await manager
|
||||||
|
.getRepository(QqbotMessageEvent)
|
||||||
|
.find({ where: { sourceKey: NETWORK_STUN_SOURCE } });
|
||||||
|
const eventIds = events
|
||||||
|
.filter((event) => event.payload.publicIpv4 === input.appliedAddress)
|
||||||
|
.map((event) => event.id);
|
||||||
|
if (!eventIds.length) return 0;
|
||||||
|
const result = await manager.getRepository(QqbotMessageDelivery).update(
|
||||||
|
{
|
||||||
|
messageEventId: In(eventIds),
|
||||||
|
status: 'waiting_ddns',
|
||||||
|
subscriptionId: In(subscriptionIds),
|
||||||
|
},
|
||||||
|
{ nextAttemptAt: new KtDateTime() },
|
||||||
|
);
|
||||||
|
return result.affected || 0;
|
||||||
|
});
|
||||||
|
if (advanced > 0) this.requestDrain();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Runs ordered fan-out then delivery passes, isolating each runner's failure and backlog signal. */
|
||||||
|
private async drainLoop(): Promise<void> {
|
||||||
|
while (!this.destroyed && this.drainRequested) {
|
||||||
|
this.drainRequested = false;
|
||||||
|
const fanout = await this.runBounded('fan-out', () =>
|
||||||
|
this.fanoutRunner.runOnce(),
|
||||||
|
);
|
||||||
|
const delivery = await this.runBounded('delivery', () =>
|
||||||
|
this.deliveryRunner.runOnce(),
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
fanout === SYSTEM_MESSAGE_BATCH_SIZE ||
|
||||||
|
delivery === SYSTEM_MESSAGE_BATCH_SIZE
|
||||||
|
)
|
||||||
|
this.drainRequested = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Executes one runner without allowing its rejection to starve the other durable queue. */
|
||||||
|
private async runBounded(
|
||||||
|
name: string,
|
||||||
|
runner: () => Promise<number>,
|
||||||
|
): Promise<number> {
|
||||||
|
try {
|
||||||
|
return await runner();
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn(
|
||||||
|
`System message ${name} scan failed`,
|
||||||
|
error instanceof Error ? error.message : undefined,
|
||||||
|
);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,438 @@
|
|||||||
|
import { Injectable } from '@nestjs/common';
|
||||||
|
import { Brackets, DataSource } from 'typeorm';
|
||||||
|
import { KtDateTime } from '@/common';
|
||||||
|
import {
|
||||||
|
SystemMessageContractError,
|
||||||
|
type SystemMessageDeliveryReadiness,
|
||||||
|
} from '../../contract/message-push/qqbot-message-push.types';
|
||||||
|
import { QqbotAccount } from '../../infrastructure/persistence/account/qqbot-account.entity';
|
||||||
|
import { QqbotMessageDelivery } from '../../infrastructure/persistence/message-push/qqbot-message-delivery.entity';
|
||||||
|
import { QqbotMessageEvent } from '../../infrastructure/persistence/message-push/qqbot-message-event.entity';
|
||||||
|
import { QqbotMessagePublishBinding } from '../../infrastructure/persistence/message-push/qqbot-message-publish-binding.entity';
|
||||||
|
import { QqbotMessagePublishTarget } from '../../infrastructure/persistence/message-push/qqbot-message-publish-target.entity';
|
||||||
|
import { QqbotMessageSubscription } from '../../infrastructure/persistence/message-push/qqbot-message-subscription.entity';
|
||||||
|
import { QqbotSendAttemptError } from '../send/qqbot-send.error';
|
||||||
|
import { QqbotSendService } from '../send/qqbot-send.service';
|
||||||
|
import {
|
||||||
|
SYSTEM_MESSAGE_BATCH_SIZE,
|
||||||
|
SYSTEM_MESSAGE_DDNS_RECHECK_MS,
|
||||||
|
SYSTEM_MESSAGE_LEASE_MS,
|
||||||
|
SYSTEM_MESSAGE_RETRY_BASE_MS,
|
||||||
|
SYSTEM_MESSAGE_RETRY_MAX_MS,
|
||||||
|
} from './system-message-runner.constants';
|
||||||
|
import { SystemMessageSourceRegistry } from './system-message-source.registry';
|
||||||
|
import { SystemMessageTemplateRendererService } from './system-message-template-renderer.service';
|
||||||
|
|
||||||
|
const DELIVERY_EXPIRED = 'delivery_expired';
|
||||||
|
const TRANSIENT_ERROR = 'delivery_transient_error';
|
||||||
|
|
||||||
|
type ClaimToken = {
|
||||||
|
attempt: number;
|
||||||
|
delivery: QqbotMessageDelivery;
|
||||||
|
leaseUntil: KtDateTime;
|
||||||
|
};
|
||||||
|
|
||||||
|
type PreparedDelivery =
|
||||||
|
| { kind: 'send'; delivery: QqbotMessageDelivery }
|
||||||
|
| { kind: 'stale' }
|
||||||
|
| {
|
||||||
|
code: string;
|
||||||
|
kind: 'finish';
|
||||||
|
status: 'cancelled' | 'failed' | 'superseded' | 'waiting_ddns';
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Calculates the bounded delay for a one-based durable delivery attempt. */
|
||||||
|
export function deliveryRetryDelayMs(attemptCount: number): number {
|
||||||
|
return Math.min(
|
||||||
|
SYSTEM_MESSAGE_RETRY_BASE_MS * 2 ** Math.max(0, attemptCount - 1),
|
||||||
|
SYSTEM_MESSAGE_RETRY_MAX_MS,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Claims, rechecks, and sends a bounded set of durable system-message deliveries. */
|
||||||
|
@Injectable()
|
||||||
|
export class SystemMessageDeliveryRunnerService {
|
||||||
|
/** Creates the runner from Core persistence, source, renderer, and strict-send boundaries. */
|
||||||
|
constructor(
|
||||||
|
private readonly dataSource: DataSource,
|
||||||
|
private readonly sourceRegistry: SystemMessageSourceRegistry,
|
||||||
|
private readonly templateRenderer: SystemMessageTemplateRendererService,
|
||||||
|
private readonly sendService: QqbotSendService,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
/** Processes up to one bounded batch of due deliveries and returns its claim count. */
|
||||||
|
async runOnce(now: Date = new Date()): Promise<number> {
|
||||||
|
let claimed = 0;
|
||||||
|
for (let index = 0; index < SYSTEM_MESSAGE_BATCH_SIZE; index += 1) {
|
||||||
|
const token = await this.claimOne(now);
|
||||||
|
if (!token) break;
|
||||||
|
claimed += 1;
|
||||||
|
try {
|
||||||
|
await this.processClaim(token, now);
|
||||||
|
} catch {
|
||||||
|
await this.handleUnexpectedClaimFailure(token, now);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return claimed;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Claims one oldest due row in a short transaction and returns its exact owner token. */
|
||||||
|
private async claimOne(now: Date): Promise<ClaimToken | null> {
|
||||||
|
return this.dataSource.transaction(async (manager) => {
|
||||||
|
const deliveries = manager.getRepository(QqbotMessageDelivery);
|
||||||
|
const delivery = await deliveries
|
||||||
|
.createQueryBuilder('delivery')
|
||||||
|
.setLock('pessimistic_write')
|
||||||
|
.setOnLocked('skip_locked')
|
||||||
|
.where(
|
||||||
|
new Brackets((where) => {
|
||||||
|
where
|
||||||
|
.where(
|
||||||
|
'delivery.status IN (:...due) AND delivery.nextAttemptAt <= :now',
|
||||||
|
{ due: ['pending', 'retry', 'waiting_ddns'], now },
|
||||||
|
)
|
||||||
|
.orWhere(
|
||||||
|
'delivery.status = :processing AND delivery.processingLeaseUntil <= :now',
|
||||||
|
{ processing: 'processing', now },
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.orderBy('delivery.nextAttemptAt', 'ASC')
|
||||||
|
.addOrderBy('delivery.id', 'ASC')
|
||||||
|
.take(1)
|
||||||
|
.getOne();
|
||||||
|
if (!delivery) return null;
|
||||||
|
const leaseUntil = new KtDateTime(
|
||||||
|
now.getTime() + SYSTEM_MESSAGE_LEASE_MS,
|
||||||
|
);
|
||||||
|
delivery.attemptCount += 1;
|
||||||
|
delivery.nextAttemptAt = null;
|
||||||
|
delivery.processingLeaseUntil = leaseUntil;
|
||||||
|
delivery.status = 'processing';
|
||||||
|
await deliveries.save(delivery);
|
||||||
|
return { attempt: delivery.attemptCount, delivery, leaseUntil };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Rechecks one claimed delivery before doing external I/O and then performs its final CAS. */
|
||||||
|
private async processClaim(token: ClaimToken, now: Date): Promise<void> {
|
||||||
|
if (now.getTime() >= token.delivery.expiresAt.getTime()) {
|
||||||
|
await this.finish(
|
||||||
|
token,
|
||||||
|
'failed',
|
||||||
|
DELIVERY_EXPIRED,
|
||||||
|
'delivery deadline reached',
|
||||||
|
null,
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const prepared = await this.prepare(token, now);
|
||||||
|
if (prepared.kind === 'stale') return;
|
||||||
|
if (prepared.kind === 'finish') {
|
||||||
|
if (prepared.status === 'waiting_ddns') {
|
||||||
|
await this.finish(
|
||||||
|
token,
|
||||||
|
'waiting_ddns',
|
||||||
|
prepared.code,
|
||||||
|
'DDNS is not synchronized',
|
||||||
|
null,
|
||||||
|
new KtDateTime(now.getTime() + SYSTEM_MESSAGE_DDNS_RECHECK_MS),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
await this.finish(
|
||||||
|
token,
|
||||||
|
prepared.status,
|
||||||
|
prepared.code,
|
||||||
|
prepared.code,
|
||||||
|
null,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const result = await this.sendService.sendStrictPlainText({
|
||||||
|
attemptNumber: token.attempt,
|
||||||
|
deliveryId: token.delivery.id,
|
||||||
|
message: prepared.delivery.renderedMessage,
|
||||||
|
selfId: prepared.delivery.selfId,
|
||||||
|
targetId: prepared.delivery.targetId,
|
||||||
|
targetType: prepared.delivery.targetType,
|
||||||
|
});
|
||||||
|
await this.finish(token, 'success', null, null, String(result.logId));
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof QqbotSendAttemptError) {
|
||||||
|
if (!error.retryable) {
|
||||||
|
await this.finish(
|
||||||
|
token,
|
||||||
|
'failed',
|
||||||
|
error.code,
|
||||||
|
this.safeMessage(error),
|
||||||
|
error.sendLogId,
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await this.retryOrFail(
|
||||||
|
token,
|
||||||
|
now,
|
||||||
|
error.code,
|
||||||
|
this.safeMessage(error),
|
||||||
|
error.sendLogId,
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await this.retryOrFail(
|
||||||
|
token,
|
||||||
|
now,
|
||||||
|
TRANSIENT_ERROR,
|
||||||
|
'delivery transport unavailable',
|
||||||
|
null,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Locks current configuration before locking the delivery and returns a safe next action. */
|
||||||
|
private async prepare(
|
||||||
|
token: ClaimToken,
|
||||||
|
now: Date,
|
||||||
|
): Promise<PreparedDelivery> {
|
||||||
|
return this.dataSource.transaction(async (manager) => {
|
||||||
|
const event = await manager.getRepository(QqbotMessageEvent).findOne({
|
||||||
|
where: { id: token.delivery.messageEventId },
|
||||||
|
lock: { mode: 'pessimistic_read' },
|
||||||
|
});
|
||||||
|
const subscription = await manager
|
||||||
|
.getRepository(QqbotMessageSubscription)
|
||||||
|
.findOne({
|
||||||
|
where: { id: token.delivery.subscriptionId },
|
||||||
|
lock: { mode: 'pessimistic_read' },
|
||||||
|
});
|
||||||
|
const binding = await manager
|
||||||
|
.getRepository(QqbotMessagePublishBinding)
|
||||||
|
.findOne({
|
||||||
|
where: { id: token.delivery.bindingId },
|
||||||
|
lock: { mode: 'pessimistic_read' },
|
||||||
|
});
|
||||||
|
const target = await manager
|
||||||
|
.getRepository(QqbotMessagePublishTarget)
|
||||||
|
.findOne({
|
||||||
|
where: { id: token.delivery.publishTargetId },
|
||||||
|
lock: { mode: 'pessimistic_read' },
|
||||||
|
});
|
||||||
|
const account = binding
|
||||||
|
? await manager.getRepository(QqbotAccount).findOne({
|
||||||
|
where: { id: binding.accountId },
|
||||||
|
lock: { mode: 'pessimistic_read' },
|
||||||
|
})
|
||||||
|
: null;
|
||||||
|
const delivery = await manager
|
||||||
|
.getRepository(QqbotMessageDelivery)
|
||||||
|
.findOne({
|
||||||
|
where: { id: token.delivery.id },
|
||||||
|
lock: { mode: 'pessimistic_write' },
|
||||||
|
});
|
||||||
|
if (!delivery || !this.owns(delivery, token)) return { kind: 'stale' };
|
||||||
|
if (!event)
|
||||||
|
return { code: 'event_invalid', kind: 'finish', status: 'failed' };
|
||||||
|
if (delivery.targetType !== 'group' && delivery.targetType !== 'private')
|
||||||
|
return {
|
||||||
|
code: 'invalid_target_type',
|
||||||
|
kind: 'finish',
|
||||||
|
status: 'failed',
|
||||||
|
};
|
||||||
|
if (
|
||||||
|
!subscription ||
|
||||||
|
!binding ||
|
||||||
|
!target ||
|
||||||
|
!account ||
|
||||||
|
subscription.isDeleted ||
|
||||||
|
binding.isDeleted ||
|
||||||
|
target.isDeleted ||
|
||||||
|
account.isDeleted ||
|
||||||
|
!subscription.enabled ||
|
||||||
|
!binding.enabled ||
|
||||||
|
!target.enabled ||
|
||||||
|
!account.enabled ||
|
||||||
|
subscription.sourceKey !== event.sourceKey ||
|
||||||
|
binding.subscriptionId !== delivery.subscriptionId ||
|
||||||
|
target.bindingId !== delivery.bindingId ||
|
||||||
|
target.targetId !== delivery.targetId ||
|
||||||
|
target.targetType !== delivery.targetType ||
|
||||||
|
binding.selfId !== delivery.selfId ||
|
||||||
|
account.selfId !== delivery.selfId
|
||||||
|
) {
|
||||||
|
return {
|
||||||
|
code: 'delivery_configuration_cancelled',
|
||||||
|
kind: 'finish',
|
||||||
|
status: 'cancelled',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const adapter = this.sourceRegistry.get(event.sourceKey);
|
||||||
|
let eventPayload: Record<string, boolean | null | number | string>;
|
||||||
|
try {
|
||||||
|
eventPayload = adapter.validateEventPayload(event.payload);
|
||||||
|
this.validateFrozen(delivery, event.sourceKey);
|
||||||
|
} catch (error) {
|
||||||
|
if (!(error instanceof SystemMessageContractError)) throw error;
|
||||||
|
return {
|
||||||
|
code: error.code,
|
||||||
|
kind: 'finish',
|
||||||
|
status: 'failed',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const readiness: SystemMessageDeliveryReadiness =
|
||||||
|
await adapter.resolveDelivery({
|
||||||
|
eventPayload,
|
||||||
|
subscriptionConfig: subscription.sourceConfig,
|
||||||
|
});
|
||||||
|
if (readiness.status === 'waiting_ddns') {
|
||||||
|
if (
|
||||||
|
now.getTime() + SYSTEM_MESSAGE_DDNS_RECHECK_MS >=
|
||||||
|
delivery.expiresAt.getTime()
|
||||||
|
)
|
||||||
|
return { code: DELIVERY_EXPIRED, kind: 'finish', status: 'failed' };
|
||||||
|
return {
|
||||||
|
code: readiness.reasonCode,
|
||||||
|
kind: 'finish',
|
||||||
|
status: 'waiting_ddns',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (readiness.status === 'cancelled' || readiness.status === 'superseded')
|
||||||
|
return {
|
||||||
|
code: readiness.reasonCode,
|
||||||
|
kind: 'finish',
|
||||||
|
status: readiness.status,
|
||||||
|
};
|
||||||
|
return { delivery, kind: 'send' };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Validates only immutable frozen content; current templates never replace historical work. */
|
||||||
|
private validateFrozen(
|
||||||
|
delivery: QqbotMessageDelivery,
|
||||||
|
sourceKey: string,
|
||||||
|
): void {
|
||||||
|
const definitions = this.sourceRegistry.get(sourceKey).definition.variables;
|
||||||
|
const allowed = definitions.map((item) => item.key);
|
||||||
|
const snapshot = delivery.variableSnapshot;
|
||||||
|
if (
|
||||||
|
!snapshot ||
|
||||||
|
typeof snapshot !== 'object' ||
|
||||||
|
Array.isArray(snapshot) ||
|
||||||
|
Object.keys(snapshot).length !== definitions.length ||
|
||||||
|
definitions.some(({ key, type }) => {
|
||||||
|
if (!Object.prototype.hasOwnProperty.call(snapshot, key)) return true;
|
||||||
|
const value = snapshot[key];
|
||||||
|
return (
|
||||||
|
value === null ||
|
||||||
|
typeof value !== type ||
|
||||||
|
(type === 'number' &&
|
||||||
|
(typeof value !== 'number' || !Number.isFinite(value)))
|
||||||
|
);
|
||||||
|
}) ||
|
||||||
|
Object.keys(snapshot).some((key) => !allowed.includes(key))
|
||||||
|
)
|
||||||
|
throw new SystemMessageContractError('template_invalid');
|
||||||
|
this.templateRenderer.validate(delivery.templateContent, allowed);
|
||||||
|
if (
|
||||||
|
this.templateRenderer.render(
|
||||||
|
delivery.templateContent,
|
||||||
|
snapshot as Record<string, boolean | number | string>,
|
||||||
|
) !== delivery.renderedMessage
|
||||||
|
)
|
||||||
|
throw new SystemMessageContractError('rendered_message_mismatch');
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Best-effort fences an unexpected dependency failure without starving later claims. */
|
||||||
|
private async handleUnexpectedClaimFailure(
|
||||||
|
token: ClaimToken,
|
||||||
|
now: Date,
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await this.retryOrFail(
|
||||||
|
token,
|
||||||
|
now,
|
||||||
|
TRANSIENT_ERROR,
|
||||||
|
'delivery dependency unavailable',
|
||||||
|
null,
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
// A later scan recovers the still-processing row after its lease expires.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Persists a terminal, wait, or successful state only while this worker owns its lease. */
|
||||||
|
private async finish(
|
||||||
|
token: ClaimToken,
|
||||||
|
status: 'cancelled' | 'failed' | 'success' | 'superseded' | 'waiting_ddns',
|
||||||
|
code: null | string,
|
||||||
|
message: null | string,
|
||||||
|
sendLogId: null | string,
|
||||||
|
nextAttemptAt: KtDateTime | null = null,
|
||||||
|
): Promise<void> {
|
||||||
|
const result = await this.dataSource
|
||||||
|
.getRepository(QqbotMessageDelivery)
|
||||||
|
.update(this.ownerWhere(token), {
|
||||||
|
lastErrorCode: code,
|
||||||
|
lastErrorMessage: message,
|
||||||
|
nextAttemptAt,
|
||||||
|
processingLeaseUntil: null,
|
||||||
|
sendLogId: sendLogId ?? token.delivery.sendLogId,
|
||||||
|
status,
|
||||||
|
});
|
||||||
|
if (result.affected !== 1) return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Retries until the event-derived expiration boundary and otherwise writes a final failure. */
|
||||||
|
private async retryOrFail(
|
||||||
|
token: ClaimToken,
|
||||||
|
now: Date,
|
||||||
|
code: string,
|
||||||
|
message: string,
|
||||||
|
sendLogId: null | string,
|
||||||
|
): Promise<void> {
|
||||||
|
const next = new KtDateTime(
|
||||||
|
now.getTime() + deliveryRetryDelayMs(token.attempt),
|
||||||
|
);
|
||||||
|
if (next.getTime() >= token.delivery.expiresAt.getTime()) {
|
||||||
|
await this.finish(token, 'failed', code, message, sendLogId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const result = await this.dataSource
|
||||||
|
.getRepository(QqbotMessageDelivery)
|
||||||
|
.update(this.ownerWhere(token), {
|
||||||
|
lastErrorCode: code,
|
||||||
|
lastErrorMessage: message,
|
||||||
|
nextAttemptAt: next,
|
||||||
|
processingLeaseUntil: null,
|
||||||
|
sendLogId: sendLogId ?? token.delivery.sendLogId,
|
||||||
|
status: 'retry',
|
||||||
|
});
|
||||||
|
if (result.affected !== 1) return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Builds the exact attempt-and-lease owner fence used by every post-claim mutation. */
|
||||||
|
private ownerWhere(token: ClaimToken) {
|
||||||
|
return {
|
||||||
|
attemptCount: token.attempt,
|
||||||
|
id: token.delivery.id,
|
||||||
|
processingLeaseUntil: token.leaseUntil,
|
||||||
|
status: 'processing' as const,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Tests whether a locked delivery still belongs to the detached claim owner. */
|
||||||
|
private owns(delivery: QqbotMessageDelivery, token: ClaimToken): boolean {
|
||||||
|
return (
|
||||||
|
delivery.status === 'processing' &&
|
||||||
|
delivery.attemptCount === token.attempt &&
|
||||||
|
delivery.processingLeaseUntil?.getTime() === token.leaseUntil.getTime()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Converts unexpected error text into a bounded persistence-safe summary. */
|
||||||
|
private safeMessage(error: Error): string {
|
||||||
|
return String(error.message || 'delivery failed')
|
||||||
|
.replace(/[\r\n]/g, ' ')
|
||||||
|
.slice(0, 500);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -50,12 +50,17 @@ import { QqbotMessageTemplate } from '@/modules/qqbot/core/infrastructure/persis
|
|||||||
import { SystemMessageSourceRegistry } from './application/message-push/system-message-source.registry';
|
import { SystemMessageSourceRegistry } from './application/message-push/system-message-source.registry';
|
||||||
import { SystemMessageEventStagerService } from './application/message-push/system-message-event-stager.service';
|
import { SystemMessageEventStagerService } from './application/message-push/system-message-event-stager.service';
|
||||||
import { SystemMessageFanoutService } from './application/message-push/system-message-fanout.service';
|
import { SystemMessageFanoutService } from './application/message-push/system-message-fanout.service';
|
||||||
|
import { SystemMessageDeliveryRunnerService } from './application/message-push/system-message-delivery-runner.service';
|
||||||
|
import { SystemMessageDeliveryCoordinatorService } from './application/message-push/system-message-delivery-coordinator.service';
|
||||||
import { SystemMessageTemplateRendererService } from './application/message-push/system-message-template-renderer.service';
|
import { SystemMessageTemplateRendererService } from './application/message-push/system-message-template-renderer.service';
|
||||||
import { QqbotMessageSubscriptionService } from './application/message-push/qqbot-message-subscription.service';
|
import { QqbotMessageSubscriptionService } from './application/message-push/qqbot-message-subscription.service';
|
||||||
import { QqbotMessageTemplateService } from './application/message-push/qqbot-message-template.service';
|
import { QqbotMessageTemplateService } from './application/message-push/qqbot-message-template.service';
|
||||||
import { QqbotAccountMessagePushService } from './application/message-push/qqbot-account-message-push.service';
|
import { QqbotAccountMessagePushService } from './application/message-push/qqbot-account-message-push.service';
|
||||||
import { QqbotMessageTargetOptionsService } from './application/message-push/qqbot-message-target-options.service';
|
import { QqbotMessageTargetOptionsService } from './application/message-push/qqbot-message-target-options.service';
|
||||||
import { SYSTEM_MESSAGE_EVENT_STAGER } from './contract/message-push/qqbot-message-push.types';
|
import {
|
||||||
|
SYSTEM_MESSAGE_DELIVERY_COORDINATOR,
|
||||||
|
SYSTEM_MESSAGE_EVENT_STAGER,
|
||||||
|
} from './contract/message-push/qqbot-message-push.types';
|
||||||
|
|
||||||
export { QQBOT_CORE_DOMAIN_CONTRACT } from './contract/qqbot-core.contract';
|
export { QQBOT_CORE_DOMAIN_CONTRACT } from './contract/qqbot-core.contract';
|
||||||
|
|
||||||
@ -94,10 +99,16 @@ export const QQBOT_CORE_PROVIDERS = [
|
|||||||
SystemMessageSourceRegistry,
|
SystemMessageSourceRegistry,
|
||||||
SystemMessageEventStagerService,
|
SystemMessageEventStagerService,
|
||||||
SystemMessageFanoutService,
|
SystemMessageFanoutService,
|
||||||
|
SystemMessageDeliveryRunnerService,
|
||||||
|
SystemMessageDeliveryCoordinatorService,
|
||||||
{
|
{
|
||||||
provide: SYSTEM_MESSAGE_EVENT_STAGER,
|
provide: SYSTEM_MESSAGE_EVENT_STAGER,
|
||||||
useExisting: SystemMessageEventStagerService,
|
useExisting: SystemMessageEventStagerService,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
provide: SYSTEM_MESSAGE_DELIVERY_COORDINATOR,
|
||||||
|
useExisting: SystemMessageDeliveryCoordinatorService,
|
||||||
|
},
|
||||||
SystemMessageTemplateRendererService,
|
SystemMessageTemplateRendererService,
|
||||||
QqbotMessageSubscriptionService,
|
QqbotMessageSubscriptionService,
|
||||||
QqbotMessageTemplateService,
|
QqbotMessageTemplateService,
|
||||||
@ -124,6 +135,7 @@ export const QQBOT_CORE_PROVIDERS = [
|
|||||||
|
|
||||||
export const QQBOT_CORE_EXPORTS = [
|
export const QQBOT_CORE_EXPORTS = [
|
||||||
SYSTEM_MESSAGE_EVENT_STAGER,
|
SYSTEM_MESSAGE_EVENT_STAGER,
|
||||||
|
SYSTEM_MESSAGE_DELIVERY_COORDINATOR,
|
||||||
SystemMessageSourceRegistry,
|
SystemMessageSourceRegistry,
|
||||||
SystemMessageTemplateRendererService,
|
SystemMessageTemplateRendererService,
|
||||||
QqbotMessageSubscriptionService,
|
QqbotMessageSubscriptionService,
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
import { existsSync } from 'fs';
|
import { existsSync, readFileSync } from 'fs';
|
||||||
import { join } from 'path';
|
import { join } from 'path';
|
||||||
import { MODULE_METADATA } from '@nestjs/common/constants';
|
import { MODULE_METADATA } from '@nestjs/common/constants';
|
||||||
import { ConfigModule } from '@nestjs/config';
|
import { ConfigModule } from '@nestjs/config';
|
||||||
@ -16,7 +16,12 @@ import { QqbotRuleController } from '../../../../src/modules/qqbot/core/contract
|
|||||||
import { QqbotSendController } from '../../../../src/modules/qqbot/core/contract/send/qqbot-send.controller';
|
import { QqbotSendController } from '../../../../src/modules/qqbot/core/contract/send/qqbot-send.controller';
|
||||||
import { SystemMessageEventStagerService } from '../../../../src/modules/qqbot/core/application/message-push/system-message-event-stager.service';
|
import { SystemMessageEventStagerService } from '../../../../src/modules/qqbot/core/application/message-push/system-message-event-stager.service';
|
||||||
import { SystemMessageFanoutService } from '../../../../src/modules/qqbot/core/application/message-push/system-message-fanout.service';
|
import { SystemMessageFanoutService } from '../../../../src/modules/qqbot/core/application/message-push/system-message-fanout.service';
|
||||||
import { SYSTEM_MESSAGE_EVENT_STAGER } from '../../../../src/modules/qqbot/core/contract/message-push/qqbot-message-push.types';
|
import { SystemMessageDeliveryCoordinatorService } from '../../../../src/modules/qqbot/core/application/message-push/system-message-delivery-coordinator.service';
|
||||||
|
import { SystemMessageDeliveryRunnerService } from '../../../../src/modules/qqbot/core/application/message-push/system-message-delivery-runner.service';
|
||||||
|
import {
|
||||||
|
SYSTEM_MESSAGE_DELIVERY_COORDINATOR,
|
||||||
|
SYSTEM_MESSAGE_EVENT_STAGER,
|
||||||
|
} from '../../../../src/modules/qqbot/core/contract/message-push/qqbot-message-push.types';
|
||||||
import {
|
import {
|
||||||
QQBOT_CORE_CONTROLLERS,
|
QQBOT_CORE_CONTROLLERS,
|
||||||
QQBOT_CORE_ENTITIES,
|
QQBOT_CORE_ENTITIES,
|
||||||
@ -173,10 +178,16 @@ describe('QQBot core module contract', () => {
|
|||||||
expect.arrayContaining([
|
expect.arrayContaining([
|
||||||
SystemMessageEventStagerService,
|
SystemMessageEventStagerService,
|
||||||
SystemMessageFanoutService,
|
SystemMessageFanoutService,
|
||||||
|
SystemMessageDeliveryRunnerService,
|
||||||
|
SystemMessageDeliveryCoordinatorService,
|
||||||
{
|
{
|
||||||
provide: SYSTEM_MESSAGE_EVENT_STAGER,
|
provide: SYSTEM_MESSAGE_EVENT_STAGER,
|
||||||
useExisting: SystemMessageEventStagerService,
|
useExisting: SystemMessageEventStagerService,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
provide: SYSTEM_MESSAGE_DELIVERY_COORDINATOR,
|
||||||
|
useExisting: SystemMessageDeliveryCoordinatorService,
|
||||||
|
},
|
||||||
]),
|
]),
|
||||||
);
|
);
|
||||||
expect(
|
expect(
|
||||||
@ -184,8 +195,45 @@ describe('QQBot core module contract', () => {
|
|||||||
(provider) => provider === SystemMessageFanoutService,
|
(provider) => provider === SystemMessageFanoutService,
|
||||||
),
|
),
|
||||||
).toHaveLength(1);
|
).toHaveLength(1);
|
||||||
|
expect(
|
||||||
|
QQBOT_CORE_PROVIDERS.filter(
|
||||||
|
(provider) => provider === SystemMessageDeliveryRunnerService,
|
||||||
|
),
|
||||||
|
).toHaveLength(1);
|
||||||
|
expect(
|
||||||
|
QQBOT_CORE_PROVIDERS.filter(
|
||||||
|
(provider) => provider === SystemMessageDeliveryCoordinatorService,
|
||||||
|
),
|
||||||
|
).toHaveLength(1);
|
||||||
expect(QQBOT_CORE_EXPORTS).toEqual(
|
expect(QQBOT_CORE_EXPORTS).toEqual(
|
||||||
expect.arrayContaining([SYSTEM_MESSAGE_EVENT_STAGER]),
|
expect.arrayContaining([
|
||||||
|
SYSTEM_MESSAGE_EVENT_STAGER,
|
||||||
|
SYSTEM_MESSAGE_DELIVERY_COORDINATOR,
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
expect(QQBOT_CORE_EXPORTS).not.toContain(
|
||||||
|
SystemMessageDeliveryCoordinatorService,
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
QQBOT_CORE_PROVIDERS.filter(
|
||||||
|
(provider) =>
|
||||||
|
typeof provider === 'object' &&
|
||||||
|
provider !== null &&
|
||||||
|
'provide' in provider &&
|
||||||
|
provider.provide === SYSTEM_MESSAGE_DELIVERY_COORDINATOR,
|
||||||
|
),
|
||||||
|
).toEqual([
|
||||||
|
{
|
||||||
|
provide: SYSTEM_MESSAGE_DELIVERY_COORDINATOR,
|
||||||
|
useExisting: SystemMessageDeliveryCoordinatorService,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
const coreModuleSource = readFileSync(
|
||||||
|
join(process.cwd(), 'src/modules/qqbot/core/qqbot-core.module.ts'),
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
expect(coreModuleSource).not.toMatch(
|
||||||
|
/network-(?:agent-mqtt|ddns|stun-message-source)/,
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user