feat(processor): 实现批量写库容错机制并添加失败统计

添加数据库批量写入失败处理逻辑,当批量写入失败时自动切换为逐条写入
记录失败数据并统计失败数量,同时更新相关测试和统计模块
符合新增的批量写库容错需求规范
This commit is contained in:
2026-01-20 08:22:55 +08:00
parent 41301f9ce5
commit b90faf4aa4
5 changed files with 111 additions and 44 deletions

View File

@@ -1,6 +1,6 @@
class StatsCounters {
constructor() {
this._minuteBuf = new SharedArrayBuffer(BigInt64Array.BYTES_PER_ELEMENT * 3);
this._minuteBuf = new SharedArrayBuffer(BigInt64Array.BYTES_PER_ELEMENT * 4);
this._minute = new BigInt64Array(this._minuteBuf);
}
@@ -22,11 +22,18 @@ class StatsCounters {
Atomics.add(this._minute, 2, v);
}
incDbWriteFailed(n = 1) {
const v = BigInt(Math.max(0, Number(n) || 0));
if (v === 0n) return;
Atomics.add(this._minute, 3, v);
}
snapshotAndResetMinute() {
const dbWritten = Atomics.exchange(this._minute, 0, 0n);
const filtered = Atomics.exchange(this._minute, 1, 0n);
const kafkaPulled = Atomics.exchange(this._minute, 2, 0n);
return { dbWritten, filtered, kafkaPulled };
const dbWriteFailed = Atomics.exchange(this._minute, 3, 0n);
return { dbWritten, filtered, kafkaPulled, dbWriteFailed };
}
}
@@ -50,6 +57,7 @@ class StatsReporter {
this.stats = stats;
this._timer = null;
this._running = false;
this._lastFlushMinute = null;
}
start() {
@@ -68,9 +76,16 @@ class StatsReporter {
flushOnce() {
if (!this.redis?.isEnabled?.()) return;
const { dbWritten, filtered, kafkaPulled } = this.stats.snapshotAndResetMinute();
const now = Date.now();
const minuteKey = Math.floor(now / 60_000);
if (this._lastFlushMinute === minuteKey) {
return;
}
const { dbWritten, filtered, kafkaPulled, dbWriteFailed } = this.stats.snapshotAndResetMinute();
this._lastFlushMinute = minuteKey;
const ts = formatTimestamp(new Date());
this.redis.pushConsoleLog?.({ level: 'info', message: `[STATS] ${ts} 数据库写入量: ${dbWritten}`, metadata: { module: 'stats' } });
this.redis.pushConsoleLog?.({ level: 'info', message: `[STATS] ${ts} 数据库写入失败量: ${dbWriteFailed}`, metadata: { module: 'stats' } });
this.redis.pushConsoleLog?.({ level: 'info', message: `[STATS] ${ts} 数据过滤量: ${filtered}`, metadata: { module: 'stats' } });
this.redis.pushConsoleLog?.({ level: 'info', message: `[STATS] ${ts} Kafka拉取量: ${kafkaPulled}`, metadata: { module: 'stats' } });
}