feat(processor): 实现批量写库容错机制并添加失败统计
添加数据库批量写入失败处理逻辑,当批量写入失败时自动切换为逐条写入 记录失败数据并统计失败数量,同时更新相关测试和统计模块 符合新增的批量写库容错需求规范
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
class StatsCounters {
|
||||
constructor() {
|
||||
this._minuteBuf = new SharedArrayBuffer(BigInt64Array.BYTES_PER_ELEMENT * 3);
|
||||
this._minuteBuf = new SharedArrayBuffer(BigInt64Array.BYTES_PER_ELEMENT * 4);
|
||||
this._minute = new BigInt64Array(this._minuteBuf);
|
||||
}
|
||||
|
||||
@@ -22,11 +22,18 @@ class StatsCounters {
|
||||
Atomics.add(this._minute, 2, v);
|
||||
}
|
||||
|
||||
incDbWriteFailed(n = 1) {
|
||||
const v = BigInt(Math.max(0, Number(n) || 0));
|
||||
if (v === 0n) return;
|
||||
Atomics.add(this._minute, 3, v);
|
||||
}
|
||||
|
||||
snapshotAndResetMinute() {
|
||||
const dbWritten = Atomics.exchange(this._minute, 0, 0n);
|
||||
const filtered = Atomics.exchange(this._minute, 1, 0n);
|
||||
const kafkaPulled = Atomics.exchange(this._minute, 2, 0n);
|
||||
return { dbWritten, filtered, kafkaPulled };
|
||||
const dbWriteFailed = Atomics.exchange(this._minute, 3, 0n);
|
||||
return { dbWritten, filtered, kafkaPulled, dbWriteFailed };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,6 +57,7 @@ class StatsReporter {
|
||||
this.stats = stats;
|
||||
this._timer = null;
|
||||
this._running = false;
|
||||
this._lastFlushMinute = null;
|
||||
}
|
||||
|
||||
start() {
|
||||
@@ -68,9 +76,16 @@ class StatsReporter {
|
||||
|
||||
flushOnce() {
|
||||
if (!this.redis?.isEnabled?.()) return;
|
||||
const { dbWritten, filtered, kafkaPulled } = this.stats.snapshotAndResetMinute();
|
||||
const now = Date.now();
|
||||
const minuteKey = Math.floor(now / 60_000);
|
||||
if (this._lastFlushMinute === minuteKey) {
|
||||
return;
|
||||
}
|
||||
const { dbWritten, filtered, kafkaPulled, dbWriteFailed } = this.stats.snapshotAndResetMinute();
|
||||
this._lastFlushMinute = minuteKey;
|
||||
const ts = formatTimestamp(new Date());
|
||||
this.redis.pushConsoleLog?.({ level: 'info', message: `[STATS] ${ts} 数据库写入量: ${dbWritten}条`, metadata: { module: 'stats' } });
|
||||
this.redis.pushConsoleLog?.({ level: 'info', message: `[STATS] ${ts} 数据库写入失败量: ${dbWriteFailed}条`, metadata: { module: 'stats' } });
|
||||
this.redis.pushConsoleLog?.({ level: 'info', message: `[STATS] ${ts} 数据过滤量: ${filtered}条`, metadata: { module: 'stats' } });
|
||||
this.redis.pushConsoleLog?.({ level: 'info', message: `[STATS] ${ts} Kafka拉取量: ${kafkaPulled}条`, metadata: { module: 'stats' } });
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user