- 添加Kafka消费者组件用于消费升级事件数据 - 实现数据处理器进行数据验证和转换 - 添加数据库写入组件支持批量写入G5数据库 - 配置环境变量管理连接参数 - 添加日志记录和错误处理机制 - 实现优雅关闭和流控功能
54 lines
1.3 KiB
JavaScript
54 lines
1.3 KiB
JavaScript
class OffsetTracker {
|
|
constructor() {
|
|
this.partitions = new Map();
|
|
}
|
|
|
|
add(topic, partition, offset) {
|
|
const key = `${topic}-${partition}`;
|
|
if (!this.partitions.has(key)) {
|
|
this.partitions.set(key, { nextCommitOffset: null, done: new Set() });
|
|
}
|
|
|
|
const state = this.partitions.get(key);
|
|
const numericOffset = Number(offset);
|
|
if (!Number.isFinite(numericOffset)) return;
|
|
|
|
if (state.nextCommitOffset === null) {
|
|
state.nextCommitOffset = numericOffset;
|
|
} else if (numericOffset < state.nextCommitOffset) {
|
|
state.nextCommitOffset = numericOffset;
|
|
}
|
|
}
|
|
|
|
markDone(topic, partition, offset) {
|
|
const key = `${topic}-${partition}`;
|
|
const state = this.partitions.get(key);
|
|
if (!state) return null;
|
|
|
|
const numericOffset = Number(offset);
|
|
if (!Number.isFinite(numericOffset)) return null;
|
|
|
|
state.done.add(numericOffset);
|
|
|
|
if (state.nextCommitOffset === null) {
|
|
state.nextCommitOffset = numericOffset;
|
|
}
|
|
|
|
let advanced = false;
|
|
while (state.nextCommitOffset !== null && state.done.has(state.nextCommitOffset)) {
|
|
state.done.delete(state.nextCommitOffset);
|
|
state.nextCommitOffset += 1;
|
|
advanced = true;
|
|
}
|
|
|
|
if (!advanced) return null;
|
|
return state.nextCommitOffset;
|
|
}
|
|
|
|
clear() {
|
|
this.partitions.clear();
|
|
}
|
|
}
|
|
|
|
export { OffsetTracker };
|