feat: 更新 Kafka 配置和数据库管理逻辑

- 在 .env.example 中添加 Kafka 配置项:KAFKA_FETCH_MAX_BYTES, KAFKA_FETCH_MIN_BYTES, KAFKA_FETCH_MAX_WAIT_MS。
- 删除 room_status_sync 提案及相关文档。
- 删除 fix_uint64_overflow 提案及相关文档。
- 更新数据库管理器以支持使用 COPY 语句进行高效数据写入,替换批量 INSERT 逻辑。
- 实现心跳数据的整数溢出处理,确保无效数据被持久化到 heartbeat_events_errors 表。
- 更新处理器规范,确保心跳数据成功写入历史表后触发 room_status 同步。
- 添加新文档,描述新的分区方法案例。
- 归档旧的提案和规范文档以保持项目整洁。
This commit is contained in:
2026-03-03 18:22:12 +08:00
parent d0c4940e01
commit c0cdc9ea66
19 changed files with 188 additions and 215 deletions

View File

@@ -33,36 +33,24 @@ AS $$
SELECT format('heartbeat_events_%s', to_char(p_day, 'YYYYMMDD'));
$$;
-- 创建单日分区(若不存在)并创建该分区上的索引
-- 创建单日分区(幂等);父表索引自动继承到子表,无需手动建索引
CREATE OR REPLACE FUNCTION heartbeat.create_daily_partition(p_day date)
RETURNS void
LANGUAGE plpgsql
AS $$
DECLARE
start_ms bigint;
end_ms bigint;
end_ms bigint;
part_name text;
BEGIN
start_ms := heartbeat.day_start_ms_shanghai(p_day);
end_ms := start_ms + 86400000;
start_ms := heartbeat.day_start_ms_shanghai(p_day);
end_ms := start_ms + 86400000;
part_name := heartbeat.partition_name_for_day(p_day);
IF to_regclass(format('heartbeat.%I', part_name)) IS NOT NULL THEN
RETURN;
END IF;
EXECUTE format(
'CREATE TABLE heartbeat.%I PARTITION OF heartbeat.heartbeat_events FOR VALUES FROM (%s) TO (%s);',
'CREATE TABLE IF NOT EXISTS heartbeat.%I PARTITION OF heartbeat.heartbeat_events FOR VALUES FROM (%s) TO (%s)',
part_name, start_ms, end_ms
);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON heartbeat.%I (hotel_id);', 'idx_'||part_name||'_hotel_id', part_name);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON heartbeat.%I (power_state);', 'idx_'||part_name||'_power_state', part_name);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON heartbeat.%I (guest_type);', 'idx_'||part_name||'_guest_type', part_name);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON heartbeat.%I (device_id);', 'idx_'||part_name||'_device_id', part_name);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON heartbeat.%I USING BRIN (service_mask);', 'idx_'||part_name||'_service_mask_brin', part_name);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON heartbeat.%I ((service_mask & 1));', 'idx_'||part_name||'_service_mask_first_bit', part_name);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON heartbeat.%I (hotel_id, ts_ms);', 'idx_'||part_name||'_hotel_ts', part_name);
END;
$$;
@@ -78,10 +66,9 @@ BEGIN
RAISE EXCEPTION 'p_end_day (%) must be >= p_start_day (%)', p_end_day, p_start_day;
END IF;
d := p_start_day;
WHILE d <= p_end_day LOOP
FOR d IN SELECT generate_series(p_start_day, p_end_day, interval '1 day')::date
LOOP
PERFORM heartbeat.create_daily_partition(d);
d := d + 1;
END LOOP;
END;
$$;

View File

@@ -26,7 +26,7 @@ const args = parseArgs(process.argv);
if (args.help) usageAndExit(0);
const processor = new HeartbeatProcessor(
{ batchSize: 9999, batchTimeout: 1000 },
{ batchSize: 30000, batchTimeout: 5000 },
{ insertHeartbeatEvents: async () => {} }
);