feat: 新增 G4 热表独立双写能力
- 新增配置项以支持旧/新明细表的独立写入开关及目标表名。 - 重构 DatabaseManager,抽象通用批量 COPY 写入内核,支持不同目标表的复用。 - 新增双明细写入编排器,支持旧/新表独立执行、重试及 fallback。 - 调整 HeartbeatProcessor.processBatch(),确保 room_status 独立执行。 - 错误表仅记录新表写入失败,旧表失败不再写入错误表。 - 重新定义消费暂停策略,基于当前启用的关键 sink 判断。 - 补充按 sink 维度的统计项与启动日志。 新增 G4 热表相关的数据库规范与处理逻辑,确保系统在双写模式下的稳定性与可扩展性。
This commit is contained in:
81
.env.example
81
.env.example
@@ -1,13 +1,29 @@
|
||||
# 应用配置
|
||||
# =========================
|
||||
# 应用基础配置
|
||||
# =========================
|
||||
NODE_ENV=production
|
||||
PORT=3000
|
||||
|
||||
# Kafka配置
|
||||
# 日志级别:debug | info | warn | error
|
||||
LOG_LEVEL=info
|
||||
LOG_FORMAT=json
|
||||
|
||||
# =========================
|
||||
# Kafka 配置
|
||||
# =========================
|
||||
# 多个 broker 用英文逗号分隔
|
||||
KAFKA_BROKERS=localhost:9092
|
||||
KAFKA_TOPIC=blwlog4Nodejs-rcu-heartbeat-topic
|
||||
KAFKA_TOPICS=
|
||||
KAFKA_GROUP_ID=bls-heartbeat-consumer
|
||||
KAFKA_CLIENT_ID=bls-heartbeat
|
||||
KAFKA_GROUP_ID=bls-heartbeat-consumer
|
||||
|
||||
# 推荐使用 KAFKA_TOPICS;多个 topic 用英文逗号分隔
|
||||
KAFKA_TOPICS=blwlog4Nodejs-rcu-heartbeat-topic
|
||||
|
||||
# 兼容旧写法;如已使用 KAFKA_TOPICS,可不填写
|
||||
# KAFKA_TOPIC=blwlog4Nodejs-rcu-heartbeat-topic
|
||||
|
||||
# latest / earliest
|
||||
KAFKA_FROM_OFFSET=latest
|
||||
KAFKA_CONSUMER_INSTANCES=6
|
||||
KAFKA_AUTO_COMMIT=true
|
||||
KAFKA_AUTO_COMMIT_INTERVAL_MS=5000
|
||||
@@ -18,16 +34,16 @@ KAFKA_FETCH_MAX_BYTES=10485760
|
||||
KAFKA_FETCH_MIN_BYTES=1
|
||||
KAFKA_FETCH_MAX_WAIT_MS=100
|
||||
|
||||
# Kafka SASL配置(如果需要)
|
||||
# SASL/SSL 按实际环境开启
|
||||
KAFKA_SASL_ENABLED=false
|
||||
KAFKA_SASL_MECHANISM=plain
|
||||
KAFKA_SASL_USERNAME=
|
||||
KAFKA_SASL_PASSWORD=
|
||||
|
||||
# Kafka SSL配置(如果需要)
|
||||
KAFKA_SSL_ENABLED=false
|
||||
|
||||
# PostgreSQL配置
|
||||
# =========================
|
||||
# PostgreSQL 配置
|
||||
# =========================
|
||||
POSTGRES_HOST=127.0.0.1
|
||||
POSTGRES_PORT=5432
|
||||
POSTGRES_DATABASE=log_platform
|
||||
@@ -35,38 +51,47 @@ POSTGRES_USER=log_admin
|
||||
POSTGRES_PASSWORD=your_password
|
||||
POSTGRES_IDLE_TIMEOUT_MS=30000
|
||||
|
||||
# PostgreSQL环境变量(兼容性)
|
||||
PGHOST=
|
||||
PGPORT=
|
||||
PGTARGETDB=
|
||||
PGUSER=
|
||||
PGPASSWORD=
|
||||
# 可选:兼容标准 PG 环境变量;留空则优先使用上面的 POSTGRES_*
|
||||
# PGHOST=
|
||||
# PGPORT=
|
||||
# PGTARGETDB=
|
||||
# PGUSER=
|
||||
# PGPASSWORD=
|
||||
|
||||
# 数据库重试配置
|
||||
# 数据库失败重试
|
||||
DB_RETRY_ATTEMPTS=3
|
||||
DB_RETRY_DELAY=1000
|
||||
DB_PARTITION_MAINTENANCE_ENABLED=true
|
||||
DB_PARTITION_FUTURE_DAYS=30
|
||||
DB_PARTITION_INTERVAL_HOURS=6
|
||||
|
||||
# Redis配置
|
||||
# 双写开关
|
||||
# 旧明细表:heartbeat.heartbeat_events
|
||||
DB_LEGACY_HEARTBEAT_ENABLED=true
|
||||
# 新热表:heartbeat.heartbeat_events_g4_hot
|
||||
DB_G4_HOT_HEARTBEAT_ENABLED=false
|
||||
# room_status 写入开关
|
||||
DB_ROOM_STATUS_ENABLED=true
|
||||
|
||||
# 如无特殊需要,保持默认表名即可
|
||||
DB_LEGACY_TABLE=heartbeat.heartbeat_events
|
||||
DB_G4_HOT_TABLE=heartbeat.heartbeat_events_g4_hot
|
||||
|
||||
# =========================
|
||||
# Redis 配置
|
||||
# =========================
|
||||
REDIS_ENABLED=true
|
||||
REDIS_HOST=10.8.8.109
|
||||
REDIS_PORT=6379
|
||||
REDIS_PASSWORD=
|
||||
REDIS_DB=15
|
||||
|
||||
# 若提供 REDIS_URL,则业务侧可优先使用统一连接串
|
||||
REDIS_URL=
|
||||
REDIS_CONNECT_TIMEOUT_MS=5000
|
||||
REDIS_PROJECT_NAME=BLS主机心跳日志
|
||||
REDIS_HEARTBEAT_INTERVAL_MS=3000
|
||||
REDIS_HEARTBEAT_TTL_SECONDS=30
|
||||
REDIS_API_BASE_URL=http://127.0.0.1:3000
|
||||
REDIS_CONSOLE_MAX_LEN=
|
||||
|
||||
# 处理器配置
|
||||
# =========================
|
||||
# 批处理配置
|
||||
# =========================
|
||||
PROCESSOR_BATCH_SIZE=100
|
||||
PROCESSOR_BATCH_TIMEOUT=5000
|
||||
|
||||
# 日志配置
|
||||
LOG_LEVEL=info
|
||||
LOG_FORMAT=json
|
||||
PROCESSOR_BATCH_TIMEOUT=5000
|
||||
Reference in New Issue
Block a user