feat: 增强Kafka消费者配置并完善action_type分类逻辑

- 支持多实例Kafka消费者,增加并发处理能力
- 新增Kafka配置参数:最大飞行中消息数、获取字节数、等待时间等
- 完善action_type分类逻辑,根据dev_type映射为"用户操作"或"设备回路状态"
- 临时支持hex格式udp_raw自动转换为base64存储
- 增加extra字段支持上游扩展数据传递
- 优化数据库初始化脚本查找路径
- 更新PM2配置,修改应用名称和端口
- 清理开发环境日志文件,将dist目录加入.gitignore
- 更新相关文档说明
This commit is contained in:
2026-01-30 20:09:46 +08:00
parent e0c3728b42
commit 0e6c5c3cc3
18 changed files with 459 additions and 57 deletions

View File

@@ -3,7 +3,7 @@ import { config } from './config/config.js';
import dbManager from './db/databaseManager.js';
import dbInitializer from './db/initializer.js';
import partitionManager from './db/partitionManager.js';
import { createKafkaConsumer } from './kafka/consumer.js';
import { createKafkaConsumers } from './kafka/consumer.js';
import { processKafkaMessage } from './processor/index.js';
import { createRedisClient } from './redis/redisClient.js';
import { RedisIntegration } from './redis/redisIntegration.js';
@@ -79,13 +79,23 @@ const bootstrap = async () => {
const messageKey = Buffer.isBuffer(message.key)
? message.key.toString('utf8')
: message.key;
logger.info('Kafka message received', {
topic: message.topic,
partition: message.partition,
offset: message.offset,
key: messageKey,
value: messageValue
});
if (config.kafka.logMessages) {
logger.info('Kafka message received', {
topic: message.topic,
partition: message.partition,
offset: message.offset,
key: messageKey,
value: messageValue
});
} else {
logger.info('Kafka message received', {
topic: message.topic,
partition: message.partition,
offset: message.offset,
key: messageKey,
valueLength: typeof messageValue === 'string' ? messageValue.length : null
});
}
const inserted = await processKafkaMessage({ message, dbManager, config });
metricCollector.increment('db_inserted');
logger.info('Kafka message processed', { inserted });
@@ -143,7 +153,7 @@ const bootstrap = async () => {
}
};
const consumer = createKafkaConsumer({
const consumers = createKafkaConsumers({
kafkaConfig: config.kafka,
onMessage: handleMessage,
onError: handleError
@@ -170,9 +180,9 @@ const bootstrap = async () => {
try {
// 1. Close Kafka Consumer
if (consumer) {
await new Promise((resolve) => consumer.close(true, resolve));
logger.info('Kafka consumer closed');
if (consumers && consumers.length > 0) {
await Promise.all(consumers.map(c => new Promise((resolve) => c.close(true, resolve))));
logger.info('Kafka consumer closed', { count: consumers.length });
}
// 2. Stop Redis Heartbeat (if method exists, otherwise just close client)