feat: 增强Kafka消费者配置并完善action_type分类逻辑
- 支持多实例Kafka消费者,增加并发处理能力 - 新增Kafka配置参数:最大飞行中消息数、获取字节数、等待时间等 - 完善action_type分类逻辑,根据dev_type映射为"用户操作"或"设备回路状态" - 临时支持hex格式udp_raw自动转换为base64存储 - 增加extra字段支持上游扩展数据传递 - 优化数据库初始化脚本查找路径 - 更新PM2配置,修改应用名称和端口 - 清理开发环境日志文件,将dist目录加入.gitignore - 更新相关文档说明
This commit is contained in:
@@ -3,7 +3,7 @@ import { config } from './config/config.js';
|
||||
import dbManager from './db/databaseManager.js';
|
||||
import dbInitializer from './db/initializer.js';
|
||||
import partitionManager from './db/partitionManager.js';
|
||||
import { createKafkaConsumer } from './kafka/consumer.js';
|
||||
import { createKafkaConsumers } from './kafka/consumer.js';
|
||||
import { processKafkaMessage } from './processor/index.js';
|
||||
import { createRedisClient } from './redis/redisClient.js';
|
||||
import { RedisIntegration } from './redis/redisIntegration.js';
|
||||
@@ -79,13 +79,23 @@ const bootstrap = async () => {
|
||||
const messageKey = Buffer.isBuffer(message.key)
|
||||
? message.key.toString('utf8')
|
||||
: message.key;
|
||||
logger.info('Kafka message received', {
|
||||
topic: message.topic,
|
||||
partition: message.partition,
|
||||
offset: message.offset,
|
||||
key: messageKey,
|
||||
value: messageValue
|
||||
});
|
||||
if (config.kafka.logMessages) {
|
||||
logger.info('Kafka message received', {
|
||||
topic: message.topic,
|
||||
partition: message.partition,
|
||||
offset: message.offset,
|
||||
key: messageKey,
|
||||
value: messageValue
|
||||
});
|
||||
} else {
|
||||
logger.info('Kafka message received', {
|
||||
topic: message.topic,
|
||||
partition: message.partition,
|
||||
offset: message.offset,
|
||||
key: messageKey,
|
||||
valueLength: typeof messageValue === 'string' ? messageValue.length : null
|
||||
});
|
||||
}
|
||||
const inserted = await processKafkaMessage({ message, dbManager, config });
|
||||
metricCollector.increment('db_inserted');
|
||||
logger.info('Kafka message processed', { inserted });
|
||||
@@ -143,7 +153,7 @@ const bootstrap = async () => {
|
||||
}
|
||||
};
|
||||
|
||||
const consumer = createKafkaConsumer({
|
||||
const consumers = createKafkaConsumers({
|
||||
kafkaConfig: config.kafka,
|
||||
onMessage: handleMessage,
|
||||
onError: handleError
|
||||
@@ -170,9 +180,9 @@ const bootstrap = async () => {
|
||||
|
||||
try {
|
||||
// 1. Close Kafka Consumer
|
||||
if (consumer) {
|
||||
await new Promise((resolve) => consumer.close(true, resolve));
|
||||
logger.info('Kafka consumer closed');
|
||||
if (consumers && consumers.length > 0) {
|
||||
await Promise.all(consumers.map(c => new Promise((resolve) => c.close(true, resolve))));
|
||||
logger.info('Kafka consumer closed', { count: consumers.length });
|
||||
}
|
||||
|
||||
// 2. Stop Redis Heartbeat (if method exists, otherwise just close client)
|
||||
|
||||
Reference in New Issue
Block a user