XuJiacheng
8337c60f98
引入批量处理机制，将消息缓冲并按批次写入数据库，显著提高消费性能。调整Kafka配置参数，优化消费者并发与提交策略。新增分区索引自动创建功能，并重构处理器以支持批量操作。添加降级写入逻辑以处理数据错误，同时增强指标收集以监控批量处理效果。
42 lines
1.0 KiB
Bash
42 lines
1.0 KiB
Bash
KAFKA_BROKERS=kafka.blv-oa.com:9092
|
|
KAFKA_CLIENT_ID=bls-onoffline-producer
|
|
KAFKA_GROUP_ID=bls-onoffline-consumer
|
|
KAFKA_TOPICS=blwlog4Nodejs-rcu-onoffline-topic-0
|
|
KAFKA_AUTO_COMMIT=false
|
|
KAFKA_AUTO_COMMIT_INTERVAL_MS=5000
|
|
KAFKA_SASL_ENABLED=true
|
|
KAFKA_SASL_MECHANISM=plain
|
|
KAFKA_SASL_USERNAME=blwmomo
|
|
KAFKA_SASL_PASSWORD=blwmomo
|
|
KAFKA_SSL_ENABLED=false
|
|
KAFKA_CONSUMER_INSTANCES=3
|
|
KAFKA_MAX_IN_FLIGHT=5000
|
|
KAFKA_BATCH_SIZE=1000
|
|
KAFKA_BATCH_TIMEOUT_MS=20
|
|
KAFKA_COMMIT_INTERVAL_MS=200
|
|
KAFKA_COMMIT_ON_ATTEMPT=true
|
|
KAFKA_FETCH_MAX_BYTES=10485760
|
|
KAFKA_FETCH_MAX_WAIT_MS=100
|
|
KAFKA_FETCH_MIN_BYTES=1
|
|
|
|
POSTGRES_HOST=10.8.8.109
|
|
POSTGRES_PORT=5433
|
|
POSTGRES_DATABASE=log_platform
|
|
POSTGRES_USER=log_admin
|
|
POSTGRES_PASSWORD=YourActualStrongPasswordForPostgres!
|
|
POSTGRES_MAX_CONNECTIONS=6
|
|
POSTGRES_IDLE_TIMEOUT_MS=30000
|
|
DB_SCHEMA=onoffline
|
|
DB_TABLE=onoffline_record
|
|
|
|
PORT=3001
|
|
LOG_LEVEL=info
|
|
|
|
# Redis connection
|
|
REDIS_HOST=10.8.8.109
|
|
REDIS_PORT=6379
|
|
REDIS_PASSWORD=
|
|
REDIS_DB=15
|
|
REDIS_CONNECT_TIMEOUT_MS=5000
|
|
REDIS_PROJECT_NAME=bls-onoffline
|