feat: 实现Kafka批量消费与写入以提升吞吐量

引入批量处理机制，将消息缓冲并按批次写入数据库，显著提高消费性能。调整Kafka配置参数，优化消费者并发与提交策略。新增分区索引自动创建功能，并重构处理器以支持批量操作。添加降级写入逻辑以处理数据错误，同时增强指标收集以监控批量处理效果。

bls-onoffline-backend/src/db/databaseManager.js

@@ -35,22 +35,27 @@ export class DatabaseManager {
     if (!rows || rows.length === 0) {
       return;
     }
-    const values = [];
-    const placeholders = rows.map((row, rowIndex) => {
-      const offset = rowIndex * columns.length;
-      columns.forEach((column) => {
-        values.push(row[column] ?? null);
-      });
-      const params = columns.map((_, columnIndex) => `$${offset + columnIndex + 1}`);
-      return `(${params.join(', ')})`;
-    });
     const statement = `
       INSERT INTO ${schema}.${table} (${columns.join(', ')}) 
-      VALUES ${placeholders.join(', ')}
+      SELECT *
+      FROM UNNEST(
+        $1::text[],
+        $2::int8[],
+        $3::int8[],
+        $4::int2[],
+        $5::text[],
+        $6::text[],
+        $7::text[],
+        $8::text[],
+        $9::text[],
+        $10::text[],
+        $11::text[]
+      )
       ON CONFLICT DO NOTHING
     `;
     try {
-      await this.pool.query(statement, values);
+      const params = columns.map((column) => rows.map((row) => row[column] ?? null));
+      await this.pool.query(statement, params);
     } catch (error) {
       logger.error('Database insert failed', {
         error: error?.message,

bls-onoffline-backend/src/db/initializer.js

@@ -23,6 +23,7 @@ class DatabaseInitializer {
 
     // 3. Ensure Partitions for the next month
     await partitionManager.ensurePartitions(30);
+    await partitionManager.ensureIndexesForExistingPartitions();
     
     console.log('Database initialization completed successfully.');
     logger.info('Database initialization completed successfully.');

bls-onoffline-backend/src/db/partitionManager.js

@@ -26,6 +26,75 @@ class PartitionManager {
     return { startMs, endMs, partitionSuffix };
   }
 
+  async ensurePartitionIndexes(client, schema, table, partitionSuffix) {
+    const startedAt = Date.now();
+    const partitionName = `${schema}.${table}_${partitionSuffix}`;
+    const indexBase = `${table}_${partitionSuffix}`;
+
+    const indexSpecs = [
+      { name: `idx_${indexBase}_ts`, column: 'ts_ms' },
+      { name: `idx_${indexBase}_hid`, column: 'hotel_id' },
+      { name: `idx_${indexBase}_mac`, column: 'mac' },
+      { name: `idx_${indexBase}_did`, column: 'device_id' },
+      { name: `idx_${indexBase}_rid`, column: 'room_id' },
+      { name: `idx_${indexBase}_cs`, column: 'current_status' }
+    ];
+
+    for (const spec of indexSpecs) {
+      await client.query(`CREATE INDEX IF NOT EXISTS ${spec.name} ON ${partitionName} (${spec.column});`);
+    }
+
+    await client.query(`ANALYZE ${partitionName};`);
+
+    const elapsedMs = Date.now() - startedAt;
+    if (elapsedMs > 1000) {
+      logger.warn(`Partition index ensure slow`, { partitionName, elapsedMs });
+    }
+  }
+
+  async ensureIndexesForExistingPartitions() {
+    const startedAt = Date.now();
+    const client = await dbManager.pool.connect();
+    try {
+      const schema = config.db.schema;
+      const table = config.db.table;
+
+      const res = await client.query(
+        `
+          SELECT c.relname AS relname
+          FROM pg_inherits i
+          JOIN pg_class p ON i.inhparent = p.oid
+          JOIN pg_namespace pn ON pn.oid = p.relnamespace
+          JOIN pg_class c ON i.inhrelid = c.oid
+          WHERE pn.nspname = $1 AND p.relname = $2
+          ORDER BY c.relname;
+        `,
+        [schema, table]
+      );
+
+      const suffixes = new Set();
+      const pattern = new RegExp(`^${table}_(\\d{8})$`);
+      for (const row of res.rows) {
+        const relname = row?.relname;
+        if (typeof relname !== 'string') continue;
+        const match = relname.match(pattern);
+        if (!match) continue;
+        suffixes.add(match[1]);
+      }
+
+      for (const suffix of suffixes) {
+        await this.ensurePartitionIndexes(client, schema, table, suffix);
+      }
+
+      const elapsedMs = Date.now() - startedAt;
+      if (elapsedMs > 5000) {
+        logger.warn('Ensure existing partition indexes slow', { schema, table, partitions: suffixes.size, elapsedMs });
+      }
+    } finally {
+      client.release();
+    }
+  }
+
   /**
    * Ensure partitions exist for the past M days and next N days.
    * @param {number} daysAhead - Number of days to pre-create.
@@ -63,6 +132,8 @@ class PartitionManager {
           `;
           await client.query(createSql);
         }
+
+        await this.ensurePartitionIndexes(client, schema, table, partitionSuffix);
       }
       logger.info('Partition check completed.');
     } catch (err) {
@@ -72,6 +143,55 @@ class PartitionManager {
       client.release();
     }
   }
+
+  async ensurePartitionsForTimestamps(tsMsList) {
+    if (!Array.isArray(tsMsList) || tsMsList.length === 0) return;
+
+    const uniqueSuffixes = new Set();
+    for (const ts of tsMsList) {
+      const numericTs = typeof ts === 'string' ? Number(ts) : ts;
+      if (!Number.isFinite(numericTs)) continue;
+      const date = new Date(numericTs);
+      if (Number.isNaN(date.getTime())) continue;
+      const { partitionSuffix } = this.getPartitionInfo(date);
+      uniqueSuffixes.add(partitionSuffix);
+      if (uniqueSuffixes.size >= 400) break;
+    }
+
+    if (uniqueSuffixes.size === 0) return;
+
+    const client = await dbManager.pool.connect();
+    try {
+      const schema = config.db.schema;
+      const table = config.db.table;
+
+      for (const partitionSuffix of uniqueSuffixes) {
+        const yyyy = Number(partitionSuffix.slice(0, 4));
+        const mm = Number(partitionSuffix.slice(4, 6));
+        const dd = Number(partitionSuffix.slice(6, 8));
+        if (!Number.isFinite(yyyy) || !Number.isFinite(mm) || !Number.isFinite(dd)) continue;
+        const targetDate = new Date(yyyy, mm - 1, dd);
+        if (Number.isNaN(targetDate.getTime())) continue;
+
+        const { startMs, endMs } = this.getPartitionInfo(targetDate);
+        const partitionName = `${schema}.${table}_${partitionSuffix}`;
+
+        const checkRes = await client.query(`SELECT to_regclass($1) as exists;`, [partitionName]);
+        if (!checkRes.rows[0].exists) {
+          logger.info(`Creating partition ${partitionName} for range [${startMs}, ${endMs})`);
+          await client.query(`
+            CREATE TABLE IF NOT EXISTS ${partitionName}
+            PARTITION OF ${schema}.${table}
+            FOR VALUES FROM (${startMs}) TO (${endMs});
+          `);
+        }
+
+        await this.ensurePartitionIndexes(client, schema, table, partitionSuffix);
+      }
+    } finally {
+      client.release();
+    }
+  }
 }
 
 export default new PartitionManager();