feat: 处理整数溢出并持久化无法处理的数据

- 更新 heartbeatProcessor.js 以处理所有数字溢出类型（int16、int32、int64）并使用二进制补码。
- 防止仅与数据相关的 PostgreSQL 失败抛出个别回退错误。
- 在 databaseManager.js 中添加 insertHeartbeatEventsErrors 方法以存储被拒绝的记录。
- 更新 heartbeatProcessor.js 中的 _emitRejectedRecord 方法，直接将所有无法处理的心跳数据写入 heartbeat_events_errors 数据库。
- 更新 openspec 规范以支持新的溢出和验证回退状态。
- 添加测试文件以验证大整数处理。

docs/heartbeat_events_errors.sql

@@ -0,0 +1,37 @@
+/*
+ Navicat Premium Dump SQL
+
+ Source Server         : FnOS 109
+ Source Server Type    : PostgreSQL
+ Source Server Version : 150004 (150004)
+ Source Host           : 10.8.8.109:5433
+ Source Catalog        : log_platform
+ Source Schema         : heartbeat
+
+ Target Server Type    : PostgreSQL
+ Target Server Version : 150004 (150004)
+ File Encoding         : 65001
+
+ Date: 02/03/2026 09:36:55
+*/
+
+
+-- ----------------------------
+-- Table structure for heartbeat_events_errors
+-- ----------------------------
+DROP TABLE IF EXISTS "heartbeat"."heartbeat_events_errors";
+CREATE TABLE "heartbeat"."heartbeat_events_errors" (
+  "error_id" int4 NOT NULL DEFAULT nextval('"heartbeat".heartbeat_events_errors_error_id_seq'::regclass),
+  "error_time" timestamp(6) DEFAULT now(),
+  "hotel_id" int4,
+  "room_id" int4,
+  "original_data" jsonb,
+  "error_code" text COLLATE "pg_catalog"."default",
+  "error_message" text COLLATE "pg_catalog"."default"
+)
+;
+
+-- ----------------------------
+-- Primary Key structure for table heartbeat_events_errors
+-- ----------------------------
+ALTER TABLE "heartbeat"."heartbeat_events_errors" ADD CONSTRAINT "heartbeat_events_errors_pkey" PRIMARY KEY ("error_id");

openspec/changes/fix-uint64-overflow/proposal.md

@@ -0,0 +1,15 @@
+# Change: Handle integer overflows and persist unprocessable data
+
+## Why
+Hardware devices occasionally report `service_mask` or other bitmasks and counters (like `power_state`) where their unsigned values exceed PostgreSQL's signed boundaries (e.g. `uint64`, `uint32`, `uint16`). This triggers out of range database insertion errors which causes batch failure and falls back to individual row insertions. Previously, rows that failed due to data range constraints directly crashed out or were only logged to Redis, meaning fully invalid data or boundary constraint violations were fundamentally lost from DB history.
+
+## What Changes
+- Safely map completely oversized integers to signed `int64`, `int32`, `int16` 2's complement equivalents natively in Javascript (e.g. `(v << 16) >> 16` for `int2`).
+- Refine the loop mechanism in `databaseManager.js` to avoid throwing errors exclusively built from data-level constraint mismatches when doing individual row fallback.
+- Extend `_emitRejectedRecord` to persist any unprocessable, validation-failing, or insert-failing raw records directly into a dedicated error database table: `heartbeat_events_errors`.
+
+## Impact
+- Affected specs: `processor`
+- Affected code:
+  - `src/processor/heartbeatProcessor.js`
+  - `src/db/databaseManager.js`

openspec/changes/fix-uint64-overflow/specs/processor/spec.md

@@ -0,0 +1,17 @@
+## MODIFIED Requirements
+### Requirement: 心跳数据转换
+系统 MUST 能够将解包后的心跳数据转换为数据库存储格式。
+
+#### Scenario: 转换为 v2 明细表字段
+- **WHEN** 心跳数据验证通过时
+- **THEN** 系统应输出与 v2 明细表字段一致的数据结构
+- **AND** 添加必要的元数据
+
+#### Scenario: 缺失必填字段
+- **WHEN** 心跳数据缺失必填字段时
+- **THEN** 系统应判定为无效数据并丢弃
+
+#### Scenario: 兼容超限的 uint64 字段转换为有符号 bigint
+- **WHEN** `service_mask` 等字段由于无符号位满转为十进制后超出 PostgreSQL 的 signed bigint 上限
+- **THEN** 系统 MUST 将该长数字利用补码映射转换其等价二进制结构对应的有符号负数
+- **AND** 保证原始 Bit 掩码结构完整并确保能顺利存入数据库

openspec/changes/fix-uint64-overflow/tasks.md

@@ -0,0 +1,6 @@
+## 1. Implementation
+- [x] 1.1 Update `heartbeatProcessor.js` to handle all numeric overflow types (`int16`, `int32`, `int64`) with two's complement.
+- [x] 1.2 Prevent purely data-related Postgres failures from throwing away individual fallbacks within `databaseManager.js`.
+- [x] 1.3 Add `insertHeartbeatEventsErrors` to `databaseManager.js` to sink rejected records.
+- [x] 1.4 Wire `_emitRejectedRecord` in `heartbeatProcessor.js` to directly write all completely unprocessable heartbeats into the `heartbeat_events_errors` DB.
+- [x] 1.5 Update the `openspec` specs with these newly supported overflow & validation fallback states.

src/db/databaseManager.js

@@ -570,15 +570,55 @@ class DatabaseManager {
         await this.pool.query(singleSql, toRowValues(event));
         insertedCount += 1;
       } catch (error) {
+        const connCodes = ['57P03', '08006', '08001', '08003', '08004', '08007'];
+        if (connCodes.includes(error?.code) || (error?.message && /ECONNREFUSED|ETIMEDOUT|connection/i.test(error.message))) {
+          throw error;
+        }
         failedRecords.push({ error, record: event });
       }
     }
 
     if (insertedCount === 0 && failedRecords.length === events.length) {
+      // 只有在确定是纯网络断开时（所有单独重试都触发同一级别的连接错误），才向外抛出网络错误以重试批次。
+      const connCodes = ['57P03', '08006', '08001', '08003', '08004', '08007'];
+      const isConnError = connCodes.includes(lastError?.code) ||
+        (lastError?.message && /ECONNREFUSED|ETIMEDOUT|connection/i.test(lastError.message));
+      if (isConnError) {
         throw lastError;
       }
+    }
 
-    return { insertedCount, failedRecords, batchError: lastError };
+    return { insertedCount, failedRecords, batchError: (insertedCount === 0 && failedRecords.length === events.length) ? lastError : null };
+  }
+
+  async insertHeartbeatEventsErrors(errorsList) {
+    if (!errorsList || errorsList.length === 0) return;
+    try {
+      const placeholders = [];
+      const values = [];
+      let i = 1;
+      for (const err of errorsList) {
+        let hotel_id = Number(err.hotel_id);
+        if (!Number.isFinite(hotel_id)) hotel_id = null;
+
+        let room_id = Number(err.room_id);
+        if (!Number.isFinite(room_id)) room_id = null;
+
+        let original_data = null;
+        try { original_data = JSON.stringify(err.original_data); } catch { /* ignore */ }
+
+        const error_code = String(err.error_code || '');
+        const error_message = String(err.error_message || '');
+
+        placeholders.push(`($${i}, $${i + 1}, $${i + 2}, $${i + 3}, $${i + 4})`);
+        values.push(hotel_id, room_id, original_data, error_code, error_message);
+        i += 5;
+      }
+      const sql = `INSERT INTO heartbeat.heartbeat_events_errors (hotel_id, room_id, original_data, error_code, error_message) VALUES ${placeholders.join(', ')}`;
+      await this.pool.query(sql, values);
+    } catch (e) {
+      console.warn('Failed to insert into heartbeat_events_errors:', e);
+    }
   }
 
   async insertHeartbeatData(data) {

src/processor/heartbeatProcessor.js

@@ -202,11 +202,16 @@ class HeartbeatProcessor {
     this._batchInFlight = true;
     let hasMore = false;
     let batchData = null;
+    let batchEventCount = 0;
+    let batchMessageCount = 0;
+    let batchMessages = [];
 
     try {
-      const { batchEventCount, batchMessageCount } = this.computeNextBatchWindow();
+      const window = this.computeNextBatchWindow();
+      batchEventCount = window.batchEventCount;
+      batchMessageCount = window.batchMessageCount;
       batchData = this.batchQueue.slice(0, batchEventCount);
-      const batchMessages = this.batchMessageQueue.slice(0, batchMessageCount);
+      batchMessages = this.batchMessageQueue.slice(0, batchMessageCount);
 
       let insertedCount = 0;
       let failedRecords = [];
@@ -243,13 +248,12 @@ class HeartbeatProcessor {
       }
 
       if (failedRecords.length > 0) {
-        for (const item of failedRecords) {
+        const rejects = failedRecords.map(item => ({
-          this._emitRejectedRecord({
           errorId: 'db_write_failed',
           error: item?.error,
           rawData: item?.record,
-          });
+        }));
-        }
+        this._emitRejectedRecords(rejects);
         this.stats?.incDbWriteFailed?.(failedRecords.length);
       }
       this.stats?.incDbWritten?.(insertedCount);
@@ -353,7 +357,38 @@ class HeartbeatProcessor {
   }
 
   _emitRejectedRecord({ errorId, error, rawData }) {
+    this._emitRejectedRecords([{ errorId, error, rawData }]);
+  }
+
+  _emitRejectedRecords(records) {
+    if (!records || records.length === 0) return;
+
+    if (typeof this.databaseManager?.insertHeartbeatEventsErrors === 'function') {
+      const dbPayload = records.map(r => {
+        let hotel_id = null;
+        let room_id = null;
+        const effective = r.rawData?.effective || r.rawData?.record || r.rawData;
+        if (effective && typeof effective === 'object') {
+          hotel_id = effective.hotel_id ?? effective.hotelId;
+          room_id = effective.room_id ?? effective.roomId;
+          if (typeof room_id === 'string' && /^-?\d+$/.test(room_id)) room_id = Number(room_id);
+        }
+        return {
+          hotel_id,
+          room_id,
+          original_data: r.rawData,
+          error_code: String(r.errorId),
+          error_message: r.error ? String(r.error?.message ?? r.error) : 'Rejected record'
+        };
+      });
+      this.databaseManager.insertHeartbeatEventsErrors(dbPayload).catch(() => { });
+    }
+
     if (!this.redis?.isEnabled?.()) return;
+
+    // Batch log to redis (at most first 10 to avoid noise)
+    const logs = records.slice(0, 10);
+    for (const { errorId, error, rawData } of logs) {
       const ts = formatTimestamp(new Date());
       const errMsg = error ? String(error?.stack ?? error?.message ?? error) : undefined;
       const payload = this._safeStringify({
@@ -365,7 +400,7 @@ class HeartbeatProcessor {
       const maxChunkChars = 50_000;
       if (payload.length <= maxChunkChars) {
         this.redis.pushConsoleLog?.({ level: 'warn', message: `${base}${payload}`, metadata: { module: 'processor' } });
-      return;
+        continue;
       }
       const parts = Math.ceil(payload.length / maxChunkChars);
       for (let i = 0; i < parts; i += 1) {
@@ -377,6 +412,7 @@ class HeartbeatProcessor {
         });
       }
     }
+  }
 
   _extractRawKafkaValue(message) {
     try {
@@ -607,50 +643,68 @@ class HeartbeatProcessor {
       return s.length === 0 ? undefined : s;
     };
 
-    const toIntOrUndefined = (v) => {
+    const toInt32OrUndefined = (v) => {
       if (v === undefined || v === null) return v;
-      if (typeof v === 'number') {
+      let val = null;
-        if (!Number.isFinite(v)) return undefined;
+      if (typeof v === 'number' && Number.isFinite(v)) val = Math.trunc(v);
-        return Math.trunc(v);
+      else {
-      }
         const s = String(v).trim();
-      if (s.length === 0) return undefined;
+        if (s.length === 0 || !/^-?\d+$/.test(s)) return undefined;
-      if (!/^-?\d+$/.test(s)) return undefined;
+        val = Number(s);
-      const n = Number(s);
+      }
-      if (!Number.isFinite(n)) return undefined;
+      if (val === null || !Number.isFinite(val)) return undefined;
-      return Math.trunc(n);
+      return val | 0; // 强制补码映射到 32位有符号并去小数
+    };
+
+    const toInt16OrUndefined = (v) => {
+      const val = toInt32OrUndefined(v);
+      if (val === undefined) return undefined;
+      return (val << 16) >> 16; // 强制补码映射到 16位有符号
     };
 
     const toBigintParamOrUndefined = (v) => {
       if (v === undefined || v === null) return v;
+
+      let s;
       if (typeof v === 'number') {
         if (!Number.isFinite(v)) return undefined;
         const n = Math.trunc(v);
-        return Number.isSafeInteger(n) ? n : String(n);
+        // 如果是安全整数范围内，原样返回（作为 number 进库没问题）
-      }
+        if (Number.isSafeInteger(n)) return n;
-      const s = String(v).trim();
+        // 如果超出了上限（比如 uint64 最大值附近的数因精度变为 float），转成字符串走下面转换
+        s = BigInt(n).toString();
+      } else {
+        s = String(v).trim();
         if (s.length === 0) return undefined;
         if (!/^-?\d+$/.test(s)) return undefined;
+      }
+
+      try {
+        // 把可能超过有符号 64 位整数上限的无符号数（高位置 1 时），
+        // 转换为对应的有符号负数，以完美适应 PostgreSQL bigint 并保留完整的 bit 掩码。
+        return BigInt.asIntN(64, BigInt(s)).toString();
+      } catch (err) {
         return s;
+      }
     };
 
     normalized.ts_ms = toBigintParamOrUndefined(normalized.ts_ms);
-    normalized.hotel_id = toIntOrUndefined(normalized.hotel_id);
+    normalized.hotel_id = toInt16OrUndefined(normalized.hotel_id);
     normalized.room_id = toTrimmedStringOrUndefined(normalized.room_id);
     normalized.device_id = toTrimmedStringOrUndefined(normalized.device_id);
     normalized.ip = toTrimmedStringOrUndefined(normalized.ip);
-    normalized.power_state = toIntOrUndefined(normalized.power_state);
+    normalized.power_state = toInt16OrUndefined(normalized.power_state);
-    normalized.guest_type = toIntOrUndefined(normalized.guest_type);
+    normalized.guest_type = toInt16OrUndefined(normalized.guest_type);
-    normalized.cardless_state = toIntOrUndefined(normalized.cardless_state);
+    normalized.cardless_state = toInt16OrUndefined(normalized.cardless_state);
     normalized.service_mask = toBigintParamOrUndefined(normalized.service_mask);
-    normalized.pms_state = toIntOrUndefined(normalized.pms_state);
+    normalized.pms_state = toInt16OrUndefined(normalized.pms_state);
-    normalized.carbon_state = toIntOrUndefined(normalized.carbon_state);
+    normalized.carbon_state = toInt16OrUndefined(normalized.carbon_state);
-    normalized.device_count = toIntOrUndefined(normalized.device_count);
+    normalized.device_count = toInt16OrUndefined(normalized.device_count);
-    normalized.comm_seq = toIntOrUndefined(normalized.comm_seq);
+    normalized.comm_seq = toInt32OrUndefined(normalized.comm_seq); // int4
-    normalized.insert_card = toIntOrUndefined(normalized.insert_card);
+    normalized.insert_card = toInt16OrUndefined(normalized.insert_card);
-    const bg = toIntOrUndefined(normalized.bright_g);
+    const bg = toInt16OrUndefined(normalized.bright_g);
     normalized.bright_g = bg === -1 ? undefined : bg;
-    normalized.version = toIntOrUndefined(normalized.version);
+    normalized.version = toInt32OrUndefined(normalized.version);
 
     // 其余未知字段塞进 extra（避免丢信息），但不覆盖显式 extra
     if (!normalized.extra || typeof normalized.extra !== 'object') {
@@ -703,13 +757,20 @@ class HeartbeatProcessor {
       const num = Number(s);
       return Number.isFinite(num) ? num : null;
     };
-    const toIntOrNull = (v) => {
+    const toInt16OrNull = (v) => {
       if (v === undefined || v === null) return null;
-      if (typeof v === 'number') return Number.isFinite(v) ? Math.trunc(v) : null;
+      let val = null;
+      if (typeof v === 'number') {
+        if (Number.isFinite(v)) val = Math.trunc(v);
+      } else {
         const s = String(v).trim();
-      if (!/^-?\d+$/.test(s)) return null;
+        if (/^-?\d+$/.test(s)) {
           const num = Number(s);
-      return Number.isFinite(num) ? Math.trunc(num) : null;
+          if (Number.isFinite(num)) val = Math.trunc(num);
+        }
+      }
+      if (val === null) return null;
+      return (val << 16) >> 16;
     };
     const out = {};
     const elec = Array.isArray(n.electricity) ? n.electricity : null;
@@ -725,12 +786,12 @@ class HeartbeatProcessor {
     const ac = Array.isArray(n.air_conditioner) ? n.air_conditioner : null;
     if (ac && ac.length) {
       out.air_address = ac.map((x) => toStrOrNull(x?.address));
-      out.state = ac.map((x) => toIntOrNull(x?.state));
+      out.state = ac.map((x) => toInt16OrNull(x?.state));
-      out.model = ac.map((x) => toIntOrNull(x?.model));
+      out.model = ac.map((x) => toInt16OrNull(x?.model));
-      out.speed = ac.map((x) => toIntOrNull(x?.speed));
+      out.speed = ac.map((x) => toInt16OrNull(x?.speed));
-      out.set_temp = ac.map((x) => toIntOrNull(x?.set_temp));
+      out.set_temp = ac.map((x) => toInt16OrNull(x?.set_temp));
-      out.now_temp = ac.map((x) => toIntOrNull(x?.now_temp));
+      out.now_temp = ac.map((x) => toInt16OrNull(x?.now_temp));
-      out.solenoid_valve = ac.map((x) => toIntOrNull(x?.solenoid_valve));
+      out.solenoid_valve = ac.map((x) => toInt16OrNull(x?.solenoid_valve));
     }
     return out;
   }

test.js

@@ -0,0 +1,2 @@
+const n = JSON.parse('{\
+mask\: 9223372054034653000 }').mask; console.log(n, typeof n, String(Math.trunc(n)), BigInt.asIntN(64, BigInt(Math.trunc(n))));