feat: 将批量升级改为逐个主机顺序升级并添加超时控制

- 修改升级流程,从同时触发所有主机改为逐个主机顺序触发和轮询
- 添加 session_id 字段到 upgrade_log 表以区分不同主机的升级会话
- 引入 upgradePollTimeoutSeconds 配置项控制单个主机轮询超时时间
- 添加数据库迁移脚本以更新表结构和主键约束
- 实现运行组状态跟踪,防止同一组并发执行
- 改进错误处理和日志记录,为每个主机独立记录升级结果
This commit is contained in:
2026-01-23 18:22:05 +08:00
parent 834ac02da5
commit fc134b0108
5 changed files with 167 additions and 94 deletions

View File

@@ -5,6 +5,7 @@
CREATE TABLE IF NOT EXISTS upgrade_log ( CREATE TABLE IF NOT EXISTS upgrade_log (
uuid UUID NOT NULL, uuid UUID NOT NULL,
session_id UUID,
start_time TIMESTAMP NOT NULL, start_time TIMESTAMP NOT NULL,
roomtype_id INTEGER NOT NULL, roomtype_id INTEGER NOT NULL,
host_str TEXT NOT NULL, host_str TEXT NOT NULL,

View File

@@ -0,0 +1,34 @@
const db = require('../src/db');
const updateSchema = async () => {
try {
console.log('Updating database schema...');
await db.query(`
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'upgrade_log' AND column_name = 'session_id') THEN
ALTER TABLE upgrade_log ADD COLUMN session_id UUID;
END IF;
END $$;
`);
console.log('Added session_id column.');
try {
await db.query(`ALTER TABLE upgrade_log DROP CONSTRAINT IF EXISTS upgrade_log_pkey`);
console.log('Dropped old primary key constraint.');
} catch (e) {
console.log('Primary key might not exist or different name:', e.message);
}
await db.query(`ALTER TABLE upgrade_log ADD PRIMARY KEY (uuid)`);
console.log('Set uuid as Primary Key.');
console.log('Schema update completed successfully.');
process.exit(0);
} catch (error) {
console.error('Schema update failed:', error);
process.exit(1);
}
};
updateSchema();

View File

@@ -41,5 +41,6 @@ module.exports = {
runOnStartup: String(process.env.RUN_ON_STARTUP || 'false').toLowerCase() === 'true', runOnStartup: String(process.env.RUN_ON_STARTUP || 'false').toLowerCase() === 'true',
upgradeWaitSeconds: Number(process.env.UPGRADE_WAIT_SECONDS || 45), upgradeWaitSeconds: Number(process.env.UPGRADE_WAIT_SECONDS || 45),
upgradePollIntervalSeconds: Number(process.env.UPGRADE_POLL_INTERVAL_SECONDS || 45), upgradePollIntervalSeconds: Number(process.env.UPGRADE_POLL_INTERVAL_SECONDS || 45),
upgradePollTimeoutSeconds: Number(process.env.UPGRADE_POLL_TIMEOUT_SECONDS || 300),
upgradeConfig: parseUpgradeConfig(process.env.UPGRADE_CONFIG) upgradeConfig: parseUpgradeConfig(process.env.UPGRADE_CONFIG)
}; };

View File

@@ -1,13 +1,16 @@
const { v4: uuidv4 } = require('uuid');
const db = require('./db'); const db = require('./db');
const logHostResult = async (data) => { const logHostResult = async (data) => {
const entryId = uuidv4();
const query = ` const query = `
INSERT INTO upgrade_log ( INSERT INTO upgrade_log (
uuid, start_time, roomtype_id, host_str, filename, status, uuid, session_id, start_time, roomtype_id, host_str, filename, status,
end_time, file_type, config_version, firmware_version end_time, file_type, config_version, firmware_version
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
`; `;
const values = [ const values = [
entryId,
data.uuid, data.uuid,
data.start_time, data.start_time,
data.roomtype_id, data.roomtype_id,

View File

@@ -5,113 +5,147 @@ const config = require('./config');
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
const runningGroups = new Map();
const processGroup = async (group, groupIdx) => { const processGroup = async (group, groupIdx) => {
const stateKey = `group_${groupIdx}`; const stateKey = `group_${groupIdx}`;
let state = await loggerService.getUpgradeState(stateKey); if (runningGroups.get(stateKey)) {
if (!state) { console.log(`[${stateKey}] Skipping scheduled run because group is still running.`);
state = { current_roomtype_index: 0, execution_count: 0 };
}
const roomtype = group.roomtypes[state.current_roomtype_index];
const roomtype_id = roomtype.roomtype_id;
const fileName = roomtype.fileName;
const upgradeCountLimit = roomtype.upgrade_count || 2;
let nextState = { ...state };
nextState.execution_count += 1;
if (nextState.execution_count >= upgradeCountLimit) {
nextState.execution_count = 0;
nextState.current_roomtype_index = (state.current_roomtype_index + 1) % group.roomtypes.length;
}
const sessionUuid = uuidv4();
const startTime = new Date();
const hostList = group.hosts;
console.log(`[${stateKey}] Starting upgrade. Roomtype: ${roomtype_id}. File: ${fileName}. Hosts: ${hostList}`);
try {
const upgradeRes = await apiClient.triggerUpgrade(roomtype_id, hostList, fileName);
if (!upgradeRes.IsSuccess) {
console.error(`[${stateKey}] Upgrade trigger failed: ${upgradeRes.Message}`);
return;
}
} catch (e) {
console.error(`[${stateKey}] Upgrade trigger error:`, e.message);
return; return;
} }
const waitSeconds = config.upgradeWaitSeconds || 45; runningGroups.set(stateKey, true);
const pollIntervalSeconds = config.upgradePollIntervalSeconds || 45;
console.log(`[${stateKey}] Waiting ${waitSeconds}s...`);
await sleep(waitSeconds * 1000);
const timeout = 5 * 60 * 1000; try {
const interval = pollIntervalSeconds * 1000; let state = await loggerService.getUpgradeState(stateKey);
const pollStartTime = Date.now(); if (!state) {
state = { current_roomtype_index: 0, execution_count: 0 };
}
const allHosts = new Set(hostList.map(String)); const roomtype = group.roomtypes[state.current_roomtype_index];
const lastStatusMap = new Map(); const roomtype_id = roomtype.roomtype_id;
const fileName = roomtype.fileName;
const upgradeCountLimit = roomtype.upgrade_count || 2;
while (Date.now() - pollStartTime < timeout) { let nextState = { ...state };
try { nextState.execution_count += 1;
const queryRes = await apiClient.queryStatus(hostList); if (nextState.execution_count >= upgradeCountLimit) {
nextState.execution_count = 0;
nextState.current_roomtype_index = (state.current_roomtype_index + 1) % group.roomtypes.length;
}
if (queryRes && Array.isArray(queryRes.Response)) { const hostQueue = group.hosts.map(String);
for (const hostStatus of queryRes.Response) { console.log(`[${stateKey}] Starting queue. Roomtype: ${roomtype_id}. File: ${fileName}. Hosts: ${hostQueue.join(', ')}`);
const hid = String(hostStatus.HostID);
if (allHosts.has(hid)) { const waitSeconds = config.upgradeWaitSeconds || 45;
lastStatusMap.set(hid, { const pollIntervalSeconds = config.upgradePollIntervalSeconds || 45;
status: hostStatus.Upgrade_status, const pollTimeoutSeconds = config.upgradePollTimeoutSeconds || 300;
file_type: hostStatus.UpgradeFileType,
config_version: hostStatus.ConfiguraVersion, for (const hostId of hostQueue) {
firmware_version: hostStatus.Version const sessionUuid = uuidv4();
}); const startTime = new Date();
}
console.log(`[${stateKey}] Triggering upgrade for host ${hostId}. Roomtype: ${roomtype_id}. File: ${fileName}.`);
try {
const upgradeRes = await apiClient.triggerUpgrade(roomtype_id, [hostId], fileName);
if (!upgradeRes.IsSuccess) {
console.error(`[${stateKey}] Upgrade trigger failed for host ${hostId}: ${upgradeRes.Message}`);
await loggerService.logHostResult({
uuid: sessionUuid,
start_time: startTime.toISOString(),
roomtype_id: roomtype_id,
host_str: hostId,
filename: fileName,
status: '触发失败',
end_time: new Date().toISOString(),
file_type: '',
config_version: '',
firmware_version: ''
});
continue;
} }
} catch (e) {
console.error(`[${stateKey}] Upgrade trigger error for host ${hostId}:`, e.message);
await loggerService.logHostResult({
uuid: sessionUuid,
start_time: startTime.toISOString(),
roomtype_id: roomtype_id,
host_str: hostId,
filename: fileName,
status: '触发失败',
end_time: new Date().toISOString(),
file_type: '',
config_version: '',
firmware_version: ''
});
continue;
} }
} catch (e) {
console.error(`[${stateKey}] Poll error:`, e.message);
}
if (lastStatusMap.size === allHosts.size) { console.log(`[${stateKey}] Waiting ${waitSeconds}s for host ${hostId}...`);
const allDone = Array.from(lastStatusMap.values()).every((item) => await sleep(waitSeconds * 1000);
item.status === '升级完成' || item.status === '超时失败'
); const pollStartTime = Date.now();
if (allDone) { let completed = false;
console.log(`[${stateKey}] All hosts completed or timeout failed.`);
break; while (Date.now() - pollStartTime < pollTimeoutSeconds * 1000) {
try {
const queryRes = await apiClient.queryStatus([hostId]);
if (queryRes && Array.isArray(queryRes.Response)) {
const hostStatus = queryRes.Response.find((item) => String(item.HostID) === String(hostId));
if (hostStatus) {
const rawStatus = hostStatus.Upgrade_status || '';
const status = rawStatus.trim();
if (['升级完成', '超时失败', '升级失败'].includes(status)) {
await loggerService.logHostResult({
uuid: sessionUuid,
start_time: startTime.toISOString(),
roomtype_id: roomtype_id,
host_str: hostId,
filename: fileName,
status: status,
end_time: new Date().toISOString(),
file_type: hostStatus.UpgradeFileType,
config_version: hostStatus.ConfiguraVersion,
firmware_version: hostStatus.Version
});
console.log(`[${stateKey}] Host ${hostId} logged to DB with status: '${status}'.`);
completed = true;
break;
}
}
} else {
console.warn(`[${stateKey}] Invalid query response for host ${hostId}:`, queryRes);
}
} catch (e) {
console.error(`[${stateKey}] Poll error for host ${hostId}:`, e.message);
}
await sleep(pollIntervalSeconds * 1000);
}
if (!completed) {
await loggerService.logHostResult({
uuid: sessionUuid,
start_time: startTime.toISOString(),
roomtype_id: roomtype_id,
host_str: hostId,
filename: fileName,
status: '超时失败',
end_time: new Date().toISOString(),
file_type: 'Unknown',
config_version: '',
firmware_version: ''
});
console.log(`[${stateKey}] Host ${hostId} timeout failed.`);
} }
} }
await sleep(interval); await loggerService.updateUpgradeState(stateKey, nextState.current_roomtype_index, nextState.execution_count);
} finally {
runningGroups.set(stateKey, false);
} }
for (const hid of allHosts) {
const data = lastStatusMap.get(hid) || {
status: '超时失败',
file_type: 'Unknown',
config_version: '',
firmware_version: ''
};
await loggerService.logHostResult({
uuid: sessionUuid,
start_time: startTime.toISOString(),
roomtype_id: roomtype_id,
host_str: hid,
filename: fileName,
status: data.status,
end_time: new Date().toISOString(),
file_type: data.file_type,
config_version: data.config_version,
firmware_version: data.firmware_version
});
console.log(`[${stateKey}] Host ${hid} final status: ${data.status}.`);
}
await loggerService.updateUpgradeState(stateKey, nextState.current_roomtype_index, nextState.execution_count);
}; };
const run = async () => { const run = async () => {