From 4551ae573311ff1b2b571bda9f94b0a150fb4baf Mon Sep 17 00:00:00 2001 From: XuJiacheng Date: Tue, 20 Jan 2026 08:23:07 +0800 Subject: [PATCH] =?UTF-8?q?feat(=E6=97=A5=E5=BF=97):=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BF=AE=E5=89=AA=E9=A1=B9=E7=9B=AE=E6=8E=A7?= =?UTF-8?q?=E5=88=B6=E5=8F=B0=E6=97=A5=E5=BF=97=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 实现每小时自动修剪项目控制台日志列表至1000条的功能 新增日志修剪工具函数和相应测试用例 在服务器关闭时清理日志修剪定时器 --- src/backend/routes/logs.js | 47 +++++++++++++++-- .../routes/projects.integration.test.js | 51 +++++++++++++++++++ src/backend/server.js | 19 ++++++- 3 files changed, 113 insertions(+), 4 deletions(-) diff --git a/src/backend/routes/logs.js b/src/backend/routes/logs.js index 7436c61..2bd2d91 100644 --- a/src/backend/routes/logs.js +++ b/src/backend/routes/logs.js @@ -9,6 +9,45 @@ import { getProjectsList } from '../services/migrateHeartbeatData.js'; const LOGS_MAX_LEN = 1000; const LOG_TTL_MS = 24 * 60 * 60 * 1000; +async function listConsoleLogKeys(redis) { + const pattern = '*_项目控制台'; + if (typeof redis.scanIterator === 'function') { + const keys = []; + for await (const key of redis.scanIterator({ MATCH: pattern, COUNT: 500 })) { + keys.push(key); + } + return keys; + } + + if (typeof redis.keys === 'function') { + const list = await redis.keys(pattern); + return Array.isArray(list) ? list : []; + } + + return []; +} + +export async function trimProjectConsoleLogsByLength(redis, options = {}) { + const maxLen = + typeof options.maxLen === 'number' && Number.isFinite(options.maxLen) + ? Math.max(1, Math.trunc(options.maxLen)) + : LOGS_MAX_LEN; + + const keys = await listConsoleLogKeys(redis); + let trimmedKeys = 0; + + for (const key of keys) { + // eslint-disable-next-line no-await-in-loop + const len = await redis.lLen(key); + if (!Number.isFinite(len) || len <= maxLen) continue; + // eslint-disable-next-line no-await-in-loop + await redis.lTrim(key, -maxLen, -1); + trimmedKeys += 1; + } + + return { keysScanned: keys.length, keysTrimmed: trimmedKeys, maxLen }; +} + function parsePositiveInt(value, defaultValue) { const num = Number.parseInt(String(value), 10); if (!Number.isFinite(num) || num <= 0) return defaultValue; @@ -144,9 +183,11 @@ async function pruneAndReadLogsAtomically(redis, key, limit) { await redis.watch(key); const rawItems = await redis.lRange(key, 0, -1); - const kept = rawItems - .filter((raw) => shouldKeepRawLog(raw, cutoffMs)) - .slice(-maxLen); + const ttlKept = rawItems.filter((raw) => shouldKeepRawLog(raw, cutoffMs)); + const kept = + ttlKept.length > 0 + ? ttlKept.slice(-maxLen) + : rawItems.slice(-Math.min(maxLen, rawItems.length)); const needsRewrite = rawItems.length !== kept.length || rawItems.length > maxLen; if (!needsRewrite) return kept.slice(-effectiveLimit); diff --git a/src/backend/routes/projects.integration.test.js b/src/backend/routes/projects.integration.test.js index 45856fd..d90c28d 100644 --- a/src/backend/routes/projects.integration.test.js +++ b/src/backend/routes/projects.integration.test.js @@ -3,6 +3,7 @@ import request from 'supertest'; import { createApp } from '../app.js'; import { createFakeRedis } from '../test/fakeRedis.js'; +import { trimProjectConsoleLogsByLength } from './logs.js'; describe('projects API', () => { it('GET /api/projects returns projects from unified list', async () => { @@ -95,6 +96,24 @@ describe('projects API', () => { }); describe('logs API', () => { + it('trimProjectConsoleLogsByLength trims *_项目控制台 lists to 1000 items', async () => { + const projectName = 'Demo'; + const key = `${projectName}_项目控制台`; + const items = Array.from({ length: 1505 }, (_, i) => `l${i + 1}`); + const redis = createFakeRedis({ + [key]: items, + other: [ 'x', 'y' ], + }); + + const result = await trimProjectConsoleLogsByLength(redis, { maxLen: 1000 }); + expect(result).toMatchObject({ keysScanned: 1, keysTrimmed: 1, maxLen: 1000 }); + + const listItems = await redis.lRange(key, 0, -1); + expect(listItems.length).toBe(1000); + expect(listItems[0]).toBe('l506'); + expect(listItems[listItems.length - 1]).toBe('l1505'); + }); + it('GET /api/logs uses LOG_TTL_MS=24h without wiping recent logs', async () => { const prev = process.env.LOG_TTL_MS; process.env.LOG_TTL_MS = '24h'; @@ -202,6 +221,38 @@ describe('logs API', () => { expect(JSON.parse(listItems[0]).id).toBe('new'); }); + it('GET /api/logs keeps latest 1000 logs when all timestamps are expired', async () => { + const now = Date.now(); + const projectName = 'Demo'; + const key = `${projectName}_项目控制台`; + const items = Array.from({ length: 1505 }, (_, i) => + JSON.stringify({ + id: `log-${i + 1}`, + timestamp: new Date(now - 26 * 60 * 60 * 1000).toISOString(), + level: 'info', + message: `m${i + 1}`, + }), + ); + const redis = createFakeRedis({ + [key]: items, + }); + + const app = createApp({ redis }); + const resp = await request(app) + .get('/api/logs') + .query({ projectName, limit: 1000 }); + + expect(resp.status).toBe(200); + expect(resp.body.logs.length).toBe(1000); + expect(resp.body.logs[0].id).toBe('log-506'); + expect(resp.body.logs[resp.body.logs.length - 1].id).toBe('log-1505'); + + const listItems = await redis.lRange(key, 0, -1); + expect(listItems.length).toBe(1000); + expect(JSON.parse(listItems[0]).id).toBe('log-506'); + expect(JSON.parse(listItems[listItems.length - 1]).id).toBe('log-1505'); + }); + it('GET /api/logs keeps unix-second timestamps and prunes correctly', async () => { const now = Date.now(); const nowSec = Math.floor(now / 1000); diff --git a/src/backend/server.js b/src/backend/server.js index e27d199..bd0ef40 100644 --- a/src/backend/server.js +++ b/src/backend/server.js @@ -1,6 +1,6 @@ import express from 'express'; import cors from 'cors'; -import logRoutes from './routes/logs.js'; +import logRoutes, { trimProjectConsoleLogsByLength } from './routes/logs.js'; import commandRoutes from './routes/commands.js'; import projectRoutes from './routes/projects.js'; import { getRedisClient } from './services/redisClient.js'; @@ -42,6 +42,20 @@ const server = app.listen(PORT, async () => { void err; } }, intervalMs); + + let trimRunning = false; + const logsTrimIntervalMs = 60 * 60 * 1000; + app.locals.logsTrimInterval = setInterval(async () => { + if (trimRunning) return; + trimRunning = true; + try { + await trimProjectConsoleLogsByLength(redis, { maxLen: 1000 }); + } catch (err) { + void err; + } finally { + trimRunning = false; + } + }, logsTrimIntervalMs); } catch (err) { console.error('[redis] failed to connect on startup', err); } @@ -52,6 +66,9 @@ process.on('SIGINT', async () => { if (app.locals.heartbeatPruneInterval) { clearInterval(app.locals.heartbeatPruneInterval); } + if (app.locals.logsTrimInterval) { + clearInterval(app.locals.logsTrimInterval); + } if (app.locals.redis) { await app.locals.redis.quit(); }