feat(日志): 添加定时修剪项目控制台日志功能

实现每小时自动修剪项目控制台日志列表至1000条的功能
新增日志修剪工具函数和相应测试用例
在服务器关闭时清理日志修剪定时器
This commit is contained in:
2026-01-20 08:23:07 +08:00
parent 7ac3949dfa
commit 4551ae5733
3 changed files with 113 additions and 4 deletions

View File

@@ -9,6 +9,45 @@ import { getProjectsList } from '../services/migrateHeartbeatData.js';
const LOGS_MAX_LEN = 1000; const LOGS_MAX_LEN = 1000;
const LOG_TTL_MS = 24 * 60 * 60 * 1000; const LOG_TTL_MS = 24 * 60 * 60 * 1000;
async function listConsoleLogKeys(redis) {
const pattern = '*_项目控制台';
if (typeof redis.scanIterator === 'function') {
const keys = [];
for await (const key of redis.scanIterator({ MATCH: pattern, COUNT: 500 })) {
keys.push(key);
}
return keys;
}
if (typeof redis.keys === 'function') {
const list = await redis.keys(pattern);
return Array.isArray(list) ? list : [];
}
return [];
}
export async function trimProjectConsoleLogsByLength(redis, options = {}) {
const maxLen =
typeof options.maxLen === 'number' && Number.isFinite(options.maxLen)
? Math.max(1, Math.trunc(options.maxLen))
: LOGS_MAX_LEN;
const keys = await listConsoleLogKeys(redis);
let trimmedKeys = 0;
for (const key of keys) {
// eslint-disable-next-line no-await-in-loop
const len = await redis.lLen(key);
if (!Number.isFinite(len) || len <= maxLen) continue;
// eslint-disable-next-line no-await-in-loop
await redis.lTrim(key, -maxLen, -1);
trimmedKeys += 1;
}
return { keysScanned: keys.length, keysTrimmed: trimmedKeys, maxLen };
}
function parsePositiveInt(value, defaultValue) { function parsePositiveInt(value, defaultValue) {
const num = Number.parseInt(String(value), 10); const num = Number.parseInt(String(value), 10);
if (!Number.isFinite(num) || num <= 0) return defaultValue; if (!Number.isFinite(num) || num <= 0) return defaultValue;
@@ -144,9 +183,11 @@ async function pruneAndReadLogsAtomically(redis, key, limit) {
await redis.watch(key); await redis.watch(key);
const rawItems = await redis.lRange(key, 0, -1); const rawItems = await redis.lRange(key, 0, -1);
const kept = rawItems const ttlKept = rawItems.filter((raw) => shouldKeepRawLog(raw, cutoffMs));
.filter((raw) => shouldKeepRawLog(raw, cutoffMs)) const kept =
.slice(-maxLen); ttlKept.length > 0
? ttlKept.slice(-maxLen)
: rawItems.slice(-Math.min(maxLen, rawItems.length));
const needsRewrite = rawItems.length !== kept.length || rawItems.length > maxLen; const needsRewrite = rawItems.length !== kept.length || rawItems.length > maxLen;
if (!needsRewrite) return kept.slice(-effectiveLimit); if (!needsRewrite) return kept.slice(-effectiveLimit);

View File

@@ -3,6 +3,7 @@ import request from 'supertest';
import { createApp } from '../app.js'; import { createApp } from '../app.js';
import { createFakeRedis } from '../test/fakeRedis.js'; import { createFakeRedis } from '../test/fakeRedis.js';
import { trimProjectConsoleLogsByLength } from './logs.js';
describe('projects API', () => { describe('projects API', () => {
it('GET /api/projects returns projects from unified list', async () => { it('GET /api/projects returns projects from unified list', async () => {
@@ -95,6 +96,24 @@ describe('projects API', () => {
}); });
describe('logs API', () => { describe('logs API', () => {
it('trimProjectConsoleLogsByLength trims *_项目控制台 lists to 1000 items', async () => {
const projectName = 'Demo';
const key = `${projectName}_项目控制台`;
const items = Array.from({ length: 1505 }, (_, i) => `l${i + 1}`);
const redis = createFakeRedis({
[key]: items,
other: [ 'x', 'y' ],
});
const result = await trimProjectConsoleLogsByLength(redis, { maxLen: 1000 });
expect(result).toMatchObject({ keysScanned: 1, keysTrimmed: 1, maxLen: 1000 });
const listItems = await redis.lRange(key, 0, -1);
expect(listItems.length).toBe(1000);
expect(listItems[0]).toBe('l506');
expect(listItems[listItems.length - 1]).toBe('l1505');
});
it('GET /api/logs uses LOG_TTL_MS=24h without wiping recent logs', async () => { it('GET /api/logs uses LOG_TTL_MS=24h without wiping recent logs', async () => {
const prev = process.env.LOG_TTL_MS; const prev = process.env.LOG_TTL_MS;
process.env.LOG_TTL_MS = '24h'; process.env.LOG_TTL_MS = '24h';
@@ -202,6 +221,38 @@ describe('logs API', () => {
expect(JSON.parse(listItems[0]).id).toBe('new'); expect(JSON.parse(listItems[0]).id).toBe('new');
}); });
it('GET /api/logs keeps latest 1000 logs when all timestamps are expired', async () => {
const now = Date.now();
const projectName = 'Demo';
const key = `${projectName}_项目控制台`;
const items = Array.from({ length: 1505 }, (_, i) =>
JSON.stringify({
id: `log-${i + 1}`,
timestamp: new Date(now - 26 * 60 * 60 * 1000).toISOString(),
level: 'info',
message: `m${i + 1}`,
}),
);
const redis = createFakeRedis({
[key]: items,
});
const app = createApp({ redis });
const resp = await request(app)
.get('/api/logs')
.query({ projectName, limit: 1000 });
expect(resp.status).toBe(200);
expect(resp.body.logs.length).toBe(1000);
expect(resp.body.logs[0].id).toBe('log-506');
expect(resp.body.logs[resp.body.logs.length - 1].id).toBe('log-1505');
const listItems = await redis.lRange(key, 0, -1);
expect(listItems.length).toBe(1000);
expect(JSON.parse(listItems[0]).id).toBe('log-506');
expect(JSON.parse(listItems[listItems.length - 1]).id).toBe('log-1505');
});
it('GET /api/logs keeps unix-second timestamps and prunes correctly', async () => { it('GET /api/logs keeps unix-second timestamps and prunes correctly', async () => {
const now = Date.now(); const now = Date.now();
const nowSec = Math.floor(now / 1000); const nowSec = Math.floor(now / 1000);

View File

@@ -1,6 +1,6 @@
import express from 'express'; import express from 'express';
import cors from 'cors'; import cors from 'cors';
import logRoutes from './routes/logs.js'; import logRoutes, { trimProjectConsoleLogsByLength } from './routes/logs.js';
import commandRoutes from './routes/commands.js'; import commandRoutes from './routes/commands.js';
import projectRoutes from './routes/projects.js'; import projectRoutes from './routes/projects.js';
import { getRedisClient } from './services/redisClient.js'; import { getRedisClient } from './services/redisClient.js';
@@ -42,6 +42,20 @@ const server = app.listen(PORT, async () => {
void err; void err;
} }
}, intervalMs); }, intervalMs);
let trimRunning = false;
const logsTrimIntervalMs = 60 * 60 * 1000;
app.locals.logsTrimInterval = setInterval(async () => {
if (trimRunning) return;
trimRunning = true;
try {
await trimProjectConsoleLogsByLength(redis, { maxLen: 1000 });
} catch (err) {
void err;
} finally {
trimRunning = false;
}
}, logsTrimIntervalMs);
} catch (err) { } catch (err) {
console.error('[redis] failed to connect on startup', err); console.error('[redis] failed to connect on startup', err);
} }
@@ -52,6 +66,9 @@ process.on('SIGINT', async () => {
if (app.locals.heartbeatPruneInterval) { if (app.locals.heartbeatPruneInterval) {
clearInterval(app.locals.heartbeatPruneInterval); clearInterval(app.locals.heartbeatPruneInterval);
} }
if (app.locals.logsTrimInterval) {
clearInterval(app.locals.logsTrimInterval);
}
if (app.locals.redis) { if (app.locals.redis) {
await app.locals.redis.quit(); await app.locals.redis.quit();
} }