feat(日志): 添加定时修剪项目控制台日志功能
实现每小时自动修剪项目控制台日志列表至1000条的功能 新增日志修剪工具函数和相应测试用例 在服务器关闭时清理日志修剪定时器
This commit is contained in:
@@ -9,6 +9,45 @@ import { getProjectsList } from '../services/migrateHeartbeatData.js';
|
|||||||
const LOGS_MAX_LEN = 1000;
|
const LOGS_MAX_LEN = 1000;
|
||||||
const LOG_TTL_MS = 24 * 60 * 60 * 1000;
|
const LOG_TTL_MS = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
async function listConsoleLogKeys(redis) {
|
||||||
|
const pattern = '*_项目控制台';
|
||||||
|
if (typeof redis.scanIterator === 'function') {
|
||||||
|
const keys = [];
|
||||||
|
for await (const key of redis.scanIterator({ MATCH: pattern, COUNT: 500 })) {
|
||||||
|
keys.push(key);
|
||||||
|
}
|
||||||
|
return keys;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof redis.keys === 'function') {
|
||||||
|
const list = await redis.keys(pattern);
|
||||||
|
return Array.isArray(list) ? list : [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function trimProjectConsoleLogsByLength(redis, options = {}) {
|
||||||
|
const maxLen =
|
||||||
|
typeof options.maxLen === 'number' && Number.isFinite(options.maxLen)
|
||||||
|
? Math.max(1, Math.trunc(options.maxLen))
|
||||||
|
: LOGS_MAX_LEN;
|
||||||
|
|
||||||
|
const keys = await listConsoleLogKeys(redis);
|
||||||
|
let trimmedKeys = 0;
|
||||||
|
|
||||||
|
for (const key of keys) {
|
||||||
|
// eslint-disable-next-line no-await-in-loop
|
||||||
|
const len = await redis.lLen(key);
|
||||||
|
if (!Number.isFinite(len) || len <= maxLen) continue;
|
||||||
|
// eslint-disable-next-line no-await-in-loop
|
||||||
|
await redis.lTrim(key, -maxLen, -1);
|
||||||
|
trimmedKeys += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return { keysScanned: keys.length, keysTrimmed: trimmedKeys, maxLen };
|
||||||
|
}
|
||||||
|
|
||||||
function parsePositiveInt(value, defaultValue) {
|
function parsePositiveInt(value, defaultValue) {
|
||||||
const num = Number.parseInt(String(value), 10);
|
const num = Number.parseInt(String(value), 10);
|
||||||
if (!Number.isFinite(num) || num <= 0) return defaultValue;
|
if (!Number.isFinite(num) || num <= 0) return defaultValue;
|
||||||
@@ -144,9 +183,11 @@ async function pruneAndReadLogsAtomically(redis, key, limit) {
|
|||||||
await redis.watch(key);
|
await redis.watch(key);
|
||||||
const rawItems = await redis.lRange(key, 0, -1);
|
const rawItems = await redis.lRange(key, 0, -1);
|
||||||
|
|
||||||
const kept = rawItems
|
const ttlKept = rawItems.filter((raw) => shouldKeepRawLog(raw, cutoffMs));
|
||||||
.filter((raw) => shouldKeepRawLog(raw, cutoffMs))
|
const kept =
|
||||||
.slice(-maxLen);
|
ttlKept.length > 0
|
||||||
|
? ttlKept.slice(-maxLen)
|
||||||
|
: rawItems.slice(-Math.min(maxLen, rawItems.length));
|
||||||
|
|
||||||
const needsRewrite = rawItems.length !== kept.length || rawItems.length > maxLen;
|
const needsRewrite = rawItems.length !== kept.length || rawItems.length > maxLen;
|
||||||
if (!needsRewrite) return kept.slice(-effectiveLimit);
|
if (!needsRewrite) return kept.slice(-effectiveLimit);
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import request from 'supertest';
|
|||||||
|
|
||||||
import { createApp } from '../app.js';
|
import { createApp } from '../app.js';
|
||||||
import { createFakeRedis } from '../test/fakeRedis.js';
|
import { createFakeRedis } from '../test/fakeRedis.js';
|
||||||
|
import { trimProjectConsoleLogsByLength } from './logs.js';
|
||||||
|
|
||||||
describe('projects API', () => {
|
describe('projects API', () => {
|
||||||
it('GET /api/projects returns projects from unified list', async () => {
|
it('GET /api/projects returns projects from unified list', async () => {
|
||||||
@@ -95,6 +96,24 @@ describe('projects API', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe('logs API', () => {
|
describe('logs API', () => {
|
||||||
|
it('trimProjectConsoleLogsByLength trims *_项目控制台 lists to 1000 items', async () => {
|
||||||
|
const projectName = 'Demo';
|
||||||
|
const key = `${projectName}_项目控制台`;
|
||||||
|
const items = Array.from({ length: 1505 }, (_, i) => `l${i + 1}`);
|
||||||
|
const redis = createFakeRedis({
|
||||||
|
[key]: items,
|
||||||
|
other: [ 'x', 'y' ],
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await trimProjectConsoleLogsByLength(redis, { maxLen: 1000 });
|
||||||
|
expect(result).toMatchObject({ keysScanned: 1, keysTrimmed: 1, maxLen: 1000 });
|
||||||
|
|
||||||
|
const listItems = await redis.lRange(key, 0, -1);
|
||||||
|
expect(listItems.length).toBe(1000);
|
||||||
|
expect(listItems[0]).toBe('l506');
|
||||||
|
expect(listItems[listItems.length - 1]).toBe('l1505');
|
||||||
|
});
|
||||||
|
|
||||||
it('GET /api/logs uses LOG_TTL_MS=24h without wiping recent logs', async () => {
|
it('GET /api/logs uses LOG_TTL_MS=24h without wiping recent logs', async () => {
|
||||||
const prev = process.env.LOG_TTL_MS;
|
const prev = process.env.LOG_TTL_MS;
|
||||||
process.env.LOG_TTL_MS = '24h';
|
process.env.LOG_TTL_MS = '24h';
|
||||||
@@ -202,6 +221,38 @@ describe('logs API', () => {
|
|||||||
expect(JSON.parse(listItems[0]).id).toBe('new');
|
expect(JSON.parse(listItems[0]).id).toBe('new');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('GET /api/logs keeps latest 1000 logs when all timestamps are expired', async () => {
|
||||||
|
const now = Date.now();
|
||||||
|
const projectName = 'Demo';
|
||||||
|
const key = `${projectName}_项目控制台`;
|
||||||
|
const items = Array.from({ length: 1505 }, (_, i) =>
|
||||||
|
JSON.stringify({
|
||||||
|
id: `log-${i + 1}`,
|
||||||
|
timestamp: new Date(now - 26 * 60 * 60 * 1000).toISOString(),
|
||||||
|
level: 'info',
|
||||||
|
message: `m${i + 1}`,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const redis = createFakeRedis({
|
||||||
|
[key]: items,
|
||||||
|
});
|
||||||
|
|
||||||
|
const app = createApp({ redis });
|
||||||
|
const resp = await request(app)
|
||||||
|
.get('/api/logs')
|
||||||
|
.query({ projectName, limit: 1000 });
|
||||||
|
|
||||||
|
expect(resp.status).toBe(200);
|
||||||
|
expect(resp.body.logs.length).toBe(1000);
|
||||||
|
expect(resp.body.logs[0].id).toBe('log-506');
|
||||||
|
expect(resp.body.logs[resp.body.logs.length - 1].id).toBe('log-1505');
|
||||||
|
|
||||||
|
const listItems = await redis.lRange(key, 0, -1);
|
||||||
|
expect(listItems.length).toBe(1000);
|
||||||
|
expect(JSON.parse(listItems[0]).id).toBe('log-506');
|
||||||
|
expect(JSON.parse(listItems[listItems.length - 1]).id).toBe('log-1505');
|
||||||
|
});
|
||||||
|
|
||||||
it('GET /api/logs keeps unix-second timestamps and prunes correctly', async () => {
|
it('GET /api/logs keeps unix-second timestamps and prunes correctly', async () => {
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const nowSec = Math.floor(now / 1000);
|
const nowSec = Math.floor(now / 1000);
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import express from 'express';
|
import express from 'express';
|
||||||
import cors from 'cors';
|
import cors from 'cors';
|
||||||
import logRoutes from './routes/logs.js';
|
import logRoutes, { trimProjectConsoleLogsByLength } from './routes/logs.js';
|
||||||
import commandRoutes from './routes/commands.js';
|
import commandRoutes from './routes/commands.js';
|
||||||
import projectRoutes from './routes/projects.js';
|
import projectRoutes from './routes/projects.js';
|
||||||
import { getRedisClient } from './services/redisClient.js';
|
import { getRedisClient } from './services/redisClient.js';
|
||||||
@@ -42,6 +42,20 @@ const server = app.listen(PORT, async () => {
|
|||||||
void err;
|
void err;
|
||||||
}
|
}
|
||||||
}, intervalMs);
|
}, intervalMs);
|
||||||
|
|
||||||
|
let trimRunning = false;
|
||||||
|
const logsTrimIntervalMs = 60 * 60 * 1000;
|
||||||
|
app.locals.logsTrimInterval = setInterval(async () => {
|
||||||
|
if (trimRunning) return;
|
||||||
|
trimRunning = true;
|
||||||
|
try {
|
||||||
|
await trimProjectConsoleLogsByLength(redis, { maxLen: 1000 });
|
||||||
|
} catch (err) {
|
||||||
|
void err;
|
||||||
|
} finally {
|
||||||
|
trimRunning = false;
|
||||||
|
}
|
||||||
|
}, logsTrimIntervalMs);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('[redis] failed to connect on startup', err);
|
console.error('[redis] failed to connect on startup', err);
|
||||||
}
|
}
|
||||||
@@ -52,6 +66,9 @@ process.on('SIGINT', async () => {
|
|||||||
if (app.locals.heartbeatPruneInterval) {
|
if (app.locals.heartbeatPruneInterval) {
|
||||||
clearInterval(app.locals.heartbeatPruneInterval);
|
clearInterval(app.locals.heartbeatPruneInterval);
|
||||||
}
|
}
|
||||||
|
if (app.locals.logsTrimInterval) {
|
||||||
|
clearInterval(app.locals.logsTrimInterval);
|
||||||
|
}
|
||||||
if (app.locals.redis) {
|
if (app.locals.redis) {
|
||||||
await app.locals.redis.quit();
|
await app.locals.redis.quit();
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user