refactor: 重构分区索引策略,移除显式创建索引的方法并更新相关流程
This commit is contained in:
66
bls-onoffline-backend/dist/index.js
vendored
66
bls-onoffline-backend/dist/index.js
vendored
@@ -193,65 +193,6 @@ class PartitionManager {
|
|||||||
const endMs = end.getTime();
|
const endMs = end.getTime();
|
||||||
return { startMs, endMs, partitionSuffix };
|
return { startMs, endMs, partitionSuffix };
|
||||||
}
|
}
|
||||||
async ensurePartitionIndexes(client, schema, table, partitionSuffix) {
|
|
||||||
const startedAt = Date.now();
|
|
||||||
const partitionName = `${schema}.${table}_${partitionSuffix}`;
|
|
||||||
const indexBase = `${table}_${partitionSuffix}`;
|
|
||||||
const indexSpecs = [
|
|
||||||
{ name: `idx_${indexBase}_ts`, column: "ts_ms" },
|
|
||||||
{ name: `idx_${indexBase}_hid`, column: "hotel_id" },
|
|
||||||
{ name: `idx_${indexBase}_mac`, column: "mac" },
|
|
||||||
{ name: `idx_${indexBase}_did`, column: "device_id" },
|
|
||||||
{ name: `idx_${indexBase}_rid`, column: "room_id" },
|
|
||||||
{ name: `idx_${indexBase}_cs`, column: "current_status" }
|
|
||||||
];
|
|
||||||
for (const spec of indexSpecs) {
|
|
||||||
await client.query(`CREATE INDEX IF NOT EXISTS ${spec.name} ON ${partitionName} (${spec.column});`);
|
|
||||||
}
|
|
||||||
await client.query(`ANALYZE ${partitionName};`);
|
|
||||||
const elapsedMs = Date.now() - startedAt;
|
|
||||||
if (elapsedMs > 1e3) {
|
|
||||||
logger.warn(`Partition index ensure slow`, { partitionName, elapsedMs });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
async ensureIndexesForExistingPartitions() {
|
|
||||||
const startedAt = Date.now();
|
|
||||||
const client = await dbManager.pool.connect();
|
|
||||||
try {
|
|
||||||
const schema = config.db.schema;
|
|
||||||
const table = config.db.table;
|
|
||||||
const res = await client.query(
|
|
||||||
`
|
|
||||||
SELECT c.relname AS relname
|
|
||||||
FROM pg_inherits i
|
|
||||||
JOIN pg_class p ON i.inhparent = p.oid
|
|
||||||
JOIN pg_namespace pn ON pn.oid = p.relnamespace
|
|
||||||
JOIN pg_class c ON i.inhrelid = c.oid
|
|
||||||
WHERE pn.nspname = $1 AND p.relname = $2
|
|
||||||
ORDER BY c.relname;
|
|
||||||
`,
|
|
||||||
[schema, table]
|
|
||||||
);
|
|
||||||
const suffixes = /* @__PURE__ */ new Set();
|
|
||||||
const pattern = new RegExp(`^${table}_(\\d{8})$`);
|
|
||||||
for (const row of res.rows) {
|
|
||||||
const relname = row?.relname;
|
|
||||||
if (typeof relname !== "string") continue;
|
|
||||||
const match = relname.match(pattern);
|
|
||||||
if (!match) continue;
|
|
||||||
suffixes.add(match[1]);
|
|
||||||
}
|
|
||||||
for (const suffix of suffixes) {
|
|
||||||
await this.ensurePartitionIndexes(client, schema, table, suffix);
|
|
||||||
}
|
|
||||||
const elapsedMs = Date.now() - startedAt;
|
|
||||||
if (elapsedMs > 5e3) {
|
|
||||||
logger.warn("Ensure existing partition indexes slow", { schema, table, partitions: suffixes.size, elapsedMs });
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
client.release();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/**
|
/**
|
||||||
* Ensure partitions exist for the past M days and next N days.
|
* Ensure partitions exist for the past M days and next N days.
|
||||||
* @param {number} daysAhead - Number of days to pre-create.
|
* @param {number} daysAhead - Number of days to pre-create.
|
||||||
@@ -284,7 +225,6 @@ class PartitionManager {
|
|||||||
`;
|
`;
|
||||||
await client.query(createSql);
|
await client.query(createSql);
|
||||||
}
|
}
|
||||||
await this.ensurePartitionIndexes(client, schema, table, partitionSuffix);
|
|
||||||
}
|
}
|
||||||
logger.info("Partition check completed.");
|
logger.info("Partition check completed.");
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -329,7 +269,6 @@ class PartitionManager {
|
|||||||
FOR VALUES FROM (${startMs}) TO (${endMs});
|
FOR VALUES FROM (${startMs}) TO (${endMs});
|
||||||
`);
|
`);
|
||||||
}
|
}
|
||||||
await this.ensurePartitionIndexes(client, schema, table, partitionSuffix);
|
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
client.release();
|
client.release();
|
||||||
@@ -345,7 +284,6 @@ class DatabaseInitializer {
|
|||||||
await this.ensureDatabaseExists();
|
await this.ensureDatabaseExists();
|
||||||
await this.ensureSchemaAndTable();
|
await this.ensureSchemaAndTable();
|
||||||
await partitionManager.ensurePartitions(30);
|
await partitionManager.ensurePartitions(30);
|
||||||
await partitionManager.ensureIndexesForExistingPartitions();
|
|
||||||
console.log("Database initialization completed successfully.");
|
console.log("Database initialization completed successfully.");
|
||||||
logger.info("Database initialization completed successfully.");
|
logger.info("Database initialization completed successfully.");
|
||||||
}
|
}
|
||||||
@@ -875,9 +813,9 @@ const bootstrap = async () => {
|
|||||||
const metrics = metricCollector.getAndReset();
|
const metrics = metricCollector.getAndReset();
|
||||||
const flushAvgMs = metrics.batch_flush_count > 0 ? (metrics.batch_flush_ms_sum / metrics.batch_flush_count).toFixed(1) : "0.0";
|
const flushAvgMs = metrics.batch_flush_count > 0 ? (metrics.batch_flush_ms_sum / metrics.batch_flush_count).toFixed(1) : "0.0";
|
||||||
const dbAvgMs = metrics.db_insert_count > 0 ? (metrics.db_insert_ms_sum / metrics.db_insert_count).toFixed(1) : "0.0";
|
const dbAvgMs = metrics.db_insert_count > 0 ? (metrics.db_insert_ms_sum / metrics.db_insert_count).toFixed(1) : "0.0";
|
||||||
const report = `[Minute Metrics] Pulled: ${metrics.kafka_pulled}, Parse Error: ${metrics.parse_error}, Inserted: ${metrics.db_inserted}, Failed: ${metrics.db_failed}, FlushAvgMs: ${flushAvgMs}, DbAvgMs: ${dbAvgMs}, PulledByPartition: ${JSON.stringify(metrics.keyed?.kafka_pulled_by_partition || {})}, InsertedByPartition: ${JSON.stringify(metrics.keyed?.db_inserted_by_partition || {})}, FailedByPartition: ${JSON.stringify(metrics.keyed?.db_failed_by_partition || {})}, InsertedByDay: ${JSON.stringify(metrics.keyed?.db_inserted_by_day || {})}, DbMsByDay: ${JSON.stringify(metrics.keyed?.db_insert_ms_sum_by_day || {})}`;
|
const report = `[Metrics] Pulled:${metrics.kafka_pulled} ParseErr:${metrics.parse_error} Inserted:${metrics.db_inserted} Failed:${metrics.db_failed} FlushAvg:${flushAvgMs}ms DbAvg:${dbAvgMs}ms`;
|
||||||
console.log(report);
|
console.log(report);
|
||||||
logger.info(report, metrics);
|
logger.info(report);
|
||||||
try {
|
try {
|
||||||
await redisIntegration.info("Minute Metrics", metrics);
|
await redisIntegration.info("Minute Metrics", metrics);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
# Proposal: Refactor Partition Indexes
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
利用 PostgreSQL 默认的支持,改变每日分区创立时的索引策略,不再在代码中对每个分区单独创建索引。
|
||||||
|
|
||||||
|
## Context
|
||||||
|
当前 `PartitionManager` 在动态创建子分区后,会隐式调用查询在子分区上创建六个单列索引。由于我们使用的是 PostgreSQL 11+,且我们在初始化脚本中的主分区表 `onoffline.onoffline_record` 上已经创建了所有的索引,此主表上的索引会自动应用于所有的子分区,不需要我们在创建分区时另外手动添加。
|
||||||
|
|
||||||
|
## Proposed Changes
|
||||||
|
1. 在 `src/db/partitionManager.js` 中移除子分区显式创建索引的方法 `ensurePartitionIndexes` 以及针对已有子分区的循环索引检查函数 `ensureIndexesForExistingPartitions`。
|
||||||
|
2. 在更新分区流程 `ensurePartitions` 以及 `ensurePartitionsForTimestamps` 中,移除对 `ensurePartitionIndexes` 的调用。
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
# Spec Delta: onoffline-backend
|
||||||
|
|
||||||
|
## MODIFIED Requirements
|
||||||
|
|
||||||
|
### Requirement: 数据库分区策略
|
||||||
|
系统 SHALL 使用 Range Partitioning 按天分区,并自动维护未来 30 天的分区表,子表依赖 PostgreSQL 原生机制继承主表索引。
|
||||||
|
|
||||||
|
#### Scenario: 分区预创建
|
||||||
|
- **GIVEN** 系统启动或每日凌晨
|
||||||
|
- **WHEN** 运行分区维护任务
|
||||||
|
- **THEN** 确保数据库中存在未来 30 天的分区表,无需对子表显式创建单列表索引
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# Tasks: Refactor Partition Indexes
|
||||||
|
|
||||||
|
- [x] refactor `src/db/partitionManager.js`: remove `ensurePartitionIndexes` and `ensureIndexesForExistingPartitions`.
|
||||||
|
- [x] refactor `src/db/partitionManager.js`: update `ensurePartitions` and `ensurePartitionsForTimestamps` to remove calls to `ensurePartitionIndexes`.
|
||||||
|
- [x] refactor `src/db/initializer.js` (and any other occurrences) to reflect the removal.
|
||||||
|
- [x] update openspec requirements to clarify that index propagation relies on PostgreSQL parent-table indexes.
|
||||||
@@ -23,7 +23,6 @@ class DatabaseInitializer {
|
|||||||
|
|
||||||
// 3. Ensure Partitions for the next month
|
// 3. Ensure Partitions for the next month
|
||||||
await partitionManager.ensurePartitions(30);
|
await partitionManager.ensurePartitions(30);
|
||||||
await partitionManager.ensureIndexesForExistingPartitions();
|
|
||||||
|
|
||||||
console.log('Database initialization completed successfully.');
|
console.log('Database initialization completed successfully.');
|
||||||
logger.info('Database initialization completed successfully.');
|
logger.info('Database initialization completed successfully.');
|
||||||
|
|||||||
@@ -26,75 +26,6 @@ class PartitionManager {
|
|||||||
return { startMs, endMs, partitionSuffix };
|
return { startMs, endMs, partitionSuffix };
|
||||||
}
|
}
|
||||||
|
|
||||||
async ensurePartitionIndexes(client, schema, table, partitionSuffix) {
|
|
||||||
const startedAt = Date.now();
|
|
||||||
const partitionName = `${schema}.${table}_${partitionSuffix}`;
|
|
||||||
const indexBase = `${table}_${partitionSuffix}`;
|
|
||||||
|
|
||||||
const indexSpecs = [
|
|
||||||
{ name: `idx_${indexBase}_ts`, column: 'ts_ms' },
|
|
||||||
{ name: `idx_${indexBase}_hid`, column: 'hotel_id' },
|
|
||||||
{ name: `idx_${indexBase}_mac`, column: 'mac' },
|
|
||||||
{ name: `idx_${indexBase}_did`, column: 'device_id' },
|
|
||||||
{ name: `idx_${indexBase}_rid`, column: 'room_id' },
|
|
||||||
{ name: `idx_${indexBase}_cs`, column: 'current_status' }
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const spec of indexSpecs) {
|
|
||||||
await client.query(`CREATE INDEX IF NOT EXISTS ${spec.name} ON ${partitionName} (${spec.column});`);
|
|
||||||
}
|
|
||||||
|
|
||||||
await client.query(`ANALYZE ${partitionName};`);
|
|
||||||
|
|
||||||
const elapsedMs = Date.now() - startedAt;
|
|
||||||
if (elapsedMs > 1000) {
|
|
||||||
logger.warn(`Partition index ensure slow`, { partitionName, elapsedMs });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async ensureIndexesForExistingPartitions() {
|
|
||||||
const startedAt = Date.now();
|
|
||||||
const client = await dbManager.pool.connect();
|
|
||||||
try {
|
|
||||||
const schema = config.db.schema;
|
|
||||||
const table = config.db.table;
|
|
||||||
|
|
||||||
const res = await client.query(
|
|
||||||
`
|
|
||||||
SELECT c.relname AS relname
|
|
||||||
FROM pg_inherits i
|
|
||||||
JOIN pg_class p ON i.inhparent = p.oid
|
|
||||||
JOIN pg_namespace pn ON pn.oid = p.relnamespace
|
|
||||||
JOIN pg_class c ON i.inhrelid = c.oid
|
|
||||||
WHERE pn.nspname = $1 AND p.relname = $2
|
|
||||||
ORDER BY c.relname;
|
|
||||||
`,
|
|
||||||
[schema, table]
|
|
||||||
);
|
|
||||||
|
|
||||||
const suffixes = new Set();
|
|
||||||
const pattern = new RegExp(`^${table}_(\\d{8})$`);
|
|
||||||
for (const row of res.rows) {
|
|
||||||
const relname = row?.relname;
|
|
||||||
if (typeof relname !== 'string') continue;
|
|
||||||
const match = relname.match(pattern);
|
|
||||||
if (!match) continue;
|
|
||||||
suffixes.add(match[1]);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const suffix of suffixes) {
|
|
||||||
await this.ensurePartitionIndexes(client, schema, table, suffix);
|
|
||||||
}
|
|
||||||
|
|
||||||
const elapsedMs = Date.now() - startedAt;
|
|
||||||
if (elapsedMs > 5000) {
|
|
||||||
logger.warn('Ensure existing partition indexes slow', { schema, table, partitions: suffixes.size, elapsedMs });
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
client.release();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Ensure partitions exist for the past M days and next N days.
|
* Ensure partitions exist for the past M days and next N days.
|
||||||
* @param {number} daysAhead - Number of days to pre-create.
|
* @param {number} daysAhead - Number of days to pre-create.
|
||||||
@@ -132,8 +63,6 @@ class PartitionManager {
|
|||||||
`;
|
`;
|
||||||
await client.query(createSql);
|
await client.query(createSql);
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.ensurePartitionIndexes(client, schema, table, partitionSuffix);
|
|
||||||
}
|
}
|
||||||
logger.info('Partition check completed.');
|
logger.info('Partition check completed.');
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -185,8 +114,6 @@ class PartitionManager {
|
|||||||
FOR VALUES FROM (${startMs}) TO (${endMs});
|
FOR VALUES FROM (${startMs}) TO (${endMs});
|
||||||
`);
|
`);
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.ensurePartitionIndexes(client, schema, table, partitionSuffix);
|
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
client.release();
|
client.release();
|
||||||
|
|||||||
@@ -78,9 +78,9 @@ const bootstrap = async () => {
|
|||||||
const metrics = metricCollector.getAndReset();
|
const metrics = metricCollector.getAndReset();
|
||||||
const flushAvgMs = metrics.batch_flush_count > 0 ? (metrics.batch_flush_ms_sum / metrics.batch_flush_count).toFixed(1) : '0.0';
|
const flushAvgMs = metrics.batch_flush_count > 0 ? (metrics.batch_flush_ms_sum / metrics.batch_flush_count).toFixed(1) : '0.0';
|
||||||
const dbAvgMs = metrics.db_insert_count > 0 ? (metrics.db_insert_ms_sum / metrics.db_insert_count).toFixed(1) : '0.0';
|
const dbAvgMs = metrics.db_insert_count > 0 ? (metrics.db_insert_ms_sum / metrics.db_insert_count).toFixed(1) : '0.0';
|
||||||
const report = `[Minute Metrics] Pulled: ${metrics.kafka_pulled}, Parse Error: ${metrics.parse_error}, Inserted: ${metrics.db_inserted}, Failed: ${metrics.db_failed}, FlushAvgMs: ${flushAvgMs}, DbAvgMs: ${dbAvgMs}, PulledByPartition: ${JSON.stringify(metrics.keyed?.kafka_pulled_by_partition || {})}, InsertedByPartition: ${JSON.stringify(metrics.keyed?.db_inserted_by_partition || {})}, FailedByPartition: ${JSON.stringify(metrics.keyed?.db_failed_by_partition || {})}, InsertedByDay: ${JSON.stringify(metrics.keyed?.db_inserted_by_day || {})}, DbMsByDay: ${JSON.stringify(metrics.keyed?.db_insert_ms_sum_by_day || {})}`;
|
const report = `[Metrics] Pulled:${metrics.kafka_pulled} ParseErr:${metrics.parse_error} Inserted:${metrics.db_inserted} Failed:${metrics.db_failed} FlushAvg:${flushAvgMs}ms DbAvg:${dbAvgMs}ms`;
|
||||||
console.log(report);
|
console.log(report);
|
||||||
logger.info(report, metrics);
|
logger.info(report);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await redisIntegration.info('Minute Metrics', metrics);
|
await redisIntegration.info('Minute Metrics', metrics);
|
||||||
|
|||||||
Reference in New Issue
Block a user