feat(ops): 增加运行时性能看板
This commit is contained in:
11
apps/server/src/modules/ops/ops-permission.adapter.ts
Normal file
11
apps/server/src/modules/ops/ops-permission.adapter.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { OPS_VIEW_PERMISSION } from './ops.service';
|
||||
|
||||
@Injectable()
|
||||
export class OpsPermissionAdapter {
|
||||
assertCanViewOps(_request: unknown) {
|
||||
// V2.5 backend RBAC is not landed yet. Keep this adapter as the single
|
||||
// replacement point for a real guard instead of coupling Ops to a temporary shape.
|
||||
return { requiredPermission: OPS_VIEW_PERMISSION, enforced: false };
|
||||
}
|
||||
}
|
||||
53
apps/server/src/modules/ops/ops-runtime.store.spec.ts
Normal file
53
apps/server/src/modules/ops/ops-runtime.store.spec.ts
Normal file
@@ -0,0 +1,53 @@
|
||||
import {
|
||||
clearOpsRuntimeEventsForTests,
|
||||
getOpsRuntimeEvents,
|
||||
recordSlowPrismaQuery,
|
||||
recordSlowRequest,
|
||||
} from './ops-runtime.store';
|
||||
|
||||
describe('ops runtime store', () => {
|
||||
afterEach(() => clearOpsRuntimeEventsForTests());
|
||||
|
||||
it('records recent slow requests without leaking query-string secrets', () => {
|
||||
recordSlowRequest({
|
||||
method: 'GET',
|
||||
url: '/api/v1/config/ai?apiKey=sk-secret&visible=1',
|
||||
durationMs: 1300,
|
||||
thresholdMs: 1000,
|
||||
occurredAt: new Date('2026-07-08T08:00:00.000Z'),
|
||||
});
|
||||
|
||||
const events = getOpsRuntimeEvents();
|
||||
|
||||
expect(events.slowRequests).toEqual([
|
||||
expect.objectContaining({
|
||||
method: 'GET',
|
||||
path: '/api/v1/config/ai',
|
||||
durationMs: 1300,
|
||||
thresholdMs: 1000,
|
||||
occurredAt: '2026-07-08T08:00:00.000Z',
|
||||
}),
|
||||
]);
|
||||
expect(JSON.stringify(events)).not.toContain('sk-secret');
|
||||
});
|
||||
|
||||
it('records recent slow Prisma queries with redacted and bounded previews', () => {
|
||||
recordSlowPrismaQuery({
|
||||
query: `SELECT * FROM ai_logs WHERE metadata::text LIKE '%sk-secret-token%' ${'x'.repeat(400)}`,
|
||||
durationMs: 450,
|
||||
thresholdMs: 300,
|
||||
occurredAt: new Date('2026-07-08T08:01:00.000Z'),
|
||||
});
|
||||
|
||||
const [event] = getOpsRuntimeEvents().slowQueries;
|
||||
|
||||
expect(event).toEqual(expect.objectContaining({
|
||||
durationMs: 450,
|
||||
thresholdMs: 300,
|
||||
occurredAt: '2026-07-08T08:01:00.000Z',
|
||||
}));
|
||||
expect(event.queryPreview).toContain('[redacted]');
|
||||
expect(event.queryPreview.length).toBeLessThanOrEqual(240);
|
||||
expect(event.queryPreview).not.toContain('sk-secret-token');
|
||||
});
|
||||
});
|
||||
116
apps/server/src/modules/ops/ops-runtime.store.ts
Normal file
116
apps/server/src/modules/ops/ops-runtime.store.ts
Normal file
@@ -0,0 +1,116 @@
|
||||
const MAX_EVENTS = 50;
|
||||
const MAX_PREVIEW_LENGTH = 240;
|
||||
|
||||
export interface SlowRequestEvent {
|
||||
id: string;
|
||||
method: string;
|
||||
path: string;
|
||||
durationMs: number;
|
||||
thresholdMs: number;
|
||||
occurredAt: string;
|
||||
}
|
||||
|
||||
export interface SlowPrismaQueryEvent {
|
||||
id: string;
|
||||
queryPreview: string;
|
||||
durationMs: number;
|
||||
thresholdMs: number;
|
||||
occurredAt: string;
|
||||
}
|
||||
|
||||
export interface OpsRuntimeEvents {
|
||||
slowRequests: SlowRequestEvent[];
|
||||
slowQueries: SlowPrismaQueryEvent[];
|
||||
}
|
||||
|
||||
interface RecordSlowRequestInput {
|
||||
method: string;
|
||||
url: string;
|
||||
durationMs: number;
|
||||
thresholdMs: number;
|
||||
occurredAt?: Date;
|
||||
}
|
||||
|
||||
interface RecordSlowPrismaQueryInput {
|
||||
query: string;
|
||||
durationMs: number;
|
||||
thresholdMs: number;
|
||||
occurredAt?: Date;
|
||||
}
|
||||
|
||||
const slowRequests: SlowRequestEvent[] = [];
|
||||
const slowQueries: SlowPrismaQueryEvent[] = [];
|
||||
let nextId = 1;
|
||||
|
||||
export function recordSlowRequest(input: RecordSlowRequestInput) {
|
||||
slowRequests.unshift({
|
||||
id: makeId('req'),
|
||||
method: normalizeMethod(input.method),
|
||||
path: sanitizeRequestPath(input.url),
|
||||
durationMs: Math.round(input.durationMs),
|
||||
thresholdMs: Math.round(input.thresholdMs),
|
||||
occurredAt: (input.occurredAt ?? new Date()).toISOString(),
|
||||
});
|
||||
trim(slowRequests);
|
||||
}
|
||||
|
||||
export function recordSlowPrismaQuery(input: RecordSlowPrismaQueryInput) {
|
||||
slowQueries.unshift({
|
||||
id: makeId('qry'),
|
||||
queryPreview: truncate(redactText(input.query.replace(/\s+/g, ' ').trim()), MAX_PREVIEW_LENGTH),
|
||||
durationMs: Math.round(input.durationMs),
|
||||
thresholdMs: Math.round(input.thresholdMs),
|
||||
occurredAt: (input.occurredAt ?? new Date()).toISOString(),
|
||||
});
|
||||
trim(slowQueries);
|
||||
}
|
||||
|
||||
export function getOpsRuntimeEvents(): OpsRuntimeEvents {
|
||||
return {
|
||||
slowRequests: slowRequests.map((event) => ({ ...event })),
|
||||
slowQueries: slowQueries.map((event) => ({ ...event })),
|
||||
};
|
||||
}
|
||||
|
||||
export function clearOpsRuntimeEventsForTests() {
|
||||
slowRequests.splice(0, slowRequests.length);
|
||||
slowQueries.splice(0, slowQueries.length);
|
||||
nextId = 1;
|
||||
}
|
||||
|
||||
export function redactText(value: string, maxLength = 500): string {
|
||||
const redacted = value
|
||||
.replace(/(^|[^A-Za-z0-9])sk-[A-Za-z0-9_-]+/g, '$1[redacted]')
|
||||
.replace(/(api[_-]?key|token|secret|password|authorization)(\s*[=:]\s*)(["']?)[^&\s"']+/gi, '$1$2$3[redacted]');
|
||||
return truncate(redacted, maxLength);
|
||||
}
|
||||
|
||||
function sanitizeRequestPath(url: string): string {
|
||||
const raw = String(url || 'unknown-url');
|
||||
try {
|
||||
const parsed = new URL(raw, 'http://local.invalid');
|
||||
return redactText(parsed.pathname || '/');
|
||||
} catch {
|
||||
return redactText(raw.split('?')[0] || 'unknown-url');
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeMethod(method: string): string {
|
||||
const value = String(method || 'UNKNOWN').toUpperCase();
|
||||
return /^[A-Z]+$/.test(value) ? value : 'UNKNOWN';
|
||||
}
|
||||
|
||||
function truncate(value: string, maxLength: number): string {
|
||||
if (value.length <= maxLength) return value;
|
||||
return `${value.slice(0, Math.max(0, maxLength - 3))}...`;
|
||||
}
|
||||
|
||||
function trim<T>(items: T[]) {
|
||||
if (items.length > MAX_EVENTS) items.splice(MAX_EVENTS);
|
||||
}
|
||||
|
||||
function makeId(prefix: string): string {
|
||||
const id = `${prefix}-${Date.now().toString(36)}-${nextId.toString(36)}`;
|
||||
nextId += 1;
|
||||
return id;
|
||||
}
|
||||
19
apps/server/src/modules/ops/ops.controller.spec.ts
Normal file
19
apps/server/src/modules/ops/ops.controller.spec.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
import { OpsController } from './ops.controller';
|
||||
|
||||
describe('OpsController', () => {
|
||||
it('checks the ops:view adapter before returning runtime snapshot', async () => {
|
||||
const service = {
|
||||
getRuntimeSnapshot: jest.fn().mockResolvedValue({ ok: true }),
|
||||
};
|
||||
const permissions = {
|
||||
assertCanViewOps: jest.fn(),
|
||||
};
|
||||
const controller = new OpsController(service as any, permissions as any);
|
||||
const request = { headers: {} };
|
||||
|
||||
await expect(controller.getRuntime(request as any)).resolves.toEqual({ ok: true });
|
||||
|
||||
expect(permissions.assertCanViewOps).toHaveBeenCalledWith(request);
|
||||
expect(service.getRuntimeSnapshot).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
17
apps/server/src/modules/ops/ops.controller.ts
Normal file
17
apps/server/src/modules/ops/ops.controller.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import { Controller, Get, Req } from '@nestjs/common';
|
||||
import { OpsPermissionAdapter } from './ops-permission.adapter';
|
||||
import { OpsService } from './ops.service';
|
||||
|
||||
@Controller('ops')
|
||||
export class OpsController {
|
||||
constructor(
|
||||
private readonly ops: OpsService,
|
||||
private readonly permissions: OpsPermissionAdapter,
|
||||
) {}
|
||||
|
||||
@Get('runtime')
|
||||
getRuntime(@Req() request: unknown) {
|
||||
this.permissions.assertCanViewOps(request);
|
||||
return this.ops.getRuntimeSnapshot();
|
||||
}
|
||||
}
|
||||
16
apps/server/src/modules/ops/ops.module.ts
Normal file
16
apps/server/src/modules/ops/ops.module.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { PrismaService } from '../../prisma/prisma.service';
|
||||
import { OpsController } from './ops.controller';
|
||||
import { OpsPermissionAdapter } from './ops-permission.adapter';
|
||||
import { OPS_PRISMA, OpsService } from './ops.service';
|
||||
|
||||
@Module({
|
||||
controllers: [OpsController],
|
||||
providers: [
|
||||
{ provide: OPS_PRISMA, useExisting: PrismaService },
|
||||
OpsService,
|
||||
OpsPermissionAdapter,
|
||||
],
|
||||
exports: [OpsService, OpsPermissionAdapter],
|
||||
})
|
||||
export class OpsModule {}
|
||||
89
apps/server/src/modules/ops/ops.service.spec.ts
Normal file
89
apps/server/src/modules/ops/ops.service.spec.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import { recordSlowPrismaQuery, recordSlowRequest, clearOpsRuntimeEventsForTests } from './ops-runtime.store';
|
||||
import { OpsService } from './ops.service';
|
||||
|
||||
describe('OpsService', () => {
|
||||
afterEach(() => clearOpsRuntimeEventsForTests());
|
||||
|
||||
it('returns runtime performance counters without exposing secrets', async () => {
|
||||
const prisma = {
|
||||
backgroundJob: {
|
||||
findMany: jest.fn().mockResolvedValue([
|
||||
{
|
||||
id: 'job-queued',
|
||||
type: 'xiaobao.summary.refresh',
|
||||
status: 'queued',
|
||||
attempts: 0,
|
||||
maxAttempts: 5,
|
||||
availableAt: new Date('2026-07-08T08:00:00.000Z'),
|
||||
lockedUntil: null,
|
||||
lastError: null,
|
||||
updatedAt: new Date('2026-07-08T08:00:00.000Z'),
|
||||
},
|
||||
{
|
||||
id: 'job-running',
|
||||
type: 'xiaobao.ai.interpret',
|
||||
status: 'running',
|
||||
attempts: 1,
|
||||
maxAttempts: 3,
|
||||
availableAt: new Date('2026-07-08T08:01:00.000Z'),
|
||||
lockedUntil: new Date('2026-07-08T08:05:00.000Z'),
|
||||
lastError: null,
|
||||
updatedAt: new Date('2026-07-08T08:02:00.000Z'),
|
||||
},
|
||||
{
|
||||
id: 'job-failed',
|
||||
type: 'xiaobao.ai.interpret',
|
||||
status: 'failed',
|
||||
attempts: 3,
|
||||
maxAttempts: 3,
|
||||
availableAt: new Date('2026-07-08T08:03:00.000Z'),
|
||||
lockedUntil: null,
|
||||
lastError: 'provider failed with sk-secret-token',
|
||||
updatedAt: new Date('2026-07-08T08:04:00.000Z'),
|
||||
},
|
||||
]),
|
||||
},
|
||||
xiaobaoRiskSummary: {
|
||||
count: jest.fn().mockResolvedValue(7),
|
||||
},
|
||||
};
|
||||
recordSlowRequest({
|
||||
method: 'POST',
|
||||
url: '/api/v1/ai/risk-interpret?token=sk-secret-token',
|
||||
durationMs: 1500,
|
||||
thresholdMs: 1000,
|
||||
occurredAt: new Date('2026-07-08T08:05:00.000Z'),
|
||||
});
|
||||
recordSlowPrismaQuery({
|
||||
query: 'SELECT * FROM background_jobs WHERE last_error = "sk-secret-token"',
|
||||
durationMs: 420,
|
||||
thresholdMs: 300,
|
||||
occurredAt: new Date('2026-07-08T08:06:00.000Z'),
|
||||
});
|
||||
const service = new OpsService(prisma as any);
|
||||
|
||||
const snapshot = await service.getRuntimeSnapshot(new Date('2026-07-08T08:07:00.000Z'));
|
||||
|
||||
expect(snapshot.dirtySummaryCount).toBe(7);
|
||||
expect(snapshot.jobQueue.totals).toEqual({ queued: 1, running: 1, succeeded: 0, failed: 1, total: 3 });
|
||||
expect(snapshot.jobQueue.byType).toEqual([
|
||||
expect.objectContaining({ type: 'xiaobao.ai.interpret', running: 1, failed: 1, total: 2 }),
|
||||
expect.objectContaining({ type: 'xiaobao.summary.refresh', queued: 1, total: 1 }),
|
||||
]);
|
||||
expect(snapshot.jobQueue.recentFailures).toEqual([
|
||||
expect.objectContaining({
|
||||
id: 'job-failed',
|
||||
type: 'xiaobao.ai.interpret',
|
||||
lastError: expect.stringContaining('[redacted]'),
|
||||
}),
|
||||
]);
|
||||
expect(snapshot.slowRequests[0].path).toBe('/api/v1/ai/risk-interpret');
|
||||
expect(snapshot.slowQueries[0].queryPreview).toContain('[redacted]');
|
||||
expect(JSON.stringify(snapshot)).not.toContain('sk-secret-token');
|
||||
expect(snapshot.access).toEqual({
|
||||
requiredPermission: 'ops:view',
|
||||
backendEnforced: false,
|
||||
adapter: 'OpsPermissionAdapter',
|
||||
});
|
||||
});
|
||||
});
|
||||
148
apps/server/src/modules/ops/ops.service.ts
Normal file
148
apps/server/src/modules/ops/ops.service.ts
Normal file
@@ -0,0 +1,148 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { resolveApiSlowRequestThreshold } from '../../common/interceptors/api-timing.interceptor';
|
||||
import { resolvePrismaSlowQueryThreshold } from '../../prisma/prisma-monitoring';
|
||||
import { getOpsRuntimeEvents, redactText } from './ops-runtime.store';
|
||||
|
||||
export const OPS_PRISMA = 'OPS_PRISMA';
|
||||
export const OPS_VIEW_PERMISSION = 'ops:view';
|
||||
|
||||
type JobStatus = 'queued' | 'running' | 'succeeded' | 'failed';
|
||||
|
||||
interface BackgroundJobRow {
|
||||
id: string;
|
||||
type: string;
|
||||
status: string;
|
||||
attempts?: number | null;
|
||||
maxAttempts?: number | null;
|
||||
availableAt?: Date | string | null;
|
||||
lockedUntil?: Date | string | null;
|
||||
lastError?: string | null;
|
||||
updatedAt?: Date | string | null;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class OpsService {
|
||||
constructor(@Inject(OPS_PRISMA) private readonly prisma: any) {}
|
||||
|
||||
async getRuntimeSnapshot(now = new Date()) {
|
||||
const events = getOpsRuntimeEvents();
|
||||
const database = { ok: true, error: undefined as string | undefined };
|
||||
let jobRows: BackgroundJobRow[] = [];
|
||||
let dirtySummaryCount = 0;
|
||||
|
||||
try {
|
||||
[jobRows, dirtySummaryCount] = await Promise.all([
|
||||
this.prisma.backgroundJob.findMany({
|
||||
select: {
|
||||
id: true,
|
||||
type: true,
|
||||
status: true,
|
||||
attempts: true,
|
||||
maxAttempts: true,
|
||||
availableAt: true,
|
||||
lockedUntil: true,
|
||||
lastError: true,
|
||||
updatedAt: true,
|
||||
},
|
||||
orderBy: { updatedAt: 'desc' },
|
||||
take: 200,
|
||||
}),
|
||||
this.prisma.xiaobaoRiskSummary.count({ where: { dirty: true } }),
|
||||
]);
|
||||
} catch (error) {
|
||||
database.ok = false;
|
||||
database.error = redactText(error instanceof Error ? error.message : String(error), 300);
|
||||
}
|
||||
|
||||
return {
|
||||
collectedAt: now.toISOString(),
|
||||
thresholds: {
|
||||
apiSlowRequestMs: resolveApiSlowRequestThreshold(),
|
||||
prismaSlowQueryMs: resolvePrismaSlowQueryThreshold(),
|
||||
},
|
||||
database,
|
||||
slowRequests: events.slowRequests,
|
||||
slowQueries: events.slowQueries,
|
||||
jobQueue: buildJobQueue(jobRows),
|
||||
dirtySummaryCount,
|
||||
access: {
|
||||
requiredPermission: OPS_VIEW_PERMISSION,
|
||||
backendEnforced: false,
|
||||
adapter: 'OpsPermissionAdapter',
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function buildJobQueue(rows: BackgroundJobRow[]) {
|
||||
const totals = emptyStatusCounts();
|
||||
const byType = new Map<string, ReturnType<typeof emptyTypeCounts>>();
|
||||
const recentFailures = rows
|
||||
.filter((row) => normalizeStatus(row.status) === 'failed')
|
||||
.slice(0, 10)
|
||||
.map((row) => ({
|
||||
id: row.id,
|
||||
type: row.type,
|
||||
attempts: row.attempts ?? 0,
|
||||
maxAttempts: row.maxAttempts ?? 0,
|
||||
lastError: redactText(row.lastError ?? '', 300),
|
||||
updatedAt: toIso(row.updatedAt),
|
||||
}));
|
||||
|
||||
for (const row of rows) {
|
||||
const status = normalizeStatus(row.status);
|
||||
totals[status] += 1;
|
||||
totals.total += 1;
|
||||
|
||||
const current = byType.get(row.type) ?? emptyTypeCounts(row.type);
|
||||
current[status] += 1;
|
||||
current.total += 1;
|
||||
if (status === 'queued') {
|
||||
current.oldestQueuedAt = minIso(current.oldestQueuedAt, toIso(row.availableAt));
|
||||
}
|
||||
if (status === 'running') {
|
||||
current.nextLeaseExpiresAt = minIso(current.nextLeaseExpiresAt, toIso(row.lockedUntil));
|
||||
}
|
||||
byType.set(row.type, current);
|
||||
}
|
||||
|
||||
return {
|
||||
totals,
|
||||
byType: Array.from(byType.values()).sort((a, b) => b.total - a.total || a.type.localeCompare(b.type)),
|
||||
recentFailures,
|
||||
};
|
||||
}
|
||||
|
||||
function emptyStatusCounts(): Record<JobStatus, number> & { total: number } {
|
||||
return { queued: 0, running: 0, succeeded: 0, failed: 0, total: 0 };
|
||||
}
|
||||
|
||||
function emptyTypeCounts(type: string) {
|
||||
return {
|
||||
type,
|
||||
queued: 0,
|
||||
running: 0,
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
total: 0,
|
||||
oldestQueuedAt: undefined as string | undefined,
|
||||
nextLeaseExpiresAt: undefined as string | undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeStatus(status: string): JobStatus {
|
||||
return status === 'running' || status === 'succeeded' || status === 'failed' ? status : 'queued';
|
||||
}
|
||||
|
||||
function toIso(value: Date | string | null | undefined): string | undefined {
|
||||
if (value instanceof Date) return Number.isFinite(value.getTime()) ? value.toISOString() : undefined;
|
||||
if (typeof value !== 'string') return undefined;
|
||||
const date = new Date(value);
|
||||
return Number.isFinite(date.getTime()) ? date.toISOString() : undefined;
|
||||
}
|
||||
|
||||
function minIso(current: string | undefined, next: string | undefined): string | undefined {
|
||||
if (!next) return current;
|
||||
if (!current) return next;
|
||||
return next < current ? next : current;
|
||||
}
|
||||
Reference in New Issue
Block a user