feat(ops): 增加运行时性能看板

This commit is contained in:
2026-07-08 17:47:36 +08:00
parent 58c98a3a3d
commit 15653b5b35
19 changed files with 933 additions and 2 deletions

View File

@@ -0,0 +1,11 @@
import { Injectable } from '@nestjs/common';
import { OPS_VIEW_PERMISSION } from './ops.service';
@Injectable()
export class OpsPermissionAdapter {
assertCanViewOps(_request: unknown) {
// V2.5 backend RBAC is not landed yet. Keep this adapter as the single
// replacement point for a real guard instead of coupling Ops to a temporary shape.
return { requiredPermission: OPS_VIEW_PERMISSION, enforced: false };
}
}

View File

@@ -0,0 +1,53 @@
import {
clearOpsRuntimeEventsForTests,
getOpsRuntimeEvents,
recordSlowPrismaQuery,
recordSlowRequest,
} from './ops-runtime.store';
describe('ops runtime store', () => {
afterEach(() => clearOpsRuntimeEventsForTests());
it('records recent slow requests without leaking query-string secrets', () => {
recordSlowRequest({
method: 'GET',
url: '/api/v1/config/ai?apiKey=sk-secret&visible=1',
durationMs: 1300,
thresholdMs: 1000,
occurredAt: new Date('2026-07-08T08:00:00.000Z'),
});
const events = getOpsRuntimeEvents();
expect(events.slowRequests).toEqual([
expect.objectContaining({
method: 'GET',
path: '/api/v1/config/ai',
durationMs: 1300,
thresholdMs: 1000,
occurredAt: '2026-07-08T08:00:00.000Z',
}),
]);
expect(JSON.stringify(events)).not.toContain('sk-secret');
});
it('records recent slow Prisma queries with redacted and bounded previews', () => {
recordSlowPrismaQuery({
query: `SELECT * FROM ai_logs WHERE metadata::text LIKE '%sk-secret-token%' ${'x'.repeat(400)}`,
durationMs: 450,
thresholdMs: 300,
occurredAt: new Date('2026-07-08T08:01:00.000Z'),
});
const [event] = getOpsRuntimeEvents().slowQueries;
expect(event).toEqual(expect.objectContaining({
durationMs: 450,
thresholdMs: 300,
occurredAt: '2026-07-08T08:01:00.000Z',
}));
expect(event.queryPreview).toContain('[redacted]');
expect(event.queryPreview.length).toBeLessThanOrEqual(240);
expect(event.queryPreview).not.toContain('sk-secret-token');
});
});

View File

@@ -0,0 +1,116 @@
const MAX_EVENTS = 50;
const MAX_PREVIEW_LENGTH = 240;
export interface SlowRequestEvent {
id: string;
method: string;
path: string;
durationMs: number;
thresholdMs: number;
occurredAt: string;
}
export interface SlowPrismaQueryEvent {
id: string;
queryPreview: string;
durationMs: number;
thresholdMs: number;
occurredAt: string;
}
export interface OpsRuntimeEvents {
slowRequests: SlowRequestEvent[];
slowQueries: SlowPrismaQueryEvent[];
}
interface RecordSlowRequestInput {
method: string;
url: string;
durationMs: number;
thresholdMs: number;
occurredAt?: Date;
}
interface RecordSlowPrismaQueryInput {
query: string;
durationMs: number;
thresholdMs: number;
occurredAt?: Date;
}
const slowRequests: SlowRequestEvent[] = [];
const slowQueries: SlowPrismaQueryEvent[] = [];
let nextId = 1;
export function recordSlowRequest(input: RecordSlowRequestInput) {
slowRequests.unshift({
id: makeId('req'),
method: normalizeMethod(input.method),
path: sanitizeRequestPath(input.url),
durationMs: Math.round(input.durationMs),
thresholdMs: Math.round(input.thresholdMs),
occurredAt: (input.occurredAt ?? new Date()).toISOString(),
});
trim(slowRequests);
}
export function recordSlowPrismaQuery(input: RecordSlowPrismaQueryInput) {
slowQueries.unshift({
id: makeId('qry'),
queryPreview: truncate(redactText(input.query.replace(/\s+/g, ' ').trim()), MAX_PREVIEW_LENGTH),
durationMs: Math.round(input.durationMs),
thresholdMs: Math.round(input.thresholdMs),
occurredAt: (input.occurredAt ?? new Date()).toISOString(),
});
trim(slowQueries);
}
export function getOpsRuntimeEvents(): OpsRuntimeEvents {
return {
slowRequests: slowRequests.map((event) => ({ ...event })),
slowQueries: slowQueries.map((event) => ({ ...event })),
};
}
export function clearOpsRuntimeEventsForTests() {
slowRequests.splice(0, slowRequests.length);
slowQueries.splice(0, slowQueries.length);
nextId = 1;
}
export function redactText(value: string, maxLength = 500): string {
const redacted = value
.replace(/(^|[^A-Za-z0-9])sk-[A-Za-z0-9_-]+/g, '$1[redacted]')
.replace(/(api[_-]?key|token|secret|password|authorization)(\s*[=:]\s*)(["']?)[^&\s"']+/gi, '$1$2$3[redacted]');
return truncate(redacted, maxLength);
}
function sanitizeRequestPath(url: string): string {
const raw = String(url || 'unknown-url');
try {
const parsed = new URL(raw, 'http://local.invalid');
return redactText(parsed.pathname || '/');
} catch {
return redactText(raw.split('?')[0] || 'unknown-url');
}
}
function normalizeMethod(method: string): string {
const value = String(method || 'UNKNOWN').toUpperCase();
return /^[A-Z]+$/.test(value) ? value : 'UNKNOWN';
}
function truncate(value: string, maxLength: number): string {
if (value.length <= maxLength) return value;
return `${value.slice(0, Math.max(0, maxLength - 3))}...`;
}
function trim<T>(items: T[]) {
if (items.length > MAX_EVENTS) items.splice(MAX_EVENTS);
}
function makeId(prefix: string): string {
const id = `${prefix}-${Date.now().toString(36)}-${nextId.toString(36)}`;
nextId += 1;
return id;
}

View File

@@ -0,0 +1,19 @@
import { OpsController } from './ops.controller';
describe('OpsController', () => {
it('checks the ops:view adapter before returning runtime snapshot', async () => {
const service = {
getRuntimeSnapshot: jest.fn().mockResolvedValue({ ok: true }),
};
const permissions = {
assertCanViewOps: jest.fn(),
};
const controller = new OpsController(service as any, permissions as any);
const request = { headers: {} };
await expect(controller.getRuntime(request as any)).resolves.toEqual({ ok: true });
expect(permissions.assertCanViewOps).toHaveBeenCalledWith(request);
expect(service.getRuntimeSnapshot).toHaveBeenCalled();
});
});

View File

@@ -0,0 +1,17 @@
import { Controller, Get, Req } from '@nestjs/common';
import { OpsPermissionAdapter } from './ops-permission.adapter';
import { OpsService } from './ops.service';
@Controller('ops')
export class OpsController {
constructor(
private readonly ops: OpsService,
private readonly permissions: OpsPermissionAdapter,
) {}
@Get('runtime')
getRuntime(@Req() request: unknown) {
this.permissions.assertCanViewOps(request);
return this.ops.getRuntimeSnapshot();
}
}

View File

@@ -0,0 +1,16 @@
import { Module } from '@nestjs/common';
import { PrismaService } from '../../prisma/prisma.service';
import { OpsController } from './ops.controller';
import { OpsPermissionAdapter } from './ops-permission.adapter';
import { OPS_PRISMA, OpsService } from './ops.service';
@Module({
controllers: [OpsController],
providers: [
{ provide: OPS_PRISMA, useExisting: PrismaService },
OpsService,
OpsPermissionAdapter,
],
exports: [OpsService, OpsPermissionAdapter],
})
export class OpsModule {}

View File

@@ -0,0 +1,89 @@
import { recordSlowPrismaQuery, recordSlowRequest, clearOpsRuntimeEventsForTests } from './ops-runtime.store';
import { OpsService } from './ops.service';
describe('OpsService', () => {
afterEach(() => clearOpsRuntimeEventsForTests());
it('returns runtime performance counters without exposing secrets', async () => {
const prisma = {
backgroundJob: {
findMany: jest.fn().mockResolvedValue([
{
id: 'job-queued',
type: 'xiaobao.summary.refresh',
status: 'queued',
attempts: 0,
maxAttempts: 5,
availableAt: new Date('2026-07-08T08:00:00.000Z'),
lockedUntil: null,
lastError: null,
updatedAt: new Date('2026-07-08T08:00:00.000Z'),
},
{
id: 'job-running',
type: 'xiaobao.ai.interpret',
status: 'running',
attempts: 1,
maxAttempts: 3,
availableAt: new Date('2026-07-08T08:01:00.000Z'),
lockedUntil: new Date('2026-07-08T08:05:00.000Z'),
lastError: null,
updatedAt: new Date('2026-07-08T08:02:00.000Z'),
},
{
id: 'job-failed',
type: 'xiaobao.ai.interpret',
status: 'failed',
attempts: 3,
maxAttempts: 3,
availableAt: new Date('2026-07-08T08:03:00.000Z'),
lockedUntil: null,
lastError: 'provider failed with sk-secret-token',
updatedAt: new Date('2026-07-08T08:04:00.000Z'),
},
]),
},
xiaobaoRiskSummary: {
count: jest.fn().mockResolvedValue(7),
},
};
recordSlowRequest({
method: 'POST',
url: '/api/v1/ai/risk-interpret?token=sk-secret-token',
durationMs: 1500,
thresholdMs: 1000,
occurredAt: new Date('2026-07-08T08:05:00.000Z'),
});
recordSlowPrismaQuery({
query: 'SELECT * FROM background_jobs WHERE last_error = "sk-secret-token"',
durationMs: 420,
thresholdMs: 300,
occurredAt: new Date('2026-07-08T08:06:00.000Z'),
});
const service = new OpsService(prisma as any);
const snapshot = await service.getRuntimeSnapshot(new Date('2026-07-08T08:07:00.000Z'));
expect(snapshot.dirtySummaryCount).toBe(7);
expect(snapshot.jobQueue.totals).toEqual({ queued: 1, running: 1, succeeded: 0, failed: 1, total: 3 });
expect(snapshot.jobQueue.byType).toEqual([
expect.objectContaining({ type: 'xiaobao.ai.interpret', running: 1, failed: 1, total: 2 }),
expect.objectContaining({ type: 'xiaobao.summary.refresh', queued: 1, total: 1 }),
]);
expect(snapshot.jobQueue.recentFailures).toEqual([
expect.objectContaining({
id: 'job-failed',
type: 'xiaobao.ai.interpret',
lastError: expect.stringContaining('[redacted]'),
}),
]);
expect(snapshot.slowRequests[0].path).toBe('/api/v1/ai/risk-interpret');
expect(snapshot.slowQueries[0].queryPreview).toContain('[redacted]');
expect(JSON.stringify(snapshot)).not.toContain('sk-secret-token');
expect(snapshot.access).toEqual({
requiredPermission: 'ops:view',
backendEnforced: false,
adapter: 'OpsPermissionAdapter',
});
});
});

View File

@@ -0,0 +1,148 @@
import { Inject, Injectable } from '@nestjs/common';
import { resolveApiSlowRequestThreshold } from '../../common/interceptors/api-timing.interceptor';
import { resolvePrismaSlowQueryThreshold } from '../../prisma/prisma-monitoring';
import { getOpsRuntimeEvents, redactText } from './ops-runtime.store';
export const OPS_PRISMA = 'OPS_PRISMA';
export const OPS_VIEW_PERMISSION = 'ops:view';
type JobStatus = 'queued' | 'running' | 'succeeded' | 'failed';
interface BackgroundJobRow {
id: string;
type: string;
status: string;
attempts?: number | null;
maxAttempts?: number | null;
availableAt?: Date | string | null;
lockedUntil?: Date | string | null;
lastError?: string | null;
updatedAt?: Date | string | null;
}
@Injectable()
export class OpsService {
constructor(@Inject(OPS_PRISMA) private readonly prisma: any) {}
async getRuntimeSnapshot(now = new Date()) {
const events = getOpsRuntimeEvents();
const database = { ok: true, error: undefined as string | undefined };
let jobRows: BackgroundJobRow[] = [];
let dirtySummaryCount = 0;
try {
[jobRows, dirtySummaryCount] = await Promise.all([
this.prisma.backgroundJob.findMany({
select: {
id: true,
type: true,
status: true,
attempts: true,
maxAttempts: true,
availableAt: true,
lockedUntil: true,
lastError: true,
updatedAt: true,
},
orderBy: { updatedAt: 'desc' },
take: 200,
}),
this.prisma.xiaobaoRiskSummary.count({ where: { dirty: true } }),
]);
} catch (error) {
database.ok = false;
database.error = redactText(error instanceof Error ? error.message : String(error), 300);
}
return {
collectedAt: now.toISOString(),
thresholds: {
apiSlowRequestMs: resolveApiSlowRequestThreshold(),
prismaSlowQueryMs: resolvePrismaSlowQueryThreshold(),
},
database,
slowRequests: events.slowRequests,
slowQueries: events.slowQueries,
jobQueue: buildJobQueue(jobRows),
dirtySummaryCount,
access: {
requiredPermission: OPS_VIEW_PERMISSION,
backendEnforced: false,
adapter: 'OpsPermissionAdapter',
},
};
}
}
function buildJobQueue(rows: BackgroundJobRow[]) {
const totals = emptyStatusCounts();
const byType = new Map<string, ReturnType<typeof emptyTypeCounts>>();
const recentFailures = rows
.filter((row) => normalizeStatus(row.status) === 'failed')
.slice(0, 10)
.map((row) => ({
id: row.id,
type: row.type,
attempts: row.attempts ?? 0,
maxAttempts: row.maxAttempts ?? 0,
lastError: redactText(row.lastError ?? '', 300),
updatedAt: toIso(row.updatedAt),
}));
for (const row of rows) {
const status = normalizeStatus(row.status);
totals[status] += 1;
totals.total += 1;
const current = byType.get(row.type) ?? emptyTypeCounts(row.type);
current[status] += 1;
current.total += 1;
if (status === 'queued') {
current.oldestQueuedAt = minIso(current.oldestQueuedAt, toIso(row.availableAt));
}
if (status === 'running') {
current.nextLeaseExpiresAt = minIso(current.nextLeaseExpiresAt, toIso(row.lockedUntil));
}
byType.set(row.type, current);
}
return {
totals,
byType: Array.from(byType.values()).sort((a, b) => b.total - a.total || a.type.localeCompare(b.type)),
recentFailures,
};
}
function emptyStatusCounts(): Record<JobStatus, number> & { total: number } {
return { queued: 0, running: 0, succeeded: 0, failed: 0, total: 0 };
}
function emptyTypeCounts(type: string) {
return {
type,
queued: 0,
running: 0,
succeeded: 0,
failed: 0,
total: 0,
oldestQueuedAt: undefined as string | undefined,
nextLeaseExpiresAt: undefined as string | undefined,
};
}
function normalizeStatus(status: string): JobStatus {
return status === 'running' || status === 'succeeded' || status === 'failed' ? status : 'queued';
}
function toIso(value: Date | string | null | undefined): string | undefined {
if (value instanceof Date) return Number.isFinite(value.getTime()) ? value.toISOString() : undefined;
if (typeof value !== 'string') return undefined;
const date = new Date(value);
return Number.isFinite(date.getTime()) ? date.toISOString() : undefined;
}
function minIso(current: string | undefined, next: string | undefined): string | undefined {
if (!next) return current;
if (!current) return next;
return next < current ? next : current;
}