496 lines
20 KiB
Java
496 lines
20 KiB
Java
package com.codechecker.cache.analyze;
|
||
|
||
import com.codechecker.cache.config.CheckerConfig;
|
||
import com.codechecker.cache.detector.RedisWritePointDetector;
|
||
import com.codechecker.cache.detector.WritePoint;
|
||
import com.codechecker.cache.diff.ChangeType;
|
||
import com.codechecker.cache.diff.SchemaChange;
|
||
import com.codechecker.cache.diff.SchemaDiffer;
|
||
import com.codechecker.cache.diff.Severity;
|
||
import com.codechecker.cache.git.GitDiffScanner;
|
||
import com.codechecker.cache.git.GitException;
|
||
import com.codechecker.cache.report.CheckReport;
|
||
import com.codechecker.cache.report.KeyStructureChange;
|
||
import com.codechecker.cache.schema.JavaSchemaExtractor;
|
||
import com.codechecker.cache.schema.SkeletonJsonRenderer;
|
||
import com.codechecker.cache.schema.SourceIndex;
|
||
import com.codechecker.cache.schema.TypeSchema;
|
||
import com.github.javaparser.StaticJavaParser;
|
||
import com.github.javaparser.ast.CompilationUnit;
|
||
import com.github.javaparser.ast.body.TypeDeclaration;
|
||
|
||
import java.nio.file.Path;
|
||
import java.util.ArrayList;
|
||
import java.util.Collections;
|
||
import java.util.HashSet;
|
||
import java.util.LinkedHashMap;
|
||
import java.util.LinkedHashSet;
|
||
import java.util.List;
|
||
import java.util.Map;
|
||
import java.util.Set;
|
||
import java.util.regex.Pattern;
|
||
|
||
/**
|
||
* 端到端分析编排：git diff → 定位写入点 → 双版本 Schema 提取 → 结构对比 → 生成报告。
|
||
*/
|
||
public class SchemaCheckAnalyzer {
|
||
|
||
private final CheckerConfig config;
|
||
private final Path repoRoot;
|
||
|
||
public SchemaCheckAnalyzer(CheckerConfig config, Path repoRoot) {
|
||
this.config = config;
|
||
this.repoRoot = repoRoot;
|
||
}
|
||
|
||
public CheckReport analyze(String oldSha, String newSha) throws GitException {
|
||
GitDiffScanner scanner = new GitDiffScanner(repoRoot);
|
||
|
||
List<String> changedRaw = scanner.changedJavaFiles(oldSha, newSha);
|
||
Set<String> changedFiles = new LinkedHashSet<>();
|
||
FileScanner fileScanner = new FileScanner(repoRoot, config.getIncludeModules(), config.getExcludeModules());
|
||
for (String path : changedRaw) {
|
||
if (path.contains("/src/main/java/") && fileScanner.moduleAllowed(path) && !isFileIgnored(path)) {
|
||
changedFiles.add(path);
|
||
}
|
||
}
|
||
|
||
// 当前工作树（= newSha 检出）内容
|
||
Map<String, String> newContents = fileScanner.scan();
|
||
|
||
// 旧版本内容：以工作树为基线，用 old 版本覆盖变更文件
|
||
Map<String, String> oldContents = new LinkedHashMap<>(newContents);
|
||
for (String path : changedFiles) {
|
||
String oldContent = scanner.fileContentAt(oldSha, path);
|
||
if (oldContent == null) {
|
||
oldContents.remove(path); // 新增文件在旧版本不存在
|
||
} else {
|
||
oldContents.put(path, oldContent);
|
||
}
|
||
}
|
||
|
||
SourceIndex newIndex = buildIndex(newContents.values());
|
||
SourceIndex oldIndex = buildIndex(oldContents.values());
|
||
|
||
// 变更涉及的类型简单名（用于扩展候选写入点文件）
|
||
Set<String> changedTypeNames = new HashSet<>();
|
||
for (String path : changedFiles) {
|
||
collectTypeNames(newContents.get(path), changedTypeNames);
|
||
collectTypeNames(oldContents.get(path), changedTypeNames);
|
||
}
|
||
|
||
Set<String> candidates = new LinkedHashSet<>(changedFiles);
|
||
Pattern typePattern = buildTypePattern(changedTypeNames);
|
||
if (typePattern != null) {
|
||
for (Map.Entry<String, String> e : newContents.entrySet()) {
|
||
if (!candidates.contains(e.getKey()) && typePattern.matcher(e.getValue()).find()) {
|
||
candidates.add(e.getKey());
|
||
}
|
||
}
|
||
}
|
||
|
||
Set<String> patterns = new HashSet<>(config.getDetection().getPatterns());
|
||
RedisWritePointDetector detectorNew = new RedisWritePointDetector(newIndex, patterns);
|
||
RedisWritePointDetector detectorOld = new RedisWritePointDetector(oldIndex, patterns);
|
||
JavaSchemaExtractor extractorNew = new JavaSchemaExtractor(newIndex, config.getDetection().getMaxFieldDepth());
|
||
JavaSchemaExtractor extractorOld = new JavaSchemaExtractor(oldIndex, config.getDetection().getMaxFieldDepth());
|
||
SchemaDiffer differ = new SchemaDiffer();
|
||
SkeletonJsonRenderer skeletonRenderer = new SkeletonJsonRenderer();
|
||
|
||
List<SchemaChange> allChanges = new ArrayList<>();
|
||
Map<String, KeyStructureChange> keyChanges = new LinkedHashMap<>();
|
||
|
||
for (String path : candidates) {
|
||
String newContent = newContents.get(path);
|
||
if (newContent == null) {
|
||
continue;
|
||
}
|
||
boolean fileChanged = changedFiles.contains(path);
|
||
String oldContent = fileChanged ? oldContents.get(path) : newContent;
|
||
|
||
List<WritePoint> newWps = detectorNew.detect(path, newContent);
|
||
List<WritePoint> oldWps = oldContent == null
|
||
? new ArrayList<>() : detectorOld.detect(path, oldContent);
|
||
newWps.forEach(this::applyManualMappings);
|
||
oldWps.forEach(this::applyManualMappings);
|
||
|
||
Map<String, WritePoint> oldBySig = new LinkedHashMap<>();
|
||
for (WritePoint wp : oldWps) {
|
||
oldBySig.put(wp.signature(), wp);
|
||
}
|
||
Set<String> newSigs = new HashSet<>();
|
||
|
||
for (WritePoint nw : newWps) {
|
||
newSigs.add(nw.signature());
|
||
if (isKeyIgnored(nw.getResolvedKeyPattern()) || isWriterIgnored(nw)) {
|
||
continue;
|
||
}
|
||
WritePoint ow = oldBySig.get(nw.signature());
|
||
if (ow != null) {
|
||
TypeSchema oldSchema = extractorOld.extract(ow.getResolvedValueType(), ow.isRootArray());
|
||
TypeSchema newSchema = extractorNew.extract(nw.getResolvedValueType(), nw.isRootArray());
|
||
List<SchemaChange> changes = differ.diff(oldSchema, newSchema);
|
||
if (changes.isEmpty()) {
|
||
continue;
|
||
}
|
||
double confidence = min(nw.getConfidence(), ow.getConfidence(),
|
||
oldSchema.getConfidence(), newSchema.getConfidence());
|
||
enrich(changes, nw, confidence);
|
||
allChanges.addAll(changes);
|
||
mergeKeyChange(keyChanges, nw, changes,
|
||
skeletonRenderer.render(oldSchema, protectedPaths(changes),
|
||
SkeletonJsonRenderer.DEFAULT_MAX_LEN),
|
||
skeletonRenderer.render(newSchema, protectedPaths(changes),
|
||
SkeletonJsonRenderer.DEFAULT_MAX_LEN));
|
||
} else if (fileChanged) {
|
||
SchemaChange c = new SchemaChange(ChangeType.WRITE_POINT_ADDED);
|
||
fillFromWritePoint(c, nw);
|
||
c.setMessage("新增缓存写入点，value 类型: " + displayType(nw));
|
||
allChanges.add(c);
|
||
TypeSchema newSchema = extractorNew.extract(nw.getResolvedValueType(), nw.isRootArray());
|
||
mergeKeyChange(keyChanges, nw,
|
||
Collections.singletonList(c),
|
||
"",
|
||
skeletonRenderer.render(newSchema, protectedPaths(
|
||
Collections.singletonList(c)),
|
||
SkeletonJsonRenderer.DEFAULT_MAX_LEN));
|
||
}
|
||
}
|
||
|
||
if (fileChanged) {
|
||
for (WritePoint ow : oldWps) {
|
||
if (!newSigs.contains(ow.signature())
|
||
&& !isKeyIgnored(ow.getResolvedKeyPattern()) && !isWriterIgnored(ow)) {
|
||
SchemaChange c = new SchemaChange(ChangeType.WRITE_POINT_REMOVED);
|
||
fillFromWritePoint(c, ow);
|
||
c.setMessage("删除缓存写入点，原 value 类型: " + displayType(ow));
|
||
allChanges.add(c);
|
||
TypeSchema oldSchema = extractorOld.extract(ow.getResolvedValueType(), ow.isRootArray());
|
||
mergeKeyChange(keyChanges, ow,
|
||
Collections.singletonList(c),
|
||
skeletonRenderer.render(oldSchema, protectedPaths(
|
||
Collections.singletonList(c)),
|
||
SkeletonJsonRenderer.DEFAULT_MAX_LEN),
|
||
"");
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
List<SchemaChange> finalChanges = postProcess(allChanges);
|
||
Map<String, KeyStructureChange> finalKeys = filterKeyChanges(keyChanges, finalChanges);
|
||
return buildReport(oldSha, newSha, finalChanges, finalKeys);
|
||
}
|
||
|
||
private void mergeKeyChange(Map<String, KeyStructureChange> keyChanges, WritePoint wp,
|
||
List<SchemaChange> changes, String oldSkeleton, String newSkeleton) {
|
||
String aggKey = aggregationKey(wp);
|
||
KeyStructureChange kc = keyChanges.computeIfAbsent(aggKey, k -> {
|
||
KeyStructureChange n = new KeyStructureChange();
|
||
n.setKeyPattern(wp.getResolvedKeyPattern());
|
||
n.setKeyExpression(wp.getKeyExpression());
|
||
n.setWriteLocation(wp.location());
|
||
n.setValueType(displayType(wp));
|
||
n.setKeyUnresolved(isUnresolvedKey(wp.getResolvedKeyPattern()));
|
||
return n;
|
||
});
|
||
if (kc.getWriteLocation() == null || kc.getWriteLocation().isEmpty()) {
|
||
kc.setWriteLocation(wp.location());
|
||
}
|
||
if (kc.getValueType() == null || kc.getValueType().isEmpty()) {
|
||
kc.setValueType(displayType(wp));
|
||
}
|
||
if (kc.getKeyExpression() == null || kc.getKeyExpression().isEmpty()) {
|
||
kc.setKeyExpression(wp.getKeyExpression());
|
||
}
|
||
kc.getFieldDetails().addAll(changes);
|
||
for (SchemaChange c : changes) {
|
||
kc.raiseSeverity(c.getSeverity());
|
||
}
|
||
if (oldSkeleton != null && !oldSkeleton.isEmpty()) {
|
||
kc.setOldSkeletonJson(oldSkeleton);
|
||
}
|
||
if (newSkeleton != null && !newSkeleton.isEmpty()) {
|
||
kc.setNewSkeletonJson(newSkeleton);
|
||
}
|
||
if (kc.getOldSkeletonJson() == null) {
|
||
kc.setOldSkeletonJson(oldSkeleton == null ? "" : oldSkeleton);
|
||
}
|
||
if (kc.getNewSkeletonJson() == null) {
|
||
kc.setNewSkeletonJson(newSkeleton == null ? "" : newSkeleton);
|
||
}
|
||
}
|
||
|
||
/** 已解析 key 按模式聚合；未解析按「位置+表达式」拆分，避免串单。 */
|
||
private String aggregationKey(WritePoint wp) {
|
||
String pattern = wp.getResolvedKeyPattern();
|
||
if (!isUnresolvedKey(pattern)) {
|
||
return pattern == null ? "<unknown>" : pattern;
|
||
}
|
||
return "unknown|" + nvl(wp.location()) + "|" + nvl(wp.getKeyExpression());
|
||
}
|
||
|
||
private boolean isUnresolvedKey(String keyPattern) {
|
||
return keyPattern == null
|
||
|| keyPattern.isEmpty()
|
||
|| "unknown-key".equals(keyPattern)
|
||
|| "<unknown>".equals(keyPattern);
|
||
}
|
||
|
||
private String displayType(WritePoint wp) {
|
||
String simple = shortType(wp.getResolvedValueType());
|
||
if (wp.isRootArray()) {
|
||
return "List<" + simple + ">";
|
||
}
|
||
return simple;
|
||
}
|
||
|
||
private void fillFromWritePoint(SchemaChange c, WritePoint wp) {
|
||
c.setKeyPattern(wp.getResolvedKeyPattern());
|
||
c.setKeyExpression(wp.getKeyExpression());
|
||
c.setWriteLocation(wp.location());
|
||
c.setValueType(displayType(wp));
|
||
}
|
||
|
||
private String nvl(String s) {
|
||
return s == null ? "" : s;
|
||
}
|
||
|
||
private Set<String> protectedPaths(List<SchemaChange> changes) {
|
||
Set<String> paths = new LinkedHashSet<>();
|
||
for (SchemaChange c : changes) {
|
||
if (c.getFieldPath() != null && !c.getFieldPath().isEmpty()) {
|
||
paths.add(c.getFieldPath());
|
||
}
|
||
// 路径迁移时 oldValue 为旧路径
|
||
if (c.getChangeType() == ChangeType.FIELD_PATH_MOVED
|
||
&& c.getOldValue() != null && !c.getOldValue().isEmpty()) {
|
||
paths.add(c.getOldValue());
|
||
}
|
||
if (c.getChangeType() == ChangeType.WRAPPER_ADDED
|
||
&& c.getFieldPath() != null) {
|
||
paths.add(c.getFieldPath());
|
||
}
|
||
}
|
||
return paths;
|
||
}
|
||
|
||
private Map<String, KeyStructureChange> filterKeyChanges(
|
||
Map<String, KeyStructureChange> keyChanges, List<SchemaChange> finalChanges) {
|
||
Set<String> liveDedup = new HashSet<>();
|
||
for (SchemaChange c : finalChanges) {
|
||
liveDedup.add(changeDedupKey(c));
|
||
}
|
||
Map<String, KeyStructureChange> result = new LinkedHashMap<>();
|
||
for (Map.Entry<String, KeyStructureChange> e : keyChanges.entrySet()) {
|
||
KeyStructureChange kc = e.getValue();
|
||
List<SchemaChange> retained = new ArrayList<>();
|
||
Set<String> seen = new HashSet<>();
|
||
for (SchemaChange c : kc.getFieldDetails()) {
|
||
String dk = changeDedupKey(c);
|
||
if (liveDedup.contains(dk) && seen.add(dk)) {
|
||
retained.add(c);
|
||
}
|
||
}
|
||
if (retained.isEmpty()) {
|
||
continue;
|
||
}
|
||
kc.getFieldDetails().clear();
|
||
kc.getFieldDetails().addAll(retained);
|
||
// 从明细回填通用展示字段（若聚合时未带上）
|
||
for (SchemaChange c : retained) {
|
||
if ((kc.getWriteLocation() == null || kc.getWriteLocation().isEmpty())
|
||
&& c.getWriteLocation() != null) {
|
||
kc.setWriteLocation(c.getWriteLocation());
|
||
}
|
||
if ((kc.getValueType() == null || kc.getValueType().isEmpty())
|
||
&& c.getValueType() != null) {
|
||
kc.setValueType(c.getValueType());
|
||
}
|
||
if ((kc.getKeyExpression() == null || kc.getKeyExpression().isEmpty())
|
||
&& c.getKeyExpression() != null) {
|
||
kc.setKeyExpression(c.getKeyExpression());
|
||
}
|
||
}
|
||
Severity max = Severity.P2;
|
||
for (SchemaChange c : retained) {
|
||
if (c.getSeverity() != null && c.getSeverity().ordinal() < max.ordinal()) {
|
||
max = c.getSeverity();
|
||
}
|
||
}
|
||
kc.setSeverity(max);
|
||
result.put(e.getKey(), kc);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
private String changeDedupKey(SchemaChange c) {
|
||
return c.getChangeType() + "|" + c.getKeyPattern() + "|"
|
||
+ c.getWriteLocation() + "|" + c.getFieldPath();
|
||
}
|
||
|
||
private void enrich(List<SchemaChange> changes, WritePoint wp, double confidence) {
|
||
boolean lowConfidence = confidence < config.getDetection().getMinConfidence();
|
||
String type = displayType(wp);
|
||
for (SchemaChange c : changes) {
|
||
fillFromWritePoint(c, wp);
|
||
c.setValueType(type);
|
||
if (lowConfidence) {
|
||
c.setSeverity(Severity.P2);
|
||
c.setMessage(c.getMessage() + "（低置信度，建议人工确认）");
|
||
}
|
||
}
|
||
}
|
||
|
||
private List<SchemaChange> postProcess(List<SchemaChange> changes) {
|
||
List<SchemaChange> result = new ArrayList<>();
|
||
Set<String> seen = new HashSet<>();
|
||
for (SchemaChange c : changes) {
|
||
applySeverityOverride(c);
|
||
if (isSuppressed(c)) {
|
||
continue;
|
||
}
|
||
String dedupKey = c.getChangeType() + "|" + c.getKeyPattern() + "|"
|
||
+ c.getWriteLocation() + "|" + c.getFieldPath();
|
||
if (seen.add(dedupKey)) {
|
||
result.add(c);
|
||
}
|
||
}
|
||
return result;
|
||
}
|
||
|
||
private void applySeverityOverride(SchemaChange c) {
|
||
String override = config.getSeverityOverrides().get(c.getChangeType().name());
|
||
if (override != null) {
|
||
try {
|
||
c.setSeverity(Severity.valueOf(override.trim().toUpperCase()));
|
||
} catch (IllegalArgumentException ignored) {
|
||
// 无效级别忽略
|
||
}
|
||
}
|
||
}
|
||
|
||
private boolean isSuppressed(SchemaChange c) {
|
||
for (CheckerConfig.Suppression s : config.getSuppressions()) {
|
||
boolean keyMatch = s.getKeyPattern() == null
|
||
|| s.getKeyPattern().equals(c.getKeyPattern());
|
||
boolean typeMatch = s.getChangeTypes() == null || s.getChangeTypes().isEmpty()
|
||
|| s.getChangeTypes().contains(c.getChangeType().name());
|
||
if (keyMatch && typeMatch && (s.getKeyPattern() != null
|
||
|| (s.getChangeTypes() != null && !s.getChangeTypes().isEmpty()))) {
|
||
return true;
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
private CheckReport buildReport(String oldSha, String newSha, List<SchemaChange> changes,
|
||
Map<String, KeyStructureChange> keyChanges) {
|
||
CheckReport report = new CheckReport();
|
||
report.setOldSha(oldSha);
|
||
report.setNewSha(newSha);
|
||
report.setMode(config.getMode());
|
||
report.getChanges().addAll(changes);
|
||
report.getKeyChanges().addAll(keyChanges.values());
|
||
|
||
boolean blocked = config.isBlockMode() && !changes.isEmpty();
|
||
report.setBlocked(blocked);
|
||
report.setExitCode(blocked ? 1 : 0);
|
||
return report;
|
||
}
|
||
|
||
private SourceIndex buildIndex(Iterable<String> contents) {
|
||
SourceIndex index = new SourceIndex();
|
||
List<String> list = new ArrayList<>();
|
||
for (String content : contents) {
|
||
list.add(content);
|
||
}
|
||
index.addSources(list);
|
||
return index;
|
||
}
|
||
|
||
private void applyManualMappings(WritePoint wp) {
|
||
String location = wp.getEnclosingClass() + "#" + wp.getEnclosingMethod();
|
||
for (CheckerConfig.ManualMapping mapping : config.getManualMappings()) {
|
||
if (mapping.getWriterMethod() == null || !mapping.getWriterMethod().equals(location)) {
|
||
continue;
|
||
}
|
||
if (mapping.getKeyPattern() != null && !mapping.getKeyPattern().isEmpty()) {
|
||
wp.setResolvedKeyPattern(mapping.getKeyPattern());
|
||
}
|
||
if (mapping.getValueType() != null && !mapping.getValueType().isEmpty()) {
|
||
wp.setResolvedValueType(mapping.getValueType());
|
||
wp.setConfidence(Math.max(wp.getConfidence(), 1.0));
|
||
}
|
||
}
|
||
}
|
||
|
||
private void collectTypeNames(String content, Set<String> out) {
|
||
if (content == null || content.isEmpty()) {
|
||
return;
|
||
}
|
||
try {
|
||
CompilationUnit cu = StaticJavaParser.parse(content);
|
||
for (TypeDeclaration<?> type : cu.findAll(TypeDeclaration.class)) {
|
||
out.add(type.getNameAsString());
|
||
}
|
||
} catch (RuntimeException ignored) {
|
||
// 解析失败跳过
|
||
}
|
||
}
|
||
|
||
private Pattern buildTypePattern(Set<String> typeNames) {
|
||
List<String> valid = new ArrayList<>();
|
||
for (String name : typeNames) {
|
||
if (name != null && name.matches("[A-Za-z_][A-Za-z0-9_]*")) {
|
||
valid.add(Pattern.quote(name));
|
||
}
|
||
}
|
||
if (valid.isEmpty()) {
|
||
return null;
|
||
}
|
||
return Pattern.compile("\\b(" + String.join("|", valid) + ")\\b");
|
||
}
|
||
|
||
private boolean isFileIgnored(String path) {
|
||
for (String glob : config.getIgnore().getFilePatterns()) {
|
||
if (GlobMatcher.matches(glob, path)) {
|
||
return true;
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
private boolean isKeyIgnored(String keyPattern) {
|
||
if (keyPattern == null) {
|
||
return false;
|
||
}
|
||
for (String glob : config.getIgnore().getKeyPatterns()) {
|
||
if (GlobMatcher.matches(glob, keyPattern)) {
|
||
return true;
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
private boolean isWriterIgnored(WritePoint wp) {
|
||
String sig = wp.getEnclosingClass() + "#" + wp.getEnclosingMethod();
|
||
return config.getIgnore().getWriterMethods().contains(sig);
|
||
}
|
||
|
||
private String shortType(String fqn) {
|
||
if (fqn == null) {
|
||
return "<未解析>";
|
||
}
|
||
return fqn.contains(".") ? fqn.substring(fqn.lastIndexOf('.') + 1) : fqn;
|
||
}
|
||
|
||
private double min(double... values) {
|
||
double m = 1.0;
|
||
for (double v : values) {
|
||
m = Math.min(m, v);
|
||
}
|
||
return m;
|
||
}
|
||
}
|