initial-commit

This commit is contained in:
Guillaume GRABÉ
2026-05-12 17:06:43 +02:00
commit 051a080dfa
110 changed files with 26377 additions and 0 deletions
+132
View File
@@ -0,0 +1,132 @@
use sha2::{Digest, Sha256};
use crate::scanners::NormalizedFinding;
/// Compute a stable fingerprint for deduplication.
///
/// Priority:
/// 1. Scanner-provided fingerprint (e.g., SARIF partialFingerprints) — most stable,
/// survives line shifts because scanners compute it from code content, not position.
/// 2. Fallback: hash of (scanner + rule_id + file_path) — deliberately excludes line number
/// to avoid ghost resolve/create when lines shift. Trade-off: if the same rule fires
/// twice in the same file, they collapse into one finding.
pub fn compute_fingerprint(
scanner: &str,
finding: &NormalizedFinding,
) -> String {
if let Some(ref fp) = finding.scanner_fingerprint {
return fp.clone();
}
let mut hasher = Sha256::new();
hasher.update(scanner.as_bytes());
hasher.update(b"|");
hasher.update(finding.rule_id.as_bytes());
hasher.update(b"|");
if let Some(ref path) = finding.file_path {
hasher.update(path.as_bytes());
}
let hash = hasher.finalize();
hex::encode(hash)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::scanners::{NormalizedFinding, Severity};
fn make_finding(rule_id: &str, file: Option<&str>, line: Option<u32>) -> NormalizedFinding {
NormalizedFinding {
title: "test".to_string(),
description: String::new(),
severity: Severity::Medium,
scanner: "test-scanner".to_string(),
rule_id: rule_id.to_string(),
file_path: file.map(|s| s.to_string()),
line_start: line,
line_end: None,
cwe: None,
cve: None,
cvss_score: None,
package_name: None,
package_version: None,
fixed_version: None,
details_url: None,
tags: vec![],
scanner_fingerprint: None,
}
}
#[test]
fn test_same_input_same_fingerprint() {
let f1 = make_finding("RULE-1", Some("src/main.rs"), Some(42));
let f2 = make_finding("RULE-1", Some("src/main.rs"), Some(42));
let fp1 = compute_fingerprint("scanner", &f1);
let fp2 = compute_fingerprint("scanner", &f2);
assert_eq!(fp1, fp2);
}
#[test]
fn test_line_shift_same_fingerprint() {
let f1 = make_finding("RULE-1", Some("src/main.rs"), Some(42));
let f2 = make_finding("RULE-1", Some("src/main.rs"), Some(99));
let fp1 = compute_fingerprint("scanner", &f1);
let fp2 = compute_fingerprint("scanner", &f2);
assert_eq!(fp1, fp2, "line number should NOT affect fallback fingerprint");
}
#[test]
fn test_different_rule_different_fingerprint() {
let f1 = make_finding("RULE-1", Some("src/main.rs"), Some(42));
let f2 = make_finding("RULE-2", Some("src/main.rs"), Some(42));
let fp1 = compute_fingerprint("scanner", &f1);
let fp2 = compute_fingerprint("scanner", &f2);
assert_ne!(fp1, fp2);
}
#[test]
fn test_different_file_different_fingerprint() {
let f1 = make_finding("RULE-1", Some("src/main.rs"), None);
let f2 = make_finding("RULE-1", Some("src/lib.rs"), None);
let fp1 = compute_fingerprint("scanner", &f1);
let fp2 = compute_fingerprint("scanner", &f2);
assert_ne!(fp1, fp2);
}
#[test]
fn test_scanner_fingerprint_takes_priority() {
let mut f1 = make_finding("RULE-1", Some("src/main.rs"), Some(42));
f1.scanner_fingerprint = Some("scanner-provided-hash-abc123".to_string());
let mut f2 = make_finding("RULE-1", Some("src/main.rs"), Some(99));
f2.scanner_fingerprint = Some("scanner-provided-hash-abc123".to_string());
let fp1 = compute_fingerprint("scanner", &f1);
let fp2 = compute_fingerprint("scanner", &f2);
assert_eq!(fp1, "scanner-provided-hash-abc123");
assert_eq!(fp1, fp2);
}
#[test]
fn test_no_file_path_stable() {
let f1 = make_finding("RULE-1", None, None);
let f2 = make_finding("RULE-1", None, None);
let fp1 = compute_fingerprint("scanner", &f1);
let fp2 = compute_fingerprint("scanner", &f2);
assert_eq!(fp1, fp2);
}
}
+246
View File
@@ -0,0 +1,246 @@
use std::sync::Arc;
use tonic::{Request, Response, Status};
use tracing::{error, info, instrument};
use uuid::Uuid;
use crate::charybdis::ingestion::ingestion_service_server::IngestionService;
use crate::charybdis::ingestion::{
DryRunScanRequest, DryRunScanResponse, FindingResult, ImportScanRequest, ImportScanResponse,
ReconciliationSummary,
};
use crate::database::EntityRepository;
use crate::findings::reconciler::{ReconciliationEngine, ReconciledFinding};
use crate::scanners::{ParserRegistry, Severity};
pub struct MyIngestionService {
repository: Arc<EntityRepository>,
reconciler: ReconciliationEngine,
parser_registry: Arc<ParserRegistry>,
}
impl MyIngestionService {
pub fn new(repository: Arc<EntityRepository>, parser_registry: Arc<ParserRegistry>) -> Self {
Self {
repository: repository.clone(),
reconciler: ReconciliationEngine::new(repository),
parser_registry,
}
}
/// Resolve a component_ref (UUID or entity name) to a stable UUID.
/// If it's already a valid UUID, use it directly.
/// Otherwise, search for an entity with that name.
async fn resolve_component_id(&self, component_ref: &str) -> Result<String, Status> {
if uuid::Uuid::parse_str(component_ref).is_ok() {
// Verify entity exists
self.repository
.get_by_id(component_ref)
.await
.map_err(|e| Status::internal(format!("Database error: {}", e)))?
.ok_or_else(|| {
Status::not_found(format!("Component not found: {}", component_ref))
})?;
return Ok(component_ref.to_string());
}
// O(1) lookup by kind+name
if let Some(entity) = self
.repository
.get_by_kind_and_name("Component", component_ref)
.await
.map_err(|e| Status::internal(format!("Database error: {}", e)))?
{
return Ok(entity.id.clone());
}
Err(Status::not_found(format!(
"Component not found: {}",
component_ref
)))
}
}
#[tonic::async_trait]
impl IngestionService for MyIngestionService {
#[instrument(skip(self, request), fields(component_ref, lifecycle, format))]
async fn import_scan(
&self,
request: Request<ImportScanRequest>,
) -> Result<Response<ImportScanResponse>, Status> {
let req = request.into_inner();
tracing::Span::current()
.record("component_ref", &req.component_ref)
.record("lifecycle", &req.lifecycle)
.record("format", &req.format);
info!(
component_ref = %req.component_ref,
lifecycle = %req.lifecycle,
format = %req.format,
"Received ImportScan request"
);
let component_id = self.resolve_component_id(&req.component_ref).await?;
let parser = self
.parser_registry
.get(&req.format)
.ok_or_else(|| Status::invalid_argument(format!("Unsupported format: {}", req.format)))?;
let report = parser
.parse(&req.data)
.map_err(|e| Status::invalid_argument(format!("Failed to parse scan data: {}", e)))?;
let scanner_name = if req.scanner_name.is_empty() {
report.scanner_name.clone()
} else {
req.scanner_name.clone()
};
let scan_id = if req.scan_id.is_empty() {
Uuid::new_v4().to_string()
} else {
req.scan_id.clone()
};
let result = self
.reconciler
.reconcile(&component_id, &req.lifecycle, &scanner_name, &report.findings)
.await
.map_err(|e| {
error!(error = %e, "Reconciliation failed");
Status::internal("Reconciliation failed")
})?;
self.reconciler
.apply(
&component_id,
&req.lifecycle,
&scanner_name,
&scan_id,
&result,
&report.findings,
)
.await
.map_err(|e| {
error!(error = %e, "Failed to apply reconciliation");
Status::internal("Failed to apply reconciliation result")
})?;
let summary = ReconciliationSummary {
total_parsed: report.findings.len() as u32,
new_count: result.new_findings.len() as u32,
unchanged_count: result.unchanged.len() as u32,
resolved_count: result.resolved.len() as u32,
reopened_count: result.reopened.len() as u32,
};
info!(
total = summary.total_parsed,
new = summary.new_count,
resolved = summary.resolved_count,
reopened = summary.reopened_count,
"ImportScan completed"
);
Ok(Response::new(ImportScanResponse {
summary: Some(summary),
new_findings: result.new_findings.iter().map(to_finding_result).collect(),
resolved_findings: result.resolved.iter().map(to_finding_result).collect(),
reopened_findings: result.reopened.iter().map(to_finding_result).collect(),
}))
}
#[instrument(skip(self, request), fields(component_ref, lifecycle, format))]
async fn dry_run_scan(
&self,
request: Request<DryRunScanRequest>,
) -> Result<Response<DryRunScanResponse>, Status> {
let req = request.into_inner();
tracing::Span::current()
.record("component_ref", &req.component_ref)
.record("lifecycle", &req.lifecycle)
.record("format", &req.format);
info!(
component_ref = %req.component_ref,
lifecycle = %req.lifecycle,
format = %req.format,
"Received DryRunScan request"
);
let component_id = self.resolve_component_id(&req.component_ref).await?;
let parser = self
.parser_registry
.get(&req.format)
.ok_or_else(|| Status::invalid_argument(format!("Unsupported format: {}", req.format)))?;
let report = parser
.parse(&req.data)
.map_err(|e| Status::invalid_argument(format!("Failed to parse scan data: {}", e)))?;
let scanner_name = if req.scanner_name.is_empty() {
report.scanner_name.clone()
} else {
req.scanner_name.clone()
};
let result = self
.reconciler
.reconcile(&component_id, &req.lifecycle, &scanner_name, &report.findings)
.await
.map_err(|e| {
error!(error = %e, "Reconciliation failed");
Status::internal("Reconciliation failed")
})?;
let summary = ReconciliationSummary {
total_parsed: report.findings.len() as u32,
new_count: result.new_findings.len() as u32,
unchanged_count: result.unchanged.len() as u32,
resolved_count: result.resolved.len() as u32,
reopened_count: result.reopened.len() as u32,
};
info!(
total = summary.total_parsed,
new = summary.new_count,
resolved = summary.resolved_count,
"DryRunScan completed (no changes persisted)"
);
Ok(Response::new(DryRunScanResponse {
summary: Some(summary),
new_findings: result.new_findings.iter().map(to_finding_result).collect(),
resolved_findings: result.resolved.iter().map(to_finding_result).collect(),
reopened_findings: result.reopened.iter().map(to_finding_result).collect(),
}))
}
}
fn to_finding_result(reconciled: &ReconciledFinding) -> FindingResult {
FindingResult {
title: reconciled.title.clone(),
severity: domain_severity_to_proto_i32(reconciled.severity),
rule_id: reconciled.rule_id.clone(),
file_path: reconciled.file_path.clone().unwrap_or_default(),
line_start: reconciled.line_start.unwrap_or(0),
fingerprint: reconciled.fingerprint.clone(),
entity_id: reconciled.entity_id.clone().unwrap_or_default(),
}
}
fn domain_severity_to_proto_i32(severity: Severity) -> i32 {
match severity {
Severity::Info => 1,
Severity::Low => 2,
Severity::Medium => 3,
Severity::High => 4,
Severity::Critical => 5,
}
}
+3
View File
@@ -0,0 +1,3 @@
pub mod fingerprint;
pub mod ingestion;
pub mod reconciler;
+362
View File
@@ -0,0 +1,362 @@
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use anyhow::Result;
use chrono::Utc;
use tracing::{info, instrument};
use crate::charybdis::core::{FindingMetadata, FindingSpec, FindingState, Severity as ProtoSeverity};
use crate::charybdis::entities::entity::{Metadata, Spec};
use crate::charybdis::entities::Entity;
use crate::database::EntityRepository;
use crate::findings::fingerprint::compute_fingerprint;
use crate::scanners::{NormalizedFinding, Severity};
/// Result of reconciliation — what changed between existing state and incoming scan
#[derive(Debug)]
pub struct ReconciliationResult {
pub new_findings: Vec<ReconciledFinding>,
pub unchanged: Vec<ReconciledFinding>,
pub resolved: Vec<ReconciledFinding>,
pub reopened: Vec<ReconciledFinding>,
}
#[derive(Debug, Clone)]
pub struct ReconciledFinding {
pub entity_id: Option<String>,
pub title: String,
pub severity: Severity,
pub rule_id: String,
pub file_path: Option<String>,
pub line_start: Option<u32>,
pub fingerprint: String,
}
pub struct ReconciliationEngine {
repository: Arc<EntityRepository>,
}
impl ReconciliationEngine {
pub fn new(repository: Arc<EntityRepository>) -> Self {
Self { repository }
}
/// Reconcile incoming findings against existing findings for a (component, lifecycle) scope.
/// Does NOT persist anything — caller decides whether to apply or return as dry-run.
#[instrument(skip(self, incoming), fields(component_ref, lifecycle, incoming_count = incoming.len()))]
pub async fn reconcile(
&self,
component_ref: &str,
lifecycle: &str,
scanner: &str,
incoming: &[NormalizedFinding],
) -> Result<ReconciliationResult> {
let existing = self.load_existing_findings(component_ref, lifecycle).await?;
let mut existing_by_fingerprint: HashMap<String, Entity> = HashMap::new();
for entity in existing {
if let Some(Spec::FindingSpec(ref spec)) = entity.spec {
existing_by_fingerprint.insert(spec.fingerprint.clone(), entity.clone());
}
}
let mut new_findings = Vec::new();
let mut unchanged = Vec::new();
let mut reopened = Vec::new();
let mut seen_fingerprints = HashSet::new();
for finding in incoming {
let fp = compute_fingerprint(scanner, finding);
seen_fingerprints.insert(fp.clone());
if let Some(existing_entity) = existing_by_fingerprint.get(&fp) {
let state = existing_entity
.spec
.as_ref()
.and_then(|s| match s {
Spec::FindingSpec(spec) => Some(spec.state),
_ => None,
})
.unwrap_or(0);
let reconciled = ReconciledFinding {
entity_id: Some(existing_entity.id.clone()),
title: finding.title.clone(),
severity: finding.severity,
rule_id: finding.rule_id.clone(),
file_path: finding.file_path.clone(),
line_start: finding.line_start,
fingerprint: fp,
};
if state == FindingState::Resolved as i32
|| state == FindingState::FalsePositive as i32
{
reopened.push(reconciled);
} else {
unchanged.push(reconciled);
}
} else {
new_findings.push(ReconciledFinding {
entity_id: None,
title: finding.title.clone(),
severity: finding.severity,
rule_id: finding.rule_id.clone(),
file_path: finding.file_path.clone(),
line_start: finding.line_start,
fingerprint: fp,
});
}
}
// Findings in DB that weren't in the incoming scan → resolved
let resolved: Vec<ReconciledFinding> = existing_by_fingerprint
.iter()
.filter(|(fp, _)| !seen_fingerprints.contains(*fp))
.filter(|(_, entity)| {
let state = entity
.spec
.as_ref()
.and_then(|s| match s {
Spec::FindingSpec(spec) => Some(spec.state),
_ => None,
})
.unwrap_or(0);
state == FindingState::Active as i32 || state == FindingState::Reopened as i32
})
.map(|(fp, entity)| {
let (title, severity, rule_id, file_path, line_start) = entity
.spec
.as_ref()
.map(|s| match s {
Spec::FindingSpec(spec) => (
entity
.metadata
.as_ref()
.and_then(|m| match m {
Metadata::FindingMetadata(meta) => Some(meta.title.clone()),
_ => None,
})
.unwrap_or_default(),
proto_severity_to_domain(spec.severity),
spec.rule_id.clone(),
if spec.file_path.is_empty() { None } else { Some(spec.file_path.clone()) },
if spec.line_start == 0 { None } else { Some(spec.line_start) },
),
_ => (String::new(), Severity::Medium, String::new(), None, None),
})
.unwrap_or((String::new(), Severity::Medium, String::new(), None, None));
ReconciledFinding {
entity_id: Some(entity.id.clone()),
title,
severity,
rule_id,
file_path,
line_start,
fingerprint: fp.clone(),
}
})
.collect();
info!(
new = new_findings.len(),
unchanged = unchanged.len(),
resolved = resolved.len(),
reopened = reopened.len(),
"Reconciliation complete"
);
Ok(ReconciliationResult {
new_findings,
unchanged,
resolved,
reopened,
})
}
/// Apply reconciliation result: create, update, and resolve findings in the database
#[instrument(skip(self, result, incoming), fields(component_ref, lifecycle))]
pub async fn apply(
&self,
component_ref: &str,
lifecycle: &str,
scanner: &str,
scan_id: &str,
result: &ReconciliationResult,
incoming: &[NormalizedFinding],
) -> Result<()> {
let now = Utc::now();
let timestamp = prost_types::Timestamp {
seconds: now.timestamp(),
nanos: now.timestamp_subsec_nanos() as i32,
};
// Create new findings
for reconciled in &result.new_findings {
let normalized = incoming
.iter()
.find(|f| compute_fingerprint(scanner, f) == reconciled.fingerprint);
if let Some(finding) = normalized {
let entity = build_finding_entity(
finding,
component_ref,
lifecycle,
scanner,
scan_id,
&reconciled.fingerprint,
&timestamp,
);
self.repository.create(&entity).await?;
}
}
// Update last_seen on unchanged findings
for reconciled in &result.unchanged {
if let Some(ref entity_id) = reconciled.entity_id {
let mut annotations = std::collections::HashMap::new();
annotations.insert(
"charybdis.io/last-seen".to_string(),
now.to_rfc3339(),
);
annotations.insert(
"charybdis.io/scan-id".to_string(),
scan_id.to_string(),
);
self.repository.update_annotations(entity_id, annotations).await?;
}
}
// Resolve findings no longer detected
for reconciled in &result.resolved {
if let Some(ref entity_id) = reconciled.entity_id {
self.mark_finding_state(entity_id, FindingState::Resolved).await?;
}
}
// Reopen findings detected again
for reconciled in &result.reopened {
if let Some(ref entity_id) = reconciled.entity_id {
self.mark_finding_state(entity_id, FindingState::Reopened).await?;
}
}
Ok(())
}
async fn load_existing_findings(
&self,
component_ref: &str,
lifecycle: &str,
) -> Result<Vec<Entity>> {
let finding_entities = self.repository.list_by_kind("Finding").await?;
let findings: Vec<Entity> = finding_entities
.into_iter()
.filter(|e| {
e.spec
.as_ref()
.map(|s| match s {
Spec::FindingSpec(spec) => {
spec.component_ref == component_ref && spec.lifecycle == lifecycle
}
_ => false,
})
.unwrap_or(false)
})
.collect();
Ok(findings)
}
async fn mark_finding_state(&self, entity_id: &str, state: FindingState) -> Result<()> {
let entity = self.repository.get_by_id(entity_id).await?;
if let Some(mut entity) = entity {
if let Some(Spec::FindingSpec(ref mut spec)) = entity.spec {
spec.state = state as i32;
if state == FindingState::Resolved {
spec.resolved_at = Some(prost_types::Timestamp {
seconds: Utc::now().timestamp(),
nanos: Utc::now().timestamp_subsec_nanos() as i32,
});
}
}
self.repository.update(entity_id, &entity).await?;
}
Ok(())
}
}
fn build_finding_entity(
finding: &NormalizedFinding,
component_ref: &str,
lifecycle: &str,
scanner: &str,
scan_id: &str,
fingerprint: &str,
timestamp: &prost_types::Timestamp,
) -> Entity {
let metadata = FindingMetadata {
title: finding.title.clone(),
namespace: "default".to_string(),
description: finding.description.clone(),
labels: std::collections::HashMap::new(),
tags: finding.tags.clone(),
};
let spec = FindingSpec {
component_ref: component_ref.to_string(),
lifecycle: lifecycle.to_string(),
severity: domain_severity_to_proto(finding.severity) as i32,
state: FindingState::Active as i32,
scanner: scanner.to_string(),
rule_id: finding.rule_id.clone(),
fingerprint: fingerprint.to_string(),
file_path: finding.file_path.clone().unwrap_or_default(),
line_start: finding.line_start.unwrap_or(0),
line_end: finding.line_end.unwrap_or(0),
cwe: finding.cwe.clone().unwrap_or_default(),
cve: finding.cve.clone().unwrap_or_default(),
cvss_score: finding.cvss_score.unwrap_or(0.0),
package_name: finding.package_name.clone().unwrap_or_default(),
package_version: finding.package_version.clone().unwrap_or_default(),
fixed_version: finding.fixed_version.clone().unwrap_or_default(),
details_url: finding.details_url.clone().unwrap_or_default(),
first_seen: Some(timestamp.clone()),
last_seen: Some(timestamp.clone()),
resolved_at: None,
scan_id: scan_id.to_string(),
};
Entity {
id: String::new(),
kind: "Finding".to_string(),
metadata: Some(Metadata::FindingMetadata(metadata)),
spec: Some(Spec::FindingSpec(spec)),
annotations: std::collections::HashMap::new(),
created_at: None,
updated_at: None,
}
}
fn domain_severity_to_proto(severity: Severity) -> ProtoSeverity {
match severity {
Severity::Info => ProtoSeverity::Info,
Severity::Low => ProtoSeverity::Low,
Severity::Medium => ProtoSeverity::Medium,
Severity::High => ProtoSeverity::High,
Severity::Critical => ProtoSeverity::Critical,
}
}
fn proto_severity_to_domain(proto: i32) -> Severity {
match proto {
1 => Severity::Info,
2 => Severity::Low,
3 => Severity::Medium,
4 => Severity::High,
5 => Severity::Critical,
_ => Severity::Medium,
}
}