From 60783afa2573f5043469d02727eaa86e7765980c Mon Sep 17 00:00:00 2001 From: Alex Mabe Date: Wed, 12 Aug 2026 11:17:25 -0400 Subject: [PATCH] fix(core): enforce analysis input limits --- CHANGELOG.md | 4 + crates/flowscope-core/src/analyzer.rs | 6 +- crates/flowscope-core/src/analyzer/input.rs | 152 ++++++++++++++++++++ crates/flowscope-core/src/analyzer/tests.rs | 59 ++++++++ crates/flowscope-core/src/lib.rs | 1 + crates/flowscope-core/src/limits.rs | 12 ++ crates/flowscope-core/src/types/request.rs | 14 +- crates/flowscope-core/src/types/response.rs | 7 +- docs/api_schema.json | 8 +- docs/core-engine-spec.md | 8 ++ packages/core/src/generated/api-types.ts | 18 ++- 11 files changed, 280 insertions(+), 9 deletions(-) create mode 100644 crates/flowscope-core/src/limits.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e630f98..1ff480f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +#### Core Engine (flowscope-core) + +- **Bounded analysis inputs** — enforced UTF-8 byte limits before templating and parsing for inline and multi-file requests, with a 10 MiB per-source limit, a 100 MiB aggregate limit, and structured source-attributed errors + #### TypeScript API, Web App, and VS Code Extension - **Stable React store subscriptions** — added selector overloads for lineage state and actions, stabilized the legacy `useLineage()` object references, and prevented prop synchronization effects from rerunning on unrelated store updates diff --git a/crates/flowscope-core/src/analyzer.rs b/crates/flowscope-core/src/analyzer.rs index 6c606b5d..46ba3b95 100644 --- a/crates/flowscope-core/src/analyzer.rs +++ b/crates/flowscope-core/src/analyzer.rs @@ -37,7 +37,7 @@ use descriptions::DescriptionKey; use helpers::{ build_column_schemas_with_constraints, find_identifier_span, find_relation_occurrence_spans, }; -use input::{collect_statements, StatementInput}; +use input::{collect_statements, validate_analysis_input_sizes, StatementInput}; use schema_registry::SchemaRegistry; use statements::{ detect_dbt_model_materialization, extract_model_name, DbtMaterializationDetection, @@ -51,6 +51,10 @@ pub(crate) use schema_registry::TableResolution; /// Main entry point for SQL analysis #[must_use] pub fn analyze(request: &AnalyzeRequest) -> AnalyzeResult { + if let Err(issue) = validate_analysis_input_sizes(request) { + return AnalyzeResult::from_issue(*issue); + } + #[cfg(feature = "tracing")] let _span = info_span!("analyze_request", statement_count = %request.sql.matches(';').count() + 1) diff --git a/crates/flowscope-core/src/analyzer/input.rs b/crates/flowscope-core/src/analyzer/input.rs index aef6a8df..e9d43c49 100644 --- a/crates/flowscope-core/src/analyzer/input.rs +++ b/crates/flowscope-core/src/analyzer/input.rs @@ -3,6 +3,7 @@ //! This module handles the parsing and collection of SQL statements from analysis requests, //! supporting both file-based and inline SQL inputs. +use crate::limits::{MAX_ANALYSIS_SOURCE_BYTES, MAX_ANALYSIS_TOTAL_BYTES}; use crate::parser::{parse_sql_with_dialect, parse_sql_with_dialect_output}; use crate::types::{issue_codes, AnalyzeRequest, Dialect, Issue, Span}; use sqlparser::ast::Statement; @@ -20,6 +21,115 @@ use crate::templater::{template_sql, TemplateMode}; /// on malformed SQL input. const MAX_MERGE_ITERATIONS: usize = 10_000; +#[derive(Clone, Copy)] +enum AnalysisSource<'a> { + Inline(Option<&'a str>), + File(&'a str), +} + +impl<'a> AnalysisSource<'a> { + fn name(self) -> Option<&'a str> { + match self { + Self::Inline(name) => name, + Self::File(name) => Some(name), + } + } + + fn description(self) -> String { + match self { + Self::Inline(Some(name)) | Self::File(name) => format!("SQL source \"{name}\""), + Self::Inline(None) => "Inline SQL".to_string(), + } + } +} + +#[derive(Clone, Copy)] +struct AnalysisSourceSize<'a> { + source: AnalysisSource<'a>, + bytes: usize, +} + +/// Validates raw SQL byte sizes before schema initialization, templating, or parsing. +pub(super) fn validate_analysis_input_sizes(request: &AnalyzeRequest) -> Result<(), Box> { + validate_analysis_input_sizes_with_limits( + request, + MAX_ANALYSIS_SOURCE_BYTES, + MAX_ANALYSIS_TOTAL_BYTES, + ) +} + +fn validate_analysis_input_sizes_with_limits( + request: &AnalyzeRequest, + max_source_bytes: usize, + max_total_bytes: usize, +) -> Result<(), Box> { + let inline = std::iter::once(AnalysisSourceSize { + source: AnalysisSource::Inline(request.source_name.as_deref()), + bytes: request.sql.len(), + }); + let files = request + .files + .iter() + .flatten() + .map(|file| AnalysisSourceSize { + source: AnalysisSource::File(&file.name), + bytes: file.content.len(), + }); + + validate_analysis_source_sizes(inline.chain(files), max_source_bytes, max_total_bytes) +} + +fn validate_analysis_source_sizes<'a>( + sources: impl IntoIterator>, + max_source_bytes: usize, + max_total_bytes: usize, +) -> Result<(), Box> { + let mut total_bytes = 0usize; + + for source in sources { + if source.bytes > max_source_bytes { + let mut issue = Issue::error( + issue_codes::INVALID_REQUEST, + format!( + "{} exceeds the maximum analysis source size of {} bytes ({} bytes provided)", + source.source.description(), + max_source_bytes, + source.bytes + ), + ); + if let Some(name) = source.source.name() { + issue = issue.with_source_name(name); + } + return Err(Box::new(issue)); + } + + total_bytes = match total_bytes.checked_add(source.bytes) { + Some(total) => total, + None => { + return Err(Box::new(Issue::error( + issue_codes::INVALID_REQUEST, + format!( + "Aggregate SQL input exceeds the maximum analysis size of {} bytes", + max_total_bytes + ), + ))); + } + }; + + if total_bytes > max_total_bytes { + return Err(Box::new(Issue::error( + issue_codes::INVALID_REQUEST, + format!( + "Aggregate SQL input exceeds the maximum analysis size of {} bytes ({} bytes provided)", + max_total_bytes, total_bytes + ), + ))); + } + } + + Ok(()) +} + /// Creates an issue for a template rendering error. #[cfg(feature = "templating")] fn template_error_issue( @@ -981,6 +1091,48 @@ mod tests { } } + #[test] + fn analysis_source_size_limit_uses_utf8_bytes_and_is_inclusive() { + let mut request = base_request(); + request.sql = "é".repeat(5); + assert_eq!(request.sql.chars().count(), 5); + assert_eq!(request.sql.len(), 10); + assert!(validate_analysis_input_sizes_with_limits(&request, 10, 100).is_ok()); + + request.sql.push('x'); + let issue = + validate_analysis_input_sizes_with_limits(&request, 10, 100).expect_err("oversized"); + assert_eq!(issue.code, issue_codes::INVALID_REQUEST); + assert!(issue.message.contains("11 bytes provided")); + } + + #[test] + fn aggregate_limit_includes_inline_and_multibyte_multi_file_sources() { + let mut request = base_request(); + request.sql = "é".repeat(2); + request.files = Some(vec![ + crate::types::FileSource { + name: "first.sql".to_string(), + content: "日".repeat(2), + }, + crate::types::FileSource { + name: "second.sql".to_string(), + content: "SELECT 1".to_string(), + }, + ]); + assert_eq!(request.sql.len(), 4); + assert_eq!(request.files.as_ref().unwrap()[0].content.len(), 6); + assert_eq!(request.files.as_ref().unwrap()[1].content.len(), 8); + assert!(validate_analysis_input_sizes_with_limits(&request, 10, 18).is_ok()); + + request.files.as_mut().unwrap()[1].content.push('é'); + let issue = + validate_analysis_input_sizes_with_limits(&request, 10, 18).expect_err("oversized"); + assert_eq!(issue.code, issue_codes::INVALID_REQUEST); + assert!(issue.message.contains("Aggregate SQL input")); + assert!(issue.message.contains("20 bytes provided")); + } + #[test] fn collects_file_and_inline_statements() { let mut request = base_request(); diff --git a/crates/flowscope-core/src/analyzer/tests.rs b/crates/flowscope-core/src/analyzer/tests.rs index b152a858..ba677995 100644 --- a/crates/flowscope-core/src/analyzer/tests.rs +++ b/crates/flowscope-core/src/analyzer/tests.rs @@ -1,11 +1,15 @@ use super::*; use crate::test_utils::{load_schema_fixture, load_sql_fixture}; use crate::{ + limits::MAX_ANALYSIS_SOURCE_BYTES, types::{AnalysisOptions, LintConfidence, LintFallbackSource}, LintConfig, }; use std::collections::{BTreeSet, HashMap, HashSet}; +#[cfg(feature = "templating")] +use crate::templater::{TemplateConfig, TemplateMode}; + fn make_request(sql: &str) -> AnalyzeRequest { AnalyzeRequest { sql: sql.to_string(), @@ -33,6 +37,61 @@ fn make_request_with_options( request } +#[test] +fn analyze_rejects_oversized_inline_sql_before_templating_or_parsing() { + let mut sql = "{{ unclosed ".to_string(); + sql.push_str(&"x".repeat(MAX_ANALYSIS_SOURCE_BYTES + 1 - sql.len())); + let mut request = make_request(&sql); + #[cfg(feature = "templating")] + { + request.template_config = Some(TemplateConfig { + mode: TemplateMode::Jinja, + context: HashMap::new(), + }); + } + let result = analyze(&request); + + assert!(result.statements.is_empty()); + assert!(result.nodes.is_empty()); + assert_eq!(result.issues.len(), 1); + assert_eq!(result.issues[0].code, issue_codes::INVALID_REQUEST); + assert!(result.issues[0].message.contains("Inline SQL")); + assert!(result.summary.has_errors); +} + +#[test] +fn analyze_uses_utf8_bytes_for_inline_size_limit() { + let sql = "é".repeat(MAX_ANALYSIS_SOURCE_BYTES / "é".len() + 1); + assert!(sql.chars().count() < MAX_ANALYSIS_SOURCE_BYTES); + assert!(sql.len() > MAX_ANALYSIS_SOURCE_BYTES); + + let result = analyze(&make_request(&sql)); + + assert_eq!(result.issues.len(), 1); + assert_eq!(result.issues[0].code, issue_codes::INVALID_REQUEST); + assert!(result.issues[0].message.contains(&sql.len().to_string())); +} + +#[test] +fn analyze_rejects_oversized_file_with_source_attribution() { + let mut request = make_request(""); + request.files = Some(vec![FileSource { + name: "oversized.sql".to_string(), + content: "x".repeat(MAX_ANALYSIS_SOURCE_BYTES + 1), + }]); + + let result = analyze(&request); + + assert!(result.statements.is_empty()); + assert_eq!(result.issues.len(), 1); + assert_eq!(result.issues[0].code, issue_codes::INVALID_REQUEST); + assert_eq!( + result.issues[0].source_name.as_deref(), + Some("oversized.sql") + ); + assert!(result.issues[0].message.contains("oversized.sql")); +} + fn schema_with_known_table() -> SchemaMetadata { SchemaMetadata { default_catalog: None, diff --git a/crates/flowscope-core/src/lib.rs b/crates/flowscope-core/src/lib.rs index eb5298ed..10ef317f 100644 --- a/crates/flowscope-core/src/lib.rs +++ b/crates/flowscope-core/src/lib.rs @@ -3,6 +3,7 @@ pub mod completion; pub mod error; pub mod extractors; pub mod generated; +mod limits; pub mod linter; pub mod parser; #[cfg(feature = "templating")] diff --git a/crates/flowscope-core/src/limits.rs b/crates/flowscope-core/src/limits.rs new file mode 100644 index 00000000..5f27e913 --- /dev/null +++ b/crates/flowscope-core/src/limits.rs @@ -0,0 +1,12 @@ +//! Resource limits for FlowScope analysis requests. + +/// Maximum UTF-8 size of one SQL source: 10 MiB. +/// +/// An inline `AnalyzeRequest::sql` value and each `FileSource::content` value +/// are separate sources. +pub(crate) const MAX_ANALYSIS_SOURCE_BYTES: usize = 10 * 1024 * 1024; + +/// Maximum aggregate UTF-8 size of all SQL sources in one analysis: 100 MiB. +/// +/// The total includes inline SQL plus every file's content. +pub(crate) const MAX_ANALYSIS_TOTAL_BYTES: usize = 100 * 1024 * 1024; diff --git a/crates/flowscope-core/src/types/request.rs b/crates/flowscope-core/src/types/request.rs index b423c230..3c55bf4d 100644 --- a/crates/flowscope-core/src/types/request.rs +++ b/crates/flowscope-core/src/types/request.rs @@ -13,13 +13,21 @@ pub use crate::templater::{TemplateConfig, TemplateError, TemplateMode}; /// /// This is the main entry point for the analysis API. It accepts SQL code along with /// optional dialect and schema information to produce accurate lineage graphs. +/// +/// Raw SQL is limited by UTF-8 byte length before templating or parsing: inline SQL and +/// each file may contain at most 10 MiB (10,485,760 bytes), and all sources combined +/// may contain at most 100 MiB (104,857,600 bytes). Inputs exactly at either limit are +/// accepted. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct AnalyzeRequest { - /// The SQL code to analyze (UTF-8 string, multi-statement supported) + /// Inline SQL to analyze (UTF-8 string, multi-statement supported). + /// + /// This may be empty when `files` contains at least one source. When both are + /// provided, file statements are analyzed first and inline statements last. pub sql: String, - /// Optional list of source files to analyze (alternative to single `sql` field) + /// Optional source files to analyze, either alone or together with inline `sql`. #[serde(default, skip_serializing_if = "Option::is_none")] pub files: Option>, @@ -80,7 +88,9 @@ pub struct StatementSplitRequest { #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct FileSource { + /// Source identifier used for grouping and issue attribution. pub name: String, + /// UTF-8 SQL content, subject to the per-source and aggregate analysis limits. pub content: String, } diff --git a/crates/flowscope-core/src/types/response.rs b/crates/flowscope-core/src/types/response.rs index 25a2873b..6425cd23 100644 --- a/crates/flowscope-core/src/types/response.rs +++ b/crates/flowscope-core/src/types/response.rs @@ -81,11 +81,16 @@ impl AnalyzeResult { /// Create an error result with a single issue. /// Useful for returning errors from WASM boundary or other entry points. pub fn from_error(code: impl Into, message: impl Into) -> Self { + Self::from_issue(Issue::error(code, message)) + } + + /// Create an error result from an existing structured error issue. + pub(crate) fn from_issue(issue: Issue) -> Self { Self { statements: Vec::new(), nodes: Vec::new(), edges: Vec::new(), - issues: vec![Issue::error(code, message)], + issues: vec![issue], summary: Summary { statement_count: 0, table_count: 0, diff --git a/docs/api_schema.json b/docs/api_schema.json index cf25e49e..72f5b1ed 100644 --- a/docs/api_schema.json +++ b/docs/api_schema.json @@ -2,15 +2,15 @@ "AnalyzeRequest": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "AnalyzeRequest", - "description": "A request to analyze SQL for data lineage.\n\nThis is the main entry point for the analysis API. It accepts SQL code along with\noptional dialect and schema information to produce accurate lineage graphs.", + "description": "A request to analyze SQL for data lineage.\n\nThis is the main entry point for the analysis API. It accepts SQL code along with\noptional dialect and schema information to produce accurate lineage graphs.\n\nRaw SQL is limited by UTF-8 byte length before templating or parsing: inline SQL and\neach file may contain at most 10 MiB (10,485,760 bytes), and all sources combined\nmay contain at most 100 MiB (104,857,600 bytes). Inputs exactly at either limit are\naccepted.", "type": "object", "properties": { "sql": { - "description": "The SQL code to analyze (UTF-8 string, multi-statement supported)", + "description": "Inline SQL to analyze (UTF-8 string, multi-statement supported).\n\nThis may be empty when `files` contains at least one source. When both are\nprovided, file statements are analyzed first and inline statements last.", "type": "string" }, "files": { - "description": "Optional list of source files to analyze (alternative to single `sql` field)", + "description": "Optional source files to analyze, either alone or together with inline `sql`.", "type": ["array", "null"], "items": { "$ref": "#/definitions/FileSource" @@ -68,9 +68,11 @@ "type": "object", "properties": { "name": { + "description": "Source identifier used for grouping and issue attribution.", "type": "string" }, "content": { + "description": "UTF-8 SQL content, subject to the per-source and aggregate analysis limits.", "type": "string" } }, diff --git a/docs/core-engine-spec.md b/docs/core-engine-spec.md index 064390ff..a6e921db 100644 --- a/docs/core-engine-spec.md +++ b/docs/core-engine-spec.md @@ -65,6 +65,14 @@ Analysis produces a single flat graph in `AnalyzeResult.nodes` / - Issues include severity, code, message, span, and statement index. - `Summary` includes counts (statements, tables, columns, joins), a complexity score, and per-severity issue counts. +## Analysis Input Limits + +- Limits use UTF-8 byte lengths, not Unicode character counts. +- Inline SQL and each file are separate sources, each limited to 10 MiB (10,485,760 bytes). +- Inline SQL and all file contents together are limited to 100 MiB (104,857,600 bytes). +- Values exactly at either limit are accepted. The aggregate includes every source even when both inline SQL and files are supplied. +- Size validation runs before schema initialization, templating, statement splitting, tokenization, and parsing. An oversized request returns an error issue with code `INVALID_REQUEST`; named sources also populate the issue's `sourceName`. + ## Performance Expectations - The engine favors deterministic behavior and stable output for identical inputs. diff --git a/packages/core/src/generated/api-types.ts b/packages/core/src/generated/api-types.ts index 01514ad9..d1a58fdd 100644 --- a/packages/core/src/generated/api-types.ts +++ b/packages/core/src/generated/api-types.ts @@ -14,14 +14,22 @@ * * This is the main entry point for the analysis API. It accepts SQL code along with * optional dialect and schema information to produce accurate lineage graphs. + * + * Raw SQL is limited by UTF-8 byte length before templating or parsing: inline SQL and + * each file may contain at most 10 MiB (10,485,760 bytes), and all sources combined + * may contain at most 100 MiB (104,857,600 bytes). Inputs exactly at either limit are + * accepted. */ export interface AnalyzeRequest { /** - * The SQL code to analyze (UTF-8 string, multi-statement supported) + * Inline SQL to analyze (UTF-8 string, multi-statement supported). + * + * This may be empty when `files` contains at least one source. When both are + * provided, file statements are analyzed first and inline statements last. */ sql: string; /** - * Optional list of source files to analyze (alternative to single `sql` field) + * Optional source files to analyze, either alone or together with inline `sql`. */ files?: FileSource[]; /** @@ -47,7 +55,13 @@ export interface AnalyzeRequest { } export interface FileSource { + /** + * Source identifier used for grouping and issue attribution. + */ name: string; + /** + * UTF-8 SQL content, subject to the per-source and aggregate analysis limits. + */ content: string; }