diff --git a/crates/squawk_ide/src/semantic_tokens.rs b/crates/squawk_ide/src/semantic_tokens.rs index 1b16c7e5..6ceb38aa 100644 --- a/crates/squawk_ide/src/semantic_tokens.rs +++ b/crates/squawk_ide/src/semantic_tokens.rs @@ -1,11 +1,12 @@ use rowan::{NodeOrToken, TextRange}; +use rustc_hash::FxHashSet; use salsa::Database as Db; use squawk_syntax::{ - SyntaxElement, SyntaxKind, + SyntaxElement, SyntaxKind, SyntaxNode, ast::{self, AstNode}, }; -use crate::db::{File, parse}; +use crate::db::{EmbeddedFile, File, FileId, embedded_body, parse}; use crate::file::InFile; use crate::goto_definition::goto_definition; use crate::location::LocationKind; @@ -270,6 +271,26 @@ impl TryFrom for SemanticTokenType { } } +fn embedded_token_type(kind: SyntaxKind) -> Option { + match kind { + SyntaxKind::TRUE_KW | SyntaxKind::FALSE_KW => Some(SemanticTokenType::Bool), + SyntaxKind::BIT_STRING + | SyntaxKind::BYTE_STRING + | SyntaxKind::DOLLAR_QUOTED_STRING + | SyntaxKind::ESC_STRING + | SyntaxKind::NATIONAL_STRING + | SyntaxKind::STRING + | SyntaxKind::UNICODE_ESC_STRING => Some(SemanticTokenType::String), + SyntaxKind::INT_NUMBER | SyntaxKind::NUMERIC_NUMBER => Some(SemanticTokenType::Number), + SyntaxKind::COMMENT => Some(SemanticTokenType::Comment), + SyntaxKind::IDENT => Some(SemanticTokenType::NameRef), + _ if kind.is_keyword() => Some(SemanticTokenType::Keyword), + _ if kind.is_punctuation() => Some(SemanticTokenType::Punctuation), + _ if kind.is_operator() => Some(SemanticTokenType::Operator), + _ => None, + } +} + fn token_type_for_node(db: &dyn Db, node: InFile<&T>) -> Option { let offset = node.value.syntax().text_range().start(); let location = goto_definition(db, InFile::new(node.file_id, offset)) @@ -282,6 +303,7 @@ fn token_type_for_node(db: &dyn Db, node: InFile<&T>) -> Option, + highlighted_ranges: FxHashSet, } impl SemanticTokenBuilder { @@ -299,13 +321,22 @@ impl SemanticTokenBuilder { self.push_token(syntax_element, SemanticTokenType::Type); } + fn contains_range(&self, range: TextRange) -> bool { + self.highlighted_ranges.contains(&range) + } + fn push_token(&mut self, syntax_element: SyntaxElement, token_type: SemanticTokenType) { - self.tokens.push(SemanticToken { + self.push(SemanticToken { range: syntax_element.text_range(), token_type, modifiers: None, }); } + + fn push(&mut self, token: SemanticToken) { + self.highlighted_ranges.insert(token.range); + self.tokens.push(token); + } } #[salsa::tracked] @@ -334,7 +365,17 @@ pub fn semantic_tokens( }; let mut out = SemanticTokenBuilder::default(); + highlight(db, file.into(), &root, range_to_highlight, &mut out); + out.build() +} +fn highlight( + db: &dyn Db, + file: FileId, + root: &SyntaxNode, + range_to_highlight: TextRange, + out: &mut SemanticTokenBuilder, +) { // Taken from: https://github.com/rust-lang/rust-analyzer/blob/2efc80078029894eec0699f62ec8d5c1a56af763/crates/ide/src/syntax_highlighting.rs#L267C21-L267C21 let preorder = root.preorder_with_tokens(); for event in preorder { @@ -358,11 +399,15 @@ pub fn semantic_tokens( } if let Some(ty) = ast::Type::cast(node.clone()) { - highlight_type(&mut out, ty); + highlight_type(out, ty); } if let Some(mode) = ast::ParamMode::cast(node.clone()) { - highlight_param_mode(&mut out, mode); + highlight_param_mode(out, mode); + } + + if let Some(literal) = ast::Literal::cast(node.clone()) { + highlight_embedded(db, file, literal, range_to_highlight, out); } // Cleanup various operators that the textmate grammar @@ -390,13 +435,50 @@ pub fn semantic_tokens( } if token.kind() == SyntaxKind::POSITIONAL_PARAM { out.push_token(token.into(), SemanticTokenType::PositionalParam); + } else if matches!(file, FileId::Embedded(_)) + && !out.contains_range(token.text_range()) + && let Some(token_type) = embedded_token_type(token.kind()) + { + out.push_token(token.into(), token_type); } } Leave(_) => {} } } +} - out.build() +fn highlight_embedded( + db: &dyn Db, + file: FileId, + literal: ast::Literal, + range_to_highlight: TextRange, + out: &mut SemanticTokenBuilder, +) { + let Some(parent) = literal.syntax().parent() else { + return; + }; + if !ast::AsDefinition::can_cast(parent.kind()) { + return; + } + let embedded = EmbeddedFile::new(db, file, literal.syntax().text_range()); + let Some(body) = embedded_body(db, embedded) else { + return; + }; + let root = body.syntax(); + let mut embedded_out = SemanticTokenBuilder::default(); + highlight( + db, + FileId::Embedded(embedded), + &root, + root.text_range(), + &mut embedded_out, + ); + for token in embedded_out.tokens { + let range = body.source_range(token.range); + if range_to_highlight.intersect(range).is_some() { + out.push(SemanticToken { range, ..token }); + } + } } #[cfg(test)] @@ -457,6 +539,10 @@ language sql; "d" @ 103..104: Parameter "int" @ 105..108: Type "int" @ 121..124: Type + "select" @ 129..135: Keyword + "$1" @ 136..138: PositionalParam + "+" @ 139..140: Operator + "$2" @ 141..143: PositionalParam "#); } @@ -682,6 +768,8 @@ language sql; "f" @ 17..18: Function "setof" @ 29..34: Type "int" @ 35..38: Type + "select" @ 43..49: Keyword + "1" @ 50..51: Number "#); } @@ -808,6 +896,8 @@ select f(); ), @r#" "f" @ 17..18: Function "int" @ 29..32: Type + "select" @ 37..43: Keyword + "1" @ 44..45: Number "f" @ 68..69: Function "#); } @@ -827,6 +917,8 @@ select b(t), t.b from t; "b" @ 40..41: Function "t" @ 42..43: Type "int" @ 53..56: Type + "select" @ 61..67: Keyword + "1" @ 68..69: Number "b" @ 92..93: Function "t" @ 94..95: Table "t" @ 98..99: Table @@ -851,6 +943,8 @@ create policy p on t "x" @ 40..41: Function "t" @ 42..43: Type "int" @ 53..56: Type + "select" @ 61..67: Keyword + "1" @ 68..69: Number "t" @ 104..105: Table "t" @ 120..121: Table "x" @ 122..123: Function @@ -907,4 +1001,118 @@ select s.t.a from s.t; "t" @ 63..64: Table "#); } + + #[test] + fn create_function_embedded_sql() { + assert_snapshot!(semantic_tokens( + " +create table t(a int); +create function f(x int) returns int +as $$ select a from t where a = x and a = $1 $$ +language sql; +", + ), @r#" + "t" @ 14..15: Table + "a" @ 16..17: Column + "int" @ 18..21: Type + "f" @ 40..41: Function + "x" @ 42..43: Parameter + "int" @ 44..47: Type + "int" @ 57..60: Type + "select" @ 67..73: Keyword + "a" @ 74..75: Column + "from" @ 76..80: Keyword + "t" @ 81..82: Table + "where" @ 83..88: Keyword + "a" @ 89..90: Column + "=" @ 91..92: Operator + "x" @ 93..94: Parameter + "and" @ 95..98: Keyword + "a" @ 99..100: Column + "=" @ 101..102: Operator + "$1" @ 103..105: PositionalParam + "#); + } + + #[test] + fn create_procedure_embedded_sql_escaped() { + assert_snapshot!(semantic_tokens( + " +create table t(a text); +create procedure p() +language sql +as 'insert into t(a) select ''x'' from t'; +", + ), @r#" + "t" @ 14..15: Table + "a" @ 16..17: Column + "text" @ 18..22: Type + "p" @ 42..43: Function + "insert" @ 63..69: Keyword + "into" @ 70..74: Keyword + "t" @ 75..76: Table + "(" @ 76..77: Punctuation + "a" @ 77..78: Column + ")" @ 78..79: Punctuation + "select" @ 80..86: Keyword + "''x''" @ 87..92: String + "from" @ 93..97: Keyword + "t" @ 98..99: Table + "#); + } + + #[test] + fn create_function_embedded_non_sql() { + assert_snapshot!(semantic_tokens( + " +create table t(a int); +create function f() returns int +as $$ select a from t $$ +language plpgsql; +", + ), @r#" + "t" @ 14..15: Table + "a" @ 16..17: Column + "int" @ 18..21: Type + "f" @ 40..41: Function + "int" @ 52..55: Type + "#); + } + + #[test] + fn embedded_sql_lexical_tokens() { + assert_snapshot!(semantic_tokens( + " +create function f() returns int language sql as $body$ +-- embedded +select unknown_name, 42, 1.5, true, false, null, 'text', E'escape', B'01', X'ff'; +$body$; +", + ), @r#" + "f" @ 17..18: Function + "int" @ 29..32: Type + "-- embedded" @ 56..67: Comment + "select" @ 68..74: Keyword + "unknown_name" @ 75..87: NameRef + "," @ 87..88: Punctuation + "42" @ 89..91: Number + "," @ 91..92: Punctuation + "1.5" @ 93..96: Number + "," @ 96..97: Punctuation + "true" @ 98..102: Bool + "," @ 102..103: Punctuation + "false" @ 104..109: Bool + "," @ 109..110: Punctuation + "null" @ 111..115: Keyword + "," @ 115..116: Punctuation + "'text'" @ 117..123: String + "," @ 123..124: Punctuation + "E'escape'" @ 125..134: String + "," @ 134..135: Punctuation + "B'01'" @ 136..141: String + "," @ 141..142: Punctuation + "X'ff'" @ 143..148: String + ";" @ 148..149: Punctuation + "#); + } } diff --git a/crates/squawk_parser/src/generated/token_sets.rs b/crates/squawk_parser/src/generated/token_sets.rs index b83c3601..b63ae57b 100644 --- a/crates/squawk_parser/src/generated/token_sets.rs +++ b/crates/squawk_parser/src/generated/token_sets.rs @@ -4,6 +4,102 @@ use crate::syntax_kind::SyntaxKind; use crate::token_set::TokenSet; +pub(crate) const PUNCTUATION: TokenSet = TokenSet::new(&[ + SyntaxKind::SEMICOLON, + SyntaxKind::COMMA, + SyntaxKind::L_PAREN, + SyntaxKind::R_PAREN, + SyntaxKind::L_BRACK, + SyntaxKind::R_BRACK, + SyntaxKind::L_CURLY, + SyntaxKind::R_CURLY, + SyntaxKind::DOT, + SyntaxKind::DOT_DOT, +]); + +pub(crate) const OPERATORS: TokenSet = TokenSet::new(&[ + SyntaxKind::DOLLAR, + SyntaxKind::L_ANGLE, + SyntaxKind::LESS_LESS, + SyntaxKind::LTEQ, + SyntaxKind::NEQB, + SyntaxKind::R_ANGLE, + SyntaxKind::GREATER_GREATER, + SyntaxKind::GTEQ, + SyntaxKind::AT, + SyntaxKind::POUND, + SyntaxKind::TILDE, + SyntaxKind::QUESTION, + SyntaxKind::AMP, + SyntaxKind::PIPE, + SyntaxKind::PLUS, + SyntaxKind::STAR, + SyntaxKind::SLASH, + SyntaxKind::CARET, + SyntaxKind::PERCENT, + SyntaxKind::UNDERSCORE, + SyntaxKind::COLON, + SyntaxKind::COLON_EQ, + SyntaxKind::EQ, + SyntaxKind::FAT_ARROW, + SyntaxKind::BANG, + SyntaxKind::NEQ, + SyntaxKind::MINUS, + SyntaxKind::BACKTICK, +]); + +pub(crate) const CONTEXTUAL_KEYWORDS: TokenSet = TokenSet::new(&[ + SyntaxKind::ALIAS_KW, + SyntaxKind::ASSERT_KW, + SyntaxKind::COLUMN_NAME_KW, + SyntaxKind::CONSTANT_KW, + SyntaxKind::CONSTRAINT_NAME_KW, + SyntaxKind::DATATYPE_KW, + SyntaxKind::DEBUG_KW, + SyntaxKind::DETAIL_KW, + SyntaxKind::DIAGNOSTICS_KW, + SyntaxKind::DUMP_KW, + SyntaxKind::ELSEIF_KW, + SyntaxKind::ELSIF_KW, + SyntaxKind::ERRCODE_KW, + SyntaxKind::EXCEPTION_KW, + SyntaxKind::EXIT_KW, + SyntaxKind::FOREACH_KW, + SyntaxKind::GET_KW, + SyntaxKind::HINT_KW, + SyntaxKind::INFO_KW, + SyntaxKind::LOG_KW, + SyntaxKind::LOOP_KW, + SyntaxKind::MESSAGE_KW, + SyntaxKind::MESSAGE_TEXT_KW, + SyntaxKind::NOTICE_KW, + SyntaxKind::OPEN_KW, + SyntaxKind::PERFORM_KW, + SyntaxKind::PG_CONTEXT_KW, + SyntaxKind::PG_DATATYPE_NAME_KW, + SyntaxKind::PG_EXCEPTION_CONTEXT_KW, + SyntaxKind::PG_EXCEPTION_DETAIL_KW, + SyntaxKind::PG_EXCEPTION_HINT_KW, + SyntaxKind::PG_ROUTINE_OID_KW, + SyntaxKind::PRINT_STRICT_PARAMS_KW, + SyntaxKind::QUERY_KW, + SyntaxKind::RAISE_KW, + SyntaxKind::RETURNED_SQLSTATE_KW, + SyntaxKind::REVERSE_KW, + SyntaxKind::ROW_COUNT_KW, + SyntaxKind::ROWTYPE_KW, + SyntaxKind::SCHEMA_NAME_KW, + SyntaxKind::SLICE_KW, + SyntaxKind::SQLSTATE_KW, + SyntaxKind::STACKED_KW, + SyntaxKind::TABLE_NAME_KW, + SyntaxKind::USE_COLUMN_KW, + SyntaxKind::USE_VARIABLE_KW, + SyntaxKind::VARIABLE_CONFLICT_KW, + SyntaxKind::WARNING_KW, + SyntaxKind::WHILE_KW, +]); + pub(crate) const COLUMN_OR_TABLE_KEYWORDS: TokenSet = TokenSet::new(&[ SyntaxKind::ABORT_KW, SyntaxKind::ABSENT_KW, diff --git a/crates/squawk_parser/src/syntax_kind.rs b/crates/squawk_parser/src/syntax_kind.rs index f942be19..7e5b488c 100644 --- a/crates/squawk_parser/src/syntax_kind.rs +++ b/crates/squawk_parser/src/syntax_kind.rs @@ -22,6 +22,23 @@ impl SyntaxKind { pub fn is_trivia(self) -> bool { matches!(self, SyntaxKind::WHITESPACE | SyntaxKind::COMMENT) } + + #[inline] + pub fn is_keyword(self) -> bool { + self <= SyntaxKind::WHITESPACE + && (crate::generated::token_sets::ALL_KEYWORDS.contains(self) + || crate::generated::token_sets::CONTEXTUAL_KEYWORDS.contains(self)) + } + + #[inline] + pub fn is_operator(self) -> bool { + self <= SyntaxKind::WHITESPACE && crate::generated::token_sets::OPERATORS.contains(self) + } + + #[inline] + pub fn is_punctuation(self) -> bool { + self <= SyntaxKind::WHITESPACE && crate::generated::token_sets::PUNCTUATION.contains(self) + } } #[inline] @@ -54,4 +71,17 @@ mod tests { assert!(!is_reserved_keyword(SyntaxKind::IDENT)); assert!(!is_reserved_keyword(SyntaxKind::SELECT)); } + + #[test] + fn semantic_categories() { + assert!(SyntaxKind::SELECT_KW.is_keyword()); + assert!(SyntaxKind::PERFORM_KW.is_keyword()); + assert!(SyntaxKind::PLUS.is_operator()); + assert!(SyntaxKind::L_PAREN.is_punctuation()); + + assert!(!SyntaxKind::IDENT.is_keyword()); + assert!(!SyntaxKind::L_PAREN.is_operator()); + assert!(!SyntaxKind::PLUS.is_punctuation()); + assert!(!SyntaxKind::SELECT.is_keyword()); + } } diff --git a/crates/xtask/src/codegen.rs b/crates/xtask/src/codegen.rs index a4c851e1..42ab5be9 100644 --- a/crates/xtask/src/codegen.rs +++ b/crates/xtask/src/codegen.rs @@ -76,8 +76,9 @@ pub(crate) fn codegen() -> Result<()> { std::fs::write(ast_nodes_file, ast_nodes).context("problem writing generated nodes")?; let keyword_kinds = keyword_kinds()?; + let contextual_keywords = contextual_keywords()?; - let token_sets = generate_token_sets(&keyword_kinds)?; + let token_sets = generate_token_sets(&keyword_kinds, &contextual_keywords)?; let token_sets_file = project_root().join("crates/squawk_parser/src/generated/token_sets.rs"); std::fs::write(token_sets_file, token_sets).context("problem writing generated token sets")?; @@ -96,7 +97,7 @@ pub(crate) fn codegen() -> Result<()> { &ast_src.nodes, &grammar, keyword_kinds.all_keywords, - contextual_keywords()?, + contextual_keywords, ); let syntax_kinds = generate_syntax_kinds(kinds)?; @@ -123,7 +124,8 @@ const EOF: &str = "EOF"; const NAME_TOKEN: &str = "#name"; const LITERAL_TOKEN: &str = "@"; -/// The punctuations of the language. +const PUNCTUATION: &[&str] = &[";", ",", "(", ")", "[", "]", "{", "}", ".", ".."]; + const PUNCT: &[(&str, &str)] = &[ // KEEP THE DOLLAR AT THE TOP ITS SPECIAL ("$", "DOLLAR"), @@ -415,7 +417,26 @@ fn generate_syntax_kinds(grammar: KindsSrc) -> Result { Ok(format!("{PRELUDE}{output}")) } -fn generate_token_sets(keyword_kinds: &KeywordKinds) -> Result { +fn generate_token_sets( + keyword_kinds: &KeywordKinds, + contextual_keywords: &[String], +) -> Result { + let punctuation = PUNCT + .iter() + .filter(|(token, _)| PUNCTUATION.contains(token)) + .map(|(_, name)| format_ident!("{name}")) + .collect::>(); + let operators = PUNCT + .iter() + .filter(|(token, _)| !PUNCTUATION.contains(token)) + .map(|(_, name)| format_ident!("{name}")) + .collect::>(); + + let contextual_keywords = contextual_keywords + .iter() + .map(|key| format_ident!("{}_KW", key.to_case(Case::UpperSnake))) + .collect::>(); + let column_or_table_keywords = keyword_kinds .col_table_keywords .iter() @@ -476,6 +497,18 @@ fn generate_token_sets(keyword_kinds: &KeywordKinds) -> Result { use crate::syntax_kind::SyntaxKind; use crate::token_set::TokenSet; + pub(crate) const PUNCTUATION: TokenSet = TokenSet::new(&[ + #(SyntaxKind::#punctuation),* + ]); + + pub(crate) const OPERATORS: TokenSet = TokenSet::new(&[ + #(SyntaxKind::#operators),* + ]); + + pub(crate) const CONTEXTUAL_KEYWORDS: TokenSet = TokenSet::new(&[ + #(SyntaxKind::#contextual_keywords),* + ]); + pub(crate) const COLUMN_OR_TABLE_KEYWORDS: TokenSet = TokenSet::new(&[ #(SyntaxKind::#column_or_table_keywords),* ]);