diff --git a/README.md b/README.md index b918093..2326ee6 100644 --- a/README.md +++ b/README.md @@ -59,13 +59,15 @@ func main() { - **Three validators** — fields selection, filtering, and sorting, each gated by an allow-list of fields (nested dot-notation like `user.profile.age` supported). -- **PostgreSQL's filter grammar, or the SQL standard's** — comparison, +- **The filter grammar of the database that runs it** — comparison, logical, `IN`, `BETWEEN [SYMMETRIC]`, `LIKE`/`ILIKE` with `ESCAPE`, `SIMILAR TO`, POSIX regex, `IS NULL`, `IS [NOT] TRUE/FALSE/UNKNOWN`, and - `IS [NOT] DISTINCT FROM`. The default dialect is PostgreSQL 18's; - `WithDialect(DialectSQLStandard)` accepts SQL:2023 (ISO/IEC 9075:2023) and - refuses PostgreSQL's own spellings. See [the SQL dialect](docs/dialects.md). -- **Configurable** — choose the SQL dialect (PostgreSQL 18 or SQL:2023), and + `IS [NOT] DISTINCT FROM`, as PostgreSQL 18 runs them (the default), or as + SQL:2023 (ISO/IEC 9075:2023) or MySQL 8.4 does with `WithDialect`. MySQL's + dialect reads strings with MySQL's backslash escapes, so a filter cannot + hide a predicate from the allow-list. See [SQL Dialects](docs/dialects.md). +- **Configurable** — choose the SQL dialect (PostgreSQL 18, SQL:2023 or + MySQL 8.4), and restrict which operators or sort directions are allowed. - **Returns an AST** you can walk or render, with precise, aggregated errors. - **Concurrency-safe** parsers. @@ -88,7 +90,7 @@ flowchart LR AL(["Allow-list of fields"]) -. gates .-> FP AL -. gates .-> FI AL -. gates .-> SP - DI(["SQL dialect
PostgreSQL 18 (default) · SQL:2023"]) -. grammar .-> FI + DI(["SQL dialect
PostgreSQL 18 (default) · SQL:2023 · MySQL 8.4"]) -. grammar .-> FI FP --> FN["FieldsNode
[]string"] FI --> AST["Filter AST
(Node tree)"] @@ -121,7 +123,7 @@ Full documentation lives in [`docs/`](docs/README.md): - [Getting Started](docs/getting-started.md) — install, update, complete example - [Filtering](docs/filtering.md) — the full operator/predicate reference -- [SQL Dialects](docs/dialects.md) — PostgreSQL 18 (the default) or SQL:2023: +- [SQL Dialects](docs/dialects.md) — PostgreSQL 18 (the default), SQL:2023 or MySQL 8.4: why a filter parser needs one, choosing it, and every construct in each - [Configuration](docs/configuration.md) — choose the dialect, restrict operators and sort directions diff --git a/dialect.go b/dialect.go index dfa9c31..5c21170 100644 --- a/dialect.go +++ b/dialect.go @@ -15,8 +15,8 @@ package qfv // against what its manual calls an extension. // // A Dialect is one of the values below, each a version of its SQL: PostgreSQL -// 18, SQL:2023. The zero value is DialectPostgreSQL, the grammar this package -// has always parsed. +// 18, SQL:2023, MySQL 8.4. The zero value is DialectPostgreSQL, the grammar +// this package has always parsed. type Dialect struct { name string } @@ -49,10 +49,39 @@ var ( // has no escape character, where PostgreSQL's is the backslash. A pattern // with a backslash in it means one thing to each. DialectSQLStandard = Dialect{name: "SQL:2023"} + + // DialectMySQL is MySQL's grammar, as MySQL 8.4 runs it with its default + // sql_mode (https://dev.mysql.com/doc/refman/8.4/en/sql-statements.html). + // It is PostgreSQL's grammar without what MySQL 8.4 does not run: + // + // - ILIKE and NOT ILIKE, and the ~~, ~~*, !~~ and !~~* operators; + // - SIMILAR TO, and the POSIX operators ~, ~*, !~ and !~* (MySQL's + // regular expressions are REGEXP, which this package does not parse); + // - IS [NOT] DISTINCT FROM (MySQL's null-safe equality is <=>, which + // this package does not parse); + // - BETWEEN SYMMETRIC and BETWEEN ASYMMETRIC; + // - the ISNULL and NOTNULL shorthands (MySQL's ISNULL is a function). + // + // It keeps != and an empty ESCAPE, which MySQL 8.4 runs. + // + // And it reads a string as MySQL does: a backslash escapes the character + // after it, so 'a\'b' is one string, a'b. PostgreSQL's strings keep a + // backslash as it is, and read the same text as the string 'a\' followed + // by b'. A filter checked with PostgreSQL's rules can hide a predicate + // from the field allow-list when MySQL runs it: name = '\'' OR id > 0 #' + // is one comparison of name with a string to PostgreSQL's reading, and to + // MySQL 8.4 a comparison of name with a quote, then id > 0, then a + // comment to the end of the line. With this dialect it is refused: id is + // not allowed, and # is not part of the grammar. + // + // MySQL's sql_mode can turn backslash escapes off (NO_BACKSLASH_ESCAPES). + // The parser cannot see the server's mode: this dialect is MySQL 8.4's + // default, which has them on. + DialectMySQL = Dialect{name: "MySQL 8.4"} ) // String is the dialect's name with its version, as error messages write it: -// "PostgreSQL 18", "SQL:2023". +// "PostgreSQL 18", "SQL:2023", "MySQL 8.4". func (d Dialect) String() string { if d.name == "" { return "PostgreSQL 18" @@ -66,6 +95,37 @@ func (d Dialect) standard() bool { return d == DialectSQLStandard } +// backslashEscapes reports whether a backslash in a string escapes the +// character after it, as it does in MySQL 8.4's default sql_mode. +func (d Dialect) backslashEscapes() bool { + return d == DialectMySQL +} + +// hasSpelling reports whether the dialect has this spelling of an operator it +// has. PostgreSQL 18 has every one. +func (d Dialect) hasSpelling(spelling string) bool { + switch d { + case DialectSQLStandard: + switch spelling { + case "!=", "~~", "!~~", "ISNULL", "NOTNULL": + return false + } + case DialectMySQL: + switch spelling { + case "~~", "!~~", "ISNULL", "NOTNULL", "BETWEEN SYMMETRIC", "BETWEEN ASYMMETRIC": + return false + } + } + + return true +} + +// emptyEscape reports whether the dialect takes an empty ESCAPE string, which +// turns escaping off. PostgreSQL 18 and MySQL 8.4 do; the SQL standard does not. +func (d Dialect) emptyEscape() bool { + return !d.standard() +} + // Operators returns the operators the dialect has. The returned slice is a // copy and safe to modify. func (d Dialect) Operators() []Operator { @@ -84,13 +144,11 @@ func (d Dialect) Operators() []Operator { // dialect does not have (!= for <>, ~~ for LIKE, ISNULL for IS NULL) is // refused where the parser reads it. func (d Dialect) has(op Operator) bool { - if !d.standard() { - return true - } - - switch op { - case OpILike, OpRegexMatch: - return false + switch d { + case DialectSQLStandard: + return op != OpILike && op != OpRegexMatch + case DialectMySQL: + return op != OpILike && op != OpSimilarTo && op != OpRegexMatch && op != OpIsDistinctFrom default: return true } diff --git a/dialect_docs_test.go b/dialect_docs_test.go index 25c58fa..21c8654 100644 --- a/dialect_docs_test.go +++ b/dialect_docs_test.go @@ -7,10 +7,12 @@ import ( "testing" ) -// docs/dialects.md quotes, word for word, what Parse answers in the SQL:2023 -// dialect. A message that changes here and not there would leave the page -// telling clients to look for an error they never get, so each row of its -// table is parsed and held to the parser. +// docs/dialects.md quotes, word for word, what Parse answers in each dialect. +// A message that changes here and not there would leave the page telling +// clients to look for an error they never get, so each row of its tables is +// parsed and held to the parser: the "### SQL:2023" table in the SQL standard +// dialect, the "### MySQL 8.4" one in MySQL's. A row that names an allow-list +// ("with the allow-list `["name"]`") is parsed with it. func TestTheDialectsPageQuotesTheParser(t *testing.T) { t.Parallel() @@ -31,30 +33,52 @@ func TestTheDialectsPageQuotesTheParser(t *testing.T) { section = section[:end+1] } - row := regexp.MustCompile("(?m)^\\| (`[^|]+`(?:, `[^|]+`)*) \\| `([^`]+)` \\|$") + row := regexp.MustCompile("(?m)^\\| (`[^|]+`(?:, `[^|]+`)*) \\| `([^`]+)`(?: \\(with the allow-list `\\[([^]]*)\\]`\\))? \\|$") code := regexp.MustCompile("`([^`]+)`") - parser := NewFilterParser([]string{"id", "name"}, WithDialect(DialectSQLStandard)) + quoted := regexp.MustCompile(`"([^"]+)"`) - checked := 0 + dialects := map[string]Dialect{"SQL:2023": DialectSQLStandard, "MySQL 8.4": DialectMySQL} + checked := map[string]int{} - for _, m := range row.FindAllStringSubmatch(section, -1) { - for _, input := range code.FindAllStringSubmatch(m[1], -1) { - checked++ + for _, table := range strings.Split(section, "\n### ")[1:] { + heading, body, _ := strings.Cut(table, "\n") - _, err := parser.Parse(input[1]) - if err == nil { - t.Errorf("the page says %q is refused, and Parse accepts it", input[1]) + dialect, ok := dialects[heading] + if !ok { + t.Errorf("docs/dialects.md has a refusals table for %q, a dialect this test does not know", heading) - continue + continue + } + + for _, m := range row.FindAllStringSubmatch(body, -1) { + allowed := []string{"id", "name"} + if m[3] != "" { + allowed = nil + for _, f := range quoted.FindAllStringSubmatch(m[3], -1) { + allowed = append(allowed, f[1]) + } } - if !strings.Contains(err.Error(), m[2]) { - t.Errorf("the page says %q answers %q; Parse answers %q", input[1], m[2], err.Error()) + parser := NewFilterParser(allowed, WithDialect(dialect)) + + for _, input := range code.FindAllStringSubmatch(m[1], -1) { + checked[heading]++ + + _, err := parser.Parse(input[1]) + if err == nil { + t.Errorf("%s: the page says %q is refused, and Parse accepts it", heading, input[1]) + + continue + } + + if !strings.Contains(err.Error(), m[2]) { + t.Errorf("%s: the page says %q answers %q; Parse answers %q", heading, input[1], m[2], err.Error()) + } } } } - if checked < 9 { - t.Fatalf("read %d inputs from the page's table, want at least 9: the table's shape changed and this test reads nothing", checked) + if checked["SQL:2023"] < 9 || checked["MySQL 8.4"] < 12 { + t.Fatalf("read %v inputs from the page's tables, want at least 9 for SQL:2023 and 12 for MySQL 8.4: the tables' shape changed and this test reads nothing", checked) } } diff --git a/dialect_mysql_test.go b/dialect_mysql_test.go new file mode 100644 index 0000000..83ab98d --- /dev/null +++ b/dialect_mysql_test.go @@ -0,0 +1,174 @@ +package qfv + +import ( + "strings" + "testing" +) + +// Each row of this file was run against MySQL 8.4.13 with its default +// sql_mode (ONLY_FULL_GROUP_BY, STRICT_TRANS_TABLES, NO_ZERO_IN_DATE, +// NO_ZERO_DATE, ERROR_FOR_DIVISION_BY_ZERO, NO_ENGINE_SUBSTITUTION: no +// NO_BACKSLASH_ESCAPES), as SELECT count(*) FROM t WHERE (), on a +// table with the columns id INT, name TEXT, price DECIMAL(10,2) and active +// BOOLEAN; what it answered is in the comment. The statement was prepared +// from a hex literal, so that only the server parsed it. + +// TestTheMySQLDialectReadsStringsAsMySQLDoes: MySQL reads a backslash in a +// string as an escape, and PostgreSQL's rules read it as a character. The +// filter below is one comparison of name with a string to PostgreSQL's +// reading, which the field allow-list ["name"] lets through; MySQL 8.4, +// with a newline after it as in a SQL template, ran it as a comparison of +// name with a single quote, then id > 0, and the rest of the line a comment: +// every row, on a field the allow-list does not have. In MySQL's dialect it +// is refused. +func TestTheMySQLDialectReadsStringsAsMySQLDoes(t *testing.T) { + t.Parallel() + + const smuggled = `name = '\'' OR id > 0 #'` + + // PostgreSQL's reading: one comparison, which PostgreSQL 18 runs as one. + node, err := NewFilterParser([]string{"name"}).Parse(smuggled) + if err != nil { + t.Fatalf("PostgreSQL 18's dialect: %v", err) + } + + if _, ok := node.(*BinaryOperatorNode); !ok { + t.Fatalf("PostgreSQL 18's dialect read %T, want one comparison", node) + } + + // MySQL's reading: id is a field of its own, and # is no part of the + // grammar. + _, err = NewFilterParser([]string{"name"}, WithDialect(DialectMySQL)).Parse(smuggled) + if err == nil { + t.Fatalf("MySQL 8.4's dialect accepted %q, which MySQL runs with a predicate on id", smuggled) + } + + if !strings.Contains(err.Error(), "error on field 'id': field not allowed") { + t.Errorf("MySQL 8.4's dialect: %v, want id refused as a field the allow-list does not have", err) + } + + // The same strings, one token each, as MySQL reads them. + for input, want := range map[string]string{ + `name = 'a\'b'`: `'a\'b'`, // one string, a'b: 0 rows (the row holds a\'b) + `name = 'a\\b'`: `'a\\b'`, // one string, a\b + `name = 'a''b'`: `'a''b'`, // the doubled quote, as in every dialect + `name LIKE 'x\%'`: `'x\%'`, // 1 row: \% is a literal % + } { + node, err := NewFilterParser([]string{"name"}, WithDialect(DialectMySQL)).Parse(input) + if err != nil { + t.Errorf("Parse(%q): %v", input, err) + + continue + } + + if right := node.(*BinaryOperatorNode).Right.(*LiteralNode).Text; right != want { + t.Errorf("Parse(%q): the string is %s, want %s", input, right, want) + } + } + + // A string that ends on its backslash is not closed: 'a\' is the + // start of a string to MySQL. + if _, err := NewFilterParser([]string{"name"}, WithDialect(DialectMySQL)).Parse(`name = 'a\'`); err == nil { + t.Error(`MySQL 8.4's dialect accepted name = 'a\', an unclosed string to MySQL`) + } + + // Nor is one whose input ends on the backslash itself. + if _, err := NewFilterParser([]string{"name"}, WithDialect(DialectMySQL)).Parse(`name = 'a\`); err == nil { + t.Error(`MySQL 8.4's dialect accepted name = 'a\, an unclosed string`) + } + + // NewLexer, which callers use directly, reads strings as it always did. + l := NewLexer(`'a\'`) + l.Parse() + + if tok := l.Next(); tok.Type != TokenString || tok.Value != `'a\'` { + t.Errorf("NewLexer read %v, want the PostgreSQL string 'a\\'", tok) + } +} + +func TestTheMySQLDialect(t *testing.T) { + t.Parallel() + + parser := NewFilterParser([]string{"id", "name", "price", "active"}, WithDialect(DialectMySQL)) + + if DialectMySQL.String() != "MySQL 8.4" { + t.Errorf("String() = %q, want MySQL 8.4", DialectMySQL) + } + + for _, input := range []string{ + "id = 1", "id <> 1", "id != 1", "id < 2 AND id >= 1", // 1, 2, 2, 1 + "id = 1 OR NOT active IS TRUE", // 3 + "id BETWEEN 1 AND 2", "id NOT BETWEEN 1 AND 2", // 2, 1 + "id IN (1, 2)", "id NOT IN (1, 2)", // 2, 1 + "name LIKE 'x%'", "name NOT LIKE 'x%'", // 2, 1 + "name LIKE 'x!%' ESCAPE '!'", // 1 + "name NOT LIKE 'x!%' ESCAPE '!'", // 2 + "name LIKE 'x%' ESCAPE ''", // 2 + `name LIKE 'x\%' ESCAPE '\\'`, // 1: '\\' is one backslash + "name IS NULL", "name IS NOT NULL", // 0, 3 + "active IS TRUE", "active IS NOT FALSE", "active IS UNKNOWN", // 1, 2, 1 + "active = TRUE", "price = 1.5", "price = -2", "price = 1e3", // 1, 0, 0, 0 + "name = 'it''s'", // 0 + } { + t.Run("accepted: "+input, func(t *testing.T) { + t.Parallel() + + if _, err := parser.Parse(input); err != nil { + t.Errorf("Parse(%q): %v, want it accepted: MySQL 8.4 runs it", input, err) + } + }) + } + + for _, tt := range []struct { + input string + says string + }{ + // syntax error near 'ILIKE …', '~~ …', 'SIMILAR TO …', '~ …' + {input: "name ILIKE 'x%'", says: `operator "ILIKE" is not part of the MySQL 8.4 dialect`}, + {input: "name NOT ILIKE 'x%'", says: `operator "ILIKE" is not part of the MySQL 8.4 dialect`}, + {input: "name ~~* 'x%'", says: `operator "ILIKE" is not part of the MySQL 8.4 dialect`}, + {input: "name ~~ 'x%'", says: "~~ is not part of the MySQL 8.4 dialect: write LIKE"}, + {input: "name !~~ 'x%'", says: "!~~ is not part of the MySQL 8.4 dialect: write NOT LIKE"}, + {input: "name SIMILAR TO 'x%'", says: `operator "SIMILAR TO" is not part of the MySQL 8.4 dialect`}, + {input: "name NOT SIMILAR TO 'x%'", says: `operator "SIMILAR TO" is not part of the MySQL 8.4 dialect`}, + {input: "name ~ '^x'", says: `operator "~" is not part of the MySQL 8.4 dialect`}, + {input: "name !~ '^x'", says: `operator "~" is not part of the MySQL 8.4 dialect`}, + // syntax error near 'ISNULL)', 'NOTNULL)' + {input: "name ISNULL", says: "ISNULL is not part of the MySQL 8.4 dialect: write IS NULL"}, + {input: "name NOTNULL", says: "NOTNULL is not part of the MySQL 8.4 dialect: write IS NOT NULL"}, + // syntax error near 'DISTINCT FROM 1)' + {input: "id IS DISTINCT FROM 1", says: `operator "IS DISTINCT FROM" is not part of the MySQL 8.4 dialect`}, + {input: "id IS NOT DISTINCT FROM 1", says: `operator "IS DISTINCT FROM" is not part of the MySQL 8.4 dialect`}, + // syntax error near '2 AND 1)' + {input: "id BETWEEN SYMMETRIC 2 AND 1", says: "BETWEEN SYMMETRIC is not part of the MySQL 8.4 dialect: write BETWEEN, the lower bound first"}, + {input: "id BETWEEN ASYMMETRIC 1 AND 2", says: "BETWEEN ASYMMETRIC is not part of the MySQL 8.4 dialect: write BETWEEN"}, + // Incorrect arguments to ESCAPE (1210) + {input: "name LIKE 'x%' ESCAPE '!!'", says: "must be empty or one character"}, + } { + t.Run("refused: "+tt.input, func(t *testing.T) { + t.Parallel() + + _, err := parser.Parse(tt.input) + if err == nil { + t.Fatalf("Parse(%q) accepted, want it refused: MySQL 8.4 does not run it", tt.input) + } + + if !strings.Contains(err.Error(), tt.says) { + t.Errorf("Parse(%q) = %q, want it to say %q", tt.input, err.Error(), tt.says) + } + }) + } +} + +// One quote, written doubled, is one character in every dialect: an ESCAPE +// string of four single quotes is a quote, not an empty string. MySQL 8.4 and +// PostgreSQL 18 each answered 1 row for the filter below (the row 'x%'). +func TestADoubledQuoteIsOneEscapeCharacter(t *testing.T) { + t.Parallel() + + for _, d := range []Dialect{DialectPostgreSQL, DialectSQLStandard, DialectMySQL} { + if _, err := NewFilterParser([]string{"name"}, WithDialect(d)).Parse("name LIKE 'x''%' ESCAPE ''''"); err != nil { + t.Errorf("%s: %v, want ESCAPE '''' accepted as one quote", d, err) + } + } +} diff --git a/docs.go b/docs.go index 9e45433..78fc8e1 100644 --- a/docs.go +++ b/docs.go @@ -7,7 +7,10 @@ // what the SQL it will run as accepts: by default PostgreSQL 18's grammar, as // PostgreSQL 18 runs it; with WithDialect(DialectSQLStandard), the SQL // standard's (ISO/IEC 9075:2023), which refuses PostgreSQL's own spellings -// such as != and ILIKE. WithAllowedOperators and WithAllowedOperatorGroups +// such as != and ILIKE; with WithDialect(DialectMySQL), MySQL 8.4's, which +// reads strings with MySQL's backslash escapes, as MySQL does, so that a +// filter cannot hide a predicate from the allow-list. WithAllowedOperators +// and WithAllowedOperatorGroups // narrow the operators further. docs/dialects.md explains the dialects. // // Every parser is safe for concurrent use. The documentation, with the full diff --git a/docs/README.md b/docs/README.md index dffe8f4..d2c69af 100644 --- a/docs/README.md +++ b/docs/README.md @@ -25,7 +25,7 @@ flowchart TB allow -. gates .-> FilterParser allow -. gates .-> SortParser - dialect(["SQL dialect
PostgreSQL 18 (default) · SQL:2023"]) + dialect(["SQL dialect
PostgreSQL 18 (default) · SQL:2023 · MySQL 8.4"]) dialect -. grammar .-> FilterParser ``` @@ -41,7 +41,7 @@ flowchart TB | --- | --- | | [Getting Started](getting-started.md) | Install, update, and a complete runnable example of the three parsers | | [Filtering](filtering.md) | The full filter grammar — every operator/predicate, `ESCAPE`, dotted fields, and worked examples | -| [SQL Dialects](dialects.md) | PostgreSQL 18 (the default) or SQL:2023: why a filter parser needs one, choosing it, how a filter is checked, every construct in each | +| [SQL Dialects](dialects.md) | PostgreSQL 18 (the default), SQL:2023 or MySQL 8.4: why a filter parser needs one, choosing it, how a filter is checked, every construct in each | | [Configuration](configuration.md) | Choosing the dialect, and restricting which operators and sort directions are allowed | | [Error Handling](error-handling.md) | Error shapes and how to inspect validation failures | | [Migration Guide](migration.md) | Breaking changes and how to upgrade between versions | diff --git a/docs/configuration.md b/docs/configuration.md index 787c8bc..baaf7ab 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -8,8 +8,9 @@ time. ## The SQL dialect The filter parser checks a filter in the grammar of the SQL it will run as: -PostgreSQL 18's by default, or the SQL standard's (SQL:2023) with -`WithDialect`. Everything about it is on its own page, [SQL Dialects](dialects.md): +PostgreSQL 18's by default, or, with `WithDialect`, the SQL standard's +(SQL:2023) or MySQL 8.4's, whose strings are read with MySQL's backslash +escapes. Everything about it is on its own page, [SQL Dialects](dialects.md): why there is one, how to choose it, how it is checked, a table of every construct in each, and what it does not do. @@ -18,6 +19,11 @@ p := qfv.NewFilterParser(fields, qfv.WithDialect(qfv.DialectSQLStandard)) p.Parse("id <> 1 AND name LIKE 'x!%' ESCAPE '!'") // ok p.Parse("id != 1") // error: != is not part of the SQL:2023 dialect: write <> + +m := qfv.NewFilterParser(fields, qfv.WithDialect(qfv.DialectMySQL)) + +m.Parse(`id != 1 AND name = 'it\'s'`) // ok: MySQL 8.4 runs it +m.Parse("name ILIKE 'x%'") // error: operator "ILIKE" is not part of the MySQL 8.4 dialect ``` The dialect comes before the allow-list below: an operator the dialect does not diff --git a/docs/dialects.md b/docs/dialects.md index f83f449..36a1579 100644 --- a/docs/dialects.md +++ b/docs/dialects.md @@ -9,6 +9,7 @@ write instead. | --- | --- | --- | --- | | PostgreSQL 18 (the default) | `qfv.DialectPostgreSQL` | `PostgreSQL 18` | PostgreSQL 18, and its manual's [SQL Syntax](https://www.postgresql.org/docs/18/sql-syntax.html) chapter | | SQL:2023 | `qfv.DialectSQLStandard` | `SQL:2023` | ISO/IEC 9075:2023, Part 2 (Foundation), as PostgreSQL's [SQL Conformance](https://www.postgresql.org/docs/18/features-sql-standard.html) appendix names its features | +| MySQL 8.4 | `qfv.DialectMySQL` | `MySQL 8.4` | MySQL 8.4 with its default `sql_mode`, and its manual's [SQL Statements](https://dev.mysql.com/doc/refman/8.4/en/sql-statements.html) | ```go // PostgreSQL 18: the default, no option needed. @@ -16,6 +17,9 @@ p := qfv.NewFilterParser(fields) // The SQL standard. s := qfv.NewFilterParser(fields, qfv.WithDialect(qfv.DialectSQLStandard)) + +// MySQL 8.4. +m := qfv.NewFilterParser(fields, qfv.WithDialect(qfv.DialectMySQL)) ``` ## Why a filter parser needs a dialect @@ -25,16 +29,53 @@ way to use it is to validate the `filter` query parameter and then put that same text in a `WHERE` clause. So "accepted" has to mean "the database runs it", and every SQL accepts something different: -- `id != 1` runs on PostgreSQL 18 and is not SQL:2023, which writes `id <> 1`. -- `name ILIKE 'x%'` runs on PostgreSQL 18 and is not SQL:2023, which has no - case-insensitive `LIKE`. -- `name LIKE 'x%' ESCAPE ''` runs on PostgreSQL 18; SQL:2023 disallows an - empty escape. +- `id != 1` runs on PostgreSQL 18 and MySQL 8.4, and is not SQL:2023, which + writes `id <> 1`. +- `name ILIKE 'x%'` runs on PostgreSQL 18, and on neither of the others. +- `name = 'a\'b'` is one string, `a'b`, to MySQL 8.4, and to PostgreSQL 18 the + string `a\` followed by `b'`. When the parser and the database disagree, the best outcome is a database error on a request the parser called valid. The worst is a filter the parser -read as one predicate and the database reads as another. The dialect is how -you tell the parser which database it is speaking for. +read as one predicate and the database reads as another, and the next section +shows one. The dialect is how you tell the parser which database it is +speaking for. + +## A string is read the way the database reads it + +MySQL 8.4 reads a backslash inside a string as an escape: `\'` is a quote that +does not end the string. PostgreSQL 18 and the SQL standard read it as a +character. A parser that reads strings one way, in front of a database that +reads them the other, can be shown a filter whose predicates it never sees. + +With the field allow-list `["name"]`, PostgreSQL's reading of this filter is +one comparison of `name` with a string: + +```text +name = '\'' OR id > 0 #' +``` + +MySQL 8.4 runs the same text, followed by a newline as in a SQL template, as a +comparison of `name` with a single quote, then `id > 0`, then a comment to the +end of the line. It returns every row, filtered on `id`, a field the allow-list +does not have. + +```mermaid +flowchart TB + T["name = '\'' OR id > 0 #'"] + T --> PG["read as PostgreSQL 18 reads it:
name = ‹a string›
one predicate, on name"] + T --> MY["read as MySQL 8.4 reads it:
name = ‹a quote› OR id > 0 ‹comment›
a predicate on id"] + PG --> A1["DialectPostgreSQL: accepted
right for PostgreSQL 18, which runs
one predicate"] + MY --> A2["DialectMySQL: refused
error on field 'id': field not allowed"] +``` + +`DialectMySQL` reads strings as MySQL 8.4 does, so it sees `id` and refuses +it. Before `v1.2.0` there was no MySQL dialect: a parser in front of MySQL read +strings as PostgreSQL does, and accepted that filter. + +MySQL's `sql_mode` can turn backslash escapes off (`NO_BACKSLASH_ESCAPES`). The +parser cannot see the server's mode: `DialectMySQL` is MySQL 8.4's default, +which has them on. ## Choosing a dialect @@ -42,36 +83,40 @@ you tell the parser which database it is speaking for. flowchart TD Q{"What runs the filter's text?"} Q -->|"PostgreSQL 18"| PG["DialectPostgreSQL
(the default: pass nothing)"] + Q -->|"MySQL 8.4, default sql_mode"| MY["DialectMySQL"] Q -->|"several databases, or portable SQL"| STD["DialectSQLStandard"] - Q -->|"another database (MySQL 8.4, …)"| NONE["no dialect yet:
see Adding a dialect"] + Q -->|"another database"| NONE["no dialect yet:
see Adding a dialect"] STD --> NOTE["the filter uses only the standard's spellings:
it runs on PostgreSQL 18 too"] ``` - **PostgreSQL 18** is the default and the grammar every version of this package has parsed. Use it when PostgreSQL runs the filter. -- **SQL:2023** accepts a subset of PostgreSQL 18's grammar: everything it +- **MySQL 8.4** is PostgreSQL's grammar without what MySQL does not run, with + strings read as MySQL reads them. Use it when MySQL runs the filter. It is + not optional there: see the section above. +- **SQL:2023** accepts a subset of PostgreSQL 18's grammar, and everything it accepts runs on PostgreSQL 18 as well. Use it when the filter must stay portable, or when you want clients to write the standard's spellings. -- Only these two exist. `Dialect` is a struct with unexported fields, so +- Only these three exist. `Dialect` is a struct with unexported fields, so there is no way to name a dialect the package does not have. Its zero value is `DialectPostgreSQL`. ## How a filter is checked against it -The lexer turns the text into tokens. The parser then checks every field -against the allow-list, and every operator against the dialect, first as an -operator and then in the spelling it was written in. Only then does it check -the operator against the operator allow-list. Every problem is collected, and -`Parse` returns them joined. +The lexer turns the text into tokens, reading strings as the dialect does. The +parser then checks every field against the allow-list, and every operator +against the dialect, first as an operator and then in the spelling it was +written in. Only then does it check the operator against the operator +allow-list. Every problem is collected, and `Parse` returns them joined. ```mermaid flowchart TD - IN["filter text"] --> LEX["Lexer: tokens"] + IN["filter text"] --> LEX["Lexer: tokens
(strings read as the dialect reads them)"] LEX --> OP["an operator, as written
(e.g. !=, ILIKE, ~~, ISNULL)"] OP --> D1{"does the dialect have
the operator?"} - D1 -->|"no: ILIKE, ~ in SQL:2023"| E1["operator "ILIKE" is not part of
the SQL:2023 dialect"] + D1 -->|"no: ILIKE, ~ in SQL:2023 and MySQL 8.4;
SIMILAR TO, IS DISTINCT FROM in MySQL 8.4"| E1["operator "ILIKE" is not part of
the SQL:2023 dialect"] D1 -->|"yes"| D2{"does it have
this spelling of it?"} - D2 -->|"no: != ~~ !~~ ISNULL NOTNULL
in SQL:2023"| E2["!= is not part of the SQL:2023
dialect: write <>"] + D2 -->|"no: != in SQL:2023; ~~ !~~ ISNULL NOTNULL
in both; BETWEEN SYMMETRIC in MySQL 8.4"| E2["!= is not part of the SQL:2023
dialect: write <>"] D2 -->|"yes"| A{"allowed by
WithAllowedOperators?"} A -->|"no"| E3["operator "LIKE" is not allowed"] A -->|"yes, or no allow-list"| OK["into the AST"] @@ -85,7 +130,7 @@ flowchart TD The dialect says what the SQL has; `WithAllowedOperators` and `WithAllowedOperatorGroups` say what you let clients use. An operator must pass both, so the dialect always wins: allowing `OpILike` does not put -`ILIKE` in SQL:2023. +`ILIKE` in SQL:2023 or MySQL 8.4. ```mermaid flowchart LR @@ -100,40 +145,53 @@ allow-list or showing clients what they may write. ## What each dialect accepts Every construct the filter parser knows, in each dialect. A ✅ is accepted -and a ❌ refused, with the spelling the dialect does have. +and a ❌ refused, with the spelling the dialect does have. The MySQL 8.4 column +was run against MySQL 8.4.13 with its default `sql_mode`. -| Construct | PostgreSQL 18 | SQL:2023 | SQL:2023 feature | -| --- | --- | --- | --- | -| `=`, `<`, `<=`, `>`, `>=` | ✅ | ✅ | E061-01 | -| `<>` | ✅ | ✅ | E061-01 | -| `!=` | ✅ | ❌ write `<>` | — | -| `AND`, `OR`, `NOT`, `( … )` | ✅ | ✅ | — | -| `BETWEEN`, `NOT BETWEEN` | ✅ | ✅ | E061-02 | -| `BETWEEN SYMMETRIC`, `BETWEEN ASYMMETRIC` | ✅ | ✅ | T461 | -| `IN ( … )`, `NOT IN ( … )` | ✅ | ✅ | E061-03 | -| `LIKE`, `NOT LIKE` | ✅ | ✅ | E061-04 | -| `LIKE … ESCAPE 'c'` | ✅ | ✅ | E061-05 | -| `LIKE … ESCAPE ''` (escaping off) | ✅ | ❌ one character | — | -| `ILIKE`, `NOT ILIKE` | ✅ | ❌ no such predicate | — | -| `~~`, `!~~` | ✅ | ❌ write `LIKE`, `NOT LIKE` | — | -| `~~*`, `!~~*` | ✅ | ❌ no such predicate | — | -| `SIMILAR TO`, `NOT SIMILAR TO`, with `ESCAPE` | ✅ | ✅ | T141 | -| `~`, `~*`, `!~`, `!~*` | ✅ | ❌ the standard's is `LIKE_REGEX`, which the parser does not read | — | -| `IS NULL`, `IS NOT NULL` | ✅ | ✅ | E061-06 | -| `ISNULL`, `NOTNULL` | ✅ | ❌ write `IS NULL`, `IS NOT NULL` | — | -| `IS [NOT] TRUE`, `FALSE`, `UNKNOWN` | ✅ | ✅ | F571 | -| `IS [NOT] DISTINCT FROM` | ✅ | ✅ | T151, T152 | -| string, number and boolean literals | ✅ | ✅ | — | - -Where the ❌ comes from: PostgreSQL's own manual calls each of these its own. -`<>` is "the standard SQL notation for not equal", and `!=` is an alias. -`ILIKE` is "not in the SQL standard but is a PostgreSQL extension". The `~~` -operators "are PostgreSQL-specific". `ISNULL` and `NOTNULL` are "nonstandard -syntax". "A zero-length ESCAPE value is disallowed" by the standard. +| Construct | PostgreSQL 18 | SQL:2023 | SQL:2023 feature | MySQL 8.4 | +| --- | --- | --- | --- | --- | +| `=`, `<`, `<=`, `>`, `>=` | ✅ | ✅ | E061-01 | ✅ | +| `<>` | ✅ | ✅ | E061-01 | ✅ | +| `!=` | ✅ | ❌ write `<>` | — | ✅ | +| `AND`, `OR`, `NOT`, `( … )` | ✅ | ✅ | — | ✅ | +| `BETWEEN`, `NOT BETWEEN` | ✅ | ✅ | E061-02 | ✅ | +| `BETWEEN SYMMETRIC`, `BETWEEN ASYMMETRIC` | ✅ | ✅ | T461 | ❌ write `BETWEEN` | +| `IN ( … )`, `NOT IN ( … )` | ✅ | ✅ | E061-03 | ✅ | +| `LIKE`, `NOT LIKE` | ✅ | ✅ | E061-04 | ✅ | +| `LIKE … ESCAPE 'c'` | ✅ | ✅ | E061-05 | ✅ | +| `LIKE … ESCAPE ''` (escaping off) | ✅ | ❌ one character | — | ✅ | +| `ILIKE`, `NOT ILIKE` | ✅ | ❌ no such predicate | — | ❌ no such predicate | +| `~~`, `!~~` | ✅ | ❌ write `LIKE`, `NOT LIKE` | — | ❌ write `LIKE`, `NOT LIKE` | +| `~~*`, `!~~*` | ✅ | ❌ no such predicate | — | ❌ no such predicate | +| `SIMILAR TO`, `NOT SIMILAR TO`, with `ESCAPE` | ✅ | ✅ | T141 | ❌ no such predicate | +| `~`, `~*`, `!~`, `!~*` | ✅ | ❌ the standard's is `LIKE_REGEX` | — | ❌ MySQL's is `REGEXP` | +| `IS NULL`, `IS NOT NULL` | ✅ | ✅ | E061-06 | ✅ | +| `ISNULL`, `NOTNULL` | ✅ | ❌ write `IS NULL`, `IS NOT NULL` | — | ❌ write `IS NULL`, `IS NOT NULL` | +| `IS [NOT] TRUE`, `FALSE`, `UNKNOWN` | ✅ | ✅ | F571 | ✅ | +| `IS [NOT] DISTINCT FROM` | ✅ | ✅ | T151, T152 | ❌ MySQL's is `<=>` | +| string, number and boolean literals | ✅ | ✅ | — | ✅, strings read with backslash escapes | + +`LIKE_REGEX`, `REGEXP` and `<=>` are not part of this package's grammar, in +any dialect. + +Where the ❌ comes from: + +- **SQL:2023:** PostgreSQL's own manual calls each of these its own. `<>` is + "the standard SQL notation for not equal", and `!=` is an alias. `ILIKE` is + "not in the SQL standard but is a PostgreSQL extension". The `~~` operators + "are PostgreSQL-specific". `ISNULL` and `NOTNULL` are "nonstandard syntax". + "A zero-length ESCAPE value is disallowed" by the standard. +- **MySQL 8.4:** each one was run against MySQL 8.4.13, and each is a syntax + error there except an escape of two characters (`Incorrect arguments to + ESCAPE`). ## What the refusals say -The SQL:2023 dialect's refusals, as `Parse` writes them: +The refusals, as `Parse` writes them. Each is a `*qfv.QFVFilterError`, joined +with any other problem in the same filter (see +[Error Handling](error-handling.md)). + +### SQL:2023 | Input | Error | | --- | --- | @@ -146,40 +204,57 @@ The SQL:2023 dialect's refusals, as `Parse` writes them: | `name ~ '^x'` | `operator "~" is not part of the SQL:2023 dialect` | | `name LIKE 'x%' ESCAPE ''` | `an empty ESCAPE is not part of the SQL:2023 dialect: the escape character is one character` | -Each is a `*qfv.QFVFilterError`, joined with any other problem in the same -filter (see [Error Handling](error-handling.md)). +### MySQL 8.4 + +| Input | Error | +| --- | --- | +| `name ~~ 'x%'` | `~~ is not part of the MySQL 8.4 dialect: write LIKE` | +| `name !~~ 'x%'` | `!~~ is not part of the MySQL 8.4 dialect: write NOT LIKE` | +| `name ISNULL` | `ISNULL is not part of the MySQL 8.4 dialect: write IS NULL` | +| `name NOTNULL` | `NOTNULL is not part of the MySQL 8.4 dialect: write IS NOT NULL` | +| `id BETWEEN SYMMETRIC 2 AND 1` | `BETWEEN SYMMETRIC is not part of the MySQL 8.4 dialect: write BETWEEN, the lower bound first` | +| `id BETWEEN ASYMMETRIC 1 AND 2` | `BETWEEN ASYMMETRIC is not part of the MySQL 8.4 dialect: write BETWEEN` | +| `name ILIKE 'x%'`, `name ~~* 'x%'` | `operator "ILIKE" is not part of the MySQL 8.4 dialect` | +| `name SIMILAR TO 'x%'` | `operator "SIMILAR TO" is not part of the MySQL 8.4 dialect` | +| `name ~ '^x'` | `operator "~" is not part of the MySQL 8.4 dialect` | +| `id IS DISTINCT FROM 1` | `operator "IS DISTINCT FROM" is not part of the MySQL 8.4 dialect` | +| `name = '\'' OR id > 0 #'` | `error on field 'id': field not allowed` (with the allow-list `["name"]`) | ## What a dialect does not do -- **It does not translate.** A filter written for PostgreSQL 18 is not - rewritten for another database. Choose the dialect of the database that - runs it. +- **It does not translate.** A filter written for one database is not + rewritten for another. Choose the dialect of the database that runs it. - **It does not know your columns.** Types are the database's: `id = 'x'` is - accepted in both dialects and fails on an integer column. Whether `LIKE` - is case-sensitive depends on the column's collation. + accepted in every dialect and fails on an integer column in PostgreSQL. + Whether `LIKE` is case-sensitive depends on the column's collation. +- **It does not see the server's settings.** `DialectMySQL` is MySQL 8.4's + default `sql_mode`. A server with `NO_BACKSLASH_ESCAPES` reads strings as + PostgreSQL does, and `ANSI_QUOTES` or `PIPES_AS_CONCAT` change the grammar + further. - **It does not change what a pattern means.** Without `ESCAPE`, PostgreSQL - 18's escape character is the backslash, and SQL:2023 has none, so - `name LIKE '100\%'` matches `100%` on PostgreSQL 18, and `100\` followed - by anything under the standard. The parser cannot see which you - meant, so write an `ESCAPE` clause when it matters. + 18's and MySQL 8.4's escape character is the backslash, and SQL:2023 has + none, so `name LIKE '100\%'` matches `100%` on PostgreSQL 18 and MySQL 8.4, + and `100\` followed by anything under the standard. The parser cannot see + which you meant, so write an `ESCAPE` clause when it matters. +- **It is stricter than the database in places, never looser.** MySQL 8.4 + takes `ESCAPE 1` and strings in double quotes. The parser refuses both, + in every dialect. - **It does not apply to sorting or field selection.** The sort parser reads `field [ASC|DESC]` and the fields parser reads a list of names. Both are the same in every SQL, so they take no dialect. ## Adding a dialect -A dialect is a set of operators and spellings, and sometimes lexing rules, -checked against the database it names. The next one planned is **MySQL -8.4** ([reference](https://dev.mysql.com/doc/refman/8.4/en/sql-statements.html)), -and it shows what a dialect can take: - -- **Operators it lacks:** `ILIKE`, `SIMILAR TO`, the `~` and `~~` families, - `ISNULL`/`NOTNULL` as postfix operators, `BETWEEN SYMMETRIC`, - `IS DISTINCT FROM`. -- **Operators it adds:** `<=>` (null-safe equality) and `REGEXP`/`RLIKE`, - which need new tokens and nodes. -- **Lexing rules:** MySQL reads a backslash inside a string as an escape by - default. The lexer must read strings the way MySQL does, or the parser and - the database could read the same text as different predicates. -- **Tests run against the database:** every accepted and refused row checked - against a running MySQL 8.4, as PostgreSQL 18's are. +A dialect is a set of operators and spellings, and sometimes rules for reading +strings, each checked against the database it names, with every accepted and +refused row run against it. MySQL 8.4 shows what one can take: + +- **Operators it lacks**, refused as the database refuses them. +- **Spellings it lacks**, refused with the one it has. +- **How it reads strings.** MySQL's backslash escapes made the lexer take the + dialect, since reading strings the wrong way lets a predicate past the + allow-list. + +What MySQL 8.4 has and this package does not parse yet: `<=>` (null-safe +equality) and `REGEXP`/`RLIKE`. Each would be a new operator and a new node, +and so a minor version of its own. diff --git a/docs/error-handling.md b/docs/error-handling.md index 2b3fcf0..b605142 100644 --- a/docs/error-handling.md +++ b/docs/error-handling.md @@ -92,9 +92,9 @@ The filter parser rejects, among others: - unknown fields (not in the allow-list), - disallowed operators (see [Configuration](configuration.md)), - operators and spellings the dialect does not have (`!=` or `ILIKE` in the - SQL standard's; see [the SQL dialect](dialects.md)), + SQL standard's, `SIMILAR TO` in MySQL 8.4's; see [SQL Dialects](dialects.md)), - a pattern that is not a string, and an `ESCAPE` that is not a string of - one character (empty is PostgreSQL's only), + one character (empty is PostgreSQL 18's and MySQL 8.4's only), - trailing or incomplete input — the **entire** string must form one expression (`first_name = 'John' garbage` is an error), - unterminated strings, unbalanced parentheses, and illegal characters. diff --git a/docs/filtering.md b/docs/filtering.md index 6fabeeb..b3f95da 100644 --- a/docs/filtering.md +++ b/docs/filtering.md @@ -5,9 +5,9 @@ grammar is PostgreSQL's by default; see the PostgreSQL 18 references for [SQL Syntax](https://www.postgresql.org/docs/18/sql-syntax.html), [Comparison Functions and Operators](https://www.postgresql.org/docs/18/functions-comparison.html) and [Pattern Matching](https://www.postgresql.org/docs/18/functions-matching.html). -With `WithDialect(qfv.DialectSQLStandard)` it is the SQL standard's (ISO/IEC -9075:2023), which this page's PostgreSQL extensions are not part of: see -[the SQL dialect](dialects.md). +With `WithDialect` it is the SQL standard's (ISO/IEC 9075:2023) or MySQL +8.4's, and neither has all of this page's PostgreSQL extensions: see +[SQL Dialects](dialects.md), which also has what each one accepts. ```go filterParser := qfv.NewFilterParser(allowedFields) diff --git a/docs/getting-started.md b/docs/getting-started.md index 34d97a2..268c00e 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -34,7 +34,9 @@ invalid or malicious input reaching your database. The filter parser checks an expression in the grammar of the SQL it will run as, and does not translate it: **PostgreSQL 18's by default**, or the SQL -standard's with `WithDialect(qfv.DialectSQLStandard)`. See +standard's (`qfv.DialectSQLStandard`) or MySQL 8.4's (`qfv.DialectMySQL`) +with `WithDialect`. In front of MySQL, choose MySQL's: it reads strings as +MySQL does. See [the SQL dialect](dialects.md). All three parsers are **safe for concurrent use** — build them once and share @@ -135,6 +137,6 @@ func main() { ## Next steps - [Filtering](filtering.md) — the full set of operators and predicates -- [SQL Dialects](dialects.md) — PostgreSQL 18 or SQL:2023, and what each accepts +- [SQL Dialects](dialects.md) — PostgreSQL 18, SQL:2023 or MySQL 8.4, and what each accepts - [Configuration](configuration.md) — the SQL dialect, and restricting operators and sort directions - [Error Handling](error-handling.md) — inspecting validation failures diff --git a/docs/migration.md b/docs/migration.md index b5bee5c..1f86a35 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -1,5 +1,27 @@ # Migration Guide +## v1.1.1 → v1.2.0 + +**`DialectMySQL`**: MySQL 8.4's grammar, with its default `sql_mode`. Nothing +changes for a caller that does not pass it. + +- **It reads strings as MySQL does**, with backslash escapes. A parser that + reads them as PostgreSQL does, in front of MySQL, accepted + `name = '\'' OR id > 0 #'` as one comparison of `name` with a string, and + MySQL 8.4 runs it with a predicate on `id`, outside the allow-list. **If + your filters run on MySQL, pass `WithDialect(qfv.DialectMySQL)`.** +- It refuses what MySQL 8.4 does not run: `ILIKE`, the `~~` operators, + `SIMILAR TO`, the `~` operators, `IS [NOT] DISTINCT FROM`, + `BETWEEN SYMMETRIC`/`ASYMMETRIC`, `ISNULL`/`NOTNULL`. It keeps `!=` and an + empty `ESCAPE`, which MySQL runs. +- `NewLexer` reads strings as before, as PostgreSQL does. +- An `ESCAPE` string is measured in characters with its doubled quotes + counted once in every dialect: an escape string of four single quotes is + one quote, where it was counted as empty. + +Every row is in [SQL Dialects](dialects.md), each one run against MySQL +8.4.13. + ## v1.1.0 → v1.1.1 `DialectPostgreSQL.String()` is `PostgreSQL 18`, where it was `PostgreSQL`: diff --git a/example_doc_test.go b/example_doc_test.go index dfb26a7..e4a1711 100644 --- a/example_doc_test.go +++ b/example_doc_test.go @@ -90,6 +90,27 @@ func ExampleWithDialect() { // error: != is not part of the SQL:2023 dialect: write <> } +// ExampleWithDialect_mySQL checks filters in MySQL 8.4's grammar. Its strings +// are read as MySQL reads them, with backslash escapes, so the second filter +// is refused: MySQL runs it with a predicate on a field the allow-list does +// not have, which a parser reading strings as PostgreSQL does would miss. +func ExampleWithDialect_mySQL() { + parser := qfv.NewFilterParser([]string{"name"}, qfv.WithDialect(qfv.DialectMySQL)) + + node, err := parser.Parse(`name = 'it\'s' AND name != 'x'`) + if err != nil { + fmt.Println("error:", err) + return + } + fmt.Println(node.String()) + + _, err = parser.Parse(`name = '\'' OR id > 0 #'`) + fmt.Println(err != nil) + // Output: + // ((name = 'it\'s') AND (name != 'x')) + // true +} + // ExampleFilterParser_errorInspection shows how to inspect validation failures. // Parse aggregates every problem with errors.Join, so errors.As extracts the // first *QFVFilterError cause. diff --git a/filter_lexer.go b/filter_lexer.go index 5b6972b..6bee05e 100644 --- a/filter_lexer.go +++ b/filter_lexer.go @@ -13,10 +13,20 @@ type Lexer struct { inputLen int pos int tokens []Token + // backslashEscapes is whether a backslash in a string escapes the + // character after it (MySQL 8.4's default), or is a character like any + // other (PostgreSQL 18, the SQL standard). + backslashEscapes bool } -// NewLexer creates a new lexer +// NewLexer creates a new lexer that reads strings as PostgreSQL 18 does: a +// backslash in a string is a character like any other. func NewLexer(input string) *Lexer { + return newLexer(input, DialectPostgreSQL) +} + +// newLexer creates a lexer that reads strings as the dialect does. +func newLexer(input string, dialect Dialect) *Lexer { var s scanner.Scanner s.Init(strings.NewReader(input)) // Customize scanner: recognize identifiers, numbers, strings. @@ -30,7 +40,7 @@ func NewLexer(input string) *Lexer { return ch == '_' || unicode.IsLetter(ch) || (i > 0 && (unicode.IsDigit(ch) || ch == '.')) } - l := &Lexer{s: s, input: input, pos: -1} // Start at -1, first Next moves to 0 + l := &Lexer{s: s, input: input, pos: -1, backslashEscapes: dialect.backslashEscapes()} // Start at -1, first Next moves to 0 l.inputLen = len(input) return l @@ -138,6 +148,18 @@ func (l *Lexer) Parse() { // Reached EOF without closing quote isValid = false break // Exit loop + } else if char == '\\' && l.backslashEscapes { + // MySQL 8.4: the backslash escapes the character after + // it, a quote included, and both are kept as written. A + // string that ends on its backslash is not closed. + escaped := l.s.Next() + if escaped == scanner.EOF { + isValid = false + break + } + + _, _ = sb.WriteRune(char) + _, _ = sb.WriteRune(escaped) } else if char == '\'' { // Found a quote. Is it an escaped quote ('') or the end? if l.s.Peek() == '\'' { diff --git a/filter_parser.go b/filter_parser.go index e58234d..fabd03f 100644 --- a/filter_parser.go +++ b/filter_parser.go @@ -7,7 +7,6 @@ import ( "strconv" "strings" "text/scanner" - "unicode/utf8" ) type QFVFilterError struct { @@ -76,7 +75,7 @@ func (p *FilterParser) Parse(input string) (Node, error) { allowedFields: p.allowedFields, allowedOps: p.allowedOps, dialect: p.dialect, - lexer: NewLexer(input), + lexer: newLexer(input, p.dialect), } st.lexer.Parse() @@ -156,9 +155,9 @@ func (p *filterParseState) requireOp(op Operator) { // requireSpelling refuses a spelling of an operator the dialect does not // have, naming the one it does: PostgreSQL's != is the standard's <>. -func (p *filterParseState) requireSpelling(spelling, standard string) { - if p.dialect.standard() { - p.addError(&QFVFilterError{Message: fmt.Sprintf("%s is not part of the %s dialect: write %s", spelling, p.dialect, standard)}) +func (p *filterParseState) requireSpelling(spelling, instead string) { + if !p.dialect.hasSpelling(spelling) { + p.addError(&QFVFilterError{Message: fmt.Sprintf("%s is not part of the %s dialect: write %s", spelling, p.dialect, instead)}) } } @@ -546,12 +545,10 @@ func (p *filterParseState) parseEscape(operator TokenType, keyword bool) Node { return escape } - text, _ := literal.Value.(string) - - switch length := utf8.RuneCountInString(text); { + switch length := literalLength(literal.Text, p.dialect.backslashEscapes()); { case length > 1: p.addError(&QFVFilterError{Message: "the escape character after ESCAPE must be empty or one character"}) - case length == 0 && p.dialect.standard(): + case length == 0 && !p.dialect.emptyEscape(): // PostgreSQL's manual: "a zero-length ESCAPE value is disallowed" // by the SQL standard. p.addError(&QFVFilterError{Message: fmt.Sprintf("an empty ESCAPE is not part of the %s dialect: the escape character is one character", p.dialect)}) @@ -610,8 +607,10 @@ func (p *filterParseState) parseBetweenOperator(field Node, isNot bool) Node { switch asciiUpper(p.currentToken.Value) { case "SYMMETRIC": isSymmetric = true + p.requireSpelling("BETWEEN SYMMETRIC", "BETWEEN, the lower bound first") p.nextToken() case "ASYMMETRIC": + p.requireSpelling("BETWEEN ASYMMETRIC", "BETWEEN") p.nextToken() } } @@ -794,3 +793,27 @@ func (p *filterParseState) parsePrimary() Node { } } } + +// literalLength is how many characters the string literal written as text +// holds: its quotes taken off, a doubled quote counted once, and, where a +// backslash escapes the character after it, the two counted once. It is how +// the escape character after ESCAPE is held to one: MySQL's ESCAPE '\\' is +// one backslash, and an ESCAPE string of four single quotes is one quote in +// every dialect. +func literalLength(text string, backslashEscapes bool) int { + runes := []rune(strings.TrimSuffix(strings.TrimPrefix(text, "'"), "'")) + length := 0 + + for i := 0; i < len(runes); i++ { + switch { + case runes[i] == '\'' && i+1 < len(runes) && runes[i+1] == '\'': + i++ + case backslashEscapes && runes[i] == '\\' && i+1 < len(runes): + i++ + } + + length++ + } + + return length +}