From aee2b514bd9491512e01571f32b6202162a408a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Gonz=C3=A1lez=20Di=20Antonio?= Date: Sat, 24 Oct 2026 19:30:50 +0200 Subject: [PATCH] docs: the SQL dialects have a page of their own What a dialect is was a section of the configuration guide. It has its own page now, docs/dialects.md: why a filter parser needs one (it checks and does not translate, so accepted has to mean the database runs it), how to choose it, how a filter is checked against it, the dialect and the allow-list as two narrowings, every construct in PostgreSQL 18 and SQL:2023 with the standard's feature ids and the manual's words, what each refusal says, what a dialect does not do, and what adding one (MySQL 8.4 next) takes. Its three diagrams were rendered. Every page that pointed at the section points at it, and the configuration guide keeps a summary. TestTheDialectsPageQuotesTheParser reads the page's table of refusals and holds each message to what Parse answers. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- dialect_docs_test.go | 60 +++++++++++++ docs.go | 2 +- docs/README.md | 3 +- docs/configuration.md | 47 ++-------- docs/dialects.md | 185 ++++++++++++++++++++++++++++++++++++++++ docs/error-handling.md | 2 +- docs/filtering.md | 2 +- docs/getting-started.md | 3 +- docs/migration.md | 6 +- 10 files changed, 265 insertions(+), 47 deletions(-) create mode 100644 dialect_docs_test.go create mode 100644 docs/dialects.md diff --git a/README.md b/README.md index 7adf0d0..a373b5c 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ func main() { `SIMILAR TO`, POSIX regex, `IS NULL`, `IS [NOT] TRUE/FALSE/UNKNOWN`, and `IS [NOT] DISTINCT FROM`. The default dialect is PostgreSQL 18's; `WithDialect(DialectSQLStandard)` accepts SQL:2023 (ISO/IEC 9075:2023) and - refuses PostgreSQL's own spellings. See [the SQL dialect](docs/configuration.md#the-sql-dialect). + refuses PostgreSQL's own spellings. See [the SQL dialect](docs/dialects.md). - **Configurable** — restrict which operators or sort directions are allowed. - **Returns an AST** you can walk or render, with precise, aggregated errors. - **Concurrency-safe** parsers. diff --git a/dialect_docs_test.go b/dialect_docs_test.go new file mode 100644 index 0000000..25c58fa --- /dev/null +++ b/dialect_docs_test.go @@ -0,0 +1,60 @@ +package qfv + +import ( + "os" + "regexp" + "strings" + "testing" +) + +// docs/dialects.md quotes, word for word, what Parse answers in the SQL:2023 +// dialect. A message that changes here and not there would leave the page +// telling clients to look for an error they never get, so each row of its +// table is parsed and held to the parser. +func TestTheDialectsPageQuotesTheParser(t *testing.T) { + t.Parallel() + + raw, err := os.ReadFile("docs/dialects.md") + if err != nil { + t.Fatal(err) + } + + page := string(raw) + + start := strings.Index(page, "## What the refusals say") + if start < 0 { + t.Fatal(`docs/dialects.md has no "What the refusals say" section`) + } + + section := page[start:] + if end := strings.Index(section[1:], "\n## "); end >= 0 { + section = section[:end+1] + } + + row := regexp.MustCompile("(?m)^\\| (`[^|]+`(?:, `[^|]+`)*) \\| `([^`]+)` \\|$") + code := regexp.MustCompile("`([^`]+)`") + parser := NewFilterParser([]string{"id", "name"}, WithDialect(DialectSQLStandard)) + + checked := 0 + + for _, m := range row.FindAllStringSubmatch(section, -1) { + for _, input := range code.FindAllStringSubmatch(m[1], -1) { + checked++ + + _, err := parser.Parse(input[1]) + if err == nil { + t.Errorf("the page says %q is refused, and Parse accepts it", input[1]) + + continue + } + + if !strings.Contains(err.Error(), m[2]) { + t.Errorf("the page says %q answers %q; Parse answers %q", input[1], m[2], err.Error()) + } + } + } + + if checked < 9 { + t.Fatalf("read %d inputs from the page's table, want at least 9: the table's shape changed and this test reads nothing", checked) + } +} diff --git a/docs.go b/docs.go index 21758f2..9e45433 100644 --- a/docs.go +++ b/docs.go @@ -8,7 +8,7 @@ // PostgreSQL 18 runs it; with WithDialect(DialectSQLStandard), the SQL // standard's (ISO/IEC 9075:2023), which refuses PostgreSQL's own spellings // such as != and ILIKE. WithAllowedOperators and WithAllowedOperatorGroups -// narrow the operators further. +// narrow the operators further. docs/dialects.md explains the dialects. // // Every parser is safe for concurrent use. The documentation, with the full // grammar, is in the repository's docs directory. diff --git a/docs/README.md b/docs/README.md index a436db4..dffe8f4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -41,7 +41,8 @@ flowchart TB | --- | --- | | [Getting Started](getting-started.md) | Install, update, and a complete runnable example of the three parsers | | [Filtering](filtering.md) | The full filter grammar — every operator/predicate, `ESCAPE`, dotted fields, and worked examples | -| [Configuration](configuration.md) | The SQL dialect (PostgreSQL or the SQL standard), and restricting which operators and sort directions are allowed | +| [SQL Dialects](dialects.md) | PostgreSQL 18 (the default) or SQL:2023: why a filter parser needs one, choosing it, how a filter is checked, every construct in each | +| [Configuration](configuration.md) | Choosing the dialect, and restricting which operators and sort directions are allowed | | [Error Handling](error-handling.md) | Error shapes and how to inspect validation failures | | [Migration Guide](migration.md) | Breaking changes and how to upgrade between versions | diff --git a/docs/configuration.md b/docs/configuration.md index 070f1ff..787c8bc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -7,54 +7,21 @@ time. ## The SQL dialect -A filter is checked in the grammar of the SQL it will run as: the parser -checks an expression, it does not translate one, so what it accepts has to -be what the database runs. `WithDialect` chooses it. - -| Dialect | Reference | What it accepts | -| --- | --- | --- | -| `DialectPostgreSQL` (the default) | PostgreSQL 18, and its manual's [SQL Syntax](https://www.postgresql.org/docs/18/sql-syntax.html) chapter | every operator this package knows | -| `DialectSQLStandard` | ISO/IEC 9075:2023 (SQL:2023), Part 2, as PostgreSQL's [SQL Conformance](https://www.postgresql.org/docs/18/features-sql-standard.html) appendix names its features | PostgreSQL's grammar without what PostgreSQL's manual calls its own | +The filter parser checks a filter in the grammar of the SQL it will run as: +PostgreSQL 18's by default, or the SQL standard's (SQL:2023) with +`WithDialect`. Everything about it is on its own page, [SQL Dialects](dialects.md): +why there is one, how to choose it, how it is checked, a table of every +construct in each, and what it does not do. ```go p := qfv.NewFilterParser(fields, qfv.WithDialect(qfv.DialectSQLStandard)) p.Parse("id <> 1 AND name LIKE 'x!%' ESCAPE '!'") // ok p.Parse("id != 1") // error: != is not part of the SQL:2023 dialect: write <> -p.Parse("name ILIKE 'x%'") // error: operator "ILIKE" is not part of the SQL:2023 dialect ``` -What the SQL standard dialect refuses, and what the standard writes instead: - -| PostgreSQL's | The standard's | Why | -| --- | --- | --- | -| `!=` | `<>` | "`<>` is the standard SQL notation for not equal" | -| `ILIKE`, `NOT ILIKE`, `~~*`, `!~~*` | — | "not in the SQL standard but is a PostgreSQL extension" | -| `~~`, `!~~` | `LIKE`, `NOT LIKE` | "PostgreSQL-specific" | -| `~`, `~*`, `!~`, `!~*` | `LIKE_REGEX` (not parsed by this package) | the standard's regular expressions are XQuery's | -| `ISNULL`, `NOTNULL` | `IS NULL`, `IS NOT NULL` | "nonstandard syntax" | -| `ESCAPE ''` | one character | "a zero-length `ESCAPE` value is disallowed" | - -What it keeps, each with the standard's feature: the comparison predicate -(E061-01), `BETWEEN` (E061-02) and its `SYMMETRIC` form (T461), `IN` with a -list of values (E061-03), `LIKE` (E061-04) and its `ESCAPE` clause (E061-05), -the NULL predicate (E061-06), `SIMILAR TO` (T141), `IS [NOT] DISTINCT FROM` -(T151, T152), the truth value tests `IS [NOT] TRUE/FALSE/UNKNOWN` (F571), and -`AND`, `OR`, `NOT`. - -One difference cannot be checked: without an `ESCAPE` clause the standard has -no escape character, where PostgreSQL's is the backslash, so a pattern with a -backslash in it means one thing to each. - -The dialect comes before the allow-list: an operator the dialect does not have -is refused whether `WithAllowedOperators` allows it or not. - -A dialect is a value of the `Dialect` type, and the zero value is -`DialectPostgreSQL`; there is no way to name one that does not exist. Each -names its version: `String()` is `PostgreSQL 18` or `SQL:2023`. -Another database's grammar (MySQL's, for instance: `REGEXP`, `<=>`, -backslash escapes in strings) is a dialect to add, with its own tests run -against that database. +The dialect comes before the allow-list below: an operator the dialect does not +have is refused whether it is allowed or not. ## Restricting filter operators diff --git a/docs/dialects.md b/docs/dialects.md new file mode 100644 index 0000000..f83f449 --- /dev/null +++ b/docs/dialects.md @@ -0,0 +1,185 @@ +# SQL Dialects + +A filter that the parser accepts is text you will run as SQL. The **dialect** +says which SQL: the parser accepts exactly what that database, or that +standard, accepts, and refuses the rest with a message that says what to +write instead. + +| Dialect | Value | `String()` | Reference | +| --- | --- | --- | --- | +| PostgreSQL 18 (the default) | `qfv.DialectPostgreSQL` | `PostgreSQL 18` | PostgreSQL 18, and its manual's [SQL Syntax](https://www.postgresql.org/docs/18/sql-syntax.html) chapter | +| SQL:2023 | `qfv.DialectSQLStandard` | `SQL:2023` | ISO/IEC 9075:2023, Part 2 (Foundation), as PostgreSQL's [SQL Conformance](https://www.postgresql.org/docs/18/features-sql-standard.html) appendix names its features | + +```go +// PostgreSQL 18: the default, no option needed. +p := qfv.NewFilterParser(fields) + +// The SQL standard. +s := qfv.NewFilterParser(fields, qfv.WithDialect(qfv.DialectSQLStandard)) +``` + +## Why a filter parser needs a dialect + +The parser **checks** an expression; it does not **translate** one. The usual +way to use it is to validate the `filter` query parameter and then put that +same text in a `WHERE` clause. So "accepted" has to mean "the database runs +it", and every SQL accepts something different: + +- `id != 1` runs on PostgreSQL 18 and is not SQL:2023, which writes `id <> 1`. +- `name ILIKE 'x%'` runs on PostgreSQL 18 and is not SQL:2023, which has no + case-insensitive `LIKE`. +- `name LIKE 'x%' ESCAPE ''` runs on PostgreSQL 18; SQL:2023 disallows an + empty escape. + +When the parser and the database disagree, the best outcome is a database +error on a request the parser called valid. The worst is a filter the parser +read as one predicate and the database reads as another. The dialect is how +you tell the parser which database it is speaking for. + +## Choosing a dialect + +```mermaid +flowchart TD + Q{"What runs the filter's text?"} + Q -->|"PostgreSQL 18"| PG["DialectPostgreSQL
(the default: pass nothing)"] + Q -->|"several databases, or portable SQL"| STD["DialectSQLStandard"] + Q -->|"another database (MySQL 8.4, …)"| NONE["no dialect yet:
see Adding a dialect"] + STD --> NOTE["the filter uses only the standard's spellings:
it runs on PostgreSQL 18 too"] +``` + +- **PostgreSQL 18** is the default and the grammar every version of this + package has parsed. Use it when PostgreSQL runs the filter. +- **SQL:2023** accepts a subset of PostgreSQL 18's grammar: everything it + accepts runs on PostgreSQL 18 as well. Use it when the filter must stay + portable, or when you want clients to write the standard's spellings. +- Only these two exist. `Dialect` is a struct with unexported fields, so + there is no way to name a dialect the package does not have. Its zero value + is `DialectPostgreSQL`. + +## How a filter is checked against it + +The lexer turns the text into tokens. The parser then checks every field +against the allow-list, and every operator against the dialect, first as an +operator and then in the spelling it was written in. Only then does it check +the operator against the operator allow-list. Every problem is collected, and +`Parse` returns them joined. + +```mermaid +flowchart TD + IN["filter text"] --> LEX["Lexer: tokens"] + LEX --> OP["an operator, as written
(e.g. !=, ILIKE, ~~, ISNULL)"] + OP --> D1{"does the dialect have
the operator?"} + D1 -->|"no: ILIKE, ~ in SQL:2023"| E1["operator "ILIKE" is not part of
the SQL:2023 dialect"] + D1 -->|"yes"| D2{"does it have
this spelling of it?"} + D2 -->|"no: != ~~ !~~ ISNULL NOTNULL
in SQL:2023"| E2["!= is not part of the SQL:2023
dialect: write <>"] + D2 -->|"yes"| A{"allowed by
WithAllowedOperators?"} + A -->|"no"| E3["operator "LIKE" is not allowed"] + A -->|"yes, or no allow-list"| OK["into the AST"] + E1 --> J["errors.Join: every problem"] + E2 --> J + E3 --> J +``` + +### The dialect and the operator allow-list + +The dialect says what the SQL has; `WithAllowedOperators` and +`WithAllowedOperatorGroups` say what you let clients use. An operator must +pass both, so the dialect always wins: allowing `OpILike` does not put +`ILIKE` in SQL:2023. + +```mermaid +flowchart LR + ALL(["every operator the parser knows"]) --> DIA["∩ the dialect's operators
Dialect.Operators()"] + DIA --> ALW["∩ the allow-list
(when there is one)"] + ALW --> USE(["what a filter may use"]) +``` + +`Dialect.Operators()` returns the dialect's operators, for building an +allow-list or showing clients what they may write. + +## What each dialect accepts + +Every construct the filter parser knows, in each dialect. A ✅ is accepted +and a ❌ refused, with the spelling the dialect does have. + +| Construct | PostgreSQL 18 | SQL:2023 | SQL:2023 feature | +| --- | --- | --- | --- | +| `=`, `<`, `<=`, `>`, `>=` | ✅ | ✅ | E061-01 | +| `<>` | ✅ | ✅ | E061-01 | +| `!=` | ✅ | ❌ write `<>` | — | +| `AND`, `OR`, `NOT`, `( … )` | ✅ | ✅ | — | +| `BETWEEN`, `NOT BETWEEN` | ✅ | ✅ | E061-02 | +| `BETWEEN SYMMETRIC`, `BETWEEN ASYMMETRIC` | ✅ | ✅ | T461 | +| `IN ( … )`, `NOT IN ( … )` | ✅ | ✅ | E061-03 | +| `LIKE`, `NOT LIKE` | ✅ | ✅ | E061-04 | +| `LIKE … ESCAPE 'c'` | ✅ | ✅ | E061-05 | +| `LIKE … ESCAPE ''` (escaping off) | ✅ | ❌ one character | — | +| `ILIKE`, `NOT ILIKE` | ✅ | ❌ no such predicate | — | +| `~~`, `!~~` | ✅ | ❌ write `LIKE`, `NOT LIKE` | — | +| `~~*`, `!~~*` | ✅ | ❌ no such predicate | — | +| `SIMILAR TO`, `NOT SIMILAR TO`, with `ESCAPE` | ✅ | ✅ | T141 | +| `~`, `~*`, `!~`, `!~*` | ✅ | ❌ the standard's is `LIKE_REGEX`, which the parser does not read | — | +| `IS NULL`, `IS NOT NULL` | ✅ | ✅ | E061-06 | +| `ISNULL`, `NOTNULL` | ✅ | ❌ write `IS NULL`, `IS NOT NULL` | — | +| `IS [NOT] TRUE`, `FALSE`, `UNKNOWN` | ✅ | ✅ | F571 | +| `IS [NOT] DISTINCT FROM` | ✅ | ✅ | T151, T152 | +| string, number and boolean literals | ✅ | ✅ | — | + +Where the ❌ comes from: PostgreSQL's own manual calls each of these its own. +`<>` is "the standard SQL notation for not equal", and `!=` is an alias. +`ILIKE` is "not in the SQL standard but is a PostgreSQL extension". The `~~` +operators "are PostgreSQL-specific". `ISNULL` and `NOTNULL` are "nonstandard +syntax". "A zero-length ESCAPE value is disallowed" by the standard. + +## What the refusals say + +The SQL:2023 dialect's refusals, as `Parse` writes them: + +| Input | Error | +| --- | --- | +| `id != 1` | `!= is not part of the SQL:2023 dialect: write <>` | +| `name ~~ 'x%'` | `~~ is not part of the SQL:2023 dialect: write LIKE` | +| `name !~~ 'x%'` | `!~~ is not part of the SQL:2023 dialect: write NOT LIKE` | +| `name ISNULL` | `ISNULL is not part of the SQL:2023 dialect: write IS NULL` | +| `name NOTNULL` | `NOTNULL is not part of the SQL:2023 dialect: write IS NOT NULL` | +| `name ILIKE 'x%'`, `name ~~* 'x%'` | `operator "ILIKE" is not part of the SQL:2023 dialect` | +| `name ~ '^x'` | `operator "~" is not part of the SQL:2023 dialect` | +| `name LIKE 'x%' ESCAPE ''` | `an empty ESCAPE is not part of the SQL:2023 dialect: the escape character is one character` | + +Each is a `*qfv.QFVFilterError`, joined with any other problem in the same +filter (see [Error Handling](error-handling.md)). + +## What a dialect does not do + +- **It does not translate.** A filter written for PostgreSQL 18 is not + rewritten for another database. Choose the dialect of the database that + runs it. +- **It does not know your columns.** Types are the database's: `id = 'x'` is + accepted in both dialects and fails on an integer column. Whether `LIKE` + is case-sensitive depends on the column's collation. +- **It does not change what a pattern means.** Without `ESCAPE`, PostgreSQL + 18's escape character is the backslash, and SQL:2023 has none, so + `name LIKE '100\%'` matches `100%` on PostgreSQL 18, and `100\` followed + by anything under the standard. The parser cannot see which you + meant, so write an `ESCAPE` clause when it matters. +- **It does not apply to sorting or field selection.** The sort parser reads + `field [ASC|DESC]` and the fields parser reads a list of names. Both are the + same in every SQL, so they take no dialect. + +## Adding a dialect + +A dialect is a set of operators and spellings, and sometimes lexing rules, +checked against the database it names. The next one planned is **MySQL +8.4** ([reference](https://dev.mysql.com/doc/refman/8.4/en/sql-statements.html)), +and it shows what a dialect can take: + +- **Operators it lacks:** `ILIKE`, `SIMILAR TO`, the `~` and `~~` families, + `ISNULL`/`NOTNULL` as postfix operators, `BETWEEN SYMMETRIC`, + `IS DISTINCT FROM`. +- **Operators it adds:** `<=>` (null-safe equality) and `REGEXP`/`RLIKE`, + which need new tokens and nodes. +- **Lexing rules:** MySQL reads a backslash inside a string as an escape by + default. The lexer must read strings the way MySQL does, or the parser and + the database could read the same text as different predicates. +- **Tests run against the database:** every accepted and refused row checked + against a running MySQL 8.4, as PostgreSQL 18's are. diff --git a/docs/error-handling.md b/docs/error-handling.md index 0f1d6a3..2b3fcf0 100644 --- a/docs/error-handling.md +++ b/docs/error-handling.md @@ -92,7 +92,7 @@ The filter parser rejects, among others: - unknown fields (not in the allow-list), - disallowed operators (see [Configuration](configuration.md)), - operators and spellings the dialect does not have (`!=` or `ILIKE` in the - SQL standard's; see [the SQL dialect](configuration.md#the-sql-dialect)), + SQL standard's; see [the SQL dialect](dialects.md)), - a pattern that is not a string, and an `ESCAPE` that is not a string of one character (empty is PostgreSQL's only), - trailing or incomplete input — the **entire** string must form one expression diff --git a/docs/filtering.md b/docs/filtering.md index b1e7913..6fabeeb 100644 --- a/docs/filtering.md +++ b/docs/filtering.md @@ -7,7 +7,7 @@ grammar is PostgreSQL's by default; see the PostgreSQL 18 references for and [Pattern Matching](https://www.postgresql.org/docs/18/functions-matching.html). With `WithDialect(qfv.DialectSQLStandard)` it is the SQL standard's (ISO/IEC 9075:2023), which this page's PostgreSQL extensions are not part of: see -[the SQL dialect](configuration.md#the-sql-dialect). +[the SQL dialect](dialects.md). ```go filterParser := qfv.NewFilterParser(allowedFields) diff --git a/docs/getting-started.md b/docs/getting-started.md index 7b6c917..34d97a2 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -35,7 +35,7 @@ invalid or malicious input reaching your database. The filter parser checks an expression in the grammar of the SQL it will run as, and does not translate it: **PostgreSQL 18's by default**, or the SQL standard's with `WithDialect(qfv.DialectSQLStandard)`. See -[the SQL dialect](configuration.md#the-sql-dialect). +[the SQL dialect](dialects.md). All three parsers are **safe for concurrent use** — build them once and share them across goroutines. @@ -135,5 +135,6 @@ func main() { ## Next steps - [Filtering](filtering.md) — the full set of operators and predicates +- [SQL Dialects](dialects.md) — PostgreSQL 18 or SQL:2023, and what each accepts - [Configuration](configuration.md) — the SQL dialect, and restricting operators and sort directions - [Error Handling](error-handling.md) — inspecting validation failures diff --git a/docs/migration.md b/docs/migration.md index 8943a3b..b5bee5c 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -6,6 +6,10 @@ each dialect names its version, as `DialectSQLStandard`'s `SQL:2023` does. Only a caller that compares the string changes; the grammar does not. +The dialects have a page of their own, [SQL Dialects](dialects.md): why a +filter parser needs one, how to choose it, how a filter is checked against +it, and every construct in each. + ## v1.0.5 → v1.1.0 Two additions: the SQL dialect a filter is written in, and the `ESCAPE` @@ -19,7 +23,7 @@ with `ESCAPE` accepted. `DialectSQLStandard`, ISO/IEC 9075:2023, which refuses PostgreSQL's own spellings (`!=`, `ILIKE`, the `~~` and `~` operators, `ISNULL`/`NOTNULL`, an empty `ESCAPE`). The table is in -[configuration.md](configuration.md#the-sql-dialect). Nothing changes +[dialects.md](dialects.md). Nothing changes without the option. ### The `ESCAPE` clause