From c439c27f5a4f73f531c04ae6d62047de203506dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Gonz=C3=A1lez=20Di=20Antonio?= Date: Sat, 24 Oct 2026 19:12:31 +0200 Subject: [PATCH] feat: the ESCAPE clause of the pattern operators PostgreSQL writes "string LIKE pattern [ESCAPE escape-character]", and the same after SIMILAR TO; the clause was not part of this grammar, and "name LIKE 'x!%' ESCAPE '!'" was refused. It follows LIKE, ILIKE, their negations and SIMILAR TO, written with the keyword, in any case. The escape character is a string of one character, or empty to turn escaping off. What PostgreSQL 18.6 refuses is refused: two characters (invalid escape string), a number, a boolean or a column as the escape, and the clause after the ~~ operators, which take none. BinaryOperatorNode and SimilarToNode carry it as Escape, nil without a clause, which is every filter v1.0.5 accepted; String() writes it. A new field and nothing else changed: a minor version. Co-Authored-By: Claude Opus 5.5 --- docs/filtering.md | 23 +++++-- docs/migration.md | 32 +++++++++ filter_nodes.go | 30 +++++++-- filter_parser.go | 67 ++++++++++++++++--- filter_pg_patterns_test.go | 131 ++++++++++++++++++++++++++++++++++++- 5 files changed, 259 insertions(+), 24 deletions(-) diff --git a/docs/filtering.md b/docs/filtering.md index 8f57764..4fe6899 100644 --- a/docs/filtering.md +++ b/docs/filtering.md @@ -99,9 +99,15 @@ to. PostgreSQL has no `~~` that takes one, whatever the column's type is, and answers "operator does not exist". A number written as a string is a pattern: `name LIKE '1'`. -- **The `ESCAPE` clause is not supported**; `name LIKE 'x!%' ESCAPE '!'` is - refused. A pattern is written with the default escape character, the - backslash, which a string literal keeps as it is: `name LIKE '100\%'`. +- **The `ESCAPE` clause** follows `LIKE`, `ILIKE`, their negations and + `SIMILAR TO`, written with the keyword: `name LIKE 'x!%' ESCAPE '!'`. The + escape character is a string of one character, or empty to turn escaping + off (`ESCAPE ''`). PostgreSQL refuses the rest, and so does the parser: + two characters (`invalid escape string`), a number, a boolean or a column + as the escape, and the clause after `~~`, `~~*`, `!~~` and `!~~*`, which + take none. Without the clause the escape character is the backslash, which + a string literal keeps as it is: `name LIKE '100\%'`. The node carries it + as `Escape`, nil when there is no clause. - **What is in the pattern is not looked at**: see "Patterns are not compiled", under [AST nodes](#ast-nodes). @@ -305,8 +311,15 @@ classDiagram class BinaryOperatorNode { +Left Node +Right Node + +Escape Node +Operator TokenType } + class SimilarToNode { + +Field Node + +Pattern Node + +Escape Node + +IsNot bool + } class InNode { +Field Node +Values []Node @@ -331,7 +344,7 @@ Common concrete types: | Node | Produced by | | --- | --- | -| `BinaryOperatorNode` | comparisons, `AND`/`OR`, `LIKE`/`ILIKE` | +| `BinaryOperatorNode` | comparisons, `AND`/`OR`, `LIKE`/`ILIKE` (`Escape`) | | `UnaryOperatorNode` | a standalone leading `NOT` | | `GroupNode` | parenthesized expressions | | `InNode` | `IN` / `NOT IN` (`IsNot`) | @@ -339,7 +352,7 @@ Common concrete types: | `IsNullNode` | `IS [NOT] NULL`, `ISNULL`/`NOTNULL` (`IsNot`) | | `BooleanTestNode` | `IS [NOT] TRUE/FALSE/UNKNOWN` (`Value`, `IsNot`) | | `DistinctNode` | `IS [NOT] DISTINCT FROM` (`Value`, `IsNot`) | -| `SimilarToNode` | `SIMILAR TO` / `NOT SIMILAR TO` (`IsNot`) | +| `SimilarToNode` | `SIMILAR TO` / `NOT SIMILAR TO` (`IsNot`, `Escape`) | | `RegexMatchNode` | `~ ~* !~ !~*` (`IsNot`, `IsCaseInsensitive`) | | `IdentifierNode` / `LiteralNode` | field names / literal values | diff --git a/docs/migration.md b/docs/migration.md index c775659..8e2e3c1 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -1,5 +1,37 @@ # Migration Guide +## v1.0.5 → v1.1.0 + +The `ESCAPE` clause of the pattern operators, which PostgreSQL writes +`string LIKE pattern [ESCAPE escape-character]` and the same after +`SIMILAR TO`. Each row was run against PostgreSQL 18.6. + +**Accepted now, and refused before:** + +| Input | v1.0.5 | v1.1.0 | PostgreSQL 18 | +| --- | --- | --- | --- | +| `name LIKE 'x!%' ESCAPE '!'`, after `ILIKE`, `NOT LIKE`, `NOT ILIKE`, `SIMILAR TO` and `NOT SIMILAR TO` too, the keyword in any case | ❌ error | ✅ accepted | runs | +| an empty escape: `name LIKE 'x%' ESCAPE ''` | ❌ error | ✅ accepted | runs, with escaping off | + +**Refused, as PostgreSQL refuses them:** + +| Input | v1.1.0 | PostgreSQL 18 | +| --- | --- | --- | +| two characters: `ESCAPE '!!'` | ❌ error: `must be empty or one character` | `invalid escape string` | +| a number, a boolean or a column: `ESCAPE 1`, `ESCAPE TRUE`, `ESCAPE name` | ❌ error: `expected a string escape character after ESCAPE` | `function like_escape(unknown, integer) does not exist`; `invalid escape string` per row | +| after the operator spellings: `name ~~ 'x!%' ESCAPE '!'`, and after `~~*`, `!~~`, `!~~*` | ❌ error: `ESCAPE follows a pattern written with LIKE, ILIKE or SIMILAR TO` | `syntax error at or near "ESCAPE"` | + +**For callers that walk the tree.** `BinaryOperatorNode` and `SimilarToNode` +have a new field, `Escape`: the escape character, a string `LiteralNode`, or +nil when there is no clause, which is every filter v1.0.5 accepted. Their +`String()` writes the clause when there is one. Nothing else changes, so the +version is a minor one. + +**For callers that hand the filter to PostgreSQL** as written: the clause +goes with it and runs as the database runs it. A caller that prefixes field +names in the text (`name` to `t.name`) leaves `ESCAPE` alone, since it is no +field. + ## v1.0.3 → v1.0.4 Three places where the filter parser and PostgreSQL disagreed about an diff --git a/filter_nodes.go b/filter_nodes.go index 3669cf0..4180278 100644 --- a/filter_nodes.go +++ b/filter_nodes.go @@ -40,7 +40,10 @@ type SimilarToNode struct { baseNode Field Node Pattern Node - IsNot bool // true for NOT SIMILAR TO + // Escape is the ESCAPE clause's escape character, a string LiteralNode of + // zero or one character, or nil when there is no clause. + Escape Node + IsNot bool // true for NOT SIMILAR TO } func (n *SimilarToNode) Type() NodeType { @@ -51,9 +54,9 @@ func (n *SimilarToNode) Type() NodeType { } func (n *SimilarToNode) String() string { if n.IsNot { - return fmt.Sprintf("%s NOT SIMILAR TO %s", n.Field.String(), n.Pattern.String()) + return fmt.Sprintf("%s NOT SIMILAR TO %s%s", n.Field.String(), n.Pattern.String(), escapeClause(n.Escape)) } - return fmt.Sprintf("%s SIMILAR TO %s", n.Field.String(), n.Pattern.String()) + return fmt.Sprintf("%s SIMILAR TO %s%s", n.Field.String(), n.Pattern.String(), escapeClause(n.Escape)) } func (n *SimilarToNode) Pos() scanner.Position { return n.pos } @@ -148,14 +151,29 @@ func (n *UnaryOperatorNode) Pos() scanner.Position { return n.pos } // BinaryOperatorNode represents a binary operation (AND, OR) type BinaryOperatorNode struct { baseNode - Left Node - Right Node + Left Node + Right Node + // Escape is the ESCAPE clause of LIKE, ILIKE and their negations, written + // with the keyword: a string LiteralNode of zero or one character, or nil + // when there is no clause. It is nil for every other operator, and for + // the ~~ operators, which take no clause. + Escape Node Operator TokenType } func (n *BinaryOperatorNode) Type() NodeType { return NodeTypeBinaryOperator } func (n *BinaryOperatorNode) String() string { - return fmt.Sprintf("(%s %s %s)", n.Left.String(), n.Operator, n.Right.String()) + return fmt.Sprintf("(%s %s %s%s)", n.Left.String(), n.Operator, n.Right.String(), escapeClause(n.Escape)) +} + +// escapeClause is " ESCAPE " for a pattern written with one, and +// nothing for a pattern without. +func escapeClause(escape Node) string { + if escape == nil { + return "" + } + + return " ESCAPE " + escape.String() } func (n *BinaryOperatorNode) Pos() scanner.Position { return n.pos } diff --git a/filter_parser.go b/filter_parser.go index 9b4daa5..cd2533f 100644 --- a/filter_parser.go +++ b/filter_parser.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" "text/scanner" + "unicode/utf8" ) type QFVFilterError struct { @@ -302,20 +303,22 @@ func (p *filterParseState) parseComparison() Node { return p.parseComparisonOperator(field) case TokenOperatorLike: p.requireOp(OpLike) + keyword := p.isKeyword("LIKE") p.nextToken() // Consume LIKE (or ~~) - return p.parseLikeOperator(field, TokenOperatorLike) + return p.parseLikeOperator(field, TokenOperatorLike, keyword) case TokenOperatorILike: p.requireOp(OpILike) + keyword := p.isKeyword("ILIKE") p.nextToken() // Consume ILIKE (or ~~*) - return p.parseLikeOperator(field, TokenOperatorILike) + return p.parseLikeOperator(field, TokenOperatorILike, keyword) case TokenOperatorNotLike: p.requireOp(OpLike) p.nextToken() // Consume !~~ (operator form of NOT LIKE) - return p.parseLikeOperator(field, TokenOperatorNotLike) + return p.parseLikeOperator(field, TokenOperatorNotLike, false) case TokenOperatorNotILike: p.requireOp(OpILike) p.nextToken() // Consume !~~* (operator form of NOT ILIKE) - return p.parseLikeOperator(field, TokenOperatorNotILike) + return p.parseLikeOperator(field, TokenOperatorNotILike, false) case TokenOperatorIn: p.requireOp(OpIn) p.nextToken() // Consume IN @@ -378,7 +381,7 @@ func (p *filterParseState) parseComparison() Node { p.requireOp(OpLike) p.nextToken() // Consume LIKE - return p.parseLikeOperator(field, TokenOperatorNotLike) + return p.parseLikeOperator(field, TokenOperatorNotLike, true) case TokenOperatorILike: if !p.isKeyword("ILIKE") { return p.unexpectedAfterNot(field) @@ -386,7 +389,7 @@ func (p *filterParseState) parseComparison() Node { p.requireOp(OpILike) p.nextToken() // Consume ILIKE - return p.parseLikeOperator(field, TokenOperatorNotILike) + return p.parseLikeOperator(field, TokenOperatorNotILike, true) case TokenOperatorSimilarTo: p.requireOp(OpSimilarTo) p.nextToken() // Consume SIMILAR @@ -439,6 +442,7 @@ func (p *filterParseState) parseSimilarToOperator(field Node, isNot bool) Node { baseNode: baseNode{pos: pos}, Field: field, Pattern: pattern, + Escape: p.parseEscape(operator, true), IsNot: isNot, } } @@ -452,8 +456,7 @@ func (p *filterParseState) parseSimilarToOperator(field Node, isNot bool) Node { // type is, because no ~~ takes one. So a literal of another kind is refused // here, as it already was for the POSIX operators (~, ~*, !~, !~*). // -// The ESCAPE clause is not part of this grammar: a pattern is written with -// the default escape character, the backslash. +// The ESCAPE clause that may follow it is read by parseEscape. func (p *filterParseState) parsePattern(operator TokenType) Node { pattern := p.parsePrimary() @@ -467,18 +470,62 @@ func (p *filterParseState) parsePattern(operator TokenType) Node { // parseLikeOperator parses a LIKE/ILIKE/NOT LIKE/NOT ILIKE operator. // Expects the current token to be the pattern after the operator was consumed. // operator is one of TokenOperatorLike, TokenOperatorILike, TokenOperatorNotLike, -// or TokenOperatorNotILike. -func (p *filterParseState) parseLikeOperator(field Node, operator TokenType) Node { +// or TokenOperatorNotILike; keyword is whether it was written as a word +// (LIKE, ILIKE) rather than as a ~~ operator, which takes no ESCAPE clause. +func (p *filterParseState) parseLikeOperator(field Node, operator TokenType, keyword bool) Node { pos := p.lexer.Current().Pos // Use position of the operator token (already consumed) pattern := p.parsePattern(operator) return &BinaryOperatorNode{ baseNode: baseNode{pos: pos}, Left: field, Right: pattern, + Escape: p.parseEscape(operator, keyword), Operator: operator, } } +// parseEscape reads the ESCAPE clause after a pattern, and returns its +// escape character, or nil when the pattern has no clause. +// +// PostgreSQL writes it "string LIKE pattern [ESCAPE escape-character]", and +// the same after SIMILAR TO. What it refuses is refused here, so that a +// filter the parser accepts is one PostgreSQL runs (each answer is from +// PostgreSQL 18.6): +// +// - after the ~~ operators ("syntax error at or near ESCAPE"): the clause +// belongs to the keyword spelling; +// - an escape character that is not a string literal: a number or a +// boolean has no like_escape, and a column's value is decided per row; +// - a string of more than one character ("invalid escape string"). +// +// An empty string is a clause: it turns escaping off. +func (p *filterParseState) parseEscape(operator TokenType, keyword bool) Node { + if p.currentToken.Type != TokenIdentifier || !p.isKeyword("ESCAPE") { + return nil + } + + if !keyword { + p.addError(&QFVFilterError{Message: fmt.Sprintf("ESCAPE follows a pattern written with LIKE, ILIKE or SIMILAR TO, not with the operator %s", operator)}) + } + + p.nextToken() // Consume ESCAPE + + escape := p.parsePrimary() + + literal, ok := escape.(*LiteralNode) + if !ok || literal.Kind != reflect.String { + p.addError(&QFVFilterError{Message: fmt.Sprintf("expected a string escape character after ESCAPE, got %s", escape.Type())}) + + return escape + } + + if text, _ := literal.Value.(string); utf8.RuneCountInString(text) > 1 { + p.addError(&QFVFilterError{Message: "the escape character after ESCAPE must be empty or one character"}) + } + + return escape +} + // parseInOperator parses IN operator // Expects the current token to be LPAREN after IN was consumed. func (p *filterParseState) parseInOperator(field Node, isNot bool) Node { diff --git a/filter_pg_patterns_test.go b/filter_pg_patterns_test.go index fedec68..39aa5ab 100644 --- a/filter_pg_patterns_test.go +++ b/filter_pg_patterns_test.go @@ -59,9 +59,6 @@ func TestFilterParser_APatternIsAString(t *testing.T) { // In the middle of an expression, and negated. {name: "inside a group", input: "id = 1 AND (name LIKE 2 OR id = 3)"}, {name: "under NOT", input: "NOT name ILIKE 2"}, - // The ESCAPE clause is not part of this grammar. - {name: "ESCAPE", input: "name LIKE 'x!%' ESCAPE '!'"}, - {name: "SIMILAR TO with ESCAPE", input: "name SIMILAR TO 'x!%' ESCAPE '!'"}, } for _, tt := range refused { @@ -232,3 +229,131 @@ func TestFilterParser_YesAndNoAreNames(t *testing.T) { t.Errorf(`active = 'yes': %v`, err) } } + +// TestFilterParser_TheEscapeClause: PostgreSQL writes "string LIKE pattern +// [ESCAPE escape-character]", and the same after SIMILAR TO. The clause was +// not part of this grammar, and "name LIKE 'x!%' ESCAPE '!'" was refused. +// Each row was run against PostgreSQL 18.6 on the table above, with the rows +// ('x%') and ('xa'); what it answered is in the comment. +func TestFilterParser_TheEscapeClause(t *testing.T) { + allowed := []string{"id", "name", "price", "active"} + + accepted := []struct { + input string + escape string + text string // the node's String(), with the clause + }{ + // 1: only the row 'x%' + {input: "name LIKE 'x!%' ESCAPE '!'", escape: "!", text: "(name LIKE 'x!%' ESCAPE '!')"}, + {input: "name like 'x!%' escape '!'", escape: "!", text: "(name LIKE 'x!%' ESCAPE '!')"}, + {input: "name ILIKE 'X!%' ESCAPE '!'", escape: "!", text: "(name ILIKE 'X!%' ESCAPE '!')"}, + {input: "name NOT LIKE 'x!%' ESCAPE '!'", escape: "!", text: "(name NOT LIKE 'x!%' ESCAPE '!')"}, + {input: "name NOT ILIKE 'X!%' ESCAPE '!'", escape: "!", text: "(name NOT ILIKE 'X!%' ESCAPE '!')"}, + {input: "name SIMILAR TO 'x!%' ESCAPE '!'", escape: "!", text: "name SIMILAR TO 'x!%' ESCAPE '!'"}, + {input: "name NOT SIMILAR TO 'x!%' ESCAPE '!'", escape: "!", text: "name NOT SIMILAR TO 'x!%' ESCAPE '!'"}, + // 2: an empty escape string turns escaping off, and % matches both + {input: "name LIKE 'x%' ESCAPE ''", escape: "", text: "(name LIKE 'x%' ESCAPE '')"}, + {input: "name SIMILAR TO 'x%' ESCAPE ''", escape: "", text: "name SIMILAR TO 'x%' ESCAPE ''"}, + // 2: one character is one character, whatever its bytes + {input: "name LIKE 'x%' ESCAPE 'é'", escape: "é", text: "(name LIKE 'x%' ESCAPE 'é')"}, + } + + for _, tt := range accepted { + t.Run("accepted: "+tt.input, func(t *testing.T) { + node, err := NewFilterParser(allowed).Parse(tt.input) + if err != nil { + t.Fatalf("Parse(%q): %v", tt.input, err) + } + + var escape Node + + switch n := node.(type) { + case *BinaryOperatorNode: + escape = n.Escape + case *SimilarToNode: + escape = n.Escape + default: + t.Fatalf("Parse(%q) = %T, want a pattern node", tt.input, node) + } + + literal, ok := escape.(*LiteralNode) + if !ok || literal.Kind != reflect.String || literal.Value != tt.escape { + t.Errorf("Parse(%q): the escape is %#v, want the string %q", tt.input, escape, tt.escape) + } + + if got := node.String(); got != tt.text { + t.Errorf("Parse(%q).String() = %q, want %q", tt.input, got, tt.text) + } + }) + } + + // A clause between two predicates belongs to the pattern before it. + t.Run("accepted: in an expression", func(t *testing.T) { + node, err := NewFilterParser(allowed).Parse("name LIKE 'x!%' ESCAPE '!' AND id = 1") // 1 + if err != nil { + t.Fatal(err) + } + + if got, want := node.String(), "((name LIKE 'x!%' ESCAPE '!') AND (id = 1))"; got != want { + t.Errorf("String() = %q, want %q", got, want) + } + }) + + // Without the clause there is no escape: what every caller saw before. + for _, input := range []string{"name LIKE 'x%'", "name ~~ 'x%'", "name SIMILAR TO 'x%'"} { + t.Run("no clause: "+input, func(t *testing.T) { + node, err := NewFilterParser(allowed).Parse(input) + if err != nil { + t.Fatal(err) + } + + switch n := node.(type) { + case *BinaryOperatorNode: + if n.Escape != nil { + t.Errorf("Escape = %v, want nil", n.Escape) + } + case *SimilarToNode: + if n.Escape != nil { + t.Errorf("Escape = %v, want nil", n.Escape) + } + } + }) + } + + refused := []struct { + input string + says string + }{ + // invalid escape string + {input: "name LIKE 'x%' ESCAPE '!!'", says: "must be empty or one character"}, + {input: "name SIMILAR TO 'x%' ESCAPE 'ab'", says: "must be empty or one character"}, + // function pg_catalog.like_escape(unknown, integer) does not exist + {input: "name LIKE 'x%' ESCAPE 1", says: "expected a string escape character after ESCAPE"}, + // function pg_catalog.like_escape(unknown, boolean) does not exist + {input: "name LIKE 'x%' ESCAPE TRUE", says: "expected a string escape character after ESCAPE"}, + // invalid escape string: a column's value, decided per row + {input: "name LIKE 'x%' ESCAPE name", says: "expected a string escape character after ESCAPE"}, + // syntax error at or near "ESCAPE": the ~~ operators take no clause + {input: "name ~~ 'x!%' ESCAPE '!'", says: "not with the operator LIKE"}, + {input: "name ~~* 'x!%' ESCAPE '!'", says: "not with the operator ILIKE"}, + {input: "name !~~ 'x!%' ESCAPE '!'", says: "not with the operator NOT LIKE"}, + {input: "name !~~* 'x!%' ESCAPE '!'", says: "not with the operator NOT ILIKE"}, + // syntax error at or near ")": no escape character + {input: "name LIKE 'x%' ESCAPE", says: ""}, + // syntax error at or near "ESCAPE": one clause + {input: "name LIKE 'x!%' ESCAPE '!' ESCAPE '!'", says: ""}, + } + + for _, tt := range refused { + t.Run("refused: "+tt.input, func(t *testing.T) { + node, err := NewFilterParser(allowed).Parse(tt.input) + if err == nil { + t.Fatalf("Parse(%q) = %v, want an error: PostgreSQL refuses it", tt.input, node) + } + + if !strings.Contains(err.Error(), tt.says) { + t.Errorf("Parse(%q) = %q, want it to say %q", tt.input, err.Error(), tt.says) + } + }) + } +}