From b73a865555c76d49c23061b436ad885423b3bf47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Gonz=C3=A1lez=20Di=20Antonio?= Date: Sat, 24 Oct 2026 20:40:59 +0200 Subject: [PATCH] fix: no connection character set reads a MySQL string differently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review of the MySQL dialect found that its string reading held for utf8mb4 and latin1 connections and not for gbk, gb18030, sjis, cp932 or big5, where a backslash can be the last byte of a character that starts with any byte from 0x81: MySQL reads the two as one, and the backslash escapes nothing. name = '丁\' OR id = 2 #' was one comparison to the parser, and MySQL 8.4.13 ran id = 2 with SET NAMES gbk. A backslash right after a character outside ASCII is refused, with why. 6,000 accepted filters run under all seven character sets: none read differently (before the rule, 115 under gbk and 49 under sjis). And what MySQL refuses of an ESCAPE string is refused: \% and \_ are two characters outside a pattern, and a byte that is not UTF-8 is none (the scanner reads it as U+FFFD; the input says which it was). A string's Value takes one quote off each end, where Trim took every one. Co-Authored-By: Claude Opus 5.5 --- dialect.go | 7 +++ dialect_mysql_test.go | 103 ++++++++++++++++++++++++++++++++++++++++++ docs/dialects.md | 39 ++++++++++++++-- docs/migration.md | 8 ++++ filter_lexer.go | 54 ++++++++++++++++++++-- filter_parser.go | 38 +++++++++++++--- 6 files changed, 235 insertions(+), 14 deletions(-) diff --git a/dialect.go b/dialect.go index 5c21170..707ba19 100644 --- a/dialect.go +++ b/dialect.go @@ -74,6 +74,13 @@ var ( // comment to the end of the line. With this dialect it is refused: id is // not allowed, and # is not part of the grammar. // + // MySQL reads a string in the connection's character set, and in gbk, + // gb18030, sjis, cp932 and big5 a backslash can be the last byte of a + // character that starts with any byte from 0x81: the server reads the two + // as one, and the backslash escapes nothing. So a backslash right after a + // character outside ASCII is refused, which no character set then reads + // differently. + // // MySQL's sql_mode can turn backslash escapes off (NO_BACKSLASH_ESCAPES). // The parser cannot see the server's mode: this dialect is MySQL 8.4's // default, which has them on. diff --git a/dialect_mysql_test.go b/dialect_mysql_test.go index 83ab98d..eb6681e 100644 --- a/dialect_mysql_test.go +++ b/dialect_mysql_test.go @@ -172,3 +172,106 @@ func TestADoubledQuoteIsOneEscapeCharacter(t *testing.T) { } } } + +// TestNoConnectionCharacterSetReadsAStringDifferently: MySQL reads a string +// in the connection's character set, and in gbk, gb18030, sjis, cp932 and +// big5 a backslash (0x5C) can be the last byte of a character that starts +// with any byte from 0x81. With SET NAMES gbk, MySQL 8.4.13 answered 1 row +// for the first filter below: it read the bytes of 丁 and the backslash as +// characters, the backslash escaped nothing, and id = 2 ran. With utf8mb4 it +// answered 0. A backslash right after a character outside ASCII is refused. +func TestNoConnectionCharacterSetReadsAStringDifferently(t *testing.T) { + t.Parallel() + + parser := NewFilterParser([]string{"name"}, WithDialect(DialectMySQL)) + + for _, input := range []string{ + `name = '丁\' OR id = 2 #'`, // gbk, gb18030, sjis, cp932: 1 row, on id + "name = '\xbf\\' OR id > 0 #'", // gbk, gb18030: every row + "name = '\xc3\\' OR id > 0 #'", // gbk, gb18030: every row + `name = 'é\\'`, // the same rule for \\ + `name = '\丁\'x'`, // after an escaped character too + } { + _, err := parser.Parse(input) + if err == nil { + t.Errorf("Parse(%q) accepted: a multi-byte connection character set reads the backslash as part of the character before it", input) + + continue + } + + if !strings.Contains(err.Error(), "a backslash right after a character outside ASCII") { + t.Errorf("Parse(%q) = %q, want it to say why", input, err.Error()) + } + } + + // An ASCII character between them reads the same in every character set. + for _, input := range []string{`name = '丁 \'x'`, `name = '丁'`, `name = 'é'`, `name = 'a\'丁'`} { + if _, err := parser.Parse(input); err != nil { + t.Errorf("Parse(%q): %v", input, err) + } + } + + // PostgreSQL's strings have no backslash escapes, and nothing to refuse. + if _, err := NewFilterParser([]string{"name"}).Parse(`name = '丁\'`); err != nil { + t.Errorf("PostgreSQL 18's dialect: %v", err) + } +} + +// What MySQL 8.4 refuses of an ESCAPE string, the parser refuses: \% and \_ +// are two characters outside a pattern ("Incorrect arguments to ESCAPE"; \x +// is one, and ran), and a byte that is not UTF-8 is no character (the same +// error, with a utf8mb4 connection). +func TestTheMySQLEscapeCharacterIsOneCharacterToMySQL(t *testing.T) { + t.Parallel() + + parser := NewFilterParser([]string{"name"}, WithDialect(DialectMySQL)) + + for input, says := range map[string]string{ + `name LIKE 'x%' ESCAPE '\%'`: "must be empty or one character", + `name LIKE 'x%' ESCAPE '\_'`: "must be empty or one character", + "name LIKE 'x%' ESCAPE '\xc3'": "not valid UTF-8", + "name LIKE 'x%' ESCAPE '\\\xff'": "not valid UTF-8", + } { + _, err := parser.Parse(input) + if err == nil || !strings.Contains(err.Error(), says) { + t.Errorf("Parse(%q) = %v, want it to say %q", input, err, says) + } + } + + for _, input := range []string{`name LIKE 'x%' ESCAPE '\x'`, "name LIKE 'x%' ESCAPE '�'"} { + if _, err := parser.Parse(input); err != nil { + t.Errorf("Parse(%q): %v", input, err) + } + } + + // Not UTF-8 is refused in PostgreSQL's dialect too: PostgreSQL refuses + // the statement's encoding. + if _, err := NewFilterParser([]string{"name"}).Parse("name LIKE 'x%' ESCAPE '\xc3'"); err == nil { + t.Error("PostgreSQL 18's dialect accepted an escape that is not UTF-8") + } +} + +// A string's Value is its text with one quote off each end: Trim took every +// quote, and a string that ends on an escaped quote lost it. +func TestAStringsValueKeepsItsLastQuote(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + dialect Dialect + input string + value string + }{ + {DialectMySQL, `name = 'a\''`, `a\'`}, + {DialectPostgreSQL, `name = ''''`, `''`}, + {DialectPostgreSQL, `name = 'it''s'`, `it''s`}, + } { + node, err := NewFilterParser([]string{"name"}, WithDialect(tt.dialect)).Parse(tt.input) + if err != nil { + t.Fatalf("%s: Parse(%q): %v", tt.dialect, tt.input, err) + } + + if got := node.(*BinaryOperatorNode).Right.(*LiteralNode).Value; got != tt.value { + t.Errorf("%s: Parse(%q): Value = %q, want %q", tt.dialect, tt.input, got, tt.value) + } + } +} diff --git a/docs/dialects.md b/docs/dialects.md index 71c6264..f4883ea 100644 --- a/docs/dialects.md +++ b/docs/dialects.md @@ -77,6 +77,29 @@ MySQL's `sql_mode` can turn backslash escapes off (`NO_BACKSLASH_ESCAPES`). The parser cannot see the server's mode: `DialectMySQL` is MySQL 8.4's default, which has them on. +### And in every connection character set + +MySQL reads a string in the connection's character set. In `gbk`, `gb18030`, +`sjis`, `cp932` and `big5`, a backslash (byte 0x5C) can be the last byte of a +character that starts with any byte from 0x81. The server then reads the byte +before the backslash and the backslash as one character, and the backslash +escapes nothing. `name = '丁\' OR id = 2 #'` is one comparison of `name` to +this parser's reading, and with `SET NAMES gbk` MySQL 8.4.13 ran `id = 2`; with +`utf8mb4` it did not. + +So `DialectMySQL` refuses a backslash right after a character outside ASCII +(every such character ends in a byte from 0x80 in UTF-8, and so does every +byte that is not UTF-8). A quote (0x27) is never part of a character in any of +these sets, so nothing else can be read differently. Put a space or another +ASCII character between them: `name = '丁 \'x'`. + +This was checked by running 6,000 filters the dialect accepts, written from +quotes, backslashes, comments and multi-byte and invalid bytes, through MySQL +8.4.13 under `utf8mb4`, `latin1`, `gbk`, `gb18030`, `sjis`, `cp932` and +`big5`. None was read as anything but what the parser read: no syntax error, +and no unknown column. Before this rule, `gbk` read 115 of them differently +and `sjis` 49. + ## Choosing a dialect ```mermaid @@ -219,6 +242,8 @@ with any other problem in the same filter (see | `name ~ '^x'` | `operator "~" is not part of the MySQL 8.4 dialect` | | `id IS DISTINCT FROM 1` | `operator "IS DISTINCT FROM" is not part of the MySQL 8.4 dialect` | | `name = '\'' OR id > 0 #'` | `error on field 'id': field not allowed` (with the allow-list `["name"]`) | +| `name = '丁\' OR id = 2 #'` | `illegal string: a backslash right after a character outside ASCII` | +| `name LIKE 'x%' ESCAPE '\%'`, `name LIKE 'x%' ESCAPE '\_'` | `the escape character after ESCAPE must be empty or one character` | ## What a dialect does not do @@ -230,15 +255,21 @@ with any other problem in the same filter (see - **It does not see the server's settings.** `DialectMySQL` is MySQL 8.4's default `sql_mode`. A server with `NO_BACKSLASH_ESCAPES` reads strings as PostgreSQL does, and `ANSI_QUOTES` or `PIPES_AS_CONCAT` change the grammar - further. + further. The connection's character set cannot change what a filter means + (see above), but it can make MySQL refuse one: an escape character outside + ASCII, such as `ESCAPE 'é'`, is two characters under `latin1` or `sjis`, and a + literal the column's collation cannot compare is an "Illegal mix of + collations". - **It does not change what a pattern means.** Without `ESCAPE`, PostgreSQL 18's and MySQL 8.4's escape character is the backslash, and SQL:2023 has none, so `name LIKE '100\%'` matches `100%` on PostgreSQL 18 and MySQL 8.4, and `100\` followed by anything under the standard. The parser cannot see which you meant, so write an `ESCAPE` clause when it matters. -- **It is stricter than the database in places, never looser.** MySQL 8.4 - takes `ESCAPE 1` and strings in double quotes. The parser refuses both, - in every dialect. +- **It is stricter than the database in places.** MySQL 8.4 takes `ESCAPE 1`, + strings in double quotes, and a backslash after a character outside ASCII + under `utf8mb4`. The parser refuses all three. It is looser only where the + database's answer depends on the connection, as above: never in what a + filter runs as. - **It does not apply to sorting or field selection.** The sort parser reads `field [ASC|DESC]` and the fields parser reads a list of names. Both are the same in every SQL, so they take no dialect. diff --git a/docs/migration.md b/docs/migration.md index 1f86a35..1c6048a 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -14,7 +14,15 @@ changes for a caller that does not pass it. `SIMILAR TO`, the `~` operators, `IS [NOT] DISTINCT FROM`, `BETWEEN SYMMETRIC`/`ASYMMETRIC`, `ISNULL`/`NOTNULL`. It keeps `!=` and an empty `ESCAPE`, which MySQL runs. +- It refuses a backslash right after a character outside ASCII in a string, + which MySQL reads as part of that character in a multi-byte connection + character set (`gbk`, `sjis`): put an ASCII character between them. - `NewLexer` reads strings as before, as PostgreSQL does. +- In every dialect, an `ESCAPE` string with a byte that is not UTF-8 in it is + refused, and in MySQL's, `ESCAPE '\%'` and `ESCAPE '\_'` are two characters, + as MySQL counts them. +- A string's `LiteralNode.Value` is its text with one quote taken off each end. + It took every quote, so a string ending on an escaped quote lost it. - An `ESCAPE` string is measured in characters with its doubled quotes counted once in every dialect: an escape string of four single quotes is one quote, where it was counted as empty. diff --git a/filter_lexer.go b/filter_lexer.go index 6bee05e..1f90939 100644 --- a/filter_lexer.go +++ b/filter_lexer.go @@ -4,6 +4,7 @@ import ( "strings" "text/scanner" "unicode" + "unicode/utf8" ) // Lexer breaks the input string into tokens @@ -13,6 +14,13 @@ type Lexer struct { inputLen int pos int tokens []Token + // reasons says why a token is illegal, by its index in tokens, where + // "illegal token" would not tell the caller what to write instead. + reasons map[int]string + // notUTF8 holds the byte offset of every string that had a byte which is + // not UTF-8 in it. The scanner reads such a byte as U+FFFD, so the + // string's text no longer shows it. + notUTF8 map[int]bool // backslashEscapes is whether a backslash in a string escapes the // character after it (MySQL 8.4's default), or is a character like any // other (PostgreSQL 18, the SQL standard). @@ -40,7 +48,7 @@ func newLexer(input string, dialect Dialect) *Lexer { return ch == '_' || unicode.IsLetter(ch) || (i > 0 && (unicode.IsDigit(ch) || ch == '.')) } - l := &Lexer{s: s, input: input, pos: -1, backslashEscapes: dialect.backslashEscapes()} // Start at -1, first Next moves to 0 + l := &Lexer{s: s, input: input, pos: -1, reasons: map[int]string{}, notUTF8: map[int]bool{}, backslashEscapes: dialect.backslashEscapes()} // Start at -1, first Next moves to 0 l.inputLen = len(input) return l @@ -141,18 +149,35 @@ func (l *Lexer) Parse() { case '\'': // Start of a single-quoted string literal var sb strings.Builder isValid := true + previous := rune(0) for { - char := l.s.Next() // Consume next character + char := l.nextInString(pos) if char == scanner.EOF { // Reached EOF without closing quote isValid = false break // Exit loop + } else if char == '\\' && l.backslashEscapes && previous >= utf8.RuneSelf { + // MySQL reads a string in the connection's character + // set, and in gbk, gb18030, sjis, cp932 and big5 the + // backslash (0x5C) can be the last byte of a character + // that starts with any byte from 0x81: the server reads + // the two as one character, and the backslash escapes + // nothing. 'X\' OR id > 0 #' is then a string that ends + // where this lexer reads an escaped quote. A backslash + // right after a byte outside ASCII, which every + // non-ASCII character ends with in UTF-8 and every + // invalid byte is, is refused, so that no connection + // character set reads the string differently. A quote + // (0x27) is never part of a character in any of them. + l.reasons[len(l.tokens)] = "a backslash right after a character outside ASCII, which MySQL reads as part of that character in a multi-byte connection character set (gbk, sjis): put a space or another ASCII character between them" + isValid = false + break } else if char == '\\' && l.backslashEscapes { // MySQL 8.4: the backslash escapes the character after // it, a quote included, and both are kept as written. A // string that ends on its backslash is not closed. - escaped := l.s.Next() + escaped := l.nextInString(pos) if escaped == scanner.EOF { isValid = false break @@ -160,6 +185,9 @@ func (l *Lexer) Parse() { _, _ = sb.WriteRune(char) _, _ = sb.WriteRune(escaped) + previous = escaped + + continue } else if char == '\'' { // Found a quote. Is it an escaped quote ('') or the end? if l.s.Peek() == '\'' { @@ -178,6 +206,8 @@ func (l *Lexer) Parse() { // Note: Standard Go escapes like \n are not handled here, only '' for quote escape. _, _ = sb.WriteRune(char) } + + previous = char } if isValid { @@ -398,3 +428,21 @@ func (l *Lexer) Current() Token { return l.tokens[l.pos] } + +// nextInString reads the next character of the string that starts at start, +// and notes the string when the character is a byte that is not UTF-8. The +// scanner reads such a byte as U+FFFD; the input says which it was, since +// U+FFFD itself is the three bytes EF BF BD before the scanner's position. +// (The scanner's ErrorCount cannot tell: it decodes one character ahead, and +// counts the error during the call before.) +func (l *Lexer) nextInString(start scanner.Position) rune { + char := l.s.Next() + + if char == utf8.RuneError { + if end := l.s.Pos().Offset; end < 3 || l.input[end-3:end] != string(utf8.RuneError) { + l.notUTF8[start.Offset] = true + } + } + + return char +} diff --git a/filter_parser.go b/filter_parser.go index fabd03f..95c07cc 100644 --- a/filter_parser.go +++ b/filter_parser.go @@ -80,10 +80,18 @@ func (p *FilterParser) Parse(input string) (Node, error) { st.lexer.Parse() // Check for illegal tokens in the input - for _, token := range st.lexer.tokens { - if token.Type == TokenIllegal { - st.addError(&QFVFilterError{Field: token.Value, Message: "illegal token"}) + for i, token := range st.lexer.tokens { + if token.Type != TokenIllegal { + continue } + + if reason, ok := st.lexer.reasons[i]; ok { + st.addError(&QFVFilterError{Message: "illegal string: " + reason}) + + continue + } + + st.addError(&QFVFilterError{Field: token.Value, Message: "illegal token"}) } st.nextToken() @@ -545,6 +553,15 @@ func (p *filterParseState) parseEscape(operator TokenType, keyword bool) Node { return escape } + // A byte that is not UTF-8 counts as one character here, and is none to + // the database: MySQL 8.4 answers "Incorrect arguments to ESCAPE", and + // PostgreSQL refuses the statement's encoding. + if p.lexer.notUTF8[literal.Pos().Offset] { + p.addError(&QFVFilterError{Message: "the escape character after ESCAPE is not valid UTF-8"}) + + return escape + } + switch length := literalLength(literal.Text, p.dialect.backslashEscapes()); { case length > 1: p.addError(&QFVFilterError{Message: "the escape character after ESCAPE must be empty or one character"}) @@ -735,9 +752,11 @@ func (p *filterParseState) parsePrimary() Node { case TokenString: node := &LiteralNode{ baseNode: baseNode{pos: p.currentToken.Pos}, - Value: strings.Trim(p.currentToken.Value, "'"), - Kind: reflect.String, - Text: p.currentToken.Value, + // One quote off each end, and only one: Trim took every quote, + // and 'a\'' (MySQL's a') or '''' (one quote) lost the last. + Value: strings.TrimSuffix(strings.TrimPrefix(p.currentToken.Value, "'"), "'"), + Kind: reflect.String, + Text: p.currentToken.Value, } p.nextToken() return node @@ -809,7 +828,12 @@ func literalLength(text string, backslashEscapes bool) int { case runes[i] == '\'' && i+1 < len(runes) && runes[i+1] == '\'': i++ case backslashEscapes && runes[i] == '\\' && i+1 < len(runes): - i++ + // MySQL keeps \% and \_ as two characters outside a pattern + // ("they are interpreted as literal \% and \_"), and refuses + // ESCAPE '\%' with "Incorrect arguments to ESCAPE". + if runes[i+1] != '%' && runes[i+1] != '_' { + i++ + } } length++