fix: no connection character set reads a MySQL string differently

dialect.go

@@ -74,6 +74,13 @@ var (
 	// comment to the end of the line. With this dialect it is refused: id is
 	// not allowed, and # is not part of the grammar.
 	//
+	// MySQL reads a string in the connection's character set, and in gbk,
+	// gb18030, sjis, cp932 and big5 a backslash can be the last byte of a
+	// character that starts with any byte from 0x81: the server reads the two
+	// as one, and the backslash escapes nothing. So a backslash right after a
+	// character outside ASCII is refused, which no character set then reads
+	// differently.
+	//
 	// MySQL's sql_mode can turn backslash escapes off (NO_BACKSLASH_ESCAPES).
 	// The parser cannot see the server's mode: this dialect is MySQL 8.4's
 	// default, which has them on.

dialect_mysql_test.go

@@ -172,3 +172,106 @@ func TestADoubledQuoteIsOneEscapeCharacter(t *testing.T) {
 		}
 	}
 }
+
+// TestNoConnectionCharacterSetReadsAStringDifferently: MySQL reads a string
+// in the connection's character set, and in gbk, gb18030, sjis, cp932 and
+// big5 a backslash (0x5C) can be the last byte of a character that starts
+// with any byte from 0x81. With SET NAMES gbk, MySQL 8.4.13 answered 1 row
+// for the first filter below: it read the bytes of 丁 and the backslash as
+// characters, the backslash escaped nothing, and id = 2 ran. With utf8mb4 it
+// answered 0. A backslash right after a character outside ASCII is refused.
+func TestNoConnectionCharacterSetReadsAStringDifferently(t *testing.T) {
+	t.Parallel()
+
+	parser := NewFilterParser([]string{"name"}, WithDialect(DialectMySQL))
+
+	for _, input := range []string{
+		`name = '丁\' OR id = 2 #'`,     // gbk, gb18030, sjis, cp932: 1 row, on id
+		"name = '\xbf\\' OR id > 0 #'", // gbk, gb18030: every row
+		"name = '\xc3\\' OR id > 0 #'", // gbk, gb18030: every row
+		`name = 'é\\'`,                 // the same rule for \\
+		`name = '\丁\'x'`,               // after an escaped character too
+	} {
+		_, err := parser.Parse(input)
+		if err == nil {
+			t.Errorf("Parse(%q) accepted: a multi-byte connection character set reads the backslash as part of the character before it", input)
+
+			continue
+		}
+
+		if !strings.Contains(err.Error(), "a backslash right after a character outside ASCII") {
+			t.Errorf("Parse(%q) = %q, want it to say why", input, err.Error())
+		}
+	}
+
+	// An ASCII character between them reads the same in every character set.
+	for _, input := range []string{`name = '丁 \'x'`, `name = '丁'`, `name = 'é'`, `name = 'a\'丁'`} {
+		if _, err := parser.Parse(input); err != nil {
+			t.Errorf("Parse(%q): %v", input, err)
+		}
+	}
+
+	// PostgreSQL's strings have no backslash escapes, and nothing to refuse.
+	if _, err := NewFilterParser([]string{"name"}).Parse(`name = '丁\'`); err != nil {
+		t.Errorf("PostgreSQL 18's dialect: %v", err)
+	}
+}
+
+// What MySQL 8.4 refuses of an ESCAPE string, the parser refuses: \% and \_
+// are two characters outside a pattern ("Incorrect arguments to ESCAPE"; \x
+// is one, and ran), and a byte that is not UTF-8 is no character (the same
+// error, with a utf8mb4 connection).
+func TestTheMySQLEscapeCharacterIsOneCharacterToMySQL(t *testing.T) {
+	t.Parallel()
+
+	parser := NewFilterParser([]string{"name"}, WithDialect(DialectMySQL))
+
+	for input, says := range map[string]string{
+		`name LIKE 'x%' ESCAPE '\%'`:     "must be empty or one character",
+		`name LIKE 'x%' ESCAPE '\_'`:     "must be empty or one character",
+		"name LIKE 'x%' ESCAPE '\xc3'":   "not valid UTF-8",
+		"name LIKE 'x%' ESCAPE '\\\xff'": "not valid UTF-8",
+	} {
+		_, err := parser.Parse(input)
+		if err == nil || !strings.Contains(err.Error(), says) {
+			t.Errorf("Parse(%q) = %v, want it to say %q", input, err, says)
+		}
+	}
+
+	for _, input := range []string{`name LIKE 'x%' ESCAPE '\x'`, "name LIKE 'x%' ESCAPE '�'"} {
+		if _, err := parser.Parse(input); err != nil {
+			t.Errorf("Parse(%q): %v", input, err)
+		}
+	}
+
+	// Not UTF-8 is refused in PostgreSQL's dialect too: PostgreSQL refuses
+	// the statement's encoding.
+	if _, err := NewFilterParser([]string{"name"}).Parse("name LIKE 'x%' ESCAPE '\xc3'"); err == nil {
+		t.Error("PostgreSQL 18's dialect accepted an escape that is not UTF-8")
+	}
+}
+
+// A string's Value is its text with one quote off each end: Trim took every
+// quote, and a string that ends on an escaped quote lost it.
+func TestAStringsValueKeepsItsLastQuote(t *testing.T) {
+	t.Parallel()
+
+	for _, tt := range []struct {
+		dialect Dialect
+		input   string
+		value   string
+	}{
+		{DialectMySQL, `name = 'a\''`, `a\'`},
+		{DialectPostgreSQL, `name = ''''`, `''`},
+		{DialectPostgreSQL, `name = 'it''s'`, `it''s`},
+	} {
+		node, err := NewFilterParser([]string{"name"}, WithDialect(tt.dialect)).Parse(tt.input)
+		if err != nil {
+			t.Fatalf("%s: Parse(%q): %v", tt.dialect, tt.input, err)
+		}
+
+		if got := node.(*BinaryOperatorNode).Right.(*LiteralNode).Value; got != tt.value {
+			t.Errorf("%s: Parse(%q): Value = %q, want %q", tt.dialect, tt.input, got, tt.value)
+		}
+	}
+}

docs/dialects.md

@@ -77,6 +77,29 @@ MySQL's `sql_mode` can turn backslash escapes off (`NO_BACKSLASH_ESCAPES`). The
 parser cannot see the server's mode: `DialectMySQL` is MySQL 8.4's default,
 which has them on.
 
+### And in every connection character set
+
+MySQL reads a string in the connection's character set. In `gbk`, `gb18030`,
+`sjis`, `cp932` and `big5`, a backslash (byte 0x5C) can be the last byte of a
+character that starts with any byte from 0x81. The server then reads the byte
+before the backslash and the backslash as one character, and the backslash
+escapes nothing. `name = '丁\' OR id = 2 #'` is one comparison of `name` to
+this parser's reading, and with `SET NAMES gbk` MySQL 8.4.13 ran `id = 2`; with
+`utf8mb4` it did not.
+
+So `DialectMySQL` refuses a backslash right after a character outside ASCII
+(every such character ends in a byte from 0x80 in UTF-8, and so does every
+byte that is not UTF-8). A quote (0x27) is never part of a character in any of
+these sets, so nothing else can be read differently. Put a space or another
+ASCII character between them: `name = '丁 \'x'`.
+
+This was checked by running 6,000 filters the dialect accepts, written from
+quotes, backslashes, comments and multi-byte and invalid bytes, through MySQL
+8.4.13 under `utf8mb4`, `latin1`, `gbk`, `gb18030`, `sjis`, `cp932` and
+`big5`. None was read as anything but what the parser read: no syntax error,
+and no unknown column. Before this rule, `gbk` read 115 of them differently
+and `sjis` 49.
+
 ## Choosing a dialect
 
 ```mermaid
@@ -219,6 +242,8 @@ with any other problem in the same filter (see
 | `name ~ '^x'` | `operator "~" is not part of the MySQL 8.4 dialect` |
 | `id IS DISTINCT FROM 1` | `operator "IS DISTINCT FROM" is not part of the MySQL 8.4 dialect` |
 | `name = '\'' OR id > 0 #'` | `error on field 'id': field not allowed` (with the allow-list `["name"]`) |
+| `name = '丁\' OR id = 2 #'` | `illegal string: a backslash right after a character outside ASCII` |
+| `name LIKE 'x%' ESCAPE '\%'`, `name LIKE 'x%' ESCAPE '\_'` | `the escape character after ESCAPE must be empty or one character` |
 
 ## What a dialect does not do
 
@@ -230,15 +255,21 @@ with any other problem in the same filter (see
 - **It does not see the server's settings.** `DialectMySQL` is MySQL 8.4's
   default `sql_mode`. A server with `NO_BACKSLASH_ESCAPES` reads strings as
   PostgreSQL does, and `ANSI_QUOTES` or `PIPES_AS_CONCAT` change the grammar
-  further.
+  further. The connection's character set cannot change what a filter means
+  (see above), but it can make MySQL refuse one: an escape character outside
+  ASCII, such as `ESCAPE 'é'`, is two characters under `latin1` or `sjis`, and a
+  literal the column's collation cannot compare is an "Illegal mix of
+  collations".
 - **It does not change what a pattern means.** Without `ESCAPE`, PostgreSQL
   18's and MySQL 8.4's escape character is the backslash, and SQL:2023 has
   none, so `name LIKE '100\%'` matches `100%` on PostgreSQL 18 and MySQL 8.4,
   and `100\` followed by anything under the standard. The parser cannot see
   which you meant, so write an `ESCAPE` clause when it matters.
-- **It is stricter than the database in places, never looser.** MySQL 8.4
-  takes `ESCAPE 1` and strings in double quotes. The parser refuses both,
-  in every dialect.
+- **It is stricter than the database in places.** MySQL 8.4 takes `ESCAPE 1`,
+  strings in double quotes, and a backslash after a character outside ASCII
+  under `utf8mb4`. The parser refuses all three. It is looser only where the
+  database's answer depends on the connection, as above: never in what a
+  filter runs as.
 - **It does not apply to sorting or field selection.** The sort parser reads
   `field [ASC|DESC]` and the fields parser reads a list of names. Both are the
   same in every SQL, so they take no dialect.

docs/migration.md

@@ -14,7 +14,15 @@ changes for a caller that does not pass it.
   `SIMILAR TO`, the `~` operators, `IS [NOT] DISTINCT FROM`,
   `BETWEEN SYMMETRIC`/`ASYMMETRIC`, `ISNULL`/`NOTNULL`. It keeps `!=` and an
   empty `ESCAPE`, which MySQL runs.
+- It refuses a backslash right after a character outside ASCII in a string,
+  which MySQL reads as part of that character in a multi-byte connection
+  character set (`gbk`, `sjis`): put an ASCII character between them.
 - `NewLexer` reads strings as before, as PostgreSQL does.
+- In every dialect, an `ESCAPE` string with a byte that is not UTF-8 in it is
+  refused, and in MySQL's, `ESCAPE '\%'` and `ESCAPE '\_'` are two characters,
+  as MySQL counts them.
+- A string's `LiteralNode.Value` is its text with one quote taken off each end.
+  It took every quote, so a string ending on an escaped quote lost it.
 - An `ESCAPE` string is measured in characters with its doubled quotes
   counted once in every dialect: an escape string of four single quotes is
   one quote, where it was counted as empty.

filter_lexer.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 	"text/scanner"
 	"unicode"
+	"unicode/utf8"
 )
 
 // Lexer breaks the input string into tokens
@@ -13,6 +14,13 @@ type Lexer struct {
 	inputLen int
 	pos      int
 	tokens   []Token
+	// reasons says why a token is illegal, by its index in tokens, where
+	// "illegal token" would not tell the caller what to write instead.
+	reasons map[int]string
+	// notUTF8 holds the byte offset of every string that had a byte which is
+	// not UTF-8 in it. The scanner reads such a byte as U+FFFD, so the
+	// string's text no longer shows it.
+	notUTF8 map[int]bool
 	// backslashEscapes is whether a backslash in a string escapes the
 	// character after it (MySQL 8.4's default), or is a character like any
 	// other (PostgreSQL 18, the SQL standard).
@@ -40,7 +48,7 @@ func newLexer(input string, dialect Dialect) *Lexer {
 		return ch == '_' || unicode.IsLetter(ch) || (i > 0 && (unicode.IsDigit(ch) || ch == '.'))
 	}
 
-	l := &Lexer{s: s, input: input, pos: -1, backslashEscapes: dialect.backslashEscapes()} // Start at -1, first Next moves to 0
+	l := &Lexer{s: s, input: input, pos: -1, reasons: map[int]string{}, notUTF8: map[int]bool{}, backslashEscapes: dialect.backslashEscapes()} // Start at -1, first Next moves to 0
 	l.inputLen = len(input)
 
 	return l
@@ -141,25 +149,45 @@ func (l *Lexer) Parse() {
 		case '\'': // Start of a single-quoted string literal
 			var sb strings.Builder
 			isValid := true
+			previous := rune(0)
 			for {
-				char := l.s.Next() // Consume next character
+				char := l.nextInString(pos)
 
 				if char == scanner.EOF {
 					// Reached EOF without closing quote
 					isValid = false
 					break // Exit loop
+				} else if char == '\\' && l.backslashEscapes && previous >= utf8.RuneSelf {
+					// MySQL reads a string in the connection's character
+					// set, and in gbk, gb18030, sjis, cp932 and big5 the
+					// backslash (0x5C) can be the last byte of a character
+					// that starts with any byte from 0x81: the server reads
+					// the two as one character, and the backslash escapes
+					// nothing. 'X\' OR id > 0 #' is then a string that ends
+					// where this lexer reads an escaped quote. A backslash
+					// right after a byte outside ASCII, which every
+					// non-ASCII character ends with in UTF-8 and every
+					// invalid byte is, is refused, so that no connection
+					// character set reads the string differently. A quote
+					// (0x27) is never part of a character in any of them.
+					l.reasons[len(l.tokens)] = "a backslash right after a character outside ASCII, which MySQL reads as part of that character in a multi-byte connection character set (gbk, sjis): put a space or another ASCII character between them"
+					isValid = false
+					break
 				} else if char == '\\' && l.backslashEscapes {
 					// MySQL 8.4: the backslash escapes the character after
 					// it, a quote included, and both are kept as written. A
 					// string that ends on its backslash is not closed.
-					escaped := l.s.Next()
+					escaped := l.nextInString(pos)
 					if escaped == scanner.EOF {
 						isValid = false
 						break
 					}
 
 					_, _ = sb.WriteRune(char)
 					_, _ = sb.WriteRune(escaped)
+					previous = escaped
+
+					continue
 				} else if char == '\'' {
 					// Found a quote. Is it an escaped quote ('') or the end?
 					if l.s.Peek() == '\'' {
@@ -178,6 +206,8 @@ func (l *Lexer) Parse() {
 					// Note: Standard Go escapes like \n are not handled here, only '' for quote escape.
 					_, _ = sb.WriteRune(char)
 				}
+
+				previous = char
 			}
 
 			if isValid {
@@ -398,3 +428,21 @@ func (l *Lexer) Current() Token {
 
 	return l.tokens[l.pos]
 }
+
+// nextInString reads the next character of the string that starts at start,
+// and notes the string when the character is a byte that is not UTF-8. The
+// scanner reads such a byte as U+FFFD; the input says which it was, since
+// U+FFFD itself is the three bytes EF BF BD before the scanner's position.
+// (The scanner's ErrorCount cannot tell: it decodes one character ahead, and
+// counts the error during the call before.)
+func (l *Lexer) nextInString(start scanner.Position) rune {
+	char := l.s.Next()
+
+	if char == utf8.RuneError {
+		if end := l.s.Pos().Offset; end < 3 || l.input[end-3:end] != string(utf8.RuneError) {
+			l.notUTF8[start.Offset] = true
+		}
+	}
+
+	return char
+}

filter_parser.go

@@ -80,10 +80,18 @@ func (p *FilterParser) Parse(input string) (Node, error) {
 	st.lexer.Parse()
 
 	// Check for illegal tokens in the input
-	for _, token := range st.lexer.tokens {
-		if token.Type == TokenIllegal {
-			st.addError(&QFVFilterError{Field: token.Value, Message: "illegal token"})
+	for i, token := range st.lexer.tokens {
+		if token.Type != TokenIllegal {
+			continue
 		}
+
+		if reason, ok := st.lexer.reasons[i]; ok {
+			st.addError(&QFVFilterError{Message: "illegal string: " + reason})
+
+			continue
+		}
+
+		st.addError(&QFVFilterError{Field: token.Value, Message: "illegal token"})
 	}
 
 	st.nextToken()
@@ -545,6 +553,15 @@ func (p *filterParseState) parseEscape(operator TokenType, keyword bool) Node {
 		return escape
 	}
 
+	// A byte that is not UTF-8 counts as one character here, and is none to
+	// the database: MySQL 8.4 answers "Incorrect arguments to ESCAPE", and
+	// PostgreSQL refuses the statement's encoding.
+	if p.lexer.notUTF8[literal.Pos().Offset] {
+		p.addError(&QFVFilterError{Message: "the escape character after ESCAPE is not valid UTF-8"})
+
+		return escape
+	}
+
 	switch length := literalLength(literal.Text, p.dialect.backslashEscapes()); {
 	case length > 1:
 		p.addError(&QFVFilterError{Message: "the escape character after ESCAPE must be empty or one character"})
@@ -735,9 +752,11 @@ func (p *filterParseState) parsePrimary() Node {
 	case TokenString:
 		node := &LiteralNode{
 			baseNode: baseNode{pos: p.currentToken.Pos},
-			Value:    strings.Trim(p.currentToken.Value, "'"),
-			Kind:     reflect.String,
-			Text:     p.currentToken.Value,
+			// One quote off each end, and only one: Trim took every quote,
+			// and 'a\'' (MySQL's a') or '''' (one quote) lost the last.
+			Value: strings.TrimSuffix(strings.TrimPrefix(p.currentToken.Value, "'"), "'"),
+			Kind:  reflect.String,
+			Text:  p.currentToken.Value,
 		}
 		p.nextToken()
 		return node
@@ -809,7 +828,12 @@ func literalLength(text string, backslashEscapes bool) int {
 		case runes[i] == '\'' && i+1 < len(runes) && runes[i+1] == '\'':
 			i++
 		case backslashEscapes && runes[i] == '\\' && i+1 < len(runes):
-			i++
+			// MySQL keeps \% and \_ as two characters outside a pattern
+			// ("they are interpreted as literal \% and \_"), and refuses
+			// ESCAPE '\%' with "Incorrect arguments to ESCAPE".
+			if runes[i+1] != '%' && runes[i+1] != '_' {
+				i++
+			}
 		}
 
 		length++