gh-159635: Use quote frequency to break csv.Sniffer ties

Author: MrGreyfun

Lib/csv.py

@@ -361,6 +361,7 @@ def sniff(self, sample, delimiters=None):
 
         best = None
         best_score = None
+        best_field_start_count = None
         for combo, rows in sorted(parsed):
             if rows is None:
                 continue
@@ -379,6 +380,18 @@ def sniff(self, sample, delimiters=None):
             if best_score is None or score > best_score:
                 best_score = score
                 best = combo[1:]
+                best_field_start_count = None
+            elif (score == best_score and delimiter == best[0]
+                  and escapechar == best[2] and quotechar != best[1]):
+                # Do not let quote frequency affect delimiter or escape choice.
+                if best_field_start_count is None:
+                    best_field_start_count = self._field_start_count(
+                        lines, best[1], best[0], best[2], cut)
+                field_start_count = self._field_start_count(
+                    lines, quotechar, delimiter, escapechar, cut)
+                if field_start_count > best_field_start_count:
+                    best = combo[1:]
+                    best_field_start_count = field_start_count
 
         if best is None:
             raise Error("Could not determine delimiter")
@@ -401,6 +414,109 @@ class dialect(Dialect):
 
         return dialect
 
+    def _field_start_count(self, lines, quotechar, delimiter, escapechar,
+                           cut):
+        """
+        The number of fields in the sample starting with *quotechar*:
+        at the start of a field.  Used for breaking ties between quote
+        characters.
+        """
+        if not quotechar:
+            return 0
+
+        class LineIterator:
+            def __init__(self, lines):
+                self.lines = iter(lines)
+                self.position = 0
+
+            def __iter__(self):
+                return self
+
+            def __next__(self):
+                line = next(self.lines)
+                self.position += 1
+                return line
+
+        source = LineIterator(lines)
+        record_counts = []
+        start = 0
+        # Keep the scan boundaries identical to the candidate reader's.
+        reader = self._make_reader(source, delimiter, quotechar, escapechar)
+        parse_error = False
+        try:
+            while True:
+                next(reader)
+                end = source.position
+                record = ''.join(lines[start:end])
+                record_counts.append(
+                    self._record_field_start_count(
+                        record, quotechar, delimiter, escapechar))
+                start = end
+        except StopIteration:
+            pass
+        except Error:
+            # Match _try_dialect(): a failing row is not counted.
+            parse_error = True
+        if not parse_error and cut and len(record_counts) > 1:
+            record_counts.pop()
+        return sum(record_counts)
+
+    def _record_field_start_count(self, record, quotechar, delimiter,
+                                  escapechar):
+        if not record:
+            return 0
+        skipinitialspace = delimiter != ' '
+        count = 0
+        field_start = True
+        skip_spaces = skipinitialspace
+        active_quote = None
+        pos = 0
+        # The candidate reader fixes record boundaries; recognize either
+        # quote mark within a record so embedded delimiters do not vote.
+        while pos < len(record):
+            char = record[pos]
+            if active_quote is not None:
+                if escapechar and char == escapechar:
+                    pos += 2
+                    continue
+                if char == active_quote:
+                    if pos + 1 < len(record) and record[pos + 1] == char:
+                        pos += 2
+                        continue
+                    active_quote = None
+                pos += 1
+                continue
+
+            if escapechar and char == escapechar:
+                field_start = False
+                pos += 2
+                continue
+            if field_start:
+                if skip_spaces and char == ' ':
+                    pos += 1
+                    continue
+                if char in "\"'":
+                    if char == quotechar:
+                        count += 1
+                    active_quote = char
+                    field_start = False
+                    pos += 1
+                    continue
+                if char in '\r\n':
+                    break
+                if char == delimiter:
+                    skip_spaces = skipinitialspace
+                    pos += 1
+                    continue
+                field_start = False
+            elif char == delimiter:
+                field_start = True
+                skip_spaces = skipinitialspace
+            elif char in '\r\n':
+                break
+            pos += 1
+        return count
+
     def _parses_as_single_column(self, sample):
         """
         True if the whole sample parses as a single column of quoted

Lib/test/test_csv.py

@@ -1855,6 +1855,29 @@ def test_sniff_regex_backtracking(self):
         sample = '"",' * 100 + '"' * 100 + '0' + '"' * 100 + '0'
         self.assertEqual(sniffer.sniff(sample).delimiter, ',')
 
+    def test_sniff_quotechar_tie_break(self):
+        # When both quote characters fit the sample equally well, the
+        # one starting more fields wins, not the one tried first.
+        # A single row of double-quoted fields used to make the sniffer
+        # report '"' even for a file quoted with "'".
+        sniffer = csv.Sniffer()
+        sample = "\n".join(["'a','b'"] * 99 + ['"k","l"']) + "\n"
+        dialect = sniffer.sniff(sample)
+        self.assertEqual(dialect.quotechar, "'")
+        self.assertEqual(list(csv.reader(StringIO(sample), dialect))[0],
+                         ['a', 'b'])
+        # The mirror sample is not ambiguous: '"' starts far more fields.
+        sample = "\n".join(['"a","b"'] * 99 + ["'k','l'"]) + "\n"
+        dialect = sniffer.sniff(sample)
+        self.assertEqual(dialect.quotechar, '"')
+        # A single unquoted field starting with a double quote does not
+        # outweigh the single-quoted fields.
+        sample = ("id,note\n"
+                  + "'1','a'\n" * 20
+                  + "5,\"N/A\"\n")
+        dialect = sniffer.sniff(sample)
+        self.assertEqual(dialect.quotechar, "'")
+
     def test_doublequote(self):
         sniffer = csv.Sniffer()
         dialect = sniffer.sniff(self.header1)

Misc/NEWS.d/next/Library/2026-10-17-13-14-56.gh-issue-159635.qU7fNc.rst

@@ -0,0 +1,5 @@
+In ``csv.Sniffer.sniff()``, break ties between quote characters which fit
+the sample equally well by the number of fields starting with each quote
+character, instead of always preferring ``'"'``.  This prevents misdetection
+of the quote character for files quoted with ``"'"`` that contain a row of
+double-quoted fields.