From a45aca25b3d3f61320480871f0387b85b66fb89f Mon Sep 17 00:00:00 2001 From: Punisheroot <44579963+Punisheroot@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:47:16 +0200 Subject: [PATCH 1/2] gh-109638: Avoid pathological backtracking in csv.Sniffer --- Lib/csv.py | 57 ++++++++++--- Lib/test/test_csv.py | 81 +++++++++++++++++++ ...-07-29-13-00-00.gh-issue-109638.SnifF1.rst | 3 + 3 files changed, 131 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-29-13-00-00.gh-issue-109638.SnifF1.rst diff --git a/Lib/csv.py b/Lib/csv.py index 6cae34c705777d..074ef2a90e371d 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -328,20 +328,57 @@ def _guess_quote_and_delimiter(self, data, delimiters): delim = '' skipinitialspace = 0 - # if we see an extra quote between delimiters, we've got a - # double quoted format - dq_regexp = re.compile( - r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \ - {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE) + doublequote = self._detect_doublequote(data, delim, quotechar) + return (quotechar, doublequote, delim, skipinitialspace) - if dq_regexp.search(data): - doublequote = True - else: - doublequote = False + def _detect_doublequote(self, data, delimiter, quotechar): + """ + Return whether a doubled quote occurs inside a quoted field. - return (quotechar, doublequote, delim, skipinitialspace) + The first regexp is a fast, linear pre-filter. Since it is not + anchored, a match can start at a delimiter inside another quoted + field. The second regexp rules out well-formed input without doubled + quotes. Both regexps use possessive repetition to avoid backtracking. + """ + import re + + escaped_delimiter = re.escape(delimiter) + escaped_quote = re.escape(quotechar) + values = {'delim': escaped_delimiter, 'quote': escaped_quote} + if delimiter: + candidate = re.compile( + r"(?:%(delim)s|\r|^) *+%(quote)s" + r"[^%(quote)s]*+%(quote)s%(quote)s" + r"(?:%(quote)s%(quote)s|[^%(quote)s]++)*+" + r"%(quote)s(?:%(delim)s|(?=\r)|$)" + % values, re.MULTILINE) + separator = rf"(?:{escaped_delimiter}|\r\n|\r|\n)" + plain = ( + rf"(?! *+{escaped_quote})" + rf"[^{escaped_delimiter}\r\n]*+") + else: + # An empty delimiter must not create a zero-width alternative + # which makes re.search() retry the pattern at every position. + candidate = re.compile( + r"(?:[\r\n]|\A) *+%(quote)s" + r"[^%(quote)s]*+%(quote)s%(quote)s" + r"(?:%(quote)s%(quote)s|[^%(quote)s]++)*+" + r"%(quote)s(?=[\r\n]|\Z)" + % values, re.MULTILINE) + separator = r"(?:\r\n|\r|\n)" + plain = rf"(?! *+{escaped_quote})[^\r\n]*+" + + if candidate.search(data) is None: + return False + + quoted = ( + rf" *+{escaped_quote}[^{escaped_quote}]*+{escaped_quote}") + field = rf"(?:{quoted}|{plain})" + without_doublequote = re.compile( + rf"\A{field}(?:{separator}{field})*+\Z") + return without_doublequote.match(data) is None def _guess_delimiter(self, data, delimiters): diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 2ab529b51c207d..290aeddff13bfc 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1501,6 +1501,12 @@ def test_delimiters(self): self.assertEqual(dialect.delimiter, ',') self.assertEqual(dialect.quotechar, '"') + def test_sniff_regex_backtracking(self): + # gh-109638: this artificial sample used to take minutes. + sniffer = csv.Sniffer() + sample = '"",' * 100 + '"' * 100 + '0' + '"' * 100 + '0' + self.assertEqual(sniffer.sniff(sample).delimiter, ',') + def test_doublequote(self): sniffer = csv.Sniffer() dialect = sniffer.sniff(self.header1) @@ -1514,6 +1520,81 @@ def test_doublequote(self): dialect = sniffer.sniff(self.sample9) self.assertTrue(dialect.doublequote) + def test_doublequote_without_delimiter(self): + sniffer = csv.Sniffer() + for quotechar in ('"', "'"): + with self.subTest(quotechar=quotechar): + self.assertEqual( + sniffer._guess_quote_and_delimiter( + f'{quotechar}a{quotechar}{quotechar}b{quotechar}', + None, + ), + (quotechar, True, '', 0), + ) + self.assertEqual( + sniffer._guess_quote_and_delimiter( + f'{quotechar}ab{quotechar}', + None, + ), + (quotechar, False, '', 0), + ) + self.assertEqual( + sniffer._guess_quote_and_delimiter( + f'{quotechar}a{quotechar}\n' + f'x{quotechar}{quotechar}{quotechar}{quotechar}x', + None, + ), + (quotechar, False, '', 0), + ) + + def test_doublequote_with_carriage_return(self): + sniffer = csv.Sniffer() + for record_delimiter in ('\r\n', '\r'): + for quotechar in ('"', "'"): + with self.subTest( + record_delimiter=record_delimiter, + quotechar=quotechar, + ): + data = ( + f'x,{quotechar}plain{quotechar}{record_delimiter}' + f'{quotechar}a{quotechar}{quotechar}b{quotechar},y' + f'{record_delimiter}' + ) + self.assertEqual( + sniffer._guess_quote_and_delimiter(data, None), + (quotechar, True, ',', 0), + ) + self.assertIs(sniffer.sniff(data).doublequote, True) + + def test_doublequote_across_quoted_fields(self): + sniffer = csv.Sniffer() + for delimiter in (',', ' ', ''): + for quotechar in ('"', "'"): + with self.subTest( + delimiter=delimiter, + quotechar=quotechar, + ): + separator = delimiter or '\n' + data = separator.join(( + f'{quotechar}{separator}{quotechar}', + quotechar * 2, + f'{quotechar}{separator}{quotechar}', + )) + result = sniffer._guess_quote_and_delimiter(data, None) + self.assertEqual(result[0], quotechar) + self.assertIs(result[1], False) + self.assertEqual(result[2], delimiter) + if delimiter: + self.assertIs( + sniffer.sniff(data).doublequote, + False, + ) + self.assertFalse(sniffer._detect_doublequote( + f'a{quotechar}b{separator}{data}', + delimiter, + quotechar, + )) + def test_guess_delimiter_crlf_not_chosen(self): # Ensure that we pick the real delimiter ("|") over "\r" in a tie. sniffer = csv.Sniffer() diff --git a/Misc/NEWS.d/next/Library/2026-07-29-13-00-00.gh-issue-109638.SnifF1.rst b/Misc/NEWS.d/next/Library/2026-07-29-13-00-00.gh-issue-109638.SnifF1.rst new file mode 100644 index 00000000000000..1481f192346255 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-29-13-00-00.gh-issue-109638.SnifF1.rst @@ -0,0 +1,3 @@ +Prevent pathological regular expression backtracking in :class:`csv.Sniffer` +when detecting doubled quote characters. Improve detection for quoted fields, +avoid matching across quoted fields, and handle CRLF or CR record delimiters. From 91680c671bdb851cafedbfe3f74646607ecb0112 Mon Sep 17 00:00:00 2001 From: Punisheroot <44579963+Punisheroot@users.noreply.github.com> Date: Thu, 30 Jul 2026 11:52:34 +0200 Subject: [PATCH 2/2] gh-109638: Test literal quotes in unquoted fields --- Lib/test/test_csv.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 290aeddff13bfc..ac27b88307d5e6 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1589,11 +1589,16 @@ def test_doublequote_across_quoted_fields(self): sniffer.sniff(data).doublequote, False, ) - self.assertFalse(sniffer._detect_doublequote( - f'a{quotechar}b{separator}{data}', - delimiter, - quotechar, - )) + for literal_quote_count in range(1, 5): + with self.subTest( + literal_quote_count=literal_quote_count, + ): + self.assertFalse(sniffer._detect_doublequote( + f'a{quotechar * literal_quote_count}b' + f'{separator}{data}', + delimiter, + quotechar, + )) def test_guess_delimiter_crlf_not_chosen(self): # Ensure that we pick the real delimiter ("|") over "\r" in a tie.