Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 9 additions & 8 deletions Lib/csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -404,18 +404,19 @@ def _guess_delimiter(self, data, delimiters):

# build a list of possible delimiters
modeList = modes.items()
total = float(min(chunkLength * iteration, len(data)))
# (rows of consistent data) / (number of rows) = 100%
consistency = 1.0
# minimum consistency threshold
threshold = 0.9
while len(delims) == 0 and consistency >= threshold:
total = min(chunkLength * iteration, len(data))
# (rows of consistent data) / (number of rows) = 100%, counted down
# in whole percent to a minimum consistency threshold of 90%.
# Integer arithmetic: subtracting 0.01 from a float stopped at
# 0.9099999999999999, so the pass at the threshold never ran.
for consistency in range(100, 89, -1):
if delims:
break
for k, v in modeList:
if v[0] > 0 and v[1] > 0:
if ((v[1]/total) >= consistency and
if (v[1] * 100 >= consistency * total and
(delimiters is None or k in delimiters)):
delims[k] = v
consistency -= 0.01

if len(delims) == 1:
delim = list(delims.keys())[0]
Expand Down
15 changes: 15 additions & 0 deletions Lib/test/test_csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -1521,6 +1521,21 @@ def test_guess_delimiter_crlf_not_chosen(self):
self.assertEqual(sniffer.sniff(sample).delimiter, "|")
self.assertNotEqual(sniffer.sniff(sample).delimiter, "\r")

def test_guess_delimiter_at_minimum_consistency(self):
# gh-111487: the consistency counter was stepped down by repeatedly
# subtracting 0.01 from a float, so it stopped at 0.9099999999999999
# and the pass at the documented 0.9 threshold never ran. Here ";" is
# modal on 19 of the 20 rows, which scores (19 - 1) / 20 == 90% once
# the non-matching row is subtracted -- exactly the threshold.
# Every row uses a different letter so nothing else is consistent, and
# the odd row sits in the first chunk, which keeps that chunk below the
# threshold and defers the decision to the full 20-row chunk.
rows = [(chr(ord('a') + i) + ';') * 3 + chr(ord('a') + i)
for i in range(20)]
rows[4] = 'e;e;e'
sniffer = csv.Sniffer()
self.assertEqual(sniffer.sniff('\n'.join(rows)).delimiter, ';')

def test_zero_mode_tie_order_independence(self):
sniffer = csv.Sniffer()
# ":" appears in half the rows (1, 0, 1, 0) - a tie between
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Fix :meth:`csv.Sniffer.sniff` never testing the documented minimum delimiter
consistency of 90%. The consistency counter was stepped down by repeatedly
subtracting ``0.01`` from a float, so it stopped at ``0.9099999999999999`` and
the final pass at the threshold was skipped; it now counts down in whole
percent using exact integer arithmetic.
Loading