Skip to content

Commit fd78b56

Browse files
[3.10] gh-98820: Fix quadratic time in csv.Sniffer for quoted fields (GH-154867) (#155539)
1 parent 12dcbd7 commit fd78b56

3 files changed

Lines changed: 20 additions & 5 deletions

File tree

Lib/csv.py

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -215,12 +215,18 @@ def _guess_quote_and_delimiter(self, data, delimiters):
215215
this way.
216216
"""
217217

218+
# The body of a quoted field ends at the first quote which is
219+
# not doubled, as it does for a reader. A lazy ".*?" scans to
220+
# the end of the sample instead, from every start: quadratically.
221+
# As an unrolled loop it is unambiguous, so it does not backtrack.
222+
other = r'(?:(?!(?P=quote)).)*'
223+
body = r'%s(?:(?P=quote){2}%s)*' % (other, other)
218224
matches = []
219-
for restr in (r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?P=delim)', # ,".*?",
220-
r'(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)', # ".*?",
221-
r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\n)', # ,".*?"
222-
r'(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?:$|\n)'): # ".*?" (no delim, no space)
223-
regexp = re.compile(restr, re.DOTALL | re.MULTILINE)
225+
for restr in (r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\'])%s(?P=quote)(?P=delim)', # ,"...",
226+
r'(?:^|\n)(?P<quote>["\'])%s(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)', # "...",
227+
r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\'])%s(?P=quote)(?:$|\n)', # ,"..."
228+
r'(?:^|\n)(?P<quote>["\'])%s(?P=quote)(?:$|\n)'): # "..." (no delim, no space)
229+
regexp = re.compile(restr % body, re.DOTALL | re.MULTILINE)
224230
matches = regexp.findall(data)
225231
if matches:
226232
break

Lib/test/test_csv.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1232,6 +1232,13 @@ def test_sniff_space_delimiter(self):
12321232
self.assertEqual(dialect.delimiter, ' ')
12331233
self.assertIs(dialect.doublequote, False)
12341234

1235+
def test_sniff_quoted_single_column(self):
1236+
# gh-98820: this sample used to take minutes.
1237+
sniffer = csv.Sniffer()
1238+
sample = '"abcdefghijklmnopqrstuvwxyz"\n' * 10000
1239+
with self.assertRaisesRegex(csv.Error, "Could not determine delimiter"):
1240+
sniffer.sniff(sample, delimiters=',:|\t')
1241+
12351242

12361243
class NUL:
12371244
def write(s, *args):
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix quadratic time in :meth:`csv.Sniffer.sniff` for a sample which contains
2+
quoted fields, in particular for a single column of quoted fields.

0 commit comments

Comments
 (0)