diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 87c2d4702e74086..36db964a8bbfe44 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -312,6 +312,22 @@ The :mod:`!csv` module defines the following classes: is given, it is interpreted as a string containing possible valid delimiter characters. + The dialect is deduced by parsing the sample with every plausible + combination of parameters + and choosing the combination which splits the sample into rows + with the most consistent number of fields, + so the returned dialect is consistent with how :func:`reader` + will parse the sample. + Raise :exc:`Error` if no combination fits the sample, + in particular if it is a single column, + so there is no delimiter to find. + + .. versionchanged:: next + The dialect is now deduced by trial parsing + and the results may differ from those of earlier Python versions. + The *escapechar* parameter can now be detected, + and the requested *delimiters* are not restricted to ASCII. + .. method:: has_header(sample) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index da98f1ad6b9bc3b..98b795de8250d90 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -96,6 +96,21 @@ codecs even when a built-in codec of the same name exists. (Contributed by Serhiy Storchaka in :gh:`152997`.) +csv +--- + +* :meth:`csv.Sniffer.sniff` now deduces the dialect by trial parsing + instead of heuristics based on matching isolated fragments + and on character frequencies, + so the result is consistent with how :func:`csv.reader` will parse the sample. + It can now detect the *escapechar* parameter, + accepts non-ASCII delimiters in the *delimiters* argument, + no longer misdetects the delimiter + when the sample contains delimiter characters inside quoted fields, + and no longer takes quadratic time on quoted samples. + The results may differ from those of earlier Python versions. + (Contributed by Serhiy Storchaka in :gh:`83273`.) + curses ------ diff --git a/Lib/csv.py b/Lib/csv.py index b2aaf5fd9fa91e6..505e9e501e6772b 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -233,6 +233,11 @@ class Sniffer: "Sniffs" the format of a CSV file (i.e. delimiter, quotechar) Returns a Dialect object. ''' + # Characters which can be guessed as a delimiter if the delimiters + # argument is not specified. + _delimiter_candidates = [c for c in map(chr, range(128)) + if not c.isalnum()] + def __init__(self): # in case there is more than one possible delimiter self.preferred = [',', '\t', ';', ' ', ':'] @@ -240,214 +245,336 @@ def __init__(self): def sniff(self, sample, delimiters=None): """ - Returns a dialect (or None) corresponding to the sample + Analyze the sample and return a Dialect subclass reflecting the + parameters found. If the optional delimiters parameter is + given, it is interpreted as a string containing possible valid + delimiter characters. Raises Error if the dialect cannot be + determined. + + The dialect is guessed by parsing the sample with every + plausible combination of delimiter, quotechar and escapechar, + and choosing the combination which splits the sample into rows + with the most consistent number of fields. + + A large sample is parsed incrementally: first only its + beginning, then, after eliminating the combinations which are + clearly worse than the leader, a several times larger part, + and so on. """ + import re + from collections import defaultdict - quotechar, doublequote, delimiter, skipinitialspace = \ - self._guess_quote_and_delimiter(sample, delimiters) - if not delimiter: - delimiter, skipinitialspace = self._guess_delimiter(sample, - delimiters) + if self._parses_as_single_column(sample): + # There is no delimiter to find; any combination could + # only find a bogus one inside the quoted fields. + raise Error("Could not determine delimiter") - if not delimiter: + chars = set(sample) + if delimiters is None: + delimiters = self._delimiter_candidates + delimiters = [d for d in delimiters + if d in chars and d not in '\r\n"\'\\'] + # Combinations to try, numbered by preference for breaking + # ties. The unquoted combinations are parsed from the start; + # the rest stay dormant until the quote character occurs at + # the start of a field (see _split_dormant). + groups = defaultdict(list) + order = 0 + # Only '\\' is tried as an escape character: others are not + # seen in the wild. + for escapechar in ('', '\\') if '\\' in chars else ('',): + for quotechar in '"', "'", '': + if quotechar and quotechar not in chars: + continue + for delimiter in delimiters: + groups[quotechar].append( + (order, delimiter, quotechar, escapechar)) + order += 1 + active = groups.pop('', []) + # Only non-empty groups were created; a plain dict cannot + # grow one by accident. + dormant = dict(groups) + + # The initial window should cover the minimal number of rows + # required for elimination (see _eliminate_worse) at a typical + # line length, so that the first round can already eliminate. + window = 2000 + # A line with its line break: '\r', '\n' or '\r\n' (the + # reader treats other line boundary characters as ordinary + # data, but does not support a bare '\r' inside a chunk). + # The \z alternative produces one final empty match. + line_re = re.compile(r'[^\r\n]*(?:\r\n|[\r\n]|\z)') + parsed = [] + lines = [] + first_round = True + while active or dormant: + end = min(window, len(sample)) + part = sample[:end] + lines = line_re.findall(part) + del lines[-1] + cut = not part.endswith(('\r', '\n')) + for quotechar in list(dormant): + activated, still = self._split_dormant( + part, quotechar, dormant[quotechar]) + active += activated + if still: + dormant[quotechar] = still + else: + del dormant[quotechar] + parsed = [(combo, self._try_dialect(lines, cut, *combo[1:])) + for combo in active] + if end == len(sample): + break + active = self._eliminate_worse(parsed, not first_round) + first_round = False + if len(active) <= 3: + # Quoted data most often leaves three survivors: the + # true dialect, its equally consistent unquoted shadow, + # and one accident. Parsing the whole sample with them + # is cheaper than another elimination round. + window = len(sample) + else: + # Too small a factor would increase the total + # re-parsing cost, too large -- the cost of the next + # round if this one did not eliminate enough. + window *= 4 + + best = None + best_score = None + for combo, rows in sorted(parsed): + if rows is None: + continue + _, delimiter, quotechar, escapechar = combo + nfields, share = self._modal_share(rows) + if nfields < 2: + # The delimiter does not delimit anything. + continue + try: + preference = -self.preferred.index(delimiter) + except ValueError: + preference = -len(self.preferred) + # A successful quoted parse is direct evidence; the preferred + # delimiters list is only a nudge. + score = (share, len(rows), bool(quotechar), preference) + if best_score is None or score > best_score: + best_score = score + best = combo[1:] + + if best is None: raise Error("Could not determine delimiter") + delimiter, quotechar, escapechar = best + doublequote = self._detect_doublequote(lines, *best) + skipinitialspace = self._detect_skipinitialspace(lines, *best, + doublequote) class dialect(Dialect): _name = "sniffed" lineterminator = '\r\n' quoting = QUOTE_MINIMAL - # escapechar = '' - dialect.doublequote = doublequote dialect.delimiter = delimiter # _csv.reader won't accept a quotechar of '' dialect.quotechar = quotechar or '"' + dialect.escapechar = escapechar or None + dialect.doublequote = doublequote dialect.skipinitialspace = skipinitialspace return dialect - - def _guess_quote_and_delimiter(self, data, delimiters): + def _parses_as_single_column(self, sample): """ - Looks for text enclosed between two identical quotes - (the probable quotechar) which are preceded and followed - by the same character (the probable delimiter). - For example: - ,'some text', - The quote with the most wins, same with the delimiter. - If there is no quotechar the delimiter can't be determined - this way. + True if the whole sample parses as a single column of quoted + fields (the last one may be cut off in the middle), so there + is no delimiter to find. """ import re - matches = [] - for restr in (r'(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?P=delim)', # ,".*?", - r'(?:^|\n)(?P["\']).*?(?P=quote)(?P[^\w\n"\'])(?P ?)', # ".*?", - r'(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?:$|\n)', # ,".*?" - r'(?:^|\n)(?P["\']).*?(?P=quote)(?:$|\n)'): # ".*?" (no delim, no space) - regexp = re.compile(restr, re.DOTALL | re.MULTILINE) - matches = regexp.findall(data) - if matches: - break - - if not matches: - # (quotechar, doublequote, delimiter, skipinitialspace) - return ('', False, None, 0) - quotes = {} - delims = {} - spaces = 0 - groupindex = regexp.groupindex - for m in matches: - n = groupindex['quote'] - 1 - key = m[n] - if key: - quotes[key] = quotes.get(key, 0) + 1 - try: - n = groupindex['delim'] - 1 - key = m[n] - except KeyError: - continue - if key and (delimiters is None or key in delimiters): - delims[key] = delims.get(key, 0) + 1 - try: - n = groupindex['space'] - 1 - except KeyError: - continue - if m[n]: - spaces += 1 - - quotechar = max(quotes, key=quotes.get) - - if delims: - delim = max(delims, key=delims.get) - skipinitialspace = delims[delim] == spaces - if delim == '\n': # most likely a file with a single column - delim = '' - else: - # there is *no* delimiter, it's a single column of quoted data - delim = '' - skipinitialspace = 0 - - # if we see an extra quote between delimiters, we've got a - # double quoted format - dq_regexp = re.compile( - r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \ - {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE) - + for q in '"', "'": + if q in sample: + row_re = (fr' *+{q}(?:[^{q}]|{q}{q})*+' + fr'(?:{q} *+(?:[\r\n]++|\z)|\z)') + if re.fullmatch(fr'(?:{row_re})++', sample): + return True + return False + def _split_dormant(self, part, quotechar, combos): + """ + Split the dormant combinations into those ready for trial + parsing and the rest. + + A combination is ready when its quote character occurs in + *part* at the start of a field, i.e. at the start of a line or + after its delimiter; until then parsing would not differ from + the unquoted variant. Spaces before the quote are allowed even + for the space delimiter: a false activation only costs a trial + parse. + """ + import re - if dq_regexp.search(data): - doublequote = True + remaining = {combo[1] for combo in combos} + found = set() + pos = 0 + while remaining: + # Include only the delimiters not found yet, so that the + # search skips over the found ones; the compiled patterns + # come from the re cache. + cls = re.escape(''.join(sorted(remaining))) + m = re.compile(fr'(?:^|([\r\n{cls}]))' + fr' *{quotechar}').search(part, pos) + if m is None: + break + pre = m[1] + if pre is None or pre in '\r\n': + # A quote at the start of a line starts a field for + # every delimiter. + found |= remaining + break + found.add(pre) + remaining.discard(pre) + pos = m.end() + activated = [combo for combo in combos if combo[1] in found] + still_dormant = [combo for combo in combos if combo[1] not in found] + return activated, still_dormant + + def _make_reader(self, lines, delimiter, quotechar, escapechar, + doublequote=True, skipinitialspace=None): + """ + Create a reader for trial parsing. quotechar '' means no + quoting and escapechar '' means no escape character. + """ + if skipinitialspace is None: + # Be lenient to spaces after a delimiter, unless the + # delimiter is a space itself. + skipinitialspace = delimiter != ' ' + return reader(lines, delimiter=delimiter, + quotechar=quotechar or '"', + quoting=QUOTE_MINIMAL if quotechar else QUOTE_NONE, + escapechar=escapechar or None, + doublequote=doublequote, + skipinitialspace=skipinitialspace, + strict=True) + + def _try_dialect(self, lines, cut, delimiter, quotechar, escapechar): + """ + Parse the sample, pre-split into *lines*, and return the list + of the number of fields in every parsed row, or None if not a + single row was parsed. + + If the sample cannot be parsed to the end (for example it is + cut off in the middle of a quoted field, or the combination + does not fit the sample), the rows parsed so far are counted. + The last row is not counted if *cut* is true: the sample can + be cut off in the middle of it. + """ + rows = [] + try: + rows.extend(map(len, self._make_reader(lines, delimiter, + quotechar, escapechar))) + except Error: + # The row which failed to parse is not counted. + pass else: - doublequote = False + if cut and len(rows) > 1: + rows.pop() + if 0 in rows: + # Blank lines produce empty rows. + rows = [nfields for nfields in rows if nfields] + return rows or None + + def _eliminate_worse(self, parsed, judge_hopeless): + """ + Return the combinations from *parsed* (a list of (combination, + rows) pairs) without those which are clearly worse than the + leader. Combinations with too few parsed rows (e.g. if the + parsed part ends in the middle of a large quoted field) are + not judged yet. + + If *judge_hopeless* is false, keep the combinations whose + delimiter does not delimit anything. Unlike the comparison + with the leader, which self-normalizes when the parsed part is + not representative, this verdict is absolute and irreversible, + so it is not trusted to the first part, which covers the least + representative beginning of the sample (titles, headers, + preamble). + """ + # Judging a combination by fewer rows is too noisy. + min_rows = 16 + hopeless = set() + scores = {} + for combo, rows in parsed: + if rows is not None and len(rows) >= min_rows: + nfields, share = self._modal_share(rows) + if nfields < 2: + if judge_hopeless: + hopeless.add(combo) + else: + scores[combo] = share + threshold = max(scores.values(), default=0.0) - 0.1 + return [combo for combo, _ in parsed + if combo not in hopeless + and scores.get(combo, threshold) >= threshold] - return (quotechar, doublequote, delim, skipinitialspace) + def _modal_share(self, rows): + """ + The most common number of fields in a row and its share of all + rows. Prefer the smaller number of fields in the case of a + tie: a candidate delimiter which delimits only half of the rows + is not convincing. + """ + from collections import Counter + counts = Counter(rows) + nfields = max(counts, key=lambda n: (counts[n], -n)) + return nfields, counts[nfields] / len(rows) - def _guess_delimiter(self, data, delimiters): + def _detect_doublequote(self, lines, delimiter, quotechar, escapechar): """ - The delimiter /should/ occur the same number of times on - each row. However, due to malformed data, it may not. We don't want - an all or nothing approach, so we allow for small variations in this - number. - 1) build a table of the frequency of each character on every line. - 2) build a table of frequencies of this frequency (meta-frequency?), - e.g. 'x occurred 5 times in 10 rows, 6 times in 1000 rows, - 7 times in 2 rows' - 3) use the mode of the meta-frequency to determine the /expected/ - frequency for that character - 4) find out how often the character actually meets that goal - 5) the character that best meets its goal is the delimiter - For performance reasons, the data is evaluated in chunks, so it can - try and evaluate the smallest portion of the data possible, evaluating - additional chunks as necessary. + True if a doubled quote character represents a single quote + character in the sample: interpreting it so changes the result + of parsing. """ - from collections import Counter, defaultdict - - data = list(filter(None, data.split('\n'))) - - # build frequency tables - chunkLength = min(10, len(data)) - iteration = 0 - num_lines = 0 - # {char -> {count_per_line -> num_lines_with_that_count}} - char_frequency = defaultdict(Counter) - modes = {} - delims = {} - start, end = 0, chunkLength - while start < len(data): - iteration += 1 - for line in data[start:end]: - num_lines += 1 - for char, count in Counter(line).items(): - if char.isascii(): - char_frequency[char][count] += 1 - - for char, counts in char_frequency.items(): - items = list(counts.items()) - missed_lines = num_lines - sum(counts.values()) - if missed_lines: - # Store the number of lines 'char' was missing from. - items.append((0, missed_lines)) - if len(items) == 1 and items[0][0] == 0: - continue - # get the mode of the frequencies - if len(items) > 1: - modes[char] = max(items, key=lambda x: x[1]) - # adjust the mode - subtract the sum of all - # other frequencies - items.remove(modes[char]) - modes[char] = (modes[char][0], modes[char][1] - - sum(item[1] for item in items)) - else: - modes[char] = items[0] - - # build a list of possible delimiters - modeList = modes.items() - total = float(min(chunkLength * iteration, len(data))) - # (rows of consistent data) / (number of rows) = 100% - consistency = 1.0 - # minimum consistency threshold - threshold = 0.9 - while len(delims) == 0 and consistency >= threshold: - for k, v in modeList: - if v[0] > 0 and v[1] > 0: - if ((v[1]/total) >= consistency and - (delimiters is None or k in delimiters)): - delims[k] = v - consistency -= 0.01 - - if len(delims) == 1: - delim = list(delims.keys())[0] - skipinitialspace = (data[0].count(delim) == - data[0].count("%c " % delim)) - return (delim, skipinitialspace) - - # analyze another chunkLength lines - start = end - end += chunkLength - - if not delims: - return ('', 0) - - # if there's more than one, fall back to a 'preferred' list - if len(delims) > 1: - for d in self.preferred: - if d in delims.keys(): - skipinitialspace = (data[0].count(d) == - data[0].count("%c " % d)) - return (d, skipinitialspace) - - # nothing else indicates a preference, pick the character that - # dominates(?) - items = [(v,k) for (k,v) in delims.items()] - items.sort() - delim = items[-1][1] - - skipinitialspace = (data[0].count(delim) == - data[0].count("%c " % delim)) - return (delim, skipinitialspace) - + if not quotechar or not any(quotechar * 2 in line + for line in lines): + return False + readers = [self._make_reader( + lines, delimiter, quotechar, escapechar, + doublequote=doublequote) + for doublequote in (False, True)] + while True: + rows = [] + for rdr in readers: + try: + rows.append(next(rdr)) + except (StopIteration, Error): + # Ending cleanly and failing are equivalent here: + # after equal rows both readers are at the same + # position, so they cannot end for different + # reasons. + rows.append(None) + if rows[0] != rows[1]: + return True + if rows == [None, None]: + return False + + def _detect_skipinitialspace(self, lines, delimiter, quotechar, + escapechar, doublequote): + """ + True only if every field following a delimiter starts with + a space. + """ + skipinitialspace = False + try: + for row in self._make_reader(lines, delimiter, quotechar, + escapechar, + doublequote=doublequote, + skipinitialspace=False): + for field in row[1:]: + if not field.startswith(' '): + return False + skipinitialspace = True + except Error: + pass + return skipinitialspace def has_header(self, sample): # Creates a dictionary of types of data in each column. If any diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 7327c1bd5f50530..0e95b127320674b 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1443,13 +1443,16 @@ def test_has_header_checks_20_rows(self): def test_guess_quote_and_delimiter(self): sniffer = csv.Sniffer() - for header in (";'123;4';", "'123;4';", ";'123;4'", "'123;4'"): + for header in (";'123;4';", "'123;4';", ";'123;4'"): with self.subTest(header): dialect = sniffer.sniff(header, ",;") self.assertEqual(dialect.delimiter, ';') self.assertEqual(dialect.quotechar, "'") self.assertIs(dialect.doublequote, False) self.assertIs(dialect.skipinitialspace, False) + # A single quoted field is a single column without a delimiter. + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, "'123;4'", ",;") def test_sniff(self): sniffer = csv.Sniffer() @@ -1495,6 +1498,196 @@ def test_delimiters(self): self.assertRaisesRegex(csv.Error, "Could not determine delimiter", sniffer.sniff, self.sample16) + def test_sniff_escapechar(self): + # gh-83273: escaped delimiters make the delimiter frequencies + # inconsistent, but the escape character can be detected by trial + # parsing. + sniffer = csv.Sniffer() + sample = 'ab,cd\\,ef\ngh\\,ij,kl\nmn,op\n' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.escapechar, '\\') + self.assertIs(dialect.doublequote, False) + self.assertEqual(list(csv.reader(StringIO(sample), dialect)), + [['ab', 'cd,ef'], ['gh,ij', 'kl'], ['mn', 'op']]) + # No escape character in the sample -- none is detected. + dialect = sniffer.sniff('ab,cd\ngh,ij\n') + self.assertEqual(dialect.delimiter, ',') + self.assertIsNone(dialect.escapechar) + + def test_sniff_quoted_rows_among_unquoted(self): + # Rows which happen to consist of a single quoted or unquoted + # field must not be mistaken for a single column of quoted fields. + sniffer = csv.Sniffer() + sample = '"header line"\na|b\nc|d\ne|f\n' + self.assertEqual(sniffer.sniff(sample).delimiter, '|') + # Even if an unterminated quote breaks the quoted parse. + sample = '"header"\na|"b c\nd|e\nf|g\n' + self.assertEqual(sniffer.sniff(sample).delimiter, '|') + + def test_sniff_single_column_quoted(self): + # gh-98820: a sample consisting of a single column of quoted fields + # has no delimiter to detect, even if the quoted content contains + # characters which could pass for one. It also used to take + # quadratic time before failing. + sniffer = csv.Sniffer() + sample = '\n'.join('"%02d-%02d-%02d"' % (i, i, i) for i in range(50)) + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, sample) + # Even if none of the requested delimiters occurs in the sample. + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, sample, ',:|\t') + # A mix of quoted and unquoted single-field rows. + sample = '"abc"\ndef\n"ghi"\njkl\n' * 10 + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, sample) + # Even if the quoted fields contain delimiter characters. + sample = '"a,b"\ncd\n' * 20 + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, sample) + # Even if the content of the quoted fields parses consistently + # with the other quote character. + sample = '''"a-'b'-c"\n"d-'e'-f"\n''' * 10 + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, sample) + + def test_sniff_non_ascii_delimiter(self): + # gh-111820: an explicitly requested delimiter can be non-ASCII. + sniffer = csv.Sniffer() + sample = 'aaa\xacbbb\xacccc\nddd\xaceee\xacfff\nggg\xachhh\xaciii\n' + dialect = sniffer.sniff(sample, delimiters='\xac') + self.assertEqual(dialect.delimiter, '\xac') + + def test_sniff_preamble(self): + # A preamble (title or comment lines) before the data must not + # prevent detection, even if it is larger than the part of the + # sample which is parsed first. It is enough for the data rows + # to slightly outnumber the preamble lines. + sniffer = csv.Sniffer() + for n in 3, 24, 80, 320: + preamble = ''.join(f'Comment line {i:05}\n' for i in range(n)) + for ndata in n + 2, 2 * n + 5: + with self.subTest(preamble_lines=n, data_lines=ndata): + sample = preamble + 'aaa,bbb,ccc\n' * ndata + self.assertEqual(sniffer.sniff(sample).delimiter, ',') + + def test_sniff_line_boundary_characters(self): + # Only '\r', '\n' and '\r\n' are row separators; characters like + # '\x1c' or '\x85', which str.splitlines() treats as line + # boundaries, are ordinary data for the reader -- '\x1c' can + # even be the delimiter. + sniffer = csv.Sniffer() + sample = 'aa\x1cbb\ncc\x1cdd\nee\x1cff\n' + self.assertEqual(sniffer.sniff(sample).delimiter, '\x1c') + sample = 'a\x85b,c\nd,e\nf,g\n' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(next(csv.reader(StringIO(sample), dialect)), + ['a\x85b', 'c']) + + def test_sniff_delimiter_in_quoted_field(self): + # gh-97611: a delimiter inside a quoted field should not win over + # the delimiter which actually separates the fields. + sniffer = csv.Sniffer() + sample = ( + 'Surname;First Name;Year of birth\n' + '"\tDoe;Jane"\t;1971\n' + ) + dialect = sniffer.sniff(sample, delimiters=',;\t') + self.assertEqual(dialect.delimiter, ';') + sample = ( + 'Surname;First Name;Year of birth\n' + '"\t";"\t";1971\n' + '"Le Trec";"Mary Ann";1486\n' + ) + dialect = sniffer.sniff(sample, delimiters=',;\t') + self.assertEqual(dialect.delimiter, ';') + self.assertEqual(dialect.quotechar, '"') + + def test_sniff_embedded_lists(self): + # gh-119123: commas in bracketed lists adjacent to quotes should + # not be mistaken for the delimiter. + sniffer = csv.Sniffer() + sample = ( + "id;is_sort;cost;group;merge\n" + "1;True;62.25;['345'];UNKNOWN\n" + "2;True;54.00;['235'];UNKNOWN\n" + "3;True;237.00;['567', '568'];UNKNOWN\n" + "4;True;46.50;['112', '112'];UNKNOWN\n" + ) + dialect = sniffer.sniff(sample, delimiters=',;') + self.assertEqual(dialect.delimiter, ';') + + def test_sniff_space_adjacent_to_quotes(self): + # gh-88843: a space adjacent to stray quotes should not be + # detected as the delimiter. + sniffer = csv.Sniffer() + sample = "a|b\nc| 'd\ne|' f" + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, '|') + + def test_sniff_crlf_lineterminator(self): + # gh-103925: a quote at the end of a \r\n-terminated line. + sniffer = csv.Sniffer() + sample = ( + 'Timestamp,URL,Title\r\n' + '2020-10-01 17:17:37+08:00,' + 'https://www.mozilla.org/en-US/firefox/welcome/2/,' + '"Pocket - Save news, videos, stories and more"\r\n' + ) + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.quotechar, '"') + + def test_sniff_excel_tab_with_quotes(self): + # gh-62029: tab-delimited data with a quoted field containing + # spaces and doubled quotes. + sniffer = csv.Sniffer() + sample = ('foo\tbar\t"baz ""quoted"" here"\tspam eggs\n' + 'ham\teggs\tx y\tz w\n') + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, '\t') + self.assertEqual(dialect.quotechar, '"') + self.assertIs(dialect.doublequote, True) + + def test_sniff_truncated_sample(self): + # A sample cut off in the middle of a quoted field should not + # spoil the detection. + sniffer = csv.Sniffer() + sample = '"a,a";"b"\n"c";"d"\n"e";"f,f\n' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ';') + self.assertEqual(dialect.quotechar, '"') + # Cut off in the middle of the last row. + sample = 'a,b,c\nd,e,f\ng,h,i\nj,k' + self.assertEqual(sniffer.sniff(sample).delimiter, ',') + # Cut off in the middle of an escaped sequence. + sample = 'ab,cd\\,ef\ngh\\,ij,kl\nmn,op\nqr,st\\' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.escapechar, '\\') + # Cut off at a line end in the middle of a quoted field. + sample = '"a","b"\n"c","d"\n"e","multi\nline' + self.assertEqual(sniffer.sniff(sample).delimiter, ',') + # An unclosed quote consumes everything to the end of the sample, + # but the rows before it still count. + sample = 'a|b\nc|d\ne|f\ng|"h\ni|j\nk|l' + self.assertEqual(sniffer.sniff(sample).delimiter, '|') + + def test_sniff_skipinitialspace_quoted(self): + sniffer = csv.Sniffer() + sample = "'a': 'b': 'c'\n'd': 'e': 'f'\n" + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ':') + self.assertEqual(dialect.quotechar, "'") + self.assertIs(dialect.skipinitialspace, True) + + def test_sniff_regex_backtracking(self): + # gh-109638: this artificial sample used to take minutes. + sniffer = csv.Sniffer() + sample = '"",' * 100 + '"' * 100 + '0' + '"' * 100 + '0' + self.assertEqual(sniffer.sniff(sample).delimiter, ',') + def test_doublequote(self): sniffer = csv.Sniffer() dialect = sniffer.sniff(self.header1) @@ -1507,6 +1700,15 @@ def test_doublequote(self): self.assertFalse(dialect.doublequote) dialect = sniffer.sniff(self.sample9) self.assertTrue(dialect.doublequote) + # A doubled quote character in an unquoted field or an empty + # quoted field is not evidence of doubling. + dialect = sniffer.sniff('"x",a""b\n"y",c\n') + self.assertIs(dialect.doublequote, False) + dialect = sniffer.sniff('a,"",c\nd,"",f\n') + self.assertIs(dialect.doublequote, False) + # A doubled quote character inside a quoted field is. + dialect = sniffer.sniff('"a""b",c\n"d",e\n') + self.assertIs(dialect.doublequote, True) def test_guess_delimiter_crlf_not_chosen(self): # Ensure that we pick the real delimiter ("|") over "\r" in a tie. diff --git a/Misc/NEWS.d/next/Library/2026-07-14-12-00-00.gh-issue-83273.SnifQ7.rst b/Misc/NEWS.d/next/Library/2026-07-14-12-00-00.gh-issue-83273.SnifQ7.rst new file mode 100644 index 000000000000000..789503913048f94 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-14-12-00-00.gh-issue-83273.SnifQ7.rst @@ -0,0 +1,10 @@ +:meth:`csv.Sniffer.sniff` now deduces the dialect by trial parsing with the +actual CSV parser instead of heuristics based on matching isolated fragments +and on character frequencies. It can now +detect the *escapechar* parameter, accepts non-ASCII delimiters in the +*delimiters* argument, no longer misdetects the delimiter when the sample +contains delimiter characters inside quoted fields, handles samples +truncated at an arbitrary point, and no longer takes quadratic time on +quoted samples. A sample consisting of a single column of quoted fields +now raises :exc:`csv.Error` instead of guessing a delimiter from the +content of the fields.