From 6f4601926b8d848229896d3c44a759644d78e32c Mon Sep 17 00:00:00 2001 From: Max Vorobev Date: Sun, 2 Jul 2017 00:53:44 +0300 Subject: [PATCH 1/2] bpo-30825: guess lineterminator in csv.Sniffer --- Lib/csv.py | 24 +++++++++++++++++++----- Lib/test/test_csv.py | 11 +++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/Lib/csv.py b/Lib/csv.py index 6a8587674fe02b..efd27d70fd7778 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -13,6 +13,7 @@ from collections import OrderedDict from io import StringIO +import os __all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", "Error", "Dialect", "__doc__", "excel", "excel_tab", @@ -165,7 +166,7 @@ def writerows(self, rowdicts): class Sniffer: ''' - "Sniffs" the format of a CSV file (i.e. delimiter, quotechar) + "Sniffs" the format of a CSV file (i.e. delimiter, quotechar, lineterminator) Returns a Dialect object. ''' def __init__(self): @@ -178,18 +179,19 @@ def sniff(self, sample, delimiters=None): Returns a dialect (or None) corresponding to the sample """ + lineterminator = self._guess_lineterminator(sample) quotechar, doublequote, delimiter, skipinitialspace = \ self._guess_quote_and_delimiter(sample, delimiters) if not delimiter: delimiter, skipinitialspace = self._guess_delimiter(sample, - delimiters) + delimiters, + lineterminator) if not delimiter: raise Error("Could not determine delimiter") class dialect(Dialect): _name = "sniffed" - lineterminator = '\r\n' quoting = QUOTE_MINIMAL # escapechar = '' @@ -198,6 +200,7 @@ class dialect(Dialect): # _csv.reader won't accept a quotechar of '' dialect.quotechar = quotechar or '"' dialect.skipinitialspace = skipinitialspace + dialect.lineterminator = lineterminator return dialect @@ -278,7 +281,7 @@ def _guess_quote_and_delimiter(self, data, delimiters): return (quotechar, doublequote, delim, skipinitialspace) - def _guess_delimiter(self, data, delimiters): + def _guess_delimiter(self, data, delimiters, lineterminator): """ The delimiter /should/ occur the same number of times on each row. However, due to malformed data, it may not. We don't want @@ -297,7 +300,7 @@ def _guess_delimiter(self, data, delimiters): additional chunks as necessary. """ - data = list(filter(None, data.split('\n'))) + data = list(filter(None, data.split(lineterminator))) ascii = [chr(c) for c in range(127)] # 7-bit ASCII @@ -447,3 +450,14 @@ def has_header(self, sample): hasHeader -= 1 return hasHeader > 0 + + + def _guess_lineterminator(self, sample): + # Guess line terminator based on presence + terminators = ('\r\n', '\n', '\r') + + for terminator in terminators: + if terminator in sample: + return terminator + + return os.linesep diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 03ab1840dd0df9..b76a7120336fff 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -954,6 +954,7 @@ class TestSniffer(unittest.TestCase): 'Tommy''s Place'+ Blue Island'+ 'IL'+ '12/28/02'+ 'Blue Sunday/White Crow' 'Stonecutters ''Seafood'' and Chop House'+ 'Lemont'+ 'IL'+ '12/19/02'+ 'Week Back' """ + sample10 = "Date;Value\r2010-01-01;10\r2010-01-01;20" def test_has_header(self): sniffer = csv.Sniffer() @@ -1018,6 +1019,16 @@ def test_doublequote(self): dialect = sniffer.sniff(self.sample9) self.assertTrue(dialect.doublequote) + def test_guess_lineterminator(self): + sniffer = csv.Sniffer() + dialect = sniffer.sniff(self.sample5) + self.assertEqual(dialect.lineterminator, '\r\n') + dialect = sniffer.sniff(self.sample8) + self.assertEqual(dialect.lineterminator, '\n') + dialect = sniffer.sniff(self.sample10) + self.assertEqual(dialect.lineterminator, '\r') + + class NUL: def write(s, *args): pass From 524ff18e0c99447121b1bd0ebaf3a9d5b292f65f Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 12 Feb 2018 05:11:04 -0700 Subject: [PATCH 2/2] Fix unit test to cover weird or no terminator case (os.linesep) --- Lib/test/test_csv.py | 12 ++++++++---- .../Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index b76a7120336fff..1fbf9a18650546 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -9,6 +9,7 @@ import csv import gc import pickle +import os from test import support from itertools import permutations from textwrap import dedent @@ -954,7 +955,6 @@ class TestSniffer(unittest.TestCase): 'Tommy''s Place'+ Blue Island'+ 'IL'+ '12/28/02'+ 'Blue Sunday/White Crow' 'Stonecutters ''Seafood'' and Chop House'+ 'Lemont'+ 'IL'+ '12/19/02'+ 'Week Back' """ - sample10 = "Date;Value\r2010-01-01;10\r2010-01-01;20" def test_has_header(self): sniffer = csv.Sniffer() @@ -1021,12 +1021,16 @@ def test_doublequote(self): def test_guess_lineterminator(self): sniffer = csv.Sniffer() - dialect = sniffer.sniff(self.sample5) + dialect = sniffer.sniff(r"Date;Value\r\n2010-01-01;10") self.assertEqual(dialect.lineterminator, '\r\n') - dialect = sniffer.sniff(self.sample8) + dialect = sniffer.sniff(r"Date;Value\n2010-01-01;10") self.assertEqual(dialect.lineterminator, '\n') - dialect = sniffer.sniff(self.sample10) + dialect = sniffer.sniff(r"Date;Value\r2010-01-01;10") self.assertEqual(dialect.lineterminator, '\r') + dialect = sniffer.sniff(r"Date;Value\v2010-01-01;10") + self.assertEqual(dialect.lineterminator, os.linesep) + dialect = sniffer.sniff(r"Date;Value") + self.assertEqual(dialect.lineterminator, os.linesep) class NUL: diff --git a/Misc/NEWS.d/next/Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst b/Misc/NEWS.d/next/Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst new file mode 100644 index 00000000000000..70a3ef14479a4f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst @@ -0,0 +1 @@ +csv.Sniffer now detects line terminator instead of defaulting to '\r\n'