From fec87c9992c22dc7333cdff280f1a8f30f12b85f Mon Sep 17 00:00:00 2001 From: Max Vorobev Date: Sun, 2 Jul 2017 00:53:44 +0300 Subject: [PATCH 1/4] bpo-30825: guess lineterminator in csv.Sniffer --- Lib/csv.py | 24 +++++++++++++++++++----- Lib/test/test_csv.py | 11 +++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/Lib/csv.py b/Lib/csv.py index dc85077f3ec6636..0aad7595e0415c4 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -12,6 +12,7 @@ from _csv import Dialect as _Dialect from io import StringIO +import os __all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", "Error", "Dialect", "__doc__", "excel", "excel_tab", @@ -164,7 +165,7 @@ def writerows(self, rowdicts): class Sniffer: ''' - "Sniffs" the format of a CSV file (i.e. delimiter, quotechar) + "Sniffs" the format of a CSV file (i.e. delimiter, quotechar, lineterminator) Returns a Dialect object. ''' def __init__(self): @@ -177,18 +178,19 @@ def sniff(self, sample, delimiters=None): Returns a dialect (or None) corresponding to the sample """ + lineterminator = self._guess_lineterminator(sample) quotechar, doublequote, delimiter, skipinitialspace = \ self._guess_quote_and_delimiter(sample, delimiters) if not delimiter: delimiter, skipinitialspace = self._guess_delimiter(sample, - delimiters) + delimiters, + lineterminator) if not delimiter: raise Error("Could not determine delimiter") class dialect(Dialect): _name = "sniffed" - lineterminator = '\r\n' quoting = QUOTE_MINIMAL # escapechar = '' @@ -197,6 +199,7 @@ class dialect(Dialect): # _csv.reader won't accept a quotechar of '' dialect.quotechar = quotechar or '"' dialect.skipinitialspace = skipinitialspace + dialect.lineterminator = lineterminator return dialect @@ -277,7 +280,7 @@ def _guess_quote_and_delimiter(self, data, delimiters): return (quotechar, doublequote, delim, skipinitialspace) - def _guess_delimiter(self, data, delimiters): + def _guess_delimiter(self, data, delimiters, lineterminator): """ The delimiter /should/ occur the same number of times on each row. However, due to malformed data, it may not. We don't want @@ -296,7 +299,7 @@ def _guess_delimiter(self, data, delimiters): additional chunks as necessary. """ - data = list(filter(None, data.split('\n'))) + data = list(filter(None, data.split(lineterminator))) ascii = [chr(c) for c in range(127)] # 7-bit ASCII @@ -446,3 +449,14 @@ def has_header(self, sample): hasHeader -= 1 return hasHeader > 0 + + + def _guess_lineterminator(self, sample): + # Guess line terminator based on presence + terminators = ('\r\n', '\n', '\r') + + for terminator in terminators: + if terminator in sample: + return terminator + + return os.linesep diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index a16d14019f341f6..c62891c9a13d0b6 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -979,6 +979,7 @@ class TestSniffer(unittest.TestCase): 'Tommy''s Place'+ Blue Island'+ 'IL'+ '12/28/02'+ 'Blue Sunday/White Crow' 'Stonecutters ''Seafood'' and Chop House'+ 'Lemont'+ 'IL'+ '12/19/02'+ 'Week Back' """ + sample10 = "Date;Value\r2010-01-01;10\r2010-01-01;20" def test_has_header(self): sniffer = csv.Sniffer() @@ -1051,6 +1052,16 @@ def test_doublequote(self): dialect = sniffer.sniff(self.sample9) self.assertTrue(dialect.doublequote) + def test_guess_lineterminator(self): + sniffer = csv.Sniffer() + dialect = sniffer.sniff(self.sample5) + self.assertEqual(dialect.lineterminator, '\r\n') + dialect = sniffer.sniff(self.sample8) + self.assertEqual(dialect.lineterminator, '\n') + dialect = sniffer.sniff(self.sample10) + self.assertEqual(dialect.lineterminator, '\r') + + class NUL: def write(s, *args): pass From f99b927d622ba0ba9d61f21d98a1c5a283b274a5 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 12 Feb 2018 05:11:04 -0700 Subject: [PATCH 2/4] Fix unit test to cover weird or no terminator case (os.linesep) --- Lib/test/test_csv.py | 12 ++++++++---- .../Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index c62891c9a13d0b6..20fc947bdf9eb3c 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -9,6 +9,7 @@ import csv import gc import pickle +import os from test import support from itertools import permutations from textwrap import dedent @@ -979,7 +980,6 @@ class TestSniffer(unittest.TestCase): 'Tommy''s Place'+ Blue Island'+ 'IL'+ '12/28/02'+ 'Blue Sunday/White Crow' 'Stonecutters ''Seafood'' and Chop House'+ 'Lemont'+ 'IL'+ '12/19/02'+ 'Week Back' """ - sample10 = "Date;Value\r2010-01-01;10\r2010-01-01;20" def test_has_header(self): sniffer = csv.Sniffer() @@ -1054,12 +1054,16 @@ def test_doublequote(self): def test_guess_lineterminator(self): sniffer = csv.Sniffer() - dialect = sniffer.sniff(self.sample5) + dialect = sniffer.sniff(r"Date;Value\r\n2010-01-01;10") self.assertEqual(dialect.lineterminator, '\r\n') - dialect = sniffer.sniff(self.sample8) + dialect = sniffer.sniff(r"Date;Value\n2010-01-01;10") self.assertEqual(dialect.lineterminator, '\n') - dialect = sniffer.sniff(self.sample10) + dialect = sniffer.sniff(r"Date;Value\r2010-01-01;10") self.assertEqual(dialect.lineterminator, '\r') + dialect = sniffer.sniff(r"Date;Value\v2010-01-01;10") + self.assertEqual(dialect.lineterminator, os.linesep) + dialect = sniffer.sniff(r"Date;Value") + self.assertEqual(dialect.lineterminator, os.linesep) class NUL: diff --git a/Misc/NEWS.d/next/Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst b/Misc/NEWS.d/next/Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst new file mode 100644 index 000000000000000..70a3ef14479a4fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst @@ -0,0 +1 @@ +csv.Sniffer now detects line terminator instead of defaulting to '\r\n' From 1f9bdc601ab60fef9879e1ba543f992ece80a4af Mon Sep 17 00:00:00 2001 From: Max Vorobev Date: Mon, 3 Feb 2020 22:29:34 +0300 Subject: [PATCH 3/4] Do not use raw strings in tests --- Lib/test/test_csv.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 20fc947bdf9eb3c..965d90c43c1ad46 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1054,15 +1054,15 @@ def test_doublequote(self): def test_guess_lineterminator(self): sniffer = csv.Sniffer() - dialect = sniffer.sniff(r"Date;Value\r\n2010-01-01;10") + dialect = sniffer.sniff('Date;Value\r\n2010-01-01;10') self.assertEqual(dialect.lineterminator, '\r\n') - dialect = sniffer.sniff(r"Date;Value\n2010-01-01;10") + dialect = sniffer.sniff('Date;Value\n2010-01-01;10') self.assertEqual(dialect.lineterminator, '\n') - dialect = sniffer.sniff(r"Date;Value\r2010-01-01;10") + dialect = sniffer.sniff('Date;Value\r2010-01-01;10') self.assertEqual(dialect.lineterminator, '\r') - dialect = sniffer.sniff(r"Date;Value\v2010-01-01;10") + dialect = sniffer.sniff('Date;Value\v2010-01-01;10') self.assertEqual(dialect.lineterminator, os.linesep) - dialect = sniffer.sniff(r"Date;Value") + dialect = sniffer.sniff('Date;Value') self.assertEqual(dialect.lineterminator, os.linesep) From e1e53168c7fac64d8ee0ab749e68905b2a4d0611 Mon Sep 17 00:00:00 2001 From: Max Vorobev Date: Thu, 10 Oct 2024 08:38:58 +0000 Subject: [PATCH 4/4] address review comments --- Lib/csv.py | 4 ++-- Lib/test/test_csv.py | 6 +++--- .../next/Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/csv.py b/Lib/csv.py index 0aad7595e0415c4..e60e6dbb65b2715 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -12,7 +12,6 @@ from _csv import Dialect as _Dialect from io import StringIO -import os __all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", "Error", "Dialect", "__doc__", "excel", "excel_tab", @@ -459,4 +458,5 @@ def _guess_lineterminator(self, sample): if terminator in sample: return terminator - return os.linesep + from os import linesep + return linesep diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 965d90c43c1ad46..fd58cb9683afe49 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -9,7 +9,6 @@ import csv import gc import pickle -import os from test import support from itertools import permutations from textwrap import dedent @@ -1053,6 +1052,7 @@ def test_doublequote(self): self.assertTrue(dialect.doublequote) def test_guess_lineterminator(self): + from os import linesep sniffer = csv.Sniffer() dialect = sniffer.sniff('Date;Value\r\n2010-01-01;10') self.assertEqual(dialect.lineterminator, '\r\n') @@ -1061,9 +1061,9 @@ def test_guess_lineterminator(self): dialect = sniffer.sniff('Date;Value\r2010-01-01;10') self.assertEqual(dialect.lineterminator, '\r') dialect = sniffer.sniff('Date;Value\v2010-01-01;10') - self.assertEqual(dialect.lineterminator, os.linesep) + self.assertEqual(dialect.lineterminator, linesep) dialect = sniffer.sniff('Date;Value') - self.assertEqual(dialect.lineterminator, os.linesep) + self.assertEqual(dialect.lineterminator, linesep) class NUL: diff --git a/Misc/NEWS.d/next/Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst b/Misc/NEWS.d/next/Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst index 70a3ef14479a4fb..f59df8a8df41e73 100644 --- a/Misc/NEWS.d/next/Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst +++ b/Misc/NEWS.d/next/Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst @@ -1 +1 @@ -csv.Sniffer now detects line terminator instead of defaulting to '\r\n' +:class:`csv.Sniffer` now detects line terminator instead of defaulting to '\r\n'