"Sniffs" the format of a CSV file (i.e. delimiter, quotechar)
Returns a Dialect object.
'''
+ # Characters which can be guessed as a delimiter if the delimiters
+ # argument is not specified.
+ _delimiter_candidates = [c for c in map(chr, range(128))
+ if not c.isalnum()]
+
def __init__(self):
# in case there is more than one possible delimiter
self.preferred = [',', '\t', ';', ' ', ':']
def sniff(self, sample, delimiters=None):
"""
- Returns a dialect (or None) corresponding to the sample
+ Analyze the sample and return a Dialect subclass reflecting the
+ parameters found. If the optional delimiters parameter is
+ given, it is interpreted as a string containing possible valid
+ delimiter characters. Raises Error if the dialect cannot be
+ determined.
+
+ The dialect is guessed by parsing the sample with every
+ plausible combination of delimiter, quotechar and escapechar,
+ and choosing the combination which splits the sample into rows
+ with the most consistent number of fields.
+
+ A large sample is parsed incrementally: first only its
+ beginning, then, after eliminating the combinations which are
+ clearly worse than the leader, a several times larger part,
+ and so on.
"""
+ import re
+ from collections import defaultdict
- quotechar, doublequote, delimiter, skipinitialspace = \
- self._guess_quote_and_delimiter(sample, delimiters)
- if not delimiter:
- delimiter, skipinitialspace = self._guess_delimiter(sample,
- delimiters)
+ if self._parses_as_single_column(sample):
+ # There is no delimiter to find; any combination could
+ # only find a bogus one inside the quoted fields.
+ raise Error("Could not determine delimiter")
- if not delimiter:
+ chars = set(sample)
+ if delimiters is None:
+ delimiters = self._delimiter_candidates
+ delimiters = [d for d in delimiters
+ if d in chars and d not in '\r\n"\'\\']
+ # Combinations to try, numbered by preference for breaking
+ # ties. The unquoted combinations are parsed from the start;
+ # the rest stay dormant until the quote character occurs at
+ # the start of a field (see _split_dormant).
+ groups = defaultdict(list)
+ order = 0
+ # Only '\\' is tried as an escape character: others are not
+ # seen in the wild.
+ for escapechar in ('', '\\') if '\\' in chars else ('',):
+ for quotechar in '"', "'", '':
+ if quotechar and quotechar not in chars:
+ continue
+ for delimiter in delimiters:
+ groups[quotechar].append(
+ (order, delimiter, quotechar, escapechar))
+ order += 1
+ active = groups.pop('', [])
+ # Only non-empty groups were created; a plain dict cannot
+ # grow one by accident.
+ dormant = dict(groups)
+
+ # The initial window should cover the minimal number of rows
+ # required for elimination (see _eliminate_worse) at a typical
+ # line length, so that the first round can already eliminate.
+ window = 2000
+ # A line with its line break: '\r', '\n' or '\r\n' (the
+ # reader treats other line boundary characters as ordinary
+ # data, but does not support a bare '\r' inside a chunk).
+ # The \z alternative produces one final empty match.
+ line_re = re.compile(r'[^\r\n]*(?:\r\n|[\r\n]|\z)')
+ parsed = []
+ lines = []
+ first_round = True
+ while active or dormant:
+ end = min(window, len(sample))
+ part = sample[:end]
+ lines = line_re.findall(part)
+ del lines[-1]
+ cut = not part.endswith(('\r', '\n'))
+ for quotechar in list(dormant):
+ activated, still = self._split_dormant(
+ part, quotechar, dormant[quotechar])
+ active += activated
+ if still:
+ dormant[quotechar] = still
+ else:
+ del dormant[quotechar]
+ parsed = [(combo, self._try_dialect(lines, cut, *combo[1:]))
+ for combo in active]
+ if end == len(sample):
+ break
+ active = self._eliminate_worse(parsed, not first_round)
+ first_round = False
+ if len(active) <= 3:
+ # Quoted data most often leaves three survivors: the
+ # true dialect, its equally consistent unquoted shadow,
+ # and one accident. Parsing the whole sample with them
+ # is cheaper than another elimination round.
+ window = len(sample)
+ else:
+ # Too small a factor would increase the total
+ # re-parsing cost, too large -- the cost of the next
+ # round if this one did not eliminate enough.
+ window *= 4
+
+ best = None
+ best_score = None
+ for combo, rows in sorted(parsed):
+ if rows is None:
+ continue
+ _, delimiter, quotechar, escapechar = combo
+ nfields, share = self._modal_share(rows)
+ if nfields < 2:
+ # The delimiter does not delimit anything.
+ continue
+ try:
+ preference = -self.preferred.index(delimiter)
+ except ValueError:
+ preference = -len(self.preferred)
+ # A successful quoted parse is direct evidence; the preferred
+ # delimiters list is only a nudge.
+ score = (share, len(rows), bool(quotechar), preference)
+ if best_score is None or score > best_score:
+ best_score = score
+ best = combo[1:]
+
+ if best is None:
raise Error("Could not determine delimiter")
+ delimiter, quotechar, escapechar = best
+ doublequote = self._detect_doublequote(lines, *best)
+ skipinitialspace = self._detect_skipinitialspace(lines, *best,
+ doublequote)
class dialect(Dialect):
_name = "sniffed"
lineterminator = '\r\n'
quoting = QUOTE_MINIMAL
- # escapechar = ''
- dialect.doublequote = doublequote
dialect.delimiter = delimiter
# _csv.reader won't accept a quotechar of ''
dialect.quotechar = quotechar or '"'
+ dialect.escapechar = escapechar or None
+ dialect.doublequote = doublequote
dialect.skipinitialspace = skipinitialspace
return dialect
-
- def _guess_quote_and_delimiter(self, data, delimiters):
+ def _parses_as_single_column(self, sample):
"""
- Looks for text enclosed between two identical quotes
- (the probable quotechar) which are preceded and followed
- by the same character (the probable delimiter).
- For example:
- ,'some text',
- The quote with the most wins, same with the delimiter.
- If there is no quotechar the delimiter can't be determined
- this way.
+ True if the whole sample parses as a single column of quoted
+ fields (the last one may be cut off in the middle), so there
+ is no delimiter to find.
"""
import re
- matches = []
- for restr in (r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?P=delim)', # ,".*?",
- r'(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)', # ".*?",
- r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\r|\n)', # ,".*?"
- r'(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?:$|\r|\n)'): # ".*?" (no delim, no space)
- regexp = re.compile(restr, re.DOTALL | re.MULTILINE)
- matches = regexp.findall(data)
- if matches:
- break
-
- if not matches:
- # (quotechar, doublequote, delimiter, skipinitialspace)
- return ('', False, None, 0)
- quotes = {}
- delims = {}
- spaces = 0
- groupindex = regexp.groupindex
- for m in matches:
- n = groupindex['quote'] - 1
- key = m[n]
- if key:
- quotes[key] = quotes.get(key, 0) + 1
- try:
- n = groupindex['delim'] - 1
- key = m[n]
- except KeyError:
- continue
- if key and (delimiters is None or key in delimiters):
- delims[key] = delims.get(key, 0) + 1
- try:
- n = groupindex['space'] - 1
- except KeyError:
- continue
- if m[n]:
- spaces += 1
-
- quotechar = max(quotes, key=quotes.get)
-
- if delims:
- delim = max(delims, key=delims.get)
- skipinitialspace = delims[delim] == spaces
- if delim == '\n': # most likely a file with a single column
- delim = ''
- else:
- # there is *no* delimiter, it's a single column of quoted data
- delim = ''
- skipinitialspace = 0
-
- # if we see an extra quote between delimiters, we've got a
- # double quoted format
- dq_regexp = re.compile(
- r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \
- {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE)
-
+ for q in '"', "'":
+ if q in sample:
+ row_re = (fr' *+{q}(?:[^{q}]|{q}{q})*+'
+ fr'(?:{q} *+(?:[\r\n]++|\z)|\z)')
+ if re.fullmatch(fr'(?:{row_re})++', sample):
+ return True
+ return False
+ def _split_dormant(self, part, quotechar, combos):
+ """
+ Split the dormant combinations into those ready for trial
+ parsing and the rest.
+
+ A combination is ready when its quote character occurs in
+ *part* at the start of a field, i.e. at the start of a line or
+ after its delimiter; until then parsing would not differ from
+ the unquoted variant. Spaces before the quote are allowed even
+ for the space delimiter: a false activation only costs a trial
+ parse.
+ """
+ import re
- if dq_regexp.search(data):
- doublequote = True
+ remaining = {combo[1] for combo in combos}
+ found = set()
+ pos = 0
+ while remaining:
+ # Include only the delimiters not found yet, so that the
+ # search skips over the found ones; the compiled patterns
+ # come from the re cache.
+ cls = re.escape(''.join(sorted(remaining)))
+ m = re.compile(fr'(?:^|([\r\n{cls}]))'
+ fr' *{quotechar}').search(part, pos)
+ if m is None:
+ break
+ pre = m[1]
+ if pre is None or pre in '\r\n':
+ # A quote at the start of a line starts a field for
+ # every delimiter.
+ found |= remaining
+ break
+ found.add(pre)
+ remaining.discard(pre)
+ pos = m.end()
+ activated = [combo for combo in combos if combo[1] in found]
+ still_dormant = [combo for combo in combos if combo[1] not in found]
+ return activated, still_dormant
+
+ def _make_reader(self, lines, delimiter, quotechar, escapechar,
+ doublequote=True, skipinitialspace=None):
+ """
+ Create a reader for trial parsing. quotechar '' means no
+ quoting and escapechar '' means no escape character.
+ """
+ if skipinitialspace is None:
+ # Be lenient to spaces after a delimiter, unless the
+ # delimiter is a space itself.
+ skipinitialspace = delimiter != ' '
+ return reader(lines, delimiter=delimiter,
+ quotechar=quotechar or '"',
+ quoting=QUOTE_MINIMAL if quotechar else QUOTE_NONE,
+ escapechar=escapechar or None,
+ doublequote=doublequote,
+ skipinitialspace=skipinitialspace,
+ strict=True)
+
+ def _try_dialect(self, lines, cut, delimiter, quotechar, escapechar):
+ """
+ Parse the sample, pre-split into *lines*, and return the list
+ of the number of fields in every parsed row, or None if not a
+ single row was parsed.
+
+ If the sample cannot be parsed to the end (for example it is
+ cut off in the middle of a quoted field, or the combination
+ does not fit the sample), the rows parsed so far are counted.
+ The last row is not counted if *cut* is true: the sample can
+ be cut off in the middle of it.
+ """
+ rows = []
+ try:
+ rows.extend(map(len, self._make_reader(lines, delimiter,
+ quotechar, escapechar)))
+ except Error:
+ # The row which failed to parse is not counted.
+ pass
else:
- doublequote = False
+ if cut and len(rows) > 1:
+ rows.pop()
+ if 0 in rows:
+ # Blank lines produce empty rows.
+ rows = [nfields for nfields in rows if nfields]
+ return rows or None
+
+ def _eliminate_worse(self, parsed, judge_hopeless):
+ """
+ Return the combinations from *parsed* (a list of (combination,
+ rows) pairs) without those which are clearly worse than the
+ leader. Combinations with too few parsed rows (e.g. if the
+ parsed part ends in the middle of a large quoted field) are
+ not judged yet.
+
+ If *judge_hopeless* is false, keep the combinations whose
+ delimiter does not delimit anything. Unlike the comparison
+ with the leader, which self-normalizes when the parsed part is
+ not representative, this verdict is absolute and irreversible,
+ so it is not trusted to the first part, which covers the least
+ representative beginning of the sample (titles, headers,
+ preamble).
+ """
+ # Judging a combination by fewer rows is too noisy.
+ min_rows = 16
+ hopeless = set()
+ scores = {}
+ for combo, rows in parsed:
+ if rows is not None and len(rows) >= min_rows:
+ nfields, share = self._modal_share(rows)
+ if nfields < 2:
+ if judge_hopeless:
+ hopeless.add(combo)
+ else:
+ scores[combo] = share
+ threshold = max(scores.values(), default=0.0) - 0.1
+ return [combo for combo, _ in parsed
+ if combo not in hopeless
+ and scores.get(combo, threshold) >= threshold]
- return (quotechar, doublequote, delim, skipinitialspace)
+ def _modal_share(self, rows):
+ """
+ The most common number of fields in a row and its share of all
+ rows. Prefer the smaller number of fields in the case of a
+ tie: a candidate delimiter which delimits only half of the rows
+ is not convincing.
+ """
+ from collections import Counter
+ counts = Counter(rows)
+ nfields = max(counts, key=lambda n: (counts[n], -n))
+ return nfields, counts[nfields] / len(rows)
- def _guess_delimiter(self, data, delimiters):
+ def _detect_doublequote(self, lines, delimiter, quotechar, escapechar):
"""
- The delimiter /should/ occur the same number of times on
- each row. However, due to malformed data, it may not. We don't want
- an all or nothing approach, so we allow for small variations in this
- number.
- 1) build a table of the frequency of each character on every line.
- 2) build a table of frequencies of this frequency (meta-frequency?),
- e.g. 'x occurred 5 times in 10 rows, 6 times in 1000 rows,
- 7 times in 2 rows'
- 3) use the mode of the meta-frequency to determine the /expected/
- frequency for that character
- 4) find out how often the character actually meets that goal
- 5) the character that best meets its goal is the delimiter
- For performance reasons, the data is evaluated in chunks, so it can
- try and evaluate the smallest portion of the data possible, evaluating
- additional chunks as necessary.
+ True if a doubled quote character represents a single quote
+ character in the sample: interpreting it so changes the result
+ of parsing.
"""
- from collections import Counter, defaultdict
-
- data = list(filter(None, data.split('\n')))
-
- # build frequency tables
- chunkLength = min(10, len(data))
- iteration = 0
- num_lines = 0
- # {char -> {count_per_line -> num_lines_with_that_count}}
- char_frequency = defaultdict(Counter)
- modes = {}
- delims = {}
- start, end = 0, chunkLength
- while start < len(data):
- iteration += 1
- for line in data[start:end]:
- num_lines += 1
- for char, count in Counter(line).items():
- if char.isascii():
- char_frequency[char][count] += 1
-
- for char, counts in char_frequency.items():
- items = list(counts.items())
- missed_lines = num_lines - sum(counts.values())
- if missed_lines:
- # Store the number of lines 'char' was missing from.
- items.append((0, missed_lines))
- if len(items) == 1 and items[0][0] == 0:
- continue
- # get the mode of the frequencies
- if len(items) > 1:
- modes[char] = max(items, key=lambda x: x[1])
- # adjust the mode - subtract the sum of all
- # other frequencies
- items.remove(modes[char])
- modes[char] = (modes[char][0], modes[char][1]
- - sum(item[1] for item in items))
- else:
- modes[char] = items[0]
-
- # build a list of possible delimiters
- modeList = modes.items()
- total = float(min(chunkLength * iteration, len(data)))
- # (rows of consistent data) / (number of rows) = 100%
- consistency = 1.0
- # minimum consistency threshold
- threshold = 0.9
- while len(delims) == 0 and consistency >= threshold:
- for k, v in modeList:
- if v[0] > 0 and v[1] > 0:
- if ((v[1]/total) >= consistency and
- (delimiters is None or k in delimiters)):
- delims[k] = v
- consistency -= 0.01
-
- if len(delims) == 1:
- delim = list(delims.keys())[0]
- skipinitialspace = (data[0].count(delim) ==
- data[0].count("%c " % delim))
- return (delim, skipinitialspace)
-
- # analyze another chunkLength lines
- start = end
- end += chunkLength
-
- if not delims:
- return ('', 0)
-
- # if there's more than one, fall back to a 'preferred' list
- if len(delims) > 1:
- for d in self.preferred:
- if d in delims.keys():
- skipinitialspace = (data[0].count(d) ==
- data[0].count("%c " % d))
- return (d, skipinitialspace)
-
- # nothing else indicates a preference, pick the character that
- # dominates(?)
- items = [(v,k) for (k,v) in delims.items()]
- items.sort()
- delim = items[-1][1]
-
- skipinitialspace = (data[0].count(delim) ==
- data[0].count("%c " % delim))
- return (delim, skipinitialspace)
-
+ if not quotechar or not any(quotechar * 2 in line
+ for line in lines):
+ return False
+ readers = [self._make_reader(
+ lines, delimiter, quotechar, escapechar,
+ doublequote=doublequote)
+ for doublequote in (False, True)]
+ while True:
+ rows = []
+ for rdr in readers:
+ try:
+ rows.append(next(rdr))
+ except (StopIteration, Error):
+ # Ending cleanly and failing are equivalent here:
+ # after equal rows both readers are at the same
+ # position, so they cannot end for different
+ # reasons.
+ rows.append(None)
+ if rows[0] != rows[1]:
+ return True
+ if rows == [None, None]:
+ return False
+
+ def _detect_skipinitialspace(self, lines, delimiter, quotechar,
+ escapechar, doublequote):
+ """
+ True only if every field following a delimiter starts with
+ a space.
+ """
+ skipinitialspace = False
+ try:
+ for row in self._make_reader(lines, delimiter, quotechar,
+ escapechar,
+ doublequote=doublequote,
+ skipinitialspace=False):
+ for field in row[1:]:
+ if not field.startswith(' '):
+ return False
+ skipinitialspace = True
+ except Error:
+ pass
+ return skipinitialspace
def has_header(self, sample):
# Creates a dictionary of types of data in each column. If any
def test_guess_quote_and_delimiter(self):
sniffer = csv.Sniffer()
- for header in (";'123;4';", "'123;4';", ";'123;4'", "'123;4'"):
+ for header in (";'123;4';", "'123;4';", ";'123;4'"):
with self.subTest(header):
dialect = sniffer.sniff(header, ",;")
self.assertEqual(dialect.delimiter, ';')
self.assertEqual(dialect.quotechar, "'")
self.assertIs(dialect.doublequote, False)
self.assertIs(dialect.skipinitialspace, False)
+ # A single quoted field is a single column without a delimiter.
+ self.assertRaisesRegex(csv.Error, "Could not determine delimiter",
+ sniffer.sniff, "'123;4'", ",;")
def test_sniff(self):
sniffer = csv.Sniffer()
self.assertEqual(dialect.delimiter, ',')
self.assertEqual(dialect.quotechar, '"')
+ def test_sniff_escapechar(self):
+ # gh-83273: escaped delimiters make the delimiter frequencies
+ # inconsistent, but the escape character can be detected by trial
+ # parsing.
+ sniffer = csv.Sniffer()
+ sample = 'ab,cd\\,ef\ngh\\,ij,kl\nmn,op\n'
+ dialect = sniffer.sniff(sample)
+ self.assertEqual(dialect.delimiter, ',')
+ self.assertEqual(dialect.escapechar, '\\')
+ self.assertIs(dialect.doublequote, False)
+ self.assertEqual(list(csv.reader(StringIO(sample), dialect)),
+ [['ab', 'cd,ef'], ['gh,ij', 'kl'], ['mn', 'op']])
+ # No escape character in the sample -- none is detected.
+ dialect = sniffer.sniff('ab,cd\ngh,ij\n')
+ self.assertEqual(dialect.delimiter, ',')
+ self.assertIsNone(dialect.escapechar)
+
+ def test_sniff_quoted_rows_among_unquoted(self):
+ # Rows which happen to consist of a single quoted or unquoted
+ # field must not be mistaken for a single column of quoted fields.
+ sniffer = csv.Sniffer()
+ sample = '"header line"\na|b\nc|d\ne|f\n'
+ self.assertEqual(sniffer.sniff(sample).delimiter, '|')
+ # Even if an unterminated quote breaks the quoted parse.
+ sample = '"header"\na|"b c\nd|e\nf|g\n'
+ self.assertEqual(sniffer.sniff(sample).delimiter, '|')
+
+ def test_sniff_single_column_quoted(self):
+ # gh-98820: a sample consisting of a single column of quoted fields
+ # has no delimiter to detect, even if the quoted content contains
+ # characters which could pass for one. It also used to take
+ # quadratic time before failing.
+ sniffer = csv.Sniffer()
+ sample = '\n'.join('"%02d-%02d-%02d"' % (i, i, i) for i in range(50))
+ self.assertRaisesRegex(csv.Error, "Could not determine delimiter",
+ sniffer.sniff, sample)
+ # Even if none of the requested delimiters occurs in the sample.
+ self.assertRaisesRegex(csv.Error, "Could not determine delimiter",
+ sniffer.sniff, sample, ',:|\t')
+ # A mix of quoted and unquoted single-field rows.
+ sample = '"abc"\ndef\n"ghi"\njkl\n' * 10
+ self.assertRaisesRegex(csv.Error, "Could not determine delimiter",
+ sniffer.sniff, sample)
+ # Even if the quoted fields contain delimiter characters.
+ sample = '"a,b"\ncd\n' * 20
+ self.assertRaisesRegex(csv.Error, "Could not determine delimiter",
+ sniffer.sniff, sample)
+ # Even if the content of the quoted fields parses consistently
+ # with the other quote character.
+ sample = '''"a-'b'-c"\n"d-'e'-f"\n''' * 10
+ self.assertRaisesRegex(csv.Error, "Could not determine delimiter",
+ sniffer.sniff, sample)
+
+ def test_sniff_non_ascii_delimiter(self):
+ # gh-111820: an explicitly requested delimiter can be non-ASCII.
+ sniffer = csv.Sniffer()
+ sample = 'aaa\xacbbb\xacccc\nddd\xaceee\xacfff\nggg\xachhh\xaciii\n'
+ dialect = sniffer.sniff(sample, delimiters='\xac')
+ self.assertEqual(dialect.delimiter, '\xac')
+
+ def test_sniff_preamble(self):
+ # A preamble (title or comment lines) before the data must not
+ # prevent detection, even if it is larger than the part of the
+ # sample which is parsed first. It is enough for the data rows
+ # to slightly outnumber the preamble lines.
+ sniffer = csv.Sniffer()
+ for n in 3, 24, 80, 320:
+ preamble = ''.join(f'Comment line {i:05}\n' for i in range(n))
+ for ndata in n + 2, 2 * n + 5:
+ with self.subTest(preamble_lines=n, data_lines=ndata):
+ sample = preamble + 'aaa,bbb,ccc\n' * ndata
+ self.assertEqual(sniffer.sniff(sample).delimiter, ',')
+
+ def test_sniff_line_boundary_characters(self):
+ # Only '\r', '\n' and '\r\n' are row separators; characters like
+ # '\x1c' or '\x85', which str.splitlines() treats as line
+ # boundaries, are ordinary data for the reader -- '\x1c' can
+ # even be the delimiter.
+ sniffer = csv.Sniffer()
+ sample = 'aa\x1cbb\ncc\x1cdd\nee\x1cff\n'
+ self.assertEqual(sniffer.sniff(sample).delimiter, '\x1c')
+ sample = 'a\x85b,c\nd,e\nf,g\n'
+ dialect = sniffer.sniff(sample)
+ self.assertEqual(dialect.delimiter, ',')
+ self.assertEqual(next(csv.reader(StringIO(sample), dialect)),
+ ['a\x85b', 'c'])
+
+ def test_sniff_delimiter_in_quoted_field(self):
+ # gh-97611: a delimiter inside a quoted field should not win over
+ # the delimiter which actually separates the fields.
+ sniffer = csv.Sniffer()
+ sample = (
+ 'Surname;First Name;Year of birth\n'
+ '"\tDoe;Jane"\t;1971\n'
+ )
+ dialect = sniffer.sniff(sample, delimiters=',;\t')
+ self.assertEqual(dialect.delimiter, ';')
+ sample = (
+ 'Surname;First Name;Year of birth\n'
+ '"\t";"\t";1971\n'
+ '"Le Trec";"Mary Ann";1486\n'
+ )
+ dialect = sniffer.sniff(sample, delimiters=',;\t')
+ self.assertEqual(dialect.delimiter, ';')
+ self.assertEqual(dialect.quotechar, '"')
+
+ def test_sniff_embedded_lists(self):
+ # gh-119123: commas in bracketed lists adjacent to quotes should
+ # not be mistaken for the delimiter.
+ sniffer = csv.Sniffer()
+ sample = (
+ "id;is_sort;cost;group;merge\n"
+ "1;True;62.25;['345'];UNKNOWN\n"
+ "2;True;54.00;['235'];UNKNOWN\n"
+ "3;True;237.00;['567', '568'];UNKNOWN\n"
+ "4;True;46.50;['112', '112'];UNKNOWN\n"
+ )
+ dialect = sniffer.sniff(sample, delimiters=',;')
+ self.assertEqual(dialect.delimiter, ';')
+
+ def test_sniff_space_adjacent_to_quotes(self):
+ # gh-88843: a space adjacent to stray quotes should not be
+ # detected as the delimiter.
+ sniffer = csv.Sniffer()
+ sample = "a|b\nc| 'd\ne|' f"
+ dialect = sniffer.sniff(sample)
+ self.assertEqual(dialect.delimiter, '|')
+
+ def test_sniff_crlf_lineterminator(self):
+ # gh-103925: a quote at the end of a \r\n-terminated line.
+ sniffer = csv.Sniffer()
+ sample = (
+ 'Timestamp,URL,Title\r\n'
+ '2020-10-01 17:17:37+08:00,'
+ 'https://www.mozilla.org/en-US/firefox/welcome/2/,'
+ '"Pocket - Save news, videos, stories and more"\r\n'
+ )
+ dialect = sniffer.sniff(sample)
+ self.assertEqual(dialect.delimiter, ',')
+ self.assertEqual(dialect.quotechar, '"')
+
+ def test_sniff_excel_tab_with_quotes(self):
+ # gh-62029: tab-delimited data with a quoted field containing
+ # spaces and doubled quotes.
+ sniffer = csv.Sniffer()
+ sample = ('foo\tbar\t"baz ""quoted"" here"\tspam eggs\n'
+ 'ham\teggs\tx y\tz w\n')
+ dialect = sniffer.sniff(sample)
+ self.assertEqual(dialect.delimiter, '\t')
+ self.assertEqual(dialect.quotechar, '"')
+ self.assertIs(dialect.doublequote, True)
+
+ def test_sniff_truncated_sample(self):
+ # A sample cut off in the middle of a quoted field should not
+ # spoil the detection.
+ sniffer = csv.Sniffer()
+ sample = '"a,a";"b"\n"c";"d"\n"e";"f,f\n'
+ dialect = sniffer.sniff(sample)
+ self.assertEqual(dialect.delimiter, ';')
+ self.assertEqual(dialect.quotechar, '"')
+ # Cut off in the middle of the last row.
+ sample = 'a,b,c\nd,e,f\ng,h,i\nj,k'
+ self.assertEqual(sniffer.sniff(sample).delimiter, ',')
+ # Cut off in the middle of an escaped sequence.
+ sample = 'ab,cd\\,ef\ngh\\,ij,kl\nmn,op\nqr,st\\'
+ dialect = sniffer.sniff(sample)
+ self.assertEqual(dialect.delimiter, ',')
+ self.assertEqual(dialect.escapechar, '\\')
+ # Cut off at a line end in the middle of a quoted field.
+ sample = '"a","b"\n"c","d"\n"e","multi\nline'
+ self.assertEqual(sniffer.sniff(sample).delimiter, ',')
+ # An unclosed quote consumes everything to the end of the sample,
+ # but the rows before it still count.
+ sample = 'a|b\nc|d\ne|f\ng|"h\ni|j\nk|l'
+ self.assertEqual(sniffer.sniff(sample).delimiter, '|')
+
+ def test_sniff_skipinitialspace_quoted(self):
+ sniffer = csv.Sniffer()
+ sample = "'a': 'b': 'c'\n'd': 'e': 'f'\n"
+ dialect = sniffer.sniff(sample)
+ self.assertEqual(dialect.delimiter, ':')
+ self.assertEqual(dialect.quotechar, "'")
+ self.assertIs(dialect.skipinitialspace, True)
+
+ def test_sniff_regex_backtracking(self):
+ # gh-109638: this artificial sample used to take minutes.
+ sniffer = csv.Sniffer()
+ sample = '"",' * 100 + '"' * 100 + '0' + '"' * 100 + '0'
+ self.assertEqual(sniffer.sniff(sample).delimiter, ',')
+
def test_doublequote(self):
sniffer = csv.Sniffer()
dialect = sniffer.sniff(self.header1)
self.assertFalse(dialect.doublequote)
dialect = sniffer.sniff(self.sample9)
self.assertTrue(dialect.doublequote)
+ # A doubled quote character in an unquoted field or an empty
+ # quoted field is not evidence of doubling.
+ dialect = sniffer.sniff('"x",a""b\n"y",c\n')
+ self.assertIs(dialect.doublequote, False)
+ dialect = sniffer.sniff('a,"",c\nd,"",f\n')
+ self.assertIs(dialect.doublequote, False)
+ # A doubled quote character inside a quoted field is.
+ dialect = sniffer.sniff('"a""b",c\n"d",e\n')
+ self.assertIs(dialect.doublequote, True)
def test_guess_delimiter_crlf_not_chosen(self):
# Ensure that we pick the real delimiter ("|") over "\r" in a tie.
projects / thirdparty / Python / cpython.git / commitdiff
