From: Mike Bayer Date: Wed, 17 Feb 2016 18:31:29 +0000 (-0500) Subject: - All string formatting of bound parameter sets and result rows for X-Git-Tag: rel_1_1_0b1~98^2~22 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=591e0cf08a798fb16e0ee9b56df5c3141aa48959;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git - All string formatting of bound parameter sets and result rows for logging, exception, and ``repr()`` purposes now truncate very large scalar values within each collection, including an "N characters truncated" notation, similar to how the display for large multiple-parameter sets are themselves truncated. fixes #2837 --- diff --git a/doc/build/changelog/changelog_11.rst b/doc/build/changelog/changelog_11.rst index 95a1275793..37ed3470c9 100644 --- a/doc/build/changelog/changelog_11.rst +++ b/doc/build/changelog/changelog_11.rst @@ -21,6 +21,22 @@ .. changelog:: :version: 1.1.0b1 + .. change:: + :tags: feature, engine + :tickets: 2837 + + All string formatting of bound parameter sets and result rows for + logging, exception, and ``repr()`` purposes now truncate very large + scalar values within each collection, including an + "N characters truncated" + notation, similar to how the display for large multiple-parameter sets + are themselves truncated. + + + .. seealso:: + + :ref:`change_2837` + .. change:: :tags: feature, ext :tickets: 3297 diff --git a/doc/build/changelog/migration_11.rst b/doc/build/changelog/migration_11.rst index 9ad99ae9f4..8ed6ad6b56 100644 --- a/doc/build/changelog/migration_11.rst +++ b/doc/build/changelog/migration_11.rst @@ -883,6 +883,45 @@ this CHECK constraint can now be disabled using the new :ticket:`3095` +.. _change_2837: + +Large parameter and row values are now truncated in logging and exception displays +---------------------------------------------------------------------------------- + +A large value present as a bound parameter for a SQL statement, as well as a +large value present in a result row, will now be truncated during display +within logging, exception reporting, as well as ``repr()`` of the row itself:: + + >>> from sqlalchemy import create_engine + >>> import random + >>> e = create_engine("sqlite://", echo='debug') + >>> some_value = ''.join(chr(random.randint(52, 85)) for i in range(5000)) + >>> row = e.execute("select ?", [some_value]).first() + ... (lines are wrapped for clarity) ... + 2016-02-17 13:23:03,027 INFO sqlalchemy.engine.base.Engine select ? + 2016-02-17 13:23:03,027 INFO sqlalchemy.engine.base.Engine + ('E6@?>9HPOJB<:=TSTLA;9K;9FPM4M8M@;NM6GU + LUAEBT9QGHNHTHR5EP75@OER4?SKC;D:TFUMD:M>;C6U:JLM6R67GEK4=4:P + GJ7HQ6 ... (4702 characters truncated) ... J6IK546AJMB4N6S9L;;9AKI;=RJP + HDSSOTNBUEEC9@Q:RCL:I@5?FO<9K>KJAGAO@E6@A7JI8O:J7B69T6<8;F:S;4BEIJS9HM + K:;5OLPM@JR;R:J6Q>7T@I::OTDC:CC<=NGP6C>BC8N',) + 2016-02-17 13:23:03,027 DEBUG sqlalchemy.engine.base.Engine Col ('?',) + 2016-02-17 13:23:03,027 DEBUG sqlalchemy.engine.base.Engine + Row (u'E6@?>9HPOJB<:=TSTLA;9K;9FPM4M8M@; + NM6GULUAEBT9QGHNHTHR5EP75@OER4?SKC;D:TFUMD:M>;C6U:JLM6R67GEK4=4:PGJ7HQ ... (4703 characters truncated) ... J6IK546AJMB4N6S9L;;9AKI;= + RJPHDSSOTNBUEEC9@Q:RCL:I@5?FO<9K>KJAGAO@E6@A7JI8O:J7B69T6<8;F:S;4BEIJS9HM + K:;5OLPM@JR;R:J6Q>7T@I::OTDC:CC<=NGP6C>BC8N',) + >>> print row + (u'E6@?>9HPOJB<:=TSTLA;9K;9FPM4M8M@;NM6 + GULUAEBT9QGHNHTHR5EP75@OER4?SKC;D:TFUMD:M>;C6U:JLM6R67GEK4 + =4:PGJ7HQ ... (4703 characters truncated) ... J6IK546AJMB4N6S9L;;9AKI; + =RJPHDSSOTNBUEEC9@Q:RCL:I@5?FO<9K>KJAGAO@E6@A7JI8O:J7B69T6<8;F:S;4BEIJS9H + MK:;5OLPM@JR;R:J6Q>7T@I::OTDC:CC<=NGP6C>BC8N',) + + +:ticket:`2837` + .. _change_2528: A UNION or similar of SELECTs with LIMIT/OFFSET/ORDER BY now parenthesizes the embedded selects diff --git a/lib/sqlalchemy/engine/result.py b/lib/sqlalchemy/engine/result.py index 3305c4ce5f..c069fcedf2 100644 --- a/lib/sqlalchemy/engine/result.py +++ b/lib/sqlalchemy/engine/result.py @@ -10,7 +10,7 @@ and :class:`.RowProxy.""" from .. import exc, util -from ..sql import expression, sqltypes +from ..sql import expression, sqltypes, util as sql_util import collections import operator @@ -153,7 +153,7 @@ class RowProxy(BaseRowProxy): return self._op(other, operator.ne) def __repr__(self): - return repr(tuple(self)) + return repr(sql_util._repr_row(self)) def has_key(self, key): """Return True if this RowProxy contains the given key.""" @@ -1080,7 +1080,7 @@ class ResultProxy(object): log = self.context.engine.logger.debug l = [] for row in rows: - log("Row %r", row) + log("Row %r", sql_util._repr_row(row)) l.append(process_row(metadata, row, processors, keymap)) return l else: diff --git a/lib/sqlalchemy/sql/util.py b/lib/sqlalchemy/sql/util.py index 7e294d85f5..98d9cc9447 100644 --- a/lib/sqlalchemy/sql/util.py +++ b/lib/sqlalchemy/sql/util.py @@ -279,28 +279,125 @@ def _quote_ddl_expr(element): return repr(element) -class _repr_params(object): - """A string view of bound parameters, truncating - display to the given number of 'multi' parameter sets. +class _repr_base(object): + _LIST = 0 + _TUPLE = 1 + _DICT = 2 + + __slots__ = 'max_chars', + + def trunc(self, value): + rep = repr(value) + lenrep = len(rep) + if lenrep > self.max_chars: + segment_length = self.max_chars // 2 + rep = ( + rep[0:segment_length] + + (" ... (%d characters truncated) ... " + % (lenrep - self.max_chars)) + + rep[-segment_length:] + ) + return rep + + +class _repr_row(_repr_base): + """Provide a string view of a row.""" + + __slots__ = 'row', + + def __init__(self, row, max_chars=300): + self.row = row + self.max_chars = max_chars + + def __repr__(self): + trunc = self.trunc + return "(%s)" % ( + "".join(trunc(value) + "," for value in self.row) + ) + + +class _repr_params(_repr_base): + """Provide a string view of bound parameters. + + Truncates display to a given numnber of 'multi' parameter sets, + as well as long values to a given number of characters. """ - def __init__(self, params, batches): + __slots__ = 'params', 'batches', + + def __init__(self, params, batches, max_chars=300): self.params = params self.batches = batches + self.max_chars = max_chars def __repr__(self): - if isinstance(self.params, (list, tuple)) and \ - len(self.params) > self.batches and \ - isinstance(self.params[0], (list, dict, tuple)): + if isinstance(self.params, list): + typ = self._LIST + ismulti = self.params and isinstance( + self.params[0], (list, dict, tuple)) + elif isinstance(self.params, tuple): + typ = self._TUPLE + ismulti = self.params and isinstance( + self.params[0], (list, dict, tuple)) + elif isinstance(self.params, dict): + typ = self._DICT + ismulti = False + else: + assert False, "Unknown parameter type %s" % (type(self.params), ) + + if ismulti and len(self.params) > self.batches: msg = " ... displaying %i of %i total bound parameter sets ... " return ' '.join(( - repr(self.params[:self.batches - 2])[0:-1], + self._repr_multi(self.params[:self.batches - 2], typ)[0:-1], msg % (self.batches, len(self.params)), - repr(self.params[-2:])[1:] + self._repr_multi(self.params[-2:], typ)[1:] )) + elif ismulti: + return self._repr_multi(self.params, typ) + else: + return self._repr_params(self.params, typ) + + def _repr_multi(self, multi_params, typ): + if multi_params: + if isinstance(multi_params[0], list): + elem_type = self._LIST + elif isinstance(multi_params[0], tuple): + elem_type = self._TUPLE + elif isinstance(multi_params[0], dict): + elem_type = self._DICT + else: + assert False, \ + "Unknown parameter type %s" % (type(multi_params[0])) + + elements = ", ".join( + self._repr_params(params, elem_type) + for params in multi_params) + else: + elements = "" + + if typ == self._LIST: + return "[%s]" % elements + else: + return "(%s)" % elements + + def _repr_params(self, params, typ): + trunc = self.trunc + if typ is self._DICT: + return "{%s}" % ( + ", ".join( + "%r: %s" % (key, trunc(value)) + for key, value in params.items() + ) + ) + elif typ is self._TUPLE: + return "(%s)" % ( + "".join(trunc(value) + "," for value in params) + ) else: - return repr(self.params) + return "[%s]" % ( + ", ".join(trunc(value) for value in params) + ) def adapt_criterion_to_null(crit, nulls): diff --git a/test/engine/test_logging.py b/test/engine/test_logging.py index 180ea93888..847ba06c4e 100644 --- a/test/engine/test_logging.py +++ b/test/engine/test_logging.py @@ -6,7 +6,7 @@ import logging.handlers from sqlalchemy.testing import fixtures from sqlalchemy.testing import mock from sqlalchemy.testing.util import lazy_gc - +from sqlalchemy import util class LogParamsTest(fixtures.TestBase): __only_on__ = 'sqlite' @@ -53,6 +53,90 @@ class LogParamsTest(fixtures.TestBase): "bound parameter sets ... ('98',), ('99',)]" ) + def test_log_large_parameter_single(self): + import random + largeparam = ''.join(chr(random.randint(52, 85)) for i in range(5000)) + + self.eng.execute( + "INSERT INTO foo (data) values (?)", + (largeparam, ) + ) + + eq_( + self.buf.buffer[1].message, + "('%s ... (4702 characters truncated) ... %s',)" % ( + largeparam[0:149], largeparam[-149:] + ) + ) + + def test_log_large_parameter_multiple(self): + import random + lp1 = ''.join(chr(random.randint(52, 85)) for i in range(5000)) + lp2 = ''.join(chr(random.randint(52, 85)) for i in range(200)) + lp3 = ''.join(chr(random.randint(52, 85)) for i in range(670)) + + self.eng.execute( + "INSERT INTO foo (data) values (?)", + [(lp1, ), (lp2, ), (lp3, )] + ) + + eq_( + self.buf.buffer[1].message, + "[('%s ... (4702 characters truncated) ... %s',), ('%s',), " + "('%s ... (372 characters truncated) ... %s',)]" % ( + lp1[0:149], lp1[-149:], lp2, lp3[0:149], lp3[-149:] + ) + ) + + def test_result_large_param(self): + import random + largeparam = ''.join(chr(random.randint(52, 85)) for i in range(5000)) + + self.eng.echo = 'debug' + result = self.eng.execute( + "SELECT ?", + (largeparam, ) + ) + + row = result.first() + + eq_( + self.buf.buffer[1].message, + "('%s ... (4702 characters truncated) ... %s',)" % ( + largeparam[0:149], largeparam[-149:] + ) + ) + + if util.py3k: + eq_( + self.buf.buffer[3].message, + "Row ('%s ... (4702 characters truncated) ... %s',)" % ( + largeparam[0:149], largeparam[-149:] + ) + ) + else: + eq_( + self.buf.buffer[3].message, + "Row (u'%s ... (4703 characters truncated) ... %s',)" % ( + largeparam[0:148], largeparam[-149:] + ) + ) + + if util.py3k: + eq_( + repr(row), + "('%s ... (4702 characters truncated) ... %s',)" % ( + largeparam[0:149], largeparam[-149:] + ) + ) + else: + eq_( + repr(row), + "(u'%s ... (4703 characters truncated) ... %s',)" % ( + largeparam[0:148], largeparam[-149:] + ) + ) + def test_error_large_dict(self): assert_raises_message( tsa.exc.DBAPIError,