operation; most DBAPIs support this correctly now.
+.. _feature_3176:
+
+New KeyedTuple implementation dramatically faster
+-------------------------------------------------
+
+We took a look into the :class:`.KeyedTuple` implementation in the hopes
+of improving queries like this::
+
+ rows = sess.query(Foo.a, Foo.b, Foo.c).all()
+
+The :class:`.KeyedTuple` class is used rather than Python's
+``collections.namedtuple()``, because the latter has a very complex
+type-creation routine that benchmarks much slower than :class:`.KeyedTuple`.
+However, when fetching hundreds of thousands of rows,
+``collections.namedtuple()`` quickly overtakes :class:`.KeyedTuple` which
+becomes dramatically slower as instance invocation goes up. What to do?
+A new type that hedges between the approaches of both. Benching
+all three types for "size" (number of rows returned) and "num"
+(number of distinct queries), the new "lightweight keyed tuple" either
+outperforms both, or lags very slightly behind the faster object, based on
+which scenario. In the "sweet spot", where we are both creating a good number
+of new types as well as fetching a good number of rows, the lightweight
+object totally smokes both namedtuple and KeyedTuple::
+
+ -----------------
+ size=10 num=10000 # few rows, lots of queries
+ namedtuple: 3.60302400589 # namedtuple falls over
+ keyedtuple: 0.255059957504 # KeyedTuple very fast
+ lw keyed tuple: 0.582715034485 # lw keyed trails right on KeyedTuple
+ -----------------
+ size=100 num=1000 # <--- sweet spot
+ namedtuple: 0.365247011185
+ keyedtuple: 0.24896979332
+ lw keyed tuple: 0.0889317989349 # lw keyed blows both away!
+ -----------------
+ size=10000 num=100
+ namedtuple: 0.572599887848
+ keyedtuple: 2.54251694679
+ lw keyed tuple: 0.613876104355
+ -----------------
+ size=1000000 num=10 # few queries, lots of rows
+ namedtuple: 5.79669594765 # namedtuple very fast
+ keyedtuple: 28.856498003 # KeyedTuple falls over
+ lw keyed tuple: 6.74346804619 # lw keyed trails right on namedtuple
+
+
+:ticket:`3176`
+
+
.. _feature_2963:
.info dictionary improvements
EMPTY_SET = frozenset()
-class KeyedTuple(tuple):
+class AbstractKeyedTuple(tuple):
+ def keys(self):
+ """Return a list of string key names for this :class:`.KeyedTuple`.
+
+ .. seealso::
+
+ :attr:`.KeyedTuple._fields`
+
+ """
+
+ return list(self._fields)
+
+
+class KeyedTuple(AbstractKeyedTuple):
"""``tuple`` subclass that adds labeled names.
E.g.::
def __new__(cls, vals, labels=None):
t = tuple.__new__(cls, vals)
- t._labels = []
if labels:
t.__dict__.update(zip(labels, vals))
- t._labels = labels
+ else:
+ labels = []
+ t.__dict__['_labels'] = labels
return t
- def keys(self):
- """Return a list of string key names for this :class:`.KeyedTuple`.
-
- .. seealso::
-
- :attr:`.KeyedTuple._fields`
-
- """
-
- return [l for l in self._labels if l is not None]
-
@property
def _fields(self):
"""Return a tuple of string key names for this :class:`.KeyedTuple`.
:meth:`.KeyedTuple.keys`
"""
- return tuple(self.keys())
+ return tuple([l for l in self._labels if l is not None])
+
+ def __setattr__(self, key, value):
+ raise AttributeError("Can't set attribute: %s" % key)
def _asdict(self):
"""Return the contents of this :class:`.KeyedTuple` as a dictionary.
return dict((key, self.__dict__[key]) for key in self.keys())
+class _LW(AbstractKeyedTuple):
+ __slots__ = ()
+
+ def __new__(cls, vals):
+ return tuple.__new__(cls, vals)
+
+ def __reduce__(self):
+ # for pickling, degrade down to the regular
+ # KeyedTuple, thus avoiding anonymous class pickling
+ # difficulties
+ return KeyedTuple, (list(self), self._real_fields)
+
+ def _asdict(self):
+ """Return the contents of this :class:`.KeyedTuple` as a dictionary."""
+
+ d = dict(zip(self._real_fields, self))
+ d.pop(None, None)
+ return d
+
+
+def lightweight_named_tuple(name, fields):
+
+ tp_cls = type(name, (_LW,), {})
+ for idx, field in enumerate(fields):
+ if field is None:
+ continue
+ setattr(tp_cls, field, property(operator.itemgetter(idx)))
+
+ tp_cls._real_fields = fields
+ tp_cls._fields = tuple([f for f in fields if f is not None])
+
+ return tp_cls
+
+
class ImmutableContainer(object):
def _immutable(self, *arg, **kw):
raise TypeError("%s object is immutable" % self.__class__.__name__)
from sqlalchemy.util import classproperty, WeakSequence, get_callable_argspec
from sqlalchemy.sql import column
-class KeyedTupleTest():
+
+class _KeyedTupleTest(object):
+
+ def _fixture(self, values, labels):
+ raise NotImplementedError()
def test_empty(self):
- keyed_tuple = util.KeyedTuple([])
- eq_(type(keyed_tuple), util.KeyedTuple)
+ keyed_tuple = self._fixture([], [])
eq_(str(keyed_tuple), '()')
eq_(len(keyed_tuple), 0)
- eq_(keyed_tuple.__dict__, {'_labels': []})
eq_(list(keyed_tuple.keys()), [])
eq_(keyed_tuple._fields, ())
eq_(keyed_tuple._asdict(), {})
def test_values_but_no_labels(self):
- keyed_tuple = util.KeyedTuple([1, 2])
- eq_(type(keyed_tuple), util.KeyedTuple)
+ keyed_tuple = self._fixture([1, 2], [])
eq_(str(keyed_tuple), '(1, 2)')
eq_(len(keyed_tuple), 2)
- eq_(keyed_tuple.__dict__, {'_labels': []})
eq_(list(keyed_tuple.keys()), [])
eq_(keyed_tuple._fields, ())
eq_(keyed_tuple._asdict(), {})
eq_(keyed_tuple[1], 2)
def test_basic_creation(self):
- keyed_tuple = util.KeyedTuple([1, 2], ['a', 'b'])
+ keyed_tuple = self._fixture([1, 2], ['a', 'b'])
eq_(str(keyed_tuple), '(1, 2)')
eq_(list(keyed_tuple.keys()), ['a', 'b'])
eq_(keyed_tuple._fields, ('a', 'b'))
eq_(keyed_tuple._asdict(), {'a': 1, 'b': 2})
def test_basic_index_access(self):
- keyed_tuple = util.KeyedTuple([1, 2], ['a', 'b'])
+ keyed_tuple = self._fixture([1, 2], ['a', 'b'])
eq_(keyed_tuple[0], 1)
eq_(keyed_tuple[1], 2)
assert_raises(IndexError, should_raise)
def test_basic_attribute_access(self):
- keyed_tuple = util.KeyedTuple([1, 2], ['a', 'b'])
+ keyed_tuple = self._fixture([1, 2], ['a', 'b'])
eq_(keyed_tuple.a, 1)
eq_(keyed_tuple.b, 2)
assert_raises(AttributeError, should_raise)
def test_none_label(self):
- keyed_tuple = util.KeyedTuple([1, 2, 3], ['a', None, 'b'])
+ keyed_tuple = self._fixture([1, 2, 3], ['a', None, 'b'])
eq_(str(keyed_tuple), '(1, 2, 3)')
- # TODO: consider not allowing None labels
- expected = {'a': 1, None: 2, 'b': 3, '_labels': ['a', None, 'b']}
- eq_(keyed_tuple.__dict__, expected)
eq_(list(keyed_tuple.keys()), ['a', 'b'])
eq_(keyed_tuple._fields, ('a', 'b'))
eq_(keyed_tuple._asdict(), {'a': 1, 'b': 3})
eq_(keyed_tuple[2], 3)
def test_duplicate_labels(self):
- keyed_tuple = util.KeyedTuple([1, 2, 3], ['a', 'b', 'b'])
+ keyed_tuple = self._fixture([1, 2, 3], ['a', 'b', 'b'])
eq_(str(keyed_tuple), '(1, 2, 3)')
- # TODO: consider not allowing duplicate labels
- expected = {'a': 1, 'b': 3, '_labels': ['a', 'b', 'b']}
- eq_(keyed_tuple.__dict__, expected)
eq_(list(keyed_tuple.keys()), ['a', 'b', 'b'])
eq_(keyed_tuple._fields, ('a', 'b', 'b'))
eq_(keyed_tuple._asdict(), {'a': 1, 'b': 3})
eq_(keyed_tuple[2], 3)
def test_immutable(self):
- keyed_tuple = util.KeyedTuple([1, 2], ['a', 'b'])
+ keyed_tuple = self._fixture([1, 2], ['a', 'b'])
eq_(str(keyed_tuple), '(1, 2)')
- # attribute access: mutable
eq_(keyed_tuple.a, 1)
- keyed_tuple.a = 100
- eq_(keyed_tuple.a, 100)
- keyed_tuple.c = 300
- eq_(keyed_tuple.c, 300)
- # index access: immutable
+ assert_raises(AttributeError, setattr, keyed_tuple, "a", 5)
+
def should_raise():
keyed_tuple[0] = 100
assert_raises(TypeError, should_raise)
+ def test_serialize(self):
+
+ keyed_tuple = self._fixture([1, 2, 3], ['a', None, 'b'])
+
+ for loads, dumps in picklers():
+ kt = loads(dumps(keyed_tuple))
+
+ eq_(str(kt), '(1, 2, 3)')
+
+ eq_(list(kt.keys()), ['a', 'b'])
+ eq_(kt._fields, ('a', 'b'))
+ eq_(kt._asdict(), {'a': 1, 'b': 3})
+
+
+class KeyedTupleTest(_KeyedTupleTest, fixtures.TestBase):
+ def _fixture(self, values, labels):
+ return util.KeyedTuple(values, labels)
+
+
+class LWKeyedTupleTest(_KeyedTupleTest, fixtures.TestBase):
+ def _fixture(self, values, labels):
+ return util.lightweight_named_tuple('n', labels)(values)
+
+
class WeakSequenceTest(fixtures.TestBase):
@testing.requires.predictable_gc
def test_cleanout_elements(self):
projects / thirdparty / sqlalchemy / sqlalchemy.git / commitdiff
