From: Mike Bayer Date: Sat, 4 May 2013 18:59:26 +0000 (-0400) Subject: - unicode literals need to just be handled differently if they have utf-8 X-Git-Tag: rel_0_9_0b1~304^2~13^2~51 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=2a99b770ddf144c279ad8b42ad8593b3439cd2de;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git - unicode literals need to just be handled differently if they have utf-8 encoded in them vs. unicode escaping. not worth figuring out how to combine these right now --- diff --git a/lib/sqlalchemy/dialects/sqlite/base.py b/lib/sqlalchemy/dialects/sqlite/base.py index d742a71b5e..1ca8f4e64f 100644 --- a/lib/sqlalchemy/dialects/sqlite/base.py +++ b/lib/sqlalchemy/dialects/sqlite/base.py @@ -812,7 +812,7 @@ class SQLiteDialect(default.DefaultDialect): coltype = sqltypes.NullType() if default is not None: - default = str(default) + default = util.text_type(default) return { 'name': name, diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py index d51dd625a4..c3aea159a2 100644 --- a/lib/sqlalchemy/sql/compiler.py +++ b/lib/sqlalchemy/sql/compiler.py @@ -1868,11 +1868,11 @@ class DDLCompiler(engine.Compiled): first_pk = True except exc.CompileError as ce: util.raise_from_cause( - exc.CompileError(util.u("(in table '%s', column '%s'): %s" % ( + exc.CompileError(util.u("(in table '%s', column '%s'): %s") % ( table.description, column.name, ce.args[0] - )))) + ))) const = self.create_table_constraints(table) if const: @@ -2344,7 +2344,7 @@ class IdentifierPreparer(object): lc_value = value.lower() return (lc_value in self.reserved_words or value[0] in self.illegal_initial_characters - or not self.legal_characters.match(str(value)) + or not self.legal_characters.match(util.text_type(value)) or (lc_value != value)) def quote_schema(self, schema, force): diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py index 72baa9f368..b1f1a2db69 100644 --- a/lib/sqlalchemy/util/__init__.py +++ b/lib/sqlalchemy/util/__init__.py @@ -8,7 +8,7 @@ from .compat import callable, cmp, reduce, \ threading, py3k, py2k, jython, pypy, cpython, win32, \ pickle, dottedgetter, parse_qsl, namedtuple, next, WeakSet, reraise, \ raise_from_cause, text_type, string_types, int_types, binary_type, \ - quote_plus, with_metaclass, print_, itertools_filterfalse, u, b,\ + quote_plus, with_metaclass, print_, itertools_filterfalse, u, ue, b,\ unquote_plus from ._collections import KeyedTuple, ImmutableContainer, immutabledict, \ diff --git a/lib/sqlalchemy/util/compat.py b/lib/sqlalchemy/util/compat.py index bc7a0fe213..94b35f0195 100644 --- a/lib/sqlalchemy/util/compat.py +++ b/lib/sqlalchemy/util/compat.py @@ -46,6 +46,9 @@ if py3k: def u(s): return s + def ue(s): + return s + def b(s): return s.encode("latin-1") @@ -79,6 +82,13 @@ else: return (ord(byte) for byte in buf) def u(s): + # this differs from what six does, which doesn't support non-ASCII + # strings - we only use u() with + # literal source strings, and all our source files with non-ascii + # in them (all are tests) are utf-8 encoded. + return unicode(s, "utf-8") + + def ue(s): return unicode(s, "unicode_escape") def b(s): diff --git a/test/sql/test_compiler.py b/test/sql/test_compiler.py index e3f7df7026..9cb0e6bbe8 100644 --- a/test/sql/test_compiler.py +++ b/test/sql/test_compiler.py @@ -20,6 +20,7 @@ from sqlalchemy import Integer, String, MetaData, Table, Column, select, \ intersect, union_all, Boolean, distinct, join, outerjoin, asc, desc,\ over, subquery, case import decimal +from sqlalchemy.util import u from sqlalchemy import exc, sql, util, types, schema from sqlalchemy.sql import table, column, label from sqlalchemy.sql.expression import ClauseList, _literal_as_text, HasPrefixes @@ -2680,11 +2681,11 @@ class DDLTest(fixtures.TestBase, AssertsCompiledSQL): def test_reraise_of_column_spec_issue_unicode(self): MyType = self._illegal_type_fixture() t1 = Table('t', MetaData(), - Column('méil', MyType()) + Column(u('méil'), MyType()) ) assert_raises_message( exc.CompileError, - r"\(in table 't', column 'méil'\): Couldn't compile type", + u(r"\(in table 't', column 'méil'\): Couldn't compile type"), schema.CreateTable(t1).compile ) diff --git a/test/sql/test_unicode.py b/test/sql/test_unicode.py index 2ab081dade..ffcef903f3 100644 --- a/test/sql/test_unicode.py +++ b/test/sql/test_unicode.py @@ -7,6 +7,7 @@ from sqlalchemy import testing from sqlalchemy.testing.engines import utf8_engine from sqlalchemy.sql import column from sqlalchemy.testing.schema import Table, Column +from sqlalchemy.util import u, ue class UnicodeSchemaTest(fixtures.TestBase): __requires__ = ('unicode_ddl',) @@ -18,14 +19,14 @@ class UnicodeSchemaTest(fixtures.TestBase): unicode_bind = utf8_engine() metadata = MetaData(unicode_bind) - t1 = Table('unitable1', metadata, - Column('méil', Integer, primary_key=True), - Column('\u6e2c\u8a66', Integer), + t1 = Table(u('unitable1'), metadata, + Column(u('méil'), Integer, primary_key=True), + Column(ue('\u6e2c\u8a66'), Integer), test_needs_fk=True, ) - t2 = Table('Unitéble2', metadata, - Column('méil', Integer, primary_key=True, key="a"), - Column('\u6e2c\u8a66', Integer, ForeignKey('unitable1.méil'), + t2 = Table(u('Unitéble2'), metadata, + Column(u('méil'), Integer, primary_key=True, key="a"), + Column(ue('\u6e2c\u8a66'), Integer, ForeignKey(u('unitable1.méil')), key="b" ), test_needs_fk=True, @@ -33,27 +34,27 @@ class UnicodeSchemaTest(fixtures.TestBase): # Few DBs support Unicode foreign keys if testing.against('sqlite'): - t3 = Table('\u6e2c\u8a66', metadata, - Column('\u6e2c\u8a66_id', Integer, primary_key=True, + t3 = Table(ue('\u6e2c\u8a66'), metadata, + Column(ue('\u6e2c\u8a66_id'), Integer, primary_key=True, autoincrement=False), - Column('unitable1_\u6e2c\u8a66', Integer, - ForeignKey('unitable1.\u6e2c\u8a66') + Column(ue('unitable1_\u6e2c\u8a66'), Integer, + ForeignKey(ue('unitable1.\u6e2c\u8a66')) ), - Column('Unitéble2_b', Integer, - ForeignKey('Unitéble2.b') + Column(u('Unitéble2_b'), Integer, + ForeignKey(u('Unitéble2.b')) ), - Column('\u6e2c\u8a66_self', Integer, - ForeignKey('\u6e2c\u8a66.\u6e2c\u8a66_id') + Column(ue('\u6e2c\u8a66_self'), Integer, + ForeignKey(ue('\u6e2c\u8a66.\u6e2c\u8a66_id')) ), test_needs_fk=True, ) else: - t3 = Table('\u6e2c\u8a66', metadata, - Column('\u6e2c\u8a66_id', Integer, primary_key=True, + t3 = Table(ue('\u6e2c\u8a66'), metadata, + Column(ue('\u6e2c\u8a66_id'), Integer, primary_key=True, autoincrement=False), - Column('unitable1_\u6e2c\u8a66', Integer), - Column('Unitéble2_b', Integer), - Column('\u6e2c\u8a66_self', Integer), + Column(ue('unitable1_\u6e2c\u8a66'), Integer), + Column(u('Unitéble2_b'), Integer), + Column(ue('\u6e2c\u8a66_self'), Integer), test_needs_fk=True, ) metadata.create_all() @@ -72,42 +73,42 @@ class UnicodeSchemaTest(fixtures.TestBase): del unicode_bind def test_insert(self): - t1.insert().execute({'méil':1, '\u6e2c\u8a66':5}) - t2.insert().execute({'a':1, 'b':1}) - t3.insert().execute({'\u6e2c\u8a66_id': 1, - 'unitable1_\u6e2c\u8a66': 5, - 'Unitéble2_b': 1, - '\u6e2c\u8a66_self': 1}) + t1.insert().execute({u('méil'):1, ue('\u6e2c\u8a66'):5}) + t2.insert().execute({u('a'):1, u('b'):1}) + t3.insert().execute({ue('\u6e2c\u8a66_id'): 1, + ue('unitable1_\u6e2c\u8a66'): 5, + u('Unitéble2_b'): 1, + ue('\u6e2c\u8a66_self'): 1}) assert t1.select().execute().fetchall() == [(1, 5)] assert t2.select().execute().fetchall() == [(1, 1)] assert t3.select().execute().fetchall() == [(1, 5, 1, 1)] def test_reflect(self): - t1.insert().execute({'méil':2, '\u6e2c\u8a66':7}) - t2.insert().execute({'a':2, 'b':2}) - t3.insert().execute({'\u6e2c\u8a66_id': 2, - 'unitable1_\u6e2c\u8a66': 7, - 'Unitéble2_b': 2, - '\u6e2c\u8a66_self': 2}) + t1.insert().execute({u('méil'):2, ue('\u6e2c\u8a66'):7}) + t2.insert().execute({u('a'):2, u('b'):2}) + t3.insert().execute({ue('\u6e2c\u8a66_id'): 2, + ue('unitable1_\u6e2c\u8a66'): 7, + u('Unitéble2_b'): 2, + ue('\u6e2c\u8a66_self'): 2}) meta = MetaData(unicode_bind) tt1 = Table(t1.name, meta, autoload=True) tt2 = Table(t2.name, meta, autoload=True) tt3 = Table(t3.name, meta, autoload=True) - tt1.insert().execute({'méil':1, '\u6e2c\u8a66':5}) - tt2.insert().execute({'méil':1, '\u6e2c\u8a66':1}) - tt3.insert().execute({'\u6e2c\u8a66_id': 1, - 'unitable1_\u6e2c\u8a66': 5, - 'Unitéble2_b': 1, - '\u6e2c\u8a66_self': 1}) + tt1.insert().execute({u('méil'):1, ue('\u6e2c\u8a66'):5}) + tt2.insert().execute({u('méil'):1, ue('\u6e2c\u8a66'):1}) + tt3.insert().execute({ue('\u6e2c\u8a66_id'): 1, + ue('unitable1_\u6e2c\u8a66'): 5, + u('Unitéble2_b'): 1, + ue('\u6e2c\u8a66_self'): 1}) - self.assert_(tt1.select(order_by=desc('méil')).execute().fetchall() == + self.assert_(tt1.select(order_by=desc(u('méil'))).execute().fetchall() == [(2, 7), (1, 5)]) - self.assert_(tt2.select(order_by=desc('méil')).execute().fetchall() == + self.assert_(tt2.select(order_by=desc(u('méil'))).execute().fetchall() == [(2, 2), (1, 1)]) - self.assert_(tt3.select(order_by=desc('\u6e2c\u8a66_id')). + self.assert_(tt3.select(order_by=desc(ue('\u6e2c\u8a66_id'))). execute().fetchall() == [(2, 7, 2, 2), (1, 5, 1, 1)]) meta.drop_all()