From c2a00153f1a8d4591de6ebe4f75e8595d7193226 Mon Sep 17 00:00:00 2001 From: Tony Locke Date: Sat, 26 Jul 2014 16:19:09 +0100 Subject: [PATCH 01/55] With pg8000-1.9.13 passes engine/test_reconnect The pg8000 dialect checks the text of the exception to determine if the connection is closed. I'd (recklessly!) changed the text of the exception in a recent version of the pg8000 driver adding capitalization and a full stop. I've changed it back now so all works. --- test/requirements.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/test/requirements.py b/test/requirements.py index e8705d1452..bf9b8f5263 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -366,16 +366,6 @@ class DefaultRequirements(SuiteRequirements): no_support("postgresql+pg8000", "not supported and/or hangs") ]) - @property - def graceful_disconnects(self): - """Target driver must raise a DBAPI-level exception, such as - InterfaceError, when the underlying connection has been closed - and the execute() method is called. - """ - return fails_on( - "postgresql+pg8000", "Driver crashes" - ) - @property def views(self): """Target database must support VIEWs.""" From f586754c864056a7739ef515b8aece3001a377fc Mon Sep 17 00:00:00 2001 From: Tony Locke Date: Sat, 26 Jul 2014 18:15:06 +0100 Subject: [PATCH 02/55] PEP8 tidy of test/engine/test_reconnect --- test/engine/test_reconnect.py | 110 ++++++++++++++++------------------ 1 file changed, 52 insertions(+), 58 deletions(-) diff --git a/test/engine/test_reconnect.py b/test/engine/test_reconnect.py index f92b874da8..c82cca5a15 100644 --- a/test/engine/test_reconnect.py +++ b/test/engine/test_reconnect.py @@ -1,23 +1,24 @@ from sqlalchemy.testing import eq_, assert_raises, assert_raises_message import time -from sqlalchemy import select, MetaData, Integer, String, create_engine, pool +from sqlalchemy import ( + select, MetaData, Integer, String, create_engine, pool, exc, util) from sqlalchemy.testing.schema import Table, Column import sqlalchemy as tsa from sqlalchemy import testing from sqlalchemy.testing import engines -from sqlalchemy.testing.util import gc_collect -from sqlalchemy import exc, util from sqlalchemy.testing import fixtures from sqlalchemy.testing.engines import testing_engine -from sqlalchemy.testing import is_not_ from sqlalchemy.testing.mock import Mock, call + class MockError(Exception): pass + class MockDisconnect(MockError): pass + def mock_connection(): def mock_cursor(): def execute(*args, **kwargs): @@ -25,10 +26,12 @@ def mock_connection(): raise MockDisconnect("Lost the DB connection on execute") elif conn.explode in ('execute_no_disconnect', ): raise MockError( - "something broke on execute but we didn't lose the connection") + "something broke on execute but we didn't lose the " + "connection") elif conn.explode in ('rollback', 'rollback_no_disconnect'): raise MockError( - "something broke on execute but we didn't lose the connection") + "something broke on execute but we didn't lose the " + "connection") elif args and "SELECT" in args[0]: cursor.description = [('foo', None, None, None, None, None)] else: @@ -38,9 +41,8 @@ def mock_connection(): cursor.fetchall = cursor.fetchone = \ Mock(side_effect=MockError("cursor closed")) cursor = Mock( - execute=Mock(side_effect=execute), - close=Mock(side_effect=close) - ) + execute=Mock(side_effect=execute), + close=Mock(side_effect=close)) return cursor def cursor(): @@ -52,18 +54,20 @@ def mock_connection(): raise MockDisconnect("Lost the DB connection on rollback") if conn.explode == 'rollback_no_disconnect': raise MockError( - "something broke on rollback but we didn't lose the connection") + "something broke on rollback but we didn't lose the " + "connection") else: return conn = Mock( - rollback=Mock(side_effect=rollback), - cursor=Mock(side_effect=cursor()) - ) + rollback=Mock(side_effect=rollback), + cursor=Mock(side_effect=cursor())) return conn + def MockDBAPI(): connections = [] + def connect(): while True: conn = mock_connection() @@ -80,13 +84,12 @@ def MockDBAPI(): connections[:] = [] return Mock( - connect=Mock(side_effect=connect()), - shutdown=Mock(side_effect=shutdown), - dispose=Mock(side_effect=dispose), - paramstyle='named', - connections=connections, - Error=MockError - ) + connect=Mock(side_effect=connect()), + shutdown=Mock(side_effect=shutdown), + dispose=Mock(side_effect=dispose), + paramstyle='named', + connections=connections, + Error=MockError) class MockReconnectTest(fixtures.TestBase): @@ -94,13 +97,14 @@ class MockReconnectTest(fixtures.TestBase): self.dbapi = MockDBAPI() self.db = testing_engine( - 'postgresql://foo:bar@localhost/test', - options=dict(module=self.dbapi, _initialize=False)) + 'postgresql://foo:bar@localhost/test', + options=dict(module=self.dbapi, _initialize=False)) - self.mock_connect = call(host='localhost', password='bar', - user='foo', database='test') + self.mock_connect = call( + host='localhost', password='bar', user='foo', database='test') # monkeypatch disconnect checker - self.db.dialect.is_disconnect = lambda e, conn, cursor: isinstance(e, MockDisconnect) + self.db.dialect.is_disconnect = \ + lambda e, conn, cursor: isinstance(e, MockDisconnect) def teardown(self): self.dbapi.dispose() @@ -194,10 +198,8 @@ class MockReconnectTest(fixtures.TestBase): assert_raises_message( tsa.exc.InvalidRequestError, - "Can't reconnect until invalid transaction is " - "rolled back", - trans.commit - ) + "Can't reconnect until invalid transaction is rolled back", + trans.commit) assert trans.is_active trans.rollback() @@ -351,16 +353,16 @@ class MockReconnectTest(fixtures.TestBase): ) def test_dialect_initialize_once(self): - from sqlalchemy.engine.base import Engine from sqlalchemy.engine.url import URL from sqlalchemy.engine.default import DefaultDialect - from sqlalchemy.pool import QueuePool dbapi = self.dbapi mock_dialect = Mock() + class MyURL(URL): def get_dialect(self): return Dialect + class Dialect(DefaultDialect): initialize = Mock() @@ -371,7 +373,6 @@ class MockReconnectTest(fixtures.TestBase): eq_(Dialect.initialize.call_count, 1) - class CursorErrTest(fixtures.TestBase): # this isn't really a "reconnect" test, it's more of # a generic "recovery". maybe this test suite should have been @@ -394,29 +395,24 @@ class CursorErrTest(fixtures.TestBase): description=[], close=Mock(side_effect=Exception("explode")), ) + def connect(): while True: yield Mock( - spec=['cursor', 'commit', 'rollback', 'close'], - cursor=Mock(side_effect=cursor()), - ) + spec=['cursor', 'commit', 'rollback', 'close'], + cursor=Mock(side_effect=cursor()),) return Mock( - Error = DBAPIError, - paramstyle='qmark', - connect=Mock(side_effect=connect()) - ) + Error=DBAPIError, paramstyle='qmark', + connect=Mock(side_effect=connect())) dbapi = MockDBAPI() from sqlalchemy.engine import default url = Mock( - get_dialect=lambda: default.DefaultDialect, - translate_connect_args=lambda: {}, - query={}, - ) + get_dialect=lambda: default.DefaultDialect, + translate_connect_args=lambda: {}, query={},) eng = testing_engine( - url, - options=dict(module=dbapi, _initialize=initialize)) + url, options=dict(module=dbapi, _initialize=initialize)) eng.pool.logger = Mock() return eng @@ -508,7 +504,6 @@ class RealReconnectTest(fixtures.TestBase): # pool isn't replaced assert self.engine.pool is p2 - def test_ensure_is_disconnect_gets_connection(self): def is_disconnect(e, conn, cursor): # connection is still present @@ -556,6 +551,7 @@ class RealReconnectTest(fixtures.TestBase): "Crashes on py3k+cx_oracle") def test_explode_in_initializer(self): engine = engines.testing_engine() + def broken_initialize(connection): connection.execute("select fake_stuff from _fake_table") @@ -569,6 +565,7 @@ class RealReconnectTest(fixtures.TestBase): "Crashes on py3k+cx_oracle") def test_explode_in_initializer_disconnect(self): engine = engines.testing_engine() + def broken_initialize(connection): connection.execute("select fake_stuff from _fake_table") @@ -584,7 +581,6 @@ class RealReconnectTest(fixtures.TestBase): # invalidate() also doesn't screw up assert_raises(exc.DBAPIError, engine.connect) - def test_null_pool(self): engine = \ engines.reconnecting_engine(options=dict(poolclass=pool.NullPool)) @@ -623,10 +619,8 @@ class RealReconnectTest(fixtures.TestBase): assert trans.is_active assert_raises_message( tsa.exc.StatementError, - "Can't reconnect until invalid transaction is "\ - "rolled back", - conn.execute, select([1]) - ) + "Can't reconnect until invalid transaction is rolled back", + conn.execute, select([1])) assert trans.is_active assert_raises_message( tsa.exc.InvalidRequestError, @@ -640,13 +634,14 @@ class RealReconnectTest(fixtures.TestBase): eq_(conn.execute(select([1])).scalar(), 1) assert not conn.invalidated + class RecycleTest(fixtures.TestBase): __backend__ = True def test_basic(self): for threadlocal in False, True: engine = engines.reconnecting_engine( - options={'pool_threadlocal': threadlocal}) + options={'pool_threadlocal': threadlocal}) conn = engine.contextual_connect() eq_(conn.execute(select([1])).scalar(), 1) @@ -671,13 +666,15 @@ class RecycleTest(fixtures.TestBase): eq_(conn.execute(select([1])).scalar(), 1) conn.close() + class InvalidateDuringResultTest(fixtures.TestBase): __backend__ = True def setup(self): self.engine = engines.reconnecting_engine() self.meta = MetaData(self.engine) - table = Table('sometable', self.meta, + table = Table( + 'sometable', self.meta, Column('id', Integer, primary_key=True), Column('name', String(50))) self.meta.create_all() @@ -690,10 +687,8 @@ class InvalidateDuringResultTest(fixtures.TestBase): self.engine.dispose() @testing.fails_if([ - '+mysqlconnector', '+mysqldb', - '+cymysql', '+pymysql', '+pg8000' - ], "Buffers the result set and doesn't check for " - "connection close") + '+mysqlconnector', '+mysqldb', '+cymysql', '+pymysql', '+pg8000'], + "Buffers the result set and doesn't check for connection close") def test_invalidate_on_results(self): conn = self.engine.connect() result = conn.execute('select * from sometable') @@ -702,4 +697,3 @@ class InvalidateDuringResultTest(fixtures.TestBase): self.engine.test_shutdown() _assert_invalidated(result.fetchone) assert conn.invalidated - From 2e44749b76af4e9e1a2fd6e52dd329dc1e980216 Mon Sep 17 00:00:00 2001 From: Tony Locke Date: Sat, 26 Jul 2014 18:56:56 +0100 Subject: [PATCH 03/55] Remove spurious print statements in pg8000 dialect --- lib/sqlalchemy/dialects/postgresql/pg8000.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/sqlalchemy/dialects/postgresql/pg8000.py b/lib/sqlalchemy/dialects/postgresql/pg8000.py index 68da5b6d7c..2793b048da 100644 --- a/lib/sqlalchemy/dialects/postgresql/pg8000.py +++ b/lib/sqlalchemy/dialects/postgresql/pg8000.py @@ -172,11 +172,9 @@ class PGDialect_pg8000(PGDialect): ) def do_begin_twophase(self, connection, xid): - print("begin twophase", xid) connection.connection.tpc_begin((0, xid, '')) def do_prepare_twophase(self, connection, xid): - print("prepare twophase", xid) connection.connection.tpc_prepare() def do_rollback_twophase( From ed1bbbed272d6413561a2b5a29873f1021890c0d Mon Sep 17 00:00:00 2001 From: Tony Locke Date: Sat, 26 Jul 2014 20:10:36 +0100 Subject: [PATCH 04/55] two_phase_recover, COMMIT PREPARED in transaction In test/engine/test_transaction/test_two_phase_recover(), a COMMIT PREPARED is issued while in a transaction. This causes an error, and a prepared transaction is left hanging around which causes the subsequent test to hang. I've altered the test to execute the offending query with autocommit=true, then when it gets to the COMMIT PRPARED it can go ahead. There's another complication for pg8000 because its tpc_recover() method started a transaction if one wasn't already in progress. I've decided that this is incorrect behaviour and so from pg8000-1.9.13 this method never starts or stops a transaction. --- test/engine/test_transaction.py | 7 ++++--- test/requirements.py | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/engine/test_transaction.py b/test/engine/test_transaction.py index f9744444d2..8a53036422 100644 --- a/test/engine/test_transaction.py +++ b/test/engine/test_transaction.py @@ -347,9 +347,10 @@ class TransactionTest(fixtures.TestBase): connection.invalidate() connection2 = testing.db.connect() - eq_(connection2.execute(select([users.c.user_id]). - order_by(users.c.user_id)).fetchall(), - []) + eq_( + connection2.execution_options(autocommit=True). + execute(select([users.c.user_id]). + order_by(users.c.user_id)).fetchall(), []) recoverables = connection2.recover_twophase() assert transaction.xid in recoverables connection2.commit_prepared(transaction.xid, recover=True) diff --git a/test/requirements.py b/test/requirements.py index bf9b8f5263..7eeabef2b4 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -363,7 +363,6 @@ class DefaultRequirements(SuiteRequirements): 'need separate XA implementation'), exclude('mysql', '<', (5, 0, 3), 'two-phase xact not supported by database'), - no_support("postgresql+pg8000", "not supported and/or hangs") ]) @property From 0dbe9d9aaf22d69e44c486472ff3b412a96cf216 Mon Sep 17 00:00:00 2001 From: Tony Locke Date: Sat, 2 Aug 2014 16:19:46 +0100 Subject: [PATCH 05/55] pg8000 now supports sane_multi_rowcount From pg8000-1.9.14 sane_multi_rowcount is supported so this commit updates the dialect accordingly. --- lib/sqlalchemy/dialects/postgresql/pg8000.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sqlalchemy/dialects/postgresql/pg8000.py b/lib/sqlalchemy/dialects/postgresql/pg8000.py index 2793b048da..909b41b825 100644 --- a/lib/sqlalchemy/dialects/postgresql/pg8000.py +++ b/lib/sqlalchemy/dialects/postgresql/pg8000.py @@ -119,7 +119,7 @@ class PGDialect_pg8000(PGDialect): supports_unicode_binds = True default_paramstyle = 'format' - supports_sane_multi_rowcount = False + supports_sane_multi_rowcount = True execution_ctx_cls = PGExecutionContext_pg8000 statement_compiler = PGCompiler_pg8000 preparer = PGIdentifierPreparer_pg8000 From 2645c8427729733fcd3db044abe7901412890214 Mon Sep 17 00:00:00 2001 From: Matt Chisholm Date: Sun, 27 Jul 2014 12:15:51 +0200 Subject: [PATCH 06/55] add update() support to MutableDict --- lib/sqlalchemy/ext/mutable.py | 4 ++++ test/ext/test_mutable.py | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/lib/sqlalchemy/ext/mutable.py b/lib/sqlalchemy/ext/mutable.py index 7469bcbdae..3ef2f979d2 100644 --- a/lib/sqlalchemy/ext/mutable.py +++ b/lib/sqlalchemy/ext/mutable.py @@ -621,6 +621,10 @@ class MutableDict(Mutable, dict): dict.__delitem__(self, key) self.changed() + def update(self, *a, **kw): + dict.update(self, *a, **kw) + self.changed() + def clear(self): dict.clear(self) self.changed() diff --git a/test/ext/test_mutable.py b/test/ext/test_mutable.py index 32b3e11dd5..9065f3d039 100644 --- a/test/ext/test_mutable.py +++ b/test/ext/test_mutable.py @@ -86,6 +86,18 @@ class _MutableDictTestBase(object): eq_(f1.data, {}) + def test_update(self): + sess = Session() + + f1 = Foo(data={'a': 'b'}) + sess.add(f1) + sess.commit() + + f1.data.update({'a': 'z'}) + sess.commit() + + eq_(f1.data, {'a': 'z'}) + def test_setdefault(self): sess = Session() From 88f7ec6a0efe68305d5d1ee429565c1778ec6a87 Mon Sep 17 00:00:00 2001 From: Matt Chisholm Date: Sun, 27 Jul 2014 12:15:36 +0200 Subject: [PATCH 07/55] fix MutableDict.coerce If a class inherited from MutableDict (say, for instance, to add an update() method), coerce() would give back an instance of MutableDict instead of an instance of the derived class. --- lib/sqlalchemy/ext/mutable.py | 6 ++-- test/ext/test_mutable.py | 53 +++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/lib/sqlalchemy/ext/mutable.py b/lib/sqlalchemy/ext/mutable.py index 7469bcbdae..1a4568f237 100644 --- a/lib/sqlalchemy/ext/mutable.py +++ b/lib/sqlalchemy/ext/mutable.py @@ -627,10 +627,10 @@ class MutableDict(Mutable, dict): @classmethod def coerce(cls, key, value): - """Convert plain dictionary to MutableDict.""" - if not isinstance(value, MutableDict): + """Convert plain dictionary to instance of this class.""" + if not isinstance(value, cls): if isinstance(value, dict): - return MutableDict(value) + return cls(value) return Mutable.coerce(key, value) else: return value diff --git a/test/ext/test_mutable.py b/test/ext/test_mutable.py index 32b3e11dd5..e81b91d93a 100644 --- a/test/ext/test_mutable.py +++ b/test/ext/test_mutable.py @@ -299,6 +299,59 @@ class MutableAssociationScalarJSONTest(_MutableDictTestBase, fixtures.MappedTest ) +class CustomMutableAssociationScalarJSONTest(_MutableDictTestBase, fixtures.MappedTest): + + CustomMutableDict = None + + @classmethod + def _type_fixture(cls): + if not(getattr(cls, 'CustomMutableDict')): + MutableDict = super(CustomMutableAssociationScalarJSONTest, cls)._type_fixture() + class CustomMutableDict(MutableDict): + pass + cls.CustomMutableDict = CustomMutableDict + return cls.CustomMutableDict + + @classmethod + def define_tables(cls, metadata): + import json + + class JSONEncodedDict(TypeDecorator): + impl = VARCHAR(50) + + def process_bind_param(self, value, dialect): + if value is not None: + value = json.dumps(value) + + return value + + def process_result_value(self, value, dialect): + if value is not None: + value = json.loads(value) + return value + + CustomMutableDict = cls._type_fixture() + CustomMutableDict.associate_with(JSONEncodedDict) + + Table('foo', metadata, + Column('id', Integer, primary_key=True, + test_needs_autoincrement=True), + Column('data', JSONEncodedDict), + Column('unrelated_data', String(50)) + ) + + def test_pickle_parent(self): + # Picklers don't know how to pickle CustomMutableDict, but we aren't testing that here + pass + + def test_coerce(self): + sess = Session() + f1 = Foo(data={'a': 'b'}) + sess.add(f1) + sess.flush() + eq_(type(f1.data), self._type_fixture()) + + class _CompositeTestBase(object): @classmethod def define_tables(cls, metadata): From 6ca694610b6d3d791a6bde3daea6c80ef8373426 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 14 Aug 2014 13:25:09 -0400 Subject: [PATCH 08/55] - updates --- doc/build/changelog/migration_10.rst | 65 +++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index 06fccd1dde..22005e062a 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -102,6 +102,25 @@ symbol, and no change to the object's state occurs. Behavioral Changes - Core ========================= +.. _change_3027: + +``autoload_with`` now implies ``autoload=True`` +----------------------------------------------- + +A :class:`.Table` can be set up for reflection by passing ``autoload_with`` +alone:: + + my_table = Table('my_table', metadata, autoload_with=some_engine) + +:ticket:`3027` + +``raise_on_warnings`` defaults to False for mysql-connector-python +------------------------------------------------------------------ + +The Mysql-connector-Python dialect now sets ``raise_on_warnings`` to +``False``, matching the default of the DBAPI itself. + +:ticket:`2515` New Features ============ @@ -131,11 +150,55 @@ wishes to support the new feature should now call upon the ``._limit_clause`` and ``._offset_clause`` attributes to receive the full SQL expression, rather than the integer value. -.. _feature_3076: +.. _feature_get_enums: + +New get_enums() method with Postgresql Dialect +---------------------------------------------- + +The :func:`.inspect` method returns a :class:`.PGInspector` object in the +case of Postgresql, which includes a new :meth:`.PGInspector.get_enums` +method that returns information on all available ``ENUM`` types:: + + from sqlalchemy import inspect, create_engine + + engine = create_engine("postgresql+psycopg2://host/dbname") + insp = inspect(engine) + print(insp.get_enums()) + +.. seealso:: + + :meth:`.PGInspector.get_enums` Behavioral Improvements ======================= +.. _feature_2963: + +.info dictionary improvements +----------------------------- + +The :attr:`.InspectionAttr.info` collection is now available on every kind +of object that one would retrieve from the :attr:`.Mapper.all_orm_descriptors` +collection:: + + class SomeObject(Base): + # ... + + @hybrid_property(self): + def some_prop(self): + return self.value + 5 + + + inspect(SomeObject).all_orm_descriptors.some_prop.info['foo'] = 'bar' + +It is also available as a constructor argument for all :class:`.SchemaItem` +objects (e.g. :class:`.ForeignKey`, :class:`.UniqueConstraint` etc.) as well +as remaining ORM constructs such as :func:`.orm.synonym`. + +:ticket:`2971` + +:ticket:`2963` + Dialect Changes =============== From fe1a09029b6b92f8125d1e09cfdc9eea9f0f9024 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 14 Aug 2014 14:15:46 -0400 Subject: [PATCH 09/55] on second thought we need to prioritize what really needs to be here and what's just in changelog --- doc/build/changelog/migration_10.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index 22005e062a..cf81d41fdc 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -114,13 +114,6 @@ alone:: :ticket:`3027` -``raise_on_warnings`` defaults to False for mysql-connector-python ------------------------------------------------------------------- - -The Mysql-connector-Python dialect now sets ``raise_on_warnings`` to -``False``, matching the default of the DBAPI itself. - -:ticket:`2515` New Features ============ From c0c6aaa58ad1bf01345189a917029c40edc3c8eb Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 14 Aug 2014 14:21:03 -0400 Subject: [PATCH 10/55] pep8 --- test/base/test_events.py | 77 ++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/test/base/test_events.py b/test/base/test_events.py index 4ae89fe175..41ccfbc353 100644 --- a/test/base/test_events.py +++ b/test/base/test_events.py @@ -8,6 +8,7 @@ from sqlalchemy.testing.util import gc_collect from sqlalchemy.testing.mock import Mock, call from sqlalchemy import testing + class EventsTest(fixtures.TestBase): """Test class- and instance-level event registration.""" @@ -155,8 +156,8 @@ class EventsTest(fixtures.TestBase): t2.dispatch.event_one(5, 6) is_( t1.dispatch.__dict__['event_one'], - self.Target.dispatch.event_one.\ - _empty_listeners[self.Target] + self.Target.dispatch.event_one. + _empty_listeners[self.Target] ) @event.listens_for(t1, "event_one") @@ -164,13 +165,13 @@ class EventsTest(fixtures.TestBase): pass is_not_( t1.dispatch.__dict__['event_one'], - self.Target.dispatch.event_one.\ - _empty_listeners[self.Target] + self.Target.dispatch.event_one. + _empty_listeners[self.Target] ) is_( t2.dispatch.__dict__['event_one'], - self.Target.dispatch.event_one.\ - _empty_listeners[self.Target] + self.Target.dispatch.event_one. + _empty_listeners[self.Target] ) def test_immutable_methods(self): @@ -188,6 +189,7 @@ class EventsTest(fixtures.TestBase): meth ) + class NamedCallTest(fixtures.TestBase): def setUp(self): @@ -206,8 +208,8 @@ class NamedCallTest(fixtures.TestBase): self.TargetOne = TargetOne def tearDown(self): - event.base._remove_dispatcher(self.TargetOne.__dict__['dispatch'].events) - + event.base._remove_dispatcher( + self.TargetOne.__dict__['dispatch'].events) def test_kw_accept(self): canary = Mock() @@ -255,7 +257,6 @@ class NamedCallTest(fixtures.TestBase): class LegacySignatureTest(fixtures.TestBase): """test adaption of legacy args""" - def setUp(self): class TargetEventsOne(event.Events): @@ -267,18 +268,19 @@ class LegacySignatureTest(fixtures.TestBase): def event_four(self, x, y, z, q, **kw): pass - @event._legacy_signature("0.9", ["x", "y", "z", "q"], - lambda x, y: (x, y, x + y, x * y)) + @event._legacy_signature( + "0.9", ["x", "y", "z", "q"], + lambda x, y: (x, y, x + y, x * y)) def event_six(self, x, y): pass - class TargetOne(object): dispatch = event.dispatcher(TargetEventsOne) self.TargetOne = TargetOne def tearDown(self): - event.base._remove_dispatcher(self.TargetOne.__dict__['dispatch'].events) + event.base._remove_dispatcher( + self.TargetOne.__dict__['dispatch'].events) def test_legacy_accept(self): canary = Mock() @@ -306,6 +308,7 @@ class LegacySignatureTest(fixtures.TestBase): canary = Mock() inst = self.TargetOne() + @event.listens_for(inst, "event_four") def handler1(x, y, **kw): canary(x, y, kw) @@ -313,6 +316,7 @@ class LegacySignatureTest(fixtures.TestBase): def test_legacy_accept_partial(self): canary = Mock() + def evt(a, x, y, **kw): canary(a, x, y, **kw) from functools import partial @@ -330,7 +334,6 @@ class LegacySignatureTest(fixtures.TestBase): [call(5, 4, 5, foo="bar")] ) - def _test_legacy_accept_kw(self, target, canary): target.dispatch.event_four(4, 5, 6, 7, foo="bar") @@ -410,22 +413,19 @@ class LegacySignatureTest(fixtures.TestBase): class ClsLevelListenTest(fixtures.TestBase): - def tearDown(self): - event.base._remove_dispatcher(self.TargetOne.__dict__['dispatch'].events) + event.base._remove_dispatcher( + self.TargetOne.__dict__['dispatch'].events) def setUp(self): class TargetEventsOne(event.Events): def event_one(self, x, y): pass + class TargetOne(object): dispatch = event.dispatcher(TargetEventsOne) self.TargetOne = TargetOne - def tearDown(self): - event.base._remove_dispatcher( - self.TargetOne.__dict__['dispatch'].events) - def test_lis_subcalss_lis(self): @event.listens_for(self.TargetOne, "event_one") def handler1(x, y): @@ -470,12 +470,14 @@ class ClsLevelListenTest(fixtures.TestBase): def test_two_sub_lis(self): class SubTarget1(self.TargetOne): pass + class SubTarget2(self.TargetOne): pass @event.listens_for(self.TargetOne, "event_one") def handler1(x, y): pass + @event.listens_for(SubTarget1, "event_one") def handler2(x, y): pass @@ -510,8 +512,10 @@ class AcceptTargetsTest(fixtures.TestBase): self.TargetTwo = TargetTwo def tearDown(self): - event.base._remove_dispatcher(self.TargetOne.__dict__['dispatch'].events) - event.base._remove_dispatcher(self.TargetTwo.__dict__['dispatch'].events) + event.base._remove_dispatcher( + self.TargetOne.__dict__['dispatch'].events) + event.base._remove_dispatcher( + self.TargetTwo.__dict__['dispatch'].events) def test_target_accept(self): """Test that events of the same name are routed to the correct @@ -560,6 +564,7 @@ class AcceptTargetsTest(fixtures.TestBase): [listen_two, listen_four] ) + class CustomTargetsTest(fixtures.TestBase): """Test custom target acceptance.""" @@ -599,6 +604,7 @@ class CustomTargetsTest(fixtures.TestBase): listen, "event_one", self.Target ) + class SubclassGrowthTest(fixtures.TestBase): """test that ad-hoc subclasses are garbage collected.""" @@ -625,7 +631,8 @@ class SubclassGrowthTest(fixtures.TestBase): class ListenOverrideTest(fixtures.TestBase): - """Test custom listen functions which change the listener function signature.""" + """Test custom listen functions which change the listener function + signature.""" def setUp(self): class TargetEvents(event.Events): @@ -715,7 +722,6 @@ class PropagateTest(fixtures.TestBase): dispatch = event.dispatcher(TargetEvents) self.Target = Target - def test_propagate(self): listen_one = Mock() listen_two = Mock() @@ -741,6 +747,7 @@ class PropagateTest(fixtures.TestBase): [] ) + class JoinTest(fixtures.TestBase): def setUp(self): class TargetEvents(event.Events): @@ -767,7 +774,8 @@ class JoinTest(fixtures.TestBase): self.TargetElement = TargetElement def tearDown(self): - for cls in (self.TargetElement, + for cls in ( + self.TargetElement, self.TargetFactory, self.BaseTarget): if 'dispatch' in cls.__dict__: event.base._remove_dispatcher(cls.__dict__['dispatch'].events) @@ -780,6 +788,7 @@ class JoinTest(fixtures.TestBase): def test_kw_ok(self): l1 = Mock() + def listen(**kw): l1(kw) @@ -789,8 +798,10 @@ class JoinTest(fixtures.TestBase): element.run_event(2) eq_( l1.mock_calls, - [call({"target": element, "arg": 1}), - call({"target": element, "arg": 2}),] + [ + call({"target": element, "arg": 1}), + call({"target": element, "arg": 2}), + ] ) def test_parent_class_only(self): @@ -895,7 +906,6 @@ class JoinTest(fixtures.TestBase): [call(element, 1), call(element, 2), call(element, 3)] ) - def test_parent_instance_child_class_apply_after(self): l1 = Mock() l2 = Mock() @@ -969,6 +979,7 @@ class JoinTest(fixtures.TestBase): [call(element, 1), call(element, 2), call(element, 3)] ) + class RemovalTest(fixtures.TestBase): def _fixture(self): class TargetEvents(event.Events): @@ -1003,6 +1014,7 @@ class RemovalTest(fixtures.TestBase): def test_clslevel_subclass(self): Target = self._fixture() + class SubTarget(Target): pass @@ -1097,8 +1109,10 @@ class RemovalTest(fixtures.TestBase): t2.dispatch.event_two("t2e2y") eq_(m1.mock_calls, - [call('t1e1x'), call('t1e2x'), - call('t2e1x')]) + [ + call('t1e1x'), call('t1e2x'), + call('t2e1x') + ]) @testing.requires.predictable_gc def test_listener_collection_removed_cleanup(self): @@ -1140,7 +1154,8 @@ class RemovalTest(fixtures.TestBase): event.remove(t1, "event_one", m1) assert_raises_message( exc.InvalidRequestError, - r"No listeners found for event <.*Target.*> / 'event_two' / ", + r"No listeners found for event <.*Target.*> / " + r"'event_two' / ", event.remove, t1, "event_two", m1 ) From 4a4cccfee5a2eb78380e56eb9476e91658656676 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 14 Aug 2014 14:40:28 -0400 Subject: [PATCH 11/55] - Removing (or adding) an event listener at the same time that the event is being run itself, either from inside the listener or from a concurrent thread, now raises a RuntimeError, as the collection used is now an instance of ``colletions.deque()`` and does not support changes while being iterated. Previously, a plain Python list was used where removal from inside the event itself would produce silent failures. fixes #3163 --- doc/build/changelog/changelog_10.rst | 11 ++++++ lib/sqlalchemy/event/api.py | 54 ++++++++++++++++++++++++++++ lib/sqlalchemy/event/attr.py | 15 ++++---- lib/sqlalchemy/event/registry.py | 2 +- test/base/test_events.py | 34 ++++++++++++++++++ 5 files changed, 108 insertions(+), 8 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 815de72c71..fb14279ac1 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -16,6 +16,17 @@ .. changelog:: :version: 1.0.0 + .. change:: + :tags: engine, bug + :tickets: 3163 + + Removing (or adding) an event listener at the same time that the event + is being run itself, either from inside the listener or from a + concurrent thread, now raises a RuntimeError, as the collection used is + now an instance of ``colletions.deque()`` and does not support changes + while being iterated. Previously, a plain Python list was used where + removal from inside the event itself would produce silent failures. + .. change:: :tags: orm, feature :tickets: 2963 diff --git a/lib/sqlalchemy/event/api.py b/lib/sqlalchemy/event/api.py index 270e95c9c3..b3d79bcf4e 100644 --- a/lib/sqlalchemy/event/api.py +++ b/lib/sqlalchemy/event/api.py @@ -58,6 +58,32 @@ def listen(target, identifier, fn, *args, **kw): .. versionadded:: 0.9.4 Added ``once=True`` to :func:`.event.listen` and :func:`.event.listens_for`. + .. note:: + + The :func:`.listen` function cannot be called at the same time + that the target event is being run. This has implications + for thread safety, and also means an event cannot be added + from inside the listener function for itself. The list of + events to be run are present inside of a mutable collection + that can't be changed during iteration. + + Event registration and removal is not intended to be a "high + velocity" operation; it is a configurational operation. For + systems that need to quickly associate and deassociate with + events at high scale, use a mutable structure that is handled + from inside of a single listener. + + .. versionchanged:: 1.0.0 - a ``collections.deque()`` object is now + used as the container for the list of events, which explicitly + disallows collection mutation while the collection is being + iterated. + + .. seealso:: + + :func:`.listens_for` + + :func:`.remove` + """ _event_key(target, identifier, fn).listen(*args, **kw) @@ -89,6 +115,10 @@ def listens_for(target, identifier, *args, **kw): .. versionadded:: 0.9.4 Added ``once=True`` to :func:`.event.listen` and :func:`.event.listens_for`. + .. seealso:: + + :func:`.listen` - general description of event listening + """ def decorate(fn): listen(target, identifier, fn, *args, **kw) @@ -120,6 +150,30 @@ def remove(target, identifier, fn): .. versionadded:: 0.9.0 + .. note:: + + The :func:`.remove` function cannot be called at the same time + that the target event is being run. This has implications + for thread safety, and also means an event cannot be removed + from inside the listener function for itself. The list of + events to be run are present inside of a mutable collection + that can't be changed during iteration. + + Event registration and removal is not intended to be a "high + velocity" operation; it is a configurational operation. For + systems that need to quickly associate and deassociate with + events at high scale, use a mutable structure that is handled + from inside of a single listener. + + .. versionchanged:: 1.0.0 - a ``collections.deque()`` object is now + used as the container for the list of events, which explicitly + disallows collection mutation while the collection is being + iterated. + + .. seealso:: + + :func:`.listen` + """ _event_key(target, identifier, fn).remove() diff --git a/lib/sqlalchemy/event/attr.py b/lib/sqlalchemy/event/attr.py index 7641b595a8..dba1063cfd 100644 --- a/lib/sqlalchemy/event/attr.py +++ b/lib/sqlalchemy/event/attr.py @@ -37,6 +37,7 @@ from . import registry from . import legacy from itertools import chain import weakref +import collections class RefCollection(object): @@ -96,8 +97,8 @@ class _DispatchDescriptor(RefCollection): self.update_subclass(cls) else: if cls not in self._clslevel: - self._clslevel[cls] = [] - self._clslevel[cls].insert(0, event_key._listen_fn) + self._clslevel[cls] = collections.deque() + self._clslevel[cls].appendleft(event_key._listen_fn) registry._stored_in_collection(event_key, self) def append(self, event_key, propagate): @@ -113,13 +114,13 @@ class _DispatchDescriptor(RefCollection): self.update_subclass(cls) else: if cls not in self._clslevel: - self._clslevel[cls] = [] + self._clslevel[cls] = collections.deque() self._clslevel[cls].append(event_key._listen_fn) registry._stored_in_collection(event_key, self) def update_subclass(self, target): if target not in self._clslevel: - self._clslevel[target] = [] + self._clslevel[target] = collections.deque() clslevel = self._clslevel[target] for cls in target.__mro__[1:]: if cls in self._clslevel: @@ -145,7 +146,7 @@ class _DispatchDescriptor(RefCollection): to_clear = set() for dispatcher in self._clslevel.values(): to_clear.update(dispatcher) - dispatcher[:] = [] + dispatcher.clear() registry._clear(self, to_clear) def for_modify(self, obj): @@ -287,7 +288,7 @@ class _ListenerCollection(RefCollection, _CompoundListener): self.parent_listeners = parent._clslevel[target_cls] self.parent = parent self.name = parent.__name__ - self.listeners = [] + self.listeners = collections.deque() self.propagate = set() def for_modify(self, obj): @@ -337,7 +338,7 @@ class _ListenerCollection(RefCollection, _CompoundListener): def clear(self): registry._clear(self, self.listeners) self.propagate.clear() - self.listeners[:] = [] + self.listeners.clear() class _JoinedDispatchDescriptor(object): diff --git a/lib/sqlalchemy/event/registry.py b/lib/sqlalchemy/event/registry.py index a34de3cd75..ba2f671a34 100644 --- a/lib/sqlalchemy/event/registry.py +++ b/lib/sqlalchemy/event/registry.py @@ -243,4 +243,4 @@ class _EventKey(object): def prepend_to_list(self, owner, list_): _stored_in_collection(self, owner) - list_.insert(0, self._listen_fn) + list_.appendleft(self._listen_fn) diff --git a/test/base/test_events.py b/test/base/test_events.py index 41ccfbc353..30b728cd39 100644 --- a/test/base/test_events.py +++ b/test/base/test_events.py @@ -1160,3 +1160,37 @@ class RemovalTest(fixtures.TestBase): ) event.remove(t1, "event_three", m1) + + def test_no_remove_in_event(self): + Target = self._fixture() + + t1 = Target() + + def evt(): + event.remove(t1, "event_one", evt) + + event.listen(t1, "event_one", evt) + + assert_raises_message( + Exception, + "deque mutated during iteration", + t1.dispatch.event_one + ) + + def test_no_add_in_event(self): + Target = self._fixture() + + t1 = Target() + + m1 = Mock() + + def evt(): + event.listen(t1, "event_one", m1) + + event.listen(t1, "event_one", evt) + + assert_raises_message( + Exception, + "deque mutated during iteration", + t1.dispatch.event_one + ) From 6a21f9e328361d5185fd616e7992a183030f9a10 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 14 Aug 2014 20:00:35 -0400 Subject: [PATCH 12/55] - The string keys that are used to determine the columns impacted for an INSERT or UPDATE are now sorted when they contribute towards the "compiled cache" cache key. These keys were previously not deterministically ordered, meaning the same statement could be cached multiple times on equivalent keys, costing both in terms of memory as well as performance. fixes #3165 --- doc/build/changelog/changelog_09.rst | 12 +++++++ lib/sqlalchemy/engine/base.py | 2 +- test/engine/test_execute.py | 48 ++++++++++++++++++++++++++-- 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index a797bfa295..0f92fb254c 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -13,6 +13,18 @@ .. changelog:: :version: 0.9.8 + .. change:: + :tags: bug, engine + :versions: 1.0.0 + :tickets: 3165 + + The string keys that are used to determine the columns impacted + for an INSERT or UPDATE are now sorted when they contribute towards + the "compiled cache" cache key. These keys were previously not + deterministically ordered, meaning the same statement could be + cached multiple times on equivalent keys, costing both in terms of + memory as well as performance. + .. change:: :tags: bug, postgresql :versions: 1.0.0 diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index 2dc4d43f2d..65753b6dcf 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -805,7 +805,7 @@ class Connection(Connectable): dialect = self.dialect if 'compiled_cache' in self._execution_options: - key = dialect, elem, tuple(keys), len(distilled_params) > 1 + key = dialect, elem, tuple(sorted(keys)), len(distilled_params) > 1 if key in self._execution_options['compiled_cache']: compiled_sql = self._execution_options['compiled_cache'][key] else: diff --git a/test/engine/test_execute.py b/test/engine/test_execute.py index 291aee2f34..f65168552a 100644 --- a/test/engine/test_execute.py +++ b/test/engine/test_execute.py @@ -688,6 +688,7 @@ class CompiledCacheTest(fixtures.TestBase): Column('user_id', INT, primary_key=True, test_needs_autoincrement=True), Column('user_name', VARCHAR(20)), + Column("extra_data", VARCHAR(20)) ) metadata.create_all() @@ -705,12 +706,53 @@ class CompiledCacheTest(fixtures.TestBase): cached_conn = conn.execution_options(compiled_cache=cache) ins = users.insert() - cached_conn.execute(ins, {'user_name': 'u1'}) - cached_conn.execute(ins, {'user_name': 'u2'}) - cached_conn.execute(ins, {'user_name': 'u3'}) + with patch.object( + ins, "compile", + Mock(side_effect=ins.compile)) as compile_mock: + cached_conn.execute(ins, {'user_name': 'u1'}) + cached_conn.execute(ins, {'user_name': 'u2'}) + cached_conn.execute(ins, {'user_name': 'u3'}) + eq_(compile_mock.call_count, 1) assert len(cache) == 1 eq_(conn.execute("select count(*) from users").scalar(), 3) + def test_keys_independent_of_ordering(self): + conn = testing.db.connect() + conn.execute( + users.insert(), + {"user_id": 1, "user_name": "u1", "extra_data": "e1"}) + cache = {} + cached_conn = conn.execution_options(compiled_cache=cache) + + upd = users.update().where(users.c.user_id == bindparam("b_user_id")) + + with patch.object( + upd, "compile", + Mock(side_effect=upd.compile)) as compile_mock: + cached_conn.execute( + upd, util.OrderedDict([ + ("b_user_id", 1), + ("user_name", "u2"), + ("extra_data", "e2") + ]) + ) + cached_conn.execute( + upd, util.OrderedDict([ + ("b_user_id", 1), + ("extra_data", "e3"), + ("user_name", "u3"), + ]) + ) + cached_conn.execute( + upd, util.OrderedDict([ + ("extra_data", "e4"), + ("user_name", "u4"), + ("b_user_id", 1), + ]) + ) + eq_(compile_mock.call_count, 1) + eq_(len(cache), 1) + class MockStrategyTest(fixtures.TestBase): From 253523c57f76c515ade82e4db30cff2536fb2a92 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 14 Aug 2014 20:01:59 -0400 Subject: [PATCH 13/55] pep8 --- lib/sqlalchemy/orm/persistence.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index 295d4a3d0a..17ce2e6247 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -309,10 +309,10 @@ def _collect_update_commands(base_mapper, uowtransaction, if col is mapper.version_id_col: params[col._label] = \ mapper._get_committed_state_attr_by_column( - row_switch or state, - row_switch and row_switch.dict - or state_dict, - col) + row_switch or state, + row_switch and row_switch.dict + or state_dict, + col) prop = mapper._columntoproperty[col] history = state.manager[prop.key].impl.get_history( @@ -417,8 +417,8 @@ def _collect_post_update_commands(base_mapper, uowtransaction, table, if col in pks: params[col._label] = \ mapper._get_state_attr_by_column( - state, - state_dict, col) + state, + state_dict, col) elif col in post_update_cols: prop = mapper._columntoproperty[col] @@ -453,7 +453,7 @@ def _collect_delete_commands(base_mapper, uowtransaction, table, params[col.key] = \ value = \ mapper._get_committed_state_attr_by_column( - state, state_dict, col) + state, state_dict, col) if value is None: raise orm_exc.FlushError( "Can't delete from table " @@ -464,8 +464,8 @@ def _collect_delete_commands(base_mapper, uowtransaction, table, table.c.contains_column(mapper.version_id_col): params[mapper.version_id_col.key] = \ mapper._get_committed_state_attr_by_column( - state, state_dict, - mapper.version_id_col) + state, state_dict, + mapper.version_id_col) return delete From f0a56bc5aa8cb0ca7b642eaa99093ade2065d4b5 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 14 Aug 2014 20:37:28 -0400 Subject: [PATCH 14/55] pep8 --- test/orm/test_unitofworkv2.py | 1008 +++++++++++++++++---------------- 1 file changed, 526 insertions(+), 482 deletions(-) diff --git a/test/orm/test_unitofworkv2.py b/test/orm/test_unitofworkv2.py index 9fedc95901..9c9296786e 100644 --- a/test/orm/test_unitofworkv2.py +++ b/test/orm/test_unitofworkv2.py @@ -1,4 +1,4 @@ -from sqlalchemy.testing import eq_, assert_raises, assert_raises_message +from sqlalchemy.testing import eq_, assert_raises_message from sqlalchemy import testing from sqlalchemy.testing import engines from sqlalchemy.testing.schema import Table, Column @@ -7,12 +7,14 @@ from sqlalchemy import exc from sqlalchemy.testing import fixtures from sqlalchemy import Integer, String, ForeignKey, func from sqlalchemy.orm import mapper, relationship, backref, \ - create_session, unitofwork, attributes,\ - Session, class_mapper, sync, exc as orm_exc + create_session, unitofwork, attributes,\ + Session, exc as orm_exc + +from sqlalchemy.testing.assertsql import AllOf, CompiledSQL -from sqlalchemy.testing.assertsql import AllOf, CompiledSQL, Or class AssertsUOW(object): + def _get_test_uow(self, session): uow = unitofwork.UOWTransaction(session) deleted = set(session._deleted) @@ -24,26 +26,29 @@ class AssertsUOW(object): uow.register_object(d, isdelete=True) return uow - def _assert_uow_size(self, session, expected ): + def _assert_uow_size(self, session, expected): uow = self._get_test_uow(session) postsort_actions = uow._generate_actions() print(postsort_actions) eq_(len(postsort_actions), expected, postsort_actions) -class UOWTest(_fixtures.FixtureTest, - testing.AssertsExecutionResults, AssertsUOW): + +class UOWTest( + _fixtures.FixtureTest, + testing.AssertsExecutionResults, AssertsUOW): run_inserts = None + class RudimentaryFlushTest(UOWTest): def test_one_to_many_save(self): users, Address, addresses, User = (self.tables.users, - self.classes.Address, - self.tables.addresses, - self.classes.User) + self.classes.Address, + self.tables.addresses, + self.classes.User) mapper(User, users, properties={ - 'addresses':relationship(Address), + 'addresses': relationship(Address), }) mapper(Address, addresses) sess = create_session() @@ -53,32 +58,32 @@ class RudimentaryFlushTest(UOWTest): sess.add(u1) self.assert_sql_execution( - testing.db, - sess.flush, - CompiledSQL( - "INSERT INTO users (name) VALUES (:name)", - {'name': 'u1'} - ), - CompiledSQL( - "INSERT INTO addresses (user_id, email_address) " - "VALUES (:user_id, :email_address)", - lambda ctx: {'email_address': 'a1', 'user_id':u1.id} - ), - CompiledSQL( - "INSERT INTO addresses (user_id, email_address) " - "VALUES (:user_id, :email_address)", - lambda ctx: {'email_address': 'a2', 'user_id':u1.id} - ), - ) + testing.db, + sess.flush, + CompiledSQL( + "INSERT INTO users (name) VALUES (:name)", + {'name': 'u1'} + ), + CompiledSQL( + "INSERT INTO addresses (user_id, email_address) " + "VALUES (:user_id, :email_address)", + lambda ctx: {'email_address': 'a1', 'user_id': u1.id} + ), + CompiledSQL( + "INSERT INTO addresses (user_id, email_address) " + "VALUES (:user_id, :email_address)", + lambda ctx: {'email_address': 'a2', 'user_id': u1.id} + ), + ) def test_one_to_many_delete_all(self): users, Address, addresses, User = (self.tables.users, - self.classes.Address, - self.tables.addresses, - self.classes.User) + self.classes.Address, + self.tables.addresses, + self.classes.User) mapper(User, users, properties={ - 'addresses':relationship(Address), + 'addresses': relationship(Address), }) mapper(Address, addresses) sess = create_session() @@ -91,26 +96,26 @@ class RudimentaryFlushTest(UOWTest): sess.delete(a1) sess.delete(a2) self.assert_sql_execution( - testing.db, - sess.flush, - CompiledSQL( - "DELETE FROM addresses WHERE addresses.id = :id", - [{'id':a1.id},{'id':a2.id}] - ), - CompiledSQL( - "DELETE FROM users WHERE users.id = :id", - {'id':u1.id} - ), + testing.db, + sess.flush, + CompiledSQL( + "DELETE FROM addresses WHERE addresses.id = :id", + [{'id': a1.id}, {'id': a2.id}] + ), + CompiledSQL( + "DELETE FROM users WHERE users.id = :id", + {'id': u1.id} + ), ) def test_one_to_many_delete_parent(self): users, Address, addresses, User = (self.tables.users, - self.classes.Address, - self.tables.addresses, - self.classes.User) + self.classes.Address, + self.tables.addresses, + self.classes.User) mapper(User, users, properties={ - 'addresses':relationship(Address), + 'addresses': relationship(Address), }) mapper(Address, addresses) sess = create_session() @@ -121,76 +126,75 @@ class RudimentaryFlushTest(UOWTest): sess.delete(u1) self.assert_sql_execution( - testing.db, - sess.flush, - CompiledSQL( - "UPDATE addresses SET user_id=:user_id WHERE " - "addresses.id = :addresses_id", - lambda ctx: [{'addresses_id': a1.id, 'user_id': None}] - ), - CompiledSQL( - "UPDATE addresses SET user_id=:user_id WHERE " - "addresses.id = :addresses_id", - lambda ctx: [{'addresses_id': a2.id, 'user_id': None}] - ), - CompiledSQL( - "DELETE FROM users WHERE users.id = :id", - {'id':u1.id} - ), + testing.db, + sess.flush, + CompiledSQL( + "UPDATE addresses SET user_id=:user_id WHERE " + "addresses.id = :addresses_id", + lambda ctx: [{'addresses_id': a1.id, 'user_id': None}] + ), + CompiledSQL( + "UPDATE addresses SET user_id=:user_id WHERE " + "addresses.id = :addresses_id", + lambda ctx: [{'addresses_id': a2.id, 'user_id': None}] + ), + CompiledSQL( + "DELETE FROM users WHERE users.id = :id", + {'id': u1.id} + ), ) def test_many_to_one_save(self): users, Address, addresses, User = (self.tables.users, - self.classes.Address, - self.tables.addresses, - self.classes.User) - + self.classes.Address, + self.tables.addresses, + self.classes.User) mapper(User, users) mapper(Address, addresses, properties={ - 'user':relationship(User) + 'user': relationship(User) }) sess = create_session() u1 = User(name='u1') a1, a2 = Address(email_address='a1', user=u1), \ - Address(email_address='a2', user=u1) + Address(email_address='a2', user=u1) sess.add_all([a1, a2]) self.assert_sql_execution( - testing.db, - sess.flush, - CompiledSQL( - "INSERT INTO users (name) VALUES (:name)", - {'name': 'u1'} - ), - CompiledSQL( - "INSERT INTO addresses (user_id, email_address) " - "VALUES (:user_id, :email_address)", - lambda ctx: {'email_address': 'a1', 'user_id':u1.id} - ), - CompiledSQL( - "INSERT INTO addresses (user_id, email_address) " - "VALUES (:user_id, :email_address)", - lambda ctx: {'email_address': 'a2', 'user_id':u1.id} - ), - ) + testing.db, + sess.flush, + CompiledSQL( + "INSERT INTO users (name) VALUES (:name)", + {'name': 'u1'} + ), + CompiledSQL( + "INSERT INTO addresses (user_id, email_address) " + "VALUES (:user_id, :email_address)", + lambda ctx: {'email_address': 'a1', 'user_id': u1.id} + ), + CompiledSQL( + "INSERT INTO addresses (user_id, email_address) " + "VALUES (:user_id, :email_address)", + lambda ctx: {'email_address': 'a2', 'user_id': u1.id} + ), + ) def test_many_to_one_delete_all(self): users, Address, addresses, User = (self.tables.users, - self.classes.Address, - self.tables.addresses, - self.classes.User) + self.classes.Address, + self.tables.addresses, + self.classes.User) mapper(User, users) mapper(Address, addresses, properties={ - 'user':relationship(User) + 'user': relationship(User) }) sess = create_session() u1 = User(name='u1') a1, a2 = Address(email_address='a1', user=u1), \ - Address(email_address='a2', user=u1) + Address(email_address='a2', user=u1) sess.add_all([a1, a2]) sess.flush() @@ -198,71 +202,71 @@ class RudimentaryFlushTest(UOWTest): sess.delete(a1) sess.delete(a2) self.assert_sql_execution( - testing.db, - sess.flush, - CompiledSQL( - "DELETE FROM addresses WHERE addresses.id = :id", - [{'id':a1.id},{'id':a2.id}] - ), - CompiledSQL( - "DELETE FROM users WHERE users.id = :id", - {'id':u1.id} - ), + testing.db, + sess.flush, + CompiledSQL( + "DELETE FROM addresses WHERE addresses.id = :id", + [{'id': a1.id}, {'id': a2.id}] + ), + CompiledSQL( + "DELETE FROM users WHERE users.id = :id", + {'id': u1.id} + ), ) def test_many_to_one_delete_target(self): users, Address, addresses, User = (self.tables.users, - self.classes.Address, - self.tables.addresses, - self.classes.User) + self.classes.Address, + self.tables.addresses, + self.classes.User) mapper(User, users) mapper(Address, addresses, properties={ - 'user':relationship(User) + 'user': relationship(User) }) sess = create_session() u1 = User(name='u1') a1, a2 = Address(email_address='a1', user=u1), \ - Address(email_address='a2', user=u1) + Address(email_address='a2', user=u1) sess.add_all([a1, a2]) sess.flush() sess.delete(u1) a1.user = a2.user = None self.assert_sql_execution( - testing.db, - sess.flush, - CompiledSQL( - "UPDATE addresses SET user_id=:user_id WHERE " - "addresses.id = :addresses_id", - lambda ctx: [{'addresses_id': a1.id, 'user_id': None}] - ), - CompiledSQL( - "UPDATE addresses SET user_id=:user_id WHERE " - "addresses.id = :addresses_id", - lambda ctx: [{'addresses_id': a2.id, 'user_id': None}] - ), - CompiledSQL( - "DELETE FROM users WHERE users.id = :id", - {'id':u1.id} - ), + testing.db, + sess.flush, + CompiledSQL( + "UPDATE addresses SET user_id=:user_id WHERE " + "addresses.id = :addresses_id", + lambda ctx: [{'addresses_id': a1.id, 'user_id': None}] + ), + CompiledSQL( + "UPDATE addresses SET user_id=:user_id WHERE " + "addresses.id = :addresses_id", + lambda ctx: [{'addresses_id': a2.id, 'user_id': None}] + ), + CompiledSQL( + "DELETE FROM users WHERE users.id = :id", + {'id': u1.id} + ), ) def test_many_to_one_delete_unloaded(self): users, Address, addresses, User = (self.tables.users, - self.classes.Address, - self.tables.addresses, - self.classes.User) + self.classes.Address, + self.tables.addresses, + self.classes.User) mapper(User, users) mapper(Address, addresses, properties={ - 'parent':relationship(User) + 'parent': relationship(User) }) parent = User(name='p1') c1, c2 = Address(email_address='c1', parent=parent), \ - Address(email_address='c2', parent=parent) + Address(email_address='c2', parent=parent) session = Session() session.add_all([c1, c2]) @@ -295,16 +299,20 @@ class RudimentaryFlushTest(UOWTest): # the User row might be handled before or the addresses # are loaded so need to use AllOf CompiledSQL( - "SELECT addresses.id AS addresses_id, addresses.user_id AS " + "SELECT addresses.id AS addresses_id, " + "addresses.user_id AS " "addresses_user_id, addresses.email_address AS " - "addresses_email_address FROM addresses WHERE addresses.id = " + "addresses_email_address FROM addresses " + "WHERE addresses.id = " ":param_1", lambda ctx: {'param_1': c1id} ), CompiledSQL( - "SELECT addresses.id AS addresses_id, addresses.user_id AS " + "SELECT addresses.id AS addresses_id, " + "addresses.user_id AS " "addresses_user_id, addresses.email_address AS " - "addresses_email_address FROM addresses WHERE addresses.id = " + "addresses_email_address FROM addresses " + "WHERE addresses.id = " ":param_1", lambda ctx: {'param_1': c2id} ), @@ -326,18 +334,18 @@ class RudimentaryFlushTest(UOWTest): def test_many_to_one_delete_childonly_unloaded(self): users, Address, addresses, User = (self.tables.users, - self.classes.Address, - self.tables.addresses, - self.classes.User) + self.classes.Address, + self.tables.addresses, + self.classes.User) mapper(User, users) mapper(Address, addresses, properties={ - 'parent':relationship(User) + 'parent': relationship(User) }) parent = User(name='p1') c1, c2 = Address(email_address='c1', parent=parent), \ - Address(email_address='c2', parent=parent) + Address(email_address='c2', parent=parent) session = Session() session.add_all([c1, c2]) @@ -345,7 +353,7 @@ class RudimentaryFlushTest(UOWTest): session.flush() - pid = parent.id + #pid = parent.id c1id = c1.id c2id = c2.id @@ -360,18 +368,23 @@ class RudimentaryFlushTest(UOWTest): session.flush, AllOf( # [ticket:2049] - we aren't deleting User, - # relationship is simple m2o, no SELECT should be emitted for it. + # relationship is simple m2o, no SELECT should be emitted for + # it. CompiledSQL( - "SELECT addresses.id AS addresses_id, addresses.user_id AS " + "SELECT addresses.id AS addresses_id, " + "addresses.user_id AS " "addresses_user_id, addresses.email_address AS " - "addresses_email_address FROM addresses WHERE addresses.id = " + "addresses_email_address FROM addresses " + "WHERE addresses.id = " ":param_1", lambda ctx: {'param_1': c1id} ), CompiledSQL( - "SELECT addresses.id AS addresses_id, addresses.user_id AS " + "SELECT addresses.id AS addresses_id, " + "addresses.user_id AS " "addresses_user_id, addresses.email_address AS " - "addresses_email_address FROM addresses WHERE addresses.id = " + "addresses_email_address FROM addresses " + "WHERE addresses.id = " ":param_1", lambda ctx: {'param_1': c2id} ), @@ -384,18 +397,18 @@ class RudimentaryFlushTest(UOWTest): def test_many_to_one_delete_childonly_unloaded_expired(self): users, Address, addresses, User = (self.tables.users, - self.classes.Address, - self.tables.addresses, - self.classes.User) + self.classes.Address, + self.tables.addresses, + self.classes.User) mapper(User, users) mapper(Address, addresses, properties={ - 'parent':relationship(User) + 'parent': relationship(User) }) parent = User(name='p1') c1, c2 = Address(email_address='c1', parent=parent), \ - Address(email_address='c2', parent=parent) + Address(email_address='c2', parent=parent) session = Session() session.add_all([c1, c2]) @@ -403,7 +416,7 @@ class RudimentaryFlushTest(UOWTest): session.flush() - pid = parent.id + #pid = parent.id c1id = c1.id c2id = c2.id @@ -420,16 +433,20 @@ class RudimentaryFlushTest(UOWTest): AllOf( # the parent User is expired, so it gets loaded here. CompiledSQL( - "SELECT addresses.id AS addresses_id, addresses.user_id AS " + "SELECT addresses.id AS addresses_id, " + "addresses.user_id AS " "addresses_user_id, addresses.email_address AS " - "addresses_email_address FROM addresses WHERE addresses.id = " + "addresses_email_address FROM addresses " + "WHERE addresses.id = " ":param_1", lambda ctx: {'param_1': c1id} ), CompiledSQL( - "SELECT addresses.id AS addresses_id, addresses.user_id AS " + "SELECT addresses.id AS addresses_id, " + "addresses.user_id AS " "addresses_user_id, addresses.email_address AS " - "addresses_email_address FROM addresses WHERE addresses.id = " + "addresses_email_address FROM addresses " + "WHERE addresses.id = " ":param_1", lambda ctx: {'param_1': c2id} ), @@ -441,17 +458,17 @@ class RudimentaryFlushTest(UOWTest): ) def test_natural_ordering(self): - """test that unconnected items take relationship() into account regardless.""" + """test that unconnected items take relationship() + into account regardless.""" users, Address, addresses, User = (self.tables.users, - self.classes.Address, - self.tables.addresses, - self.classes.User) - + self.classes.Address, + self.tables.addresses, + self.classes.User) mapper(User, users) mapper(Address, addresses, properties={ - 'parent':relationship(User) + 'parent': relationship(User) }) sess = create_session() @@ -465,7 +482,7 @@ class RudimentaryFlushTest(UOWTest): sess.flush, CompiledSQL( "INSERT INTO users (id, name) VALUES (:id, :name)", - {'id':1, 'name':'u1'}), + {'id': 1, 'name': 'u1'}), CompiledSQL( "INSERT INTO addresses (id, user_id, email_address) " "VALUES (:id, :user_id, :email_address)", @@ -489,13 +506,13 @@ class RudimentaryFlushTest(UOWTest): ) def test_natural_selfref(self): - """test that unconnected items take relationship() into account regardless.""" + """test that unconnected items take relationship() + into account regardless.""" Node, nodes = self.classes.Node, self.tables.nodes - mapper(Node, nodes, properties={ - 'children':relationship(Node) + 'children': relationship(Node) }) sess = create_session() @@ -515,20 +532,18 @@ class RudimentaryFlushTest(UOWTest): "INSERT INTO nodes (id, parent_id, data) VALUES " "(:id, :parent_id, :data)", [{'parent_id': None, 'data': None, 'id': 1}, - {'parent_id': 1, 'data': None, 'id': 2}, - {'parent_id': 2, 'data': None, 'id': 3}] - ), + {'parent_id': 1, 'data': None, 'id': 2}, + {'parent_id': 2, 'data': None, 'id': 3}] + ), ) def test_many_to_many(self): - keywords, items, item_keywords, Keyword, Item = (self.tables.keywords, - self.tables.items, - self.tables.item_keywords, - self.classes.Keyword, - self.classes.Item) + keywords, items, item_keywords, Keyword, Item = ( + self.tables.keywords, self.tables.items, self.tables.item_keywords, + self.classes.Keyword, self.classes.Item) mapper(Item, items, properties={ - 'keywords':relationship(Keyword, secondary=item_keywords) + 'keywords': relationship(Keyword, secondary=item_keywords) }) mapper(Keyword, keywords) @@ -537,45 +552,45 @@ class RudimentaryFlushTest(UOWTest): i1 = Item(description='i1', keywords=[k1]) sess.add(i1) self.assert_sql_execution( - testing.db, - sess.flush, - AllOf( - CompiledSQL( + testing.db, + sess.flush, + AllOf( + CompiledSQL( "INSERT INTO keywords (name) VALUES (:name)", - {'name':'k1'} - ), - CompiledSQL( - "INSERT INTO items (description) VALUES (:description)", - {'description':'i1'} - ), + {'name': 'k1'} ), CompiledSQL( - "INSERT INTO item_keywords (item_id, keyword_id) " - "VALUES (:item_id, :keyword_id)", - lambda ctx:{'item_id':i1.id, 'keyword_id':k1.id} - ) + "INSERT INTO items (description) VALUES (:description)", + {'description': 'i1'} + ), + ), + CompiledSQL( + "INSERT INTO item_keywords (item_id, keyword_id) " + "VALUES (:item_id, :keyword_id)", + lambda ctx: {'item_id': i1.id, 'keyword_id': k1.id} + ) ) # test that keywords collection isn't loaded sess.expire(i1, ['keywords']) i1.description = 'i2' self.assert_sql_execution( - testing.db, - sess.flush, - CompiledSQL("UPDATE items SET description=:description " - "WHERE items.id = :items_id", - lambda ctx:{'description':'i2', 'items_id':i1.id}) + testing.db, + sess.flush, + CompiledSQL("UPDATE items SET description=:description " + "WHERE items.id = :items_id", + lambda ctx: {'description': 'i2', 'items_id': i1.id}) ) def test_m2o_flush_size(self): users, Address, addresses, User = (self.tables.users, - self.classes.Address, - self.tables.addresses, - self.classes.User) + self.classes.Address, + self.tables.addresses, + self.classes.User) mapper(User, users) mapper(Address, addresses, properties={ - 'user':relationship(User, passive_updates=True) + 'user': relationship(User, passive_updates=True) }) sess = create_session() u1 = User(name='ed') @@ -584,12 +599,12 @@ class RudimentaryFlushTest(UOWTest): def test_o2m_flush_size(self): users, Address, addresses, User = (self.tables.users, - self.classes.Address, - self.tables.addresses, - self.classes.User) + self.classes.Address, + self.tables.addresses, + self.classes.User) mapper(User, users, properties={ - 'addresses':relationship(Address), + 'addresses': relationship(Address), }) mapper(Address, addresses) @@ -600,7 +615,7 @@ class RudimentaryFlushTest(UOWTest): sess.flush() - u1.name='jack' + u1.name = 'jack' self._assert_uow_size(sess, 2) sess.flush() @@ -617,7 +632,7 @@ class RudimentaryFlushTest(UOWTest): sess = create_session() u1 = sess.query(User).first() - u1.name='ed' + u1.name = 'ed' self._assert_uow_size(sess, 2) u1.addresses @@ -625,6 +640,7 @@ class RudimentaryFlushTest(UOWTest): class SingleCycleTest(UOWTest): + def teardown(self): engines.testing_reaper.rollback_all() # mysql can't handle delete from nodes @@ -639,7 +655,7 @@ class SingleCycleTest(UOWTest): Node, nodes = self.classes.Node, self.tables.nodes mapper(Node, nodes, properties={ - 'children':relationship(Node) + 'children': relationship(Node) }) sess = create_session() @@ -649,15 +665,15 @@ class SingleCycleTest(UOWTest): sess.add(n1) self.assert_sql_execution( - testing.db, - sess.flush, + testing.db, + sess.flush, - CompiledSQL( - "INSERT INTO nodes (parent_id, data) VALUES " - "(:parent_id, :data)", - {'parent_id': None, 'data': 'n1'} - ), - AllOf( + CompiledSQL( + "INSERT INTO nodes (parent_id, data) VALUES " + "(:parent_id, :data)", + {'parent_id': None, 'data': 'n1'} + ), + AllOf( CompiledSQL( "INSERT INTO nodes (parent_id, data) VALUES " "(:parent_id, :data)", @@ -668,14 +684,14 @@ class SingleCycleTest(UOWTest): "(:parent_id, :data)", lambda ctx: {'parent_id': n1.id, 'data': 'n3'} ), - ) ) + ) def test_one_to_many_delete_all(self): Node, nodes = self.classes.Node, self.tables.nodes mapper(Node, nodes, properties={ - 'children':relationship(Node) + 'children': relationship(Node) }) sess = create_session() @@ -689,19 +705,19 @@ class SingleCycleTest(UOWTest): sess.delete(n2) sess.delete(n3) self.assert_sql_execution( - testing.db, - sess.flush, - CompiledSQL("DELETE FROM nodes WHERE nodes.id = :id", - lambda ctx:[{'id':n2.id}, {'id':n3.id}]), - CompiledSQL("DELETE FROM nodes WHERE nodes.id = :id", - lambda ctx: {'id':n1.id}) + testing.db, + sess.flush, + CompiledSQL("DELETE FROM nodes WHERE nodes.id = :id", + lambda ctx: [{'id': n2.id}, {'id': n3.id}]), + CompiledSQL("DELETE FROM nodes WHERE nodes.id = :id", + lambda ctx: {'id': n1.id}) ) def test_one_to_many_delete_parent(self): Node, nodes = self.classes.Node, self.tables.nodes mapper(Node, nodes, properties={ - 'children':relationship(Node) + 'children': relationship(Node) }) sess = create_session() @@ -713,25 +729,25 @@ class SingleCycleTest(UOWTest): sess.delete(n1) self.assert_sql_execution( - testing.db, - sess.flush, - AllOf( - CompiledSQL("UPDATE nodes SET parent_id=:parent_id " - "WHERE nodes.id = :nodes_id", - lambda ctx: {'nodes_id':n3.id, 'parent_id':None}), - CompiledSQL("UPDATE nodes SET parent_id=:parent_id " - "WHERE nodes.id = :nodes_id", - lambda ctx: {'nodes_id':n2.id, 'parent_id':None}), - ), - CompiledSQL("DELETE FROM nodes WHERE nodes.id = :id", - lambda ctx:{'id':n1.id}) - ) + testing.db, sess.flush, AllOf( + CompiledSQL( + "UPDATE nodes SET parent_id=:parent_id " + "WHERE nodes.id = :nodes_id", lambda ctx: { + 'nodes_id': n3.id, 'parent_id': None}), + CompiledSQL( + "UPDATE nodes SET parent_id=:parent_id " + "WHERE nodes.id = :nodes_id", lambda ctx: { + 'nodes_id': n2.id, 'parent_id': None}), + ), + CompiledSQL( + "DELETE FROM nodes WHERE nodes.id = :id", lambda ctx: { + 'id': n1.id})) def test_many_to_one_save(self): Node, nodes = self.classes.Node, self.tables.nodes mapper(Node, nodes, properties={ - 'parent':relationship(Node, remote_side=nodes.c.id) + 'parent': relationship(Node, remote_side=nodes.c.id) }) sess = create_session() @@ -741,15 +757,15 @@ class SingleCycleTest(UOWTest): sess.add_all([n2, n3]) self.assert_sql_execution( - testing.db, - sess.flush, + testing.db, + sess.flush, - CompiledSQL( - "INSERT INTO nodes (parent_id, data) VALUES " - "(:parent_id, :data)", - {'parent_id': None, 'data': 'n1'} - ), - AllOf( + CompiledSQL( + "INSERT INTO nodes (parent_id, data) VALUES " + "(:parent_id, :data)", + {'parent_id': None, 'data': 'n1'} + ), + AllOf( CompiledSQL( "INSERT INTO nodes (parent_id, data) VALUES " "(:parent_id, :data)", @@ -760,14 +776,14 @@ class SingleCycleTest(UOWTest): "(:parent_id, :data)", lambda ctx: {'parent_id': n1.id, 'data': 'n3'} ), - ) ) + ) def test_many_to_one_delete_all(self): Node, nodes = self.classes.Node, self.tables.nodes mapper(Node, nodes, properties={ - 'parent':relationship(Node, remote_side=nodes.c.id) + 'parent': relationship(Node, remote_side=nodes.c.id) }) sess = create_session() @@ -781,19 +797,19 @@ class SingleCycleTest(UOWTest): sess.delete(n2) sess.delete(n3) self.assert_sql_execution( - testing.db, - sess.flush, - CompiledSQL("DELETE FROM nodes WHERE nodes.id = :id", - lambda ctx:[{'id':n2.id},{'id':n3.id}]), - CompiledSQL("DELETE FROM nodes WHERE nodes.id = :id", - lambda ctx: {'id':n1.id}) + testing.db, + sess.flush, + CompiledSQL("DELETE FROM nodes WHERE nodes.id = :id", + lambda ctx: [{'id': n2.id}, {'id': n3.id}]), + CompiledSQL("DELETE FROM nodes WHERE nodes.id = :id", + lambda ctx: {'id': n1.id}) ) def test_many_to_one_set_null_unloaded(self): Node, nodes = self.classes.Node, self.tables.nodes mapper(Node, nodes, properties={ - 'parent':relationship(Node, remote_side=nodes.c.id) + 'parent': relationship(Node, remote_side=nodes.c.id) }) sess = create_session() n1 = Node(data='n1') @@ -810,7 +826,7 @@ class SingleCycleTest(UOWTest): CompiledSQL( "UPDATE nodes SET parent_id=:parent_id WHERE " "nodes.id = :nodes_id", - lambda ctx: {"parent_id":None, "nodes_id":n2.id} + lambda ctx: {"parent_id": None, "nodes_id": n2.id} ) ) @@ -818,7 +834,7 @@ class SingleCycleTest(UOWTest): Node, nodes = self.classes.Node, self.tables.nodes mapper(Node, nodes, properties={ - 'children':relationship(Node) + 'children': relationship(Node) }) sess = create_session() @@ -836,9 +852,9 @@ class SingleCycleTest(UOWTest): Node, nodes = self.classes.Node, self.tables.nodes mapper(Node, nodes, properties={ - 'children':relationship(Node, - backref=backref('parent', - remote_side=nodes.c.id)) + 'children': relationship(Node, + backref=backref('parent', + remote_side=nodes.c.id)) }) sess = create_session() @@ -857,11 +873,15 @@ class SingleCycleTest(UOWTest): def test_bidirectional_multilevel_save(self): Node, nodes = self.classes.Node, self.tables.nodes - mapper(Node, nodes, properties={ - 'children':relationship(Node, - backref=backref('parent', remote_side=nodes.c.id) - ) - }) + mapper( + Node, + nodes, + properties={ + 'children': relationship( + Node, + backref=backref( + 'parent', + remote_side=nodes.c.id))}) sess = create_session() n1 = Node(data='n1') n1.children.append(Node(data='n11')) @@ -878,37 +898,37 @@ class SingleCycleTest(UOWTest): CompiledSQL( "INSERT INTO nodes (parent_id, data) VALUES " "(:parent_id, :data)", - lambda ctx:{'parent_id':None, 'data':'n1'} + lambda ctx: {'parent_id': None, 'data': 'n1'} ), CompiledSQL( "INSERT INTO nodes (parent_id, data) VALUES " "(:parent_id, :data)", - lambda ctx:{'parent_id':n1.id, 'data':'n11'} + lambda ctx: {'parent_id': n1.id, 'data': 'n11'} ), CompiledSQL( "INSERT INTO nodes (parent_id, data) VALUES " "(:parent_id, :data)", - lambda ctx:{'parent_id':n1.id, 'data':'n12'} + lambda ctx: {'parent_id': n1.id, 'data': 'n12'} ), CompiledSQL( "INSERT INTO nodes (parent_id, data) VALUES " "(:parent_id, :data)", - lambda ctx:{'parent_id':n1.id, 'data':'n13'} + lambda ctx: {'parent_id': n1.id, 'data': 'n13'} ), CompiledSQL( "INSERT INTO nodes (parent_id, data) VALUES " "(:parent_id, :data)", - lambda ctx:{'parent_id':n12.id, 'data':'n121'} + lambda ctx: {'parent_id': n12.id, 'data': 'n121'} ), CompiledSQL( "INSERT INTO nodes (parent_id, data) VALUES " "(:parent_id, :data)", - lambda ctx:{'parent_id':n12.id, 'data':'n122'} + lambda ctx: {'parent_id': n12.id, 'data': 'n122'} ), CompiledSQL( "INSERT INTO nodes (parent_id, data) VALUES " "(:parent_id, :data)", - lambda ctx:{'parent_id':n12.id, 'data':'n123'} + lambda ctx: {'parent_id': n12.id, 'data': 'n123'} ), ) @@ -916,7 +936,7 @@ class SingleCycleTest(UOWTest): Node, nodes = self.classes.Node, self.tables.nodes mapper(Node, nodes, properties={ - 'children':relationship(Node) + 'children': relationship(Node) }) sess = create_session() n1 = Node(data='ed') @@ -925,7 +945,7 @@ class SingleCycleTest(UOWTest): sess.flush() - n1.data='jack' + n1.data = 'jack' self._assert_uow_size(sess, 2) sess.flush() @@ -942,18 +962,17 @@ class SingleCycleTest(UOWTest): sess = create_session() n1 = sess.query(Node).first() - n1.data='ed' + n1.data = 'ed' self._assert_uow_size(sess, 2) n1.children self._assert_uow_size(sess, 2) - def test_delete_unloaded_m2o(self): Node, nodes = self.classes.Node, self.tables.nodes mapper(Node, nodes, properties={ - 'parent':relationship(Node, remote_side=nodes.c.id) + 'parent': relationship(Node, remote_side=nodes.c.id) }) parent = Node() @@ -1022,35 +1041,38 @@ class SingleCycleTest(UOWTest): ) +class SingleCyclePlusAttributeTest( + fixtures.MappedTest, + testing.AssertsExecutionResults, + AssertsUOW): -class SingleCyclePlusAttributeTest(fixtures.MappedTest, - testing.AssertsExecutionResults, AssertsUOW): @classmethod def define_tables(cls, metadata): Table('nodes', metadata, - Column('id', Integer, primary_key=True, - test_needs_autoincrement=True), - Column('parent_id', Integer, ForeignKey('nodes.id')), - Column('data', String(30)) - ) + Column('id', Integer, primary_key=True, + test_needs_autoincrement=True), + Column('parent_id', Integer, ForeignKey('nodes.id')), + Column('data', String(30)) + ) Table('foobars', metadata, - Column('id', Integer, primary_key=True, - test_needs_autoincrement=True), - Column('parent_id', Integer, ForeignKey('nodes.id')), - ) + Column('id', Integer, primary_key=True, + test_needs_autoincrement=True), + Column('parent_id', Integer, ForeignKey('nodes.id')), + ) def test_flush_size(self): foobars, nodes = self.tables.foobars, self.tables.nodes class Node(fixtures.ComparableEntity): pass + class FooBar(fixtures.ComparableEntity): pass mapper(Node, nodes, properties={ - 'children':relationship(Node), - 'foobars':relationship(FooBar) + 'children': relationship(Node), + 'foobars': relationship(FooBar) }) mapper(FooBar, foobars) @@ -1070,25 +1092,30 @@ class SingleCyclePlusAttributeTest(fixtures.MappedTest, sess.flush() + class SingleCycleM2MTest(fixtures.MappedTest, - testing.AssertsExecutionResults, AssertsUOW): + testing.AssertsExecutionResults, AssertsUOW): @classmethod def define_tables(cls, metadata): - nodes = Table('nodes', metadata, - Column('id', Integer, - primary_key=True, - test_needs_autoincrement=True), - Column('data', String(30)), - Column('favorite_node_id', Integer, ForeignKey('nodes.id')) - ) + Table( + 'nodes', metadata, + Column( + 'id', Integer, primary_key=True, + test_needs_autoincrement=True), + Column( + 'data', String(30)), Column( + 'favorite_node_id', Integer, ForeignKey('nodes.id'))) - node_to_nodes =Table('node_to_nodes', metadata, - Column('left_node_id', Integer, - ForeignKey('nodes.id'),primary_key=True), - Column('right_node_id', Integer, - ForeignKey('nodes.id'),primary_key=True), - ) + Table( + 'node_to_nodes', metadata, + Column( + 'left_node_id', Integer, + ForeignKey('nodes.id'), primary_key=True), + Column( + 'right_node_id', Integer, + ForeignKey('nodes.id'), primary_key=True), + ) def test_many_to_many_one(self): nodes, node_to_nodes = self.tables.nodes, self.tables.node_to_nodes @@ -1096,14 +1123,19 @@ class SingleCycleM2MTest(fixtures.MappedTest, class Node(fixtures.ComparableEntity): pass - mapper(Node, nodes, properties={ - 'children':relationship(Node, secondary=node_to_nodes, - primaryjoin=nodes.c.id==node_to_nodes.c.left_node_id, - secondaryjoin=nodes.c.id==node_to_nodes.c.right_node_id, - backref='parents' - ), - 'favorite':relationship(Node, remote_side=nodes.c.id) - }) + mapper( + Node, + nodes, + properties={ + 'children': relationship( + Node, + secondary=node_to_nodes, + primaryjoin=nodes.c.id == node_to_nodes.c.left_node_id, + secondaryjoin=nodes.c.id == node_to_nodes.c.right_node_id, + backref='parents'), + 'favorite': relationship( + Node, + remote_side=nodes.c.id)}) sess = create_session() n1 = Node(data='n1') @@ -1128,46 +1160,46 @@ class SingleCycleM2MTest(fixtures.MappedTest, sess.flush() eq_( sess.query(node_to_nodes.c.left_node_id, - node_to_nodes.c.right_node_id).\ - order_by(node_to_nodes.c.left_node_id, - node_to_nodes.c.right_node_id).\ - all(), + node_to_nodes.c.right_node_id). + order_by(node_to_nodes.c.left_node_id, + node_to_nodes.c.right_node_id). + all(), sorted([ - (n1.id, n2.id), (n1.id, n3.id), (n1.id, n4.id), - (n2.id, n3.id), (n2.id, n5.id), - (n3.id, n5.id), (n3.id, n4.id) - ]) + (n1.id, n2.id), (n1.id, n3.id), (n1.id, n4.id), + (n2.id, n3.id), (n2.id, n5.id), + (n3.id, n5.id), (n3.id, n4.id) + ]) ) sess.delete(n1) self.assert_sql_execution( - testing.db, - sess.flush, - # this is n1.parents firing off, as it should, since - # passive_deletes is False for n1.parents - CompiledSQL( - "SELECT nodes.id AS nodes_id, nodes.data AS nodes_data, " - "nodes.favorite_node_id AS nodes_favorite_node_id FROM " - "nodes, node_to_nodes WHERE :param_1 = " - "node_to_nodes.right_node_id AND nodes.id = " - "node_to_nodes.left_node_id" , - lambda ctx:{'param_1': n1.id}, - ), - CompiledSQL( - "DELETE FROM node_to_nodes WHERE " - "node_to_nodes.left_node_id = :left_node_id AND " - "node_to_nodes.right_node_id = :right_node_id", - lambda ctx:[ - {'right_node_id': n2.id, 'left_node_id': n1.id}, - {'right_node_id': n3.id, 'left_node_id': n1.id}, - {'right_node_id': n4.id, 'left_node_id': n1.id} - ] - ), - CompiledSQL( - "DELETE FROM nodes WHERE nodes.id = :id", - lambda ctx:{'id': n1.id} - ), + testing.db, + sess.flush, + # this is n1.parents firing off, as it should, since + # passive_deletes is False for n1.parents + CompiledSQL( + "SELECT nodes.id AS nodes_id, nodes.data AS nodes_data, " + "nodes.favorite_node_id AS nodes_favorite_node_id FROM " + "nodes, node_to_nodes WHERE :param_1 = " + "node_to_nodes.right_node_id AND nodes.id = " + "node_to_nodes.left_node_id", + lambda ctx: {'param_1': n1.id}, + ), + CompiledSQL( + "DELETE FROM node_to_nodes WHERE " + "node_to_nodes.left_node_id = :left_node_id AND " + "node_to_nodes.right_node_id = :right_node_id", + lambda ctx: [ + {'right_node_id': n2.id, 'left_node_id': n1.id}, + {'right_node_id': n3.id, 'left_node_id': n1.id}, + {'right_node_id': n4.id, 'left_node_id': n1.id} + ] + ), + CompiledSQL( + "DELETE FROM nodes WHERE nodes.id = :id", + lambda ctx: {'id': n1.id} + ), ) for n in [n2, n3, n4, n5]: @@ -1185,7 +1217,7 @@ class SingleCycleM2MTest(fixtures.MappedTest, "DELETE FROM node_to_nodes WHERE node_to_nodes.left_node_id " "= :left_node_id AND node_to_nodes.right_node_id = " ":right_node_id", - lambda ctx:[ + lambda ctx: [ {'right_node_id': n5.id, 'left_node_id': n3.id}, {'right_node_id': n4.id, 'left_node_id': n3.id}, {'right_node_id': n3.id, 'left_node_id': n2.id}, @@ -1194,38 +1226,41 @@ class SingleCycleM2MTest(fixtures.MappedTest, ), CompiledSQL( "DELETE FROM nodes WHERE nodes.id = :id", - lambda ctx:[{'id': n4.id}, {'id': n5.id}] + lambda ctx: [{'id': n4.id}, {'id': n5.id}] ), CompiledSQL( "DELETE FROM nodes WHERE nodes.id = :id", - lambda ctx:[{'id': n2.id}, {'id': n3.id}] + lambda ctx: [{'id': n2.id}, {'id': n3.id}] ), ) + class RowswitchAccountingTest(fixtures.MappedTest): + @classmethod def define_tables(cls, metadata): Table('parent', metadata, - Column('id', Integer, primary_key=True), - Column('data', Integer) - ) + Column('id', Integer, primary_key=True), + Column('data', Integer) + ) Table('child', metadata, - Column('id', Integer, ForeignKey('parent.id'), primary_key=True), - Column('data', Integer) - ) + Column('id', Integer, ForeignKey('parent.id'), primary_key=True), + Column('data', Integer) + ) def _fixture(self): parent, child = self.tables.parent, self.tables.child class Parent(fixtures.BasicEntity): pass + class Child(fixtures.BasicEntity): pass mapper(Parent, parent, properties={ - 'child':relationship(Child, uselist=False, - cascade="all, delete-orphan", - backref="parent") + 'child': relationship(Child, uselist=False, + cascade="all, delete-orphan", + backref="parent") }) mapper(Child, child) return Parent, Child @@ -1274,8 +1309,10 @@ class RowswitchAccountingTest(fixtures.MappedTest): eq_(sess.scalar(self.tables.parent.count()), 0) + class RowswitchM2OTest(fixtures.MappedTest): # tests for #3060 and related issues + @classmethod def define_tables(cls, metadata): Table( @@ -1299,17 +1336,18 @@ class RowswitchM2OTest(fixtures.MappedTest): class A(fixtures.BasicEntity): pass + class B(fixtures.BasicEntity): pass + class C(fixtures.BasicEntity): pass - mapper(A, a, properties={ - 'bs': relationship(B, cascade="all, delete-orphan") + 'bs': relationship(B, cascade="all, delete-orphan") }) mapper(B, b, properties={ - 'c': relationship(C) + 'c': relationship(C) }) mapper(C, c) return A, B, C @@ -1391,29 +1429,31 @@ class RowswitchM2OTest(fixtures.MappedTest): class BasicStaleChecksTest(fixtures.MappedTest): + @classmethod def define_tables(cls, metadata): Table('parent', metadata, - Column('id', Integer, primary_key=True), - Column('data', Integer) - ) + Column('id', Integer, primary_key=True), + Column('data', Integer) + ) Table('child', metadata, - Column('id', Integer, ForeignKey('parent.id'), primary_key=True), - Column('data', Integer) - ) + Column('id', Integer, ForeignKey('parent.id'), primary_key=True), + Column('data', Integer) + ) def _fixture(self, confirm_deleted_rows=True): parent, child = self.tables.parent, self.tables.child class Parent(fixtures.BasicEntity): pass + class Child(fixtures.BasicEntity): pass mapper(Parent, parent, properties={ - 'child':relationship(Child, uselist=False, - cascade="all, delete-orphan", - backref="parent"), + 'child': relationship(Child, uselist=False, + cascade="all, delete-orphan", + backref="parent"), }, confirm_deleted_rows=confirm_deleted_rows) mapper(Child, child) return Parent, Child @@ -1431,7 +1471,7 @@ class BasicStaleChecksTest(fixtures.MappedTest): assert_raises_message( orm_exc.StaleDataError, "UPDATE statement on table 'parent' expected to " - "update 1 row\(s\); 0 were matched.", + "update 1 row\(s\); 0 were matched.", sess.flush ) @@ -1451,7 +1491,7 @@ class BasicStaleChecksTest(fixtures.MappedTest): assert_raises_message( exc.SAWarning, "DELETE statement on table 'parent' expected to " - "delete 2 row\(s\); 0 were matched.", + "delete 2 row\(s\); 0 were matched.", sess.flush ) @@ -1471,14 +1511,15 @@ class BasicStaleChecksTest(fixtures.MappedTest): class BatchInsertsTest(fixtures.MappedTest, testing.AssertsExecutionResults): + @classmethod def define_tables(cls, metadata): Table('t', metadata, - Column('id', Integer, primary_key=True, - test_needs_autoincrement=True), - Column('data', String(50)), - Column('def_', String(50), server_default='def1') - ) + Column('id', Integer, primary_key=True, + test_needs_autoincrement=True), + Column('data', String(50)), + Column('def_', String(50), server_default='def1') + ) def test_batch_interaction(self): """test batching groups same-structured, primary @@ -1532,8 +1573,8 @@ class BatchInsertsTest(fixtures.MappedTest, testing.AssertsExecutionResults): ), CompiledSQL( "INSERT INTO t (id, data, def_) VALUES (:id, :data, :def_)", - [{'data': 't9', 'id': 9, 'def_':'def2'}, - {'data': 't10', 'id': 10, 'def_':'def3'}] + [{'data': 't9', 'id': 9, 'def_': 'def2'}, + {'data': 't10', 'id': 10, 'def_': 'def3'}] ), CompiledSQL( "INSERT INTO t (id, data) VALUES (:id, :data)", @@ -1541,126 +1582,129 @@ class BatchInsertsTest(fixtures.MappedTest, testing.AssertsExecutionResults): ), ) + class LoadersUsingCommittedTest(UOWTest): - """Test that events which occur within a flush() - get the same attribute loading behavior as on the outside - of the flush, and that the unit of work itself uses the - "committed" version of primary/foreign key attributes - when loading a collection for historical purposes (this typically - has importance for when primary key values change). + + """Test that events which occur within a flush() + get the same attribute loading behavior as on the outside + of the flush, and that the unit of work itself uses the + "committed" version of primary/foreign key attributes + when loading a collection for historical purposes (this typically + has importance for when primary key values change). + + """ + + def _mapper_setup(self, passive_updates=True): + users, Address, addresses, User = (self.tables.users, + self.classes.Address, + self.tables.addresses, + self.classes.User) + + mapper(User, users, properties={ + 'addresses': relationship(Address, + order_by=addresses.c.email_address, + passive_updates=passive_updates, + backref='user') + }) + mapper(Address, addresses) + return create_session(autocommit=False) + + def test_before_update_m2o(self): + """Expect normal many to one attribute load behavior + (should not get committed value) + from within public 'before_update' event""" + sess = self._mapper_setup() + + Address, User = self.classes.Address, self.classes.User + + def before_update(mapper, connection, target): + # if get committed is used to find target.user, then + # it will be still be u1 instead of u2 + assert target.user.id == target.user_id == u2.id + from sqlalchemy import event + event.listen(Address, 'before_update', before_update) + + a1 = Address(email_address='a1') + u1 = User(name='u1', addresses=[a1]) + sess.add(u1) + + u2 = User(name='u2') + sess.add(u2) + sess.commit() + + sess.expunge_all() + # lookup an address and move it to the other user + a1 = sess.query(Address).get(a1.id) + + # move address to another user's fk + assert a1.user_id == u1.id + a1.user_id = u2.id + + sess.flush() + + def test_before_update_o2m_passive(self): + """Expect normal one to many attribute load behavior + (should not get committed value) + from within public 'before_update' event""" + self._test_before_update_o2m(True) + + def test_before_update_o2m_notpassive(self): + """Expect normal one to many attribute load behavior + (should not get committed value) + from within public 'before_update' event with + passive_updates=False """ + self._test_before_update_o2m(False) - def _mapper_setup(self, passive_updates=True): - users, Address, addresses, User = (self.tables.users, - self.classes.Address, - self.tables.addresses, - self.classes.User) + def _test_before_update_o2m(self, passive_updates): + sess = self._mapper_setup(passive_updates=passive_updates) - mapper(User, users, properties={ - 'addresses': relationship(Address, - order_by=addresses.c.email_address, - passive_updates=passive_updates, - backref='user') - }) - mapper(Address, addresses) - return create_session(autocommit=False) + Address, User = self.classes.Address, self.classes.User - def test_before_update_m2o(self): - """Expect normal many to one attribute load behavior - (should not get committed value) - from within public 'before_update' event""" - sess = self._mapper_setup() + class AvoidReferencialError(Exception): - Address, User = self.classes.Address, self.classes.User - - def before_update(mapper, connection, target): - # if get committed is used to find target.user, then - # it will be still be u1 instead of u2 - assert target.user.id == target.user_id == u2.id - from sqlalchemy import event - event.listen(Address, 'before_update', before_update) - - a1 = Address(email_address='a1') - u1 = User(name='u1', addresses=[a1]) - sess.add(u1) - - u2 = User(name='u2') - sess.add(u2) - sess.commit() - - sess.expunge_all() - # lookup an address and move it to the other user - a1 = sess.query(Address).get(a1.id) - - # move address to another user's fk - assert a1.user_id == u1.id - a1.user_id = u2.id - - sess.flush() - - def test_before_update_o2m_passive(self): - """Expect normal one to many attribute load behavior - (should not get committed value) - from within public 'before_update' event""" - self._test_before_update_o2m(True) - - def test_before_update_o2m_notpassive(self): - """Expect normal one to many attribute load behavior - (should not get committed value) - from within public 'before_update' event with - passive_updates=False + """the test here would require ON UPDATE CASCADE on FKs + for the flush to fully succeed; this exception is used + to cancel the flush before we get that far. """ - self._test_before_update_o2m(False) - def _test_before_update_o2m(self, passive_updates): - sess = self._mapper_setup(passive_updates=passive_updates) + def before_update(mapper, connection, target): + if passive_updates: + # we shouldn't be using committed value. + # so, having switched target's primary key, + # we expect no related items in the collection + # since we are using passive_updates + # this is a behavior change since #2350 + assert 'addresses' not in target.__dict__ + eq_(target.addresses, []) + else: + # in contrast with passive_updates=True, + # here we expect the orm to have looked up the addresses + # with the committed value (it needs to in order to + # update the foreign keys). So we expect addresses + # collection to move with the user, + # (just like they will be after the update) - Address, User = self.classes.Address, self.classes.User + # collection is already loaded + assert 'addresses' in target.__dict__ + eq_([a.id for a in target.addresses], + [a.id for a in [a1, a2]]) + raise AvoidReferencialError() + from sqlalchemy import event + event.listen(User, 'before_update', before_update) - class AvoidReferencialError(Exception): - """the test here would require ON UPDATE CASCADE on FKs - for the flush to fully succeed; this exception is used - to cancel the flush before we get that far. + a1 = Address(email_address='jack1') + a2 = Address(email_address='jack2') + u1 = User(id=1, name='jack', addresses=[a1, a2]) + sess.add(u1) + sess.commit() - """ - - def before_update(mapper, connection, target): - if passive_updates: - # we shouldn't be using committed value. - # so, having switched target's primary key, - # we expect no related items in the collection - # since we are using passive_updates - # this is a behavior change since #2350 - assert 'addresses' not in target.__dict__ - eq_(target.addresses, []) - else: - # in contrast with passive_updates=True, - # here we expect the orm to have looked up the addresses - # with the committed value (it needs to in order to - # update the foreign keys). So we expect addresses - # collection to move with the user, - # (just like they will be after the update) - - # collection is already loaded - assert 'addresses' in target.__dict__ - eq_([a.id for a in target.addresses], - [a.id for a in [a1, a2]]) - raise AvoidReferencialError() - from sqlalchemy import event - event.listen(User, 'before_update', before_update) - - a1 = Address(email_address='jack1') - a2 = Address(email_address='jack2') - u1 = User(id=1, name='jack', addresses=[a1, a2]) - sess.add(u1) - sess.commit() - - sess.expunge_all() - u1 = sess.query(User).get(u1.id) - u1.id = 2 - try: - sess.flush() - except AvoidReferencialError: - pass + sess.expunge_all() + u1 = sess.query(User).get(u1.id) + u1.id = 2 + try: + sess.flush() + except AvoidReferencialError: + pass From bc509dd50d7b65e35412e2be67bd37a6c19e7119 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 14 Aug 2014 20:47:49 -0400 Subject: [PATCH 15/55] - UPDATE statements can now be batched within an ORM flush into more performant executemany() call, similarly to how INSERT statements can be batched; this will be invoked within flush to the degree that subsequent UPDATE statements for the same mapping and table involve the identical columns within the VALUES clause, as well as that no VALUES-level SQL expressions are embedded. - some other inlinings within persistence.py --- doc/build/changelog/changelog_10.rst | 11 ++++ lib/sqlalchemy/orm/persistence.py | 95 +++++++++++++++++----------- test/orm/test_unitofwork.py | 9 +-- test/orm/test_unitofworkv2.py | 31 ++++----- 4 files changed, 87 insertions(+), 59 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index fb14279ac1..439d02c479 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -16,6 +16,17 @@ .. changelog:: :version: 1.0.0 + .. change:: + :tags: orm, feature + + UPDATE statements can now be batched within an ORM flush + into more performant executemany() call, similarly to how INSERT + statements can be batched; this will be invoked within flush + to the degree that subsequent UPDATE statements for the + same mapping and table involve the identical columns within the + VALUES clause, as well as that no VALUES-level SQL expressions + are embedded. + .. change:: :tags: engine, bug :tickets: 3163 diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index 17ce2e6247..9d39c39b07 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -248,9 +248,10 @@ def _collect_insert_commands(base_mapper, uowtransaction, table, has_all_pks = True has_all_defaults = True + has_version_id_generator = mapper.version_id_generator is not False \ + and mapper.version_id_col is not None for col in mapper._cols_by_table[table]: - if col is mapper.version_id_col and \ - mapper.version_id_generator is not False: + if has_version_id_generator and col is mapper.version_id_col: val = mapper.version_id_generator(None) params[col.key] = val else: @@ -305,6 +306,7 @@ def _collect_update_commands(base_mapper, uowtransaction, value_params = {} hasdata = hasnull = False + for col in mapper._cols_by_table[table]: if col is mapper.version_id_col: params[col._label] = \ @@ -341,6 +343,7 @@ def _collect_update_commands(base_mapper, uowtransaction, prop = mapper._columntoproperty[col] history = state.manager[prop.key].impl.get_history( state, state_dict, + attributes.PASSIVE_OFF if col in pks else attributes.PASSIVE_NO_INITIALIZE) if history.added: if isinstance(history.added[0], @@ -381,8 +384,7 @@ def _collect_update_commands(base_mapper, uowtransaction, else: hasdata = True elif col in pks: - value = state.manager[prop.key].impl.get( - state, state_dict) + value = history.unchanged[0] if value is None: hasnull = True params[col._label] = value @@ -500,41 +502,63 @@ def _emit_update_statements(base_mapper, uowtransaction, statement = base_mapper._memo(('update', table), update_stmt) - rows = 0 - for state, state_dict, params, mapper, \ - connection, value_params in update: + for (connection, paramkeys, hasvalue), \ + records in groupby( + update, + lambda rec: ( + rec[4], + tuple(sorted(rec[2])), + bool(rec[5])) + ): - if value_params: - c = connection.execute( - statement.values(value_params), - params) + rows = 0 + records = list(records) + if hasvalue: + for state, state_dict, params, mapper, \ + connection, value_params in records: + c = connection.execute( + statement.values(value_params), + params) + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params) + rows += c.rowcount else: + multiparams = [rec[2] for rec in records] c = cached_connections[connection].\ - execute(statement, params) + execute(statement, multiparams) - _postfetch( - mapper, - uowtransaction, - table, - state, - state_dict, - c, - c.context.compiled_parameters[0], - value_params) - rows += c.rowcount + rows += c.rowcount + for state, state_dict, params, mapper, \ + connection, value_params in records: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params) - if connection.dialect.supports_sane_rowcount: - if rows != len(update): - raise orm_exc.StaleDataError( - "UPDATE statement on table '%s' expected to " - "update %d row(s); %d were matched." % - (table.description, len(update), rows)) + if connection.dialect.supports_sane_rowcount: + if rows != len(records): + raise orm_exc.StaleDataError( + "UPDATE statement on table '%s' expected to " + "update %d row(s); %d were matched." % + (table.description, len(records), rows)) - elif needs_version_id: - util.warn("Dialect %s does not support updated rowcount " - "- versioning cannot be verified." % - c.dialect.dialect_description, - stacklevel=12) + elif needs_version_id: + util.warn("Dialect %s does not support updated rowcount " + "- versioning cannot be verified." % + c.dialect.dialect_description, + stacklevel=12) def _emit_insert_statements(base_mapper, uowtransaction, @@ -833,15 +857,12 @@ def _connections_for_states(base_mapper, uowtransaction, states): connection_callable = \ uowtransaction.session.connection_callable else: - connection = None + connection = uowtransaction.transaction.connection(base_mapper) connection_callable = None for state in _sort_states(states): if connection_callable: connection = connection_callable(base_mapper, state.obj()) - elif not connection: - connection = uowtransaction.transaction.connection( - base_mapper) mapper = _state_mapper(state) diff --git a/test/orm/test_unitofwork.py b/test/orm/test_unitofwork.py index 6eb7632130..a54097b03e 100644 --- a/test/orm/test_unitofwork.py +++ b/test/orm/test_unitofwork.py @@ -1126,11 +1126,12 @@ class OneToManyTest(_fixtures.FixtureTest): ("UPDATE addresses SET user_id=:user_id " "WHERE addresses.id = :addresses_id", - {'user_id': None, 'addresses_id': a1.id}), + [ + {'user_id': None, 'addresses_id': a1.id}, + {'user_id': u1.id, 'addresses_id': a3.id} + ]), - ("UPDATE addresses SET user_id=:user_id " - "WHERE addresses.id = :addresses_id", - {'user_id': u1.id, 'addresses_id': a3.id})]) + ]) def test_child_move(self): """Moving a child from one parent to another, with a delete. diff --git a/test/orm/test_unitofworkv2.py b/test/orm/test_unitofworkv2.py index 9c9296786e..c643e6a870 100644 --- a/test/orm/test_unitofworkv2.py +++ b/test/orm/test_unitofworkv2.py @@ -131,12 +131,10 @@ class RudimentaryFlushTest(UOWTest): CompiledSQL( "UPDATE addresses SET user_id=:user_id WHERE " "addresses.id = :addresses_id", - lambda ctx: [{'addresses_id': a1.id, 'user_id': None}] - ), - CompiledSQL( - "UPDATE addresses SET user_id=:user_id WHERE " - "addresses.id = :addresses_id", - lambda ctx: [{'addresses_id': a2.id, 'user_id': None}] + lambda ctx: [ + {'addresses_id': a1.id, 'user_id': None}, + {'addresses_id': a2.id, 'user_id': None} + ] ), CompiledSQL( "DELETE FROM users WHERE users.id = :id", @@ -240,12 +238,10 @@ class RudimentaryFlushTest(UOWTest): CompiledSQL( "UPDATE addresses SET user_id=:user_id WHERE " "addresses.id = :addresses_id", - lambda ctx: [{'addresses_id': a1.id, 'user_id': None}] - ), - CompiledSQL( - "UPDATE addresses SET user_id=:user_id WHERE " - "addresses.id = :addresses_id", - lambda ctx: [{'addresses_id': a2.id, 'user_id': None}] + lambda ctx: [ + {'addresses_id': a1.id, 'user_id': None}, + {'addresses_id': a2.id, 'user_id': None} + ] ), CompiledSQL( "DELETE FROM users WHERE users.id = :id", @@ -732,12 +728,11 @@ class SingleCycleTest(UOWTest): testing.db, sess.flush, AllOf( CompiledSQL( "UPDATE nodes SET parent_id=:parent_id " - "WHERE nodes.id = :nodes_id", lambda ctx: { - 'nodes_id': n3.id, 'parent_id': None}), - CompiledSQL( - "UPDATE nodes SET parent_id=:parent_id " - "WHERE nodes.id = :nodes_id", lambda ctx: { - 'nodes_id': n2.id, 'parent_id': None}), + "WHERE nodes.id = :nodes_id", lambda ctx: [ + {'nodes_id': n3.id, 'parent_id': None}, + {'nodes_id': n2.id, 'parent_id': None} + ] + ) ), CompiledSQL( "DELETE FROM nodes WHERE nodes.id = :id", lambda ctx: { From b952d892d690ec808829aede84769b2bf089f94d Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 15 Aug 2014 00:19:32 -0400 Subject: [PATCH 16/55] - modify how class state is tracked here as it seems like things are a little more crazy under xdist mode --- lib/sqlalchemy/testing/plugin/pytestplugin.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/sqlalchemy/testing/plugin/pytestplugin.py b/lib/sqlalchemy/testing/plugin/pytestplugin.py index fd06163276..f4c9efd554 100644 --- a/lib/sqlalchemy/testing/plugin/pytestplugin.py +++ b/lib/sqlalchemy/testing/plugin/pytestplugin.py @@ -115,7 +115,6 @@ def pytest_pycollect_makeitem(collector, name, obj): _current_class = None - def pytest_runtest_setup(item): # here we seem to get called only based on what we collected # in pytest_collection_modifyitems. So to do class-based stuff @@ -126,16 +125,18 @@ def pytest_runtest_setup(item): return # ... so we're doing a little dance here to figure it out... - if item.parent.parent is not _current_class: - + if _current_class is None: class_setup(item.parent.parent) _current_class = item.parent.parent # this is needed for the class-level, to ensure that the # teardown runs after the class is completed with its own # class-level teardown... - item.parent.parent.addfinalizer( - lambda: class_teardown(item.parent.parent)) + def finalize(): + global _current_class + class_teardown(item.parent.parent) + _current_class = None + item.parent.parent.addfinalizer(finalize) test_setup(item) From b0411e80df13d347104a60c512aeb18b6479bb12 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 15 Aug 2014 00:19:57 -0400 Subject: [PATCH 17/55] - other test fixes --- lib/sqlalchemy/engine/base.py | 2 +- test/orm/test_naturalpks.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index 65753b6dcf..3728b59fd0 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -799,7 +799,7 @@ class Connection(Connectable): if distilled_params: # note this is usually dict but we support RowProxy # as well; but dict.keys() as an iterator is OK - keys = distilled_params[0].keys() + keys = list(distilled_params[0].keys()) else: keys = [] diff --git a/test/orm/test_naturalpks.py b/test/orm/test_naturalpks.py index 53b661a49b..a4e982f84b 100644 --- a/test/orm/test_naturalpks.py +++ b/test/orm/test_naturalpks.py @@ -184,7 +184,7 @@ class NaturalPKTest(fixtures.MappedTest): if not passive_updates: # test passive_updates=False; #load addresses, update user, update 2 addresses - self.assert_sql_count(testing.db, go, 4) + self.assert_sql_count(testing.db, go, 3) else: # test passive_updates=True; update user self.assert_sql_count(testing.db, go, 1) @@ -239,7 +239,7 @@ class NaturalPKTest(fixtures.MappedTest): def go(): sess.flush() - self.assert_sql_count(testing.db, go, 3) + self.assert_sql_count(testing.db, go, 2) def _test_manytoone(self, passive_updates): users, Address, addresses, User = (self.tables.users, @@ -270,7 +270,7 @@ class NaturalPKTest(fixtures.MappedTest): if passive_updates: self.assert_sql_count(testing.db, go, 1) else: - self.assert_sql_count(testing.db, go, 3) + self.assert_sql_count(testing.db, go, 2) def go(): sess.flush() @@ -366,7 +366,8 @@ class NaturalPKTest(fixtures.MappedTest): if passive_updates: self.assert_sql_count(testing.db, go, 1) else: - self.assert_sql_count(testing.db, go, 3) + # two updates bundled + self.assert_sql_count(testing.db, go, 2) eq_([Address(username='ed'), Address(username='ed')], [ad1, ad2]) sess.expunge_all() eq_( @@ -383,7 +384,8 @@ class NaturalPKTest(fixtures.MappedTest): if passive_updates: self.assert_sql_count(testing.db, go, 1) else: - self.assert_sql_count(testing.db, go, 3) + # two updates bundled + self.assert_sql_count(testing.db, go, 2) sess.expunge_all() eq_( [Address(username='fred'), Address(username='fred')], @@ -789,8 +791,8 @@ class NonPKCascadeTest(fixtures.MappedTest): sess.flush() if not passive_updates: # test passive_updates=False; load addresses, - # update user, update 2 addresses - self.assert_sql_count(testing.db, go, 4) + # update user, update 2 addresses (in one executemany) + self.assert_sql_count(testing.db, go, 3) else: # test passive_updates=True; update user self.assert_sql_count(testing.db, go, 1) From d768ec2c266ec462a8ff0b782516c494c451f2db Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 15 Aug 2014 14:27:12 -0400 Subject: [PATCH 18/55] - don't add the parent attach event within _on_table_attach if we already have a table; this prevents reentrant calls and we aren't supporting columns/etc being moved around between different parents --- lib/sqlalchemy/sql/schema.py | 3 ++- test/sql/test_metadata.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/sqlalchemy/sql/schema.py b/lib/sqlalchemy/sql/schema.py index 8099dca759..c8e815d244 100644 --- a/lib/sqlalchemy/sql/schema.py +++ b/lib/sqlalchemy/sql/schema.py @@ -1269,7 +1269,8 @@ class Column(SchemaItem, ColumnClause): def _on_table_attach(self, fn): if self.table is not None: fn(self, self.table) - event.listen(self, 'after_parent_attach', fn) + else: + event.listen(self, 'after_parent_attach', fn) def copy(self, **kw): """Create a copy of this ``Column``, unitialized. diff --git a/test/sql/test_metadata.py b/test/sql/test_metadata.py index ff2755ab1d..4a484dbac8 100644 --- a/test/sql/test_metadata.py +++ b/test/sql/test_metadata.py @@ -349,6 +349,20 @@ class MetaDataTest(fixtures.TestBase, ComparesTables): assert t.c.x.default is s2 assert m1._sequences['x_seq'] is s2 + + def test_sequence_attach_to_table(self): + m1 = MetaData() + s1 = Sequence("s") + t = Table('a', m1, Column('x', Integer, s1)) + assert s1.metadata is m1 + + def test_sequence_attach_to_existing_table(self): + m1 = MetaData() + s1 = Sequence("s") + t = Table('a', m1, Column('x', Integer)) + t.c.x._init_items(s1) + assert s1.metadata is m1 + def test_pickle_metadata_sequence_implicit(self): m1 = MetaData() Table('a', m1, From 961217aa923562c21a0113fae41d6841276e6ca5 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 15 Aug 2014 14:38:33 -0400 Subject: [PATCH 19/55] - clean up provision and keep sqlite on memory DBs if thats what we start with --- lib/sqlalchemy/testing/plugin/provision.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/sqlalchemy/testing/plugin/provision.py b/lib/sqlalchemy/testing/plugin/provision.py index baec8a299f..c6b9030f57 100644 --- a/lib/sqlalchemy/testing/plugin/provision.py +++ b/lib/sqlalchemy/testing/plugin/provision.py @@ -36,14 +36,8 @@ class register(object): def create_follower_db(follower_ident): for cfg in _configs_for_db_operation(): - url = cfg.db.url - backend = url.get_backend_name() _create_db(cfg, cfg.db, follower_ident) - new_url = sa_url.make_url(str(url)) - - new_url.database = follower_ident - def configure_follower(follower_ident): for cfg in config.Config.all_configs(): @@ -63,7 +57,6 @@ def setup_config(db_url, db_opts, options, file_config, follower_ident): def drop_follower_db(follower_ident): for cfg in _configs_for_db_operation(): - url = cfg.db.url _drop_db(cfg, cfg.db, follower_ident) @@ -110,9 +103,13 @@ def _follower_url_from_main(url, ident): return url -#@_follower_url_from_main.for_db("sqlite") -#def _sqlite_follower_url_from_main(url, ident): -# return sa_url.make_url("sqlite:///%s.db" % ident) +@_follower_url_from_main.for_db("sqlite") +def _sqlite_follower_url_from_main(url, ident): + url = sa_url.make_url(url) + if not url.database or url.database == ':memory:': + return url + else: + return sa_url.make_url("sqlite:///%s.db" % ident) @_create_db.for_db("postgresql") From 5a68f856daee59caf4c9da7d06880eada9d70302 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 15 Aug 2014 14:57:29 -0400 Subject: [PATCH 20/55] - TIL that dict.keys() in py3K is not an iterator, it is an iterable view. So copy collections.OrderedDict and use MutableMapping to set up keys, items, values on our own OrderedDict. Conflicts: lib/sqlalchemy/engine/base.py --- lib/sqlalchemy/engine/base.py | 4 +-- lib/sqlalchemy/util/_collections.py | 46 ++++------------------------- 2 files changed, 7 insertions(+), 43 deletions(-) diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index 3728b59fd0..d2cc8890fd 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -798,8 +798,8 @@ class Connection(Connectable): distilled_params = _distill_params(multiparams, params) if distilled_params: # note this is usually dict but we support RowProxy - # as well; but dict.keys() as an iterator is OK - keys = list(distilled_params[0].keys()) + # as well; but dict.keys() as an iterable is OK + keys = distilled_params[0].keys() else: keys = [] diff --git a/lib/sqlalchemy/util/_collections.py b/lib/sqlalchemy/util/_collections.py index 5236d0120f..fa27897a11 100644 --- a/lib/sqlalchemy/util/_collections.py +++ b/lib/sqlalchemy/util/_collections.py @@ -13,6 +13,7 @@ import operator from .compat import threading, itertools_filterfalse from . import py2k import types +from collections import MutableMapping EMPTY_SET = frozenset() @@ -264,13 +265,11 @@ class OrderedDict(dict): def __iter__(self): return iter(self._list) + keys = MutableMapping.keys + values = MutableMapping.values + items = MutableMapping.items + if py2k: - def values(self): - return [self[key] for key in self._list] - - def keys(self): - return self._list - def itervalues(self): return iter([self[key] for key in self._list]) @@ -280,41 +279,6 @@ class OrderedDict(dict): def iteritems(self): return iter(self.items()) - def items(self): - return [(key, self[key]) for key in self._list] - else: - def values(self): - # return (self[key] for key in self) - return (self[key] for key in self._list) - - def keys(self): - # return iter(self) - return iter(self._list) - - def items(self): - # return ((key, self[key]) for key in self) - return ((key, self[key]) for key in self._list) - - _debug_iter = False - if _debug_iter: - # normally disabled to reduce function call - # overhead - def __iter__(self): - len_ = len(self._list) - for item in self._list: - yield item - assert len_ == len(self._list), \ - "Dictionary changed size during iteration" - - def values(self): - return (self[key] for key in self) - - def keys(self): - return iter(self) - - def items(self): - return ((key, self[key]) for key in self) - def __setitem__(self, key, object): if key not in self: try: From 652a24f0303b9bb0e7a326b05709d7660793f90b Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 15 Aug 2014 15:13:13 -0400 Subject: [PATCH 21/55] - The :class:`.IdentityMap` exposed from :class:`.Session.identity` now returns lists for ``items()`` and ``values()`` in Py3K. Early porting to Py3K here had these returning iterators, when they technically should be "iterable views"..for now, lists are OK. --- doc/build/changelog/changelog_10.rst | 8 +++++++ lib/sqlalchemy/orm/identity.py | 34 +++++++--------------------- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 439d02c479..fb639ddf73 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -16,6 +16,14 @@ .. changelog:: :version: 1.0.0 + .. change:: + :tags: bug, orm, py3k + + The :class:`.IdentityMap` exposed from :class:`.Session.identity` + now returns lists for ``items()`` and ``values()`` in Py3K. + Early porting to Py3K here had these returning iterators, when + they technically should be "iterable views"..for now, lists are OK. + .. change:: :tags: orm, feature diff --git a/lib/sqlalchemy/orm/identity.py b/lib/sqlalchemy/orm/identity.py index d9cdd791f1..4425fc3a6e 100644 --- a/lib/sqlalchemy/orm/identity.py +++ b/lib/sqlalchemy/orm/identity.py @@ -150,7 +150,7 @@ class WeakInstanceDict(IdentityMap): return default return o - def _items(self): + def items(self): values = self.all_states() result = [] for state in values: @@ -159,7 +159,7 @@ class WeakInstanceDict(IdentityMap): result.append((state.key, value)) return result - def _values(self): + def values(self): values = self.all_states() result = [] for state in values: @@ -169,9 +169,10 @@ class WeakInstanceDict(IdentityMap): return result + def __iter__(self): + return iter(self.keys()) + if util.py2k: - items = _items - values = _values def iteritems(self): return iter(self.items()) @@ -179,24 +180,8 @@ class WeakInstanceDict(IdentityMap): def itervalues(self): return iter(self.values()) - def __iter__(self): - return iter(self.keys()) - - else: - def items(self): - return iter(self._items()) - - def values(self): - return iter(self._values()) - - def __iter__(self): - return self.keys() - def all_states(self): - if util.py2k: - return self._dict.values() - else: - return list(self._dict.values()) + return self._dict.values() def discard(self, state): if state.key in self._dict: @@ -217,11 +202,8 @@ class StrongInstanceDict(IdentityMap): def iteritems(self): return self._dict.iteritems() - def __iter__(self): - return iter(self.keys()) - else: - def __iter__(self): - return self.keys() + def __iter__(self): + return iter(self.dict_) def __getitem__(self, key): return self._dict[key] From 239464a98a4b1a2d6e5e39d998911ec7a8fe3666 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 15 Aug 2014 18:33:42 -0400 Subject: [PATCH 22/55] - port the _collect_insert_commands optimizations from ticket_3100 --- lib/sqlalchemy/orm/mapper.py | 35 +++++++++++++++ lib/sqlalchemy/orm/persistence.py | 75 +++++++++++++++---------------- 2 files changed, 72 insertions(+), 38 deletions(-) diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index 06ec2bf144..fc15769cd4 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -1892,6 +1892,41 @@ class Mapper(InspectionAttr): """ + @_memoized_configured_property + def _col_to_propkey(self): + return dict( + ( + table, + [ + (col, self._columntoproperty[col].key) + for col in columns + ] + ) + for table, columns in self._cols_by_table.items() + ) + + @_memoized_configured_property + def _pk_keys_by_table(self): + return dict( + ( + table, + frozenset([col.key for col in pks]) + ) + for table, pks in self._pks_by_table.items() + ) + + @_memoized_configured_property + def _server_default_cols(self): + return dict( + ( + table, + frozenset([ + col for col in columns + if col.server_default is not None]) + ) + for table, columns in self._cols_by_table.items() + ) + @property def selectable(self): """The :func:`.select` construct this :class:`.Mapper` selects from diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index 9d39c39b07..b7283e48e2 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -18,7 +18,7 @@ import operator from itertools import groupby from .. import sql, util, exc as sa_exc, schema from . import attributes, sync, exc as orm_exc, evaluator -from .base import _state_mapper, state_str, _attr_as_key +from .base import state_str, _attr_as_key from ..sql import expression from . import loading @@ -141,6 +141,7 @@ def _organize_states_for_save(base_mapper, states, uowtransaction): states): has_identity = bool(state.key) + instance_key = state.key or mapper._identity_key_from_state(state) row_switch = None @@ -183,12 +184,12 @@ def _organize_states_for_save(base_mapper, states, uowtransaction): if not has_identity and not row_switch: states_to_insert.append( (state, dict_, mapper, connection, - has_identity, instance_key, row_switch) + has_identity, row_switch) ) else: states_to_update.append( (state, dict_, mapper, connection, - has_identity, instance_key, row_switch) + has_identity, row_switch) ) return states_to_insert, states_to_update @@ -237,43 +238,41 @@ def _collect_insert_commands(base_mapper, uowtransaction, table, """ insert = [] for state, state_dict, mapper, connection, has_identity, \ - instance_key, row_switch in states_to_insert: + row_switch in states_to_insert: + if table not in mapper._pks_by_table: continue - pks = mapper._pks_by_table[table] - params = {} value_params = {} - - has_all_pks = True - has_all_defaults = True - has_version_id_generator = mapper.version_id_generator is not False \ - and mapper.version_id_col is not None - for col in mapper._cols_by_table[table]: - if has_version_id_generator and col is mapper.version_id_col: - val = mapper.version_id_generator(None) - params[col.key] = val - else: - # pull straight from the dict for - # pending objects - prop = mapper._columntoproperty[col] - value = state_dict.get(prop.key, None) - - if value is None: - if col in pks: - has_all_pks = False - elif col.default is None and \ - col.server_default is None: - params[col.key] = value - elif col.server_default is not None and \ - mapper.base_mapper.eager_defaults: - has_all_defaults = False - - elif isinstance(value, sql.ClauseElement): - value_params[col] = value - else: + for col, propkey in mapper._col_to_propkey[table]: + if propkey in state_dict: + value = state_dict[propkey] + if isinstance(value, sql.ClauseElement): + value_params[col.key] = value + elif value is not None or ( + not col.primary_key and + not col.server_default and + not col.default): params[col.key] = value + else: + if not col.server_default \ + and not col.default and not col.primary_key: + params[col.key] = None + + has_all_pks = mapper._pk_keys_by_table[table].issubset(params) + + if base_mapper.eager_defaults: + has_all_defaults = mapper._server_default_cols[table].\ + issubset(params) + else: + has_all_defaults = True + + if mapper.version_id_generator is not False \ + and mapper.version_id_col is not None and \ + mapper.version_id_col in mapper._cols_by_table[table]: + params[mapper.version_id_col.key] = \ + mapper.version_id_generator(None) insert.append((state, state_dict, params, mapper, connection, value_params, has_all_pks, @@ -296,7 +295,7 @@ def _collect_update_commands(base_mapper, uowtransaction, update = [] for state, state_dict, mapper, connection, has_identity, \ - instance_key, row_switch in states_to_update: + row_switch in states_to_update: if table not in mapper._pks_by_table: continue @@ -571,7 +570,7 @@ def _emit_insert_statements(base_mapper, uowtransaction, for (connection, pkeys, hasvalue, has_all_pks, has_all_defaults), \ records in groupby(insert, lambda rec: (rec[4], - list(rec[2].keys()), + tuple(sorted(rec[2].keys())), bool(rec[5]), rec[6], rec[7]) ): @@ -762,7 +761,7 @@ def _finalize_insert_update_commands(base_mapper, uowtransaction, """ for state, state_dict, mapper, connection, has_identity, \ - instance_key, row_switch in states_to_insert + \ + row_switch in states_to_insert + \ states_to_update: if mapper._readonly_props: @@ -864,7 +863,7 @@ def _connections_for_states(base_mapper, uowtransaction, states): if connection_callable: connection = connection_callable(base_mapper, state.obj()) - mapper = _state_mapper(state) + mapper = state.manager.mapper yield state, state.dict, mapper, connection From ca69e4560333a1a7e3a2dafd746be851cc89228c Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 15 Aug 2014 18:39:26 -0400 Subject: [PATCH 23/55] - mutablemapping adds compiler overhead, so screw it --- lib/sqlalchemy/util/_collections.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/sqlalchemy/util/_collections.py b/lib/sqlalchemy/util/_collections.py index fa27897a11..0904d454eb 100644 --- a/lib/sqlalchemy/util/_collections.py +++ b/lib/sqlalchemy/util/_collections.py @@ -13,7 +13,6 @@ import operator from .compat import threading, itertools_filterfalse from . import py2k import types -from collections import MutableMapping EMPTY_SET = frozenset() @@ -265,13 +264,18 @@ class OrderedDict(dict): def __iter__(self): return iter(self._list) - keys = MutableMapping.keys - values = MutableMapping.values - items = MutableMapping.items + def keys(self): + return list(self) + + def values(self): + return [self[key] for key in self._list] + + def items(self): + return [(key, self[key]) for key in self._list] if py2k: def itervalues(self): - return iter([self[key] for key in self._list]) + return iter(self.values()) def iterkeys(self): return iter(self) From 7fa595221400d168a7bb78551d45379290db195f Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 16 Aug 2014 13:33:02 -0400 Subject: [PATCH 24/55] - max failures 25 - guard against some potential pytest snarkiness --- lib/sqlalchemy/testing/plugin/pytestplugin.py | 3 +++ setup.cfg | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/sqlalchemy/testing/plugin/pytestplugin.py b/lib/sqlalchemy/testing/plugin/pytestplugin.py index f4c9efd554..0059429133 100644 --- a/lib/sqlalchemy/testing/plugin/pytestplugin.py +++ b/lib/sqlalchemy/testing/plugin/pytestplugin.py @@ -74,6 +74,9 @@ def pytest_collection_modifyitems(session, config, items): # new classes to a module on the fly. rebuilt_items = collections.defaultdict(list) + items[:] = [ + item for item in + items if isinstance(item.parent, pytest.Instance)] test_classes = set(item.parent for item in items) for test_class in test_classes: for sub_cls in plugin_base.generate_sub_tests( diff --git a/setup.cfg b/setup.cfg index 7517220a66..698c4b037f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,7 +9,7 @@ first-package-wins = true where = test [pytest] -addopts= --tb native -v -r fxX +addopts= --tb native -v -r fxX --maxfail=25 python_files=test/*test_*.py [upload] From e220ea11de931e86bbbaf373b49a26b906bbffdf Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 16 Aug 2014 13:37:49 -0400 Subject: [PATCH 25/55] - need list() here for py3k --- lib/sqlalchemy/orm/identity.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/sqlalchemy/orm/identity.py b/lib/sqlalchemy/orm/identity.py index 4425fc3a6e..0fa5411947 100644 --- a/lib/sqlalchemy/orm/identity.py +++ b/lib/sqlalchemy/orm/identity.py @@ -181,7 +181,10 @@ class WeakInstanceDict(IdentityMap): return iter(self.values()) def all_states(self): - return self._dict.values() + if util.py2k: + return self._dict.values() + else: + return list(self._dict.values()) def discard(self, state): if state.key in self._dict: From 4b288a9553b3eb44fef44eb1d649ca7dc0007e2d Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 16 Aug 2014 13:46:15 -0400 Subject: [PATCH 26/55] - support dialects w/o sane multi row count again --- lib/sqlalchemy/orm/persistence.py | 48 +++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index b7283e48e2..8c9b677fea 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -529,22 +529,40 @@ def _emit_update_statements(base_mapper, uowtransaction, value_params) rows += c.rowcount else: - multiparams = [rec[2] for rec in records] - c = cached_connections[connection].\ - execute(statement, multiparams) + if needs_version_id and \ + not connection.dialect.supports_sane_multi_rowcount and \ + connection.dialect.supports_sane_rowcount: + for state, state_dict, params, mapper, \ + connection, value_params in records: + c = cached_connections[connection].\ + execute(statement, params) + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params) + rows += c.rowcount + else: + multiparams = [rec[2] for rec in records] + c = cached_connections[connection].\ + execute(statement, multiparams) - rows += c.rowcount - for state, state_dict, params, mapper, \ - connection, value_params in records: - _postfetch( - mapper, - uowtransaction, - table, - state, - state_dict, - c, - c.context.compiled_parameters[0], - value_params) + rows += c.rowcount + for state, state_dict, params, mapper, \ + connection, value_params in records: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params) if connection.dialect.supports_sane_rowcount: if rows != len(records): From 589f205d53f031ceb297af760f2acfc777a5bc5d Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 16 Aug 2014 13:57:46 -0400 Subject: [PATCH 27/55] - changelog for pullreq github:125 - add pg8000 version detection for the "sane multi rowcount" feature --- doc/build/changelog/changelog_09.rst | 10 ++++++++++ lib/sqlalchemy/dialects/postgresql/pg8000.py | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index 0f92fb254c..b6eec2e9db 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -13,6 +13,16 @@ .. changelog:: :version: 0.9.8 + .. change:: + :tags: feature, postgresql, pg8000 + :versions: 1.0.0 + :pullreq: github:125 + + Support is added for "sane multi row count" with the pg8000 driver, + which applies mostly to when using versioning with the ORM. + The feature is version-detected based on pg8000 1.9.14 or greater + in use. Pull request courtesy Tony Locke. + .. change:: :tags: bug, engine :versions: 1.0.0 diff --git a/lib/sqlalchemy/dialects/postgresql/pg8000.py b/lib/sqlalchemy/dialects/postgresql/pg8000.py index 909b41b825..4ccc90208f 100644 --- a/lib/sqlalchemy/dialects/postgresql/pg8000.py +++ b/lib/sqlalchemy/dialects/postgresql/pg8000.py @@ -133,6 +133,16 @@ class PGDialect_pg8000(PGDialect): } ) + def initialize(self, connection): + if self.dbapi and hasattr(self.dbapi, '__version__'): + self._dbapi_version = tuple([ + int(x) for x in + self.dbapi.__version__.split(".")]) + else: + self._dbapi_version = (99, 99, 99) + self.supports_sane_multi_rowcount = self._dbapi_version >= (1, 9, 14) + super(PGDialect_pg8000, self).initialize(connection) + @classmethod def dbapi(cls): return __import__('pg8000') From b577afcb2bdcd94581606bc911968d8885509769 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 16 Aug 2014 19:49:07 -0400 Subject: [PATCH 28/55] - rework profiling, zoomark tests into single tests so that they can be used under xdist --- lib/sqlalchemy/testing/engines.py | 112 ----- lib/sqlalchemy/testing/profiling.py | 214 ++++----- test/aaa_profiling/test_compiler.py | 2 +- test/aaa_profiling/test_zoomark.py | 153 +++---- test/aaa_profiling/test_zoomark_orm.py | 229 ++++------ test/profiles.txt | 609 +++++++------------------ 6 files changed, 400 insertions(+), 919 deletions(-) diff --git a/lib/sqlalchemy/testing/engines.py b/lib/sqlalchemy/testing/engines.py index 9052df5707..67c13231e9 100644 --- a/lib/sqlalchemy/testing/engines.py +++ b/lib/sqlalchemy/testing/engines.py @@ -7,15 +7,12 @@ from __future__ import absolute_import -import types import weakref -from collections import deque from . import config from .util import decorator from .. import event, pool import re import warnings -from .. import util class ConnectionKiller(object): @@ -339,112 +336,3 @@ def proxying_engine(conn_cls=DBAPIProxyConnection, return testing_engine(options={'creator': mock_conn}) -class ReplayableSession(object): - """A simple record/playback tool. - - This is *not* a mock testing class. It only records a session for later - playback and makes no assertions on call consistency whatsoever. It's - unlikely to be suitable for anything other than DB-API recording. - - """ - - Callable = object() - NoAttribute = object() - - if util.py2k: - Natives = set([getattr(types, t) - for t in dir(types) if not t.startswith('_')]).\ - difference([getattr(types, t) - for t in ('FunctionType', 'BuiltinFunctionType', - 'MethodType', 'BuiltinMethodType', - 'LambdaType', 'UnboundMethodType',)]) - else: - Natives = set([getattr(types, t) - for t in dir(types) if not t.startswith('_')]).\ - union([type(t) if not isinstance(t, type) - else t for t in __builtins__.values()]).\ - difference([getattr(types, t) - for t in ('FunctionType', 'BuiltinFunctionType', - 'MethodType', 'BuiltinMethodType', - 'LambdaType', )]) - - def __init__(self): - self.buffer = deque() - - def recorder(self, base): - return self.Recorder(self.buffer, base) - - def player(self): - return self.Player(self.buffer) - - class Recorder(object): - def __init__(self, buffer, subject): - self._buffer = buffer - self._subject = subject - - def __call__(self, *args, **kw): - subject, buffer = [object.__getattribute__(self, x) - for x in ('_subject', '_buffer')] - - result = subject(*args, **kw) - if type(result) not in ReplayableSession.Natives: - buffer.append(ReplayableSession.Callable) - return type(self)(buffer, result) - else: - buffer.append(result) - return result - - @property - def _sqla_unwrap(self): - return self._subject - - def __getattribute__(self, key): - try: - return object.__getattribute__(self, key) - except AttributeError: - pass - - subject, buffer = [object.__getattribute__(self, x) - for x in ('_subject', '_buffer')] - try: - result = type(subject).__getattribute__(subject, key) - except AttributeError: - buffer.append(ReplayableSession.NoAttribute) - raise - else: - if type(result) not in ReplayableSession.Natives: - buffer.append(ReplayableSession.Callable) - return type(self)(buffer, result) - else: - buffer.append(result) - return result - - class Player(object): - def __init__(self, buffer): - self._buffer = buffer - - def __call__(self, *args, **kw): - buffer = object.__getattribute__(self, '_buffer') - result = buffer.popleft() - if result is ReplayableSession.Callable: - return self - else: - return result - - @property - def _sqla_unwrap(self): - return None - - def __getattribute__(self, key): - try: - return object.__getattribute__(self, key) - except AttributeError: - pass - buffer = object.__getattribute__(self, '_buffer') - result = buffer.popleft() - if result is ReplayableSession.Callable: - return self - elif result is ReplayableSession.NoAttribute: - raise AttributeError(key) - else: - return result diff --git a/lib/sqlalchemy/testing/profiling.py b/lib/sqlalchemy/testing/profiling.py index 75baec9876..fcb888f861 100644 --- a/lib/sqlalchemy/testing/profiling.py +++ b/lib/sqlalchemy/testing/profiling.py @@ -14,13 +14,12 @@ in a more fine-grained way than nose's profiling plugin. import os import sys -from .util import gc_collect, decorator +from .util import gc_collect from . import config from .plugin.plugin_base import SkipTest import pstats -import time import collections -from .. import util +import contextlib try: import cProfile @@ -30,64 +29,8 @@ from ..util import jython, pypy, win32, update_wrapper _current_test = None - -def profiled(target=None, **target_opts): - """Function profiling. - - @profiled() - or - @profiled(report=True, sort=('calls',), limit=20) - - Outputs profiling info for a decorated function. - - """ - - profile_config = {'targets': set(), - 'report': True, - 'print_callers': False, - 'print_callees': False, - 'graphic': False, - 'sort': ('time', 'calls'), - 'limit': None} - if target is None: - target = 'anonymous_target' - - @decorator - def decorate(fn, *args, **kw): - elapsed, load_stats, result = _profile( - fn, *args, **kw) - - graphic = target_opts.get('graphic', profile_config['graphic']) - if graphic: - os.system("runsnake %s" % filename) - else: - report = target_opts.get('report', profile_config['report']) - if report: - sort_ = target_opts.get('sort', profile_config['sort']) - limit = target_opts.get('limit', profile_config['limit']) - print(("Profile report for target '%s'" % ( - target, ) - )) - - stats = load_stats() - stats.sort_stats(*sort_) - if limit: - stats.print_stats(limit) - else: - stats.print_stats() - - print_callers = target_opts.get( - 'print_callers', profile_config['print_callers']) - if print_callers: - stats.print_callers() - - print_callees = target_opts.get( - 'print_callees', profile_config['print_callees']) - if print_callees: - stats.print_callees() - - return result - return decorate +# ProfileStatsFile instance, set up in plugin_base +_profile_stats = None class ProfileStatsFile(object): @@ -177,20 +120,23 @@ class ProfileStatsFile(object): self._write() def _header(self): - return \ - "# %s\n"\ - "# This file is written out on a per-environment basis.\n"\ - "# For each test in aaa_profiling, the corresponding function and \n"\ - "# environment is located within this file. If it doesn't exist,\n"\ - "# the test is skipped.\n"\ - "# If a callcount does exist, it is compared to what we received. \n"\ - "# assertions are raised if the counts do not match.\n"\ - "# \n"\ - "# To add a new callcount test, apply the function_call_count \n"\ - "# decorator and re-run the tests using the --write-profiles \n"\ - "# option - this file will be rewritten including the new count.\n"\ - "# \n"\ - "" % (self.fname) + return ( + "# %s\n" + "# This file is written out on a per-environment basis.\n" + "# For each test in aaa_profiling, the corresponding " + "function and \n" + "# environment is located within this file. " + "If it doesn't exist,\n" + "# the test is skipped.\n" + "# If a callcount does exist, it is compared " + "to what we received. \n" + "# assertions are raised if the counts do not match.\n" + "# \n" + "# To add a new callcount test, apply the function_call_count \n" + "# decorator and re-run the tests using the --write-profiles \n" + "# option - this file will be rewritten including the new count.\n" + "# \n" + ) % (self.fname) def _read(self): try: @@ -239,72 +185,66 @@ def function_call_count(variance=0.05): def decorate(fn): def wrap(*args, **kw): - - if cProfile is None: - raise SkipTest("cProfile is not installed") - - if not _profile_stats.has_stats() and not _profile_stats.write: - # run the function anyway, to support dependent tests - # (not a great idea but we have these in test_zoomark) - fn(*args, **kw) - raise SkipTest("No profiling stats available on this " - "platform for this function. Run tests with " - "--write-profiles to add statistics to %s for " - "this platform." % _profile_stats.short_fname) - - gc_collect() - - timespent, load_stats, fn_result = _profile( - fn, *args, **kw - ) - stats = load_stats() - callcount = stats.total_calls - - expected = _profile_stats.result(callcount) - if expected is None: - expected_count = None - else: - line_no, expected_count = expected - - print(("Pstats calls: %d Expected %s" % ( - callcount, - expected_count - ) - )) - stats.print_stats() - # stats.print_callers() - - if expected_count: - deviance = int(callcount * variance) - failed = abs(callcount - expected_count) > deviance - - if failed: - if _profile_stats.write: - _profile_stats.replace(callcount) - else: - raise AssertionError( - "Adjusted function call count %s not within %s%% " - "of expected %s. Rerun with --write-profiles to " - "regenerate this callcount." - % ( - callcount, (variance * 100), - expected_count)) - return fn_result + with count_functions(variance=variance): + return fn(*args, **kw) return update_wrapper(wrap, fn) return decorate -def _profile(fn, *args, **kw): - filename = "%s.prof" % fn.__name__ +@contextlib.contextmanager +def count_functions(variance=0.05): + if cProfile is None: + raise SkipTest("cProfile is not installed") - def load_stats(): - st = pstats.Stats(filename) - os.unlink(filename) - return st + if not _profile_stats.has_stats() and not _profile_stats.write: + raise SkipTest("No profiling stats available on this " + "platform for this function. Run tests with " + "--write-profiles to add statistics to %s for " + "this platform." % _profile_stats.short_fname) + + gc_collect() + + pr = cProfile.Profile() + pr.enable() + #began = time.time() + yield + #ended = time.time() + pr.disable() + + #s = compat.StringIO() + stats = pstats.Stats(pr, stream=sys.stdout) + + #timespent = ended - began + callcount = stats.total_calls + + expected = _profile_stats.result(callcount) + if expected is None: + expected_count = None + else: + line_no, expected_count = expected + + print(("Pstats calls: %d Expected %s" % ( + callcount, + expected_count + ) + )) + stats.sort_stats("cumulative") + stats.print_stats() + + if expected_count: + deviance = int(callcount * variance) + failed = abs(callcount - expected_count) > deviance + + if failed: + if _profile_stats.write: + _profile_stats.replace(callcount) + else: + raise AssertionError( + "Adjusted function call count %s not within %s%% " + "of expected %s. Rerun with --write-profiles to " + "regenerate this callcount." + % ( + callcount, (variance * 100), + expected_count)) - began = time.time() - cProfile.runctx('result = fn(*args, **kw)', globals(), locals(), - filename=filename) - ended = time.time() - return ended - began, load_stats, locals()['result'] diff --git a/test/aaa_profiling/test_compiler.py b/test/aaa_profiling/test_compiler.py index 47a412e73c..5eece46025 100644 --- a/test/aaa_profiling/test_compiler.py +++ b/test/aaa_profiling/test_compiler.py @@ -42,7 +42,7 @@ class CompileTest(fixtures.TestBase, AssertsExecutionResults): def test_insert(self): t1.insert().compile(dialect=self.dialect) - @profiling.function_call_count() + @profiling.function_call_count(variance=.15) def test_update(self): t1.update().compile(dialect=self.dialect) diff --git a/test/aaa_profiling/test_zoomark.py b/test/aaa_profiling/test_zoomark.py index 4c47085037..5b8a0f7853 100644 --- a/test/aaa_profiling/test_zoomark.py +++ b/test/aaa_profiling/test_zoomark.py @@ -7,43 +7,42 @@ An adaptation of Robert Brewers' ZooMark speed tests. """ import datetime from sqlalchemy import Table, Column, Integer, Unicode, Date, \ - DateTime, Time, Float, MetaData, Sequence, ForeignKey, create_engine, \ + DateTime, Time, Float, Sequence, ForeignKey, \ select, join, and_, outerjoin, func -from sqlalchemy.testing import fixtures, engines, profiling -from sqlalchemy import testing +from sqlalchemy.testing import replay_fixture + ITERATIONS = 1 -dbapi_session = engines.ReplayableSession() -metadata = None -class ZooMarkTest(fixtures.TestBase): +class ZooMarkTest(replay_fixture.ReplayFixtureTest): - """Runs the ZooMark and squawks if method counts vary from the norm. + """Runs the ZooMark and squawks if method counts vary from the norm.""" - Each test has an associated `call_range`, the total number of - accepted function calls made during the test. The count can vary - between Python 2.4 and 2.5. - - Unlike a unit test, this is a ordered collection of steps. Running - components individually will fail. - - """ __requires__ = 'cpython', __only_on__ = 'postgresql+psycopg2' - def test_baseline_0_setup(self): - global metadata - creator = testing.db.pool._creator - recorder = lambda: dbapi_session.recorder(creator()) - engine = engines.testing_engine(options={'creator': recorder, - 'use_reaper': False}) - metadata = MetaData(engine) - engine.connect() + def _run_steps(self, ctx): + self._baseline_1_create_tables() + with ctx(): + self._baseline_1a_populate() + with ctx(): + self._baseline_2_insert() + with ctx(): + self._baseline_3_properties() + with ctx(): + self._baseline_4_expressions() + with ctx(): + self._baseline_5_aggregates() + with ctx(): + self._baseline_6_editing() + with ctx(): + self._baseline_7_multiview() + self._baseline_8_drop() - def test_baseline_1_create_tables(self): + def _baseline_1_create_tables(self): Table( 'Zoo', - metadata, + self.metadata, Column('ID', Integer, Sequence('zoo_id_seq'), primary_key=True, index=True), Column('Name', Unicode(255)), @@ -54,7 +53,7 @@ class ZooMarkTest(fixtures.TestBase): ) Table( 'Animal', - metadata, + self.metadata, Column('ID', Integer, Sequence('animal_id_seq'), primary_key=True), Column('ZooID', Integer, ForeignKey('Zoo.ID'), index=True), @@ -67,12 +66,12 @@ class ZooMarkTest(fixtures.TestBase): Column('PreferredFoodID', Integer), Column('AlternateFoodID', Integer), ) - metadata.create_all() + self.metadata.create_all() - def test_baseline_1a_populate(self): - Zoo = metadata.tables['Zoo'] - Animal = metadata.tables['Animal'] - engine = metadata.bind + def _baseline_1a_populate(self): + Zoo = self.metadata.tables['Zoo'] + Animal = self.metadata.tables['Animal'] + engine = self.metadata.bind wap = engine.execute(Zoo.insert(), Name='Wild Animal Park', Founded=datetime.date(2000, 1, 1), Opens=datetime.time(8, 15, 59), @@ -137,16 +136,16 @@ class ZooMarkTest(fixtures.TestBase): engine.execute(Animal.insert(inline=True), Species='Ape', Name='Hua Mei', Legs=2, MotherID=bai_yun) - def test_baseline_2_insert(self): - Animal = metadata.tables['Animal'] + def _baseline_2_insert(self): + Animal = self.metadata.tables['Animal'] i = Animal.insert(inline=True) for x in range(ITERATIONS): i.execute(Species='Tick', Name='Tick %d' % x, Legs=8) - def test_baseline_3_properties(self): - Zoo = metadata.tables['Zoo'] - Animal = metadata.tables['Animal'] - engine = metadata.bind + def _baseline_3_properties(self): + Zoo = self.metadata.tables['Zoo'] + Animal = self.metadata.tables['Animal'] + engine = self.metadata.bind def fullobject(select): """Iterate over the full result row.""" @@ -171,10 +170,10 @@ class ZooMarkTest(fixtures.TestBase): fullobject(Animal.select(Animal.c.Legs == 1000000)) fullobject(Animal.select(Animal.c.Species == 'Tick')) - def test_baseline_4_expressions(self): - Zoo = metadata.tables['Zoo'] - Animal = metadata.tables['Animal'] - engine = metadata.bind + def _baseline_4_expressions(self): + Zoo = self.metadata.tables['Zoo'] + Animal = self.metadata.tables['Animal'] + engine = self.metadata.bind def fulltable(select): """Iterate over the full result table.""" @@ -280,10 +279,10 @@ class ZooMarkTest(fixtures.TestBase): 'day', Animal.c.LastEscape) == 21))) == 1 - def test_baseline_5_aggregates(self): - Animal = metadata.tables['Animal'] - Zoo = metadata.tables['Zoo'] - engine = metadata.bind + def _baseline_5_aggregates(self): + Animal = self.metadata.tables['Animal'] + Zoo = self.metadata.tables['Zoo'] + engine = self.metadata.bind for x in range(ITERATIONS): @@ -327,9 +326,9 @@ class ZooMarkTest(fixtures.TestBase): distinct=True)).fetchall()] legs.sort() - def test_baseline_6_editing(self): - Zoo = metadata.tables['Zoo'] - engine = metadata.bind + def _baseline_6_editing(self): + Zoo = self.metadata.tables['Zoo'] + engine = self.metadata.bind for x in range(ITERATIONS): # Edit @@ -364,10 +363,10 @@ class ZooMarkTest(fixtures.TestBase): )).first() assert SDZ['Founded'] == datetime.date(1935, 9, 13) - def test_baseline_7_multiview(self): - Zoo = metadata.tables['Zoo'] - Animal = metadata.tables['Animal'] - engine = metadata.bind + def _baseline_7_multiview(self): + Zoo = self.metadata.tables['Zoo'] + Animal = self.metadata.tables['Animal'] + engine = self.metadata.bind def fulltable(select): """Iterate over the full result table.""" @@ -403,52 +402,6 @@ class ZooMarkTest(fixtures.TestBase): Zoo.c.Name, Animal.c.Species], from_obj=[outerjoin(Animal, Zoo)])) - def test_baseline_8_drop(self): - metadata.drop_all() + def _baseline_8_drop(self): + self.metadata.drop_all() - # Now, run all of these tests again with the DB-API driver factored - # out: the ReplayableSession playback stands in for the database. - # - # How awkward is this in a unittest framework? Very. - - def test_profile_0(self): - global metadata - player = lambda: dbapi_session.player() - engine = create_engine('postgresql:///', creator=player, - use_native_hstore=False) - metadata = MetaData(engine) - engine.connect() - - def test_profile_1_create_tables(self): - self.test_baseline_1_create_tables() - - @profiling.function_call_count() - def test_profile_1a_populate(self): - self.test_baseline_1a_populate() - - @profiling.function_call_count() - def test_profile_2_insert(self): - self.test_baseline_2_insert() - - @profiling.function_call_count() - def test_profile_3_properties(self): - self.test_baseline_3_properties() - - @profiling.function_call_count() - def test_profile_4_expressions(self): - self.test_baseline_4_expressions() - - @profiling.function_call_count() - def test_profile_5_aggregates(self): - self.test_baseline_5_aggregates() - - @profiling.function_call_count() - def test_profile_6_editing(self): - self.test_baseline_6_editing() - - @profiling.function_call_count() - def test_profile_7_multiview(self): - self.test_baseline_7_multiview() - - def test_profile_8_drop(self): - self.test_baseline_8_drop() diff --git a/test/aaa_profiling/test_zoomark_orm.py b/test/aaa_profiling/test_zoomark_orm.py index 6b781af9b7..500d7c2cb1 100644 --- a/test/aaa_profiling/test_zoomark_orm.py +++ b/test/aaa_profiling/test_zoomark_orm.py @@ -7,48 +7,52 @@ An adaptation of Robert Brewers' ZooMark speed tests. """ import datetime from sqlalchemy import Table, Column, Integer, Unicode, Date, \ - DateTime, Time, Float, MetaData, Sequence, ForeignKey, create_engine, \ + DateTime, Time, Float, Sequence, ForeignKey, \ select, and_, func -from sqlalchemy.orm import sessionmaker, mapper -from sqlalchemy.testing import fixtures, engines, profiling -from sqlalchemy import testing +from sqlalchemy.orm import mapper +from sqlalchemy.testing import replay_fixture + ITERATIONS = 1 -dbapi_session = engines.ReplayableSession() -metadata = None Zoo = Animal = session = None -class ZooMarkTest(fixtures.TestBase): +class ZooMarkTest(replay_fixture.ReplayFixtureTest): """Runs the ZooMark and squawks if method counts vary from the norm. - Each test has an associated `call_range`, the total number of - accepted function calls made during the test. The count can vary - between Python 2.4 and 2.5. - - Unlike a unit test, this is a ordered collection of steps. Running - components individually will fail. """ __requires__ = 'cpython', __only_on__ = 'postgresql+psycopg2' - def test_baseline_0_setup(self): - global metadata, session - creator = testing.db.pool._creator - recorder = lambda: dbapi_session.recorder(creator()) - engine = engines.testing_engine( - options={'creator': recorder, 'use_reaper': False}) - metadata = MetaData(engine) - session = sessionmaker(engine)() - engine.connect() + def _run_steps(self, ctx): + #self._baseline_1_create_tables() + with ctx(): + self._baseline_1a_populate() + with ctx(): + self._baseline_2_insert() + with ctx(): + self._baseline_3_properties() + with ctx(): + self._baseline_4_expressions() + with ctx(): + self._baseline_5_aggregates() + with ctx(): + self._baseline_6_editing() + #self._baseline_7_drop() - def test_baseline_1_create_tables(self): + def setup_engine(self): + self._baseline_1_create_tables() + + def teardown_engine(self): + self._baseline_7_drop() + + def _baseline_1_create_tables(self): zoo = Table( 'Zoo', - metadata, + self.metadata, Column('ID', Integer, Sequence('zoo_id_seq'), primary_key=True, index=True), Column('Name', Unicode(255)), @@ -59,7 +63,7 @@ class ZooMarkTest(fixtures.TestBase): ) animal = Table( 'Animal', - metadata, + self.metadata, Column('ID', Integer, Sequence('animal_id_seq'), primary_key=True), Column('ZooID', Integer, ForeignKey('Zoo.ID'), index=True), @@ -72,7 +76,7 @@ class ZooMarkTest(fixtures.TestBase): Column('PreferredFoodID', Integer), Column('AlternateFoodID', Integer), ) - metadata.create_all() + self.metadata.create_all() global Zoo, Animal class Zoo(object): @@ -90,131 +94,129 @@ class ZooMarkTest(fixtures.TestBase): mapper(Zoo, zoo) mapper(Animal, animal) - def test_baseline_1a_populate(self): + def _baseline_1a_populate(self): wap = Zoo( Name='Wild Animal Park', Founded=datetime.date( 2000, 1, 1), Opens=datetime.time( 8, 15, 59), LastEscape=datetime.datetime( 2004, 7, 29, 5, 6, 7, ), Admission=4.95) - session.add(wap) + self.session.add(wap) sdz = Zoo( Name='San Diego Zoo', Founded=datetime.date( 1835, 9, 13), Opens=datetime.time( 9, 0, 0), Admission=0) - session.add(sdz) + self.session.add(sdz) bio = Zoo(Name='Montr\xe9al Biod\xf4me', Founded=datetime.date(1992, 6, 19), Opens=datetime.time(9, 0, 0), Admission=11.75) - session.add(bio) + self.session.add(bio) seaworld = Zoo(Name='Sea_World', Admission=60) - session.add(seaworld) + self.session.add(seaworld) # Let's add a crazy futuristic Zoo to test large date values. lp = Zoo(Name='Luna Park', Founded=datetime.date(2072, 7, 17), Opens=datetime.time(0, 0, 0), Admission=134.95) - session.add(lp) - session.flush() + self.session.add(lp) # Animals leopard = Animal(Species='Leopard', Lifespan=73.5) - session.add(leopard) + self.session.add(leopard) leopard.ZooID = wap.ID leopard.LastEscape = \ datetime.datetime(2004, 12, 21, 8, 15, 0, 999907, ) - session.add(Animal(Species='Lion', ZooID=wap.ID)) - session.add(Animal(Species='Slug', Legs=1, Lifespan=.75)) - session.add(Animal(Species='Tiger', ZooID=sdz.ID)) + self.session.add(Animal(Species='Lion', ZooID=wap.ID)) + self.session.add(Animal(Species='Slug', Legs=1, Lifespan=.75)) + self.session.add(Animal(Species='Tiger', ZooID=sdz.ID)) # Override Legs.default with itself just to make sure it works. - session.add(Animal(Species='Bear', Legs=4)) - session.add(Animal(Species='Ostrich', Legs=2, Lifespan=103.2)) - session.add(Animal(Species='Centipede', Legs=100)) - session.add(Animal(Species='Emperor Penguin', Legs=2, + self.session.add(Animal(Species='Bear', Legs=4)) + self.session.add(Animal(Species='Ostrich', Legs=2, Lifespan=103.2)) + self.session.add(Animal(Species='Centipede', Legs=100)) + self.session.add(Animal(Species='Emperor Penguin', Legs=2, ZooID=seaworld.ID)) - session.add(Animal(Species='Adelie Penguin', Legs=2, + self.session.add(Animal(Species='Adelie Penguin', Legs=2, ZooID=seaworld.ID)) - session.add(Animal(Species='Millipede', Legs=1000000, + self.session.add(Animal(Species='Millipede', Legs=1000000, ZooID=sdz.ID)) # Add a mother and child to test relationships bai_yun = Animal(Species='Ape', Nameu='Bai Yun', Legs=2) - session.add(bai_yun) - session.add(Animal(Species='Ape', Name='Hua Mei', Legs=2, + self.session.add(bai_yun) + self.session.add(Animal(Species='Ape', Name='Hua Mei', Legs=2, MotherID=bai_yun.ID)) - session.flush() - session.commit() + self.session.commit() - def test_baseline_2_insert(self): + def _baseline_2_insert(self): for x in range(ITERATIONS): - session.add(Animal(Species='Tick', Name='Tick %d' % x, + self.session.add(Animal(Species='Tick', Name='Tick %d' % x, Legs=8)) - session.flush() + self.session.flush() - def test_baseline_3_properties(self): + def _baseline_3_properties(self): for x in range(ITERATIONS): # Zoos - list(session.query(Zoo).filter( + list(self.session.query(Zoo).filter( Zoo.Name == 'Wild Animal Park')) list( - session.query(Zoo).filter( + self.session.query(Zoo).filter( Zoo.Founded == datetime.date( 1835, 9, 13))) list( - session.query(Zoo).filter( + self.session.query(Zoo).filter( Zoo.Name == 'Montr\xe9al Biod\xf4me')) - list(session.query(Zoo).filter(Zoo.Admission == float(60))) + list(self.session.query(Zoo).filter(Zoo.Admission == float(60))) # Animals - list(session.query(Animal).filter(Animal.Species == 'Leopard')) - list(session.query(Animal).filter(Animal.Species == 'Ostrich')) - list(session.query(Animal).filter(Animal.Legs == 1000000)) - list(session.query(Animal).filter(Animal.Species == 'Tick')) + list(self.session.query(Animal).filter(Animal.Species == 'Leopard')) + list(self.session.query(Animal).filter(Animal.Species == 'Ostrich')) + list(self.session.query(Animal).filter(Animal.Legs == 1000000)) + list(self.session.query(Animal).filter(Animal.Species == 'Tick')) - def test_baseline_4_expressions(self): + def _baseline_4_expressions(self): for x in range(ITERATIONS): - assert len(list(session.query(Zoo))) == 5 - assert len(list(session.query(Animal))) == ITERATIONS + 12 - assert len(list(session.query(Animal).filter(Animal.Legs + assert len(list(self.session.query(Zoo))) == 5 + assert len(list(self.session.query(Animal))) == ITERATIONS + 12 + assert len(list(self.session.query(Animal).filter(Animal.Legs == 4))) == 4 - assert len(list(session.query(Animal).filter(Animal.Legs + assert len(list(self.session.query(Animal).filter(Animal.Legs == 2))) == 5 assert len( list( - session.query(Animal).filter( + self.session.query(Animal).filter( and_( Animal.Legs >= 2, Animal.Legs < 20)))) == ITERATIONS + 9 - assert len(list(session.query(Animal).filter(Animal.Legs + assert len(list(self.session.query(Animal).filter(Animal.Legs > 10))) == 2 - assert len(list(session.query(Animal).filter(Animal.Lifespan + assert len(list(self.session.query(Animal).filter(Animal.Lifespan > 70))) == 2 - assert len(list(session.query(Animal). + assert len(list(self.session.query(Animal). filter(Animal.Species.like('L%')))) == 2 - assert len(list(session.query(Animal). + assert len(list(self.session.query(Animal). filter(Animal.Species.like('%pede')))) == 2 - assert len(list(session.query(Animal).filter(Animal.LastEscape + assert len(list(self.session.query(Animal).filter(Animal.LastEscape != None))) == 1 assert len( list( - session.query(Animal).filter( + self.session.query(Animal).filter( Animal.LastEscape == None))) == ITERATIONS + 11 # In operator (containedby) - assert len(list(session.query(Animal).filter( + assert len(list(self.session.query(Animal).filter( Animal.Species.like('%pede%')))) == 2 assert len( list( - session.query(Animal). filter( + self.session.query(Animal). filter( Animal.Species.in_( ('Lion', 'Tiger', 'Bear'))))) == 3 @@ -224,17 +226,17 @@ class ZooMarkTest(fixtures.TestBase): pet, pet2 = thing(), thing() pet.Name, pet2.Name = 'Slug', 'Ostrich' - assert len(list(session.query(Animal). + assert len(list(self.session.query(Animal). filter(Animal.Species.in_((pet.Name, pet2.Name))))) == 2 # logic and other functions name = 'Lion' - assert len(list(session.query(Animal). + assert len(list(self.session.query(Animal). filter(func.length(Animal.Species) == len(name)))) == ITERATIONS + 3 - assert len(list(session.query(Animal). + assert len(list(self.session.query(Animal). filter(Animal.Species.like('%i%' )))) == ITERATIONS + 7 @@ -242,29 +244,29 @@ class ZooMarkTest(fixtures.TestBase): assert len( list( - session.query(Zoo).filter( + self.session.query(Zoo).filter( and_( Zoo.Founded != None, Zoo.Founded < func.now())))) == 3 - assert len(list(session.query(Animal).filter(Animal.LastEscape + assert len(list(self.session.query(Animal).filter(Animal.LastEscape == func.now()))) == 0 - assert len(list(session.query(Animal).filter( + assert len(list(self.session.query(Animal).filter( func.date_part('year', Animal.LastEscape) == 2004))) == 1 assert len( list( - session.query(Animal). filter( + self.session.query(Animal). filter( func.date_part( 'month', Animal.LastEscape) == 12))) == 1 - assert len(list(session.query(Animal).filter( + assert len(list(self.session.query(Animal).filter( func.date_part('day', Animal.LastEscape) == 21))) == 1 - def test_baseline_5_aggregates(self): - Animal = metadata.tables['Animal'] - Zoo = metadata.tables['Zoo'] + def _baseline_5_aggregates(self): + Animal = self.metadata.tables['Animal'] + Zoo = self.metadata.tables['Zoo'] # TODO: convert to ORM - engine = metadata.bind + engine = self.metadata.bind for x in range(ITERATIONS): # views @@ -307,12 +309,12 @@ class ZooMarkTest(fixtures.TestBase): distinct=True)).fetchall()] legs.sort() - def test_baseline_6_editing(self): + def _baseline_6_editing(self): for x in range(ITERATIONS): # Edit - SDZ = session.query(Zoo).filter(Zoo.Name == 'San Diego Zoo' + SDZ = self.session.query(Zoo).filter(Zoo.Name == 'San Diego Zoo' ).one() SDZ.Name = 'The San Diego Zoo' SDZ.Founded = datetime.date(1900, 1, 1) @@ -321,7 +323,7 @@ class ZooMarkTest(fixtures.TestBase): # Test edits - SDZ = session.query(Zoo).filter(Zoo.Name + SDZ = self.session.query(Zoo).filter(Zoo.Name == 'The San Diego Zoo').one() assert SDZ.Founded == datetime.date(1900, 1, 1), SDZ.Founded @@ -334,55 +336,12 @@ class ZooMarkTest(fixtures.TestBase): # Test re-edits - SDZ = session.query(Zoo).filter(Zoo.Name == 'San Diego Zoo' + SDZ = self.session.query(Zoo).filter(Zoo.Name == 'San Diego Zoo' ).one() assert SDZ.Founded == datetime.date(1835, 9, 13), \ SDZ.Founded - def test_baseline_7_drop(self): - session.rollback() - metadata.drop_all() + def _baseline_7_drop(self): + self.session.rollback() + self.metadata.drop_all() - # Now, run all of these tests again with the DB-API driver factored - # out: the ReplayableSession playback stands in for the database. - # - # How awkward is this in a unittest framework? Very. - - def test_profile_0(self): - global metadata, session - player = lambda: dbapi_session.player() - engine = create_engine('postgresql:///', creator=player, - use_native_hstore=False) - metadata = MetaData(engine) - session = sessionmaker(engine)() - engine.connect() - - def test_profile_1_create_tables(self): - self.test_baseline_1_create_tables() - - @profiling.function_call_count() - def test_profile_1a_populate(self): - self.test_baseline_1a_populate() - - @profiling.function_call_count() - def test_profile_2_insert(self): - self.test_baseline_2_insert() - - @profiling.function_call_count() - def test_profile_3_properties(self): - self.test_baseline_3_properties() - - @profiling.function_call_count() - def test_profile_4_expressions(self): - self.test_baseline_4_expressions() - - @profiling.function_call_count() - def test_profile_5_aggregates(self): - self.test_baseline_5_aggregates() - - @profiling.function_call_count() - def test_profile_6_editing(self): - self.test_baseline_6_editing() - - def test_profile_7_drop(self): - self.test_baseline_7_drop() diff --git a/test/profiles.txt b/test/profiles.txt index 59ce23db3a..ca84cdc263 100644 --- a/test/profiles.txt +++ b/test/profiles.txt @@ -13,507 +13,248 @@ # TEST: test.aaa_profiling.test_compiler.CompileTest.test_insert -test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_mysql_mysqlconnector_cextensions 73 -test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_mysql_mysqlconnector_nocextensions 73 -test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_postgresql_psycopg2_cextensions 73 -test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_postgresql_psycopg2_nocextensions 73 -test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_sqlite_pysqlite_cextensions 73 -test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_sqlite_pysqlite_nocextensions 73 -test.aaa_profiling.test_compiler.CompileTest.test_insert 3.3_mysql_mysqlconnector_cextensions 78 -test.aaa_profiling.test_compiler.CompileTest.test_insert 3.3_mysql_mysqlconnector_nocextensions 78 -test.aaa_profiling.test_compiler.CompileTest.test_insert 3.3_postgresql_psycopg2_cextensions 78 -test.aaa_profiling.test_compiler.CompileTest.test_insert 3.3_postgresql_psycopg2_nocextensions 78 -test.aaa_profiling.test_compiler.CompileTest.test_insert 3.3_sqlite_pysqlite_cextensions 78 -test.aaa_profiling.test_compiler.CompileTest.test_insert 3.3_sqlite_pysqlite_nocextensions 78 -test.aaa_profiling.test_compiler.CompileTest.test_insert 3.4_mysql_mysqlconnector_cextensions 78 -test.aaa_profiling.test_compiler.CompileTest.test_insert 3.4_mysql_mysqlconnector_nocextensions 78 -test.aaa_profiling.test_compiler.CompileTest.test_insert 3.4_postgresql_psycopg2_cextensions 78 -test.aaa_profiling.test_compiler.CompileTest.test_insert 3.4_postgresql_psycopg2_nocextensions 78 -test.aaa_profiling.test_compiler.CompileTest.test_insert 3.4_sqlite_pysqlite_cextensions 78 -test.aaa_profiling.test_compiler.CompileTest.test_insert 3.4_sqlite_pysqlite_nocextensions 78 +test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_mysql_mysqlconnector_cextensions 74 +test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_mysql_mysqlconnector_nocextensions 74 +test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_postgresql_psycopg2_cextensions 74 +test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_postgresql_psycopg2_nocextensions 74 +test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_sqlite_pysqlite_cextensions 74 +test.aaa_profiling.test_compiler.CompileTest.test_insert 2.7_sqlite_pysqlite_nocextensions 74 +test.aaa_profiling.test_compiler.CompileTest.test_insert 3.3_mysql_mysqlconnector_cextensions 77 +test.aaa_profiling.test_compiler.CompileTest.test_insert 3.3_mysql_mysqlconnector_nocextensions 77 +test.aaa_profiling.test_compiler.CompileTest.test_insert 3.3_postgresql_psycopg2_cextensions 77 +test.aaa_profiling.test_compiler.CompileTest.test_insert 3.3_postgresql_psycopg2_nocextensions 77 +test.aaa_profiling.test_compiler.CompileTest.test_insert 3.3_sqlite_pysqlite_cextensions 77 +test.aaa_profiling.test_compiler.CompileTest.test_insert 3.3_sqlite_pysqlite_nocextensions 77 # TEST: test.aaa_profiling.test_compiler.CompileTest.test_select -test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_mysql_mysqlconnector_cextensions 151 -test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_mysql_mysqlconnector_nocextensions 151 -test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_postgresql_psycopg2_cextensions 151 -test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_postgresql_psycopg2_nocextensions 151 -test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_sqlite_pysqlite_cextensions 151 -test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_sqlite_pysqlite_nocextensions 151 -test.aaa_profiling.test_compiler.CompileTest.test_select 3.3_mysql_mysqlconnector_cextensions 166 -test.aaa_profiling.test_compiler.CompileTest.test_select 3.3_mysql_mysqlconnector_nocextensions 166 -test.aaa_profiling.test_compiler.CompileTest.test_select 3.3_postgresql_psycopg2_cextensions 166 -test.aaa_profiling.test_compiler.CompileTest.test_select 3.3_postgresql_psycopg2_nocextensions 166 -test.aaa_profiling.test_compiler.CompileTest.test_select 3.3_sqlite_pysqlite_cextensions 166 -test.aaa_profiling.test_compiler.CompileTest.test_select 3.3_sqlite_pysqlite_nocextensions 166 -test.aaa_profiling.test_compiler.CompileTest.test_select 3.4_mysql_mysqlconnector_cextensions 166 -test.aaa_profiling.test_compiler.CompileTest.test_select 3.4_mysql_mysqlconnector_nocextensions 166 -test.aaa_profiling.test_compiler.CompileTest.test_select 3.4_postgresql_psycopg2_cextensions 166 -test.aaa_profiling.test_compiler.CompileTest.test_select 3.4_postgresql_psycopg2_nocextensions 166 -test.aaa_profiling.test_compiler.CompileTest.test_select 3.4_sqlite_pysqlite_cextensions 166 -test.aaa_profiling.test_compiler.CompileTest.test_select 3.4_sqlite_pysqlite_nocextensions 166 +test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_mysql_mysqlconnector_cextensions 152 +test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_mysql_mysqlconnector_nocextensions 152 +test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_postgresql_psycopg2_cextensions 152 +test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_postgresql_psycopg2_nocextensions 152 +test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_sqlite_pysqlite_cextensions 152 +test.aaa_profiling.test_compiler.CompileTest.test_select 2.7_sqlite_pysqlite_nocextensions 152 +test.aaa_profiling.test_compiler.CompileTest.test_select 3.3_mysql_mysqlconnector_cextensions 165 +test.aaa_profiling.test_compiler.CompileTest.test_select 3.3_mysql_mysqlconnector_nocextensions 165 +test.aaa_profiling.test_compiler.CompileTest.test_select 3.3_postgresql_psycopg2_cextensions 165 +test.aaa_profiling.test_compiler.CompileTest.test_select 3.3_postgresql_psycopg2_nocextensions 165 +test.aaa_profiling.test_compiler.CompileTest.test_select 3.3_sqlite_pysqlite_cextensions 165 +test.aaa_profiling.test_compiler.CompileTest.test_select 3.3_sqlite_pysqlite_nocextensions 165 # TEST: test.aaa_profiling.test_compiler.CompileTest.test_select_labels -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_mysql_mysqlconnector_cextensions 185 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_mysql_mysqlconnector_nocextensions 185 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_postgresql_psycopg2_cextensions 185 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_postgresql_psycopg2_nocextensions 185 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_sqlite_pysqlite_cextensions 185 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_sqlite_pysqlite_nocextensions 185 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.3_mysql_mysqlconnector_cextensions 200 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.3_mysql_mysqlconnector_nocextensions 200 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.3_postgresql_psycopg2_cextensions 200 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.3_postgresql_psycopg2_nocextensions 200 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.3_sqlite_pysqlite_cextensions 200 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.3_sqlite_pysqlite_nocextensions 200 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.4_mysql_mysqlconnector_cextensions 200 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.4_mysql_mysqlconnector_nocextensions 200 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.4_postgresql_psycopg2_cextensions 200 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.4_postgresql_psycopg2_nocextensions 200 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.4_sqlite_pysqlite_cextensions 200 -test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.4_sqlite_pysqlite_nocextensions 200 +test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_mysql_mysqlconnector_cextensions 186 +test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_mysql_mysqlconnector_nocextensions 186 +test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_postgresql_psycopg2_cextensions 186 +test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_postgresql_psycopg2_nocextensions 186 +test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_sqlite_pysqlite_cextensions 186 +test.aaa_profiling.test_compiler.CompileTest.test_select_labels 2.7_sqlite_pysqlite_nocextensions 186 +test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.3_mysql_mysqlconnector_cextensions 199 +test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.3_mysql_mysqlconnector_nocextensions 199 +test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.3_postgresql_psycopg2_cextensions 199 +test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.3_postgresql_psycopg2_nocextensions 199 +test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.3_sqlite_pysqlite_cextensions 199 +test.aaa_profiling.test_compiler.CompileTest.test_select_labels 3.3_sqlite_pysqlite_nocextensions 199 # TEST: test.aaa_profiling.test_compiler.CompileTest.test_update -test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_mysql_mysqlconnector_cextensions 78 -test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_mysql_mysqlconnector_nocextensions 78 -test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_postgresql_psycopg2_cextensions 76 -test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_postgresql_psycopg2_nocextensions 76 -test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_sqlite_pysqlite_cextensions 76 -test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_sqlite_pysqlite_nocextensions 76 -test.aaa_profiling.test_compiler.CompileTest.test_update 3.3_mysql_mysqlconnector_cextensions 81 -test.aaa_profiling.test_compiler.CompileTest.test_update 3.3_mysql_mysqlconnector_nocextensions 81 -test.aaa_profiling.test_compiler.CompileTest.test_update 3.3_postgresql_psycopg2_cextensions 79 -test.aaa_profiling.test_compiler.CompileTest.test_update 3.3_postgresql_psycopg2_nocextensions 79 -test.aaa_profiling.test_compiler.CompileTest.test_update 3.3_sqlite_pysqlite_cextensions 79 -test.aaa_profiling.test_compiler.CompileTest.test_update 3.3_sqlite_pysqlite_nocextensions 79 -test.aaa_profiling.test_compiler.CompileTest.test_update 3.4_mysql_mysqlconnector_cextensions 81 -test.aaa_profiling.test_compiler.CompileTest.test_update 3.4_mysql_mysqlconnector_nocextensions 81 -test.aaa_profiling.test_compiler.CompileTest.test_update 3.4_postgresql_psycopg2_cextensions 79 -test.aaa_profiling.test_compiler.CompileTest.test_update 3.4_postgresql_psycopg2_nocextensions 79 -test.aaa_profiling.test_compiler.CompileTest.test_update 3.4_sqlite_pysqlite_cextensions 79 -test.aaa_profiling.test_compiler.CompileTest.test_update 3.4_sqlite_pysqlite_nocextensions 79 +test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_mysql_mysqlconnector_cextensions 79 +test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_mysql_mysqlconnector_nocextensions 79 +test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_postgresql_psycopg2_cextensions 77 +test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_postgresql_psycopg2_nocextensions 77 +test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_sqlite_pysqlite_cextensions 77 +test.aaa_profiling.test_compiler.CompileTest.test_update 2.7_sqlite_pysqlite_nocextensions 77 +test.aaa_profiling.test_compiler.CompileTest.test_update 3.3_mysql_mysqlconnector_cextensions 80 +test.aaa_profiling.test_compiler.CompileTest.test_update 3.3_mysql_mysqlconnector_nocextensions 80 +test.aaa_profiling.test_compiler.CompileTest.test_update 3.3_postgresql_psycopg2_cextensions 78 +test.aaa_profiling.test_compiler.CompileTest.test_update 3.3_postgresql_psycopg2_nocextensions 78 +test.aaa_profiling.test_compiler.CompileTest.test_update 3.3_sqlite_pysqlite_cextensions 78 +test.aaa_profiling.test_compiler.CompileTest.test_update 3.3_sqlite_pysqlite_nocextensions 78 # TEST: test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 2.7_mysql_mysqlconnector_cextensions 147 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 2.7_mysql_mysqlconnector_nocextensions 147 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 2.7_postgresql_psycopg2_cextensions 147 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 2.7_postgresql_psycopg2_nocextensions 147 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 2.7_sqlite_pysqlite_cextensions 147 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 2.7_sqlite_pysqlite_nocextensions 147 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.3_mysql_mysqlconnector_cextensions 149 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.3_mysql_mysqlconnector_nocextensions 149 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.3_postgresql_psycopg2_cextensions 149 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.3_postgresql_psycopg2_nocextensions 149 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.3_sqlite_pysqlite_cextensions 149 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.3_sqlite_pysqlite_nocextensions 149 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.4_mysql_mysqlconnector_cextensions 149 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.4_mysql_mysqlconnector_nocextensions 149 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.4_postgresql_psycopg2_cextensions 149 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.4_postgresql_psycopg2_nocextensions 149 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.4_sqlite_pysqlite_cextensions 149 -test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.4_sqlite_pysqlite_nocextensions 149 +test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 2.7_mysql_mysqlconnector_cextensions 148 +test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 2.7_mysql_mysqlconnector_nocextensions 148 +test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 2.7_postgresql_psycopg2_cextensions 148 +test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 2.7_postgresql_psycopg2_nocextensions 148 +test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 2.7_sqlite_pysqlite_cextensions 148 +test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 2.7_sqlite_pysqlite_nocextensions 148 +test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.3_mysql_mysqlconnector_cextensions 148 +test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.3_mysql_mysqlconnector_nocextensions 148 +test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.3_postgresql_psycopg2_cextensions 148 +test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.3_postgresql_psycopg2_nocextensions 148 +test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.3_sqlite_pysqlite_cextensions 148 +test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.3_sqlite_pysqlite_nocextensions 148 # TEST: test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_sqlite_pysqlite_cextensions 4265 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_sqlite_pysqlite_nocextensions 4265 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_mysql_mysqlconnector_cextensions 4266 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_mysql_mysqlconnector_nocextensions 4266 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_postgresql_psycopg2_cextensions 4266 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_postgresql_psycopg2_nocextensions 4266 +test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_sqlite_pysqlite_nocextensions 4260 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_sqlite_pysqlite_cextensions 4266 test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.3_sqlite_pysqlite_nocextensions 4266 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.4_mysql_mysqlconnector_cextensions 4266 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.4_postgresql_psycopg2_cextensions 4266 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.4_sqlite_pysqlite_cextensions 4266 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 3.4_sqlite_pysqlite_nocextensions 4266 # TEST: test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove -test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 2.7_sqlite_pysqlite_cextensions 6525 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 2.7_sqlite_pysqlite_nocextensions 6525 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_mysql_mysqlconnector_cextensions 6527 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_mysql_mysqlconnector_nocextensions 6527 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_postgresql_psycopg2_cextensions 6527 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_postgresql_psycopg2_nocextensions 6527 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_sqlite_pysqlite_cextensions 6527 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_sqlite_pysqlite_nocextensions 6527 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.4_mysql_mysqlconnector_cextensions 6527 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.4_postgresql_psycopg2_cextensions 6527 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.4_sqlite_pysqlite_cextensions 6527 -test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.4_sqlite_pysqlite_nocextensions 6527 +test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 2.7_sqlite_pysqlite_cextensions 6426 +test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 2.7_sqlite_pysqlite_nocextensions 6426 +test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_sqlite_pysqlite_cextensions 6428 +test.aaa_profiling.test_orm.AttributeOverheadTest.test_collection_append_remove 3.3_sqlite_pysqlite_nocextensions 6428 # TEST: test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_cextensions 31372 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_nocextensions 40389 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_mysql_mysqlconnector_cextensions 111690 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_mysql_mysqlconnector_nocextensions 120693 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_postgresql_psycopg2_cextensions 32222 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_postgresql_psycopg2_nocextensions 41225 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_cextensions 32411 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_nocextensions 41414 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.4_mysql_mysqlconnector_cextensions 91564 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.4_postgresql_psycopg2_cextensions 32222 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.4_sqlite_pysqlite_cextensions 32411 -test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.4_sqlite_pysqlite_nocextensions 41414 +test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_cextensions 31373 +test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_nocextensions 40336 +test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_cextensions 32398 +test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 3.3_sqlite_pysqlite_nocextensions 41401 # TEST: test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_cextensions 31164 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_nocextensions 34169 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_mysql_mysqlconnector_cextensions 57315 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_mysql_mysqlconnector_nocextensions 60318 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_postgresql_psycopg2_cextensions 32099 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_postgresql_psycopg2_nocextensions 35102 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_cextensions 32210 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_nocextensions 35213 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.4_mysql_mysqlconnector_cextensions 55266 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.4_postgresql_psycopg2_cextensions 32099 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.4_sqlite_pysqlite_cextensions 32210 -test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.4_sqlite_pysqlite_nocextensions 35213 +test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_cextensions 31165 +test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_nocextensions 34170 +test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_cextensions 32197 +test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 3.3_sqlite_pysqlite_nocextensions 35200 # TEST: test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 2.7_sqlite_pysqlite_cextensions 17987 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 2.7_sqlite_pysqlite_nocextensions 17987 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_mysql_mysqlconnector_cextensions 18987 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_mysql_mysqlconnector_nocextensions 18987 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_postgresql_psycopg2_cextensions 18987 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_postgresql_psycopg2_nocextensions 18987 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_sqlite_pysqlite_cextensions 18987 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_sqlite_pysqlite_nocextensions 18987 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.4_mysql_mysqlconnector_cextensions 18987 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.4_postgresql_psycopg2_cextensions 18987 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.4_sqlite_pysqlite_cextensions 18987 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.4_sqlite_pysqlite_nocextensions 18987 +test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 2.7_sqlite_pysqlite_cextensions 17988 +test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 2.7_sqlite_pysqlite_nocextensions 17988 +test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_sqlite_pysqlite_cextensions 18988 +test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 3.3_sqlite_pysqlite_nocextensions 18988 # TEST: test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_sqlite_pysqlite_cextensions 162360 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_sqlite_pysqlite_nocextensions 165110 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_mysql_mysqlconnector_cextensions 203865 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_mysql_mysqlconnector_nocextensions 205567 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_postgresql_psycopg2_cextensions 127615 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_postgresql_psycopg2_nocextensions 129365 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_sqlite_pysqlite_cextensions 170115 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_sqlite_pysqlite_nocextensions 171865 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.4_mysql_mysqlconnector_cextensions 184817 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.4_postgresql_psycopg2_cextensions 127567 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.4_sqlite_pysqlite_cextensions 170067 -test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.4_sqlite_pysqlite_nocextensions 171865 +test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_sqlite_pysqlite_cextensions 162315 +test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 2.7_sqlite_pysqlite_nocextensions 165111 +test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_sqlite_pysqlite_cextensions 169566 +test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_no_identity 3.3_sqlite_pysqlite_nocextensions 171364 # TEST: test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks -test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_sqlite_pysqlite_cextensions 22448 -test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_sqlite_pysqlite_nocextensions 22662 -test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_mysql_mysqlconnector_cextensions 26042 -test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_mysql_mysqlconnector_nocextensions 26246 -test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_postgresql_psycopg2_cextensions 20541 -test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_postgresql_psycopg2_nocextensions 20685 -test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_sqlite_pysqlite_cextensions 23330 -test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_sqlite_pysqlite_nocextensions 23534 -test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.4_mysql_mysqlconnector_cextensions 24861 -test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.4_postgresql_psycopg2_cextensions 20377 -test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.4_sqlite_pysqlite_cextensions 23282 -test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.4_sqlite_pysqlite_nocextensions 23452 +test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_sqlite_pysqlite_cextensions 22288 +test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 2.7_sqlite_pysqlite_nocextensions 22530 +test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_sqlite_pysqlite_cextensions 23067 +test.aaa_profiling.test_orm.MergeBackrefsTest.test_merge_pending_with_all_pks 3.3_sqlite_pysqlite_nocextensions 23271 # TEST: test.aaa_profiling.test_orm.MergeTest.test_merge_load -test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_sqlite_pysqlite_cextensions 1600 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_sqlite_pysqlite_nocextensions 1625 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_mysql_mysqlconnector_cextensions 2268 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_mysql_mysqlconnector_nocextensions 2283 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_postgresql_psycopg2_cextensions 1394 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_postgresql_psycopg2_nocextensions 1409 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_sqlite_pysqlite_cextensions 1669 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_sqlite_pysqlite_nocextensions 1684 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.4_mysql_mysqlconnector_cextensions 2139 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.4_postgresql_psycopg2_cextensions 1394 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.4_sqlite_pysqlite_cextensions 1669 -test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.4_sqlite_pysqlite_nocextensions 1684 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_sqlite_pysqlite_cextensions 1601 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 2.7_sqlite_pysqlite_nocextensions 1626 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_sqlite_pysqlite_cextensions 1656 +test.aaa_profiling.test_orm.MergeTest.test_merge_load 3.3_sqlite_pysqlite_nocextensions 1671 # TEST: test.aaa_profiling.test_orm.MergeTest.test_merge_no_load -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_sqlite_pysqlite_cextensions 116,17 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_sqlite_pysqlite_nocextensions 116,17 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_mysql_mysqlconnector_cextensions 128,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_mysql_mysqlconnector_nocextensions 128,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_postgresql_psycopg2_cextensions 128,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_postgresql_psycopg2_nocextensions 128,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_sqlite_pysqlite_cextensions 128,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_sqlite_pysqlite_nocextensions 128,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_mysql_mysqlconnector_cextensions 128,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_postgresql_psycopg2_cextensions 128,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_sqlite_pysqlite_cextensions 128,18 -test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.4_sqlite_pysqlite_nocextensions 128,18 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_sqlite_pysqlite_cextensions 117,18 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 2.7_sqlite_pysqlite_nocextensions 117,18 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_sqlite_pysqlite_cextensions 122,19 +test.aaa_profiling.test_orm.MergeTest.test_merge_no_load 3.3_sqlite_pysqlite_nocextensions 122,19 # TEST: test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect -test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 2.7_sqlite_pysqlite_cextensions 90 -test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 2.7_sqlite_pysqlite_nocextensions 90 -test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 3.3_mysql_mysqlconnector_cextensions 77 -test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 3.3_mysql_mysqlconnector_nocextensions 77 -test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 3.3_postgresql_psycopg2_cextensions 77 -test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 3.3_postgresql_psycopg2_nocextensions 77 -test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 3.3_sqlite_pysqlite_cextensions 77 -test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 3.3_sqlite_pysqlite_nocextensions 77 -test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 3.4_mysql_mysqlconnector_cextensions 77 -test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 3.4_postgresql_psycopg2_cextensions 77 -test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 3.4_sqlite_pysqlite_cextensions 77 -test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 3.4_sqlite_pysqlite_nocextensions 77 +test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 2.7_sqlite_pysqlite_cextensions 91 +test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 2.7_sqlite_pysqlite_nocextensions 91 +test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 3.3_sqlite_pysqlite_cextensions 78 +test.aaa_profiling.test_pool.QueuePoolTest.test_first_connect 3.3_sqlite_pysqlite_nocextensions 78 # TEST: test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect -test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 2.7_sqlite_pysqlite_cextensions 30 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 2.7_sqlite_pysqlite_nocextensions 30 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 3.3_mysql_mysqlconnector_cextensions 23 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 3.3_mysql_mysqlconnector_nocextensions 23 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 3.3_postgresql_psycopg2_cextensions 23 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 3.3_postgresql_psycopg2_nocextensions 23 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 3.3_sqlite_pysqlite_cextensions 23 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 3.3_sqlite_pysqlite_nocextensions 23 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 3.4_mysql_mysqlconnector_cextensions 23 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 3.4_postgresql_psycopg2_cextensions 23 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 3.4_sqlite_pysqlite_cextensions 23 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 3.4_sqlite_pysqlite_nocextensions 23 +test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 2.7_sqlite_pysqlite_cextensions 31 +test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 2.7_sqlite_pysqlite_nocextensions 31 +test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 3.3_sqlite_pysqlite_cextensions 24 +test.aaa_profiling.test_pool.QueuePoolTest.test_second_connect 3.3_sqlite_pysqlite_nocextensions 24 # TEST: test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect -test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 2.7_sqlite_pysqlite_cextensions 7 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 2.7_sqlite_pysqlite_nocextensions 7 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 3.3_mysql_mysqlconnector_cextensions 8 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 3.3_mysql_mysqlconnector_nocextensions 8 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 3.3_postgresql_psycopg2_cextensions 8 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 3.3_postgresql_psycopg2_nocextensions 8 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 3.3_sqlite_pysqlite_cextensions 8 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 3.3_sqlite_pysqlite_nocextensions 8 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 3.4_mysql_mysqlconnector_cextensions 8 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 3.4_postgresql_psycopg2_cextensions 8 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 3.4_sqlite_pysqlite_cextensions 8 -test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 3.4_sqlite_pysqlite_nocextensions 8 +test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 2.7_sqlite_pysqlite_cextensions 8 +test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 2.7_sqlite_pysqlite_nocextensions 8 +test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 3.3_sqlite_pysqlite_cextensions 9 +test.aaa_profiling.test_pool.QueuePoolTest.test_second_samethread_connect 3.3_sqlite_pysqlite_nocextensions 9 # TEST: test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_mysql_mysqlconnector_cextensions 42 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_mysql_mysqlconnector_nocextensions 44 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_postgresql_psycopg2_cextensions 42 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_postgresql_psycopg2_nocextensions 44 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_sqlite_pysqlite_cextensions 42 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_sqlite_pysqlite_nocextensions 44 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_mysql_mysqlconnector_cextensions 42 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_mysql_mysqlconnector_nocextensions 42 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_postgresql_psycopg2_cextensions 42 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_postgresql_psycopg2_nocextensions 42 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_sqlite_pysqlite_cextensions 42 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_sqlite_pysqlite_nocextensions 42 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_mysql_mysqlconnector_cextensions 42 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_mysql_mysqlconnector_nocextensions 42 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_postgresql_psycopg2_cextensions 42 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_postgresql_psycopg2_nocextensions 42 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_sqlite_pysqlite_cextensions 42 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.4_sqlite_pysqlite_nocextensions 42 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_mysql_mysqlconnector_cextensions 43 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_mysql_mysqlconnector_nocextensions 45 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_postgresql_psycopg2_cextensions 43 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_postgresql_psycopg2_nocextensions 45 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_sqlite_pysqlite_cextensions 43 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 2.7_sqlite_pysqlite_nocextensions 45 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_mysql_mysqlconnector_cextensions 43 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_mysql_mysqlconnector_nocextensions 43 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_postgresql_psycopg2_cextensions 43 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_postgresql_psycopg2_nocextensions 43 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_sqlite_pysqlite_cextensions 43 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_connection_execute 3.3_sqlite_pysqlite_nocextensions 43 # TEST: test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_mysql_mysqlconnector_cextensions 77 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_mysql_mysqlconnector_nocextensions 79 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_postgresql_psycopg2_cextensions 77 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_postgresql_psycopg2_nocextensions 79 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_sqlite_pysqlite_cextensions 77 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_sqlite_pysqlite_nocextensions 79 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_mysql_mysqlconnector_cextensions 77 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_mysql_mysqlconnector_nocextensions 77 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_postgresql_psycopg2_cextensions 77 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_postgresql_psycopg2_nocextensions 77 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_sqlite_pysqlite_cextensions 77 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_sqlite_pysqlite_nocextensions 77 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_mysql_mysqlconnector_cextensions 77 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_mysql_mysqlconnector_nocextensions 77 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_postgresql_psycopg2_cextensions 77 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_postgresql_psycopg2_nocextensions 77 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_sqlite_pysqlite_cextensions 77 -test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.4_sqlite_pysqlite_nocextensions 77 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_mysql_mysqlconnector_cextensions 78 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_mysql_mysqlconnector_nocextensions 80 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_postgresql_psycopg2_cextensions 78 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_postgresql_psycopg2_nocextensions 80 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_sqlite_pysqlite_cextensions 78 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 2.7_sqlite_pysqlite_nocextensions 80 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_mysql_mysqlconnector_cextensions 78 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_mysql_mysqlconnector_nocextensions 78 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_postgresql_psycopg2_cextensions 78 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_postgresql_psycopg2_nocextensions 78 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_sqlite_pysqlite_cextensions 78 +test.aaa_profiling.test_resultset.ExecutionTest.test_minimal_engine_execute 3.3_sqlite_pysqlite_nocextensions 78 # TEST: test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_mysql_mysqlconnector_cextensions 14 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_mysql_mysqlconnector_nocextensions 14 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_postgresql_psycopg2_cextensions 14 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_postgresql_psycopg2_nocextensions 14 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_sqlite_pysqlite_cextensions 14 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_sqlite_pysqlite_nocextensions 14 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_mysql_mysqlconnector_cextensions 15 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_mysql_mysqlconnector_nocextensions 15 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_postgresql_psycopg2_cextensions 15 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_postgresql_psycopg2_nocextensions 15 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_sqlite_pysqlite_cextensions 15 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_sqlite_pysqlite_nocextensions 15 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_mysql_mysqlconnector_cextensions 15 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_mysql_mysqlconnector_nocextensions 15 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_postgresql_psycopg2_cextensions 15 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_postgresql_psycopg2_nocextensions 15 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_sqlite_pysqlite_cextensions 15 -test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.4_sqlite_pysqlite_nocextensions 15 +test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_mysql_mysqlconnector_cextensions 15 +test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_mysql_mysqlconnector_nocextensions 15 +test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_postgresql_psycopg2_cextensions 15 +test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_postgresql_psycopg2_nocextensions 15 +test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_sqlite_pysqlite_cextensions 15 +test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 2.7_sqlite_pysqlite_nocextensions 15 +test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_mysql_mysqlconnector_cextensions 16 +test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_mysql_mysqlconnector_nocextensions 16 +test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_postgresql_psycopg2_cextensions 16 +test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_postgresql_psycopg2_nocextensions 16 +test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_sqlite_pysqlite_cextensions 16 +test.aaa_profiling.test_resultset.ResultSetTest.test_contains_doesnt_compile 3.3_sqlite_pysqlite_nocextensions 16 # TEST: test.aaa_profiling.test_resultset.ResultSetTest.test_string -test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_mysql_mysqlconnector_cextensions 92958 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_mysql_mysqlconnector_nocextensions 107978 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_postgresql_psycopg2_cextensions 20500 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_postgresql_psycopg2_nocextensions 35520 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_sqlite_pysqlite_cextensions 456 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_sqlite_pysqlite_nocextensions 15476 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_mysql_mysqlconnector_cextensions 109145 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_mysql_mysqlconnector_nocextensions 123145 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_postgresql_psycopg2_cextensions 498 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_postgresql_psycopg2_nocextensions 14498 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_sqlite_pysqlite_cextensions 471 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_sqlite_pysqlite_nocextensions 14471 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_mysql_mysqlconnector_cextensions 79885 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_mysql_mysqlconnector_nocextensions 93885 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_postgresql_psycopg2_cextensions 498 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_postgresql_psycopg2_nocextensions 14498 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_sqlite_pysqlite_cextensions 471 -test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.4_sqlite_pysqlite_nocextensions 14471 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_mysql_mysqlconnector_cextensions 92959 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_mysql_mysqlconnector_nocextensions 107979 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_postgresql_psycopg2_cextensions 20501 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_postgresql_psycopg2_nocextensions 35521 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_sqlite_pysqlite_cextensions 457 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 2.7_sqlite_pysqlite_nocextensions 15477 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_mysql_mysqlconnector_cextensions 109136 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_mysql_mysqlconnector_nocextensions 123136 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_postgresql_psycopg2_cextensions 489 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_postgresql_psycopg2_nocextensions 14489 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_sqlite_pysqlite_cextensions 462 +test.aaa_profiling.test_resultset.ResultSetTest.test_string 3.3_sqlite_pysqlite_nocextensions 14462 # TEST: test.aaa_profiling.test_resultset.ResultSetTest.test_unicode -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_mysql_mysqlconnector_cextensions 92958 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_mysql_mysqlconnector_nocextensions 107978 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_postgresql_psycopg2_cextensions 20500 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_postgresql_psycopg2_nocextensions 35520 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_sqlite_pysqlite_cextensions 456 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_sqlite_pysqlite_nocextensions 15476 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_mysql_mysqlconnector_cextensions 109145 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_mysql_mysqlconnector_nocextensions 123145 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_postgresql_psycopg2_cextensions 498 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_postgresql_psycopg2_nocextensions 14498 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_sqlite_pysqlite_cextensions 471 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_sqlite_pysqlite_nocextensions 14471 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_mysql_mysqlconnector_cextensions 79885 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_mysql_mysqlconnector_nocextensions 93885 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_postgresql_psycopg2_cextensions 498 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_postgresql_psycopg2_nocextensions 14498 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_sqlite_pysqlite_cextensions 471 -test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.4_sqlite_pysqlite_nocextensions 14471 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_mysql_mysqlconnector_cextensions 92959 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_mysql_mysqlconnector_nocextensions 107979 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_postgresql_psycopg2_cextensions 20501 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_postgresql_psycopg2_nocextensions 35521 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_sqlite_pysqlite_cextensions 457 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 2.7_sqlite_pysqlite_nocextensions 15477 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_mysql_mysqlconnector_cextensions 109136 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_mysql_mysqlconnector_nocextensions 123136 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_postgresql_psycopg2_cextensions 489 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_postgresql_psycopg2_nocextensions 14489 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_sqlite_pysqlite_cextensions 462 +test.aaa_profiling.test_resultset.ResultSetTest.test_unicode 3.3_sqlite_pysqlite_nocextensions 14462 -# TEST: test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_1a_populate +# TEST: test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_1a_populate 2.7_postgresql_psycopg2_cextensions 5562 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_1a_populate 2.7_postgresql_psycopg2_nocextensions 5606 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_1a_populate 3.3_postgresql_psycopg2_cextensions 5381 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_1a_populate 3.3_postgresql_psycopg2_nocextensions 5403 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_1a_populate 3.4_postgresql_psycopg2_cextensions 5381 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_1a_populate 3.4_postgresql_psycopg2_nocextensions 5403 +test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_cextensions 5562,277,3697,11893,1106,1968,2433 +test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_nocextensions 5606,277,3929,13595,1223,2011,2692 +test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.3_postgresql_psycopg2_cextensions 5238,259,3577,11529,1077,1886,2439 +test.aaa_profiling.test_zoomark.ZooMarkTest.test_invocation 3.3_postgresql_psycopg2_nocextensions 5260,259,3673,12701,1171,1893,2631 -# TEST: test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_2_insert +# TEST: test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_invocation -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_2_insert 2.7_postgresql_psycopg2_cextensions 277 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_2_insert 2.7_postgresql_psycopg2_nocextensions 277 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_2_insert 3.3_postgresql_psycopg2_cextensions 269 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_2_insert 3.3_postgresql_psycopg2_nocextensions 269 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_2_insert 3.4_postgresql_psycopg2_cextensions 269 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_2_insert 3.4_postgresql_psycopg2_nocextensions 269 - -# TEST: test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_3_properties - -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_3_properties 2.7_postgresql_psycopg2_cextensions 3697 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_3_properties 2.7_postgresql_psycopg2_nocextensions 3929 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_3_properties 3.3_postgresql_psycopg2_cextensions 3641 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_3_properties 3.3_postgresql_psycopg2_nocextensions 3737 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_3_properties 3.4_postgresql_psycopg2_cextensions 3641 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_3_properties 3.4_postgresql_psycopg2_nocextensions 3737 - -# TEST: test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_4_expressions - -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_4_expressions 2.7_postgresql_psycopg2_cextensions 11893 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_4_expressions 2.7_postgresql_psycopg2_nocextensions 13595 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_4_expressions 3.3_postgresql_psycopg2_cextensions 11751 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_4_expressions 3.3_postgresql_psycopg2_nocextensions 12923 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_4_expressions 3.4_postgresql_psycopg2_cextensions 11751 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_4_expressions 3.4_postgresql_psycopg2_nocextensions 12923 - -# TEST: test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_5_aggregates - -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_5_aggregates 2.7_postgresql_psycopg2_cextensions 1106 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_5_aggregates 2.7_postgresql_psycopg2_nocextensions 1223 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_5_aggregates 3.3_postgresql_psycopg2_cextensions 1077 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_5_aggregates 3.3_postgresql_psycopg2_nocextensions 1171 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_5_aggregates 3.4_postgresql_psycopg2_cextensions 1077 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_5_aggregates 3.4_postgresql_psycopg2_nocextensions 1171 - -# TEST: test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_6_editing - -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_6_editing 2.7_postgresql_psycopg2_cextensions 1968 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_6_editing 2.7_postgresql_psycopg2_nocextensions 2011 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_6_editing 3.3_postgresql_psycopg2_cextensions 1913 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_6_editing 3.3_postgresql_psycopg2_nocextensions 1920 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_6_editing 3.4_postgresql_psycopg2_cextensions 1913 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_6_editing 3.4_postgresql_psycopg2_nocextensions 1920 - -# TEST: test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_7_multiview - -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_7_multiview 2.7_postgresql_psycopg2_cextensions 2433 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_7_multiview 2.7_postgresql_psycopg2_nocextensions 2692 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_7_multiview 3.3_postgresql_psycopg2_cextensions 2449 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_7_multiview 3.3_postgresql_psycopg2_nocextensions 2641 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_7_multiview 3.4_postgresql_psycopg2_cextensions 2449 -test.aaa_profiling.test_zoomark.ZooMarkTest.test_profile_7_multiview 3.4_postgresql_psycopg2_nocextensions 2641 - -# TEST: test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_1a_populate - -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_1a_populate 2.7_postgresql_psycopg2_cextensions 6276 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_1a_populate 2.7_postgresql_psycopg2_nocextensions 6395 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_1a_populate 3.3_postgresql_psycopg2_cextensions 6412 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_1a_populate 3.3_postgresql_psycopg2_nocextensions 6497 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_1a_populate 3.4_postgresql_psycopg2_cextensions 6412 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_1a_populate 3.4_postgresql_psycopg2_nocextensions 6497 - -# TEST: test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_2_insert - -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_2_insert 2.7_postgresql_psycopg2_cextensions 403 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_2_insert 2.7_postgresql_psycopg2_nocextensions 410 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_2_insert 3.3_postgresql_psycopg2_cextensions 401 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_2_insert 3.3_postgresql_psycopg2_nocextensions 406 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_2_insert 3.4_postgresql_psycopg2_cextensions 401 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_2_insert 3.4_postgresql_psycopg2_nocextensions 406 - -# TEST: test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_3_properties - -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_3_properties 2.7_postgresql_psycopg2_cextensions 6878 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_3_properties 2.7_postgresql_psycopg2_nocextensions 7110 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_3_properties 3.3_postgresql_psycopg2_cextensions 7008 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_3_properties 3.3_postgresql_psycopg2_nocextensions 7112 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_3_properties 3.4_postgresql_psycopg2_cextensions 7008 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_3_properties 3.4_postgresql_psycopg2_nocextensions 7112 - -# TEST: test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_4_expressions - -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_4_expressions 2.7_postgresql_psycopg2_cextensions 19521 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_4_expressions 2.7_postgresql_psycopg2_nocextensions 20952 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_4_expressions 3.3_postgresql_psycopg2_cextensions 19868 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_4_expressions 3.3_postgresql_psycopg2_nocextensions 20895 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_4_expressions 3.4_postgresql_psycopg2_cextensions 19868 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_4_expressions 3.4_postgresql_psycopg2_nocextensions 20895 - -# TEST: test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_5_aggregates - -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_5_aggregates 2.7_postgresql_psycopg2_cextensions 1118 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_5_aggregates 2.7_postgresql_psycopg2_nocextensions 1226 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_5_aggregates 3.3_postgresql_psycopg2_cextensions 1091 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_5_aggregates 3.3_postgresql_psycopg2_nocextensions 1177 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_5_aggregates 3.4_postgresql_psycopg2_cextensions 1091 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_5_aggregates 3.4_postgresql_psycopg2_nocextensions 1177 - -# TEST: test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_6_editing - -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_6_editing 2.7_postgresql_psycopg2_cextensions 2733 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_6_editing 2.7_postgresql_psycopg2_nocextensions 2796 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_6_editing 3.3_postgresql_psycopg2_cextensions 2784 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_6_editing 3.3_postgresql_psycopg2_nocextensions 2811 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_6_editing 3.4_postgresql_psycopg2_cextensions 2784 -test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_profile_6_editing 3.4_postgresql_psycopg2_nocextensions 2811 +test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_cextensions 5908,396,6878,19521,1118,2725 +test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_invocation 2.7_postgresql_psycopg2_nocextensions 5999,401,7110,20952,1226,2790 +test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_invocation 3.3_postgresql_psycopg2_cextensions 5816,383,6928,19676,1091,2753 +test.aaa_profiling.test_zoomark_orm.ZooMarkTest.test_invocation 3.3_postgresql_psycopg2_nocextensions 5886,388,7032,20703,1177,2782 From ef6042ff461e490c2a3040f18f0a3688b2e601a0 Mon Sep 17 00:00:00 2001 From: Malik Diarra Date: Wed, 13 Aug 2014 01:39:09 +0200 Subject: [PATCH 29/55] Adding a tablespace options for postgresql create table --- lib/sqlalchemy/dialects/postgresql/base.py | 10 +++++++++- test/dialect/postgresql/test_compiler.py | 6 ++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index c2b1d66f43..0f008642e8 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -1448,6 +1448,13 @@ class PGDDLCompiler(compiler.DDLCompiler): text += self.define_constraint_deferrability(constraint) return text + def post_create_table(self, table): + table_opts = [] + if table.dialect_options['postgresql']['tablespace']: + table_opts.append('TABLESPACE %s' % table.dialect_options['postgresql']['tablespace']) + + return ' '.join(table_opts) + class PGTypeCompiler(compiler.GenericTypeCompiler): @@ -1707,7 +1714,8 @@ class PGDialect(default.DefaultDialect): "ops": {} }), (schema.Table, { - "ignore_search_path": False + "ignore_search_path": False, + "tablespace": None }) ] diff --git a/test/dialect/postgresql/test_compiler.py b/test/dialect/postgresql/test_compiler.py index b08fb01603..301f80fd44 100644 --- a/test/dialect/postgresql/test_compiler.py +++ b/test/dialect/postgresql/test_compiler.py @@ -166,6 +166,12 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): "VARCHAR(1), CHECK (somecolumn IN ('x', " "'y', 'z')))") + def test_create_table_with_tablespace(self): + m = MetaData() + tbl = Table('atable', m, Column("id", Integer), postgresql_tablespace = 'sometablespace') + self.assert_compile(schema.CreateTable(tbl), + "CREATE TABLE atable (id INTEGER)TABLESPACE sometablespace") + def test_create_partial_index(self): m = MetaData() tbl = Table('testtbl', m, Column('data', Integer)) From d6873904c40134df787ffe5459d61d3663bf5d5f Mon Sep 17 00:00:00 2001 From: Malik Diarra Date: Sun, 17 Aug 2014 00:39:36 +0200 Subject: [PATCH 30/55] Adding oids and on_commit table options --- lib/sqlalchemy/dialects/postgresql/base.py | 12 +++++++++++- test/dialect/postgresql/test_compiler.py | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index 0f008642e8..9b30bf8949 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -1450,6 +1450,14 @@ class PGDDLCompiler(compiler.DDLCompiler): def post_create_table(self, table): table_opts = [] + if table.dialect_options['postgresql']['withoids'] is not None: + if table.dialect_options['postgresql']['withoids']: + table_opts.append('WITH OIDS') + else: + table_opts.append('WITHOUT OIDS') + if table.dialect_options['postgresql']['on_commit']: + on_commit_options = table.dialect_options['postgresql']['on_commit'].replace("_", " ").upper() + table_opts.append('ON COMMIT %s' % on_commit_options) if table.dialect_options['postgresql']['tablespace']: table_opts.append('TABLESPACE %s' % table.dialect_options['postgresql']['tablespace']) @@ -1715,7 +1723,9 @@ class PGDialect(default.DefaultDialect): }), (schema.Table, { "ignore_search_path": False, - "tablespace": None + "tablespace": None, + "withoids" : None, + "on_commit" : None, }) ] diff --git a/test/dialect/postgresql/test_compiler.py b/test/dialect/postgresql/test_compiler.py index 301f80fd44..b439ab916f 100644 --- a/test/dialect/postgresql/test_compiler.py +++ b/test/dialect/postgresql/test_compiler.py @@ -172,6 +172,28 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): self.assert_compile(schema.CreateTable(tbl), "CREATE TABLE atable (id INTEGER)TABLESPACE sometablespace") + def test_create_table_with_oids(self): + m = MetaData() + tbl = Table('atable', m, Column("id", Integer), postgresql_withoids = True, ) + self.assert_compile(schema.CreateTable(tbl), + "CREATE TABLE atable (id INTEGER)WITH OIDS") + + tbl2 = Table('anothertable', m, Column("id", Integer), postgresql_withoids = False, ) + self.assert_compile(schema.CreateTable(tbl2), + "CREATE TABLE anothertable (id INTEGER)WITHOUT OIDS") + + def create_table_with_oncommit_option(self): + m = MetaData() + tbl = Table('atable', m, Column("id", Integer), postgresql_on_commit = "drop") + self.assert_compile(schema.CreateTable(tbl), + "CREATE TABLE atable (id INTEGER) ON COMMIT DROP") + + def create_table_with_multiple_options(self): + m = MetaData() + tbl = Table('atable', m, Column("id", Integer), postgresql_tablespace = 'sometablespace', postgresql_withoids = False, postgresql_on_commit = "preserve_rows") + self.assert_compile(schema.CreateTable(tbl), + "CREATE TABLE atable (id INTEGER)WITHOUT OIDS ON COMMIT PRESERVE ROWS TABLESPACE sometablespace") + def test_create_partial_index(self): m = MetaData() tbl = Table('testtbl', m, Column('data', Integer)) From 8e03430acdb98d7e5fef4a48a3120b928ed3266d Mon Sep 17 00:00:00 2001 From: Malik Diarra Date: Sun, 17 Aug 2014 01:39:14 +0200 Subject: [PATCH 31/55] quoting tablespace name in create table command in postgresql dialect --- lib/sqlalchemy/dialects/postgresql/base.py | 3 ++- test/dialect/postgresql/test_compiler.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index 9b30bf8949..b09eaba725 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -1459,7 +1459,8 @@ class PGDDLCompiler(compiler.DDLCompiler): on_commit_options = table.dialect_options['postgresql']['on_commit'].replace("_", " ").upper() table_opts.append('ON COMMIT %s' % on_commit_options) if table.dialect_options['postgresql']['tablespace']: - table_opts.append('TABLESPACE %s' % table.dialect_options['postgresql']['tablespace']) + tablespace_name = table.dialect_options['postgresql']['tablespace'] + table_opts.append('TABLESPACE %s' % self.preparer.quote(tablespace_name)) return ' '.join(table_opts) diff --git a/test/dialect/postgresql/test_compiler.py b/test/dialect/postgresql/test_compiler.py index b439ab916f..e4a8c02eb0 100644 --- a/test/dialect/postgresql/test_compiler.py +++ b/test/dialect/postgresql/test_compiler.py @@ -172,6 +172,11 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): self.assert_compile(schema.CreateTable(tbl), "CREATE TABLE atable (id INTEGER)TABLESPACE sometablespace") + # testing quoting of tablespace name + tbl = Table('anothertable', m, Column("id", Integer), postgresql_tablespace = 'table') + self.assert_compile(schema.CreateTable(tbl), + 'CREATE TABLE anothertable (id INTEGER)TABLESPACE "table"') + def test_create_table_with_oids(self): m = MetaData() tbl = Table('atable', m, Column("id", Integer), postgresql_withoids = True, ) From 2de7f94739ec1873e1dce48797e1e6f12044cf4c Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 16 Aug 2014 20:17:08 -0400 Subject: [PATCH 32/55] - oldest screwup in the book, forgot the file --- lib/sqlalchemy/testing/replay_fixture.py | 167 +++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 lib/sqlalchemy/testing/replay_fixture.py diff --git a/lib/sqlalchemy/testing/replay_fixture.py b/lib/sqlalchemy/testing/replay_fixture.py new file mode 100644 index 0000000000..b8a0f6df1a --- /dev/null +++ b/lib/sqlalchemy/testing/replay_fixture.py @@ -0,0 +1,167 @@ +from . import fixtures +from . import profiling +from .. import util +import types +from collections import deque +import contextlib +from . import config +from sqlalchemy import MetaData +from sqlalchemy import create_engine +from sqlalchemy.orm import Session + + +class ReplayFixtureTest(fixtures.TestBase): + + @contextlib.contextmanager + def _dummy_ctx(self, *arg, **kw): + yield + + def test_invocation(self): + + dbapi_session = ReplayableSession() + creator = config.db.pool._creator + recorder = lambda: dbapi_session.recorder(creator()) + engine = create_engine( + config.db.url, creator=recorder, + use_native_hstore=False) + self.metadata = MetaData(engine) + self.engine = engine + self.session = Session(engine) + + self.setup_engine() + self._run_steps(ctx=self._dummy_ctx) + self.teardown_engine() + engine.dispose() + + player = lambda: dbapi_session.player() + engine = create_engine( + config.db.url, creator=player, + use_native_hstore=False) + + self.metadata = MetaData(engine) + self.engine = engine + self.session = Session(engine) + + self.setup_engine() + self._run_steps(ctx=profiling.count_functions) + self.teardown_engine() + + def setup_engine(self): + pass + + def teardown_engine(self): + pass + + def _run_steps(self, ctx): + raise NotImplementedError() + + +class ReplayableSession(object): + """A simple record/playback tool. + + This is *not* a mock testing class. It only records a session for later + playback and makes no assertions on call consistency whatsoever. It's + unlikely to be suitable for anything other than DB-API recording. + + """ + + Callable = object() + NoAttribute = object() + + if util.py2k: + Natives = set([getattr(types, t) + for t in dir(types) if not t.startswith('_')]).\ + difference([getattr(types, t) + for t in ('FunctionType', 'BuiltinFunctionType', + 'MethodType', 'BuiltinMethodType', + 'LambdaType', 'UnboundMethodType',)]) + else: + Natives = set([getattr(types, t) + for t in dir(types) if not t.startswith('_')]).\ + union([type(t) if not isinstance(t, type) + else t for t in __builtins__.values()]).\ + difference([getattr(types, t) + for t in ('FunctionType', 'BuiltinFunctionType', + 'MethodType', 'BuiltinMethodType', + 'LambdaType', )]) + + def __init__(self): + self.buffer = deque() + + def recorder(self, base): + return self.Recorder(self.buffer, base) + + def player(self): + return self.Player(self.buffer) + + class Recorder(object): + def __init__(self, buffer, subject): + self._buffer = buffer + self._subject = subject + + def __call__(self, *args, **kw): + subject, buffer = [object.__getattribute__(self, x) + for x in ('_subject', '_buffer')] + + result = subject(*args, **kw) + if type(result) not in ReplayableSession.Natives: + buffer.append(ReplayableSession.Callable) + return type(self)(buffer, result) + else: + buffer.append(result) + return result + + @property + def _sqla_unwrap(self): + return self._subject + + def __getattribute__(self, key): + try: + return object.__getattribute__(self, key) + except AttributeError: + pass + + subject, buffer = [object.__getattribute__(self, x) + for x in ('_subject', '_buffer')] + try: + result = type(subject).__getattribute__(subject, key) + except AttributeError: + buffer.append(ReplayableSession.NoAttribute) + raise + else: + if type(result) not in ReplayableSession.Natives: + buffer.append(ReplayableSession.Callable) + return type(self)(buffer, result) + else: + buffer.append(result) + return result + + class Player(object): + def __init__(self, buffer): + self._buffer = buffer + + def __call__(self, *args, **kw): + buffer = object.__getattribute__(self, '_buffer') + result = buffer.popleft() + if result is ReplayableSession.Callable: + return self + else: + return result + + @property + def _sqla_unwrap(self): + return None + + def __getattribute__(self, key): + try: + return object.__getattribute__(self, key) + except AttributeError: + pass + buffer = object.__getattribute__(self, '_buffer') + result = buffer.popleft() + if result is ReplayableSession.Callable: + return self + elif result is ReplayableSession.NoAttribute: + raise AttributeError(key) + else: + return result From 9eacc8d42ad49527c7fd0fe7e37100edba9eb1dc Mon Sep 17 00:00:00 2001 From: Malik Diarra Date: Sun, 17 Aug 2014 02:48:21 +0200 Subject: [PATCH 33/55] Correcting options name from withoids to with_oids --- lib/sqlalchemy/dialects/postgresql/base.py | 6 +++--- test/dialect/postgresql/test_compiler.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index b09eaba725..057a5e0727 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -1450,8 +1450,8 @@ class PGDDLCompiler(compiler.DDLCompiler): def post_create_table(self, table): table_opts = [] - if table.dialect_options['postgresql']['withoids'] is not None: - if table.dialect_options['postgresql']['withoids']: + if table.dialect_options['postgresql']['with_oids'] is not None: + if table.dialect_options['postgresql']['with_oids']: table_opts.append('WITH OIDS') else: table_opts.append('WITHOUT OIDS') @@ -1725,7 +1725,7 @@ class PGDialect(default.DefaultDialect): (schema.Table, { "ignore_search_path": False, "tablespace": None, - "withoids" : None, + "with_oids" : None, "on_commit" : None, }) ] diff --git a/test/dialect/postgresql/test_compiler.py b/test/dialect/postgresql/test_compiler.py index e4a8c02eb0..5d1fddb49f 100644 --- a/test/dialect/postgresql/test_compiler.py +++ b/test/dialect/postgresql/test_compiler.py @@ -179,11 +179,11 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): def test_create_table_with_oids(self): m = MetaData() - tbl = Table('atable', m, Column("id", Integer), postgresql_withoids = True, ) + tbl = Table('atable', m, Column("id", Integer), postgresql_with_oids = True, ) self.assert_compile(schema.CreateTable(tbl), "CREATE TABLE atable (id INTEGER)WITH OIDS") - tbl2 = Table('anothertable', m, Column("id", Integer), postgresql_withoids = False, ) + tbl2 = Table('anothertable', m, Column("id", Integer), postgresql_with_oids = False, ) self.assert_compile(schema.CreateTable(tbl2), "CREATE TABLE anothertable (id INTEGER)WITHOUT OIDS") @@ -195,7 +195,7 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): def create_table_with_multiple_options(self): m = MetaData() - tbl = Table('atable', m, Column("id", Integer), postgresql_tablespace = 'sometablespace', postgresql_withoids = False, postgresql_on_commit = "preserve_rows") + tbl = Table('atable', m, Column("id", Integer), postgresql_tablespace = 'sometablespace', postgresql_with_oids = False, postgresql_on_commit = "preserve_rows") self.assert_compile(schema.CreateTable(tbl), "CREATE TABLE atable (id INTEGER)WITHOUT OIDS ON COMMIT PRESERVE ROWS TABLESPACE sometablespace") From faa5a9067661039dcc8663e00bdcea2d098c9989 Mon Sep 17 00:00:00 2001 From: Malik Diarra Date: Sun, 17 Aug 2014 16:56:53 +0200 Subject: [PATCH 34/55] Adding postgres create table options documentation --- lib/sqlalchemy/dialects/postgresql/base.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index 057a5e0727..34932520f4 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -417,6 +417,22 @@ of :class:`.PGInspector`, which offers additional methods:: .. autoclass:: PGInspector :members: +PostgreSQL specific table options +--------------------------------- + +PostgreSQL provides several CREATE TABLE specific options allowing to +specify how table data are stored. The following options are currently +supported: ``TABLESPACE``, ``ON COMMIT``, ``WITH OIDS``. + +``postgresql_tablespace`` is probably the more common and allows to specify +where in the filesystem the data files for the table will be created (see +http://www.postgresql.org/docs/9.3/static/manage-ag-tablespaces.html) + +.. seealso:: + + `Postgresql CREATE TABLE options + `_ - + on the PostgreSQL website """ from collections import defaultdict From 530d3f07e0c1e70e0f9b80d3b5986253e06dcaf2 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 17 Aug 2014 20:06:16 -0400 Subject: [PATCH 35/55] - Fixed bug where attribute "set" events or columns with ``@validates`` would have events triggered within the flush process, when those columns were the targets of a "fetch and populate" operation, such as an autoincremented primary key, a Python side default, or a server-side default "eagerly" fetched via RETURNING. fixes #3167 --- doc/build/changelog/changelog_10.rst | 10 ++++++ doc/build/orm/mapper_config.rst | 6 ++++ lib/sqlalchemy/orm/mapper.py | 29 ++++++++++++------ lib/sqlalchemy/orm/persistence.py | 12 ++------ test/orm/test_unitofworkv2.py | 46 +++++++++++++++++++++++++++- 5 files changed, 83 insertions(+), 20 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index fb639ddf73..1cbbec3b3b 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -16,6 +16,16 @@ .. changelog:: :version: 1.0.0 + .. change:: + :tags: bug, orm + :tickets: 3167 + + Fixed bug where attribute "set" events or columns with + ``@validates`` would have events triggered within the flush process, + when those columns were the targets of a "fetch and populate" + operation, such as an autoincremented primary key, a Python side + default, or a server-side default "eagerly" fetched via RETURNING. + .. change:: :tags: bug, orm, py3k diff --git a/doc/build/orm/mapper_config.rst b/doc/build/orm/mapper_config.rst index 9139b53f0f..d0679c7212 100644 --- a/doc/build/orm/mapper_config.rst +++ b/doc/build/orm/mapper_config.rst @@ -667,6 +667,12 @@ issued when the ORM is populating the object:: assert '@' in address return address +.. versionchanged:: 1.0.0 - validators are no longer triggered within + the flush process when the newly fetched values for primary key + columns as well as some python- or server-side defaults are fetched. + Prior to 1.0, validators may be triggered in those cases as well. + + Validators also receive collection append events, when items are added to a collection:: diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index fc15769cd4..1e12918570 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -1189,14 +1189,6 @@ class Mapper(InspectionAttr): util.ordered_column_set(t.c).\ intersection(all_cols) - # determine cols that aren't expressed within our tables; mark these - # as "read only" properties which are refreshed upon INSERT/UPDATE - self._readonly_props = set( - self._columntoproperty[col] - for col in self._columntoproperty - if not hasattr(col, 'table') or - col.table not in self._cols_by_table) - # if explicit PK argument sent, add those columns to the # primary key mappings if self._primary_key_argument: @@ -1247,6 +1239,15 @@ class Mapper(InspectionAttr): self.primary_key = tuple(primary_key) self._log("Identified primary key columns: %s", primary_key) + # determine cols that aren't expressed within our tables; mark these + # as "read only" properties which are refreshed upon INSERT/UPDATE + self._readonly_props = set( + self._columntoproperty[col] + for col in self._columntoproperty + if self._columntoproperty[col] not in self._primary_key_props and + (not hasattr(col, 'table') or + col.table not in self._cols_by_table)) + def _configure_properties(self): # Column and other ClauseElement objects which are mapped @@ -2342,18 +2343,26 @@ class Mapper(InspectionAttr): dict_ = state.dict manager = state.manager return [ - manager[self._columntoproperty[col].key]. + manager[prop.key]. impl.get(state, dict_, attributes.PASSIVE_RETURN_NEVER_SET) - for col in self.primary_key + for prop in self._primary_key_props ] + @_memoized_configured_property + def _primary_key_props(self): + return [self._columntoproperty[col] for col in self.primary_key] + def _get_state_attr_by_column( self, state, dict_, column, passive=attributes.PASSIVE_RETURN_NEVER_SET): prop = self._columntoproperty[column] return state.manager[prop.key].impl.get(state, dict_, passive=passive) + def _set_committed_state_attr_by_column(self, state, dict_, column, value): + prop = self._columntoproperty[column] + state.manager[prop.key].impl.set_committed_value(state, dict_, value) + def _set_state_attr_by_column(self, state, dict_, column, value): prop = self._columntoproperty[column] state.manager[prop.key].impl.set(state, dict_, value, None) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index 8c9b677fea..d511c0816c 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -645,13 +645,7 @@ def _emit_insert_statements(base_mapper, uowtransaction, mapper._pks_by_table[table]): prop = mapper_rec._columntoproperty[col] if state_dict.get(prop.key) is None: - # TODO: would rather say: - # state_dict[prop.key] = pk - mapper_rec._set_state_attr_by_column( - state, - state_dict, - col, pk) - + state_dict[prop.key] = pk _postfetch( mapper_rec, uowtransaction, @@ -836,11 +830,11 @@ def _postfetch(mapper, uowtransaction, table, for col in returning_cols: if col.primary_key: continue - mapper._set_state_attr_by_column(state, dict_, col, row[col]) + dict_[mapper._columntoproperty[col].key] = row[col] for c in prefetch_cols: if c.key in params and c in mapper._columntoproperty: - mapper._set_state_attr_by_column(state, dict_, c, params[c.key]) + dict_[mapper._columntoproperty[c].key] = params[c.key] if postfetch_cols: state._expire_attributes(state.dict, diff --git a/test/orm/test_unitofworkv2.py b/test/orm/test_unitofworkv2.py index c643e6a870..122fe2514b 100644 --- a/test/orm/test_unitofworkv2.py +++ b/test/orm/test_unitofworkv2.py @@ -9,8 +9,9 @@ from sqlalchemy import Integer, String, ForeignKey, func from sqlalchemy.orm import mapper, relationship, backref, \ create_session, unitofwork, attributes,\ Session, exc as orm_exc - +from sqlalchemy.testing.mock import Mock from sqlalchemy.testing.assertsql import AllOf, CompiledSQL +from sqlalchemy import event class AssertsUOW(object): @@ -1703,3 +1704,46 @@ class LoadersUsingCommittedTest(UOWTest): sess.flush() except AvoidReferencialError: pass + + +class NoAttrEventInFlushTest(fixtures.MappedTest): + """test [ticket:3167]""" + + __backend__ = True + + @classmethod + def define_tables(cls, metadata): + Table( + 'test', metadata, + Column('id', Integer, primary_key=True, + test_needs_autoincrement=True), + Column('prefetch_val', Integer, default=5), + Column('returning_val', Integer, server_default="5") + ) + + @classmethod + def setup_classes(cls): + class Thing(cls.Basic): + pass + + @classmethod + def setup_mappers(cls): + Thing = cls.classes.Thing + + mapper(Thing, cls.tables.test, eager_defaults=True) + + def test_no_attr_events_flush(self): + Thing = self.classes.Thing + mock = Mock() + event.listen(Thing.id, "set", mock.id) + event.listen(Thing.prefetch_val, "set", mock.prefetch_val) + event.listen(Thing.returning_val, "set", mock.prefetch_val) + t1 = Thing() + s = Session() + s.add(t1) + s.flush() + + eq_(len(mock.mock_calls), 0) + eq_(t1.id, 1) + eq_(t1.prefetch_val, 5) + eq_(t1.returning_val, 5) From d39927ec20dd0b66f4ab3aab3e4e67b3814186ce Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 18 Aug 2014 12:50:29 -0400 Subject: [PATCH 36/55] - major simplification of _collect_update_commands. in particular, we only call upon the history API fully for primary key columns. We also now skip the whole step of looking at PK columns and using any history at all if no net changes are detected on the object. --- lib/sqlalchemy/orm/mapper.py | 13 +++ lib/sqlalchemy/orm/persistence.py | 138 ++++++++++++------------------ test/orm/test_dynamic.py | 10 +-- test/orm/test_unitofworkv2.py | 2 + 4 files changed, 76 insertions(+), 87 deletions(-) diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index 1e12918570..14dc5d7f87 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -1893,6 +1893,19 @@ class Mapper(InspectionAttr): """ + @_memoized_configured_property + def _propkey_to_col(self): + return dict( + ( + table, + dict( + (self._columntoproperty[col].key, col) + for col in columns + ) + ) + for table, columns in self._cols_by_table.items() + ) + @_memoized_configured_property def _col_to_propkey(self): return dict( diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index d511c0816c..228cfef3aa 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -304,96 +304,70 @@ def _collect_update_commands(base_mapper, uowtransaction, params = {} value_params = {} - hasdata = hasnull = False + propkey_to_col = mapper._propkey_to_col[table] - for col in mapper._cols_by_table[table]: - if col is mapper.version_id_col: - params[col._label] = \ - mapper._get_committed_state_attr_by_column( - row_switch or state, - row_switch and row_switch.dict - or state_dict, - col) + for propkey in set(propkey_to_col).intersection(state.committed_state): + value = state_dict[propkey] + col = propkey_to_col[propkey] - prop = mapper._columntoproperty[col] - history = state.manager[prop.key].impl.get_history( - state, state_dict, attributes.PASSIVE_NO_INITIALIZE - ) - if history.added: - params[col.key] = history.added[0] - hasdata = True + if not state.manager[propkey].impl.is_equal( + value, state.committed_state[propkey]): + if isinstance(value, sql.ClauseElement): + value_params[col] = value else: - if mapper.version_id_generator is not False: - val = mapper.version_id_generator(params[col._label]) - params[col.key] = val + params[col.key] = value - # HACK: check for history, in case the - # history is only - # in a different table than the one - # where the version_id_col is. - for prop in mapper._columntoproperty.values(): - history = ( - state.manager[prop.key].impl.get_history( - state, state_dict, - attributes.PASSIVE_NO_INITIALIZE)) - if history.added: - hasdata = True + if mapper.version_id_col is not None: + col = mapper.version_id_col + params[col._label] = \ + mapper._get_committed_state_attr_by_column( + row_switch if row_switch else state, + row_switch.dict if row_switch else state_dict, + col) + + if col.key not in params and \ + mapper.version_id_generator is not False: + val = mapper.version_id_generator(params[col._label]) + params[col.key] = val + + if not (params or value_params): + continue + + pk_params = {} + for col in pks: + propkey = mapper._columntoproperty[col].key + history = state.manager[propkey].impl.get_history( + state, state_dict, attributes.PASSIVE_OFF) + + if row_switch and not history.deleted and history.added: + # row switch present. convert a row that thought + # it would be an INSERT into an UPDATE, by removing + # the PK value from the SET clause and instead putting + # it in the WHERE clause. + del params[col.key] + pk_params[col._label] = history.added[0] + elif history.added: + # we're updating the PK value. + assert history.deleted, ( + "New PK value without an old one not " + "possible for an UPDATE") + # check if an UPDATE of the PK value + # has already occurred as a result of ON UPDATE CASCADE. + # If so, use the new value to locate the row. + if ("pk_cascaded", state, col) in uowtransaction.attributes: + pk_params[col._label] = history.added[0] + else: + # else, use the old value to locate the row + pk_params[col._label] = history.deleted[0] else: - prop = mapper._columntoproperty[col] - history = state.manager[prop.key].impl.get_history( - state, state_dict, - attributes.PASSIVE_OFF if col in pks else - attributes.PASSIVE_NO_INITIALIZE) - if history.added: - if isinstance(history.added[0], - sql.ClauseElement): - value_params[col] = history.added[0] - else: - value = history.added[0] - params[col.key] = value + pk_params[col._label] = history.unchanged[0] - if col in pks: - if history.deleted and \ - not row_switch: - # if passive_updates and sync detected - # this was a pk->pk sync, use the new - # value to locate the row, since the - # DB would already have set this - if ("pk_cascaded", state, col) in \ - uowtransaction.attributes: - value = history.added[0] - params[col._label] = value - else: - # use the old value to - # locate the row - value = history.deleted[0] - params[col._label] = value - hasdata = True - else: - # row switch logic can reach us here - # remove the pk from the update params - # so the update doesn't - # attempt to include the pk in the - # update statement - del params[col.key] - value = history.added[0] - params[col._label] = value - if value is None: - hasnull = True - else: - hasdata = True - elif col in pks: - value = history.unchanged[0] - if value is None: - hasnull = True - params[col._label] = value - - if hasdata: - if hasnull: + if params or value_params: + if None in pk_params.values(): raise orm_exc.FlushError( - "Can't update table " - "using NULL for primary " + "Can't update table using NULL for primary " "key value") + params.update(pk_params) update.append((state, state_dict, params, mapper, connection, value_params)) return update diff --git a/test/orm/test_dynamic.py b/test/orm/test_dynamic.py index bc47ba3f3c..950ff19539 100644 --- a/test/orm/test_dynamic.py +++ b/test/orm/test_dynamic.py @@ -509,10 +509,6 @@ class UOWTest( self.assert_sql_execution( testing.db, sess.flush, - CompiledSQL( - "SELECT users.id AS users_id, users.name AS users_name " - "FROM users WHERE users.id = :param_1", - lambda ctx: [{"param_1": u1_id}]), CompiledSQL( "SELECT addresses.id AS addresses_id, addresses.email_address " "AS addresses_email_address FROM addresses " @@ -523,7 +519,11 @@ class UOWTest( "UPDATE addresses SET user_id=:user_id WHERE addresses.id = " ":addresses_id", lambda ctx: [{'addresses_id': a2_id, 'user_id': None}] - ) + ), + CompiledSQL( + "SELECT users.id AS users_id, users.name AS users_name " + "FROM users WHERE users.id = :param_1", + lambda ctx: [{"param_1": u1_id}]), ) def test_rollback(self): diff --git a/test/orm/test_unitofworkv2.py b/test/orm/test_unitofworkv2.py index 122fe2514b..374a77237c 100644 --- a/test/orm/test_unitofworkv2.py +++ b/test/orm/test_unitofworkv2.py @@ -1277,6 +1277,8 @@ class RowswitchAccountingTest(fixtures.MappedTest): old = attributes.get_history(p3, 'child')[2][0] assert old in sess + # essentially no SQL should emit here, + # because we've replaced the row with another identical one sess.flush() assert p3.child._sa_instance_state.session_id == sess.hash_key From 06dec268e53e999bd348ef2ca148def066ca30d6 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 18 Aug 2014 16:32:48 -0400 Subject: [PATCH 37/55] - organize persistence methods in terms of generators, narrow down argument lists and generator items for each function down to just what each function needs. This will help for them to be of more multipurpose use for bulk operations --- lib/sqlalchemy/orm/persistence.py | 179 +++++++++++++++--------------- 1 file changed, 90 insertions(+), 89 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index 228cfef3aa..c7850ac1da 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -40,32 +40,58 @@ def save_obj(base_mapper, states, uowtransaction, single=False): save_obj(base_mapper, [state], uowtransaction, single=True) return - states_to_insert, states_to_update = _organize_states_for_save( - base_mapper, - states, - uowtransaction) - + states_to_update = [] + states_to_insert = [] cached_connections = _cached_connection_dict(base_mapper) + for (state, dict_, mapper, connection, + has_identity, row_switch) in _organize_states_for_save( + base_mapper, states, uowtransaction + ): + if has_identity or row_switch: + states_to_update.append( + (state, dict_, mapper, connection, + has_identity, row_switch) + ) + else: + states_to_insert.append( + (state, dict_, mapper, connection, + has_identity, row_switch) + ) + for table, mapper in base_mapper._sorted_tables.items(): - insert = _collect_insert_commands(base_mapper, uowtransaction, - table, states_to_insert) + if table not in mapper._pks_by_table: + continue + insert = ( + (state, state_dict, mapper, connection) + for state, state_dict, mapper, connection, has_identity, + row_switch in states_to_insert + ) + insert = _collect_insert_commands(table, insert) - update = _collect_update_commands(base_mapper, uowtransaction, - table, states_to_update) + update = ( + (state, state_dict, mapper, connection, row_switch) + for state, state_dict, mapper, connection, has_identity, + row_switch in states_to_update + ) + update = _collect_update_commands(uowtransaction, table, update) - if update: - _emit_update_statements(base_mapper, uowtransaction, - cached_connections, - mapper, table, update) + _emit_update_statements(base_mapper, uowtransaction, + cached_connections, + mapper, table, update) - if insert: - _emit_insert_statements(base_mapper, uowtransaction, - cached_connections, - mapper, table, insert) + _emit_insert_statements(base_mapper, uowtransaction, + cached_connections, + mapper, table, insert) - _finalize_insert_update_commands(base_mapper, uowtransaction, - states_to_insert, states_to_update) + _finalize_insert_update_commands( + base_mapper, uowtransaction, + ( + (state, state_dict, mapper, connection, has_identity) + for state, state_dict, mapper, connection, has_identity, + row_switch in states_to_insert + states_to_update + ) + ) def post_update(base_mapper, states, uowtransaction, post_update_cols): @@ -75,19 +101,20 @@ def post_update(base_mapper, states, uowtransaction, post_update_cols): """ cached_connections = _cached_connection_dict(base_mapper) - states_to_update = _organize_states_for_post_update( + states_to_update = list(_organize_states_for_post_update( base_mapper, - states, uowtransaction) + states, uowtransaction)) for table, mapper in base_mapper._sorted_tables.items(): + if table not in mapper._pks_by_table: + continue update = _collect_post_update_commands(base_mapper, uowtransaction, table, states_to_update, post_update_cols) - if update: - _emit_post_update_statements(base_mapper, uowtransaction, - cached_connections, - mapper, table, update) + _emit_post_update_statements(base_mapper, uowtransaction, + cached_connections, + mapper, table, update) def delete_obj(base_mapper, states, uowtransaction): @@ -100,19 +127,21 @@ def delete_obj(base_mapper, states, uowtransaction): cached_connections = _cached_connection_dict(base_mapper) - states_to_delete = _organize_states_for_delete( + states_to_delete = list(_organize_states_for_delete( base_mapper, states, - uowtransaction) + uowtransaction)) table_to_mapper = base_mapper._sorted_tables for table in reversed(list(table_to_mapper.keys())): + mapper = table_to_mapper[table] + if table not in mapper._pks_by_table: + continue + delete = _collect_delete_commands(base_mapper, uowtransaction, table, states_to_delete) - mapper = table_to_mapper[table] - _emit_delete_statements(base_mapper, uowtransaction, cached_connections, mapper, table, delete) @@ -133,9 +162,6 @@ def _organize_states_for_save(base_mapper, states, uowtransaction): """ - states_to_insert = [] - states_to_update = [] - for state, dict_, mapper, connection in _connections_for_states( base_mapper, uowtransaction, states): @@ -181,18 +207,8 @@ def _organize_states_for_save(base_mapper, states, uowtransaction): uowtransaction.remove_state_actions(existing) row_switch = existing - if not has_identity and not row_switch: - states_to_insert.append( - (state, dict_, mapper, connection, - has_identity, row_switch) - ) - else: - states_to_update.append( - (state, dict_, mapper, connection, - has_identity, row_switch) - ) - - return states_to_insert, states_to_update + yield (state, dict_, mapper, connection, + has_identity, row_switch) def _organize_states_for_post_update(base_mapper, states, @@ -205,8 +221,7 @@ def _organize_states_for_post_update(base_mapper, states, the execution per state. """ - return list(_connections_for_states(base_mapper, uowtransaction, - states)) + return _connections_for_states(base_mapper, uowtransaction, states) def _organize_states_for_delete(base_mapper, states, uowtransaction): @@ -217,28 +232,21 @@ def _organize_states_for_delete(base_mapper, states, uowtransaction): mapper, the connection to use for the execution per state. """ - states_to_delete = [] - for state, dict_, mapper, connection in _connections_for_states( base_mapper, uowtransaction, states): mapper.dispatch.before_delete(mapper, connection, state) - states_to_delete.append((state, dict_, mapper, - bool(state.key), connection)) - return states_to_delete + yield state, dict_, mapper, bool(state.key), connection -def _collect_insert_commands(base_mapper, uowtransaction, table, - states_to_insert): +def _collect_insert_commands(table, states_to_insert): """Identify sets of values to use in INSERT statements for a list of states. """ - insert = [] - for state, state_dict, mapper, connection, has_identity, \ - row_switch in states_to_insert: + for state, state_dict, mapper, connection in states_to_insert: if table not in mapper._pks_by_table: continue @@ -262,7 +270,7 @@ def _collect_insert_commands(base_mapper, uowtransaction, table, has_all_pks = mapper._pk_keys_by_table[table].issubset(params) - if base_mapper.eager_defaults: + if mapper.base_mapper.eager_defaults: has_all_defaults = mapper._server_default_cols[table].\ issubset(params) else: @@ -274,14 +282,13 @@ def _collect_insert_commands(base_mapper, uowtransaction, table, params[mapper.version_id_col.key] = \ mapper.version_id_generator(None) - insert.append((state, state_dict, params, mapper, - connection, value_params, has_all_pks, - has_all_defaults)) - return insert + yield ( + state, state_dict, params, mapper, + connection, value_params, has_all_pks, + has_all_defaults) -def _collect_update_commands(base_mapper, uowtransaction, - table, states_to_update): +def _collect_update_commands(uowtransaction, table, states_to_update): """Identify sets of values to use in UPDATE statements for a list of states. @@ -293,9 +300,7 @@ def _collect_update_commands(base_mapper, uowtransaction, """ - update = [] - for state, state_dict, mapper, connection, has_identity, \ - row_switch in states_to_update: + for state, state_dict, mapper, connection, row_switch in states_to_update: if table not in mapper._pks_by_table: continue @@ -368,9 +373,9 @@ def _collect_update_commands(base_mapper, uowtransaction, "Can't update table using NULL for primary " "key value") params.update(pk_params) - update.append((state, state_dict, params, mapper, - connection, value_params)) - return update + yield ( + state, state_dict, params, mapper, + connection, value_params) def _collect_post_update_commands(base_mapper, uowtransaction, table, @@ -380,7 +385,6 @@ def _collect_post_update_commands(base_mapper, uowtransaction, table, """ - update = [] for state, state_dict, mapper, connection in states_to_update: if table not in mapper._pks_by_table: continue @@ -405,9 +409,7 @@ def _collect_post_update_commands(base_mapper, uowtransaction, table, params[col.key] = value hasdata = True if hasdata: - update.append((state, state_dict, params, mapper, - connection)) - return update + yield params, connection def _collect_delete_commands(base_mapper, uowtransaction, table, @@ -415,15 +417,12 @@ def _collect_delete_commands(base_mapper, uowtransaction, table, """Identify values to use in DELETE statements for a list of states to be deleted.""" - delete = util.defaultdict(list) - for state, state_dict, mapper, has_identity, connection \ in states_to_delete: if not has_identity or table not in mapper._pks_by_table: continue params = {} - delete[connection].append(params) for col in mapper._pks_by_table[table]: params[col.key] = \ value = \ @@ -441,7 +440,7 @@ def _collect_delete_commands(base_mapper, uowtransaction, table, mapper._get_committed_state_attr_by_column( state, state_dict, mapper.version_id_col) - return delete + yield params, connection def _emit_update_statements(base_mapper, uowtransaction, @@ -481,8 +480,7 @@ def _emit_update_statements(base_mapper, uowtransaction, lambda rec: ( rec[4], tuple(sorted(rec[2])), - bool(rec[5])) - ): + bool(rec[5]))): rows = 0 records = list(records) @@ -652,11 +650,10 @@ def _emit_post_update_statements(base_mapper, uowtransaction, # also group them into common (connection, cols) sets # to support executemany(). for key, grouper in groupby( - update, lambda rec: (rec[4], list(rec[2].keys())) + update, lambda rec: (rec[1], sorted(rec[0])) ): connection = key[0] - multiparams = [params for state, state_dict, - params, mapper, conn in grouper] + multiparams = [params for params, conn in grouper] cached_connections[connection].\ execute(statement, multiparams) @@ -686,8 +683,15 @@ def _emit_delete_statements(base_mapper, uowtransaction, cached_connections, return table.delete(clause) - for connection, del_objects in delete.items(): - statement = base_mapper._memo(('delete', table), delete_stmt) + statement = base_mapper._memo(('delete', table), delete_stmt) + for connection, recs in groupby( + delete, + lambda rec: rec[1] + ): + del_objects = [ + params + for params, connection in recs + ] connection = cached_connections[connection] @@ -740,15 +744,12 @@ def _emit_delete_statements(base_mapper, uowtransaction, cached_connections, ) -def _finalize_insert_update_commands(base_mapper, uowtransaction, - states_to_insert, states_to_update): +def _finalize_insert_update_commands(base_mapper, uowtransaction, states): """finalize state on states that have been inserted or updated, including calling after_insert/after_update events. """ - for state, state_dict, mapper, connection, has_identity, \ - row_switch in states_to_insert + \ - states_to_update: + for state, state_dict, mapper, connection, has_identity in states: if mapper._readonly_props: readonly = state.unmodified_intersection( From 4ade138769a74ee2beda184e89d89238426d3741 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 18 Aug 2014 16:44:07 -0400 Subject: [PATCH 38/55] - further reorganize collect_insert_commands to distinguish between setting up given values vs. defaults. again trying to shoot for making this of more general use --- lib/sqlalchemy/orm/persistence.py | 34 +++++++++++++++++++------------ 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index c7850ac1da..f17b1d79cd 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -253,20 +253,28 @@ def _collect_insert_commands(table, states_to_insert): params = {} value_params = {} - for col, propkey in mapper._col_to_propkey[table]: - if propkey in state_dict: - value = state_dict[propkey] - if isinstance(value, sql.ClauseElement): - value_params[col.key] = value - elif value is not None or ( - not col.primary_key and - not col.server_default and - not col.default): - params[col.key] = value + + propkey_to_col = mapper._propkey_to_col[table] + + for propkey in set(propkey_to_col).intersection(state_dict): + value = state_dict[propkey] + col = propkey_to_col[propkey] + if value is None: + continue + elif isinstance(value, sql.ClauseElement): + value_params[col.key] = value else: - if not col.server_default \ - and not col.default and not col.primary_key: - params[col.key] = None + params[col.key] = value + + for colkey in ( + set( + col.key for col in + mapper._cols_by_table[table] + if not col.primary_key and + not col.server_default and not col.default + ).difference(params).difference(value_params) + ): + params[colkey] = None has_all_pks = mapper._pk_keys_by_table[table].issubset(params) From 4ed640ba907b529d79c634baf37792ce14e59805 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 18 Aug 2014 17:02:52 -0400 Subject: [PATCH 39/55] - move out checks for table in mapper._pks_by_table --- lib/sqlalchemy/orm/persistence.py | 48 ++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index f17b1d79cd..c949e47764 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -63,16 +63,18 @@ def save_obj(base_mapper, states, uowtransaction, single=False): if table not in mapper._pks_by_table: continue insert = ( - (state, state_dict, mapper, connection) - for state, state_dict, mapper, connection, has_identity, + (state, state_dict, sub_mapper, connection) + for state, state_dict, sub_mapper, connection, has_identity, row_switch in states_to_insert + if table in sub_mapper._pks_by_table ) insert = _collect_insert_commands(table, insert) update = ( - (state, state_dict, mapper, connection, row_switch) - for state, state_dict, mapper, connection, has_identity, + (state, state_dict, sub_mapper, connection, row_switch) + for state, state_dict, sub_mapper, connection, has_identity, row_switch in states_to_update + if table in sub_mapper._pks_by_table ) update = _collect_update_commands(uowtransaction, table, update) @@ -108,8 +110,16 @@ def post_update(base_mapper, states, uowtransaction, post_update_cols): for table, mapper in base_mapper._sorted_tables.items(): if table not in mapper._pks_by_table: continue + + update = ( + (state, state_dict, sub_mapper, connection) + for + state, state_dict, sub_mapper, connection in states_to_update + if table in sub_mapper._pks_by_table + ) + update = _collect_post_update_commands(base_mapper, uowtransaction, - table, states_to_update, + table, update, post_update_cols) _emit_post_update_statements(base_mapper, uowtransaction, @@ -139,8 +149,15 @@ def delete_obj(base_mapper, states, uowtransaction): if table not in mapper._pks_by_table: continue + delete = ( + (state, state_dict, sub_mapper, connection) + for state, state_dict, sub_mapper, has_identity, connection + in states_to_delete if table in sub_mapper._pks_by_table + and has_identity + ) + delete = _collect_delete_commands(base_mapper, uowtransaction, - table, states_to_delete) + table, delete) _emit_delete_statements(base_mapper, uowtransaction, cached_connections, mapper, table, delete) @@ -248,8 +265,7 @@ def _collect_insert_commands(table, states_to_insert): """ for state, state_dict, mapper, connection in states_to_insert: - if table not in mapper._pks_by_table: - continue + # assert table in mapper._pks_by_table params = {} value_params = {} @@ -309,8 +325,8 @@ def _collect_update_commands(uowtransaction, table, states_to_update): """ for state, state_dict, mapper, connection, row_switch in states_to_update: - if table not in mapper._pks_by_table: - continue + + # assert table in mapper._pks_by_table pks = mapper._pks_by_table[table] @@ -394,8 +410,9 @@ def _collect_post_update_commands(base_mapper, uowtransaction, table, """ for state, state_dict, mapper, connection in states_to_update: - if table not in mapper._pks_by_table: - continue + + # assert table in mapper._pks_by_table + pks = mapper._pks_by_table[table] params = {} hasdata = False @@ -425,10 +442,9 @@ def _collect_delete_commands(base_mapper, uowtransaction, table, """Identify values to use in DELETE statements for a list of states to be deleted.""" - for state, state_dict, mapper, has_identity, connection \ - in states_to_delete: - if not has_identity or table not in mapper._pks_by_table: - continue + for state, state_dict, mapper, connection in states_to_delete: + + # assert table in mapper._pks_by_table params = {} for col in mapper._pks_by_table[table]: From 399c03939768d4c8afb29ca1e091b046ea4fc88f Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 18 Aug 2014 17:12:06 -0400 Subject: [PATCH 40/55] - optimize collection of cols we insert as none --- lib/sqlalchemy/orm/mapper.py | 26 +++++++++++++------------- lib/sqlalchemy/orm/persistence.py | 10 ++-------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index 14dc5d7f87..89c092b580 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -1893,6 +1893,19 @@ class Mapper(InspectionAttr): """ + @_memoized_configured_property + def _insert_cols_as_none(self): + return dict( + ( + table, + frozenset( + col.key for col in columns + if not col.primary_key and + not col.server_default and not col.default) + ) + for table, columns in self._cols_by_table.items() + ) + @_memoized_configured_property def _propkey_to_col(self): return dict( @@ -1906,19 +1919,6 @@ class Mapper(InspectionAttr): for table, columns in self._cols_by_table.items() ) - @_memoized_configured_property - def _col_to_propkey(self): - return dict( - ( - table, - [ - (col, self._columntoproperty[col].key) - for col in columns - ] - ) - for table, columns in self._cols_by_table.items() - ) - @_memoized_configured_property def _pk_keys_by_table(self): return dict( diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index c949e47764..e36f87991d 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -282,14 +282,8 @@ def _collect_insert_commands(table, states_to_insert): else: params[col.key] = value - for colkey in ( - set( - col.key for col in - mapper._cols_by_table[table] - if not col.primary_key and - not col.server_default and not col.default - ).difference(params).difference(value_params) - ): + for colkey in mapper._insert_cols_as_none[table].\ + difference(params).difference(value_params): params[colkey] = None has_all_pks = mapper._pk_keys_by_table[table].issubset(params) From 28103e9a865860a46037ca82e634827f2329deb0 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 19 Aug 2014 16:14:57 -0400 Subject: [PATCH 41/55] - simplify PK logic in update for row switch --- lib/sqlalchemy/orm/mapper.py | 3 +++ lib/sqlalchemy/orm/persistence.py | 22 ++++++---------------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index 89c092b580..f22cac329c 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -2364,6 +2364,9 @@ class Mapper(InspectionAttr): @_memoized_configured_property def _primary_key_props(self): + # TODO: this should really be called "identity key props", + # as it does not necessarily include primary key columns within + # individual tables return [self._columntoproperty[col] for col in self.primary_key] def _get_state_attr_by_column( diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index e36f87991d..37b696d0f9 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -362,29 +362,19 @@ def _collect_update_commands(uowtransaction, table, states_to_update): history = state.manager[propkey].impl.get_history( state, state_dict, attributes.PASSIVE_OFF) - if row_switch and not history.deleted and history.added: - # row switch present. convert a row that thought - # it would be an INSERT into an UPDATE, by removing - # the PK value from the SET clause and instead putting - # it in the WHERE clause. - del params[col.key] - pk_params[col._label] = history.added[0] - elif history.added: - # we're updating the PK value. - assert history.deleted, ( - "New PK value without an old one not " - "possible for an UPDATE") - # check if an UPDATE of the PK value - # has already occurred as a result of ON UPDATE CASCADE. - # If so, use the new value to locate the row. - if ("pk_cascaded", state, col) in uowtransaction.attributes: + if history.added: + if not history.deleted or \ + ("pk_cascaded", state, col) in uowtransaction.attributes: pk_params[col._label] = history.added[0] + params.pop(col.key, None) else: # else, use the old value to locate the row pk_params[col._label] = history.deleted[0] + params[col.key] = history.added[0] else: pk_params[col._label] = history.unchanged[0] + if params or value_params: if None in pk_params.values(): raise orm_exc.FlushError( From cea97d1fae999001eb991ae1da9db226f2d3b5da Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 20 Aug 2014 09:08:59 -0400 Subject: [PATCH 42/55] - pep8 cleanup --- test/engine/test_execute.py | 1 - test/engine/test_logging.py | 37 +++++++++++++------------------------ 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/test/engine/test_execute.py b/test/engine/test_execute.py index f65168552a..d8e1c655e9 100644 --- a/test/engine/test_execute.py +++ b/test/engine/test_execute.py @@ -21,7 +21,6 @@ from sqlalchemy.testing import fixtures from sqlalchemy.testing.mock import Mock, call, patch from contextlib import contextmanager from sqlalchemy.util import nested -import logging.handlers # needed for logging tests to work correctly users, metadata, users_autoinc = None, None, None diff --git a/test/engine/test_logging.py b/test/engine/test_logging.py index ea2ad3964b..5d792e4701 100644 --- a/test/engine/test_logging.py +++ b/test/engine/test_logging.py @@ -1,30 +1,17 @@ -from sqlalchemy.testing import eq_, assert_raises, assert_raises_message, \ - config, is_ -import re -from sqlalchemy.testing.util import picklers -from sqlalchemy.interfaces import ConnectionProxy -from sqlalchemy import MetaData, Integer, String, INT, VARCHAR, func, \ - bindparam, select, event, TypeDecorator, create_engine, Sequence -from sqlalchemy.sql import column, literal -from sqlalchemy.testing.schema import Table, Column +from sqlalchemy.testing import eq_, assert_raises_message +from sqlalchemy import select import sqlalchemy as tsa -from sqlalchemy import testing from sqlalchemy.testing import engines -from sqlalchemy import util -from sqlalchemy.testing.engines import testing_engine import logging.handlers -from sqlalchemy.dialects.oracle.zxjdbc import ReturningParam -from sqlalchemy.engine import result as _result, default -from sqlalchemy.engine.base import Engine from sqlalchemy.testing import fixtures -from sqlalchemy.testing.mock import Mock, call, patch + class LogParamsTest(fixtures.TestBase): __only_on__ = 'sqlite' __requires__ = 'ad_hoc_engines', def setup(self): - self.eng = engines.testing_engine(options={'echo':True}) + self.eng = engines.testing_engine(options={'echo': True}) self.eng.execute("create table foo (data string)") self.buf = logging.handlers.BufferingHandler(100) for log in [ @@ -44,7 +31,7 @@ class LogParamsTest(fixtures.TestBase): def test_log_large_dict(self): self.eng.execute( "INSERT INTO foo (data) values (:data)", - [{"data":str(i)} for i in range(100)] + [{"data": str(i)} for i in range(100)] ) eq_( self.buf.buffer[1].message, @@ -76,7 +63,7 @@ class LogParamsTest(fixtures.TestBase): "100 total bound parameter sets ... {'data': '98'}, {'data': '99'}\]", lambda: self.eng.execute( "INSERT INTO nonexistent (data) values (:data)", - [{"data":str(i)} for i in range(100)] + [{"data": str(i)} for i in range(100)] ) ) @@ -94,6 +81,7 @@ class LogParamsTest(fixtures.TestBase): ) ) + class LoggingNameTest(fixtures.TestBase): __requires__ = 'ad_hoc_engines', @@ -104,7 +92,7 @@ class LoggingNameTest(fixtures.TestBase): assert name in ( 'sqlalchemy.engine.base.Engine.%s' % eng_name, 'sqlalchemy.pool.%s.%s' % - (eng.pool.__class__.__name__, pool_name) + (eng.pool.__class__.__name__, pool_name) ) def _assert_no_name_in_execute(self, eng): @@ -118,15 +106,15 @@ class LoggingNameTest(fixtures.TestBase): def _named_engine(self, **kw): options = { - 'logging_name':'myenginename', - 'pool_logging_name':'mypoolname', - 'echo':True + 'logging_name': 'myenginename', + 'pool_logging_name': 'mypoolname', + 'echo': True } options.update(kw) return engines.testing_engine(options=options) def _unnamed_engine(self, **kw): - kw.update({'echo':True}) + kw.update({'echo': True}) return engines.testing_engine(options=kw) def setup(self): @@ -183,6 +171,7 @@ class LoggingNameTest(fixtures.TestBase): eng = self._unnamed_engine(echo='debug', echo_pool='debug') self._assert_no_name_in_execute(eng) + class EchoTest(fixtures.TestBase): __requires__ = 'ad_hoc_engines', From 92b0ad0fef0b9ee3d54767cf17e2baf1fd1546da Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 20 Aug 2014 12:01:20 -0400 Subject: [PATCH 43/55] - Fixed bug in connection pool logging where the "connection checked out" debug logging message would not emit if the logging were set up using ``logging.setLevel()``, rather than using the ``echo_pool`` flag. Tests to assert this logging have been added. This is a regression that was introduced in 0.9.0. fixes #3168 --- doc/build/changelog/changelog_09.rst | 11 ++++ lib/sqlalchemy/pool.py | 22 +++---- test/engine/test_logging.py | 85 +++++++++++++++++++++++++++- 3 files changed, 106 insertions(+), 12 deletions(-) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index b6eec2e9db..6795a101c5 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -13,6 +13,17 @@ .. changelog:: :version: 0.9.8 + .. change:: + :tags: bug, pool + :versions: 1.0.0 + :tickets: 3168 + + Fixed bug in connection pool logging where the "connection checked out" + debug logging message would not emit if the logging were set up using + ``logging.setLevel()``, rather than using the ``echo_pool`` flag. + Tests to assert this logging have been added. This is a + regression that was introduced in 0.9.0. + .. change:: :tags: feature, postgresql, pg8000 :versions: 1.0.0 diff --git a/lib/sqlalchemy/pool.py b/lib/sqlalchemy/pool.py index d26bbf32c1..89cddfc312 100644 --- a/lib/sqlalchemy/pool.py +++ b/lib/sqlalchemy/pool.py @@ -443,16 +443,17 @@ class _ConnectionRecord(object): except: rec.checkin() raise - fairy = _ConnectionFairy(dbapi_connection, rec) + echo = pool._should_log_debug() + fairy = _ConnectionFairy(dbapi_connection, rec, echo) rec.fairy_ref = weakref.ref( fairy, lambda ref: _finalize_fairy and _finalize_fairy( dbapi_connection, - rec, pool, ref, pool._echo) + rec, pool, ref, echo) ) _refs.add(rec) - if pool._echo: + if echo: pool.logger.debug("Connection %r checked out from pool", dbapi_connection) return fairy @@ -560,9 +561,10 @@ def _finalize_fairy(connection, connection_record, connection) try: - fairy = fairy or _ConnectionFairy(connection, connection_record) + fairy = fairy or _ConnectionFairy( + connection, connection_record, echo) assert fairy.connection is connection - fairy._reset(pool, echo) + fairy._reset(pool) # Immediately close detached instances if not connection_record: @@ -603,9 +605,10 @@ class _ConnectionFairy(object): """ - def __init__(self, dbapi_connection, connection_record): + def __init__(self, dbapi_connection, connection_record, echo): self.connection = dbapi_connection self._connection_record = connection_record + self._echo = echo connection = None """A reference to the actual DBAPI connection being tracked.""" @@ -642,7 +645,6 @@ class _ConnectionFairy(object): fairy._pool = pool fairy._counter = 0 - fairy._echo = pool._should_log_debug() if threadconns is not None: threadconns.current = weakref.ref(fairy) @@ -684,11 +686,11 @@ class _ConnectionFairy(object): _close = _checkin - def _reset(self, pool, echo): + def _reset(self, pool): if pool.dispatch.reset: pool.dispatch.reset(self, self._connection_record) if pool._reset_on_return is reset_rollback: - if echo: + if self._echo: pool.logger.debug("Connection %s rollback-on-return%s", self.connection, ", via agent" @@ -698,7 +700,7 @@ class _ConnectionFairy(object): else: pool._dialect.do_rollback(self) elif pool._reset_on_return is reset_commit: - if echo: + if self._echo: pool.logger.debug("Connection %s commit-on-return%s", self.connection, ", via agent" diff --git a/test/engine/test_logging.py b/test/engine/test_logging.py index 5d792e4701..1432a0f7bf 100644 --- a/test/engine/test_logging.py +++ b/test/engine/test_logging.py @@ -4,6 +4,8 @@ import sqlalchemy as tsa from sqlalchemy.testing import engines import logging.handlers from sqlalchemy.testing import fixtures +from sqlalchemy.testing import mock +from sqlalchemy.testing.util import lazy_gc class LogParamsTest(fixtures.TestBase): @@ -16,7 +18,6 @@ class LogParamsTest(fixtures.TestBase): self.buf = logging.handlers.BufferingHandler(100) for log in [ logging.getLogger('sqlalchemy.engine'), - logging.getLogger('sqlalchemy.pool') ]: log.addHandler(self.buf) @@ -24,7 +25,6 @@ class LogParamsTest(fixtures.TestBase): self.eng.execute("drop table foo") for log in [ logging.getLogger('sqlalchemy.engine'), - logging.getLogger('sqlalchemy.pool') ]: log.removeHandler(self.buf) @@ -82,6 +82,87 @@ class LogParamsTest(fixtures.TestBase): ) +class PoolLoggingTest(fixtures.TestBase): + def setup(self): + self.existing_level = logging.getLogger("sqlalchemy.pool").level + + self.buf = logging.handlers.BufferingHandler(100) + for log in [ + logging.getLogger('sqlalchemy.pool') + ]: + log.addHandler(self.buf) + + def teardown(self): + for log in [ + logging.getLogger('sqlalchemy.pool') + ]: + log.removeHandler(self.buf) + logging.getLogger("sqlalchemy.pool").setLevel(self.existing_level) + + def _queuepool_echo_fixture(self): + return tsa.pool.QueuePool(creator=mock.Mock(), echo='debug') + + def _queuepool_logging_fixture(self): + logging.getLogger("sqlalchemy.pool").setLevel(logging.DEBUG) + return tsa.pool.QueuePool(creator=mock.Mock()) + + def _stpool_echo_fixture(self): + return tsa.pool.SingletonThreadPool(creator=mock.Mock(), echo='debug') + + def _stpool_logging_fixture(self): + logging.getLogger("sqlalchemy.pool").setLevel(logging.DEBUG) + return tsa.pool.SingletonThreadPool(creator=mock.Mock()) + + def _test_queuepool(self, q, dispose=True): + conn = q.connect() + conn.close() + conn = None + + conn = q.connect() + conn.close() + conn = None + + conn = q.connect() + conn = None + del conn + lazy_gc() + q.dispose() + + eq_( + [buf.msg for buf in self.buf.buffer], + [ + 'Created new connection %r', + 'Connection %r checked out from pool', + 'Connection %r being returned to pool', + 'Connection %s rollback-on-return%s', + 'Connection %r checked out from pool', + 'Connection %r being returned to pool', + 'Connection %s rollback-on-return%s', + 'Connection %r checked out from pool', + 'Connection %r being returned to pool', + 'Connection %s rollback-on-return%s', + 'Closing connection %r', + + ] + (['Pool disposed. %s'] if dispose else []) + ) + + def test_stpool_echo(self): + q = self._stpool_echo_fixture() + self._test_queuepool(q, False) + + def test_stpool_logging(self): + q = self._stpool_logging_fixture() + self._test_queuepool(q, False) + + def test_queuepool_echo(self): + q = self._queuepool_echo_fixture() + self._test_queuepool(q) + + def test_queuepool_logging(self): + q = self._queuepool_logging_fixture() + self._test_queuepool(q) + + class LoggingNameTest(fixtures.TestBase): __requires__ = 'ad_hoc_engines', From 85e75ebcee15f216ace71628f1e491e36663d5c8 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 20 Aug 2014 14:24:45 -0400 Subject: [PATCH 44/55] - factor out determination of current version id out of _collect_update_commands and _collect_delete_commands --- lib/sqlalchemy/orm/persistence.py | 110 +++++++++++++++--------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index 37b696d0f9..511a9cef0f 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -45,38 +45,26 @@ def save_obj(base_mapper, states, uowtransaction, single=False): cached_connections = _cached_connection_dict(base_mapper) for (state, dict_, mapper, connection, - has_identity, row_switch) in _organize_states_for_save( + has_identity, + row_switch, update_version_id) in _organize_states_for_save( base_mapper, states, uowtransaction ): if has_identity or row_switch: states_to_update.append( - (state, dict_, mapper, connection, - has_identity, row_switch) + (state, dict_, mapper, connection, update_version_id) ) else: states_to_insert.append( - (state, dict_, mapper, connection, - has_identity, row_switch) + (state, dict_, mapper, connection) ) for table, mapper in base_mapper._sorted_tables.items(): if table not in mapper._pks_by_table: continue - insert = ( - (state, state_dict, sub_mapper, connection) - for state, state_dict, sub_mapper, connection, has_identity, - row_switch in states_to_insert - if table in sub_mapper._pks_by_table - ) - insert = _collect_insert_commands(table, insert) + insert = _collect_insert_commands(table, states_to_insert) - update = ( - (state, state_dict, sub_mapper, connection, row_switch) - for state, state_dict, sub_mapper, connection, has_identity, - row_switch in states_to_update - if table in sub_mapper._pks_by_table - ) - update = _collect_update_commands(uowtransaction, table, update) + update = _collect_update_commands( + uowtransaction, table, states_to_update) _emit_update_statements(base_mapper, uowtransaction, cached_connections, @@ -89,9 +77,16 @@ def save_obj(base_mapper, states, uowtransaction, single=False): _finalize_insert_update_commands( base_mapper, uowtransaction, ( - (state, state_dict, mapper, connection, has_identity) - for state, state_dict, mapper, connection, has_identity, - row_switch in states_to_insert + states_to_update + (state, state_dict, mapper, connection, False) + for state, state_dict, mapper, connection in states_to_insert + ) + ) + _finalize_insert_update_commands( + base_mapper, uowtransaction, + ( + (state, state_dict, mapper, connection, True) + for state, state_dict, mapper, connection, + update_version_id in states_to_update ) ) @@ -149,21 +144,14 @@ def delete_obj(base_mapper, states, uowtransaction): if table not in mapper._pks_by_table: continue - delete = ( - (state, state_dict, sub_mapper, connection) - for state, state_dict, sub_mapper, has_identity, connection - in states_to_delete if table in sub_mapper._pks_by_table - and has_identity - ) - delete = _collect_delete_commands(base_mapper, uowtransaction, - table, delete) + table, states_to_delete) _emit_delete_statements(base_mapper, uowtransaction, cached_connections, mapper, table, delete) - for state, state_dict, mapper, has_identity, connection \ - in states_to_delete: + for state, state_dict, mapper, connection, \ + update_version_id in states_to_delete: mapper.dispatch.after_delete(mapper, connection, state) @@ -187,7 +175,7 @@ def _organize_states_for_save(base_mapper, states, uowtransaction): instance_key = state.key or mapper._identity_key_from_state(state) - row_switch = None + row_switch = update_version_id = None # call before_XXX extensions if not has_identity: @@ -224,8 +212,14 @@ def _organize_states_for_save(base_mapper, states, uowtransaction): uowtransaction.remove_state_actions(existing) row_switch = existing + if (has_identity or row_switch) and mapper.version_id_col is not None: + update_version_id = mapper._get_committed_state_attr_by_column( + row_switch if row_switch else state, + row_switch.dict if row_switch else dict_, + mapper.version_id_col) + yield (state, dict_, mapper, connection, - has_identity, row_switch) + has_identity, row_switch, update_version_id) def _organize_states_for_post_update(base_mapper, states, @@ -255,7 +249,16 @@ def _organize_states_for_delete(base_mapper, states, uowtransaction): mapper.dispatch.before_delete(mapper, connection, state) - yield state, dict_, mapper, bool(state.key), connection + if mapper.version_id_col is not None: + update_version_id = \ + mapper._get_committed_state_attr_by_column( + state, dict_, + mapper.version_id_col) + else: + update_version_id = None + + yield ( + state, dict_, mapper, connection, update_version_id) def _collect_insert_commands(table, states_to_insert): @@ -264,8 +267,8 @@ def _collect_insert_commands(table, states_to_insert): """ for state, state_dict, mapper, connection in states_to_insert: - - # assert table in mapper._pks_by_table + if table not in mapper._pks_by_table: + continue params = {} value_params = {} @@ -318,9 +321,11 @@ def _collect_update_commands(uowtransaction, table, states_to_update): """ - for state, state_dict, mapper, connection, row_switch in states_to_update: + for state, state_dict, mapper, connection, \ + update_version_id in states_to_update: - # assert table in mapper._pks_by_table + if table not in mapper._pks_by_table: + continue pks = mapper._pks_by_table[table] @@ -340,17 +345,13 @@ def _collect_update_commands(uowtransaction, table, states_to_update): else: params[col.key] = value - if mapper.version_id_col is not None: + if update_version_id is not None: col = mapper.version_id_col - params[col._label] = \ - mapper._get_committed_state_attr_by_column( - row_switch if row_switch else state, - row_switch.dict if row_switch else state_dict, - col) + params[col._label] = update_version_id if col.key not in params and \ mapper.version_id_generator is not False: - val = mapper.version_id_generator(params[col._label]) + val = mapper.version_id_generator(update_version_id) params[col.key] = val if not (params or value_params): @@ -364,7 +365,8 @@ def _collect_update_commands(uowtransaction, table, states_to_update): if history.added: if not history.deleted or \ - ("pk_cascaded", state, col) in uowtransaction.attributes: + ("pk_cascaded", state, col) in \ + uowtransaction.attributes: pk_params[col._label] = history.added[0] params.pop(col.key, None) else: @@ -374,7 +376,6 @@ def _collect_update_commands(uowtransaction, table, states_to_update): else: pk_params[col._label] = history.unchanged[0] - if params or value_params: if None in pk_params.values(): raise orm_exc.FlushError( @@ -426,9 +427,11 @@ def _collect_delete_commands(base_mapper, uowtransaction, table, """Identify values to use in DELETE statements for a list of states to be deleted.""" - for state, state_dict, mapper, connection in states_to_delete: + for state, state_dict, mapper, connection, \ + update_version_id in states_to_delete: - # assert table in mapper._pks_by_table + if table not in mapper._pks_by_table: + continue params = {} for col in mapper._pks_by_table[table]: @@ -442,12 +445,9 @@ def _collect_delete_commands(base_mapper, uowtransaction, table, "using NULL for primary " "key value") - if mapper.version_id_col is not None and \ + if update_version_id is not None and \ table.c.contains_column(mapper.version_id_col): - params[mapper.version_id_col.key] = \ - mapper._get_committed_state_attr_by_column( - state, state_dict, - mapper.version_id_col) + params[mapper.version_id_col.key] = update_version_id yield params, connection From 89ff6df7dcdfa144efbd4d7c2031c0643a266351 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 20 Aug 2014 19:12:32 -0400 Subject: [PATCH 45/55] - pep8 --- test/sql/test_returning.py | 45 ++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/test/sql/test_returning.py b/test/sql/test_returning.py index 26dbcdaa21..79a0b38a5d 100644 --- a/test/sql/test_returning.py +++ b/test/sql/test_returning.py @@ -1,13 +1,16 @@ from sqlalchemy.testing import eq_ -from sqlalchemy import * from sqlalchemy import testing from sqlalchemy.testing.schema import Table, Column from sqlalchemy.types import TypeDecorator from sqlalchemy.testing import fixtures, AssertsExecutionResults, engines, \ assert_raises_message from sqlalchemy import exc as sa_exc +from sqlalchemy import MetaData, String, Integer, Boolean, func, select, \ + Sequence import itertools +table = GoofyType = seq = None + class ReturningTest(fixtures.TestBase, AssertsExecutionResults): __requires__ = 'returning', @@ -31,11 +34,13 @@ class ReturningTest(fixtures.TestBase, AssertsExecutionResults): return value + "BAR" table = Table( - 'tables', meta, Column( - 'id', Integer, primary_key=True, test_needs_autoincrement=True), Column( - 'persons', Integer), Column( - 'full', Boolean), Column( - 'goofy', GoofyType(50))) + 'tables', meta, + Column( + 'id', Integer, primary_key=True, + test_needs_autoincrement=True), + Column('persons', Integer), + Column('full', Boolean), + Column('goofy', GoofyType(50))) table.create(checkfirst=True) def teardown(self): @@ -47,9 +52,11 @@ class ReturningTest(fixtures.TestBase, AssertsExecutionResults): row = result.first() assert row[table.c.id] == row['id'] == 1 - assert row[table.c.full] == row['full'] == False + assert row[table.c.full] == row['full'] + assert row['full'] is False - result = table.insert().values(persons=5, full=True, goofy="somegoofy").\ + result = table.insert().values( + persons=5, full=True, goofy="somegoofy").\ returning(table.c.persons, table.c.full, table.c.goofy).execute() row = result.first() assert row[table.c.persons] == row['persons'] == 5 @@ -238,11 +245,13 @@ class ReturnDefaultsTest(fixtures.TablesTest): return str(next(counter)) Table( - "t1", metadata, Column( - "id", Integer, primary_key=True, test_needs_autoincrement=True), Column( - "data", String(50)), Column( - "insdef", Integer, default=IncDefault()), Column( - "upddef", Integer, onupdate=IncDefault())) + "t1", metadata, + Column( + "id", Integer, primary_key=True, + test_needs_autoincrement=True), + Column("data", String(50)), + Column("insdef", Integer, default=IncDefault()), + Column("upddef", Integer, onupdate=IncDefault())) def test_chained_insert_pk(self): t1 = self.tables.t1 @@ -336,9 +345,10 @@ class ReturnDefaultsTest(fixtures.TablesTest): testing.db.execute( t1.insert().values(upddef=1) ) - result = testing.db.execute(t1.update(). - values(insdef=2).return_defaults( - t1.c.data, t1.c.upddef)) + result = testing.db.execute( + t1.update(). + values(insdef=2).return_defaults( + t1.c.data, t1.c.upddef)) eq_( dict(result.returned_defaults), {"data": None, 'upddef': 1} @@ -352,12 +362,14 @@ class ImplicitReturningFlag(fixtures.TestBase): e = engines.testing_engine(options={'implicit_returning': False}) assert e.dialect.implicit_returning is False c = e.connect() + c.close() assert e.dialect.implicit_returning is False def test_flag_turned_on(self): e = engines.testing_engine(options={'implicit_returning': True}) assert e.dialect.implicit_returning is True c = e.connect() + c.close() assert e.dialect.implicit_returning is True def test_flag_turned_default(self): @@ -377,4 +389,5 @@ class ImplicitReturningFlag(fixtures.TestBase): # version detection on connect sets it c = e.connect() + c.close() assert e.dialect.implicit_returning is supports[0] From 71ca494f518658676b532afaf84a4cc93025dbbb Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 20 Aug 2014 20:14:20 -0400 Subject: [PATCH 46/55] - The INSERT...FROM SELECT construct now implies ``inline=True`` on :class:`.Insert`. This helps to fix a bug where an INSERT...FROM SELECT construct would inadvertently be compiled as "implicit returning" on supporting backends, which would cause breakage in the case of an INSERT that inserts zero rows (as implicit returning expects a row), as well as arbitrary return data in the case of an INSERT that inserts multiple rows (e.g. only the first row of many). A similar change is also applied to an INSERT..VALUES with multiple parameter sets; implicit RETURNING will no longer emit for this statement either. As both of these constructs deal with varible numbers of rows, the :attr:`.ResultProxy.inserted_primary_key` accessor does not apply. Previously, there was a documentation note that one may prefer ``inline=True`` with INSERT..FROM SELECT as some databases don't support returning and therefore can't do "implicit" returning, but there's no reason an INSERT...FROM SELECT needs implicit returning in any case. Regular explicit :meth:`.Insert.returning` should be used to return variable numbers of result rows if inserted data is needed. fixes #3169 --- doc/build/changelog/changelog_10.rst | 25 ++++++ lib/sqlalchemy/sql/compiler.py | 4 +- lib/sqlalchemy/sql/dml.py | 34 +++++---- test/sql/test_insert.py | 109 ++++++++++++++++++++++++++- test/sql/test_query.py | 7 ++ 5 files changed, 163 insertions(+), 16 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index 1cbbec3b3b..3da2c63c2a 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -16,6 +16,31 @@ .. changelog:: :version: 1.0.0 + .. change:: + :tags: bug, sql + :tickets: 3169 + + The INSERT...FROM SELECT construct now implies ``inline=True`` + on :class:`.Insert`. This helps to fix a bug where an + INSERT...FROM SELECT construct would inadvertently be compiled + as "implicit returning" on supporting backends, which would + cause breakage in the case of an INSERT that inserts zero rows + (as implicit returning expects a row), as well as arbitrary + return data in the case of an INSERT that inserts multiple + rows (e.g. only the first row of many). + A similar change is also applied to an INSERT..VALUES + with multiple parameter sets; implicit RETURNING will no longer emit + for this statement either. As both of these constructs deal + with varible numbers of rows, the + :attr:`.ResultProxy.inserted_primary_key` accessor does not + apply. Previously, there was a documentation note that one + may prefer ``inline=True`` with INSERT..FROM SELECT as some databases + don't support returning and therefore can't do "implicit" returning, + but there's no reason an INSERT...FROM SELECT needs implicit returning + in any case. Regular explicit :meth:`.Insert.returning` should + be used to return variable numbers of result rows if inserted + data is needed. + .. change:: :tags: bug, orm :tickets: 3167 diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py index e45510aa4f..fac4980b09 100644 --- a/lib/sqlalchemy/sql/compiler.py +++ b/lib/sqlalchemy/sql/compiler.py @@ -1981,11 +1981,13 @@ class SQLCompiler(Compiled): need_pks = self.isinsert and \ not self.inline and \ - not stmt._returning + not stmt._returning and \ + not stmt._has_multi_parameters implicit_returning = need_pks and \ self.dialect.implicit_returning and \ stmt.table.implicit_returning + if self.isinsert: implicit_return_defaults = (implicit_returning and stmt._return_defaults) diff --git a/lib/sqlalchemy/sql/dml.py b/lib/sqlalchemy/sql/dml.py index f7e033d85c..72dd92c990 100644 --- a/lib/sqlalchemy/sql/dml.py +++ b/lib/sqlalchemy/sql/dml.py @@ -269,6 +269,13 @@ class ValuesBase(UpdateBase): .. versionadded:: 0.8 Support for multiple-VALUES INSERT statements. + .. versionchanged:: 1.0.0 an INSERT that uses a multiple-VALUES + clause, even a list of length one, + implies that the :paramref:`.Insert.inline` flag is set to + True, indicating that the statement will not attempt to fetch + the "last inserted primary key" or other defaults. The statement + deals with an arbitrary number of rows, so the + :attr:`.ResultProxy.inserted_primary_key` accessor does not apply. .. seealso:: @@ -434,8 +441,13 @@ class Insert(ValuesBase): dynamically render the VALUES clause at execution time based on the parameters passed to :meth:`.Connection.execute`. - :param inline: if True, SQL defaults will be compiled 'inline' into - the statement and not pre-executed. + :param inline: if True, no attempt will be made to retrieve the + SQL-generated default values to be provided within the statement; + in particular, + this allows SQL expressions to be rendered 'inline' within the + statement without the need to pre-execute them beforehand; for + backends that support "returning", this turns off the "implicit + returning" feature for the statement. If both `values` and compile-time bind parameters are present, the compile-time bind parameters override the information specified @@ -495,17 +507,12 @@ class Insert(ValuesBase): would normally raise an exception if these column lists don't correspond. - .. note:: - - Depending on backend, it may be necessary for the :class:`.Insert` - statement to be constructed using the ``inline=True`` flag; this - flag will prevent the implicit usage of ``RETURNING`` when the - ``INSERT`` statement is rendered, which isn't supported on a - backend such as Oracle in conjunction with an ``INSERT..SELECT`` - combination:: - - sel = select([table1.c.a, table1.c.b]).where(table1.c.c > 5) - ins = table2.insert(inline=True).from_select(['a', 'b'], sel) + .. versionchanged:: 1.0.0 an INSERT that uses FROM SELECT + implies that the :paramref:`.Insert.inline` flag is set to + True, indicating that the statement will not attempt to fetch + the "last inserted primary key" or other defaults. The statement + deals with an arbitrary number of rows, so the + :attr:`.ResultProxy.inserted_primary_key` accessor does not apply. .. note:: @@ -525,6 +532,7 @@ class Insert(ValuesBase): self._process_colparams(dict((n, Null()) for n in names)) self.select_names = names + self.inline = True self.select = _interpret_as_select(select) def _copy_internals(self, clone=_clone, **kw): diff --git a/test/sql/test_insert.py b/test/sql/test_insert.py index d59d79d890..d2fba58623 100644 --- a/test/sql/test_insert.py +++ b/test/sql/test_insert.py @@ -17,7 +17,7 @@ class _InsertTestBase(object): Column('name', String(30)), Column('description', String(30))) Table('myothertable', metadata, - Column('otherid', Integer), + Column('otherid', Integer, primary_key=True), Column('othername', String(30))) @@ -138,6 +138,23 @@ class InsertTest(_InsertTestBase, fixtures.TablesTest, AssertsCompiledSQL): dialect=default.DefaultDialect() ) + def test_insert_from_select_returning(self): + table1 = self.tables.mytable + sel = select([table1.c.myid, table1.c.name]).where( + table1.c.name == 'foo') + ins = self.tables.myothertable.insert().\ + from_select(("otherid", "othername"), sel).returning( + self.tables.myothertable.c.otherid + ) + self.assert_compile( + ins, + "INSERT INTO myothertable (otherid, othername) " + "SELECT mytable.myid, mytable.name FROM mytable " + "WHERE mytable.name = %(name_1)s RETURNING myothertable.otherid", + checkparams={"name_1": "foo"}, + dialect="postgresql" + ) + def test_insert_from_select_select(self): table1 = self.tables.mytable sel = select([table1.c.myid, table1.c.name]).where( @@ -230,7 +247,7 @@ class InsertTest(_InsertTestBase, fixtures.TablesTest, AssertsCompiledSQL): ) ins = mytable.insert().\ from_select( - [mytable.c.name, mytable.c.description], sel) + [mytable.c.name, mytable.c.description], sel) self.assert_compile( ins, "INSERT INTO mytable (name, description) " @@ -254,6 +271,94 @@ class InsertTest(_InsertTestBase, fixtures.TablesTest, AssertsCompiledSQL): ) +class InsertImplicitReturningTest( + _InsertTestBase, fixtures.TablesTest, AssertsCompiledSQL): + __dialect__ = postgresql.dialect(implicit_returning=True) + + def test_insert_select(self): + table1 = self.tables.mytable + sel = select([table1.c.myid, table1.c.name]).where( + table1.c.name == 'foo') + ins = self.tables.myothertable.insert().\ + from_select(("otherid", "othername"), sel) + self.assert_compile( + ins, + "INSERT INTO myothertable (otherid, othername) " + "SELECT mytable.myid, mytable.name FROM mytable " + "WHERE mytable.name = %(name_1)s", + checkparams={"name_1": "foo"} + ) + + def test_insert_select_return_defaults(self): + table1 = self.tables.mytable + sel = select([table1.c.myid, table1.c.name]).where( + table1.c.name == 'foo') + ins = self.tables.myothertable.insert().\ + from_select(("otherid", "othername"), sel).\ + return_defaults(self.tables.myothertable.c.otherid) + self.assert_compile( + ins, + "INSERT INTO myothertable (otherid, othername) " + "SELECT mytable.myid, mytable.name FROM mytable " + "WHERE mytable.name = %(name_1)s", + checkparams={"name_1": "foo"} + ) + + def test_insert_multiple_values(self): + ins = self.tables.myothertable.insert().values([ + {"othername": "foo"}, + {"othername": "bar"}, + ]) + self.assert_compile( + ins, + "INSERT INTO myothertable (othername) " + "VALUES (%(othername_0)s), " + "(%(othername_1)s)", + checkparams={ + 'othername_1': 'bar', + 'othername_0': 'foo'} + ) + + def test_insert_multiple_values_return_defaults(self): + # TODO: not sure if this should raise an + # error or what + ins = self.tables.myothertable.insert().values([ + {"othername": "foo"}, + {"othername": "bar"}, + ]).return_defaults(self.tables.myothertable.c.otherid) + self.assert_compile( + ins, + "INSERT INTO myothertable (othername) " + "VALUES (%(othername_0)s), " + "(%(othername_1)s)", + checkparams={ + 'othername_1': 'bar', + 'othername_0': 'foo'} + ) + + def test_insert_single_list_values(self): + ins = self.tables.myothertable.insert().values([ + {"othername": "foo"}, + ]) + self.assert_compile( + ins, + "INSERT INTO myothertable (othername) " + "VALUES (%(othername_0)s)", + checkparams={'othername_0': 'foo'} + ) + + def test_insert_single_element_values(self): + ins = self.tables.myothertable.insert().values( + {"othername": "foo"}, + ) + self.assert_compile( + ins, + "INSERT INTO myothertable (othername) " + "VALUES (%(othername)s) RETURNING myothertable.otherid", + checkparams={'othername': 'foo'} + ) + + class EmptyTest(_InsertTestBase, fixtures.TablesTest, AssertsCompiledSQL): __dialect__ = 'default' diff --git a/test/sql/test_query.py b/test/sql/test_query.py index a475b899f5..2075bcecfd 100644 --- a/test/sql/test_query.py +++ b/test/sql/test_query.py @@ -276,6 +276,13 @@ class QueryTest(fixtures.TestBase): r = t6.insert().values(manual_id=id).execute() eq_(r.inserted_primary_key, [12, 1]) + def test_implicit_id_insert_select(self): + stmt = users.insert().from_select( + (users.c.user_id, users.c.user_name), + users.select().where(users.c.user_id == 20)) + + testing.db.execute(stmt) + def test_row_iteration(self): users.insert().execute( {'user_id': 7, 'user_name': 'jack'}, From 374173e89d4e21a75bfabd8a655d17c247b6f1fc Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 21 Aug 2014 10:29:21 -0400 Subject: [PATCH 47/55] - fix link --- lib/sqlalchemy/sql/dml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sqlalchemy/sql/dml.py b/lib/sqlalchemy/sql/dml.py index 72dd92c990..06c50981da 100644 --- a/lib/sqlalchemy/sql/dml.py +++ b/lib/sqlalchemy/sql/dml.py @@ -508,7 +508,7 @@ class Insert(ValuesBase): correspond. .. versionchanged:: 1.0.0 an INSERT that uses FROM SELECT - implies that the :paramref:`.Insert.inline` flag is set to + implies that the :paramref:`.insert.inline` flag is set to True, indicating that the statement will not attempt to fetch the "last inserted primary key" or other defaults. The statement deals with an arbitrary number of rows, so the From 7c4d0a4d6611fd891fb7afda6db277e9fbf83e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnlaugur=20=C3=9E=C3=B3r=20Briem?= Date: Thu, 31 Jul 2014 23:23:56 +0000 Subject: [PATCH 48/55] Fix copy-paste error in Delete doc --- lib/sqlalchemy/sql/dml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/sqlalchemy/sql/dml.py b/lib/sqlalchemy/sql/dml.py index 06c50981da..1934d07763 100644 --- a/lib/sqlalchemy/sql/dml.py +++ b/lib/sqlalchemy/sql/dml.py @@ -736,10 +736,10 @@ class Delete(UpdateBase): :meth:`~.TableClause.delete` method on :class:`~.schema.Table`. - :param table: The table to be updated. + :param table: The table to delete rows from. :param whereclause: A :class:`.ClauseElement` describing the ``WHERE`` - condition of the ``UPDATE`` statement. Note that the + condition of the ``DELETE`` statement. Note that the :meth:`~Delete.where()` generative method may be used instead. .. seealso:: From a12fcd1487f6ae210486fa4a015d9ea71e3bb7d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnlaugur=20=C3=9E=C3=B3r=20Briem?= Date: Thu, 31 Jul 2014 23:26:18 +0000 Subject: [PATCH 49/55] Fix doc typo 'conjunection' --- lib/sqlalchemy/pool.py | 2 +- test/orm/test_options.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/sqlalchemy/pool.py b/lib/sqlalchemy/pool.py index 89cddfc312..bc9affe4a1 100644 --- a/lib/sqlalchemy/pool.py +++ b/lib/sqlalchemy/pool.py @@ -305,7 +305,7 @@ class Pool(log.Identified): """Return a new :class:`.Pool`, of the same class as this one and configured with identical creation arguments. - This method is used in conjunection with :meth:`dispose` + This method is used in conjunction with :meth:`dispose` to close out an entire :class:`.Pool` and create a new one in its place. diff --git a/test/orm/test_options.py b/test/orm/test_options.py index 6eba38d15c..1c1a797a68 100644 --- a/test/orm/test_options.py +++ b/test/orm/test_options.py @@ -497,7 +497,7 @@ class OptionsTest(PathTest, QueryTest): class OptionsNoPropTest(_fixtures.FixtureTest): """test the error messages emitted when using property - options in conjunection with column-only entities, or + options in conjunction with column-only entities, or for not existing options """ From 42a65014f8cf39d29ca9e411e511b613a3ffa446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnlaugur=20=C3=9E=C3=B3r=20Briem?= Date: Thu, 21 Aug 2014 18:42:08 +0000 Subject: [PATCH 50/55] Add note on begin_nested requiring rollback/commit Avoid confusion about rollback/commit "must be issued" after ``session.begin_nested()`` --- this might be taken to mean call must be *added*, but that's only true if not using the return value as a context manager. --- doc/build/orm/session.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/build/orm/session.rst b/doc/build/orm/session.rst index b47e70d537..78ae1ba814 100644 --- a/doc/build/orm/session.rst +++ b/doc/build/orm/session.rst @@ -1773,7 +1773,10 @@ method:: of times, which will issue a new SAVEPOINT with a unique identifier for each call. For each :meth:`~.Session.begin_nested` call, a corresponding :meth:`~.Session.rollback` or -:meth:`~.Session.commit` must be issued. +:meth:`~.Session.commit` must be issued. (But note that if the return value is +used as a context manager, i.e. in a with-statement, then this rollback/commit +is issued by the context manager upon exiting the context, and so should not be +added explicitly.) When :meth:`~.Session.begin_nested` is called, a :meth:`~.Session.flush` is unconditionally issued From b490534657229cbc44f1f5735a39539ceaf776a3 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 23 Aug 2014 15:21:16 -0400 Subject: [PATCH 51/55] - pep8 formatting for pg table opts feature, tests - add support for PG INHERITS - fix mis-named tests - changelog fixes #2051 --- doc/build/changelog/changelog_10.rst | 13 +++ lib/sqlalchemy/dialects/postgresql/base.py | 83 +++++++++++++------ test/dialect/postgresql/test_compiler.py | 95 +++++++++++++++++----- 3 files changed, 145 insertions(+), 46 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index fb639ddf73..4e75181dac 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -16,6 +16,19 @@ .. changelog:: :version: 1.0.0 + .. change:: + :tags: feature, postgresql + :tickets: 2051 + + Added support for PG table options TABLESPACE, ON COMMIT, + WITH(OUT) OIDS, and INHERITS, when rendering DDL via + the :class:`.Table` construct. Pull request courtesy + malikdiarra. + + .. seealso:: + + :ref:`postgresql_table_options` + .. change:: :tags: bug, orm, py3k diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index 34932520f4..39de0cf924 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -417,22 +417,42 @@ of :class:`.PGInspector`, which offers additional methods:: .. autoclass:: PGInspector :members: -PostgreSQL specific table options ---------------------------------- +.. postgresql_table_options: -PostgreSQL provides several CREATE TABLE specific options allowing to -specify how table data are stored. The following options are currently -supported: ``TABLESPACE``, ``ON COMMIT``, ``WITH OIDS``. +PostgreSQL Table Options +------------------------- -``postgresql_tablespace`` is probably the more common and allows to specify -where in the filesystem the data files for the table will be created (see -http://www.postgresql.org/docs/9.3/static/manage-ag-tablespaces.html) +Several options for CREATE TABLE are supported directly by the PostgreSQL +dialect in conjunction with the :class:`.Table` construct: + +* ``TABLESPACE``:: + + Table("some_table", metadata, ..., postgresql_tablespace='some_tablespace') + +* ``ON COMMIT``:: + + Table("some_table", metadata, ..., postgresql_on_commit='PRESERVE ROWS') + +* ``WITH OIDS``:: + + Table("some_table", metadata, ..., postgresql_with_oids=True) + +* ``WITHOUT OIDS``:: + + Table("some_table", metadata, ..., postgresql_with_oids=False) + +* ``INHERITS``:: + + Table("some_table", metadata, ..., postgresql_inherits="some_supertable") + + Table("some_table", metadata, ..., postgresql_inherits=("t1", "t2", ...)) + +.. versionadded:: 1.0.0 .. seealso:: `Postgresql CREATE TABLE options - `_ - - on the PostgreSQL website + `_ """ from collections import defaultdict @@ -1466,19 +1486,33 @@ class PGDDLCompiler(compiler.DDLCompiler): def post_create_table(self, table): table_opts = [] - if table.dialect_options['postgresql']['with_oids'] is not None: - if table.dialect_options['postgresql']['with_oids']: - table_opts.append('WITH OIDS') - else: - table_opts.append('WITHOUT OIDS') - if table.dialect_options['postgresql']['on_commit']: - on_commit_options = table.dialect_options['postgresql']['on_commit'].replace("_", " ").upper() - table_opts.append('ON COMMIT %s' % on_commit_options) - if table.dialect_options['postgresql']['tablespace']: - tablespace_name = table.dialect_options['postgresql']['tablespace'] - table_opts.append('TABLESPACE %s' % self.preparer.quote(tablespace_name)) + pg_opts = table.dialect_options['postgresql'] - return ' '.join(table_opts) + inherits = pg_opts.get('inherits') + if inherits is not None: + if not isinstance(inherits, (list, tuple)): + inherits = (inherits, ) + table_opts.append( + '\n INHERITS ( ' + + ', '.join(self.preparer.quote(name) for name in inherits) + + ' )') + + if pg_opts['with_oids'] is True: + table_opts.append('\n WITH OIDS') + elif pg_opts['with_oids'] is False: + table_opts.append('\n WITHOUT OIDS') + + if pg_opts['on_commit']: + on_commit_options = pg_opts['on_commit'].replace("_", " ").upper() + table_opts.append('\n ON COMMIT %s' % on_commit_options) + + if pg_opts['tablespace']: + tablespace_name = pg_opts['tablespace'] + table_opts.append( + '\n TABLESPACE %s' % self.preparer.quote(tablespace_name) + ) + + return ''.join(table_opts) class PGTypeCompiler(compiler.GenericTypeCompiler): @@ -1741,8 +1775,9 @@ class PGDialect(default.DefaultDialect): (schema.Table, { "ignore_search_path": False, "tablespace": None, - "with_oids" : None, - "on_commit" : None, + "with_oids": None, + "on_commit": None, + "inherits": None }) ] diff --git a/test/dialect/postgresql/test_compiler.py b/test/dialect/postgresql/test_compiler.py index 5d1fddb49f..6c4f3c8ccc 100644 --- a/test/dialect/postgresql/test_compiler.py +++ b/test/dialect/postgresql/test_compiler.py @@ -168,37 +168,88 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): def test_create_table_with_tablespace(self): m = MetaData() - tbl = Table('atable', m, Column("id", Integer), postgresql_tablespace = 'sometablespace') - self.assert_compile(schema.CreateTable(tbl), - "CREATE TABLE atable (id INTEGER)TABLESPACE sometablespace") + tbl = Table( + 'atable', m, Column("id", Integer), + postgresql_tablespace='sometablespace') + self.assert_compile( + schema.CreateTable(tbl), + "CREATE TABLE atable (id INTEGER) TABLESPACE sometablespace") + def test_create_table_with_tablespace_quoted(self): # testing quoting of tablespace name - tbl = Table('anothertable', m, Column("id", Integer), postgresql_tablespace = 'table') - self.assert_compile(schema.CreateTable(tbl), - 'CREATE TABLE anothertable (id INTEGER)TABLESPACE "table"') + m = MetaData() + tbl = Table( + 'anothertable', m, Column("id", Integer), + postgresql_tablespace='table') + self.assert_compile( + schema.CreateTable(tbl), + 'CREATE TABLE anothertable (id INTEGER) TABLESPACE "table"') + + def test_create_table_inherits(self): + m = MetaData() + tbl = Table( + 'atable', m, Column("id", Integer), + postgresql_inherits='i1') + self.assert_compile( + schema.CreateTable(tbl), + "CREATE TABLE atable (id INTEGER) INHERITS ( i1 )") + + def test_create_table_inherits_tuple(self): + m = MetaData() + tbl = Table( + 'atable', m, Column("id", Integer), + postgresql_inherits=('i1', 'i2')) + self.assert_compile( + schema.CreateTable(tbl), + "CREATE TABLE atable (id INTEGER) INHERITS ( i1, i2 )") + + def test_create_table_inherits_quoting(self): + m = MetaData() + tbl = Table( + 'atable', m, Column("id", Integer), + postgresql_inherits=('Quote Me', 'quote Me Too')) + self.assert_compile( + schema.CreateTable(tbl), + 'CREATE TABLE atable (id INTEGER) INHERITS ' + '( "Quote Me", "quote Me Too" )') def test_create_table_with_oids(self): m = MetaData() - tbl = Table('atable', m, Column("id", Integer), postgresql_with_oids = True, ) - self.assert_compile(schema.CreateTable(tbl), - "CREATE TABLE atable (id INTEGER)WITH OIDS") + tbl = Table( + 'atable', m, Column("id", Integer), + postgresql_with_oids=True, ) + self.assert_compile( + schema.CreateTable(tbl), + "CREATE TABLE atable (id INTEGER) WITH OIDS") - tbl2 = Table('anothertable', m, Column("id", Integer), postgresql_with_oids = False, ) - self.assert_compile(schema.CreateTable(tbl2), - "CREATE TABLE anothertable (id INTEGER)WITHOUT OIDS") + tbl2 = Table( + 'anothertable', m, Column("id", Integer), + postgresql_with_oids=False) + self.assert_compile( + schema.CreateTable(tbl2), + "CREATE TABLE anothertable (id INTEGER) WITHOUT OIDS") - def create_table_with_oncommit_option(self): + def test_create_table_with_oncommit_option(self): m = MetaData() - tbl = Table('atable', m, Column("id", Integer), postgresql_on_commit = "drop") - self.assert_compile(schema.CreateTable(tbl), - "CREATE TABLE atable (id INTEGER) ON COMMIT DROP") - - def create_table_with_multiple_options(self): + tbl = Table( + 'atable', m, Column("id", Integer), + postgresql_on_commit="drop") + self.assert_compile( + schema.CreateTable(tbl), + "CREATE TABLE atable (id INTEGER) ON COMMIT DROP") + + def test_create_table_with_multiple_options(self): m = MetaData() - tbl = Table('atable', m, Column("id", Integer), postgresql_tablespace = 'sometablespace', postgresql_with_oids = False, postgresql_on_commit = "preserve_rows") - self.assert_compile(schema.CreateTable(tbl), - "CREATE TABLE atable (id INTEGER)WITHOUT OIDS ON COMMIT PRESERVE ROWS TABLESPACE sometablespace") - + tbl = Table( + 'atable', m, Column("id", Integer), + postgresql_tablespace='sometablespace', + postgresql_with_oids=False, + postgresql_on_commit="preserve_rows") + self.assert_compile( + schema.CreateTable(tbl), + "CREATE TABLE atable (id INTEGER) WITHOUT OIDS " + "ON COMMIT PRESERVE ROWS TABLESPACE sometablespace") + def test_create_partial_index(self): m = MetaData() tbl = Table('testtbl', m, Column('data', Integer)) From be2351481fdb83d1ed02a717ecc7741a19c73f62 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 25 Aug 2014 17:00:21 -0400 Subject: [PATCH 52/55] - The "resurrect" ORM event has been removed. This event hook had no purpose since the old "mutable attribute" system was removed in 0.8. fixes #3171 --- doc/build/changelog/changelog_10.rst | 8 ++++++++ lib/sqlalchemy/orm/events.py | 12 ------------ lib/sqlalchemy/orm/mapper.py | 11 ----------- 3 files changed, 8 insertions(+), 23 deletions(-) diff --git a/doc/build/changelog/changelog_10.rst b/doc/build/changelog/changelog_10.rst index ace9569602..bff3652c58 100644 --- a/doc/build/changelog/changelog_10.rst +++ b/doc/build/changelog/changelog_10.rst @@ -16,6 +16,14 @@ .. changelog:: :version: 1.0.0 + .. change:: + :tags: bug, orm + :tickets: 3171 + + The "resurrect" ORM event has been removed. This event hook had + no purpose since the old "mutable attribute" system was removed + in 0.8. + .. change:: :tags: bug, sql :tickets: 3169 diff --git a/lib/sqlalchemy/orm/events.py b/lib/sqlalchemy/orm/events.py index aa99673bad..8edaa27449 100644 --- a/lib/sqlalchemy/orm/events.py +++ b/lib/sqlalchemy/orm/events.py @@ -293,18 +293,6 @@ class InstanceEvents(event.Events): """ - def resurrect(self, target): - """Receive an object instance as it is 'resurrected' from - garbage collection, which occurs when a "dirty" state falls - out of scope. - - :param target: the mapped instance. If - the event is configured with ``raw=True``, this will - instead be the :class:`.InstanceState` state-management - object associated with the instance. - - """ - def pickle(self, target, state_dict): """Receive an object instance when its associated state is being pickled. diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index f22cac329c..aab28ee0c5 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -1127,7 +1127,6 @@ class Mapper(InspectionAttr): event.listen(manager, 'first_init', _event_on_first_init, raw=True) event.listen(manager, 'init', _event_on_init, raw=True) - event.listen(manager, 'resurrect', _event_on_resurrect, raw=True) for key, method in util.iterate_attributes(self.class_): if isinstance(method, types.FunctionType): @@ -2762,16 +2761,6 @@ def _event_on_init(state, args, kwargs): instrumenting_mapper._set_polymorphic_identity(state) -def _event_on_resurrect(state): - # re-populate the primary key elements - # of the dict based on the mapping. - instrumenting_mapper = state.manager.info.get(_INSTRUMENTOR) - if instrumenting_mapper: - for col, val in zip(instrumenting_mapper.primary_key, state.key[1]): - instrumenting_mapper._set_state_attr_by_column( - state, state.dict, col, val) - - class _ColumnMapping(dict): """Error reporting helper for mapper._columntoproperty.""" From a16ee423e4528bd7a6ba6375cccd88b7450c58d3 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 25 Aug 2014 17:06:28 -0400 Subject: [PATCH 53/55] - mention that FOUND_ROWS is hardcoded; fixes #3146 --- lib/sqlalchemy/dialects/mysql/base.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index 3749607659..012d178e7b 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -190,15 +190,13 @@ SQLAlchemy standardizes the DBAPI ``cursor.rowcount`` attribute to be the usual definition of "number of rows matched by an UPDATE or DELETE" statement. This is in contradiction to the default setting on most MySQL DBAPI drivers, which is "number of rows actually modified/deleted". For this reason, the -SQLAlchemy MySQL dialects always set the ``constants.CLIENT.FOUND_ROWS`` flag, -or whatever is equivalent for the DBAPI in use, on connect, unless the flag -value is overridden using DBAPI-specific options -(such as ``client_flag`` for the MySQL-Python driver, ``found_rows`` for the -OurSQL driver). +SQLAlchemy MySQL dialects always add the ``constants.CLIENT.FOUND_ROWS`` +flag, or whatever is equivalent for the target dialect, upon connection. +This setting is currently hardcoded. -See also: +.. seealso:: -:attr:`.ResultProxy.rowcount` + :attr:`.ResultProxy.rowcount` CAST Support From 4c1c9d5ed7b9e68dcd3a55cda35edc287b9185db Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 25 Aug 2014 19:13:25 -0400 Subject: [PATCH 54/55] - changelog for pr bitbucket:27 --- doc/build/changelog/changelog_09.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index 6795a101c5..90a43ad598 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -13,6 +13,16 @@ .. changelog:: :version: 0.9.8 + .. change:: + :tags: bug, ext + :versions: 1.0.0 + :pullrequest: bitbucket:27 + + Fixed bug where a custom subclass of :ref:`ext.mutable.MutableDict` + would not show up in a "coerce" operation, and would instead + return a plain :ref:`ext.mutable.MutableDict`. Pull request + courtesy Matt Chisholm. + .. change:: :tags: bug, pool :versions: 1.0.0 From 8e84942aa6fa2644b3fe6407c79449715a7e2c8c Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 25 Aug 2014 19:14:47 -0400 Subject: [PATCH 55/55] - changelog for pr bitbucket:28 --- doc/build/changelog/changelog_09.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index 90a43ad598..2931916e3c 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -13,6 +13,15 @@ .. changelog:: :version: 0.9.8 + .. change:: + :tags: bug, ext + :versions: 1.0.0 + :pullrequest: bitbucket:28 + + Fixed bug where :ref:`ext.mutable.MutableDict` + failed to implement the ``update()`` dictionary method, thus + not catching changes. Pull request courtesy Matt Chisholm. + .. change:: :tags: bug, ext :versions: 1.0.0