Traversal and clause generation performance improvements

Added one traversal test, callcounts have been brought from 29754 to
5173 so far.

Change-Id: I164e9831600709ee214c1379bb215fdad73b39aa
This commit is contained in:
Mike Bayer
2019-12-14 11:39:06 -05:00
parent b63bf945fb
commit 89bf6d80a9
10 changed files with 292 additions and 154 deletions
+5 -1
View File
@@ -2209,8 +2209,12 @@ class Mapper(sql_base.HasCacheKey, InspectionAttr):
for table, columns in self._cols_by_table.items()
)
# temporarily commented out until we fix an issue in the serializer
# @_memoized_configured_property.method
def __clause_element__(self):
return self.selectable
return self.selectable # ._annotate(
# {"parententity": self, "parentmapper": self}
# )
@property
def selectable(self):
+20 -17
View File
@@ -19,23 +19,26 @@ from .. import util
class SupportsAnnotations(object):
@util.memoized_property
def _annotation_traversals(self):
return [
(
key,
InternalTraversal.dp_has_cache_key
if isinstance(value, HasCacheKey)
else InternalTraversal.dp_plain_obj,
)
for key, value in self._annotations.items()
]
def _annotations_cache_key(self):
return (
"_annotations",
tuple(
(
key,
value._gen_cache_key(None, [])
if isinstance(value, HasCacheKey)
else value,
)
for key, value in self._annotations.items()
),
)
class SupportsCloneAnnotations(SupportsAnnotations):
_annotations = util.immutabledict()
_traverse_internals = [
("_annotations", InternalTraversal.dp_annotations_state)
("_annotations_cache_key", InternalTraversal.dp_plain_obj)
]
def _annotate(self, values):
@@ -45,7 +48,7 @@ class SupportsCloneAnnotations(SupportsAnnotations):
"""
new = self._clone()
new._annotations = new._annotations.union(values)
new.__dict__.pop("_annotation_traversals", None)
new.__dict__.pop("_annotations_cache_key", None)
return new
def _with_annotations(self, values):
@@ -55,7 +58,7 @@ class SupportsCloneAnnotations(SupportsAnnotations):
"""
new = self._clone()
new._annotations = util.immutabledict(values)
new.__dict__.pop("_annotation_traversals", None)
new.__dict__.pop("_annotations_cache_key", None)
return new
def _deannotate(self, values=None, clone=False):
@@ -71,7 +74,7 @@ class SupportsCloneAnnotations(SupportsAnnotations):
# the expression for a deep deannotation
new = self._clone()
new._annotations = {}
new.__dict__.pop("_annotation_traversals", None)
new.__dict__.pop("_annotations_cache_key", None)
return new
else:
return self
@@ -146,7 +149,7 @@ class Annotated(object):
def __init__(self, element, values):
self.__dict__ = element.__dict__.copy()
self.__dict__.pop("_annotation_traversals", None)
self.__dict__.pop("_annotations_cache_key", None)
self.__element = element
self._annotations = values
self._hash = hash(element)
@@ -159,7 +162,7 @@ class Annotated(object):
def _with_annotations(self, values):
clone = self.__class__.__new__(self.__class__)
clone.__dict__ = self.__dict__.copy()
clone.__dict__.pop("_annotation_traversals", None)
clone.__dict__.pop("_annotations_cache_key", None)
clone._annotations = values
return clone
@@ -305,7 +308,7 @@ def _new_annotation_type(cls, base_cls):
if "_traverse_internals" in cls.__dict__:
anno_cls._traverse_internals = list(cls._traverse_internals) + [
("_annotations", InternalTraversal.dp_annotations_state)
("_annotations_cache_key", InternalTraversal.dp_plain_obj)
]
return anno_cls
+37 -10
View File
@@ -198,12 +198,7 @@ class ClauseElement(
_order_by_label_element = None
@property
def _cache_key_traversal(self):
try:
return self._traverse_internals
except AttributeError:
return NO_CACHE
_cache_key_traversal = None
def _clone(self):
"""Create a shallow copy of this ClauseElement.
@@ -1344,16 +1339,21 @@ class BindParameter(roles.InElementRole, ColumnElement):
return c
def _gen_cache_key(self, anon_map, bindparams):
if self in anon_map:
return (anon_map[self], self.__class__)
idself = id(self)
if idself in anon_map:
return (anon_map[idself], self.__class__)
else:
# inline of
# id_ = anon_map[idself]
anon_map[idself] = id_ = str(anon_map.index)
anon_map.index += 1
id_ = anon_map[self]
bindparams.append(self)
return (
id_,
self.__class__,
self.type._gen_cache_key,
self.type._static_cache_key,
traversals._resolve_name_for_compare(self, self.key, anon_map),
)
@@ -3239,6 +3239,33 @@ class BinaryExpression(ColumnElement):
"""
def _gen_cache_key(self, anon_map, bindparams):
# inlined for performance
idself = id(self)
if idself in anon_map:
return (anon_map[idself], self.__class__)
else:
# inline of
# id_ = anon_map[idself]
anon_map[idself] = id_ = str(anon_map.index)
anon_map.index += 1
if self._cache_key_traversal is NO_CACHE:
anon_map[NO_CACHE] = True
return None
result = (id_, self.__class__)
return result + (
("left", self.left._gen_cache_key(anon_map, bindparams)),
("right", self.right._gen_cache_key(anon_map, bindparams)),
("operator", self.operator),
("negate", self.negate),
("modifiers", self.modifiers),
)
def __init__(
self, left, right, operator, type_=None, negate=None, modifiers=None
):
+3
View File
@@ -404,6 +404,9 @@ class FunctionAsBinary(BinaryExpression):
("modifiers", InternalTraversal.dp_plain_dict),
]
def _gen_cache_key(self, anon_map, bindparams):
return ColumnElement._gen_cache_key(self, anon_map, bindparams)
def __init__(self, fn, left_index, right_index):
self.sql_function = fn
self.left_index = left_index
+8 -2
View File
@@ -3148,8 +3148,14 @@ class Select(
("_raw_columns", InternalTraversal.dp_clauseelement_list),
("_whereclause", InternalTraversal.dp_clauseelement),
("_having", InternalTraversal.dp_clauseelement),
("_order_by_clause", InternalTraversal.dp_clauseelement_list),
("_group_by_clause", InternalTraversal.dp_clauseelement_list),
(
"_order_by_clause.clauses",
InternalTraversal.dp_clauseelement_list,
),
(
"_group_by_clause.clauses",
InternalTraversal.dp_clauseelement_list,
),
("_correlate", InternalTraversal.dp_clauseelement_unordered_set),
(
"_correlate_except",
+118 -109
View File
@@ -1,5 +1,6 @@
from collections import deque
from collections import namedtuple
import operator
from . import operators
from .visitors import ExtendedInternalTraversal
@@ -11,6 +12,9 @@ SKIP_TRAVERSE = util.symbol("skip_traverse")
COMPARE_FAILED = False
COMPARE_SUCCEEDED = True
NO_CACHE = util.symbol("no_cache")
CACHE_IN_PLACE = util.symbol("cache_in_place")
CALL_GEN_CACHE_KEY = util.symbol("call_gen_cache_key")
STATIC_CACHE_KEY = util.symbol("static_cache_key")
def compare(obj1, obj2, **kw):
@@ -46,22 +50,82 @@ class HasCacheKey(object):
"""
if self in anon_map:
return (anon_map[self], self.__class__)
idself = id(self)
id_ = anon_map[self]
if anon_map is not None:
if idself in anon_map:
return (anon_map[idself], self.__class__)
else:
# inline of
# id_ = anon_map[idself]
anon_map[idself] = id_ = str(anon_map.index)
anon_map.index += 1
else:
id_ = None
if self._cache_key_traversal is NO_CACHE:
anon_map[NO_CACHE] = True
_cache_key_traversal = self._cache_key_traversal
if _cache_key_traversal is None:
try:
_cache_key_traversal = self._traverse_internals
except AttributeError:
_cache_key_traversal = NO_CACHE
if _cache_key_traversal is NO_CACHE:
if anon_map is not None:
anon_map[NO_CACHE] = True
return None
result = (id_, self.__class__)
for attrname, obj, meth in _cache_key_traversal.run_generated_dispatch(
self, self._cache_key_traversal, "_generated_cache_key_traversal"
# inline of _cache_key_traversal_visitor.run_generated_dispatch()
try:
dispatcher = self.__class__.__dict__[
"_generated_cache_key_traversal"
]
except KeyError:
dispatcher = _cache_key_traversal_visitor.generate_dispatch(
self, _cache_key_traversal, "_generated_cache_key_traversal"
)
for attrname, obj, meth in dispatcher(
self, _cache_key_traversal_visitor
):
if obj is not None:
result += meth(attrname, obj, self, anon_map, bindparams)
if meth is CACHE_IN_PLACE:
# cache in place is always going to be a Python
# tuple, dict, list, etc. so we can do a boolean check
if obj:
result += (attrname, obj)
elif meth is STATIC_CACHE_KEY:
result += (attrname, obj._static_cache_key)
elif meth is CALL_GEN_CACHE_KEY:
result += (
attrname,
obj._gen_cache_key(anon_map, bindparams),
)
elif meth is InternalTraversal.dp_clauseelement_list:
if obj:
result += (
attrname,
tuple(
[
elem._gen_cache_key(anon_map, bindparams)
for elem in obj
]
),
)
else:
# note that all the "ClauseElement" standalone cases
# here have been handled by inlines above; so we can
# safely assume the object is a standard list/tuple/dict
# which we can skip if it evaluates to false.
# improvement would be to have this as a flag delivered
# up front in the dispatcher list
if obj:
result += meth(
attrname, obj, self, anon_map, bindparams
)
return result
def _generate_cache_key(self):
@@ -118,17 +182,22 @@ def _clone(element, **kw):
class _CacheKey(ExtendedInternalTraversal):
def visit_has_cache_key(self, attrname, obj, parent, anon_map, bindparams):
return (attrname, obj._gen_cache_key(anon_map, bindparams))
# very common elements are inlined into the main _get_cache_key() method
# to produce a dramatic savings in Python function call overhead
visit_has_cache_key = visit_clauseelement = CALL_GEN_CACHE_KEY
visit_clauseelement_list = InternalTraversal.dp_clauseelement_list
visit_string = (
visit_boolean
) = visit_operator = visit_plain_obj = CACHE_IN_PLACE
visit_statement_hint_list = CACHE_IN_PLACE
visit_type = STATIC_CACHE_KEY
def visit_inspectable(self, attrname, obj, parent, anon_map, bindparams):
return self.visit_has_cache_key(
attrname, inspect(obj), parent, anon_map, bindparams
)
def visit_clauseelement(self, attrname, obj, parent, anon_map, bindparams):
return (attrname, obj._gen_cache_key(anon_map, bindparams))
def visit_multi(self, attrname, obj, parent, anon_map, bindparams):
return (
attrname,
@@ -151,6 +220,8 @@ class _CacheKey(ExtendedInternalTraversal):
def visit_has_cache_key_tuples(
self, attrname, obj, parent, anon_map, bindparams
):
if not obj:
return ()
return (
attrname,
tuple(
@@ -165,6 +236,8 @@ class _CacheKey(ExtendedInternalTraversal):
def visit_has_cache_key_list(
self, attrname, obj, parent, anon_map, bindparams
):
if not obj:
return ()
return (
attrname,
tuple(elem._gen_cache_key(anon_map, bindparams) for elem in obj),
@@ -177,14 +250,6 @@ class _CacheKey(ExtendedInternalTraversal):
attrname, [inspect(o) for o in obj], parent, anon_map, bindparams
)
def visit_clauseelement_list(
self, attrname, obj, parent, anon_map, bindparams
):
return (
attrname,
tuple(elem._gen_cache_key(anon_map, bindparams) for elem in obj),
)
def visit_clauseelement_tuples(
self, attrname, obj, parent, anon_map, bindparams
):
@@ -204,14 +269,18 @@ class _CacheKey(ExtendedInternalTraversal):
def visit_fromclause_ordered_set(
self, attrname, obj, parent, anon_map, bindparams
):
if not obj:
return ()
return (
attrname,
tuple(elem._gen_cache_key(anon_map, bindparams) for elem in obj),
tuple([elem._gen_cache_key(anon_map, bindparams) for elem in obj]),
)
def visit_clauseelement_unordered_set(
self, attrname, obj, parent, anon_map, bindparams
):
if not obj:
return ()
cache_keys = [
elem._gen_cache_key(anon_map, bindparams) for elem in obj
]
@@ -230,39 +299,40 @@ class _CacheKey(ExtendedInternalTraversal):
def visit_prefix_sequence(
self, attrname, obj, parent, anon_map, bindparams
):
if not obj:
return ()
return (
attrname,
tuple(
(clause._gen_cache_key(anon_map, bindparams), strval)
for clause, strval in obj
[
(clause._gen_cache_key(anon_map, bindparams), strval)
for clause, strval in obj
]
),
)
def visit_statement_hint_list(
self, attrname, obj, parent, anon_map, bindparams
):
return (attrname, obj)
def visit_table_hint_list(
self, attrname, obj, parent, anon_map, bindparams
):
if not obj:
return ()
return (
attrname,
tuple(
(
clause._gen_cache_key(anon_map, bindparams),
dialect_name,
text,
)
for (clause, dialect_name), text in obj.items()
[
(
clause._gen_cache_key(anon_map, bindparams),
dialect_name,
text,
)
for (clause, dialect_name), text in obj.items()
]
),
)
def visit_type(self, attrname, obj, parent, anon_map, bindparams):
return (attrname, obj._gen_cache_key)
def visit_plain_dict(self, attrname, obj, parent, anon_map, bindparams):
return (attrname, tuple((key, obj[key]) for key in sorted(obj)))
return (attrname, tuple([(key, obj[key]) for key in sorted(obj)]))
def visit_string_clauseelement_dict(
self, attrname, obj, parent, anon_map, bindparams
@@ -291,18 +361,6 @@ class _CacheKey(ExtendedInternalTraversal):
),
)
def visit_string(self, attrname, obj, parent, anon_map, bindparams):
return (attrname, obj)
def visit_boolean(self, attrname, obj, parent, anon_map, bindparams):
return (attrname, obj)
def visit_operator(self, attrname, obj, parent, anon_map, bindparams):
return (attrname, obj)
def visit_plain_obj(self, attrname, obj, parent, anon_map, bindparams):
return (attrname, obj)
def visit_fromclause_canonical_column_collection(
self, attrname, obj, parent, anon_map, bindparams
):
@@ -311,22 +369,6 @@ class _CacheKey(ExtendedInternalTraversal):
tuple(col._gen_cache_key(anon_map, bindparams) for col in obj),
)
def visit_annotations_state(
self, attrname, obj, parent, anon_map, bindparams
):
return (
attrname,
tuple(
(
key,
self.dispatch(sym)(
key, obj[key], obj, anon_map, bindparams
),
)
for key, sym in parent._annotation_traversals
),
)
def visit_unknown_structure(
self, attrname, obj, parent, anon_map, bindparams
):
@@ -334,7 +376,7 @@ class _CacheKey(ExtendedInternalTraversal):
return ()
_cache_key_traversal = _CacheKey()
_cache_key_traversal_visitor = _CacheKey()
class _CopyInternals(InternalTraversal):
@@ -489,29 +531,23 @@ class TraversalComparatorStrategy(InternalTraversal, util.MemoizedSlots):
right._traverse_internals,
fillvalue=(None, None),
):
if not compare_annotations and (
(left_attrname == "_annotations_cache_key")
or (right_attrname == "_annotations_cache_key")
):
continue
if (
left_attrname != right_attrname
or left_visit_sym is not right_visit_sym
):
if not compare_annotations and (
(
left_visit_sym
is InternalTraversal.dp_annotations_state,
)
or (
right_visit_sym
is InternalTraversal.dp_annotations_state,
)
):
continue
return False
elif left_attrname in attributes_compared:
continue
dispatch = self.dispatch(left_visit_sym)
left_child = getattr(left, left_attrname)
right_child = getattr(right, right_attrname)
left_child = operator.attrgetter(left_attrname)(left)
right_child = operator.attrgetter(right_attrname)(right)
if left_child is None:
if right_child is not None:
return False
@@ -564,33 +600,6 @@ class TraversalComparatorStrategy(InternalTraversal, util.MemoizedSlots):
return COMPARE_FAILED
self.stack.append((left[lstr], right[rstr]))
def visit_annotations_state(
self, left_parent, left, right_parent, right, **kw
):
if not kw.get("compare_annotations", False):
return
for (lstr, lmeth), (rstr, rmeth) in util.zip_longest(
left_parent._annotation_traversals,
right_parent._annotation_traversals,
fillvalue=(None, None),
):
if lstr != rstr or (lmeth is not rmeth):
return COMPARE_FAILED
dispatch = self.dispatch(lmeth)
left_child = left[lstr]
right_child = right[rstr]
if left_child is None:
if right_child is not None:
return False
else:
continue
comparison = dispatch(None, left_child, None, right_child, **kw)
if comparison is COMPARE_FAILED:
return comparison
def visit_clauseelement_tuples(
self, left_parent, left, right_parent, right, **kw
):
+1 -1
View File
@@ -535,7 +535,7 @@ class TypeEngine(Traversible):
return dialect.type_descriptor(self)
@util.memoized_property
def _gen_cache_key(self):
def _static_cache_key(self):
names = util.get_cls_kwargs(self.__class__)
return (self.__class__,) + tuple(
(k, self.__dict__[k])
+11 -8
View File
@@ -216,12 +216,20 @@ class InternalTraversal(util.with_metaclass(_InternalTraversalType, object)):
try:
dispatcher = target.__class__.__dict__[generate_dispatcher_name]
except KeyError:
dispatcher = _generate_dispatcher(
self, internal_dispatch, generate_dispatcher_name
dispatcher = self.generate_dispatch(
target, internal_dispatch, generate_dispatcher_name
)
setattr(target.__class__, generate_dispatcher_name, dispatcher)
return dispatcher(target, self)
def generate_dispatch(
self, target, internal_dispatch, generate_dispatcher_name
):
dispatcher = _generate_dispatcher(
self, internal_dispatch, generate_dispatcher_name
)
setattr(target.__class__, generate_dispatcher_name, dispatcher)
return dispatcher
dp_has_cache_key = symbol("HC")
"""Visit a :class:`.HasCacheKey` object."""
@@ -331,11 +339,6 @@ class InternalTraversal(util.with_metaclass(_InternalTraversalType, object)):
"""
dp_annotations_state = symbol("A")
"""Visit the state of the :class:`.Annotatated` version of an object.
"""
dp_named_ddl_element = symbol("DD")
"""Visit a simple named DDL element.
+79
View File
@@ -1,4 +1,16 @@
from sqlalchemy import Column
from sqlalchemy import Enum
from sqlalchemy import ForeignKey
from sqlalchemy import Integer
from sqlalchemy import MetaData
from sqlalchemy import select
from sqlalchemy import String
from sqlalchemy import Table
from sqlalchemy import testing
from sqlalchemy.orm import join as ormjoin
from sqlalchemy.orm import mapper
from sqlalchemy.orm import relationship
from sqlalchemy.testing import eq_
from sqlalchemy.testing import fixtures
from sqlalchemy.testing import profiling
from sqlalchemy.util import classproperty
@@ -35,3 +47,70 @@ class EnumTest(fixtures.TestBase):
@profiling.function_call_count()
def test_create_enum_from_pep_435_w_expensive_members(self):
Enum(self.SomeEnum)
class CacheKeyTest(fixtures.TestBase):
__requires__ = ("cpython",)
@testing.fixture(scope="class")
def mapping_fixture(self):
# note in order to work nicely with "fixture" we are emerging
# a whole new model of setup/teardown, since pytest "fixture"
# sort of purposely works badly with setup/teardown
metadata = MetaData()
parent = Table(
"parent",
metadata,
Column("id", Integer, primary_key=True),
Column("data", String(20)),
)
child = Table(
"child",
metadata,
Column("id", Integer, primary_key=True),
Column("data", String(20)),
Column(
"parent_id", Integer, ForeignKey("parent.id"), nullable=False
),
)
class Parent(testing.entities.BasicEntity):
pass
class Child(testing.entities.BasicEntity):
pass
mapper(
Parent,
parent,
properties={"children": relationship(Child, backref="parent")},
)
mapper(Child, child)
return Parent, Child
@testing.fixture(scope="function")
def stmt_fixture_one(self, mapping_fixture):
# note that by using ORM elements we will have annotations in these
# items also which is part of the performance hit
Parent, Child = mapping_fixture
return [
(
select([Parent.id, Child.id])
.select_from(ormjoin(Parent, Child, Parent.children))
.where(Child.id == 5)
)
for i in range(100)
]
@profiling.function_call_count()
def test_statement_one(self, stmt_fixture_one):
current_key = None
for stmt in stmt_fixture_one:
key = stmt._generate_cache_key()
if current_key:
eq_(key, current_key)
else:
current_key = key
+10 -6
View File
@@ -1,15 +1,15 @@
# /home/classic/dev/sqlalchemy/test/profiles.txt
# This file is written out on a per-environment basis.
# For each test in aaa_profiling, the corresponding function and
# For each test in aaa_profiling, the corresponding function and
# environment is located within this file. If it doesn't exist,
# the test is skipped.
# If a callcount does exist, it is compared to what we received.
# If a callcount does exist, it is compared to what we received.
# assertions are raised if the counts do not match.
#
# To add a new callcount test, apply the function_call_count
# decorator and re-run the tests using the --write-profiles
#
# To add a new callcount test, apply the function_call_count
# decorator and re-run the tests using the --write-profiles
# option - this file will be rewritten including the new count.
#
#
# TEST: test.aaa_profiling.test_compiler.CompileTest.test_insert
@@ -136,6 +136,10 @@ test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.7_postgre
test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.7_sqlite_pysqlite_dbapiunicode_cextensions 162
test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.7_sqlite_pysqlite_dbapiunicode_nocextensions 162
# TEST: test.aaa_profiling.test_misc.CacheKeyTest.test_statement_one
test.aaa_profiling.test_misc.CacheKeyTest.test_statement_one 3.7_sqlite_pysqlite_dbapiunicode_nocextensions 5173
# TEST: test.aaa_profiling.test_misc.EnumTest.test_create_enum_from_pep_435_w_expensive_members
test.aaa_profiling.test_misc.EnumTest.test_create_enum_from_pep_435_w_expensive_members 2.7_mssql_pyodbc_dbapiunicode_nocextensions 1325