From 49b313fa4f07a4224c2704e41d7563c378c27214 Mon Sep 17 00:00:00 2001 From: Joey Leingang Date: Fri, 16 Jun 2017 22:55:55 -0700 Subject: [PATCH 1/4] ClockedOption to TemporalOption --- temporal_sqlalchemy/__init__.py | 2 +- temporal_sqlalchemy/bases.py | 8 ++++---- temporal_sqlalchemy/clock.py | 4 ++-- temporal_sqlalchemy/core.py | 2 +- temporal_sqlalchemy/session.py | 4 ++-- tests/test_concrete_base.py | 2 +- tests/test_temporal_model_mixin.py | 2 +- tests/test_temporal_models.py | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/temporal_sqlalchemy/__init__.py b/temporal_sqlalchemy/__init__.py index 1905159..f42e58c 100644 --- a/temporal_sqlalchemy/__init__.py +++ b/temporal_sqlalchemy/__init__.py @@ -2,7 +2,7 @@ from .version import __version__ from .bases import ( Clocked, - ClockedOption, + TemporalOption, EntityClock, TemporalProperty, TemporalActivityMixin) diff --git a/temporal_sqlalchemy/bases.py b/temporal_sqlalchemy/bases.py index 5d92ef0..4e2a319 100644 --- a/temporal_sqlalchemy/bases.py +++ b/temporal_sqlalchemy/bases.py @@ -41,7 +41,7 @@ def id(self): pass -class ClockedOption(object): +class TemporalOption(object): def __init__( self, history_models: typing.Dict[T_PROPS, nine.Type[TemporalProperty]], @@ -57,14 +57,14 @@ def __init__( @property def clock_table(self): warnings.warn( - 'use ClockedOption.clock_model instead', + 'use TemporalOption.clock_model instead', PendingDeprecationWarning) return self.clock_model @property def history_tables(self): warnings.warn( - 'use ClockedOption.history_models instead', + 'use TemporalOption.history_models instead', PendingDeprecationWarning) return self.history_models @@ -178,7 +178,7 @@ class Clocked(object): vclock = sa.Column(sa.Integer, default=1) clock = None # type: orm.relationship - temporal_options = None # type: ClockedOption + temporal_options = None # type: TemporalOption first_tick = None # type: EntityClock latest_tick = None # type: EntityClock diff --git a/temporal_sqlalchemy/clock.py b/temporal_sqlalchemy/clock.py index 4948a00..9f8d01e 100644 --- a/temporal_sqlalchemy/clock.py +++ b/temporal_sqlalchemy/clock.py @@ -16,7 +16,7 @@ from temporal_sqlalchemy.bases import ( T_PROPS, Clocked, - ClockedOption, + TemporalOption, TemporalActivityMixin, EntityClock, TemporalProperty) @@ -173,7 +173,7 @@ def make_temporal(cls: nine.Type[Clocked]): ) mapper.add_property('first_tick', first_tick) - temporal_options = ClockedOption( + temporal_options = TemporalOption( temporal_props=local_props | relationship_props, history_models=history_tables, clock_model=clock_table, diff --git a/temporal_sqlalchemy/core.py b/temporal_sqlalchemy/core.py index 7504ef8..5f3900d 100644 --- a/temporal_sqlalchemy/core.py +++ b/temporal_sqlalchemy/core.py @@ -117,7 +117,7 @@ def temporal_map(mapper: orm.Mapper, cls: bases.Clocked): for p in tracked_props } - cls.temporal_options = bases.ClockedOption( + cls.temporal_options = bases.TemporalOption( temporal_props=tracked_props, history_models=history_models, clock_model=clock_model, diff --git a/temporal_sqlalchemy/session.py b/temporal_sqlalchemy/session.py index fd8d56d..9e78b41 100644 --- a/temporal_sqlalchemy/session.py +++ b/temporal_sqlalchemy/session.py @@ -5,7 +5,7 @@ import sqlalchemy.event as event import sqlalchemy.orm as orm -from temporal_sqlalchemy.bases import ClockedOption, Clocked +from temporal_sqlalchemy.bases import TemporalOption, Clocked from temporal_sqlalchemy.metadata import ( get_session_metadata, set_session_metadata @@ -14,7 +14,7 @@ def _temporal_models(session: orm.Session) -> typing.Iterable[Clocked]: for obj in session: - if isinstance(getattr(obj, 'temporal_options', None), ClockedOption): + if isinstance(getattr(obj, 'temporal_options', None), TemporalOption): yield obj diff --git a/tests/test_concrete_base.py b/tests/test_concrete_base.py index 87d7287..b484833 100644 --- a/tests/test_concrete_base.py +++ b/tests/test_concrete_base.py @@ -12,7 +12,7 @@ class TestTemporalConcreteBaseModels(shared.DatabaseTest): def test_temporal_options_class(self): options = models.SimpleConcreteChildTemporalTable.temporal_options - assert isinstance(options, temporal.ClockedOption) + assert isinstance(options, temporal.TemporalOption) clock_table = options.clock_table assert clock_table.__table__.name == "%s_clock" % ( diff --git a/tests/test_temporal_model_mixin.py b/tests/test_temporal_model_mixin.py index 75baeba..2ff9b2f 100644 --- a/tests/test_temporal_model_mixin.py +++ b/tests/test_temporal_model_mixin.py @@ -24,7 +24,7 @@ def test_create_temporal_options(): assert hasattr(m, 'temporal_options') assert m.temporal_options is models.NewStyleModel.temporal_options - assert isinstance(m.temporal_options, temporal.ClockedOption) + assert isinstance(m.temporal_options, temporal.TemporalOption) @pytest.mark.parametrize('table,expected_name,expected_cols,activity_class', ( diff --git a/tests/test_temporal_models.py b/tests/test_temporal_models.py index d9ec0d0..97fecc0 100644 --- a/tests/test_temporal_models.py +++ b/tests/test_temporal_models.py @@ -15,7 +15,7 @@ class TestTemporalModels(shared.DatabaseTest): def test_temporal_options_class(self): options = models.SimpleTableTemporal.temporal_options - assert isinstance(options, temporal.ClockedOption) + assert isinstance(options, temporal.TemporalOption) clock_table = options.clock_table assert (clock_table.__table__.name From 182fee39cda2eaf754299ad8d9fe238e19aaa819 Mon Sep 17 00:00:00 2001 From: Joey Leingang Date: Sat, 17 Jun 2017 00:18:20 -0700 Subject: [PATCH 2/4] unify everything --- temporal_sqlalchemy/__init__.py | 4 +- temporal_sqlalchemy/bases.py | 5 +- temporal_sqlalchemy/clock.py | 368 +++++++++++++++-------------- temporal_sqlalchemy/core.py | 145 +----------- temporal_sqlalchemy/util.py | 59 +++++ tests/test_builders.py | 76 +++++- tests/test_temporal_model_mixin.py | 72 +----- 7 files changed, 342 insertions(+), 387 deletions(-) diff --git a/temporal_sqlalchemy/__init__.py b/temporal_sqlalchemy/__init__.py index f42e58c..40f70e2 100644 --- a/temporal_sqlalchemy/__init__.py +++ b/temporal_sqlalchemy/__init__.py @@ -7,5 +7,7 @@ TemporalProperty, TemporalActivityMixin) from .session import temporal_session, persist_history, is_temporal_session -from .clock import add_clock, get_activity_clock_backref, get_history_model, get_history_model +from .clock import add_clock +from .util import get_activity_clock_backref, \ + get_history_model from .core import TemporalModel diff --git a/temporal_sqlalchemy/bases.py b/temporal_sqlalchemy/bases.py index 4e2a319..5c08fcb 100644 --- a/temporal_sqlalchemy/bases.py +++ b/temporal_sqlalchemy/bases.py @@ -13,7 +13,6 @@ from temporal_sqlalchemy import nine from temporal_sqlalchemy.metadata import get_session_metadata - _ClockSet = collections.namedtuple('_ClockSet', ('effective', 'vclock')) T_PROPS = typing.TypeVar( @@ -36,7 +35,7 @@ class TemporalProperty(object): class TemporalActivityMixin(object): - @abc.abstractproperty + @abc.abstractmethod def id(self): pass @@ -192,6 +191,8 @@ def date_modified(self): @contextlib.contextmanager def clock_tick(self, activity: TemporalActivityMixin = None): + warnings.warn("clock_tick is going away in 0.5.0", + PendingDeprecationWarning) """Increments vclock by 1 with changes scoped to the session""" if self.temporal_options.activity_cls is not None and activity is None: raise ValueError("activity is missing on edit") from None diff --git a/temporal_sqlalchemy/clock.py b/temporal_sqlalchemy/clock.py index 9f8d01e..e09a828 100644 --- a/temporal_sqlalchemy/clock.py +++ b/temporal_sqlalchemy/clock.py @@ -1,16 +1,13 @@ -import datetime as dt import itertools -import uuid import typing +import uuid +import warnings import sqlalchemy as sa import sqlalchemy.dialects.postgresql as sap import sqlalchemy.event as event import sqlalchemy.ext.declarative as declarative import sqlalchemy.orm as orm -import sqlalchemy.orm.attributes as attributes -import sqlalchemy.util as sa_util -import psycopg2.extras as psql_extras from temporal_sqlalchemy import nine, util from temporal_sqlalchemy.bases import ( @@ -22,32 +19,149 @@ TemporalProperty) -def effective_now() -> psql_extras.DateTimeTZRange: - utc_now = dt.datetime.now(tz=dt.timezone.utc) - return psql_extras.DateTimeTZRange(utc_now, None) +def temporal_map(*track, mapper: orm.Mapper, activity_class=None, schema=None): + assert 'vclock' not in track + cls = mapper.class_ + entity_table = mapper.local_table + # get things defined on Temporal: + tracked_props = frozenset( + mapper.get_property(prop) for prop in track + ) + # make sure all temporal properties have active_history (always loaded) + for prop in tracked_props: + getattr(cls, prop.key).impl.active_history = True -def get_activity_clock_backref( - activity: TemporalActivityMixin, - entity: Clocked) -> orm.RelationshipProperty: - """Get the backref'd clock history for a given entity.""" - assert ( - activity is entity.temporal_options.activity_cls or - isinstance(activity, entity.temporal_options.activity_cls) - ), "cannot inspect %r for mapped activity %r" % (entity, activity) + schema = schema or entity_table.schema - inspected = sa.inspect(entity.temporal_options.activity_cls) - backref = entity.temporal_options.clock_table.activity.property.backref + clock_table = build_clock_table( + entity_table, + entity_table.metadata, + schema, + activity_class + ) + clock_properties = { + 'entity': orm.relationship( + lambda: cls, backref=orm.backref('clock', lazy='dynamic') + ), + 'entity_first_tick': orm.relationship( + lambda: cls, + backref=orm.backref( + 'first_tick', + primaryjoin=sa.and_( + clock_table.join(entity_table).onclause, + clock_table.c.tick == 1 + ), + innerjoin=True, + uselist=False, # single record + viewonly=True # view only + ) + ), + 'entity_latest_tick': orm.relationship( + lambda: cls, + backref=orm.backref( + 'latest_tick', + primaryjoin=sa.and_( + clock_table.join(entity_table).onclause, + entity_table.c.vclock == clock_table.c.tick + ), + innerjoin=True, + uselist=False, # single record + viewonly=True # view only + ) + ), + '__table__': clock_table + } # used to construct a new clock model for this entity + + if activity_class: + # create a relationship to the activity from the clock model + backref_name = '%s_clock' % entity_table.name + clock_properties['activity'] = \ + orm.relationship(lambda: activity_class, backref=backref_name) + + clock_model = build_clock_class(cls.__name__, + entity_table.metadata, + clock_properties) + + history_models = { + p: build_history_class(cls, p, schema) + for p in tracked_props + } - return inspected.relationships[backref] + cls.temporal_options = TemporalOption( + temporal_props=tracked_props, + history_models=history_models, + clock_model=clock_model, + activity_cls=activity_class + ) + event.listen(cls, 'init', init_clock) -def get_history_model( - target: attributes.InstrumentedAttribute) -> TemporalProperty: - """Get the history model for given entity class.""" - assert hasattr(target.class_, 'temporal_options') - return target.class_.temporal_options.history_tables[target.property] +def init_clock(clocked: Clocked, args, kwargs): + kwargs.setdefault('vclock', 1) + initial_tick = clocked.temporal_options.clock_model( + tick=kwargs['vclock'], + entity=clocked, + ) + + if clocked.temporal_options.activity_cls and 'activity' not in kwargs: + raise ValueError("%r missing keyword argument: activity" % clocked.__class__) + + if 'activity' in kwargs: + initial_tick.activity = kwargs.pop('activity') + + materialize_defaults(clocked, kwargs) + + +def materialize_defaults(clocked, kwargs): + """Add the first clock tick when initializing. + Note: Special case for non-server side defaults""" + # Note this block is because sqlalchemy doesn't materialize default + # values on instances until after a flush but we need defaults & nulls + # before flush so we can have a consistent history + warnings.warn("this method is unnecessary with recent sqlalchemy changes", + PendingDeprecationWarning) + to_materialize = { + prop for prop + in clocked.temporal_options.history_tables.keys() + if prop.key not in kwargs + and getattr(prop.class_attribute, 'default', None) is not None} + for prop in to_materialize: + if callable(prop.class_attribute.default.arg): + value = prop.class_attribute.default.arg(clocked) + else: + value = prop.class_attribute.default.arg + setattr(clocked, prop.key, value) + + +def defaults_safety(*track, mapper): + warnings.warn("these caveats are temporary", PendingDeprecationWarning) + local_props = {mapper.get_property(prop) for prop in track} + for prop in local_props: + assert all(col.onupdate is None for col in prop.columns), \ + '%r has onupdate' % prop + assert all(col.server_default is None for col in prop.columns), \ + '%r has server_default' % prop + assert all(col.server_onupdate is None for col in prop.columns), \ + '%r has server_onupdate' % prop + + +def relationship_safety(mapper): + warnings.warn("these caveats are temporary", PendingDeprecationWarning) + cls = mapper.class_ + relationship_props = set() + for prop in mapper.relationships: + if 'temporal_on' in prop.info: + assert hasattr(cls, prop.info['temporal_on']), \ + '%r is missing a property %s' % ( + cls, prop.info['temporal_on']) + assert isinstance( + mapper.get_property(prop.info['temporal_on']), + orm.ColumnProperty), \ + '%r has %s but it is not a Column' % ( + cls, prop.info['temporal_on']) + relationship_props.add(prop) # TODO kwargs to override default clock table and history tables prefix @@ -56,160 +170,20 @@ def add_clock(*props: typing.Iterable[str], # noqa: C901 temporal_schema: typing.Optional[str] = None): """Decorator to add clock and history to an orm model.""" - def init_clock(clocked: Clocked, args, kwargs): - """Add the first clock tick when initializing. - - Note: Special case for non-server side defaults""" - # Note this block is because sqlalchemy doesn't materialize default - # values on instances until after a flush but we need defaults & nulls - # before flush so we can have a consistent history - # TODO this whole thing seems horribly complex - to_materialize = { - prop for prop - in clocked.temporal_options.history_tables.keys() - if prop.key not in kwargs - and getattr(prop.class_attribute, 'default', None) is not None} - for prop in to_materialize: - if callable(prop.class_attribute.default.arg): - value = prop.class_attribute.default.arg(clocked) - else: - value = prop.class_attribute.default.arg - setattr(clocked, prop.key, value) - - clocked.vclock = 1 - initial_clock_tick = clocked.temporal_options.clock_table( - tick=clocked.vclock) - if activity_cls is not None: - try: - initial_clock_tick.activity = kwargs.pop('activity') - except KeyError as e: - raise ValueError( - "activity is missing on create (%s)" % e) from None - - clocked.clock = [initial_clock_tick] - def make_temporal(cls: nine.Type[Clocked]): assert issubclass(cls, Clocked), "add temporal.Clocked to %r" % cls mapper = cls.__mapper__ - - local_props = {mapper.get_property(prop) for prop in props} - for prop in local_props: - assert all(col.onupdate is None for col in prop.columns), \ - '%r has onupdate' % prop - assert all(col.server_default is None for col in prop.columns), \ - '%r has server_default' % prop - assert all(col.server_onupdate is None for col in prop.columns), \ - '%r has server_onupdate' % prop - - relationship_props = set() - for prop in mapper.relationships: - # TODO: there has got to be a better way - if 'temporal_on' in prop.info: - assert hasattr(cls, prop.info['temporal_on']), \ - '%r is missing a property %s' % ( - cls, prop.info['temporal_on']) - assert isinstance( - mapper.get_property(prop.info['temporal_on']), - orm.ColumnProperty), \ - '%r has %s but it is not a Column' % ( - cls, prop.info['temporal_on']) - relationship_props.add(prop) - - # make sure all temporal properties have active_history (always loaded) - for prop in local_props | relationship_props: - getattr(cls, prop.key).impl.active_history = True - - entity_table = mapper.local_table - entity_table_name = entity_table.name - schema = temporal_schema or entity_table.schema - clock_table_name = truncate_identifier("%s_clock" % entity_table_name) - - history_tables = { - p: build_history_class(cls, p, schema) - for p in local_props | relationship_props - } - - clock_properties = dict( - __tablename__=clock_table_name, - # todo support different shape PKs - entity_id=sa.Column(sa.ForeignKey(cls.id), primary_key=True), - entity=orm.relationship( - cls, backref=orm.backref("clock", lazy='dynamic')), - ) - - if activity_cls is not None: - backref_name = '%s_clock' % entity_table_name - clock_properties['activity_id'] = sa.Column( - sa.ForeignKey(activity_cls.id), nullable=False) - clock_properties['activity'] = orm.relationship( - activity_cls, backref=backref_name) - clock_properties['__table_args__'] = ( - sa.UniqueConstraint('entity_id', 'activity_id'), - {'schema': schema} - ) - else: - clock_properties['__table_args__'] = {'schema': schema} - - clock_table = build_clock_class( - cls.__name__, cls.metadata, clock_properties) - # Add relationships for the latest and first clock ticks. These are - # often accessed in list views and should be eagerly joined on when - # doing so like this: `query.options(orm.joinedload('latest_tick'))` - latest_tick = orm.relationship( - clock_table, - primaryjoin=sa.and_(cls.id == clock_table.entity_id, - cls.vclock == clock_table.tick), - innerjoin=True, - uselist=False, # We are looking up a single child record - ) - mapper.add_property('latest_tick', latest_tick) - - first_tick = orm.relationship( - clock_table, - primaryjoin=sa.and_(cls.id == clock_table.entity_id, - clock_table.tick == 1), - innerjoin=True, - uselist=False, # We are looking up a single child record - ) - mapper.add_property('first_tick', first_tick) - - temporal_options = TemporalOption( - temporal_props=local_props | relationship_props, - history_models=history_tables, - clock_model=clock_table, - activity_cls=activity_cls, - ) - cls.temporal_options = temporal_options - event.listen(cls, 'init', init_clock) - + defaults_safety(*props, mapper=mapper) + relationship_safety(mapper) + temporal_map(*props, + mapper=mapper, + activity_class=activity_cls, + schema=temporal_schema) return cls return make_temporal -def _copy_column(column: sa.Column) -> sa.Column: - """copy a column, set some properties on it for history table creation""" - original = column - new = column.copy() - original.info['history_copy'] = new - for fk in column.foreign_keys: - new.append_foreign_key(sa.ForeignKey(fk.target_fullname)) - new.unique = False - new.default = new.server_default = None - - return new - - -def truncate_identifier(identifier: str) -> str: - """ensure identifier doesn't exceed max characters postgres allows""" - max_len = (sap.dialect.max_index_name_length - or sap.dialect.max_identifier_length) - if len(identifier) > max_len: - return "%s_%s" % (identifier[0:max_len - 8], - sa_util.md5_hex(identifier)[-4:]) - return identifier - - def build_clock_class( name: str, metadata: sa.MetaData, @@ -221,6 +195,46 @@ def build_clock_class( return type('%sClock' % name, base_classes, props) +def build_clock_table(entity_table: sa.Table, + metadata: sa.MetaData, + schema: str, + activity_class=None) -> sa.Table: + clock_table_name = util.truncate_identifier( + "%s_clock" % entity_table.name) + clock_table = sa.Table( + clock_table_name, + metadata, + sa.Column('tick', + sa.Integer, + primary_key=True, + autoincrement=False), + sa.Column('timestamp', + sa.DateTime(True), + server_default=sa.func.current_timestamp()), + schema=schema) + + entity_keys = set() + for fk in util.foreign_key_to(entity_table, primary_key=True): + # this is done to support arbitrary primary key shape on entity + clock_table.append_column(fk) + entity_keys.add(fk.key) + + if activity_class: + activity_keys = set() + # support arbitrary shaped activity primary keys + for fk in util.foreign_key_to(activity_class.__table__, + prefix='activity', + nullable=False): + clock_table.append_column(fk) + activity_keys.add(fk.key) + # ensure we have DB constraint on clock <> activity uniqueness + clock_table.append_constraint( + sa.UniqueConstraint(*(entity_keys | activity_keys)) + ) + + return clock_table + + def build_history_class( cls: declarative.DeclarativeMeta, prop: T_PROPS, @@ -273,12 +287,12 @@ def build_history_table( schema: str = None) -> sa.Table: """build a sql alchemy table for given prop""" if isinstance(prop, orm.RelationshipProperty): - columns = [_copy_column(column) for column in prop.local_columns] + columns = [util.copy_column(column) for column in prop.local_columns] else: - columns = [_copy_column(column) for column in prop.columns] + columns = [util.copy_column(column) for column in prop.columns] local_table = cls.__table__ - table_name = truncate_identifier( + table_name = util.truncate_identifier( _generate_history_table_name(local_table, columns) ) entity_foreign_keys = list(util.foreign_key_to(local_table)) @@ -289,17 +303,17 @@ def build_history_table( constraints = [ sa.Index( - truncate_identifier('%s_effective_idx' % table_name), + util.truncate_identifier('%s_effective_idx' % table_name), 'effective', postgresql_using='gist' ), sap.ExcludeConstraint( *itertools.chain(entity_constraints, [('vclock', '&&')]), - name=truncate_identifier('%s_excl_vclock' % table_name) + name=util.truncate_identifier('%s_excl_vclock' % table_name) ), sap.ExcludeConstraint( *itertools.chain(entity_constraints, [('effective', '&&')]), - name=truncate_identifier('%s_excl_effective' % table_name) + name=util.truncate_identifier('%s_excl_effective' % table_name) ), ] @@ -312,7 +326,7 @@ def build_history_table( primary_key=True), sa.Column('effective', sap.TSTZRANGE, - default=effective_now, + default=util.effective_now, nullable=False), sa.Column('vclock', sap.INT4RANGE, nullable=False), *itertools.chain(entity_foreign_keys, columns, constraints), diff --git a/temporal_sqlalchemy/core.py b/temporal_sqlalchemy/core.py index 5f3900d..93e6c4e 100644 --- a/temporal_sqlalchemy/core.py +++ b/temporal_sqlalchemy/core.py @@ -1,149 +1,24 @@ -import sqlalchemy as sa import sqlalchemy.ext.declarative as declarative import sqlalchemy.orm as orm -import sqlalchemy.event as event -from temporal_sqlalchemy import bases, clock, util +from temporal_sqlalchemy import bases, clock class TemporalModel(bases.Clocked): - @staticmethod - def build_clock_table(entity_table: sa.Table, - metadata: sa.MetaData, - schema: str, - activity_class=None) -> sa.Table: - clock_table_name = clock.truncate_identifier( - "%s_clock" % entity_table.name) - clock_table = sa.Table( - clock_table_name, - metadata, - sa.Column('tick', - sa.Integer, - primary_key=True, - autoincrement=False), - sa.Column('timestamp', - sa.DateTime(True), - server_default=sa.func.current_timestamp()), - schema=schema) - - entity_keys = set() - for fk in util.foreign_key_to(entity_table, primary_key=True): - # this is done to support arbitrary primary key shape on entity - clock_table.append_column(fk) - entity_keys.add(fk.key) - - if activity_class: - activity_keys = set() - # support arbitrary shaped activity primary keys - for fk in util.foreign_key_to(activity_class.__table__, - prefix='activity', - nullable=False): - clock_table.append_column(fk) - activity_keys.add(fk.key) - # ensure we have DB constraint on clock <> activity uniqueness - clock_table.append_constraint( - sa.UniqueConstraint(*(entity_keys | activity_keys)) - ) - - return clock_table - - @staticmethod - def temporal_map(mapper: orm.Mapper, cls: bases.Clocked): - temporal_declaration = cls.Temporal - assert 'vclock' not in temporal_declaration.track - entity_table = mapper.local_table - # get things defined on Temporal: - tracked_props = frozenset( - mapper.get_property(prop) for prop in temporal_declaration.track - ) - # make sure all temporal properties have active_history (always loaded) - for prop in tracked_props: - getattr(cls, prop.key).impl.active_history = True - - activity_class = getattr(temporal_declaration, 'activity_class', None) - schema = getattr(temporal_declaration, 'schema', entity_table.schema) - - clock_table = TemporalModel.build_clock_table( - entity_table, - entity_table.metadata, - schema, - activity_class - ) - clock_properties = { - 'entity': orm.relationship( - lambda: cls, backref=orm.backref('clock', lazy='dynamic') - ), - 'entity_first_tick': orm.relationship( - lambda: cls, - backref=orm.backref( - 'first_tick', - primaryjoin=sa.and_( - clock_table.join(entity_table).onclause, - clock_table.c.tick == 1 - ), - innerjoin=True, - uselist=False, # single record - viewonly=True # view only - ) - ), - 'entity_latest_tick': orm.relationship( - lambda: cls, - backref=orm.backref( - 'latest_tick', - primaryjoin=sa.and_( - clock_table.join(entity_table).onclause, - entity_table.c.vclock == clock_table.c.tick - ), - innerjoin=True, - uselist=False, # single record - viewonly=True # view only - ) - ), - '__table__': clock_table - } # used to construct a new clock model for this entity - - if activity_class: - # create a relationship to the activity from the clock model - backref_name = '%s_clock' % entity_table.name - clock_properties['activity'] = \ - orm.relationship(lambda: activity_class, backref=backref_name) - - clock_model = clock.build_clock_class(cls.__name__, - entity_table.metadata, - clock_properties) - - history_models = { - p: clock.build_history_class(cls, p, schema) - for p in tracked_props - } - - cls.temporal_options = bases.TemporalOption( - temporal_props=tracked_props, - history_models=history_models, - clock_model=clock_model, - activity_cls=activity_class - ) - - event.listen(cls, 'init', TemporalModel.init_clock) - - @staticmethod - def init_clock(clocked: 'TemporalModel', args, kwargs): - kwargs.setdefault('vclock', 1) - initial_tick = clocked.temporal_options.clock_model( - tick=kwargs['vclock'], - entity=clocked, - ) - - if 'activity' in kwargs: - initial_tick.activity = kwargs.pop('activity') @declarative.declared_attr def __mapper_cls__(cls): assert hasattr(cls, 'Temporal') - def mapper(cls, *args, **kwargs): - mp = orm.mapper(cls, *args, **kwargs) - cls.temporal_map(mp, cls) + def mapper(cls_, *args, **kwargs): + options = cls_.Temporal + + mp = orm.mapper(cls_, *args, **kwargs) + clock.temporal_map( + *options.track, + mapper=mp, + activity_class=getattr(options, 'activity_class'), + schema=getattr(options, 'schema')) return mp return mapper diff --git a/temporal_sqlalchemy/util.py b/temporal_sqlalchemy/util.py index 49fa0f6..c1d79a7 100644 --- a/temporal_sqlalchemy/util.py +++ b/temporal_sqlalchemy/util.py @@ -1,6 +1,14 @@ +import datetime as dt import typing +import psycopg2.extras as psql_extras import sqlalchemy as sa +import sqlalchemy.dialects.postgresql as sap +import sqlalchemy.orm as orm +import sqlalchemy.orm.attributes as attributes +import sqlalchemy.util as sa_util + +from temporal_sqlalchemy import bases def foreign_key_to(table: sa.Table, prefix='entity', **opts) -> typing.Iterable[sa.Column]: # pylint: disable=unsubscriptable-object @@ -8,3 +16,54 @@ def foreign_key_to(table: sa.Table, prefix='entity', **opts) -> typing.Iterable[ for pk in table.primary_key: name = '%s_%s' % (prefix, pk.name) yield sa.Column(name, pk.type, sa.ForeignKey(pk), **opts) + + +def effective_now() -> psql_extras.DateTimeTZRange: + utc_now = dt.datetime.now(tz=dt.timezone.utc) + return psql_extras.DateTimeTZRange(utc_now, None) + + +def copy_column(column: sa.Column) -> sa.Column: + """copy a column, set some properties on it for history table creation""" + original = column + new = column.copy() + original.info['history_copy'] = new + for fk in column.foreign_keys: + new.append_foreign_key(sa.ForeignKey(fk.target_fullname)) + new.unique = False + new.default = new.server_default = None + + return new + + +def truncate_identifier(identifier: str) -> str: + """ensure identifier doesn't exceed max characters postgres allows""" + max_len = (sap.dialect.max_index_name_length + or sap.dialect.max_identifier_length) + if len(identifier) > max_len: + return "%s_%s" % (identifier[0:max_len - 8], + sa_util.md5_hex(identifier)[-4:]) + return identifier + + +def get_activity_clock_backref( + activity: bases.TemporalActivityMixin, + entity: bases.Clocked) -> orm.RelationshipProperty: + """Get the backref'd clock history for a given entity.""" + assert ( + activity is entity.temporal_options.activity_cls or + isinstance(activity, entity.temporal_options.activity_cls) + ), "cannot inspect %r for mapped activity %r" % (entity, activity) + + inspected = sa.inspect(entity.temporal_options.activity_cls) + backref = entity.temporal_options.clock_table.activity.property.backref + + return inspected.relationships[backref] + + +def get_history_model( + target: attributes.InstrumentedAttribute) -> bases.TemporalProperty: + """Get the history model for given entity class.""" + assert hasattr(target.class_, 'temporal_options') + + return target.class_.temporal_options.history_tables[target.property] diff --git a/tests/test_builders.py b/tests/test_builders.py index 8670acd..41db0e8 100644 --- a/tests/test_builders.py +++ b/tests/test_builders.py @@ -1,8 +1,12 @@ +import pytest import sqlalchemy as sa import temporal_sqlalchemy as temporal from temporal_sqlalchemy.clock import ( - build_history_table, build_history_class, build_clock_class) + build_history_table, + build_history_class, + build_clock_class, + build_clock_table) from . import models @@ -43,3 +47,73 @@ def test_build_clock_class(): assert clock.__name__ == 'TestingClock' assert issubclass(clock, temporal.EntityClock) + + +@pytest.mark.parametrize('table,expected_name,expected_cols,activity_class', ( + ( + sa.Table( + 'bare_table_single_pk_no_activity', + sa.MetaData(), + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('description', sa.Text), + schema='bare_table_test_schema' + ), + 'bare_table_single_pk_no_activity_clock', + {'tick', 'timestamp', 'entity_id'}, + None + ), + ( + sa.Table( + 'bare_table_compositve_pk_no_activity', + sa.MetaData(), + sa.Column('num_id', sa.Integer, primary_key=True), + sa.Column('text_id', sa.Text, primary_key=True), + sa.Column('description', sa.Text), + schema='bare_table_test_schema' + ), + 'bare_table_compositve_pk_no_activity_clock', + {'tick', 'timestamp', 'entity_num_id', 'entity_text_id'}, + None + ), + ( + sa.Table( + 'bare_table_single_pk_with_activity', + sa.MetaData(), + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('description', sa.Text), + schema='bare_table_test_schema' + ), + 'bare_table_single_pk_with_activity_clock', + {'tick', 'timestamp', 'entity_id', 'activity_id'}, + models.Activity + ), + ( + sa.Table( + 'bare_table_compositve_pk_with_activity', + sa.MetaData(), + sa.Column('num_id', sa.Integer, primary_key=True), + sa.Column('text_id', sa.Text, primary_key=True), + sa.Column('description', sa.Text), + schema='bare_table_test_schema' + ), + 'bare_table_compositve_pk_with_activity_clock', + {'tick', 'timestamp', 'entity_num_id', 'entity_text_id', 'activity_id'}, + models.Activity + ) +)) +def test_build_clock_table(table, expected_name, expected_cols, activity_class): + clock_table = build_clock_table( + table, + table.metadata, + table.schema, + activity_class + ) + assert clock_table.name == expected_name + assert clock_table.metadata is table.metadata + assert {c.key for c in clock_table.c} == expected_cols + for foreign_key in clock_table.foreign_keys: + references_entity = foreign_key.references(table) + if activity_class: + assert foreign_key.references(activity_class.__table__) or references_entity + else: + assert references_entity diff --git a/tests/test_temporal_model_mixin.py b/tests/test_temporal_model_mixin.py index 2ff9b2f..1ea6604 100644 --- a/tests/test_temporal_model_mixin.py +++ b/tests/test_temporal_model_mixin.py @@ -20,83 +20,13 @@ class Error(models.Base, temporal.TemporalModel): def test_create_temporal_options(): assert hasattr(models.NewStyleModel, 'temporal_options') - m = models.NewStyleModel() + m = models.NewStyleModel(activity=models.Activity(description="Activity Description")) assert hasattr(m, 'temporal_options') assert m.temporal_options is models.NewStyleModel.temporal_options assert isinstance(m.temporal_options, temporal.TemporalOption) -@pytest.mark.parametrize('table,expected_name,expected_cols,activity_class', ( - ( - sa.Table( - 'bare_table_single_pk_no_activity', - sa.MetaData(), - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('description', sa.Text), - schema='bare_table_test_schema' - ), - 'bare_table_single_pk_no_activity_clock', - {'tick', 'timestamp', 'entity_id'}, - None - ), - ( - sa.Table( - 'bare_table_compositve_pk_no_activity', - sa.MetaData(), - sa.Column('num_id', sa.Integer, primary_key=True), - sa.Column('text_id', sa.Text, primary_key=True), - sa.Column('description', sa.Text), - schema='bare_table_test_schema' - ), - 'bare_table_compositve_pk_no_activity_clock', - {'tick', 'timestamp', 'entity_num_id', 'entity_text_id'}, - None - ), - ( - sa.Table( - 'bare_table_single_pk_with_activity', - sa.MetaData(), - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('description', sa.Text), - schema='bare_table_test_schema' - ), - 'bare_table_single_pk_with_activity_clock', - {'tick', 'timestamp', 'entity_id', 'activity_id'}, - models.Activity - ), - ( - sa.Table( - 'bare_table_compositve_pk_with_activity', - sa.MetaData(), - sa.Column('num_id', sa.Integer, primary_key=True), - sa.Column('text_id', sa.Text, primary_key=True), - sa.Column('description', sa.Text), - schema='bare_table_test_schema' - ), - 'bare_table_compositve_pk_with_activity_clock', - {'tick', 'timestamp', 'entity_num_id', 'entity_text_id', 'activity_id'}, - models.Activity - ) -)) -def test_build_clock_table(table, expected_name, expected_cols, activity_class): - clock_table = temporal.TemporalModel.build_clock_table( - table, - table.metadata, - table.schema, - activity_class - ) - assert clock_table.name == expected_name - assert clock_table.metadata is table.metadata - assert {c.key for c in clock_table.c} == expected_cols - for foreign_key in clock_table.foreign_keys: - references_entity = foreign_key.references(table) - if activity_class: - assert foreign_key.references(activity_class.__table__) or references_entity - else: - assert references_entity - - def test_creates_clock_model(): options = models.NewStyleModel.temporal_options From a0ef7344ef0c2ea7702b45d7ea5550c77fe6188d Mon Sep 17 00:00:00 2001 From: Joey Leingang Date: Mon, 26 Jun 2017 15:07:50 -0700 Subject: [PATCH 3/4] fix relationship handling --- temporal_sqlalchemy/clock.py | 45 ++++++++++++------------------------ tests/models.py | 6 ++--- tests/test_invariants.py | 21 ----------------- 3 files changed, 18 insertions(+), 54 deletions(-) diff --git a/temporal_sqlalchemy/clock.py b/temporal_sqlalchemy/clock.py index e09a828..96a5f24 100644 --- a/temporal_sqlalchemy/clock.py +++ b/temporal_sqlalchemy/clock.py @@ -98,23 +98,24 @@ def temporal_map(*track, mapper: orm.Mapper, activity_class=None, schema=None): event.listen(cls, 'init', init_clock) -def init_clock(clocked: Clocked, args, kwargs): +def init_clock(obj: Clocked, args, kwargs): kwargs.setdefault('vclock', 1) - initial_tick = clocked.temporal_options.clock_model( + initial_tick = obj.temporal_options.clock_model( tick=kwargs['vclock'], - entity=clocked, + entity=obj, ) - if clocked.temporal_options.activity_cls and 'activity' not in kwargs: - raise ValueError("%r missing keyword argument: activity" % clocked.__class__) + if obj.temporal_options.activity_cls and 'activity' not in kwargs: + raise ValueError( + "%r missing keyword argument: activity" % obj.__class__) if 'activity' in kwargs: initial_tick.activity = kwargs.pop('activity') - materialize_defaults(clocked, kwargs) + materialize_defaults(obj, kwargs) -def materialize_defaults(clocked, kwargs): +def materialize_defaults(obj: Clocked, kwargs): """Add the first clock tick when initializing. Note: Special case for non-server side defaults""" # Note this block is because sqlalchemy doesn't materialize default @@ -123,22 +124,24 @@ def materialize_defaults(clocked, kwargs): warnings.warn("this method is unnecessary with recent sqlalchemy changes", PendingDeprecationWarning) to_materialize = { - prop for prop - in clocked.temporal_options.history_tables.keys() + prop for prop in obj.temporal_options.history_models.keys() if prop.key not in kwargs - and getattr(prop.class_attribute, 'default', None) is not None} + and getattr(prop.class_attribute, 'default', None) is not None + } for prop in to_materialize: if callable(prop.class_attribute.default.arg): - value = prop.class_attribute.default.arg(clocked) + value = prop.class_attribute.default.arg(obj) else: value = prop.class_attribute.default.arg - setattr(clocked, prop.key, value) + setattr(obj, prop.key, value) def defaults_safety(*track, mapper): warnings.warn("these caveats are temporary", PendingDeprecationWarning) local_props = {mapper.get_property(prop) for prop in track} for prop in local_props: + if isinstance(prop, orm.RelationshipProperty): + continue assert all(col.onupdate is None for col in prop.columns), \ '%r has onupdate' % prop assert all(col.server_default is None for col in prop.columns), \ @@ -147,23 +150,6 @@ def defaults_safety(*track, mapper): '%r has server_onupdate' % prop -def relationship_safety(mapper): - warnings.warn("these caveats are temporary", PendingDeprecationWarning) - cls = mapper.class_ - relationship_props = set() - for prop in mapper.relationships: - if 'temporal_on' in prop.info: - assert hasattr(cls, prop.info['temporal_on']), \ - '%r is missing a property %s' % ( - cls, prop.info['temporal_on']) - assert isinstance( - mapper.get_property(prop.info['temporal_on']), - orm.ColumnProperty), \ - '%r has %s but it is not a Column' % ( - cls, prop.info['temporal_on']) - relationship_props.add(prop) - - # TODO kwargs to override default clock table and history tables prefix def add_clock(*props: typing.Iterable[str], # noqa: C901 activity_cls: nine.Type[TemporalActivityMixin] = None, @@ -174,7 +160,6 @@ def make_temporal(cls: nine.Type[Clocked]): assert issubclass(cls, Clocked), "add temporal.Clocked to %r" % cls mapper = cls.__mapper__ defaults_safety(*props, mapper=mapper) - relationship_safety(mapper) temporal_map(*props, mapper=mapper, activity_class=activity_cls, diff --git a/tests/models.py b/tests/models.py index 4fc7b4c..93cf866 100644 --- a/tests/models.py +++ b/tests/models.py @@ -97,7 +97,7 @@ class RelatedTable(Base): @temporal_sqlalchemy.add_clock( - 'prop_a', 'prop_b', 'rel_id', temporal_schema=TEMPORAL_SCHEMA) + 'prop_a', 'prop_b', 'rel_id', 'rel', temporal_schema=TEMPORAL_SCHEMA) class RelationalTemporalModel(temporal_sqlalchemy.Clocked, Base): __tablename__ = 'relational_temporal' __table_args__ = {'schema': SCHEMA} @@ -106,7 +106,7 @@ class RelationalTemporalModel(temporal_sqlalchemy.Clocked, Base): prop_a = sa.Column(sa.Integer) prop_b = sa.Column(sap.TEXT) rel_id = sa.Column(sa.ForeignKey(RelatedTable.id)) - rel = orm.relationship(RelatedTable, info={'temporal_on': 'rel_id'}) + rel = orm.relationship(RelatedTable) class Activity(temporal_sqlalchemy.TemporalActivityMixin, Base): @@ -148,7 +148,7 @@ class SimpleTable(Base): prop_a = sa.Column(sa.Integer) prop_b = sa.Column(sap.TEXT) rel_id = sa.Column(sa.ForeignKey(RelatedTable.id)) - rel = orm.relationship(RelatedTable, info={'temporal_on': 'rel_id'}) + rel = orm.relationship(RelatedTable) REALLY_REALLY = 'really_' * 5 diff --git a/tests/test_invariants.py b/tests/test_invariants.py index 2411582..c84e010 100644 --- a/tests/test_invariants.py +++ b/tests/test_invariants.py @@ -65,24 +65,3 @@ class TemporalTableWithServerOnUpdate( prop_b = sa.Column(sap.TEXT) prop_c = sa.Column(sa.DateTime, server_onupdate=sa.func.current_timestamp()) - - -def test_fails_on_bad_relation_info(): - class RelatedFail(models.ExpectedFailBase): - __tablename__ = 'related_fail' - __table_args__ = {'schema': models.SCHEMA} - - id = models.auto_uuid() - prop_a = sa.Column(sa.Integer) - - with pytest.raises(AssertionError): - @temporal.add_clock('prop_a', 'related_fail_id') - class ParentRelatedFail(temporal.Clocked, models.ExpectedFailBase): - __tablename__ = 'related_fail_parent' - __table_args__ = {'schema': models.SCHEMA} - - id = models.auto_uuid() - prop_a = sa.Column(sa.Integer) - related_fail_id = sa.Column(sa.ForeignKey(RelatedFail.id)) - related_fail = orm.relationship(RelatedFail, info={ - 'temporal_on': 'related_id'}) From 649f3dccfdae36501d40c6172a77cc747f1683a9 Mon Sep 17 00:00:00 2001 From: Joey Leingang Date: Mon, 26 Jun 2017 15:08:19 -0700 Subject: [PATCH 4/4] new version --- temporal_sqlalchemy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/temporal_sqlalchemy/version.py b/temporal_sqlalchemy/version.py index 29a4408..114c9c5 100644 --- a/temporal_sqlalchemy/version.py +++ b/temporal_sqlalchemy/version.py @@ -1,2 +1,2 @@ """Version information.""" -__version__ = '0.3.2' +__version__ = '0.4.0'