versioning and automatic migration for ndb.Model on Google App Engine

versioned_model.py

# Article for this code: https://medium.com/engineering-workzeit/e21938a2e2ec
#
# NOTE: mapreduce.operation.ndb is not part of the mapreduce library shipped with App Engine.
#       See here for an implementation: https://gist.github.com/jdiaz5513/8911930

from google.appengine.ext import ndb


class VersionedModel(ndb.Model):
    new_property = ndb.StringProperty()
    version = ndb.IntegerProperty(name='v', default=1)
    
    def _migrate_old_version(self):
        to_put = []
        to_delete = []
        # migration logic goes here
        if 'old_property' in self._values:
            self.new_property = self._values['old_property'].b_val
            del self._values['old_property']
            del self._properties['old_property']
        self.version = 1
        to_put.append(self)
        return (to_put, to_delete)
    
    @classmethod
    def _post_get_hook(cls, key, future):
        entity = future.get_result()
        if entity is not None and entity.version < 1:
            entity._migrate_old_version()


def _VersionedModel_migration(entity):
    from mapreduce.operation import ndb as op
    to_put, to_delete = entity._migrate_old_version()
    for e in to_put:
        yield op.Put(e)
    for e in to_delete:
        yield op.Delete(e)


# example MapReduce job:
#
# >>> from google.appengine.ext.mapreduce import base_handler
# >>> class MigrationMapper(base_handler.PipelineBase):
#        def run(self):
#            yield mapreduce_pipeline.MapPipeline(
#                'version-migration',
#                'versioned_model._VersionedModel_migration',
#                'mapreduce.input_readers.DatastoreInputReader',
#                params={'entity_kind': 'VersionedModel',
#                        'batch_size': 100})
#
# >>> mapper = MigrationMapper()
# >>> mapper.start()