App Engine Import Datastore Export (NDB/Backup)

import_backup.py

"""
# App Engine import data from Datastore Backup to localhost
You can use this script to import large(ish) App Engine Datastore backups to your localohst dev server.
## Getting backup files
Follow instructions from Google to use `gcloud` to export datastore to a google storage bucket

Then download the export to your local machine:
```
gsutil -m cp -R gs://your_bucket_name/your_path /local_target
```
## Reading data to your local (dev_appserver) application
Copy the downloaded datastore export into your app directory
Copy-paste this gist to your Interactive Console, set correct relative path and press `Execute`.
(default: http://localhost:8000/console)

Credits to GitHub user jehna and his gist: https://gist.github.com/jehna/3b258f5287fcc181aacf
"""
from google.appengine.api.files import records
from google.appengine.datastore import entity_pb
from google.net.proto.ProtocolBuffer import ProtocolBufferDecodeError
from google.appengine.datastore import datastore_pbs
from google.appengine.api import datastore
from google.appengine.ext import db
from google.appengine.ext import ndb
from os.path import isfile
from os.path import join
from os import listdir


##############
Insert NDB models here:
class MyKind(ndb.Model):
    name = ndb.StringProperty()
    age = ndb.IntegerProperty()
##############




def run():
    # Set your downloaded folder's path here (must be readable by dev_appserver)
    mypath = 'all_kinds'
    # Set your app's name here
    appname = "dev~None"
    
    
    # Do the harlem shake
    onlyfiles = [ f for f in listdir(mypath) if isfile(join(mypath,f)) ]
    ec = datastore_pbs.get_entity_converter()
    
    for file in onlyfiles:
        i = 0
        print "Opening file %s" % file
        try:
            raw = open(mypath + "/" + file, 'rb')
            reader = records.RecordsReader(raw)
            to_put = list()
            for record in reader:
                entity_proto = entity_pb.EntityProto(record)
                # print "entity_proto %s" % entity_proto
                entity_proto.key_.app_ = appname
                # entity = db.model_from_protobuf(entity_proto)
                # a = db.model_from_protobuf(entity_proto)
                entity = ndb.ModelAdapter().pb_to_entity(entity_proto)
                a = ndb.ModelAdapter().pb_to_entity(entity_proto)
    
                for pp in dir(a):
                   try:
                      ppp = getattr(a, "_" + pp)
                      if isinstance(ppp, db.Key):
                         ppp._Key__reference.set_app(appname)
                         ppp
                   except AttributeError:
                      """ It's okay """
                
                to_put.append(a)
                i += 1
                if i % 100 == 0:
                    print "Saved %d %ss" % (i, entity.key.kind())
                    ndb.put_multi(to_put)
                    to_put = list()
            
            ndb.put_multi(to_put)
            to_put = list()
            print "Saved %d" % i
            
        except ProtocolBufferDecodeError:
            """ All good """
 
run()