Skip to content

Instantly share code, notes, and snippets.

@mentat
Created October 27, 2011 22:47
Show Gist options
  • Save mentat/1321129 to your computer and use it in GitHub Desktop.
Save mentat/1321129 to your computer and use it in GitHub Desktop.
Mapreduce job for blob migration
def migrate_blobstore(entity):
" Migrate entities with blob fields from another app. "
import urllib, mimetypes
from google.appengine.api import files
modified = False
fields = entity.get_blob_fields()
logging.debug("Fields are: %s" % fields)
for field in fields:
if entity.get_blobref_key(field):
modified = True
try:
resp = urlfetch.fetch(
'http://blah.appspot.com/admin/migrate/blobs/%s' % urllib.quote(str(entity.get_blobref_key(field))),
deadline=30
)
except urlfetch.DownloadError:
# Try again
raise
else:
if resp.status_code == 404:
logging.info("Blob gave 404 %s" % entity.get_blobref_key(field))
else:
filename = '%s_%s_%d%s' % (
entity.__class__.__name__,
field,
entity.key().id(),
mimetypes.guess_extension(resp.headers['Content-Type']).replace('jpe','jpg')
)
logging.info("Storing blob as %s" % filename)
file_name = files.blobstore.create(
mime_type=resp.headers['Content-Type'],
_blobinfo_uploaded_filename=filename
)
with files.open(file_name, 'a') as f:
f.write(resp.content)
files.finalize(file_name)
# Get the file's blob key
blob_key = files.blobstore.get_blob_key(file_name)
setattr(entity, field, blob_key)
if modified:
# Enable high performance urls
entity.fix_serving_urls()
yield op.db.Put(entity)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment