Skip to content

Instantly share code, notes, and snippets.

@miodeqqq
Created December 8, 2016 20:11
Show Gist options
  • Select an option

  • Save miodeqqq/4e47f9e3a0c88885308594ae16418c82 to your computer and use it in GitHub Desktop.

Select an option

Save miodeqqq/4e47f9e3a0c88885308594ae16418c82 to your computer and use it in GitHub Desktop.
PyMongo basic setup for connecting with database. For example: find all PDF files and download them to local drive.
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
from gridfs import GridFS
from pymongo import MongoClient
from bson.objectid import ObjectId
MONGODB_SERVER = 'you_mongo_server.com'
MONGODB_PORT = 27017
MONGODB_DB = 'your_db_name'
MONGODB_COLLECTION = 'fs.collection_name'
def main():
connection = MongoClient(
'{server}:{port}'.format(
server=MONGODB_SERVER,
port=MONGODB_PORT,
connect=False,
)
)
db = connection[MONGODB_DB]
fs = GridFS(db)
my_db = db.get_collection(MONGODB_COLLECTION)
query = my_db.find(
{
"metadata.file_type": 'pdf'
},
{
"_id": 1
}
)
print("Extracting ID's")
ids_data = [str(item['_id']) for item in query]
print("Extracted --> {} id's".format(len(ids_data)))
for i, file_id in enumerate(ids_data):
print("Processing file {index}/{n}".format(
index=i + 1,
n=len(ids_data))
)
file_from_gridfs = fs.get(ObjectId(file_id)).read()
dir_path = os.path.join(
'pdf_files',
file_id[0:2],
file_id[2:4],
)
os.makedirs(dir_path, exist_ok=True)
file_path = os.path.join(
dir_path, "{}.pdf".format(file_id)
)
with open(file_path, "wb") as f:
f.write(file_from_gridfs)
print("Done!")
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment