unstructured-ingest-mdb.py

from unstructured.partition.auto import partition
import pymongo
from openai import AzureOpenAI

az_client = AzureOpenAI(azure_endpoint="",api_version="",api_key="")
def generate_embeddings(text, model=""): # model = "deployment_name"
    return az_client.embeddings.create(input = [text], model=model).data[0].embedding

MDB_URI = ""
DB_NAME = ""
COLLECTION_NAME = ""

print("Partioning PDF document...")
elements = partition("document.pdf")
print(elements)
print("\n\n".join([str(el) for el in elements]))

mdb_client = pymongo.MongoClient(MDB_URI)
mdb_db = mdb_client[DB_NAME]
mdb_collection = mdb_db[COLLECTION_NAME]
for element in elements:
    mdb_collection.insert_one({
        "text":str(element.text),
        "embeddings":generate_embeddings(str(element.text)),
        "metadata": {
            "raw_element":element.to_dict(),
        },
        "source":"document.pdf"
    })