add_data.py

add_data.py

values = embedd_model.encode([b['content'] for b in batch])
sparse_values = sparsed_model.encode([b['content'] for b in batch])

# Create unique IDs
ids = [str(b['metadata']['id']) for b in batch]

# Add all to upsert list
to_upsert = [{'id': i, 'values': v, 'metadata':m , 'sparse_values': sv} for (i,v,m,sv) in zip(ids,values, metas, sparse_values)]

# Upsert/insert these records to pinecone
index.upsert(vectors=to_upsert, namespace="Eiffage")