qdrant_merge.py

def scroll_merge(klass, source_names, target_name, batch_size = 100):
    Qdrant.recreate(target_name) # reset target collection

    def generator(source_name):
        offset = None

        while True:
            # Scroll https://qdrant.tech/documentation/points/#scroll-points
            # https://github.com/qdrant/qdrant-client/blob/master/qdrant_client/qdrant_client.py#L412
            result = Qdrant.client.scroll(
                collection_name=source_name,
                offset=offset,
                limit=batch_size,
                with_payload=True,
                with_vectors=True,
            )
            points = result[0]
            offset = result[1]

            if points:
                yield points

            if offset is None:
                break

    for source_name in source_names:
        for points in generator(source_name):
            print(f'Upserting {len(points)} points from {source_name} to {target_name}...', flush=True)
            Qdrant.client.upsert(
                collection_name=target_name,
                points=points,
                wait=True,
            )