work with many files sending to server

ipartition.py

import six

if six.PY2:
    from collections import Iterator
elif six.PY3:
    from collections.abc import Iterator

# https://github.com/turicas/rows/tree/05cea12a42cc46ebc6ac3d33614a3e77c497e6d4
# https://github.com/turicas/rows/blob/05cea12a42cc46ebc6ac3d33614a3e77c497e6d4/rows/plugins/utils.py#L35-L51

def ipartition(iterable, partition_size):
    if not isinstance(iterable, Iterator):
        iterator = iter(iterable)
    else:
        iterator = iterable

    finished = False
    while not finished:
        data = []
        for _ in range(partition_size):
            try:
                data.append(next(iterator))
            except StopIteration:
                finished = True
                break
        if data:
            yield data

ipartition_exemple_use.py

BULK_SIZE = 512

for bulk in ipartition(tons_of_filenames, BULK_SIZE):
    keywords = (extract_keyword(filename) for filename in bulk)
    files = tuple(File(open(filename)) for filename in bulk)

    list_m_cv = tuple(
        Model(user=user, resume=resume, resume_text=resume_text)
        for resume, resume_text in zip(files, keywords)
    )
    Model.objects.bulk_create(list_m_cv)

    for file in files:
        file.close()