create_boinc_data.py

from pathlib import Path
from tqdm import tqdm, trange
from math import ceil

input_dir = Path("OneChunk-CRR")
output_dir = Path("output")
per_file = 10000

input_files = input_dir.glob("*.txt")

current_data = []

for file in tqdm(list(input_files)):
    with file.open() as f:
        current_data = [*current_data, *f.readlines()]
current_data = list(set(current_data))

for file_index in trange(ceil(len(current_data) / per_file)):
    start = file_index * per_file
    end = ((file_index + 1) * per_file) - 1
    file = output_dir / f"{start}-{end}.txt"
    with file.open("w+") as f:
        f.writelines(current_data[start:end])