partitioning to translate.py

import math
import os

buckets_dir = './all_texts_buckets'

if not os.path.isdir(buckets_dir):
    
    os.mkdir(buckets_dir)

n_characters = 30000

accumulated = 0

positions = []

for i, (idx, text_len) in enumerate(all_texts['ltext'].str.len().iteritems()):
    
    accumulated += text_len
    
    if accumulated > n_characters:
        
        positions.append(i - 1)
        accumulated = text_len
        
for i, (i_begin, i_end) in enumerate(zip([0] + positions, positions + [None])):
    
    bucket = all_texts.iloc[i_begin:i_end, :]
    
    bucket['ltext'].to_csv(os.path.join(buckets_dir, f'all_texts_bucket_{i}.txt'), index=False)