Rename files to hf shards format

rename_shards.py

# Given list of filenames
file_names = ['chunk_1.jsonl', 'chunk_2.jsonl']

# Function to convert file names to the required format
def convert_filenames(filenames):
    total_files = len(filenames)
    new_file_names = []
    
    for i, filename in enumerate(filenames, start=1):
        # Extract the base name without extension and the chunk number
        base_name, _ = filename.split('_')
        extension = filename.split('.')[-1]
        
        # Create the new filename using the zero-padded format
        new_name = f"{base_name}-{i:05d}-of-{total_files:05d}.{extension}"
        new_file_names.append(new_name)
    
    return new_file_names

# Convert the filenames
converted_file_names = convert_filenames(file_names)
print(converted_file_names)