create_random_numpy_files.py

import os 
import numpy as np

NO_OF_FILES = 1000
NO_OF_FEATURES = 1000
MAX_ROW_LIMIT_IN_SINGLE_FILE = 370

train_dir = os.path.join(os.getcwd(), 'train')
os.makedirs(train_dir, exist_ok=True)

validation_dir = os.path.join(os.getcwd(), 'validation')
os.makedirs(validation_dir, exist_ok=True)

for i in range(NO_OF_FILES): 
    rows = np.random.randint(100, MAX_ROW_LIMIT_IN_SINGLE_FILE, size=1)[0]
    feature = np.random.rand(rows, NO_OF_FEATURES)
    
    np.save(os.path.join(train_dir, 'train_{}'.format(i+1)), feature)
    
for i in range(NO_OF_FILES):
    rows = np.random.randint(100, MAX_ROW_LIMIT_IN_SINGLE_FILE, size=1)[0]
    feature = np.random.rand(rows, NO_OF_FEATURES)
    
    np.save(os.path.join(validation_dir, 'validation_{}'.format(i+1)), feature)

# After executing this code we obtain some numpy files
# directories named train and validation.