symlink-perf-test.py

#!/usr/bin/env python3

import argparse
import os
import shutil
import subprocess
import sys
import time

from multiprocessing import Pool, Lock

parser = argparse.ArgumentParser(description='Test symlink performance')

def PositiveInt(string):
  value = int(string)
  if value > 0:
    return value
  m = 'Should be greater that zero: {}'.format(string)
  raise argparse.ArgumentTypeError(m)

parser.add_argument(
    '--files',
    type=PositiveInt,
    help='Number of files',
    required=True
)

parser.add_argument(
    '--threads',
    type=PositiveInt,
    help='Number of threads',
    required=True
)

cmd_args = parser.parse_args()

from_dir = os.path.join(os.getcwd(), '_from')
to_dir = os.path.join(os.getcwd(), '_to')
scripts_dir = os.path.join(os.getcwd(), '_scripts')

if os.path.exists(from_dir):
  shutil.rmtree(from_dir)

if os.path.exists(to_dir):
  shutil.rmtree(to_dir)

if os.path.exists(scripts_dir):
  shutil.rmtree(scripts_dir)

os.makedirs(from_dir)
os.makedirs(to_dir)
os.makedirs(scripts_dir)

print('Number of files: {}'.format(cmd_args.files))
print('Number of threads: {}'.format(cmd_args.threads))

print('Creating files...')
for i in range(cmd_args.files):
  name = os.path.join(from_dir, 'file.{}'.format(i))
  open(name, 'w')

print('Creating scripts...')

# script_head = '#!/bin/bash -e\n\nset -x\n\n'
script_head = '#!/bin/bash -e\n\n'

files_per_script = cmd_args.files // cmd_args.threads

for i in range(cmd_args.threads):
  name = os.path.join(scripts_dir, 'script-{}.sh'.format(i))
  f = open(name, 'w')
  f.write(script_head)

  is_last = (i == cmd_args.threads - 1)

  begin_index = i * files_per_script
  end_index = (i + 1) * files_per_script
  if is_last:
    end_index = cmd_args.files

  for j in range(begin_index, end_index):
    link_from = os.path.join(from_dir, 'file.{}'.format(j))
    link_to = os.path.join(to_dir, 'file.{}'.format(j))
    f.write('ln -s {} {}\n'.format(link_from, link_to))
  f.close()

def job(script_index):
  try:
    script_path = os.path.join(scripts_dir, 'script-{}.sh'.format(script_index))
    result = subprocess.run(['/bin/bash', '-e', script_path])
    return result.returncode
  except:
    return 1

print('Run jobs')

start = time.time()

pool = Pool(processes=cmd_args.threads)
x = pool.map(job, range(cmd_args.threads))

pool.close()
pool.join()

end = time.time()
print(x)
print('Time: {}'.format(end - start))