cat_and_bzip2_by_external_cmd.py

#!/usr/bin/env python

# -*- coding: utf-8 -*-
from benchmarker import Benchmarker

import os
import glob
import fnmatch
import subprocess

base_dir = '/home/mumumu/bigfile_bench'


def concat_files(target_files, dest_dir):
    concated_file_path = os.path.join(base_dir, 'concated.txt')
    target_files.insert(0, '/bin/cat')
    target_files.append('>')
    target_files.append(concated_file_path)

    cmd = ' '.join(target_files)
    return_value = subprocess.call(cmd, shell=True)
    if return_value != 0:
        raise Exception('concat files failed')
    return concated_file_path


def bzip2_file(target_file):
    cmd_and_args = '/bin/bzip2 ' + target_file
    return_value = subprocess.call(cmd_and_args, shell=True)
    if return_value != 0:
        raise Exception('bzip2 file failed')


if __name__ == '__main__':
    pattern = 'test_*.txt'
    target_dir = base_dir
    dest_file = None
    with Benchmarker(width=20) as bm:
        for delete_file in glob.glob("concated.txt*"):
            os.remove(delete_file)
        target_files = sorted([os.path.join(target_dir, f)
                               for f in os.listdir(target_dir) if fnmatch.fnmatch(f, pattern)])
        with bm('cat by /bin/cat'):
            dest_file = concat_files(target_files, base_dir)
        with bm('bz2 by /bin/bzip2'):
            bzip2_file(dest_file)