Skip to content

Instantly share code, notes, and snippets.

@tangnotes
Created July 16, 2015 14:11
Show Gist options
  • Save tangnotes/79636c531ddbc3ec1f6e to your computer and use it in GitHub Desktop.
Save tangnotes/79636c531ddbc3ec1f6e to your computer and use it in GitHub Desktop.
Python multiprocessing template for log processing
import sys
import os
import datetime
import multiprocessing
TMP_DIR='/tmp'
RAW_DIR='/raw_data/access'
def _print(msg):
print datetime.datetime.now().isoformat(), '-', msg
def get_parts():
jobs = []
return jobs
def do_process(job):
pass
def merge():
pass
if __name__ == '__main__':
date = (datetime.datetime.now() - datetime.timedelta(1)).strftime('%Y-%m-%d')
parts = get_parts()
try:
jobs = []
i = 0
for part in parts:
p = multiprocessing.Process(target=do_process, args=(part,), name='do_process-' + str(i))
i += 1
jobs.append(p)
p.start()
for p in jobs:
p.join()
_print(p.name + ' exit ' + str(p.exitcode))
#TODO after do process
except:
exc = sys.exc_info();
_print('Exception(1):' + str(exc[0]) + str(exc[1]) + 'at line' + str(exc[2].tb_lineno))
sys.exit(1)
try:
merge()
#TODO after merge
except:
exc = sys.exc_info();
_print('Exception(2):' + str(exc[0]) + str(exc[1]) + 'at line' + str(exc[2].tb_lineno))
sys.exit(1)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment