uniform_exposure.py multiprocessing patch

uniform_exposure.patch

--- uniform_exposure.py	2013-12-11 21:23:03.000000000 +0100
+++ uniform_exposure_par.py	2013-12-11 22:04:49.000000000 +0100
@@ -66,18 +66,16 @@
 # =====================================================================================
 
 import os, sys, re, time, datetime, subprocess, shlex, shutil
+from multiprocessing import Pool
 from math import *
+
+os.putenv("OMP_NUM_THREADS", "1") # optimizes the parallelisation, by allowing only one OpenMP thread per shell programm
+
 log2 = lambda x: log(x) / log(2)
 sign = lambda x: x / abs(x) if x != 0 else 0
 
 direrr = False
 
-try: os.mkdir(out_dir)
-except: print "Warning: could not create output dir '%s'" % out_dir
-
-try: os.mkdir(tmp_dir)
-except: print "Warning: could not create working dir '%s'" % tmp_dir
-
 def progress(x, interval=1):
     global _progress_first_time, _progress_last_time, _progress_message, _progress_interval
     
@@ -98,7 +96,6 @@
             _progress_last_time = time.time()
 
 def run(cmd):
-    f = open("dev.log", "a");
     try:
         p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         out = p.communicate()
@@ -108,10 +105,12 @@
             print out[1]
             raise SystemExit
             
+        f = open("dev.log", "a");
         print >> f, cmd
         print >> f, out[0]
         print >> f, out[1]
         print >> f, ""
+        f.close()
         return out[0]
     except KeyboardInterrupt:
         raise SystemExit
@@ -119,12 +118,11 @@
         raise SystemExit
     except:
         print sys.exc_info()
-    f.close()
 
 def change_ext(file, newext):
     return os.path.splitext(file)[0] + newext
 
-def file_number(f):
+def file_number(j):
     nr = re.search("([0-9][0-9][0-9][0-9]+)", j)
     if nr:
         nr = int(nr.groups()[0])
@@ -244,29 +242,13 @@
     if median >= 1:
         return
     gamma = log2(median) / log2(target_median)
-    print ("(gamma %.02f)" % gamma), ; sys.stdout.flush()
+    line = ("(gamma %.02f)" % gamma)
     cmd = 'mogrify -gamma %f "%s" ' % (gamma, file)
     run(cmd)
+    return line
 
-files = sorted(os.listdir(raw_dir))
-
-# prefer the DNG if there are two files with the same name
-for f in [f for f in files]:
-    dng = change_ext(f, ".DNG")
-    if f[0] == ".":
-        files.remove(f)
-        continue
-    if dng != f and dng in files:
-        files.remove(f)
-        continue
-    
-    # only process CR2 and DNG
-    if not(f.endswith(".CR2") or f.endswith(".cr2") or f.endswith(".DNG") or f.endswith(".dng")):
-        files.remove(f)
-        continue
-
-progress("")
-for k,f in enumerate(files):
+def main(f):
+    global ufraw_options, default_ufraw_options, overall_bias, default_overall_bias, highlight_level, default_highlight_level, midtone_level, default_midtone_level, shadow_level, default_shadow_level, samyang8ff, default_samyang8ff, fullsize, default_fullsize, target_median, default_target_median, ev_step, default_ev_step, enfuse_options, default_enfuse_options
     r  = os.path.join(raw_dir, f)
     j  = os.path.join(out_dir, change_ext(f, ".tif" if output_tif16bit else ".jpg"))
     jm = os.path.join(tmp_dir, change_ext(f, "-m.tif"))
@@ -277,15 +259,14 @@
 
     # don't overwrite existing jpeg files
     if os.path.isfile(j) or os.path.isfile(change_ext(j, "r.jpg")):
-        print "%s: output file %s already exists, skipping" % (r, j)
-        continue
+        return "%s: output file %s already exists, skipping\n" % (r, j), None
 
     # skip sub-dirs under raw_dir
     if not os.path.isfile(r):
-        continue
+        return None, None
     
     if f.lower().endswith('.jpg'):
-        continue
+        return None, None
 
     rotate_options = ""
     if os.path.isfile(lev):
@@ -295,13 +276,13 @@
         ar = "2:3" if abs(r90) == 90 else "3:2"
         rotate_options = " --rotate=%s --auto-crop --aspect-ratio %s " % (-roll, ar)
 
-    print ""
-    print "%s:" % r
+    lines = ""
+    lines += "%s:\n" % r
 
     # override settings
     override_settings(r, file_number(r))
     ufraw_options = rotate_options + ufraw_options
-    print ufraw_options
+    lines += ufraw_options + "\n"
 
     # compute percentiles
     mm, mh, ms = get_medians(r)
@@ -328,10 +309,10 @@
     else: ecs = [ecs]
 
     # print the levels
-    print "    midtones: brightness level %5d => exposure %+.2f EV" % (mm, ecm)
-    print "  highlights: brightness level %5d => exposure %s EV %s" % (mh, ",".join(["%+.2f" % e for e in ech]), "" if needs_highlight_recovery else "(skipping)")
-    print "     shadows: brightness level %5d => exposure %s EV %s" % (ms, ",".join(["%+.2f" % e for e in ecs]), "" if needs_shadow_recovery else "(skipping)")
-    print "", ; sys.stdout.flush()
+    lines += "    midtones: brightness level %5d => exposure %+.2f EV\n" % (mm, ecm)
+    lines += "  highlights: brightness level %5d => exposure %s EV %s\n" % (mh, ",".join(["%+.2f" % e for e in ech]), "" if needs_highlight_recovery else "(skipping)")
+    lines += "     shadows: brightness level %5d => exposure %s EV %s\n" % (ms, ",".join(["%+.2f" % e for e in ecs]), "" if needs_shadow_recovery else "(skipping)")
+    lines += ""
 
     # any ufraw settings file? use it when developing
     if os.path.isfile(ufr):
@@ -340,52 +321,51 @@
     # develop the raws
     shrink = 1 if fullsize == True else (2 if fullsize == False else fullsize)
     jpegs = [jm]
-    print "(midtones)", ; sys.stdout.flush()
+    lines += "(midtones)"
     cmd = 'ufraw-batch --out-type=tiff --out-depth=16 --overwrite %s --exposure=%s "%s" --output="%s" --shrink=%d' % (ufraw_options, ecm, r, jm, shrink)
     run(cmd)
     
     if needs_highlight_recovery:
         # highlight recovery
-        print "(highlights", ; sys.stdout.flush()
+        lines += "(highlights"
         for ji,e in enumerate(ech):
-            if ji > 0: print "\b.", ; sys.stdout.flush()
+            if ji > 0: lines += "\b."
             jp = change_ext(jh, "%d.tif" % ji)
             cmd = 'ufraw-batch --out-type=tiff --out-depth=16 --overwrite %s --exposure=%s "%s" --output="%s" --shrink=%d' % (ufraw_options, e, r, jp, shrink)
             run(cmd)
             jpegs.append(jp)
-        print "\b)", ; sys.stdout.flush()
+        lines += "\b)"
 
     if needs_shadow_recovery:
         # shadow recovery
-        print "(shadows", ; sys.stdout.flush()
+        lines += "(shadows"
         for ji,e in enumerate(ecs):
-            if ji > 0: print "\b.", ; sys.stdout.flush()
+            if ji > 0: lines += "\b."
             jp = change_ext(js, "%d.tif" % ji)
             cmd = 'ufraw-batch --out-type=tiff --out-depth=16 --overwrite %s --exposure=%s "%s" --output="%s" --shrink=%d' % (ufraw_options, e, r, jp, shrink)
             run(cmd)
             jpegs.append(jp)
-        print "\b)", ; sys.stdout.flush()
+        lines += "\b)"
 
     if needs_highlight_recovery or needs_shadow_recovery:
         # blend highlights and shadows
-        print "(enfuse)", ; sys.stdout.flush()
+        lines += "(enfuse)"
         compression = "--compression=DEFLATE" if output_tif16bit else ""
         cmd = 'enfuse %s %s -o "%s" %s' % (enfuse_options, compression, j, " ".join(['"%s"' % ji for ji in jpegs]))
         run(cmd)
     else:
         # nothing to blend
         if output_tif16bit:
-            print "(copy)", ; sys.stdout.flush()
+            lines += "(copy)"
             shutil.copy(jm, j)
         else:
-            print "(convert)", ; sys.stdout.flush()
+            lines += "(convert)"
             run("convert %s %s" % (jm, j))
     
     if target_median:
-        gamma_correction(j, target_median)
+        lines += gamma_correction(j, target_median)
     
-    cmd = "echo \"%s: overall_bias=%g; highlight_level=%g; midtone_level=%g; shadow_level=%g; ufraw_options='%s'; \" >> settings.log" % (f, overall_bias, highlight_level, midtone_level, shadow_level, ufraw_options)
-    run(cmd)
+    settings_cmd = "echo \"%s: overall_bias=%g; highlight_level=%g; midtone_level=%g; shadow_level=%g; ufraw_options='%s'; \" >> settings.log" % (f, overall_bias, highlight_level, midtone_level, shadow_level, ufraw_options)
 
     if 0:
         # lossless optimization of the Huffman tables
@@ -406,7 +386,41 @@
         for j in jpegs:
             os.remove(j)
 
-    print ""
-    progress((k+1) / len(files))
-
+    lines += "\n"
+    return lines, settings_cmd
 
+if __name__ == '__main__':
+    try: os.mkdir(out_dir)
+    except: print "Warning: could not create output dir '%s'" % out_dir
+
+    try: os.mkdir(tmp_dir)
+    except: print "Warning: could not create working dir '%s'" % tmp_dir
+
+    progress("")
+    files = sorted(os.listdir(raw_dir))
+
+    # prefer the DNG if there are two files with the same name
+    for f in [f for f in files]:
+        dng = change_ext(f, ".DNG")
+        if f[0] == ".":
+            files.remove(f)
+            continue
+        if dng != f and dng in files:
+            files.remove(f)
+            continue
+        
+        # only process CR2 and DNG
+        if not(f.endswith(".CR2") or f.endswith(".cr2") or f.endswith(".DNG") or f.endswith(".dng")):
+            files.remove(f)
+            continue
+
+    p = Pool()
+    it = p.imap(main, files)
+    for k,f in enumerate(files):
+        line, settings_cmd = it.next()
+        if line: 
+            print line
+        if settings_cmd: 
+            run(settings_cmd)
+        progress((k+1) / len(files))
+    p.close()