dbast · December 11, 2013 21:13
diff --git a/uniform_exposure.py b/uniform_exposure.py
 # Develop a bunch of raw pics so they look pretty much equally exposed.
 # Copyright (2013) a1ex. License: GPL.

 # Requires python, ufraw, enfuse, ImageMagick and exiftool.

 # Usage:
 # 1) Place your raw photos under a "raw" subdirectory; for example:
 #  
 #     $ ls -R
 #          .:
 #          uniform-exposure.py
 #
 #          ./raw:
 #          IMG_0001.CR2  IMG_0002.CR2 ...
 #
 # 2) run the script
 #
 #     $ python uniform-exposure.py
 #
 #          IMG_0001.CR2:
 #             midtones: brightness level  2160 => exposure +4.14 EV
 #           highlights: brightness level 58877 => exposure +3.21,+2.28,+1.36,+0.43,-0.50 EV 
 #              shadows: brightness level   289 => exposure +2.79 EV (skipping)
 #           . . . .
 #          Developing images... [5% done, ETA 0:05:02]...
 #
 # 3) the images will be placed in a directory called 'jpg'.
 #
 # 4) you will also get images optimized for highlights, for midtones and for shadows.

 from __future__ import division

 # User adjustable parameters 
 # =====================================================================================

 # Global options
 raw_dir = 'raw'                                     # place raw files here
 out_dir = 'jpg'                                     # output files will be saved here
 tmp_dir = 'tmp'                                     # temporary working folder
 output_tif16bit = False                             # enable to output 16-bit tif

 # Options that can be changed for each picture (in override_settings)
 overall_bias = 0                                    # exposure compensation, in EV
 highlight_level = 20000                             # decrease for stronger highlight recovery
 midtone_level = 20000                               # median exposure
 shadow_level = 5000                                 # increase for stronger shadow recovery
 ev_step = 1                                         # EV step size for bracketed exposures (larger values may cause halos, smaller values are very slow)
 target_median = None                                # for the final output (set to None for disabling, try around 128 for flicker-free video/timelapse)
 samyang8ff = False                                  # for Samyang 8mm on full-frame cameras: don't analyze the black borders
 fullsize = False                                    # develop full size (turn off for higher speed)
 ufraw_options = "--temperature=5500 --green=1 "     # customize ufraw parameters (tip: you can also open a file in ufraw, tweak settings in the GUI and save an ID file)
 enfuse_options = "--gray-projector=value"           # customize enfuse parameters (tip: tweak --exposure-sigma to adjust contrast and highlight recovery behavior)

 def override_settings(fname, num):
    global ufraw_options, default_ufraw_options, overall_bias, default_overall_bias, highlight_level, default_highlight_level, midtone_level, default_midtone_level, shadow_level, default_shadow_level, samyang8ff, default_samyang8ff, fullsize, default_fullsize, target_median, default_target_median, ev_step, default_ev_step, enfuse_options, default_enfuse_options
    try:    ufraw_options = default_ufraw_options; overall_bias = default_overall_bias; highlight_level = default_highlight_level; midtone_level = default_midtone_level; shadow_level = default_shadow_level; samyang8ff = default_samyang8ff; fullsize = default_fullsize; target_median = default_target_median; ev_step = default_ev_step; enfuse_options = default_enfuse_options;
    except: default_ufraw_options = ufraw_options; default_overall_bias = overall_bias; default_highlight_level = highlight_level; default_midtone_level = midtone_level; default_shadow_level = shadow_level; default_samyang8ff = samyang8ff; default_fullsize = fullsize; default_target_median = target_median; default_ev_step = ev_step; default_enfuse_options = enfuse_options;

    # override per-picture settings here
    # for example:
    #
    # if num in range(1234, 1251):
    #       overall_bias += 2
    

 # =====================================================================================

 import os, sys, re, time, datetime, subprocess, shlex, shutil
 from multiprocessing import Pool
 from math import *

 os.putenv("OMP_NUM_THREADS", "1") # optimizes the parallelisation, by allowing only one OpenMP thread per shell programm

 log2 = lambda x: log(x) / log(2)
 sign = lambda x: x / abs(x) if x != 0 else 0

 direrr = False

 def progress(x, interval=1):
    global _progress_first_time, _progress_last_time, _progress_message, _progress_interval
    
    try:
        p = float(x)
        init = False
    except:
        init = True
        
    if init:
        _progress_message = x
        _progress_last_time = time.time()
        _progress_first_time = time.time()
        _progress_interval = interval
    elif x:
        if time.time() - _progress_last_time > _progress_interval:
            print >> sys.stderr, "%s [%d%% done, ETA %s]..." % (_progress_message, int(100*p), datetime.timedelta(seconds = round((1-p)/p*(time.time()-_progress_first_time))))
            _progress_last_time = time.time()

 def run(cmd):
    try:
        p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out = p.communicate()
        if p.returncode:
            print cmd
            print out[0]
            print out[1]
            raise SystemExit
            
        f = open("dev.log", "a");
        print >> f, cmd
        print >> f, out[0]
        print >> f, out[1]
        print >> f, ""
        f.close()
        return out[0]
    except KeyboardInterrupt:
        raise SystemExit
    except SystemExit:
        raise SystemExit
    except:
        print sys.exc_info()

 def change_ext(file, newext):
    return os.path.splitext(file)[0] + newext

 def file_number(j):
    nr = re.search("([0-9][0-9][0-9][0-9]+)", j)
    if nr:
        nr = int(nr.groups()[0])
        return nr

 def get_histogram_data_work(file, cmd):

    try: 
        p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
        out = p.communicate()
        if p.returncode:
            print out[0]
            raise SystemExit
        
        lines = out[0].split("\n")

    except KeyboardInterrupt:
        raise SystemExit

    X = []
    for l in lines[1:]:
        p1 = l.find("(")
        if p1 > 0:
            p2 = l.find(",", p1)
            level = int(l[p1+1:p2])
            count = int(l[:p1-2])
            X.append((level, count))
    return X

 def get_histogram_data_for_raw(file):
    cmd1 = 'ufraw-batch --exposure=0 --gamma=1 --clip=digital --shrink=10 --grayscale=luminance --out-depth=16 --output=- --create-id=no --silent "%s"' % file
    cmd2 = "convert - -gravity Center %s -format %%c histogram:info:-" % ("-crop 67%x67%" if samyang8ff else "")
    cmd = cmd1 + " | " + cmd2
    return get_histogram_data_work(file, cmd)

 def get_histogram_data_for_jpg(file):
    cmd = 'convert "%s" -gravity Center %s -format %%c histogram:info:-' % (file, "-crop 67%x67%" if samyang8ff else "")
    return get_histogram_data_work(file, cmd)

 def get_histogram_data(file):
    ext = os.path.splitext(file)[1].lower()
    if ext in [".cr2", ".dng", ".ufraw"]:
        return get_histogram_data_for_raw(file)
    else:
        return get_histogram_data_for_jpg(file)

 def get_percentiles(file, percentiles):
    X = get_histogram_data(file)
    
    ans = []
    for percentile in percentiles:
    
        total = sum([count for level, count in X])
        target = total * percentile / 100
        
        acc = 0
        for level, count in X:
            acc += count
            if acc >= target:
                ans.append(level)
                break

    # see where these percentile levels fall on the image
    # (where exactly it meters to keep the exposure constant)
    if 0:
        level_min = min(ans)
        level_max = max(ans)
        cmd1 = 'ufraw-batch --exposure=0 --gamma=1 --clip=digital --shrink=10 --grayscale=luminance --out-depth=16 --output=- --silent "%s"' % file
        cmd_dbg = cmd1 + ' | convert - -gravity Center -level %s,%s -solarize 65534 -threshold 0 "%s" ' % (level_min-1, level_max+1, change_ext(file, "-test.jpg"))
        print cmd_dbg
        run(cmd_dbg)

    return ans

 def get_medians(file):
    return get_percentiles(file, [50, 99.99, 1])

 def frange(x, y, jump):
    if jump > 0:
        while x < y:
            yield x
            x += jump
    else:
        while x > y:
            yield x
            x += jump
    

 def expo_range(start, end, step):
    step = abs(step) * sign(end-start)
    r = list(frange(start, end+0.5*sign(step), step))
    if len(r) <= 1:
        return [end]
    step = float(end-start) / (len(r)-1)
    r = list(frange(start, end+0.5*sign(step), step))
    return r[1:]

 def parse_lev(lev):
    roll = 0
    pitch = 0
    lines = open(lev).readlines()
    for l in lines:
        m = re.match("Roll *: *([+\-0-9.]+)", l)
        if m:
            try: roll = float(m.groups()[0])
            except: pass
        m = re.match("Pitch *: *([+\-0-9.]+)", l)
        if m:
            try: pitch = float(m.groups()[0])
            except: pass
    return roll, pitch

 def gamma_correction(file, target_median):
    median = get_percentiles(file, [50])[0]
    median = median / 255
    target_median = target_median / 255
    if median >= 1:
        return
    gamma = log2(median) / log2(target_median)
    line = ("(gamma %.02f)" % gamma)
    cmd = 'mogrify -gamma %f "%s" ' % (gamma, file)
    run(cmd)
    return line

 def main(f):
    global ufraw_options, default_ufraw_options, overall_bias, default_overall_bias, highlight_level, default_highlight_level, midtone_level, default_midtone_level, shadow_level, default_shadow_level, samyang8ff, default_samyang8ff, fullsize, default_fullsize, target_median, default_target_median, ev_step, default_ev_step, enfuse_options, default_enfuse_options
    r  = os.path.join(raw_dir, f)
    j  = os.path.join(out_dir, change_ext(f, ".tif" if output_tif16bit else ".jpg"))
    jm = os.path.join(tmp_dir, change_ext(f, "-m.tif"))
    jh = os.path.join(tmp_dir, change_ext(f, "-h.tif"))
    js = os.path.join(tmp_dir, change_ext(f, "-s.tif"))
    ufr = change_ext(r, ".ufraw")
    lev = change_ext(r, ".LEV")

    # don't overwrite existing jpeg files
    if os.path.isfile(j) or os.path.isfile(change_ext(j, "r.jpg")):
        return "%s: output file %s already exists, skipping\n" % (r, j), None

    # skip sub-dirs under raw_dir
    if not os.path.isfile(r):
        return None, None
    
    if f.lower().endswith('.jpg'):
        return None, None

    rotate_options = ""
    if os.path.isfile(lev):
        roll, pitch = parse_lev(lev)
        r90 = round(roll/90.0) * 90
        roll -= r90
        ar = "2:3" if abs(r90) == 90 else "3:2"
        rotate_options = " --rotate=%s --auto-crop --aspect-ratio %s " % (-roll, ar)

    lines = ""
    lines += "%s:\n" % r

    # override settings
    override_settings(r, file_number(r))
    ufraw_options = rotate_options + ufraw_options
    lines += ufraw_options + "\n"

    # compute percentiles
    mm, mh, ms = get_medians(r)

    # normal exposure
    ecm = -log2(mm / midtone_level) + overall_bias

    # exposure for highlights
    ech = max(-log2(mh / highlight_level) + overall_bias, -0.5)
    needs_highlight_recovery = (ech < ecm - 0.5);

    # exposure for shadows
    ecs = -log2(ms / shadow_level) + overall_bias
    needs_shadow_recovery = (ecs > ecm + 0.5)

    # compensate normal exposure to avoid brightness changes when doing strong recovery (approximate)
    if needs_highlight_recovery: ecm += 0.25 * abs(ech - ecm)
    if needs_shadow_recovery: ecm -= 0.25 * abs(ecs - ecm)

    # do highlight/shadow recovery in more steps, not just one
    if needs_highlight_recovery: ech = expo_range(ecm, ech, ev_step)
    else: ech = [ech]
    if needs_shadow_recovery: ecs = expo_range(ecm, ecs, ev_step)
    else: ecs = [ecs]

    # print the levels
    lines += "    midtones: brightness level %5d => exposure %+.2f EV\n" % (mm, ecm)
    lines += "  highlights: brightness level %5d => exposure %s EV %s\n" % (mh, ",".join(["%+.2f" % e for e in ech]), "" if needs_highlight_recovery else "(skipping)")
    lines += "     shadows: brightness level %5d => exposure %s EV %s\n" % (ms, ",".join(["%+.2f" % e for e in ecs]), "" if needs_shadow_recovery else "(skipping)")
    lines += ""

    # any ufraw settings file? use it when developing
    if os.path.isfile(ufr):
        r = ufr

    # develop the raws
    shrink = 1 if fullsize == True else (2 if fullsize == False else fullsize)
    jpegs = [jm]
    lines += "(midtones)"
    cmd = 'ufraw-batch --out-type=tiff --out-depth=16 --overwrite %s --exposure=%s "%s" --output="%s" --shrink=%d' % (ufraw_options, ecm, r, jm, shrink)
    run(cmd)
    
    if needs_highlight_recovery:
        # highlight recovery
        lines += "(highlights"
        for ji,e in enumerate(ech):
            if ji > 0: lines += "\b."
            jp = change_ext(jh, "%d.tif" % ji)
            cmd = 'ufraw-batch --out-type=tiff --out-depth=16 --overwrite %s --exposure=%s "%s" --output="%s" --shrink=%d' % (ufraw_options, e, r, jp, shrink)
            run(cmd)
            jpegs.append(jp)
        lines += "\b)"

    if needs_shadow_recovery:
        # shadow recovery
        lines += "(shadows"
        for ji,e in enumerate(ecs):
            if ji > 0: lines += "\b."
            jp = change_ext(js, "%d.tif" % ji)
            cmd = 'ufraw-batch --out-type=tiff --out-depth=16 --overwrite %s --exposure=%s "%s" --output="%s" --shrink=%d' % (ufraw_options, e, r, jp, shrink)
            run(cmd)
            jpegs.append(jp)
        lines += "\b)"

    if needs_highlight_recovery or needs_shadow_recovery:
        # blend highlights and shadows
        lines += "(enfuse)"
        compression = "--compression=DEFLATE" if output_tif16bit else ""
        cmd = 'enfuse %s %s -o "%s" %s' % (enfuse_options, compression, j, " ".join(['"%s"' % ji for ji in jpegs]))
        run(cmd)
    else:
        # nothing to blend
        if output_tif16bit:
            lines += "(copy)"
            shutil.copy(jm, j)
        else:
            lines += "(convert)"
            run("convert %s %s" % (jm, j))
    
    if target_median:
        lines += gamma_correction(j, target_median)
    
    settings_cmd = "echo \"%s: overall_bias=%g; highlight_level=%g; midtone_level=%g; shadow_level=%g; ufraw_options='%s'; \" >> settings.log" % (f, overall_bias, highlight_level, midtone_level, shadow_level, ufraw_options)

    if 0:
        # lossless optimization of the Huffman tables
        cmd = 'jpegoptim "%s"' % j
        run(cmd)
    
    if 1:
        # copy over exif-data (without old preview/thumbnail-images and without orientation/icc-profile as ufraw already takes care of it), setting colorspace to sRGB and add comment with processing parameters
        comment = "overall_bias=%g; highlight_level=%g; midtone_level=%g; shadow_level=%g; ufraw_options='%s'; " % (overall_bias, highlight_level, midtone_level, shadow_level, ufraw_options)
        comment += "midtones: brightness level %5d => exposure %+.2f EV; " % (mm, ecm)
        comment += "highlights: brightness level %5d => exposure %s EV %s; " % (mh, ",".join(["%+.2f" % e for e in ech]), "" if needs_highlight_recovery else "(skipping)")
        comment += "shadows: brightness level %5d => exposure %s EV %s" % (ms, ",".join(["%+.2f" % e for e in ecs]), "" if needs_shadow_recovery else "(skipping)")
        cmd = 'exiftool -TagsFromFile "%s" -comment="%s" -ThumbnailImage= -PreviewImage= -Orientation= -icc_profile:all= -all:Colorspace=sRGB -z -overwrite_original "%s"' % (r, comment, j)
        run(cmd)
    
    if 1:
        # cleanup temporary files
        for j in jpegs:
            os.remove(j)

    lines += "\n"
    return lines, settings_cmd

 if __name__ == '__main__':
    try: os.mkdir(out_dir)
    except: print "Warning: could not create output dir '%s'" % out_dir

    try: os.mkdir(tmp_dir)
    except: print "Warning: could not create working dir '%s'" % tmp_dir

    progress("")
    files = sorted(os.listdir(raw_dir))

    # prefer the DNG if there are two files with the same name
    for f in [f for f in files]:
        dng = change_ext(f, ".DNG")
        if f[0] == ".":
            files.remove(f)
            continue
        if dng != f and dng in files:
            files.remove(f)
            continue
        
        # only process CR2 and DNG
        if not(f.endswith(".CR2") or f.endswith(".cr2") or f.endswith(".DNG") or f.endswith(".dng")):
            files.remove(f)
            continue

    p = Pool()
    it = p.imap(main, files)
    for k,f in enumerate(files):
        line, settings_cmd = it.next()
        if line: 
            print line
        if settings_cmd: 
            run(settings_cmd)
        progress((k+1) / len(files))
    p.close()
	# Develop a bunch of raw pics so they look pretty much equally exposed.
	# Copyright (2013) a1ex. License: GPL.

	# Requires python, ufraw, enfuse, ImageMagick and exiftool.

	# Usage:
	# 1) Place your raw photos under a "raw" subdirectory; for example:
	#
	# $ ls -R
	# .:
	# uniform-exposure.py
	#
	# ./raw:
	# IMG_0001.CR2 IMG_0002.CR2 ...
	#
	# 2) run the script
	#
	# $ python uniform-exposure.py
	#
	# IMG_0001.CR2:
	# midtones: brightness level 2160 => exposure +4.14 EV
	# highlights: brightness level 58877 => exposure +3.21,+2.28,+1.36,+0.43,-0.50 EV
	# shadows: brightness level 289 => exposure +2.79 EV (skipping)
	# . . . .
	# Developing images... [5% done, ETA 0:05:02]...
	#
	# 3) the images will be placed in a directory called 'jpg'.
	#
	# 4) you will also get images optimized for highlights, for midtones and for shadows.

	from __future__ import division

	# User adjustable parameters
	# =====================================================================================

	# Global options
	raw_dir = 'raw' # place raw files here
	out_dir = 'jpg' # output files will be saved here
	tmp_dir = 'tmp' # temporary working folder
	output_tif16bit = False # enable to output 16-bit tif

	# Options that can be changed for each picture (in override_settings)
	overall_bias = 0 # exposure compensation, in EV
	highlight_level = 20000 # decrease for stronger highlight recovery
	midtone_level = 20000 # median exposure
	shadow_level = 5000 # increase for stronger shadow recovery
	ev_step = 1 # EV step size for bracketed exposures (larger values may cause halos, smaller values are very slow)
	target_median = None # for the final output (set to None for disabling, try around 128 for flicker-free video/timelapse)
	samyang8ff = False # for Samyang 8mm on full-frame cameras: don't analyze the black borders
	fullsize = False # develop full size (turn off for higher speed)
	ufraw_options = "--temperature=5500 --green=1 " # customize ufraw parameters (tip: you can also open a file in ufraw, tweak settings in the GUI and save an ID file)
	enfuse_options = "--gray-projector=value" # customize enfuse parameters (tip: tweak --exposure-sigma to adjust contrast and highlight recovery behavior)

	def override_settings(fname, num):
	global ufraw_options, default_ufraw_options, overall_bias, default_overall_bias, highlight_level, default_highlight_level, midtone_level, default_midtone_level, shadow_level, default_shadow_level, samyang8ff, default_samyang8ff, fullsize, default_fullsize, target_median, default_target_median, ev_step, default_ev_step, enfuse_options, default_enfuse_options
	try: ufraw_options = default_ufraw_options; overall_bias = default_overall_bias; highlight_level = default_highlight_level; midtone_level = default_midtone_level; shadow_level = default_shadow_level; samyang8ff = default_samyang8ff; fullsize = default_fullsize; target_median = default_target_median; ev_step = default_ev_step; enfuse_options = default_enfuse_options;
	except: default_ufraw_options = ufraw_options; default_overall_bias = overall_bias; default_highlight_level = highlight_level; default_midtone_level = midtone_level; default_shadow_level = shadow_level; default_samyang8ff = samyang8ff; default_fullsize = fullsize; default_target_median = target_median; default_ev_step = ev_step; default_enfuse_options = enfuse_options;

	# override per-picture settings here
	# for example:
	#
	# if num in range(1234, 1251):
	# overall_bias += 2


	# =====================================================================================

	import os, sys, re, time, datetime, subprocess, shlex, shutil
	from multiprocessing import Pool
	from math import *

	os.putenv("OMP_NUM_THREADS", "1") # optimizes the parallelisation, by allowing only one OpenMP thread per shell programm

	log2 = lambda x: log(x) / log(2)
	sign = lambda x: x / abs(x) if x != 0 else 0

	direrr = False

	def progress(x, interval=1):
	global _progress_first_time, _progress_last_time, _progress_message, _progress_interval

	try:
	p = float(x)
	init = False
	except:
	init = True

	if init:
	_progress_message = x
	_progress_last_time = time.time()
	_progress_first_time = time.time()
	_progress_interval = interval
	elif x:
	if time.time() - _progress_last_time > _progress_interval:
	print >> sys.stderr, "%s [%d%% done, ETA %s]..." % (_progress_message, int(100p), datetime.timedelta(seconds = round((1-p)/p(time.time()-_progress_first_time))))
	_progress_last_time = time.time()

	def run(cmd):
	try:
	p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
	out = p.communicate()
	if p.returncode:
	print cmd
	print out[0]
	print out[1]
	raise SystemExit

	f = open("dev.log", "a");
	print >> f, cmd
	print >> f, out[0]
	print >> f, out[1]
	print >> f, ""
	f.close()
	return out[0]
	except KeyboardInterrupt:
	raise SystemExit
	except SystemExit:
	raise SystemExit
	except:
	print sys.exc_info()

	def change_ext(file, newext):
	return os.path.splitext(file)[0] + newext

	def file_number(j):
	nr = re.search("([0-9][0-9][0-9][0-9]+)", j)
	if nr:
	nr = int(nr.groups()[0])
	return nr

	def get_histogram_data_work(file, cmd):

	try:
	p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
	out = p.communicate()
	if p.returncode:
	print out[0]
	raise SystemExit

	lines = out[0].split("\n")

	except KeyboardInterrupt:
	raise SystemExit

	X = []
	for l in lines[1:]:
	p1 = l.find("(")
	if p1 > 0:
	p2 = l.find(",", p1)
	level = int(l[p1+1:p2])
	count = int(l[:p1-2])
	X.append((level, count))
	return X

	def get_histogram_data_for_raw(file):
	cmd1 = 'ufraw-batch --exposure=0 --gamma=1 --clip=digital --shrink=10 --grayscale=luminance --out-depth=16 --output=- --create-id=no --silent "%s"' % file
	cmd2 = "convert - -gravity Center %s -format %%c histogram:info:-" % ("-crop 67%x67%" if samyang8ff else "")
	cmd = cmd1 + " \| " + cmd2
	return get_histogram_data_work(file, cmd)

	def get_histogram_data_for_jpg(file):
	cmd = 'convert "%s" -gravity Center %s -format %%c histogram:info:-' % (file, "-crop 67%x67%" if samyang8ff else "")
	return get_histogram_data_work(file, cmd)

	def get_histogram_data(file):
	ext = os.path.splitext(file)[1].lower()
	if ext in [".cr2", ".dng", ".ufraw"]:
	return get_histogram_data_for_raw(file)
	else:
	return get_histogram_data_for_jpg(file)

	def get_percentiles(file, percentiles):
	X = get_histogram_data(file)

	ans = []
	for percentile in percentiles:

	total = sum([count for level, count in X])
	target = total * percentile / 100

	acc = 0
	for level, count in X:
	acc += count
	if acc >= target:
	ans.append(level)
	break

	# see where these percentile levels fall on the image
	# (where exactly it meters to keep the exposure constant)
	if 0:
	level_min = min(ans)
	level_max = max(ans)
	cmd1 = 'ufraw-batch --exposure=0 --gamma=1 --clip=digital --shrink=10 --grayscale=luminance --out-depth=16 --output=- --silent "%s"' % file
	cmd_dbg = cmd1 + ' \| convert - -gravity Center -level %s,%s -solarize 65534 -threshold 0 "%s" ' % (level_min-1, level_max+1, change_ext(file, "-test.jpg"))
	print cmd_dbg
	run(cmd_dbg)

	return ans

	def get_medians(file):
	return get_percentiles(file, [50, 99.99, 1])

	def frange(x, y, jump):
	if jump > 0:
	while x < y:
	yield x
	x += jump
	else:
	while x > y:
	yield x
	x += jump


	def expo_range(start, end, step):
	step = abs(step) * sign(end-start)
	r = list(frange(start, end+0.5*sign(step), step))
	if len(r) <= 1:
	return [end]
	step = float(end-start) / (len(r)-1)
	r = list(frange(start, end+0.5*sign(step), step))
	return r[1:]

	def parse_lev(lev):
	roll = 0
	pitch = 0
	lines = open(lev).readlines()
	for l in lines:
	m = re.match("Roll : ([+\-0-9.]+)", l)
	if m:
	try: roll = float(m.groups()[0])
	except: pass
	m = re.match("Pitch : ([+\-0-9.]+)", l)
	if m:
	try: pitch = float(m.groups()[0])
	except: pass
	return roll, pitch

	def gamma_correction(file, target_median):
	median = get_percentiles(file, [50])[0]
	median = median / 255
	target_median = target_median / 255
	if median >= 1:
	return
	gamma = log2(median) / log2(target_median)
	line = ("(gamma %.02f)" % gamma)
	cmd = 'mogrify -gamma %f "%s" ' % (gamma, file)
	run(cmd)
	return line

	def main(f):
	global ufraw_options, default_ufraw_options, overall_bias, default_overall_bias, highlight_level, default_highlight_level, midtone_level, default_midtone_level, shadow_level, default_shadow_level, samyang8ff, default_samyang8ff, fullsize, default_fullsize, target_median, default_target_median, ev_step, default_ev_step, enfuse_options, default_enfuse_options
	r = os.path.join(raw_dir, f)
	j = os.path.join(out_dir, change_ext(f, ".tif" if output_tif16bit else ".jpg"))
	jm = os.path.join(tmp_dir, change_ext(f, "-m.tif"))
	jh = os.path.join(tmp_dir, change_ext(f, "-h.tif"))
	js = os.path.join(tmp_dir, change_ext(f, "-s.tif"))
	ufr = change_ext(r, ".ufraw")
	lev = change_ext(r, ".LEV")

	# don't overwrite existing jpeg files
	if os.path.isfile(j) or os.path.isfile(change_ext(j, "r.jpg")):
	return "%s: output file %s already exists, skipping\n" % (r, j), None

	# skip sub-dirs under raw_dir
	if not os.path.isfile(r):
	return None, None

	if f.lower().endswith('.jpg'):
	return None, None

	rotate_options = ""
	if os.path.isfile(lev):
	roll, pitch = parse_lev(lev)
	r90 = round(roll/90.0) * 90
	roll -= r90
	ar = "2:3" if abs(r90) == 90 else "3:2"
	rotate_options = " --rotate=%s --auto-crop --aspect-ratio %s " % (-roll, ar)

	lines = ""
	lines += "%s:\n" % r

	# override settings
	override_settings(r, file_number(r))
	ufraw_options = rotate_options + ufraw_options
	lines += ufraw_options + "\n"

	# compute percentiles
	mm, mh, ms = get_medians(r)

	# normal exposure
	ecm = -log2(mm / midtone_level) + overall_bias

	# exposure for highlights
	ech = max(-log2(mh / highlight_level) + overall_bias, -0.5)
	needs_highlight_recovery = (ech < ecm - 0.5);

	# exposure for shadows
	ecs = -log2(ms / shadow_level) + overall_bias
	needs_shadow_recovery = (ecs > ecm + 0.5)

	# compensate normal exposure to avoid brightness changes when doing strong recovery (approximate)
	if needs_highlight_recovery: ecm += 0.25 * abs(ech - ecm)
	if needs_shadow_recovery: ecm -= 0.25 * abs(ecs - ecm)

	# do highlight/shadow recovery in more steps, not just one
	if needs_highlight_recovery: ech = expo_range(ecm, ech, ev_step)
	else: ech = [ech]
	if needs_shadow_recovery: ecs = expo_range(ecm, ecs, ev_step)
	else: ecs = [ecs]

	# print the levels
	lines += " midtones: brightness level %5d => exposure %+.2f EV\n" % (mm, ecm)
	lines += " highlights: brightness level %5d => exposure %s EV %s\n" % (mh, ",".join(["%+.2f" % e for e in ech]), "" if needs_highlight_recovery else "(skipping)")
	lines += " shadows: brightness level %5d => exposure %s EV %s\n" % (ms, ",".join(["%+.2f" % e for e in ecs]), "" if needs_shadow_recovery else "(skipping)")
	lines += ""

	# any ufraw settings file? use it when developing
	if os.path.isfile(ufr):
	r = ufr

	# develop the raws
	shrink = 1 if fullsize == True else (2 if fullsize == False else fullsize)
	jpegs = [jm]
	lines += "(midtones)"
	cmd = 'ufraw-batch --out-type=tiff --out-depth=16 --overwrite %s --exposure=%s "%s" --output="%s" --shrink=%d' % (ufraw_options, ecm, r, jm, shrink)
	run(cmd)

	if needs_highlight_recovery:
	# highlight recovery
	lines += "(highlights"
	for ji,e in enumerate(ech):
	if ji > 0: lines += "\b."
	jp = change_ext(jh, "%d.tif" % ji)
	cmd = 'ufraw-batch --out-type=tiff --out-depth=16 --overwrite %s --exposure=%s "%s" --output="%s" --shrink=%d' % (ufraw_options, e, r, jp, shrink)
	run(cmd)
	jpegs.append(jp)
	lines += "\b)"

	if needs_shadow_recovery:
	# shadow recovery
	lines += "(shadows"
	for ji,e in enumerate(ecs):
	if ji > 0: lines += "\b."
	jp = change_ext(js, "%d.tif" % ji)
	cmd = 'ufraw-batch --out-type=tiff --out-depth=16 --overwrite %s --exposure=%s "%s" --output="%s" --shrink=%d' % (ufraw_options, e, r, jp, shrink)
	run(cmd)
	jpegs.append(jp)
	lines += "\b)"

	if needs_highlight_recovery or needs_shadow_recovery:
	# blend highlights and shadows
	lines += "(enfuse)"
	compression = "--compression=DEFLATE" if output_tif16bit else ""
	cmd = 'enfuse %s %s -o "%s" %s' % (enfuse_options, compression, j, " ".join(['"%s"' % ji for ji in jpegs]))
	run(cmd)
	else:
	# nothing to blend
	if output_tif16bit:
	lines += "(copy)"
	shutil.copy(jm, j)
	else:
	lines += "(convert)"
	run("convert %s %s" % (jm, j))

	if target_median:
	lines += gamma_correction(j, target_median)

	settings_cmd = "echo \"%s: overall_bias=%g; highlight_level=%g; midtone_level=%g; shadow_level=%g; ufraw_options='%s'; \" >> settings.log" % (f, overall_bias, highlight_level, midtone_level, shadow_level, ufraw_options)

	if 0:
	# lossless optimization of the Huffman tables
	cmd = 'jpegoptim "%s"' % j
	run(cmd)

	if 1:
	# copy over exif-data (without old preview/thumbnail-images and without orientation/icc-profile as ufraw already takes care of it), setting colorspace to sRGB and add comment with processing parameters
	comment = "overall_bias=%g; highlight_level=%g; midtone_level=%g; shadow_level=%g; ufraw_options='%s'; " % (overall_bias, highlight_level, midtone_level, shadow_level, ufraw_options)
	comment += "midtones: brightness level %5d => exposure %+.2f EV; " % (mm, ecm)
	comment += "highlights: brightness level %5d => exposure %s EV %s; " % (mh, ",".join(["%+.2f" % e for e in ech]), "" if needs_highlight_recovery else "(skipping)")
	comment += "shadows: brightness level %5d => exposure %s EV %s" % (ms, ",".join(["%+.2f" % e for e in ecs]), "" if needs_shadow_recovery else "(skipping)")
	cmd = 'exiftool -TagsFromFile "%s" -comment="%s" -ThumbnailImage= -PreviewImage= -Orientation= -icc_profile:all= -all:Colorspace=sRGB -z -overwrite_original "%s"' % (r, comment, j)
	run(cmd)

	if 1:
	# cleanup temporary files
	for j in jpegs:
	os.remove(j)

	lines += "\n"
	return lines, settings_cmd

	if __name__ == '__main__':
	try: os.mkdir(out_dir)
	except: print "Warning: could not create output dir '%s'" % out_dir

	try: os.mkdir(tmp_dir)
	except: print "Warning: could not create working dir '%s'" % tmp_dir

	progress("")
	files = sorted(os.listdir(raw_dir))

	# prefer the DNG if there are two files with the same name
	for f in [f for f in files]:
	dng = change_ext(f, ".DNG")
	if f[0] == ".":
	files.remove(f)
	continue
	if dng != f and dng in files:
	files.remove(f)
	continue

	# only process CR2 and DNG
	if not(f.endswith(".CR2") or f.endswith(".cr2") or f.endswith(".DNG") or f.endswith(".dng")):
	files.remove(f)
	continue

	p = Pool()
	it = p.imap(main, files)
	for k,f in enumerate(files):
	line, settings_cmd = it.next()
	if line:
	print line
	if settings_cmd:
	run(settings_cmd)
	progress((k+1) / len(files))
	p.close()