hook.py

#!/usr/bin/python3

import sys
import os
import re

args = sys.argv
layout = {
    "stereo": "2",
    "5.1": "6"
}

i = 0
n = len(args)
while i < n:
    if args[i] == "-filter_complex":
        if ":ochl=" in args[i+1]:
            fc = re.search("^\[([^\]]+)\].*:ochl='([^']+)'.*:osr=([0-9]+)", args[i+1])

            if fc.group(2) in layout:
                args.pop(i)
                args.pop(i)

                for ii in range(i, len(args)):
                    if args[ii] == "-map":
                        args[ii+1] = fc.group(1)
                        break

                for ii in range(i, len(args)):
                    if "-codec:" in args[ii]:
                        stream = re.search("-codec:(.+)", args[ii])

                        args.insert(ii+2, "-ac:{}".format(stream.group(1)))
                        args.insert(ii+3, layout[fc.group(2)])
                        args.insert(ii+4, "-ar:{}".format(stream.group(1)))
                        args.insert(ii+5, fc.group(3))
                        args.insert(ii+6, "-filter:{}".format(stream.group(1)))
                        args.insert(ii+7, "acontrast=100")
                        break

                n = len(args)
                continue

        elif "-hwaccel:0" in args and "hwupload" in args[i+1] and ("overlay" in args[i+1] or "inlineass" in args[i+1]):
            if "-hwaccel_output_format:0" not in args:
                args.insert(1, "-hwaccel_output_format:0")
                args.insert(2, "vaapi")
                i += 2
                n = len(args)

            res = re.search("=w=([0-9]+):h=([0-9]+)", args[i+1])

            filters = f"[0:0]scale_vaapi=w={res.group(1)}:h={res.group(2)}:force_divisible_by=4,"
            if "tonemap" in args[i+1]:
                filters += "procamp_vaapi=b=32,tonemap_vaapi=m=bt709:p=bt709:t=bt709,"
            filters += "hwdownload[0];"
            if "overlay" in args[i+1]:
                sub = re.search("\[(0:[1-9][0-9]*)\]", args[i+1]).group(1)
                filters += f"[{sub}][0]scale2ref=flags=fast_bilinear[2][1];"
                filters += "[1][2]overlay=eval=init,"
            elif "inlineass" in args[i+1]:
                inlineass = re.search("inlineass[^\[]+", args[i+1]).group(0)
                filters += f"[0]{inlineass},"
            filters += "format=pix_fmts=nv12,hwupload[3]"

            args[i+1] = filters

            for ii in range(i, len(args)):
                if args[ii] == "-map":
                    args[ii+1] = "[3]"
                    break

    i += 1

os.execv("/usr/lib/plexmediaserver/PlexTranscoder", args)

override.conf

[Service]
ExecStartPost=+/var/lib/plexmediaserver/pre-hook.py

pre-hook.py

#!/usr/bin/python3

import os

if not os.path.islink("/usr/lib/plexmediaserver/Plex Transcoder"):
    os.replace("/usr/lib/plexmediaserver/Plex Transcoder", "/usr/lib/plexmediaserver/PlexTranscoder")
    os.symlink("/var/lib/plexmediaserver/hook.py", "/usr/lib/plexmediaserver/Plex Transcoder")