Example Script to run code that is in python package. Can be used to run Python Packages as Python job on databricks. Use this script as "python_file" and pass the job_module and the job_args as "parameter" like this: [ "--job-module=package.subpackage.module", "--job-method=run_job" ]

databricks_python_library_job_executor.py

import importlib
import argparse


def get_installed_packages():
    import pkg_resources
    installed_packages = pkg_resources.working_set
    return sorted(["%s==%s" % (i.key, i.version) for i in installed_packages])


parser = argparse.ArgumentParser()
parser.add_argument("--job-module", help="The fully qualified python module containing the job", required=True)
parser.add_argument("--job-method", help="The method within the job module executing the job", required=True)
args, argv = parser.parse_known_args()

print("Job Module: " + args.job_module)
print("Job Method: " + args.job_method)
print("Job Arguments: " + str(argv))
print("Installed packages: " + str(get_installed_packages()))
job_module = importlib.import_module(args.job_module)
job_method = getattr(job_module, args.job_method)
job_method(argv)