script.py

import os
from time import sleep

import skein
from skein.model import ApplicationState

mem_sz_gb = os.environ["LMD_PO_MEM_SIZE_GB"]
cores_number = os.environ["LMD_PO_NUM_CORES"]

spec = skein.ApplicationSpec(
    name="Portfolio optimization",
    queue="default",
    master=skein.Master(
        resources=skein.Resources(memory=f"{mem_sz_gb} GiB", vcores=int(cores_number)),
        env={
            "LMD_PO_OUTPUT_HDFS_PATH": os.environ["LMD_PO_OUTPUT_HDFS_PATH"],
            "LMD_PO_INPUT_HDFS_PATH": os.environ["LMD_PO_INPUT_HDFS_PATH"],
            "LMD_PO_NUM_CORES": cores_number,
        },
        files={"environment": "/app/environment.tar.gz"},
        script="""
source environment/bin/activate
cd environment
./run.sh
""",
    ),
)

with skein.Client() as client:
    app_id = client.submit(spec)
    while True:
        report = client.application_report(app_id)
        cur_state = report.state
        print(f"Current status of {app_id} is {cur_state}")
        if cur_state in [
            ApplicationState.FINISHED,
            ApplicationState.FAILED,
            ApplicationState.KILLED,
        ]:
            break
        else:
            sleep(10)

    print(f"Application with id {app_id} has finished with state {cur_state}")
    if cur_state == "FINISHED":
        exit(0)
    else:
        exit(1)