example_runspec.json

example_runspec.json

{
  "jobName": "deeprecs-train-job",
  "artifact": "artifact-deeprecs-train",
  "artifactTag": "latest",
  "mlApplication": "com.ebay.oss.krylov.workflow.app.GenericMLApplication",
  "executionParameters": {
    "hadoop": {
      "hadoopCluster": "ares",
      "batchUser": "b_merch"
    },
    "image": "ecr.vip.ebayc3.com/deeprecs/deeprecs-train-krylov:latest",
    "gpu": "1"
  },
  "applicationParameters": {
    "tasks": {
      "run_hadoop_train": {
        "program": "com.ebay.oss.krylov.workflow.ShellScriptProgram",
        "parameters": {
          "file": "deeprecs-train-shell/scripts/run_hadoop_jobs.sh",
          "args": [
            "0",
            "155290",
            "3979,77,32750,17076,3981,17082,155290,17085,3953,3955,32767",
            "2015-01-01",
            "2017-12-31",
            "20150101_20171231",
            "63"
          ]
        }
      },
      "run_train": {
        "program": "com.ebay.oss.krylov.workflow.PythonProgram",
        "parameters": {
          "file": "deeprecs-train-python/deeprecs-train/scripts_train/run_train.py",
          "args": [
            "--hadoop_user",
            "b_merch",
            "--hadoop_infix",
            "63",
            "--train_data_slice",
            "20150101_20171231",
            "--model_name",
            "63",
            "--gpu_no",
            "0",
            "--train_script",
            "binary_deeprecs/FFDeepNet1.clipped.py",
            "--site",
            "0",
            "--batch-size",
            "1000"
          ]
        }
      },
      "run_heuristic": {
        "program": "com.ebay.oss.krylov.workflow.ShellScriptProgram",
        "parameters": {
          "file": "deeprecs-train-shell/scripts/run_heuristic.sh",
          "args": [
            "63",
            "20150101_20171231",
            "0"
          ]
        }
      }
    },
    "flows": {
      "training_flow": {
        "run_hadoop_train": ["run_train"],
        "run_train": ["run_heuristic"]
      },
      "test_flow": {
        "run_heuristic": []
      },
      "hadoop_flow": {
        "run_hadoop_train": []
      }
    },
    "flow": "training_flow"
  }
}