Parallel example with py> operator

rand.dig

+t1_1:
  py>: rand.rand_params
  docker:
    image: 'digdag/digdag-python:3.6.8-stretch'
  n_iter: 5

+t1_2:
  echo>: ${param_list}

+t1_3:
  for_each>:
    param: ${param_list}

  _parallel: true

  _do:
    +echo:
      echo>: ${param.bootstrap}_${param.criterion}_${param.max_depth}_${param.max_features}_${param.min_samples_split}

+t2_1:
  py>: rand.rand_params2
  docker:
    image: 'digdag/digdag-python:3.6.8-stretch'
  n_iter: 20
  eta0: [5.0, 1.0, 0.5, 0.1, 0.05, 0.01, 0.001]
  reg: ['no', 'rda', 'l1', 'l2', 'elasticnet']


+t2_2:
  echo>: ${param_list}

+t2_3:
  for_each>:
    param: ${param_list}

  _parallel: true

  _do:
    +echo:
      echo>: ${param.eta0}_${param.reg}

+t3_1:
  py>: rand.RandSomething.rand_sklearn
  docker:
    image: 'digdag/digdag-python:3.6.8-stretch'
  n_iter: 20
  eta0: [5.0, 1.0, 0.5, 0.1, 0.05, 0.01, 0.001]
  reg: ['no', 'rda', 'l1', 'l2', 'elasticnet']

rand.py

import os, sys

def rand_params(n_iter):
    sys.path.insert('/home/td-user/.local/lib/python3.6/site-packages')
    os.system('pip install --user scikit-learn scipy')
    from sklearn.model_selection import ParameterSampler
    from scipy.stats import randint as sp_randint

    param_dist = {"max_depth": [3, None],
              "max_features": sp_randint(1, 11),
              "min_samples_split": sp_randint(2, 11),
              "bootstrap": [True, False],
              "criterion": ["gini", "entropy"]}

    param_list = list(ParameterSampler(param_dist, n_iter=n_iter))
    try:
        import digdag
        digdag.env.store({"param_list": param_list})

    except ImportError:
        pass

    return True

def rand_params2(n_iter, eta0, reg):
    sys.path.append('/home/td-user/.local/lib/python3.6/site-packages')
    os.system('pip install --user scikit-learn')

    from sklearn.model_selection import ParameterSampler

    param_dist = {"eta0": eta0, "reg": reg}

    param_list = list(ParameterSampler(param_dist, n_iter=n_iter))
    try:
        import digdag
        digdag.env.store({"param_list": param_list})

    except ImportError:
        pass

    return True

class RandSomething():
    def rand_sklearn(self, n_iter, eta0, reg):
        sys.path.append('/home/td-user/.local/lib/python3.6/site-packages')
        os.system('pip install --user scikit-learn')

        from sklearn.model_selection import ParameterSampler

        param_dist = {"eta0": eta0, "reg": reg}

        param_list = list(ParameterSampler(param_dist, n_iter=n_iter))
        try:
            import digdag
            digdag.env.store({"param_list": param_list})

        except ImportError:
            pass

        return True