Example of using doodad

launch_ec2.py

import doodad
from doodad.wrappers.sweeper import launcher


local_mnt = doodad.mount.MountLocal(
    local_dir='data_in',
    mount_point='./data_in',
    output=False
)


sweeper = launcher.DoodadSweeper(
    docker_img='kouroshhakha/env-python-base:latest',
    s3_bucket_name='kourosh-doodad',
    docker_output_dir='/data',
    mounts=[local_mnt],
)
args = {
    'n': [i for i in range(5)]
}
sweeper.run_sweep_aws(
    target='script/print_n.py',
    log_prefix='print_n',
    add_date_to_logname=True,
    params=args,
    instance_type='c5.large',
    ami_image='ami-0dae15d5b30f07c19',
    region='us-west-1',
)

print_n.py

# main task script: print input argument, read a file, and dump a file to the output
# it's assumed that this script is located at script/print_n.py
import argparse
import yaml
from pathlib import Path

parser = argparse.ArgumentParser()
parser.add_argument('--root', type=str, default='/data', help='prefix directory for results')
parser.add_argument('--n', type=int, help='Integer to print')
args = parser.parse_args()
print(args.n)

print(f'file location: {str(Path(__file__).absolute())}')

with open('data_in/data.yaml', 'r') as f:
    print(yaml.safe_load(f))

# the output path should be absolute and the output folder name should be unique to the experiment.
out_file = Path(f'{args.root}/out_{args.n}/data.yaml')
parent = out_file.parent
parent.mkdir(exist_ok=True, parents=True)
with open(str(out_file), 'w') as f:
    yaml.dump(dict(n=args.n), f)