juliensimon

service: sagemakerscheduler

provider:
  name: aws
  runtime: python2.7
  memorySize: 128
  iamRoleStatements:
    - Effect: "Allow"
      Action:
        - sagemaker:DescribeTrainingJob

juliensimon

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "VisualEditor0",
            "Effect": "Allow",
            "Action": [
                "logs:CreateLogStream",
                "logs:PutLogEvents",
                "logs:CreateLogGroup"

juliensimon

$ chalice deploy
$ export URL=`chalice url`
$ http post $URL/train/DEMO-imageclassification-2018-05-10-09-58-12 \
       TrainingJobName=imageclassification-2018-05-12 \
       InstanceType=ml.p3.2xlarge InstanceCount=2
{
    "ResponseMetadata": {
        "HTTPHeaders": {
            "connection": "keep-alive",
    {

juliensimon

@app.route('/train/{name}', methods=['POST'])
def train_job_by_name(name):
    job = sm.describe_training_job(TrainingJobName=name)

    body = app.current_request.json_body
    if 'TrainingJobName' not in body:
        raise BadRequestError('Missing new job name')
    else:
       job['TrainingJobName'] = body['TrainingJobName']
    if 'S3OutputPath' in body:

juliensimon

@app.route('/get/{name}')
def get_job_by_name(name):
    job = sm.describe_training_job(TrainingJobName=name)
    return {'job': str(job)}

juliensimon

$ http get localhost:8000/jobs/2
{
    "jobs": [
        [
            "sagemaker-tensorflow-py2-cpu-2018-03-10-10-41-27-307",
            "Completed"
        ],
        [
            "sagemaker-mxnet-py2-gpu-2018-03-10-09-50-33-717",
            "Completed"

juliensimon

@app.route('/list/{results}')
def list_jobs(results):
    jobs = sm.list_training_jobs(MaxResults=int(results), SortBy="CreationTime", SortOrder="Descending")
    job_names = map(lambda job: [job['TrainingJobName'], job['TrainingJobStatus']],  jobs['TrainingJobSummaries'])
    return {'jobs': list(job_names)}

juliensimon

+-----+--------------------+----------+
|label|            features|prediction|
+-----+--------------------+----------+
|  1.0|(200,[25,41,99,14...|       1.0|
|  1.0|(200,[30,41,63,91...|       1.0|
|  1.0|(200,[0,13,30,48,...|       1.0|
|  1.0|(200,[22,30,36,40...|       1.0|
|  1.0|(200,[15,17,29,32...|       1.0|
|  1.0|(200,[25,31,45,48...|       0.0|
|  1.0|(200,[25,32,34,47...|       1.0|

juliensimon

+-----+--------------------+
|label|            features|
+-----+--------------------+
|  1.0|(200,[1,29,37,48,...|
|  1.0|(200,[17,19,42,48...|
|  1.0|(200,[17,19,20,25...|
|  1.0|(200,[13,20,34,36...|
|  1.0|(200,[3,25,36,73,...|
+-----+--------------------+

juliensimon

val testData_libsvm = MLUtils.convertVectorColumnsToML(testData.toDF)

val transformedData = xgboost_model.transform(testData_libsvm)

val accuracy = 1.0 * transformedData.filter($"label"=== $"prediction").count / transformedData.count()