joannapurosto

echo "Creating a Valohai execution…"
response=$(curl -H 'Content-Type: application/json' \
         -H "Authorization: Token $VALOHAI_API_TOKEN" \
         -X POST \
         -d "$(execution_data)" \
         'https://app.valohai.com/api/v0/executions/')

execution_id=$(echo "$response" | jq '.id')
echo "Created Valohai execution $execution_id"

joannapurosto

function execution_data {
  cat << EOF
  {
    "project": "$project_id",
    "commit": "$git_sha",
    "step": "Preprocess dataset (MNIST)"
  }
EOF
}

joannapurosto

echo "Fetching the repository in Valohai..."

response=$(curl -L -H 'Content-Type: application/json' \
         -H "Authorization: Token $VALOHAI_API_TOKEN" \
         -X POST \
         -d '{"name": "fetch"}' \
         "https://app.valohai.com/api/v0/projects/$project_id/fetch")
echo "$response"

joannapurosto

#!/bin/bash
set -e

project_id="$VALOHAI_PROJECT_ID"
git_sha="$GIT_SHA"

joannapurosto

curl -H "Authorization: Token $VALOHAI_API_TOKEN" 'https://app.valohai.com/api/v0/projects/' | jq
{
  "count": 2,
  "next": null,
  "previous": null,
  "results": [
...
    {
      "id": "016c425b-94ba-d4f9-1ec9-9c8ec8f8470d",
      "name": "tensorflow-azure-pipelines",

joannapurosto

pool:
  vmImage: 'ubuntu-latest'

steps:

- bash: scripts/orchestrate_valohai.sh
  displayName: 'Orchestrate a Valohai workflow'
  env:
    GIT_SHA: $(Build.SourceVersion)
    VALOHAI_PROJECT_ID: $(VALOHAI_PROJECT_ID)

joannapurosto

- step:
    name: Train model (MNIST)
    image: tensorflow/tensorflow:1.13.1-gpu-py3
    command: python train.py {parameters}
    parameters:
      - name: max_steps
        pass-as: --max_steps={v}
        description: Number of steps to run the trainer
        type: integer
        default: 300

joannapurosto

- step:
    name: Preprocess dataset (MNIST)
    image: tensorflow/tensorflow:1.13.1-gpu-py3
    command: python preprocess.py
    inputs:
      - name: training-set-images
        default: https://valohaidemo.blob.core.windows.net/mnist/train-images-idx3-ubyte.gz
      - name: training-set-labels
        default: https://valohaidemo.blob.core.windows.net/mnist/train-labels-idx1-ubyte.gz
      - name: test-set-images

joannapurosto

import tensorflow as tf
import numpy as np
import shutil
import os
import time

def create_valohai_log_snapshot(file_writer):
    from_path = file_writer.get_logdir()
    to_path = os.getenv('VH_OUTPUTS_DIR')
    if not os.path.exists(to_path):

joannapurosto

import tensorflow as tf
import time

tf.reset_default_graph()
myvar = tf.get_variable('myvar', shape=[])
myvar_summary = tf.summary.scalar(name='Myvar', tensor=myvar)
init = tf.global_variables_initializer()

with tf.Session() as sess:
    writer = tf.summary.FileWriter('./logs/run1', sess.graph)