AutoML Tables retrieve batch prediction results.ipynb

automl-tables-retrieve-batch-prediction-results.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "AutoML Tables retrieve batch prediction results.ipynb",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPU9N3kJupBhkyKX0ufVe9G",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/ffeldhaus/79805f61b5b5202a50a08f41b380178d/automl-tables-retrieve-batch-prediction-results.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BE-te_v8DdEj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Setup variables { run: \"auto\" }\n",
        "\n",
        "#@markdown Project ID of the existing project to use\n",
        "project_id = \"myproject\" #@param {type:\"string\"}\n",
        "\n",
        "#@markdown AutoML Tables dataset location\n",
        "location = \"us-central1\" #@param [\"us-central1\", \"eu\"]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yBWyJPE3_yx4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Install packages\n",
        "\n",
        "!pip install --quiet google-cloud-automl  > /dev/null"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7k452_geBqsG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Import modules\n",
        "import os\n",
        "import json\n",
        "import base64\n",
        "import re\n",
        "from random import choice\n",
        "from string import ascii_lowercase\n",
        "\n",
        "import googleapiclient.discovery\n",
        "from google.oauth2 import service_account\n",
        "from google.cloud import automl_v1beta1 as automl\n",
        "from google.api_core import operations_v1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "replo8OeCdwZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Provide your Google Cloud user credentials \n",
        "\n",
        "#@markdown Your Google Cloud User will be used to create a service account\n",
        "\n",
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "print('Authenticated')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UjNBHGT2GHZU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Initialize IAM and Cloud Resource Manager services\n",
        "\n",
        "iam_service = googleapiclient.discovery.build(\"iam\",\"v1\",cache_discovery=False)\n",
        "crm_service = googleapiclient.discovery.build(\"cloudresourcemanager\", \"v1\",cache_discovery=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0tIkwPPQC7n2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Create Service Account if not exists and retrieve credentials\n",
        "\n",
        "sa_name = ''.join(choice(ascii_lowercase) for i in range(6))\n",
        "sa_full_name = f\"projects/{project_id}/serviceAccounts/{sa_name}@{project_id}.iam.gserviceaccount.com\"\n",
        "\n",
        "print(f\"Create service account with random name {sa_name}\")\n",
        "\n",
        "sa = iam_service.projects().serviceAccounts().create(\n",
        "  name=f\"projects/{project_id}\",\n",
        "  body={\n",
        "    'accountId': sa_name,\n",
        "  }).execute()\n",
        "print(f\"Service account {sa['name']} created\")\n",
        "\n",
        "print(\"Download service account key\")\n",
        "key = iam_service.projects().serviceAccounts().keys().create(\n",
        "    name=sa_full_name,\n",
        "    body={'privateKeyType':'TYPE_GOOGLE_CREDENTIALS_FILE'}\n",
        "  ).execute()\n",
        "\n",
        "sa_info = json.loads(base64.decodebytes(key['privateKeyData'].encode()))\n",
        "\n",
        "credentials = service_account.Credentials.from_service_account_info(\n",
        "    sa_info,\n",
        "    scopes=[\"https://www.googleapis.com/auth/cloud-platform\"]\n",
        "  )\n",
        "\n",
        "print(\"Credential for service account created\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ds7aTgO7Gc23",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Granting roles for AutoML and Logging to service account\n",
        "\n",
        "# retrieve project IAM Policy\n",
        "policy = crm_service.projects().getIamPolicy(resource=project_id).execute()\n",
        "\n",
        "# grant roles\n",
        "member = f\"serviceAccount:{sa_name}@{project_id}.iam.gserviceaccount.com\"\n",
        "\n",
        "# grant AutoML role\n",
        "role = 'roles/automl.viewer'\n",
        "binding = {\"role\": role, \"members\": [member]}\n",
        "policy[\"bindings\"].append(binding)\n",
        "# grant log viewer role to display AutoML model metrics\n",
        "role = 'roles/logging.viewer'\n",
        "binding = {\"role\": role, \"members\": [member]}\n",
        "policy[\"bindings\"].append(binding)\n",
        "\n",
        "# update project IAM policy\n",
        "response = crm_service.projects().setIamPolicy(resource=project_id, body={\"policy\": policy}).execute()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oaibdW5qG5gD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Initialize AutoML and Operations clients\n",
        "\n",
        "if location==\"eu\":\n",
        "  client_options = {'api_endpoint': 'eu-automl.googleapis.com:443'}\n",
        "  automl_client = automl.TablesClient(project=project_id, region='eu', credentials=credentials, client_options=client_options)\n",
        "else:\n",
        "  automl_client = automl.TablesClient(project=project_id, credentials=credentials)\n",
        "\n",
        "# create operations client\n",
        "operations_client = operations_v1.OperationsClient(channel=automl_client.auto_ml_client.transport.channel)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7B-_SRwFJKWC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Retrieve results of AutoML Batch Predictions\n",
        "\n",
        "model_hyperparameters = []\n",
        "\n",
        "print(f\"Retrieve all AutoML operations in project {project_id} in location {location}\")\n",
        "name = f\"projects/{project_id}/locations/{location}\"\n",
        "operations = operations_client.list_operations(name=name,filter_=\"\")\n",
        "operation_type = \"type.googleapis.com/google.cloud.automl.v1beta1.BatchPredictResult\"\n",
        "for operation in (operation for operation in operations if operation.response.type_url==operation_type):\n",
        "  operation_metadata = automl.types.OperationMetadata.FromString(operation.metadata.value)\n",
        "  print(value)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YEKpxj4_HMDL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Remove service account\n",
        "\n",
        "response = iam_service.projects().serviceAccounts().delete(name=sa_full_name).execute()\n",
        "\n",
        "print(f\"Deleted service account {sa_name}\")"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}