example use case for https://github.com/Open-EO/openeo-geopyspark-driver/issues/127

issue127.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7a25c4dd-9888-4e90-9450-d53b6ff8ee2d",
   "metadata": {},
   "source": [
    "- https://discuss.eodc.eu/t/caching-openeo-results/197/10\n",
    "- https://github.com/Open-EO/openeo-geopyspark-driver/issues/127"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f47b2eed-cb68-4a5c-8ef6-512cd0d939a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openeo\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3d5880a9-64ae-4bab-8872-e7e645becb14",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Authenticated using refresh token.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Connection to 'https://openeo-dev.vito.be/openeo/1.0/' with BearerAuth>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "con = openeo.connect(\n",
    "    \"openeo-dev.vito.be\"\n",
    ")\n",
    "con.authenticate_oidc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4f32d7c3-c43c-4d5c-a42b-536b5c29ac67",
   "metadata": {},
   "outputs": [],
   "source": [
    "download_dir = Path(\"issue127-data\")\n",
    "download_dir.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4d198d1-808b-4bb9-9622-1c9ccf7e27f3",
   "metadata": {},
   "source": [
    "## Initial batch job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a18abcdf-7a89-4481-8bc5-e64f1224a0b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "cube1 = con.load_collection(\n",
    "    \"TERRASCOPE_S2_TOC_V2\",\n",
    "    temporal_extent=[\"2021-09-01\", \"2021-09-30\"],\n",
    "    spatial_extent={\"west\": 4, \"south\": 51, \"east\": 4.1, \"north\": 51.1}\n",
    ")\n",
    "cube1 = cube1.save_result(\"GTiff\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eee91f8e-5979-4f8a-a334-2ce29783341a",
   "metadata": {},
   "source": [
    "Reuse an existing job or create a new one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22b055b8-e124-4b85-97fd-61ce4e8cc4eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "job1 = con.job(\"337fd792-bb5f-42d9-b055-26d669125fe3\")\n",
    "\n",
    "# job1 = cube1.send_job()\n",
    "# job1.start_and_wait()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ec1c2159-6ab0-44f2-8eeb-819e96c097eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "issue127-data/1-openEO_2021-09-02Z.tif\n",
      "issue127-data/1-openEO_2021-09-07Z.tif\n",
      "issue127-data/1-openEO_2021-09-12Z.tif\n",
      "issue127-data/1-openEO_2021-09-17Z.tif\n",
      "issue127-data/1-openEO_2021-09-27Z.tif\n"
     ]
    }
   ],
   "source": [
    "for asset in job1.get_results().get_assets():\n",
    "    print(asset.download(download_dir / f\"1-{asset.name}\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4aa61744-7241-45e4-a8f2-43f449c54966",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'337fd792-bb5f-42d9-b055-26d669125fe3'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_job_id = job1.job_id\n",
    "first_job_id"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04c27b8f-e729-4d7a-ae72-e019f37c5614",
   "metadata": {},
   "source": [
    "## Load job results with `load_result` and download synchronously as-is"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "25861594-1374-4786-866a-9cfb429dd35d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lippenss/src/openeo/openeo-python-client/openeo/metadata.py:240: UserWarning: No cube:dimensions metadata\n",
      "  complain(\"No cube:dimensions metadata\")\n"
     ]
    }
   ],
   "source": [
    "cube2 = con.datacube_from_process(\"load_result\", id=first_job_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "084011ce-e039-451f-ae58-cb96f9d34444",
   "metadata": {},
   "outputs": [],
   "source": [
    "cube2.download(download_dir / \"2-sync.tiff\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a4ad85f9-996d-4b4d-88b3-f03d83ede6fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "import rasterio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ff7a52a7-e429-4c44-8393-3b6ce47cdf8e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bands: 1, width: 1166, height: 1854\n",
      "data pixels: 0\n",
      "nodata pixels: 2161764\n"
     ]
    }
   ],
   "source": [
    "with rasterio.open(download_dir / \"2-sync.tiff\") as ds:\n",
    "    print(f\"bands: {ds.count}, width: {ds.width}, height: {ds.height}\")\n",
    "    data = ds.read()\n",
    "    print(\"data pixels:\", (data != ds.nodata).sum())\n",
    "    print(\"nodata pixels:\", (data == ds.nodata).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ba5d4984-7fda-4def-8194-a4ce6c28ca29",
   "metadata": {},
   "outputs": [
    {
     "ename": "OpenEoApiError",
     "evalue": "[500] unknown: / by zero",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOpenEoApiError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-e641f0a7c9a8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcube2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdownload_dir\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m\"2-sync.nc\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/src/openeo/openeo-python-client/openeo/rest/datacube.py\u001b[0m in \u001b[0;36mdownload\u001b[0;34m(self, outputfile, format, options)\u001b[0m\n\u001b[1;32m   1523\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1524\u001b[0m         \u001b[0mcube\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1525\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_connection\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcube\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflat_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1526\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1527\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mvalidate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/src/openeo/openeo-python-client/openeo/rest/connection.py\u001b[0m in \u001b[0;36mdownload\u001b[0;34m(self, graph, outputfile, timeout)\u001b[0m\n\u001b[1;32m   1014\u001b[0m         \"\"\"\n\u001b[1;32m   1015\u001b[0m         \u001b[0mrequest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_build_request_with_process_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1016\u001b[0;31m         \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpost\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"/result\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexpected_status\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1017\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1018\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0moutputfile\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/src/openeo/openeo-python-client/openeo/rest/connection.py\u001b[0m in \u001b[0;36mpost\u001b[0;34m(self, path, json, **kwargs)\u001b[0m\n\u001b[1;32m    170\u001b[0m         \u001b[0;34m:\u001b[0m\u001b[0;32mreturn\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mResponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m         \"\"\"\n\u001b[0;32m--> 172\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"post\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_redirects\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    173\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    174\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mdelete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mResponse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/src/openeo/openeo-python-client/openeo/rest/connection.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, path, headers, auth, check_error, expected_status, **kwargs)\u001b[0m\n\u001b[1;32m    118\u001b[0m         \u001b[0mexpected_status\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mensure_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexpected_status\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mexpected_status\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcheck_error\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m400\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mexpected_status\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 120\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_raise_api_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    121\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mexpected_status\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mexpected_status\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m             raise OpenEoRestError(\"Got status code {s!r} for `{m} {p}` (expected {e!r})\".format(\n",
      "\u001b[0;32m~/src/openeo/openeo-python-client/openeo/rest/connection.py\u001b[0m in \u001b[0;36m_raise_api_error\u001b[0;34m(self, response)\u001b[0m\n\u001b[1;32m    149\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    150\u001b[0m                 \u001b[0mexception\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mOpenEoApiError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhttp_status_code\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstatus_code\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 151\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    152\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    153\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mauth\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAuthBase\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mResponse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOpenEoApiError\u001b[0m: [500] unknown: / by zero"
     ]
    }
   ],
   "source": [
    "cube2.download(download_dir / \"2-sync.nc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "413b795a-c6a8-4ab7-b190-9d5a98d566be",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c490e80-d6d5-4272-8ee8-524a41b2dd60",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c19b1920-8e77-46f8-a6ba-22eb71f24bd0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efb676e8-3da7-4672-8cbc-d1a14f2492af",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "openeo-py39",
   "language": "python",
   "name": "openeo-py39"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}