stream.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import descarteslabs as dl\n",
    "import json\n",
    "import uuid\n",
    "\n",
    "class ImageStream(object):\n",
    "    \"\"\"\n",
    "    Compose metadata and storage to get a class that\n",
    "    can continuously and reliably iterate a search of \n",
    "    imagery forever.\n",
    "    \"\"\"\n",
    "    def __init__(self, params, key=None, storage_client=None, metadata_client=None):        \n",
    "        self.params = params\n",
    "\n",
    "        if key is None:\n",
    "            key = str(uuid.uuid4())\n",
    "        self.key = key\n",
    "    \n",
    "        if storage_client is None:\n",
    "            storage_client = dl.Storage()\n",
    "            \n",
    "        self.storage_client = storage_client\n",
    "        \n",
    "        if metadata_client is None:\n",
    "            metadata_client = dl.Metadata()\n",
    "\n",
    "        self.metadata_client = metadata_client\n",
    "        self._continuation_token = None\n",
    "    \n",
    "    @staticmethod\n",
    "    def _persistence_key(key):\n",
    "        return key + \"-persist\"\n",
    "\n",
    "    def persist(self):\n",
    "        params = self.storage_client.set(self._persistence_key(self.key), json.dumps(self.params))\n",
    "        return self.key\n",
    "\n",
    "    @classmethod\n",
    "    def load(cls, key, storage_client=None):\n",
    "        if storage_client is None:\n",
    "            storage_client = dl.Storage()\n",
    "            \n",
    "\n",
    "        params = storage_client.get(cls._persistence_key(key))\n",
    "        \n",
    "        return cls(params=json.loads(params), key=key)\n",
    "\n",
    "    def features(self):\n",
    "        # TODO lock for distributed runs\n",
    "        continuation_token = self.continuation_token\n",
    "        \n",
    "        # TODO switch to scenes once able to support no aoi\n",
    "        response = self.metadata_client.search(continuation_token=continuation_token, **self.params)\n",
    "    \n",
    "        if 'properties' in response and 'continuation_token' in response.properties:\n",
    "            self.continuation_token = response.properties.continuation_token\n",
    "\n",
    "        return response.features\n",
    "        \n",
    "    @property\n",
    "    def continuation_token(self):\n",
    "        # TODO lock for distributed runs\n",
    "        if self._continuation_token is None:\n",
    "            try:\n",
    "                self._continuation_token = self.storage_client.get(self.key).decode('utf-8')\n",
    "            except dl.exceptions.NotFoundError:\n",
    "                self._continuation_token = None\n",
    "                \n",
    "        return self._continuation_token\n",
    "    \n",
    "    @continuation_token.setter\n",
    "    def continuation_token(self, token):\n",
    "        # TODO lock for distributed runs\n",
    "        self._continuation_token = token\n",
    "        self.storage_client.set(self.key, value=token)\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = dict(\n",
    "    limit=1, # this is for demonstration purposes only\n",
    "    fields=['acquired'],\n",
    "    start_datetime='2018-12-03',\n",
    "    products=['sentinel-2:L1C'],\n",
    "    sort_field='acquired',\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "stream = ImageStream(params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\n",
       "  {\n",
       "    'id': 'sentinel-2:L1C:2018-12-03_53SLD_03_S2B_v1',\n",
       "    'properties': {\n",
       "      'acquired': '2018-12-03T02:06:36.503000+00:00'\n",
       "    }\n",
       "  }\n",
       "]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stream.features()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1daf6566-52eb-4815-be08-00efe7728254'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "key = stream.persist()\n",
    "key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# could run this in a cron and publish to a queue\n",
    "stream = ImageStream.load(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\n",
       "  {\n",
       "    'id': 'sentinel-2:L1C:2018-12-03_53SKD_24_S2B_v1',\n",
       "    'properties': {\n",
       "      'acquired': '2018-12-03T02:06:37.701000+00:00'\n",
       "    }\n",
       "  }\n",
       "]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stream.features()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\n",
      "  {\n",
      "    'id': 'sentinel-2:L1C:2018-12-03_52SGJ_28_S2B_v1',\n",
      "    'properties': {\n",
      "      'acquired': '2018-12-03T02:06:37.897000+00:00'\n",
      "    }\n",
      "  }\n",
      "]\n",
      "[\n",
      "  {\n",
      "    'id': 'sentinel-2:L1C:2018-12-03_52SFJ_10_S2B_v1',\n",
      "    'properties': {\n",
      "      'acquired': '2018-12-03T02:06:39.465000+00:00'\n",
      "    }\n",
      "  }\n",
      "]\n",
      "[\n",
      "  {\n",
      "    'id': 'sentinel-2:L1C:2018-12-03_53SMC_25_S2B_v1',\n",
      "    'properties': {\n",
      "      'acquired': '2018-12-03T02:06:40.551000+00:00'\n",
      "    }\n",
      "  }\n",
      "]\n",
      "[\n",
      "  {\n",
      "    'id': 'sentinel-2:L1C:2018-12-03_53SLC_89_S2B_v1',\n",
      "    'properties': {\n",
      "      'acquired': '2018-12-03T02:06:42.836000+00:00'\n",
      "    }\n",
      "  }\n",
      "]\n",
      "[\n",
      "  {\n",
      "    'id': 'sentinel-2:L1C:2018-12-03_53SKC_99_S2B_v1',\n",
      "    'properties': {\n",
      "      'acquired': '2018-12-03T02:06:45.992000+00:00'\n",
      "    }\n",
      "  }\n",
      "]\n",
      "[\n",
      "  {\n",
      "    'id': 'sentinel-2:L1C:2018-12-03_52SGH_99_S2B_v1',\n",
      "    'properties': {\n",
      "      'acquired': '2018-12-03T02:06:46.717000+00:00'\n",
      "    }\n",
      "  }\n",
      "]\n",
      "[\n",
      "  {\n",
      "    'id': 'sentinel-2:L1C:2018-12-03_52SFH_41_S2B_v1',\n",
      "    'properties': {\n",
      "      'acquired': '2018-12-03T02:06:49.436000+00:00'\n",
      "    }\n",
      "  }\n",
      "]\n",
      "[\n",
      "  {\n",
      "    'id': 'sentinel-2:L1C:2018-12-03_53SMB_14_S2B_v1',\n",
      "    'properties': {\n",
      "      'acquired': '2018-12-03T02:06:50.216000+00:00'\n",
      "    }\n",
      "  }\n",
      "]\n",
      "[\n",
      "  {\n",
      "    'id': 'sentinel-2:L1C:2018-12-03_53SLB_98_S2B_v1',\n",
      "    'properties': {\n",
      "      'acquired': '2018-12-03T02:06:56.035000+00:00'\n",
      "    }\n",
      "  }\n",
      "]\n",
      "[\n",
      "  {\n",
      "    'id': 'sentinel-2:L1C:2018-12-03_53SMA_00_S2B_v1',\n",
      "    'properties': {\n",
      "      'acquired': '2018-12-03T02:07:00.083000+00:00'\n",
      "    }\n",
      "  }\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print(ImageStream.load(key).features()) # simulating cron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}