fix datetimes

fix-datetimes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "af1ce8bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pystac_client\n",
    "import requests\n",
    "import time\n",
    "import boto3\n",
    "from datetime import datetime, timezone\n",
    "import json\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fd76b1c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "INGESTOR_ENDPOINT = 'https://stac-ingestor.maap-project.org/ingestions'\n",
    "DATETIME='2019-01-01T00:00:00+00:00'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "38dcf9bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get a token to authenticate to the ingestor\n",
    "def get_authentication_token():\n",
    "\n",
    "    # session = boto3.session.Session()\n",
    "    client = boto3.client(\"secretsmanager\", region_name=\"us-west-2\")\n",
    "    secret_id = 'SECRET_HERE'\n",
    "\n",
    "    res_secret = client.get_secret_value(SecretId=secret_id)\n",
    "\n",
    "\n",
    "    # Authentication - Get TOKEN\n",
    "    secret = json.loads(res_secret[\"SecretString\"])\n",
    "    client_secret = secret[\"client_secret\"]\n",
    "    client_id = secret[\"client_id\"]\n",
    "    cognito_domain = secret[\"cognito_domain\"]\n",
    "    scope = secret[\"scope\"]\n",
    "\n",
    "    res_token = requests.post(\n",
    "        f\"{cognito_domain}/oauth2/token\",\n",
    "        headers={\n",
    "            \"Content-Type\": \"application/x-www-form-urlencoded\",\n",
    "        },\n",
    "        auth=(client_id, client_secret),\n",
    "        data={\n",
    "            \"grant_type\": \"client_credentials\",\n",
    "            # A space-separated list of scopes to request for the generated access token.\n",
    "            \"scope\": scope,\n",
    "        },\n",
    "    )\n",
    "\n",
    "    token = res_token.json()[\"access_token\"]\n",
    "    return token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2c3809ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "client = pystac_client.Client.open('https://stac.maap-project.org/').get_collection('icesat2-boreal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a696dc2e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing boreal_agb_202302061675675202_31758: : 81it [02:43,  2.19s/it]"
     ]
    }
   ],
   "source": [
    "failures = []\n",
    "it = tqdm.tqdm(client.get_items())\n",
    "for item in it:\n",
    "    it.set_description(\"Processing %s\" % item.to_dict()['id'])\n",
    "    try:\n",
    "        item = item.to_dict()\n",
    "        item['properties']['datetime'] = DATETIME\n",
    "        result = requests.post(INGESTOR_ENDPOINT, json=item, headers={\"Authorization\": f\"bearer {get_authentication_token()}\"})\n",
    "        if result.status_code!=201:\n",
    "            raise Exception('failed ingestion')\n",
    "    except Exception as ex:\n",
    "        failures.append(item['id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12c7c70b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ingest_stac_items",
   "language": "python",
   "name": "ingest_stac_items"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}