Trying out tiledb with python, gdal, xarray

tiledb-experiments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test out tileDB\n",
    "https://docs.tiledb.io/en/stable/quickstart.html#a-simple-dense-array-example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 6, 2)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import sys\n",
    "import tiledb\n",
    "\n",
    "print(tiledb.libtiledb.version())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Name of the array to create.\n",
    "array_name = \"quickstart_dense\"\n",
    "\n",
    "def create_array():\n",
    "    # Create a TileDB context (removed to just use default)\n",
    "    # ctx = tiledb.Ctx()\n",
    "\n",
    "    # Check if the array already exists.\n",
    "    if tiledb.object_type(array_name) == \"array\":\n",
    "        print(\"Array already exists.\")\n",
    "        #sys.exit(0)\n",
    "        return\n",
    "\n",
    "    # The array will be 4x4 with dimensions \"rows\" and \"cols\", with domain [1,4].\n",
    "    dom = tiledb.Domain(tiledb.Dim(name=\"rows\", domain=(1, 4), tile=4, dtype=np.int32),\n",
    "                        tiledb.Dim(name=\"cols\", domain=(1, 4), tile=4, dtype=np.int32))\n",
    "\n",
    "    # The array will be dense with a single attribute \"a\" so each (i,j) cell can store an integer.\n",
    "    schema = tiledb.ArraySchema(domain=dom, sparse=False,\n",
    "                                attrs=[tiledb.Attr(name=\"a\", dtype=np.int32)])\n",
    "\n",
    "    # Create the (empty) array on disk.\n",
    "    tiledb.DenseArray.create(array_name, schema)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Array already exists.\n"
     ]
    }
   ],
   "source": [
    "create_array()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_array():\n",
    "    #ctx = tiledb.Ctx()\n",
    "    # Open the array and write to it.\n",
    "    with tiledb.DenseArray(array_name, mode='w') as A:\n",
    "        data = np.array(([1, 2, 3, 4],\n",
    "                         [5, 6, 7, 8],\n",
    "                         [9, 10, 11, 12],\n",
    "                         [13, 14, 15, 16]))\n",
    "        A[:] = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_array()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_array():\n",
    "    #ctx = tiledb.Ctx()\n",
    "    # Open the array and read from it.\n",
    "    with tiledb.DenseArray(array_name, mode='r') as A:\n",
    "        # Slice only rows 1, 2 and cols 2, 3, 4.\n",
    "        data = A[1:3, 2:5]\n",
    "        print(data[\"a\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2 3 4]\n",
      " [6 7 8]]\n"
     ]
    }
   ],
   "source": [
    "read_array()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use GDAL driver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDAL 3.0.1, released 2019/06/28\n"
     ]
    }
   ],
   "source": [
    "!gdalinfo --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rest.server_serialization_format CAPNP\n",
      "sm.check_coord_dups true\n",
      "sm.check_coord_oob true\n",
      "sm.check_global_order true\n",
      "sm.consolidation.amplification 1\n",
      "sm.consolidation.buffer_size 50000000\n",
      "sm.consolidation.step_max_frags 4294967295\n",
      "sm.consolidation.step_min_frags 4294967295\n",
      "sm.consolidation.step_size_ratio 0\n",
      "sm.consolidation.steps 4294967295\n",
      "sm.dedup_coords false\n",
      "sm.enable_signal_handlers true\n",
      "sm.memory_budget 5368709120\n",
      "sm.memory_budget_var 10737418240\n",
      "sm.num_async_threads 1\n",
      "sm.num_reader_threads 1\n",
      "sm.num_tbb_threads -1\n",
      "sm.num_writer_threads 1\n",
      "sm.tile_cache_size 10000000\n",
      "vfs.file.enable_filelocks true\n",
      "vfs.file.max_parallel_ops 8\n",
      "vfs.min_batch_gap 512000\n",
      "vfs.min_batch_size 20971520\n",
      "vfs.min_parallel_size 10485760\n",
      "vfs.num_threads 8\n",
      "vfs.s3.connect_max_tries 5\n",
      "vfs.s3.connect_scale_factor 25\n",
      "vfs.s3.connect_timeout_ms 3000\n",
      "vfs.s3.max_parallel_ops 8\n",
      "vfs.s3.multipart_part_size 5242880\n",
      "vfs.s3.proxy_port 0\n",
      "vfs.s3.proxy_scheme https\n",
      "vfs.s3.region us-east-1\n",
      "vfs.s3.request_timeout_ms 3000\n",
      "vfs.s3.scheme https\n",
      "vfs.s3.use_multipart_upload true\n",
      "vfs.s3.use_virtual_addressing true\n"
     ]
    }
   ],
   "source": [
    "# Write default config to a file\n",
    "# Save to file\n",
    "config = tiledb.Config()\n",
    "config.save(\"tiledb.config\")\n",
    "!cat tiledb.config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter                        | Value        \n",
      "-------------------------------- | -------------\n",
      "rest.server_serialization_format | 'CAPNP'      \n",
      "sm.check_coord_dups              | 'true'       \n",
      "sm.check_coord_oob               | 'true'       \n",
      "sm.check_global_order            | 'true'       \n",
      "sm.consolidation.amplification   | '1'          \n",
      "sm.consolidation.buffer_size     | '50000000'   \n",
      "sm.consolidation.step_max_frags  | '4294967295' \n",
      "sm.consolidation.step_min_frags  | '4294967295' \n",
      "sm.consolidation.step_size_ratio | '0'          \n",
      "sm.consolidation.steps           | '4294967295' \n",
      "sm.dedup_coords                  | 'false'      \n",
      "sm.enable_signal_handlers        | 'true'       \n",
      "sm.memory_budget                 | '5368709120' \n",
      "sm.memory_budget_var             | '10737418240'\n",
      "sm.num_async_threads             | '1'          \n",
      "sm.num_reader_threads            | '1'          \n",
      "sm.num_tbb_threads               | '-1'         \n",
      "sm.num_writer_threads            | '1'          \n",
      "sm.tile_cache_size               | '10000000'   \n",
      "vfs.file.enable_filelocks        | 'true'       \n",
      "vfs.file.max_parallel_ops        | '8'          \n",
      "vfs.hdfs.kerb_ticket_cache_path  | ''           \n",
      "vfs.hdfs.name_node_uri           | ''           \n",
      "vfs.hdfs.username                | ''           \n",
      "vfs.min_batch_gap                | '512000'     \n",
      "vfs.min_batch_size               | '20971520'   \n",
      "vfs.min_parallel_size            | '10485760'   \n",
      "vfs.num_threads                  | '8'          \n",
      "vfs.s3.aws_access_key_id         | ''           \n",
      "vfs.s3.aws_secret_access_key     | ''           \n",
      "vfs.s3.connect_max_tries         | '5'          \n",
      "vfs.s3.connect_scale_factor      | '25'         \n",
      "vfs.s3.connect_timeout_ms        | '3000'       \n",
      "vfs.s3.endpoint_override         | ''           \n",
      "vfs.s3.max_parallel_ops          | '8'          \n",
      "vfs.s3.multipart_part_size       | '5242880'    \n",
      "vfs.s3.proxy_host                | ''           \n",
      "vfs.s3.proxy_password            | ''           \n",
      "vfs.s3.proxy_port                | '0'          \n",
      "vfs.s3.proxy_scheme              | 'https'      \n",
      "vfs.s3.proxy_username            | ''           \n",
      "vfs.s3.region                    | 'us-east-1'  \n",
      "vfs.s3.request_timeout_ms        | '3000'       \n",
      "vfs.s3.scheme                    | 'https'      \n",
      "vfs.s3.use_multipart_upload      | 'true'       \n",
      "vfs.s3.use_virtual_addressing    | 'true'       \n"
     ]
    }
   ],
   "source": [
    "# Load from file\n",
    "config_load = tiledb.Config.load(\"tiledb.config\")\n",
    "print(config_load)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      Name                    Value             Type    Location\n",
      "      ----                    -----             ----    --------\n",
      "   profile                 guest-s3           manual    --profile\n",
      "access_key     ****************C2HA shared-credentials-file    \n",
      "secret_key     ****************fXt2 shared-credentials-file    \n",
      "    region                <not set>             None    None\n"
     ]
    }
   ],
   "source": [
    "!aws --profile guest-s3 configure list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                           PRE DeepBlue-SeaWiFS-1.0_L3_20100101_v004-20130604T131317Z.tiledb/\n"
     ]
    }
   ],
   "source": [
    "# Uses EC2 instance credentials\n",
    "!aws --profile guest-s3 s3 ls s3://pangeo-data-upload-virginia/gdal3-test/DeepBlue-SeaWiFS-1.0_L3_20100101_v004-20130604T131317Z.tiledb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Driver: TileDB/TileDB\n",
      "Files: DeepBlue-SeaWiFS-1.0_L3_20100101_v004-20130604T131317Z.tiledb/DeepBlue-SeaWiFS-1.0_L3_20100101_v004-20130604T131317Z.tdb\n",
      "       DeepBlue-SeaWiFS-1.0_L3_20100101_v004-20130604T131317Z.tiledb\n",
      "       DeepBlue-SeaWiFS-1.0_L3_20100101_v004-20130604T131317Z.tiledb/DeepBlue-SeaWiFS-1.0_L3_20100101_v004-20130604T131317Z.tdb.aux.xml\n",
      "Size is 512, 512\n",
      "Metadata:\n",
      "  solar_zenith_angle_long_name=solar zenith angle\n",
      "  solar_zenith_angle_standard_name=solar_zenith_angle\n",
      "  solar_zenith_angle_units=degrees\n",
      "  solar_zenith_angle_valid_range=0 90 \n",
      "  solar_zenith_angle__FillValue=-999 \n",
      "  viewing_zenith_angle_long_name=viewing zenith angle\n",
      "  viewing_zenith_angle_units=degrees\n",
      "  viewing_zenith_angle_valid_range=0 90 \n",
      "  viewing_zenith_angle__FillValue=-999 \n",
      "Subdatasets:\n",
      "  SUBDATASET_1_NAME=TILEDB:\"DeepBlue-SeaWiFS-1.0_L3_20100101_v004-20130604T131317Z.tiledb\":solar_zenith_angle\n",
      "  SUBDATASET_1_DESC=[1x360x180] solar_zenith_angle (Float32)\n",
      "  SUBDATASET_2_NAME=TILEDB:\"DeepBlue-SeaWiFS-1.0_L3_20100101_v004-20130604T131317Z.tiledb\":viewing_zenith_angle\n",
      "  SUBDATASET_2_DESC=[1x360x180] viewing_zenith_angle (Float32)\n",
      "Corner Coordinates:\n",
      "Upper Left  (    0.0,    0.0)\n",
      "Lower Left  (    0.0,  512.0)\n",
      "Upper Right (  512.0,    0.0)\n",
      "Lower Right (  512.0,  512.0)\n",
      "Center      (  256.0,  256.0)\n"
     ]
    }
   ],
   "source": [
    "!gdalinfo DeepBlue-SeaWiFS-1.0_L3_20100101_v004-20130604T131317Z.tiledb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR 4: `DeepBlue-SeaWiFS-1.0_L3_20100101_v004-20130604T131317Z.tiledb' not recognized as a supported file format.\n",
      "gdalinfo failed - unable to open 'DeepBlue-SeaWiFS-1.0_L3_20100101_v004-20130604T131317Z.tiledb'.\n"
     ]
    }
   ],
   "source": [
    "!CPL_DEBUG=ON gdalinfo -oo TILEDB_CONFIG=tiledb.config DeepBlue-SeaWiFS-1.0_L3_20100101_v004-20130604T131317Z.tiledb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:gdal3]",
   "language": "python",
   "name": "conda-env-gdal3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}