Parallel Ground (Except it Doesn't Work).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['ISISROOT'] = '/usgs/cpkgs/anaconda3_linux/envs/isis3.9.0'\n",
    "\n",
    "import ctypes\n",
    "from ctypes.util import find_library\n",
    "ctypes.CDLL(find_library('usgscsm'))\n",
    "\n",
    "from pysis import isis\n",
    "from plio.io import io_controlnetwork\n",
    "\n",
    "from autocnet.graph.network import NetworkCandidateGraph\n",
    "from autocnet.io.db.model import Measures, Points, Images\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "config_path = '/home/krodriguez/.autocnet/newpoints.yml'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "db name:  kel_oxia_palus_nw_points_test\n",
      "log dir:  /scratch/krodriguez/mars_quads/oxia_palus/NW/test/logs\n",
      "ISISROOT:  /usgs/cpkgs/anaconda3_linux/envs/isis3.9.0\n",
      "conda env:  autocnetdev\n"
     ]
    }
   ],
   "source": [
    "from autocnet.config_parser import parse_config\n",
    "config = parse_config(config_path)\n",
    "default_db = config['database']['name']\n",
    "print('db name: ', default_db)\n",
    "default_log = str(config['cluster']['cluster_log_dir'])\n",
    "print('log dir: ', default_log)\n",
    "print('ISISROOT: ', config['env']['ISISROOT'])\n",
    "print('conda env: ', config['env']['conda'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Quickly write out DB that isn't the config_path file DB \n",
    "# subpix_config = {\n",
    "#     'username': 'jay',\n",
    "#     'password': 'abcde',\n",
    "#     'host': '130.118.160.193',\n",
    "#     'port': 8085,\n",
    "#     'pgbouncer_port': 8083,\n",
    "#     'name': 'lak_oxia_palus_subpix'\n",
    "# }\n",
    "\n",
    "# ncg = NetworkCandidateGraph()\n",
    "# ncg.config = {}\n",
    "# ncg.config['database'] = subpix_config\n",
    "# ncg.config['spatial'] = config['spatial']\n",
    "# ncg._setup_database()\n",
    "# ncg.from_database()\n",
    "# ncg.plot()\n",
    "\n",
    "# path = '/scratch/krodriguez/mars_quads/oxia_palus_phase/'\n",
    "# # cnet = 'mc11_center_test_ns8_ew1_subpix_t121x61.net'\n",
    "# ncg.to_isis(os.path.join(path, 'mc11_ns8_ew1_t121x61_all.net'), sql=sql) # write out all points"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read in "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First Run - Remote Database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "remote_config = {'username':'jay',\n",
    "          'password':'abcde',\n",
    "          'host':'autocnet.wr.usgs.gov',\n",
    "          'pgbouncer_port':30001, # PREVIOUSLY 5432\n",
    "          'name':'mars'}\n",
    "\n",
    "# geom operates in the 0 to 360 Longitude domain\n",
    "# the oxia palus quad is located at 0-30N Latitude and 315-360E Longitude\n",
    "\n",
    "# # # geom = 'LINESTRING(315 0, 360 0, 360 30, 315 30, 315 0)' # full quad - 3401 images\n",
    "# # # geom_NW = 'LINESTRING(315 15, 337.5 15, 337.5 30, 315 30, 315 15)' # north west - 833 images\n",
    "# geom = 'LINESTRING(325 20, 330 20, 330 25, 325 25, 325 20)' # north west - 69 images\n",
    "\n",
    "# outpath = '/scratch/krodriguez/mars_quads/oxia_palus_phase/'\n",
    "\n",
    "# srid = 949900\n",
    "# query = f\"SELECT * FROM ctx WHERE ST_INTERSECTS(geom, ST_Polygon(ST_GeomFromText('{geom}'), {srid})) = TRUE\"\n",
    "# ncg = NetworkCandidateGraph()\n",
    "# ncg.config_from_file(config_path)\n",
    "# ncg.add_from_remote_database(remote_config, outpath, query_string=query)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Subsequent Runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cluster queue length:\n",
      "before ->  0\n"
     ]
    }
   ],
   "source": [
    "ncg = NetworkCandidateGraph()\n",
    "ncg.config_from_file(config_path)\n",
    "ncg.from_database()\n",
    "\n",
    "print('Cluster queue length:')\n",
    "print('before -> ', ncg.queue_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ncg.queue_flushdb()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after  ->  0\n"
     ]
    }
   ],
   "source": [
    "# print('Cleaning up cluster queue:')\n",
    "# ncg.queue_flushdb()\n",
    "print('after  -> ', ncg.queue_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Place Points in Overlap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "364\n"
     ]
    }
   ],
   "source": [
    "def ns(x):\n",
    "    from math import ceil\n",
    "    return ceil(round(x,1)*8)\n",
    "\n",
    "def ew(x):\n",
    "    from math import ceil\n",
    "    return ceil(round(x,1)*2)\n",
    "\n",
    "distribute_points_kwargs = {'nspts_func':ns, 'ewpts_func':ew, 'method':'classic'}\n",
    "# log_dir = default_log.replace('logs', 'ppio_logs')\n",
    "\n",
    "njobs = ncg.apply('spatial.overlap.place_points_in_overlap', \n",
    "                  on='overlaps', \n",
    "                  walltime='00:20:00',\n",
    "                  cam_type='isis',\n",
    "                  point_type=2, \n",
    "                  distribute_points_kwargs=distribute_points_kwargs,\n",
    "#                   log_dir=log_dir,\n",
    "                  arraychunk=200,\n",
    "                  chunksize=20000)\n",
    "print(njobs)\n",
    "# number of overlaps - 1016\n",
    "# no errors! Yay!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "jobs left on the queue:  728\n"
     ]
    }
   ],
   "source": [
    "redis_orphans = ncg.queue_length\n",
    "print(\"jobs left on the queue: \", redis_orphans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "364\n"
     ]
    }
   ],
   "source": [
    "def ns(x):\n",
    "    from math import ceil\n",
    "    return ceil(round(x,1)*1)\n",
    "\n",
    "def ew(x):\n",
    "    from math import ceil\n",
    "    return ceil(round(x,1)*1)\n",
    "\n",
    "distribute_points_kwargs = {'nspts_func':ns, 'ewpts_func':ew, 'method':'classic'}\n",
    "\n",
    "njobs = ncg.apply('spatial.overlap.place_points_in_overlap', \n",
    "                  on='overlaps', \n",
    "                  walltime='00:20:00',\n",
    "                  cam_type='isis',\n",
    "                  distribute_points_kwargs=distribute_points_kwargs,\n",
    "                  point_type=3, \n",
    "                  arraychunk=200,\n",
    "                  chunksize=20000)\n",
    "print(njobs)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set Reference Measures "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "334"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "gmosaic = \"/archive/projects/themis_control/BlendedQuads/DIR/mc11_oxia_palus_dir_final.cub\"\n",
    "\n",
    "subpixel_template_kwargs = {'image_size':(21, 21), 'template_size':(11,11)} # same used on kaguya mosaic, similar data in terms of offset/ resolution\n",
    "iterative_phase_kwargs = {\"size\" : (61,61), \"reduction\": 11, \"max_dist\" : 10}\n",
    "\n",
    "filters = {\"pointtype\" : 3}\n",
    "\n",
    "# run using \n",
    "njobs = ncg.apply('matcher.subpixel.compute_reference_measure',\n",
    "                  on='points',\n",
    "                  base_image=gmosaic,\n",
    "                  match_func='classic',\n",
    "                  verbose=True,\n",
    "                  match_kwargs=subpixel_template_kwargs,\n",
    "                  filters=filters,\n",
    "                  version='simple',\n",
    "                  walltime=\"00:30:00\",\n",
    "                  value_func=lambda x,y: y == min(x),\n",
    "                  threshold=.98, \n",
    "                  arraychunk=200,\n",
    "                  chunksize=20000) # maximum chunksize = 20,000\n",
    "njobs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Match to ref measures "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3799"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subpixel_template_kwargs = {'image_size':(121,121), 'template_size':(61,61)} # same used on kaguya mosaic, similar data in terms of offset/ resolution\n",
    "iterative_phase_kwargs = {\"size\" : (61,61), \"max_dist\" : 10}\n",
    "filters = {}\n",
    "# log_dir = default_log.replace('logs', 'subpix_logs_newMethod')\n",
    "\n",
    "# run using \n",
    "njobs = ncg.apply('matcher.subpixel.subpixel_register_measure',\n",
    "                  on='measures',\n",
    "                  match_func='classic',\n",
    "                  match_kwargs=subpixel_template_kwargs,\n",
    "                  filters=filters,\n",
    "                  version='simple',\n",
    "                  walltime=\"00:30:00\",\n",
    "                  size_x=61, size_y=61,\n",
    "                  cost_func=lambda x,y:1-y,\n",
    "                  threshold=.93, \n",
    "                  arraychunk=200,\n",
    "                  chunksize=20000) # maximum chunksize = 20,000\n",
    "\n",
    "njobs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ppio: Write out\n",
    "#### The default sql query will only write out the non-ignored points and measures. This sql query writes out all the points. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sql = \"\"\"\n",
    "SELECT measures.\"pointid\",\n",
    "        points.\"pointType\",\n",
    "        points.\"apriori\",\n",
    "        points.\"adjusted\",\n",
    "        points.\"pointIgnore\",\n",
    "        measures.\"id\",\n",
    "        measures.\"serialnumber\",\n",
    "        measures.\"sample\",\n",
    "        measures.\"line\",\n",
    "        measures.\"measureType\",\n",
    "        measures.\"imageid\",\n",
    "        measures.\"measureIgnore\",\n",
    "        measures.\"measureJigsawRejected\",\n",
    "        measures.\"aprioriline\",\n",
    "        measures.\"apriorisample\"\n",
    "FROM measures\n",
    "INNER JOIN points ON measures.\"pointid\" = points.\"id\"\n",
    "WHERE\n",
    "    measures.\"imageid\" NOT IN\n",
    "        (SELECT measures.\"imageid\"\n",
    "        FROM measures\n",
    "        INNER JOIN points ON measures.\"pointid\" = points.\"id\"\n",
    "        WHERE measures.\"measureIgnore\" = False and measures.\"measureJigsawRejected\" = False AND points.\"pointIgnore\" = False\n",
    "        GROUP BY measures.\"imageid\"\n",
    "        HAVING COUNT(DISTINCT measures.\"pointid\")  < 3)\n",
    "ORDER BY measures.\"pointid\", measures.\"id\";\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = '/scratch/krodriguez/mars_quads/oxia_palus/NW/test/iterations/'\n",
    "\n",
    "# ncg.to_isis(os.path.join(path, 'mc11_ns8_ew1_t121x61_active.net')) # write out active points\n",
    "ncg.to_isis(os.path.join(path, 'mc11_ns8_ew1_t121x61_new_points.net')) # write out all points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "AutocnetDev",
   "language": "python",
   "name": "autocnetdev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
(bas