Memcached client for Lima

Lima_Memcached_demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'Lima'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-8933fadce3b0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mLima\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'Lima'"
     ]
    }
   ],
   "source": [
    "import Lima"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'Lima'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-f0cddb6115ce>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mLima\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mCore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'Lima'"
     ]
    }
   ],
   "source": [
    "from Lima import Core"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Lima import Basler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import blosc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "IP = \"192.168.5.19\"\n",
    "fps = 30\n",
    "EXPO = 0.001 #sec\n",
    "scan = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "Exception",
     "evalue": "Hardware: Camera(/opt/conda/conda-bld/lima-camera-basler_1549640215705/work/src/BaslerCamera.cpp, 284): Error: Failed to discover GigE device '192.168.5.19:3956'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mException\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-f99c8f65ed8e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcam\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBasler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCamera\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mIP\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0miface\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBasler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mInterface\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcam\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mctrl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCtControl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miface\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0macq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mctrl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquisition\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0macq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetAcqNbFrames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mException\u001b[0m: Hardware: Camera(/opt/conda/conda-bld/lima-camera-basler_1549640215705/work/src/BaslerCamera.cpp, 284): Error: Failed to discover GigE device '192.168.5.19:3956'"
     ]
    }
   ],
   "source": [
    "cam = Basler.Camera(IP)\n",
    "iface = Basler.Interface(cam)\n",
    "ctrl = Core.CtControl(iface)\n",
    "acq = ctrl.acquisition()\n",
    "acq.setAcqNbFrames(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<AcquisitionStatus=AcqRunning, ImageCounters=<LastImageAcquired=-1, LastBaseImageReady=-1, LastImageReady=-1, LastImageSaved=-1, LastCounterReady=-1>\n"
     ]
    }
   ],
   "source": [
    "ctrl.prepareAcq()\n",
    "ctrl.startAcq()\n",
    "print(ctrl.getStatus())\n",
    "import time\n",
    "time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "img = ctrl.ReadImage()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "779 µs ± 41.7 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit len(blosc.compress(img.buffer.data, 2, shuffle=2, cname=\"lz4\"))/img.buffer.nbytes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img.frameNumber"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 76,  88,  72, ...,  93,  86,  87],\n",
       "       [ 65, 109,  66, ...,  83, 122,  70],\n",
       "       [ 77,  74,  71, ...,  80,  86,  90],\n",
       "       ...,\n",
       "       [127, 136, 122, ..., 126, 144, 138],\n",
       "       [138, 128, 130, ..., 140, 156, 152],\n",
       "       [134, 148, 128, ..., 157, 168, 157]], dtype=uint16)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img.buffer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<memory at 0x7f8ec5cd8120>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img.buffer.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import namedtuple\n",
    "Id = namedtuple(\"Id\", \"beamline camera scan frame timestamp shape dtype comp\")\n",
    "i=Id(\"id00\", \"basler\", 1, img.frameNumber, img.timestamp, img.buffer.shape, str(img.buffer.dtype), \"blosc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "130"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(str(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Id(beamline='id00',camera='basler',scan=1,frame=4,timestamp=5.03251314163208,shape=(966,1296),dtype='uint16',comp='blosc')\""
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(i).replace(\" \", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Id(beamline='id00', camera='basler', scan=1, frame=11, timestamp=12.032986164093018, shape=(966, 1296), dtype='uint16', comp='blosc')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "i=Id(\"id00\", \"basler\", 1, img.frameNumber, img.timestamp, img.buffer.shape, str(img.buffer.dtype), \"blosc\")\n",
    "i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import memcache\n",
    "#client = memcache.Client(\"localhost\", 11211)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MemcachedSink(Core.Processlib.SinkTaskBase):\n",
    "    def __init__(self, server, port, compression=None):\n",
    "        \"\"\"\n",
    "        :param server_port: port to which to connect (can be a unix socket)\n",
    "        :param compression\n",
    "        \"\"\"\n",
    "        Core.Processlib.SinkTaskBase.__init__(self)\n",
    "        self.client = memcache.Client([(server, port)])\n",
    "        self.compression = compression\n",
    "\n",
    "    def process(self, img) :\n",
    "        \"\"\"\n",
    "        Process a frame\n",
    "        \"\"\"\n",
    "        \n",
    "        i = Id(\"id00\", \"basler\", scan, img.frameNumber, img.timestamp, img.buffer.shape, str(img.buffer.dtype), self.compression)\n",
    "        if self.compression:\n",
    "            raw = blosc.compress(img.buffer.data, img.buffer.dtype.itemsize, shuffle=2, cname=\"lz4\")\n",
    "        else:\n",
    "            raw = img.buffer.to_bytes()\n",
    "        key = str(i).replace(\" \", \"\")\n",
    "        self.client.set(key, raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<AcquisitionStatus=AcqRunning, ImageCounters=<LastImageAcquired=28, LastBaseImageReady=28, LastImageReady=28, LastImageSaved=-1, LastCounterReady=-1>\n"
     ]
    }
   ],
   "source": [
    "print(ctrl.getStatus())\n",
    "ctrl.stopAcq()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "scan += 1 \n",
    "extMgr = ctrl.externalOperation()\n",
    "myOp = extMgr.addOp(Core.USER_SINK_TASK, \"memcached\", 0)\n",
    "memcacheTask = MemcachedSink(\"localhost\", 11211, \"blosc\")\n",
    "myOp.setSinkTask(memcacheTask)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<AcquisitionStatus=AcqRunning, ImageCounters=<LastImageAcquired=-1, LastBaseImageReady=-1, LastImageReady=-1, LastImageSaved=-1, LastCounterReady=-1>\n"
     ]
    }
   ],
   "source": [
    "acq = ctrl.acquisition()\n",
    "acq.setAcqNbFrames(0)\n",
    "acq.setAcqExpoTime(EXPO)\n",
    "ctrl.prepareAcq()\n",
    "ctrl.startAcq()\n",
    "print(ctrl.getStatus())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('localhost:11211 (11211)',\n",
       "  {'5:chunk_size': '240',\n",
       "   '5:chunks_per_page': '4369',\n",
       "   '5:total_pages': '3',\n",
       "   '5:total_chunks': '13107',\n",
       "   '5:used_chunks': '11405',\n",
       "   '5:free_chunks': '1702',\n",
       "   '5:free_chunks_end': '0',\n",
       "   '5:mem_requested': '2637427',\n",
       "   '5:get_hits': '0',\n",
       "   '5:cmd_set': '0',\n",
       "   '5:delete_hits': '0',\n",
       "   '5:incr_hits': '0',\n",
       "   '5:decr_hits': '0',\n",
       "   '5:cas_hits': '0',\n",
       "   '5:cas_badval': '0',\n",
       "   '5:touch_hits': '0',\n",
       "   '37:chunk_size': '315872',\n",
       "   '37:chunks_per_page': '3',\n",
       "   '37:total_pages': '3802',\n",
       "   '37:total_chunks': '11406',\n",
       "   '37:used_chunks': '11405',\n",
       "   '37:free_chunks': '1',\n",
       "   '37:free_chunks_end': '0',\n",
       "   '37:mem_requested': '3481861680',\n",
       "   '37:get_hits': '0',\n",
       "   '37:cmd_set': '0',\n",
       "   '37:delete_hits': '0',\n",
       "   '37:incr_hits': '0',\n",
       "   '37:decr_hits': '0',\n",
       "   '37:cas_hits': '0',\n",
       "   '37:cas_badval': '0',\n",
       "   '37:touch_hits': '0',\n",
       "   '39:chunk_size': '524288',\n",
       "   '39:chunks_per_page': '2',\n",
       "   '39:total_pages': '5703',\n",
       "   '39:total_chunks': '11406',\n",
       "   '39:used_chunks': '11405',\n",
       "   '39:free_chunks': '1',\n",
       "   '39:free_chunks_end': '0',\n",
       "   '39:mem_requested': '5979504640',\n",
       "   '39:get_hits': '24004',\n",
       "   '39:cmd_set': '11416',\n",
       "   '39:delete_hits': '0',\n",
       "   '39:incr_hits': '0',\n",
       "   '39:decr_hits': '0',\n",
       "   '39:cas_hits': '0',\n",
       "   '39:cas_badval': '0',\n",
       "   '39:touch_hits': '0',\n",
       "   'active_slabs': '3',\n",
       "   'total_malloced': '9969860608'})]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "memcacheTask.client.get_stats(\"slabs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "ctrl.stopAcq()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "memcacheTask.client.get(\"Id(beamline='id00',camera='basler',scan=1,frame=4149,timestamp=340.0730359554291,shape=(966,1296),dtype='uint16',comp='blosc')\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "memcacheTask.client.get(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "memcacheTask.client.get_slab_stats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "memcacheTask.client.get_stats(\"cachedump 39 10\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "memcacheTask.client.get(\"Id(beamline='id00',camera='basler',scan=1,frame=3836,timestamp=157.0424518585205,shape=(966,1296),dtype='uint16',comp='blosc')\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "memcacheTask.client.get_stats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "img = ctrl.ReadImage()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[25, 29, 24, ..., 18,  7, 14],\n",
       "       [25, 25, 24, ..., 17,  9, 18],\n",
       "       [12, 25, 21, ...,  8,  6, 15],\n",
       "       ...,\n",
       "       [30, 17, 27, ..., 21, 11, 12],\n",
       "       [28, 29, 27, ..., 10, 19, 16],\n",
       "       [26, 30, 16, ..., 12, 16, 11]], dtype=uint16)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img.buffer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}