Skip to content

Instantly share code, notes, and snippets.

@kif

kif/Lima_Memcached_demo.ipynb

Created November 22, 2019 12:36
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save kif/b47d4f9202b265fd95a07fc9db5104d4 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save kif/b47d4f9202b265fd95a07fc9db5104d4 to your computer and use it in GitHub Desktop.
Download ZIP
Memcached client for Lima
Display the source blob
Display the rendered blob
Raw
Lima_Memcached_demo.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"ename": "ModuleNotFoundError",
"evalue": "No module named 'Lima'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-2-8933fadce3b0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mLima\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'Lima'"
]
}
],
"source": [
"import Lima"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"ename": "ModuleNotFoundError",
"evalue": "No module named 'Lima'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-3-f0cddb6115ce>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mLima\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mCore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'Lima'"
]
}
],
"source": [
"from Lima import Core"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"from Lima import Basler"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"import blosc"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"IP = \"192.168.5.19\"\n",
"fps = 30\n",
"EXPO = 0.001 #sec\n",
"scan = 0"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"ename": "Exception",
"evalue": "Hardware: Camera(/opt/conda/conda-bld/lima-camera-basler_1549640215705/work/src/BaslerCamera.cpp, 284): Error: Failed to discover GigE device '192.168.5.19:3956'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mException\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-6-f99c8f65ed8e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcam\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBasler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCamera\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mIP\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0miface\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBasler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mInterface\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcam\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mctrl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCtControl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miface\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0macq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mctrl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquisition\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0macq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetAcqNbFrames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mException\u001b[0m: Hardware: Camera(/opt/conda/conda-bld/lima-camera-basler_1549640215705/work/src/BaslerCamera.cpp, 284): Error: Failed to discover GigE device '192.168.5.19:3956'"
]
}
],
"source": [
"cam = Basler.Camera(IP)\n",
"iface = Basler.Interface(cam)\n",
"ctrl = Core.CtControl(iface)\n",
"acq = ctrl.acquisition()\n",
"acq.setAcqNbFrames(0)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<AcquisitionStatus=AcqRunning, ImageCounters=<LastImageAcquired=-1, LastBaseImageReady=-1, LastImageReady=-1, LastImageSaved=-1, LastCounterReady=-1>\n"
]
}
],
"source": [
"ctrl.prepareAcq()\n",
"ctrl.startAcq()\n",
"print(ctrl.getStatus())\n",
"import time\n",
"time.sleep(1)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"img = ctrl.ReadImage()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"779 µs ± 41.7 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n"
]
}
],
"source": [
"%timeit len(blosc.compress(img.buffer.data, 2, shuffle=2, cname=\"lz4\"))/img.buffer.nbytes"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"4"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"img.frameNumber"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[ 76, 88, 72, ..., 93, 86, 87],\n",
" [ 65, 109, 66, ..., 83, 122, 70],\n",
" [ 77, 74, 71, ..., 80, 86, 90],\n",
" ...,\n",
" [127, 136, 122, ..., 126, 144, 138],\n",
" [138, 128, 130, ..., 140, 156, 152],\n",
" [134, 148, 128, ..., 157, 168, 157]], dtype=uint16)"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"img.buffer"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<memory at 0x7f8ec5cd8120>"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"img.buffer.data"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"from collections import namedtuple\n",
"Id = namedtuple(\"Id\", \"beamline camera scan frame timestamp shape dtype comp\")\n",
"i=Id(\"id00\", \"basler\", 1, img.frameNumber, img.timestamp, img.buffer.shape, str(img.buffer.dtype), \"blosc\")"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"130"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(str(i))"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\"Id(beamline='id00',camera='basler',scan=1,frame=4,timestamp=5.03251314163208,shape=(966,1296),dtype='uint16',comp='blosc')\""
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"str(i).replace(\" \", \"\")"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Id(beamline='id00', camera='basler', scan=1, frame=11, timestamp=12.032986164093018, shape=(966, 1296), dtype='uint16', comp='blosc')"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\n",
"i=Id(\"id00\", \"basler\", 1, img.frameNumber, img.timestamp, img.buffer.shape, str(img.buffer.dtype), \"blosc\")\n",
"i"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"import memcache\n",
"#client = memcache.Client(\"localhost\", 11211)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"class MemcachedSink(Core.Processlib.SinkTaskBase):\n",
" def __init__(self, server, port, compression=None):\n",
" \"\"\"\n",
" :param server_port: port to which to connect (can be a unix socket)\n",
" :param compression\n",
" \"\"\"\n",
" Core.Processlib.SinkTaskBase.__init__(self)\n",
" self.client = memcache.Client([(server, port)])\n",
" self.compression = compression\n",
"\n",
" def process(self, img) :\n",
" \"\"\"\n",
" Process a frame\n",
" \"\"\"\n",
" \n",
" i = Id(\"id00\", \"basler\", scan, img.frameNumber, img.timestamp, img.buffer.shape, str(img.buffer.dtype), self.compression)\n",
" if self.compression:\n",
" raw = blosc.compress(img.buffer.data, img.buffer.dtype.itemsize, shuffle=2, cname=\"lz4\")\n",
" else:\n",
" raw = img.buffer.to_bytes()\n",
" key = str(i).replace(\" \", \"\")\n",
" self.client.set(key, raw)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<AcquisitionStatus=AcqRunning, ImageCounters=<LastImageAcquired=28, LastBaseImageReady=28, LastImageReady=28, LastImageSaved=-1, LastCounterReady=-1>\n"
]
}
],
"source": [
"print(ctrl.getStatus())\n",
"ctrl.stopAcq()"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"scan += 1 \n",
"extMgr = ctrl.externalOperation()\n",
"myOp = extMgr.addOp(Core.USER_SINK_TASK, \"memcached\", 0)\n",
"memcacheTask = MemcachedSink(\"localhost\", 11211, \"blosc\")\n",
"myOp.setSinkTask(memcacheTask)\n"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<AcquisitionStatus=AcqRunning, ImageCounters=<LastImageAcquired=-1, LastBaseImageReady=-1, LastImageReady=-1, LastImageSaved=-1, LastCounterReady=-1>\n"
]
}
],
"source": [
"acq = ctrl.acquisition()\n",
"acq.setAcqNbFrames(0)\n",
"acq.setAcqExpoTime(EXPO)\n",
"ctrl.prepareAcq()\n",
"ctrl.startAcq()\n",
"print(ctrl.getStatus())\n"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[('localhost:11211 (11211)',\n",
" {'5:chunk_size': '240',\n",
" '5:chunks_per_page': '4369',\n",
" '5:total_pages': '3',\n",
" '5:total_chunks': '13107',\n",
" '5:used_chunks': '11405',\n",
" '5:free_chunks': '1702',\n",
" '5:free_chunks_end': '0',\n",
" '5:mem_requested': '2637427',\n",
" '5:get_hits': '0',\n",
" '5:cmd_set': '0',\n",
" '5:delete_hits': '0',\n",
" '5:incr_hits': '0',\n",
" '5:decr_hits': '0',\n",
" '5:cas_hits': '0',\n",
" '5:cas_badval': '0',\n",
" '5:touch_hits': '0',\n",
" '37:chunk_size': '315872',\n",
" '37:chunks_per_page': '3',\n",
" '37:total_pages': '3802',\n",
" '37:total_chunks': '11406',\n",
" '37:used_chunks': '11405',\n",
" '37:free_chunks': '1',\n",
" '37:free_chunks_end': '0',\n",
" '37:mem_requested': '3481861680',\n",
" '37:get_hits': '0',\n",
" '37:cmd_set': '0',\n",
" '37:delete_hits': '0',\n",
" '37:incr_hits': '0',\n",
" '37:decr_hits': '0',\n",
" '37:cas_hits': '0',\n",
" '37:cas_badval': '0',\n",
" '37:touch_hits': '0',\n",
" '39:chunk_size': '524288',\n",
" '39:chunks_per_page': '2',\n",
" '39:total_pages': '5703',\n",
" '39:total_chunks': '11406',\n",
" '39:used_chunks': '11405',\n",
" '39:free_chunks': '1',\n",
" '39:free_chunks_end': '0',\n",
" '39:mem_requested': '5979504640',\n",
" '39:get_hits': '24004',\n",
" '39:cmd_set': '11416',\n",
" '39:delete_hits': '0',\n",
" '39:incr_hits': '0',\n",
" '39:decr_hits': '0',\n",
" '39:cas_hits': '0',\n",
" '39:cas_badval': '0',\n",
" '39:touch_hits': '0',\n",
" 'active_slabs': '3',\n",
" 'total_malloced': '9969860608'})]"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"memcacheTask.client.get_stats(\"slabs\")"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
"ctrl.stopAcq()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"memcacheTask.client.get(\"Id(beamline='id00',camera='basler',scan=1,frame=4149,timestamp=340.0730359554291,shape=(966,1296),dtype='uint16',comp='blosc')\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"memcacheTask.client.get(\"\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"memcacheTask.client.get_slab_stats()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"memcacheTask.client.get_stats(\"cachedump 39 10\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"memcacheTask.client.get(\"Id(beamline='id00',camera='basler',scan=1,frame=3836,timestamp=157.0424518585205,shape=(966,1296),dtype='uint16',comp='blosc')\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"memcacheTask.client.get_stats()"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
"img = ctrl.ReadImage()"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[25, 29, 24, ..., 18, 7, 14],\n",
" [25, 25, 24, ..., 17, 9, 18],\n",
" [12, 25, 21, ..., 8, 6, 15],\n",
" ...,\n",
" [30, 17, 27, ..., 21, 11, 12],\n",
" [28, 29, 27, ..., 10, 19, 16],\n",
" [26, 30, 16, ..., 12, 16, 11]], dtype=uint16)"
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"img.buffer"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment