Skip to content

Instantly share code, notes, and snippets.

@andreped

andreped/custom_python_process_object_example.ipynb

Last active September 12, 2023 06:11
Show Gist options
  • Save andreped/6690d32b3e9673c4b55538bad24bf7f6 to your computer and use it in GitHub Desktop.
Save andreped/6690d32b3e9673c4b55538bad24bf7f6 to your computer and use it in GitHub Desktop.
Download ZIP
custom_python_process_object_example.ipynb
Display the source blob
Display the rendered blob
Raw
custom_python_process_object_example.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/andreped/6690d32b3e9673c4b55538bad24bf7f6/custom_python_process_object_example.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "YJBII5KZUE1X",
"outputId": "814a3829-fafa-4711-9ef6-a31f438871c3"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\r0% [Working]\r \rGet:1 http://security.ubuntu.com/ubuntu jammy-security InRelease [110 kB]\n",
"\r0% [Waiting for headers] [1 InRelease 5,481 B/110 kB 5%] [Waiting for headers] \r \rGet:2 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease [3,626 B]\n",
"\r0% [Waiting for headers] [1 InRelease 14.2 kB/110 kB 13%] [2 InRelease 3,626 B/\r0% [Waiting for headers] [1 InRelease 14.2 kB/110 kB 13%] [Connecting to ppa.la\r \rHit:3 http://archive.ubuntu.com/ubuntu jammy InRelease\n",
"\r0% [Waiting for headers] [1 InRelease 14.2 kB/110 kB 13%] [Connecting to ppa.la\r \rHit:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 InRelease\n",
"Get:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [119 kB]\n",
"Get:6 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [109 kB]\n",
"Get:7 https://ppa.launchpadcontent.net/c2d4u.team/c2d4u4.0+/ubuntu jammy InRelease [18.1 kB]\n",
"Get:8 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [965 kB]\n",
"Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,233 kB]\n",
"Get:10 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [1,064 kB]\n",
"Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [1,085 kB]\n",
"Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,254 kB]\n",
"Get:13 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [28.1 kB]\n",
"Hit:14 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease\n",
"Hit:15 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n",
"Hit:16 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n",
"Get:17 https://ppa.launchpadcontent.net/c2d4u.team/c2d4u4.0+/ubuntu jammy/main Sources [2,180 kB]\n",
"Get:18 https://ppa.launchpadcontent.net/c2d4u.team/c2d4u4.0+/ubuntu jammy/main amd64 Packages [1,119 kB]\n",
"Fetched 9,287 kB in 5s (1,943 kB/s)\n",
"Reading package lists... Done\n",
"Reading package lists... Done\n",
"Building dependency tree... Done\n",
"Reading state information... Done\n",
"Calculating upgrade... Done\n",
"The following packages have been kept back:\n",
" libcudnn8 libcudnn8-dev libnccl-dev libnccl2\n",
"The following packages will be upgraded:\n",
" base-files binutils binutils-common binutils-x86-64-linux-gnu cuda-keyring\n",
" cuda-toolkit-config-common libbinutils libcap2 libctf-nobfd0 libctf0\n",
" libldap-2.5-0 linux-libc-dev\n",
"12 upgraded, 0 newly installed, 0 to remove and 4 not upgraded.\n",
"Need to get 5,039 kB of archives.\n",
"After this operation, 47.1 kB of additional disk space will be used.\n",
"Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 base-files amd64 12ubuntu4.4 [62.6 kB]\n",
"Get:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 cuda-keyring 1.1-1 [4,332 B]\n",
"Get:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 cuda-toolkit-config-common 12.2.140-1 [16.3 kB]\n",
"Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libcap2 amd64 1:2.44-1ubuntu0.22.04.1 [18.3 kB]\n",
"Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libctf0 amd64 2.38-4ubuntu2.3 [103 kB]\n",
"Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libctf-nobfd0 amd64 2.38-4ubuntu2.3 [107 kB]\n",
"Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 binutils-x86-64-linux-gnu amd64 2.38-4ubuntu2.3 [2,327 kB]\n",
"Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libbinutils amd64 2.38-4ubuntu2.3 [662 kB]\n",
"Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 binutils amd64 2.38-4ubuntu2.3 [3,190 B]\n",
"Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 binutils-common amd64 2.38-4ubuntu2.3 [222 kB]\n",
"Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libldap-2.5-0 amd64 2.5.16+dfsg-0ubuntu0.22.04.1 [183 kB]\n",
"Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-libc-dev amd64 5.15.0-83.92 [1,330 kB]\n",
"Fetched 5,039 kB in 0s (13.2 MB/s)\n",
"(Reading database ... 120901 files and directories currently installed.)\n",
"Preparing to unpack .../base-files_12ubuntu4.4_amd64.deb ...\n",
"Unpacking base-files (12ubuntu4.4) over (12ubuntu4.3) ...\n",
"Setting up base-files (12ubuntu4.4) ...\n",
"Installing new version of config file /etc/issue ...\n",
"Installing new version of config file /etc/issue.net ...\n",
"Installing new version of config file /etc/lsb-release ...\n",
"(Reading database ... 120901 files and directories currently installed.)\n",
"Preparing to unpack .../libcap2_1%3a2.44-1ubuntu0.22.04.1_amd64.deb ...\n",
"Unpacking libcap2:amd64 (1:2.44-1ubuntu0.22.04.1) over (1:2.44-1build3) ...\n",
"Setting up libcap2:amd64 (1:2.44-1ubuntu0.22.04.1) ...\n",
"(Reading database ... 120901 files and directories currently installed.)\n",
"Preparing to unpack .../0-libctf0_2.38-4ubuntu2.3_amd64.deb ...\n",
"Unpacking libctf0:amd64 (2.38-4ubuntu2.3) over (2.38-4ubuntu2.2) ...\n",
"Preparing to unpack .../1-libctf-nobfd0_2.38-4ubuntu2.3_amd64.deb ...\n",
"Unpacking libctf-nobfd0:amd64 (2.38-4ubuntu2.3) over (2.38-4ubuntu2.2) ...\n",
"Preparing to unpack .../2-binutils-x86-64-linux-gnu_2.38-4ubuntu2.3_amd64.deb ...\n",
"Unpacking binutils-x86-64-linux-gnu (2.38-4ubuntu2.3) over (2.38-4ubuntu2.2) ...\n",
"Preparing to unpack .../3-libbinutils_2.38-4ubuntu2.3_amd64.deb ...\n",
"Unpacking libbinutils:amd64 (2.38-4ubuntu2.3) over (2.38-4ubuntu2.2) ...\n",
"Preparing to unpack .../4-binutils_2.38-4ubuntu2.3_amd64.deb ...\n",
"Unpacking binutils (2.38-4ubuntu2.3) over (2.38-4ubuntu2.2) ...\n",
"Preparing to unpack .../5-binutils-common_2.38-4ubuntu2.3_amd64.deb ...\n",
"Unpacking binutils-common:amd64 (2.38-4ubuntu2.3) over (2.38-4ubuntu2.2) ...\n",
"Preparing to unpack .../6-cuda-keyring_1.1-1_all.deb ...\n",
"Unpacking cuda-keyring (1.1-1) over (1.0-1) ...\n",
"Preparing to unpack .../7-cuda-toolkit-config-common_12.2.140-1_all.deb ...\n",
"Unpacking cuda-toolkit-config-common (12.2.140-1) over (12.1.105-1) ...\n",
"Preparing to unpack .../8-libldap-2.5-0_2.5.16+dfsg-0ubuntu0.22.04.1_amd64.deb ...\n",
"Unpacking libldap-2.5-0:amd64 (2.5.16+dfsg-0ubuntu0.22.04.1) over (2.5.14+dfsg-0ubuntu0.22.04.2) ...\n",
"Preparing to unpack .../9-linux-libc-dev_5.15.0-83.92_amd64.deb ...\n",
"Unpacking linux-libc-dev:amd64 (5.15.0-83.92) over (5.15.0-75.82) ...\n",
"Setting up cuda-toolkit-config-common (12.2.140-1) ...\n",
"Setting up binutils-common:amd64 (2.38-4ubuntu2.3) ...\n",
"Setting up linux-libc-dev:amd64 (5.15.0-83.92) ...\n",
"Setting up libctf-nobfd0:amd64 (2.38-4ubuntu2.3) ...\n",
"Setting up libldap-2.5-0:amd64 (2.5.16+dfsg-0ubuntu0.22.04.1) ...\n",
"Setting up cuda-keyring (1.1-1) ...\n",
"Setting up libbinutils:amd64 (2.38-4ubuntu2.3) ...\n",
"Setting up libctf0:amd64 (2.38-4ubuntu2.3) ...\n",
"Setting up binutils-x86-64-linux-gnu (2.38-4ubuntu2.3) ...\n",
"Setting up binutils (2.38-4ubuntu2.3) ...\n",
"Processing triggers for libc-bin (2.35-0ubuntu3.1) ...\n",
"/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
"\n",
"/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
"\n",
"/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
"\n",
"/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
"\n",
"/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
"\n",
"/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
"\n",
"Processing triggers for man-db (2.10.2-1) ...\n",
"Reading package lists... Done\n",
"Building dependency tree... Done\n",
"Reading state information... Done\n",
"xz-utils is already the newest version (5.2.5-2ubuntu1).\n",
"xz-utils set to manually installed.\n",
"clinfo is already the newest version (3.0.21.02.21-1).\n",
"ocl-icd-libopencl1 is already the newest version (2.2.14-3).\n",
"ocl-icd-libopencl1 set to manually installed.\n",
"apt-utils is already the newest version (2.4.10).\n",
"curl is already the newest version (7.81.0-1ubuntu1.13).\n",
"The following additional packages will be installed:\n",
" python3-setuptools python3-wheel\n",
"Suggested packages:\n",
" python-setuptools-doc\n",
"The following NEW packages will be installed:\n",
" opencl-headers python3-pip python3-setuptools python3-wheel\n",
"0 upgraded, 4 newly installed, 0 to remove and 4 not upgraded.\n",
"Need to get 1,679 kB of archives.\n",
"After this operation, 8,977 kB of additional disk space will be used.\n",
"Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-setuptools all 59.6.0-1.2ubuntu0.22.04.1 [339 kB]\n",
"Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python3-wheel all 0.37.1-2ubuntu0.22.04.1 [32.0 kB]\n",
"Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python3-pip all 22.0.2+dfsg-1ubuntu0.3 [1,305 kB]\n",
"Get:4 http://archive.ubuntu.com/ubuntu jammy/universe amd64 opencl-headers all 3.0~2022.01.04-1 [1,754 B]\n",
"Fetched 1,679 kB in 0s (4,043 kB/s)\n",
"Selecting previously unselected package python3-setuptools.\n",
"(Reading database ... 120901 files and directories currently installed.)\n",
"Preparing to unpack .../python3-setuptools_59.6.0-1.2ubuntu0.22.04.1_all.deb ...\n",
"Unpacking python3-setuptools (59.6.0-1.2ubuntu0.22.04.1) ...\n",
"Selecting previously unselected package python3-wheel.\n",
"Preparing to unpack .../python3-wheel_0.37.1-2ubuntu0.22.04.1_all.deb ...\n",
"Unpacking python3-wheel (0.37.1-2ubuntu0.22.04.1) ...\n",
"Selecting previously unselected package python3-pip.\n",
"Preparing to unpack .../python3-pip_22.0.2+dfsg-1ubuntu0.3_all.deb ...\n",
"Unpacking python3-pip (22.0.2+dfsg-1ubuntu0.3) ...\n",
"Selecting previously unselected package opencl-headers.\n",
"Preparing to unpack .../opencl-headers_3.0~2022.01.04-1_all.deb ...\n",
"Unpacking opencl-headers (3.0~2022.01.04-1) ...\n",
"Setting up python3-setuptools (59.6.0-1.2ubuntu0.22.04.1) ...\n",
"Setting up opencl-headers (3.0~2022.01.04-1) ...\n",
"Setting up python3-wheel (0.37.1-2ubuntu0.22.04.1) ...\n",
"Setting up python3-pip (22.0.2+dfsg-1ubuntu0.3) ...\n",
"Processing triggers for man-db (2.10.2-1) ...\n",
"Reading package lists... Done\n",
"Building dependency tree... Done\n",
"Reading state information... Done\n",
"libcairo2 is already the newest version (1.16.0-5ubuntu2).\n",
"libcairo2 set to manually installed.\n",
"libgl1 is already the newest version (1.4.0-1).\n",
"libopengl0 is already the newest version (1.4.0-1).\n",
"The following additional packages will be installed:\n",
" libusb-1.0-doc\n",
"The following NEW packages will be installed:\n",
" libusb-1.0-0-dev libusb-1.0-doc\n",
"0 upgraded, 2 newly installed, 0 to remove and 4 not upgraded.\n",
"Need to get 259 kB of archives.\n",
"After this operation, 1,940 kB of additional disk space will be used.\n",
"Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libusb-1.0-0-dev amd64 2:1.0.25-1ubuntu2 [76.3 kB]\n",
"Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libusb-1.0-doc all 2:1.0.25-1ubuntu2 [183 kB]\n",
"Fetched 259 kB in 1s (304 kB/s)\n",
"Selecting previously unselected package libusb-1.0-0-dev:amd64.\n",
"(Reading database ... 121766 files and directories currently installed.)\n",
"Preparing to unpack .../libusb-1.0-0-dev_2%3a1.0.25-1ubuntu2_amd64.deb ...\n",
"Unpacking libusb-1.0-0-dev:amd64 (2:1.0.25-1ubuntu2) ...\n",
"Selecting previously unselected package libusb-1.0-doc.\n",
"Preparing to unpack .../libusb-1.0-doc_2%3a1.0.25-1ubuntu2_all.deb ...\n",
"Unpacking libusb-1.0-doc (2:1.0.25-1ubuntu2) ...\n",
"Setting up libusb-1.0-doc (2:1.0.25-1ubuntu2) ...\n",
"Setting up libusb-1.0-0-dev:amd64 (2:1.0.25-1ubuntu2) ...\n"
]
}
],
"source": [
"# install OpenGL\n",
"!apt-get update && apt-get -y upgrade \\\n",
" && apt-get install -y \\\n",
" apt-utils \\\n",
" curl \\\n",
" xz-utils \\\n",
" ocl-icd-libopencl1 \\\n",
" opencl-headers \\\n",
" clinfo \\\n",
" python3-pip \\\n",
" ;\n",
"!mkdir -p /etc/OpenCL/vendors && \\\n",
" echo \"libnvidia-opencl.so.1\" > /etc/OpenCL/vendors/nvidia.icd\n",
"\n",
"# install OpenGL\n",
"!apt-get install -y \\\n",
" libopengl0 \\\n",
" libgl1 \\\n",
" libusb-1.0-0-dev \\\n",
" libcairo2"
]
},
{
"cell_type": "code",
"source": [
"# install pyFAST\n",
"!pip install pyFAST==4.7.1 matplotlib"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "K6yZu93tZvsZ",
"outputId": "5212c0b5-b0c6-4856-d44e-f2516ac9103f"
},
"execution_count": 2,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Collecting pyFAST==4.7.1\n",
" Downloading pyFAST-4.7.1-cp36-abi3-manylinux1_x86_64.whl (385.8 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m385.8/385.8 MB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hRequirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (3.7.1)\n",
"Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from pyFAST==4.7.1) (1.23.5)\n",
"Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.1.0)\n",
"Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (0.11.0)\n",
"Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (4.42.1)\n",
"Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.4.5)\n",
"Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (23.1)\n",
"Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (9.4.0)\n",
"Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (3.1.1)\n",
"Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (2.8.2)\n",
"Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n",
"Installing collected packages: pyFAST\n",
"Successfully installed pyFAST-4.7.1\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"!clinfo"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "VTA4NnBplcQ2",
"outputId": "cca11d70-79d2-4429-8a02-59a58e82b07d"
},
"execution_count": 3,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Number of platforms 1\n",
" Platform Name NVIDIA CUDA\n",
" Platform Vendor NVIDIA Corporation\n",
" Platform Version OpenCL 3.0 CUDA 12.0.151\n",
" Platform Profile FULL_PROFILE\n",
" Platform Extensions cl_khr_global_int32_base_atomics cl_khr_global_int32_extended_atomics cl_khr_local_int32_base_atomics cl_khr_local_int32_extended_atomics cl_khr_fp64 cl_khr_3d_image_writes cl_khr_byte_addressable_store cl_khr_icd cl_nv_compiler_options cl_nv_device_attribute_query cl_nv_pragma_unroll cl_nv_copy_opts cl_nv_create_buffer cl_khr_int64_base_atomics cl_khr_int64_extended_atomics cl_nv_kernel_attribute cl_khr_device_uuid cl_khr_pci_bus_info cl_khr_external_semaphore cl_khr_external_memory cl_khr_external_semaphore_opaque_fd cl_khr_external_memory_opaque_fd\n",
" Platform Extensions with Version cl_khr_global_int32_base_atomics 0x400000 (1.0.0)\n",
" cl_khr_global_int32_extended_atomics 0x400000 (1.0.0)\n",
" cl_khr_local_int32_base_atomics 0x400000 (1.0.0)\n",
" cl_khr_local_int32_extended_atomics 0x400000 (1.0.0)\n",
" cl_khr_fp64 0x400000 (1.0.0)\n",
" cl_khr_3d_image_writes 0x400000 (1.0.0)\n",
" cl_khr_byte_addressable_store 0x400000 (1.0.0)\n",
" cl_khr_icd 0x400000 (1.0.0)\n",
" cl_nv_compiler_options 0x400000 (1.0.0)\n",
" cl_nv_device_attribute_query 0x400000 (1.0.0)\n",
" cl_nv_pragma_unroll 0x400000 (1.0.0)\n",
" cl_nv_copy_opts 0x400000 (1.0.0)\n",
" cl_nv_create_buffer 0x400000 (1.0.0)\n",
" cl_khr_int64_base_atomics 0x400000 (1.0.0)\n",
" cl_khr_int64_extended_atomics 0x400000 (1.0.0)\n",
" cl_nv_kernel_attribute 0x400000 (1.0.0)\n",
" cl_khr_device_uuid 0x400000 (1.0.0)\n",
" cl_khr_pci_bus_info 0x400000 (1.0.0)\n",
" cl_khr_external_semaphore 0x9000 (0.9.0)\n",
" cl_khr_external_memory 0x9000 (0.9.0)\n",
" cl_khr_external_semaphore_opaque_fd 0x9000 (0.9.0)\n",
" cl_khr_external_memory_opaque_fd 0x9000 (0.9.0)\n",
" Platform Numeric Version 0xc00000 (3.0.0)\n",
" Platform Extensions function suffix NV\n",
" Platform Host timer resolution 0ns\n",
"\n",
" Platform Name NVIDIA CUDA\n",
"Number of devices 1\n",
" Device Name Tesla T4\n",
" Device Vendor NVIDIA Corporation\n",
" Device Vendor ID 0x10de\n",
" Device Version OpenCL 3.0 CUDA\n",
" Device UUID ef30eace-4268-4d58-f85d-bdcb4b2895b6\n",
" Driver UUID ef30eace-4268-4d58-f85d-bdcb4b2895b6\n",
" Valid Device LUID No\n",
" Device LUID 6d69-637300000000\n",
" Device Node Mask 0\n",
" Device Numeric Version 0xc00000 (3.0.0)\n",
" Driver Version 525.105.17\n",
" Device OpenCL C Version OpenCL C 1.2 \n",
" Device OpenCL C all versions OpenCL C 0x400000 (1.0.0)\n",
" OpenCL C 0x401000 (1.1.0)\n",
" OpenCL C 0x402000 (1.2.0)\n",
" OpenCL C 0xc00000 (3.0.0)\n",
" Device OpenCL C features __opencl_c_fp64 0xc00000 (3.0.0)\n",
" __opencl_c_images 0xc00000 (3.0.0)\n",
" __opencl_c_int64 0xc00000 (3.0.0)\n",
" __opencl_c_3d_image_writes 0xc00000 (3.0.0)\n",
" Latest comfornace test passed v2022-10-05-00\n",
" Device Type GPU\n",
" Device Topology (NV) PCI-E, 0000:00:00.4\n",
" Device Profile FULL_PROFILE\n",
" Device Available Yes\n",
" Compiler Available Yes\n",
" Linker Available Yes\n",
" Max compute units 40\n",
" Max clock frequency 1590MHz\n",
" Compute Capability (NV) 7.5\n",
" Device Partition (core)\n",
" Max number of sub-devices 1\n",
" Supported partition types None\n",
" Supported affinity domains (n/a)\n",
" Max work item dimensions 3\n",
" Max work item sizes 1024x1024x64\n",
" Max work group size 1024\n",
" Preferred work group size multiple (device) 32\n",
" Preferred work group size multiple (kernel) 32\n",
" Warp size (NV) 32\n",
" Max sub-groups per work group 0\n",
" Preferred / native vector sizes \n",
" char 1 / 1 \n",
" short 1 / 1 \n",
" int 1 / 1 \n",
" long 1 / 1 \n",
" half 0 / 0 (n/a)\n",
" float 1 / 1 \n",
" double 1 / 1 (cl_khr_fp64)\n",
" Half-precision Floating-point support (n/a)\n",
" Single-precision Floating-point support (core)\n",
" Denormals Yes\n",
" Infinity and NANs Yes\n",
" Round to nearest Yes\n",
" Round to zero Yes\n",
" Round to infinity Yes\n",
" IEEE754-2008 fused multiply-add Yes\n",
" Support is emulated in software No\n",
" Correctly-rounded divide and sqrt operations Yes\n",
" Double-precision Floating-point support (cl_khr_fp64)\n",
" Denormals Yes\n",
" Infinity and NANs Yes\n",
" Round to nearest Yes\n",
" Round to zero Yes\n",
" Round to infinity Yes\n",
" IEEE754-2008 fused multiply-add Yes\n",
" Support is emulated in software No\n",
" Address bits 64, Little-Endian\n",
" Global memory size 15835398144 (14.75GiB)\n",
" Error Correction support Yes\n",
" Max memory allocation 3958849536 (3.687GiB)\n",
" Unified memory for Host and Device No\n",
" Integrated memory (NV) No\n",
" Shared Virtual Memory (SVM) capabilities (core)\n",
" Coarse-grained buffer sharing Yes\n",
" Fine-grained buffer sharing No\n",
" Fine-grained system sharing No\n",
" Atomics No\n",
" Minimum alignment for any data type 128 bytes\n",
" Alignment of base address 4096 bits (512 bytes)\n",
" Preferred alignment for atomics \n",
" SVM 0 bytes\n",
" Global 0 bytes\n",
" Local 0 bytes\n",
" Atomic memory capabilities relaxed, work-group scope\n",
" Atomic fence capabilities relaxed, acquire/release, work-group scope\n",
" Max size for global variable 0\n",
" Preferred total size of global vars 0\n",
" Global Memory cache type Read/Write\n",
" Global Memory cache size 1310720 (1.25MiB)\n",
" Global Memory cache line size 128 bytes\n",
" Image support Yes\n",
" Max number of samplers per kernel 32\n",
" Max size for 1D images from buffer 268435456 pixels\n",
" Max 1D or 2D image array size 2048 images\n",
" Max 2D image size 32768x32768 pixels\n",
" Max 3D image size 16384x16384x16384 pixels\n",
" Max number of read image args 256\n",
" Max number of write image args 32\n",
" Max number of read/write image args 0\n",
" Pipe support No\n",
" Max number of pipe args 0\n",
" Max active pipe reservations 0\n",
" Max pipe packet size 0\n",
" Local memory type Local\n",
" Local memory size 49152 (48KiB)\n",
" Registers per block (NV) 65536\n",
" Max number of constant args 9\n",
" Max constant buffer size 65536 (64KiB)\n",
" Generic address space support No\n",
" Max size of kernel argument 4352 (4.25KiB)\n",
" Queue properties (on host) \n",
" Out-of-order execution Yes\n",
" Profiling Yes\n",
" Device enqueue capabilities (n/a)\n",
" Queue properties (on device) \n",
" Out-of-order execution No\n",
" Profiling No\n",
" Preferred size 0\n",
" Max size 0\n",
" Max queues on device 0\n",
" Max events on device 0\n",
" Prefer user sync for interop No\n",
" Profiling timer resolution 1000ns\n",
" Execution capabilities \n",
" Run OpenCL kernels Yes\n",
" Run native kernels No\n",
" Non-uniform work-groups No\n",
" Work-group collective functions No\n",
" Sub-group independent forward progress No\n",
" Kernel execution timeout (NV) No\n",
" Concurrent copy and kernel execution (NV) Yes\n",
" Number of async copy engines 3\n",
" IL version (n/a)\n",
" ILs with version <printDeviceInfo:186: get CL_DEVICE_ILS_WITH_VERSION : error -30>\n",
" printf() buffer size 1048576 (1024KiB)\n",
" Built-in kernels (n/a)\n",
" Built-in kernels with version <printDeviceInfo:190: get CL_DEVICE_BUILT_IN_KERNELS_WITH_VERSION : error -30>\n",
" Device Extensions cl_khr_global_int32_base_atomics cl_khr_global_int32_extended_atomics cl_khr_local_int32_base_atomics cl_khr_local_int32_extended_atomics cl_khr_fp64 cl_khr_3d_image_writes cl_khr_byte_addressable_store cl_khr_icd cl_nv_compiler_options cl_nv_device_attribute_query cl_nv_pragma_unroll cl_nv_copy_opts cl_nv_create_buffer cl_khr_int64_base_atomics cl_khr_int64_extended_atomics cl_nv_kernel_attribute cl_khr_device_uuid cl_khr_pci_bus_info cl_khr_external_semaphore cl_khr_external_memory cl_khr_external_semaphore_opaque_fd cl_khr_external_memory_opaque_fd\n",
" Device Extensions with Version cl_khr_global_int32_base_atomics 0x400000 (1.0.0)\n",
" cl_khr_global_int32_extended_atomics 0x400000 (1.0.0)\n",
" cl_khr_local_int32_base_atomics 0x400000 (1.0.0)\n",
" cl_khr_local_int32_extended_atomics 0x400000 (1.0.0)\n",
" cl_khr_fp64 0x400000 (1.0.0)\n",
" cl_khr_3d_image_writes 0x400000 (1.0.0)\n",
" cl_khr_byte_addressable_store 0x400000 (1.0.0)\n",
" cl_khr_icd 0x400000 (1.0.0)\n",
" cl_nv_compiler_options 0x400000 (1.0.0)\n",
" cl_nv_device_attribute_query 0x400000 (1.0.0)\n",
" cl_nv_pragma_unroll 0x400000 (1.0.0)\n",
" cl_nv_copy_opts 0x400000 (1.0.0)\n",
" cl_nv_create_buffer 0x400000 (1.0.0)\n",
" cl_khr_int64_base_atomics 0x400000 (1.0.0)\n",
" cl_khr_int64_extended_atomics 0x400000 (1.0.0)\n",
" cl_nv_kernel_attribute 0x400000 (1.0.0)\n",
" cl_khr_device_uuid 0x400000 (1.0.0)\n",
" cl_khr_pci_bus_info 0x400000 (1.0.0)\n",
" cl_khr_external_semaphore 0x9000 (0.9.0)\n",
" cl_khr_external_memory 0x9000 (0.9.0)\n",
" cl_khr_external_semaphore_opaque_fd 0x9000 (0.9.0)\n",
" cl_khr_external_memory_opaque_fd 0x9000 (0.9.0)\n",
"\n",
"NULL platform behavior\n",
" clGetPlatformInfo(NULL, CL_PLATFORM_NAME, ...) No platform\n",
" clGetDeviceIDs(NULL, CL_DEVICE_TYPE_ALL, ...) No platform\n",
" clCreateContext(NULL, ...) [default] No platform\n",
" clCreateContext(NULL, ...) [other] Success [NV]\n",
" clCreateContextFromType(NULL, CL_DEVICE_TYPE_DEFAULT) No platform\n",
" clCreateContextFromType(NULL, CL_DEVICE_TYPE_CPU) No devices found in platform\n",
" clCreateContextFromType(NULL, CL_DEVICE_TYPE_GPU) No platform\n",
" clCreateContextFromType(NULL, CL_DEVICE_TYPE_ACCELERATOR) No devices found in platform\n",
" clCreateContextFromType(NULL, CL_DEVICE_TYPE_CUSTOM) Invalid device type for platform\n",
" clCreateContextFromType(NULL, CL_DEVICE_TYPE_ALL) No platform\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# download test WSI\n",
"!wget https://folk.ntnu.no/andpeder/FastPathology/test_data/WSI/CMU-1.svs"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "mVVJS7ODwWVW",
"outputId": "7b97e178-0929-4c19-f323-ecb678980436"
},
"execution_count": 4,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"--2023-09-12 06:09:32-- https://folk.ntnu.no/andpeder/FastPathology/test_data/WSI/CMU-1.svs\n",
"Resolving folk.ntnu.no (folk.ntnu.no)... 129.241.56.95, 2001:700:300:3::95\n",
"Connecting to folk.ntnu.no (folk.ntnu.no)|129.241.56.95|:443... connected.\n",
"HTTP request sent, awaiting response... 200 OK\n",
"Length: 177552579 (169M) [image/tiff]\n",
"Saving to: ‘CMU-1.svs’\n",
"\n",
"CMU-1.svs 100%[===================>] 169.33M 22.0MB/s in 8.4s \n",
"\n",
"2023-09-12 06:09:41 (20.1 MB/s) - ‘CMU-1.svs’ saved [177552579/177552579]\n",
"\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"import fast\n",
"import numpy as np\n",
"import os\n",
"\n",
"\n",
"# enable verbose in FAST\n",
"fast.Reporter.setGlobalReportMethod(fast.Reporter.COUT)\n",
"\n",
"\n",
"class TwoInputCustomPythonPO(fast.PythonProcessObject):\n",
" def __init__(self, value=2):\n",
" super().__init__()\n",
" self.createInputPort(0)\n",
" self.createInputPort(1)\n",
" self.createOutputPort(0)\n",
" self.value = value # not actually used in this PO, but now you know how to add arguments\n",
"\n",
" def execute(self):\n",
" pred = self.getInputData(0)\n",
" new_array = np.asarray(pred)\n",
" patch = self.getInputData(1) # not actually used in this PO, just as an example\n",
"\n",
" # make some algorithm here, e.g., flips the channels and argmax to get uint8 result (one uint assigned to each pixel)\n",
" new_array = new_array[..., ::-1]\n",
" new_array = np.argmax(new_array, axis=-1, keepdims=True).astype(\"uint8\")\n",
"\n",
" # Create new fast image and add as output\n",
" new_output_image = fast.Image.createFromArray(new_array)\n",
" new_output_image.setSpacing(pred.getSpacing())\n",
" self.addOutputData(0, new_output_image)\n",
"\n",
"\n",
"# Download a nuclei segmentation model from the DataHub\n",
"model = fast.DataHub().download('nuclei-segmentation-model')\n",
"\n",
"# hyperparams\n",
"patchSize = 256\n",
"magnification = 1\n",
"overlapPercent = 0.0\n",
"scaleFactor = 1.0 / 255.0\n",
"\n",
"# pipeline with custom PO\n",
"importer = fast.WholeSlideImageImporter.create(\"./CMU-1.svs\")\n",
"\n",
"tissueSegmentation = fast.TissueSegmentation.create(threshold=70).connect(importer)\n",
"\n",
"patchGenerator = fast.PatchGenerator.create(patchSize,\n",
" patchSize,\n",
" magnification=magnification,\n",
" maskThreshold=0.02,\n",
" overlapPercent=overlapPercent)\\\n",
" .connect(0, importer)\\\n",
" .connect(1, tissueSegmentation)\n",
"\n",
"#patchStream = fast.DataStream(patchGenerator)\n",
"\n",
"network = fast.NeuralNetwork.create(scaleFactor=scaleFactor, modelFilename=model.paths[0] + '/high_res_nuclei_unet.onnx')\\\n",
" .connect(patchGenerator)\n",
"\n",
"post_process = TwoInputCustomPythonPO.create().connect(0, network).connect(1, patchGenerator)\n",
"\n",
"stitcher = fast.PatchStitcher.create().connect(post_process)\n",
"\n",
"finished = fast.RunUntilFinished.create().connect(stitcher)\n",
"\n",
"exporter = fast.TIFFImagePyramidExporter.create(\"./result.tiff\")\\\n",
" .connect(finished)\\\n",
" .run()\n",
"\n",
"print(os.path.exists(\"./result.tiff\"))"
],
"metadata": {
"id": "LCwiqDA0HeO0"
},
"execution_count": null,
"outputs": []
}
],
"metadata": {
"colab": {
"provenance": [],
"gpuType": "T4",
"authorship_tag": "ABX9TyNDx07LFoeOVyVVa/Ay2HMC",
"include_colab_link": true
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
},
"accelerator": "GPU"
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment