| import subprocess, getpass | |
| def get_gpu_usage(): | |
| """ | |
| Returns a dict which contains information about memory usage for each GPU. | |
| In the following output, the GPU with id "0" uses 5774 MB of 16280 MB. | |
| 253 MB are used by other users, which means that we are using 5774 - 253 MB. | |
| { |
| import asyncpg | |
| async def create_pool(): | |
| pool = await asyncpg.create_pool( | |
| database="research", | |
| user="jwickens", | |
| setup=setup_connection, | |
| min_size=32, | |
| max_size=32 | |
| ) |
| """Extract nested values from a JSON tree.""" | |
| def json_extract(obj, key): | |
| """Recursively fetch values from nested JSON.""" | |
| arr = [] | |
| def extract(obj, arr, key): | |
| """Recursively search for values of key in JSON tree.""" | |
| if isinstance(obj, dict): |
| # WARNING: These steps seem to not work anymore! | |
| #!/bin/bash | |
| # Purge existign CUDA first | |
| sudo apt --purge remove "cublas*" "cuda*" | |
| sudo apt --purge remove "nvidia*" | |
| # Install CUDA Toolkit 10 | |
| wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.0.130-1_amd64.deb |
| def _to_one_hot(y, num_classes): | |
| scatter_dim = len(y.size()) | |
| y_tensor = y.view(*y.size(), -1) | |
| zeros = torch.zeros(*y.size(), num_classes, dtype=y.dtype) | |
| return zeros.scatter(scatter_dim, y_tensor, 1) | |
| print(_to_one_hot(torch.as_tensor([2, 4, 7]), num_classes=10)) | |
| print(_to_one_hot(torch.as_tensor([[1, 5 ,6], [2, 4, 7]]), num_classes=10)) |
| import multiprocessing | |
| import pandas as pd | |
| import numpy as np | |
| def _apply_df(args): | |
| df, func, num, kwargs = args | |
| return num, df.apply(func, **kwargs) | |
| def apply_by_multiprocessing(df,func,**kwargs): | |
| workers=kwargs.pop('workers') |
| import multiprocessing | |
| from multiprocessing.dummy import Pool as ThreadPool | |
| import numpy as np | |
| def my_multipro(items, func, max_cpus=12): | |
| """Do an embarrassingly parallel task using multiprocessing. | |
| Use this for CPU bound tasks. |
Once in a while, you may need to cleanup resources (containers, volumes, images, networks) ...
// see: https://github.com/chadoe/docker-cleanup-volumes
$ docker volume rm $(docker volume ls -qf dangling=true)
$ docker volume ls -qf dangling=true | xargs -r docker volume rm
| # PHP / Ruby | |
| # Regex find key and value pair in JSON formated string | |
| # Match 1: Key | |
| # Match 2: Value | |
| # https://regex101.com/r/zR2vU9/4 | |
| # http://rubular.com/r/KpF3suIL10 | |
| # http://stackoverflow.com/questions/14349889/how-to-use-a-regular-expression-to-extract-json-fields/35129815#35129815 | |
| (?:\"|\')(?<key>[^"]*)(?:\"|\')(?=:)(?:\:\s*)(?:\"|\')?(?<value>true|false|[0-9a-zA-Z\+\-\,\.\$]*) |