Skip to content

Instantly share code, notes, and snippets.

@keithshep
Last active October 13, 2020 16:53
Show Gist options
  • Save keithshep/2650820 to your computer and use it in GitHub Desktop.
Save keithshep/2650820 to your computer and use it in GitHub Desktop.
python cheats

Angle between two vectors

Note that this doesn't give you the direction the angle sweeps in

def angle_deg_between_vecs(vec1, vec2):
    """
    Computes an angle in degrees between two vectors
    :param vec1: the first vector (numpy array of length n >= 2)
    :param vec2: the second vector (numpy array of length n >= 2)
    :return: the angle in radians
    """
    cos_theta = np.dot(vec1, vec2) / (np.linalg.norm(vec1) * np.linalg.norm(vec2))
    theta_rad = np.arccos(cos_theta)

    return math.degrees(theta_rad)

Random Note on Random String

import random
print(random.choice(['A', 'B', 'C', 'D', 'E', 'F', 'G']), random.randrange(1, 5))

import random
string_nums = ['1', '2', '3', '4']
notes = ['A', 'B', 'C', 'D', 'E', 'F', 'G']
def rand_note_seq(note_count, note_weight):
    last_string = random.choice(string_nums)
    yield last_string
    string_just_changed = True
    curr_note_count = 0

    while curr_note_count < note_count:
        if string_just_changed or random.random() < note_weight:
            yield random.choice(notes)
            curr_note_count += 1
            string_just_changed = False
        else:
            last_string = random.choice(list(set(string_nums) - set([last_string])))
            yield last_string
            string_just_changed = True

', '.join(rand_note_seq(5, 0.75))

Connecting a terminal to a jupyter notebook

jupyter console --existing

or if there are multiple kernels running

jupyter console --existing=/run/user/<USER_NUMBER>/jupyter/kernel-<KERNEL_ID>.json

Installing django

> curl http://python-distribute.org/distribute_setup.py | python
> easy_install-2.7 pip
> pip-2.7 install django

start a CGI server:

python -m CGIHTTPServer

A custom server:

import sys
sys.path.append('/Users/kss/projects/muga-haploqa/related-projs/pyweb')

import CGIHTTPServer
import BaseHTTPServer

class Handler(CGIHTTPServer.CGIHTTPRequestHandler):
  cgi_directories = ["/pyweb"]
  def do_GET(self):
      # sys.path.append('/Users/kss/projects/muga-haploqa/related-projs/pyweb')
      if self.path == '/':
          self.path = '/index.py'
      return CGIHTTPServer.CGIHTTPRequestHandler.do_GET(self)

PORT = 8000

httpd = BaseHTTPServer.HTTPServer(("", PORT), Handler)
print "serving at port", PORT
httpd.serve_forever()

getting script relative dir:

SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))

reading a CSV

in python 3

import csv
with open('eggs.csv', newline='') as csvfile:
    spamreader = csv.reader(csvfile, delimiter=' ', quotechar='|')
    for row in spamreader:
        print(', '.join(row))

in python 2

import csv
with open('eggs.csv', 'rb') as csvfile:
    spamreader = csv.reader(csvfile, delimiter=' ', quotechar='|')
    for row in spamreader:
        print ', '.join(row)

Render matplotlib to numpy image

import numpy

def fig2data ( fig ):
    """
    @brief Convert a Matplotlib figure to a 4D numpy array with RGBA channels and return it
    @param fig a matplotlib figure
    @return a numpy 3D array of RGBA values
    """
    # draw the renderer
    fig.canvas.draw ( )

    # Get the RGBA buffer from the figure
    w,h = fig.canvas.get_width_height()
    buf = numpy.fromstring ( fig.canvas.tostring_argb(), dtype=numpy.uint8 )
    buf.shape = ( w, h,4 )

    # canvas.tostring_argb give pixmap in ARGB mode. Roll the ALPHA channel to have it in RGBA mode
    buf = numpy.roll ( buf, 3, axis = 2 )
    return buf

This is how I was rendering side by side images

    masks = r['masks']

    mask_rgb = np.zeros(image.shape, dtype=np.uint8)
    if len(masks.shape) == 3:
        mask_x, mask_y, mask_z = masks.shape
        img_x, img_y, _ = image.shape
        if mask_x == img_x and mask_y == img_y and mask_z >= 1:
#             flatten_masks = np.amax(masks, axis=2)
#             mask_rgb[:, :, 0] = flatten_masks
#             mask_rgb[:, :, 1] = flatten_masks
#             mask_rgb[:, :, 2] = flatten_masks
#             mask_rgb *= 255
            for z_index in range(mask_z):
                mask_rgb[:, :, z_index % 3] += masks[:, :, z_index] * 255

    #img_pred = Net_.predict( np.expand_dims(np.expand_dims(frame[...,0],axis=0),axis=3), batch_size=1, verbose=0, steps=None)
    writer.append_data(np.concatenate((image, mask_rgb), axis=1))

Parallel Operations Sharing Large Datastructure

import numpy as np
from joblib import Parallel, delayed
from joblib.pool import has_shareable_memory

Parallel(n_jobs=2, max_nbytes=1e6)(
    delayed(has_shareable_memory)(np.ones(int(i)))
    for i in [1e2, 1e4, 1e6])

For more detailed controll see https://pythonhosted.org/joblib/parallel.html#writing-parallel-computation-results-in-shared-memory

Progress bars

Install tqdm for easy console progress bars

pytorch

pytorch visualization

check out https://github.com/facebookresearch/visdom for visualization

printing parameter info

total_params = 0
for name, param in model.named_parameters():
    total_params += param.numel()
    print('parameter size: {}\tnumel: {}\tname: {}'.format(
        tuple(param.size()), param.numel(), name))
    # param.requires_grad = False
print('total params:', total_params)

2D argmax

def argmax_2d(tensor):
    assert tensor.dim() >= 2
    max_col_vals, max_cols = torch.max(tensor, -1, keepdim=True)
    max_vals, max_rows = torch.max(max_col_vals, -2, keepdim=True)
    max_cols = torch.gather(max_cols, -2, max_rows)

    max_vals = max_vals.squeeze(-1).squeeze(-1)
    max_rows = max_rows.squeeze(-1).squeeze(-1)
    max_cols = max_cols.squeeze(-1).squeeze(-1)

    return max_vals, torch.stack([max_rows, max_cols], -1)

Advanced (Programatic) slicing

In [45]: x                                                                                                                                                                                           
Out[45]: 
tensor([[[0.3547, 0.5365, 0.5720, 0.0090],
         [0.7891, 0.7401, 0.2083, 0.0652],
         [0.0532, 0.1681, 0.5422, 0.6860]],

        [[0.6818, 0.5591, 0.7986, 0.6649],
         [0.4301, 0.6177, 0.9294, 0.7927],
         [0.5283, 0.2691, 0.0288, 0.0476]]])

In [46]: x[[slice(None, None), [0, 1, 2], [1, 2, 0]]]                                                                                                                                                
Out[46]: 
tensor([[0.5365, 0.2083, 0.0532],
        [0.5591, 0.9294, 0.5283]])

In [63]: x[(..., [1, 2, 0])]                                                                                                                                                                         
Out[63]: 
tensor([[[0.5365, 0.5720, 0.3547],
         [0.7401, 0.2083, 0.7891],
         [0.1681, 0.5422, 0.0532]],

        [[0.5591, 0.7986, 0.6818],
         [0.6177, 0.9294, 0.4301],
         [0.2691, 0.0288, 0.5283]]])

binary search

# Binary search which returns the matchin index -(insert_index + 1) in the case
# that no match is found
# @param array a sorted sorted array
# @param x the value we're searching for
# @param comp_func the function used to compare values for the binary
#        search. This function takes two parameters (x, y) and will return a number
#        less than, greater than or equal to zero depending on whether x is less than,
#        greater than or equal to y respectively
# @return {number} the index
def binary_search(array, x, comp_func=None):

    if comp_func is None:
        comp_func = lambda x, y: x - y

    low = 0
    high = len(array) - 1
    while low <= high:
        mid = (low + high) // 1
        midVal = array[mid]
        compVal = comp_func(midVal, x)
        if compVal < 0:
            low = mid + 1
        elif compVal > 0:
            high = mid - 1
        else:
            return mid

    return -(low + 1)

h5py

# create a jagged array structure in HDF5
vlen_dataset = f.create_dataset('vlen_dbl', (3,), h5py.special_dtype(vlen=np.double))
for i in range(3):
    vlen_dataset[i] = np.random.rand(i * 2)

simple interpolation

def interpolate(arr_1d, target_len):
    in_len = len(arr_1d)

    for tgt_index in range(target_len):
        interp_index = tgt_index * (in_len - 1) / (target_len - 1)
        if interp_index.is_integer():
            yield arr_1d[int(interp_index)]
        else:
            ceil_val = arr_1d[int(math.ceil(interp_index))]
            floor_val = arr_1d[int(math.floor(interp_index))]

            ceil_proportion = interp_index - floor_val

            yield ceil_val * ceil_proportion + floor_val * (1.0 - ceil_proportion)

matplotlib animation in jupyter

hmpf = np.ones([4,4])
hmpf[2][1] = 0
imagelist = [ hmpf*i*255./19. for i in range(20) ]

fig = plt.figure() # make figure

# make axesimage object
# the vmin and vmax here are very important to get the color map correct
im = plt.imshow(imagelist[0], cmap=plt.get_cmap('jet'), vmin=0, vmax=255)

# function to update figure
def updatefig(j):
    # set the data in the axesimage object
    im.set_array(imagelist[j])
    # return the artists set
    return [im]

# kick off the animation
anim = animation.FuncAnimation(fig, updatefig, frames=range(20), 
                              interval=50, blit=True)
#plt.show()
plt.close(anim._fig)

HTML(anim.to_html5_video())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment