martinsotir

"""
Utility script to visualize embeddings using the tensorboard projector module.

Usage
-----

Dependencies : numpy, pillow, pandas, tensorflow

Call `prepare_projection(embedding, metadata, image_paths, ...)`, where :
  - `embedding` is a 2D numpy array (`n_sample` x `dim_embedding`)

martinsotir

# Instructions (requires docker):
# docker build -t patchwork .
# docker run -it --rm patchwork

FROM openjdk:8

RUN apt-get update
RUN apt-get install apt-transport-https

# Install sbt

martinsotir

import itertools
import pandas as pd

def flatten_df(df, list_col, elem_col_name="elem"):
    """Convert a series of list to individual rows, within a dataframe.
    Adapted from https://stackoverflow.com/a/48532692
    This function can be used on a dask dataframe:
    
    ```python
    df.map_partitions(lambda x: flatten_df(x, "list_col", elem_col_name="elem")).clear_divisions()

martinsotir

martinsotir

import torch
from torch.utils.data import Dataset, DataLoader
import tarfile
import zipfile
from pathlib import Path
from PIL import Image
from tqdm import tqdm
from torchvision import transforms
import mmap
import torch.multiprocessing as mp

martinsotir

Enabling conda in Windows Powershell

First, in an administrator command prompt, enable unrestricted Powershell script execution (see About Execution Policies):

set-executionpolicy unrestricted

Then makes sure that the conda Script directory in is your Path.

martinsotir

#!/bin/bash
# ssh-multi
# D.Kovalov
# Based on http://linuxpixies.blogspot.jp/2011/06/tmux-copy-mode-and-how-to-control.html

# a script to ssh multiple servers over multiple tmux panes


starttmux() {
    if [ -z "$HOSTS" ]; then

martinsotir

import pandas as pd

def join_part(A, B, cond, left_on, right_on):
    C = A.merge(B, left_on=left_on, right_on=right_on, how="inner", copy=False)
    return C[cond].copy()

def conditional_join(A, B, cond, left_on, right_on, batch_size=50000):
    indices = range(0, len(A) + batch_size, batch_size)
    batches = (A.iloc[b_start : b_start + batch_size] for b_start in indices)
    merges = (join_part(subset, B, cond, left_on, right_on) for subset in batches)

martinsotir

# WIP
# Inspired from Keras and https://towardsdatascience.com/how-to-visualize-convolutional-features-in-40-lines-of-code-70b7d87b0030

from pathlib import Path
import torch
import torchvision.utils as vutils
import matplotlib.pyplot as plt
from torchvision.utils import make_grid
import cv2
import numpy as np

martinsotir

Disk I/O benchmarking

Required packages:

sudo apt install fio ioping smartmontools

List volume and partitions: