martinsotir

import pandas as pd

def join_part(A, B, cond, left_on, right_on):
    C = A.merge(B, left_on=left_on, right_on=right_on, how="inner", copy=False)
    return C[cond].copy()

def conditional_join(A, B, cond, left_on, right_on, batch_size=50000):
    indices = range(0, len(A) + batch_size, batch_size)
    batches = (A.iloc[b_start : b_start + batch_size] for b_start in indices)
    merges = (join_part(subset, B, cond, left_on, right_on) for subset in batches)

martinsotir

#!/bin/bash
# ssh-multi
# D.Kovalov
# Based on http://linuxpixies.blogspot.jp/2011/06/tmux-copy-mode-and-how-to-control.html

# a script to ssh multiple servers over multiple tmux panes


starttmux() {
    if [ -z "$HOSTS" ]; then

martinsotir

Enabling conda in Windows Powershell

First, in an administrator command prompt, enable unrestricted Powershell script execution (see About Execution Policies):

set-executionpolicy unrestricted

Then makes sure that the conda Script directory in is your Path.

martinsotir

import torch
from torch.utils.data import Dataset, DataLoader
import tarfile
import zipfile
from pathlib import Path
from PIL import Image
from tqdm import tqdm
from torchvision import transforms
import mmap
import torch.multiprocessing as mp

martinsotir

martinsotir

import itertools
import pandas as pd

def flatten_df(df, list_col, elem_col_name="elem"):
    """Convert a series of list to individual rows, within a dataframe.
    Adapted from https://stackoverflow.com/a/48532692
    This function can be used on a dask dataframe:
    
    ```python
    df.map_partitions(lambda x: flatten_df(x, "list_col", elem_col_name="elem")).clear_divisions()

martinsotir

# Instructions (requires docker):
# docker build -t patchwork .
# docker run -it --rm patchwork

FROM openjdk:8

RUN apt-get update
RUN apt-get install apt-transport-https

# Install sbt

martinsotir

"""
Utility script to visualize embeddings using the tensorboard projector module.

Usage
-----

Dependencies : numpy, pillow, pandas, tensorflow

Call `prepare_projection(embedding, metadata, image_paths, ...)`, where :
  - `embedding` is a 2D numpy array (`n_sample` x `dim_embedding`)