jiqiujia

import torch
import torch.distributed as dist
import torch.multiprocessing as mp

from torch.utils.data import IterableDataset, DataLoader


class DistributedIterableDataset(IterableDataset):
    """
    Example implementation of an IterableDataset that handles both multiprocessing (num_workers > 0) 

jiqiujia

#!/usr/bin/env sh

sudo apt-get update -y && \
sudo apt-get upgrade -y && \
sudo apt-get dist-upgrade -y && \
sudo apt-get install build-essential software-properties-common -y && \
sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y && \
sudo apt-get update -y && \
sudo apt-get install gcc-9 g++-9 -y && \
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-9 && \

jiqiujia

/*
* function to validate input to be of type `T`
*/
template<typename T>
T& validateInput(T& val, string prompt)
{
    while (true) {
        cout << prompt;
        if (cin >> val) {

jiqiujia

# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""
import matplotlib
import matplotlib.pyplot as plt
import networkx as nx
import numpy as np

jiqiujia

import java.io.File;
import java.util.ArrayList;
import java.util.List;

import edu.berkeley.nlp.lm.ConfigOptions;
import edu.berkeley.nlp.lm.StringWordIndexer;
import edu.berkeley.nlp.lm.io.ArpaLmReader;
import edu.berkeley.nlp.lm.io.LmReaders;
import edu.berkeley.nlp.lm.util.Logger;

jiqiujia

import org.apache.commons.io.FileUtils;
import org.tensorflow.Graph;
import org.tensorflow.SavedModelBundle;
import org.tensorflow.Session;
import org.tensorflow.Tensor;

import java.io.File;
import java.io.IOException;
import java.util.List;

jiqiujia

import numpy as np

def make_lsh_model(nb_tables, nb_bits, nb_dimensions, vector_sample):
    # vector_sample: np arr w/ shape (2 * nb_tables * nb_tables, nb_dimensions).
    # normals, midpoints: np arrs w/ shape (nb_bits, nb_dimensions)
    # thresholds: np arrs w/ shape (nb_bits)
    # all_normals, all_thresholds: lists w/ one normal, one threshold per table.
    all_normals, all_thresholds = [], []
    for i in range(0, len(vector_sample), 2 * nb_bits):
        vector_sample_a = vector_sample[i:i + nb_bits]

jiqiujia

# -*- coding: utf-8 -*-

import struct
import os

# 由于原代码不适用python3且有大量bug
# 以及有函数没有必要使用且一些代码书写不太规范或冗余
# 所以本人在原有的大框架基本不动的情况下作了大量的细节更改。
# 使得没有乱码出现，文件夹导入更方便等等。
# Author：Ling Yue, Taiyuan U of Tech

jiqiujia

##+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
## Created by: Hang Zhang, Rutgers University, Email: [email protected]
## Modified by Thomas Wolf, HuggingFace Inc., Email: [email protected]
## Copyright (c) 2017-2018
##
## This source code is licensed under the MIT-style license found in the
## LICENSE file in the root directory of this source tree
##+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

"""Encoding Data Parallel"""

jiqiujia

Compile the C++ code creating a shared library (or shared object in UNIX)

$ clang++ TestJNI.cpp -o libTestJNI.so -fPIC -shared -std=c++11 -I$HOME/opt/java/include -I$HOME/opt/java/include/linux

Run the application

$ scala -save load.scala 
dir = /home/archbox/opengl/jni/libTestJNI.so
Hello world java
i = 0