Skip to content

Instantly share code, notes, and snippets.

@jiqiujia
jiqiujia / BertModel.java
Created July 2, 2019 03:07
bert serving
import org.apache.commons.io.FileUtils;
import org.tensorflow.Graph;
import org.tensorflow.SavedModelBundle;
import org.tensorflow.Session;
import org.tensorflow.Tensor;
import java.io.File;
import java.io.IOException;
import java.util.List;
@jiqiujia
jiqiujia / BuildLM.java
Created September 4, 2019 06:17
berkeley lm
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import edu.berkeley.nlp.lm.ConfigOptions;
import edu.berkeley.nlp.lm.StringWordIndexer;
import edu.berkeley.nlp.lm.io.ArpaLmReader;
import edu.berkeley.nlp.lm.io.LmReaders;
import edu.berkeley.nlp.lm.util.Logger;
# -*- coding: utf-8 -*-
"""
Spyder Editor
This is a temporary script file.
"""
import matplotlib
import matplotlib.pyplot as plt
import networkx as nx
import numpy as np
@jiqiujia
jiqiujia / validate_input.h
Created June 4, 2023 10:39
c++ code snippets
/*
* function to validate input to be of type `T`
*/
template<typename T>
T& validateInput(T& val, string prompt)
{
while (true) {
cout << prompt;
if (cin >> val) {
@jiqiujia
jiqiujia / install-gcc-9.sh
Created June 6, 2023 14:37 — forked from alexandreelise/README.md
Install gcc 9 on Ubuntu LTS 12.04,14.04,16.04 and 18.04
#!/usr/bin/env sh
sudo apt-get update -y && \
sudo apt-get upgrade -y && \
sudo apt-get dist-upgrade -y && \
sudo apt-get install build-essential software-properties-common -y && \
sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y && \
sudo apt-get update -y && \
sudo apt-get install gcc-9 g++-9 -y && \
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-9 && \
@jiqiujia
jiqiujia / iterable_dataset_dist.py
Created November 1, 2024 09:04 — forked from kklemon/iterable_dataset_dist.py
PyTorch IterableDataset implementation with multiprocessing and distributed training support
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from torch.utils.data import IterableDataset, DataLoader
class DistributedIterableDataset(IterableDataset):
"""
Example implementation of an IterableDataset that handles both multiprocessing (num_workers > 0)