Skip to content

Instantly share code, notes, and snippets.

@jerinphilip
Last active March 24, 2021 18:58
Show Gist options
  • Save jerinphilip/f1caac37ac1eced45e6e9d94380bc4d0 to your computer and use it in GitHub Desktop.
Save jerinphilip/f1caac37ac1eced45e6e9d94380bc4d0 to your computer and use it in GitHub Desktop.
Some bergamot-makefile
# This Makefile is being prepared to cache installation setups on
# [email protected] and ensure that WASM pipeline builds and route
# through regression tests.
THREADS=40
SHELL := /bin/bash
ROOT := /mnt/Storage/jphilip/bergamot
BERGAMOT := $(ROOT)/bergamot-translator
EMSDK := $(ROOT)/emsdk
MODELS := $(ROOT)/models
BUILD := $(ROOT)/build
NATIVE_BUILD := $(BUILD)/native
WASM_BUILD := $(BUILD)/wasm
MODELS_GZ := $(BUILD)/models.gz.d
# Requires CMake > 3.12 or something. Locally compiled and added.
# 3.20.0-rc1 is used while testing this script
CMAKE := /home/jphilip/.local/bin/cmake
BUILD_TYPE ?= Release
# Parameterize builds by branches
BRANCH ?= jp/absorb-batch-translator
.PHONY: emsdk bergamot models dirs
dirs:
mkdir -p $(ROOT) $(BUILD) $(NATIVE_BUILD) $(WASM_BUILD) $(MODELS_GZ)
emsdk:
git -C $(EMSDK) pull || git clone https://github.com/emscripten-core/emsdk.git $(EMSDK)
$(EMSDK)/emsdk install latest
bergamot:
git -C $(BERGAMOT) pull || git clone https://github.com/browsermt/bergamot-translator $(BERGAMOT)
git -C $(BERGAMOT) checkout $(BRANCH)
git -C $(BERGAMOT) submodule update --init --recursive
models: dirs
git -C $(MODELS) pull || git clone https://github.com/mozilla-applied-ml/bergamot-models $(MODELS)
cp -r $(MODELS)/* $(MODELS_GZ)/
gunzip $(MODELS_GZ)/*/*
first-setup: emsdk dirs models
wasm: dirs bergamot
$(EMSDK)/emsdk activate latest &> /tmp/error.log || cat /tmp/error.log
source $(EMSDK)/emsdk_env.sh && cd $(WASM_BUILD) && \
emcmake $(CMAKE) -L \
-DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
-DCOMPILE_WASM=on \
$(BERGAMOT) && \
cd $(WASM_BUILD) && emmake make -f $(WASM_BUILD)/Makefile -j$(THREADS)
native: dirs bergamot
cd $(NATIVE_BUILD) && \
$(CMAKE) \
-L \
-DCOMPILE_CUDA=off -DUSE_WASM_COMPATIBLE_SOURCES=off\
-DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
$(BERGAMOT)
cd $(NATIVE_BUILD) && make -f $(NATIVE_BUILD)/Makefile -j$(THREADS)
clean:
rm $(BUILD) -rv
clean-native:
rm $(NATIVE_BUILD) -rv
clean-wasm:
rm $(WASM_BUILD) -rv
@jerinphilip
Copy link
Author

@abhi-agg:

		$(CMAKE) \
			-L \
			-DCOMPILE_CUDA=off -DCOMPILE_DECODER_ONLY=on -DCOMPILE_LIBRARY_ONLY=off \
			-DUSE_MKL=on  -DCOMPILE_THREAD_VARIANT=on -DCOMPILE_MAIN_EIGEN=off  \
			-DONNXJS_SGEMM=off -DUSE_WASM_COMPATIBLE_BLAS=off \
			-DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
		   	$(BERGAMOT)

is too complicated, can you switch this to one switch for WASM.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment