josephrocca

josephrocca

{
  "activation_dropout": 0.0,
  "activation_function": "gelu",
  "architectures": [
    "eBart"
  ],
  "attention_dropout": 0.0,
  "bos_token_id": 16385,
  "d_model": 1024,
  "decoder_attention_heads": 16,

josephrocca

@josephrocca ➜ /workspaces/tokenizers/bindings/python (pyodide) $ RUSTUP_TOOLCHAIN=nightly maturin build --release -o dist --target wasm32-unknown-emscripten -i python3.10
⚠️  Warning: `build-backend` in pyproject.toml is not set to `maturin`, packaging tools such as pip will not use maturin to build this project.
🔗 Found pyo3 bindings
🐍 Found cross compiling target CPython 3.10
   Compiling autocfg v1.1.0
   Compiling cfg-if v1.0.0
   Compiling proc-macro2 v1.0.36
   Compiling libc v0.2.121
   Compiling unicode-xid v0.2.2
   Compiling syn v1.0.89

josephrocca

@josephrocca ➜ /workspaces/tokenizers/bindings/python (master) $ RUSTUP_TOOLCHAIN=nightly maturin build --release -o dist --target wasm32-unknown-emscripten -i python3.10
⚠️  Warning: `build-backend` in pyproject.toml is not set to `maturin`, packaging tools such as pip will not use maturin to build this project.
🔗 Found pyo3 bindings
🐍 Found cross compiling target CPython 3.10
   Compiling libc v0.2.121
   Compiling cfg-if v1.0.0
   Compiling autocfg v1.1.0
   Compiling proc-macro2 v1.0.36
   Compiling unicode-xid v0.2.2
   Compiling syn v1.0.89

josephrocca

@josephrocca ➜ /workspaces/tokenizers/bindings/python (pyodide) $ RUSTUP_TOOLCHAIN=nightly maturin build --release -o dist --target wasm32-unknown-emscripten -i python3.10
⚠️  Warning: `build-backend` in pyproject.toml is not set to `maturin`, packaging tools such as pip will not use maturin to build this project.
🔗 Found pyo3 bindings
🐍 Found cross compiling target CPython 3.10
error: "/home/codespace/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/Cargo.lock" does not exist, unable to build with the standard library, try:
        rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu
💥 maturin failed
  Caused by: Failed to build a native library through cargo
  Caused by: Cargo build finished with "exit status: 101": `"cargo" "rustc" "--release" "--target" "wasm32-unknown-emscripten" "--message-format" "json" "-Z" "build-std" "--lib" "--" "-C" "link-arg=-sSIDE_MODULE=2" "-C" "link-arg=-sWASM_BIGINT"`

josephrocca

@josephrocca ➜ /workspaces/tokenizers/bindings/python (pyodide ✗) $ RUSTUP_TOOLCHAIN=nightly maturin build --release -o dist --target wasm32-unknown-emscripten -i python3.10
⚠️  Warning: `build-backend` in pyproject.toml is not set to `maturin`, packaging tools such as pip will not use maturin to build this project.
🔗 Found pyo3 bindings
🐍 Found cross compiling target CPython 3.10
  Downloaded rustc-demangle v0.1.21
  Downloaded memchr v2.4.1
  Downloaded miniz_oxide v0.4.0
  Downloaded cc v1.0.69
  Downloaded unicode-width v0.1.8
  Downloaded object v0.26.2

josephrocca

josephrocca

josephrocca

josephrocca

[package]
name = "tokenizers-python"
version = "0.11.0"
authors = ["Anthony MOI <[email protected]>"]
edition = "2018"

[lib]
name = "tokenizers"
crate-type = ["cdylib"]