lhoestq

import math
import tempfile
from functools import partial

import dask.dataframe as dd
import pandas as pd
from huggingface_hub import CommitOperationAdd, HfFileSystem


def _preupload(df: pd.DataFrame, path: str, filesystem: HfFileSystem, **kwargs) -> pd.DataFrame:

lhoestq

function getJsonVal () { 
    python -c "import json,sys;sys.stdout.write(str(json.load(sys.stdin)$1))"; 
}
prUrl=$1
apiUrl=$(echo "$prUrl" | sed -e 's/pull/pulls/g' -e 's/github.com/api.github.com\/repos/g')
upstreamUrl=$(echo "$prUrl" | sed -re 's/\/pull\/[0-9]+/.git/g')
prData=`curl "$apiUrl"`
userName=$(echo $prData | getJsonVal "['head']['repo']['owner']['login']")
repoName=$(echo $prData | getJsonVal "['head']['repo']['name']")
repoFullName=$(echo $prData | getJsonVal "['head']['repo']['full_name']")

lhoestq

from nlp import load_dataset
from tqdm.auto import tqdm

wiki = load_dataset('wikipedia', '20200501.en', split="train")

batch_size = 1000
total_length = 0
for i in tqdm(range(0, len(wiki), batch_size)):  # loop takes ~1min to run
    batch = wiki[i:i + batch_size]
    total_length += sum(len(sample_text) for sample_text in batch["text"])