bart-preprocess-example.sh

source ~/miniconda3/bin/activate allen

LANG=en
TASK=qa_en_small
for SPLIT in train valid
  do
    python -m examples.roberta.multiprocessing_bpe_encoder \
    --encoder-json encoder.json \
    --vocab-bpe vocab.bpe \
    --inputs "$TASK/$SPLIT.$LANG" \
    --outputs "$TASK/$SPLIT.bpe.$LANG" \
    --workers 8 \
    --keep-empty;
done

DESTDIR="${TASK}-bin/"
rm -rf $DESTDIR

fairseq-preprocess \
  --source-lang "en" \
  --trainpref "${TASK}/train.bpe" \
  --validpref "${TASK}/valid.bpe" \
  --destdir ${DESTDIR} \
  --workers 8 \
  --srcdict dict.txt \
  --joined-dictionary \
  --only-source

cp dict.txt qa_en_small-bin/dict.txt

rename s/en-None.en.// qa_en_small-bin/*