This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import argparse | |
import logging | |
import os | |
import copy | |
from transformers import MBart50Tokenizer | |
from transformers import MBartForConditionalGeneration, AutoTokenizer | |
# from transformers.modeling_bart import shift_tokens_right | |
from longformer_encoder_decoder import LongformerSelfAttentionForMBart, LongformerEncoderDecoderConfig | |
from longformer_encoder_decoder import LongformerEncoderDecoderForConditionalGeneration |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas as pd | |
import urllib.request | |
from tqdm import tqdm | |
from multiprocessing.pool import ThreadPool | |
import os.path | |
# | |
# This gist downloads all series from TMDB. You can easily modify this to download all movies. | |
# It uses threads to parallelize downloads and speed up this process. | |
# Depends on python 3. Tested on anaconda. |