Created
January 11, 2023 20:08
-
-
Save soldni/df31ad7f8b6f23ba5fced6b265077387 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# must install smashed[remote] via `pip install smashed[remote] | |
from smashed.utils.io_utils import upload_on_success | |
# Option 1: use upload_on_success as context manager | |
def run_training(): | |
with upload_on_success("s3://ai2-s2-research/<your handle>/<exp name>") as local_path: | |
# example if using huggingface trainer | |
trainer = Trainer(args=TrainingArguments(output_dir=local_path)) | |
trainer.train() | |
# Option 2: use upload_on_sucess to decorate function | |
@upload_on_success("s3://ai2-s2-research/<your handle>/<exp name>") | |
def run_training(local_path, ...) | |
# example if using huggingface trainer | |
trainer = Trainer(args=TrainingArguments(output_dir=local_path)) | |
trainer.train() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment