Skip to content

Instantly share code, notes, and snippets.

@RobertArbon
Created March 7, 2021 17:57
Show Gist options
  • Save RobertArbon/8a08d150dd8f2c203974a1bd8eb5bf92 to your computer and use it in GitHub Desktop.
Save RobertArbon/8a08d150dd8f2c203974a1bd8eb5bf92 to your computer and use it in GitHub Desktop.
this uploads all files in a directory which meet certain given conditions.
import osfclient
from osfclient.models.storage import Storage
import os
from pathlib import Path
from typing import Callable, Optional
def get_storage(username: str, password: str, project_id: str, storage_provider: str = 'osfstorage') -> Storage:
"""
project_id is the alpha-numeric sequence at the end of the project url on OSF, e.g., osf.io/ab123 has project id of ab123
username and password pertain to your OSF account.
"""
osf = osfclient.OSF(username=username, password=password)
proj = osf.project(project_id)
storage = proj.storage(provider=storage_provider)
return storage
def upload_project_files(project_root: Path, storage: Storage, is_uploadable: Callable, dry_run: Optional[bool]=True) -> None:
"""
'is_uploadable' is a function with signature: is_uploadable(path: Path, file: Path) -> bool. This just determines
which files to upload based on the path to the containing directory and the file name.
'project_root' is the absolute path to the local project folder.
"""
for (root, dirs, files) in os.walk(project_root, topdown=True):
for file in files:
root = Path(root)
file = Path(file)
if is_uploadable(root, file):
file_path = root.joinpath(file)
new_file_path = root.relative_to(project_root).joinpath(file)
print(f' uploading: {file_path}\n to: {new_file_path}\n at storage id: {storage.id}\n')
if not dry_run:
with file_path.open(mode='rb') as f:
storage.create_file(path=str(new_file_path), fp=f)
# usage (uploads all files):
# define file/path matching function
def all_files(path, file):
return True
# get the storage object:
storage = get_storage(my_osf_username, my_osf_password, my_osf_project_id )
# upload the files dry-run:
upload_project_files(Path('path/to/project/'), storage, all_files)
# upload for real:
upload_project_files(Path('path/to/project/'), storage, all_files, dry_run=False)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment