upload_b2.sh

#!/bin/bash
# Upload folders from s3_folders.txt to AWS S3 storage
#
# Requirements:
# + AWS CLI 2.26.6+
#   + https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html
#
# Notes:
# + Increase terminal window width if upload progress causes scrolling due to overlong line ("completing...")
#

# Globals
S3_BUCKET="s3://meshcapade-datasets/PS_datasets/BEDLAM2_train"
UPLOAD_JOBLIST="/home/mkocabas/all_seq_folders.txt"
B2_DIR="/ps/project/bedlam/v2/render/images"

# Check if folder list exists
if [ ! -f "$UPLOAD_JOBLIST" ]; then
    echo "Joblist file '$UPLOAD_JOBLIST' not found!"
    exit 1
fi

# Read joblist line by line
while read -r basename || [ -n "$basename" ]; do
    
    foldername=$B2_DIR/$basename
    echo $foldername
    # Skip empty lines
    if [ -z "$foldername" ]; then
        continue
    fi

    # Skip comment lines starting with #
    if [[ "$foldername" == \#* ]]; then
        echo "Ignoring: '$foldername'"
        continue
    fi

    # Check if local folder exists
    if [ -d "$foldername" ]; then
        echo "Uploading $foldername to $S3_BUCKET/$basename"

        # Exclude exr_depth
        aws s3 sync "$foldername" "${S3_BUCKET}/${basename}" --exclude "*exr_depth*" --exclude "*png*" --exact-timestamps # --only-show-errors

        # Sync all
#        aws s3 sync "$foldername/" "${S3_BUCKET}/${foldername}/" --exact-timestamps # --only-show-errors

        sync_status=$?
        if [ $sync_status -eq 0 ]; then
            echo "  [OK]"
        else
            echo "ERROR: aws s3 sync failed [$sync_status]"
            exit 1
        fi
    else
        echo "ERROR: Local folder not found: $LOCAL_FOLDER"
        exit 1
    fi
done < "$UPLOAD_JOBLIST"

echo "[SUCCESS]: All folders from '$UPLOAD_JOBLIST' uploaded."
exit 0

echo "Uploading motions"
aws s3 cp /ps/archive/bedlam/v2/bedlam_clothing/motions/b2/b2v0/11_whitelisted/b2v0_motions_npz.zip $S3_BUCKET/source_data/
aws s3 cp /ps/archive/bedlam/v2/bedlam_clothing/motions/b2/b2v1/11_whitelisted/b2v1_motions_npz.zip $S3_BUCKET/source_data/
aws s3 cp /ps/archive/bedlam/v2/bedlam_clothing/motions/b2/b2v2/11_whitelisted/b2v2_motions_npz.zip $S3_BUCKET/source_data/
aws s3 cp /ps/archive/bedlam/v2/bedlam_clothing/motions/b2/b2v4/11_whitelisted/b2v4_motions_npz.zip $S3_BUCKET/source_data/