UploadToS3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "import glob\n",
    "import os\n",
    "import boto3\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "UPLOAD_DIR = '/usgs/cpkgs/isis3/isis_testData/'  # Where are the data to be uploaded?\n",
    "BUCKET_NAME = 'asc-isisdata'  # What is the name of the bucket in S3?\n",
    "PREFIX = 'isis_testData'  # What is the prefix inside the bucket to add the data?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Retrieve the list of existing buckets\n",
    "def upload_file(fname, bucket=BUCKET_NAME, prefix=''):\n",
    "    \"\"\"\n",
    "    upload a path to S3\n",
    "    \"\"\"\n",
    "    basename = os.path.basename(fname)\n",
    "    print(bucket, os.path.join(prefix, basename))\n",
    "    with open(fname, 'rb') as f:\n",
    "        s3.upload_fileobj(f, bucket, os.path.join(prefix, basename))\n",
    "\n",
    "        \n",
    "def list_objects(bucket=BUCKET_NAME, prefix=''):\n",
    "    \"\"\"\n",
    "    List objects in a S3 bucket\n",
    "    \"\"\"\n",
    "    resp = s3.list_objects(Bucket=bucket, Prefix=prefix)\n",
    "    try:\n",
    "        return resp['Contents']\n",
    "    except:\n",
    "        return []\n",
    "\n",
    "\n",
    "# Setup the login items\n",
    "s3 = boto3.client('s3',\n",
    "                 aws_access_key_id='', # Enter your access ID here\n",
    "                 aws_secret_access_key='') # Enter your access key here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only get Mro\n",
    "files_to_upload = glob.glob(f'{UPLOAD_DIR}/**/*/**', recursive=True)  \n",
    "\n",
    "# remove /ugsgs/cpkgs/isis3/isis_testData and remove non files\n",
    "files_to_upload = [f for f in files_to_upload if os.path.isfile(f)] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for f in files_to_upload:\n",
    "    # we want the path in the bucket to match the ISIS test data root, so trim \"/usgs/cpkgs/isis3/isis_testData\" \n",
    "    # from the original path. As the bucket prefix is already isis_testData. Normally, prefixes don't have \n",
    "    # to match local. \n",
    "    filedir =  os.path.dirname('/'.join(f.split('/')[5:]))\n",
    "    prefix = os.path.join(PREFIX, filedir)\n",
    "    upload_file(f, prefix=prefix)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "AutocnetDev",
   "language": "python",
   "name": "autocnetdev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}