TechDay_UPE_Acesso_AWS.ipynb

techday_upe_acesso_aws.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TechDay_UPE_Acesso_AWS.ipynb",
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/leocosta037/70496cfae56e0b2416ac00aad9756b4a/techday_upe_acesso_aws.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QG0_FuDKPDV6"
      },
      "source": [
        "# Notebook para acesso aos arquivos da Amazon S3"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1VSQa6OQPnAD"
      },
      "source": [
        "Material para consulta"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ColhS7m9oI7r"
      },
      "source": [
        "https://towardsdatascience.com/working-with-amazon-s3-buckets-with-boto3-785252ea22e0\n",
        "\n",
        "https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#examples\n",
        "\n",
        "https://realpython.com/python-boto3-aws-s3/#downloading-a-file\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jXwXFX9R_wvc"
      },
      "source": [
        "# Inicialização"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FQdgv6IEB5sB"
      },
      "source": [
        "pip install boto3"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lKDkhmdOU1d3"
      },
      "source": [
        "# Importação do Google Drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XclrDSSGdqvn"
      },
      "source": [
        "##Recurso"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kz6iw2gYduWL"
      },
      "source": [
        "password = '' # Inserir o valor de \"SECRET_KEY\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x2CFsm3mePTY"
      },
      "source": [
        "# Carregamento"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B9FU3xsr9j4v"
      },
      "source": [
        "# Variáveis iniciais\n",
        "import boto3\n",
        "import pandas as pd\n",
        "\n",
        "BUCKET = '' # Inserir o nome do bucket\n",
        "#PATH = '' # Caminho do arquivo, caso esteja numa pasta\n",
        "ACCESS_KEY = '' # Inserir chave de acesso da AWS\n",
        "SECRET_KEY = password # Senha da chave de acesso AWS"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T1_Ig3Gu9phW"
      },
      "source": [
        "# Carregando as funções \"client\" e \"resource\" do boto3\n",
        "client = boto3.client('s3', aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY)\n",
        "resource = boto3.resource('s3', aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4z6VHMEEIFJT"
      },
      "source": [
        "# Essencial para a função \"bucket\"\n",
        "bucket_resource = resource.Bucket(BUCKET)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ol7_L7oFAlNn"
      },
      "source": [
        "# Funções"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DoQQBiIxJ9X6"
      },
      "source": [
        "Listar os objetos"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iWEQApYDI9sw"
      },
      "source": [
        "for obj in bucket_resource.objects.all():\n",
        "    print(obj.key)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TjCd3UopLTuD"
      },
      "source": [
        "Listar as pastas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vGfOBVlkKeD2"
      },
      "source": [
        "paginator = client.get_paginator('list_objects')\n",
        "result = paginator.paginate(Bucket=BUCKET, Delimiter='/')\n",
        "for prefix in result.search('CommonPrefixes'):\n",
        "    print(prefix.get('Prefix'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bqfCl4F_A2Fo"
      },
      "source": [
        "Listar os buckets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UU-_W8WhMWpa"
      },
      "source": [
        "bucket_response = client.list_buckets()\n",
        "buckets = bucket_response[\"Buckets\"]\n",
        "buckets"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3L3-k1AiLhum"
      },
      "source": [
        "# List objects present in a bucket\n",
        "response = client.list_objects(Bucket=BUCKET)\n",
        "response"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "REOBKmN1orkZ"
      },
      "source": [
        "OBS: Inserir em:\n",
        "- \"Filename\" o caminho do arquivo no PC;\n",
        "- \"Key\" o caminho do arquivo no Bucket do AWS S3."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oitGVAbWLoCr"
      },
      "source": [
        "# Downloading files\n",
        "client.download_file(Filename='', Bucket=BUCKET, Key='')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LNdG9mBIS0bc"
      },
      "source": [
        "# Uploading files\n",
        "client.upload_file(Filename='', Bucket=BUCKET, Key='')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YifyQuJtNAMq"
      },
      "source": [
        "# Get object's metadata (last modification time, size in bytes etc.)\n",
        "response = client.head_object(Bucket=BUCKET, Key='')\n",
        "response"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}