Big_file_copy_from_gdrive.ipynb

big_file_copy_from_gdrive.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Big_file_copy_from_gdrive.ipynb",
      "provenance": [],
      "mount_file_id": "1Rpmt7jw0YlXzxcnrnOelcq6MgB_q6gD0",
      "authorship_tag": "ABX9TyP9vcjVTAHCgaMLDayTE4L5",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/jamm1985/df7b8c0abb346d3dbe403bf06ae4f46f/big_file_copy_from_gdrive.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Bypassing the Google Drive file cache in Colab. Just put it in colab cell to copy large files from google drive to Colab local storage. You can then load the dataset from local storage, which will greatly speed up training."
      ],
      "metadata": {
        "id": "gIngKSdvsvdN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "sh = \"\"\"\n",
        "#!/bin/bash\n",
        "\n",
        "BIGFILE=/content/drive/MyDrive/DATA/combined/2021-12-29-combined.h5\n",
        "OUTFILE=/content/2021-12-29-combined.h5\n",
        "PARTSIZE=$((1*1024*1024*1024)) # 1 GiB\n",
        "FILESIZE=$(stat -c%s \"$BIGFILE\")\n",
        "echo \"Size of $BIGFILE = $FILESIZE bytes.\"\n",
        "\n",
        "CHUNKS=$((FILESIZE / PARTSIZE))\n",
        "if (( FILESIZE % PARTSIZE == 0 ))\n",
        "then\n",
        "    echo \"CHUNKS = $CHUNKS\"\n",
        "else\n",
        "    ((CHUNKS = CHUNKS + 1))\n",
        "    echo \"CHUNKS = $CHUNKS\"\n",
        "fi\n",
        "\n",
        "for ((i=0;i<$CHUNKS;i++));\n",
        "do\n",
        "\techo \"Transfer CHUNK $i out of $CHUNKS\"\n",
        "  dd if=$BIGFILE bs=$PARTSIZE skip=$i count=1 iflag=sync oflag=sync >> $OUTFILE\n",
        "  sleep 10\n",
        "  sync\n",
        "done\n",
        "\"\"\"\n",
        "with open('script.sh', 'w') as file:\n",
        "  file.write(sh)\n",
        "\n",
        "!bash script.sh\n"
      ],
      "metadata": {
        "id": "jAEPhMe2Q0ht"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}