Extract All Website Links.ipynb

how-to-extract-all-website-links.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Extract All Website Links.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/mryap/0a709af0de617468cfb4cfd95370ea79/how-to-extract-all-website-links.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PHelxJDtdDCf"
      },
      "source": [
        "### Extract All Website Links "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e6ZkI3F_jHTo",
        "outputId": "7a12728d-fec2-4352-a7a8-cc984b6b5dce"
      },
      "source": [
        "!pip install requests bs4"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (2.23.0)\n",
            "Requirement already satisfied: bs4 in /usr/local/lib/python3.7/dist-packages (0.0.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests) (2.10)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.7/dist-packages (from bs4) (4.6.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s9O2sc6-npXv"
      },
      "source": [
        "import requests\n",
        "from urllib.parse import urlparse, urljoin\n",
        "from bs4 import BeautifulSoup"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y9iYM1qxn2e1"
      },
      "source": [
        "def get_all_website_links(url):\n",
        "    \"\"\"\n",
        "    Returns all URLs that is found on `url` in which it belongs to the same website\n",
        "    \"\"\"\n",
        "\n",
        "    def is_valid(url):\n",
        "        parsed = urlparse(url)\n",
        "        return bool(parsed.netloc) and bool(parsed.scheme)\n",
        "\n",
        "    urls = set()\n",
        "    soup = BeautifulSoup(requests.get(url).content, \"html.parser\")\n",
        "\n",
        "    for a_tag in soup.findAll(\"a\"):\n",
        "        href = a_tag.attrs.get(\"href\")\n",
        "\n",
        "        if href == \"\" or href is None:\n",
        "            # href empty tag\n",
        "            continue\n",
        "\n",
        "        # join the URL if it's relative (not absolute link)\n",
        "        href = urljoin(url, href)\n",
        "        parsed_href = urlparse(href)\n",
        "\n",
        "        # remove URL GET parameters, URL fragments, etc.\n",
        "        href = parsed_href.scheme + \"://\" + parsed_href.netloc + parsed_href.path\n",
        "\n",
        "        if is_valid(href):\n",
        "            urls.add(href)\n",
        "        \n",
        "\n",
        "    return urls"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c8LnQHhhn6B9",
        "outputId": "b58fd088-a6bc-4037-ca44-1fd53f04dcd1"
      },
      "source": [
        "get_all_website_links(\"https://www.testandoptimize.com\")"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'https://twitter.com/testandoptimize',\n",
              " 'https://www.iubenda.com/privacy-policy/21524025',\n",
              " 'https://www.linkedin.com/in/ssyap/',\n",
              " 'https://www.testandoptimize.com/',\n",
              " 'https://www.testandoptimize.com/about',\n",
              " 'https://www.testandoptimize.com/blog',\n",
              " 'https://www.testandoptimize.com/blog/',\n",
              " 'https://www.testandoptimize.com/contact',\n",
              " 'https://www.testandoptimize.com/posts/challenges-facing-marketer/',\n",
              " 'https://www.testandoptimize.com/posts/getting-ready-for-server-side-tracking/',\n",
              " 'https://www.testandoptimize.com/posts/measure-and-report-your-real-users-core-web-vitals/',\n",
              " 'https://www.testandoptimize.com/posts/sample-ratio-mismatch-srm-in-a-b-testing/'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    }
  ]
}