Skip to content

Instantly share code, notes, and snippets.

@Geobm
Created August 30, 2020 12:22
Show Gist options
  • Save Geobm/ee1c60556e54d1b5652282471289dc84 to your computer and use it in GitHub Desktop.
Save Geobm/ee1c60556e54d1b5652282471289dc84 to your computer and use it in GitHub Desktop.
Hackmty.ipynb
Display the source blob
Display the rendered blob
Raw
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "Hackmty.ipynb",
"provenance": [],
"collapsed_sections": [],
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/Geobm/ee1c60556e54d1b5652282471289dc84/hackmty.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"metadata": {
"id": "s3QA3QL_dCxR",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 367
},
"outputId": "da700237-d458-49c1-8ac7-02b56dcd3d75"
},
"source": [
"!pip install nltk\n",
"\n",
"!pip install gensim\n",
"import nltk\n",
"import json \n",
"from nltk.corpus import stopwords \n",
"from nltk.tokenize import word_tokenize \n",
"import pandas as pd\n",
"from nltk.tokenize import word_tokenize\n",
"import numpy as np\n",
"import re\n",
"import nltk\n",
"from nltk.corpus import stopwords\n",
"from nltk.stem.porter import PorterStemmer\n",
"from nltk \n",
"import difflib\n",
"import requests\n",
"import os\n",
"from pprint import pprint\n",
"import difflib\n",
"import itertools\n",
"from flask import Flask, jsonify"
],
"execution_count": 155,
"outputs": [
{
"output_type": "stream",
"text": [
"Requirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (3.2.5)\n",
"Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk) (1.15.0)\n",
"Requirement already satisfied: gensim in /usr/local/lib/python3.6/dist-packages (3.6.0)\n",
"Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.4.1)\n",
"Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.6/dist-packages (from gensim) (2.1.0)\n",
"Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.15.0)\n",
"Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.18.5)\n",
"Requirement already satisfied: boto in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (2.49.0)\n",
"Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (2.23.0)\n",
"Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (1.14.48)\n",
"Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (2.10)\n",
"Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (1.24.3)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (2020.6.20)\n",
"Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (3.0.4)\n",
"Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim) (0.10.0)\n",
"Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim) (0.3.3)\n",
"Requirement already satisfied: botocore<1.18.0,>=1.17.48 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim) (1.17.48)\n",
"Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.18.0,>=1.17.48->boto3->smart-open>=1.2.1->gensim) (2.8.1)\n",
"Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.18.0,>=1.17.48->boto3->smart-open>=1.2.1->gensim) (0.15.2)\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "S8gNF3ctP9hH",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 34
},
"outputId": "cf212c17-808d-4a0f-b52f-ffa41622a006"
},
"source": [
"\n",
"github_token = \"99ad5dd512e16341e613145d10c8f603c615d633\"\n",
"\n",
"token = os.getenv('GITHUB_TOKEN', github_token)\n",
"\n",
"usernames = \"jg-11,andreamoronzavala,irving-estrada,geobm\"\n",
"\n",
"names = usernames.split(',')\n",
"\n",
"repo_description = []\n",
"repo_name = []\n",
"aux = 0\n",
"final_list = []\n",
"\n",
"for name in names:\n",
" query_url = f\"https://api.github.com/users/{name}/repos\"\n",
" params = {\n",
" \"state\": \"open\",\n",
" }\n",
" headers = {'Authorization': f'token {token}'}\n",
" response = requests.get(query_url, headers=headers, params=params)\n",
" result = response.json()\n",
"\n",
"for i in range(len(result)):\n",
" if result[i]['description'] is not None:\n",
" repo_name.append(result[i]['name'])\n",
" repo_description.append(result[i]['description'])\n",
"final_results = list(zip(repo_name,repo_description))\n",
"#print(json.dumps([dict(final_results)]))\n",
"\n",
"\n",
"tokenized_text = [word for word in nltk.tokenize.word_tokenize(str(repo_description)[1:]) if len(word) > 1]\n",
"#print(tokenized_text)\n",
"fdist = FreqDist(tokenized_text)\n",
"text1 = ''.split(str(tokenized_text))\n",
"fdist1 = nltk.FreqDist(tokenized_text)\n",
"print (fdist1.most_common(10))"
],
"execution_count": 202,
"outputs": [
{
"output_type": "stream",
"text": [
"[('training', 2), ('course', 2), ('the', 2), ('and', 2), ('for', 2), (\"'edX\", 1), (\"'Angular\", 1), ('CRUD', 1), ('front-end', 1), (\"'Advanced\", 1)]\n"
],
"name": "stdout"
}
]
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment