This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import logging | |
import pexpect | |
from airflow.hooks.base_hook import BaseHook | |
class SFTP(object): | |
""" | |
Requires openssh_client. Spawns process to execute sftp command. | |
""" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas as pd | |
import requests | |
import feedparser | |
import time | |
import requests | |
def parse_rss_feed(url): | |
# Read feed xml data | |
# Try 3 times requesting the url if error | |
for i in range(0, 4): |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from bs4 import BeautifulSoup | |
import requests | |
def detect_feeds_in_HTML(html): | |
""" | |
Extract Feed URL from a given HTML page | |
This is achieved by detecting all ``link`` tags that reference a feed in HTML. | |
""" | |
# check if really an input stream | |
result = [] |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Gets the webpage | |
Converts the HTML to a readable HTML using readability | |
Extracts the text and saves it to a text file. | |
usage - python url_to_txt.py http://example.com | |
""" | |
from readability import Document | |
from bs4 import BeautifulSoup | |
from urllib.parse import urlparse |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import sqlite3 | |
db_name = "database.db" | |
def create_connection(db_file): | |
""" create a database connection to the SQLite database | |
specified by db_file | |
:param db_file: database file | |
:return: Connection object or None |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import glob | |
import pandas as pd | |
import os | |
def get_file_names(path=os.getcwd()): | |
return glob.glob(path + "/ubersuggest_*.csv") | |
def get_file(filename): |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// original gist: https://gist.github.com/willpatera/ee41ae374d3c9839c2d6 | |
function doGet(e){ | |
return handleResponse(e); | |
} | |
// Enter sheet name where data is to be written below | |
var SHEET_NAME = "Sheet1"; | |
var SCRIPT_PROP = PropertiesService.getScriptProperties(); // new property service |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# pip install verify-email (before running this) | |
import pandas as pd | |
from verify_email import verify_email | |
import pickle | |
final_row = ["S.No", "Name of Student", "Email", "valid"] | |
def get_file(filename): | |
return pd.read_csv(filename) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import glob | |
import pandas as pd | |
import os | |
def get_file_names(path=os.getcwd()): | |
return glob.glob(path + "/ubersuggest_*.csv") | |
def get_file(filename): |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import spacy | |
import pandas as pd | |
import sys | |
import re | |
nlp = spacy.load("en_core_web_md") | |
class Error(Exception): | |
"""Base class for other exceptions""" |