This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import typing as t | |
from concurrent.futures import ThreadPoolExecutor, as_completed | |
from functools import wraps | |
import requests | |
def parallelize(f: t.Callable = None, max_workers: int = 5): | |
def decorator(func): | |
@wraps(func) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from urllib.request import urlopen | |
from urllib.error import HTTPError | |
from concurrent.futures import ThreadPoolExecutor, as_completed | |
class HttpError(Exception): | |
def __init__(self, url: str, code: int) -> None: | |
self.url = url | |
self.code = code |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import datetime as dt | |
from urllib.parse import urljoin | |
import requests | |
from airflow.models import DAG | |
from airflow.operators.python import PythonOperator | |
default_args = { | |
'owner': 'airflow', |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import requests | |
from http import HTTPStatus | |
import pprint | |
class HttpError: | |
def __init__(self, status_code: int): | |
self.status_code = status_code | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import logging | |
from logging import Handler, LogRecord | |
from airflow.operators.python import get_current_context | |
class RowsAffectedHandler(Handler): | |
def emit(self, record: LogRecord) -> None: | |
msg = self.format(record) | |
context = get_current_context() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import re | |
from operator import add, mul | |
from functools import reduce | |
from typing import List | |
def multiply(iin: str, weights: List[int]) -> int: | |
result = reduce( | |
add, | |
map(lambda i: mul(*i), zip(map(int, iin), weights)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import csv | |
import luigi | |
from luigi.format import UTF8 | |
import requests | |
import pandas as pd | |
from bs4 import BeautifulSoup | |
class AggregateMovieRatingTask(luigi.Task): |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -*- coding: utf8 -*- | |
import re | |
import string | |
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer | |
import requests | |
from bs4 import BeautifulSoup | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
# | |
# Vkontatke OAuth 2.0 wrapper | |
# Copyright 2011, Adil Khashtamov [[email protected]] | |
# http://khashtamov.kz | |
# | |
# | |
import logging |