This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from io import StringIO | |
import sys | |
from typing import Dict, Optional | |
from langchain.agents import load_tools | |
from langchain.agents import initialize_agent | |
from langchain.agents.tools import Tool | |
from langchain.llms import OpenAI | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -*- coding: utf-8 -*- | |
from airflow.operators.http_operator import SimpleHttpOperator | |
from airflow.operators.postgres_operator import PostgresOperator | |
from airflow.operators.subdag_operator import SubDagOperator | |
from airflow.operators.sensors import SqlSensor | |
from airflow.hooks.postgres_hook import PostgresHook | |
from airflow.operators.python_operator import PythonOperator | |
from airflow.models import Variable, DAG |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
$ sudo apt-get install wget | |
$ sudo apt-get install build-essential | |
$ sudo apt-get install libc6-dev | |
$ wget ftp://ftp.freetds.org/pub/freetds/stable/freetds-1.00.27.tar.gz | |
$ tar -xzf freetds-1.00.27.tar.gz | |
$ cd freetds-1.00.27 | |
$ ./configure --prefix=/usr/local --with-tdsver=7.3 | |
$ make | |
$ make install |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Airflow Operator to download results of a sql query to a file on the worker | |
# Pass chunksize parameter to download large tables without the | |
# worker running out of memory | |
import logging | |
from airflow.hooks.postgres_hook import PostgresHook | |
from airflow.models import BaseOperator | |
from airflow.utils.decorators import apply_defaults |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Note – this is not a bash script (some of the steps require reboot) | |
# I named it .sh just so Github does correct syntax highlighting. | |
# | |
# This is also available as an AMI in us-east-1 (virginia): ami-cf5028a5 | |
# | |
# The CUDA part is mostly based on this excellent blog post: | |
# http://tleyden.github.io/blog/2014/10/25/cuda-6-dot-5-on-aws-gpu-instance-running-ubuntu-14-dot-04/ | |
# Install various packages | |
sudo apt-get update |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import boto | |
import boto.s3 | |
import os.path | |
import sys | |
# Fill these in - you get them when you sign up for S3 | |
AWS_ACCESS_KEY_ID = '' | |
AWS_ACCESS_KEY_SECRET = '' | |
# Fill in info on data to upload |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def does_exist(tbl, where = None, where_field = 'id'): | |
"""Builds a MySQL SELECT statement | |
""" | |
where_clause = "WHERE `%s` = %%s" % where_field | |
query = "SELECT COUNT(`id`) AS total FROM %s %s" % ( | |
tbl, where_clause | |
) | |
cur.execute(query, (where,)) | |
return cur.rownumber != 0 |