Skip to content

Instantly share code, notes, and snippets.

@brianvanderlugt
Last active July 3, 2018 14:40
Show Gist options
  • Save brianvanderlugt/34c5c39f79afd6510319e433405e36ef to your computer and use it in GitHub Desktop.
Save brianvanderlugt/34c5c39f79afd6510319e433405e36ef to your computer and use it in GitHub Desktop.
TDS Error on SQL Data Warehouse
import atexit
import ctds
import os
DATAWAREHOUSE_HOST = os.environ.get('DATAWAREHOUSE_HOST') # Server
DATAWAREHOUSE_NAME = os.environ.get('DATAWAREHOUSE_NAME') # DBName
DATAWAREHOUSE_USER = os.environ.get('DATAWAREHOUSE_USER')
DATAWAREHOUSE_PASSWORD = os.environ.get('DATAWAREHOUSE_PASSWORD')
connection = ctds.connect(
DATAWAREHOUSE_HOST,
user=DATAWAREHOUSE_USER,
password=DATAWAREHOUSE_PASSWORD,
database=DATAWAREHOUSE_NAME,
autocommit=True,
tds_version='7.3',
timeout=60 * 60
)
data = [
[74, "EAB79-0080E1032342BEC2", "TestClass", 1509321600, "2018-08-18", 0, 0],
[74, "EAB79-0080E1032342BEC2", "TestClass", 1509374160, "2018-08-18", 52560, None],
[74, "EAB79-0080E1032342BEC2", "TestClass", 1509374248, "2018-08-18", 52648, 0],
[74, "EAB79-0080E1032342BEC2", "TestClass", 1509374464, "2018-08-18", 52864, None]
]
# create table
create_statement = '''CREATE TABLE test_table
(
column1 BIGINT NOT NULL,
column2 VARCHAR(100) NOT NULL,
column3 VARCHAR(45) NOT NULL,
column4 BIGINT NOT NULL,
column5 DATE NOT NULL,
column6 BIGINT NOT NULL,
column7 TINYINT NULL
)'''
drop_statement = "DROP TABLE test_table"
def cleanup():
print("Cleaning up...")
connection.cursor().execute(drop_statement)
atexit.register(cleanup)
connection.autocommit = True
connection.cursor().execute(create_statement)
table = "test_table"
row_count = connection.bulk_insert(table, data)
print(len(row_count))
# docker build -f Dockerfile -t datawarehous-insert-test .
# docker run -it --rm --env-file=env.azure datawarehous-insert-test
# env.azure has the connection details
FROM python:3.6-jessie
WORKDIR /usr/src/app
COPY install_ctds.sh requirements.txt ./
RUN ["/bin/bash", "-c", "./install_ctds.sh"]
RUN pip install --no-cache-dir -r requirements.txt
COPY datawarehouse_test.py ./
CMD ["python","datawarehouse_test.py"]
#!/bin/bash
apt-get install libc6-dev
wget 'ftp://ftp.freetds.org/pub/freetds/stable/freetds-patched.tar.gz'
tar -xzf freetds-patched.tar.gz
pushd freetds-*
./configure --prefix "$(dirname $(pwd))" --with-openssl='/usr/lib/ssl' && make && make install
popd
echo "FreeTDS Installation complete."
pip install --global-option=build_ext \
--global-option="--include-dirs=$(pwd)/include" \
--global-option=build_ext \
--global-option="--library-dirs=$(pwd)/lib" \
--global-option=build_ext --global-option="--rpath=./lib" \
ctds
echo "CTDS installation complete."
exit 0
alembic==0.9.1
appdirs==1.4.3
appnope==0.1.0
arrow==0.10.0
chardet==2.3.0
click==6.7
cycler==0.10.0
dataset==0.8.0
decorator==4.0.11
dill==0.2.6
dj-database-url==0.4.2
Django==1.10
django-mysql==1.1.0
enum34==1.1.6
future==0.16.0
gapic-google-cloud-logging-v2==0.91.3
google-auth==0.8.0
google-auth-httplib2==0.0.2
google-cloud-bigquery==0.23.0
google-cloud-core==0.23.1
google-cloud-logging==0.23.0
google-gax==0.15.8
googleapis-common-protos==1.5.2
grpcio==1.1.3
httplib2==0.10.3
infinity==1.4
intervals==0.8.0
ipdb==0.10.2
ipython==5.3.0
ipython-genutils==0.2.0
Mako==1.0.6
MarkupSafe==1.0
matplotlib==2.0.0
mysqlclient==1.3.10
normality==0.4.0
numpy==1.12.1
oauth2client==3.0.0
packaging==16.8
pandas==0.19.2
pexpect==4.2.1
pickleshare==0.7.4
ply==3.8
prompt-toolkit==1.0.13
proto-google-cloud-logging-v2==0.91.3
protobuf==3.2.0
ptyprocess==0.5.1
pyasn1==0.2.3
pyasn1-modules==0.0.8
Pygments==2.2.0
pyparsing==2.2.0
python-dateutil==2.6.0
python-editor==1.0.3
pytz==2016.10
PyYAML==3.12
rsa==3.4.2
scipy==0.19.0
seaborn==0.7.1
simplegeneric==0.8.1
six==1.10.0
sortedcontainers==1.5.7
SQLAlchemy==1.1.6
traitlets==4.3.2
wcwidth==0.1.7
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment