Last active
July 3, 2018 14:40
-
-
Save brianvanderlugt/34c5c39f79afd6510319e433405e36ef to your computer and use it in GitHub Desktop.
TDS Error on SQL Data Warehouse
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import atexit | |
import ctds | |
import os | |
DATAWAREHOUSE_HOST = os.environ.get('DATAWAREHOUSE_HOST') # Server | |
DATAWAREHOUSE_NAME = os.environ.get('DATAWAREHOUSE_NAME') # DBName | |
DATAWAREHOUSE_USER = os.environ.get('DATAWAREHOUSE_USER') | |
DATAWAREHOUSE_PASSWORD = os.environ.get('DATAWAREHOUSE_PASSWORD') | |
connection = ctds.connect( | |
DATAWAREHOUSE_HOST, | |
user=DATAWAREHOUSE_USER, | |
password=DATAWAREHOUSE_PASSWORD, | |
database=DATAWAREHOUSE_NAME, | |
autocommit=True, | |
tds_version='7.3', | |
timeout=60 * 60 | |
) | |
data = [ | |
[74, "EAB79-0080E1032342BEC2", "TestClass", 1509321600, "2018-08-18", 0, 0], | |
[74, "EAB79-0080E1032342BEC2", "TestClass", 1509374160, "2018-08-18", 52560, None], | |
[74, "EAB79-0080E1032342BEC2", "TestClass", 1509374248, "2018-08-18", 52648, 0], | |
[74, "EAB79-0080E1032342BEC2", "TestClass", 1509374464, "2018-08-18", 52864, None] | |
] | |
# create table | |
create_statement = '''CREATE TABLE test_table | |
( | |
column1 BIGINT NOT NULL, | |
column2 VARCHAR(100) NOT NULL, | |
column3 VARCHAR(45) NOT NULL, | |
column4 BIGINT NOT NULL, | |
column5 DATE NOT NULL, | |
column6 BIGINT NOT NULL, | |
column7 TINYINT NULL | |
)''' | |
drop_statement = "DROP TABLE test_table" | |
def cleanup(): | |
print("Cleaning up...") | |
connection.cursor().execute(drop_statement) | |
atexit.register(cleanup) | |
connection.autocommit = True | |
connection.cursor().execute(create_statement) | |
table = "test_table" | |
row_count = connection.bulk_insert(table, data) | |
print(len(row_count)) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# docker build -f Dockerfile -t datawarehous-insert-test . | |
# docker run -it --rm --env-file=env.azure datawarehous-insert-test | |
# env.azure has the connection details | |
FROM python:3.6-jessie | |
WORKDIR /usr/src/app | |
COPY install_ctds.sh requirements.txt ./ | |
RUN ["/bin/bash", "-c", "./install_ctds.sh"] | |
RUN pip install --no-cache-dir -r requirements.txt | |
COPY datawarehouse_test.py ./ | |
CMD ["python","datawarehouse_test.py"] |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
apt-get install libc6-dev | |
wget 'ftp://ftp.freetds.org/pub/freetds/stable/freetds-patched.tar.gz' | |
tar -xzf freetds-patched.tar.gz | |
pushd freetds-* | |
./configure --prefix "$(dirname $(pwd))" --with-openssl='/usr/lib/ssl' && make && make install | |
popd | |
echo "FreeTDS Installation complete." | |
pip install --global-option=build_ext \ | |
--global-option="--include-dirs=$(pwd)/include" \ | |
--global-option=build_ext \ | |
--global-option="--library-dirs=$(pwd)/lib" \ | |
--global-option=build_ext --global-option="--rpath=./lib" \ | |
ctds | |
echo "CTDS installation complete." | |
exit 0 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
alembic==0.9.1 | |
appdirs==1.4.3 | |
appnope==0.1.0 | |
arrow==0.10.0 | |
chardet==2.3.0 | |
click==6.7 | |
cycler==0.10.0 | |
dataset==0.8.0 | |
decorator==4.0.11 | |
dill==0.2.6 | |
dj-database-url==0.4.2 | |
Django==1.10 | |
django-mysql==1.1.0 | |
enum34==1.1.6 | |
future==0.16.0 | |
gapic-google-cloud-logging-v2==0.91.3 | |
google-auth==0.8.0 | |
google-auth-httplib2==0.0.2 | |
google-cloud-bigquery==0.23.0 | |
google-cloud-core==0.23.1 | |
google-cloud-logging==0.23.0 | |
google-gax==0.15.8 | |
googleapis-common-protos==1.5.2 | |
grpcio==1.1.3 | |
httplib2==0.10.3 | |
infinity==1.4 | |
intervals==0.8.0 | |
ipdb==0.10.2 | |
ipython==5.3.0 | |
ipython-genutils==0.2.0 | |
Mako==1.0.6 | |
MarkupSafe==1.0 | |
matplotlib==2.0.0 | |
mysqlclient==1.3.10 | |
normality==0.4.0 | |
numpy==1.12.1 | |
oauth2client==3.0.0 | |
packaging==16.8 | |
pandas==0.19.2 | |
pexpect==4.2.1 | |
pickleshare==0.7.4 | |
ply==3.8 | |
prompt-toolkit==1.0.13 | |
proto-google-cloud-logging-v2==0.91.3 | |
protobuf==3.2.0 | |
ptyprocess==0.5.1 | |
pyasn1==0.2.3 | |
pyasn1-modules==0.0.8 | |
Pygments==2.2.0 | |
pyparsing==2.2.0 | |
python-dateutil==2.6.0 | |
python-editor==1.0.3 | |
pytz==2016.10 | |
PyYAML==3.12 | |
rsa==3.4.2 | |
scipy==0.19.0 | |
seaborn==0.7.1 | |
simplegeneric==0.8.1 | |
six==1.10.0 | |
sortedcontainers==1.5.7 | |
SQLAlchemy==1.1.6 | |
traitlets==4.3.2 | |
wcwidth==0.1.7 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment