Last active
January 29, 2022 14:10
-
-
Save martinyung/8210204d74d6c160e0bbbabe10f0925c to your computer and use it in GitHub Desktop.
python etl
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def main(): | |
print('starting etl') | |
# establish connection for target database (sql-server) | |
target_cnx = pyodbc.connect(**datawarehouse_db_config) | |
# loop through credentials | |
# mysql | |
for config in mysql_db_config: | |
try: | |
print("loading db: " + config['database']) | |
etl_process(mysql_queries, target_cnx, config, 'mysql') | |
except Exception as error: | |
print("etl for {} has error".format(config['database'])) | |
print('error message: {}'.format(error)) | |
continue | |
# sql-server | |
for config in sqlserver_db_config: | |
try: | |
print("loading db: " + config['database']) | |
etl_process(sqlserver_queries, target_cnx, config, 'sqlserver') | |
except Exception as error: | |
print("etl for {} has error".format(config['database'])) | |
print('error message: {}'.format(error)) | |
continue | |
# firebird | |
for config in fbd_db_config: | |
try: | |
print("loading db: " + config['database']) | |
etl_process(fbd_queries, target_cnx, config, 'firebird') | |
except Exception as error: | |
print("etl for {} has error".format(config['database'])) | |
print('error message: {}'.format(error)) | |
continue | |
target_cnx.close() | |
if __name__ == "__main__": | |
main() |
i did this gist for a blog, refer tot he full blog to better understand it. thanks.
https://codeburst.io/using-python-script-for-data-etl-53138c567906
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Isn't you have to import other modules and variables?