-
-
Save infinityhacks/e87f661c1f1246aa26bce01e36f51cd7 to your computer and use it in GitHub Desktop.
Medium Blog - Athena S3 - python athena example
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import boto3 | |
import time | |
import re | |
client = boto3.client('athena') | |
# run athen query | |
response = client.start_query_execution( | |
QueryString=athena_query, | |
QueryExecutionContext={'Database': ATHENA_DATABASE_NAME}, | |
ResultConfiguration={ | |
'OutputLocation': ATHENA_RESULT_S3_BUCKET | |
} | |
) | |
# wait for query results for max 20 seconds | |
execution_id = response['QueryExecutionId'] | |
state = 'RUNNING' | |
max_wait = 20 | |
while (max_wait > 0 and state in ['RUNNING', 'QUEUED']): | |
max_wait = max_wait - 1 | |
response = client.get_query_execution( | |
QueryExecutionId=execution_id) | |
if 'QueryExecution' in response and \ | |
'Status' in response['QueryExecution'] and \ | |
'State' in response['QueryExecution']['Status']: | |
state = response['QueryExecution']['Status']['State'] | |
if state == 'SUCCEEDED': | |
s3_path = response['QueryExecution']['ResultConfiguration']['OutputLocation'] | |
filename = re.findall('.*\/(.*)', s3_path)[0] | |
return filename | |
time.sleep(1) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment