Created
November 3, 2024 09:33
-
-
Save huynhbaoan/1281f774a1352a8f12a5905d8bfc1a00 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def wait_for_all_queries_to_complete(query_ids, timeout=3600): | |
start_time = time.time() | |
sys.stdout.write('\tQuerying: ') | |
while True: | |
all_complete = True # This will only remain True if all queries are complete | |
try: | |
# Describe all active queries for the log group | |
response = LOG_CLIENT.describe_queries(logGroupName=LOG_GROUP) | |
except Exception as e: | |
print(f"\nError describing queries: {e}") | |
all_complete = False | |
time.sleep(5) | |
continue | |
# Check the status of each query in our list | |
for query_id in query_ids: | |
# Find the status of the specific query_id | |
query_status = next((q['status'] for q in response['queries'] if q['queryId'] == query_id), None) | |
if query_status is None: | |
print(f"\nWarning: Query ID {query_id} not found.") | |
query_ids.remove(query_id) | |
continue # Move on to the next query | |
if query_status in ['Failed', 'Cancelled', 'Timeout']: | |
print(f"\nQuery {query_id} {query_status}. Skipping this query.") | |
query_ids.remove(query_id) # Remove failed query from list | |
continue # Move on to the next query | |
elif query_status != 'Complete': | |
all_complete = False # At least one query is not complete | |
# Exit loop if all queries are confirmed complete | |
if all_complete: | |
break | |
# Check for timeout | |
if time.time() - start_time > timeout: | |
print("\nWarning: Timeout reached. Some queries may still be incomplete.") | |
break | |
sys.stdout.write('.') | |
sys.stdout.flush() | |
time.sleep(5) # Check more frequently if short-running queries are expected | |
print("\n\tTotal querying time: %s seconds" % (time.time() - start_time)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment