Skip to content

Instantly share code, notes, and snippets.

@huynhbaoan
Created November 3, 2024 09:33
Show Gist options
  • Save huynhbaoan/1281f774a1352a8f12a5905d8bfc1a00 to your computer and use it in GitHub Desktop.
Save huynhbaoan/1281f774a1352a8f12a5905d8bfc1a00 to your computer and use it in GitHub Desktop.
def wait_for_all_queries_to_complete(query_ids, timeout=3600):
start_time = time.time()
sys.stdout.write('\tQuerying: ')
while True:
all_complete = True # This will only remain True if all queries are complete
try:
# Describe all active queries for the log group
response = LOG_CLIENT.describe_queries(logGroupName=LOG_GROUP)
except Exception as e:
print(f"\nError describing queries: {e}")
all_complete = False
time.sleep(5)
continue
# Check the status of each query in our list
for query_id in query_ids:
# Find the status of the specific query_id
query_status = next((q['status'] for q in response['queries'] if q['queryId'] == query_id), None)
if query_status is None:
print(f"\nWarning: Query ID {query_id} not found.")
query_ids.remove(query_id)
continue # Move on to the next query
if query_status in ['Failed', 'Cancelled', 'Timeout']:
print(f"\nQuery {query_id} {query_status}. Skipping this query.")
query_ids.remove(query_id) # Remove failed query from list
continue # Move on to the next query
elif query_status != 'Complete':
all_complete = False # At least one query is not complete
# Exit loop if all queries are confirmed complete
if all_complete:
break
# Check for timeout
if time.time() - start_time > timeout:
print("\nWarning: Timeout reached. Some queries may still be incomplete.")
break
sys.stdout.write('.')
sys.stdout.flush()
time.sleep(5) # Check more frequently if short-running queries are expected
print("\n\tTotal querying time: %s seconds" % (time.time() - start_time))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment