justinnaldzin

# Delete all tables matching a grep pattern in a BigQuery dataset

DATASET=my_dataset
TABLE_PATTERN=my_table_

# Confirm the table names before deleting
for TABLE in `bq ls --max_results=10000 $DATASET | grep TABLE | grep $TABLE_PATTERN | awk '{print $1}'`; do echo $TABLE; done

# Delete the tables; USE WITH CAUTION!
for TABLE in `bq ls --max_results=10000 $DATASET | grep TABLE | grep $TABLE_PATTERN | awk '{print $1}'`; do echo $TABLE; bq rm -f -t $DATASET.$TABLE; done

justinnaldzin

import os
import sys
import logging


FORMATTER = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(module)s - %(message)s')
LOG_FOLDER = 'log'


def get_console_handler():

justinnaldzin

LOG_GROUP='my/log/group'
AWS_PROFILE='my-profile'

# Delete all log streams within a log group
aws logs describe-log-streams --profile $AWS_PROFILE --log-group-name $LOG_GROUP --query 'logStreams[*].logStreamName' --output table | awk '{print $2}' | grep -v ^$ | while read x; do aws logs delete-log-stream --profile $AWS_PROFILE --log-group-name $LOG_GROUP --log-stream-name $x; done

# Delete log streams starting with string
aws logs describe-log-streams --profile $AWS_PROFILE --log-group-name $LOG_GROUP --query 'logStreams[?starts_with(logStreamName,`2020/04/3`)].logStreamName' --output table | awk '{print $2}' | grep -v ^$ | while read x; do aws logs delete-log-stream --profile $AWS_PROFILE --log-group-name $LOG_GROUP --log-stream-name $x; done

justinnaldzin

import boto3


def get_matching_s3_objects(bucket, prefix="", suffix=""):
    """
    Generate objects in an S3 bucket.

    :param bucket: Name of the S3 bucket.
    :param prefix: Only fetch objects whose key starts with this prefix (optional).
    :param suffix: Only fetch objects whose keys end with this suffix (optional).