- Setup aws profile for both source and destination
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import argparse | |
import logging | |
import os | |
import sys | |
parser = argparse.ArgumentParser(description='Read a teflon event file') | |
parser.add_argument("--log", choices=["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], default="INFO", | |
help="Log level") | |
args = parser.parse_args() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import re | |
list_one = ["apple", "orange", "banana", "pineapple", "watermelon"] | |
list_two = ["apple", "water"] | |
filter_func = lambda x, lst: all(i not in x for i in lst) | |
select_func = lambda x, lst: any(i in x for i in lst) | |
def intercept(list_one, list_two, select=True): | |
func = select_func if select else filter_func | |
return filter(lambda x: func(x, list_two), list_one) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class DictAsObject: | |
def __init__(self, dic): | |
self._dict = dic | |
def __getattr__(self, name): | |
if name == "value": | |
return self._dict | |
v = self._dict[name] | |
return DictAsObject(v) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
function es_proxy () { | |
if [ ! -z $1 ] | |
then | |
profile="-e AWS_PROFILE=$1" | |
echo $1 profile will be used | |
fi | |
if [ -z $REGION ] | |
then | |
REGION="us-west-2" | |
echo $REGION region is used |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from flask import Flask, request | |
app = Flask(__name__) | |
import logging, sys | |
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) | |
logger = logging.getLogger(__name__) | |
@app.route("/", methods=["POST", "GET"]) | |
def call(): |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import boto3 | |
from requests_aws4auth import AWS4Auth | |
from elasticsearch import Elasticsearch, RequestsHttpConnection | |
class AwsElasticSearchClient(Elasticsearch): | |
def __init__(self, **args): | |
session = boto3.Session() | |
creds = session.get_credentials().get_frozen_credentials() | |
awsauth = AWS4Auth( | |
creds.access_key, creds.secret_key, args["region"], "es", session_token=creds.token |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
import base64 | |
def each_data(stream): | |
for line in stream.readlines(): | |
data = json.loads(line) | |
data['errorMessage'] = json.loads(data['errorMessage']) | |
yield data, json.loads(base64.b64decode(data['rawData']).decode()) | |
import boto3 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class Record: | |
def __init__(self, record): | |
self.record = record | |
def decoded_data(self): | |
return base64.b64decode(self.record['data']) | |
def merge(self, new_data): | |
return { | |
'recordId': self.record['recordId'], |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
read -s PASSWORD && security -v unlock-keychain -p $PASSWORD "/Users/plee/Library/Keychains/login.keychain-db" 2>&1 | grep -v 'unlock-keychain' |