shreyasms17

---
Version: '2012-10-17'
Statement:
- Sid: VisualEditor0
  Effect: Allow
  Action:
    - ec2:DescribeSecurityGroups
    - ec2:DescribeSubnets
    - ec2:DescribeVpcs
    - ec2:DescribeNetworkInterfaces

shreyasms17

{
       "arn:aws:sqs:us-east-1:000000000000:PageVisitEventQueue": "PageVisitEventStream",
       "arn:aws:sqs:us-east-1:000000000000:ClickedEventQueue": "ClickedEventStream"
}

shreyasms17

import boto3
import traceback
import json
import os
import uuid

kinesis_client = boto3.client('kinesis')
config_file_path = 'sqs_to_kinesis_mapping.json'
dlq_stream_name = 'DLQStream'

shreyasms17

---
Version: '2008-10-17'
Id: arn:aws:sqs:us-east-1:000000000000:PageVisitQueue/SQSDefaultPolicy
Statement:
- Sid: AllowedSQSPermissions
  Effect: Allow
  Principal:
    AWS: arn:aws:iam::111111111111:role/crossaccount_sqs_lambda_role
  Action:
    - SQS:ReceiveMessage

shreyasms17

import json

class AutoFlatten:
    def __init__(self, json_schema):
        self.fields_in_json = self.get_fields_in_json(json_schema)
        self.all_fields = {}
        self.cols_to_explode = set()
        self.structure = {}
        self.order = []
        self.bottom_to_top = {}

shreyasms17

class AutoFlatten:
    def __init__(self, json_schema):
        self.fields_in_json = self.get_fields_in_json(json_schema)
        self.all_fields = {}
        self.cols_to_explode = set()
        self.structure = {}
        self.order = []
        self.bottom_to_top = {}
        self.rest = set()

shreyasms17

def get_fields_in_json(self, json_schema):
    '''
    Description: 
    This function takes in the schema in json format and returns the metadata of the schema

    :param json_schema: [type : str] a string containing path to raw data
    :return fields: [type : dict] contains metadata of the schema
    '''
    a = json_schema.json()
    schema_dict = json.loads(a)

shreyasms17

def is_leaf(self, data):
    '''
    Description: 
    This function checks if the particular field in the schema is a leaf or not.
    Types not considered as a leaf : struct, array

    :param data: [type: dict] a dictionary containing metadata about a field

    :return leaf: [type: bool] indicates whether a given field is a leaf or not
    '''

shreyasms17

def unnest_dict(self, json, cur_path):
    '''
    Description: 
    This function unnests the dictionaries in the json schema recursively 
    and maps the hierarchical path to the field to the column name when it encounters a leaf node

    :param json: [type: dict] contains metadata about the field
    :param cur_path: [type: str] contains hierarchical path to that field, each parent separated by a '.'
    '''
    if self.is_leaf(json):

shreyasms17

def get_structure(self, col_list):
    '''
    Description: 
    This function gets the structure to the traversal to array field in the schema

    :param col_list: [type: list] contains list of fields that are to be exploded

    :return structure: [type: dict] contains the hierarchical mapping for array fields
    '''
    structure = {'json' : {}}