shreyasms17

def get_fields_in_json(self, json_schema):
    '''
    Description: 
    This function takes in the schema in json format and returns the metadata of the schema

    :param json_schema: [type : str] a string containing path to raw data
    :return fields: [type : dict] contains metadata of the schema
    '''
    a = json_schema.json()
    schema_dict = json.loads(a)

shreyasms17

class AutoFlatten:
    def __init__(self, json_schema):
        self.fields_in_json = self.get_fields_in_json(json_schema)
        self.all_fields = {}
        self.cols_to_explode = set()
        self.structure = {}
        self.order = []
        self.bottom_to_top = {}
        self.rest = set()

shreyasms17

import json

class AutoFlatten:
    def __init__(self, json_schema):
        self.fields_in_json = self.get_fields_in_json(json_schema)
        self.all_fields = {}
        self.cols_to_explode = set()
        self.structure = {}
        self.order = []
        self.bottom_to_top = {}

shreyasms17

---
Version: '2008-10-17'
Id: arn:aws:sqs:us-east-1:000000000000:PageVisitQueue/SQSDefaultPolicy
Statement:
- Sid: AllowedSQSPermissions
  Effect: Allow
  Principal:
    AWS: arn:aws:iam::111111111111:role/crossaccount_sqs_lambda_role
  Action:
    - SQS:ReceiveMessage

shreyasms17

import boto3
import traceback
import json
import os
import uuid

kinesis_client = boto3.client('kinesis')
config_file_path = 'sqs_to_kinesis_mapping.json'
dlq_stream_name = 'DLQStream'

shreyasms17

{
       "arn:aws:sqs:us-east-1:000000000000:PageVisitEventQueue": "PageVisitEventStream",
       "arn:aws:sqs:us-east-1:000000000000:ClickedEventQueue": "ClickedEventStream"
}

shreyasms17

---
Version: '2012-10-17'
Statement:
- Sid: VisualEditor0
  Effect: Allow
  Action:
    - ec2:DescribeSecurityGroups
    - ec2:DescribeSubnets
    - ec2:DescribeVpcs
    - ec2:DescribeNetworkInterfaces