Created
December 19, 2013 10:26
-
-
Save shantanuo/8037193 to your computer and use it in GitHub Desktop.
Amazon Kinesis is a managed service that scales elastically for real time processing of streaming big data. Here is python script to operate the basic functions.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # put and get data | |
| import boto | |
| mykin = boto.connect_kinesis(aws_access_key_id='access', | |
| aws_secret_access_key='passWord') | |
| myput = mykin.put_record(stream_name='mytest', data='abcdefghij223344', | |
| partition_key='parti11223344', b64_encode=True) | |
| myiterator = mykin.get_shard_iterator(stream_name='mytest', | |
| shard_id='shardId-000000000000', | |
| shard_iterator_type='TRIM_HORIZON') | |
| myget=mykin.get_records(shard_iterator=myiterator['ShardIterator']) | |
| # store the data to mysql | |
| import MySQLdb as mdb | |
| con = mdb.connect('localhost', 'dba', 'dba', 'test'); | |
| cur = con.cursor() | |
| sql = "INSERT INTO aws ( `PartitionKey`, `Data`, `SequenceNumber` ) VALUES ( %(PartitionKey)s, %(Data)s, %(SequenceNumber)s )" | |
| cur.executemany( sql, myget['Records'] ) | |
| con.commit() | |
| """ | |
| # mysql table structure: | |
| drop table aws; | |
| create table aws ( | |
| PartitionKey varchar(255), | |
| Data varchar(255), | |
| SequenceNumber varchar(255) | |
| ); | |
| """ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment