Last active
March 22, 2018 16:05
-
-
Save onema/0a8587d290c883ac4652c6b1e8555d7f to your computer and use it in GitHub Desktop.
My own solution of the LambdaSharp challenge March2018-DynamoDB
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Using sbt 1.1.1 | |
lazy val root = (project in file(".")) | |
.settings( | |
organization := "onema", | |
name := "lambda-dynamodb-meetup", | |
version := "0.1.0", | |
scalaVersion := "2.12.5", | |
libraryDependencies ++= { | |
Seq( | |
// AWS Clients and Events | |
"com.amazonaws" % "aws-lambda-java-core" % "1.2.0", | |
"com.amazonaws" % "aws-java-sdk-s3" % "1.11.298", | |
"com.amazonaws" % "aws-java-sdk-dynamodb" % "1.11.298", | |
"com.amazonaws" % "aws-lambda-java-events" % "2.1.0", | |
// Logging | |
"com.typesafe.scala-logging"% "scala-logging_2.12" % "3.7.2", | |
"ch.qos.logback" % "logback-classic" % "1.1.7", | |
// Testing | |
"org.scalatest" %% "scalatest" % "3.0.4" % "test", | |
"org.mockito" % "mockito-core" % "2.12.0" % "test" | |
) | |
} | |
) | |
// Assembly | |
assemblyJarName in assembly := "app.jar" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Parameters: | |
TableName: | |
Type: String | |
Default: LambdaDynamodbMeetup | |
Description: The name of the dynamodb table | |
PartitionKey: | |
Type: String | |
Default: Country | |
Description: The name of the main parition key | |
SortKey: | |
Type: String | |
Default: Time | |
Description: The name of the primary sort key | |
ReadCapacityUnits: | |
Default: 5 | |
Description: Dynamo read capacity units | |
Type: Number | |
WriteCapacityUnits: | |
Default: 5 | |
Description: Dynamo write capacity units | |
Type: Number | |
Resources: | |
DynamoDBTable: | |
Type: AWS::DynamoDB::Table | |
Properties: | |
TableName: | |
Ref: TableName | |
AttributeDefinitions: | |
- AttributeName: | |
Ref: PartitionKey | |
AttributeType: S | |
- AttributeName: | |
Ref: SortKey | |
AttributeType: S | |
KeySchema: | |
- AttributeName: | |
Ref: PartitionKey | |
KeyType: HASH | |
- AttributeName: | |
Ref: SortKey | |
KeyType: RANGE | |
ProvisionedThroughput: | |
ReadCapacityUnits: | |
Ref: ReadCapacityUnits | |
WriteCapacityUnits: | |
Ref: WriteCapacityUnits | |
TimeToLiveSpecification: | |
AttributeName: ExpirationTime | |
Enabled: true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/** | |
* See https://github.com/LambdaSharp/March2018-DynamoDB | |
*/ | |
package onema | |
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsyncClientBuilder | |
import com.amazonaws.services.lambda.runtime.Context | |
import collection.JavaConverters._ | |
import com.amazonaws.services.lambda.runtime.events.S3Event | |
import com.amazonaws.services.s3.AmazonS3ClientBuilder | |
import com.typesafe.scalalogging.Logger | |
import onema.core.json.Implicits._ | |
class Function { | |
//--- Fields --- | |
protected val log = Logger("lambda-handler") | |
private val dynamoClient = AmazonDynamoDBAsyncClientBuilder.defaultClient() | |
private val s3Client = AmazonS3ClientBuilder.defaultClient() | |
//--- Methods --- | |
def lambdaHandler(event: S3Event, context: Context): Unit = { | |
log.info(event.javaClassToJson) | |
val s3Info = event.getRecords.asScala.head.getS3 | |
val logic = new Logic(s3Info, dynamoClient, s3Client) | |
logic.handleRequest() | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package onema | |
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB | |
import com.amazonaws.services.dynamodbv2.document.{DynamoDB, Item, PutItemOutcome} | |
import com.amazonaws.services.s3.AmazonS3 | |
import com.amazonaws.services.s3.event.S3EventNotification.S3Entity | |
import com.amazonaws.services.s3.model.S3Object | |
import com.typesafe.scalalogging.Logger | |
import onema.Logic._ | |
import onema.core.json.Implicits._ | |
import scala.io.Source | |
object Logic { | |
//--- Classes --- | |
case class TimelineData(time: String, formattedTime: String, formattedAxisTime: String, value: List[Double], hasData: List[Boolean], formattedValue: List[String]) | |
case class Default( timelineData: List[TimelineData], averages: List[String]) | |
case class R00tJsonObject(default: Default) | |
case class Stats(max: Double, min: Double, average: Double, standardDeviation: Double) | |
implicit class ListStats(list: List[Double]) { | |
//--- Methods --- | |
val average: Double = list.sum / list.size | |
val mean: Double = average | |
def standardDeviation: Double = math.sqrt(list.map(x => x - mean).sum/list.size) | |
} | |
} | |
class Logic(val s3Info: S3Entity, val dynamoClient: AmazonDynamoDB, val s3Client: AmazonS3) { | |
//--- Fields --- | |
protected val log = Logger("lambda-handler") | |
private val filename = s3Info.getObject.getKey | |
private val bucket = s3Info.getBucket.getName | |
private val dynabodb = new DynamoDB(dynamoClient) | |
val country: String = filename.split('.').head.split('-').last | |
val keyword: String = filename.split('-').head | |
//--- Methods --- | |
def handleRequest(): Unit = { | |
val data = content | |
log.info(data) | |
log.info(s"Country $country") | |
log.info(s"Keyword $keyword") | |
val root = data.jsonParse[R00tJsonObject] | |
root.default.timelineData.foreach(recordEntry) | |
} | |
def content: String = { | |
val response: S3Object = s3Client.getObject(bucket, filename) | |
val objectData = response.getObjectContent | |
Source.fromInputStream(objectData).mkString | |
} | |
def recordEntry(data: TimelineData): PutItemOutcome = { | |
val stats = Stats(data.value.max, data.value.min, data.value.average, data.value.standardDeviation).toJson | |
val table = dynabodb.getTable("LambdaDynamodbMeetup") | |
val item = new Item() | |
.withPrimaryKey("Country", country) | |
.withString("Time", data.time) | |
.withString("Value", data.formattedValue.head) | |
.withString("Keyword", keyword) | |
.withJSON("Stats", stats) | |
table.putItem(item) | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Deploy using: serverless deploy --stage your-name | |
# The bucket created is: "<your-name>-lambda-dynamodb-meetup" | |
# To remove: empty bucket and then run: serverless remove --stage your-name | |
service: lamdba-dynamo-meetup | |
provider: | |
name: aws | |
runtime: java8 | |
profile: ${opt:profile, 'default'} | |
timeout: 30 | |
versionFunctions: false | |
# you can overwrite defaults here | |
stage: dev | |
region: us-east-1 | |
# you can add statements to the Lambda function's IAM Role here | |
iamRoleStatements: | |
- Effect: Allow | |
Action: | |
- dynamodb:PutItem | |
- dynamodb:GetItem | |
- dynamodb:Query | |
Resource: | |
- Fn::GetAtt: [ DynamoDBTable, Arn ] | |
- Fn::Join: ["/", [ Fn::GetAtt: [DynamoDBTable, Arn], "index", "*"]] | |
- Effect: Allow | |
Action: | |
- s3:* | |
Resource: | |
- "arn:aws:s3:::${self:custom.bucketName}/" | |
- "arn:aws:s3:::${self:custom.bucketName}/*" | |
# you can define service wide environment variables here | |
environment: | |
ENVIRONMENT_NAME: ${self:custom.environmentName} | |
BUCKET_NAME: ${self:custom.bucketName} | |
# Custom values. These can be referenced in the Cloud Formation template | |
custom: | |
environmentName: ${opt:stage, self:provider.stage} | |
bucketName: "${self:custom.environmentName}-lambda-dynamodb-meetup" | |
# you can add packaging information here | |
# Make sure to run "sbt assembly" to create a jar file | |
# with all your dependencies and put that jar file name here. | |
package: | |
artifact: target/scala-2.12/app.jar | |
functions: | |
# functions | |
lambda-dynamo-meetup: | |
handler: onema.Function::lambdaHandler | |
events: | |
- s3: ${self:custom.bucketName} | |
# you can add CloudFormation resource templates here | |
resources: | |
Parameters: | |
TableName: | |
Type: String | |
Default: LambdaDynamodbMeetup | |
Description: The name of the dynamodb table | |
PartitionKey: | |
Type: String | |
Default: Country | |
Description: The name of the main parition key | |
SortKey: | |
Type: String | |
Default: Time | |
Description: The name of the primary sort key | |
ReadCapacityUnits: | |
Default: 5 | |
Description: Dynamo read capacity units | |
Type: Number | |
WriteCapacityUnits: | |
Default: 5 | |
Description: Dynamo write capacity units | |
Type: Number | |
Resources: ${file(dynamodb-table_cfn.yml):Resources} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Here is the original challenge https://github.com/LambdaSharp/March2018-DynamoDB