Skip to content

Instantly share code, notes, and snippets.

@onema
Last active March 22, 2018 16:05
Show Gist options
  • Save onema/0a8587d290c883ac4652c6b1e8555d7f to your computer and use it in GitHub Desktop.
Save onema/0a8587d290c883ac4652c6b1e8555d7f to your computer and use it in GitHub Desktop.
My own solution of the LambdaSharp challenge March2018-DynamoDB
// Using sbt 1.1.1
lazy val root = (project in file("."))
.settings(
organization := "onema",
name := "lambda-dynamodb-meetup",
version := "0.1.0",
scalaVersion := "2.12.5",
libraryDependencies ++= {
Seq(
// AWS Clients and Events
"com.amazonaws" % "aws-lambda-java-core" % "1.2.0",
"com.amazonaws" % "aws-java-sdk-s3" % "1.11.298",
"com.amazonaws" % "aws-java-sdk-dynamodb" % "1.11.298",
"com.amazonaws" % "aws-lambda-java-events" % "2.1.0",
// Logging
"com.typesafe.scala-logging"% "scala-logging_2.12" % "3.7.2",
"ch.qos.logback" % "logback-classic" % "1.1.7",
// Testing
"org.scalatest" %% "scalatest" % "3.0.4" % "test",
"org.mockito" % "mockito-core" % "2.12.0" % "test"
)
}
)
// Assembly
assemblyJarName in assembly := "app.jar"
Parameters:
TableName:
Type: String
Default: LambdaDynamodbMeetup
Description: The name of the dynamodb table
PartitionKey:
Type: String
Default: Country
Description: The name of the main parition key
SortKey:
Type: String
Default: Time
Description: The name of the primary sort key
ReadCapacityUnits:
Default: 5
Description: Dynamo read capacity units
Type: Number
WriteCapacityUnits:
Default: 5
Description: Dynamo write capacity units
Type: Number
Resources:
DynamoDBTable:
Type: AWS::DynamoDB::Table
Properties:
TableName:
Ref: TableName
AttributeDefinitions:
- AttributeName:
Ref: PartitionKey
AttributeType: S
- AttributeName:
Ref: SortKey
AttributeType: S
KeySchema:
- AttributeName:
Ref: PartitionKey
KeyType: HASH
- AttributeName:
Ref: SortKey
KeyType: RANGE
ProvisionedThroughput:
ReadCapacityUnits:
Ref: ReadCapacityUnits
WriteCapacityUnits:
Ref: WriteCapacityUnits
TimeToLiveSpecification:
AttributeName: ExpirationTime
Enabled: true
/**
* See https://github.com/LambdaSharp/March2018-DynamoDB
*/
package onema
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsyncClientBuilder
import com.amazonaws.services.lambda.runtime.Context
import collection.JavaConverters._
import com.amazonaws.services.lambda.runtime.events.S3Event
import com.amazonaws.services.s3.AmazonS3ClientBuilder
import com.typesafe.scalalogging.Logger
import onema.core.json.Implicits._
class Function {
//--- Fields ---
protected val log = Logger("lambda-handler")
private val dynamoClient = AmazonDynamoDBAsyncClientBuilder.defaultClient()
private val s3Client = AmazonS3ClientBuilder.defaultClient()
//--- Methods ---
def lambdaHandler(event: S3Event, context: Context): Unit = {
log.info(event.javaClassToJson)
val s3Info = event.getRecords.asScala.head.getS3
val logic = new Logic(s3Info, dynamoClient, s3Client)
logic.handleRequest()
}
}
package onema
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB
import com.amazonaws.services.dynamodbv2.document.{DynamoDB, Item, PutItemOutcome}
import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.event.S3EventNotification.S3Entity
import com.amazonaws.services.s3.model.S3Object
import com.typesafe.scalalogging.Logger
import onema.Logic._
import onema.core.json.Implicits._
import scala.io.Source
object Logic {
//--- Classes ---
case class TimelineData(time: String, formattedTime: String, formattedAxisTime: String, value: List[Double], hasData: List[Boolean], formattedValue: List[String])
case class Default( timelineData: List[TimelineData], averages: List[String])
case class R00tJsonObject(default: Default)
case class Stats(max: Double, min: Double, average: Double, standardDeviation: Double)
implicit class ListStats(list: List[Double]) {
//--- Methods ---
val average: Double = list.sum / list.size
val mean: Double = average
def standardDeviation: Double = math.sqrt(list.map(x => x - mean).sum/list.size)
}
}
class Logic(val s3Info: S3Entity, val dynamoClient: AmazonDynamoDB, val s3Client: AmazonS3) {
//--- Fields ---
protected val log = Logger("lambda-handler")
private val filename = s3Info.getObject.getKey
private val bucket = s3Info.getBucket.getName
private val dynabodb = new DynamoDB(dynamoClient)
val country: String = filename.split('.').head.split('-').last
val keyword: String = filename.split('-').head
//--- Methods ---
def handleRequest(): Unit = {
val data = content
log.info(data)
log.info(s"Country $country")
log.info(s"Keyword $keyword")
val root = data.jsonParse[R00tJsonObject]
root.default.timelineData.foreach(recordEntry)
}
def content: String = {
val response: S3Object = s3Client.getObject(bucket, filename)
val objectData = response.getObjectContent
Source.fromInputStream(objectData).mkString
}
def recordEntry(data: TimelineData): PutItemOutcome = {
val stats = Stats(data.value.max, data.value.min, data.value.average, data.value.standardDeviation).toJson
val table = dynabodb.getTable("LambdaDynamodbMeetup")
val item = new Item()
.withPrimaryKey("Country", country)
.withString("Time", data.time)
.withString("Value", data.formattedValue.head)
.withString("Keyword", keyword)
.withJSON("Stats", stats)
table.putItem(item)
}
}
# Deploy using: serverless deploy --stage your-name
# The bucket created is: "<your-name>-lambda-dynamodb-meetup"
# To remove: empty bucket and then run: serverless remove --stage your-name
service: lamdba-dynamo-meetup
provider:
name: aws
runtime: java8
profile: ${opt:profile, 'default'}
timeout: 30
versionFunctions: false
# you can overwrite defaults here
stage: dev
region: us-east-1
# you can add statements to the Lambda function's IAM Role here
iamRoleStatements:
- Effect: Allow
Action:
- dynamodb:PutItem
- dynamodb:GetItem
- dynamodb:Query
Resource:
- Fn::GetAtt: [ DynamoDBTable, Arn ]
- Fn::Join: ["/", [ Fn::GetAtt: [DynamoDBTable, Arn], "index", "*"]]
- Effect: Allow
Action:
- s3:*
Resource:
- "arn:aws:s3:::${self:custom.bucketName}/"
- "arn:aws:s3:::${self:custom.bucketName}/*"
# you can define service wide environment variables here
environment:
ENVIRONMENT_NAME: ${self:custom.environmentName}
BUCKET_NAME: ${self:custom.bucketName}
# Custom values. These can be referenced in the Cloud Formation template
custom:
environmentName: ${opt:stage, self:provider.stage}
bucketName: "${self:custom.environmentName}-lambda-dynamodb-meetup"
# you can add packaging information here
# Make sure to run "sbt assembly" to create a jar file
# with all your dependencies and put that jar file name here.
package:
artifact: target/scala-2.12/app.jar
functions:
# functions
lambda-dynamo-meetup:
handler: onema.Function::lambdaHandler
events:
- s3: ${self:custom.bucketName}
# you can add CloudFormation resource templates here
resources:
Parameters:
TableName:
Type: String
Default: LambdaDynamodbMeetup
Description: The name of the dynamodb table
PartitionKey:
Type: String
Default: Country
Description: The name of the main parition key
SortKey:
Type: String
Default: Time
Description: The name of the primary sort key
ReadCapacityUnits:
Default: 5
Description: Dynamo read capacity units
Type: Number
WriteCapacityUnits:
Default: 5
Description: Dynamo write capacity units
Type: Number
Resources: ${file(dynamodb-table_cfn.yml):Resources}
@onema
Copy link
Author

onema commented Mar 22, 2018

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment