Working with AWS Lambda and Chalice

_README.md

AWS Lambda

1. Create a new bucket in S3
2. Create a local Python project and activate a virtual environment.
3. Install the following packages:
   • boto3
   • chalice
4. Create a new DynamoDB table named s3-linecount with a partition key named fileid (a STRING).
5. Open your AWS Console.

app.py

import os
import boto3
from chalice import Chalice

app = Chalice(app_name='s3-linecount')
app.debug = True

# Set the value of BUCKET_NAME in the .chalice/config.json file.
S3_BUCKET = os.environ.get('BUCKET_NAME', '')

s3 = boto3.client('s3')
ddb = boto3.resource('dynamodb')
table = ddb.Table('s3-linecount')

@app.on_s3_event(bucket=S3_BUCKET, events=['s3:ObjectCreated:*'])
def s3_handler(event):
    app.log.debug("Received event for bucket: %s, key: %s",
                  event.bucket, event.key)

    response = s3.get_object(Bucket=event.bucket, Key=event.key)
    body = response['Body'].read().decode('utf-8')
    line_count = len(body.splitlines())

    table.put_item(
        Item={
            'fileid': event.key,
            'line_count': line_count,
        }
    )

    app.log.debug("Wrote %d lines for %s to s3-linecount table",
                  line_count, event.key)

generate-files.py

"""
Generates a CSV file with a random 12-character filename,
fills it with a random number of lines of random words,
uploads it to an S3 bucket, and deletes the local file.

Usage:
    python generate-and-upload.py <bucket-name> <count>
"""

import sys
import os
import random
import string
import boto3

WORDS = [
    "apple", "river", "mountain", "cloud", "forest", "bridge", "garden",
    "silver", "thunder", "meadow", "crystal", "voyage", "harbor", "sunset",
    "lantern", "compass", "falcon", "marble", "shadow", "breeze", "canyon",
    "ember", "glacier", "horizon", "island", "jungle", "kettle", "lighthouse",
    "nectar", "orchid", "pebble", "quartz", "ribbon", "sapphire", "tornado",
    "umbrella", "velvet", "willow", "zenith", "anchor", "blossom", "comet",
    "dolphin", "eclipse", "fountain", "granite", "hummingbird", "ivory",
]


def generate_filename():
    """Return a random 12-character alphanumeric filename with .csv extension."""
    chars = string.ascii_lowercase + string.digits
    name = "".join(random.choices(chars, k=12))
    return f"{name}.csv"


def generate_csv(filename):
    """Write a CSV with a random number of lines (100-700) of random words."""
    num_lines = random.randint(100, 700)
    with open(filename, "w") as f:
        f.write("col1,col2,col3,col4,col5\n")
        for _ in range(num_lines):
            line = ",".join(random.choices(WORDS, k=5))
            f.write(line + "\n")
    print(f"Created {filename} with {num_lines} lines")
    return filename


def upload_to_s3(filename, bucket):
    """Upload the file to the given S3 bucket and delete the local copy."""
    s3 = boto3.client("s3")
    s3.upload_file(filename, bucket, filename)
    print(f"Uploaded {filename} to s3://{bucket}/{filename}")
    os.remove(filename)
    print(f"Deleted local file {filename}")


if __name__ == "__main__":
    if len(sys.argv) != 3:
        print("Usage: python generate-and-upload.py <bucket-name> <count>")
        sys.exit(1)

    bucket_name = sys.argv[1]
    count = int(sys.argv[2])
    for i in range(count):
        fname = generate_filename()
        generate_csv(fname)
        upload_to_s3(fname, bucket_name)
    print(f"Done. Uploaded {count} files to s3://{bucket_name}/")

iam-policy.json

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "S3GetOnS3LinecountBucket",
      "Effect": "Allow",
      "Action": "s3:Get*",
      "Resource": [
        "arn:aws:s3:::s3-linecount",
        "arn:aws:s3:::s3-linecount/*"
      ]
    },
    {
      "Sid": "DynamoDBPutOnS3LinecountTable",
      "Effect": "Allow",
      "Action": "dynamodb:PutItem",
      "Resource": "arn:aws:dynamodb:*:*:table/s3-linecount"
    },
    {
      "Sid": "CloudWatchLogsForLambda",
      "Effect": "Allow",
      "Action": [
        "logs:CreateLogGroup",
        "logs:CreateLogStream",
        "logs:PutLogEvents"
      ],
      "Resource": "arn:aws:logs:*:*:*"
    }
  ]
}