Skip to content

Instantly share code, notes, and snippets.

@rms1000watt
Last active May 31, 2019 00:10
Show Gist options
  • Save rms1000watt/c28dcd812b555f7a6f840b7d7d4d41e6 to your computer and use it in GitHub Desktop.
Save rms1000watt/c28dcd812b555f7a6f840b7d7d4d41e6 to your computer and use it in GitHub Desktop.
Create Table in AWS Athena for ALB -> S3 records
CREATE EXTERNAL TABLE IF NOT EXISTS alb_table_name_here (
type string,
time string,
elb string,
client_ip string,
client_port int,
target_ip string,
target_port int,
request_processing_time double,
target_processing_time double,
response_processing_time double,
elb_status_code string,
target_status_code string,
received_bytes bigint,
sent_bytes bigint,
request_verb string,
request_url string,
request_proto string,
user_agent string,
ssl_cipher string,
ssl_protocol string,
target_group_arn string,
trace_id string,
domain_name string,
chosen_cert_arn string,
matched_rule_priority string,
request_creation_time string,
actions_executed string,
redirect_url string,
lambda_error_reason string,
new_field string
)
PARTITIONED BY (year string, month string, day string)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = '1',
'input.regex' =
'([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) ([^ ]*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\"($| \"[^ ]*\")(.*)')
LOCATION 's3://name-of-s3-bucket/path/to/partition/root';
ALTER TABLE app_api_alb_prod ADD PARTITION (year='2019',month='05',day='30') location 's3://name-of-s3-bucket/path/to/parition/root/2019/05/30'
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment