Created
June 18, 2018 22:42
-
-
Save alexjurkiewicz/4a4b1e613cca6174ec3e10b91ec3e000 to your computer and use it in GitHub Desktop.
CPO ttyrec offload logic
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Compress old ttyrecs | |
| 0 17 * * * root find /opt/dgl-chroot/var/ttyrec/ -type f -mtime +1 -not -name '*.gz' -not -name '*.xz' -exec nice -n 19 xz -1 {} \; | |
| # Upload old ttyrecs | |
| 0 18 * * * root aws s3 sync --exclude '*' --include '*.xz' --include '*.gz' /space/ttyrec/ s3://dcss-cpo-ttyrecs/ | |
| # Delete old ttyrecs | |
| 0 19 * * * root find /space/ttyrec -type f -mtime +7 -delete |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash -eu | |
| # A CGI script run as FastCGI (via fcgiwrap) to list a user's local and remote TTYrec files | |
| TTYREC_BASE_PATH="/opt/dgl-chroot/var/ttyrec" | |
| S3_HOSTNAME="http://dcss-cpo-ttyrecs.s3-website-us-west-2.amazonaws.com" | |
| CACHE_DIR="/tmp/ttyrec-listing-cache" | |
| STATUS="200 OK" | |
| MESSAGE="" | |
| # Validate args | |
| if ! [[ $REQUEST_URI =~ ^/ttyrec/[a-zA-Z0-9]*/?$ ]] ; then | |
| STATUS="400 Bad Request" | |
| MESSAGE="Couldn't recognise username" | |
| fi | |
| USERNAME=$(echo $REQUEST_URI | cut -d\/ -f3) | |
| if ! [[ -d "$TTYREC_BASE_PATH/$USERNAME" ]] ; then | |
| STATUS="404 Not Found" | |
| MESSAGE="User not found" | |
| fi | |
| CACHE_FILE="$CACHE_DIR/$USERNAME" | |
| if [[ -f $CACHE_FILE && -n $(find $CACHE_FILE -type f -mtime -1) ]] ; then | |
| cat $CACHE_FILE | |
| exit | |
| fi | |
| add_to_output() { | |
| echo "$@" | tee -a $CACHE_FILE | |
| } | |
| # Headers | |
| add_to_output "Status: $STATUS" | |
| add_to_output "Content-Type: text/html" | |
| add_to_output "" | |
| # Show an error page if needed | |
| if [[ $STATUS != "200 OK" ]] ; then | |
| if [[ -n $MESSAGE ]] ; then | |
| add_to_output "$MESSAGE" | |
| else | |
| add_to_output "$STATUS" | |
| fi | |
| exit | |
| fi | |
| add_to_output "<html> | |
| <head><title>Index of $REQUEST_URI</title></head> | |
| <body bgcolor=\"white\"> | |
| <h1>Index of $REQUEST_URI</h1><hr> | |
| <table style=\"width: 100%\"> | |
| <tr> | |
| <th style=\"width: 80%; text-align: left\">Name</th> | |
| <th>Modified</th> | |
| <th>Size</th> | |
| </tr> | |
| <tr> | |
| <td><a href=\"../\">../</a></td> | |
| <td></td> | |
| <td></td> | |
| </tr> | |
| " | |
| for ttyrec in $(find $TTYREC_BASE_PATH/$USERNAME -mindepth 1 | sort -r) ; do | |
| uri=$(basename "$ttyrec") | |
| pretty_date=$(date -ud "@$(/usr/bin/stat -c '%Y' "$ttyrec")" "+%Y-%m-%d %H:%M:%S") # AWS format | |
| size=$(/usr/bin/stat -c '%s' "$ttyrec") | |
| add_to_output "<tr><td><a href=\"$uri\">$uri</a></td><td>$pretty_date</td><td>$size</td></tr>" | |
| done | |
| add_to_output "<tr><td>Remote files below</td></tr>" | |
| aws s3 ls s3://dcss-cpo-ttyrecs/$USERNAME/ | sort -r | while read -r line ; do | |
| if [[ $line = *PRE* ]] ; then continue ; fi | |
| pretty_date=$(echo "$line" | awk '{print $1" "$2}') | |
| size=$(echo "$line" | awk '{print $3}') | |
| filename=$(echo "$line" | awk '{print $4}') | |
| add_to_output "<tr><td><a href=\"$S3_HOSTNAME/$USERNAME/$filename\">$filename</a></td><td>$pretty_date</td><td>$size</td></tr>" | |
| done | |
| add_to_output "</table>" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment