gistfile1.txt

#!/bin/bash
cd /home/ubuntu
export AWS_DEFAULT_REGION=us-west-2
export QUEUE=backlog_crawlBot
export WIP_Q=wip_crawlBot
export ART_NAME=SmashEndurance.jar
export ART_BKT=endurance-crawl-artifacts
export OUT_BKT=endurance-crawl-output

sudo usermod -a -G docker ec2-user
sudo service docker restart

pip install boto3

# start wappalyzer cli
docker run --rm wappalyzer/cli https://www.wappalyzer.com

# create wrapper script
cat << WPR >> wrapper.py
import boto3
import os
import pdb

s3 = boto3.resource('s3')
sqs = boto3.client('sqs', region_name='us-west-2')
jar_art = s3.Object(os.getenv('ART_BKT'), os.getenv('ART_NAME')).download_file(os.getenv('ART_NAME'))

# count the backlog messages
def msg_count():
	n = sqs.get_queue_attributes(QueueUrl=os.getenv('QUEUE'),AttributeNames=['ApproximateNumberOfMessages'])['Attributes']['ApproximateNumberOfMessages']
	print("{} msgs in queue".format(n))
	return int(n)

# get a message
# run the jar on the message
# put the results in the results bucket
# delete the message from the backlog because we just finished it
def work():
	print("working")
	try:
		msg = sqs.receive_message(QueueUrl=os.getenv('QUEUE'), WaitTimeSeconds=20, MaxNumberOfMessages=1, VisibilityTimeout=120)['Messages'][0]
	except Exception as e:
		print("Exception getting message: {}".format(e))
		return

	sid, surl = msg['Body'].replace("'",'').split(',')
	res_name = "{}.json".format(sid)
	invoke_jar_command = "java -jar SmashEndurance.jar {} {}".format(sid, surl)
	print("invoked jar with {}".format(invoke_jar_command))
	ex_code = os.system(invoke_jar_command)
	if ex_code == 0:
		s3.Object(os.getenv('OUT_BKT'), res_name).upload_file(res_name)
		sqs.delete_message(QueueUrl=os.getenv('QUEUE'), ReceiptHandle=msg['ReceiptHandle'])
		os.remove(res_name)
	else:
		print("jar error->{}->{}->{}".format(ex_code, sid, surl))

# always stay busy and break if there is no work left
while True:
	if msg_count() > 0:
		work()
	else:
		break

WPR
# make wrapper runnable
chmod +x wrapper.py
# run the wrapper
python wrapper.py