adam-phillipps

FROM python:3.6

RUN pip install google-cloud-storage polystores[gcs]

WORKDIR /usr/src/app

COPY . /usr/src/app/
###################################
# build like this, from a directory that has this file and the script
#   docker build -t testeroo .

adam-phillipps

args = (
            [["State", "Name"]],
            [
                "InstanceId", "VpcId", "PublicIpAddress", "PrivateIpAddress",
                "PublicDnsName", "PrivateDnsName", "ImageId", "SubnetId",
                "KeyName", "InstanceType", "LaunchTime",
                ["State", "Name"]
            ]
        )

adam-phillipps

FROM python:2

WORKDIR /usr/src/app

COPY requirements.txt ./

RUN pip install --no-cache-dir -r requirements.txt

COPY . .

adam-phillipps

class YourHandler:
    # ...
    CEREBRO_URL = 'https://cerebro-backend/api/v1/'
    def is_handling_this_request(event, context):
        return True if the event came from your trigger

    def handle_request(event, context):
        resource_list = []
        def _get_id(hostname):

adam-phillipps

import boto3
import csv
import json
import os
import request

def handler(event, context):
    cerebro_api_url = os.getenv('CEREBRO_API_URL')
    s3          = boto3.client('s3')
    bucket      = event['Records'][0]['s3']['bucket']['name']

adam-phillipps

#!/bin/bash

# Export environment vars for nvm
export NVM_DIR="$HOME/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  # This loads nvm
[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"  # This loads nvm bash_completion

# Bash completion for nvm
[[ -r $NVM_DIR/bash_completion ]] && \. $NVM_DIR/bash_completion

adam-phillipps

import json
import webrecon.gcse as gcse


def handler(context, event):
    return build_response(gsce.search(event))

def build_response(body=[], status=200):
    response_object = {
        "isBase64Encoded": "false",

adam-phillipps

"""
From a raw corpus, iterate through each article, do some basic preprocessing and
yield each sentence.
"""
def create_iterable_corpus(raw_corpus):
    pdb.set_trace()
    for article in raw_corpus:
        # concatenate all section titles and texts of each Wikipedia article into a single "sentence"
        doc = '\n'.join(itertools.chain.from_iterable(zip(article['section_titles'], article['section_texts'])))

adam-phillipps

#!/bin/bash
cd /home/ubuntu
export AWS_DEFAULT_REGION=us-west-2
export QUEUE=backlog_crawlBot
export WIP_Q=wip_crawlBot
export ART_NAME=SmashEndurance.jar
export ART_BKT=endurance-crawl-artifacts
export OUT_BKT=endurance-crawl-output

sudo usermod -a -G docker ec2-user

adam-phillipps

root@ip-172-31-16-137:/home/ubuntu# python wrapper.py
jar_art SmashEndurance.jar
4 msgs in queue
working
invoked jar with java -jar SmashEndurance.jar 1 http://www.rabo.org.ro/
java.io.IOException: Cannot run program "/usr/local/bin/docker": error=2, No such file or directory
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1048)
	at java.lang.Runtime.exec(Runtime.java:620)
	at java.lang.Runtime.exec(Runtime.java:450)
	at java.lang.Runtime.exec(Runtime.java:347)