This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.io.IOException; | |
import java.net.URISyntaxException; | |
import java.nio.file.Files; | |
import java.nio.file.Paths; | |
import java.security.KeyFactory; | |
import java.security.NoSuchAlgorithmException; | |
import java.security.PrivateKey; | |
import java.security.interfaces.RSAPublicKey; | |
import java.security.spec.InvalidKeySpecException; | |
import java.security.spec.PKCS8EncodedKeySpec; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
val s3Paths = "s3://yourbucket/path/to/file1.txt,s3://yourbucket/path/to/directory" | |
val pageLength = 100 | |
val key = "YOURKEY" | |
val secret = "YOUR_SECRET" | |
import com.amazonaws.services.s3._, model._ | |
import com.amazonaws.auth.BasicAWSCredentials | |
import com.amazonaws.services.s3.model.ObjectListing | |
import scala.collection.JavaConverters._ | |
import scala.io.Source |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash -ex | |
if [ "$(cat /mnt/var/lib/info/instance.json | jq -r .isMaster)" == "true" ]; then | |
# Install Git | |
sudo yum -y install git | |
# Install Maven | |
wget -P /tmp http://apache.mirrors.spacedump.net/maven/maven-3/3.3.3/binaries/apache-maven-3.3.3-bin.tar.gz | |
sudo mkdir /opt/apache-maven | |
sudo tar -xvzf /tmp/apache-maven-3.3.3-bin.tar.gz -C /opt/apache-maven |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// in the spark shell here i load the file from S3 | |
val myFile = sc.textFile("s3://some-s3-bucket/us-constitution.txt") | |
// Classic wordcount | |
val counts = myFile.flatMap(line => line.toLowerCase().replace(".", " ").replace(",", " ").split(" ")).map(word => (word, 1L)).reduceByKey(_ + _) | |
// create tuples for the words | |
val sorted_counts = counts.collect().sortBy(wc => -wc._2) | |
// print out a sample of 10 to see results |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
sudo wget http://repos.fedorapeople.org/repos/dchen/apache-maven/epel-apache-maven.repo -O /etc/yum.repos.d/epel-apache-maven.repo | |
sudo sed -i s/\$releasever/6/g /etc/yum.repos.d/epel-apache-maven.repo | |
sudo yum install -y apache-maven | |
mvn --version |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package utils | |
import groovy.json.JsonBuilder | |
import groovy.json.JsonSlurper | |
class HDD { | |
static save(Object content, String filePath) { | |
new File(filePath).write(new JsonBuilder(content).toPrettyString()) | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package com.neolitec.examples; | |
import org.apache.commons.codec.binary.Base64; | |
import org.apache.commons.lang.StringUtils; | |
import org.slf4j.Logger; | |
import org.slf4j.LoggerFactory; | |
import javax.servlet.*; | |
import javax.servlet.http.HttpServletRequest; | |
import javax.servlet.http.HttpServletResponse; |