| import java.io.IOException; | |
| import java.util.Vector; | |
| /** | |
| * Created by moshee | |
| * on: 07/06/17 | |
| * to compile in place: `javac ConsumeHeap.java` | |
| * Execute: `java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/app-`date +%s`-pid$$.hprof -XX:OnOutOfMemoryError=/opt/app/bin/upload_dump_s3.sh -Xmx2m ConsumeHeap` | |
| * HeapDumpOnOutOfMemoryError specifies to automatically create a dump when OOM occures | |
| * HeapDumpPath supplies a path to put that file |
| #!/usr/bin/env bash | |
| # | |
| # Author: Stefan Buck | |
| # License: MIT | |
| # https://gist.github.com/stefanbuck/ce788fee19ab6eb0b4447a85fc99f447 | |
| # | |
| # | |
| # This script accepts the following parameters: | |
| # | |
| # * owner |
| import java.security.MessageDigest | |
| import java.util | |
| import javax.crypto.Cipher | |
| import javax.crypto.spec.SecretKeySpec | |
| import org.apache.commons.codec.binary.Base64 | |
| /** | |
| * Sample: | |
| * {{{ | |
| * scala> val key = "My very own, very private key here!" |
Suppose you have a key like (page, geo, day) and you want to make rollups/datacube so you can query for all pages, or all geos or all days.
Here is how you do it:
def opts[T](t: T): Seq[Option[T]] = Seq(Some(t), None)
val p: TypedPipe[(String, String, Int)] = ...
p.sumByLocalKeys
| import com.twitter.scalding._ | |
| import com.twitter.algebird.{ MinHasher, MinHasher32, MinHashSignature } | |
| /** | |
| * Computes similar items (with a string itemId), based on approximate | |
| * Jaccard similarity, using LSH. | |
| * | |
| * Assumes an input data TSV file of the following format: | |
| * | |
| * itemId userId |
| public class HCatInputFormat extends InputFormat<SerializableWritable<Writable>, HCatRecord> { | |
| private final org.apache.hcatalog.mapreduce.HCatInputFormat input; | |
| public HCatInputFormat() { | |
| input = new org.apache.hcatalog.mapreduce.HCatInputFormat(); | |
| } | |
| @Override | |
| public RecordReader<SerializableWritable<Writable>, HCatRecord> createRecordReader( |
| """ | |
| Console module provide `copen` method for opening interactive python shell in | |
| the runtime. | |
| """ | |
| import code | |
| import readline | |
| import rlcompleter | |
| def copen(_globals, _locals): |
| /* | |
| Copyright 2018 Viktor Klang | |
| Licensed under the Apache License, Version 2.0 (the "License"); | |
| you may not use this file except in compliance with the License. | |
| You may obtain a copy of the License at | |
| http://www.apache.org/licenses/LICENSE-2.0 | |
| Unless required by applicable law or agreed to in writing, software |
| #!/bin/sh | |
| ### | |
| # SOME COMMANDS WILL NOT WORK ON macOS (Sierra or newer) | |
| # For Sierra or newer, see https://github.com/mathiasbynens/dotfiles/blob/master/.macos | |
| ### | |
| # Alot of these configs have been taken from the various places | |
| # on the web, most from here | |
| # https://github.com/mathiasbynens/dotfiles/blob/5b3c8418ed42d93af2e647dc9d122f25cc034871/.osx |