ashwanthkumar

matsya {
  # Path on the local FS to write the Cluster states and Time series data we create
  # ${working-dir}/state and ${working-dir}/history respectively for the same
  working-dir = "local_run"

  clusters = [{
    # Unique Identifier - Please don't change once assigned
    name = "Staging Hadoop Cluster"

    # (Mandatory) AutoScaling group backing the spot machines

ashwanthkumar

<pipeline name="gocd-janitor" labeltemplate="cleanup-${COUNT}" isLocked="false">
  <timer>0 30 5 ? * MON</timer>
  <stage name="cleanup">
    <approval type="manual" />
    <jobs>
      <job name="cleanup">
        <tasks>
          <exec command="wget">
            <arg>https://github.com/ashwanthkumar/gocd-janitor/releases/download/v0.0.1/gocd-janitor-0.0.1-jar-with-dependencies.jar</arg>
          </exec>

ashwanthkumar

# This assumes that jq is installed on your machine

JT_MACHINE="jt-host"
NN_MACHINE="nn-host"

# Get all the TTs
curl "http://${JT_MACHINE}:50030/jmx?qry=hadoop:service=JobTracker,name=JobTrackerInfo" | jq -r .beans[].AliveNodesInfoJson | jq -r .[].hostname | sort > tasktrackers
# Get all the DNs
curl "http://${NN_MACHINE}:50070/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo" | jq -r .beans[].LiveNodes | jq -r 'keys | .[]' | sort > datanodes

ashwanthkumar

The Ying-Yang of CEOs and Engineers

All successful startups begin with two founders. One guy is the Engineer, and the other guy is the business dude. Over the years of working with various people, I've learned what makes a good engineer, and what makes a good business dude, and the two are complete opposites of each other.

CEO	Engineer

ashwanthkumar

import com.twitter.scalding.{Tsv, Job, Args}
import scala.collection.immutable.TreeSet

/*
  Lets assume we are reading a file of format (a,b) where a,b denote that node a and node b are connected in a graph.
  For simplicity we will assume that a and b are ints. We want to find the mapping of all the nodes on a fixed point.
 */
class FixedPointJob(args: Args) extends Job(args) {
  val input = args("input")
  val outputBaseDir = args("output-base-dir")

ashwanthkumar

/*
 * Copyright 2006 Sun Microsystems, Inc.  All Rights Reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *   - Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *

ashwanthkumar

struct Vehicle {
    1: string name
    2: string model
}

struct Person {
    1: string name
    2: i32 age
    // Empty collections can be specified using []. Same works for map and set as well. 
    3: list<Vehicle> cars = []

ashwanthkumar

include 'Java'
...

admin = HBaseAdmin.new(HBaseConfiguration.create)
table_regions = admin.getTableRegions(Bytes.toBytes("test_table"))

table_regions.each do |table_region|
  puts "#{table_region.getRegionNameAsString())}"
}

ashwanthkumar

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;

ashwanthkumar

import org.apache.hadoop.fs.Path
import org.apache.hadoop.hbase.io.hfile.{ HFile, HFileScanner, CacheConfig }
import org.apache.hadoop.hbase.io.hfile.HFile.Reader
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.KeyValue
import org.apache.hadoop.mapreduce.{ JobContext, InputSplit, TaskAttemptContext, RecordReader }
import org.apache.hadoop.mapreduce.lib.input.{ FileInputFormat, FileSplit }
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics

/**