anuvrat

add file s3://hive-udf-script/test.pl;

select transform(full_name) using 'test.pl' as (first_name, last_name) from names_tables;

anuvrat

hive>list file;
mnt/var/lib/hive_07_1/downloaded_resources/test.pl

hadoop>ls -la mnt/var/lib/hive_07_1/downloaded_resources/test.pl
-rwxrwxrwx 1 hadoop hadoop 2727 Mar  1 10:35 /mnt/var/lib/hive_07_1/downloaded_resources/test.pl

anuvrat

add file s3://hive-udf-script/test.pl;

select transform(full_name) using 'perl test.pl' as (first_name, last_name) from names_tables;

anuvrat

df | awk 'NR==1; NR > 1 {print $0 | "sort -nr -k2 -T ."}' | awk '{ print $6 }' | head -2 | tail -1

$> df -h
Filesystem            Size  Used Avail Use% Mounted on
/dev/sda1             9.7G  3.3G  6.0G  35% /
/dev/sda2             253G  3.0G  237G   2% /local
tmpfs                 7.9G     0  7.9G   0% /dev/shm

$> df | awk 'NR==1; NR > 1 {print $0 | "sort -nr -k2 -T ."}' 
Filesystem           1K-blocks      Used Available Use% Mounted on

anuvrat

public String cleanUrl(String url, String locale, String asin) {
  String result;

  if(url != null && url.indefOf("%3F") != -1) {
    result url.substring(0, url.indexOf("%3F"));
  } else {
    result = "www.amazon." + getAmazonDomain(locale) + "/gp/product/" + asin;
  }

  return result;

anuvrat

public String cleanUrlNew(String url, String locale, String asin) {
  String result;
  
  try {
    URL tempUrl = new URL(url);
    result = tempUrl.getProtocol() + "://" + tempUrl.getAuthority() + tempUrl.getPath();
  } catch (MalformedURLException e) {
    // This should never happen. If it happens we should file a bug with PAAPI team.
    logger.warn("PAAPI returned an invalid url - " + url);
    result = cleanUrl(url, locale, asin);

anuvrat

create table names_table (
  first_name varchar(20), 
  last_name varchar(20), 
  city varchar(20));

load data infile '/tmp/anu/20120219' 
into table names_table 
fields terminated by '\t' 
lines terminated by '\n' 
(first_name, last_name) 

anuvrat

create table names_table (
  first_name varchar(20), 
  last_name varchar(20), 
  birth_date date);

load data infile '/tmp/anu/20120219' 
into table names_table 
fields terminated by '\t' 
lines terminated by '\n' 
(first_name, last_name, @var1) 

anuvrat

/**
 * Perform the Knuth-Morris-Pratt string search. The algorithm first pre-processes the word to create a Partial Match
 * Table. The table is creatd in O(n) where n is the length of the word. After the pre-processing, the actual search can
 * be performed in O(k) where k is the size of the sentence.
 */
public class KMPStringSearch {
	/**
	 * Searches for all occurances of the word in the sentence. Runs in O(n+k) where n is the word length and k is the
	 * sentence length.
	 * 

anuvrat

        /*
         * The memory size of the process is the basis for the badness.
         */
        points = p->mm->total_vm;