robenalt

framework 'Foundation'
class String
  def language
    CFStringTokenizerCopyBestStringLanguage(self, CFRangeMake(0, self.size))
  end
  
  def tokens
      str_array = []
      stok = CFStringTokenizerCreate(nil,self,[0,self.length],0,nil)
      CFStringTokenizerGetCurrentTokenRange(stok)

robenalt

IO.popen("grep -i what", "w").write ( IO.popen('find . ').read )

robenalt

#Mac OS X - Notes

Mensch coding font

Apps

• Chrome
• Firefox
• iTerm
• Sublime Text

robenalt

package mllib


import scala.util.Random
import org.jblas.DoubleMatrix

import org.apache.spark.SparkContext
import org.apache.spark.rdd._
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext._

robenalt

# Load a Remote file to hadoop through ssh
cat /some/path/tofile.csv | ssh user@host  "hadoop fs -put - /some_hdfs/path/for/the/file.csv"

robenalt

on write_to_file(this_data, target_file, append_data)
	try
		set the target_file to the target_file as string
		set the open_target_file to open for access file target_file with write permission
		if append_data is false then set eof of the open_target_file to 0
		write this_data to the open_target_file starting at eof
		close access the open_target_file
		return true
	on error
		try

robenalt

import scala.sys.process._
//"ls -la".!!
val result = "ls -la".!!

robenalt

# pyspark --packages com.databricks:spark-avro_2.10:1.0.0
# read avro files from 1.3.0 spark
df = sqlCtx.load("/path/to/my_avro", "com.databricks.spark.avro")

robenalt

// Set logging level for spark scala
Logger.getLogger("org").setLevel(Level.WARN)
Logger.getLogger("akka").setLevel(Level.WARN)

robenalt

#from pyspark.sql import HiveContext
#sqlContext = HiveContext(sc)
query = """
select * from db.sometable where col>50
"""
results = sqlContext.sql(query)
result_writer = pyspark.sql.DataFrameWriter(results)
result_writer.saveAsTable('db.new_table_name',format='parquet', mode='overwrite',path='/path/to/new/data/files')