Skip to content

Instantly share code, notes, and snippets.

View butlermh's full-sized avatar

Mark H. Butler butlermh

  • Santa Clara, United States
View GitHub Profile
@butlermh
butlermh / hadoopexample6
Created June 9, 2011 10:39
Changing Mapper for new Hadoop API 5
@Overide
public void configure(JobConf job) {
super.configure(job);
config = job;
BECOMES
@Override
public void setup(Mapper<InputKey, InputValue, OutputKey, OutputValue>.Context context)
throws InterruptedException, IOException {
super.setup(context);
config = context.getConfiguration();
@butlermh
butlermh / hadoopexample5
Created June 9, 2011 10:37
Changing mapper to new Hadoop API 4
@Override
public void close() throws IOException {
BECOMES
@Override
public void cleanup(Context context) throws IOException, InterruptedException {
@butlermh
butlermh / hadoopexample4
Created June 9, 2011 10:34
Changing mapper to new Hadoop API 3
reporter.incrCounter(family, type, 1);
BECOMES
context.getCounter(family, type).increment(1);
output.collect(key, value);
BECOMES
context.write(key, value);
@butlermh
butlermh / hadoopexample3
Created June 9, 2011 10:30
Changing Mapper for the new Hadoop API 2
@Override
public void map(InputKey key, InputValue value,
OutputCollector<OutputKey, OutputValue> output, Reporter reporter)
throws IOException {
BECOMES
@Override
public void map(InputKey key, InputValue value,
Mapper<InputKey, InputValue, OutputKey, OutputValue>.Context context)
throws IOException, InterruptedException {
@butlermh
butlermh / hadoopexample2
Created June 8, 2011 23:29
Changing Mapper for new Hadoop API 1
public class MyMapper extends MapReduceBase implements
Mapper<InputKey, InputValue, OutputKey, OutputValue> {
BECOMES
public class MyMapper extends
Mapper<InputKey, InputValue, OutputKey, OutputValue> {
@butlermh
butlermh / hadoopexample1
Created June 8, 2011 23:26
Changing imports for new Hadoop API
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Mapper;
BECOME
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
BECOME
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
@butlermh
butlermh / mavenexample1
Created June 8, 2011 23:00
Jena Schemagen in Maven
<build>
<plugins>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<phase>compile</phase>
<configuration>
<tasks>
<property name="runtime_classpath" refid="maven.runtime.classpath" />
@butlermh
butlermh / antexample3
Created June 8, 2011 14:55
Using uptodate to speed up builds 2
<target name="compile" depends="resolve" description="--> compile the project" unless="module.uptodate">
<mkdir dir="${classes.dir}" />
<javac srcdir="${src.dir}" destdir="${classes.dir}" classpathref="lib.path.id" debug="true" includeantruntime="false"/>
</target>
<target name="jar" depends="version, compile, copyclasses" description="--> make a jar file for this project" unless="module.uptodate">
<jar destfile="${jar.file}">
<fileset dir="${classes.dir}" />
<manifest>
<attribute name="Built-By" value="${user.name}"/>
@butlermh
butlermh / antexample2
Created June 8, 2011 14:50
Using uptodate to speed up builds
<target name="check.retrieve.necessary" description="Only retrieve jars with Ivy if necessary">
<ivy:resolve file="${ivy.file}" transitive="true"/>
<uptodate property="libs.uptodate">
<srcfiles dir="." includes="${ivy.file}"/>
<mapper type="merge" to="{lib.dir}/.done"/>
</uptodate>
</target>
<target name="resolve" depends="check.retrieve.necessary" description="--> resolve and retrieve dependencies with ivy"
unless="libs.uptodate">
@butlermh
butlermh / antexample1
Created June 8, 2011 14:01
Using configurations in Ant
<!-- resolve the job and test dependencies to different directories -->
<target name="resolve" depends="clean-lib" description="--> resolve and retrieve dependencies with ivy">
<mkdir dir="${job.lib.dir}"/> <!-- not usually necessary, ivy creates the directory IF there are dependencies -->
<mkdir dir="${test.lib.dir}"/>
<ivy:resolve file="${ivy.file}" transitive="true"/>
<ivy:retrieve pattern="${job.lib.dir}/[artifact]-[revision].[ext]" conf="job" />
<ivy:retrieve pattern="${test.lib.dir}/[artifact]-[revision].[ext]" conf="test" />
</target>