Created
March 27, 2012 18:19
-
-
Save mushkevych/2218657 to your computer and use it in GitHub Desktop.
Exemplary Hadoop Mapper dealing with OutOfMemoryError
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/** | |
* @author Bohdan Mushkevych | |
* date: 16 Mar 2012 | |
* Description: presents OutOfMemoryError recovery in Hadoop | |
*/ | |
public class ExemplaryMapper extends Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, ImmutableBytesWritable> { | |
private static Logger log = Logger.getLogger(ExemplaryMapper.class); | |
@Override | |
protected void map(ImmutableBytesWritable key, Result result, Context context) throws IOException, InterruptedException { | |
try { | |
// do something very useful here | |
// keep all references in try scope of visibility | |
byte[] bigArray = new byte[BIG_NUMBER]; | |
} catch (java.lang.OutOfMemoryError e) { | |
System.gc(); | |
log.warn(String.format("OutOfMemoryError was thrown for user id=%d", Bytes.toInt(key.get()))); | |
Counter counter = context.getCounter(SynergyCounter.NUMBER_OF_OOM_ERRORS); | |
counter.increment(1); | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment