Last active
November 25, 2019 11:24
-
-
Save igeligel/c3a72f8bd16984368584e280ac85d5cf to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
public class DynamicFrameDecodeBase64 { | |
public static DynamicFrame unBase64( | |
DynamicFrame df, | |
final String base64ColumnName, | |
final String dataColumnName | |
) { | |
RDD unboxRecords = df | |
.records() | |
.mapPartitions( | |
new MapFunctionUnbase64(base64ColumnName, dataColumnName), | |
true, | |
scala.reflect.ClassTag$.MODULE$.apply(DynamicRecord.class) | |
); | |
return DynamicFrame.newFrameWithErrors( | |
df, | |
unboxRecords, | |
df.getName(), | |
"", | |
null, | |
0, | |
0 | |
); | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
public class MapFunctionUnbase64 | |
extends scala.runtime.AbstractFunction1<scala.collection.Iterator<DynamicRecord>, scala.collection.Iterator<DynamicRecord>> | |
implements scala.Serializable { | |
public final scala.collection.Iterator<DynamicRecord> apply( | |
scala.collection.Iterator<DynamicRecord> iter | |
) { | |
Base64.Decoder base64decoder = Base64.getDecoder(); | |
return iter.map( | |
new scala.runtime.AbstractFunction1<com.amazonaws.services.glue.DynamicRecord, DynamicRecord>() { | |
public final DynamicRecord apply(DynamicRecord record) { | |
String base64ColumnName = "base64"; | |
String raw = record | |
.dropField(base64ColumnName) | |
.get() | |
.getValue() | |
.toString(); | |
byte[] decodedString; | |
String data; | |
try { | |
decodedString = | |
base64decoder.decode(new String(raw).getBytes("UTF-8")); | |
data = new String(decodedString, StandardCharsets.UTF_8); | |
} catch (java.io.UnsupportedEncodingException e) { | |
data = ""; | |
} | |
record.addField( | |
"data", | |
com.amazonaws.services.glue.types.StringNode.apply(data) | |
); | |
return record; | |
} | |
} | |
); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment