-
-
Save rocel/ff3530fe9f1afea173fabf215f0094bc to your computer and use it in GitHub Desktop.
Flatten Spark data frame fields structure, via SQL in Java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class Toto | |
{ | |
public void Main() | |
{ | |
final DataFrame source = GetDataFrame(); | |
final String querySelectSQL = flattenSchema(source.schema(), null); | |
source.registerTempTable("source"); | |
final DataFrame flattenData = sqlContext.sql("SELECT " + querySelectSQL + " FROM source") | |
} | |
/** | |
* Generate SQL to select columns as flat. | |
*/ | |
public String flattenSchema(StructType schema, String prefix) | |
{ | |
final StringBuilder selectSQLQuery = new StringBuilder(); | |
for (StructField field : schema.fields()) | |
{ | |
final String fieldName = field.name(); | |
if (fieldName.startsWith("@")) | |
{ | |
continue; | |
} | |
String colName = prefix == null ? fieldName : (prefix + "." + fieldName); | |
String colNameTarget = colName.replace(".", "_"); | |
if (field.dataType().getClass().equals(StructType.class)) | |
{ | |
selectSQLQuery.append(flattenSchema((StructType) field.dataType(), colName)); | |
} | |
else | |
{ | |
selectSQLQuery.append(colName); | |
selectSQLQuery.append(" as "); | |
selectSQLQuery.append(colNameTarget); | |
} | |
selectSQLQuery.append(","); | |
} | |
if (selectSQLQuery.length() > 0) | |
{ | |
selectSQLQuery.deleteCharAt(selectSQLQuery.length() - 1); | |
} | |
return selectSQLQuery.toString(); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment