Skip to content

Instantly share code, notes, and snippets.

@nsivabalan
Created August 12, 2021 15:01
Show Gist options
  • Save nsivabalan/96c7ecedb191fc870439d09713b61660 to your computer and use it in GitHub Desktop.
Save nsivabalan/96c7ecedb191fc870439d09713b61660 to your computer and use it in GitHub Desktop.
public class SparkKeyGenUtils {
public static String getPartitionColumns(Map<String, String> parameters) throws IOException {
TypedProperties props = new TypedProperties();
props.putAll(parameters);
return getPartitionColumns(props);
}
public static String getPartitionColumns(TypedProperties props) throws IOException {
KeyGenerator keyGen = HoodieSparkKeyGeneratorFactory.createKeyGenerator(props);
return getPartitionColumns(keyGen);
}
public static String getPartitionColumns(KeyGenerator keyGenerator) {
if (keyGenerator instanceof BaseKeyGenerator) {
if (keyGenerator instanceof CustomKeyGenerator || keyGenerator instanceof CustomAvroKeyGenerator) {
BaseKeyGenerator baseKeyGenerator = (BaseKeyGenerator) keyGenerator;
List<String> partitionCols = baseKeyGenerator.getPartitionPathFields().stream().map(partitionField -> {
String[] splits = partitionField.split(CustomAvroKeyGenerator.SPLIT_REGEX);
if (splits.length != 2) {
throw new IllegalArgumentException("Partition path config is not in right format for CustomKeyGenerator " + partitionField);
} else {
return splits[0];
}
}).collect(Collectors.toList());
return String.join(",", partitionCols);
} else {
return String.join(",", ((BaseKeyGenerator)keyGenerator).getPartitionPathFields());
}
} else {
return null;
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment