Skip to content

Instantly share code, notes, and snippets.

@jzhuge
Created April 10, 2025 16:49
Show Gist options
  • Save jzhuge/7d894ce1940b95fc222f138db2c1564e to your computer and use it in GitHub Desktop.
Save jzhuge/7d894ce1940b95fc222f138db2c1564e to your computer and use it in GitHub Desktop.
Implement --extra-properties-file
diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
index c434941a58..23ee4d6605 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
@@ -48,6 +48,7 @@ abstract class AbstractCommandBuilder {
String mainClass;
String master;
protected String propertiesFile;
+ protected List<String> extraPropertiesFiles = new ArrayList<>();
final List<String> appArgs;
final List<String> jars;
final List<String> files;
@@ -264,7 +265,7 @@ abstract class AbstractCommandBuilder {
Map<String, String> getEffectiveConfig() throws IOException {
if (effectiveConfig == null) {
effectiveConfig = new HashMap<>(conf);
- Properties p = loadPropertiesFile();
+ Properties p = loadPropertiesFiles();
p.stringPropertyNames().forEach(key ->
effectiveConfig.computeIfAbsent(key, p::getProperty));
}
@@ -276,8 +277,16 @@ abstract class AbstractCommandBuilder {
* user-specified properties file, or the spark-defaults.conf file under the Spark configuration
* directory.
*/
- private Properties loadPropertiesFile() throws IOException {
+ private Properties loadPropertiesFiles() throws IOException {
Properties props = new Properties();
+ addPropertiesFromFile(propertiesFile, props);
+ for (String extraFile : extraPropertiesFiles) {
+ addPropertiesFromFile(extraFile, props);
+ }
+ return props;
+ }
+
+ private void addPropertiesFromFile(String propertiesFile, Properties props) throws IOException {
File propsFile;
if (propertiesFile != null) {
propsFile = new File(propertiesFile);
@@ -286,16 +295,16 @@ abstract class AbstractCommandBuilder {
propsFile = new File(getConfDir(), DEFAULT_PROPERTIES_FILE);
}
+ Properties temp = new Properties();
if (propsFile.isFile()) {
try (InputStreamReader isr = new InputStreamReader(
new FileInputStream(propsFile), StandardCharsets.UTF_8)) {
- props.load(isr);
- for (Map.Entry<Object, Object> e : props.entrySet()) {
- e.setValue(e.getValue().toString().trim());
+ temp.load(isr);
+ for (String property : temp.stringPropertyNames()) {
+ props.setProperty(property, temp.getProperty(property).trim());
}
}
}
- return props;
}
private String getConfDir() {
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment