Created
April 10, 2025 16:49
-
-
Save jzhuge/7d894ce1940b95fc222f138db2c1564e to your computer and use it in GitHub Desktop.
Implement --extra-properties-file
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java | |
index c434941a58..23ee4d6605 100644 | |
--- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java | |
+++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java | |
@@ -48,6 +48,7 @@ abstract class AbstractCommandBuilder { | |
String mainClass; | |
String master; | |
protected String propertiesFile; | |
+ protected List<String> extraPropertiesFiles = new ArrayList<>(); | |
final List<String> appArgs; | |
final List<String> jars; | |
final List<String> files; | |
@@ -264,7 +265,7 @@ abstract class AbstractCommandBuilder { | |
Map<String, String> getEffectiveConfig() throws IOException { | |
if (effectiveConfig == null) { | |
effectiveConfig = new HashMap<>(conf); | |
- Properties p = loadPropertiesFile(); | |
+ Properties p = loadPropertiesFiles(); | |
p.stringPropertyNames().forEach(key -> | |
effectiveConfig.computeIfAbsent(key, p::getProperty)); | |
} | |
@@ -276,8 +277,16 @@ abstract class AbstractCommandBuilder { | |
* user-specified properties file, or the spark-defaults.conf file under the Spark configuration | |
* directory. | |
*/ | |
- private Properties loadPropertiesFile() throws IOException { | |
+ private Properties loadPropertiesFiles() throws IOException { | |
Properties props = new Properties(); | |
+ addPropertiesFromFile(propertiesFile, props); | |
+ for (String extraFile : extraPropertiesFiles) { | |
+ addPropertiesFromFile(extraFile, props); | |
+ } | |
+ return props; | |
+ } | |
+ | |
+ private void addPropertiesFromFile(String propertiesFile, Properties props) throws IOException { | |
File propsFile; | |
if (propertiesFile != null) { | |
propsFile = new File(propertiesFile); | |
@@ -286,16 +295,16 @@ abstract class AbstractCommandBuilder { | |
propsFile = new File(getConfDir(), DEFAULT_PROPERTIES_FILE); | |
} | |
+ Properties temp = new Properties(); | |
if (propsFile.isFile()) { | |
try (InputStreamReader isr = new InputStreamReader( | |
new FileInputStream(propsFile), StandardCharsets.UTF_8)) { | |
- props.load(isr); | |
- for (Map.Entry<Object, Object> e : props.entrySet()) { | |
- e.setValue(e.getValue().toString().trim()); | |
+ temp.load(isr); | |
+ for (String property : temp.stringPropertyNames()) { | |
+ props.setProperty(property, temp.getProperty(property).trim()); | |
} | |
} | |
} | |
- return props; | |
} | |
private String getConfDir() { |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment