Miscellaneous Spark utilities

SparkUtils.java

import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
  
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class SparkUtils {

  private static final Logger LOG = LoggerFactory.getLogger(SparkUtils.class);

  /**
   * Given a Hadoop configuration and appId, use the YARN API (via the {@link YarnClient}) to get the application
   * report, which includes the trackingUrl.
   *
   * @param hadoopConf the Hadoop {@link org.apache.hadoop.conf.Configuration}
   * @param appId the YARN application ID
   * @return the app trackingUrl
   */
  public static String getYarnApplicationTrackingUrl(org.apache.hadoop.conf.Configuration hadoopConf, String appId) {

    final YarnClient yarnClient = YarnClient.createYarnClient();
    try {
      yarnClient.init(hadoopConf);
      yarnClient.start();

      final ApplicationReport report = yarnClient.getApplicationReport(ConverterUtils.toApplicationId(appId));
      return report.getTrackingUrl();
    } catch (YarnException | IOException e) {
      LOG.warn("{} attempting to get report for YARN appId {}", e.getClass().getSimpleName(), appId, e);
      return null;
    } finally {
      yarnClient.stop();
    }
  }
}