Some notes on our process on debugging Flink AWS parameter name

debug_flink_aws.scala

// START DEBUG LOG



// https://github.com/Aloisius/hadoop-s3a/blob/master/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java#L150
// Try to get our credentials or just connect anonymously
String accessKey = conf.get(NEW_ACCESS_KEY, conf.get(OLD_ACCESS_KEY, null));
String secretKey = conf.get(NEW_SECRET_KEY, conf.get(OLD_SECRET_KEY, null));

String userInfo = name.getUserInfo();
if (userInfo != null) {
  int index = userInfo.indexOf(':');
  if (index != -1) {
    accessKey = userInfo.substring(0, index);
    secretKey = userInfo.substring(index + 1);
  } else {
    accessKey = userInfo;
  }
}

AWSCredentialsProviderChain credentials = new AWSCredentialsProviderChain(
    new BasicAWSCredentialsProvider(accessKey, secretKey),
    new InstanceProfileCredentialsProvider(),
    new AnonymousAWSCredentialsProvider()
);

// Let's reproduce this code:

// load default
import org.apache.hadoop.conf.Configuration
val conf = new Configuration(true)
println("Config key: %s, fs=%s".format(
  conf.get("fs.s3.awsSecretAccessKey"),
  conf.get("fs.defaultFS")
  ))

println("Access key: %s".format(ACCESS_KEY))

/**
 The above actually returns the right value...
 **/
import org.apache.hadoop.fs.s3a.Constants._
println("Access key: %s".format(NEW_ACCESS_KEY))
XXX: the above doesnt work....
println("Access key: %s".format(ACCESS_KEY))

/**
Oh, this says `fs.s3a.access.key`
💩💩💩💩💩

SEE https://github.com/Aloisius/hadoop-s3a/commit/47ba5266090d82ff408adb5bd56ef951c498f0f9
for the change...
**/

// END DEBUG LOG