Hadoopの不具合をJIRAに書く前の下書き・校正メモ。

HDFS-4344.1.patch

diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 93bf5ff..154fce5 100644
--- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -967,7 +967,7 @@ private DatanodeID parseDNFromHostsEntry(String hostLine) {
       port = DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT;
     } else {
       hostStr = hostLine.substring(0, idx);
-      port = Integer.valueOf(hostLine.substring(idx));
+      port = Integer.valueOf(hostLine.substring(idx + 1));
     }
 
     if (InetAddresses.isInetAddress(hostStr)) {
diff --git hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java
index 934342e..1238cdf 100644
--- hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java
+++ hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java
@@ -27,6 +27,10 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -151,6 +155,45 @@ public void testChangeIpcPort() throws Exception {
     }
   }
   
+  /**
+   * Test for HDFS-4344.
+   * @throws Exception
+   */
+  @Test
+  public void testDfsHostsWithPort() throws Exception {
+
+    MiniDFSCluster cluster = null;
+    try {
+
+      HdfsConfiguration conf = new HdfsConfiguration();
+
+      LocalFileSystem fileSystem = FileSystem.getLocal(conf);
+      Path dir = new Path(fileSystem.getWorkingDirectory(), "build/test/data/work-dir/hdfs-4344");
+      Path excludeFile = new Path(dir, "exclude");
+      conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath());
+
+      FSDataOutputStream excludeStream = fileSystem.create(excludeFile, true);
+      excludeStream.writeUTF("dummy-host:9999");
+      excludeStream.close();
+
+      cluster = new MiniDFSCluster.Builder(conf).build();
+      cluster.waitActive();
+
+      DatanodeManager dm =
+          cluster.getNamesystem().getBlockManager().getDatanodeManager();
+
+      dm.getDatanodeListForReport(DatanodeReportType.ALL);
+      dm.getDatanodeListForReport(DatanodeReportType.LIVE);
+      dm.getDatanodeListForReport(DatanodeReportType.DEAD);
+
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+    }
+
+  }
+
   @Test
   public void testChangeStorageID() throws Exception {
     final String DN_IP_ADDR = "127.0.0.1";

ticket.md

Title: dfshealth.jsp throws NumberFormatException when dfs.hosts/dfs.hosts.exclude includes port number

Description:

dfs.hosts and dfs.hosts.exclude files cannot contain a port number of host.

If contained, and access a dfshealth.jsp on a webui, we got a NumberFormatException.

How to reproduce:

$ cat /tmp/include.txt
salve-host1:9999

$ cat /tmp/exclude.txt
slave-host1:9999

$ hdfs namenode -Ddfs.hosts=/tmp/include.txt -Ddfs.hosts.exclude=/tmp/exclude.txt

Error:

Problem accessing /dfshealth.jsp. Reason:

    For input string: ":9999"
Caused by:

java.lang.NumberFormatException: For input string: ":9999"
     at java.lang.NumberFormatException.forInputString(NumberFormatException.java:48)
     at java.lang.Integer.parseInt(Integer.java:449)
     at java.lang.Integer.valueOf(Integer.java:554)
     at org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager.parseDNFromHostsEntry(DatanodeManager.java:970)
     at org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager.getDatanodeListForReport(DatanodeManager.java:1039)
     at org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager.fetchDatanodes(DatanodeManager.java:892)
     at org.apache.hadoop.hdfs.server.namenode.NamenodeJspHelper$HealthJsp.generateHealthReport(NamenodeJspHelper.java:288)
     at org.apache.hadoop.hdfs.server.namenode.dfshealth_jsp._jspService(dfshealth_jsp.java:109)
     at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:98)
     at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
     at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
     at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
     at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109)
     at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
     at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:1071)
     at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
     at org.apache.hadoop.http.NoCacheFilter.doFilter(NoCacheFilter.java:45)
     at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
     at org.apache.hadoop.http.NoCacheFilter.doFilter(NoCacheFilter.java:45)
     at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
     at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
     at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
     at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
     at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
     at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
     at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
     at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
     at org.mortbay.jetty.Server.handle(Server.java:326)
     at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
     at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
     at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
     at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
     at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
     at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
     at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)

It's probably because DatanodeManager.parseDNFromHostsEntry() doesn't parse host:port string correctly.

  private DatanodeID parseDNFromHostsEntry(String hostLine) {
    DatanodeID dnId;
    String hostStr;
    int port;
    int idx = hostLine.indexOf(':');

    if (-1 == idx) {
      hostStr = hostLine;
      port = DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT;
    } else {
      hostStr = hostLine.substring(0, idx);
      port = Integer.valueOf(hostLine.substring(idx));
    }

correct it as the below.

port = Integer.valueOf(hostLine.substring(idx + 1));

タイトルはこんな感じ
dfshealth.jsp throws NumberFormatException when dfs.hosts/dfs.hosts.exclude includes port number

原文だと「ポート番号を入れられない」と書いてあるだけで、何が発生しているのかが伝わりにくそうだったので上記のように例外が飛ぶという部分を主体にしてみました。

I think the following code is wrong.

ここはもうちょっと具体的に、こんな感じでどうでしょう？
It's probably because DatanodeManager.parseDNFromHostsEntry() doesn't parse host:port string correctly.

なるほど！ 的確すぎて返す言葉もありません。
ありがとうございます！！＼(＾o＾)／

指摘分を更新！(｀・ω・´)

ついでに修正パッチ＆テストも書いてみた。

チケットは HDFS-4344
https://issues.apache.org/jira/browse/HDFS-4344