gistfile2.txt

sqoop.patch

diff -urB sqoop-1.1.0+8/src/java/com/cloudera/sqoop/hive/HiveImport.java sqoopfixed/src/java/com/cloudera/sqoop/hive/HiveImport.java
--- sqoop-1.1.0+8/src/java/com/cloudera/sqoop/hive/HiveImport.java	2010-10-10 18:43:50.000000000 -0400
+++ sqoopfixed/src/java/com/cloudera/sqoop/hive/HiveImport.java	2010-10-21 10:46:12.000000000 -0400
@@ -99,12 +99,7 @@
   private void removeTempLogs(String tableName) throws IOException {
     FileSystem fs = FileSystem.get(configuration);
     String warehouseDir = options.getWarehouseDir();
-    Path tablePath; 
-    if (warehouseDir != null) {
-      tablePath = new Path(new Path(warehouseDir), tableName);
-    } else {
-      tablePath = new Path(tableName);
-    }
+    Path tablePath = new Path(tableName);
 
     Path logsPath = new Path(tablePath, "_logs");
     if (fs.exists(logsPath)) {
diff -urB sqoop-1.1.0+8/src/java/com/cloudera/sqoop/hive/TableDefWriter.java sqoopfixed/src/java/com/cloudera/sqoop/hive/TableDefWriter.java
--- sqoop-1.1.0+8/src/java/com/cloudera/sqoop/hive/TableDefWriter.java	2010-10-10 18:43:50.000000000 -0400
+++ sqoopfixed/src/java/com/cloudera/sqoop/hive/TableDefWriter.java	2010-10-21 10:36:53.000000000 -0400
@@ -177,14 +177,7 @@
    * @return the LOAD DATA statement to import the data in HDFS into hive.
    */
   public String getLoadDataStmt() throws IOException { 
-    String warehouseDir = options.getWarehouseDir();
-    if (null == warehouseDir) {
-      warehouseDir = "";
-    } else if (!warehouseDir.endsWith(File.separator)) {
-      warehouseDir = warehouseDir + File.separator;
-    }
-
-    String tablePath = warehouseDir + inputTableName;
+    String tablePath = inputTableName;
     FileSystem fs = FileSystem.get(configuration);
     Path finalPath = new Path(tablePath).makeQualified(fs);
     String finalPathStr = finalPath.toString();
diff -urB sqoop-1.1.0+8/src/java/com/cloudera/sqoop/util/AppendUtils.java sqoopfixed/src/java/com/cloudera/sqoop/util/AppendUtils.java
--- sqoop-1.1.0+8/src/java/com/cloudera/sqoop/util/AppendUtils.java	2010-10-10 18:43:50.000000000 -0400
+++ sqoopfixed/src/java/com/cloudera/sqoop/util/AppendUtils.java	2010-10-21 11:03:04.000000000 -0400
@@ -68,9 +68,6 @@
     Path userDestDir = null;
     if (options.getTargetDir() != null) {
       userDestDir = new Path(options.getTargetDir());
-    } else if (options.getWarehouseDir() != null) {
-      userDestDir = new Path(options.getWarehouseDir(),
-          context.getTableName());
     } else {
       userDestDir = new Path(context.getTableName());
     }
@@ -94,7 +91,20 @@
     } else {
       LOG.info("Appending to directory " + userDestDir.getName());
       // Get the right next partition for the imported files
-      nextPartition = getNextPartition(fs, userDestDir);
+
+      // if the warehousedir is specified and a hive import is going to occur,
+      // then we need the next partition based on the files in the hive DB 
+      // folder, not based on the ones in userDestDir (because that's not the
+      // final resting place for the files)
+      Path finalDestDir = userDestDir;
+      String hiveTable = options.getHiveTableName();
+      if (hiveTable == null)
+	  hiveTable = context.getTableName();
+      if (options.getWarehouseDir() != null && options.doHiveImport()) 
+	  finalDestDir = new Path(options.getWarehouseDir(), hiveTable);
+
+      LOG.debug("Checking " + finalDestDir.toString() + " for next partition");
+      nextPartition = getNextPartition(fs, finalDestDir);
     }
 
     // move files