Patch for Android Pie ART oatdump to convert CompactDex to StandardDex before exporting

art_9.0_oatdump_cdex.patch

diff --git a/oatdump/Android.bp b/oatdump/Android.bp
index 71e276d..3d50229 100644
--- a/oatdump/Android.bp
+++ b/oatdump/Android.bp
@@ -35,6 +35,7 @@ art_cc_binary {
     shared_libs: [
         "libart",
         "libart-compiler",
+        "libart-dexlayout",
         "libart-disassembler",
         "libdexfile",
         "libbase",
@@ -50,6 +51,7 @@ art_cc_binary {
     shared_libs: [
         "libartd",
         "libartd-compiler",
+        "libartd-dexlayout",
         "libartd-disassembler",
         "libdexfiled",
         "libbase",
@@ -78,6 +80,7 @@ art_cc_binary {
         "libart",
         "libdexfile",
         "libart-compiler",
+        "libart-dexlayout",
         "libart-disassembler",
         "libvixl-arm",
         "libvixl-arm64",
@@ -109,6 +112,7 @@ art_cc_binary {
         "libartd",
         "libdexfiled",
         "libartd-compiler",
+        "libartd-dexlayout",
         "libartd-disassembler",
         "libvixld-arm",
         "libvixld-arm64",
diff --git a/oatdump/oatdump.cc b/oatdump/oatdump.cc
index 547a24c..f34d3f3 100644
--- a/oatdump/oatdump.cc
+++ b/oatdump/oatdump.cc
@@ -44,11 +44,13 @@
 #include "debug/debug_info.h"
 #include "debug/elf_debug_writer.h"
 #include "debug/method_debug_info.h"
+#include "dex/art_dex_file_loader.h"
 #include "dex/code_item_accessors-inl.h"
 #include "dex/descriptors_names.h"
 #include "dex/dex_file-inl.h"
 #include "dex/dex_instruction-inl.h"
 #include "dex/string_reference.h"
+#include "dexlayout.h"
 #include "disassembler.h"
 #include "gc/accounting/space_bitmap-inl.h"
 #include "gc/space/image_space.h"
@@ -626,8 +628,60 @@ class OatDumper {
         const OatFile::OatDexFile* oat_dex_file = oat_dex_files_[i];
         CHECK(oat_dex_file != nullptr);
         CHECK(vdex_dex_file != nullptr);
-        if (!ExportDexFile(os, *oat_dex_file, vdex_dex_file.get())) {
-          success = false;
+
+        // If a CompactDex file is detected within a Vdex container, DexLayout is used to convert
+        // back to a StandardDex file. Since the converted DexFile will most likely not reproduce
+        // original input Dex file, ExportDexFile should know if DexLayout was used to adjust the
+        // checksum logic before writing the file on disk.
+        if (vdex_dex_file->IsCompactDexFile()) {
+          Options options;
+          options.compact_dex_level_ = CompactDexLevel::kCompactDexLevelNone;
+          options.update_checksum_ = true;
+          DexLayout dex_layout(options, /*info*/ nullptr, /*out_file*/ nullptr, /*header*/ nullptr);
+          std::unique_ptr<art::DexContainer> dex_container;
+          bool result = dex_layout.ProcessDexFile(vdex_dex_file->GetLocation().c_str(),
+                                                  vdex_dex_file.get(),
+                                                  i,
+                                                  &dex_container,
+                                                  &error_msg);
+          if (!result) {
+            os << "DexLayout failed to process Dex file: " + error_msg;
+            success = false;
+            break;
+          }
+          DexContainer::Section* main_section = dex_container->GetMainSection();
+          CHECK_EQ(dex_container->GetDataSection()->Size(), 0u);
+
+          const ArtDexFileLoader dex_file_loader;
+          std::unique_ptr<const DexFile> dex(dex_file_loader.Open(
+              main_section->Begin(),
+              main_section->Size(),
+              vdex_dex_file->GetLocation(),
+              vdex_file->GetLocationChecksum(i),
+              nullptr /*oat_dex_file*/,
+              false /*verify*/,
+              true /*verify_checksum*/,
+              &error_msg));
+          if (dex == nullptr) {
+            os << "Failed to load DexFile from layout container: " + error_msg;
+            success = false;
+            break;
+          }
+          if (dex->IsCompactDexFile()) {
+            os <<"CompactDex conversion to StandardDex failed";
+            success = false;
+            break;
+          }
+
+          if (!ExportDexFile(os, *oat_dex_file, dex.get(), true /*used_dexlayout*/)) {
+            success = false;
+            break;
+          }
+        } else {
+          if (!ExportDexFile(os, *oat_dex_file, vdex_dex_file.get(), false /*used_dexlayout*/)) {
+            success = false;
+            break;
+          }
         }
         i++;
       }
@@ -1131,13 +1185,15 @@ class OatDumper {
   // Backwards compatible Dex file export. If dex_file is nullptr (valid Vdex file not present) the
   // Dex resource is extracted from the oat_dex_file and its checksum is repaired since it's not
   // unquickened. Otherwise the dex_file has been fully unquickened and is expected to verify the
-  // original checksum.
+  // original checksum. If used_dexlayout is set, it means that the dex_file has been converted from
+  // a CompactDex file via dexlayout and requires to recompute checksum.
   bool ExportDexFile(std::ostream& os,
                      const OatFile::OatDexFile& oat_dex_file,
-                     const DexFile* dex_file) {
+                     const DexFile* dex_file,
+                     bool used_dexlayout) {
     std::string error_msg;
     std::string dex_file_location = oat_dex_file.GetDexFileLocation();
-    size_t fsize = oat_dex_file.FileSize();
+    size_t fsize = dex_file == nullptr ? oat_dex_file.FileSize() : dex_file->Size();

     // Some quick checks just in case
     if (fsize == 0 || fsize < sizeof(DexFile::Header)) {
@@ -1157,27 +1213,22 @@ class OatDumper {
       reinterpret_cast<DexFile::Header*>(const_cast<uint8_t*>(dex_file->Begin()))->checksum_ =
           dex_file->CalculateChecksum();
     } else {
-      // Vdex unquicken output should match original input bytecode
-      uint32_t orig_checksum =
-          reinterpret_cast<DexFile::Header*>(const_cast<uint8_t*>(dex_file->Begin()))->checksum_;
-      CHECK_EQ(orig_checksum, dex_file->CalculateChecksum());
-      if (orig_checksum != dex_file->CalculateChecksum()) {
-        os << "Unexpected checksum from unquicken dex file '" << dex_file_location << "'\n";
-        return false;
+      // When dexlayout is used to convert CompactDex back to StandardDex, checksum is not
+      // reproducible
+      if (used_dexlayout) {
+        reinterpret_cast<DexFile::Header*>(const_cast<uint8_t*>(dex_file->Begin()))->checksum_ =
+            dex_file->CalculateChecksum();
+      } else {
+        // Vdex unquicken output should match original input bytecode
+        uint32_t orig_checksum =
+            reinterpret_cast<DexFile::Header*>(const_cast<uint8_t*>(dex_file->Begin()))->checksum_;
+        if (orig_checksum != dex_file->CalculateChecksum()) {
+          os << "Unexpected checksum from unquicken dex file '" << dex_file_location << "'\n";
+          return false;
+        }
       }
     }

-    // Update header for shared section.
-    uint32_t shared_section_offset = 0u;
-    uint32_t shared_section_size = 0u;
-    if (dex_file->IsCompactDexFile()) {
-      CompactDexFile::Header* const header =
-          reinterpret_cast<CompactDexFile::Header*>(const_cast<uint8_t*>(dex_file->Begin()));
-      shared_section_offset = header->data_off_;
-      shared_section_size = header->data_size_;
-      // The shared section will be serialized right after the dex file.
-      header->data_off_ = header->file_size_;
-    }
     // Verify output directory exists
     if (!OS::DirectoryExists(options_.export_dex_location_)) {
       // TODO: Extend OS::DirectoryExists if symlink support is required
@@ -1231,15 +1282,6 @@ class OatDumper {
       return false;
     }

-    if (shared_section_size != 0) {
-      success = file->WriteFully(dex_file->Begin() + shared_section_offset, shared_section_size);
-      if (!success) {
-        os << "Failed to write shared data section";
-        file->Erase();
-        return false;
-      }
-    }
-
     if (file->FlushCloseOrErase() != 0) {
       os << "Flush and close failed";
       return false;

Thanks for this patch!