Add ResTable_sparseTypeEntry support

Benchmarks on bullhead-userdebug show that there is a negligent
performance impact when using sparse entries on a 30% loaded
sparse type of 1000 resources.

Benchmark                                             Time           CPU Iterations
-----------------------------------------------------------------------------------
BM_SparseEntryGetResourceSparseLarge                255 ns        254 ns    2751408
BM_SparseEntryGetResourceNotSparseLarge             254 ns        254 ns    2756534

Bug: 27381711
Test: make libandroidfw_tests aapt2_tests
Change-Id: I051ea22f2f6b2bc3696e446adc9e2a34be18009f

Author: Adam Lesinski

libs/androidfw/ResourceTypes.cpp

@@ -6074,6 +6074,10 @@ bool ResTable::getResourceFlags(uint32_t resID, uint32_t* outFlags) const {
     return true;
 }
 
+static bool keyCompare(const ResTable_sparseTypeEntry& entry , uint16_t entryIdx) {
+  return dtohs(entry.idx) < entryIdx;
+}
+
 status_t ResTable::getEntry(
         const PackageGroup* packageGroup, int typeIndex, int entryIndex,
         const ResTable_config* config,
@@ -6115,6 +6119,9 @@ status_t ResTable::getEntry(
             currentTypeIsOverlay = true;
         }
 
+        // Check that the entry idx is within range of the declared entry count (ResTable_typeSpec).
+        // Particular types (ResTable_type) may be encoded with sparse entries, and so their
+        // entryCount do not need to match.
         if (static_cast<size_t>(realEntryIndex) >= typeSpec->entryCount) {
             ALOGW("For resource 0x%08x, entry index(%d) is beyond type entryCount(%d)",
                     Res_MAKEID(packageGroup->id - 1, typeIndex, entryIndex),
@@ -6169,11 +6176,37 @@ status_t ResTable::getEntry(
                 continue;
             }
 
-            // Check if there is the desired entry in this type.
             const uint32_t* const eindex = reinterpret_cast<const uint32_t*>(
                     reinterpret_cast<const uint8_t*>(thisType) + dtohs(thisType->header.headerSize));
 
-            uint32_t thisOffset = dtohl(eindex[realEntryIndex]);
+            uint32_t thisOffset;
+
+            // Check if there is the desired entry in this type.
+            if (thisType->flags & ResTable_type::FLAG_SPARSE) {
+                // This is encoded as a sparse map, so perform a binary search.
+                const ResTable_sparseTypeEntry* sparseIndices =
+                        reinterpret_cast<const ResTable_sparseTypeEntry*>(eindex);
+                const ResTable_sparseTypeEntry* result = std::lower_bound(
+                        sparseIndices, sparseIndices + dtohl(thisType->entryCount), realEntryIndex,
+                        keyCompare);
+                if (result == sparseIndices + dtohl(thisType->entryCount)
+                        || dtohs(result->idx) != realEntryIndex) {
+                    // No entry found.
+                    continue;
+                }
+
+                // Extract the offset from the entry. Each offset must be a multiple of 4
+                // so we store it as the real offset divided by 4.
+                thisOffset = dtohs(result->offset) * 4u;
+            } else {
+                if (static_cast<uint32_t>(realEntryIndex) >= dtohl(thisType->entryCount)) {
+                    // Entry does not exist.
+                    continue;
+                }
+
+                thisOffset = dtohl(eindex[realEntryIndex]);
+            }
+
             if (thisOffset == ResTable_type::NO_ENTRY) {
                 // There is no entry for this index and configuration.
                 continue;
@@ -6480,12 +6513,6 @@ status_t ResTable::parsePackage(const ResTable_package* const pkg,
                 }
 
                 Type* t = typeList.editItemAt(typeList.size() - 1);
-                if (newEntryCount != t->entryCount) {
-                    ALOGE("ResTable_type entry count inconsistent: given %d, previously %d",
-                        (int)newEntryCount, (int)t->entryCount);
-                    return (mError=BAD_TYPE);
-                }
-
                 if (t->package != package) {
                     ALOGE("No TypeSpec for type %d", type->id);
                     return (mError=BAD_TYPE);
@@ -7096,8 +7123,17 @@ void ResTable::print(bool inclValues) const
                 thisConfig.copyFromDtoH(type->config);
 
                 String8 configStr = thisConfig.toString();
-                printf("      config %s:\n", configStr.size() > 0
+                printf("      config %s", configStr.size() > 0
                         ? configStr.string() : "(default)");
+                if (type->flags != 0u) {
+                    printf(" flags=0x%02x", type->flags);
+                    if (type->flags & ResTable_type::FLAG_SPARSE) {
+                        printf(" [sparse]");
+                    }
+                }
+
+                printf(":\n");
+
                 size_t entryCount = dtohl(type->entryCount);
                 uint32_t entriesStart = dtohl(type->entriesStart);
                 if ((entriesStart&0x3) != 0) {
@@ -7109,18 +7145,30 @@ void ResTable::print(bool inclValues) const
                     printf("      NON-INTEGER ResTable_type header.size: 0x%x\n", typeSize);
                     continue;
                 }
-                for (size_t entryIndex=0; entryIndex<entryCount; entryIndex++) {
-                    const uint32_t* const eindex = (const uint32_t*)
-                        (((const uint8_t*)type) + dtohs(type->header.headerSize));
 
-                    uint32_t thisOffset = dtohl(eindex[entryIndex]);
-                    if (thisOffset == ResTable_type::NO_ENTRY) {
-                        continue;
+                const uint32_t* const eindex = (const uint32_t*)
+                        (((const uint8_t*)type) + dtohs(type->header.headerSize));
+                for (size_t entryIndex=0; entryIndex<entryCount; entryIndex++) {
+                    size_t entryId;
+                    uint32_t thisOffset;
+                    if (type->flags & ResTable_type::FLAG_SPARSE) {
+                        const ResTable_sparseTypeEntry* entry =
+                                reinterpret_cast<const ResTable_sparseTypeEntry*>(
+                                        eindex + entryIndex);
+                        entryId = dtohs(entry->idx);
+                        // Offsets are encoded as divided by 4.
+                        thisOffset = static_cast<uint32_t>(dtohs(entry->offset)) * 4u;
+                    } else {
+                        entryId = entryIndex;
+                        thisOffset = dtohl(eindex[entryIndex]);
+                        if (thisOffset == ResTable_type::NO_ENTRY) {
+                            continue;
+                        }
                     }
 
                     uint32_t resID = (0xff000000 & ((packageId)<<24))
                                 | (0x00ff0000 & ((typeIndex+1)<<16))
-                                | (0x0000ffff & (entryIndex));
+                                | (0x0000ffff & (entryId));
                     if (packageId == 0) {
                         pg->dynamicRefTable.lookupResourceId(&resID);
                     }

libs/androidfw/TypeWrappers.cpp

@@ -16,23 +16,45 @@
 
 #include <androidfw/TypeWrappers.h>
 
+#include <algorithm>
+
 namespace android {
 
+TypeVariant::TypeVariant(const ResTable_type* data) : data(data), mLength(dtohl(data->entryCount)) {
+    if (data->flags & ResTable_type::FLAG_SPARSE) {
+        const uint32_t entryCount = dtohl(data->entryCount);
+        const uintptr_t containerEnd = reinterpret_cast<uintptr_t>(data) + dtohl(data->header.size);
+        const uint32_t* const entryIndices = reinterpret_cast<const uint32_t*>(
+                reinterpret_cast<uintptr_t>(data) + dtohs(data->header.headerSize));
+        if (reinterpret_cast<uintptr_t>(entryIndices) + (sizeof(uint32_t) * entryCount)
+                > containerEnd) {
+            ALOGE("Type's entry indices extend beyond its boundaries");
+            mLength = 0;
+        } else {
+          mLength = ResTable_sparseTypeEntry{entryIndices[entryCount - 1]}.idx + 1;
+        }
+    }
+}
+
 TypeVariant::iterator& TypeVariant::iterator::operator++() {
     mIndex++;
-    if (mIndex > dtohl(mTypeVariant->data->entryCount)) {
-        mIndex = dtohl(mTypeVariant->data->entryCount);
+    if (mIndex > mTypeVariant->mLength) {
+        mIndex = mTypeVariant->mLength;
     }
     return *this;
 }
 
+static bool keyCompare(uint32_t entry, uint16_t index) {
+  return dtohs(ResTable_sparseTypeEntry{entry}.idx) < index;
+}
+
 const ResTable_entry* TypeVariant::iterator::operator*() const {
     const ResTable_type* type = mTypeVariant->data;
-    const uint32_t entryCount = dtohl(type->entryCount);
-    if (mIndex >= entryCount) {
+    if (mIndex >= mTypeVariant->mLength) {
         return NULL;
     }
 
+    const uint32_t entryCount = dtohl(mTypeVariant->data->entryCount);
     const uintptr_t containerEnd = reinterpret_cast<uintptr_t>(type)
             + dtohl(type->header.size);
     const uint32_t* const entryIndices = reinterpret_cast<const uint32_t*>(
@@ -42,7 +64,19 @@ const ResTable_entry* TypeVariant::iterator::operator*() const {
         return NULL;
     }
 
-    const uint32_t entryOffset = dtohl(entryIndices[mIndex]);
+    uint32_t entryOffset;
+    if (type->flags & ResTable_type::FLAG_SPARSE) {
+      auto iter = std::lower_bound(entryIndices, entryIndices + entryCount, mIndex, keyCompare);
+      if (iter == entryIndices + entryCount
+              || dtohs(ResTable_sparseTypeEntry{*iter}.idx) != mIndex) {
+        return NULL;
+      }
+
+      entryOffset = static_cast<uint32_t>(dtohs(ResTable_sparseTypeEntry{*iter}.offset)) * 4u;
+    } else {
+      entryOffset = dtohl(entryIndices[mIndex]);
+    }
+
     if (entryOffset == ResTable_type::NO_ENTRY) {
         return NULL;
     }

libs/androidfw/include/androidfw/ResourceTypes.h

@@ -1339,12 +1339,21 @@ struct ResTable_typeSpec
 
 /**
  * A collection of resource entries for a particular resource data
- * type. Followed by an array of uint32_t defining the resource
+ * type.
+ *
+ * If the flag FLAG_SPARSE is not set in `flags`, then this struct is
+ * followed by an array of uint32_t defining the resource
  * values, corresponding to the array of type strings in the
  * ResTable_package::typeStrings string block. Each of these hold an
  * index from entriesStart; a value of NO_ENTRY means that entry is
  * not defined.
  *
+ * If the flag FLAG_SPARSE is set in `flags`, then this struct is followed
+ * by an array of ResTable_sparseTypeEntry defining only the entries that
+ * have values for this type. Each entry is sorted by their entry ID such
+ * that a binary search can be performed over the entries. The ID and offset
+ * are encoded in a uint32_t. See ResTabe_sparseTypeEntry.
+ *
  * There may be multiple of these chunks for a particular resource type,
  * supply different configuration variations for the resource values of
  * that type.
@@ -1365,10 +1374,17 @@ struct ResTable_type
     // resource identifier).  0 is invalid.
     uint8_t id;
 
+    enum {
+        // If set, the entry is sparse, and encodes both the entry ID and offset into each entry,
+        // and a binary search is used to find the key. Only available on platforms >= O.
+        // Mark any types that use this with a v26 qualifier to prevent runtime issues on older
+        // platforms.
+        FLAG_SPARSE = 0x01,
+    };
+    uint8_t flags;
+
     // Must be 0.
-    uint8_t res0;
-    // Must be 0.
-    uint16_t res1;
+    uint16_t reserved;
 
     // Number of uint32_t entry indices that follow.
     uint32_t entryCount;
@@ -1380,6 +1396,24 @@ struct ResTable_type
     ResTable_config config;
 };
 
+/**
+ * An entry in a ResTable_type with the flag `FLAG_SPARSE` set.
+ */
+union ResTable_sparseTypeEntry {
+    // Holds the raw uint32_t encoded value. Do not read this.
+    uint32_t entry;
+    struct {
+        // The index of the entry.
+        uint16_t idx;
+
+        // The offset from ResTable_type::entriesStart, divided by 4.
+        uint16_t offset;
+    };
+};
+
+static_assert(sizeof(ResTable_sparseTypeEntry) == sizeof(uint32_t),
+        "ResTable_sparseTypeEntry must be 4 bytes in size");
+
 /**
  * This is the beginning of information about an entry in the resource
  * table.  It holds the reference to the name of this entry, and is

libs/androidfw/include/androidfw/TypeWrappers.h

@@ -23,8 +23,7 @@
 namespace android {
 
 struct TypeVariant {
-    TypeVariant(const ResTable_type* data)
-        : data(data) {}
+    TypeVariant(const ResTable_type* data);
 
     class iterator {
     public:
@@ -72,10 +71,13 @@ struct TypeVariant {
     }
 
     iterator endEntries() const {
-        return iterator(this, dtohl(data->entryCount));
+        return iterator(this, mLength);
     }
 
     const ResTable_type* data;
+
+private:
+    size_t mLength;
 };
 
 } // namespace android

libs/androidfw/tests/Android.mk

@@ -45,6 +45,8 @@ testFiles := \
 benchmarkFiles := \
     AssetManager2_bench.cpp \
     BenchMain.cpp \
+    BenchmarkHelpers.cpp \
+    SparseEntry_bench.cpp \
     TestHelpers.cpp \
     Theme_bench.cpp
 

libs/androidfw/tests/AssetManager2_bench.cpp

@@ -22,6 +22,7 @@
 #include "androidfw/AssetManager2.h"
 #include "androidfw/ResourceTypes.h"
 
+#include "BenchmarkHelpers.h"
 #include "TestHelpers.h"
 #include "data/basic/R.h"
 #include "data/libclient/R.h"
@@ -112,34 +113,6 @@ static void GetResourceBenchmark(const std::vector<std::string>& paths,
   }
 }
 
-static void GetResourceBenchmarkOld(const std::vector<std::string>& paths,
-                                    const ResTable_config* config, uint32_t resid,
-                                    benchmark::State& state) {
-  AssetManager assetmanager;
-  for (const std::string& path : paths) {
-    if (!assetmanager.addAssetPath(String8(path.c_str()), nullptr /* cookie */,
-                                   false /* appAsLib */, false /* isSystemAssets */)) {
-      state.SkipWithError(base::StringPrintf("Failed to load assets %s", path.c_str()).c_str());
-      return;
-    }
-  }
-
-  if (config != nullptr) {
-    assetmanager.setConfiguration(*config);
-  }
-
-  const ResTable& table = assetmanager.getResources(true);
-
-  Res_value value;
-  ResTable_config selected_config;
-  uint32_t flags;
-
-  while (state.KeepRunning()) {
-    table.getResource(resid, &value, false /*may_be_bag*/, 0u /*density*/, &flags,
-                      &selected_config);
-  }
-}
-
 static void BM_AssetManagerGetResource(benchmark::State& state) {
   GetResourceBenchmark({GetTestDataPath() + "/basic/basic.apk"}, nullptr /*config*/,
                        basic::R::integer::number1, state);

libs/androidfw/tests/BenchmarkHelpers.cpp

@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "BenchmarkHelpers.h"
+
+#include "android-base/stringprintf.h"
+#include "androidfw/AssetManager.h"
+
+namespace android {
+
+void GetResourceBenchmarkOld(const std::vector<std::string>& paths, const ResTable_config* config,
+                             uint32_t resid, benchmark::State& state) {
+  AssetManager assetmanager;
+  for (const std::string& path : paths) {
+    if (!assetmanager.addAssetPath(String8(path.c_str()), nullptr /* cookie */,
+                                   false /* appAsLib */, false /* isSystemAssets */)) {
+      state.SkipWithError(base::StringPrintf("Failed to load assets %s", path.c_str()).c_str());
+      return;
+    }
+  }
+
+  if (config != nullptr) {
+    assetmanager.setConfiguration(*config);
+  }
+
+  const ResTable& table = assetmanager.getResources(true);
+
+  Res_value value;
+  ResTable_config selected_config;
+  uint32_t flags;
+
+  while (state.KeepRunning()) {
+    table.getResource(resid, &value, false /*may_be_bag*/, 0u /*density*/, &flags,
+                      &selected_config);
+  }
+}
+
+}  // namespace android