diff --git a/.gitignore b/.gitignore index 6dd763603a..ae7eb7f68f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ assets/scenes +cache build .vs .vscode diff --git a/framework/common/utils.cpp b/framework/common/utils.cpp index fcdbfa3518..6a048eb1f6 100644 --- a/framework/common/utils.cpp +++ b/framework/common/utils.cpp @@ -287,4 +287,25 @@ sg::Node &add_free_camera(sg::Scene &scene, const std::string &node_name, VkExte return *camera_node; } +size_t calculate_hash(const std::vector &data) +{ + static_assert(sizeof(data[0]) == 1); + constexpr size_t chunk_size = sizeof(size_t) / sizeof(data[0]); + size_t data_hash = 0; + size_t offset = 0; + + for (; offset + chunk_size < data.size(); offset += chunk_size) + { + glm::detail::hash_combine(data_hash, *reinterpret_cast(&data[offset])); + } + + if (offset < data.size()) + { + size_t it = 0; + std::memcpy(&it, &data[offset], data.size() - offset); + glm::detail::hash_combine(data_hash, it); + } + return data_hash; +} + } // namespace vkb diff --git a/framework/common/utils.h b/framework/common/utils.h index 1ec00df83c..5faeb1b64a 100644 --- a/framework/common/utils.h +++ b/framework/common/utils.h @@ -36,6 +36,12 @@ namespace vkb * @return The extension */ std::string get_extension(const std::string &uri); +/** + * @brief Generates a hash from an array + * @param data + * @return data_hash hash of the data + */ +size_t calculate_hash(const std::vector &data); /** * @param name String to convert to snake case diff --git a/framework/gltf_loader.cpp b/framework/gltf_loader.cpp index 6e939175e9..b4d9ac31d8 100644 --- a/framework/gltf_loader.cpp +++ b/framework/gltf_loader.cpp @@ -1486,7 +1486,6 @@ std::unique_ptr GLTFLoader::parse_image(tinygltf::Image &gltf_image) { if (!device.is_image_format_supported(image->get_format())) { - LOGW("ASTC not supported: decoding {}", image->get_name()); image = std::make_unique(*image); image->generate_mipmaps(); } diff --git a/framework/scene_graph/components/hpp_image.cpp b/framework/scene_graph/components/hpp_image.cpp index e010a80afd..066bf67fa2 100644 --- a/framework/scene_graph/components/hpp_image.cpp +++ b/framework/scene_graph/components/hpp_image.cpp @@ -42,7 +42,9 @@ HPPImage::HPPImage(const std::string &name, std::vector &&d, std::vecto data{std::move(d)}, format{vk::Format::eR8G8B8A8Unorm}, mipmaps{std::move(m)} -{} +{ + update_hash(); +} std::unique_ptr HPPImage::load(const std::string &name, const std::string &uri, ContentType content_type) @@ -264,6 +266,15 @@ void HPPImage::generate_mipmaps() } } +void HPPImage::update_hash() +{ + data_hash = vkb::calculate_hash(data); +} + +void HPPImage::update_hash(size_t hash) +{ + data_hash = hash; +} const std::vector &HPPImage::get_data() const { return data; @@ -327,6 +338,7 @@ void HPPImage::set_data(const uint8_t *raw_data, size_t size) { assert(data.empty() && "HPPImage data already set"); data = {raw_data, raw_data + size}; + update_hash(); } void HPPImage::set_depth(const uint32_t depth) @@ -364,4 +376,4 @@ void HPPImage::set_width(const uint32_t width) } // namespace components } // namespace scene_graph -} // namespace vkb \ No newline at end of file +} // namespace vkb diff --git a/framework/scene_graph/components/hpp_image.h b/framework/scene_graph/components/hpp_image.h index 43c5b7b0bc..fe4f2942b9 100644 --- a/framework/scene_graph/components/hpp_image.h +++ b/framework/scene_graph/components/hpp_image.h @@ -86,6 +86,8 @@ class HPPImage : public vkb::sg::Component const std::vector> &get_offsets() const; const vkb::core::HPPImage &get_vk_image() const; const vkb::core::HPPImageView &get_vk_image_view() const; + void update_hash(size_t data_hash); + void update_hash(); protected: vkb::scene_graph::components::HPPMipmap &get_mipmap(size_t index); @@ -101,6 +103,7 @@ class HPPImage : public vkb::sg::Component private: std::vector data; + size_t data_hash{0}; vk::Format format = vk::Format::eUndefined; uint32_t layers = 1; std::vector mipmaps{{}}; diff --git a/framework/scene_graph/components/image.cpp b/framework/scene_graph/components/image.cpp index d50cd938dd..f5eddcae54 100644 --- a/framework/scene_graph/components/image.cpp +++ b/framework/scene_graph/components/image.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2024, Arm Limited and Contributors +/* Copyright (c) 2018-2025, Arm Limited and Contributors * * SPDX-License-Identifier: Apache-2.0 * @@ -17,6 +17,7 @@ #include "image.h" +#include #include #include "common/error.h" @@ -152,6 +153,7 @@ Image::Image(const std::string &name, std::vector &&d, std::vector &Image::get_data() const return data; } +size_t Image::get_data_hash() const +{ + return data_hash; +} + void Image::clear_data() { data.clear(); @@ -312,10 +319,21 @@ std::vector &Image::get_mut_data() return data; } +void Image::update_hash() +{ + data_hash = vkb::calculate_hash(data); +} + +void Image::update_hash(size_t hash) +{ + data_hash = hash; +} + void Image::set_data(const uint8_t *raw_data, size_t size) { assert(data.empty() && "Image data already set"); data = {raw_data, raw_data + size}; + update_hash(); } void Image::set_format(const VkFormat f) diff --git a/framework/scene_graph/components/image.h b/framework/scene_graph/components/image.h index 8db3d203e9..1b48b1b66e 100644 --- a/framework/scene_graph/components/image.h +++ b/framework/scene_graph/components/image.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2024, Arm Limited and Contributors +/* Copyright (c) 2018-2025, Arm Limited and Contributors * * SPDX-License-Identifier: Apache-2.0 * @@ -80,6 +80,8 @@ class Image : public Component const std::vector &get_data() const; + size_t get_data_hash() const; + void clear_data(); VkFormat get_format() const; @@ -123,9 +125,15 @@ class Image : public Component std::vector &get_mut_mipmaps(); + void update_hash(); + + void update_hash(size_t data_hash); + private: std::vector data; + size_t data_hash{0}; + VkFormat format{VK_FORMAT_UNDEFINED}; uint32_t layers{1}; diff --git a/framework/scene_graph/components/image/astc.cpp b/framework/scene_graph/components/image/astc.cpp index fbec2c74b8..98cabbb295 100644 --- a/framework/scene_graph/components/image/astc.cpp +++ b/framework/scene_graph/components/image/astc.cpp @@ -17,6 +17,7 @@ #include "scene_graph/components/image/astc.h" +#include #include #include "common/error.h" @@ -29,12 +30,21 @@ #endif #include +#include + #define MAGIC_FILE_CONSTANT 0x5CA1AB13 +#define ASTC_CACHE_DIRECTORY "cache/astc_to_bin" + +constexpr uint32_t ASTC_CACHE_HEADER_SIZE = 64; +constexpr uint32_t ASTC_CACHE_SEED = 1619; namespace vkb { namespace sg { + +using Path = std::filesystem::path; + BlockDim to_blockdim(const VkFormat format) { switch (format) @@ -86,6 +96,45 @@ BlockDim to_blockdim(const VkFormat format) } } +inline astcenc_profile to_profile(const VkFormat format) +{ + switch (format) + { + case VK_FORMAT_ASTC_4x4_UNORM_BLOCK: + case VK_FORMAT_ASTC_5x4_UNORM_BLOCK: + case VK_FORMAT_ASTC_5x5_UNORM_BLOCK: + case VK_FORMAT_ASTC_6x5_UNORM_BLOCK: + case VK_FORMAT_ASTC_6x6_UNORM_BLOCK: + case VK_FORMAT_ASTC_8x5_UNORM_BLOCK: + case VK_FORMAT_ASTC_8x6_UNORM_BLOCK: + case VK_FORMAT_ASTC_8x8_UNORM_BLOCK: + case VK_FORMAT_ASTC_10x5_UNORM_BLOCK: + case VK_FORMAT_ASTC_10x6_UNORM_BLOCK: + case VK_FORMAT_ASTC_10x8_UNORM_BLOCK: + case VK_FORMAT_ASTC_10x10_UNORM_BLOCK: + case VK_FORMAT_ASTC_12x10_UNORM_BLOCK: + case VK_FORMAT_ASTC_12x12_UNORM_BLOCK: + return ASTCENC_PRF_LDR; + case VK_FORMAT_ASTC_4x4_SRGB_BLOCK: + case VK_FORMAT_ASTC_5x4_SRGB_BLOCK: + case VK_FORMAT_ASTC_5x5_SRGB_BLOCK: + case VK_FORMAT_ASTC_6x5_SRGB_BLOCK: + case VK_FORMAT_ASTC_6x6_SRGB_BLOCK: + case VK_FORMAT_ASTC_8x5_SRGB_BLOCK: + case VK_FORMAT_ASTC_8x6_SRGB_BLOCK: + case VK_FORMAT_ASTC_8x8_SRGB_BLOCK: + case VK_FORMAT_ASTC_10x5_SRGB_BLOCK: + case VK_FORMAT_ASTC_10x6_SRGB_BLOCK: + case VK_FORMAT_ASTC_10x8_SRGB_BLOCK: + case VK_FORMAT_ASTC_10x10_SRGB_BLOCK: + case VK_FORMAT_ASTC_12x10_SRGB_BLOCK: + case VK_FORMAT_ASTC_12x12_SRGB_BLOCK: + return ASTCENC_PRF_LDR_SRGB; + default: + throw std::runtime_error{"Invalid astc format"}; + } +} + struct AstcHeader { uint8_t magic[4]; @@ -164,18 +213,136 @@ Astc::Astc(const Image &image) : { init(); + vkb::filesystem::init(); + auto fs = vkb::filesystem::get(); + + size_t key = ASTC_CACHE_SEED; + glm::detail::hash_combine(key, image.get_data_hash()); + + constexpr bool use_cache = true; + constexpr uint32_t bytes_per_pixel = 4; + constexpr const char file_cache_header[ASTC_CACHE_HEADER_SIZE] = "ASTCConvertedDataV01"; + const auto profile = to_profile(image.get_format()); + + auto can_load_from_file = [this, profile, fs, file_cache_header, bytes_per_pixel, use_cache](const Path &path, std::vector &dst_data, uint32_t width, uint32_t height, uint32_t depth) { + if (!use_cache) + { + LOGD("Device does not support ASTC format and cache is disabled. ASTC image {} will be decoded.", get_name()) + return false; + } + try + { + if (!fs->exists(path)) + { + LOGW("Device does not support ASTC format and cache file {} does not exist. ASTC image {} will be decoded.", path.string(), get_name()) + return false; + } + else + { + LOGD("Loading ASTC image {} from cache file {}", get_name(), path.string()) + } + size_t offset = 0; + + auto copy_from_file = [fs, path](void *dst, size_t *offset, size_t content_size) { + const auto bin_content = fs->read_chunk(path, *offset, content_size); + std::memcpy(dst, &bin_content[0], content_size); + *offset += content_size; + }; + + char header[ASTC_CACHE_HEADER_SIZE]; + copy_from_file(&header, &offset, ASTC_CACHE_HEADER_SIZE); + if (std::strcmp(header, file_cache_header) != 0) + { + return false; + } + + uint32_t file_width, file_height, file_depth; + copy_from_file(&file_width, &offset, sizeof(std::uint32_t)); + copy_from_file(&file_height, &offset, sizeof(std::uint32_t)); + copy_from_file(&file_depth, &offset, sizeof(std::uint32_t)); + + if (file_width != width || width == 0 || + file_height != height || height == 0 || + file_depth != depth || depth == 0) + { + return false; + } + else + { + set_width(width); + set_height(height); + set_depth(depth); + set_format(profile == ASTCENC_PRF_LDR_SRGB ? VK_FORMAT_R8G8B8A8_SRGB : VK_FORMAT_R8G8B8A8_UNORM); + } + + auto image_size = width * height * depth * bytes_per_pixel; + dst_data.resize(image_size); + copy_from_file(dst_data.data(), &offset, image_size); + + return true; + } + catch (const std::runtime_error &e) + { + LOGE("ERROR loading file {} from cache. Error: <{}>", path.string(), e.what()) + // file is truncated + return false; + } + }; + + auto save_to_file = [fs, file_cache_header, bytes_per_pixel, use_cache](const Path &path, uint8_t *dst_data, uint32_t width, uint32_t height, uint32_t depth) { + if (!use_cache) + { + return; + } + try + { + LOGI("Saving ASTC cache data to file: {}", path.string()); + + auto image_size = width * height * depth * bytes_per_pixel; + + std::vector astc_file_content; + astc_file_content.reserve(sizeof(file_cache_header) + (3 * sizeof(std::uint32_t)) + image_size); + + auto append_to_file = [](std::vector &dst_file, const std::uint8_t *content, size_t content_size) { + dst_file.insert(dst_file.end(), content, content + content_size); + }; + + append_to_file(astc_file_content, (uint8_t *) &file_cache_header, sizeof(file_cache_header)); + append_to_file(astc_file_content, (uint8_t *) &width, sizeof(uint32_t)); + append_to_file(astc_file_content, (uint8_t *) &height, sizeof(uint32_t)); + append_to_file(astc_file_content, (uint8_t *) &depth, sizeof(uint32_t)); + append_to_file(astc_file_content, (uint8_t *) dst_data, image_size); + + fs->write_file(path, astc_file_content); + } + catch (const std::runtime_error &e) + { + LOGE("ERROR: saving to file: {}\nError<{}>", path.string(), e.what()) + } + }; + // Locate mip #0 in the KTX. This is the first one in the data array for KTX1s, but the last one in KTX2s! auto mip_it = std::ranges::find_if(image.get_mipmaps(), [](auto &mip) { return mip.level == 0; }); assert(mip_it != image.get_mipmaps().end() && "Mip #0 not found"); - // When decoding ASTC on CPU (as it is the case in here), we don't decode all mips in the mip chain. - // Instead, we just decode mip #0 and re-generate the other LODs later (via image->generate_mipmaps()). - const auto blockdim = to_blockdim(image.get_format()); - const auto &extent = mip_it->extent; - auto size = extent.width * extent.height * extent.depth * 4; - const uint8_t *data_ptr = image.get_data().data() + mip_it->offset; - decode(blockdim, mip_it->extent, data_ptr, size); + const std::string path = fmt::format("{}/{}.bin", ASTC_CACHE_DIRECTORY, uint64_t(key)); + + if (!can_load_from_file(path, get_mut_data(), mip_it->extent.width, mip_it->extent.height, mip_it->extent.depth)) + { + // When decoding ASTC on CPU (as it is the case in here), we don't decode all mips in the mip chain. + // Instead, we just decode mip #0 and re-generate the other LODs later (via image->generate_mipmaps()). + const auto blockdim = to_blockdim(image.get_format()); + const auto &extent = mip_it->extent; + auto size = extent.width * extent.height * extent.depth * 4; + const uint8_t *data_ptr = image.get_data().data() + mip_it->offset; + + decode(blockdim, mip_it->extent, data_ptr, size); + + save_to_file(path, get_mut_data().data(), mip_it->extent.width, mip_it->extent.height, mip_it->extent.depth); + } + + update_hash(image.get_data_hash()); } Astc::Astc(const std::string &name, const std::vector &data) : @@ -207,6 +374,8 @@ Astc::Astc(const std::string &name, const std::vector &data) : /* depth = */ static_cast(header.zsize[0] + 256 * header.zsize[1] + 65536 * header.zsize[2])}; decode(blockdim, extent, data.data() + sizeof(AstcHeader), to_u32(data.size() - sizeof(AstcHeader))); + + update_hash(get_data_hash()); } } // namespace sg diff --git a/framework/scene_graph/components/image/ktx.cpp b/framework/scene_graph/components/image/ktx.cpp index e0563a014f..2420c6c5b2 100644 --- a/framework/scene_graph/components/image/ktx.cpp +++ b/framework/scene_graph/components/image/ktx.cpp @@ -1,5 +1,5 @@ -/* Copyright (c) 2019-2024, Arm Limited and Contributors - * Copyright (c) 2019-2024, Sascha Willems +/* Copyright (c) 2019-2025, Arm Limited and Contributors + * Copyright (c) 2019-2025, Sascha Willems * * SPDX-License-Identifier: Apache-2.0 * @@ -103,6 +103,7 @@ Ktx::Ktx(const std::string &name, const std::vector &data, ContentType set_height(texture->baseHeight); set_depth(texture->baseDepth); set_layers(texture->numLayers); + update_hash(); bool cubemap = false;