diff --git a/scripts/startup/bl_operators/object_quick_effects.py b/scripts/startup/bl_operators/object_quick_effects.py index 6cbe88d063b..6f05b080276 100644 --- a/scripts/startup/bl_operators/object_quick_effects.py +++ b/scripts/startup/bl_operators/object_quick_effects.py @@ -127,10 +127,9 @@ class QuickFur(ObjectModeOperator, Operator): with bpy.data.libraries.load( asset_library_filepath, - link=False, - clear_asset_data=True, - reuse_local_id=True, - recursive=True, + link=True, + pack=True, + set_fake=False, ) as (data_src, data_dst): # The values are assumed to exist, no inspection of the source is needed. del data_src diff --git a/scripts/startup/bl_ui/space_userpref.py b/scripts/startup/bl_ui/space_userpref.py index c2def125072..44107b2b892 100644 --- a/scripts/startup/bl_ui/space_userpref.py +++ b/scripts/startup/bl_ui/space_userpref.py @@ -2888,6 +2888,7 @@ class USERPREF_PT_developer_tools(Panel): ({"property": "use_viewport_debug"}, None), ({"property": "use_eevee_debug"}, None), ({"property": "use_extensions_debug"}, ("/blender/blender/issues/119521", "#119521")), + ({"property": "no_data_block_packing"}, ("/blender/blender/issues/132167", "#132167")), ), ) diff --git a/source/blender/asset_system/AS_asset_library.hh b/source/blender/asset_system/AS_asset_library.hh index df5342623c8..b0e72029f5b 100644 --- a/source/blender/asset_system/AS_asset_library.hh +++ b/source/blender/asset_system/AS_asset_library.hh @@ -303,3 +303,15 @@ void AS_asset_full_path_explode_from_weak_ref(const AssetWeakReference *asset_re char **r_dir, char **r_group, char **r_name); + +/** + * Updates the default import method for asset libraries based on + * #U.experimental.no_data_block_packing. + */ +void AS_asset_library_import_method_ensure_valid(Main &bmain); +/** + * This is not done as part of #AS_asset_library_import_method_ensure_valid because it changes + * run-time data only and does not need to happen during versioning (also it appears to break tests + * when run during versioning). + */ +void AS_asset_library_essential_import_method_update(); diff --git a/source/blender/asset_system/AS_essentials_library.hh b/source/blender/asset_system/AS_essentials_library.hh index 918440e20b1..91153fc8712 100644 --- a/source/blender/asset_system/AS_essentials_library.hh +++ b/source/blender/asset_system/AS_essentials_library.hh @@ -14,4 +14,4 @@ namespace blender::asset_system { StringRefNull essentials_directory_path(); -} +} // namespace blender::asset_system diff --git a/source/blender/asset_system/intern/asset_library.cc b/source/blender/asset_system/intern/asset_library.cc index 4486707a663..476a59e012b 100644 --- a/source/blender/asset_system/intern/asset_library.cc +++ b/source/blender/asset_system/intern/asset_library.cc @@ -21,11 +21,14 @@ #include "BLI_path_utils.hh" #include "BLI_string.h" +#include "DNA_space_types.h" #include "DNA_userdef_types.h" +#include "DNA_windowmanager_types.h" #include "asset_catalog_collection.hh" #include "asset_catalog_definition_file.hh" #include "asset_library_service.hh" +#include "essentials_library.hh" #include "runtime_library.hh" #include "utils.hh" @@ -161,6 +164,67 @@ void AS_asset_full_path_explode_from_weak_ref(const AssetWeakReference *asset_re } } +static void update_import_method_for_user_libraries() +{ + LISTBASE_FOREACH (bUserAssetLibrary *, library, &U.asset_libraries) { + if (U.experimental.no_data_block_packing) { + if (library->import_method == ASSET_IMPORT_PACK) { + library->import_method = ASSET_IMPORT_APPEND_REUSE; + } + } + else { + if (library->import_method == ASSET_IMPORT_APPEND_REUSE) { + library->import_method = ASSET_IMPORT_PACK; + } + } + } +} + +static void update_import_method_for_asset_browsers(Main &bmain) +{ + LISTBASE_FOREACH (bScreen *, screen, &bmain.screens) { + LISTBASE_FOREACH (ScrArea *, area, &screen->areabase) { + LISTBASE_FOREACH (SpaceLink *, sl, &area->spacedata) { + if (sl->spacetype != SPACE_FILE) { + continue; + } + SpaceFile *sfile = reinterpret_cast(sl); + if (!sfile->asset_params) { + continue; + } + if (U.experimental.no_data_block_packing) { + if (sfile->asset_params->import_method == FILE_ASSET_IMPORT_PACK) { + sfile->asset_params->import_method = FILE_ASSET_IMPORT_APPEND_REUSE; + } + } + else { + if (sfile->asset_params->import_method == FILE_ASSET_IMPORT_APPEND_REUSE) { + sfile->asset_params->import_method = FILE_ASSET_IMPORT_PACK; + } + } + } + } + } +} + +void AS_asset_library_import_method_ensure_valid(Main &bmain) +{ + update_import_method_for_user_libraries(); + update_import_method_for_asset_browsers(bmain); +} + +void AS_asset_library_essential_import_method_update() +{ + AssetLibraryReference library_ref{}; + library_ref.custom_library_index = -1; + library_ref.type = ASSET_LIBRARY_ESSENTIALS; + EssentialsAssetLibrary *library = dynamic_cast( + AS_asset_library_load(nullptr, library_ref)); + if (library) { + library->update_default_import_method(); + } +} + namespace blender::asset_system { AssetLibrary::AssetLibrary(eAssetLibraryType library_type, StringRef name, StringRef root_path) diff --git a/source/blender/asset_system/intern/library_types/essentials_library.cc b/source/blender/asset_system/intern/library_types/essentials_library.cc index 1309d7b1b82..1c589e94528 100644 --- a/source/blender/asset_system/intern/library_types/essentials_library.cc +++ b/source/blender/asset_system/intern/library_types/essentials_library.cc @@ -10,6 +10,8 @@ #include "utils.hh" +#include "DNA_userdef_types.h" + #include "AS_essentials_library.hh" #include "essentials_library.hh" @@ -20,7 +22,10 @@ EssentialsAssetLibrary::EssentialsAssetLibrary() {}, utils::normalize_directory_path(essentials_directory_path())) { - import_method_ = ASSET_IMPORT_APPEND_REUSE; + import_method_ = ASSET_IMPORT_PACK; + if (U.experimental.no_data_block_packing) { + import_method_ = ASSET_IMPORT_APPEND_REUSE; + } } std::optional EssentialsAssetLibrary::library_reference() const @@ -31,6 +36,14 @@ std::optional EssentialsAssetLibrary::library_reference() return library_ref; } +void EssentialsAssetLibrary::update_default_import_method() +{ + import_method_ = ASSET_IMPORT_PACK; + if (U.experimental.no_data_block_packing) { + import_method_ = ASSET_IMPORT_APPEND_REUSE; + } +} + StringRefNull essentials_directory_path() { static std::string path = []() { diff --git a/source/blender/asset_system/intern/library_types/essentials_library.hh b/source/blender/asset_system/intern/library_types/essentials_library.hh index 2da78739586..30e765c3a7e 100644 --- a/source/blender/asset_system/intern/library_types/essentials_library.hh +++ b/source/blender/asset_system/intern/library_types/essentials_library.hh @@ -17,6 +17,9 @@ class EssentialsAssetLibrary : public OnDiskAssetLibrary { EssentialsAssetLibrary(); std::optional library_reference() const override; + + /** Update the default import method based on whether packed data-blocks are supported. */ + void update_default_import_method(); }; } // namespace blender::asset_system diff --git a/source/blender/blenkernel/BKE_blendfile_link_append.hh b/source/blender/blenkernel/BKE_blendfile_link_append.hh index 595d6672fdc..a745cff3591 100644 --- a/source/blender/blenkernel/BKE_blendfile_link_append.hh +++ b/source/blender/blenkernel/BKE_blendfile_link_append.hh @@ -334,6 +334,14 @@ void BKE_blendfile_link_append_context_init_done(BlendfileLinkAppendContext *lap */ void BKE_blendfile_link(BlendfileLinkAppendContext *lapp_context, ReportList *reports); +/** + * Perform packing operation. + * + * The IDs processed by this functions are the one that have been linked by a previous call to + * #BKE_blendfile_link on the same `lapp_context`. + */ +void BKE_blendfile_link_pack(BlendfileLinkAppendContext *lapp_context, ReportList *reports); + /** * Perform append operation, using modern ID usage looper to detect which ID should be kept * linked, made local, duplicated as local, re-used from local etc. diff --git a/source/blender/blenkernel/BKE_id_hash.hh b/source/blender/blenkernel/BKE_id_hash.hh new file mode 100644 index 00000000000..c69e3b34900 --- /dev/null +++ b/source/blender/blenkernel/BKE_id_hash.hh @@ -0,0 +1,49 @@ +/* SPDX-FileCopyrightText: 2025 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include + +#include "BKE_main.hh" + +#include "BLI_map.hh" +#include "BLI_vector.hh" + +#include "DNA_ID.h" + +namespace blender::bke::id_hash { + +/** + * The hash of all the root IDs and their dependencies. + */ +struct ValidDeepHashes { + Map hashes; +}; + +struct DeepHashErrors { + /** + * A list of missing files paths in the case that the deep hashes could not be computed. + */ + VectorSet missing_files; + + /** + * Files that were modified since the linked ID was loaded. So the currently linked ID would not + * be matching the deep hash computed based on the source file. + */ + VectorSet updated_files; +}; + +using IDHashResult = std::variant; + +/** + * Compute a hash of the given IDs, including all their dependencies. + * This needs access to the original .blend files that the linked data-blocks come from to be able + * to compute their hash. + */ +IDHashResult compute_linked_id_deep_hashes(const Main &bmain, Span root_ids); + +/** Utility to convert the hash into a readable string. */ +std::string id_hash_to_hex(const IDHash &hash); + +} // namespace blender::bke::id_hash diff --git a/source/blender/blenkernel/BKE_lib_id.hh b/source/blender/blenkernel/BKE_lib_id.hh index 2b2079f8ea6..f7f4a6475d7 100644 --- a/source/blender/blenkernel/BKE_lib_id.hh +++ b/source/blender/blenkernel/BKE_lib_id.hh @@ -71,6 +71,11 @@ struct ID_Runtime_Remap { }; struct ID_Runtime { + /** + * The last modifification time of the source .blend file where this ID was loaded from. + */ + int64_t src_blend_modifification_time; + ID_Runtime_Remap remap = {}; /** * The depsgraph that owns this data block. This is only set on data-blocks which are @@ -249,6 +254,11 @@ enum { */ LIB_ID_COPY_SET_COPIED_ON_WRITE = 1 << 10, + /** + * Set #ID.newid pointer of the given source ID with the address of its new copy. + */ + LIB_ID_COPY_ID_NEW_SET = 1 << 11, + /* *** Specific options to some ID types or usages. *** */ /* *** May be ignored by unrelated ID copying functions. *** */ /** Object only, needed by make_local code. */ diff --git a/source/blender/blenkernel/BKE_library.hh b/source/blender/blenkernel/BKE_library.hh index 5452786da78..5895579356e 100644 --- a/source/blender/blenkernel/BKE_library.hh +++ b/source/blender/blenkernel/BKE_library.hh @@ -9,12 +9,15 @@ * API to manage `Library` data-blocks. */ +#include "DNA_ID.h" + +#include "BLI_map.hh" +#include "BLI_set.hh" #include "BLI_string_ref.hh" #include "BKE_main.hh" struct FileData; -struct Library; struct ListBase; struct Main; struct UniqueName_Map; @@ -27,6 +30,10 @@ struct LibraryRuntime { /** * Filedata (i.e. opened blendfile) source of this library data. + * + * \note: This is not always matching the library's blendfile path. E.g. for archive packed + * libraries, this will be the filedata of the packing blendfile, not of the reference/source + * library. */ FileData *filedata = nullptr; /** @@ -48,6 +55,12 @@ struct LibraryRuntime { /** Set for indirectly linked libraries, used in the outliner and while reading. */ Library *parent = nullptr; + /** + * Helper listing all archived libraries 'versions' of this library. + * Should only contain something if this library is a regular 'real' blendfile library. + */ + blender::Vector archived_libraries = {}; + /** #eLibrary_Tag. */ ushort tag = 0; @@ -67,6 +80,18 @@ struct LibraryRuntime { */ Library *search_filepath_abs(ListBase *libraries, blender::StringRef filepath_abs); +/** + * Pack given linked ID, and all the related hierarchy. + * + * Will set final embedded ID into each ID::newid pointers. + */ +void pack_linked_id_hierarchy(Main &bmain, ID &root_id); + +/** + * Cleanup references to removed/deleted archive libraries in their archive parent. + */ +void main_cleanup_parent_archives(Main &bmain); + }; // namespace blender::bke::library /** #LibraryRuntime.tag */ diff --git a/source/blender/blenkernel/CMakeLists.txt b/source/blender/blenkernel/CMakeLists.txt index e2d79497f56..8b86ae28d5c 100644 --- a/source/blender/blenkernel/CMakeLists.txt +++ b/source/blender/blenkernel/CMakeLists.txt @@ -147,6 +147,7 @@ set(SRC intern/grease_pencil_vertex_groups.cc intern/icons.cc intern/icons_rasterize.cc + intern/id_hash.cc intern/idprop.cc intern/idprop_create.cc intern/idprop_serialize.cc @@ -419,6 +420,7 @@ set(SRC BKE_grease_pencil_legacy_convert.hh BKE_grease_pencil_vertex_groups.hh BKE_icons.h + BKE_id_hash.hh BKE_idprop.hh BKE_idtype.hh BKE_image.hh diff --git a/source/blender/blenkernel/intern/blendfile_link_append.cc b/source/blender/blenkernel/intern/blendfile_link_append.cc index 85a00671645..2e6c3542a64 100644 --- a/source/blender/blenkernel/intern/blendfile_link_append.cc +++ b/source/blender/blenkernel/intern/blendfile_link_append.cc @@ -949,6 +949,51 @@ static bool foreach_libblock_link_append_common_processing( /** \} */ +/** \name Library embedding code. + * \{ */ + +void BKE_blendfile_link_pack(BlendfileLinkAppendContext *lapp_context, ReportList * /*reports*/) +{ + Main *bmain = lapp_context->params->bmain; + + /* Delete newly linked data-blocks after they have been packed. */ + blender::Vector linked_ids_to_delete; + { + ID *id; + FOREACH_MAIN_ID_BEGIN (bmain, id) { + if (ID_IS_LINKED(id) && !ID_IS_PACKED(id)) { + if (!(id->tag & ID_TAG_PRE_EXISTING)) { + linked_ids_to_delete.append(id); + } + } + } + FOREACH_MAIN_ID_END; + } + + for (BlendfileLinkAppendContextItem &item : lapp_context->items) { + ID *id = item.new_id; + BLI_assert(ID_IS_LINKED(id)); + if (!(ID_IS_PACKED(id) || (id->newid && ID_IS_PACKED(id->newid)))) { + /* No yet packed. */ + blender::bke::library::pack_linked_id_hierarchy(*bmain, *id); + } + /* Calling code may want to access newly packed embedded IDs from the link/append context + * items. */ + if (id->newid) { + item.new_id = id->newid; + } + } + BKE_main_id_newptr_and_tag_clear(bmain); + + BKE_main_id_tag_all(bmain, ID_TAG_DOIT, false); + for (ID *id : linked_ids_to_delete) { + id->tag |= ID_TAG_DOIT; + } + BKE_id_multi_tagged_delete(bmain); +} + +/** \} */ + /** \name Library append code. * \{ */ diff --git a/source/blender/blenkernel/intern/id_hash.cc b/source/blender/blenkernel/intern/id_hash.cc new file mode 100644 index 00000000000..22e75646bc7 --- /dev/null +++ b/source/blender/blenkernel/intern/id_hash.cc @@ -0,0 +1,250 @@ +/* SPDX-FileCopyrightText: 2025 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include +#include +#include + +#include "BKE_id_hash.hh" +#include "BKE_lib_id.hh" +#include "BKE_lib_query.hh" +#include "BKE_library.hh" +#include "BKE_main.hh" + +#include "BLI_fileops.hh" +#include "BLI_mmap.h" +#include "BLI_mutex.hh" +#include "BLI_set.hh" + +namespace blender::bke::id_hash { + +static std::optional> read_file(const StringRefNull path) +{ + blender::fstream stream{path.c_str(), std::ios_base::in | std::ios_base::binary}; + stream.seekg(0, std::ios_base::end); + const int64_t size = stream.tellg(); + stream.seekg(0, std::ios_base::beg); + + blender::Vector buffer(size); + stream.read(buffer.data(), size); + if (stream.bad()) { + return std::nullopt; + } + + return buffer; +} + +static std::optional compute_file_hash_with_file_read(const StringRefNull path) +{ + const std::optional> buffer = read_file(path); + if (!buffer) { + return std::nullopt; + } + return XXH3_128bits(buffer->data(), buffer->size()); +} + +static std::optional compute_file_hash_with_memory_map(const StringRefNull path) +{ + const int file = BLI_open(path.c_str(), O_BINARY | O_RDONLY, 0); + if (file == -1) { + return std::nullopt; + } + BLI_mmap_file *mmap_file = BLI_mmap_open(file); + if (!mmap_file) { + return std::nullopt; + } + BLI_SCOPED_DEFER([&]() { BLI_mmap_free(mmap_file); }); + const size_t size = BLI_mmap_get_length(mmap_file); + const void *data = BLI_mmap_get_pointer(mmap_file); + const XXH128_hash_t hash = XXH3_128bits(data, size); + if (BLI_mmap_any_io_error(mmap_file)) { + return std::nullopt; + } + return hash; +} + +static std::optional compute_file_hash(const StringRefNull path) +{ + /* First try the memory map the file, because it avoids an extra copy. */ + if (const std::optional hash = compute_file_hash_with_memory_map(path)) { + /* Make sure both code paths are tested even if memory mapping should almost always work. */ + BLI_assert(hash->low64 == compute_file_hash_with_file_read(path)->low64); + return hash; + } + if (const std::optional hash = compute_file_hash_with_file_read(path)) { + return hash; + } + return std::nullopt; +} + +struct CachedFileHash { + int64_t last_modified = 0; + XXH128_hash_t hash; +}; + +static std::optional get_source_file_hash(const ID &id, DeepHashErrors &r_errors) +{ + static Map cache; + static Mutex mutex; + + const StringRefNull path = id.lib->runtime->filepath_abs; + + BLI_stat_t stat; + if (BLI_stat(path.c_str(), &stat) == -1) { + r_errors.missing_files.add_as(path); + return std::nullopt; + } + + std::lock_guard lock(mutex); + if (const CachedFileHash *cached_hash = cache.lookup_ptr_as(path)) { + if (cached_hash->last_modified == stat.st_mtime) { + return cached_hash->hash; + } + } + + if (stat.st_mtime != id.runtime->src_blend_modifification_time) { + r_errors.updated_files.add_as(path); + return std::nullopt; + } + + if (const std::optional hash = compute_file_hash(path)) { + cache.add_overwrite(path, CachedFileHash{stat.st_mtime, *hash}); + return hash; + } + r_errors.missing_files.add_as(path); + return std::nullopt; +} + +static std::optional get_id_shallow_hash(const ID &id, DeepHashErrors &r_errors) +{ + BLI_assert(ID_IS_LINKED(&id)); + const StringRefNull id_name = id.name; + const std::optional file_hash = get_source_file_hash(id, r_errors); + if (!file_hash) { + return std::nullopt; + } + + XXH3_state_t *hash_state = XXH3_createState(); + XXH3_128bits_reset(hash_state); + XXH3_128bits_update(hash_state, id_name.data(), id_name.size()); + XXH3_128bits_update(hash_state, &*file_hash, sizeof(XXH128_hash_t)); + XXH128_hash_t shallow_hash = XXH3_128bits_digest(hash_state); + XXH3_freeState(hash_state); + return shallow_hash; +} + +static void compute_deep_hash_recursive(const Main &bmain, + const ID &id, + Set ¤t_stack, + Map &r_hashes, + DeepHashErrors &r_errors) +{ + if (r_hashes.contains(&id)) { + return; + } + if (!id.deep_hash.is_null()) { + r_hashes.add(&id, id.deep_hash); + return; + } + current_stack.add(&id); + const std::optional id_shallow_hash = get_id_shallow_hash(id, r_errors); + if (!id_shallow_hash) { + return; + } + + XXH3_state_t *hash_state = XXH3_createState(); + XXH3_128bits_reset(hash_state); + XXH3_128bits_update(hash_state, &*id_shallow_hash, sizeof(XXH128_hash_t)); + + bool success = true; + BKE_library_foreach_ID_link( + const_cast
(&bmain), + const_cast(&id), + [&](LibraryIDLinkCallbackData *cb_data) { + if (cb_data->cb_flag & IDWALK_CB_LOOPBACK) { + /* Loopback pointer (e.g. from a shapekey to its owner geometry ID, or from a collection + * to its parents) should always be ignored, as they do not represent an actual + * dependency. The dependency relationship should already have been processed from the + * owner to its dependency anyway (if applicable). */ + return IDWALK_RET_NOP; + } + if (cb_data->cb_flag & (IDWALK_CB_EMBEDDED | IDWALK_CB_EMBEDDED_NOT_OWNING)) { + /* Embedded data are part of their owner's internal data, and as such already computed as + * part of the owner's shallow hash. */ + return IDWALK_RET_NOP; + } + ID *referenced_id = *cb_data->id_pointer; + if (!referenced_id) { + /* Need to update the hash even if there is no id. There is a difference between the case + * where there is no id and the case where this callback is not called at all.*/ + const int random_data = 452942579; + XXH3_128bits_update(hash_state, &random_data, sizeof(int)); + return IDWALK_RET_NOP; + } + /* All embedded ID usages should already have been excluded above. */ + BLI_assert((referenced_id->flag & ID_FLAG_EMBEDDED_DATA) == 0); + if (current_stack.contains(referenced_id)) { + /* Somehow encode that we had a circular reference here. */ + const int random_data = 234632342; + XXH3_128bits_update(hash_state, &random_data, sizeof(int)); + return IDWALK_RET_NOP; + } + compute_deep_hash_recursive(bmain, *referenced_id, current_stack, r_hashes, r_errors); + const IDHash *referenced_id_hash = r_hashes.lookup_ptr(referenced_id); + if (!referenced_id_hash) { + success = false; + return IDWALK_RET_STOP_ITER; + } + XXH3_128bits_update(hash_state, referenced_id_hash->data, sizeof(IDHash)); + return IDWALK_RET_NOP; + }, + nullptr, + IDWALK_READONLY); + + if (!success) { + return; + } + IDHash new_deep_hash; + const XXH128_hash_t new_deep_hash_xxh128 = XXH3_128bits_digest(hash_state); + XXH3_freeState(hash_state); + static_assert(sizeof(IDHash) == sizeof(XXH128_hash_t)); + memcpy(new_deep_hash.data, &new_deep_hash_xxh128, sizeof(IDHash)); + r_hashes.add(&id, new_deep_hash); +} + +IDHashResult compute_linked_id_deep_hashes(const Main &bmain, Span ids) +{ +#ifndef NDEBUG + for (const ID *id : ids) { + BLI_assert(ID_IS_LINKED(id)); + } +#endif + + if (ids.is_empty()) { + return ValidDeepHashes{}; + } + + Map hashes; + Set current_stack; + DeepHashErrors errors; + for (const ID *id : ids) { + compute_deep_hash_recursive(bmain, *id, current_stack, hashes, errors); + } + if (!errors.missing_files.is_empty() || !errors.updated_files.is_empty()) { + return errors; + } + return ValidDeepHashes{hashes}; +} + +std::string id_hash_to_hex(const IDHash &hash) +{ + std::string hex_str; + for (const uint8_t byte : hash.data) { + hex_str += fmt::format("{:02x}", byte); + } + return hex_str; +} + +} // namespace blender::bke::id_hash diff --git a/source/blender/blenkernel/intern/lib_id.cc b/source/blender/blenkernel/intern/lib_id.cc index fc3bc9c10b2..7297dfd2413 100644 --- a/source/blender/blenkernel/intern/lib_id.cc +++ b/source/blender/blenkernel/intern/lib_id.cc @@ -1667,6 +1667,14 @@ void BKE_libblock_copy_in_lib(Main *bmain, DEG_id_type_tag(bmain, GS(new_id->name)); } + if (owner_library && *owner_library && ((*owner_library)->flag & LIBRARY_FLAG_IS_ARCHIVE) != 0) { + new_id->flag |= ID_FLAG_LINKED_AND_PACKED; + } + + if (flag & LIB_ID_COPY_ID_NEW_SET) { + ID_NEW_SET(id, new_id); + } + *new_id_p = new_id; } diff --git a/source/blender/blenkernel/intern/lib_id_delete.cc b/source/blender/blenkernel/intern/lib_id_delete.cc index 72e3828ba48..08c62b64e32 100644 --- a/source/blender/blenkernel/intern/lib_id_delete.cc +++ b/source/blender/blenkernel/intern/lib_id_delete.cc @@ -339,6 +339,10 @@ static size_t id_delete(Main *bmain, id_remapper.clear(); } + /* Remapping above may have left some Library::runtime::archived_libraries items to nullptr, + * clean this up and shrink the vector accordingly. */ + blender::bke::library::main_cleanup_parent_archives(*bmain); + /* Since we removed IDs from Main, their own other IDs usages need to be removed 'manually'. */ blender::Vector cleanup_ids{ids_to_delete.begin(), ids_to_delete.end()}; BKE_libblock_relink_multiple( diff --git a/source/blender/blenkernel/intern/library.cc b/source/blender/blenkernel/intern/library.cc index 34010597448..698d93d5dc8 100644 --- a/source/blender/blenkernel/intern/library.cc +++ b/source/blender/blenkernel/intern/library.cc @@ -14,6 +14,8 @@ /* all types are needed here, in order to do memory operations */ #include "DNA_ID.h" +#include "DNA_collection_types.h" +#include "DNA_scene_types.h" #include "BLI_utildefines.h" @@ -22,24 +24,32 @@ #include "BLI_path_utils.hh" #include "BLI_set.hh" #include "BLI_string.h" +#include "BLI_vector_set.hh" #include "BLT_translation.hh" #include "BLO_read_write.hh" #include "BKE_bpath.hh" +#include "BKE_id_hash.hh" #include "BKE_idtype.hh" +#include "BKE_key.hh" #include "BKE_lib_id.hh" #include "BKE_lib_query.hh" +#include "BKE_lib_remap.hh" #include "BKE_library.hh" #include "BKE_main.hh" +#include "BKE_main_invariants.hh" #include "BKE_main_namemap.hh" +#include "BKE_node.hh" #include "BKE_packedFile.hh" +#include "BKE_report.hh" struct BlendDataReader; static CLG_LogRef LOG = {"lib.library"}; +using namespace blender::bke; using namespace blender::bke::library; static void library_runtime_reset(Library *lib) @@ -93,7 +103,37 @@ static void library_copy_data(Main *bmain, static void library_foreach_id(ID *id, LibraryForeachIDData *data) { Library *lib = (Library *)id; + const LibraryForeachIDFlag foreach_flag = BKE_lib_query_foreachid_process_flags_get(data); BKE_LIB_FOREACHID_PROCESS_IDSUPER(data, lib->runtime->parent, IDWALK_CB_NEVER_SELF); + + if (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + /* Archive library must have a parent, this can't be nullptr. */ + if (lib->archive_parent_library) { + BKE_LIB_FOREACHID_PROCESS_ID( + data, lib->archive_parent_library, IDWALK_CB_NEVER_SELF | IDWALK_CB_NEVER_NULL); + } + + /* Archive libraries should never 'own' other archives. */ + BLI_assert(lib->runtime->archived_libraries.is_empty()); + if (foreach_flag & IDWALK_DO_INTERNAL_RUNTIME_POINTERS) { + for (Library *&lib_p : lib->runtime->archived_libraries) { + BKE_LIB_FOREACHID_PROCESS_ID( + data, lib_p, IDWALK_CB_NEVER_SELF | IDWALK_CB_INTERNAL | IDWALK_CB_LOOPBACK); + } + } + } + else { + /* Regular libraries should never have an archive parent. */ + BLI_assert(!lib->archive_parent_library); + BKE_LIB_FOREACHID_PROCESS_ID(data, lib->archive_parent_library, IDWALK_CB_NEVER_SELF); + + if (foreach_flag & IDWALK_DO_INTERNAL_RUNTIME_POINTERS) { + for (Library *&lib_p : lib->runtime->archived_libraries) { + BKE_LIB_FOREACHID_PROCESS_ID( + data, lib_p, IDWALK_CB_NEVER_SELF | IDWALK_CB_INTERNAL | IDWALK_CB_LOOPBACK); + } + } + } } static void library_foreach_path(ID *id, BPathForeachPathData *bpath_data) @@ -139,6 +179,15 @@ static void library_blend_read_data(BlendDataReader * /*reader*/, ID *id) lib->runtime = MEM_new(__func__); } +static void library_blend_read_after_liblink(BlendLibReader * /*reader*/, ID *id) +{ + Library *lib = reinterpret_cast(id); + if (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + BLI_assert(lib->archive_parent_library); + lib->archive_parent_library->runtime->archived_libraries.append(lib); + } +} + IDTypeInfo IDType_ID_LI = { /*id_code*/ Library::id_type, /*id_filter*/ FILTER_ID_LI, @@ -163,7 +212,7 @@ IDTypeInfo IDType_ID_LI = { /*blend_write*/ library_blend_write_data, /*blend_read_data*/ library_blend_read_data, - /*blend_read_after_liblink*/ nullptr, + /*blend_read_after_liblink*/ library_blend_read_after_liblink, /*blend_read_undo_preserve*/ nullptr, @@ -368,3 +417,299 @@ Library *blender::bke::library::search_filepath_abs(ListBase *libraries, } return nullptr; } + +/** + * Add a new 'archive' copy of the given reference library. It is used to store linked packed IDs. + */ +static Library *add_archive_library(Main &bmain, Library &reference_library) +{ + BLI_assert((reference_library.flag & LIBRARY_FLAG_IS_ARCHIVE) == 0); + /* Cannot copy libraries using generic ID copying functions, so create the copy manually. */ + Library *archive_library = static_cast( + BKE_id_new(&bmain, ID_LI, BKE_id_name(reference_library.id))); + + /* Like in #direct_link_library. */ + id_us_ensure_real(&archive_library->id); + + archive_library->archive_parent_library = &reference_library; + constexpr uint16_t copy_flag = ~LIBRARY_FLAG_IS_ARCHIVE; + archive_library->flag = (reference_library.flag & copy_flag) | LIBRARY_FLAG_IS_ARCHIVE; + BKE_library_filepath_set(&bmain, archive_library, reference_library.filepath); + + archive_library->runtime->parent = reference_library.runtime->parent; + /* Only copy a subset of the reference library tags. E.g. an archive library should never be + * considered as writable, so never copy #LIBRARY_ASSET_FILE_WRITABLE. This may need further + * tweaking still. */ + constexpr uint16_t copy_tag = (LIBRARY_TAG_RESYNC_REQUIRED | LIBRARY_ASSET_EDITABLE | + LIBRARY_IS_ASSET_EDIT_FILE); + archive_library->runtime->tag = reference_library.runtime->tag & copy_tag; + /* By definition, the file version of an archive library containing only packed linked data is + * the same as the one of its Main container. */ + archive_library->runtime->versionfile = bmain.versionfile; + archive_library->runtime->subversionfile = bmain.subversionfile; + + reference_library.runtime->archived_libraries.append(archive_library); + + return archive_library; +} + +static Library *get_archive_library(Main &bmain, ID *for_id, const IDHash &for_id_deep_hash) +{ + Library *reference_library = for_id->lib; + BLI_assert(reference_library && (reference_library->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0); + + Library *archive_library = nullptr; + for (Library *lib_iter : reference_library->runtime->archived_libraries) { + BLI_assert((lib_iter->flag & LIBRARY_FLAG_IS_ARCHIVE) != 0); + BLI_assert(lib_iter->archive_parent_library != nullptr); + BLI_assert(lib_iter->archive_parent_library == reference_library); + /* Check if current archive library already contains an ID of same type and name. */ + if (BKE_main_namemap_contain_name(bmain, lib_iter, GS(for_id->name), BKE_id_name(*for_id))) { +#ifndef NDEBUG + ID *packed_id = BKE_libblock_find_name_and_library( + &bmain, GS(for_id->name), BKE_id_name(*for_id), BKE_id_name(lib_iter->id)); + BLI_assert_msg( + packed_id && packed_id->deep_hash != for_id_deep_hash, + "An already packed ID with same deep hash as the one to be packed, should have already " + "be found and used (deduplication) before reaching this codepath"); +#endif + UNUSED_VARS_NDEBUG(for_id_deep_hash); + continue; + } + archive_library = lib_iter; + break; + } + if (!archive_library) { + archive_library = add_archive_library(bmain, *reference_library); + } + BLI_assert(reference_library->runtime->archived_libraries.contains(archive_library)); + return archive_library; +} + +static void pack_linked_id(Main &bmain, + ID *linked_id, + const id_hash::ValidDeepHashes &deep_hashes, + blender::Map &already_packed_ids, + blender::VectorSet &ids_to_remap, + blender::bke::id::IDRemapper &id_remapper) +{ + BLI_assert(linked_id->newid == nullptr); + + const IDHash linked_id_deep_hash = deep_hashes.hashes.lookup(linked_id); + ID *packed_id = already_packed_ids.lookup_default(linked_id_deep_hash, nullptr); + + if (packed_id) { + /* Exact same ID (and all of its dependencies) have already been linked and packed before, + * re-use these packed data. */ + + auto existing_id_process = [&deep_hashes, &id_remapper](ID *linked_id, ID *packed_id) { + BLI_assert(packed_id); + BLI_assert(ID_IS_PACKED(packed_id)); + /* Note: linked_id and packed_id may have the same deep hash while still coming from + * different original libraries. This easily happens copying an asset file such that each + * asset exists twice. */ + BLI_assert(packed_id->deep_hash == deep_hashes.hashes.lookup(linked_id)); + UNUSED_VARS_NDEBUG(deep_hashes); + + id_remapper.add(linked_id, packed_id); + linked_id->newid = packed_id; + /* No need to remap this packed ID - otherwise there would be something very wrong in + * packed IDs state. */ + }; + + existing_id_process(linked_id, packed_id); + + /* Handle 'fake-embedded' ShapeKeys IDs. */ + Key *linked_key = BKE_key_from_id(linked_id); + if (linked_key) { + Key *packed_key = BKE_key_from_id(packed_id); + BLI_assert(packed_key); + existing_id_process(&linked_key->id, &packed_key->id); + } + } + else { + /* This exact version of the ID and its dependencies have not been packed before, creates a + * new copy of it and pack it. */ + + /* Find an existing archive Library not containing a 'version' of this ID yet (to prevent names + * collisions). */ + Library *archive_lib = get_archive_library(bmain, linked_id, linked_id_deep_hash); + + auto copied_id_process = + [&archive_lib, &deep_hashes, &ids_to_remap, &id_remapper, &already_packed_ids]( + ID *linked_id, ID *packed_id) { + BLI_assert(packed_id); + BLI_assert(ID_IS_PACKED(packed_id)); + BLI_assert(packed_id->lib == archive_lib); + UNUSED_VARS_NDEBUG(archive_lib); + + packed_id->deep_hash = deep_hashes.hashes.lookup(linked_id); + id_remapper.add(linked_id, packed_id); + ids_to_remap.add(packed_id); + already_packed_ids.add(packed_id->deep_hash, packed_id); + }; + + packed_id = BKE_id_copy_in_lib(&bmain, + archive_lib, + linked_id, + std::nullopt, + nullptr, + LIB_ID_COPY_DEFAULT | LIB_ID_COPY_ID_NEW_SET | + LIB_ID_COPY_NO_ANIMDATA); + id_us_min(packed_id); + copied_id_process(linked_id, packed_id); + + /* Handle 'fake-embedded' ShapeKeys IDs. */ + Key *linked_key = BKE_key_from_id(linked_id); + if (linked_key) { + Key *embedded_key = BKE_key_from_id(packed_id); + BLI_assert(embedded_key); + copied_id_process(&linked_key->id, &embedded_key->id); + } + } +} + +/** + * Pack given linked IDs. Low-level code, assumes all given IDs are valid and safe to pack. + * + * Will set final packed ID into each ID::newid pointers. + */ +static void pack_linked_ids(Main &bmain, const blender::Set &ids_to_pack) +{ + blender::VectorSet final_ids_to_pack; + blender::VectorSet ids_to_remap; + blender::bke::id::IDRemapper id_remapper; + + for (ID *id : ids_to_pack) { + BLI_assert(ID_IS_LINKED(id)); + if (ID_IS_PACKED(id)) { + /* Should not happen, but also not critical issue. */ + CLOG_ERROR(&LOG, + "Trying to pack an already packed ID '%s' (from '%s')", + id->name, + id->lib->runtime->filepath_abs); + /* Already packed. */ + continue; + } + final_ids_to_pack.add(id); + } + + const id_hash::IDHashResult hash_result = id_hash::compute_linked_id_deep_hashes( + bmain, final_ids_to_pack.as_span()); + if (const auto *errors = std::get_if(&hash_result)) { + if (!errors->missing_files.is_empty()) { + CLOG_ERROR(&LOG, + "Trying to pack IDs that depend on missing linked libraries: %s", + errors->missing_files[0].c_str()); + } + if (!errors->updated_files.is_empty()) { + CLOG_ERROR(&LOG, + "Trying to pack linked ID that has been modified on disk: %s", + errors->updated_files[0].c_str()); + } + return; + } + const auto &deep_hashes = std::get(hash_result); + + blender::Map already_packed_ids; + { + ID *id; + FOREACH_MAIN_ID_BEGIN (&bmain, id) { + if (ID_IS_PACKED(id)) { + already_packed_ids.add(id->deep_hash, id); + } + } + FOREACH_MAIN_ID_END; + } + + for (ID *linked_id : final_ids_to_pack) { + pack_linked_id(bmain, linked_id, deep_hashes, already_packed_ids, ids_to_remap, id_remapper); + } + + BKE_libblock_relink_multiple( + &bmain, ids_to_remap.as_span(), ID_REMAP_TYPE_REMAP, id_remapper, 0); + BKE_main_ensure_invariants(bmain); +} + +void blender::bke::library::pack_linked_id_hierarchy(Main &bmain, ID &root_id) +{ + BLI_assert(ID_IS_LINKED(&root_id)); + BLI_assert(!ID_IS_PACKED(&root_id)); + + blender::Set ids_to_pack; + ids_to_pack.add(&root_id); + BKE_library_foreach_ID_link( + &bmain, + &root_id, + [&ids_to_pack](LibraryIDLinkCallbackData *cb_data) -> int { + if (cb_data->cb_flag & IDWALK_CB_LOOPBACK) { + return IDWALK_RET_NOP; + } + if (cb_data->cb_flag & (IDWALK_CB_EMBEDDED | IDWALK_CB_EMBEDDED_NOT_OWNING)) { + return IDWALK_RET_NOP; + } + + ID *self_id = cb_data->self_id; + ID *referenced_id = *cb_data->id_pointer; + if (!referenced_id) { + return IDWALK_RET_NOP; + } + if (!ID_IS_LINKED(referenced_id)) { + CLOG_ERROR(&LOG, "Linked data-block references non-linked data-block"); + return IDWALK_RET_NOP; + } + if (ID_IS_PACKED(referenced_id)) { + /* A linked ID can use another packed linked ID, as long as it is not from the same + * library. */ + BLI_assert(referenced_id->lib && referenced_id->lib->archive_parent_library); + if (referenced_id->lib->archive_parent_library == self_id->lib) { + CLOG_ERROR(&LOG, + "Non-packed data-block references packed data-block from the same library, " + "which is not allowed"); + } + return IDWALK_RET_NOP; + } + if (referenced_id->newid && ID_IS_PACKED(referenced_id->newid)) { + return IDWALK_RET_NOP; + } + if (GS(referenced_id->name) == ID_KE) { + /* Shape keys cannot be directly linked, from linking code PoV they behave as embedded + * data (i.e. their owning data is responsible to handle them). */ + return IDWALK_RET_NOP; + } + + ids_to_pack.add(referenced_id); + return IDWALK_RET_NOP; + }, + nullptr, + IDWALK_READONLY | IDWALK_RECURSE); + + pack_linked_ids(bmain, ids_to_pack); +} + +void blender::bke::library::main_cleanup_parent_archives(Main &bmain) +{ + LISTBASE_FOREACH (Library *, lib, &bmain.libraries) { + if (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + BLI_assert(!lib->runtime || lib->runtime->archived_libraries.is_empty()); + } + else { + int i_read_curr = 0; + int i_insert_curr = 0; + for (; i_read_curr < lib->runtime->archived_libraries.size(); i_read_curr++) { + if (!lib->runtime->archived_libraries[i_read_curr]) { + continue; + } + if (i_insert_curr < i_read_curr) { + lib->runtime->archived_libraries[i_insert_curr] = + lib->runtime->archived_libraries[i_read_curr]; + } + i_insert_curr++; + } + BLI_assert(i_insert_curr <= i_read_curr); + if (i_insert_curr < i_read_curr) { + lib->runtime->archived_libraries.resize(i_insert_curr); + } + } + } +} diff --git a/source/blender/blenkernel/intern/main.cc b/source/blender/blenkernel/intern/main.cc index 97fbcd779cc..c05d9b186d2 100644 --- a/source/blender/blenkernel/intern/main.cc +++ b/source/blender/blenkernel/intern/main.cc @@ -717,7 +717,15 @@ void BKE_main_library_weak_reference_add_item( BLI_assert(BKE_idtype_idcode_append_is_reusable(GS(new_id->name))); const LibWeakRefKey key{library_filepath, library_id_name}; - library_weak_reference_mapping->map.add_new(key, new_id); + /* With packed IDs and archive libraries, it is now possible to have several instances of the + * (originally) same linked ID made local at the same time in an append opeeration, so it is + * possible to get the same key several time here. And `Map::add_new` cannot be used safely + * anymore. + * + * Simply consider the first added one as valid, there is no good way to determine the 'best' one + * to keep around for append-or-reuse operations anyway - and the whole append-and-reuse may be + * depracted soon too. */ + library_weak_reference_mapping->map.add(key, new_id); BKE_main_library_weak_reference_add(new_id, library_filepath, library_id_name); } diff --git a/source/blender/blenloader/BLO_readfile.hh b/source/blender/blenloader/BLO_readfile.hh index 9029223b4b8..c4328f57390 100644 --- a/source/blender/blenloader/BLO_readfile.hh +++ b/source/blender/blenloader/BLO_readfile.hh @@ -406,6 +406,10 @@ enum eBLOLibLinkFlags { * see e.g. #BKE_blendfile_library_relocate. */ BLO_LIBLINK_COLLECTION_NO_HIERARCHY_REBUILD = 1 << 26, + /** + * Pack the linked data-blocks to keep them working even if the source file is not available. + */ + BLO_LIBLINK_PACK = 1 << 27, }; /** @@ -584,13 +588,13 @@ struct ID_Readfile_Data { }; /** - * Return `id->runtime.readfile_data->tags` if the `readfile_data` is allocated, + * Return `id->runtime->readfile_data->tags` if the `readfile_data` is allocated, * otherwise return an all-zero set of tags. */ ID_Readfile_Data::Tags BLO_readfile_id_runtime_tags(ID &id); /** - * Create the `readfile_data` if needed, and return `id->runtime.readfile_data->tags`. + * Create the `readfile_data` if needed, and return `id->runtime->readfile_data->tags`. * * Use it instead of #BLO_readfile_id_runtime_tags when tags need to be set. */ diff --git a/source/blender/blenloader/CMakeLists.txt b/source/blender/blenloader/CMakeLists.txt index eee3b15d32e..6925567427f 100644 --- a/source/blender/blenloader/CMakeLists.txt +++ b/source/blender/blenloader/CMakeLists.txt @@ -54,6 +54,7 @@ set(SRC set(LIB PRIVATE bf::animrig + PRIVATE bf::asset_system PRIVATE bf::blenkernel PRIVATE bf::blenlib PUBLIC bf::blenloader_core diff --git a/source/blender/blenloader/intern/readblenentry.cc b/source/blender/blenloader/intern/readblenentry.cc index 4dc68b65c8a..e4843b2c0ef 100644 --- a/source/blender/blenloader/intern/readblenentry.cc +++ b/source/blender/blenloader/intern/readblenentry.cc @@ -87,12 +87,21 @@ static bool blendhandle_load_id_data_and_validate(FileData *fd, BHead *bhead, bool use_assets_only, const char *&r_idname, + short &r_idflag, AssetMetaData *&r_asset_meta_data) { r_idname = blo_bhead_id_name(fd, bhead); if (!r_idname || r_idname[0] == '\0') { return false; } + r_idflag = blo_bhead_id_flag(fd, bhead); + /* Do not list (and therefore allow direct linking of) packed data. + * While supporting this is conceptually possible, it would require significant changes in + * the UI (file browser) and UX (link operation) to convey this concept and handle it + * correctly. */ + if (r_idflag & ID_FLAG_LINKED_AND_PACKED) { + return false; + } r_asset_meta_data = blo_bhead_id_asset_data_address(fd, bhead); if (use_assets_only && r_asset_meta_data == nullptr) { return false; @@ -113,9 +122,10 @@ LinkNode *BLO_blendhandle_get_datablock_names(BlendHandle *bh, for (bhead = blo_bhead_first(fd); bhead; bhead = blo_bhead_next(fd, bhead)) { if (bhead->code == ofblocktype) { const char *idname; + short idflag; AssetMetaData *asset_meta_data; if (!blendhandle_load_id_data_and_validate( - fd, bhead, use_assets_only, idname, asset_meta_data)) + fd, bhead, use_assets_only, idname, idflag, asset_meta_data)) { continue; } @@ -152,9 +162,10 @@ LinkNode *BLO_blendhandle_get_datablock_info(BlendHandle *bh, BHead *id_bhead = bhead; const char *idname; + short idflag; AssetMetaData *asset_meta_data; if (!blendhandle_load_id_data_and_validate( - fd, id_bhead, use_assets_only, idname, asset_meta_data)) + fd, id_bhead, use_assets_only, idname, idflag, asset_meta_data)) { continue; } @@ -382,8 +393,10 @@ BlendFileData *BLO_read_from_memfile(Main *oldmain, /* Build old ID map for all old IDs. */ blo_make_old_idmap_from_main(fd, oldmain); - /* Separate linked data from old main. */ - blo_split_main(oldmain); + /* Separate linked data from old main. + * WARNING: Do not split out packed IDs here, as these are handled similarly as local IDs in + * undo context. */ + blo_split_main(oldmain, false); fd->old_bmain = oldmain; /* Removed packed data from this trick - it's internal data that needs saves. */ diff --git a/source/blender/blenloader/intern/readfile.cc b/source/blender/blenloader/intern/readfile.cc index e130c354ac6..469d3d5a99a 100644 --- a/source/blender/blenloader/intern/readfile.cc +++ b/source/blender/blenloader/intern/readfile.cc @@ -390,17 +390,17 @@ void blo_join_main(Main *bmain) bmain->split_mains.reset(); } -static void split_libdata(ListBase *lb_src, Main **lib_main_array, const uint lib_main_array_len) +static void split_libdata(ListBase *lb_src, + blender::Vector
&lib_main_array, + const bool do_split_packed_ids) { for (ID *id = static_cast(lb_src->first), *idnext; id; id = idnext) { idnext = static_cast(id->next); - if (id->lib) { - if ((uint(id->lib->runtime->temp_index) < lib_main_array_len) && - /* this check should never fail, just in case 'id->lib' is a dangling pointer. */ - (lib_main_array[id->lib->runtime->temp_index]->curlib == id->lib)) - { + if (id->lib && (do_split_packed_ids || (id->lib->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0)) { + if (uint(id->lib->runtime->temp_index) < lib_main_array.size()) { Main *mainvar = lib_main_array[id->lib->runtime->temp_index]; + BLI_assert(mainvar->curlib == id->lib); ListBase *lb_dst = which_libbase(mainvar, GS(id->name)); BLI_remlink(lb_src, id); BLI_addtail(lb_dst, id); @@ -412,7 +412,7 @@ static void split_libdata(ListBase *lb_src, Main **lib_main_array, const uint li } } -void blo_split_main(Main *bmain) +void blo_split_main(Main *bmain, const bool do_split_packed_ids) { BLI_assert(!bmain->split_mains); bmain->split_mains = std::make_shared>(); @@ -432,13 +432,15 @@ void blo_split_main(Main *bmain) BKE_main_namemap_clear(*bmain); /* (Library.temp_index -> Main), lookup table */ - const uint lib_main_array_len = BLI_listbase_count(&bmain->libraries); - Main **lib_main_array = MEM_malloc_arrayN
(lib_main_array_len, __func__); + blender::Vector
lib_main_array; int i = 0; for (Library *lib = static_cast(bmain->libraries.first); lib; lib = static_cast(lib->id.next), i++) { + if (!do_split_packed_ids && (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) != 0) { + continue; + } Main *libmain = BKE_main_new(); libmain->curlib = lib; libmain->versionfile = lib->runtime->versionfile; @@ -450,7 +452,7 @@ void blo_split_main(Main *bmain) bmain->split_mains->add_new(libmain); libmain->split_mains = bmain->split_mains; lib->runtime->temp_index = i; - lib_main_array[i] = libmain; + lib_main_array.append(libmain); } MainListsArray lbarray = BKE_main_lists_get(*bmain); @@ -461,10 +463,8 @@ void blo_split_main(Main *bmain) /* No ID_LI data-block should ever be linked anyway, but just in case, better be explicit. */ continue; } - split_libdata(lbarray[i], lib_main_array, lib_main_array_len); + split_libdata(lbarray[i], lib_main_array, do_split_packed_ids); } - - MEM_freeN(lib_main_array); } static void read_file_version_and_colorspace(FileData *fd, Main *main) @@ -551,58 +551,6 @@ static void read_file_bhead_idname_map_create(FileData *fd) } } -static Main *blo_find_main(FileData *fd, const char *filepath, const char *relabase) -{ - Main *m; - Library *lib; - char filepath_abs[FILE_MAX]; - - STRNCPY(filepath_abs, filepath); - BLI_path_abs(filepath_abs, relabase); - BLI_path_normalize(filepath_abs); - - // printf("blo_find_main: relabase %s\n", relabase); - // printf("blo_find_main: original in %s\n", filepath); - // printf("blo_find_main: converted to %s\n", filepath_abs); - - for (Main *m : *fd->bmain->split_mains) { - const char *libname = (m->curlib) ? m->curlib->runtime->filepath_abs : m->filepath; - - if (BLI_path_cmp(filepath_abs, libname) == 0) { - if (G.debug & G_DEBUG) { - CLOG_DEBUG(&LOG, "Found library %s", libname); - } - return m; - } - } - - m = BKE_main_new(); - fd->bmain->split_mains->add_new(m); - m->split_mains = fd->bmain->split_mains; - - /* Add library data-block itself to 'main' Main, since libraries are **never** linked data. - * Fixes bug where you could end with all ID_LI data-blocks having the same name... */ - lib = BKE_id_new(fd->bmain, BLI_path_basename(filepath)); - - /* Important, consistency with main ID reading code from read_libblock(). */ - lib->id.us = ID_FAKE_USERS(lib); - - /* Matches direct_link_library(). */ - id_us_ensure_real(&lib->id); - - STRNCPY(lib->filepath, filepath); - STRNCPY(lib->runtime->filepath_abs, filepath_abs); - - m->curlib = lib; - - read_file_version_and_colorspace(fd, m); - - if (G.debug & G_DEBUG) { - CLOG_DEBUG(&LOG, "Added new lib %s", filepath); - } - return m; -} - void blo_readfile_invalidate(FileData *fd, Main *bmain, const char *message) { /* Tag given `bmain`, and 'root 'local' main one (in case given one is a library one) as invalid. @@ -845,6 +793,16 @@ const char *blo_bhead_id_name(FileData *fd, const BHead *bhead) return nullptr; } +short blo_bhead_id_flag(const FileData *fd, const BHead *bhead) +{ + BLI_assert(blo_bhead_is_id(bhead)); + if (fd->id_flag_offset < 0) { + return 0; + } + return *reinterpret_cast( + POINTER_OFFSET(bhead, sizeof(*bhead) + fd->id_flag_offset)); +} + AssetMetaData *blo_bhead_id_asset_data_address(const FileData *fd, const BHead *bhead) { BLI_assert(blo_bhead_is_id_valid_type(bhead)); @@ -853,6 +811,20 @@ AssetMetaData *blo_bhead_id_asset_data_address(const FileData *fd, const BHead * nullptr; } +static const IDHash *blo_bhead_id_deep_hash(const FileData *fd, const BHead *bhead) +{ + BLI_assert(blo_bhead_is_id_valid_type(bhead)); + if (fd->id_flag_offset < 0 || fd->id_deep_hash_offset < 0) { + return nullptr; + } + const short flag = blo_bhead_id_flag(fd, bhead); + if (!(flag & ID_FLAG_LINKED_AND_PACKED)) { + return nullptr; + } + return reinterpret_cast( + POINTER_OFFSET(bhead, sizeof(*bhead) + fd->id_deep_hash_offset)); +} + static void read_blender_header(FileData *fd) { const BlenderHeaderVariant header_variant = BLO_readfile_blender_header_decode(fd->file); @@ -920,6 +892,10 @@ static bool read_file_dna(FileData *fd, const char **r_error_message) BLI_assert(fd->id_name_offset != -1); fd->id_asset_data_offset = DNA_struct_member_offset_by_name_with_alias( fd->filesdna, "ID", "AssetMetaData", "*asset_data"); + fd->id_flag_offset = DNA_struct_member_offset_by_name_with_alias( + fd->filesdna, "ID", "short", "flag"); + fd->id_deep_hash_offset = DNA_struct_member_offset_by_name_with_alias( + fd->filesdna, "ID", "IDHash", "deep_hash"); fd->filesubversion = subversion; @@ -1149,6 +1125,7 @@ static FileData *filedata_new(BlendFileReadReport *reports) fd->datamap = oldnewmap_new(); fd->globmap = oldnewmap_new(); fd->libmap = oldnewmap_new(); + fd->id_by_deep_hash = std::make_shared>(); fd->reports = reports; @@ -1315,6 +1292,11 @@ static FileData *blo_filedata_from_file_descriptor(const char *filepath, FileData *fd = filedata_new(reports); fd->file = file; + BLI_stat_t stat; + if (BLI_stat(filepath, &stat) != -1) { + fd->file_stat = stat; + } + return fd; } @@ -2255,6 +2237,9 @@ static void direct_link_id_common(BlendDataReader *reader, id->session_uid = MAIN_ID_SESSION_UID_UNSET; } + if (ID_IS_PACKED(id)) { + BLI_assert(current_library->flag & LIBRARY_FLAG_IS_ARCHIVE); + } id->lib = current_library; if (id->lib) { /* Always fully clear fake user flag for linked data. */ @@ -2444,6 +2429,8 @@ static void library_filedata_release(Library *lib) { if (lib->runtime->filedata) { BLI_assert(lib->runtime->versionfile != 0); + BLI_assert_msg(!lib->runtime->is_filedata_owner || (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0, + "Packed Archive libraries should never own their filedata"); if (lib->runtime->is_filedata_owner) { blo_filedata_free(lib->runtime->filedata); } @@ -2470,6 +2457,11 @@ static void direct_link_library(FileData *fd, Library *lib, Main *main) if (!newmain->curlib) { continue; } + if (newmain->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE || lib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + /* Archive library should never be used to link new data, and there can be many such + * archive libraries for a same 'real' blendfile one. */ + continue; + } if (BLI_path_cmp(newmain->curlib->runtime->filepath_abs, lib->runtime->filepath_abs) == 0) { BLO_reportf_wrap(fd->reports, RPT_WARNING, @@ -2510,9 +2502,24 @@ static void direct_link_library(FileData *fd, Library *lib, Main *main) newmain->split_mains = fd->bmain->split_mains; newmain->curlib = lib; + if (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + /* Archive libraries contains only embedded linked IDs, which by definition have the same + * fileversion as the blendfile that contains them. */ + lib->runtime->versionfile = newmain->versionfile = fd->bmain->versionfile; + lib->runtime->subversionfile = newmain->subversionfile = fd->bmain->subversionfile; + + /* The filedata of a packed archive library should always be the one of the blendfile which + * defines the library ID and packs its linked IDs. */ + lib->runtime->filedata = fd; + lib->runtime->is_filedata_owner = false; + } + lib->runtime->parent = nullptr; id_us_ensure_real(&lib->id); + + /* Should always be null, Library IDs in Blender are always local. */ + lib->id.lib = nullptr; } /* Always call this once you have loaded new library data to set the relative paths correctly @@ -2680,6 +2687,82 @@ static BHead *read_data_into_datamap(FileData *fd, return bhead; } +/* Add a Main (and optionally create a matching Library ID), for the given filepath. + * + * - If `lib` is `nullptr`, create a new Library ID, otherwise only create a new Main for the given + * library. + * - `reference_lib` is the 'archive parent' of an archive (packed) library, can be null and will + * be ignored otherwise. */ +static Main *blo_add_main_for_library(FileData *fd, + Library *lib, + Library *reference_lib, + const char *lib_filepath, + char (&filepath_abs)[FILE_MAX], + const bool is_packed_library) +{ + Main *bmain = BKE_main_new(); + fd->bmain->split_mains->add_new(bmain); + bmain->split_mains = fd->bmain->split_mains; + + if (!lib) { + /* Add library data-block itself to 'main' Main, since libraries are **never** linked data. + * Fixes bug where you could end with all ID_LI data-blocks having the same name... */ + lib = BKE_id_new(fd->bmain, + reference_lib ? BKE_id_name(reference_lib->id) : + BLI_path_basename(lib_filepath)); + + /* Important, consistency with main ID reading code from read_libblock(). */ + lib->id.us = ID_FAKE_USERS(lib); + + /* Matches direct_link_library(). */ + id_us_ensure_real(&lib->id); + + STRNCPY(lib->filepath, lib_filepath); + STRNCPY(lib->runtime->filepath_abs, filepath_abs); + + if (is_packed_library) { + /* FIXME: This logic is very similar to the code in BKE_library dealing with archived + * libraries (e.g. #add_archive_library). Might be good to try to factorize it. */ + lib->archive_parent_library = reference_lib; + constexpr uint16_t copy_flag = ~LIBRARY_FLAG_IS_ARCHIVE; + lib->flag = (reference_lib->flag & copy_flag) | LIBRARY_FLAG_IS_ARCHIVE; + + lib->runtime->parent = reference_lib->runtime->parent; + /* Only copy a subset of the reference library tags. E.g. an archive library should never be + * considered as writable, so never copy #LIBRARY_ASSET_FILE_WRITABLE. This may need further + * tweaking still. */ + constexpr uint16_t copy_tag = (LIBRARY_TAG_RESYNC_REQUIRED | LIBRARY_ASSET_EDITABLE | + LIBRARY_IS_ASSET_EDIT_FILE); + lib->runtime->tag = reference_lib->runtime->tag & copy_tag; + + /* The filedata of a packed archive library should always be the one of the blendfile which + * defines the library ID and packs its linked IDs. */ + lib->runtime->filedata = fd; + lib->runtime->is_filedata_owner = false; + + reference_lib->runtime->archived_libraries.append(lib); + } + } + else { + if (is_packed_library) { + BLI_assert(lib->flag & LIBRARY_FLAG_IS_ARCHIVE); + BLI_assert(lib->archive_parent_library == reference_lib); + BLI_assert(reference_lib->runtime->archived_libraries.contains(lib)); + + BLI_assert(lib->runtime->filedata == nullptr); + lib->runtime->filedata = fd; + lib->runtime->is_filedata_owner = false; + } + else { + /* Should never happen currently. */ + BLI_assert_unreachable(); + } + } + + bmain->curlib = lib; + return bmain; +} + /* Verify if the datablock and all associated data is identical. */ static bool read_libblock_is_identical(FileData *fd, BHead *bhead) { @@ -2764,7 +2847,10 @@ static void read_undo_move_libmain_data(FileData *fd, Main *libmain, BHead *bhea ID *id_iter; FOREACH_MAIN_ID_BEGIN (libmain, id_iter) { - BKE_main_idmap_insert_id(fd->new_idmap_uid, id_iter); + /* Packed IDs are read from the memfile, so don't add them here already. */ + if (!ID_IS_PACKED(id_iter)) { + BKE_main_idmap_insert_id(fd->new_idmap_uid, id_iter); + } } FOREACH_MAIN_ID_END; } @@ -2796,10 +2882,13 @@ static bool read_libblock_undo_restore_library(FileData *fd, * modified should not be an issue currently. */ for (Main *libmain : fd->old_bmain->split_mains->as_span().drop_front(1)) { if (&libmain->curlib->id == id_old) { + BLI_assert(libmain->curlib); + BLI_assert((libmain->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0); CLOG_DEBUG(&LOG_UNDO, " compare with %s -> match (existing libpath: %s)", - libmain->curlib ? libmain->curlib->id.name : "", - libmain->curlib ? libmain->curlib->runtime->filepath_abs : ""); + libmain->curlib->id.name, + libmain->curlib->runtime->filepath_abs); + /* In case of a library, we need to re-add its main to fd->bmain->split_mains, * because if we have later a missing ID_LINK_PLACEHOLDER, * we need to get the correct lib it is linked to! @@ -2914,6 +3003,26 @@ static void read_libblock_undo_restore_identical( * data-blocks too. */ ob->mode &= ~OB_MODE_EDIT; } + if (GS(id_old->name) == ID_LI) { + Library *lib = reinterpret_cast(id_old); + if (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + BLI_assert(lib->runtime->filedata == nullptr); + BLI_assert(lib->archive_parent_library); + /* The 'normal' parent of this archive library should already have been moved into the new + * Main. */ + BLI_assert(BKE_main_idmap_lookup_uid(fd->new_idmap_uid, + lib->archive_parent_library->id.session_uid) == + &lib->archive_parent_library->id); + /* The archive library ID has been moved in the new Main, but not its own old split main, as + * these packed IDs should be handled like local ones in undo case. So a new split libmain + * needs to be created to contain its packed IDs. */ + blo_add_main_for_library( + fd, lib, lib->archive_parent_library, lib->filepath, lib->runtime->filepath_abs, true); + } + else { + BLI_assert_unreachable(); + } + } } /* For undo, store changed datablock at old address. */ @@ -2970,8 +3079,10 @@ static bool read_libblock_undo_restore( const bool do_partial_undo = (fd->skip_flags & BLO_READ_SKIP_UNDO_OLD_MAIN) == 0; #ifndef NDEBUG - if (do_partial_undo && (bhead->code != ID_LINK_PLACEHOLDER)) { - /* This code should only ever be reached for local data-blocks. */ + if (do_partial_undo && (bhead->code != ID_LINK_PLACEHOLDER) && + (blo_bhead_id_flag(fd, bhead) & ID_FLAG_LINKED_AND_PACKED) == 0) + { + /* This code should only ever be reached for local or packed data-blocks. */ BLI_assert(main->curlib == nullptr); } #endif @@ -2983,8 +3094,13 @@ static bool read_libblock_undo_restore( nullptr; if (bhead->code == ID_LI) { - /* Restore library datablock, if possible. */ - if (read_libblock_undo_restore_library(fd, id, id_old, bhead)) { + /* Restore library datablock, if possible. + * + * Never handle archive libraries and their packed IDs as normal ones. These are local data, + * and need to be fully handled like local IDs. */ + if (id_old && (reinterpret_cast(id_old)->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0 && + read_libblock_undo_restore_library(fd, id, id_old, bhead)) + { return true; } } @@ -3182,6 +3298,14 @@ static BHead *read_libblock(FileData *fd, if (main->id_map != nullptr) { BKE_main_idmap_insert_id(main->id_map, id_target); } + if (ID_IS_PACKED(id)) { + BLI_assert(id->deep_hash != IDHash::get_null()); + fd->id_by_deep_hash->add_new(id->deep_hash, id); + BLI_assert(main->curlib); + } + if (fd->file_stat) { + id->runtime->src_blend_modifification_time = fd->file_stat->st_mtime; + } } return bhead; @@ -3831,6 +3955,10 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) } while (bhead) { + /* If not-null after the `switch`, the BHead is an ID one and needs to be read. */ + Main *bmain_to_read_into = nullptr; + bool placeholder_set_indirect_extern = false; + switch (bhead->code) { case BLO_CODE_DATA: case BLO_CODE_DNA1: @@ -3854,42 +3982,61 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) break; case ID_LINK_PLACEHOLDER: - if (fd->skip_flags & BLO_READ_SKIP_DATA) { + if ((fd->skip_flags & BLO_READ_SKIP_DATA) != 0) { bhead = blo_bhead_next(fd, bhead); + break; } - else { - /* Add link placeholder to the main of the library it belongs to. - * The library is the most recently loaded #ID_LI block, according - * to the file format definition. So we can use the entry at the - * end of `fd->bmain->split_mains`, typically the one last added in - * #direct_link_library. */ - - Main *libmain = (*fd->bmain->split_mains)[fd->bmain->split_mains->size() - 1]; - bhead = read_libblock(fd, libmain, bhead, 0, {}, true, nullptr); - } + /* Add link placeholder to the main of the library it belongs to. + * + * The library is the most recently loaded #ID_LI block, according to the file format + * definition. So we can use the entry at the end of `fd->bmain->split_mains`, typically + * the one last added in #direct_link_library. */ + bmain_to_read_into = (*fd->bmain->split_mains)[fd->bmain->split_mains->size() - 1]; + placeholder_set_indirect_extern = true; + break; + case ID_LI: + if ((fd->skip_flags & BLO_READ_SKIP_DATA) != 0) { + bhead = blo_bhead_next(fd, bhead); + break; + } + /* Library IDs are always read into the first (aka 'local') Main, even if they are written + * in 'library' blendfile-space (for archive libraries e.g.). */ + bmain_to_read_into = fd->bmain; break; - /* in 2.50+ files, the file identifier for screens is patched, forward compatibility */ case ID_SCRN: + /* in 2.50+ files, the file identifier for screens is patched, forward compatibility */ bhead->code = ID_SCR; /* pass on to default */ ATTR_FALLTHROUGH; default: { - if (blo_bhead_is_id_valid_type(bhead)) { - /* BHead is a valid known ID type one, read the whole ID and its sub-data, unless reading - * actual data is skipped. */ - if (fd->skip_flags & BLO_READ_SKIP_DATA) { - bhead = blo_bhead_next(fd, bhead); - } - else { - bhead = read_libblock(fd, bfd->main, bhead, ID_TAG_LOCAL, {}, false, nullptr); - } - } - else { - /* Unknown BHead type (or ID type), ignore it and skip to next BHead. */ + if ((fd->skip_flags & BLO_READ_SKIP_DATA) != 0 || !blo_bhead_is_id_valid_type(bhead)) { bhead = blo_bhead_next(fd, bhead); + break; } + /* Put read real ID into the main of the library it belongs to. + * + * Local IDs should all be written before any Library in the blendfile, so this code will + * always select `fd->bmain` for these. + * + * Packed linked IDs are real ID data in the currently read blendfile (unlike placeholders + * for regular linked data). But they are in their archive library 'name space' and + * 'blendfile space', so this follows the same logic as for placeholders to select the + * Main. + * + * The library is the most recently loaded #ID_LI block, according to the file format + * definition. So we can use the entry at the end of `fd->bmain->split_mains`, typically + * the one last added in #direct_link_library. */ + bmain_to_read_into = (*fd->bmain->split_mains)[fd->bmain->split_mains->size() - 1]; + BLI_assert_msg((bmain_to_read_into == fd->bmain || + (blo_bhead_id_flag(fd, bhead) & ID_FLAG_LINKED_AND_PACKED) != 0), + "Local IDs should always be put in the first Main split data-base, not in " + "a 'linked data' one"); } } + if (bmain_to_read_into) { + bhead = read_libblock( + fd, bmain_to_read_into, bhead, 0, {}, placeholder_set_indirect_extern, nullptr); + } if (bfd->main->is_read_invalid) { return bfd; @@ -3916,6 +4063,14 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) * of its items. */ blender::Vector
old_main_split_mains = {old_main->split_mains->as_span()}; for (Main *libmain : old_main_split_mains.as_span().drop_front(1)) { + BLI_assert(libmain->curlib); + if (libmain->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + /* Never move archived libraries and their content, these are 'local' data in undo context, + * so all packed linked IDs should have been handled like local ones undo-wise, and if + * packed libraries remain unused at this point, then they are indeed fully unused/removed + * from the new main. */ + continue; + } read_undo_move_libmain_data(fd, libmain, nullptr); } } @@ -3958,7 +4113,36 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) /* Do versioning before read_libraries, but skip in undo case. */ if (!is_undo) { if ((fd->skip_flags & BLO_READ_SKIP_DATA) == 0) { - do_versions(fd, nullptr, bfd->main); + for (Main *bmain : *fd->bmain->split_mains) { + /* Packed IDs are stored in the current .blend file, but belong to dedicated 'archive + * library' Mains, not the first, 'local' Main. So they do need versioning here, as for + * local IDs, which is why all the split Mains in the list need to be checked. + * + * Placeholders (of 'real' linked data) can't be versioned yet. Since they also belong to + * dedicated 'library' Mains, and are not mixed with the 'packed' ones, these Mains can be + * entirely skipped. */ + const bool contains_link_placeholder = (bmain->curlib != nullptr && + (bmain->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) == + 0); +#ifndef NDEBUG + MainListsArray lbarray = BKE_main_lists_get(*bmain); + for (ListBase *lb_array : lbarray) { + LISTBASE_FOREACH_MUTABLE (ID *, id, lb_array) { + BLI_assert_msg((id->runtime->readfile_data->tags.is_link_placeholder == + contains_link_placeholder), + contains_link_placeholder ? + "Real Library split Main contains non-placeholder IDs" : + (bmain->curlib == nullptr ? + "Local data split Main contains placeholder IDs" : + "Archive Library split Main contains placeholder IDs")); + } + } +#endif + if (contains_link_placeholder) { + continue; + } + do_versions(fd, bmain->curlib, bmain); + } } if ((fd->skip_flags & BLO_READ_SKIP_USERDEF) == 0) { @@ -4027,7 +4211,9 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) } LISTBASE_FOREACH_MUTABLE (Library *, lib, &bfd->main->libraries) { - /* Now we can clear this runtime library filedata, it is not needed anymore. */ + /* Now we can clear this runtime library filedata, it is not needed anymore. + * + * NOTE: This is also important to do for archive libraries. */ library_filedata_release(lib); /* If no data-blocks were read from a library (should only happen when all references to a * library's data are `ID_FLAG_INDIRECT_WEAK_LINK`), its versionfile will still be zero and @@ -4038,8 +4224,12 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) * placeholders IDs created will reference the library ID, and the library ID will have a * valid version number as the file was read to search for the linked IDs. * - In case the library blendfile does not exist, its local Library ID will get the version - * of the current local Main (i.e. the loaded blendfile). */ - if (lib->runtime->versionfile == 0) { + * of the current local Main (i.e. the loaded blendfile). + * - In case it is a reference library for archived ones, its runtime #archived_libraries + * vector will not be empty, and it must be kept, even if no data is directly linked from + * it anymore. + */ + if (lib->runtime->versionfile == 0 && lib->runtime->archived_libraries.is_empty()) { #ifndef NDEBUG ID *id_iter; FOREACH_MAIN_ID_BEGIN (bfd->main, id_iter) { @@ -4247,21 +4437,41 @@ static BHead *find_bhead_from_idname(FileData *fd, const char *idname) return find_bhead_from_code_name(fd, id_code_old, idname + 2); } -static ID *library_id_is_yet_read(FileData *fd, Main *mainvar, BHead *bhead) +static ID *library_id_is_yet_read_deep_hash(FileData *fd, BHead *bhead) +{ + if (const IDHash *deep_hash = blo_bhead_id_deep_hash(fd, bhead)) { + if (ID *existing_id = fd->id_by_deep_hash->lookup_default(*deep_hash, nullptr)) { + return existing_id; + } + } + return nullptr; +} + +static ID *library_id_is_yet_read_main(Main *mainvar, const char *idname) { if (mainvar->id_map == nullptr) { mainvar->id_map = BKE_main_idmap_create(mainvar, false, nullptr, MAIN_IDMAP_TYPE_NAME); } BLI_assert(BKE_main_idmap_main_get(mainvar->id_map) == mainvar); + ID *existing_id = BKE_main_idmap_lookup_name( + mainvar->id_map, GS(idname), idname + 2, mainvar->curlib); + BLI_assert(existing_id == + BLI_findstring(which_libbase(mainvar, GS(idname)), idname, offsetof(ID, name))); + return existing_id; +} + +static ID *library_id_is_yet_read(FileData *fd, Main *mainvar, BHead *bhead) +{ + if (ID *existing_id = library_id_is_yet_read_deep_hash(fd, bhead)) { + return existing_id; + } + const char *idname = blo_bhead_id_name(fd, bhead); if (!idname) { return nullptr; } - - ID *id = BKE_main_idmap_lookup_name(mainvar->id_map, GS(idname), idname + 2, mainvar->curlib); - BLI_assert(id == BLI_findstring(which_libbase(mainvar, GS(idname)), idname, offsetof(ID, name))); - return id; + return library_id_is_yet_read_main(mainvar, idname); } static void read_libraries_report_invalid_id_names(FileData *fd, @@ -4307,15 +4517,132 @@ struct BlendExpander { BLOExpandDoitCallback callback; }; +/* Find the existing Main matching the given blendfile library filepath, or create a new one (with + * the matching Library ID) if needed. + * + * NOTE: The process is a bit more complex for packed linked IDs and their archive libraries, as + * in this case, this function also needs to find or create a new suitable archive library, i.e. + * one which does not contain yet the given ID (from its name & type). */ +static Main *blo_find_main_for_library_and_idname(FileData *fd, + const char *lib_filepath, + const char *relabase, + const BHead *id_bhead, + const char *id_name, + const bool is_packed_id) +{ + Library *parent_lib = nullptr; + char filepath_abs[FILE_MAX]; + + STRNCPY(filepath_abs, lib_filepath); + BLI_path_abs(filepath_abs, relabase); + BLI_path_normalize(filepath_abs); + + for (Main *main_it : *fd->bmain->split_mains) { + const char *libname = (main_it->curlib) ? main_it->curlib->runtime->filepath_abs : + main_it->filepath; + + if (BLI_path_cmp(filepath_abs, libname) == 0) { + CLOG_DEBUG(&LOG, + "Found library '%s' for file path '%s'", + main_it->curlib ? main_it->curlib->id.name : "", + lib_filepath); + /* Due to how parent and archive libraries are created and written in the blend-file, + * the first library matching a given filepath should never be an archive one. */ + BLI_assert(!main_it->curlib || (main_it->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0); + if (!is_packed_id) { + return main_it; + } + /* For packed IDs, the Main of the main owner library is not a valid one. Another loop is + * needed into all the Mains matching the archive libraries of this main library. */ + BLI_assert(main_it->curlib); + parent_lib = main_it->curlib; + break; + } + } + + if (is_packed_id) { + if (parent_lib) { + /* Try to find an 'available' existing archive Main library, i.e. one that does not yet + * contain an ID of the same type and name. */ + for (Main *main_it : *fd->bmain->split_mains) { + if (!main_it->curlib || (main_it->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0 || + main_it->curlib->archive_parent_library != parent_lib) + { + continue; + } + if (ID *packed_id = library_id_is_yet_read_main(main_it, id_name)) { + /* Archive Main library already contains a 'same' ID - but it should have a different + * deep_hash. Otherwise, a previous call to `library_id_is_yet_read()` should have + * returned this ID, and this code should not be reached. */ + BLI_assert(packed_id->deep_hash != *blo_bhead_id_deep_hash(fd, id_bhead)); + UNUSED_VARS_NDEBUG(packed_id, id_bhead); + continue; + } + BLI_assert(ELEM(main_it->curlib->runtime->filedata, fd, nullptr)); + main_it->curlib->runtime->filedata = fd; + main_it->curlib->runtime->is_filedata_owner = false; + BLI_assert(main_it->versionfile != 0); + CLOG_DEBUG(&LOG, + "Found archive library '%s' for the packed ID '%s'", + main_it->curlib->id.name, + id_name); + return main_it; + } + } + else { + /* An archive library requires an existing parent library, create an empty, 'virtual' one if + * needed. */ + Main *reference_bmain = blo_add_main_for_library( + fd, nullptr, nullptr, lib_filepath, filepath_abs, false); + parent_lib = reference_bmain->curlib; + CLOG_DEBUG(&LOG, + "Added new parent library '%s' for file path '%s'", + parent_lib->id.name, + lib_filepath); + } + } + BLI_assert(parent_lib || !is_packed_id); + + Main *bmain = blo_add_main_for_library( + fd, nullptr, parent_lib, lib_filepath, filepath_abs, is_packed_id); + + read_file_version_and_colorspace(fd, bmain); + + if (is_packed_id) { + CLOG_DEBUG(&LOG, + "Added new archive library '%s' for the packed ID '%s'", + bmain->curlib->id.name, + id_name); + } + else { + CLOG_DEBUG( + &LOG, "Added new library '%s' for file path '%s'", bmain->curlib->id.name, lib_filepath); + } + return bmain; +} + +/* Actually load an ID from a library. There are three possible cases here: + * - `existing_id` is non-null: calling code already found a suitable existing ID, this function + * essentially then only updates the mappings for `bhead->old` address to point to the given + * ID. This is the only case where `libmain` may be `nullptr`. + * - The given bhead has an already loaded matching ID (found by a call to + * `library_id_is_yet_read`), then once that ID is found behavior is as in the previous case. + * - No matching existing ID is found, then a new one is actually read from the given FileData. + */ static void read_id_in_lib(FileData *fd, std::queue &ids_to_expand, Main *libmain, Library *parent_lib, BHead *bhead, + ID *existing_id, ID_Readfile_Data::Tags id_read_tags) { - ID *id = library_id_is_yet_read(fd, libmain, bhead); + ID *id = existing_id; + if (id == nullptr) { + BLI_assert(libmain); + id = library_id_is_yet_read(fd, libmain, bhead); + } if (id == nullptr) { /* ID has not been read yet, add placeholder to the main of the * library it belongs to, so that it will be read later. */ @@ -4392,44 +4719,96 @@ static void expand_doit_library(void *fdhandle, if (!blo_bhead_is_id_valid_type(bhead)) { return; } - if (!blo_bhead_id_name(fd, bhead)) { + const char *id_name = blo_bhead_id_name(fd, bhead); + if (!id_name) { /* Do not allow linking ID which names are invalid (likely coming from a future version of * Blender allowing longer names). */ return; } + const bool is_packed_id = (blo_bhead_id_flag(fd, bhead) & ID_FLAG_LINKED_AND_PACKED) != 0; + + BLI_assert_msg(!is_packed_id || bhead->code != ID_LINK_PLACEHOLDER, + "A link placeholder ID (aka reference to some ID linked from another library) " + "should never be packed."); if (bhead->code == ID_LINK_PLACEHOLDER) { /* Placeholder link to data-block in another library. */ BHead *bheadlib = find_previous_lib(fd, bhead); if (bheadlib == nullptr) { + BLO_reportf_wrap(fd->reports, + RPT_ERROR, + RPT_("LIB: .blend file %s seems corrupted, no owner 'Library' data found " + "for the linked data-block %s"), + mainvar->curlib->runtime->filepath_abs, + id_name ? id_name : ""); return; } Library *lib = reinterpret_cast( read_id_struct(fd, bheadlib, "Data for Library ID type", INDEX_ID_NULL)); - Main *libmain = blo_find_main(fd, lib->filepath, fd->relabase); + Main *libmain = blo_find_main_for_library_and_idname( + fd, lib->filepath, fd->relabase, nullptr, nullptr, false); MEM_freeN(lib); if (libmain->curlib == nullptr) { - const char *idname = blo_bhead_id_name(fd, bhead); - BLO_reportf_wrap(fd->reports, RPT_WARNING, RPT_("LIB: Data refers to main .blend file: '%s' from %s"), - idname ? idname : "", + id_name ? id_name : "", mainvar->curlib->runtime->filepath_abs); return; } /* Placeholders never need expanding, as they are a mere reference to ID from another * library/blendfile. */ - read_id_in_lib(fd, ids_to_expand, libmain, mainvar->curlib, bhead, {}); + read_id_in_lib(fd, ids_to_expand, libmain, mainvar->curlib, bhead, nullptr, {}); + } + else if (is_packed_id) { + /* Packed data-block from another library. */ + + /* That exact same packed ID may have already been read before. */ + if (ID *existing_id = library_id_is_yet_read_deep_hash(fd, bhead)) { + /* Ensure that the current BHead's `old` pointer will also be remapped to the found existing + * ID. */ + read_id_in_lib(fd, ids_to_expand, nullptr, nullptr, bhead, existing_id, {}); + return; + } + + BHead *bheadlib = find_previous_lib(fd, bhead); + if (bheadlib == nullptr) { + BLO_reportf_wrap(fd->reports, + RPT_ERROR, + RPT_("LIB: .blend file %s seems corrupted, no owner 'Library' data found " + "for the packed linked data-block %s"), + mainvar->curlib->runtime->filepath_abs, + id_name ? id_name : ""); + return; + } + + Library *lib = reinterpret_cast( + read_id_struct(fd, bheadlib, "Data for Library ID type", INDEX_ID_NULL)); + Main *libmain = blo_find_main_for_library_and_idname( + fd, lib->filepath, fd->relabase, bhead, id_name, is_packed_id); + MEM_freeN(lib); + + if (libmain->curlib == nullptr) { + BLO_reportf_wrap(fd->reports, + RPT_WARNING, + RPT_("LIB: Data refers to main .blend file: '%s' from %s"), + id_name ? id_name : "", + mainvar->curlib->runtime->filepath_abs); + return; + } + + ID_Readfile_Data::Tags id_read_tags{}; + id_read_tags.needs_expanding = true; + read_id_in_lib(fd, ids_to_expand, libmain, nullptr, bhead, nullptr, id_read_tags); } else { /* Data-block in same library. */ ID_Readfile_Data::Tags id_read_tags{}; id_read_tags.needs_expanding = true; - read_id_in_lib(fd, ids_to_expand, mainvar, nullptr, bhead, id_read_tags); + read_id_in_lib(fd, ids_to_expand, mainvar, nullptr, bhead, nullptr, id_read_tags); } } @@ -4476,6 +4855,13 @@ static void expand_main(void *fdhandle, Main *mainvar, BLOExpandDoitCallback cal FileData *fd = static_cast(fdhandle); BlendExpander expander = {fd, {}, mainvar, callback}; + /* Note: Packed IDs are the only current case where IDs read/loaded from a library blendfile will + * end up in another Main (outside of placeholders, which never need to be expanded). This is not + * a problem for initialization of the 'to be expanded' queue though, as no packed ID can be + * directly linked currently, they are only brough in indirectly, i.e. during the expansion + * process itself. + * + * So just looping on the 'main'/root Main of the read library is fine here currently. */ ID *id_iter; FOREACH_MAIN_ID_BEGIN (mainvar, id_iter) { if (BLO_readfile_id_runtime_tags(*id_iter).needs_expanding) { @@ -4600,11 +4986,22 @@ static Main *library_link_begin(Main *mainvar, fd->bmain = mainvar; + /* Add already existing packed data-blocks to map so that they are not loaded again. */ + ID *id; + FOREACH_MAIN_ID_BEGIN (mainvar, id) { + if (ID_IS_PACKED(id)) { + fd->id_by_deep_hash->add(id->deep_hash, id); + } + } + FOREACH_MAIN_ID_END; + /* make mains */ blo_split_main(mainvar); - /* which one do we need? */ - mainl = blo_find_main(fd, filepath, BKE_main_blendfile_path(mainvar)); + /* Find or create a Main matching the current library filepath. */ + /* Note: Directly linking packed IDs is not supported currently. */ + mainl = blo_find_main_for_library_and_idname( + fd, filepath, BKE_main_blendfile_path(mainvar), nullptr, nullptr, false); fd->fd_bmain = mainl; if (mainl->curlib) { mainl->curlib->runtime->filedata = fd; @@ -4737,7 +5134,7 @@ static void library_link_end(Main *mainl, FileData **fd, const int flag, ReportL Main *main_newid = BKE_main_new(); for (Main *mainlib : mainvar->split_mains->as_span().drop_front(1)) { - BLI_assert(mainlib->versionfile != 0); + BLI_assert(mainlib->versionfile != 0 || BKE_main_is_empty(mainlib)); /* We need to split out IDs already existing, * or they will go again through do_versions - bad, very bad! */ split_main_newid(mainlib, main_newid); @@ -5054,6 +5451,7 @@ static FileData *read_library_file_data(FileData *basefd, Main *bmain, Main *lib } fd->libmap = oldnewmap_new(); + fd->id_by_deep_hash = basefd->id_by_deep_hash; lib_bmain->curlib->runtime->filedata = fd; lib_bmain->curlib->runtime->is_filedata_owner = true; @@ -5104,6 +5502,13 @@ static void read_libraries(FileData *basefd) * this list gets longer as more indirectly library blends are found. */ for (int i = 1; i < bmain->split_mains->size(); i++) { Main *libmain = (*bmain->split_mains)[i]; + BLI_assert(libmain->curlib); + /* Always skip archived libraries here, these should _never_ need to be processed here, as + * their data is local data from a blendfile perspective. */ + if (libmain->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + BLI_assert(!has_linked_ids_to_read(libmain)); + continue; + } /* Does this library have any more linked data-blocks we need to read? */ if (has_linked_ids_to_read(libmain)) { CLOG_DEBUG(&LOG, diff --git a/source/blender/blenloader/intern/readfile.hh b/source/blender/blenloader/intern/readfile.hh index 9f18152221f..dad2889cf68 100644 --- a/source/blender/blenloader/intern/readfile.hh +++ b/source/blender/blenloader/intern/readfile.hh @@ -16,6 +16,7 @@ # include "BLI_winstuff.h" #endif +#include "BLI_fileops.h" #include "BLI_filereader.h" #include "BLI_map.hh" @@ -82,6 +83,7 @@ struct FileData { BlenderHeader blender_header = {}; FileReader *file = nullptr; + std::optional file_stat; /** * Whether we are undoing (< 0) or redoing (> 0), used to choose which 'unchanged' flag to use @@ -118,6 +120,8 @@ struct FileData { /** Used to retrieve asset data from (bhead+1). NOTE: This may not be available in old files, * will be -1 then! */ int id_asset_data_offset = 0; + int id_flag_offset = 0; + int id_deep_hash_offset = 0; /** For do_versions patching. */ int globalf = 0; int fileflags = 0; @@ -135,6 +139,8 @@ struct FileData { OldNewMap *datamap = nullptr; OldNewMap *globmap = nullptr; + /** Used to keep track of already loaded packed IDs to avoid loading them multiple times. */ + std::shared_ptr> id_by_deep_hash; /** * Store mapping from old ID pointers (the values they have in the .blend file) to new ones, @@ -189,9 +195,22 @@ struct FileData { void *storage_handle = nullptr; }; -/***/ +/** + * Split a single main into a vector of Mains, each containing only IDs from a given library. + * + * The vector is accessible in all of the split mains through the shared pointer + * #Main::split_mains. + * + * The first Main of the vector is the same as the given `main`, and contains local IDs. + * + * If `do_split_packed_ids` is `false`, packed linked IDs remain in the local (first) main as well. + */ +void blo_split_main(Main *bmain, bool do_split_packed_ids = true); +/** + * Join the set of split mains (found in given `main` #Main::split_mains vector shared pointer) + * back into that 'main' main. + */ void blo_join_main(Main *bmain); -void blo_split_main(Main *bmain); BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) ATTR_NONNULL(1, 2); @@ -232,6 +251,13 @@ BHead *blo_bhead_prev(FileData *fd, BHead *thisblock) ATTR_NONNULL(1, 2); * it was saved in a version of Blender with higher MAX_ID_NAME value). */ const char *blo_bhead_id_name(FileData *fd, const BHead *bhead); +/** + * Warning! It's the caller's responsibility to ensure that the given bhead **is** an ID one! + * + * Returns the ID flag value (or `0` if the blendfile is too old and the offset of the ID::flag + * member could not be computed). + */ +short blo_bhead_id_flag(const FileData *fd, const BHead *bhead); /** * Warning! Caller's responsibility to ensure given bhead **is** an ID one! */ diff --git a/source/blender/blenloader/intern/versioning_500.cc b/source/blender/blenloader/intern/versioning_500.cc index e561723e5f5..b599f4a74e1 100644 --- a/source/blender/blenloader/intern/versioning_500.cc +++ b/source/blender/blenloader/intern/versioning_500.cc @@ -72,6 +72,8 @@ #include "WM_api.hh" +#include "AS_asset_library.hh" + #include "readfile.hh" #include "versioning_common.hh" @@ -3679,4 +3681,7 @@ void blo_do_versions_500(FileData *fd, Library * /*lib*/, Main *bmain) LISTBASE_FOREACH (Mesh *, mesh, &bmain->meshes) { bke::mesh_freestyle_marks_to_generic(*mesh); } + + /* TODO: Can be moved to subversion bump. */ + AS_asset_library_import_method_ensure_valid(*bmain); } diff --git a/source/blender/blenloader/intern/writefile.cc b/source/blender/blenloader/intern/writefile.cc index 0236a3cf40b..4198a5b02ef 100644 --- a/source/blender/blenloader/intern/writefile.cc +++ b/source/blender/blenloader/intern/writefile.cc @@ -1164,6 +1164,11 @@ static void write_libraries(WriteData *wd, Main *bmain) if (id->us == 0) { continue; } + if (ID_IS_PACKED(id)) { + BLI_assert(library.flag & LIBRARY_FLAG_IS_ARCHIVE); + ids_used_from_library.append(id); + continue; + } if (id->tag & ID_TAG_EXTERN) { ids_used_from_library.append(id); continue; @@ -1178,6 +1183,15 @@ static void write_libraries(WriteData *wd, Main *bmain) if (library.packedfile) { should_write_library = true; } + else if (!library.runtime->archived_libraries.is_empty()) { + /* Reference 'real' blendfile library of archived 'copies' of it containing packed linked + * IDs should always be written. */ + /* FIXME: A bit weak, as it could be that all archive libs are now empty (if all related + * packed linked IDs have been deleted e.g.)... + * Could be fixed by either adding more checks here, or ensuring empty archive libs are + * deleted when no ID uses them anymore? */ + should_write_library = true; + } else if (wd->use_memfile) { /* When writing undo step we always write all existing libraries. That makes reading undo * step much easier when dealing with purely indirectly used libraries. */ @@ -1194,16 +1208,22 @@ static void write_libraries(WriteData *wd, Main *bmain) write_id(wd, &library.id); - /* Write placeholders for linked data-blocks that are used. */ + /* Write placeholders for linked data-blocks that are used, and real IDs for the packed linked + * ones. */ for (ID *id : ids_used_from_library) { - if (!BKE_idtype_idcode_is_linkable(GS(id->name))) { - CLOG_ERROR(&LOG, - "Data-block '%s' from lib '%s' is not linkable, but is flagged as " - "directly linked", - id->name, - library.runtime->filepath_abs); + if (ID_IS_PACKED(id)) { + write_id(wd, id); + } + else { + if (!BKE_idtype_idcode_is_linkable(GS(id->name))) { + CLOG_ERROR(&LOG, + "Data-block '%s' from lib '%s' is not linkable, but is flagged as " + "directly linked", + id->name, + library.runtime->filepath_abs); + } + write_id_placeholder(wd, id); } - write_id_placeholder(wd, id); } } diff --git a/source/blender/editors/asset/intern/asset_import.cc b/source/blender/editors/asset/intern/asset_import.cc index 4a9380469ac..6cf765d644e 100644 --- a/source/blender/editors/asset/intern/asset_import.cc +++ b/source/blender/editors/asset/intern/asset_import.cc @@ -32,11 +32,14 @@ ID *asset_local_id_ensure_imported(Main &bmain, } const eAssetImportMethod method = [&]() { + const bool no_packing = U.experimental.no_data_block_packing; if (import_method) { - return *import_method; + return (no_packing && *import_method == ASSET_IMPORT_PACK) ? ASSET_IMPORT_APPEND_REUSE : + *import_method; } if (std::optional asset_method = asset.get_import_method()) { - return *asset_method; + return (no_packing && *asset_method == ASSET_IMPORT_PACK) ? ASSET_IMPORT_APPEND_REUSE : + *asset_method; } return ASSET_IMPORT_APPEND_REUSE; }(); @@ -51,6 +54,16 @@ ID *asset_local_id_ensure_imported(Main &bmain, asset.get_id_type(), asset.get_name().c_str(), (asset.get_use_relative_path() ? FILE_RELPATH : 0)); + case ASSET_IMPORT_PACK: + return WM_file_link_datablock(&bmain, + nullptr, + nullptr, + nullptr, + blend_path.c_str(), + asset.get_id_type(), + asset.get_name().c_str(), + BLO_LIBLINK_PACK | + (asset.get_use_relative_path() ? FILE_RELPATH : 0)); case ASSET_IMPORT_APPEND: return WM_file_append_datablock(&bmain, nullptr, diff --git a/source/blender/editors/asset/intern/asset_shelf_asset_view.cc b/source/blender/editors/asset/intern/asset_shelf_asset_view.cc index 5c628f93c87..88c37db87d9 100644 --- a/source/blender/editors/asset/intern/asset_shelf_asset_view.cc +++ b/source/blender/editors/asset/intern/asset_shelf_asset_view.cc @@ -406,8 +406,11 @@ void *AssetDragController::create_drag_data() const return static_cast(local_id); } - const eAssetImportMethod import_method = asset_.get_import_method().value_or( - ASSET_IMPORT_APPEND_REUSE); + eAssetImportMethod import_method = asset_.get_import_method().value_or(ASSET_IMPORT_PACK); + if (U.experimental.no_data_block_packing && import_method == ASSET_IMPORT_PACK) { + import_method = ASSET_IMPORT_APPEND_REUSE; + } + AssetImportSettings import_settings{}; import_settings.method = import_method; import_settings.use_instance_collections = false; diff --git a/source/blender/editors/interface/interface_icons.cc b/source/blender/editors/interface/interface_icons.cc index d81277205ed..767b74c42cd 100644 --- a/source/blender/editors/interface/interface_icons.cc +++ b/source/blender/editors/interface/interface_icons.cc @@ -1962,6 +1962,9 @@ int ui_id_icon_get(const bContext *C, ID *id, const bool big) int UI_icon_from_library(const ID *id) { if (ID_IS_LINKED(id)) { + if (ID_IS_PACKED(id)) { + return ICON_PACKAGE; + } if (id->tag & ID_TAG_MISSING) { return ICON_LIBRARY_DATA_BROKEN; } diff --git a/source/blender/editors/interface/interface_ops.cc b/source/blender/editors/interface/interface_ops.cc index b46ea31a2ce..36ec88cae43 100644 --- a/source/blender/editors/interface/interface_ops.cc +++ b/source/blender/editors/interface/interface_ops.cc @@ -717,6 +717,10 @@ static bool override_idtemplate_poll(bContext *C, const bool is_create_op) return false; } + if (ID_IS_PACKED(id)) { + return false; + } + if (is_create_op) { if (!ID_IS_LINKED(id) && !ID_IS_OVERRIDE_LIBRARY_REAL(id)) { return false; diff --git a/source/blender/editors/interface/templates/interface_template_id.cc b/source/blender/editors/interface/templates/interface_template_id.cc index a75a6bae274..4c6309f6120 100644 --- a/source/blender/editors/interface/templates/interface_template_id.cc +++ b/source/blender/editors/interface/templates/interface_template_id.cc @@ -1119,7 +1119,21 @@ static void template_ID(const bContext *C, if (!hide_buttons && !(idfrom && ID_IS_LINKED(idfrom))) { if (ID_IS_LINKED(id)) { const bool disabled = !BKE_idtype_idcode_is_localizable(GS(id->name)); - if (id->tag & ID_TAG_INDIRECT) { + if (ID_IS_PACKED(id)) { + but = uiDefIconBut(block, + ButType::But, + 0, + ICON_PACKAGE, + 0, + 0, + UI_UNIT_X, + UI_UNIT_Y, + nullptr, + 0, + 0, + TIP_("Packed library data-block, click to unpack and make local")); + } + else if (id->tag & ID_TAG_INDIRECT) { but = uiDefIconBut(block, ButType::But, 0, diff --git a/source/blender/editors/screen/screen_ops.cc b/source/blender/editors/screen/screen_ops.cc index fb16e037a2f..86e79a5250f 100644 --- a/source/blender/editors/screen/screen_ops.cc +++ b/source/blender/editors/screen/screen_ops.cc @@ -6959,6 +6959,8 @@ static std::string screen_drop_scene_tooltip(bContext * /*C*/, switch (asset_drag->import_settings.method) { case ASSET_IMPORT_LINK: return fmt::format(fmt::runtime(TIP_("Link {}")), dragged_scene_name); + case ASSET_IMPORT_PACK: + return fmt::format(fmt::runtime(TIP_("Pack {}")), dragged_scene_name); case ASSET_IMPORT_APPEND: return fmt::format(fmt::runtime(TIP_("Append {}")), dragged_scene_name); case ASSET_IMPORT_APPEND_REUSE: diff --git a/source/blender/editors/space_file/file_draw.cc b/source/blender/editors/space_file/file_draw.cc index 42532d1172b..f3e24c262af 100644 --- a/source/blender/editors/space_file/file_draw.cc +++ b/source/blender/editors/space_file/file_draw.cc @@ -410,7 +410,7 @@ static void file_but_enable_drag(uiBut *but, import_settings.method = eAssetImportMethod(import_method); import_settings.use_instance_collections = (sfile->asset_params->import_flags & - (import_method == ASSET_IMPORT_LINK ? + (ELEM(import_method, ASSET_IMPORT_LINK, ASSET_IMPORT_PACK) ? FILE_ASSET_IMPORT_INSTANCE_COLLECTIONS_ON_LINK : FILE_ASSET_IMPORT_INSTANCE_COLLECTIONS_ON_APPEND)) != 0; diff --git a/source/blender/editors/space_file/filesel.cc b/source/blender/editors/space_file/filesel.cc index 43225c544b7..1ea7cc7cc99 100644 --- a/source/blender/editors/space_file/filesel.cc +++ b/source/blender/editors/space_file/filesel.cc @@ -535,6 +535,8 @@ int ED_fileselect_asset_import_method_get(const SpaceFile *sfile, const FileDirE return ASSET_IMPORT_APPEND; case FILE_ASSET_IMPORT_APPEND_REUSE: return ASSET_IMPORT_APPEND_REUSE; + case FILE_ASSET_IMPORT_PACK: + return ASSET_IMPORT_PACK; /* Should be handled above already. Break and fail below. */ case FILE_ASSET_IMPORT_FOLLOW_PREFS: diff --git a/source/blender/editors/space_outliner/outliner_draw.cc b/source/blender/editors/space_outliner/outliner_draw.cc index 7d87187e3ed..4c17bf873d8 100644 --- a/source/blender/editors/space_outliner/outliner_draw.cc +++ b/source/blender/editors/space_outliner/outliner_draw.cc @@ -2571,6 +2571,9 @@ static BIFIconID tree_element_get_icon_from_id(const ID *id) if (id->tag & ID_TAG_MISSING) { return ICON_LIBRARY_DATA_BROKEN; } + else if (reinterpret_cast(id)->flag & LIBRARY_FLAG_IS_ARCHIVE) { + return ICON_PACKAGE; + } else if (((Library *)id)->runtime->parent) { return ICON_LIBRARY_DATA_INDIRECT; } @@ -2875,8 +2878,16 @@ TreeElementIcon tree_element_get_icon(TreeStoreElem *tselem, TreeElement *te) const PointerRNA &ptr = te_rna_struct->get_pointer_rna(); if (RNA_struct_is_ID(ptr.type)) { - data.drag_id = static_cast(ptr.data); - data.icon = RNA_struct_ui_icon(ptr.type); + ID *id = static_cast(ptr.data); + data.drag_id = id; + if (id && GS(id->name) == ID_LI && + id_cast(id)->flag & LIBRARY_FLAG_IS_ARCHIVE) + { + data.icon = ICON_PACKAGE; + } + else { + data.icon = RNA_struct_ui_icon(ptr.type); + } } else { data.icon = RNA_struct_ui_icon(ptr.type); diff --git a/source/blender/editors/space_outliner/tree/tree_display_libraries.cc b/source/blender/editors/space_outliner/tree/tree_display_libraries.cc index ff90f065789..42e7ab00770 100644 --- a/source/blender/editors/space_outliner/tree/tree_display_libraries.cc +++ b/source/blender/editors/space_outliner/tree/tree_display_libraries.cc @@ -71,12 +71,14 @@ ListBase TreeDisplayLibraries::build_tree(const TreeSourceData &source_data) TreeStoreElem *tselem = TREESTORE(ten); Library *lib = (Library *)tselem->id; BLI_assert(!lib || (GS(lib->id.name) == ID_LI)); - if (!lib || !lib->runtime->parent) { + if (!lib || !(lib->runtime->parent || lib->archive_parent_library)) { continue; } /* A library with a non-null `parent` is always strictly indirectly linked. */ - TreeElement *parent = reinterpret_cast(lib->runtime->parent->id.newid); + TreeElement *parent = reinterpret_cast( + (lib->archive_parent_library ? lib->archive_parent_library : lib->runtime->parent) + ->id.newid); BLI_remlink(&tree, ten); BLI_addtail(&parent->subtree, ten); ten->parent = parent; diff --git a/source/blender/makesdna/DNA_ID.h b/source/blender/makesdna/DNA_ID.h index 55e96221bfe..0d3ae436616 100644 --- a/source/blender/makesdna/DNA_ID.h +++ b/source/blender/makesdna/DNA_ID.h @@ -18,6 +18,7 @@ /** Workaround to forward-declare C++ type in C header. */ #ifdef __cplusplus +# include # include namespace blender::bke::id { @@ -43,10 +44,6 @@ typedef struct IDPropertyGroupChildrenSet IDPropertyGroupChildrenSet; typedef struct ID_RuntimeHandle ID_RuntimeHandle; #endif -#ifdef __cplusplus -extern "C" { -#endif - struct FileData; struct GHash; struct ID; @@ -383,6 +380,37 @@ enum { ID_REMAP_IS_USER_ONE_SKIPPED = 1 << 1, }; +typedef struct IDHash { + char data[16]; + +#ifdef __cplusplus + uint64_t hash() const + { + return *reinterpret_cast(this->data); + } + + static constexpr IDHash get_null() + { + return {}; + } + bool is_null() const + { + return *this == IDHash::get_null(); + } + + friend bool operator==(const IDHash &a, const IDHash &b) + { + return memcmp(a.data, b.data, sizeof(a.data)) == 0; + } + + friend bool operator!=(const IDHash &a, const IDHash &b) + { + return !(a == b); + } + +#endif +} IDHash; + typedef struct ID { /* There's a nasty circular dependency here.... 'void *' to the rescue! I * really wonder why this is needed. */ @@ -433,6 +461,18 @@ typedef struct ID { */ unsigned int session_uid; + /** + * This is only available on packed linked data-blocks. It is a hash of the contents the + * data-block including all its dependencies. It is computed when first packing the data-block + * and is not changed afterwards. It can be used to detect that packed data-blocks in two + * separate .blend files are the same. + * + * Two data-blocks with the same deep hash are assumed to be interchangeable, but not necessarily + * exactly the same. For example, it's possible to change node positions on packed data-blocks + * without changing the deep hash. + */ + IDHash deep_hash; + /** * User-defined custom properties storage. Typically Accessed through the 'dict' syntax from * Python. @@ -511,6 +551,24 @@ typedef struct Library { /** Path name used for reading, can be relative and edited in the outliner. */ char filepath[/*FILE_MAX*/ 1024]; + /** Flags defining specific characteristics of a library. See #LibraryFlag. */ + uint16_t flag; + char _pad[6]; + + /** + * For archive library only (#LIBRARY_FLAG_IS_ARCHIVE): The main library owning it. + * + * `archive_parent_library` and `packedfile` should never be both non-null in a same Library ID. + */ + struct Library *archive_parent_library; + + /** + * Packed blendfile of the library, nullptr if not packed. + * + * \note Individual IDs may be packed even if the entire library is not packed. + * + * `archive_parent_library` and `packedfile` should never be both non-null in a same Library ID. + */ struct PackedFile *packedfile; /** @@ -519,8 +577,22 @@ typedef struct Library { * Typically allocated when creating a new Library or reading it from a blendfile. */ LibraryRuntimeHandle *runtime; + + void *_pad2; } Library; +/** + * #Library.flag + * + * Some of these flags define a 'virtual' library, which may not be an actual blendfile, store + * 'archived' embedded data, etc. IDs contained in these virtual libraries are _not_ managed by + * regular linking code. + */ +enum LibraryFlag { + /** The library is an 'archive' that only contains embedded linked data. */ + LIBRARY_FLAG_IS_ARCHIVE = 1 << 0, +}; + /** * A weak library/ID reference for local data that has been appended, to allow re-using that local * data instead of creating a new copy of it in future appends. @@ -618,6 +690,12 @@ typedef struct PreviewImage { #define ID_MISSING(_id) ((((const ID *)(_id))->tag & ID_TAG_MISSING) != 0) #define ID_IS_LINKED(_id) (((const ID *)(_id))->lib != NULL) +/** + * Indicates that this ID is linked but also packed into the current .blend file. Note that this + * just means that this specific ID and its dependencies are packed, not the entire library. So + * this is separate from #Library::packedfile. + */ +#define ID_IS_PACKED(_id) (ID_IS_LINKED(_id) && ((_id)->flag & ID_FLAG_LINKED_AND_PACKED)) #define ID_TYPE_SUPPORTS_ASSET_EDITABLE(id_type) \ ELEM(id_type, ID_BR, ID_TE, ID_NT, ID_IM, ID_PC, ID_MA) @@ -717,6 +795,11 @@ enum { * so it must be treated as dirty. */ ID_FLAG_CLIPBOARD_MARK = 1 << 14, + /** + * Indicates that this linked ID is packed into the current .blend file. This should never be set + * on local ID (without)one with a null `ID::lib` pointer). + */ + ID_FLAG_LINKED_AND_PACKED = 1 << 15, }; /** @@ -1274,10 +1357,6 @@ typedef enum eID_Index { #define INDEX_ID_MAX (INDEX_ID_NULL + 1) -#ifdef __cplusplus -} -#endif - #ifdef __cplusplus namespace blender::dna { namespace detail { diff --git a/source/blender/makesdna/DNA_asset_types.h b/source/blender/makesdna/DNA_asset_types.h index 829eaaae7f1..6ad8b21a8bb 100644 --- a/source/blender/makesdna/DNA_asset_types.h +++ b/source/blender/makesdna/DNA_asset_types.h @@ -118,6 +118,8 @@ typedef enum eAssetImportMethod { * heavy data dependencies (e.g. the image data-blocks of a material, the mesh of an object) may * be reused from an earlier append. */ ASSET_IMPORT_APPEND_REUSE = 2, + /** Link data-block, but also pack it as read-only data. */ + ASSET_IMPORT_PACK = 3, } eAssetImportMethod; # diff --git a/source/blender/makesdna/DNA_object_types.h b/source/blender/makesdna/DNA_object_types.h index 30a99ab9377..1be3a9769c9 100644 --- a/source/blender/makesdna/DNA_object_types.h +++ b/source/blender/makesdna/DNA_object_types.h @@ -227,7 +227,6 @@ typedef struct Object { bAnimVizSettings avs; /** Motion path cache for this object. */ bMotionPath *mpath; - void *_pad0; ListBase effect DNA_DEPRECATED; /* XXX deprecated... keep for readfile */ ListBase defbase DNA_DEPRECATED; /* Only for versioning, moved to object data. */ diff --git a/source/blender/makesdna/DNA_particle_types.h b/source/blender/makesdna/DNA_particle_types.h index 0f1204019e8..0c970b4a09a 100644 --- a/source/blender/makesdna/DNA_particle_types.h +++ b/source/blender/makesdna/DNA_particle_types.h @@ -298,7 +298,6 @@ typedef struct ParticleSettings { float rad_root, rad_tip, rad_scale; struct CurveMapping *twistcurve; - void *_pad7; } ParticleSettings; typedef struct ParticleSystem { diff --git a/source/blender/makesdna/DNA_space_enums.h b/source/blender/makesdna/DNA_space_enums.h index 77bbe6455a9..0b36f64c1ca 100644 --- a/source/blender/makesdna/DNA_space_enums.h +++ b/source/blender/makesdna/DNA_space_enums.h @@ -477,6 +477,11 @@ typedef enum eFileAssetImportMethod { FILE_ASSET_IMPORT_APPEND_REUSE = 2, /** Default: Follow the preference setting for this asset library. */ FILE_ASSET_IMPORT_FOLLOW_PREFS = 3, + /** + * Link the data-block, but also pack it in the current file to keep it working even if the + * source file is not available anymore. + */ + FILE_ASSET_IMPORT_PACK = 4, } eFileAssetImportMethod; typedef enum eFileAssetImportFlags { diff --git a/source/blender/makesdna/DNA_userdef_types.h b/source/blender/makesdna/DNA_userdef_types.h index adfac3f84af..ec1092ad752 100644 --- a/source/blender/makesdna/DNA_userdef_types.h +++ b/source/blender/makesdna/DNA_userdef_types.h @@ -224,6 +224,7 @@ typedef struct UserDef_Experimental { char use_extensions_debug; char use_recompute_usercount_on_save_debug; char write_legacy_blend_file_format; + char no_data_block_packing; char SANITIZE_AFTER_HERE; /* The following options are automatically sanitized (set to 0) * when the release cycle is not alpha. */ @@ -233,7 +234,7 @@ typedef struct UserDef_Experimental { char use_new_volume_nodes; char use_shader_node_previews; char use_geometry_nodes_lists; - char _pad[6]; + char _pad[5]; } UserDef_Experimental; #define USER_EXPERIMENTAL_TEST(userdef, member) (((userdef)->experimental).member) diff --git a/source/blender/makesrna/intern/rna_ID.cc b/source/blender/makesrna/intern/rna_ID.cc index d8c3c635d37..85c2f13a45f 100644 --- a/source/blender/makesrna/intern/rna_ID.cc +++ b/source/blender/makesrna/intern/rna_ID.cc @@ -300,7 +300,12 @@ static int rna_ID_name_editable(const PointerRNA *ptr, const char **r_info) * and could be useful in some cases. */ if (!ID_IS_EDITABLE(id)) { if (r_info) { - *r_info = N_("Linked data-blocks cannot be renamed"); + if (ID_IS_PACKED(id)) { + *r_info = N_("Packed data-blocks cannot be renamed"); + } + else { + *r_info = N_("Linked data-blocks cannot be renamed"); + } } return 0; } @@ -1539,6 +1544,52 @@ static void rna_Library_version_get(PointerRNA *ptr, int *value) value[2] = lib->runtime->subversionfile; } +static void rna_Library_archive_libraries_begin(CollectionPropertyIterator *iter, PointerRNA *ptr) +{ + iter->parent = *ptr; + Library *lib = static_cast(ptr->data); + + Library **archive_libraries_iter = lib->runtime->archived_libraries.begin(); + iter->internal.custom = archive_libraries_iter; + + iter->valid = archive_libraries_iter != lib->runtime->archived_libraries.end(); +} + +static void rna_Library_archive_libraries_next(CollectionPropertyIterator *iter) +{ + Library *lib = static_cast(iter->parent.data); + Library **archive_libraries_iter = static_cast(iter->internal.custom); + + archive_libraries_iter++; + iter->internal.custom = archive_libraries_iter; + + iter->valid = archive_libraries_iter != lib->runtime->archived_libraries.end(); +} + +static PointerRNA rna_Library_archive_libraries_get(CollectionPropertyIterator *iter) +{ + Library **archive_libraries_iter = static_cast(iter->internal.custom); + return RNA_pointer_create_with_parent(iter->parent, &RNA_Library, *archive_libraries_iter); +} + +static int rna_Library_archive_libraries_length(PointerRNA *ptr) +{ + Library *lib = static_cast(ptr->data); + return int(lib->runtime->archived_libraries.size()); +} + +static bool rna_Library_archive_libraries_lookupint(PointerRNA *ptr, int key, PointerRNA *r_ptr) +{ + Library *lib = static_cast(ptr->data); + if (key < 0 || key >= lib->runtime->archived_libraries.size()) { + return false; + } + + Library *archive_library = lib->runtime->archived_libraries[key]; + rna_pointer_create_with_ancestors(*ptr, &RNA_Library, archive_library, *r_ptr); + return true; +} + static PointerRNA rna_Library_parent_get(PointerRNA *ptr) { Library *lib = ptr->data_as(); @@ -2345,6 +2396,12 @@ static void rna_def_ID(BlenderRNA *brna) "This data-block is not an independent one, but is actually a sub-data of another ID " "(typical example: root node trees or master collections)"); + prop = RNA_def_property(srna, "is_linked_packed", PROP_BOOLEAN, PROP_NONE); + RNA_def_property_boolean_sdna(prop, nullptr, "flag", ID_FLAG_LINKED_AND_PACKED); + RNA_def_property_clear_flag(prop, PROP_EDITABLE); + RNA_def_property_ui_text( + prop, "Linked Packed", "This data-block is linked and packed into the .blend file"); + prop = RNA_def_property(srna, "is_missing", PROP_BOOLEAN, PROP_NONE); RNA_def_property_boolean_sdna(prop, nullptr, "tag", ID_TAG_MISSING); RNA_def_property_clear_flag(prop, PROP_EDITABLE); @@ -2667,6 +2724,39 @@ static void rna_def_library(BlenderRNA *brna) "Data-blocks in this library are editable despite being linked. " "Used by brush assets and their dependencies."); + prop = RNA_def_property(srna, "is_archive", PROP_BOOLEAN, PROP_NONE); + RNA_def_property_boolean_sdna(prop, nullptr, "flag", LIBRARY_FLAG_IS_ARCHIVE); + RNA_def_property_clear_flag(prop, PROP_EDITABLE); + RNA_def_property_ui_text(prop, + "Is Archive", + "This library is an 'archive' storage for packed linked IDs " + "originally linked from its 'archive parent' library."); + + prop = RNA_def_property(srna, "archive_parent_library", PROP_POINTER, PROP_NONE); + RNA_def_property_pointer_sdna(prop, nullptr, "archive_parent_library"); + RNA_def_property_struct_type(prop, "Library"); + RNA_def_property_override_flag(prop, PROPOVERRIDE_NO_COMPARISON); + RNA_def_property_ui_text(prop, + "Parent Archive Library", + "Source library from which this archive of packed IDs was generated"); + + prop = RNA_def_property(srna, "archive_libraries", PROP_COLLECTION, PROP_NONE); + RNA_def_property_struct_type(prop, "Library"); + RNA_def_property_collection_funcs(prop, + "rna_Library_archive_libraries_begin", + "rna_Library_archive_libraries_next", + nullptr, + "rna_Library_archive_libraries_get", + "rna_Library_archive_libraries_length", + "rna_Library_archive_libraries_lookupint", + nullptr, + nullptr); + RNA_def_property_override_flag(prop, PROPOVERRIDE_NO_COMPARISON); + RNA_def_property_ui_text( + prop, + "Archive Libraries", + "Archive libraries of packed IDs, generated (and owned) by this source library"); + func = RNA_def_function(srna, "reload", "rna_Library_reload"); RNA_def_function_flag(func, FUNC_USE_REPORTS | FUNC_USE_CONTEXT); RNA_def_function_ui_description(func, "Reload this library and all its linked data-blocks"); diff --git a/source/blender/makesrna/intern/rna_main_api.cc b/source/blender/makesrna/intern/rna_main_api.cc index 395d6d24303..0efb058b60f 100644 --- a/source/blender/makesrna/intern/rna_main_api.cc +++ b/source/blender/makesrna/intern/rna_main_api.cc @@ -34,6 +34,7 @@ # include "BKE_image.hh" # include "BKE_lattice.hh" # include "BKE_lib_remap.hh" +# include "BKE_library.hh" # include "BKE_light.h" # include "BKE_lightprobe.h" # include "BKE_linestyle.h" @@ -146,6 +147,28 @@ static void rna_Main_ID_remove(Main *bmain, } } +static ID *rna_Main_pack_linked_ids_hierarchy(struct BlendData *blenddata, + ReportList *reports, + ID *root_id) +{ + if (!ID_IS_LINKED(root_id)) { + BKE_reportf(reports, RPT_ERROR, "Only linked IDs can be packed"); + return nullptr; + } + if (ID_IS_PACKED(root_id)) { + /* Nothing to do. */ + return root_id; + } + + Main *bmain = reinterpret_cast
(blenddata); + blender::bke::library::pack_linked_id_hierarchy(*bmain, *root_id); + + ID *packed_root_id = root_id->newid; + BKE_main_id_newptr_and_tag_clear(bmain); + + return packed_root_id; +} + static Camera *rna_Main_cameras_new(Main *bmain, const char *name) { char safe_name[MAX_ID_NAME - 2]; @@ -869,12 +892,12 @@ RNA_MAIN_ID_TAG_FUNCS_DEF(volumes, volumes, ID_VO) #else -void RNA_api_main(StructRNA * /*srna*/) +void RNA_api_main(StructRNA *srna) { -# if 0 FunctionRNA *func; PropertyRNA *parm; +# if 0 /* maybe we want to add functions in 'bpy.data' still? * for now they are all in collections bpy.data.images.new(...) */ func = RNA_def_function(srna, "add_image", "rna_Main_add_image"); @@ -885,6 +908,15 @@ void RNA_api_main(StructRNA * /*srna*/) parm = RNA_def_pointer(func, "image", "Image", "", "New image"); RNA_def_function_return(func, parm); # endif + + func = RNA_def_function(srna, "pack_linked_ids_hierarchy", "rna_Main_pack_linked_ids_hierarchy"); + RNA_def_function_ui_description( + func, "Pack the given linked ID and its dependencies into current blendfile"); + RNA_def_function_flag(func, FUNC_USE_REPORTS); + parm = RNA_def_pointer(func, "root_id", "ID", "", "Root linked ID to pack"); + RNA_def_parameter_flags(parm, PropertyFlag(0), PARM_REQUIRED); + parm = RNA_def_pointer(func, "packed_id", "ID", "", "The packed ID matching the given root ID"); + RNA_def_function_return(func, parm); } void RNA_def_main_cameras(BlenderRNA *brna, PropertyRNA *cprop) diff --git a/source/blender/makesrna/intern/rna_space.cc b/source/blender/makesrna/intern/rna_space.cc index a9561bb9f02..7f6d3bd9769 100644 --- a/source/blender/makesrna/intern/rna_space.cc +++ b/source/blender/makesrna/intern/rna_space.cc @@ -358,6 +358,40 @@ const EnumPropertyItem rna_enum_fileselect_params_sort_items[] = { {0, nullptr, 0, nullptr, nullptr}, }; +static const EnumPropertyItem rna_enum_asset_import_method_items[] = { + {FILE_ASSET_IMPORT_FOLLOW_PREFS, + "FOLLOW_PREFS", + 0, + "Follow Preferences", + "Use the import method set in the Preferences for this asset library, don't override it " + "for this Asset Browser"}, + {FILE_ASSET_IMPORT_LINK, + "LINK", + ICON_LINK_BLEND, + "Link", + "Import the assets as linked data-block"}, + {FILE_ASSET_IMPORT_APPEND, + "APPEND", + ICON_APPEND_BLEND, + "Append", + "Import the asset as copied data-block, with no link to the original asset data-block"}, + {FILE_ASSET_IMPORT_APPEND_REUSE, + "APPEND_REUSE", + ICON_APPEND_BLEND, + "Append (Reuse Data)", + "Import the asset as copied data-block while avoiding multiple copies of nested, " + "typically heavy data. For example the textures of a material asset, or the mesh of an " + "object asset, don't have to be copied every time this asset is imported. The instances of " + "the asset share the data instead"}, + {FILE_ASSET_IMPORT_PACK, + "PACK", + ICON_PACKAGE, + "Pack", + "Import the asset as linked data-block, and pack it in the current file (ensures that it " + "remains unchanged in case the library data is modified, is not available anymore, etc.)"}, + {0, nullptr, 0, nullptr, nullptr}, +}; + #ifndef RNA_RUNTIME static const EnumPropertyItem stereo3d_eye_items[] = { {STEREO_LEFT_ID, "LEFT_EYE", ICON_NONE, "Left Eye"}, @@ -3662,6 +3696,37 @@ static void rna_FileAssetSelectParams_catalog_id_set(PointerRNA *ptr, const char params->asset_catalog_visibility = FILE_SHOW_ASSETS_FROM_CATALOG; } +static const EnumPropertyItem *rna_FileAssetSelectParams_import_method_itemf( + bContext * /*C*/, PointerRNA * /*ptr*/, PropertyRNA * /*prop*/, bool *r_free) +{ + EnumPropertyItem *items = nullptr; + int items_num = 0; + for (const EnumPropertyItem *item = rna_enum_asset_import_method_items; item->identifier; item++) + { + switch (eFileAssetImportMethod(item->value)) { + case FILE_ASSET_IMPORT_APPEND_REUSE: { + if (U.experimental.no_data_block_packing) { + RNA_enum_item_add(&items, &items_num, item); + } + break; + } + case FILE_ASSET_IMPORT_PACK: { + if (!U.experimental.no_data_block_packing) { + RNA_enum_item_add(&items, &items_num, item); + } + break; + } + default: { + RNA_enum_item_add(&items, &items_num, item); + break; + } + } + } + RNA_enum_item_end(&items, &items_num); + *r_free = true; + return items; +} + #else static const EnumPropertyItem dt_uv_items[] = { @@ -7426,30 +7491,6 @@ static void rna_def_fileselect_asset_params(BlenderRNA *brna) StructRNA *srna; PropertyRNA *prop; - static const EnumPropertyItem asset_import_method_items[] = { - {FILE_ASSET_IMPORT_FOLLOW_PREFS, - "FOLLOW_PREFS", - 0, - "Follow Preferences", - "Use the import method set in the Preferences for this asset library, don't override it " - "for this Asset Browser"}, - {FILE_ASSET_IMPORT_LINK, "LINK", 0, "Link", "Import the assets as linked data-block"}, - {FILE_ASSET_IMPORT_APPEND, - "APPEND", - 0, - "Append", - "Import the assets as copied data-block, with no link to the original asset data-block"}, - {FILE_ASSET_IMPORT_APPEND_REUSE, - "APPEND_REUSE", - 0, - "Append (Reuse Data)", - "Import the assets as copied data-block while avoiding multiple copies of nested, " - "typically heavy data. For example the textures of a material asset, or the mesh of an " - "object asset, don't have to be copied every time this asset is imported. The instances of " - "the asset share the data instead"}, - {0, nullptr, 0, nullptr, nullptr}, - }; - srna = RNA_def_struct(brna, "FileAssetSelectParams", "FileSelectParams"); RNA_def_struct_ui_text( srna, "Asset Select Parameters", "Settings for the file selection in Asset Browser mode"); @@ -7478,7 +7519,9 @@ static void rna_def_fileselect_asset_params(BlenderRNA *brna) "Which asset types to show/hide, when browsing an asset library"); prop = RNA_def_property(srna, "import_method", PROP_ENUM, PROP_NONE); - RNA_def_property_enum_items(prop, asset_import_method_items); + RNA_def_property_enum_items(prop, rna_enum_asset_import_method_items); + RNA_def_property_enum_funcs( + prop, nullptr, nullptr, "rna_FileAssetSelectParams_import_method_itemf"); RNA_def_property_ui_text(prop, "Import Method", "Determine how the asset will be imported"); /* Asset drag info saved by buttons stores the import method, so the space must redraw when * import method changes. */ diff --git a/source/blender/makesrna/intern/rna_userdef.cc b/source/blender/makesrna/intern/rna_userdef.cc index 6f73994f7ec..de5100912a2 100644 --- a/source/blender/makesrna/intern/rna_userdef.cc +++ b/source/blender/makesrna/intern/rna_userdef.cc @@ -180,6 +180,30 @@ static const EnumPropertyItem rna_enum_preferences_extension_repo_source_type_it {0, nullptr, 0, nullptr, nullptr}, }; +static const EnumPropertyItem rna_enum_preferences_asset_import_method_items[] = { + {ASSET_IMPORT_LINK, "LINK", ICON_LINK_BLEND, "Link", "Import the assets as linked data-block"}, + {ASSET_IMPORT_APPEND, + "APPEND", + ICON_APPEND_BLEND, + "Append", + "Import the assets as copied data-block, with no link to the original asset data-block"}, + {ASSET_IMPORT_APPEND_REUSE, + "APPEND_REUSE", + ICON_APPEND_BLEND, + "Append (Reuse Data)", + "Import the assets as copied data-block while avoiding multiple copies of nested, " + "typically heavy data. For example the textures of a material asset, or the mesh of an " + "object asset, don't have to be copied every time this asset is imported. The instances of " + "the asset share the data instead."}, + {ASSET_IMPORT_PACK, + "PACK", + ICON_PACKAGE, + "Pack", + "Import the asset as linked data-block, and pack it in the current file (ensures that it " + "remains unchanged in case the library data is modified, is not available anymore, etc.)"}, + {0, nullptr, 0, nullptr, nullptr}, +}; + #ifdef RNA_RUNTIME # include "BLI_math_vector.h" @@ -223,6 +247,8 @@ static const EnumPropertyItem rna_enum_preferences_extension_repo_source_type_it # include "UI_interface.hh" +# include "AS_asset_library.hh" + static void rna_userdef_version_get(PointerRNA *ptr, int *value) { UserDef *userdef = (UserDef *)ptr->data; @@ -360,7 +386,7 @@ static void rna_userdef_asset_library_path_set(PointerRNA *ptr, const char *valu BKE_preferences_asset_library_path_set(library, value); } -static void rna_userdef_asset_library_path_update(bContext *C, PointerRNA *ptr) +static void rna_userdef_asset_library_update(bContext *C, PointerRNA *ptr) { blender::ed::asset::list::clear_all_library(C); rna_userdef_update(CTX_data_main(C), CTX_data_scene(C), ptr); @@ -1482,6 +1508,49 @@ static void rna_preference_gpu_preferred_device_set(PointerRNA *ptr, int value) preferences->gpu_preferred_device_id = 0u; } +static const EnumPropertyItem *rna_preference_asset_libray_import_method_itemf( + bContext * /*C*/, PointerRNA * /*ptr*/, PropertyRNA * /*prop*/, bool *r_free) +{ + EnumPropertyItem *items = nullptr; + int items_num = 0; + for (const EnumPropertyItem *item = rna_enum_preferences_asset_import_method_items; + item->identifier; + item++) + { + switch (eAssetImportMethod(item->value)) { + case ASSET_IMPORT_APPEND_REUSE: { + if (U.experimental.no_data_block_packing) { + RNA_enum_item_add(&items, &items_num, item); + } + break; + } + case ASSET_IMPORT_PACK: { + if (!U.experimental.no_data_block_packing) { + RNA_enum_item_add(&items, &items_num, item); + } + break; + } + default: { + RNA_enum_item_add(&items, &items_num, item); + break; + } + } + } + RNA_enum_item_end(&items, &items_num); + *r_free = true; + return items; +} + +static void rna_experimental_no_data_block_packing_update(bContext *C, PointerRNA *ptr) +{ + Main *bmain = CTX_data_main(C); + Scene *scene = CTX_data_scene(C); + rna_userdef_update(bmain, scene, ptr); + AS_asset_library_import_method_ensure_valid(*bmain); + AS_asset_library_essential_import_method_update(); + rna_userdef_asset_library_update(C, ptr); +} + #else # define USERDEF_TAG_DIRTY_PROPERTY_UPDATE_ENABLE \ @@ -6657,32 +6726,18 @@ static void rna_def_userdef_filepaths_asset_library(BlenderRNA *brna) RNA_def_property_translation_context(prop, BLT_I18NCONTEXT_EDITOR_FILEBROWSER); RNA_def_property_string_funcs(prop, nullptr, nullptr, "rna_userdef_asset_library_path_set"); RNA_def_property_flag(prop, PROP_CONTEXT_UPDATE); - RNA_def_property_update(prop, 0, "rna_userdef_asset_library_path_update"); + RNA_def_property_update(prop, 0, "rna_userdef_asset_library_update"); - static const EnumPropertyItem import_method_items[] = { - {ASSET_IMPORT_LINK, "LINK", 0, "Link", "Import the assets as linked data-block"}, - {ASSET_IMPORT_APPEND, - "APPEND", - 0, - "Append", - "Import the assets as copied data-block, with no link to the original asset data-block"}, - {ASSET_IMPORT_APPEND_REUSE, - "APPEND_REUSE", - 0, - "Append (Reuse Data)", - "Import the assets as copied data-block while avoiding multiple copies of nested, " - "typically heavy data. For example the textures of a material asset, or the mesh of an " - "object asset, don't have to be copied every time this asset is imported. The instances of " - "the asset share the data instead."}, - {0, nullptr, 0, nullptr, nullptr}, - }; prop = RNA_def_property(srna, "import_method", PROP_ENUM, PROP_NONE); - RNA_def_property_enum_items(prop, import_method_items); + RNA_def_property_enum_items(prop, rna_enum_preferences_asset_import_method_items); + RNA_def_property_enum_funcs( + prop, nullptr, nullptr, "rna_preference_asset_libray_import_method_itemf"); RNA_def_property_ui_text( prop, "Default Import Method", "Determine how the asset will be imported, unless overridden by the Asset Browser"); - RNA_def_property_update(prop, 0, "rna_userdef_update"); + RNA_def_property_flag(prop, PROP_CONTEXT_UPDATE); + RNA_def_property_update(prop, 0, "rna_userdef_asset_library_update"); prop = RNA_def_property(srna, "use_relative_path", PROP_BOOLEAN, PROP_NONE); RNA_def_property_boolean_sdna(prop, nullptr, "flag", ASSET_LIBRARY_RELATIVE_PATH); @@ -7308,6 +7363,13 @@ static void rna_def_userdef_experimental(BlenderRNA *brna) "Use file format used before Blender 5.0. This format is more limited " "but it may have better compatibility with tools that don't support the new format yet"); + prop = RNA_def_property(srna, "no_data_block_packing", PROP_BOOLEAN, PROP_NONE); + RNA_def_property_boolean_sdna(prop, nullptr, "no_data_block_packing", 1); + RNA_def_property_ui_text( + prop, "No Data-Block Packing", "Fall-back to appending instead of packing data-blocks"); + RNA_def_property_flag(prop, PROP_CONTEXT_UPDATE); + RNA_def_property_update(prop, 0, "rna_experimental_no_data_block_packing_update"); + prop = RNA_def_property(srna, "use_all_linked_data_direct", PROP_BOOLEAN, PROP_NONE); RNA_def_property_ui_text( prop, diff --git a/source/blender/python/intern/bpy_library_load.cc b/source/blender/python/intern/bpy_library_load.cc index dd05798f941..c5041e43f64 100644 --- a/source/blender/python/intern/bpy_library_load.cc +++ b/source/blender/python/intern/bpy_library_load.cc @@ -223,6 +223,9 @@ PyDoc_STRVAR( " :type filepath: str | bytes\n" " :arg link: When False reference to the original file is lost.\n" " :type link: bool\n" + " :arg pack: If True, and ``link`` is also True, pack linked data-blocks into the current " + "blend-file.\n" + " :type pack: bool\n" " :arg relative: When True the path is stored relative to the open blend file.\n" " :type relative: bool\n" " :arg set_fake: If True, set fake user on appended IDs.\n" @@ -260,6 +263,7 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k */ struct { BoolFlagPair is_link = {false, FILE_LINK}; + BoolFlagPair is_pack = {false, BLO_LIBLINK_PACK}; BoolFlagPair is_relative = {false, FILE_RELPATH}; BoolFlagPair set_fake = {false, BLO_LIBLINK_APPEND_SET_FAKEUSER}; BoolFlagPair recursive = {false, BLO_LIBLINK_APPEND_RECURSIVE}; @@ -279,6 +283,7 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k static const char *_keywords[] = { "filepath", "link", + "pack", "relative", "set_fake", "recursive", @@ -296,6 +301,7 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k /* Optional keyword only arguments. */ "|$" "O&" /* `link` */ + "O&" /* `pack` */ "O&" /* `relative` */ "O&" /* `recursive` */ "O&" /* `set_fake` */ @@ -317,6 +323,8 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k PyC_ParseBool, &flag_vars.is_link, PyC_ParseBool, + &flag_vars.is_pack, + PyC_ParseBool, &flag_vars.is_relative, PyC_ParseBool, &flag_vars.recursive, @@ -389,10 +397,18 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k PyErr_SetString(PyExc_ValueError, "`link` is False but `create_liboverrides` is True"); return nullptr; } + if (flag_vars.is_pack.value) { + PyErr_SetString(PyExc_ValueError, "`pack` must be False if `link` is False"); + return nullptr; + } } if (create_liboverrides) { /* Library overrides. */ + if (flag_vars.is_pack.value) { + PyErr_SetString(PyExc_ValueError, "`create_liboverrides` must be False if `pack` is True"); + return nullptr; + } } else { /* Library overrides (disabled). */ @@ -639,6 +655,7 @@ static bool bpy_lib_exit_lapp_context_items_cb(BlendfileLinkAppendContext *lapp_ static PyObject *bpy_lib_exit(BPy_Library *self, PyObject * /*args*/) { Main *bmain = self->bmain; + const bool do_pack = ((self->flag & BLO_LIBLINK_PACK) != 0); const bool do_append = ((self->flag & FILE_LINK) == 0); const bool create_liboverrides = self->create_liboverrides; /* Code in #bpy_lib_load should have raised exception in case of incompatible parameter values. @@ -709,7 +726,10 @@ static PyObject *bpy_lib_exit(BPy_Library *self, PyObject * /*args*/) BKE_blendfile_link_append_context_init_done(lapp_context); BKE_blendfile_link(lapp_context, nullptr); - if (do_append) { + if (do_pack) { + BKE_blendfile_link_pack(lapp_context, nullptr); + } + else if (do_append) { BKE_blendfile_append(lapp_context, nullptr); } else if (create_liboverrides) { diff --git a/source/blender/windowmanager/intern/wm_dragdrop.cc b/source/blender/windowmanager/intern/wm_dragdrop.cc index c2e9d48a43e..c57bbdc044c 100644 --- a/source/blender/windowmanager/intern/wm_dragdrop.cc +++ b/source/blender/windowmanager/intern/wm_dragdrop.cc @@ -753,6 +753,16 @@ ID *WM_drag_asset_id_import(const bContext *C, wmDragAsset *asset_drag, const in idtype, name, flag | (use_relative_path ? FILE_RELPATH : 0)); + case ASSET_IMPORT_PACK: + return WM_file_link_datablock(bmain, + scene, + view_layer, + view3d, + blend_path.c_str(), + idtype, + name, + flag | (use_relative_path ? FILE_RELPATH : 0) | + BLO_LIBLINK_PACK); case ASSET_IMPORT_APPEND: return WM_file_append_datablock(bmain, scene, @@ -787,7 +797,7 @@ bool WM_drag_asset_will_import_linked(const wmDrag *drag) } const wmDragAsset *asset_drag = WM_drag_get_asset_data(drag, 0); - return asset_drag->import_settings.method == ASSET_IMPORT_LINK; + return ELEM(asset_drag->import_settings.method, ASSET_IMPORT_LINK, ASSET_IMPORT_PACK); } ID *WM_drag_get_local_ID_or_import_from_asset(const bContext *C, const wmDrag *drag, int idcode) diff --git a/source/blender/windowmanager/intern/wm_files_link.cc b/source/blender/windowmanager/intern/wm_files_link.cc index 281c78421c3..28fac0f93be 100644 --- a/source/blender/windowmanager/intern/wm_files_link.cc +++ b/source/blender/windowmanager/intern/wm_files_link.cc @@ -725,6 +725,7 @@ static ID *wm_file_link_append_datablock_ex(Main *bmain, BLI_path_cmp(BKE_main_blendfile_path(bmain), filepath) != 0, "Calling code should ensure it does not attempt to link/append from current blendfile"); + const bool do_pack = (flag & BLO_LIBLINK_PACK) != 0; const bool do_append = (flag & FILE_LINK) == 0; /* Tag everything so we can make local only the new datablock. */ BKE_main_id_tag_all(bmain, ID_TAG_PRE_EXISTING, true); @@ -747,7 +748,10 @@ static ID *wm_file_link_append_datablock_ex(Main *bmain, /* Link datablock. */ BKE_blendfile_link(lapp_context, nullptr); - if (do_append) { + if (do_pack) { + BKE_blendfile_link_pack(lapp_context, nullptr); + } + else if (do_append) { BKE_blendfile_append(lapp_context, nullptr); } diff --git a/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1.blend b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1.blend new file mode 100644 index 00000000000..ba5cc3f86b4 --- /dev/null +++ b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1.blend @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edf4656bbac61e8b88c97c4928b922a53137f2a5ef1157db17d62237ce0c39b8 +size 587139 diff --git a/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1_with_hair.blend b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1_with_hair.blend new file mode 100644 index 00000000000..317d56bd240 --- /dev/null +++ b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1_with_hair.blend @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e68ba924578fa50c6ad09f256a1b8e369a87161ee972d298b6da58718ddeb5d +size 2946843 diff --git a/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2.blend b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2.blend new file mode 100644 index 00000000000..d3d47e10047 --- /dev/null +++ b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2.blend @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff82a19c1be5a1213b480106574ce510896a97f21834c4afae6854adc57ff60 +size 594473 diff --git a/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2_with_hair.blend b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2_with_hair.blend new file mode 100644 index 00000000000..e08e90ccff2 --- /dev/null +++ b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2_with_hair.blend @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1566e3e827e619f5c64644c103dc1904fd08d1fcea36044c8f665365e6025e5a +size 2912433 diff --git a/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/both_bodies_with_hair.blend b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/both_bodies_with_hair.blend new file mode 100644 index 00000000000..e63d75d2d7f --- /dev/null +++ b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/both_bodies_with_hair.blend @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b97107fbfbb68b2314e1555cf66d56e9c697214233804a14d05c21a6a39330d +size 3889258 diff --git a/tests/python/bl_blendfile_liblink.py b/tests/python/bl_blendfile_liblink.py index cd189c8adef..d0003b0fe25 100644 --- a/tests/python/bl_blendfile_liblink.py +++ b/tests/python/bl_blendfile_liblink.py @@ -486,6 +486,202 @@ class TestBlendLibAppendReuseID(TestBlendLibLinkHelper): self.assertEqual(len(bpy.data.collections), 0) # Scene's master collection is not listed here +class TestBlendLibPackedLinkedID(TestBlendLibLinkHelper): + + def __init__(self, args): + super().__init__(args) + + def test_link_pack_basic(self): + output_dir = self.args.output_dir + output_lib_path = self.init_lib_data_basic() + + # Link of a single Object, and make it packed. + self.reset_blender() + + link_dir = os.path.join(output_lib_path, "Object") + bpy.ops.wm.link(directory=link_dir, filename="LibMesh", instance_object_data=False) + + self.assertEqual(len(bpy.data.libraries), 1) + library = bpy.data.libraries[0] + + self.assertEqual(len(bpy.data.meshes), 1) + for me in bpy.data.meshes: + self.assertEqual(me.library, library) + self.assertEqual(me.users, 1) + self.assertEqual(len(bpy.data.objects), 1) + for ob in bpy.data.objects: + self.assertEqual(ob.library, library) + self.assertEqual(len(bpy.data.collections), 0) # Scene's master collection is not listed here + + ob_packed = bpy.data.pack_linked_ids_hierarchy(bpy.data.objects[0]) + + # Need to ensure that the newly packed linked object is used, and kept in the scene. + bpy.data.scenes[0].collection.objects.link(ob_packed) + + self.assertEqual(len(bpy.data.libraries), 2) + library = bpy.data.libraries[0] + archive_library = bpy.data.libraries[1] + + def check_valid(): + self.assertFalse(library.is_archive) + self.assertEqual(len(library.archive_libraries), 1) + self.assertEqual(library.archive_libraries[0], archive_library) + self.assertTrue(archive_library.is_archive) + self.assertEqual(archive_library.archive_parent_library, library) + + self.assertEqual(len(bpy.data.meshes), 2) + self.assertEqual(bpy.data.meshes[0].library, library) + self.assertEqual(bpy.data.meshes[0].users, 1) + self.assertEqual(bpy.data.meshes[1].library, archive_library) + self.assertEqual(bpy.data.meshes[1].users, 1) + + self.assertEqual(len(bpy.data.objects), 2) + self.assertEqual(bpy.data.objects[0].library, library) + self.assertEqual(bpy.data.objects[0].data, bpy.data.meshes[0]) + self.assertEqual(bpy.data.objects[1].library, archive_library) + self.assertEqual(bpy.data.objects[1].data, bpy.data.meshes[1]) + + check_valid() + + output_work_path = os.path.join(output_dir, self.unique_blendfile_name("blendfile")) + bpy.ops.wm.save_as_mainfile(filepath=output_work_path, check_existing=False, compress=False) + + self.reset_blender() + + bpy.ops.wm.open_mainfile(filepath=output_work_path, load_ui=False) + + self.assertEqual(len(bpy.data.libraries), 2) + library = bpy.data.libraries[0] + archive_library = bpy.data.libraries[1] + + check_valid() + + def test_link_pack_indirect(self): + # Test handling of indirectly linked packed data (when packed in another library), + # packing linked data using other packed linked data, etc. + output_dir = self.args.output_dir + output_lib_path = self.init_lib_data_packed_indirect_lib() + + # Link of a single Object, and make it packed. + self.reset_blender() + + link_dir = os.path.join(output_lib_path, "Object") + bpy.ops.wm.link(directory=link_dir, filename="LibMesh", instance_object_data=False) + + # Directly linked library, indirectly linked one (though empty), and its packed archive version. + self.assertEqual(len(bpy.data.libraries), 3) + library = bpy.data.libraries[0] + library_indirect = bpy.data.libraries[1] + library_indirect_archive = bpy.data.libraries[2] + + def check_valid(): + self.assertFalse(library.is_archive) + self.assertFalse(library_indirect.is_archive) + self.assertTrue(library_indirect_archive.is_archive) + self.assertTrue(library_indirect_archive.name in library_indirect.archive_libraries) + + self.assertEqual(len(bpy.data.images), 1) + for im in bpy.data.images: + self.assertEqual(im.library, library_indirect_archive) + self.assertEqual(im.users, 1) + self.assertTrue(im.is_linked_packed) + + self.assertEqual(len(bpy.data.materials), 1) + for ma in bpy.data.materials: + self.assertEqual(ma.library, library_indirect_archive) + self.assertEqual(ma.users, 1) + self.assertTrue(ma.is_linked_packed) + + self.assertEqual(len(bpy.data.meshes), 1) + for me in bpy.data.meshes: + self.assertEqual(me.library, library) + self.assertEqual(me.users, 1) + + self.assertEqual(len(bpy.data.objects), 1) + for ob in bpy.data.objects: + self.assertEqual(ob.library, library) + + self.assertEqual(len(bpy.data.collections), 0) # Scene's master collection is not listed here + + check_valid() + + output_work_path = os.path.join(output_dir, self.unique_blendfile_name("blendfile")) + bpy.ops.wm.save_as_mainfile(filepath=output_work_path, check_existing=False, compress=False) + + self.reset_blender() + + bpy.ops.wm.open_mainfile(filepath=output_work_path, load_ui=False) + + self.assertEqual(len(bpy.data.libraries), 3) + library = bpy.data.libraries[0] + library_indirect = bpy.data.libraries[1] + library_indirect_archive = bpy.data.libraries[2] + + check_valid() + + ob_packed = bpy.data.pack_linked_ids_hierarchy(bpy.data.objects[0]) + + # Need to ensure that the newly packed linked object is used, and kept in the scene. + bpy.data.scenes[0].collection.objects.link(ob_packed) + + self.assertEqual(len(bpy.data.libraries), 4) + # Due to ID name sorting, the newly ceratedt archive library should be second now, after its parent one. + archive_library = bpy.data.libraries[1] + + def check_valid(): + self.assertFalse(library.is_archive) + self.assertEqual(len(library.archive_libraries), 1) + self.assertEqual(library.archive_libraries[0], archive_library) + self.assertTrue(archive_library.is_archive) + self.assertEqual(archive_library.archive_parent_library, library) + + self.assertFalse(library_indirect.is_archive) + self.assertTrue(library_indirect_archive.is_archive) + self.assertTrue(library_indirect_archive.name in library_indirect.archive_libraries) + + self.assertEqual(len(bpy.data.images), 1) + for im in bpy.data.images: + self.assertEqual(im.library, library_indirect_archive) + self.assertEqual(im.users, 1) + self.assertTrue(im.is_linked_packed) + + self.assertEqual(len(bpy.data.materials), 1) + for ma in bpy.data.materials: + self.assertEqual(ma.library, library_indirect_archive) + self.assertEqual(ma.users, 2) + self.assertTrue(ma.is_linked_packed) + + self.assertEqual(len(bpy.data.meshes), 2) + self.assertEqual(bpy.data.meshes[0].library, library) + self.assertEqual(bpy.data.meshes[0].users, 1) + self.assertEqual(bpy.data.meshes[1].library, archive_library) + self.assertEqual(bpy.data.meshes[1].users, 1) + + self.assertEqual(len(bpy.data.objects), 2) + self.assertEqual(bpy.data.objects[0].library, library) + self.assertEqual(bpy.data.objects[0].data, bpy.data.meshes[0]) + self.assertEqual(bpy.data.objects[1].library, archive_library) + self.assertEqual(bpy.data.objects[1].data, bpy.data.meshes[1]) + + self.assertEqual(len(bpy.data.collections), 0) # Scene's master collection is not listed here + + check_valid() + + bpy.ops.wm.save_as_mainfile(filepath=output_work_path, check_existing=False, compress=False) + + self.reset_blender() + + bpy.ops.wm.open_mainfile(filepath=output_work_path, load_ui=False) + + self.assertEqual(len(bpy.data.libraries), 4) + library = bpy.data.libraries[0] + archive_library = bpy.data.libraries[1] + library_indirect = bpy.data.libraries[2] + library_indirect_archive = bpy.data.libraries[3] + + check_valid() + + class TestBlendLibLibraryReload(TestBlendLibLinkHelper): def __init__(self, args): @@ -610,6 +806,34 @@ class TestBlendLibDataLibrariesLoadLink(TestBlendLibDataLibrariesLoad): self.assertIsNotNone(bpy.data.collections[0].library) +class TestBlendLibDataLibrariesLoadPack(TestBlendLibDataLibrariesLoad): + + def test_libload_pack(self): + output_lib_path = self.do_libload_init() + # Cannot create overrides on packed linked data currently. + self.assertRaises(ValueError, + self.do_libload, filepath=output_lib_path, link=True, pack=True, create_liboverrides=True) + self.do_libload(filepath=output_lib_path, link=True, pack=True, create_liboverrides=False) + + self.assertEqual(len(bpy.data.meshes), 1) + self.assertEqual(len(bpy.data.objects), 1) # This code does no instantiation. + self.assertEqual(len(bpy.data.collections), 1) + # One archive library for the packed data-blocks and the reference library. + self.assertEqual(len(bpy.data.libraries), 2) + + # Packed dat should be owned by archive library. + packed_mesh = bpy.data.meshes[0] + packed_object = bpy.data.objects[0] + packed_collection = bpy.data.collections[0] + link_library = bpy.data.libraries[0] + archive_library = bpy.data.libraries[1] + self.assertEqual(packed_mesh.library, archive_library) + self.assertEqual(packed_object.library, archive_library) + self.assertEqual(packed_collection.library, archive_library) + self.assertTrue(archive_library.is_archive) + self.assertFalse(link_library.is_archive) + + class TestBlendLibDataLibrariesLoadLibOverride(TestBlendLibDataLibrariesLoad): def test_libload_liboverride(self): @@ -741,11 +965,14 @@ TESTS = ( TestBlendLibAppendBasic, TestBlendLibAppendReuseID, + TestBlendLibPackedLinkedID, + TestBlendLibLibraryReload, TestBlendLibLibraryRelocate, TestBlendLibDataLibrariesLoadAppend, TestBlendLibDataLibrariesLoadLink, + TestBlendLibDataLibrariesLoadPack, TestBlendLibDataLibrariesLoadLibOverride, ) diff --git a/tests/python/bl_blendfile_utils.py b/tests/python/bl_blendfile_utils.py index 65bccd54ec9..c369aef550e 100644 --- a/tests/python/bl_blendfile_utils.py +++ b/tests/python/bl_blendfile_utils.py @@ -182,3 +182,52 @@ class TestBlendLibLinkHelper(TestHelper): bpy.ops.wm.save_as_mainfile(filepath=output_lib_path, check_existing=False, compress=False) return output_lib_path + + def init_lib_data_packed_indirect_lib(self): + output_dir = self.args.output_dir + self.ensure_path(output_dir) + + # Create an indirect library containing a material, and an image texture. + self.reset_blender() + + self.gen_indirect_library_data_() + + # Take care to keep the name unique so multiple test jobs can run at once. + output_lib_path = os.path.join(output_dir, self.unique_blendfile_name("blendlib_indirect_material")) + + bpy.ops.wm.save_as_mainfile(filepath=output_lib_path, check_existing=False, compress=False) + + # Create a main library containing object etc., and linking material from indirect library. + self.reset_blender() + + self.gen_library_data_() + + link_dir = os.path.join(output_lib_path, "Material") + bpy.ops.wm.link(directory=link_dir, filename="LibMaterial") + + ma = bpy.data.pack_linked_ids_hierarchy(bpy.data.materials[0]) + + me = bpy.data.meshes[0] + me.materials.append(ma) + + bpy.ops.outliner.orphans_purge() + + self.assertEqual(len(bpy.data.materials), 1) + self.assertTrue(bpy.data.materials[0].is_linked_packed) + + self.assertEqual(len(bpy.data.images), 1) + self.assertTrue(bpy.data.images[0].is_linked_packed) + + self.assertEqual(len(bpy.data.libraries), 2) + self.assertFalse(bpy.data.libraries[0].is_archive) + self.assertTrue(bpy.data.libraries[1].is_archive) + self.assertIn(bpy.data.libraries[1].name, bpy.data.libraries[0].archive_libraries) + + output_dir = self.args.output_dir + self.ensure_path(output_dir) + # Take care to keep the name unique so multiple test jobs can run at once. + output_lib_path = os.path.join(output_dir, self.unique_blendfile_name("blendlib_indirect_main")) + + bpy.ops.wm.save_as_mainfile(filepath=output_lib_path, check_existing=False, compress=False) + + return output_lib_path