From 4e4976804e772ec697752fa2ecd2b84c66926c58 Mon Sep 17 00:00:00 2001 From: Jacques Lucke Date: Fri, 26 Sep 2025 10:53:40 +0200 Subject: [PATCH] Core: Add packed linked data-blocks This adds support for packed linked data. This is a key part of an improved asset workflow in Blender. Packed IDs remain considered as linked data (i.e. they cannot be edited), but they are stored in the current blendfile. This means that they: * Are not lost in case the library data becomes unavailable. * Are not changed in case the library data is updated. These packed IDs are de-duplicated across blend-files, so e.g. if a shot file and several of its dependencies all use the same util geometry node, there will be a single copy of that geometry node in the shot file. In case there are several versions of a same ID (e.g. linked at different moments from a same library, which has been modified in-between), there will be several packed IDs. Name collisions are averted by storing these packed IDs into a new type of 'archive' libraries (and their namespaces). These libraries: * Only contain packed IDs. * Are owned and managed by their 'real' library data-block, called an 'archive parent'. For more in-depth, technical design: #132167 UI/UX design: #140870 Co-authored-by: Bastien Montagne Pull Request: https://projects.blender.org/blender/blender/pulls/133801 --- .../bl_operators/object_quick_effects.py | 7 +- scripts/startup/bl_ui/space_userpref.py | 1 + .../blender/asset_system/AS_asset_library.hh | 12 + .../asset_system/AS_essentials_library.hh | 2 +- .../asset_system/intern/asset_library.cc | 64 ++ .../library_types/essentials_library.cc | 15 +- .../library_types/essentials_library.hh | 3 + .../blenkernel/BKE_blendfile_link_append.hh | 8 + source/blender/blenkernel/BKE_id_hash.hh | 49 ++ source/blender/blenkernel/BKE_lib_id.hh | 10 + source/blender/blenkernel/BKE_library.hh | 27 +- source/blender/blenkernel/CMakeLists.txt | 2 + .../intern/blendfile_link_append.cc | 45 ++ source/blender/blenkernel/intern/id_hash.cc | 250 +++++++ source/blender/blenkernel/intern/lib_id.cc | 8 + .../blenkernel/intern/lib_id_delete.cc | 4 + source/blender/blenkernel/intern/library.cc | 347 +++++++++- source/blender/blenkernel/intern/main.cc | 10 +- source/blender/blenloader/BLO_readfile.hh | 8 +- source/blender/blenloader/CMakeLists.txt | 1 + .../blenloader/intern/readblenentry.cc | 21 +- source/blender/blenloader/intern/readfile.cc | 637 ++++++++++++++---- source/blender/blenloader/intern/readfile.hh | 30 +- .../blenloader/intern/versioning_500.cc | 5 + source/blender/blenloader/intern/writefile.cc | 36 +- .../editors/asset/intern/asset_import.cc | 17 +- .../asset/intern/asset_shelf_asset_view.cc | 7 +- .../editors/interface/interface_icons.cc | 3 + .../editors/interface/interface_ops.cc | 4 + .../templates/interface_template_id.cc | 16 +- source/blender/editors/screen/screen_ops.cc | 2 + .../blender/editors/space_file/file_draw.cc | 2 +- source/blender/editors/space_file/filesel.cc | 2 + .../editors/space_outliner/outliner_draw.cc | 15 +- .../tree/tree_display_libraries.cc | 6 +- source/blender/makesdna/DNA_ID.h | 95 ++- source/blender/makesdna/DNA_asset_types.h | 2 + source/blender/makesdna/DNA_object_types.h | 1 - source/blender/makesdna/DNA_particle_types.h | 1 - source/blender/makesdna/DNA_space_enums.h | 5 + source/blender/makesdna/DNA_userdef_types.h | 3 +- source/blender/makesrna/intern/rna_ID.cc | 92 ++- .../blender/makesrna/intern/rna_main_api.cc | 36 +- source/blender/makesrna/intern/rna_space.cc | 93 ++- source/blender/makesrna/intern/rna_userdef.cc | 104 ++- .../blender/python/intern/bpy_library_load.cc | 22 +- .../windowmanager/intern/wm_dragdrop.cc | 12 +- .../windowmanager/intern/wm_files_link.cc | 6 +- .../two_objects_with_hair/body_1.blend | 3 + .../body_1_with_hair.blend | 3 + .../two_objects_with_hair/body_2.blend | 3 + .../body_2_with_hair.blend | 3 + .../both_bodies_with_hair.blend | 3 + tests/python/bl_blendfile_liblink.py | 227 +++++++ tests/python/bl_blendfile_utils.py | 49 ++ 55 files changed, 2225 insertions(+), 214 deletions(-) create mode 100644 source/blender/blenkernel/BKE_id_hash.hh create mode 100644 source/blender/blenkernel/intern/id_hash.cc create mode 100644 tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1.blend create mode 100644 tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1_with_hair.blend create mode 100644 tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2.blend create mode 100644 tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2_with_hair.blend create mode 100644 tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/both_bodies_with_hair.blend diff --git a/scripts/startup/bl_operators/object_quick_effects.py b/scripts/startup/bl_operators/object_quick_effects.py index 6cbe88d063b..6f05b080276 100644 --- a/scripts/startup/bl_operators/object_quick_effects.py +++ b/scripts/startup/bl_operators/object_quick_effects.py @@ -127,10 +127,9 @@ class QuickFur(ObjectModeOperator, Operator): with bpy.data.libraries.load( asset_library_filepath, - link=False, - clear_asset_data=True, - reuse_local_id=True, - recursive=True, + link=True, + pack=True, + set_fake=False, ) as (data_src, data_dst): # The values are assumed to exist, no inspection of the source is needed. del data_src diff --git a/scripts/startup/bl_ui/space_userpref.py b/scripts/startup/bl_ui/space_userpref.py index c2def125072..44107b2b892 100644 --- a/scripts/startup/bl_ui/space_userpref.py +++ b/scripts/startup/bl_ui/space_userpref.py @@ -2888,6 +2888,7 @@ class USERPREF_PT_developer_tools(Panel): ({"property": "use_viewport_debug"}, None), ({"property": "use_eevee_debug"}, None), ({"property": "use_extensions_debug"}, ("/blender/blender/issues/119521", "#119521")), + ({"property": "no_data_block_packing"}, ("/blender/blender/issues/132167", "#132167")), ), ) diff --git a/source/blender/asset_system/AS_asset_library.hh b/source/blender/asset_system/AS_asset_library.hh index df5342623c8..b0e72029f5b 100644 --- a/source/blender/asset_system/AS_asset_library.hh +++ b/source/blender/asset_system/AS_asset_library.hh @@ -303,3 +303,15 @@ void AS_asset_full_path_explode_from_weak_ref(const AssetWeakReference *asset_re char **r_dir, char **r_group, char **r_name); + +/** + * Updates the default import method for asset libraries based on + * #U.experimental.no_data_block_packing. + */ +void AS_asset_library_import_method_ensure_valid(Main &bmain); +/** + * This is not done as part of #AS_asset_library_import_method_ensure_valid because it changes + * run-time data only and does not need to happen during versioning (also it appears to break tests + * when run during versioning). + */ +void AS_asset_library_essential_import_method_update(); diff --git a/source/blender/asset_system/AS_essentials_library.hh b/source/blender/asset_system/AS_essentials_library.hh index 918440e20b1..91153fc8712 100644 --- a/source/blender/asset_system/AS_essentials_library.hh +++ b/source/blender/asset_system/AS_essentials_library.hh @@ -14,4 +14,4 @@ namespace blender::asset_system { StringRefNull essentials_directory_path(); -} +} // namespace blender::asset_system diff --git a/source/blender/asset_system/intern/asset_library.cc b/source/blender/asset_system/intern/asset_library.cc index 4486707a663..476a59e012b 100644 --- a/source/blender/asset_system/intern/asset_library.cc +++ b/source/blender/asset_system/intern/asset_library.cc @@ -21,11 +21,14 @@ #include "BLI_path_utils.hh" #include "BLI_string.h" +#include "DNA_space_types.h" #include "DNA_userdef_types.h" +#include "DNA_windowmanager_types.h" #include "asset_catalog_collection.hh" #include "asset_catalog_definition_file.hh" #include "asset_library_service.hh" +#include "essentials_library.hh" #include "runtime_library.hh" #include "utils.hh" @@ -161,6 +164,67 @@ void AS_asset_full_path_explode_from_weak_ref(const AssetWeakReference *asset_re } } +static void update_import_method_for_user_libraries() +{ + LISTBASE_FOREACH (bUserAssetLibrary *, library, &U.asset_libraries) { + if (U.experimental.no_data_block_packing) { + if (library->import_method == ASSET_IMPORT_PACK) { + library->import_method = ASSET_IMPORT_APPEND_REUSE; + } + } + else { + if (library->import_method == ASSET_IMPORT_APPEND_REUSE) { + library->import_method = ASSET_IMPORT_PACK; + } + } + } +} + +static void update_import_method_for_asset_browsers(Main &bmain) +{ + LISTBASE_FOREACH (bScreen *, screen, &bmain.screens) { + LISTBASE_FOREACH (ScrArea *, area, &screen->areabase) { + LISTBASE_FOREACH (SpaceLink *, sl, &area->spacedata) { + if (sl->spacetype != SPACE_FILE) { + continue; + } + SpaceFile *sfile = reinterpret_cast(sl); + if (!sfile->asset_params) { + continue; + } + if (U.experimental.no_data_block_packing) { + if (sfile->asset_params->import_method == FILE_ASSET_IMPORT_PACK) { + sfile->asset_params->import_method = FILE_ASSET_IMPORT_APPEND_REUSE; + } + } + else { + if (sfile->asset_params->import_method == FILE_ASSET_IMPORT_APPEND_REUSE) { + sfile->asset_params->import_method = FILE_ASSET_IMPORT_PACK; + } + } + } + } + } +} + +void AS_asset_library_import_method_ensure_valid(Main &bmain) +{ + update_import_method_for_user_libraries(); + update_import_method_for_asset_browsers(bmain); +} + +void AS_asset_library_essential_import_method_update() +{ + AssetLibraryReference library_ref{}; + library_ref.custom_library_index = -1; + library_ref.type = ASSET_LIBRARY_ESSENTIALS; + EssentialsAssetLibrary *library = dynamic_cast( + AS_asset_library_load(nullptr, library_ref)); + if (library) { + library->update_default_import_method(); + } +} + namespace blender::asset_system { AssetLibrary::AssetLibrary(eAssetLibraryType library_type, StringRef name, StringRef root_path) diff --git a/source/blender/asset_system/intern/library_types/essentials_library.cc b/source/blender/asset_system/intern/library_types/essentials_library.cc index 1309d7b1b82..1c589e94528 100644 --- a/source/blender/asset_system/intern/library_types/essentials_library.cc +++ b/source/blender/asset_system/intern/library_types/essentials_library.cc @@ -10,6 +10,8 @@ #include "utils.hh" +#include "DNA_userdef_types.h" + #include "AS_essentials_library.hh" #include "essentials_library.hh" @@ -20,7 +22,10 @@ EssentialsAssetLibrary::EssentialsAssetLibrary() {}, utils::normalize_directory_path(essentials_directory_path())) { - import_method_ = ASSET_IMPORT_APPEND_REUSE; + import_method_ = ASSET_IMPORT_PACK; + if (U.experimental.no_data_block_packing) { + import_method_ = ASSET_IMPORT_APPEND_REUSE; + } } std::optional EssentialsAssetLibrary::library_reference() const @@ -31,6 +36,14 @@ std::optional EssentialsAssetLibrary::library_reference() return library_ref; } +void EssentialsAssetLibrary::update_default_import_method() +{ + import_method_ = ASSET_IMPORT_PACK; + if (U.experimental.no_data_block_packing) { + import_method_ = ASSET_IMPORT_APPEND_REUSE; + } +} + StringRefNull essentials_directory_path() { static std::string path = []() { diff --git a/source/blender/asset_system/intern/library_types/essentials_library.hh b/source/blender/asset_system/intern/library_types/essentials_library.hh index 2da78739586..30e765c3a7e 100644 --- a/source/blender/asset_system/intern/library_types/essentials_library.hh +++ b/source/blender/asset_system/intern/library_types/essentials_library.hh @@ -17,6 +17,9 @@ class EssentialsAssetLibrary : public OnDiskAssetLibrary { EssentialsAssetLibrary(); std::optional library_reference() const override; + + /** Update the default import method based on whether packed data-blocks are supported. */ + void update_default_import_method(); }; } // namespace blender::asset_system diff --git a/source/blender/blenkernel/BKE_blendfile_link_append.hh b/source/blender/blenkernel/BKE_blendfile_link_append.hh index 595d6672fdc..a745cff3591 100644 --- a/source/blender/blenkernel/BKE_blendfile_link_append.hh +++ b/source/blender/blenkernel/BKE_blendfile_link_append.hh @@ -334,6 +334,14 @@ void BKE_blendfile_link_append_context_init_done(BlendfileLinkAppendContext *lap */ void BKE_blendfile_link(BlendfileLinkAppendContext *lapp_context, ReportList *reports); +/** + * Perform packing operation. + * + * The IDs processed by this functions are the one that have been linked by a previous call to + * #BKE_blendfile_link on the same `lapp_context`. + */ +void BKE_blendfile_link_pack(BlendfileLinkAppendContext *lapp_context, ReportList *reports); + /** * Perform append operation, using modern ID usage looper to detect which ID should be kept * linked, made local, duplicated as local, re-used from local etc. diff --git a/source/blender/blenkernel/BKE_id_hash.hh b/source/blender/blenkernel/BKE_id_hash.hh new file mode 100644 index 00000000000..c69e3b34900 --- /dev/null +++ b/source/blender/blenkernel/BKE_id_hash.hh @@ -0,0 +1,49 @@ +/* SPDX-FileCopyrightText: 2025 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include + +#include "BKE_main.hh" + +#include "BLI_map.hh" +#include "BLI_vector.hh" + +#include "DNA_ID.h" + +namespace blender::bke::id_hash { + +/** + * The hash of all the root IDs and their dependencies. + */ +struct ValidDeepHashes { + Map hashes; +}; + +struct DeepHashErrors { + /** + * A list of missing files paths in the case that the deep hashes could not be computed. + */ + VectorSet missing_files; + + /** + * Files that were modified since the linked ID was loaded. So the currently linked ID would not + * be matching the deep hash computed based on the source file. + */ + VectorSet updated_files; +}; + +using IDHashResult = std::variant; + +/** + * Compute a hash of the given IDs, including all their dependencies. + * This needs access to the original .blend files that the linked data-blocks come from to be able + * to compute their hash. + */ +IDHashResult compute_linked_id_deep_hashes(const Main &bmain, Span root_ids); + +/** Utility to convert the hash into a readable string. */ +std::string id_hash_to_hex(const IDHash &hash); + +} // namespace blender::bke::id_hash diff --git a/source/blender/blenkernel/BKE_lib_id.hh b/source/blender/blenkernel/BKE_lib_id.hh index 2b2079f8ea6..f7f4a6475d7 100644 --- a/source/blender/blenkernel/BKE_lib_id.hh +++ b/source/blender/blenkernel/BKE_lib_id.hh @@ -71,6 +71,11 @@ struct ID_Runtime_Remap { }; struct ID_Runtime { + /** + * The last modifification time of the source .blend file where this ID was loaded from. + */ + int64_t src_blend_modifification_time; + ID_Runtime_Remap remap = {}; /** * The depsgraph that owns this data block. This is only set on data-blocks which are @@ -249,6 +254,11 @@ enum { */ LIB_ID_COPY_SET_COPIED_ON_WRITE = 1 << 10, + /** + * Set #ID.newid pointer of the given source ID with the address of its new copy. + */ + LIB_ID_COPY_ID_NEW_SET = 1 << 11, + /* *** Specific options to some ID types or usages. *** */ /* *** May be ignored by unrelated ID copying functions. *** */ /** Object only, needed by make_local code. */ diff --git a/source/blender/blenkernel/BKE_library.hh b/source/blender/blenkernel/BKE_library.hh index 5452786da78..5895579356e 100644 --- a/source/blender/blenkernel/BKE_library.hh +++ b/source/blender/blenkernel/BKE_library.hh @@ -9,12 +9,15 @@ * API to manage `Library` data-blocks. */ +#include "DNA_ID.h" + +#include "BLI_map.hh" +#include "BLI_set.hh" #include "BLI_string_ref.hh" #include "BKE_main.hh" struct FileData; -struct Library; struct ListBase; struct Main; struct UniqueName_Map; @@ -27,6 +30,10 @@ struct LibraryRuntime { /** * Filedata (i.e. opened blendfile) source of this library data. + * + * \note: This is not always matching the library's blendfile path. E.g. for archive packed + * libraries, this will be the filedata of the packing blendfile, not of the reference/source + * library. */ FileData *filedata = nullptr; /** @@ -48,6 +55,12 @@ struct LibraryRuntime { /** Set for indirectly linked libraries, used in the outliner and while reading. */ Library *parent = nullptr; + /** + * Helper listing all archived libraries 'versions' of this library. + * Should only contain something if this library is a regular 'real' blendfile library. + */ + blender::Vector archived_libraries = {}; + /** #eLibrary_Tag. */ ushort tag = 0; @@ -67,6 +80,18 @@ struct LibraryRuntime { */ Library *search_filepath_abs(ListBase *libraries, blender::StringRef filepath_abs); +/** + * Pack given linked ID, and all the related hierarchy. + * + * Will set final embedded ID into each ID::newid pointers. + */ +void pack_linked_id_hierarchy(Main &bmain, ID &root_id); + +/** + * Cleanup references to removed/deleted archive libraries in their archive parent. + */ +void main_cleanup_parent_archives(Main &bmain); + }; // namespace blender::bke::library /** #LibraryRuntime.tag */ diff --git a/source/blender/blenkernel/CMakeLists.txt b/source/blender/blenkernel/CMakeLists.txt index e2d79497f56..8b86ae28d5c 100644 --- a/source/blender/blenkernel/CMakeLists.txt +++ b/source/blender/blenkernel/CMakeLists.txt @@ -147,6 +147,7 @@ set(SRC intern/grease_pencil_vertex_groups.cc intern/icons.cc intern/icons_rasterize.cc + intern/id_hash.cc intern/idprop.cc intern/idprop_create.cc intern/idprop_serialize.cc @@ -419,6 +420,7 @@ set(SRC BKE_grease_pencil_legacy_convert.hh BKE_grease_pencil_vertex_groups.hh BKE_icons.h + BKE_id_hash.hh BKE_idprop.hh BKE_idtype.hh BKE_image.hh diff --git a/source/blender/blenkernel/intern/blendfile_link_append.cc b/source/blender/blenkernel/intern/blendfile_link_append.cc index 85a00671645..2e6c3542a64 100644 --- a/source/blender/blenkernel/intern/blendfile_link_append.cc +++ b/source/blender/blenkernel/intern/blendfile_link_append.cc @@ -949,6 +949,51 @@ static bool foreach_libblock_link_append_common_processing( /** \} */ +/** \name Library embedding code. + * \{ */ + +void BKE_blendfile_link_pack(BlendfileLinkAppendContext *lapp_context, ReportList * /*reports*/) +{ + Main *bmain = lapp_context->params->bmain; + + /* Delete newly linked data-blocks after they have been packed. */ + blender::Vector linked_ids_to_delete; + { + ID *id; + FOREACH_MAIN_ID_BEGIN (bmain, id) { + if (ID_IS_LINKED(id) && !ID_IS_PACKED(id)) { + if (!(id->tag & ID_TAG_PRE_EXISTING)) { + linked_ids_to_delete.append(id); + } + } + } + FOREACH_MAIN_ID_END; + } + + for (BlendfileLinkAppendContextItem &item : lapp_context->items) { + ID *id = item.new_id; + BLI_assert(ID_IS_LINKED(id)); + if (!(ID_IS_PACKED(id) || (id->newid && ID_IS_PACKED(id->newid)))) { + /* No yet packed. */ + blender::bke::library::pack_linked_id_hierarchy(*bmain, *id); + } + /* Calling code may want to access newly packed embedded IDs from the link/append context + * items. */ + if (id->newid) { + item.new_id = id->newid; + } + } + BKE_main_id_newptr_and_tag_clear(bmain); + + BKE_main_id_tag_all(bmain, ID_TAG_DOIT, false); + for (ID *id : linked_ids_to_delete) { + id->tag |= ID_TAG_DOIT; + } + BKE_id_multi_tagged_delete(bmain); +} + +/** \} */ + /** \name Library append code. * \{ */ diff --git a/source/blender/blenkernel/intern/id_hash.cc b/source/blender/blenkernel/intern/id_hash.cc new file mode 100644 index 00000000000..22e75646bc7 --- /dev/null +++ b/source/blender/blenkernel/intern/id_hash.cc @@ -0,0 +1,250 @@ +/* SPDX-FileCopyrightText: 2025 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include +#include +#include + +#include "BKE_id_hash.hh" +#include "BKE_lib_id.hh" +#include "BKE_lib_query.hh" +#include "BKE_library.hh" +#include "BKE_main.hh" + +#include "BLI_fileops.hh" +#include "BLI_mmap.h" +#include "BLI_mutex.hh" +#include "BLI_set.hh" + +namespace blender::bke::id_hash { + +static std::optional> read_file(const StringRefNull path) +{ + blender::fstream stream{path.c_str(), std::ios_base::in | std::ios_base::binary}; + stream.seekg(0, std::ios_base::end); + const int64_t size = stream.tellg(); + stream.seekg(0, std::ios_base::beg); + + blender::Vector buffer(size); + stream.read(buffer.data(), size); + if (stream.bad()) { + return std::nullopt; + } + + return buffer; +} + +static std::optional compute_file_hash_with_file_read(const StringRefNull path) +{ + const std::optional> buffer = read_file(path); + if (!buffer) { + return std::nullopt; + } + return XXH3_128bits(buffer->data(), buffer->size()); +} + +static std::optional compute_file_hash_with_memory_map(const StringRefNull path) +{ + const int file = BLI_open(path.c_str(), O_BINARY | O_RDONLY, 0); + if (file == -1) { + return std::nullopt; + } + BLI_mmap_file *mmap_file = BLI_mmap_open(file); + if (!mmap_file) { + return std::nullopt; + } + BLI_SCOPED_DEFER([&]() { BLI_mmap_free(mmap_file); }); + const size_t size = BLI_mmap_get_length(mmap_file); + const void *data = BLI_mmap_get_pointer(mmap_file); + const XXH128_hash_t hash = XXH3_128bits(data, size); + if (BLI_mmap_any_io_error(mmap_file)) { + return std::nullopt; + } + return hash; +} + +static std::optional compute_file_hash(const StringRefNull path) +{ + /* First try the memory map the file, because it avoids an extra copy. */ + if (const std::optional hash = compute_file_hash_with_memory_map(path)) { + /* Make sure both code paths are tested even if memory mapping should almost always work. */ + BLI_assert(hash->low64 == compute_file_hash_with_file_read(path)->low64); + return hash; + } + if (const std::optional hash = compute_file_hash_with_file_read(path)) { + return hash; + } + return std::nullopt; +} + +struct CachedFileHash { + int64_t last_modified = 0; + XXH128_hash_t hash; +}; + +static std::optional get_source_file_hash(const ID &id, DeepHashErrors &r_errors) +{ + static Map cache; + static Mutex mutex; + + const StringRefNull path = id.lib->runtime->filepath_abs; + + BLI_stat_t stat; + if (BLI_stat(path.c_str(), &stat) == -1) { + r_errors.missing_files.add_as(path); + return std::nullopt; + } + + std::lock_guard lock(mutex); + if (const CachedFileHash *cached_hash = cache.lookup_ptr_as(path)) { + if (cached_hash->last_modified == stat.st_mtime) { + return cached_hash->hash; + } + } + + if (stat.st_mtime != id.runtime->src_blend_modifification_time) { + r_errors.updated_files.add_as(path); + return std::nullopt; + } + + if (const std::optional hash = compute_file_hash(path)) { + cache.add_overwrite(path, CachedFileHash{stat.st_mtime, *hash}); + return hash; + } + r_errors.missing_files.add_as(path); + return std::nullopt; +} + +static std::optional get_id_shallow_hash(const ID &id, DeepHashErrors &r_errors) +{ + BLI_assert(ID_IS_LINKED(&id)); + const StringRefNull id_name = id.name; + const std::optional file_hash = get_source_file_hash(id, r_errors); + if (!file_hash) { + return std::nullopt; + } + + XXH3_state_t *hash_state = XXH3_createState(); + XXH3_128bits_reset(hash_state); + XXH3_128bits_update(hash_state, id_name.data(), id_name.size()); + XXH3_128bits_update(hash_state, &*file_hash, sizeof(XXH128_hash_t)); + XXH128_hash_t shallow_hash = XXH3_128bits_digest(hash_state); + XXH3_freeState(hash_state); + return shallow_hash; +} + +static void compute_deep_hash_recursive(const Main &bmain, + const ID &id, + Set ¤t_stack, + Map &r_hashes, + DeepHashErrors &r_errors) +{ + if (r_hashes.contains(&id)) { + return; + } + if (!id.deep_hash.is_null()) { + r_hashes.add(&id, id.deep_hash); + return; + } + current_stack.add(&id); + const std::optional id_shallow_hash = get_id_shallow_hash(id, r_errors); + if (!id_shallow_hash) { + return; + } + + XXH3_state_t *hash_state = XXH3_createState(); + XXH3_128bits_reset(hash_state); + XXH3_128bits_update(hash_state, &*id_shallow_hash, sizeof(XXH128_hash_t)); + + bool success = true; + BKE_library_foreach_ID_link( + const_cast
(&bmain), + const_cast(&id), + [&](LibraryIDLinkCallbackData *cb_data) { + if (cb_data->cb_flag & IDWALK_CB_LOOPBACK) { + /* Loopback pointer (e.g. from a shapekey to its owner geometry ID, or from a collection + * to its parents) should always be ignored, as they do not represent an actual + * dependency. The dependency relationship should already have been processed from the + * owner to its dependency anyway (if applicable). */ + return IDWALK_RET_NOP; + } + if (cb_data->cb_flag & (IDWALK_CB_EMBEDDED | IDWALK_CB_EMBEDDED_NOT_OWNING)) { + /* Embedded data are part of their owner's internal data, and as such already computed as + * part of the owner's shallow hash. */ + return IDWALK_RET_NOP; + } + ID *referenced_id = *cb_data->id_pointer; + if (!referenced_id) { + /* Need to update the hash even if there is no id. There is a difference between the case + * where there is no id and the case where this callback is not called at all.*/ + const int random_data = 452942579; + XXH3_128bits_update(hash_state, &random_data, sizeof(int)); + return IDWALK_RET_NOP; + } + /* All embedded ID usages should already have been excluded above. */ + BLI_assert((referenced_id->flag & ID_FLAG_EMBEDDED_DATA) == 0); + if (current_stack.contains(referenced_id)) { + /* Somehow encode that we had a circular reference here. */ + const int random_data = 234632342; + XXH3_128bits_update(hash_state, &random_data, sizeof(int)); + return IDWALK_RET_NOP; + } + compute_deep_hash_recursive(bmain, *referenced_id, current_stack, r_hashes, r_errors); + const IDHash *referenced_id_hash = r_hashes.lookup_ptr(referenced_id); + if (!referenced_id_hash) { + success = false; + return IDWALK_RET_STOP_ITER; + } + XXH3_128bits_update(hash_state, referenced_id_hash->data, sizeof(IDHash)); + return IDWALK_RET_NOP; + }, + nullptr, + IDWALK_READONLY); + + if (!success) { + return; + } + IDHash new_deep_hash; + const XXH128_hash_t new_deep_hash_xxh128 = XXH3_128bits_digest(hash_state); + XXH3_freeState(hash_state); + static_assert(sizeof(IDHash) == sizeof(XXH128_hash_t)); + memcpy(new_deep_hash.data, &new_deep_hash_xxh128, sizeof(IDHash)); + r_hashes.add(&id, new_deep_hash); +} + +IDHashResult compute_linked_id_deep_hashes(const Main &bmain, Span ids) +{ +#ifndef NDEBUG + for (const ID *id : ids) { + BLI_assert(ID_IS_LINKED(id)); + } +#endif + + if (ids.is_empty()) { + return ValidDeepHashes{}; + } + + Map hashes; + Set current_stack; + DeepHashErrors errors; + for (const ID *id : ids) { + compute_deep_hash_recursive(bmain, *id, current_stack, hashes, errors); + } + if (!errors.missing_files.is_empty() || !errors.updated_files.is_empty()) { + return errors; + } + return ValidDeepHashes{hashes}; +} + +std::string id_hash_to_hex(const IDHash &hash) +{ + std::string hex_str; + for (const uint8_t byte : hash.data) { + hex_str += fmt::format("{:02x}", byte); + } + return hex_str; +} + +} // namespace blender::bke::id_hash diff --git a/source/blender/blenkernel/intern/lib_id.cc b/source/blender/blenkernel/intern/lib_id.cc index fc3bc9c10b2..7297dfd2413 100644 --- a/source/blender/blenkernel/intern/lib_id.cc +++ b/source/blender/blenkernel/intern/lib_id.cc @@ -1667,6 +1667,14 @@ void BKE_libblock_copy_in_lib(Main *bmain, DEG_id_type_tag(bmain, GS(new_id->name)); } + if (owner_library && *owner_library && ((*owner_library)->flag & LIBRARY_FLAG_IS_ARCHIVE) != 0) { + new_id->flag |= ID_FLAG_LINKED_AND_PACKED; + } + + if (flag & LIB_ID_COPY_ID_NEW_SET) { + ID_NEW_SET(id, new_id); + } + *new_id_p = new_id; } diff --git a/source/blender/blenkernel/intern/lib_id_delete.cc b/source/blender/blenkernel/intern/lib_id_delete.cc index 72e3828ba48..08c62b64e32 100644 --- a/source/blender/blenkernel/intern/lib_id_delete.cc +++ b/source/blender/blenkernel/intern/lib_id_delete.cc @@ -339,6 +339,10 @@ static size_t id_delete(Main *bmain, id_remapper.clear(); } + /* Remapping above may have left some Library::runtime::archived_libraries items to nullptr, + * clean this up and shrink the vector accordingly. */ + blender::bke::library::main_cleanup_parent_archives(*bmain); + /* Since we removed IDs from Main, their own other IDs usages need to be removed 'manually'. */ blender::Vector cleanup_ids{ids_to_delete.begin(), ids_to_delete.end()}; BKE_libblock_relink_multiple( diff --git a/source/blender/blenkernel/intern/library.cc b/source/blender/blenkernel/intern/library.cc index 34010597448..698d93d5dc8 100644 --- a/source/blender/blenkernel/intern/library.cc +++ b/source/blender/blenkernel/intern/library.cc @@ -14,6 +14,8 @@ /* all types are needed here, in order to do memory operations */ #include "DNA_ID.h" +#include "DNA_collection_types.h" +#include "DNA_scene_types.h" #include "BLI_utildefines.h" @@ -22,24 +24,32 @@ #include "BLI_path_utils.hh" #include "BLI_set.hh" #include "BLI_string.h" +#include "BLI_vector_set.hh" #include "BLT_translation.hh" #include "BLO_read_write.hh" #include "BKE_bpath.hh" +#include "BKE_id_hash.hh" #include "BKE_idtype.hh" +#include "BKE_key.hh" #include "BKE_lib_id.hh" #include "BKE_lib_query.hh" +#include "BKE_lib_remap.hh" #include "BKE_library.hh" #include "BKE_main.hh" +#include "BKE_main_invariants.hh" #include "BKE_main_namemap.hh" +#include "BKE_node.hh" #include "BKE_packedFile.hh" +#include "BKE_report.hh" struct BlendDataReader; static CLG_LogRef LOG = {"lib.library"}; +using namespace blender::bke; using namespace blender::bke::library; static void library_runtime_reset(Library *lib) @@ -93,7 +103,37 @@ static void library_copy_data(Main *bmain, static void library_foreach_id(ID *id, LibraryForeachIDData *data) { Library *lib = (Library *)id; + const LibraryForeachIDFlag foreach_flag = BKE_lib_query_foreachid_process_flags_get(data); BKE_LIB_FOREACHID_PROCESS_IDSUPER(data, lib->runtime->parent, IDWALK_CB_NEVER_SELF); + + if (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + /* Archive library must have a parent, this can't be nullptr. */ + if (lib->archive_parent_library) { + BKE_LIB_FOREACHID_PROCESS_ID( + data, lib->archive_parent_library, IDWALK_CB_NEVER_SELF | IDWALK_CB_NEVER_NULL); + } + + /* Archive libraries should never 'own' other archives. */ + BLI_assert(lib->runtime->archived_libraries.is_empty()); + if (foreach_flag & IDWALK_DO_INTERNAL_RUNTIME_POINTERS) { + for (Library *&lib_p : lib->runtime->archived_libraries) { + BKE_LIB_FOREACHID_PROCESS_ID( + data, lib_p, IDWALK_CB_NEVER_SELF | IDWALK_CB_INTERNAL | IDWALK_CB_LOOPBACK); + } + } + } + else { + /* Regular libraries should never have an archive parent. */ + BLI_assert(!lib->archive_parent_library); + BKE_LIB_FOREACHID_PROCESS_ID(data, lib->archive_parent_library, IDWALK_CB_NEVER_SELF); + + if (foreach_flag & IDWALK_DO_INTERNAL_RUNTIME_POINTERS) { + for (Library *&lib_p : lib->runtime->archived_libraries) { + BKE_LIB_FOREACHID_PROCESS_ID( + data, lib_p, IDWALK_CB_NEVER_SELF | IDWALK_CB_INTERNAL | IDWALK_CB_LOOPBACK); + } + } + } } static void library_foreach_path(ID *id, BPathForeachPathData *bpath_data) @@ -139,6 +179,15 @@ static void library_blend_read_data(BlendDataReader * /*reader*/, ID *id) lib->runtime = MEM_new(__func__); } +static void library_blend_read_after_liblink(BlendLibReader * /*reader*/, ID *id) +{ + Library *lib = reinterpret_cast(id); + if (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + BLI_assert(lib->archive_parent_library); + lib->archive_parent_library->runtime->archived_libraries.append(lib); + } +} + IDTypeInfo IDType_ID_LI = { /*id_code*/ Library::id_type, /*id_filter*/ FILTER_ID_LI, @@ -163,7 +212,7 @@ IDTypeInfo IDType_ID_LI = { /*blend_write*/ library_blend_write_data, /*blend_read_data*/ library_blend_read_data, - /*blend_read_after_liblink*/ nullptr, + /*blend_read_after_liblink*/ library_blend_read_after_liblink, /*blend_read_undo_preserve*/ nullptr, @@ -368,3 +417,299 @@ Library *blender::bke::library::search_filepath_abs(ListBase *libraries, } return nullptr; } + +/** + * Add a new 'archive' copy of the given reference library. It is used to store linked packed IDs. + */ +static Library *add_archive_library(Main &bmain, Library &reference_library) +{ + BLI_assert((reference_library.flag & LIBRARY_FLAG_IS_ARCHIVE) == 0); + /* Cannot copy libraries using generic ID copying functions, so create the copy manually. */ + Library *archive_library = static_cast( + BKE_id_new(&bmain, ID_LI, BKE_id_name(reference_library.id))); + + /* Like in #direct_link_library. */ + id_us_ensure_real(&archive_library->id); + + archive_library->archive_parent_library = &reference_library; + constexpr uint16_t copy_flag = ~LIBRARY_FLAG_IS_ARCHIVE; + archive_library->flag = (reference_library.flag & copy_flag) | LIBRARY_FLAG_IS_ARCHIVE; + BKE_library_filepath_set(&bmain, archive_library, reference_library.filepath); + + archive_library->runtime->parent = reference_library.runtime->parent; + /* Only copy a subset of the reference library tags. E.g. an archive library should never be + * considered as writable, so never copy #LIBRARY_ASSET_FILE_WRITABLE. This may need further + * tweaking still. */ + constexpr uint16_t copy_tag = (LIBRARY_TAG_RESYNC_REQUIRED | LIBRARY_ASSET_EDITABLE | + LIBRARY_IS_ASSET_EDIT_FILE); + archive_library->runtime->tag = reference_library.runtime->tag & copy_tag; + /* By definition, the file version of an archive library containing only packed linked data is + * the same as the one of its Main container. */ + archive_library->runtime->versionfile = bmain.versionfile; + archive_library->runtime->subversionfile = bmain.subversionfile; + + reference_library.runtime->archived_libraries.append(archive_library); + + return archive_library; +} + +static Library *get_archive_library(Main &bmain, ID *for_id, const IDHash &for_id_deep_hash) +{ + Library *reference_library = for_id->lib; + BLI_assert(reference_library && (reference_library->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0); + + Library *archive_library = nullptr; + for (Library *lib_iter : reference_library->runtime->archived_libraries) { + BLI_assert((lib_iter->flag & LIBRARY_FLAG_IS_ARCHIVE) != 0); + BLI_assert(lib_iter->archive_parent_library != nullptr); + BLI_assert(lib_iter->archive_parent_library == reference_library); + /* Check if current archive library already contains an ID of same type and name. */ + if (BKE_main_namemap_contain_name(bmain, lib_iter, GS(for_id->name), BKE_id_name(*for_id))) { +#ifndef NDEBUG + ID *packed_id = BKE_libblock_find_name_and_library( + &bmain, GS(for_id->name), BKE_id_name(*for_id), BKE_id_name(lib_iter->id)); + BLI_assert_msg( + packed_id && packed_id->deep_hash != for_id_deep_hash, + "An already packed ID with same deep hash as the one to be packed, should have already " + "be found and used (deduplication) before reaching this codepath"); +#endif + UNUSED_VARS_NDEBUG(for_id_deep_hash); + continue; + } + archive_library = lib_iter; + break; + } + if (!archive_library) { + archive_library = add_archive_library(bmain, *reference_library); + } + BLI_assert(reference_library->runtime->archived_libraries.contains(archive_library)); + return archive_library; +} + +static void pack_linked_id(Main &bmain, + ID *linked_id, + const id_hash::ValidDeepHashes &deep_hashes, + blender::Map &already_packed_ids, + blender::VectorSet &ids_to_remap, + blender::bke::id::IDRemapper &id_remapper) +{ + BLI_assert(linked_id->newid == nullptr); + + const IDHash linked_id_deep_hash = deep_hashes.hashes.lookup(linked_id); + ID *packed_id = already_packed_ids.lookup_default(linked_id_deep_hash, nullptr); + + if (packed_id) { + /* Exact same ID (and all of its dependencies) have already been linked and packed before, + * re-use these packed data. */ + + auto existing_id_process = [&deep_hashes, &id_remapper](ID *linked_id, ID *packed_id) { + BLI_assert(packed_id); + BLI_assert(ID_IS_PACKED(packed_id)); + /* Note: linked_id and packed_id may have the same deep hash while still coming from + * different original libraries. This easily happens copying an asset file such that each + * asset exists twice. */ + BLI_assert(packed_id->deep_hash == deep_hashes.hashes.lookup(linked_id)); + UNUSED_VARS_NDEBUG(deep_hashes); + + id_remapper.add(linked_id, packed_id); + linked_id->newid = packed_id; + /* No need to remap this packed ID - otherwise there would be something very wrong in + * packed IDs state. */ + }; + + existing_id_process(linked_id, packed_id); + + /* Handle 'fake-embedded' ShapeKeys IDs. */ + Key *linked_key = BKE_key_from_id(linked_id); + if (linked_key) { + Key *packed_key = BKE_key_from_id(packed_id); + BLI_assert(packed_key); + existing_id_process(&linked_key->id, &packed_key->id); + } + } + else { + /* This exact version of the ID and its dependencies have not been packed before, creates a + * new copy of it and pack it. */ + + /* Find an existing archive Library not containing a 'version' of this ID yet (to prevent names + * collisions). */ + Library *archive_lib = get_archive_library(bmain, linked_id, linked_id_deep_hash); + + auto copied_id_process = + [&archive_lib, &deep_hashes, &ids_to_remap, &id_remapper, &already_packed_ids]( + ID *linked_id, ID *packed_id) { + BLI_assert(packed_id); + BLI_assert(ID_IS_PACKED(packed_id)); + BLI_assert(packed_id->lib == archive_lib); + UNUSED_VARS_NDEBUG(archive_lib); + + packed_id->deep_hash = deep_hashes.hashes.lookup(linked_id); + id_remapper.add(linked_id, packed_id); + ids_to_remap.add(packed_id); + already_packed_ids.add(packed_id->deep_hash, packed_id); + }; + + packed_id = BKE_id_copy_in_lib(&bmain, + archive_lib, + linked_id, + std::nullopt, + nullptr, + LIB_ID_COPY_DEFAULT | LIB_ID_COPY_ID_NEW_SET | + LIB_ID_COPY_NO_ANIMDATA); + id_us_min(packed_id); + copied_id_process(linked_id, packed_id); + + /* Handle 'fake-embedded' ShapeKeys IDs. */ + Key *linked_key = BKE_key_from_id(linked_id); + if (linked_key) { + Key *embedded_key = BKE_key_from_id(packed_id); + BLI_assert(embedded_key); + copied_id_process(&linked_key->id, &embedded_key->id); + } + } +} + +/** + * Pack given linked IDs. Low-level code, assumes all given IDs are valid and safe to pack. + * + * Will set final packed ID into each ID::newid pointers. + */ +static void pack_linked_ids(Main &bmain, const blender::Set &ids_to_pack) +{ + blender::VectorSet final_ids_to_pack; + blender::VectorSet ids_to_remap; + blender::bke::id::IDRemapper id_remapper; + + for (ID *id : ids_to_pack) { + BLI_assert(ID_IS_LINKED(id)); + if (ID_IS_PACKED(id)) { + /* Should not happen, but also not critical issue. */ + CLOG_ERROR(&LOG, + "Trying to pack an already packed ID '%s' (from '%s')", + id->name, + id->lib->runtime->filepath_abs); + /* Already packed. */ + continue; + } + final_ids_to_pack.add(id); + } + + const id_hash::IDHashResult hash_result = id_hash::compute_linked_id_deep_hashes( + bmain, final_ids_to_pack.as_span()); + if (const auto *errors = std::get_if(&hash_result)) { + if (!errors->missing_files.is_empty()) { + CLOG_ERROR(&LOG, + "Trying to pack IDs that depend on missing linked libraries: %s", + errors->missing_files[0].c_str()); + } + if (!errors->updated_files.is_empty()) { + CLOG_ERROR(&LOG, + "Trying to pack linked ID that has been modified on disk: %s", + errors->updated_files[0].c_str()); + } + return; + } + const auto &deep_hashes = std::get(hash_result); + + blender::Map already_packed_ids; + { + ID *id; + FOREACH_MAIN_ID_BEGIN (&bmain, id) { + if (ID_IS_PACKED(id)) { + already_packed_ids.add(id->deep_hash, id); + } + } + FOREACH_MAIN_ID_END; + } + + for (ID *linked_id : final_ids_to_pack) { + pack_linked_id(bmain, linked_id, deep_hashes, already_packed_ids, ids_to_remap, id_remapper); + } + + BKE_libblock_relink_multiple( + &bmain, ids_to_remap.as_span(), ID_REMAP_TYPE_REMAP, id_remapper, 0); + BKE_main_ensure_invariants(bmain); +} + +void blender::bke::library::pack_linked_id_hierarchy(Main &bmain, ID &root_id) +{ + BLI_assert(ID_IS_LINKED(&root_id)); + BLI_assert(!ID_IS_PACKED(&root_id)); + + blender::Set ids_to_pack; + ids_to_pack.add(&root_id); + BKE_library_foreach_ID_link( + &bmain, + &root_id, + [&ids_to_pack](LibraryIDLinkCallbackData *cb_data) -> int { + if (cb_data->cb_flag & IDWALK_CB_LOOPBACK) { + return IDWALK_RET_NOP; + } + if (cb_data->cb_flag & (IDWALK_CB_EMBEDDED | IDWALK_CB_EMBEDDED_NOT_OWNING)) { + return IDWALK_RET_NOP; + } + + ID *self_id = cb_data->self_id; + ID *referenced_id = *cb_data->id_pointer; + if (!referenced_id) { + return IDWALK_RET_NOP; + } + if (!ID_IS_LINKED(referenced_id)) { + CLOG_ERROR(&LOG, "Linked data-block references non-linked data-block"); + return IDWALK_RET_NOP; + } + if (ID_IS_PACKED(referenced_id)) { + /* A linked ID can use another packed linked ID, as long as it is not from the same + * library. */ + BLI_assert(referenced_id->lib && referenced_id->lib->archive_parent_library); + if (referenced_id->lib->archive_parent_library == self_id->lib) { + CLOG_ERROR(&LOG, + "Non-packed data-block references packed data-block from the same library, " + "which is not allowed"); + } + return IDWALK_RET_NOP; + } + if (referenced_id->newid && ID_IS_PACKED(referenced_id->newid)) { + return IDWALK_RET_NOP; + } + if (GS(referenced_id->name) == ID_KE) { + /* Shape keys cannot be directly linked, from linking code PoV they behave as embedded + * data (i.e. their owning data is responsible to handle them). */ + return IDWALK_RET_NOP; + } + + ids_to_pack.add(referenced_id); + return IDWALK_RET_NOP; + }, + nullptr, + IDWALK_READONLY | IDWALK_RECURSE); + + pack_linked_ids(bmain, ids_to_pack); +} + +void blender::bke::library::main_cleanup_parent_archives(Main &bmain) +{ + LISTBASE_FOREACH (Library *, lib, &bmain.libraries) { + if (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + BLI_assert(!lib->runtime || lib->runtime->archived_libraries.is_empty()); + } + else { + int i_read_curr = 0; + int i_insert_curr = 0; + for (; i_read_curr < lib->runtime->archived_libraries.size(); i_read_curr++) { + if (!lib->runtime->archived_libraries[i_read_curr]) { + continue; + } + if (i_insert_curr < i_read_curr) { + lib->runtime->archived_libraries[i_insert_curr] = + lib->runtime->archived_libraries[i_read_curr]; + } + i_insert_curr++; + } + BLI_assert(i_insert_curr <= i_read_curr); + if (i_insert_curr < i_read_curr) { + lib->runtime->archived_libraries.resize(i_insert_curr); + } + } + } +} diff --git a/source/blender/blenkernel/intern/main.cc b/source/blender/blenkernel/intern/main.cc index 97fbcd779cc..c05d9b186d2 100644 --- a/source/blender/blenkernel/intern/main.cc +++ b/source/blender/blenkernel/intern/main.cc @@ -717,7 +717,15 @@ void BKE_main_library_weak_reference_add_item( BLI_assert(BKE_idtype_idcode_append_is_reusable(GS(new_id->name))); const LibWeakRefKey key{library_filepath, library_id_name}; - library_weak_reference_mapping->map.add_new(key, new_id); + /* With packed IDs and archive libraries, it is now possible to have several instances of the + * (originally) same linked ID made local at the same time in an append opeeration, so it is + * possible to get the same key several time here. And `Map::add_new` cannot be used safely + * anymore. + * + * Simply consider the first added one as valid, there is no good way to determine the 'best' one + * to keep around for append-or-reuse operations anyway - and the whole append-and-reuse may be + * depracted soon too. */ + library_weak_reference_mapping->map.add(key, new_id); BKE_main_library_weak_reference_add(new_id, library_filepath, library_id_name); } diff --git a/source/blender/blenloader/BLO_readfile.hh b/source/blender/blenloader/BLO_readfile.hh index 9029223b4b8..c4328f57390 100644 --- a/source/blender/blenloader/BLO_readfile.hh +++ b/source/blender/blenloader/BLO_readfile.hh @@ -406,6 +406,10 @@ enum eBLOLibLinkFlags { * see e.g. #BKE_blendfile_library_relocate. */ BLO_LIBLINK_COLLECTION_NO_HIERARCHY_REBUILD = 1 << 26, + /** + * Pack the linked data-blocks to keep them working even if the source file is not available. + */ + BLO_LIBLINK_PACK = 1 << 27, }; /** @@ -584,13 +588,13 @@ struct ID_Readfile_Data { }; /** - * Return `id->runtime.readfile_data->tags` if the `readfile_data` is allocated, + * Return `id->runtime->readfile_data->tags` if the `readfile_data` is allocated, * otherwise return an all-zero set of tags. */ ID_Readfile_Data::Tags BLO_readfile_id_runtime_tags(ID &id); /** - * Create the `readfile_data` if needed, and return `id->runtime.readfile_data->tags`. + * Create the `readfile_data` if needed, and return `id->runtime->readfile_data->tags`. * * Use it instead of #BLO_readfile_id_runtime_tags when tags need to be set. */ diff --git a/source/blender/blenloader/CMakeLists.txt b/source/blender/blenloader/CMakeLists.txt index eee3b15d32e..6925567427f 100644 --- a/source/blender/blenloader/CMakeLists.txt +++ b/source/blender/blenloader/CMakeLists.txt @@ -54,6 +54,7 @@ set(SRC set(LIB PRIVATE bf::animrig + PRIVATE bf::asset_system PRIVATE bf::blenkernel PRIVATE bf::blenlib PUBLIC bf::blenloader_core diff --git a/source/blender/blenloader/intern/readblenentry.cc b/source/blender/blenloader/intern/readblenentry.cc index 4dc68b65c8a..e4843b2c0ef 100644 --- a/source/blender/blenloader/intern/readblenentry.cc +++ b/source/blender/blenloader/intern/readblenentry.cc @@ -87,12 +87,21 @@ static bool blendhandle_load_id_data_and_validate(FileData *fd, BHead *bhead, bool use_assets_only, const char *&r_idname, + short &r_idflag, AssetMetaData *&r_asset_meta_data) { r_idname = blo_bhead_id_name(fd, bhead); if (!r_idname || r_idname[0] == '\0') { return false; } + r_idflag = blo_bhead_id_flag(fd, bhead); + /* Do not list (and therefore allow direct linking of) packed data. + * While supporting this is conceptually possible, it would require significant changes in + * the UI (file browser) and UX (link operation) to convey this concept and handle it + * correctly. */ + if (r_idflag & ID_FLAG_LINKED_AND_PACKED) { + return false; + } r_asset_meta_data = blo_bhead_id_asset_data_address(fd, bhead); if (use_assets_only && r_asset_meta_data == nullptr) { return false; @@ -113,9 +122,10 @@ LinkNode *BLO_blendhandle_get_datablock_names(BlendHandle *bh, for (bhead = blo_bhead_first(fd); bhead; bhead = blo_bhead_next(fd, bhead)) { if (bhead->code == ofblocktype) { const char *idname; + short idflag; AssetMetaData *asset_meta_data; if (!blendhandle_load_id_data_and_validate( - fd, bhead, use_assets_only, idname, asset_meta_data)) + fd, bhead, use_assets_only, idname, idflag, asset_meta_data)) { continue; } @@ -152,9 +162,10 @@ LinkNode *BLO_blendhandle_get_datablock_info(BlendHandle *bh, BHead *id_bhead = bhead; const char *idname; + short idflag; AssetMetaData *asset_meta_data; if (!blendhandle_load_id_data_and_validate( - fd, id_bhead, use_assets_only, idname, asset_meta_data)) + fd, id_bhead, use_assets_only, idname, idflag, asset_meta_data)) { continue; } @@ -382,8 +393,10 @@ BlendFileData *BLO_read_from_memfile(Main *oldmain, /* Build old ID map for all old IDs. */ blo_make_old_idmap_from_main(fd, oldmain); - /* Separate linked data from old main. */ - blo_split_main(oldmain); + /* Separate linked data from old main. + * WARNING: Do not split out packed IDs here, as these are handled similarly as local IDs in + * undo context. */ + blo_split_main(oldmain, false); fd->old_bmain = oldmain; /* Removed packed data from this trick - it's internal data that needs saves. */ diff --git a/source/blender/blenloader/intern/readfile.cc b/source/blender/blenloader/intern/readfile.cc index e130c354ac6..469d3d5a99a 100644 --- a/source/blender/blenloader/intern/readfile.cc +++ b/source/blender/blenloader/intern/readfile.cc @@ -390,17 +390,17 @@ void blo_join_main(Main *bmain) bmain->split_mains.reset(); } -static void split_libdata(ListBase *lb_src, Main **lib_main_array, const uint lib_main_array_len) +static void split_libdata(ListBase *lb_src, + blender::Vector
&lib_main_array, + const bool do_split_packed_ids) { for (ID *id = static_cast(lb_src->first), *idnext; id; id = idnext) { idnext = static_cast(id->next); - if (id->lib) { - if ((uint(id->lib->runtime->temp_index) < lib_main_array_len) && - /* this check should never fail, just in case 'id->lib' is a dangling pointer. */ - (lib_main_array[id->lib->runtime->temp_index]->curlib == id->lib)) - { + if (id->lib && (do_split_packed_ids || (id->lib->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0)) { + if (uint(id->lib->runtime->temp_index) < lib_main_array.size()) { Main *mainvar = lib_main_array[id->lib->runtime->temp_index]; + BLI_assert(mainvar->curlib == id->lib); ListBase *lb_dst = which_libbase(mainvar, GS(id->name)); BLI_remlink(lb_src, id); BLI_addtail(lb_dst, id); @@ -412,7 +412,7 @@ static void split_libdata(ListBase *lb_src, Main **lib_main_array, const uint li } } -void blo_split_main(Main *bmain) +void blo_split_main(Main *bmain, const bool do_split_packed_ids) { BLI_assert(!bmain->split_mains); bmain->split_mains = std::make_shared>(); @@ -432,13 +432,15 @@ void blo_split_main(Main *bmain) BKE_main_namemap_clear(*bmain); /* (Library.temp_index -> Main), lookup table */ - const uint lib_main_array_len = BLI_listbase_count(&bmain->libraries); - Main **lib_main_array = MEM_malloc_arrayN
(lib_main_array_len, __func__); + blender::Vector
lib_main_array; int i = 0; for (Library *lib = static_cast(bmain->libraries.first); lib; lib = static_cast(lib->id.next), i++) { + if (!do_split_packed_ids && (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) != 0) { + continue; + } Main *libmain = BKE_main_new(); libmain->curlib = lib; libmain->versionfile = lib->runtime->versionfile; @@ -450,7 +452,7 @@ void blo_split_main(Main *bmain) bmain->split_mains->add_new(libmain); libmain->split_mains = bmain->split_mains; lib->runtime->temp_index = i; - lib_main_array[i] = libmain; + lib_main_array.append(libmain); } MainListsArray lbarray = BKE_main_lists_get(*bmain); @@ -461,10 +463,8 @@ void blo_split_main(Main *bmain) /* No ID_LI data-block should ever be linked anyway, but just in case, better be explicit. */ continue; } - split_libdata(lbarray[i], lib_main_array, lib_main_array_len); + split_libdata(lbarray[i], lib_main_array, do_split_packed_ids); } - - MEM_freeN(lib_main_array); } static void read_file_version_and_colorspace(FileData *fd, Main *main) @@ -551,58 +551,6 @@ static void read_file_bhead_idname_map_create(FileData *fd) } } -static Main *blo_find_main(FileData *fd, const char *filepath, const char *relabase) -{ - Main *m; - Library *lib; - char filepath_abs[FILE_MAX]; - - STRNCPY(filepath_abs, filepath); - BLI_path_abs(filepath_abs, relabase); - BLI_path_normalize(filepath_abs); - - // printf("blo_find_main: relabase %s\n", relabase); - // printf("blo_find_main: original in %s\n", filepath); - // printf("blo_find_main: converted to %s\n", filepath_abs); - - for (Main *m : *fd->bmain->split_mains) { - const char *libname = (m->curlib) ? m->curlib->runtime->filepath_abs : m->filepath; - - if (BLI_path_cmp(filepath_abs, libname) == 0) { - if (G.debug & G_DEBUG) { - CLOG_DEBUG(&LOG, "Found library %s", libname); - } - return m; - } - } - - m = BKE_main_new(); - fd->bmain->split_mains->add_new(m); - m->split_mains = fd->bmain->split_mains; - - /* Add library data-block itself to 'main' Main, since libraries are **never** linked data. - * Fixes bug where you could end with all ID_LI data-blocks having the same name... */ - lib = BKE_id_new(fd->bmain, BLI_path_basename(filepath)); - - /* Important, consistency with main ID reading code from read_libblock(). */ - lib->id.us = ID_FAKE_USERS(lib); - - /* Matches direct_link_library(). */ - id_us_ensure_real(&lib->id); - - STRNCPY(lib->filepath, filepath); - STRNCPY(lib->runtime->filepath_abs, filepath_abs); - - m->curlib = lib; - - read_file_version_and_colorspace(fd, m); - - if (G.debug & G_DEBUG) { - CLOG_DEBUG(&LOG, "Added new lib %s", filepath); - } - return m; -} - void blo_readfile_invalidate(FileData *fd, Main *bmain, const char *message) { /* Tag given `bmain`, and 'root 'local' main one (in case given one is a library one) as invalid. @@ -845,6 +793,16 @@ const char *blo_bhead_id_name(FileData *fd, const BHead *bhead) return nullptr; } +short blo_bhead_id_flag(const FileData *fd, const BHead *bhead) +{ + BLI_assert(blo_bhead_is_id(bhead)); + if (fd->id_flag_offset < 0) { + return 0; + } + return *reinterpret_cast( + POINTER_OFFSET(bhead, sizeof(*bhead) + fd->id_flag_offset)); +} + AssetMetaData *blo_bhead_id_asset_data_address(const FileData *fd, const BHead *bhead) { BLI_assert(blo_bhead_is_id_valid_type(bhead)); @@ -853,6 +811,20 @@ AssetMetaData *blo_bhead_id_asset_data_address(const FileData *fd, const BHead * nullptr; } +static const IDHash *blo_bhead_id_deep_hash(const FileData *fd, const BHead *bhead) +{ + BLI_assert(blo_bhead_is_id_valid_type(bhead)); + if (fd->id_flag_offset < 0 || fd->id_deep_hash_offset < 0) { + return nullptr; + } + const short flag = blo_bhead_id_flag(fd, bhead); + if (!(flag & ID_FLAG_LINKED_AND_PACKED)) { + return nullptr; + } + return reinterpret_cast( + POINTER_OFFSET(bhead, sizeof(*bhead) + fd->id_deep_hash_offset)); +} + static void read_blender_header(FileData *fd) { const BlenderHeaderVariant header_variant = BLO_readfile_blender_header_decode(fd->file); @@ -920,6 +892,10 @@ static bool read_file_dna(FileData *fd, const char **r_error_message) BLI_assert(fd->id_name_offset != -1); fd->id_asset_data_offset = DNA_struct_member_offset_by_name_with_alias( fd->filesdna, "ID", "AssetMetaData", "*asset_data"); + fd->id_flag_offset = DNA_struct_member_offset_by_name_with_alias( + fd->filesdna, "ID", "short", "flag"); + fd->id_deep_hash_offset = DNA_struct_member_offset_by_name_with_alias( + fd->filesdna, "ID", "IDHash", "deep_hash"); fd->filesubversion = subversion; @@ -1149,6 +1125,7 @@ static FileData *filedata_new(BlendFileReadReport *reports) fd->datamap = oldnewmap_new(); fd->globmap = oldnewmap_new(); fd->libmap = oldnewmap_new(); + fd->id_by_deep_hash = std::make_shared>(); fd->reports = reports; @@ -1315,6 +1292,11 @@ static FileData *blo_filedata_from_file_descriptor(const char *filepath, FileData *fd = filedata_new(reports); fd->file = file; + BLI_stat_t stat; + if (BLI_stat(filepath, &stat) != -1) { + fd->file_stat = stat; + } + return fd; } @@ -2255,6 +2237,9 @@ static void direct_link_id_common(BlendDataReader *reader, id->session_uid = MAIN_ID_SESSION_UID_UNSET; } + if (ID_IS_PACKED(id)) { + BLI_assert(current_library->flag & LIBRARY_FLAG_IS_ARCHIVE); + } id->lib = current_library; if (id->lib) { /* Always fully clear fake user flag for linked data. */ @@ -2444,6 +2429,8 @@ static void library_filedata_release(Library *lib) { if (lib->runtime->filedata) { BLI_assert(lib->runtime->versionfile != 0); + BLI_assert_msg(!lib->runtime->is_filedata_owner || (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0, + "Packed Archive libraries should never own their filedata"); if (lib->runtime->is_filedata_owner) { blo_filedata_free(lib->runtime->filedata); } @@ -2470,6 +2457,11 @@ static void direct_link_library(FileData *fd, Library *lib, Main *main) if (!newmain->curlib) { continue; } + if (newmain->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE || lib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + /* Archive library should never be used to link new data, and there can be many such + * archive libraries for a same 'real' blendfile one. */ + continue; + } if (BLI_path_cmp(newmain->curlib->runtime->filepath_abs, lib->runtime->filepath_abs) == 0) { BLO_reportf_wrap(fd->reports, RPT_WARNING, @@ -2510,9 +2502,24 @@ static void direct_link_library(FileData *fd, Library *lib, Main *main) newmain->split_mains = fd->bmain->split_mains; newmain->curlib = lib; + if (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + /* Archive libraries contains only embedded linked IDs, which by definition have the same + * fileversion as the blendfile that contains them. */ + lib->runtime->versionfile = newmain->versionfile = fd->bmain->versionfile; + lib->runtime->subversionfile = newmain->subversionfile = fd->bmain->subversionfile; + + /* The filedata of a packed archive library should always be the one of the blendfile which + * defines the library ID and packs its linked IDs. */ + lib->runtime->filedata = fd; + lib->runtime->is_filedata_owner = false; + } + lib->runtime->parent = nullptr; id_us_ensure_real(&lib->id); + + /* Should always be null, Library IDs in Blender are always local. */ + lib->id.lib = nullptr; } /* Always call this once you have loaded new library data to set the relative paths correctly @@ -2680,6 +2687,82 @@ static BHead *read_data_into_datamap(FileData *fd, return bhead; } +/* Add a Main (and optionally create a matching Library ID), for the given filepath. + * + * - If `lib` is `nullptr`, create a new Library ID, otherwise only create a new Main for the given + * library. + * - `reference_lib` is the 'archive parent' of an archive (packed) library, can be null and will + * be ignored otherwise. */ +static Main *blo_add_main_for_library(FileData *fd, + Library *lib, + Library *reference_lib, + const char *lib_filepath, + char (&filepath_abs)[FILE_MAX], + const bool is_packed_library) +{ + Main *bmain = BKE_main_new(); + fd->bmain->split_mains->add_new(bmain); + bmain->split_mains = fd->bmain->split_mains; + + if (!lib) { + /* Add library data-block itself to 'main' Main, since libraries are **never** linked data. + * Fixes bug where you could end with all ID_LI data-blocks having the same name... */ + lib = BKE_id_new(fd->bmain, + reference_lib ? BKE_id_name(reference_lib->id) : + BLI_path_basename(lib_filepath)); + + /* Important, consistency with main ID reading code from read_libblock(). */ + lib->id.us = ID_FAKE_USERS(lib); + + /* Matches direct_link_library(). */ + id_us_ensure_real(&lib->id); + + STRNCPY(lib->filepath, lib_filepath); + STRNCPY(lib->runtime->filepath_abs, filepath_abs); + + if (is_packed_library) { + /* FIXME: This logic is very similar to the code in BKE_library dealing with archived + * libraries (e.g. #add_archive_library). Might be good to try to factorize it. */ + lib->archive_parent_library = reference_lib; + constexpr uint16_t copy_flag = ~LIBRARY_FLAG_IS_ARCHIVE; + lib->flag = (reference_lib->flag & copy_flag) | LIBRARY_FLAG_IS_ARCHIVE; + + lib->runtime->parent = reference_lib->runtime->parent; + /* Only copy a subset of the reference library tags. E.g. an archive library should never be + * considered as writable, so never copy #LIBRARY_ASSET_FILE_WRITABLE. This may need further + * tweaking still. */ + constexpr uint16_t copy_tag = (LIBRARY_TAG_RESYNC_REQUIRED | LIBRARY_ASSET_EDITABLE | + LIBRARY_IS_ASSET_EDIT_FILE); + lib->runtime->tag = reference_lib->runtime->tag & copy_tag; + + /* The filedata of a packed archive library should always be the one of the blendfile which + * defines the library ID and packs its linked IDs. */ + lib->runtime->filedata = fd; + lib->runtime->is_filedata_owner = false; + + reference_lib->runtime->archived_libraries.append(lib); + } + } + else { + if (is_packed_library) { + BLI_assert(lib->flag & LIBRARY_FLAG_IS_ARCHIVE); + BLI_assert(lib->archive_parent_library == reference_lib); + BLI_assert(reference_lib->runtime->archived_libraries.contains(lib)); + + BLI_assert(lib->runtime->filedata == nullptr); + lib->runtime->filedata = fd; + lib->runtime->is_filedata_owner = false; + } + else { + /* Should never happen currently. */ + BLI_assert_unreachable(); + } + } + + bmain->curlib = lib; + return bmain; +} + /* Verify if the datablock and all associated data is identical. */ static bool read_libblock_is_identical(FileData *fd, BHead *bhead) { @@ -2764,7 +2847,10 @@ static void read_undo_move_libmain_data(FileData *fd, Main *libmain, BHead *bhea ID *id_iter; FOREACH_MAIN_ID_BEGIN (libmain, id_iter) { - BKE_main_idmap_insert_id(fd->new_idmap_uid, id_iter); + /* Packed IDs are read from the memfile, so don't add them here already. */ + if (!ID_IS_PACKED(id_iter)) { + BKE_main_idmap_insert_id(fd->new_idmap_uid, id_iter); + } } FOREACH_MAIN_ID_END; } @@ -2796,10 +2882,13 @@ static bool read_libblock_undo_restore_library(FileData *fd, * modified should not be an issue currently. */ for (Main *libmain : fd->old_bmain->split_mains->as_span().drop_front(1)) { if (&libmain->curlib->id == id_old) { + BLI_assert(libmain->curlib); + BLI_assert((libmain->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0); CLOG_DEBUG(&LOG_UNDO, " compare with %s -> match (existing libpath: %s)", - libmain->curlib ? libmain->curlib->id.name : "", - libmain->curlib ? libmain->curlib->runtime->filepath_abs : ""); + libmain->curlib->id.name, + libmain->curlib->runtime->filepath_abs); + /* In case of a library, we need to re-add its main to fd->bmain->split_mains, * because if we have later a missing ID_LINK_PLACEHOLDER, * we need to get the correct lib it is linked to! @@ -2914,6 +3003,26 @@ static void read_libblock_undo_restore_identical( * data-blocks too. */ ob->mode &= ~OB_MODE_EDIT; } + if (GS(id_old->name) == ID_LI) { + Library *lib = reinterpret_cast(id_old); + if (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + BLI_assert(lib->runtime->filedata == nullptr); + BLI_assert(lib->archive_parent_library); + /* The 'normal' parent of this archive library should already have been moved into the new + * Main. */ + BLI_assert(BKE_main_idmap_lookup_uid(fd->new_idmap_uid, + lib->archive_parent_library->id.session_uid) == + &lib->archive_parent_library->id); + /* The archive library ID has been moved in the new Main, but not its own old split main, as + * these packed IDs should be handled like local ones in undo case. So a new split libmain + * needs to be created to contain its packed IDs. */ + blo_add_main_for_library( + fd, lib, lib->archive_parent_library, lib->filepath, lib->runtime->filepath_abs, true); + } + else { + BLI_assert_unreachable(); + } + } } /* For undo, store changed datablock at old address. */ @@ -2970,8 +3079,10 @@ static bool read_libblock_undo_restore( const bool do_partial_undo = (fd->skip_flags & BLO_READ_SKIP_UNDO_OLD_MAIN) == 0; #ifndef NDEBUG - if (do_partial_undo && (bhead->code != ID_LINK_PLACEHOLDER)) { - /* This code should only ever be reached for local data-blocks. */ + if (do_partial_undo && (bhead->code != ID_LINK_PLACEHOLDER) && + (blo_bhead_id_flag(fd, bhead) & ID_FLAG_LINKED_AND_PACKED) == 0) + { + /* This code should only ever be reached for local or packed data-blocks. */ BLI_assert(main->curlib == nullptr); } #endif @@ -2983,8 +3094,13 @@ static bool read_libblock_undo_restore( nullptr; if (bhead->code == ID_LI) { - /* Restore library datablock, if possible. */ - if (read_libblock_undo_restore_library(fd, id, id_old, bhead)) { + /* Restore library datablock, if possible. + * + * Never handle archive libraries and their packed IDs as normal ones. These are local data, + * and need to be fully handled like local IDs. */ + if (id_old && (reinterpret_cast(id_old)->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0 && + read_libblock_undo_restore_library(fd, id, id_old, bhead)) + { return true; } } @@ -3182,6 +3298,14 @@ static BHead *read_libblock(FileData *fd, if (main->id_map != nullptr) { BKE_main_idmap_insert_id(main->id_map, id_target); } + if (ID_IS_PACKED(id)) { + BLI_assert(id->deep_hash != IDHash::get_null()); + fd->id_by_deep_hash->add_new(id->deep_hash, id); + BLI_assert(main->curlib); + } + if (fd->file_stat) { + id->runtime->src_blend_modifification_time = fd->file_stat->st_mtime; + } } return bhead; @@ -3831,6 +3955,10 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) } while (bhead) { + /* If not-null after the `switch`, the BHead is an ID one and needs to be read. */ + Main *bmain_to_read_into = nullptr; + bool placeholder_set_indirect_extern = false; + switch (bhead->code) { case BLO_CODE_DATA: case BLO_CODE_DNA1: @@ -3854,42 +3982,61 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) break; case ID_LINK_PLACEHOLDER: - if (fd->skip_flags & BLO_READ_SKIP_DATA) { + if ((fd->skip_flags & BLO_READ_SKIP_DATA) != 0) { bhead = blo_bhead_next(fd, bhead); + break; } - else { - /* Add link placeholder to the main of the library it belongs to. - * The library is the most recently loaded #ID_LI block, according - * to the file format definition. So we can use the entry at the - * end of `fd->bmain->split_mains`, typically the one last added in - * #direct_link_library. */ - - Main *libmain = (*fd->bmain->split_mains)[fd->bmain->split_mains->size() - 1]; - bhead = read_libblock(fd, libmain, bhead, 0, {}, true, nullptr); - } + /* Add link placeholder to the main of the library it belongs to. + * + * The library is the most recently loaded #ID_LI block, according to the file format + * definition. So we can use the entry at the end of `fd->bmain->split_mains`, typically + * the one last added in #direct_link_library. */ + bmain_to_read_into = (*fd->bmain->split_mains)[fd->bmain->split_mains->size() - 1]; + placeholder_set_indirect_extern = true; + break; + case ID_LI: + if ((fd->skip_flags & BLO_READ_SKIP_DATA) != 0) { + bhead = blo_bhead_next(fd, bhead); + break; + } + /* Library IDs are always read into the first (aka 'local') Main, even if they are written + * in 'library' blendfile-space (for archive libraries e.g.). */ + bmain_to_read_into = fd->bmain; break; - /* in 2.50+ files, the file identifier for screens is patched, forward compatibility */ case ID_SCRN: + /* in 2.50+ files, the file identifier for screens is patched, forward compatibility */ bhead->code = ID_SCR; /* pass on to default */ ATTR_FALLTHROUGH; default: { - if (blo_bhead_is_id_valid_type(bhead)) { - /* BHead is a valid known ID type one, read the whole ID and its sub-data, unless reading - * actual data is skipped. */ - if (fd->skip_flags & BLO_READ_SKIP_DATA) { - bhead = blo_bhead_next(fd, bhead); - } - else { - bhead = read_libblock(fd, bfd->main, bhead, ID_TAG_LOCAL, {}, false, nullptr); - } - } - else { - /* Unknown BHead type (or ID type), ignore it and skip to next BHead. */ + if ((fd->skip_flags & BLO_READ_SKIP_DATA) != 0 || !blo_bhead_is_id_valid_type(bhead)) { bhead = blo_bhead_next(fd, bhead); + break; } + /* Put read real ID into the main of the library it belongs to. + * + * Local IDs should all be written before any Library in the blendfile, so this code will + * always select `fd->bmain` for these. + * + * Packed linked IDs are real ID data in the currently read blendfile (unlike placeholders + * for regular linked data). But they are in their archive library 'name space' and + * 'blendfile space', so this follows the same logic as for placeholders to select the + * Main. + * + * The library is the most recently loaded #ID_LI block, according to the file format + * definition. So we can use the entry at the end of `fd->bmain->split_mains`, typically + * the one last added in #direct_link_library. */ + bmain_to_read_into = (*fd->bmain->split_mains)[fd->bmain->split_mains->size() - 1]; + BLI_assert_msg((bmain_to_read_into == fd->bmain || + (blo_bhead_id_flag(fd, bhead) & ID_FLAG_LINKED_AND_PACKED) != 0), + "Local IDs should always be put in the first Main split data-base, not in " + "a 'linked data' one"); } } + if (bmain_to_read_into) { + bhead = read_libblock( + fd, bmain_to_read_into, bhead, 0, {}, placeholder_set_indirect_extern, nullptr); + } if (bfd->main->is_read_invalid) { return bfd; @@ -3916,6 +4063,14 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) * of its items. */ blender::Vector
old_main_split_mains = {old_main->split_mains->as_span()}; for (Main *libmain : old_main_split_mains.as_span().drop_front(1)) { + BLI_assert(libmain->curlib); + if (libmain->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + /* Never move archived libraries and their content, these are 'local' data in undo context, + * so all packed linked IDs should have been handled like local ones undo-wise, and if + * packed libraries remain unused at this point, then they are indeed fully unused/removed + * from the new main. */ + continue; + } read_undo_move_libmain_data(fd, libmain, nullptr); } } @@ -3958,7 +4113,36 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) /* Do versioning before read_libraries, but skip in undo case. */ if (!is_undo) { if ((fd->skip_flags & BLO_READ_SKIP_DATA) == 0) { - do_versions(fd, nullptr, bfd->main); + for (Main *bmain : *fd->bmain->split_mains) { + /* Packed IDs are stored in the current .blend file, but belong to dedicated 'archive + * library' Mains, not the first, 'local' Main. So they do need versioning here, as for + * local IDs, which is why all the split Mains in the list need to be checked. + * + * Placeholders (of 'real' linked data) can't be versioned yet. Since they also belong to + * dedicated 'library' Mains, and are not mixed with the 'packed' ones, these Mains can be + * entirely skipped. */ + const bool contains_link_placeholder = (bmain->curlib != nullptr && + (bmain->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) == + 0); +#ifndef NDEBUG + MainListsArray lbarray = BKE_main_lists_get(*bmain); + for (ListBase *lb_array : lbarray) { + LISTBASE_FOREACH_MUTABLE (ID *, id, lb_array) { + BLI_assert_msg((id->runtime->readfile_data->tags.is_link_placeholder == + contains_link_placeholder), + contains_link_placeholder ? + "Real Library split Main contains non-placeholder IDs" : + (bmain->curlib == nullptr ? + "Local data split Main contains placeholder IDs" : + "Archive Library split Main contains placeholder IDs")); + } + } +#endif + if (contains_link_placeholder) { + continue; + } + do_versions(fd, bmain->curlib, bmain); + } } if ((fd->skip_flags & BLO_READ_SKIP_USERDEF) == 0) { @@ -4027,7 +4211,9 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) } LISTBASE_FOREACH_MUTABLE (Library *, lib, &bfd->main->libraries) { - /* Now we can clear this runtime library filedata, it is not needed anymore. */ + /* Now we can clear this runtime library filedata, it is not needed anymore. + * + * NOTE: This is also important to do for archive libraries. */ library_filedata_release(lib); /* If no data-blocks were read from a library (should only happen when all references to a * library's data are `ID_FLAG_INDIRECT_WEAK_LINK`), its versionfile will still be zero and @@ -4038,8 +4224,12 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) * placeholders IDs created will reference the library ID, and the library ID will have a * valid version number as the file was read to search for the linked IDs. * - In case the library blendfile does not exist, its local Library ID will get the version - * of the current local Main (i.e. the loaded blendfile). */ - if (lib->runtime->versionfile == 0) { + * of the current local Main (i.e. the loaded blendfile). + * - In case it is a reference library for archived ones, its runtime #archived_libraries + * vector will not be empty, and it must be kept, even if no data is directly linked from + * it anymore. + */ + if (lib->runtime->versionfile == 0 && lib->runtime->archived_libraries.is_empty()) { #ifndef NDEBUG ID *id_iter; FOREACH_MAIN_ID_BEGIN (bfd->main, id_iter) { @@ -4247,21 +4437,41 @@ static BHead *find_bhead_from_idname(FileData *fd, const char *idname) return find_bhead_from_code_name(fd, id_code_old, idname + 2); } -static ID *library_id_is_yet_read(FileData *fd, Main *mainvar, BHead *bhead) +static ID *library_id_is_yet_read_deep_hash(FileData *fd, BHead *bhead) +{ + if (const IDHash *deep_hash = blo_bhead_id_deep_hash(fd, bhead)) { + if (ID *existing_id = fd->id_by_deep_hash->lookup_default(*deep_hash, nullptr)) { + return existing_id; + } + } + return nullptr; +} + +static ID *library_id_is_yet_read_main(Main *mainvar, const char *idname) { if (mainvar->id_map == nullptr) { mainvar->id_map = BKE_main_idmap_create(mainvar, false, nullptr, MAIN_IDMAP_TYPE_NAME); } BLI_assert(BKE_main_idmap_main_get(mainvar->id_map) == mainvar); + ID *existing_id = BKE_main_idmap_lookup_name( + mainvar->id_map, GS(idname), idname + 2, mainvar->curlib); + BLI_assert(existing_id == + BLI_findstring(which_libbase(mainvar, GS(idname)), idname, offsetof(ID, name))); + return existing_id; +} + +static ID *library_id_is_yet_read(FileData *fd, Main *mainvar, BHead *bhead) +{ + if (ID *existing_id = library_id_is_yet_read_deep_hash(fd, bhead)) { + return existing_id; + } + const char *idname = blo_bhead_id_name(fd, bhead); if (!idname) { return nullptr; } - - ID *id = BKE_main_idmap_lookup_name(mainvar->id_map, GS(idname), idname + 2, mainvar->curlib); - BLI_assert(id == BLI_findstring(which_libbase(mainvar, GS(idname)), idname, offsetof(ID, name))); - return id; + return library_id_is_yet_read_main(mainvar, idname); } static void read_libraries_report_invalid_id_names(FileData *fd, @@ -4307,15 +4517,132 @@ struct BlendExpander { BLOExpandDoitCallback callback; }; +/* Find the existing Main matching the given blendfile library filepath, or create a new one (with + * the matching Library ID) if needed. + * + * NOTE: The process is a bit more complex for packed linked IDs and their archive libraries, as + * in this case, this function also needs to find or create a new suitable archive library, i.e. + * one which does not contain yet the given ID (from its name & type). */ +static Main *blo_find_main_for_library_and_idname(FileData *fd, + const char *lib_filepath, + const char *relabase, + const BHead *id_bhead, + const char *id_name, + const bool is_packed_id) +{ + Library *parent_lib = nullptr; + char filepath_abs[FILE_MAX]; + + STRNCPY(filepath_abs, lib_filepath); + BLI_path_abs(filepath_abs, relabase); + BLI_path_normalize(filepath_abs); + + for (Main *main_it : *fd->bmain->split_mains) { + const char *libname = (main_it->curlib) ? main_it->curlib->runtime->filepath_abs : + main_it->filepath; + + if (BLI_path_cmp(filepath_abs, libname) == 0) { + CLOG_DEBUG(&LOG, + "Found library '%s' for file path '%s'", + main_it->curlib ? main_it->curlib->id.name : "", + lib_filepath); + /* Due to how parent and archive libraries are created and written in the blend-file, + * the first library matching a given filepath should never be an archive one. */ + BLI_assert(!main_it->curlib || (main_it->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0); + if (!is_packed_id) { + return main_it; + } + /* For packed IDs, the Main of the main owner library is not a valid one. Another loop is + * needed into all the Mains matching the archive libraries of this main library. */ + BLI_assert(main_it->curlib); + parent_lib = main_it->curlib; + break; + } + } + + if (is_packed_id) { + if (parent_lib) { + /* Try to find an 'available' existing archive Main library, i.e. one that does not yet + * contain an ID of the same type and name. */ + for (Main *main_it : *fd->bmain->split_mains) { + if (!main_it->curlib || (main_it->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0 || + main_it->curlib->archive_parent_library != parent_lib) + { + continue; + } + if (ID *packed_id = library_id_is_yet_read_main(main_it, id_name)) { + /* Archive Main library already contains a 'same' ID - but it should have a different + * deep_hash. Otherwise, a previous call to `library_id_is_yet_read()` should have + * returned this ID, and this code should not be reached. */ + BLI_assert(packed_id->deep_hash != *blo_bhead_id_deep_hash(fd, id_bhead)); + UNUSED_VARS_NDEBUG(packed_id, id_bhead); + continue; + } + BLI_assert(ELEM(main_it->curlib->runtime->filedata, fd, nullptr)); + main_it->curlib->runtime->filedata = fd; + main_it->curlib->runtime->is_filedata_owner = false; + BLI_assert(main_it->versionfile != 0); + CLOG_DEBUG(&LOG, + "Found archive library '%s' for the packed ID '%s'", + main_it->curlib->id.name, + id_name); + return main_it; + } + } + else { + /* An archive library requires an existing parent library, create an empty, 'virtual' one if + * needed. */ + Main *reference_bmain = blo_add_main_for_library( + fd, nullptr, nullptr, lib_filepath, filepath_abs, false); + parent_lib = reference_bmain->curlib; + CLOG_DEBUG(&LOG, + "Added new parent library '%s' for file path '%s'", + parent_lib->id.name, + lib_filepath); + } + } + BLI_assert(parent_lib || !is_packed_id); + + Main *bmain = blo_add_main_for_library( + fd, nullptr, parent_lib, lib_filepath, filepath_abs, is_packed_id); + + read_file_version_and_colorspace(fd, bmain); + + if (is_packed_id) { + CLOG_DEBUG(&LOG, + "Added new archive library '%s' for the packed ID '%s'", + bmain->curlib->id.name, + id_name); + } + else { + CLOG_DEBUG( + &LOG, "Added new library '%s' for file path '%s'", bmain->curlib->id.name, lib_filepath); + } + return bmain; +} + +/* Actually load an ID from a library. There are three possible cases here: + * - `existing_id` is non-null: calling code already found a suitable existing ID, this function + * essentially then only updates the mappings for `bhead->old` address to point to the given + * ID. This is the only case where `libmain` may be `nullptr`. + * - The given bhead has an already loaded matching ID (found by a call to + * `library_id_is_yet_read`), then once that ID is found behavior is as in the previous case. + * - No matching existing ID is found, then a new one is actually read from the given FileData. + */ static void read_id_in_lib(FileData *fd, std::queue &ids_to_expand, Main *libmain, Library *parent_lib, BHead *bhead, + ID *existing_id, ID_Readfile_Data::Tags id_read_tags) { - ID *id = library_id_is_yet_read(fd, libmain, bhead); + ID *id = existing_id; + if (id == nullptr) { + BLI_assert(libmain); + id = library_id_is_yet_read(fd, libmain, bhead); + } if (id == nullptr) { /* ID has not been read yet, add placeholder to the main of the * library it belongs to, so that it will be read later. */ @@ -4392,44 +4719,96 @@ static void expand_doit_library(void *fdhandle, if (!blo_bhead_is_id_valid_type(bhead)) { return; } - if (!blo_bhead_id_name(fd, bhead)) { + const char *id_name = blo_bhead_id_name(fd, bhead); + if (!id_name) { /* Do not allow linking ID which names are invalid (likely coming from a future version of * Blender allowing longer names). */ return; } + const bool is_packed_id = (blo_bhead_id_flag(fd, bhead) & ID_FLAG_LINKED_AND_PACKED) != 0; + + BLI_assert_msg(!is_packed_id || bhead->code != ID_LINK_PLACEHOLDER, + "A link placeholder ID (aka reference to some ID linked from another library) " + "should never be packed."); if (bhead->code == ID_LINK_PLACEHOLDER) { /* Placeholder link to data-block in another library. */ BHead *bheadlib = find_previous_lib(fd, bhead); if (bheadlib == nullptr) { + BLO_reportf_wrap(fd->reports, + RPT_ERROR, + RPT_("LIB: .blend file %s seems corrupted, no owner 'Library' data found " + "for the linked data-block %s"), + mainvar->curlib->runtime->filepath_abs, + id_name ? id_name : ""); return; } Library *lib = reinterpret_cast( read_id_struct(fd, bheadlib, "Data for Library ID type", INDEX_ID_NULL)); - Main *libmain = blo_find_main(fd, lib->filepath, fd->relabase); + Main *libmain = blo_find_main_for_library_and_idname( + fd, lib->filepath, fd->relabase, nullptr, nullptr, false); MEM_freeN(lib); if (libmain->curlib == nullptr) { - const char *idname = blo_bhead_id_name(fd, bhead); - BLO_reportf_wrap(fd->reports, RPT_WARNING, RPT_("LIB: Data refers to main .blend file: '%s' from %s"), - idname ? idname : "", + id_name ? id_name : "", mainvar->curlib->runtime->filepath_abs); return; } /* Placeholders never need expanding, as they are a mere reference to ID from another * library/blendfile. */ - read_id_in_lib(fd, ids_to_expand, libmain, mainvar->curlib, bhead, {}); + read_id_in_lib(fd, ids_to_expand, libmain, mainvar->curlib, bhead, nullptr, {}); + } + else if (is_packed_id) { + /* Packed data-block from another library. */ + + /* That exact same packed ID may have already been read before. */ + if (ID *existing_id = library_id_is_yet_read_deep_hash(fd, bhead)) { + /* Ensure that the current BHead's `old` pointer will also be remapped to the found existing + * ID. */ + read_id_in_lib(fd, ids_to_expand, nullptr, nullptr, bhead, existing_id, {}); + return; + } + + BHead *bheadlib = find_previous_lib(fd, bhead); + if (bheadlib == nullptr) { + BLO_reportf_wrap(fd->reports, + RPT_ERROR, + RPT_("LIB: .blend file %s seems corrupted, no owner 'Library' data found " + "for the packed linked data-block %s"), + mainvar->curlib->runtime->filepath_abs, + id_name ? id_name : ""); + return; + } + + Library *lib = reinterpret_cast( + read_id_struct(fd, bheadlib, "Data for Library ID type", INDEX_ID_NULL)); + Main *libmain = blo_find_main_for_library_and_idname( + fd, lib->filepath, fd->relabase, bhead, id_name, is_packed_id); + MEM_freeN(lib); + + if (libmain->curlib == nullptr) { + BLO_reportf_wrap(fd->reports, + RPT_WARNING, + RPT_("LIB: Data refers to main .blend file: '%s' from %s"), + id_name ? id_name : "", + mainvar->curlib->runtime->filepath_abs); + return; + } + + ID_Readfile_Data::Tags id_read_tags{}; + id_read_tags.needs_expanding = true; + read_id_in_lib(fd, ids_to_expand, libmain, nullptr, bhead, nullptr, id_read_tags); } else { /* Data-block in same library. */ ID_Readfile_Data::Tags id_read_tags{}; id_read_tags.needs_expanding = true; - read_id_in_lib(fd, ids_to_expand, mainvar, nullptr, bhead, id_read_tags); + read_id_in_lib(fd, ids_to_expand, mainvar, nullptr, bhead, nullptr, id_read_tags); } } @@ -4476,6 +4855,13 @@ static void expand_main(void *fdhandle, Main *mainvar, BLOExpandDoitCallback cal FileData *fd = static_cast(fdhandle); BlendExpander expander = {fd, {}, mainvar, callback}; + /* Note: Packed IDs are the only current case where IDs read/loaded from a library blendfile will + * end up in another Main (outside of placeholders, which never need to be expanded). This is not + * a problem for initialization of the 'to be expanded' queue though, as no packed ID can be + * directly linked currently, they are only brough in indirectly, i.e. during the expansion + * process itself. + * + * So just looping on the 'main'/root Main of the read library is fine here currently. */ ID *id_iter; FOREACH_MAIN_ID_BEGIN (mainvar, id_iter) { if (BLO_readfile_id_runtime_tags(*id_iter).needs_expanding) { @@ -4600,11 +4986,22 @@ static Main *library_link_begin(Main *mainvar, fd->bmain = mainvar; + /* Add already existing packed data-blocks to map so that they are not loaded again. */ + ID *id; + FOREACH_MAIN_ID_BEGIN (mainvar, id) { + if (ID_IS_PACKED(id)) { + fd->id_by_deep_hash->add(id->deep_hash, id); + } + } + FOREACH_MAIN_ID_END; + /* make mains */ blo_split_main(mainvar); - /* which one do we need? */ - mainl = blo_find_main(fd, filepath, BKE_main_blendfile_path(mainvar)); + /* Find or create a Main matching the current library filepath. */ + /* Note: Directly linking packed IDs is not supported currently. */ + mainl = blo_find_main_for_library_and_idname( + fd, filepath, BKE_main_blendfile_path(mainvar), nullptr, nullptr, false); fd->fd_bmain = mainl; if (mainl->curlib) { mainl->curlib->runtime->filedata = fd; @@ -4737,7 +5134,7 @@ static void library_link_end(Main *mainl, FileData **fd, const int flag, ReportL Main *main_newid = BKE_main_new(); for (Main *mainlib : mainvar->split_mains->as_span().drop_front(1)) { - BLI_assert(mainlib->versionfile != 0); + BLI_assert(mainlib->versionfile != 0 || BKE_main_is_empty(mainlib)); /* We need to split out IDs already existing, * or they will go again through do_versions - bad, very bad! */ split_main_newid(mainlib, main_newid); @@ -5054,6 +5451,7 @@ static FileData *read_library_file_data(FileData *basefd, Main *bmain, Main *lib } fd->libmap = oldnewmap_new(); + fd->id_by_deep_hash = basefd->id_by_deep_hash; lib_bmain->curlib->runtime->filedata = fd; lib_bmain->curlib->runtime->is_filedata_owner = true; @@ -5104,6 +5502,13 @@ static void read_libraries(FileData *basefd) * this list gets longer as more indirectly library blends are found. */ for (int i = 1; i < bmain->split_mains->size(); i++) { Main *libmain = (*bmain->split_mains)[i]; + BLI_assert(libmain->curlib); + /* Always skip archived libraries here, these should _never_ need to be processed here, as + * their data is local data from a blendfile perspective. */ + if (libmain->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) { + BLI_assert(!has_linked_ids_to_read(libmain)); + continue; + } /* Does this library have any more linked data-blocks we need to read? */ if (has_linked_ids_to_read(libmain)) { CLOG_DEBUG(&LOG, diff --git a/source/blender/blenloader/intern/readfile.hh b/source/blender/blenloader/intern/readfile.hh index 9f18152221f..dad2889cf68 100644 --- a/source/blender/blenloader/intern/readfile.hh +++ b/source/blender/blenloader/intern/readfile.hh @@ -16,6 +16,7 @@ # include "BLI_winstuff.h" #endif +#include "BLI_fileops.h" #include "BLI_filereader.h" #include "BLI_map.hh" @@ -82,6 +83,7 @@ struct FileData { BlenderHeader blender_header = {}; FileReader *file = nullptr; + std::optional file_stat; /** * Whether we are undoing (< 0) or redoing (> 0), used to choose which 'unchanged' flag to use @@ -118,6 +120,8 @@ struct FileData { /** Used to retrieve asset data from (bhead+1). NOTE: This may not be available in old files, * will be -1 then! */ int id_asset_data_offset = 0; + int id_flag_offset = 0; + int id_deep_hash_offset = 0; /** For do_versions patching. */ int globalf = 0; int fileflags = 0; @@ -135,6 +139,8 @@ struct FileData { OldNewMap *datamap = nullptr; OldNewMap *globmap = nullptr; + /** Used to keep track of already loaded packed IDs to avoid loading them multiple times. */ + std::shared_ptr> id_by_deep_hash; /** * Store mapping from old ID pointers (the values they have in the .blend file) to new ones, @@ -189,9 +195,22 @@ struct FileData { void *storage_handle = nullptr; }; -/***/ +/** + * Split a single main into a vector of Mains, each containing only IDs from a given library. + * + * The vector is accessible in all of the split mains through the shared pointer + * #Main::split_mains. + * + * The first Main of the vector is the same as the given `main`, and contains local IDs. + * + * If `do_split_packed_ids` is `false`, packed linked IDs remain in the local (first) main as well. + */ +void blo_split_main(Main *bmain, bool do_split_packed_ids = true); +/** + * Join the set of split mains (found in given `main` #Main::split_mains vector shared pointer) + * back into that 'main' main. + */ void blo_join_main(Main *bmain); -void blo_split_main(Main *bmain); BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) ATTR_NONNULL(1, 2); @@ -232,6 +251,13 @@ BHead *blo_bhead_prev(FileData *fd, BHead *thisblock) ATTR_NONNULL(1, 2); * it was saved in a version of Blender with higher MAX_ID_NAME value). */ const char *blo_bhead_id_name(FileData *fd, const BHead *bhead); +/** + * Warning! It's the caller's responsibility to ensure that the given bhead **is** an ID one! + * + * Returns the ID flag value (or `0` if the blendfile is too old and the offset of the ID::flag + * member could not be computed). + */ +short blo_bhead_id_flag(const FileData *fd, const BHead *bhead); /** * Warning! Caller's responsibility to ensure given bhead **is** an ID one! */ diff --git a/source/blender/blenloader/intern/versioning_500.cc b/source/blender/blenloader/intern/versioning_500.cc index e561723e5f5..b599f4a74e1 100644 --- a/source/blender/blenloader/intern/versioning_500.cc +++ b/source/blender/blenloader/intern/versioning_500.cc @@ -72,6 +72,8 @@ #include "WM_api.hh" +#include "AS_asset_library.hh" + #include "readfile.hh" #include "versioning_common.hh" @@ -3679,4 +3681,7 @@ void blo_do_versions_500(FileData *fd, Library * /*lib*/, Main *bmain) LISTBASE_FOREACH (Mesh *, mesh, &bmain->meshes) { bke::mesh_freestyle_marks_to_generic(*mesh); } + + /* TODO: Can be moved to subversion bump. */ + AS_asset_library_import_method_ensure_valid(*bmain); } diff --git a/source/blender/blenloader/intern/writefile.cc b/source/blender/blenloader/intern/writefile.cc index 0236a3cf40b..4198a5b02ef 100644 --- a/source/blender/blenloader/intern/writefile.cc +++ b/source/blender/blenloader/intern/writefile.cc @@ -1164,6 +1164,11 @@ static void write_libraries(WriteData *wd, Main *bmain) if (id->us == 0) { continue; } + if (ID_IS_PACKED(id)) { + BLI_assert(library.flag & LIBRARY_FLAG_IS_ARCHIVE); + ids_used_from_library.append(id); + continue; + } if (id->tag & ID_TAG_EXTERN) { ids_used_from_library.append(id); continue; @@ -1178,6 +1183,15 @@ static void write_libraries(WriteData *wd, Main *bmain) if (library.packedfile) { should_write_library = true; } + else if (!library.runtime->archived_libraries.is_empty()) { + /* Reference 'real' blendfile library of archived 'copies' of it containing packed linked + * IDs should always be written. */ + /* FIXME: A bit weak, as it could be that all archive libs are now empty (if all related + * packed linked IDs have been deleted e.g.)... + * Could be fixed by either adding more checks here, or ensuring empty archive libs are + * deleted when no ID uses them anymore? */ + should_write_library = true; + } else if (wd->use_memfile) { /* When writing undo step we always write all existing libraries. That makes reading undo * step much easier when dealing with purely indirectly used libraries. */ @@ -1194,16 +1208,22 @@ static void write_libraries(WriteData *wd, Main *bmain) write_id(wd, &library.id); - /* Write placeholders for linked data-blocks that are used. */ + /* Write placeholders for linked data-blocks that are used, and real IDs for the packed linked + * ones. */ for (ID *id : ids_used_from_library) { - if (!BKE_idtype_idcode_is_linkable(GS(id->name))) { - CLOG_ERROR(&LOG, - "Data-block '%s' from lib '%s' is not linkable, but is flagged as " - "directly linked", - id->name, - library.runtime->filepath_abs); + if (ID_IS_PACKED(id)) { + write_id(wd, id); + } + else { + if (!BKE_idtype_idcode_is_linkable(GS(id->name))) { + CLOG_ERROR(&LOG, + "Data-block '%s' from lib '%s' is not linkable, but is flagged as " + "directly linked", + id->name, + library.runtime->filepath_abs); + } + write_id_placeholder(wd, id); } - write_id_placeholder(wd, id); } } diff --git a/source/blender/editors/asset/intern/asset_import.cc b/source/blender/editors/asset/intern/asset_import.cc index 4a9380469ac..6cf765d644e 100644 --- a/source/blender/editors/asset/intern/asset_import.cc +++ b/source/blender/editors/asset/intern/asset_import.cc @@ -32,11 +32,14 @@ ID *asset_local_id_ensure_imported(Main &bmain, } const eAssetImportMethod method = [&]() { + const bool no_packing = U.experimental.no_data_block_packing; if (import_method) { - return *import_method; + return (no_packing && *import_method == ASSET_IMPORT_PACK) ? ASSET_IMPORT_APPEND_REUSE : + *import_method; } if (std::optional asset_method = asset.get_import_method()) { - return *asset_method; + return (no_packing && *asset_method == ASSET_IMPORT_PACK) ? ASSET_IMPORT_APPEND_REUSE : + *asset_method; } return ASSET_IMPORT_APPEND_REUSE; }(); @@ -51,6 +54,16 @@ ID *asset_local_id_ensure_imported(Main &bmain, asset.get_id_type(), asset.get_name().c_str(), (asset.get_use_relative_path() ? FILE_RELPATH : 0)); + case ASSET_IMPORT_PACK: + return WM_file_link_datablock(&bmain, + nullptr, + nullptr, + nullptr, + blend_path.c_str(), + asset.get_id_type(), + asset.get_name().c_str(), + BLO_LIBLINK_PACK | + (asset.get_use_relative_path() ? FILE_RELPATH : 0)); case ASSET_IMPORT_APPEND: return WM_file_append_datablock(&bmain, nullptr, diff --git a/source/blender/editors/asset/intern/asset_shelf_asset_view.cc b/source/blender/editors/asset/intern/asset_shelf_asset_view.cc index 5c628f93c87..88c37db87d9 100644 --- a/source/blender/editors/asset/intern/asset_shelf_asset_view.cc +++ b/source/blender/editors/asset/intern/asset_shelf_asset_view.cc @@ -406,8 +406,11 @@ void *AssetDragController::create_drag_data() const return static_cast(local_id); } - const eAssetImportMethod import_method = asset_.get_import_method().value_or( - ASSET_IMPORT_APPEND_REUSE); + eAssetImportMethod import_method = asset_.get_import_method().value_or(ASSET_IMPORT_PACK); + if (U.experimental.no_data_block_packing && import_method == ASSET_IMPORT_PACK) { + import_method = ASSET_IMPORT_APPEND_REUSE; + } + AssetImportSettings import_settings{}; import_settings.method = import_method; import_settings.use_instance_collections = false; diff --git a/source/blender/editors/interface/interface_icons.cc b/source/blender/editors/interface/interface_icons.cc index d81277205ed..767b74c42cd 100644 --- a/source/blender/editors/interface/interface_icons.cc +++ b/source/blender/editors/interface/interface_icons.cc @@ -1962,6 +1962,9 @@ int ui_id_icon_get(const bContext *C, ID *id, const bool big) int UI_icon_from_library(const ID *id) { if (ID_IS_LINKED(id)) { + if (ID_IS_PACKED(id)) { + return ICON_PACKAGE; + } if (id->tag & ID_TAG_MISSING) { return ICON_LIBRARY_DATA_BROKEN; } diff --git a/source/blender/editors/interface/interface_ops.cc b/source/blender/editors/interface/interface_ops.cc index b46ea31a2ce..36ec88cae43 100644 --- a/source/blender/editors/interface/interface_ops.cc +++ b/source/blender/editors/interface/interface_ops.cc @@ -717,6 +717,10 @@ static bool override_idtemplate_poll(bContext *C, const bool is_create_op) return false; } + if (ID_IS_PACKED(id)) { + return false; + } + if (is_create_op) { if (!ID_IS_LINKED(id) && !ID_IS_OVERRIDE_LIBRARY_REAL(id)) { return false; diff --git a/source/blender/editors/interface/templates/interface_template_id.cc b/source/blender/editors/interface/templates/interface_template_id.cc index a75a6bae274..4c6309f6120 100644 --- a/source/blender/editors/interface/templates/interface_template_id.cc +++ b/source/blender/editors/interface/templates/interface_template_id.cc @@ -1119,7 +1119,21 @@ static void template_ID(const bContext *C, if (!hide_buttons && !(idfrom && ID_IS_LINKED(idfrom))) { if (ID_IS_LINKED(id)) { const bool disabled = !BKE_idtype_idcode_is_localizable(GS(id->name)); - if (id->tag & ID_TAG_INDIRECT) { + if (ID_IS_PACKED(id)) { + but = uiDefIconBut(block, + ButType::But, + 0, + ICON_PACKAGE, + 0, + 0, + UI_UNIT_X, + UI_UNIT_Y, + nullptr, + 0, + 0, + TIP_("Packed library data-block, click to unpack and make local")); + } + else if (id->tag & ID_TAG_INDIRECT) { but = uiDefIconBut(block, ButType::But, 0, diff --git a/source/blender/editors/screen/screen_ops.cc b/source/blender/editors/screen/screen_ops.cc index fb16e037a2f..86e79a5250f 100644 --- a/source/blender/editors/screen/screen_ops.cc +++ b/source/blender/editors/screen/screen_ops.cc @@ -6959,6 +6959,8 @@ static std::string screen_drop_scene_tooltip(bContext * /*C*/, switch (asset_drag->import_settings.method) { case ASSET_IMPORT_LINK: return fmt::format(fmt::runtime(TIP_("Link {}")), dragged_scene_name); + case ASSET_IMPORT_PACK: + return fmt::format(fmt::runtime(TIP_("Pack {}")), dragged_scene_name); case ASSET_IMPORT_APPEND: return fmt::format(fmt::runtime(TIP_("Append {}")), dragged_scene_name); case ASSET_IMPORT_APPEND_REUSE: diff --git a/source/blender/editors/space_file/file_draw.cc b/source/blender/editors/space_file/file_draw.cc index 42532d1172b..f3e24c262af 100644 --- a/source/blender/editors/space_file/file_draw.cc +++ b/source/blender/editors/space_file/file_draw.cc @@ -410,7 +410,7 @@ static void file_but_enable_drag(uiBut *but, import_settings.method = eAssetImportMethod(import_method); import_settings.use_instance_collections = (sfile->asset_params->import_flags & - (import_method == ASSET_IMPORT_LINK ? + (ELEM(import_method, ASSET_IMPORT_LINK, ASSET_IMPORT_PACK) ? FILE_ASSET_IMPORT_INSTANCE_COLLECTIONS_ON_LINK : FILE_ASSET_IMPORT_INSTANCE_COLLECTIONS_ON_APPEND)) != 0; diff --git a/source/blender/editors/space_file/filesel.cc b/source/blender/editors/space_file/filesel.cc index 43225c544b7..1ea7cc7cc99 100644 --- a/source/blender/editors/space_file/filesel.cc +++ b/source/blender/editors/space_file/filesel.cc @@ -535,6 +535,8 @@ int ED_fileselect_asset_import_method_get(const SpaceFile *sfile, const FileDirE return ASSET_IMPORT_APPEND; case FILE_ASSET_IMPORT_APPEND_REUSE: return ASSET_IMPORT_APPEND_REUSE; + case FILE_ASSET_IMPORT_PACK: + return ASSET_IMPORT_PACK; /* Should be handled above already. Break and fail below. */ case FILE_ASSET_IMPORT_FOLLOW_PREFS: diff --git a/source/blender/editors/space_outliner/outliner_draw.cc b/source/blender/editors/space_outliner/outliner_draw.cc index 7d87187e3ed..4c17bf873d8 100644 --- a/source/blender/editors/space_outliner/outliner_draw.cc +++ b/source/blender/editors/space_outliner/outliner_draw.cc @@ -2571,6 +2571,9 @@ static BIFIconID tree_element_get_icon_from_id(const ID *id) if (id->tag & ID_TAG_MISSING) { return ICON_LIBRARY_DATA_BROKEN; } + else if (reinterpret_cast(id)->flag & LIBRARY_FLAG_IS_ARCHIVE) { + return ICON_PACKAGE; + } else if (((Library *)id)->runtime->parent) { return ICON_LIBRARY_DATA_INDIRECT; } @@ -2875,8 +2878,16 @@ TreeElementIcon tree_element_get_icon(TreeStoreElem *tselem, TreeElement *te) const PointerRNA &ptr = te_rna_struct->get_pointer_rna(); if (RNA_struct_is_ID(ptr.type)) { - data.drag_id = static_cast(ptr.data); - data.icon = RNA_struct_ui_icon(ptr.type); + ID *id = static_cast(ptr.data); + data.drag_id = id; + if (id && GS(id->name) == ID_LI && + id_cast(id)->flag & LIBRARY_FLAG_IS_ARCHIVE) + { + data.icon = ICON_PACKAGE; + } + else { + data.icon = RNA_struct_ui_icon(ptr.type); + } } else { data.icon = RNA_struct_ui_icon(ptr.type); diff --git a/source/blender/editors/space_outliner/tree/tree_display_libraries.cc b/source/blender/editors/space_outliner/tree/tree_display_libraries.cc index ff90f065789..42e7ab00770 100644 --- a/source/blender/editors/space_outliner/tree/tree_display_libraries.cc +++ b/source/blender/editors/space_outliner/tree/tree_display_libraries.cc @@ -71,12 +71,14 @@ ListBase TreeDisplayLibraries::build_tree(const TreeSourceData &source_data) TreeStoreElem *tselem = TREESTORE(ten); Library *lib = (Library *)tselem->id; BLI_assert(!lib || (GS(lib->id.name) == ID_LI)); - if (!lib || !lib->runtime->parent) { + if (!lib || !(lib->runtime->parent || lib->archive_parent_library)) { continue; } /* A library with a non-null `parent` is always strictly indirectly linked. */ - TreeElement *parent = reinterpret_cast(lib->runtime->parent->id.newid); + TreeElement *parent = reinterpret_cast( + (lib->archive_parent_library ? lib->archive_parent_library : lib->runtime->parent) + ->id.newid); BLI_remlink(&tree, ten); BLI_addtail(&parent->subtree, ten); ten->parent = parent; diff --git a/source/blender/makesdna/DNA_ID.h b/source/blender/makesdna/DNA_ID.h index 55e96221bfe..0d3ae436616 100644 --- a/source/blender/makesdna/DNA_ID.h +++ b/source/blender/makesdna/DNA_ID.h @@ -18,6 +18,7 @@ /** Workaround to forward-declare C++ type in C header. */ #ifdef __cplusplus +# include # include namespace blender::bke::id { @@ -43,10 +44,6 @@ typedef struct IDPropertyGroupChildrenSet IDPropertyGroupChildrenSet; typedef struct ID_RuntimeHandle ID_RuntimeHandle; #endif -#ifdef __cplusplus -extern "C" { -#endif - struct FileData; struct GHash; struct ID; @@ -383,6 +380,37 @@ enum { ID_REMAP_IS_USER_ONE_SKIPPED = 1 << 1, }; +typedef struct IDHash { + char data[16]; + +#ifdef __cplusplus + uint64_t hash() const + { + return *reinterpret_cast(this->data); + } + + static constexpr IDHash get_null() + { + return {}; + } + bool is_null() const + { + return *this == IDHash::get_null(); + } + + friend bool operator==(const IDHash &a, const IDHash &b) + { + return memcmp(a.data, b.data, sizeof(a.data)) == 0; + } + + friend bool operator!=(const IDHash &a, const IDHash &b) + { + return !(a == b); + } + +#endif +} IDHash; + typedef struct ID { /* There's a nasty circular dependency here.... 'void *' to the rescue! I * really wonder why this is needed. */ @@ -433,6 +461,18 @@ typedef struct ID { */ unsigned int session_uid; + /** + * This is only available on packed linked data-blocks. It is a hash of the contents the + * data-block including all its dependencies. It is computed when first packing the data-block + * and is not changed afterwards. It can be used to detect that packed data-blocks in two + * separate .blend files are the same. + * + * Two data-blocks with the same deep hash are assumed to be interchangeable, but not necessarily + * exactly the same. For example, it's possible to change node positions on packed data-blocks + * without changing the deep hash. + */ + IDHash deep_hash; + /** * User-defined custom properties storage. Typically Accessed through the 'dict' syntax from * Python. @@ -511,6 +551,24 @@ typedef struct Library { /** Path name used for reading, can be relative and edited in the outliner. */ char filepath[/*FILE_MAX*/ 1024]; + /** Flags defining specific characteristics of a library. See #LibraryFlag. */ + uint16_t flag; + char _pad[6]; + + /** + * For archive library only (#LIBRARY_FLAG_IS_ARCHIVE): The main library owning it. + * + * `archive_parent_library` and `packedfile` should never be both non-null in a same Library ID. + */ + struct Library *archive_parent_library; + + /** + * Packed blendfile of the library, nullptr if not packed. + * + * \note Individual IDs may be packed even if the entire library is not packed. + * + * `archive_parent_library` and `packedfile` should never be both non-null in a same Library ID. + */ struct PackedFile *packedfile; /** @@ -519,8 +577,22 @@ typedef struct Library { * Typically allocated when creating a new Library or reading it from a blendfile. */ LibraryRuntimeHandle *runtime; + + void *_pad2; } Library; +/** + * #Library.flag + * + * Some of these flags define a 'virtual' library, which may not be an actual blendfile, store + * 'archived' embedded data, etc. IDs contained in these virtual libraries are _not_ managed by + * regular linking code. + */ +enum LibraryFlag { + /** The library is an 'archive' that only contains embedded linked data. */ + LIBRARY_FLAG_IS_ARCHIVE = 1 << 0, +}; + /** * A weak library/ID reference for local data that has been appended, to allow re-using that local * data instead of creating a new copy of it in future appends. @@ -618,6 +690,12 @@ typedef struct PreviewImage { #define ID_MISSING(_id) ((((const ID *)(_id))->tag & ID_TAG_MISSING) != 0) #define ID_IS_LINKED(_id) (((const ID *)(_id))->lib != NULL) +/** + * Indicates that this ID is linked but also packed into the current .blend file. Note that this + * just means that this specific ID and its dependencies are packed, not the entire library. So + * this is separate from #Library::packedfile. + */ +#define ID_IS_PACKED(_id) (ID_IS_LINKED(_id) && ((_id)->flag & ID_FLAG_LINKED_AND_PACKED)) #define ID_TYPE_SUPPORTS_ASSET_EDITABLE(id_type) \ ELEM(id_type, ID_BR, ID_TE, ID_NT, ID_IM, ID_PC, ID_MA) @@ -717,6 +795,11 @@ enum { * so it must be treated as dirty. */ ID_FLAG_CLIPBOARD_MARK = 1 << 14, + /** + * Indicates that this linked ID is packed into the current .blend file. This should never be set + * on local ID (without)one with a null `ID::lib` pointer). + */ + ID_FLAG_LINKED_AND_PACKED = 1 << 15, }; /** @@ -1274,10 +1357,6 @@ typedef enum eID_Index { #define INDEX_ID_MAX (INDEX_ID_NULL + 1) -#ifdef __cplusplus -} -#endif - #ifdef __cplusplus namespace blender::dna { namespace detail { diff --git a/source/blender/makesdna/DNA_asset_types.h b/source/blender/makesdna/DNA_asset_types.h index 829eaaae7f1..6ad8b21a8bb 100644 --- a/source/blender/makesdna/DNA_asset_types.h +++ b/source/blender/makesdna/DNA_asset_types.h @@ -118,6 +118,8 @@ typedef enum eAssetImportMethod { * heavy data dependencies (e.g. the image data-blocks of a material, the mesh of an object) may * be reused from an earlier append. */ ASSET_IMPORT_APPEND_REUSE = 2, + /** Link data-block, but also pack it as read-only data. */ + ASSET_IMPORT_PACK = 3, } eAssetImportMethod; # diff --git a/source/blender/makesdna/DNA_object_types.h b/source/blender/makesdna/DNA_object_types.h index 30a99ab9377..1be3a9769c9 100644 --- a/source/blender/makesdna/DNA_object_types.h +++ b/source/blender/makesdna/DNA_object_types.h @@ -227,7 +227,6 @@ typedef struct Object { bAnimVizSettings avs; /** Motion path cache for this object. */ bMotionPath *mpath; - void *_pad0; ListBase effect DNA_DEPRECATED; /* XXX deprecated... keep for readfile */ ListBase defbase DNA_DEPRECATED; /* Only for versioning, moved to object data. */ diff --git a/source/blender/makesdna/DNA_particle_types.h b/source/blender/makesdna/DNA_particle_types.h index 0f1204019e8..0c970b4a09a 100644 --- a/source/blender/makesdna/DNA_particle_types.h +++ b/source/blender/makesdna/DNA_particle_types.h @@ -298,7 +298,6 @@ typedef struct ParticleSettings { float rad_root, rad_tip, rad_scale; struct CurveMapping *twistcurve; - void *_pad7; } ParticleSettings; typedef struct ParticleSystem { diff --git a/source/blender/makesdna/DNA_space_enums.h b/source/blender/makesdna/DNA_space_enums.h index 77bbe6455a9..0b36f64c1ca 100644 --- a/source/blender/makesdna/DNA_space_enums.h +++ b/source/blender/makesdna/DNA_space_enums.h @@ -477,6 +477,11 @@ typedef enum eFileAssetImportMethod { FILE_ASSET_IMPORT_APPEND_REUSE = 2, /** Default: Follow the preference setting for this asset library. */ FILE_ASSET_IMPORT_FOLLOW_PREFS = 3, + /** + * Link the data-block, but also pack it in the current file to keep it working even if the + * source file is not available anymore. + */ + FILE_ASSET_IMPORT_PACK = 4, } eFileAssetImportMethod; typedef enum eFileAssetImportFlags { diff --git a/source/blender/makesdna/DNA_userdef_types.h b/source/blender/makesdna/DNA_userdef_types.h index adfac3f84af..ec1092ad752 100644 --- a/source/blender/makesdna/DNA_userdef_types.h +++ b/source/blender/makesdna/DNA_userdef_types.h @@ -224,6 +224,7 @@ typedef struct UserDef_Experimental { char use_extensions_debug; char use_recompute_usercount_on_save_debug; char write_legacy_blend_file_format; + char no_data_block_packing; char SANITIZE_AFTER_HERE; /* The following options are automatically sanitized (set to 0) * when the release cycle is not alpha. */ @@ -233,7 +234,7 @@ typedef struct UserDef_Experimental { char use_new_volume_nodes; char use_shader_node_previews; char use_geometry_nodes_lists; - char _pad[6]; + char _pad[5]; } UserDef_Experimental; #define USER_EXPERIMENTAL_TEST(userdef, member) (((userdef)->experimental).member) diff --git a/source/blender/makesrna/intern/rna_ID.cc b/source/blender/makesrna/intern/rna_ID.cc index d8c3c635d37..85c2f13a45f 100644 --- a/source/blender/makesrna/intern/rna_ID.cc +++ b/source/blender/makesrna/intern/rna_ID.cc @@ -300,7 +300,12 @@ static int rna_ID_name_editable(const PointerRNA *ptr, const char **r_info) * and could be useful in some cases. */ if (!ID_IS_EDITABLE(id)) { if (r_info) { - *r_info = N_("Linked data-blocks cannot be renamed"); + if (ID_IS_PACKED(id)) { + *r_info = N_("Packed data-blocks cannot be renamed"); + } + else { + *r_info = N_("Linked data-blocks cannot be renamed"); + } } return 0; } @@ -1539,6 +1544,52 @@ static void rna_Library_version_get(PointerRNA *ptr, int *value) value[2] = lib->runtime->subversionfile; } +static void rna_Library_archive_libraries_begin(CollectionPropertyIterator *iter, PointerRNA *ptr) +{ + iter->parent = *ptr; + Library *lib = static_cast(ptr->data); + + Library **archive_libraries_iter = lib->runtime->archived_libraries.begin(); + iter->internal.custom = archive_libraries_iter; + + iter->valid = archive_libraries_iter != lib->runtime->archived_libraries.end(); +} + +static void rna_Library_archive_libraries_next(CollectionPropertyIterator *iter) +{ + Library *lib = static_cast(iter->parent.data); + Library **archive_libraries_iter = static_cast(iter->internal.custom); + + archive_libraries_iter++; + iter->internal.custom = archive_libraries_iter; + + iter->valid = archive_libraries_iter != lib->runtime->archived_libraries.end(); +} + +static PointerRNA rna_Library_archive_libraries_get(CollectionPropertyIterator *iter) +{ + Library **archive_libraries_iter = static_cast(iter->internal.custom); + return RNA_pointer_create_with_parent(iter->parent, &RNA_Library, *archive_libraries_iter); +} + +static int rna_Library_archive_libraries_length(PointerRNA *ptr) +{ + Library *lib = static_cast(ptr->data); + return int(lib->runtime->archived_libraries.size()); +} + +static bool rna_Library_archive_libraries_lookupint(PointerRNA *ptr, int key, PointerRNA *r_ptr) +{ + Library *lib = static_cast(ptr->data); + if (key < 0 || key >= lib->runtime->archived_libraries.size()) { + return false; + } + + Library *archive_library = lib->runtime->archived_libraries[key]; + rna_pointer_create_with_ancestors(*ptr, &RNA_Library, archive_library, *r_ptr); + return true; +} + static PointerRNA rna_Library_parent_get(PointerRNA *ptr) { Library *lib = ptr->data_as(); @@ -2345,6 +2396,12 @@ static void rna_def_ID(BlenderRNA *brna) "This data-block is not an independent one, but is actually a sub-data of another ID " "(typical example: root node trees or master collections)"); + prop = RNA_def_property(srna, "is_linked_packed", PROP_BOOLEAN, PROP_NONE); + RNA_def_property_boolean_sdna(prop, nullptr, "flag", ID_FLAG_LINKED_AND_PACKED); + RNA_def_property_clear_flag(prop, PROP_EDITABLE); + RNA_def_property_ui_text( + prop, "Linked Packed", "This data-block is linked and packed into the .blend file"); + prop = RNA_def_property(srna, "is_missing", PROP_BOOLEAN, PROP_NONE); RNA_def_property_boolean_sdna(prop, nullptr, "tag", ID_TAG_MISSING); RNA_def_property_clear_flag(prop, PROP_EDITABLE); @@ -2667,6 +2724,39 @@ static void rna_def_library(BlenderRNA *brna) "Data-blocks in this library are editable despite being linked. " "Used by brush assets and their dependencies."); + prop = RNA_def_property(srna, "is_archive", PROP_BOOLEAN, PROP_NONE); + RNA_def_property_boolean_sdna(prop, nullptr, "flag", LIBRARY_FLAG_IS_ARCHIVE); + RNA_def_property_clear_flag(prop, PROP_EDITABLE); + RNA_def_property_ui_text(prop, + "Is Archive", + "This library is an 'archive' storage for packed linked IDs " + "originally linked from its 'archive parent' library."); + + prop = RNA_def_property(srna, "archive_parent_library", PROP_POINTER, PROP_NONE); + RNA_def_property_pointer_sdna(prop, nullptr, "archive_parent_library"); + RNA_def_property_struct_type(prop, "Library"); + RNA_def_property_override_flag(prop, PROPOVERRIDE_NO_COMPARISON); + RNA_def_property_ui_text(prop, + "Parent Archive Library", + "Source library from which this archive of packed IDs was generated"); + + prop = RNA_def_property(srna, "archive_libraries", PROP_COLLECTION, PROP_NONE); + RNA_def_property_struct_type(prop, "Library"); + RNA_def_property_collection_funcs(prop, + "rna_Library_archive_libraries_begin", + "rna_Library_archive_libraries_next", + nullptr, + "rna_Library_archive_libraries_get", + "rna_Library_archive_libraries_length", + "rna_Library_archive_libraries_lookupint", + nullptr, + nullptr); + RNA_def_property_override_flag(prop, PROPOVERRIDE_NO_COMPARISON); + RNA_def_property_ui_text( + prop, + "Archive Libraries", + "Archive libraries of packed IDs, generated (and owned) by this source library"); + func = RNA_def_function(srna, "reload", "rna_Library_reload"); RNA_def_function_flag(func, FUNC_USE_REPORTS | FUNC_USE_CONTEXT); RNA_def_function_ui_description(func, "Reload this library and all its linked data-blocks"); diff --git a/source/blender/makesrna/intern/rna_main_api.cc b/source/blender/makesrna/intern/rna_main_api.cc index 395d6d24303..0efb058b60f 100644 --- a/source/blender/makesrna/intern/rna_main_api.cc +++ b/source/blender/makesrna/intern/rna_main_api.cc @@ -34,6 +34,7 @@ # include "BKE_image.hh" # include "BKE_lattice.hh" # include "BKE_lib_remap.hh" +# include "BKE_library.hh" # include "BKE_light.h" # include "BKE_lightprobe.h" # include "BKE_linestyle.h" @@ -146,6 +147,28 @@ static void rna_Main_ID_remove(Main *bmain, } } +static ID *rna_Main_pack_linked_ids_hierarchy(struct BlendData *blenddata, + ReportList *reports, + ID *root_id) +{ + if (!ID_IS_LINKED(root_id)) { + BKE_reportf(reports, RPT_ERROR, "Only linked IDs can be packed"); + return nullptr; + } + if (ID_IS_PACKED(root_id)) { + /* Nothing to do. */ + return root_id; + } + + Main *bmain = reinterpret_cast
(blenddata); + blender::bke::library::pack_linked_id_hierarchy(*bmain, *root_id); + + ID *packed_root_id = root_id->newid; + BKE_main_id_newptr_and_tag_clear(bmain); + + return packed_root_id; +} + static Camera *rna_Main_cameras_new(Main *bmain, const char *name) { char safe_name[MAX_ID_NAME - 2]; @@ -869,12 +892,12 @@ RNA_MAIN_ID_TAG_FUNCS_DEF(volumes, volumes, ID_VO) #else -void RNA_api_main(StructRNA * /*srna*/) +void RNA_api_main(StructRNA *srna) { -# if 0 FunctionRNA *func; PropertyRNA *parm; +# if 0 /* maybe we want to add functions in 'bpy.data' still? * for now they are all in collections bpy.data.images.new(...) */ func = RNA_def_function(srna, "add_image", "rna_Main_add_image"); @@ -885,6 +908,15 @@ void RNA_api_main(StructRNA * /*srna*/) parm = RNA_def_pointer(func, "image", "Image", "", "New image"); RNA_def_function_return(func, parm); # endif + + func = RNA_def_function(srna, "pack_linked_ids_hierarchy", "rna_Main_pack_linked_ids_hierarchy"); + RNA_def_function_ui_description( + func, "Pack the given linked ID and its dependencies into current blendfile"); + RNA_def_function_flag(func, FUNC_USE_REPORTS); + parm = RNA_def_pointer(func, "root_id", "ID", "", "Root linked ID to pack"); + RNA_def_parameter_flags(parm, PropertyFlag(0), PARM_REQUIRED); + parm = RNA_def_pointer(func, "packed_id", "ID", "", "The packed ID matching the given root ID"); + RNA_def_function_return(func, parm); } void RNA_def_main_cameras(BlenderRNA *brna, PropertyRNA *cprop) diff --git a/source/blender/makesrna/intern/rna_space.cc b/source/blender/makesrna/intern/rna_space.cc index a9561bb9f02..7f6d3bd9769 100644 --- a/source/blender/makesrna/intern/rna_space.cc +++ b/source/blender/makesrna/intern/rna_space.cc @@ -358,6 +358,40 @@ const EnumPropertyItem rna_enum_fileselect_params_sort_items[] = { {0, nullptr, 0, nullptr, nullptr}, }; +static const EnumPropertyItem rna_enum_asset_import_method_items[] = { + {FILE_ASSET_IMPORT_FOLLOW_PREFS, + "FOLLOW_PREFS", + 0, + "Follow Preferences", + "Use the import method set in the Preferences for this asset library, don't override it " + "for this Asset Browser"}, + {FILE_ASSET_IMPORT_LINK, + "LINK", + ICON_LINK_BLEND, + "Link", + "Import the assets as linked data-block"}, + {FILE_ASSET_IMPORT_APPEND, + "APPEND", + ICON_APPEND_BLEND, + "Append", + "Import the asset as copied data-block, with no link to the original asset data-block"}, + {FILE_ASSET_IMPORT_APPEND_REUSE, + "APPEND_REUSE", + ICON_APPEND_BLEND, + "Append (Reuse Data)", + "Import the asset as copied data-block while avoiding multiple copies of nested, " + "typically heavy data. For example the textures of a material asset, or the mesh of an " + "object asset, don't have to be copied every time this asset is imported. The instances of " + "the asset share the data instead"}, + {FILE_ASSET_IMPORT_PACK, + "PACK", + ICON_PACKAGE, + "Pack", + "Import the asset as linked data-block, and pack it in the current file (ensures that it " + "remains unchanged in case the library data is modified, is not available anymore, etc.)"}, + {0, nullptr, 0, nullptr, nullptr}, +}; + #ifndef RNA_RUNTIME static const EnumPropertyItem stereo3d_eye_items[] = { {STEREO_LEFT_ID, "LEFT_EYE", ICON_NONE, "Left Eye"}, @@ -3662,6 +3696,37 @@ static void rna_FileAssetSelectParams_catalog_id_set(PointerRNA *ptr, const char params->asset_catalog_visibility = FILE_SHOW_ASSETS_FROM_CATALOG; } +static const EnumPropertyItem *rna_FileAssetSelectParams_import_method_itemf( + bContext * /*C*/, PointerRNA * /*ptr*/, PropertyRNA * /*prop*/, bool *r_free) +{ + EnumPropertyItem *items = nullptr; + int items_num = 0; + for (const EnumPropertyItem *item = rna_enum_asset_import_method_items; item->identifier; item++) + { + switch (eFileAssetImportMethod(item->value)) { + case FILE_ASSET_IMPORT_APPEND_REUSE: { + if (U.experimental.no_data_block_packing) { + RNA_enum_item_add(&items, &items_num, item); + } + break; + } + case FILE_ASSET_IMPORT_PACK: { + if (!U.experimental.no_data_block_packing) { + RNA_enum_item_add(&items, &items_num, item); + } + break; + } + default: { + RNA_enum_item_add(&items, &items_num, item); + break; + } + } + } + RNA_enum_item_end(&items, &items_num); + *r_free = true; + return items; +} + #else static const EnumPropertyItem dt_uv_items[] = { @@ -7426,30 +7491,6 @@ static void rna_def_fileselect_asset_params(BlenderRNA *brna) StructRNA *srna; PropertyRNA *prop; - static const EnumPropertyItem asset_import_method_items[] = { - {FILE_ASSET_IMPORT_FOLLOW_PREFS, - "FOLLOW_PREFS", - 0, - "Follow Preferences", - "Use the import method set in the Preferences for this asset library, don't override it " - "for this Asset Browser"}, - {FILE_ASSET_IMPORT_LINK, "LINK", 0, "Link", "Import the assets as linked data-block"}, - {FILE_ASSET_IMPORT_APPEND, - "APPEND", - 0, - "Append", - "Import the assets as copied data-block, with no link to the original asset data-block"}, - {FILE_ASSET_IMPORT_APPEND_REUSE, - "APPEND_REUSE", - 0, - "Append (Reuse Data)", - "Import the assets as copied data-block while avoiding multiple copies of nested, " - "typically heavy data. For example the textures of a material asset, or the mesh of an " - "object asset, don't have to be copied every time this asset is imported. The instances of " - "the asset share the data instead"}, - {0, nullptr, 0, nullptr, nullptr}, - }; - srna = RNA_def_struct(brna, "FileAssetSelectParams", "FileSelectParams"); RNA_def_struct_ui_text( srna, "Asset Select Parameters", "Settings for the file selection in Asset Browser mode"); @@ -7478,7 +7519,9 @@ static void rna_def_fileselect_asset_params(BlenderRNA *brna) "Which asset types to show/hide, when browsing an asset library"); prop = RNA_def_property(srna, "import_method", PROP_ENUM, PROP_NONE); - RNA_def_property_enum_items(prop, asset_import_method_items); + RNA_def_property_enum_items(prop, rna_enum_asset_import_method_items); + RNA_def_property_enum_funcs( + prop, nullptr, nullptr, "rna_FileAssetSelectParams_import_method_itemf"); RNA_def_property_ui_text(prop, "Import Method", "Determine how the asset will be imported"); /* Asset drag info saved by buttons stores the import method, so the space must redraw when * import method changes. */ diff --git a/source/blender/makesrna/intern/rna_userdef.cc b/source/blender/makesrna/intern/rna_userdef.cc index 6f73994f7ec..de5100912a2 100644 --- a/source/blender/makesrna/intern/rna_userdef.cc +++ b/source/blender/makesrna/intern/rna_userdef.cc @@ -180,6 +180,30 @@ static const EnumPropertyItem rna_enum_preferences_extension_repo_source_type_it {0, nullptr, 0, nullptr, nullptr}, }; +static const EnumPropertyItem rna_enum_preferences_asset_import_method_items[] = { + {ASSET_IMPORT_LINK, "LINK", ICON_LINK_BLEND, "Link", "Import the assets as linked data-block"}, + {ASSET_IMPORT_APPEND, + "APPEND", + ICON_APPEND_BLEND, + "Append", + "Import the assets as copied data-block, with no link to the original asset data-block"}, + {ASSET_IMPORT_APPEND_REUSE, + "APPEND_REUSE", + ICON_APPEND_BLEND, + "Append (Reuse Data)", + "Import the assets as copied data-block while avoiding multiple copies of nested, " + "typically heavy data. For example the textures of a material asset, or the mesh of an " + "object asset, don't have to be copied every time this asset is imported. The instances of " + "the asset share the data instead."}, + {ASSET_IMPORT_PACK, + "PACK", + ICON_PACKAGE, + "Pack", + "Import the asset as linked data-block, and pack it in the current file (ensures that it " + "remains unchanged in case the library data is modified, is not available anymore, etc.)"}, + {0, nullptr, 0, nullptr, nullptr}, +}; + #ifdef RNA_RUNTIME # include "BLI_math_vector.h" @@ -223,6 +247,8 @@ static const EnumPropertyItem rna_enum_preferences_extension_repo_source_type_it # include "UI_interface.hh" +# include "AS_asset_library.hh" + static void rna_userdef_version_get(PointerRNA *ptr, int *value) { UserDef *userdef = (UserDef *)ptr->data; @@ -360,7 +386,7 @@ static void rna_userdef_asset_library_path_set(PointerRNA *ptr, const char *valu BKE_preferences_asset_library_path_set(library, value); } -static void rna_userdef_asset_library_path_update(bContext *C, PointerRNA *ptr) +static void rna_userdef_asset_library_update(bContext *C, PointerRNA *ptr) { blender::ed::asset::list::clear_all_library(C); rna_userdef_update(CTX_data_main(C), CTX_data_scene(C), ptr); @@ -1482,6 +1508,49 @@ static void rna_preference_gpu_preferred_device_set(PointerRNA *ptr, int value) preferences->gpu_preferred_device_id = 0u; } +static const EnumPropertyItem *rna_preference_asset_libray_import_method_itemf( + bContext * /*C*/, PointerRNA * /*ptr*/, PropertyRNA * /*prop*/, bool *r_free) +{ + EnumPropertyItem *items = nullptr; + int items_num = 0; + for (const EnumPropertyItem *item = rna_enum_preferences_asset_import_method_items; + item->identifier; + item++) + { + switch (eAssetImportMethod(item->value)) { + case ASSET_IMPORT_APPEND_REUSE: { + if (U.experimental.no_data_block_packing) { + RNA_enum_item_add(&items, &items_num, item); + } + break; + } + case ASSET_IMPORT_PACK: { + if (!U.experimental.no_data_block_packing) { + RNA_enum_item_add(&items, &items_num, item); + } + break; + } + default: { + RNA_enum_item_add(&items, &items_num, item); + break; + } + } + } + RNA_enum_item_end(&items, &items_num); + *r_free = true; + return items; +} + +static void rna_experimental_no_data_block_packing_update(bContext *C, PointerRNA *ptr) +{ + Main *bmain = CTX_data_main(C); + Scene *scene = CTX_data_scene(C); + rna_userdef_update(bmain, scene, ptr); + AS_asset_library_import_method_ensure_valid(*bmain); + AS_asset_library_essential_import_method_update(); + rna_userdef_asset_library_update(C, ptr); +} + #else # define USERDEF_TAG_DIRTY_PROPERTY_UPDATE_ENABLE \ @@ -6657,32 +6726,18 @@ static void rna_def_userdef_filepaths_asset_library(BlenderRNA *brna) RNA_def_property_translation_context(prop, BLT_I18NCONTEXT_EDITOR_FILEBROWSER); RNA_def_property_string_funcs(prop, nullptr, nullptr, "rna_userdef_asset_library_path_set"); RNA_def_property_flag(prop, PROP_CONTEXT_UPDATE); - RNA_def_property_update(prop, 0, "rna_userdef_asset_library_path_update"); + RNA_def_property_update(prop, 0, "rna_userdef_asset_library_update"); - static const EnumPropertyItem import_method_items[] = { - {ASSET_IMPORT_LINK, "LINK", 0, "Link", "Import the assets as linked data-block"}, - {ASSET_IMPORT_APPEND, - "APPEND", - 0, - "Append", - "Import the assets as copied data-block, with no link to the original asset data-block"}, - {ASSET_IMPORT_APPEND_REUSE, - "APPEND_REUSE", - 0, - "Append (Reuse Data)", - "Import the assets as copied data-block while avoiding multiple copies of nested, " - "typically heavy data. For example the textures of a material asset, or the mesh of an " - "object asset, don't have to be copied every time this asset is imported. The instances of " - "the asset share the data instead."}, - {0, nullptr, 0, nullptr, nullptr}, - }; prop = RNA_def_property(srna, "import_method", PROP_ENUM, PROP_NONE); - RNA_def_property_enum_items(prop, import_method_items); + RNA_def_property_enum_items(prop, rna_enum_preferences_asset_import_method_items); + RNA_def_property_enum_funcs( + prop, nullptr, nullptr, "rna_preference_asset_libray_import_method_itemf"); RNA_def_property_ui_text( prop, "Default Import Method", "Determine how the asset will be imported, unless overridden by the Asset Browser"); - RNA_def_property_update(prop, 0, "rna_userdef_update"); + RNA_def_property_flag(prop, PROP_CONTEXT_UPDATE); + RNA_def_property_update(prop, 0, "rna_userdef_asset_library_update"); prop = RNA_def_property(srna, "use_relative_path", PROP_BOOLEAN, PROP_NONE); RNA_def_property_boolean_sdna(prop, nullptr, "flag", ASSET_LIBRARY_RELATIVE_PATH); @@ -7308,6 +7363,13 @@ static void rna_def_userdef_experimental(BlenderRNA *brna) "Use file format used before Blender 5.0. This format is more limited " "but it may have better compatibility with tools that don't support the new format yet"); + prop = RNA_def_property(srna, "no_data_block_packing", PROP_BOOLEAN, PROP_NONE); + RNA_def_property_boolean_sdna(prop, nullptr, "no_data_block_packing", 1); + RNA_def_property_ui_text( + prop, "No Data-Block Packing", "Fall-back to appending instead of packing data-blocks"); + RNA_def_property_flag(prop, PROP_CONTEXT_UPDATE); + RNA_def_property_update(prop, 0, "rna_experimental_no_data_block_packing_update"); + prop = RNA_def_property(srna, "use_all_linked_data_direct", PROP_BOOLEAN, PROP_NONE); RNA_def_property_ui_text( prop, diff --git a/source/blender/python/intern/bpy_library_load.cc b/source/blender/python/intern/bpy_library_load.cc index dd05798f941..c5041e43f64 100644 --- a/source/blender/python/intern/bpy_library_load.cc +++ b/source/blender/python/intern/bpy_library_load.cc @@ -223,6 +223,9 @@ PyDoc_STRVAR( " :type filepath: str | bytes\n" " :arg link: When False reference to the original file is lost.\n" " :type link: bool\n" + " :arg pack: If True, and ``link`` is also True, pack linked data-blocks into the current " + "blend-file.\n" + " :type pack: bool\n" " :arg relative: When True the path is stored relative to the open blend file.\n" " :type relative: bool\n" " :arg set_fake: If True, set fake user on appended IDs.\n" @@ -260,6 +263,7 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k */ struct { BoolFlagPair is_link = {false, FILE_LINK}; + BoolFlagPair is_pack = {false, BLO_LIBLINK_PACK}; BoolFlagPair is_relative = {false, FILE_RELPATH}; BoolFlagPair set_fake = {false, BLO_LIBLINK_APPEND_SET_FAKEUSER}; BoolFlagPair recursive = {false, BLO_LIBLINK_APPEND_RECURSIVE}; @@ -279,6 +283,7 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k static const char *_keywords[] = { "filepath", "link", + "pack", "relative", "set_fake", "recursive", @@ -296,6 +301,7 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k /* Optional keyword only arguments. */ "|$" "O&" /* `link` */ + "O&" /* `pack` */ "O&" /* `relative` */ "O&" /* `recursive` */ "O&" /* `set_fake` */ @@ -317,6 +323,8 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k PyC_ParseBool, &flag_vars.is_link, PyC_ParseBool, + &flag_vars.is_pack, + PyC_ParseBool, &flag_vars.is_relative, PyC_ParseBool, &flag_vars.recursive, @@ -389,10 +397,18 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k PyErr_SetString(PyExc_ValueError, "`link` is False but `create_liboverrides` is True"); return nullptr; } + if (flag_vars.is_pack.value) { + PyErr_SetString(PyExc_ValueError, "`pack` must be False if `link` is False"); + return nullptr; + } } if (create_liboverrides) { /* Library overrides. */ + if (flag_vars.is_pack.value) { + PyErr_SetString(PyExc_ValueError, "`create_liboverrides` must be False if `pack` is True"); + return nullptr; + } } else { /* Library overrides (disabled). */ @@ -639,6 +655,7 @@ static bool bpy_lib_exit_lapp_context_items_cb(BlendfileLinkAppendContext *lapp_ static PyObject *bpy_lib_exit(BPy_Library *self, PyObject * /*args*/) { Main *bmain = self->bmain; + const bool do_pack = ((self->flag & BLO_LIBLINK_PACK) != 0); const bool do_append = ((self->flag & FILE_LINK) == 0); const bool create_liboverrides = self->create_liboverrides; /* Code in #bpy_lib_load should have raised exception in case of incompatible parameter values. @@ -709,7 +726,10 @@ static PyObject *bpy_lib_exit(BPy_Library *self, PyObject * /*args*/) BKE_blendfile_link_append_context_init_done(lapp_context); BKE_blendfile_link(lapp_context, nullptr); - if (do_append) { + if (do_pack) { + BKE_blendfile_link_pack(lapp_context, nullptr); + } + else if (do_append) { BKE_blendfile_append(lapp_context, nullptr); } else if (create_liboverrides) { diff --git a/source/blender/windowmanager/intern/wm_dragdrop.cc b/source/blender/windowmanager/intern/wm_dragdrop.cc index c2e9d48a43e..c57bbdc044c 100644 --- a/source/blender/windowmanager/intern/wm_dragdrop.cc +++ b/source/blender/windowmanager/intern/wm_dragdrop.cc @@ -753,6 +753,16 @@ ID *WM_drag_asset_id_import(const bContext *C, wmDragAsset *asset_drag, const in idtype, name, flag | (use_relative_path ? FILE_RELPATH : 0)); + case ASSET_IMPORT_PACK: + return WM_file_link_datablock(bmain, + scene, + view_layer, + view3d, + blend_path.c_str(), + idtype, + name, + flag | (use_relative_path ? FILE_RELPATH : 0) | + BLO_LIBLINK_PACK); case ASSET_IMPORT_APPEND: return WM_file_append_datablock(bmain, scene, @@ -787,7 +797,7 @@ bool WM_drag_asset_will_import_linked(const wmDrag *drag) } const wmDragAsset *asset_drag = WM_drag_get_asset_data(drag, 0); - return asset_drag->import_settings.method == ASSET_IMPORT_LINK; + return ELEM(asset_drag->import_settings.method, ASSET_IMPORT_LINK, ASSET_IMPORT_PACK); } ID *WM_drag_get_local_ID_or_import_from_asset(const bContext *C, const wmDrag *drag, int idcode) diff --git a/source/blender/windowmanager/intern/wm_files_link.cc b/source/blender/windowmanager/intern/wm_files_link.cc index 281c78421c3..28fac0f93be 100644 --- a/source/blender/windowmanager/intern/wm_files_link.cc +++ b/source/blender/windowmanager/intern/wm_files_link.cc @@ -725,6 +725,7 @@ static ID *wm_file_link_append_datablock_ex(Main *bmain, BLI_path_cmp(BKE_main_blendfile_path(bmain), filepath) != 0, "Calling code should ensure it does not attempt to link/append from current blendfile"); + const bool do_pack = (flag & BLO_LIBLINK_PACK) != 0; const bool do_append = (flag & FILE_LINK) == 0; /* Tag everything so we can make local only the new datablock. */ BKE_main_id_tag_all(bmain, ID_TAG_PRE_EXISTING, true); @@ -747,7 +748,10 @@ static ID *wm_file_link_append_datablock_ex(Main *bmain, /* Link datablock. */ BKE_blendfile_link(lapp_context, nullptr); - if (do_append) { + if (do_pack) { + BKE_blendfile_link_pack(lapp_context, nullptr); + } + else if (do_append) { BKE_blendfile_append(lapp_context, nullptr); } diff --git a/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1.blend b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1.blend new file mode 100644 index 00000000000..ba5cc3f86b4 --- /dev/null +++ b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1.blend @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edf4656bbac61e8b88c97c4928b922a53137f2a5ef1157db17d62237ce0c39b8 +size 587139 diff --git a/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1_with_hair.blend b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1_with_hair.blend new file mode 100644 index 00000000000..317d56bd240 --- /dev/null +++ b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1_with_hair.blend @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e68ba924578fa50c6ad09f256a1b8e369a87161ee972d298b6da58718ddeb5d +size 2946843 diff --git a/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2.blend b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2.blend new file mode 100644 index 00000000000..d3d47e10047 --- /dev/null +++ b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2.blend @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff82a19c1be5a1213b480106574ce510896a97f21834c4afae6854adc57ff60 +size 594473 diff --git a/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2_with_hair.blend b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2_with_hair.blend new file mode 100644 index 00000000000..e08e90ccff2 --- /dev/null +++ b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2_with_hair.blend @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1566e3e827e619f5c64644c103dc1904fd08d1fcea36044c8f665365e6025e5a +size 2912433 diff --git a/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/both_bodies_with_hair.blend b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/both_bodies_with_hair.blend new file mode 100644 index 00000000000..e63d75d2d7f --- /dev/null +++ b/tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/both_bodies_with_hair.blend @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b97107fbfbb68b2314e1555cf66d56e9c697214233804a14d05c21a6a39330d +size 3889258 diff --git a/tests/python/bl_blendfile_liblink.py b/tests/python/bl_blendfile_liblink.py index cd189c8adef..d0003b0fe25 100644 --- a/tests/python/bl_blendfile_liblink.py +++ b/tests/python/bl_blendfile_liblink.py @@ -486,6 +486,202 @@ class TestBlendLibAppendReuseID(TestBlendLibLinkHelper): self.assertEqual(len(bpy.data.collections), 0) # Scene's master collection is not listed here +class TestBlendLibPackedLinkedID(TestBlendLibLinkHelper): + + def __init__(self, args): + super().__init__(args) + + def test_link_pack_basic(self): + output_dir = self.args.output_dir + output_lib_path = self.init_lib_data_basic() + + # Link of a single Object, and make it packed. + self.reset_blender() + + link_dir = os.path.join(output_lib_path, "Object") + bpy.ops.wm.link(directory=link_dir, filename="LibMesh", instance_object_data=False) + + self.assertEqual(len(bpy.data.libraries), 1) + library = bpy.data.libraries[0] + + self.assertEqual(len(bpy.data.meshes), 1) + for me in bpy.data.meshes: + self.assertEqual(me.library, library) + self.assertEqual(me.users, 1) + self.assertEqual(len(bpy.data.objects), 1) + for ob in bpy.data.objects: + self.assertEqual(ob.library, library) + self.assertEqual(len(bpy.data.collections), 0) # Scene's master collection is not listed here + + ob_packed = bpy.data.pack_linked_ids_hierarchy(bpy.data.objects[0]) + + # Need to ensure that the newly packed linked object is used, and kept in the scene. + bpy.data.scenes[0].collection.objects.link(ob_packed) + + self.assertEqual(len(bpy.data.libraries), 2) + library = bpy.data.libraries[0] + archive_library = bpy.data.libraries[1] + + def check_valid(): + self.assertFalse(library.is_archive) + self.assertEqual(len(library.archive_libraries), 1) + self.assertEqual(library.archive_libraries[0], archive_library) + self.assertTrue(archive_library.is_archive) + self.assertEqual(archive_library.archive_parent_library, library) + + self.assertEqual(len(bpy.data.meshes), 2) + self.assertEqual(bpy.data.meshes[0].library, library) + self.assertEqual(bpy.data.meshes[0].users, 1) + self.assertEqual(bpy.data.meshes[1].library, archive_library) + self.assertEqual(bpy.data.meshes[1].users, 1) + + self.assertEqual(len(bpy.data.objects), 2) + self.assertEqual(bpy.data.objects[0].library, library) + self.assertEqual(bpy.data.objects[0].data, bpy.data.meshes[0]) + self.assertEqual(bpy.data.objects[1].library, archive_library) + self.assertEqual(bpy.data.objects[1].data, bpy.data.meshes[1]) + + check_valid() + + output_work_path = os.path.join(output_dir, self.unique_blendfile_name("blendfile")) + bpy.ops.wm.save_as_mainfile(filepath=output_work_path, check_existing=False, compress=False) + + self.reset_blender() + + bpy.ops.wm.open_mainfile(filepath=output_work_path, load_ui=False) + + self.assertEqual(len(bpy.data.libraries), 2) + library = bpy.data.libraries[0] + archive_library = bpy.data.libraries[1] + + check_valid() + + def test_link_pack_indirect(self): + # Test handling of indirectly linked packed data (when packed in another library), + # packing linked data using other packed linked data, etc. + output_dir = self.args.output_dir + output_lib_path = self.init_lib_data_packed_indirect_lib() + + # Link of a single Object, and make it packed. + self.reset_blender() + + link_dir = os.path.join(output_lib_path, "Object") + bpy.ops.wm.link(directory=link_dir, filename="LibMesh", instance_object_data=False) + + # Directly linked library, indirectly linked one (though empty), and its packed archive version. + self.assertEqual(len(bpy.data.libraries), 3) + library = bpy.data.libraries[0] + library_indirect = bpy.data.libraries[1] + library_indirect_archive = bpy.data.libraries[2] + + def check_valid(): + self.assertFalse(library.is_archive) + self.assertFalse(library_indirect.is_archive) + self.assertTrue(library_indirect_archive.is_archive) + self.assertTrue(library_indirect_archive.name in library_indirect.archive_libraries) + + self.assertEqual(len(bpy.data.images), 1) + for im in bpy.data.images: + self.assertEqual(im.library, library_indirect_archive) + self.assertEqual(im.users, 1) + self.assertTrue(im.is_linked_packed) + + self.assertEqual(len(bpy.data.materials), 1) + for ma in bpy.data.materials: + self.assertEqual(ma.library, library_indirect_archive) + self.assertEqual(ma.users, 1) + self.assertTrue(ma.is_linked_packed) + + self.assertEqual(len(bpy.data.meshes), 1) + for me in bpy.data.meshes: + self.assertEqual(me.library, library) + self.assertEqual(me.users, 1) + + self.assertEqual(len(bpy.data.objects), 1) + for ob in bpy.data.objects: + self.assertEqual(ob.library, library) + + self.assertEqual(len(bpy.data.collections), 0) # Scene's master collection is not listed here + + check_valid() + + output_work_path = os.path.join(output_dir, self.unique_blendfile_name("blendfile")) + bpy.ops.wm.save_as_mainfile(filepath=output_work_path, check_existing=False, compress=False) + + self.reset_blender() + + bpy.ops.wm.open_mainfile(filepath=output_work_path, load_ui=False) + + self.assertEqual(len(bpy.data.libraries), 3) + library = bpy.data.libraries[0] + library_indirect = bpy.data.libraries[1] + library_indirect_archive = bpy.data.libraries[2] + + check_valid() + + ob_packed = bpy.data.pack_linked_ids_hierarchy(bpy.data.objects[0]) + + # Need to ensure that the newly packed linked object is used, and kept in the scene. + bpy.data.scenes[0].collection.objects.link(ob_packed) + + self.assertEqual(len(bpy.data.libraries), 4) + # Due to ID name sorting, the newly ceratedt archive library should be second now, after its parent one. + archive_library = bpy.data.libraries[1] + + def check_valid(): + self.assertFalse(library.is_archive) + self.assertEqual(len(library.archive_libraries), 1) + self.assertEqual(library.archive_libraries[0], archive_library) + self.assertTrue(archive_library.is_archive) + self.assertEqual(archive_library.archive_parent_library, library) + + self.assertFalse(library_indirect.is_archive) + self.assertTrue(library_indirect_archive.is_archive) + self.assertTrue(library_indirect_archive.name in library_indirect.archive_libraries) + + self.assertEqual(len(bpy.data.images), 1) + for im in bpy.data.images: + self.assertEqual(im.library, library_indirect_archive) + self.assertEqual(im.users, 1) + self.assertTrue(im.is_linked_packed) + + self.assertEqual(len(bpy.data.materials), 1) + for ma in bpy.data.materials: + self.assertEqual(ma.library, library_indirect_archive) + self.assertEqual(ma.users, 2) + self.assertTrue(ma.is_linked_packed) + + self.assertEqual(len(bpy.data.meshes), 2) + self.assertEqual(bpy.data.meshes[0].library, library) + self.assertEqual(bpy.data.meshes[0].users, 1) + self.assertEqual(bpy.data.meshes[1].library, archive_library) + self.assertEqual(bpy.data.meshes[1].users, 1) + + self.assertEqual(len(bpy.data.objects), 2) + self.assertEqual(bpy.data.objects[0].library, library) + self.assertEqual(bpy.data.objects[0].data, bpy.data.meshes[0]) + self.assertEqual(bpy.data.objects[1].library, archive_library) + self.assertEqual(bpy.data.objects[1].data, bpy.data.meshes[1]) + + self.assertEqual(len(bpy.data.collections), 0) # Scene's master collection is not listed here + + check_valid() + + bpy.ops.wm.save_as_mainfile(filepath=output_work_path, check_existing=False, compress=False) + + self.reset_blender() + + bpy.ops.wm.open_mainfile(filepath=output_work_path, load_ui=False) + + self.assertEqual(len(bpy.data.libraries), 4) + library = bpy.data.libraries[0] + archive_library = bpy.data.libraries[1] + library_indirect = bpy.data.libraries[2] + library_indirect_archive = bpy.data.libraries[3] + + check_valid() + + class TestBlendLibLibraryReload(TestBlendLibLinkHelper): def __init__(self, args): @@ -610,6 +806,34 @@ class TestBlendLibDataLibrariesLoadLink(TestBlendLibDataLibrariesLoad): self.assertIsNotNone(bpy.data.collections[0].library) +class TestBlendLibDataLibrariesLoadPack(TestBlendLibDataLibrariesLoad): + + def test_libload_pack(self): + output_lib_path = self.do_libload_init() + # Cannot create overrides on packed linked data currently. + self.assertRaises(ValueError, + self.do_libload, filepath=output_lib_path, link=True, pack=True, create_liboverrides=True) + self.do_libload(filepath=output_lib_path, link=True, pack=True, create_liboverrides=False) + + self.assertEqual(len(bpy.data.meshes), 1) + self.assertEqual(len(bpy.data.objects), 1) # This code does no instantiation. + self.assertEqual(len(bpy.data.collections), 1) + # One archive library for the packed data-blocks and the reference library. + self.assertEqual(len(bpy.data.libraries), 2) + + # Packed dat should be owned by archive library. + packed_mesh = bpy.data.meshes[0] + packed_object = bpy.data.objects[0] + packed_collection = bpy.data.collections[0] + link_library = bpy.data.libraries[0] + archive_library = bpy.data.libraries[1] + self.assertEqual(packed_mesh.library, archive_library) + self.assertEqual(packed_object.library, archive_library) + self.assertEqual(packed_collection.library, archive_library) + self.assertTrue(archive_library.is_archive) + self.assertFalse(link_library.is_archive) + + class TestBlendLibDataLibrariesLoadLibOverride(TestBlendLibDataLibrariesLoad): def test_libload_liboverride(self): @@ -741,11 +965,14 @@ TESTS = ( TestBlendLibAppendBasic, TestBlendLibAppendReuseID, + TestBlendLibPackedLinkedID, + TestBlendLibLibraryReload, TestBlendLibLibraryRelocate, TestBlendLibDataLibrariesLoadAppend, TestBlendLibDataLibrariesLoadLink, + TestBlendLibDataLibrariesLoadPack, TestBlendLibDataLibrariesLoadLibOverride, ) diff --git a/tests/python/bl_blendfile_utils.py b/tests/python/bl_blendfile_utils.py index 65bccd54ec9..c369aef550e 100644 --- a/tests/python/bl_blendfile_utils.py +++ b/tests/python/bl_blendfile_utils.py @@ -182,3 +182,52 @@ class TestBlendLibLinkHelper(TestHelper): bpy.ops.wm.save_as_mainfile(filepath=output_lib_path, check_existing=False, compress=False) return output_lib_path + + def init_lib_data_packed_indirect_lib(self): + output_dir = self.args.output_dir + self.ensure_path(output_dir) + + # Create an indirect library containing a material, and an image texture. + self.reset_blender() + + self.gen_indirect_library_data_() + + # Take care to keep the name unique so multiple test jobs can run at once. + output_lib_path = os.path.join(output_dir, self.unique_blendfile_name("blendlib_indirect_material")) + + bpy.ops.wm.save_as_mainfile(filepath=output_lib_path, check_existing=False, compress=False) + + # Create a main library containing object etc., and linking material from indirect library. + self.reset_blender() + + self.gen_library_data_() + + link_dir = os.path.join(output_lib_path, "Material") + bpy.ops.wm.link(directory=link_dir, filename="LibMaterial") + + ma = bpy.data.pack_linked_ids_hierarchy(bpy.data.materials[0]) + + me = bpy.data.meshes[0] + me.materials.append(ma) + + bpy.ops.outliner.orphans_purge() + + self.assertEqual(len(bpy.data.materials), 1) + self.assertTrue(bpy.data.materials[0].is_linked_packed) + + self.assertEqual(len(bpy.data.images), 1) + self.assertTrue(bpy.data.images[0].is_linked_packed) + + self.assertEqual(len(bpy.data.libraries), 2) + self.assertFalse(bpy.data.libraries[0].is_archive) + self.assertTrue(bpy.data.libraries[1].is_archive) + self.assertIn(bpy.data.libraries[1].name, bpy.data.libraries[0].archive_libraries) + + output_dir = self.args.output_dir + self.ensure_path(output_dir) + # Take care to keep the name unique so multiple test jobs can run at once. + output_lib_path = os.path.join(output_dir, self.unique_blendfile_name("blendlib_indirect_main")) + + bpy.ops.wm.save_as_mainfile(filepath=output_lib_path, check_existing=False, compress=False) + + return output_lib_path