griefith/test
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Core: Add packed linked data-blocks

Browse Source 
This adds support for packed linked data. This is a key part of an improved
asset workflow in Blender.

Packed IDs remain considered as linked data (i.e. they cannot be edited),
but they are stored in the current blendfile. This means that they:
* Are not lost in case the library data becomes unavailable.
* Are not changed in case the library data is updated.

These packed IDs are de-duplicated across blend-files, so e.g. if a shot
file and several of its dependencies all use the same util geometry node,
there will be a single copy of that geometry node in the shot file.

In case there are several versions of a same ID (e.g. linked at different
moments from a same library, which has been modified in-between), there
will be several packed IDs.

Name collisions are averted by storing these packed IDs into a new type of
'archive' libraries (and their namespaces). These libraries:
* Only contain packed IDs.
* Are owned and managed by their 'real' library data-block, called an
  'archive parent'.

For more in-depth, technical design: #132167
UI/UX design: #140870

Co-authored-by: Bastien Montagne <bastien@blender.org>
Pull Request: https://projects.blender.org/blender/blender/pulls/133801
This commit is contained in:
Jacques Lucke
2025-09-26 10:53:40 +02:00
committed by Bastien Montagne
parent 44194579a5
commit 4e4976804e
55 changed files with 2225 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
scripts/startup/bl_operators/object_quick_effects.py
View File

@@ -127,10 +127,9 @@ class QuickFur(ObjectModeOperator, Operator):
with bpy.data.libraries.load(
asset_library_filepath,
link=False,
clear_asset_data=True,
reuse_local_id=True,
recursive=True,
link=True,
pack=True,
set_fake=False,
) as (data_src, data_dst):
# The values are assumed to exist, no inspection of the source is needed.
del data_src

1
scripts/startup/bl_ui/space_userpref.py
View File

@@ -2888,6 +2888,7 @@ class USERPREF_PT_developer_tools(Panel):
({"property": "use_viewport_debug"}, None),
({"property": "use_eevee_debug"}, None),
({"property": "use_extensions_debug"}, ("/blender/blender/issues/119521", "#119521")),
({"property": "no_data_block_packing"}, ("/blender/blender/issues/132167", "#132167")),
),
)

12
source/blender/asset_system/AS_asset_library.hh
View File

@@ -303,3 +303,15 @@ void AS_asset_full_path_explode_from_weak_ref(const AssetWeakReference *asset_re
char **r_dir,
char **r_group,
char **r_name);
/**
* Updates the default import method for asset libraries based on
* #U.experimental.no_data_block_packing.
*/
void AS_asset_library_import_method_ensure_valid(Main &bmain);
/**
* This is not done as part of #AS_asset_library_import_method_ensure_valid because it changes
* run-time data only and does not need to happen during versioning (also it appears to break tests
* when run during versioning).
*/
void AS_asset_library_essential_import_method_update();

2
source/blender/asset_system/AS_essentials_library.hh
View File

@@ -14,4 +14,4 @@ namespace blender::asset_system {
StringRefNull essentials_directory_path();
}
} // namespace blender::asset_system

64
source/blender/asset_system/intern/asset_library.cc
View File

@@ -21,11 +21,14 @@
#include "BLI_path_utils.hh"
#include "BLI_string.h"
#include "DNA_space_types.h"
#include "DNA_userdef_types.h"
#include "DNA_windowmanager_types.h"
#include "asset_catalog_collection.hh"
#include "asset_catalog_definition_file.hh"
#include "asset_library_service.hh"
#include "essentials_library.hh"
#include "runtime_library.hh"
#include "utils.hh"
@@ -161,6 +164,67 @@ void AS_asset_full_path_explode_from_weak_ref(const AssetWeakReference *asset_re
}
}
static void update_import_method_for_user_libraries()
{
LISTBASE_FOREACH (bUserAssetLibrary *, library, &U.asset_libraries) {
if (U.experimental.no_data_block_packing) {
if (library->import_method == ASSET_IMPORT_PACK) {
library->import_method = ASSET_IMPORT_APPEND_REUSE;
}
}
else {
if (library->import_method == ASSET_IMPORT_APPEND_REUSE) {
library->import_method = ASSET_IMPORT_PACK;
}
}
}
}
static void update_import_method_for_asset_browsers(Main &bmain)
{
LISTBASE_FOREACH (bScreen *, screen, &bmain.screens) {
LISTBASE_FOREACH (ScrArea *, area, &screen->areabase) {
LISTBASE_FOREACH (SpaceLink *, sl, &area->spacedata) {
if (sl->spacetype != SPACE_FILE) {
continue;
}
SpaceFile *sfile = reinterpret_cast<SpaceFile *>(sl);
if (!sfile->asset_params) {
continue;
}
if (U.experimental.no_data_block_packing) {
if (sfile->asset_params->import_method == FILE_ASSET_IMPORT_PACK) {
sfile->asset_params->import_method = FILE_ASSET_IMPORT_APPEND_REUSE;
}
}
else {
if (sfile->asset_params->import_method == FILE_ASSET_IMPORT_APPEND_REUSE) {
sfile->asset_params->import_method = FILE_ASSET_IMPORT_PACK;
}
}
}
}
}
}
void AS_asset_library_import_method_ensure_valid(Main &bmain)
{
update_import_method_for_user_libraries();
update_import_method_for_asset_browsers(bmain);
}
void AS_asset_library_essential_import_method_update()
{
AssetLibraryReference library_ref{};
library_ref.custom_library_index = -1;
library_ref.type = ASSET_LIBRARY_ESSENTIALS;
EssentialsAssetLibrary *library = dynamic_cast<EssentialsAssetLibrary *>(
AS_asset_library_load(nullptr, library_ref));
if (library) {
library->update_default_import_method();
}
}
namespace blender::asset_system {
AssetLibrary::AssetLibrary(eAssetLibraryType library_type, StringRef name, StringRef root_path)

15
source/blender/asset_system/intern/library_types/essentials_library.cc
View File

@@ -10,6 +10,8 @@
#include "utils.hh"
#include "DNA_userdef_types.h"
#include "AS_essentials_library.hh"
#include "essentials_library.hh"
@@ -20,7 +22,10 @@ EssentialsAssetLibrary::EssentialsAssetLibrary()
{},
utils::normalize_directory_path(essentials_directory_path()))
{
import_method_ = ASSET_IMPORT_APPEND_REUSE;
import_method_ = ASSET_IMPORT_PACK;
if (U.experimental.no_data_block_packing) {
import_method_ = ASSET_IMPORT_APPEND_REUSE;
}
}
std::optional<AssetLibraryReference> EssentialsAssetLibrary::library_reference() const
@@ -31,6 +36,14 @@ std::optional<AssetLibraryReference> EssentialsAssetLibrary::library_reference()
return library_ref;
}
void EssentialsAssetLibrary::update_default_import_method()
{
import_method_ = ASSET_IMPORT_PACK;
if (U.experimental.no_data_block_packing) {
import_method_ = ASSET_IMPORT_APPEND_REUSE;
}
}
StringRefNull essentials_directory_path()
{
static std::string path = []() {

3
source/blender/asset_system/intern/library_types/essentials_library.hh
View File

@@ -17,6 +17,9 @@ class EssentialsAssetLibrary : public OnDiskAssetLibrary {
EssentialsAssetLibrary();
std::optional<AssetLibraryReference> library_reference() const override;
/** Update the default import method based on whether packed data-blocks are supported. */
void update_default_import_method();
};
} // namespace blender::asset_system

8
source/blender/blenkernel/BKE_blendfile_link_append.hh
View File

@@ -334,6 +334,14 @@ void BKE_blendfile_link_append_context_init_done(BlendfileLinkAppendContext *lap
*/
void BKE_blendfile_link(BlendfileLinkAppendContext *lapp_context, ReportList *reports);
/**
* Perform packing operation.
*
* The IDs processed by this functions are the one that have been linked by a previous call to
* #BKE_blendfile_link on the same `lapp_context`.
*/
void BKE_blendfile_link_pack(BlendfileLinkAppendContext *lapp_context, ReportList *reports);
/**
* Perform append operation, using modern ID usage looper to detect which ID should be kept
* linked, made local, duplicated as local, re-used from local etc.

49
source/blender/blenkernel/BKE_id_hash.hh Normal file
View File

@@ -0,0 +1,49 @@
/* SPDX-FileCopyrightText: 2025 Blender Authors
*
* SPDX-License-Identifier: GPL-2.0-or-later */
#include <string>
#include <variant>
#include "BKE_main.hh"
#include "BLI_map.hh"
#include "BLI_vector.hh"
#include "DNA_ID.h"
namespace blender::bke::id_hash {
/**
* The hash of all the root IDs and their dependencies.
*/
struct ValidDeepHashes {
Map<const ID *, IDHash> hashes;
};
struct DeepHashErrors {
/**
* A list of missing files paths in the case that the deep hashes could not be computed.
*/
VectorSet<std::string> missing_files;
/**
* Files that were modified since the linked ID was loaded. So the currently linked ID would not
* be matching the deep hash computed based on the source file.
*/
VectorSet<std::string> updated_files;
};
using IDHashResult = std::variant<ValidDeepHashes, DeepHashErrors>;
/**
* Compute a hash of the given IDs, including all their dependencies.
* This needs access to the original .blend files that the linked data-blocks come from to be able
* to compute their hash.
*/
IDHashResult compute_linked_id_deep_hashes(const Main &bmain, Span<const ID *> root_ids);
/** Utility to convert the hash into a readable string. */
std::string id_hash_to_hex(const IDHash &hash);
} // namespace blender::bke::id_hash

10
source/blender/blenkernel/BKE_lib_id.hh
View File

@@ -71,6 +71,11 @@ struct ID_Runtime_Remap {
};
struct ID_Runtime {
/**
* The last modifification time of the source .blend file where this ID was loaded from.
*/
int64_t src_blend_modifification_time;
ID_Runtime_Remap remap = {};
/**
* The depsgraph that owns this data block. This is only set on data-blocks which are
@@ -249,6 +254,11 @@ enum {
*/
LIB_ID_COPY_SET_COPIED_ON_WRITE = 1 << 10,
/**
* Set #ID.newid pointer of the given source ID with the address of its new copy.
*/
LIB_ID_COPY_ID_NEW_SET = 1 << 11,
/* *** Specific options to some ID types or usages. *** */
/* *** May be ignored by unrelated ID copying functions. *** */
/** Object only, needed by make_local code. */

27
source/blender/blenkernel/BKE_library.hh
View File

@@ -9,12 +9,15 @@
* API to manage `Library` data-blocks.
*/
#include "DNA_ID.h"
#include "BLI_map.hh"
#include "BLI_set.hh"
#include "BLI_string_ref.hh"
#include "BKE_main.hh"
struct FileData;
struct Library;
struct ListBase;
struct Main;
struct UniqueName_Map;
@@ -27,6 +30,10 @@ struct LibraryRuntime {
/**
* Filedata (i.e. opened blendfile) source of this library data.
*
* \note: This is not always matching the library's blendfile path. E.g. for archive packed
* libraries, this will be the filedata of the packing blendfile, not of the reference/source
* library.
*/
FileData *filedata = nullptr;
/**
@@ -48,6 +55,12 @@ struct LibraryRuntime {
/** Set for indirectly linked libraries, used in the outliner and while reading. */
Library *parent = nullptr;
/**
* Helper listing all archived libraries 'versions' of this library.
* Should only contain something if this library is a regular 'real' blendfile library.
*/
blender::Vector<Library *> archived_libraries = {};
/** #eLibrary_Tag. */
ushort tag = 0;
@@ -67,6 +80,18 @@ struct LibraryRuntime {
*/
Library *search_filepath_abs(ListBase *libraries, blender::StringRef filepath_abs);
/**
* Pack given linked ID, and all the related hierarchy.
*
* Will set final embedded ID into each ID::newid pointers.
*/
void pack_linked_id_hierarchy(Main &bmain, ID &root_id);
/**
* Cleanup references to removed/deleted archive libraries in their archive parent.
*/
void main_cleanup_parent_archives(Main &bmain);
}; // namespace blender::bke::library
/** #LibraryRuntime.tag */

2
source/blender/blenkernel/CMakeLists.txt
View File

@@ -147,6 +147,7 @@ set(SRC
intern/grease_pencil_vertex_groups.cc
intern/icons.cc
intern/icons_rasterize.cc
intern/id_hash.cc
intern/idprop.cc
intern/idprop_create.cc
intern/idprop_serialize.cc
@@ -419,6 +420,7 @@ set(SRC
BKE_grease_pencil_legacy_convert.hh
BKE_grease_pencil_vertex_groups.hh
BKE_icons.h
BKE_id_hash.hh
BKE_idprop.hh
BKE_idtype.hh
BKE_image.hh

45
source/blender/blenkernel/intern/blendfile_link_append.cc
View File

@@ -949,6 +949,51 @@ static bool foreach_libblock_link_append_common_processing(
/** \} */
/** \name Library embedding code.
* \{ */
void BKE_blendfile_link_pack(BlendfileLinkAppendContext *lapp_context, ReportList * /*reports*/)
{
Main *bmain = lapp_context->params->bmain;
/* Delete newly linked data-blocks after they have been packed. */
blender::Vector<ID *> linked_ids_to_delete;
{
ID *id;
FOREACH_MAIN_ID_BEGIN (bmain, id) {
if (ID_IS_LINKED(id) && !ID_IS_PACKED(id)) {
if (!(id->tag & ID_TAG_PRE_EXISTING)) {
linked_ids_to_delete.append(id);
}
}
}
FOREACH_MAIN_ID_END;
}
for (BlendfileLinkAppendContextItem &item : lapp_context->items) {
ID *id = item.new_id;
BLI_assert(ID_IS_LINKED(id));
if (!(ID_IS_PACKED(id) || (id->newid && ID_IS_PACKED(id->newid)))) {
/* No yet packed. */
blender::bke::library::pack_linked_id_hierarchy(*bmain, *id);
}
/* Calling code may want to access newly packed embedded IDs from the link/append context
* items. */
if (id->newid) {
item.new_id = id->newid;
}
}
BKE_main_id_newptr_and_tag_clear(bmain);
BKE_main_id_tag_all(bmain, ID_TAG_DOIT, false);
for (ID *id : linked_ids_to_delete) {
id->tag |= ID_TAG_DOIT;
}
BKE_id_multi_tagged_delete(bmain);
}
/** \} */
/** \name Library append code.
* \{ */

250
source/blender/blenkernel/intern/id_hash.cc Normal file
View File

@@ -0,0 +1,250 @@
/* SPDX-FileCopyrightText: 2025 Blender Authors
*
* SPDX-License-Identifier: GPL-2.0-or-later */
#include <fcntl.h>
#include <fmt/format.h>
#include <mutex>
#include <xxhash.h>
#include "BKE_id_hash.hh"
#include "BKE_lib_id.hh"
#include "BKE_lib_query.hh"
#include "BKE_library.hh"
#include "BKE_main.hh"
#include "BLI_fileops.hh"
#include "BLI_mmap.h"
#include "BLI_mutex.hh"
#include "BLI_set.hh"
namespace blender::bke::id_hash {
static std::optional<Vector<char>> read_file(const StringRefNull path)
{
blender::fstream stream{path.c_str(), std::ios_base::in | std::ios_base::binary};
stream.seekg(0, std::ios_base::end);
const int64_t size = stream.tellg();
stream.seekg(0, std::ios_base::beg);
blender::Vector<char> buffer(size);
stream.read(buffer.data(), size);
if (stream.bad()) {
return std::nullopt;
}
return buffer;
}
static std::optional<XXH128_hash_t> compute_file_hash_with_file_read(const StringRefNull path)
{
const std::optional<Vector<char>> buffer = read_file(path);
if (!buffer) {
return std::nullopt;
}
return XXH3_128bits(buffer->data(), buffer->size());
}
static std::optional<XXH128_hash_t> compute_file_hash_with_memory_map(const StringRefNull path)
{
const int file = BLI_open(path.c_str(), O_BINARY | O_RDONLY, 0);
if (file == -1) {
return std::nullopt;
}
BLI_mmap_file *mmap_file = BLI_mmap_open(file);
if (!mmap_file) {
return std::nullopt;
}
BLI_SCOPED_DEFER([&]() { BLI_mmap_free(mmap_file); });
const size_t size = BLI_mmap_get_length(mmap_file);
const void *data = BLI_mmap_get_pointer(mmap_file);
const XXH128_hash_t hash = XXH3_128bits(data, size);
if (BLI_mmap_any_io_error(mmap_file)) {
return std::nullopt;
}
return hash;
}
static std::optional<XXH128_hash_t> compute_file_hash(const StringRefNull path)
{
/* First try the memory map the file, because it avoids an extra copy. */
if (const std::optional<XXH128_hash_t> hash = compute_file_hash_with_memory_map(path)) {
/* Make sure both code paths are tested even if memory mapping should almost always work. */
BLI_assert(hash->low64 == compute_file_hash_with_file_read(path)->low64);
return hash;
}
if (const std::optional<XXH128_hash_t> hash = compute_file_hash_with_file_read(path)) {
return hash;
}
return std::nullopt;
}
struct CachedFileHash {
int64_t last_modified = 0;
XXH128_hash_t hash;
};
static std::optional<XXH128_hash_t> get_source_file_hash(const ID &id, DeepHashErrors &r_errors)
{
static Map<std::string, CachedFileHash> cache;
static Mutex mutex;
const StringRefNull path = id.lib->runtime->filepath_abs;
BLI_stat_t stat;
if (BLI_stat(path.c_str(), &stat) == -1) {
r_errors.missing_files.add_as(path);
return std::nullopt;
}
std::lock_guard lock(mutex);
if (const CachedFileHash *cached_hash = cache.lookup_ptr_as(path)) {
if (cached_hash->last_modified == stat.st_mtime) {
return cached_hash->hash;
}
}
if (stat.st_mtime != id.runtime->src_blend_modifification_time) {
r_errors.updated_files.add_as(path);
return std::nullopt;
}
if (const std::optional<XXH128_hash_t> hash = compute_file_hash(path)) {
cache.add_overwrite(path, CachedFileHash{stat.st_mtime, *hash});
return hash;
}
r_errors.missing_files.add_as(path);
return std::nullopt;
}
static std::optional<XXH128_hash_t> get_id_shallow_hash(const ID &id, DeepHashErrors &r_errors)
{
BLI_assert(ID_IS_LINKED(&id));
const StringRefNull id_name = id.name;
const std::optional<XXH128_hash_t> file_hash = get_source_file_hash(id, r_errors);
if (!file_hash) {
return std::nullopt;
}
XXH3_state_t *hash_state = XXH3_createState();
XXH3_128bits_reset(hash_state);
XXH3_128bits_update(hash_state, id_name.data(), id_name.size());
XXH3_128bits_update(hash_state, &*file_hash, sizeof(XXH128_hash_t));
XXH128_hash_t shallow_hash = XXH3_128bits_digest(hash_state);
XXH3_freeState(hash_state);
return shallow_hash;
}
static void compute_deep_hash_recursive(const Main &bmain,
const ID &id,
Set<const ID *> &current_stack,
Map<const ID *, IDHash> &r_hashes,
DeepHashErrors &r_errors)
{
if (r_hashes.contains(&id)) {
return;
}
if (!id.deep_hash.is_null()) {
r_hashes.add(&id, id.deep_hash);
return;
}
current_stack.add(&id);
const std::optional<XXH128_hash_t> id_shallow_hash = get_id_shallow_hash(id, r_errors);
if (!id_shallow_hash) {
return;
}
XXH3_state_t *hash_state = XXH3_createState();
XXH3_128bits_reset(hash_state);
XXH3_128bits_update(hash_state, &*id_shallow_hash, sizeof(XXH128_hash_t));
bool success = true;
BKE_library_foreach_ID_link(
const_cast<Main *>(&bmain),
const_cast<ID *>(&id),
[&](LibraryIDLinkCallbackData *cb_data) {
if (cb_data->cb_flag & IDWALK_CB_LOOPBACK) {
/* Loopback pointer (e.g. from a shapekey to its owner geometry ID, or from a collection
* to its parents) should always be ignored, as they do not represent an actual
* dependency. The dependency relationship should already have been processed from the
* owner to its dependency anyway (if applicable). */
return IDWALK_RET_NOP;
}
if (cb_data->cb_flag & (IDWALK_CB_EMBEDDED | IDWALK_CB_EMBEDDED_NOT_OWNING)) {
/* Embedded data are part of their owner's internal data, and as such already computed as
* part of the owner's shallow hash. */
return IDWALK_RET_NOP;
}
ID *referenced_id = *cb_data->id_pointer;
if (!referenced_id) {
/* Need to update the hash even if there is no id. There is a difference between the case
* where there is no id and the case where this callback is not called at all.*/
const int random_data = 452942579;
XXH3_128bits_update(hash_state, &random_data, sizeof(int));
return IDWALK_RET_NOP;
}
/* All embedded ID usages should already have been excluded above. */
BLI_assert((referenced_id->flag & ID_FLAG_EMBEDDED_DATA) == 0);
if (current_stack.contains(referenced_id)) {
/* Somehow encode that we had a circular reference here. */
const int random_data = 234632342;
XXH3_128bits_update(hash_state, &random_data, sizeof(int));
return IDWALK_RET_NOP;
}
compute_deep_hash_recursive(bmain, *referenced_id, current_stack, r_hashes, r_errors);
const IDHash *referenced_id_hash = r_hashes.lookup_ptr(referenced_id);
if (!referenced_id_hash) {
success = false;
return IDWALK_RET_STOP_ITER;
}
XXH3_128bits_update(hash_state, referenced_id_hash->data, sizeof(IDHash));
return IDWALK_RET_NOP;
},
nullptr,
IDWALK_READONLY);
if (!success) {
return;
}
IDHash new_deep_hash;
const XXH128_hash_t new_deep_hash_xxh128 = XXH3_128bits_digest(hash_state);
XXH3_freeState(hash_state);
static_assert(sizeof(IDHash) == sizeof(XXH128_hash_t));
memcpy(new_deep_hash.data, &new_deep_hash_xxh128, sizeof(IDHash));
r_hashes.add(&id, new_deep_hash);
}
IDHashResult compute_linked_id_deep_hashes(const Main &bmain, Span<const ID *> ids)
{
#ifndef NDEBUG
for (const ID *id : ids) {
BLI_assert(ID_IS_LINKED(id));
}
#endif
if (ids.is_empty()) {
return ValidDeepHashes{};
}
Map<const ID *, IDHash> hashes;
Set<const ID *> current_stack;
DeepHashErrors errors;
for (const ID *id : ids) {
compute_deep_hash_recursive(bmain, *id, current_stack, hashes, errors);
}
if (!errors.missing_files.is_empty() || !errors.updated_files.is_empty()) {
return errors;
}
return ValidDeepHashes{hashes};
}
std::string id_hash_to_hex(const IDHash &hash)
{
std::string hex_str;
for (const uint8_t byte : hash.data) {
hex_str += fmt::format("{:02x}", byte);
}
return hex_str;
}
} // namespace blender::bke::id_hash

8
source/blender/blenkernel/intern/lib_id.cc
View File

@@ -1667,6 +1667,14 @@ void BKE_libblock_copy_in_lib(Main *bmain,
DEG_id_type_tag(bmain, GS(new_id->name));
}
if (owner_library && *owner_library && ((*owner_library)->flag & LIBRARY_FLAG_IS_ARCHIVE) != 0) {
new_id->flag |= ID_FLAG_LINKED_AND_PACKED;
}
if (flag & LIB_ID_COPY_ID_NEW_SET) {
ID_NEW_SET(id, new_id);
}
*new_id_p = new_id;
}

4
source/blender/blenkernel/intern/lib_id_delete.cc
View File

@@ -339,6 +339,10 @@ static size_t id_delete(Main *bmain,
id_remapper.clear();
}
/* Remapping above may have left some Library::runtime::archived_libraries items to nullptr,
* clean this up and shrink the vector accordingly. */
blender::bke::library::main_cleanup_parent_archives(*bmain);
/* Since we removed IDs from Main, their own other IDs usages need to be removed 'manually'. */
blender::Vector<ID *> cleanup_ids{ids_to_delete.begin(), ids_to_delete.end()};
BKE_libblock_relink_multiple(

347
source/blender/blenkernel/intern/library.cc
View File

@@ -14,6 +14,8 @@
/* all types are needed here, in order to do memory operations */
#include "DNA_ID.h"
#include "DNA_collection_types.h"
#include "DNA_scene_types.h"
#include "BLI_utildefines.h"
@@ -22,24 +24,32 @@
#include "BLI_path_utils.hh"
#include "BLI_set.hh"
#include "BLI_string.h"
#include "BLI_vector_set.hh"
#include "BLT_translation.hh"
#include "BLO_read_write.hh"
#include "BKE_bpath.hh"
#include "BKE_id_hash.hh"
#include "BKE_idtype.hh"
#include "BKE_key.hh"
#include "BKE_lib_id.hh"
#include "BKE_lib_query.hh"
#include "BKE_lib_remap.hh"
#include "BKE_library.hh"
#include "BKE_main.hh"
#include "BKE_main_invariants.hh"
#include "BKE_main_namemap.hh"
#include "BKE_node.hh"
#include "BKE_packedFile.hh"
#include "BKE_report.hh"
struct BlendDataReader;
static CLG_LogRef LOG = {"lib.library"};
using namespace blender::bke;
using namespace blender::bke::library;
static void library_runtime_reset(Library *lib)
@@ -93,7 +103,37 @@ static void library_copy_data(Main *bmain,
static void library_foreach_id(ID *id, LibraryForeachIDData *data)
{
Library *lib = (Library *)id;
const LibraryForeachIDFlag foreach_flag = BKE_lib_query_foreachid_process_flags_get(data);
BKE_LIB_FOREACHID_PROCESS_IDSUPER(data, lib->runtime->parent, IDWALK_CB_NEVER_SELF);
if (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) {
/* Archive library must have a parent, this can't be nullptr. */
if (lib->archive_parent_library) {
BKE_LIB_FOREACHID_PROCESS_ID(
data, lib->archive_parent_library, IDWALK_CB_NEVER_SELF | IDWALK_CB_NEVER_NULL);
}
/* Archive libraries should never 'own' other archives. */
BLI_assert(lib->runtime->archived_libraries.is_empty());
if (foreach_flag & IDWALK_DO_INTERNAL_RUNTIME_POINTERS) {
for (Library *&lib_p : lib->runtime->archived_libraries) {
BKE_LIB_FOREACHID_PROCESS_ID(
data, lib_p, IDWALK_CB_NEVER_SELF | IDWALK_CB_INTERNAL | IDWALK_CB_LOOPBACK);
}
}
}
else {
/* Regular libraries should never have an archive parent. */
BLI_assert(!lib->archive_parent_library);
BKE_LIB_FOREACHID_PROCESS_ID(data, lib->archive_parent_library, IDWALK_CB_NEVER_SELF);
if (foreach_flag & IDWALK_DO_INTERNAL_RUNTIME_POINTERS) {
for (Library *&lib_p : lib->runtime->archived_libraries) {
BKE_LIB_FOREACHID_PROCESS_ID(
data, lib_p, IDWALK_CB_NEVER_SELF | IDWALK_CB_INTERNAL | IDWALK_CB_LOOPBACK);
}
}
}
}
static void library_foreach_path(ID *id, BPathForeachPathData *bpath_data)
@@ -139,6 +179,15 @@ static void library_blend_read_data(BlendDataReader * /*reader*/, ID *id)
lib->runtime = MEM_new<LibraryRuntime>(__func__);
}
static void library_blend_read_after_liblink(BlendLibReader * /*reader*/, ID *id)
{
Library *lib = reinterpret_cast<Library *>(id);
if (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) {
BLI_assert(lib->archive_parent_library);
lib->archive_parent_library->runtime->archived_libraries.append(lib);
}
}
IDTypeInfo IDType_ID_LI = {
/*id_code*/ Library::id_type,
/*id_filter*/ FILTER_ID_LI,
@@ -163,7 +212,7 @@ IDTypeInfo IDType_ID_LI = {
/*blend_write*/ library_blend_write_data,
/*blend_read_data*/ library_blend_read_data,
/*blend_read_after_liblink*/ nullptr,
/*blend_read_after_liblink*/ library_blend_read_after_liblink,
/*blend_read_undo_preserve*/ nullptr,
@@ -368,3 +417,299 @@ Library *blender::bke::library::search_filepath_abs(ListBase *libraries,
}
return nullptr;
}
/**
* Add a new 'archive' copy of the given reference library. It is used to store linked packed IDs.
*/
static Library *add_archive_library(Main &bmain, Library &reference_library)
{
BLI_assert((reference_library.flag & LIBRARY_FLAG_IS_ARCHIVE) == 0);
/* Cannot copy libraries using generic ID copying functions, so create the copy manually. */
Library *archive_library = static_cast<Library *>(
BKE_id_new(&bmain, ID_LI, BKE_id_name(reference_library.id)));
/* Like in #direct_link_library. */
id_us_ensure_real(&archive_library->id);
archive_library->archive_parent_library = &reference_library;
constexpr uint16_t copy_flag = ~LIBRARY_FLAG_IS_ARCHIVE;
archive_library->flag = (reference_library.flag & copy_flag) | LIBRARY_FLAG_IS_ARCHIVE;
BKE_library_filepath_set(&bmain, archive_library, reference_library.filepath);
archive_library->runtime->parent = reference_library.runtime->parent;
/* Only copy a subset of the reference library tags. E.g. an archive library should never be
* considered as writable, so never copy #LIBRARY_ASSET_FILE_WRITABLE. This may need further
* tweaking still. */
constexpr uint16_t copy_tag = (LIBRARY_TAG_RESYNC_REQUIRED | LIBRARY_ASSET_EDITABLE |
LIBRARY_IS_ASSET_EDIT_FILE);
archive_library->runtime->tag = reference_library.runtime->tag & copy_tag;
/* By definition, the file version of an archive library containing only packed linked data is
* the same as the one of its Main container. */
archive_library->runtime->versionfile = bmain.versionfile;
archive_library->runtime->subversionfile = bmain.subversionfile;
reference_library.runtime->archived_libraries.append(archive_library);
return archive_library;
}
static Library *get_archive_library(Main &bmain, ID *for_id, const IDHash &for_id_deep_hash)
{
Library *reference_library = for_id->lib;
BLI_assert(reference_library && (reference_library->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0);
Library *archive_library = nullptr;
for (Library *lib_iter : reference_library->runtime->archived_libraries) {
BLI_assert((lib_iter->flag & LIBRARY_FLAG_IS_ARCHIVE) != 0);
BLI_assert(lib_iter->archive_parent_library != nullptr);
BLI_assert(lib_iter->archive_parent_library == reference_library);
/* Check if current archive library already contains an ID of same type and name. */
if (BKE_main_namemap_contain_name(bmain, lib_iter, GS(for_id->name), BKE_id_name(*for_id))) {
#ifndef NDEBUG
ID *packed_id = BKE_libblock_find_name_and_library(
&bmain, GS(for_id->name), BKE_id_name(*for_id), BKE_id_name(lib_iter->id));
BLI_assert_msg(
packed_id && packed_id->deep_hash != for_id_deep_hash,
"An already packed ID with same deep hash as the one to be packed, should have already "
"be found and used (deduplication) before reaching this codepath");
#endif
UNUSED_VARS_NDEBUG(for_id_deep_hash);
continue;
}
archive_library = lib_iter;
break;
}
if (!archive_library) {
archive_library = add_archive_library(bmain, *reference_library);
}
BLI_assert(reference_library->runtime->archived_libraries.contains(archive_library));
return archive_library;
}
static void pack_linked_id(Main &bmain,
ID *linked_id,
const id_hash::ValidDeepHashes &deep_hashes,
blender::Map<IDHash, ID *> &already_packed_ids,
blender::VectorSet<ID *> &ids_to_remap,
blender::bke::id::IDRemapper &id_remapper)
{
BLI_assert(linked_id->newid == nullptr);
const IDHash linked_id_deep_hash = deep_hashes.hashes.lookup(linked_id);
ID *packed_id = already_packed_ids.lookup_default(linked_id_deep_hash, nullptr);
if (packed_id) {
/* Exact same ID (and all of its dependencies) have already been linked and packed before,
* re-use these packed data. */
auto existing_id_process = [&deep_hashes, &id_remapper](ID *linked_id, ID *packed_id) {
BLI_assert(packed_id);
BLI_assert(ID_IS_PACKED(packed_id));
/* Note: linked_id and packed_id may have the same deep hash while still coming from
* different original libraries. This easily happens copying an asset file such that each
* asset exists twice. */
BLI_assert(packed_id->deep_hash == deep_hashes.hashes.lookup(linked_id));
UNUSED_VARS_NDEBUG(deep_hashes);
id_remapper.add(linked_id, packed_id);
linked_id->newid = packed_id;
/* No need to remap this packed ID - otherwise there would be something very wrong in
* packed IDs state. */
};
existing_id_process(linked_id, packed_id);
/* Handle 'fake-embedded' ShapeKeys IDs. */
Key *linked_key = BKE_key_from_id(linked_id);
if (linked_key) {
Key *packed_key = BKE_key_from_id(packed_id);
BLI_assert(packed_key);
existing_id_process(&linked_key->id, &packed_key->id);
}
}
else {
/* This exact version of the ID and its dependencies have not been packed before, creates a
* new copy of it and pack it. */
/* Find an existing archive Library not containing a 'version' of this ID yet (to prevent names
* collisions). */
Library *archive_lib = get_archive_library(bmain, linked_id, linked_id_deep_hash);
auto copied_id_process =
[&archive_lib, &deep_hashes, &ids_to_remap, &id_remapper, &already_packed_ids](
ID *linked_id, ID *packed_id) {
BLI_assert(packed_id);
BLI_assert(ID_IS_PACKED(packed_id));
BLI_assert(packed_id->lib == archive_lib);
UNUSED_VARS_NDEBUG(archive_lib);
packed_id->deep_hash = deep_hashes.hashes.lookup(linked_id);
id_remapper.add(linked_id, packed_id);
ids_to_remap.add(packed_id);
already_packed_ids.add(packed_id->deep_hash, packed_id);
};
packed_id = BKE_id_copy_in_lib(&bmain,
archive_lib,
linked_id,
std::nullopt,
nullptr,
LIB_ID_COPY_DEFAULT | LIB_ID_COPY_ID_NEW_SET |
LIB_ID_COPY_NO_ANIMDATA);
id_us_min(packed_id);
copied_id_process(linked_id, packed_id);
/* Handle 'fake-embedded' ShapeKeys IDs. */
Key *linked_key = BKE_key_from_id(linked_id);
if (linked_key) {
Key *embedded_key = BKE_key_from_id(packed_id);
BLI_assert(embedded_key);
copied_id_process(&linked_key->id, &embedded_key->id);
}
}
}
/**
* Pack given linked IDs. Low-level code, assumes all given IDs are valid and safe to pack.
*
* Will set final packed ID into each ID::newid pointers.
*/
static void pack_linked_ids(Main &bmain, const blender::Set<ID *> &ids_to_pack)
{
blender::VectorSet<ID *> final_ids_to_pack;
blender::VectorSet<ID *> ids_to_remap;
blender::bke::id::IDRemapper id_remapper;
for (ID *id : ids_to_pack) {
BLI_assert(ID_IS_LINKED(id));
if (ID_IS_PACKED(id)) {
/* Should not happen, but also not critical issue. */
CLOG_ERROR(&LOG,
"Trying to pack an already packed ID '%s' (from '%s')",
id->name,
id->lib->runtime->filepath_abs);
/* Already packed. */
continue;
}
final_ids_to_pack.add(id);
}
const id_hash::IDHashResult hash_result = id_hash::compute_linked_id_deep_hashes(
bmain, final_ids_to_pack.as_span());
if (const auto *errors = std::get_if<id_hash::DeepHashErrors>(&hash_result)) {
if (!errors->missing_files.is_empty()) {
CLOG_ERROR(&LOG,
"Trying to pack IDs that depend on missing linked libraries: %s",
errors->missing_files[0].c_str());
}
if (!errors->updated_files.is_empty()) {
CLOG_ERROR(&LOG,
"Trying to pack linked ID that has been modified on disk: %s",
errors->updated_files[0].c_str());
}
return;
}
const auto &deep_hashes = std::get<id_hash::ValidDeepHashes>(hash_result);
blender::Map<IDHash, ID *> already_packed_ids;
{
ID *id;
FOREACH_MAIN_ID_BEGIN (&bmain, id) {
if (ID_IS_PACKED(id)) {
already_packed_ids.add(id->deep_hash, id);
}
}
FOREACH_MAIN_ID_END;
}
for (ID *linked_id : final_ids_to_pack) {
pack_linked_id(bmain, linked_id, deep_hashes, already_packed_ids, ids_to_remap, id_remapper);
}
BKE_libblock_relink_multiple(
&bmain, ids_to_remap.as_span(), ID_REMAP_TYPE_REMAP, id_remapper, 0);
BKE_main_ensure_invariants(bmain);
}
void blender::bke::library::pack_linked_id_hierarchy(Main &bmain, ID &root_id)
{
BLI_assert(ID_IS_LINKED(&root_id));
BLI_assert(!ID_IS_PACKED(&root_id));
blender::Set<ID *> ids_to_pack;
ids_to_pack.add(&root_id);
BKE_library_foreach_ID_link(
&bmain,
&root_id,
[&ids_to_pack](LibraryIDLinkCallbackData *cb_data) -> int {
if (cb_data->cb_flag & IDWALK_CB_LOOPBACK) {
return IDWALK_RET_NOP;
}
if (cb_data->cb_flag & (IDWALK_CB_EMBEDDED | IDWALK_CB_EMBEDDED_NOT_OWNING)) {
return IDWALK_RET_NOP;
}
ID *self_id = cb_data->self_id;
ID *referenced_id = *cb_data->id_pointer;
if (!referenced_id) {
return IDWALK_RET_NOP;
}
if (!ID_IS_LINKED(referenced_id)) {
CLOG_ERROR(&LOG, "Linked data-block references non-linked data-block");
return IDWALK_RET_NOP;
}
if (ID_IS_PACKED(referenced_id)) {
/* A linked ID can use another packed linked ID, as long as it is not from the same
* library. */
BLI_assert(referenced_id->lib && referenced_id->lib->archive_parent_library);
if (referenced_id->lib->archive_parent_library == self_id->lib) {
CLOG_ERROR(&LOG,
"Non-packed data-block references packed data-block from the same library, "
"which is not allowed");
}
return IDWALK_RET_NOP;
}
if (referenced_id->newid && ID_IS_PACKED(referenced_id->newid)) {
return IDWALK_RET_NOP;
}
if (GS(referenced_id->name) == ID_KE) {
/* Shape keys cannot be directly linked, from linking code PoV they behave as embedded
* data (i.e. their owning data is responsible to handle them). */
return IDWALK_RET_NOP;
}
ids_to_pack.add(referenced_id);
return IDWALK_RET_NOP;
},
nullptr,
IDWALK_READONLY | IDWALK_RECURSE);
pack_linked_ids(bmain, ids_to_pack);
}
void blender::bke::library::main_cleanup_parent_archives(Main &bmain)
{
LISTBASE_FOREACH (Library *, lib, &bmain.libraries) {
if (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) {
BLI_assert(!lib->runtime || lib->runtime->archived_libraries.is_empty());
}
else {
int i_read_curr = 0;
int i_insert_curr = 0;
for (; i_read_curr < lib->runtime->archived_libraries.size(); i_read_curr++) {
if (!lib->runtime->archived_libraries[i_read_curr]) {
continue;
}
if (i_insert_curr < i_read_curr) {
lib->runtime->archived_libraries[i_insert_curr] =
lib->runtime->archived_libraries[i_read_curr];
}
i_insert_curr++;
}
BLI_assert(i_insert_curr <= i_read_curr);
if (i_insert_curr < i_read_curr) {
lib->runtime->archived_libraries.resize(i_insert_curr);
}
}
}
}

10
source/blender/blenkernel/intern/main.cc
View File

@@ -717,7 +717,15 @@ void BKE_main_library_weak_reference_add_item(
BLI_assert(BKE_idtype_idcode_append_is_reusable(GS(new_id->name)));
const LibWeakRefKey key{library_filepath, library_id_name};
library_weak_reference_mapping->map.add_new(key, new_id);
/* With packed IDs and archive libraries, it is now possible to have several instances of the
* (originally) same linked ID made local at the same time in an append opeeration, so it is
* possible to get the same key several time here. And `Map::add_new` cannot be used safely
* anymore.
*
* Simply consider the first added one as valid, there is no good way to determine the 'best' one
* to keep around for append-or-reuse operations anyway - and the whole append-and-reuse may be
* depracted soon too. */
library_weak_reference_mapping->map.add(key, new_id);
BKE_main_library_weak_reference_add(new_id, library_filepath, library_id_name);
}

8
source/blender/blenloader/BLO_readfile.hh
View File

@@ -406,6 +406,10 @@ enum eBLOLibLinkFlags {
* see e.g. #BKE_blendfile_library_relocate.
*/
BLO_LIBLINK_COLLECTION_NO_HIERARCHY_REBUILD = 1 << 26,
/**
* Pack the linked data-blocks to keep them working even if the source file is not available.
*/
BLO_LIBLINK_PACK = 1 << 27,
};
/**
@@ -584,13 +588,13 @@ struct ID_Readfile_Data {
};
/**
* Return `id->runtime.readfile_data->tags` if the `readfile_data` is allocated,
* Return `id->runtime->readfile_data->tags` if the `readfile_data` is allocated,
* otherwise return an all-zero set of tags.
*/
ID_Readfile_Data::Tags BLO_readfile_id_runtime_tags(ID &id);
/**
* Create the `readfile_data` if needed, and return `id->runtime.readfile_data->tags`.
* Create the `readfile_data` if needed, and return `id->runtime->readfile_data->tags`.
*
* Use it instead of #BLO_readfile_id_runtime_tags when tags need to be set.
*/

1
source/blender/blenloader/CMakeLists.txt
View File

@@ -54,6 +54,7 @@ set(SRC
set(LIB
PRIVATE bf::animrig
PRIVATE bf::asset_system
PRIVATE bf::blenkernel
PRIVATE bf::blenlib
PUBLIC bf::blenloader_core

21
source/blender/blenloader/intern/readblenentry.cc
View File

@@ -87,12 +87,21 @@ static bool blendhandle_load_id_data_and_validate(FileData *fd,
BHead *bhead,
bool use_assets_only,
const char *&r_idname,
short &r_idflag,
AssetMetaData *&r_asset_meta_data)
{
r_idname = blo_bhead_id_name(fd, bhead);
if (!r_idname || r_idname[0] == '\0') {
return false;
}
r_idflag = blo_bhead_id_flag(fd, bhead);
/* Do not list (and therefore allow direct linking of) packed data.
* While supporting this is conceptually possible, it would require significant changes in
* the UI (file browser) and UX (link operation) to convey this concept and handle it
* correctly. */
if (r_idflag & ID_FLAG_LINKED_AND_PACKED) {
return false;
}
r_asset_meta_data = blo_bhead_id_asset_data_address(fd, bhead);
if (use_assets_only && r_asset_meta_data == nullptr) {
return false;
@@ -113,9 +122,10 @@ LinkNode *BLO_blendhandle_get_datablock_names(BlendHandle *bh,
for (bhead = blo_bhead_first(fd); bhead; bhead = blo_bhead_next(fd, bhead)) {
if (bhead->code == ofblocktype) {
const char *idname;
short idflag;
AssetMetaData *asset_meta_data;
if (!blendhandle_load_id_data_and_validate(
fd, bhead, use_assets_only, idname, asset_meta_data))
fd, bhead, use_assets_only, idname, idflag, asset_meta_data))
{
continue;
}
@@ -152,9 +162,10 @@ LinkNode *BLO_blendhandle_get_datablock_info(BlendHandle *bh,
BHead *id_bhead = bhead;
const char *idname;
short idflag;
AssetMetaData *asset_meta_data;
if (!blendhandle_load_id_data_and_validate(
fd, id_bhead, use_assets_only, idname, asset_meta_data))
fd, id_bhead, use_assets_only, idname, idflag, asset_meta_data))
{
continue;
}
@@ -382,8 +393,10 @@ BlendFileData *BLO_read_from_memfile(Main *oldmain,
/* Build old ID map for all old IDs. */
blo_make_old_idmap_from_main(fd, oldmain);
/* Separate linked data from old main. */
blo_split_main(oldmain);
/* Separate linked data from old main.
* WARNING: Do not split out packed IDs here, as these are handled similarly as local IDs in
* undo context. */
blo_split_main(oldmain, false);
fd->old_bmain = oldmain;
/* Removed packed data from this trick - it's internal data that needs saves. */

637
source/blender/blenloader/intern/readfile.cc
View File

@@ -390,17 +390,17 @@ void blo_join_main(Main *bmain)
bmain->split_mains.reset();
}
static void split_libdata(ListBase *lb_src, Main **lib_main_array, const uint lib_main_array_len)
static void split_libdata(ListBase *lb_src,
blender::Vector<Main *> &lib_main_array,
const bool do_split_packed_ids)
{
for (ID *id = static_cast<ID *>(lb_src->first), *idnext; id; id = idnext) {
idnext = static_cast<ID *>(id->next);
if (id->lib) {
if ((uint(id->lib->runtime->temp_index) < lib_main_array_len) &&
/* this check should never fail, just in case 'id->lib' is a dangling pointer. */
(lib_main_array[id->lib->runtime->temp_index]->curlib == id->lib))
{
if (id->lib && (do_split_packed_ids || (id->lib->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0)) {
if (uint(id->lib->runtime->temp_index) < lib_main_array.size()) {
Main *mainvar = lib_main_array[id->lib->runtime->temp_index];
BLI_assert(mainvar->curlib == id->lib);
ListBase *lb_dst = which_libbase(mainvar, GS(id->name));
BLI_remlink(lb_src, id);
BLI_addtail(lb_dst, id);
@@ -412,7 +412,7 @@ static void split_libdata(ListBase *lb_src, Main **lib_main_array, const uint li
}
}
void blo_split_main(Main *bmain)
void blo_split_main(Main *bmain, const bool do_split_packed_ids)
{
BLI_assert(!bmain->split_mains);
bmain->split_mains = std::make_shared<blender::VectorSet<Main *>>();
@@ -432,13 +432,15 @@ void blo_split_main(Main *bmain)
BKE_main_namemap_clear(*bmain);
/* (Library.temp_index -> Main), lookup table */
const uint lib_main_array_len = BLI_listbase_count(&bmain->libraries);
Main **lib_main_array = MEM_malloc_arrayN<Main *>(lib_main_array_len, __func__);
blender::Vector<Main *> lib_main_array;
int i = 0;
for (Library *lib = static_cast<Library *>(bmain->libraries.first); lib;
lib = static_cast<Library *>(lib->id.next), i++)
{
if (!do_split_packed_ids && (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) != 0) {
continue;
}
Main *libmain = BKE_main_new();
libmain->curlib = lib;
libmain->versionfile = lib->runtime->versionfile;
@@ -450,7 +452,7 @@ void blo_split_main(Main *bmain)
bmain->split_mains->add_new(libmain);
libmain->split_mains = bmain->split_mains;
lib->runtime->temp_index = i;
lib_main_array[i] = libmain;
lib_main_array.append(libmain);
}
MainListsArray lbarray = BKE_main_lists_get(*bmain);
@@ -461,10 +463,8 @@ void blo_split_main(Main *bmain)
/* No ID_LI data-block should ever be linked anyway, but just in case, better be explicit. */
continue;
}
split_libdata(lbarray[i], lib_main_array, lib_main_array_len);
split_libdata(lbarray[i], lib_main_array, do_split_packed_ids);
}
MEM_freeN(lib_main_array);
}
static void read_file_version_and_colorspace(FileData *fd, Main *main)
@@ -551,58 +551,6 @@ static void read_file_bhead_idname_map_create(FileData *fd)
}
}
static Main *blo_find_main(FileData *fd, const char *filepath, const char *relabase)
{
Main *m;
Library *lib;
char filepath_abs[FILE_MAX];
STRNCPY(filepath_abs, filepath);
BLI_path_abs(filepath_abs, relabase);
BLI_path_normalize(filepath_abs);
// printf("blo_find_main: relabase %s\n", relabase);
// printf("blo_find_main: original in %s\n", filepath);
// printf("blo_find_main: converted to %s\n", filepath_abs);
for (Main *m : *fd->bmain->split_mains) {
const char *libname = (m->curlib) ? m->curlib->runtime->filepath_abs : m->filepath;
if (BLI_path_cmp(filepath_abs, libname) == 0) {
if (G.debug & G_DEBUG) {
CLOG_DEBUG(&LOG, "Found library %s", libname);
}
return m;
}
}
m = BKE_main_new();
fd->bmain->split_mains->add_new(m);
m->split_mains = fd->bmain->split_mains;
/* Add library data-block itself to 'main' Main, since libraries are **never** linked data.
* Fixes bug where you could end with all ID_LI data-blocks having the same name... */
lib = BKE_id_new<Library>(fd->bmain, BLI_path_basename(filepath));
/* Important, consistency with main ID reading code from read_libblock(). */
lib->id.us = ID_FAKE_USERS(lib);
/* Matches direct_link_library(). */
id_us_ensure_real(&lib->id);
STRNCPY(lib->filepath, filepath);
STRNCPY(lib->runtime->filepath_abs, filepath_abs);
m->curlib = lib;
read_file_version_and_colorspace(fd, m);
if (G.debug & G_DEBUG) {
CLOG_DEBUG(&LOG, "Added new lib %s", filepath);
}
return m;
}
void blo_readfile_invalidate(FileData *fd, Main *bmain, const char *message)
{
/* Tag given `bmain`, and 'root 'local' main one (in case given one is a library one) as invalid.
@@ -845,6 +793,16 @@ const char *blo_bhead_id_name(FileData *fd, const BHead *bhead)
return nullptr;
}
short blo_bhead_id_flag(const FileData *fd, const BHead *bhead)
{
BLI_assert(blo_bhead_is_id(bhead));
if (fd->id_flag_offset < 0) {
return 0;
}
return *reinterpret_cast<const short *>(
POINTER_OFFSET(bhead, sizeof(*bhead) + fd->id_flag_offset));
}
AssetMetaData *blo_bhead_id_asset_data_address(const FileData *fd, const BHead *bhead)
{
BLI_assert(blo_bhead_is_id_valid_type(bhead));
@@ -853,6 +811,20 @@ AssetMetaData *blo_bhead_id_asset_data_address(const FileData *fd, const BHead *
nullptr;
}
static const IDHash *blo_bhead_id_deep_hash(const FileData *fd, const BHead *bhead)
{
BLI_assert(blo_bhead_is_id_valid_type(bhead));
if (fd->id_flag_offset < 0 || fd->id_deep_hash_offset < 0) {
return nullptr;
}
const short flag = blo_bhead_id_flag(fd, bhead);
if (!(flag & ID_FLAG_LINKED_AND_PACKED)) {
return nullptr;
}
return reinterpret_cast<const IDHash *>(
POINTER_OFFSET(bhead, sizeof(*bhead) + fd->id_deep_hash_offset));
}
static void read_blender_header(FileData *fd)
{
const BlenderHeaderVariant header_variant = BLO_readfile_blender_header_decode(fd->file);
@@ -920,6 +892,10 @@ static bool read_file_dna(FileData *fd, const char **r_error_message)
BLI_assert(fd->id_name_offset != -1);
fd->id_asset_data_offset = DNA_struct_member_offset_by_name_with_alias(
fd->filesdna, "ID", "AssetMetaData", "*asset_data");
fd->id_flag_offset = DNA_struct_member_offset_by_name_with_alias(
fd->filesdna, "ID", "short", "flag");
fd->id_deep_hash_offset = DNA_struct_member_offset_by_name_with_alias(
fd->filesdna, "ID", "IDHash", "deep_hash");
fd->filesubversion = subversion;
@@ -1149,6 +1125,7 @@ static FileData *filedata_new(BlendFileReadReport *reports)
fd->datamap = oldnewmap_new();
fd->globmap = oldnewmap_new();
fd->libmap = oldnewmap_new();
fd->id_by_deep_hash = std::make_shared<blender::Map<IDHash, ID *>>();
fd->reports = reports;
@@ -1315,6 +1292,11 @@ static FileData *blo_filedata_from_file_descriptor(const char *filepath,
FileData *fd = filedata_new(reports);
fd->file = file;
BLI_stat_t stat;
if (BLI_stat(filepath, &stat) != -1) {
fd->file_stat = stat;
}
return fd;
}
@@ -2255,6 +2237,9 @@ static void direct_link_id_common(BlendDataReader *reader,
id->session_uid = MAIN_ID_SESSION_UID_UNSET;
}
if (ID_IS_PACKED(id)) {
BLI_assert(current_library->flag & LIBRARY_FLAG_IS_ARCHIVE);
}
id->lib = current_library;
if (id->lib) {
/* Always fully clear fake user flag for linked data. */
@@ -2444,6 +2429,8 @@ static void library_filedata_release(Library *lib)
{
if (lib->runtime->filedata) {
BLI_assert(lib->runtime->versionfile != 0);
BLI_assert_msg(!lib->runtime->is_filedata_owner || (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0,
"Packed Archive libraries should never own their filedata");
if (lib->runtime->is_filedata_owner) {
blo_filedata_free(lib->runtime->filedata);
}
@@ -2470,6 +2457,11 @@ static void direct_link_library(FileData *fd, Library *lib, Main *main)
if (!newmain->curlib) {
continue;
}
if (newmain->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE || lib->flag & LIBRARY_FLAG_IS_ARCHIVE) {
/* Archive library should never be used to link new data, and there can be many such
* archive libraries for a same 'real' blendfile one. */
continue;
}
if (BLI_path_cmp(newmain->curlib->runtime->filepath_abs, lib->runtime->filepath_abs) == 0) {
BLO_reportf_wrap(fd->reports,
RPT_WARNING,
@@ -2510,9 +2502,24 @@ static void direct_link_library(FileData *fd, Library *lib, Main *main)
newmain->split_mains = fd->bmain->split_mains;
newmain->curlib = lib;
if (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) {
/* Archive libraries contains only embedded linked IDs, which by definition have the same
* fileversion as the blendfile that contains them. */
lib->runtime->versionfile = newmain->versionfile = fd->bmain->versionfile;
lib->runtime->subversionfile = newmain->subversionfile = fd->bmain->subversionfile;
/* The filedata of a packed archive library should always be the one of the blendfile which
* defines the library ID and packs its linked IDs. */
lib->runtime->filedata = fd;
lib->runtime->is_filedata_owner = false;
}
lib->runtime->parent = nullptr;
id_us_ensure_real(&lib->id);
/* Should always be null, Library IDs in Blender are always local. */
lib->id.lib = nullptr;
}
/* Always call this once you have loaded new library data to set the relative paths correctly
@@ -2680,6 +2687,82 @@ static BHead *read_data_into_datamap(FileData *fd,
return bhead;
}
/* Add a Main (and optionally create a matching Library ID), for the given filepath.
*
* - If `lib` is `nullptr`, create a new Library ID, otherwise only create a new Main for the given
* library.
* - `reference_lib` is the 'archive parent' of an archive (packed) library, can be null and will
* be ignored otherwise. */
static Main *blo_add_main_for_library(FileData *fd,
Library *lib,
Library *reference_lib,
const char *lib_filepath,
char (&filepath_abs)[FILE_MAX],
const bool is_packed_library)
{
Main *bmain = BKE_main_new();
fd->bmain->split_mains->add_new(bmain);
bmain->split_mains = fd->bmain->split_mains;
if (!lib) {
/* Add library data-block itself to 'main' Main, since libraries are **never** linked data.
* Fixes bug where you could end with all ID_LI data-blocks having the same name... */
lib = BKE_id_new<Library>(fd->bmain,
reference_lib ? BKE_id_name(reference_lib->id) :
BLI_path_basename(lib_filepath));
/* Important, consistency with main ID reading code from read_libblock(). */
lib->id.us = ID_FAKE_USERS(lib);
/* Matches direct_link_library(). */
id_us_ensure_real(&lib->id);
STRNCPY(lib->filepath, lib_filepath);
STRNCPY(lib->runtime->filepath_abs, filepath_abs);
if (is_packed_library) {
/* FIXME: This logic is very similar to the code in BKE_library dealing with archived
* libraries (e.g. #add_archive_library). Might be good to try to factorize it. */
lib->archive_parent_library = reference_lib;
constexpr uint16_t copy_flag = ~LIBRARY_FLAG_IS_ARCHIVE;
lib->flag = (reference_lib->flag & copy_flag) | LIBRARY_FLAG_IS_ARCHIVE;
lib->runtime->parent = reference_lib->runtime->parent;
/* Only copy a subset of the reference library tags. E.g. an archive library should never be
* considered as writable, so never copy #LIBRARY_ASSET_FILE_WRITABLE. This may need further
* tweaking still. */
constexpr uint16_t copy_tag = (LIBRARY_TAG_RESYNC_REQUIRED | LIBRARY_ASSET_EDITABLE |
LIBRARY_IS_ASSET_EDIT_FILE);
lib->runtime->tag = reference_lib->runtime->tag & copy_tag;
/* The filedata of a packed archive library should always be the one of the blendfile which
* defines the library ID and packs its linked IDs. */
lib->runtime->filedata = fd;
lib->runtime->is_filedata_owner = false;
reference_lib->runtime->archived_libraries.append(lib);
}
}
else {
if (is_packed_library) {
BLI_assert(lib->flag & LIBRARY_FLAG_IS_ARCHIVE);
BLI_assert(lib->archive_parent_library == reference_lib);
BLI_assert(reference_lib->runtime->archived_libraries.contains(lib));
BLI_assert(lib->runtime->filedata == nullptr);
lib->runtime->filedata = fd;
lib->runtime->is_filedata_owner = false;
}
else {
/* Should never happen currently. */
BLI_assert_unreachable();
}
}
bmain->curlib = lib;
return bmain;
}
/* Verify if the datablock and all associated data is identical. */
static bool read_libblock_is_identical(FileData *fd, BHead *bhead)
{
@@ -2764,7 +2847,10 @@ static void read_undo_move_libmain_data(FileData *fd, Main *libmain, BHead *bhea
ID *id_iter;
FOREACH_MAIN_ID_BEGIN (libmain, id_iter) {
BKE_main_idmap_insert_id(fd->new_idmap_uid, id_iter);
/* Packed IDs are read from the memfile, so don't add them here already. */
if (!ID_IS_PACKED(id_iter)) {
BKE_main_idmap_insert_id(fd->new_idmap_uid, id_iter);
}
}
FOREACH_MAIN_ID_END;
}
@@ -2796,10 +2882,13 @@ static bool read_libblock_undo_restore_library(FileData *fd,
* modified should not be an issue currently. */
for (Main *libmain : fd->old_bmain->split_mains->as_span().drop_front(1)) {
if (&libmain->curlib->id == id_old) {
BLI_assert(libmain->curlib);
BLI_assert((libmain->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0);
CLOG_DEBUG(&LOG_UNDO,
" compare with %s -> match (existing libpath: %s)",
libmain->curlib ? libmain->curlib->id.name : "<none>",
libmain->curlib ? libmain->curlib->runtime->filepath_abs : "<none>");
libmain->curlib->id.name,
libmain->curlib->runtime->filepath_abs);
/* In case of a library, we need to re-add its main to fd->bmain->split_mains,
* because if we have later a missing ID_LINK_PLACEHOLDER,
* we need to get the correct lib it is linked to!
@@ -2914,6 +3003,26 @@ static void read_libblock_undo_restore_identical(
* data-blocks too. */
ob->mode &= ~OB_MODE_EDIT;
}
if (GS(id_old->name) == ID_LI) {
Library *lib = reinterpret_cast<Library *>(id_old);
if (lib->flag & LIBRARY_FLAG_IS_ARCHIVE) {
BLI_assert(lib->runtime->filedata == nullptr);
BLI_assert(lib->archive_parent_library);
/* The 'normal' parent of this archive library should already have been moved into the new
* Main. */
BLI_assert(BKE_main_idmap_lookup_uid(fd->new_idmap_uid,
lib->archive_parent_library->id.session_uid) ==
&lib->archive_parent_library->id);
/* The archive library ID has been moved in the new Main, but not its own old split main, as
* these packed IDs should be handled like local ones in undo case. So a new split libmain
* needs to be created to contain its packed IDs. */
blo_add_main_for_library(
fd, lib, lib->archive_parent_library, lib->filepath, lib->runtime->filepath_abs, true);
}
else {
BLI_assert_unreachable();
}
}
}
/* For undo, store changed datablock at old address. */
@@ -2970,8 +3079,10 @@ static bool read_libblock_undo_restore(
const bool do_partial_undo = (fd->skip_flags & BLO_READ_SKIP_UNDO_OLD_MAIN) == 0;
#ifndef NDEBUG
if (do_partial_undo && (bhead->code != ID_LINK_PLACEHOLDER)) {
/* This code should only ever be reached for local data-blocks. */
if (do_partial_undo && (bhead->code != ID_LINK_PLACEHOLDER) &&
(blo_bhead_id_flag(fd, bhead) & ID_FLAG_LINKED_AND_PACKED) == 0)
{
/* This code should only ever be reached for local or packed data-blocks. */
BLI_assert(main->curlib == nullptr);
}
#endif
@@ -2983,8 +3094,13 @@ static bool read_libblock_undo_restore(
nullptr;
if (bhead->code == ID_LI) {
/* Restore library datablock, if possible. */
if (read_libblock_undo_restore_library(fd, id, id_old, bhead)) {
/* Restore library datablock, if possible.
*
* Never handle archive libraries and their packed IDs as normal ones. These are local data,
* and need to be fully handled like local IDs. */
if (id_old && (reinterpret_cast<Library *>(id_old)->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0 &&
read_libblock_undo_restore_library(fd, id, id_old, bhead))
{
return true;
}
}
@@ -3182,6 +3298,14 @@ static BHead *read_libblock(FileData *fd,
if (main->id_map != nullptr) {
BKE_main_idmap_insert_id(main->id_map, id_target);
}
if (ID_IS_PACKED(id)) {
BLI_assert(id->deep_hash != IDHash::get_null());
fd->id_by_deep_hash->add_new(id->deep_hash, id);
BLI_assert(main->curlib);
}
if (fd->file_stat) {
id->runtime->src_blend_modifification_time = fd->file_stat->st_mtime;
}
}
return bhead;
@@ -3831,6 +3955,10 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
}
while (bhead) {
/* If not-null after the `switch`, the BHead is an ID one and needs to be read. */
Main *bmain_to_read_into = nullptr;
bool placeholder_set_indirect_extern = false;
switch (bhead->code) {
case BLO_CODE_DATA:
case BLO_CODE_DNA1:
@@ -3854,42 +3982,61 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
break;
case ID_LINK_PLACEHOLDER:
if (fd->skip_flags & BLO_READ_SKIP_DATA) {
if ((fd->skip_flags & BLO_READ_SKIP_DATA) != 0) {
bhead = blo_bhead_next(fd, bhead);
break;
}
else {
/* Add link placeholder to the main of the library it belongs to.
* The library is the most recently loaded #ID_LI block, according
* to the file format definition. So we can use the entry at the
* end of `fd->bmain->split_mains`, typically the one last added in
* #direct_link_library. */
Main *libmain = (*fd->bmain->split_mains)[fd->bmain->split_mains->size() - 1];
bhead = read_libblock(fd, libmain, bhead, 0, {}, true, nullptr);
}
/* Add link placeholder to the main of the library it belongs to.
*
* The library is the most recently loaded #ID_LI block, according to the file format
* definition. So we can use the entry at the end of `fd->bmain->split_mains`, typically
* the one last added in #direct_link_library. */
bmain_to_read_into = (*fd->bmain->split_mains)[fd->bmain->split_mains->size() - 1];
placeholder_set_indirect_extern = true;
break;
case ID_LI:
if ((fd->skip_flags & BLO_READ_SKIP_DATA) != 0) {
bhead = blo_bhead_next(fd, bhead);
break;
}
/* Library IDs are always read into the first (aka 'local') Main, even if they are written
* in 'library' blendfile-space (for archive libraries e.g.). */
bmain_to_read_into = fd->bmain;
break;
/* in 2.50+ files, the file identifier for screens is patched, forward compatibility */
case ID_SCRN:
/* in 2.50+ files, the file identifier for screens is patched, forward compatibility */
bhead->code = ID_SCR;
/* pass on to default */
ATTR_FALLTHROUGH;
default: {
if (blo_bhead_is_id_valid_type(bhead)) {
/* BHead is a valid known ID type one, read the whole ID and its sub-data, unless reading
* actual data is skipped. */
if (fd->skip_flags & BLO_READ_SKIP_DATA) {
bhead = blo_bhead_next(fd, bhead);
}
else {
bhead = read_libblock(fd, bfd->main, bhead, ID_TAG_LOCAL, {}, false, nullptr);
}
}
else {
/* Unknown BHead type (or ID type), ignore it and skip to next BHead. */
if ((fd->skip_flags & BLO_READ_SKIP_DATA) != 0 || !blo_bhead_is_id_valid_type(bhead)) {
bhead = blo_bhead_next(fd, bhead);
break;
}
/* Put read real ID into the main of the library it belongs to.
*
* Local IDs should all be written before any Library in the blendfile, so this code will
* always select `fd->bmain` for these.
*
* Packed linked IDs are real ID data in the currently read blendfile (unlike placeholders
* for regular linked data). But they are in their archive library 'name space' and
* 'blendfile space', so this follows the same logic as for placeholders to select the
* Main.
*
* The library is the most recently loaded #ID_LI block, according to the file format
* definition. So we can use the entry at the end of `fd->bmain->split_mains`, typically
* the one last added in #direct_link_library. */
bmain_to_read_into = (*fd->bmain->split_mains)[fd->bmain->split_mains->size() - 1];
BLI_assert_msg((bmain_to_read_into == fd->bmain ||
(blo_bhead_id_flag(fd, bhead) & ID_FLAG_LINKED_AND_PACKED) != 0),
"Local IDs should always be put in the first Main split data-base, not in "
"a 'linked data' one");
}
}
if (bmain_to_read_into) {
bhead = read_libblock(
fd, bmain_to_read_into, bhead, 0, {}, placeholder_set_indirect_extern, nullptr);
}
if (bfd->main->is_read_invalid) {
return bfd;
@@ -3916,6 +4063,14 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
* of its items. */
blender::Vector<Main *> old_main_split_mains = {old_main->split_mains->as_span()};
for (Main *libmain : old_main_split_mains.as_span().drop_front(1)) {
BLI_assert(libmain->curlib);
if (libmain->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) {
/* Never move archived libraries and their content, these are 'local' data in undo context,
* so all packed linked IDs should have been handled like local ones undo-wise, and if
* packed libraries remain unused at this point, then they are indeed fully unused/removed
* from the new main. */
continue;
}
read_undo_move_libmain_data(fd, libmain, nullptr);
}
}
@@ -3958,7 +4113,36 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
/* Do versioning before read_libraries, but skip in undo case. */
if (!is_undo) {
if ((fd->skip_flags & BLO_READ_SKIP_DATA) == 0) {
do_versions(fd, nullptr, bfd->main);
for (Main *bmain : *fd->bmain->split_mains) {
/* Packed IDs are stored in the current .blend file, but belong to dedicated 'archive
* library' Mains, not the first, 'local' Main. So they do need versioning here, as for
* local IDs, which is why all the split Mains in the list need to be checked.
*
* Placeholders (of 'real' linked data) can't be versioned yet. Since they also belong to
* dedicated 'library' Mains, and are not mixed with the 'packed' ones, these Mains can be
* entirely skipped. */
const bool contains_link_placeholder = (bmain->curlib != nullptr &&
(bmain->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) ==
0);
#ifndef NDEBUG
MainListsArray lbarray = BKE_main_lists_get(*bmain);
for (ListBase *lb_array : lbarray) {
LISTBASE_FOREACH_MUTABLE (ID *, id, lb_array) {
BLI_assert_msg((id->runtime->readfile_data->tags.is_link_placeholder ==
contains_link_placeholder),
contains_link_placeholder ?
"Real Library split Main contains non-placeholder IDs" :
(bmain->curlib == nullptr ?
"Local data split Main contains placeholder IDs" :
"Archive Library split Main contains placeholder IDs"));
}
}
#endif
if (contains_link_placeholder) {
continue;
}
do_versions(fd, bmain->curlib, bmain);
}
}
if ((fd->skip_flags & BLO_READ_SKIP_USERDEF) == 0) {
@@ -4027,7 +4211,9 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
}
LISTBASE_FOREACH_MUTABLE (Library *, lib, &bfd->main->libraries) {
/* Now we can clear this runtime library filedata, it is not needed anymore. */
/* Now we can clear this runtime library filedata, it is not needed anymore.
*
* NOTE: This is also important to do for archive libraries. */
library_filedata_release(lib);
/* If no data-blocks were read from a library (should only happen when all references to a
* library's data are `ID_FLAG_INDIRECT_WEAK_LINK`), its versionfile will still be zero and
@@ -4038,8 +4224,12 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
* placeholders IDs created will reference the library ID, and the library ID will have a
* valid version number as the file was read to search for the linked IDs.
* - In case the library blendfile does not exist, its local Library ID will get the version
* of the current local Main (i.e. the loaded blendfile). */
if (lib->runtime->versionfile == 0) {
* of the current local Main (i.e. the loaded blendfile).
* - In case it is a reference library for archived ones, its runtime #archived_libraries
* vector will not be empty, and it must be kept, even if no data is directly linked from
* it anymore.
*/
if (lib->runtime->versionfile == 0 && lib->runtime->archived_libraries.is_empty()) {
#ifndef NDEBUG
ID *id_iter;
FOREACH_MAIN_ID_BEGIN (bfd->main, id_iter) {
@@ -4247,21 +4437,41 @@ static BHead *find_bhead_from_idname(FileData *fd, const char *idname)
return find_bhead_from_code_name(fd, id_code_old, idname + 2);
}
static ID *library_id_is_yet_read(FileData *fd, Main *mainvar, BHead *bhead)
static ID *library_id_is_yet_read_deep_hash(FileData *fd, BHead *bhead)
{
if (const IDHash *deep_hash = blo_bhead_id_deep_hash(fd, bhead)) {
if (ID *existing_id = fd->id_by_deep_hash->lookup_default(*deep_hash, nullptr)) {
return existing_id;
}
}
return nullptr;
}
static ID *library_id_is_yet_read_main(Main *mainvar, const char *idname)
{
if (mainvar->id_map == nullptr) {
mainvar->id_map = BKE_main_idmap_create(mainvar, false, nullptr, MAIN_IDMAP_TYPE_NAME);
}
BLI_assert(BKE_main_idmap_main_get(mainvar->id_map) == mainvar);
ID *existing_id = BKE_main_idmap_lookup_name(
mainvar->id_map, GS(idname), idname + 2, mainvar->curlib);
BLI_assert(existing_id ==
BLI_findstring(which_libbase(mainvar, GS(idname)), idname, offsetof(ID, name)));
return existing_id;
}
static ID *library_id_is_yet_read(FileData *fd, Main *mainvar, BHead *bhead)
{
if (ID *existing_id = library_id_is_yet_read_deep_hash(fd, bhead)) {
return existing_id;
}
const char *idname = blo_bhead_id_name(fd, bhead);
if (!idname) {
return nullptr;
}
ID *id = BKE_main_idmap_lookup_name(mainvar->id_map, GS(idname), idname + 2, mainvar->curlib);
BLI_assert(id == BLI_findstring(which_libbase(mainvar, GS(idname)), idname, offsetof(ID, name)));
return id;
return library_id_is_yet_read_main(mainvar, idname);
}
static void read_libraries_report_invalid_id_names(FileData *fd,
@@ -4307,15 +4517,132 @@ struct BlendExpander {
BLOExpandDoitCallback callback;
};
/* Find the existing Main matching the given blendfile library filepath, or create a new one (with
* the matching Library ID) if needed.
*
* NOTE: The process is a bit more complex for packed linked IDs and their archive libraries, as
* in this case, this function also needs to find or create a new suitable archive library, i.e.
* one which does not contain yet the given ID (from its name & type). */
static Main *blo_find_main_for_library_and_idname(FileData *fd,
const char *lib_filepath,
const char *relabase,
const BHead *id_bhead,
const char *id_name,
const bool is_packed_id)
{
Library *parent_lib = nullptr;
char filepath_abs[FILE_MAX];
STRNCPY(filepath_abs, lib_filepath);
BLI_path_abs(filepath_abs, relabase);
BLI_path_normalize(filepath_abs);
for (Main *main_it : *fd->bmain->split_mains) {
const char *libname = (main_it->curlib) ? main_it->curlib->runtime->filepath_abs :
main_it->filepath;
if (BLI_path_cmp(filepath_abs, libname) == 0) {
CLOG_DEBUG(&LOG,
"Found library '%s' for file path '%s'",
main_it->curlib ? main_it->curlib->id.name : "<None>",
lib_filepath);
/* Due to how parent and archive libraries are created and written in the blend-file,
* the first library matching a given filepath should never be an archive one. */
BLI_assert(!main_it->curlib || (main_it->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0);
if (!is_packed_id) {
return main_it;
}
/* For packed IDs, the Main of the main owner library is not a valid one. Another loop is
* needed into all the Mains matching the archive libraries of this main library. */
BLI_assert(main_it->curlib);
parent_lib = main_it->curlib;
break;
}
}
if (is_packed_id) {
if (parent_lib) {
/* Try to find an 'available' existing archive Main library, i.e. one that does not yet
* contain an ID of the same type and name. */
for (Main *main_it : *fd->bmain->split_mains) {
if (!main_it->curlib || (main_it->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) == 0 ||
main_it->curlib->archive_parent_library != parent_lib)
{
continue;
}
if (ID *packed_id = library_id_is_yet_read_main(main_it, id_name)) {
/* Archive Main library already contains a 'same' ID - but it should have a different
* deep_hash. Otherwise, a previous call to `library_id_is_yet_read()` should have
* returned this ID, and this code should not be reached. */
BLI_assert(packed_id->deep_hash != *blo_bhead_id_deep_hash(fd, id_bhead));
UNUSED_VARS_NDEBUG(packed_id, id_bhead);
continue;
}
BLI_assert(ELEM(main_it->curlib->runtime->filedata, fd, nullptr));
main_it->curlib->runtime->filedata = fd;
main_it->curlib->runtime->is_filedata_owner = false;
BLI_assert(main_it->versionfile != 0);
CLOG_DEBUG(&LOG,
"Found archive library '%s' for the packed ID '%s'",
main_it->curlib->id.name,
id_name);
return main_it;
}
}
else {
/* An archive library requires an existing parent library, create an empty, 'virtual' one if
* needed. */
Main *reference_bmain = blo_add_main_for_library(
fd, nullptr, nullptr, lib_filepath, filepath_abs, false);
parent_lib = reference_bmain->curlib;
CLOG_DEBUG(&LOG,
"Added new parent library '%s' for file path '%s'",
parent_lib->id.name,
lib_filepath);
}
}
BLI_assert(parent_lib || !is_packed_id);
Main *bmain = blo_add_main_for_library(
fd, nullptr, parent_lib, lib_filepath, filepath_abs, is_packed_id);
read_file_version_and_colorspace(fd, bmain);
if (is_packed_id) {
CLOG_DEBUG(&LOG,
"Added new archive library '%s' for the packed ID '%s'",
bmain->curlib->id.name,
id_name);
}
else {
CLOG_DEBUG(
&LOG, "Added new library '%s' for file path '%s'", bmain->curlib->id.name, lib_filepath);
}
return bmain;
}
/* Actually load an ID from a library. There are three possible cases here:
* - `existing_id` is non-null: calling code already found a suitable existing ID, this function
* essentially then only updates the mappings for `bhead->old` address to point to the given
* ID. This is the only case where `libmain` may be `nullptr`.
* - The given bhead has an already loaded matching ID (found by a call to
* `library_id_is_yet_read`), then once that ID is found behavior is as in the previous case.
* - No matching existing ID is found, then a new one is actually read from the given FileData.
*/
static void read_id_in_lib(FileData *fd,
std::queue<ID *> &ids_to_expand,
Main *libmain,
Library *parent_lib,
BHead *bhead,
ID *existing_id,
ID_Readfile_Data::Tags id_read_tags)
{
ID *id = library_id_is_yet_read(fd, libmain, bhead);
ID *id = existing_id;
if (id == nullptr) {
BLI_assert(libmain);
id = library_id_is_yet_read(fd, libmain, bhead);
}
if (id == nullptr) {
/* ID has not been read yet, add placeholder to the main of the
* library it belongs to, so that it will be read later. */
@@ -4392,44 +4719,96 @@ static void expand_doit_library(void *fdhandle,
if (!blo_bhead_is_id_valid_type(bhead)) {
return;
}
if (!blo_bhead_id_name(fd, bhead)) {
const char *id_name = blo_bhead_id_name(fd, bhead);
if (!id_name) {
/* Do not allow linking ID which names are invalid (likely coming from a future version of
* Blender allowing longer names). */
return;
}
const bool is_packed_id = (blo_bhead_id_flag(fd, bhead) & ID_FLAG_LINKED_AND_PACKED) != 0;
BLI_assert_msg(!is_packed_id || bhead->code != ID_LINK_PLACEHOLDER,
"A link placeholder ID (aka reference to some ID linked from another library) "
"should never be packed.");
if (bhead->code == ID_LINK_PLACEHOLDER) {
/* Placeholder link to data-block in another library. */
BHead *bheadlib = find_previous_lib(fd, bhead);
if (bheadlib == nullptr) {
BLO_reportf_wrap(fd->reports,
RPT_ERROR,
RPT_("LIB: .blend file %s seems corrupted, no owner 'Library' data found "
"for the linked data-block %s"),
mainvar->curlib->runtime->filepath_abs,
id_name ? id_name : "<InvalidIDName>");
return;
}
Library *lib = reinterpret_cast<Library *>(
read_id_struct(fd, bheadlib, "Data for Library ID type", INDEX_ID_NULL));
Main *libmain = blo_find_main(fd, lib->filepath, fd->relabase);
Main *libmain = blo_find_main_for_library_and_idname(
fd, lib->filepath, fd->relabase, nullptr, nullptr, false);
MEM_freeN(lib);
if (libmain->curlib == nullptr) {
const char *idname = blo_bhead_id_name(fd, bhead);
BLO_reportf_wrap(fd->reports,
RPT_WARNING,
RPT_("LIB: Data refers to main .blend file: '%s' from %s"),
idname ? idname : "<InvalidIDName>",
id_name ? id_name : "<InvalidIDName>",
mainvar->curlib->runtime->filepath_abs);
return;
}
/* Placeholders never need expanding, as they are a mere reference to ID from another
* library/blendfile. */
read_id_in_lib(fd, ids_to_expand, libmain, mainvar->curlib, bhead, {});
read_id_in_lib(fd, ids_to_expand, libmain, mainvar->curlib, bhead, nullptr, {});
}
else if (is_packed_id) {
/* Packed data-block from another library. */
/* That exact same packed ID may have already been read before. */
if (ID *existing_id = library_id_is_yet_read_deep_hash(fd, bhead)) {
/* Ensure that the current BHead's `old` pointer will also be remapped to the found existing
* ID. */
read_id_in_lib(fd, ids_to_expand, nullptr, nullptr, bhead, existing_id, {});
return;
}
BHead *bheadlib = find_previous_lib(fd, bhead);
if (bheadlib == nullptr) {
BLO_reportf_wrap(fd->reports,
RPT_ERROR,
RPT_("LIB: .blend file %s seems corrupted, no owner 'Library' data found "
"for the packed linked data-block %s"),
mainvar->curlib->runtime->filepath_abs,
id_name ? id_name : "<InvalidIDName>");
return;
}
Library *lib = reinterpret_cast<Library *>(
read_id_struct(fd, bheadlib, "Data for Library ID type", INDEX_ID_NULL));
Main *libmain = blo_find_main_for_library_and_idname(
fd, lib->filepath, fd->relabase, bhead, id_name, is_packed_id);
MEM_freeN(lib);
if (libmain->curlib == nullptr) {
BLO_reportf_wrap(fd->reports,
RPT_WARNING,
RPT_("LIB: Data refers to main .blend file: '%s' from %s"),
id_name ? id_name : "<InvalidIDName>",
mainvar->curlib->runtime->filepath_abs);
return;
}
ID_Readfile_Data::Tags id_read_tags{};
id_read_tags.needs_expanding = true;
read_id_in_lib(fd, ids_to_expand, libmain, nullptr, bhead, nullptr, id_read_tags);
}
else {
/* Data-block in same library. */
ID_Readfile_Data::Tags id_read_tags{};
id_read_tags.needs_expanding = true;
read_id_in_lib(fd, ids_to_expand, mainvar, nullptr, bhead, id_read_tags);
read_id_in_lib(fd, ids_to_expand, mainvar, nullptr, bhead, nullptr, id_read_tags);
}
}
@@ -4476,6 +4855,13 @@ static void expand_main(void *fdhandle, Main *mainvar, BLOExpandDoitCallback cal
FileData *fd = static_cast<FileData *>(fdhandle);
BlendExpander expander = {fd, {}, mainvar, callback};
/* Note: Packed IDs are the only current case where IDs read/loaded from a library blendfile will
* end up in another Main (outside of placeholders, which never need to be expanded). This is not
* a problem for initialization of the 'to be expanded' queue though, as no packed ID can be
* directly linked currently, they are only brough in indirectly, i.e. during the expansion
* process itself.
*
* So just looping on the 'main'/root Main of the read library is fine here currently. */
ID *id_iter;
FOREACH_MAIN_ID_BEGIN (mainvar, id_iter) {
if (BLO_readfile_id_runtime_tags(*id_iter).needs_expanding) {
@@ -4600,11 +4986,22 @@ static Main *library_link_begin(Main *mainvar,
fd->bmain = mainvar;
/* Add already existing packed data-blocks to map so that they are not loaded again. */
ID *id;
FOREACH_MAIN_ID_BEGIN (mainvar, id) {
if (ID_IS_PACKED(id)) {
fd->id_by_deep_hash->add(id->deep_hash, id);
}
}
FOREACH_MAIN_ID_END;
/* make mains */
blo_split_main(mainvar);
/* which one do we need? */
mainl = blo_find_main(fd, filepath, BKE_main_blendfile_path(mainvar));
/* Find or create a Main matching the current library filepath. */
/* Note: Directly linking packed IDs is not supported currently. */
mainl = blo_find_main_for_library_and_idname(
fd, filepath, BKE_main_blendfile_path(mainvar), nullptr, nullptr, false);
fd->fd_bmain = mainl;
if (mainl->curlib) {
mainl->curlib->runtime->filedata = fd;
@@ -4737,7 +5134,7 @@ static void library_link_end(Main *mainl, FileData **fd, const int flag, ReportL
Main *main_newid = BKE_main_new();
for (Main *mainlib : mainvar->split_mains->as_span().drop_front(1)) {
BLI_assert(mainlib->versionfile != 0);
BLI_assert(mainlib->versionfile != 0 || BKE_main_is_empty(mainlib));
/* We need to split out IDs already existing,
* or they will go again through do_versions - bad, very bad! */
split_main_newid(mainlib, main_newid);
@@ -5054,6 +5451,7 @@ static FileData *read_library_file_data(FileData *basefd, Main *bmain, Main *lib
}
fd->libmap = oldnewmap_new();
fd->id_by_deep_hash = basefd->id_by_deep_hash;
lib_bmain->curlib->runtime->filedata = fd;
lib_bmain->curlib->runtime->is_filedata_owner = true;
@@ -5104,6 +5502,13 @@ static void read_libraries(FileData *basefd)
* this list gets longer as more indirectly library blends are found. */
for (int i = 1; i < bmain->split_mains->size(); i++) {
Main *libmain = (*bmain->split_mains)[i];
BLI_assert(libmain->curlib);
/* Always skip archived libraries here, these should _never_ need to be processed here, as
* their data is local data from a blendfile perspective. */
if (libmain->curlib->flag & LIBRARY_FLAG_IS_ARCHIVE) {
BLI_assert(!has_linked_ids_to_read(libmain));
continue;
}
/* Does this library have any more linked data-blocks we need to read? */
if (has_linked_ids_to_read(libmain)) {
CLOG_DEBUG(&LOG,

30
source/blender/blenloader/intern/readfile.hh
View File

@@ -16,6 +16,7 @@
# include "BLI_winstuff.h"
#endif
#include "BLI_fileops.h"
#include "BLI_filereader.h"
#include "BLI_map.hh"
@@ -82,6 +83,7 @@ struct FileData {
BlenderHeader blender_header = {};
FileReader *file = nullptr;
std::optional<BLI_stat_t> file_stat;
/**
* Whether we are undoing (< 0) or redoing (> 0), used to choose which 'unchanged' flag to use
@@ -118,6 +120,8 @@ struct FileData {
/** Used to retrieve asset data from (bhead+1). NOTE: This may not be available in old files,
* will be -1 then! */
int id_asset_data_offset = 0;
int id_flag_offset = 0;
int id_deep_hash_offset = 0;
/** For do_versions patching. */
int globalf = 0;
int fileflags = 0;
@@ -135,6 +139,8 @@ struct FileData {
OldNewMap *datamap = nullptr;
OldNewMap *globmap = nullptr;
/** Used to keep track of already loaded packed IDs to avoid loading them multiple times. */
std::shared_ptr<blender::Map<IDHash, ID *>> id_by_deep_hash;
/**
* Store mapping from old ID pointers (the values they have in the .blend file) to new ones,
@@ -189,9 +195,22 @@ struct FileData {
void *storage_handle = nullptr;
};
/***/
/**
* Split a single main into a vector of Mains, each containing only IDs from a given library.
*
* The vector is accessible in all of the split mains through the shared pointer
* #Main::split_mains.
*
* The first Main of the vector is the same as the given `main`, and contains local IDs.
*
* If `do_split_packed_ids` is `false`, packed linked IDs remain in the local (first) main as well.
*/
void blo_split_main(Main *bmain, bool do_split_packed_ids = true);
/**
* Join the set of split mains (found in given `main` #Main::split_mains vector shared pointer)
* back into that 'main' main.
*/
void blo_join_main(Main *bmain);
void blo_split_main(Main *bmain);
BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) ATTR_NONNULL(1, 2);
@@ -232,6 +251,13 @@ BHead *blo_bhead_prev(FileData *fd, BHead *thisblock) ATTR_NONNULL(1, 2);
* it was saved in a version of Blender with higher MAX_ID_NAME value).
*/
const char *blo_bhead_id_name(FileData *fd, const BHead *bhead);
/**
* Warning! It's the caller's responsibility to ensure that the given bhead **is** an ID one!
*
* Returns the ID flag value (or `0` if the blendfile is too old and the offset of the ID::flag
* member could not be computed).
*/
short blo_bhead_id_flag(const FileData *fd, const BHead *bhead);
/**
* Warning! Caller's responsibility to ensure given bhead **is** an ID one!
*/

5
source/blender/blenloader/intern/versioning_500.cc
View File

@@ -72,6 +72,8 @@
#include "WM_api.hh"
#include "AS_asset_library.hh"
#include "readfile.hh"
#include "versioning_common.hh"
@@ -3679,4 +3681,7 @@ void blo_do_versions_500(FileData *fd, Library * /*lib*/, Main *bmain)
LISTBASE_FOREACH (Mesh *, mesh, &bmain->meshes) {
bke::mesh_freestyle_marks_to_generic(*mesh);
}
/* TODO: Can be moved to subversion bump. */
AS_asset_library_import_method_ensure_valid(*bmain);
}

36
source/blender/blenloader/intern/writefile.cc
View File

@@ -1164,6 +1164,11 @@ static void write_libraries(WriteData *wd, Main *bmain)
if (id->us == 0) {
continue;
}
if (ID_IS_PACKED(id)) {
BLI_assert(library.flag & LIBRARY_FLAG_IS_ARCHIVE);
ids_used_from_library.append(id);
continue;
}
if (id->tag & ID_TAG_EXTERN) {
ids_used_from_library.append(id);
continue;
@@ -1178,6 +1183,15 @@ static void write_libraries(WriteData *wd, Main *bmain)
if (library.packedfile) {
should_write_library = true;
}
else if (!library.runtime->archived_libraries.is_empty()) {
/* Reference 'real' blendfile library of archived 'copies' of it containing packed linked
* IDs should always be written. */
/* FIXME: A bit weak, as it could be that all archive libs are now empty (if all related
* packed linked IDs have been deleted e.g.)...
* Could be fixed by either adding more checks here, or ensuring empty archive libs are
* deleted when no ID uses them anymore? */
should_write_library = true;
}
else if (wd->use_memfile) {
/* When writing undo step we always write all existing libraries. That makes reading undo
* step much easier when dealing with purely indirectly used libraries. */
@@ -1194,16 +1208,22 @@ static void write_libraries(WriteData *wd, Main *bmain)
write_id(wd, &library.id);
/* Write placeholders for linked data-blocks that are used. */
/* Write placeholders for linked data-blocks that are used, and real IDs for the packed linked
* ones. */
for (ID *id : ids_used_from_library) {
if (!BKE_idtype_idcode_is_linkable(GS(id->name))) {
CLOG_ERROR(&LOG,
"Data-block '%s' from lib '%s' is not linkable, but is flagged as "
"directly linked",
id->name,
library.runtime->filepath_abs);
if (ID_IS_PACKED(id)) {
write_id(wd, id);
}
else {
if (!BKE_idtype_idcode_is_linkable(GS(id->name))) {
CLOG_ERROR(&LOG,
"Data-block '%s' from lib '%s' is not linkable, but is flagged as "
"directly linked",
id->name,
library.runtime->filepath_abs);
}
write_id_placeholder(wd, id);
}
write_id_placeholder(wd, id);
}
}

17
source/blender/editors/asset/intern/asset_import.cc
View File

@@ -32,11 +32,14 @@ ID *asset_local_id_ensure_imported(Main &bmain,
}
const eAssetImportMethod method = [&]() {
const bool no_packing = U.experimental.no_data_block_packing;
if (import_method) {
return *import_method;
return (no_packing && *import_method == ASSET_IMPORT_PACK) ? ASSET_IMPORT_APPEND_REUSE :
*import_method;
}
if (std::optional asset_method = asset.get_import_method()) {
return *asset_method;
return (no_packing && *asset_method == ASSET_IMPORT_PACK) ? ASSET_IMPORT_APPEND_REUSE :
*asset_method;
}
return ASSET_IMPORT_APPEND_REUSE;
}();
@@ -51,6 +54,16 @@ ID *asset_local_id_ensure_imported(Main &bmain,
asset.get_id_type(),
asset.get_name().c_str(),
(asset.get_use_relative_path() ? FILE_RELPATH : 0));
case ASSET_IMPORT_PACK:
return WM_file_link_datablock(&bmain,
nullptr,
nullptr,
nullptr,
blend_path.c_str(),
asset.get_id_type(),
asset.get_name().c_str(),
BLO_LIBLINK_PACK |
(asset.get_use_relative_path() ? FILE_RELPATH : 0));
case ASSET_IMPORT_APPEND:
return WM_file_append_datablock(&bmain,
nullptr,

7
source/blender/editors/asset/intern/asset_shelf_asset_view.cc
View File

@@ -406,8 +406,11 @@ void *AssetDragController::create_drag_data() const
return static_cast<void *>(local_id);
}
const eAssetImportMethod import_method = asset_.get_import_method().value_or(
ASSET_IMPORT_APPEND_REUSE);
eAssetImportMethod import_method = asset_.get_import_method().value_or(ASSET_IMPORT_PACK);
if (U.experimental.no_data_block_packing && import_method == ASSET_IMPORT_PACK) {
import_method = ASSET_IMPORT_APPEND_REUSE;
}
AssetImportSettings import_settings{};
import_settings.method = import_method;
import_settings.use_instance_collections = false;

3
source/blender/editors/interface/interface_icons.cc
View File

@@ -1962,6 +1962,9 @@ int ui_id_icon_get(const bContext *C, ID *id, const bool big)
int UI_icon_from_library(const ID *id)
{
if (ID_IS_LINKED(id)) {
if (ID_IS_PACKED(id)) {
return ICON_PACKAGE;
}
if (id->tag & ID_TAG_MISSING) {
return ICON_LIBRARY_DATA_BROKEN;
}

4
source/blender/editors/interface/interface_ops.cc
View File

@@ -717,6 +717,10 @@ static bool override_idtemplate_poll(bContext *C, const bool is_create_op)
return false;
}
if (ID_IS_PACKED(id)) {
return false;
}
if (is_create_op) {
if (!ID_IS_LINKED(id) && !ID_IS_OVERRIDE_LIBRARY_REAL(id)) {
return false;

16
source/blender/editors/interface/templates/interface_template_id.cc
View File

@@ -1119,7 +1119,21 @@ static void template_ID(const bContext *C,
if (!hide_buttons && !(idfrom && ID_IS_LINKED(idfrom))) {
if (ID_IS_LINKED(id)) {
const bool disabled = !BKE_idtype_idcode_is_localizable(GS(id->name));
if (id->tag & ID_TAG_INDIRECT) {
if (ID_IS_PACKED(id)) {
but = uiDefIconBut(block,
ButType::But,
0,
ICON_PACKAGE,
0,
0,
UI_UNIT_X,
UI_UNIT_Y,
nullptr,
0,
0,
TIP_("Packed library data-block, click to unpack and make local"));
}
else if (id->tag & ID_TAG_INDIRECT) {
but = uiDefIconBut(block,
ButType::But,
0,

2
source/blender/editors/screen/screen_ops.cc
View File

@@ -6959,6 +6959,8 @@ static std::string screen_drop_scene_tooltip(bContext * /*C*/,
switch (asset_drag->import_settings.method) {
case ASSET_IMPORT_LINK:
return fmt::format(fmt::runtime(TIP_("Link {}")), dragged_scene_name);
case ASSET_IMPORT_PACK:
return fmt::format(fmt::runtime(TIP_("Pack {}")), dragged_scene_name);
case ASSET_IMPORT_APPEND:
return fmt::format(fmt::runtime(TIP_("Append {}")), dragged_scene_name);
case ASSET_IMPORT_APPEND_REUSE:

2
source/blender/editors/space_file/file_draw.cc
View File

@@ -410,7 +410,7 @@ static void file_but_enable_drag(uiBut *but,
import_settings.method = eAssetImportMethod(import_method);
import_settings.use_instance_collections =
(sfile->asset_params->import_flags &
(import_method == ASSET_IMPORT_LINK ?
(ELEM(import_method, ASSET_IMPORT_LINK, ASSET_IMPORT_PACK) ?
FILE_ASSET_IMPORT_INSTANCE_COLLECTIONS_ON_LINK :
FILE_ASSET_IMPORT_INSTANCE_COLLECTIONS_ON_APPEND)) != 0;

2
source/blender/editors/space_file/filesel.cc
View File

@@ -535,6 +535,8 @@ int ED_fileselect_asset_import_method_get(const SpaceFile *sfile, const FileDirE
return ASSET_IMPORT_APPEND;
case FILE_ASSET_IMPORT_APPEND_REUSE:
return ASSET_IMPORT_APPEND_REUSE;
case FILE_ASSET_IMPORT_PACK:
return ASSET_IMPORT_PACK;
/* Should be handled above already. Break and fail below. */
case FILE_ASSET_IMPORT_FOLLOW_PREFS:

15
source/blender/editors/space_outliner/outliner_draw.cc
View File

@@ -2571,6 +2571,9 @@ static BIFIconID tree_element_get_icon_from_id(const ID *id)
if (id->tag & ID_TAG_MISSING) {
return ICON_LIBRARY_DATA_BROKEN;
}
else if (reinterpret_cast<const Library *>(id)->flag & LIBRARY_FLAG_IS_ARCHIVE) {
return ICON_PACKAGE;
}
else if (((Library *)id)->runtime->parent) {
return ICON_LIBRARY_DATA_INDIRECT;
}
@@ -2875,8 +2878,16 @@ TreeElementIcon tree_element_get_icon(TreeStoreElem *tselem, TreeElement *te)
const PointerRNA &ptr = te_rna_struct->get_pointer_rna();
if (RNA_struct_is_ID(ptr.type)) {
data.drag_id = static_cast<ID *>(ptr.data);
data.icon = RNA_struct_ui_icon(ptr.type);
ID *id = static_cast<ID *>(ptr.data);
data.drag_id = id;
if (id && GS(id->name) == ID_LI &&
id_cast<Library *>(id)->flag & LIBRARY_FLAG_IS_ARCHIVE)
{
data.icon = ICON_PACKAGE;
}
else {
data.icon = RNA_struct_ui_icon(ptr.type);
}
}
else {
data.icon = RNA_struct_ui_icon(ptr.type);

6
source/blender/editors/space_outliner/tree/tree_display_libraries.cc
View File

@@ -71,12 +71,14 @@ ListBase TreeDisplayLibraries::build_tree(const TreeSourceData &source_data)
TreeStoreElem *tselem = TREESTORE(ten);
Library *lib = (Library *)tselem->id;
BLI_assert(!lib || (GS(lib->id.name) == ID_LI));
if (!lib || !lib->runtime->parent) {
if (!lib || !(lib->runtime->parent || lib->archive_parent_library)) {
continue;
}
/* A library with a non-null `parent` is always strictly indirectly linked. */
TreeElement *parent = reinterpret_cast<TreeElement *>(lib->runtime->parent->id.newid);
TreeElement *parent = reinterpret_cast<TreeElement *>(
(lib->archive_parent_library ? lib->archive_parent_library : lib->runtime->parent)
->id.newid);
BLI_remlink(&tree, ten);
BLI_addtail(&parent->subtree, ten);
ten->parent = parent;

95
source/blender/makesdna/DNA_ID.h
View File

@@ -18,6 +18,7 @@
/** Workaround to forward-declare C++ type in C header. */
#ifdef __cplusplus
# include <cstring>
# include <type_traits>
namespace blender::bke::id {
@@ -43,10 +44,6 @@ typedef struct IDPropertyGroupChildrenSet IDPropertyGroupChildrenSet;
typedef struct ID_RuntimeHandle ID_RuntimeHandle;
#endif
#ifdef __cplusplus
extern "C" {
#endif
struct FileData;
struct GHash;
struct ID;
@@ -383,6 +380,37 @@ enum {
ID_REMAP_IS_USER_ONE_SKIPPED = 1 << 1,
};
typedef struct IDHash {
char data[16];
#ifdef __cplusplus
uint64_t hash() const
{
return *reinterpret_cast<const uint64_t *>(this->data);
}
static constexpr IDHash get_null()
{
return {};
}
bool is_null() const
{
return *this == IDHash::get_null();
}
friend bool operator==(const IDHash &a, const IDHash &b)
{
return memcmp(a.data, b.data, sizeof(a.data)) == 0;
}
friend bool operator!=(const IDHash &a, const IDHash &b)
{
return !(a == b);
}
#endif
} IDHash;
typedef struct ID {
/* There's a nasty circular dependency here.... 'void *' to the rescue! I
* really wonder why this is needed. */
@@ -433,6 +461,18 @@ typedef struct ID {
*/
unsigned int session_uid;
/**
* This is only available on packed linked data-blocks. It is a hash of the contents the
* data-block including all its dependencies. It is computed when first packing the data-block
* and is not changed afterwards. It can be used to detect that packed data-blocks in two
* separate .blend files are the same.
*
* Two data-blocks with the same deep hash are assumed to be interchangeable, but not necessarily
* exactly the same. For example, it's possible to change node positions on packed data-blocks
* without changing the deep hash.
*/
IDHash deep_hash;
/**
* User-defined custom properties storage. Typically Accessed through the 'dict' syntax from
* Python.
@@ -511,6 +551,24 @@ typedef struct Library {
/** Path name used for reading, can be relative and edited in the outliner. */
char filepath[/*FILE_MAX*/ 1024];
/** Flags defining specific characteristics of a library. See #LibraryFlag. */
uint16_t flag;
char _pad[6];
/**
* For archive library only (#LIBRARY_FLAG_IS_ARCHIVE): The main library owning it.
*
* `archive_parent_library` and `packedfile` should never be both non-null in a same Library ID.
*/
struct Library *archive_parent_library;
/**
* Packed blendfile of the library, nullptr if not packed.
*
* \note Individual IDs may be packed even if the entire library is not packed.
*
* `archive_parent_library` and `packedfile` should never be both non-null in a same Library ID.
*/
struct PackedFile *packedfile;
/**
@@ -519,8 +577,22 @@ typedef struct Library {
* Typically allocated when creating a new Library or reading it from a blendfile.
*/
LibraryRuntimeHandle *runtime;
void *_pad2;
} Library;
/**
* #Library.flag
*
* Some of these flags define a 'virtual' library, which may not be an actual blendfile, store
* 'archived' embedded data, etc. IDs contained in these virtual libraries are _not_ managed by
* regular linking code.
*/
enum LibraryFlag {
/** The library is an 'archive' that only contains embedded linked data. */
LIBRARY_FLAG_IS_ARCHIVE = 1 << 0,
};
/**
* A weak library/ID reference for local data that has been appended, to allow re-using that local
* data instead of creating a new copy of it in future appends.
@@ -618,6 +690,12 @@ typedef struct PreviewImage {
#define ID_MISSING(_id) ((((const ID *)(_id))->tag & ID_TAG_MISSING) != 0)
#define ID_IS_LINKED(_id) (((const ID *)(_id))->lib != NULL)
/**
* Indicates that this ID is linked but also packed into the current .blend file. Note that this
* just means that this specific ID and its dependencies are packed, not the entire library. So
* this is separate from #Library::packedfile.
*/
#define ID_IS_PACKED(_id) (ID_IS_LINKED(_id) && ((_id)->flag & ID_FLAG_LINKED_AND_PACKED))
#define ID_TYPE_SUPPORTS_ASSET_EDITABLE(id_type) \
ELEM(id_type, ID_BR, ID_TE, ID_NT, ID_IM, ID_PC, ID_MA)
@@ -717,6 +795,11 @@ enum {
* so it must be treated as dirty.
*/
ID_FLAG_CLIPBOARD_MARK = 1 << 14,
/**
* Indicates that this linked ID is packed into the current .blend file. This should never be set
* on local ID (without)one with a null `ID::lib` pointer).
*/
ID_FLAG_LINKED_AND_PACKED = 1 << 15,
};
/**
@@ -1274,10 +1357,6 @@ typedef enum eID_Index {
#define INDEX_ID_MAX (INDEX_ID_NULL + 1)
#ifdef __cplusplus
}
#endif
#ifdef __cplusplus
namespace blender::dna {
namespace detail {

2
source/blender/makesdna/DNA_asset_types.h
View File

@@ -118,6 +118,8 @@ typedef enum eAssetImportMethod {
* heavy data dependencies (e.g. the image data-blocks of a material, the mesh of an object) may
* be reused from an earlier append. */
ASSET_IMPORT_APPEND_REUSE = 2,
/** Link data-block, but also pack it as read-only data. */
ASSET_IMPORT_PACK = 3,
} eAssetImportMethod;
#

1
source/blender/makesdna/DNA_object_types.h
View File

@@ -227,7 +227,6 @@ typedef struct Object {
bAnimVizSettings avs;
/** Motion path cache for this object. */
bMotionPath *mpath;
void *_pad0;
ListBase effect DNA_DEPRECATED; /* XXX deprecated... keep for readfile */
ListBase defbase DNA_DEPRECATED; /* Only for versioning, moved to object data. */

1
source/blender/makesdna/DNA_particle_types.h
View File

@@ -298,7 +298,6 @@ typedef struct ParticleSettings {
float rad_root, rad_tip, rad_scale;
struct CurveMapping *twistcurve;
void *_pad7;
} ParticleSettings;
typedef struct ParticleSystem {

5
source/blender/makesdna/DNA_space_enums.h
View File

@@ -477,6 +477,11 @@ typedef enum eFileAssetImportMethod {
FILE_ASSET_IMPORT_APPEND_REUSE = 2,
/** Default: Follow the preference setting for this asset library. */
FILE_ASSET_IMPORT_FOLLOW_PREFS = 3,
/**
* Link the data-block, but also pack it in the current file to keep it working even if the
* source file is not available anymore.
*/
FILE_ASSET_IMPORT_PACK = 4,
} eFileAssetImportMethod;
typedef enum eFileAssetImportFlags {

3
source/blender/makesdna/DNA_userdef_types.h
View File

@@ -224,6 +224,7 @@ typedef struct UserDef_Experimental {
char use_extensions_debug;
char use_recompute_usercount_on_save_debug;
char write_legacy_blend_file_format;
char no_data_block_packing;
char SANITIZE_AFTER_HERE;
/* The following options are automatically sanitized (set to 0)
* when the release cycle is not alpha. */
@@ -233,7 +234,7 @@ typedef struct UserDef_Experimental {
char use_new_volume_nodes;
char use_shader_node_previews;
char use_geometry_nodes_lists;
char _pad[6];
char _pad[5];
} UserDef_Experimental;
#define USER_EXPERIMENTAL_TEST(userdef, member) (((userdef)->experimental).member)

92
source/blender/makesrna/intern/rna_ID.cc
View File

@@ -300,7 +300,12 @@ static int rna_ID_name_editable(const PointerRNA *ptr, const char **r_info)
* and could be useful in some cases. */
if (!ID_IS_EDITABLE(id)) {
if (r_info) {
*r_info = N_("Linked data-blocks cannot be renamed");
if (ID_IS_PACKED(id)) {
*r_info = N_("Packed data-blocks cannot be renamed");
}
else {
*r_info = N_("Linked data-blocks cannot be renamed");
}
}
return 0;
}
@@ -1539,6 +1544,52 @@ static void rna_Library_version_get(PointerRNA *ptr, int *value)
value[2] = lib->runtime->subversionfile;
}
static void rna_Library_archive_libraries_begin(CollectionPropertyIterator *iter, PointerRNA *ptr)
{
iter->parent = *ptr;
Library *lib = static_cast<Library *>(ptr->data);
Library **archive_libraries_iter = lib->runtime->archived_libraries.begin();
iter->internal.custom = archive_libraries_iter;
iter->valid = archive_libraries_iter != lib->runtime->archived_libraries.end();
}
static void rna_Library_archive_libraries_next(CollectionPropertyIterator *iter)
{
Library *lib = static_cast<Library *>(iter->parent.data);
Library **archive_libraries_iter = static_cast<Library **>(iter->internal.custom);
archive_libraries_iter++;
iter->internal.custom = archive_libraries_iter;
iter->valid = archive_libraries_iter != lib->runtime->archived_libraries.end();
}
static PointerRNA rna_Library_archive_libraries_get(CollectionPropertyIterator *iter)
{
Library **archive_libraries_iter = static_cast<Library **>(iter->internal.custom);
return RNA_pointer_create_with_parent(iter->parent, &RNA_Library, *archive_libraries_iter);
}
static int rna_Library_archive_libraries_length(PointerRNA *ptr)
{
Library *lib = static_cast<Library *>(ptr->data);
return int(lib->runtime->archived_libraries.size());
}
static bool rna_Library_archive_libraries_lookupint(PointerRNA *ptr, int key, PointerRNA *r_ptr)
{
Library *lib = static_cast<Library *>(ptr->data);
if (key < 0 || key >= lib->runtime->archived_libraries.size()) {
return false;
}
Library *archive_library = lib->runtime->archived_libraries[key];
rna_pointer_create_with_ancestors(*ptr, &RNA_Library, archive_library, *r_ptr);
return true;
}
static PointerRNA rna_Library_parent_get(PointerRNA *ptr)
{
Library *lib = ptr->data_as<Library>();
@@ -2345,6 +2396,12 @@ static void rna_def_ID(BlenderRNA *brna)
"This data-block is not an independent one, but is actually a sub-data of another ID "
"(typical example: root node trees or master collections)");
prop = RNA_def_property(srna, "is_linked_packed", PROP_BOOLEAN, PROP_NONE);
RNA_def_property_boolean_sdna(prop, nullptr, "flag", ID_FLAG_LINKED_AND_PACKED);
RNA_def_property_clear_flag(prop, PROP_EDITABLE);
RNA_def_property_ui_text(
prop, "Linked Packed", "This data-block is linked and packed into the .blend file");
prop = RNA_def_property(srna, "is_missing", PROP_BOOLEAN, PROP_NONE);
RNA_def_property_boolean_sdna(prop, nullptr, "tag", ID_TAG_MISSING);
RNA_def_property_clear_flag(prop, PROP_EDITABLE);
@@ -2667,6 +2724,39 @@ static void rna_def_library(BlenderRNA *brna)
"Data-blocks in this library are editable despite being linked. "
"Used by brush assets and their dependencies.");
prop = RNA_def_property(srna, "is_archive", PROP_BOOLEAN, PROP_NONE);
RNA_def_property_boolean_sdna(prop, nullptr, "flag", LIBRARY_FLAG_IS_ARCHIVE);
RNA_def_property_clear_flag(prop, PROP_EDITABLE);
RNA_def_property_ui_text(prop,
"Is Archive",
"This library is an 'archive' storage for packed linked IDs "
"originally linked from its 'archive parent' library.");
prop = RNA_def_property(srna, "archive_parent_library", PROP_POINTER, PROP_NONE);
RNA_def_property_pointer_sdna(prop, nullptr, "archive_parent_library");
RNA_def_property_struct_type(prop, "Library");
RNA_def_property_override_flag(prop, PROPOVERRIDE_NO_COMPARISON);
RNA_def_property_ui_text(prop,
"Parent Archive Library",
"Source library from which this archive of packed IDs was generated");
prop = RNA_def_property(srna, "archive_libraries", PROP_COLLECTION, PROP_NONE);
RNA_def_property_struct_type(prop, "Library");
RNA_def_property_collection_funcs(prop,
"rna_Library_archive_libraries_begin",
"rna_Library_archive_libraries_next",
nullptr,
"rna_Library_archive_libraries_get",
"rna_Library_archive_libraries_length",
"rna_Library_archive_libraries_lookupint",
nullptr,
nullptr);
RNA_def_property_override_flag(prop, PROPOVERRIDE_NO_COMPARISON);
RNA_def_property_ui_text(
prop,
"Archive Libraries",
"Archive libraries of packed IDs, generated (and owned) by this source library");
func = RNA_def_function(srna, "reload", "rna_Library_reload");
RNA_def_function_flag(func, FUNC_USE_REPORTS | FUNC_USE_CONTEXT);
RNA_def_function_ui_description(func, "Reload this library and all its linked data-blocks");

36
source/blender/makesrna/intern/rna_main_api.cc
View File

@@ -34,6 +34,7 @@
# include "BKE_image.hh"
# include "BKE_lattice.hh"
# include "BKE_lib_remap.hh"
# include "BKE_library.hh"
# include "BKE_light.h"
# include "BKE_lightprobe.h"
# include "BKE_linestyle.h"
@@ -146,6 +147,28 @@ static void rna_Main_ID_remove(Main *bmain,
}
}
static ID *rna_Main_pack_linked_ids_hierarchy(struct BlendData *blenddata,
ReportList *reports,
ID *root_id)
{
if (!ID_IS_LINKED(root_id)) {
BKE_reportf(reports, RPT_ERROR, "Only linked IDs can be packed");
return nullptr;
}
if (ID_IS_PACKED(root_id)) {
/* Nothing to do. */
return root_id;
}
Main *bmain = reinterpret_cast<Main *>(blenddata);
blender::bke::library::pack_linked_id_hierarchy(*bmain, *root_id);
ID *packed_root_id = root_id->newid;
BKE_main_id_newptr_and_tag_clear(bmain);
return packed_root_id;
}
static Camera *rna_Main_cameras_new(Main *bmain, const char *name)
{
char safe_name[MAX_ID_NAME - 2];
@@ -869,12 +892,12 @@ RNA_MAIN_ID_TAG_FUNCS_DEF(volumes, volumes, ID_VO)
#else
void RNA_api_main(StructRNA * /*srna*/)
void RNA_api_main(StructRNA *srna)
{
# if 0
FunctionRNA *func;
PropertyRNA *parm;
# if 0
/* maybe we want to add functions in 'bpy.data' still?
* for now they are all in collections bpy.data.images.new(...) */
func = RNA_def_function(srna, "add_image", "rna_Main_add_image");
@@ -885,6 +908,15 @@ void RNA_api_main(StructRNA * /*srna*/)
parm = RNA_def_pointer(func, "image", "Image", "", "New image");
RNA_def_function_return(func, parm);
# endif
func = RNA_def_function(srna, "pack_linked_ids_hierarchy", "rna_Main_pack_linked_ids_hierarchy");
RNA_def_function_ui_description(
func, "Pack the given linked ID and its dependencies into current blendfile");
RNA_def_function_flag(func, FUNC_USE_REPORTS);
parm = RNA_def_pointer(func, "root_id", "ID", "", "Root linked ID to pack");
RNA_def_parameter_flags(parm, PropertyFlag(0), PARM_REQUIRED);
parm = RNA_def_pointer(func, "packed_id", "ID", "", "The packed ID matching the given root ID");
RNA_def_function_return(func, parm);
}
void RNA_def_main_cameras(BlenderRNA *brna, PropertyRNA *cprop)

93
source/blender/makesrna/intern/rna_space.cc
View File

@@ -358,6 +358,40 @@ const EnumPropertyItem rna_enum_fileselect_params_sort_items[] = {
{0, nullptr, 0, nullptr, nullptr},
};
static const EnumPropertyItem rna_enum_asset_import_method_items[] = {
{FILE_ASSET_IMPORT_FOLLOW_PREFS,
"FOLLOW_PREFS",
0,
"Follow Preferences",
"Use the import method set in the Preferences for this asset library, don't override it "
"for this Asset Browser"},
{FILE_ASSET_IMPORT_LINK,
"LINK",
ICON_LINK_BLEND,
"Link",
"Import the assets as linked data-block"},
{FILE_ASSET_IMPORT_APPEND,
"APPEND",
ICON_APPEND_BLEND,
"Append",
"Import the asset as copied data-block, with no link to the original asset data-block"},
{FILE_ASSET_IMPORT_APPEND_REUSE,
"APPEND_REUSE",
ICON_APPEND_BLEND,
"Append (Reuse Data)",
"Import the asset as copied data-block while avoiding multiple copies of nested, "
"typically heavy data. For example the textures of a material asset, or the mesh of an "
"object asset, don't have to be copied every time this asset is imported. The instances of "
"the asset share the data instead"},
{FILE_ASSET_IMPORT_PACK,
"PACK",
ICON_PACKAGE,
"Pack",
"Import the asset as linked data-block, and pack it in the current file (ensures that it "
"remains unchanged in case the library data is modified, is not available anymore, etc.)"},
{0, nullptr, 0, nullptr, nullptr},
};
#ifndef RNA_RUNTIME
static const EnumPropertyItem stereo3d_eye_items[] = {
{STEREO_LEFT_ID, "LEFT_EYE", ICON_NONE, "Left Eye"},
@@ -3662,6 +3696,37 @@ static void rna_FileAssetSelectParams_catalog_id_set(PointerRNA *ptr, const char
params->asset_catalog_visibility = FILE_SHOW_ASSETS_FROM_CATALOG;
}
static const EnumPropertyItem *rna_FileAssetSelectParams_import_method_itemf(
bContext * /*C*/, PointerRNA * /*ptr*/, PropertyRNA * /*prop*/, bool *r_free)
{
EnumPropertyItem *items = nullptr;
int items_num = 0;
for (const EnumPropertyItem *item = rna_enum_asset_import_method_items; item->identifier; item++)
{
switch (eFileAssetImportMethod(item->value)) {
case FILE_ASSET_IMPORT_APPEND_REUSE: {
if (U.experimental.no_data_block_packing) {
RNA_enum_item_add(&items, &items_num, item);
}
break;
}
case FILE_ASSET_IMPORT_PACK: {
if (!U.experimental.no_data_block_packing) {
RNA_enum_item_add(&items, &items_num, item);
}
break;
}
default: {
RNA_enum_item_add(&items, &items_num, item);
break;
}
}
}
RNA_enum_item_end(&items, &items_num);
*r_free = true;
return items;
}
#else
static const EnumPropertyItem dt_uv_items[] = {
@@ -7426,30 +7491,6 @@ static void rna_def_fileselect_asset_params(BlenderRNA *brna)
StructRNA *srna;
PropertyRNA *prop;
static const EnumPropertyItem asset_import_method_items[] = {
{FILE_ASSET_IMPORT_FOLLOW_PREFS,
"FOLLOW_PREFS",
0,
"Follow Preferences",
"Use the import method set in the Preferences for this asset library, don't override it "
"for this Asset Browser"},
{FILE_ASSET_IMPORT_LINK, "LINK", 0, "Link", "Import the assets as linked data-block"},
{FILE_ASSET_IMPORT_APPEND,
"APPEND",
0,
"Append",
"Import the assets as copied data-block, with no link to the original asset data-block"},
{FILE_ASSET_IMPORT_APPEND_REUSE,
"APPEND_REUSE",
0,
"Append (Reuse Data)",
"Import the assets as copied data-block while avoiding multiple copies of nested, "
"typically heavy data. For example the textures of a material asset, or the mesh of an "
"object asset, don't have to be copied every time this asset is imported. The instances of "
"the asset share the data instead"},
{0, nullptr, 0, nullptr, nullptr},
};
srna = RNA_def_struct(brna, "FileAssetSelectParams", "FileSelectParams");
RNA_def_struct_ui_text(
srna, "Asset Select Parameters", "Settings for the file selection in Asset Browser mode");
@@ -7478,7 +7519,9 @@ static void rna_def_fileselect_asset_params(BlenderRNA *brna)
"Which asset types to show/hide, when browsing an asset library");
prop = RNA_def_property(srna, "import_method", PROP_ENUM, PROP_NONE);
RNA_def_property_enum_items(prop, asset_import_method_items);
RNA_def_property_enum_items(prop, rna_enum_asset_import_method_items);
RNA_def_property_enum_funcs(
prop, nullptr, nullptr, "rna_FileAssetSelectParams_import_method_itemf");
RNA_def_property_ui_text(prop, "Import Method", "Determine how the asset will be imported");
/* Asset drag info saved by buttons stores the import method, so the space must redraw when
* import method changes. */

104
source/blender/makesrna/intern/rna_userdef.cc
View File

@@ -180,6 +180,30 @@ static const EnumPropertyItem rna_enum_preferences_extension_repo_source_type_it
{0, nullptr, 0, nullptr, nullptr},
};
static const EnumPropertyItem rna_enum_preferences_asset_import_method_items[] = {
{ASSET_IMPORT_LINK, "LINK", ICON_LINK_BLEND, "Link", "Import the assets as linked data-block"},
{ASSET_IMPORT_APPEND,
"APPEND",
ICON_APPEND_BLEND,
"Append",
"Import the assets as copied data-block, with no link to the original asset data-block"},
{ASSET_IMPORT_APPEND_REUSE,
"APPEND_REUSE",
ICON_APPEND_BLEND,
"Append (Reuse Data)",
"Import the assets as copied data-block while avoiding multiple copies of nested, "
"typically heavy data. For example the textures of a material asset, or the mesh of an "
"object asset, don't have to be copied every time this asset is imported. The instances of "
"the asset share the data instead."},
{ASSET_IMPORT_PACK,
"PACK",
ICON_PACKAGE,
"Pack",
"Import the asset as linked data-block, and pack it in the current file (ensures that it "
"remains unchanged in case the library data is modified, is not available anymore, etc.)"},
{0, nullptr, 0, nullptr, nullptr},
};
#ifdef RNA_RUNTIME
# include "BLI_math_vector.h"
@@ -223,6 +247,8 @@ static const EnumPropertyItem rna_enum_preferences_extension_repo_source_type_it
# include "UI_interface.hh"
# include "AS_asset_library.hh"
static void rna_userdef_version_get(PointerRNA *ptr, int *value)
{
UserDef *userdef = (UserDef *)ptr->data;
@@ -360,7 +386,7 @@ static void rna_userdef_asset_library_path_set(PointerRNA *ptr, const char *valu
BKE_preferences_asset_library_path_set(library, value);
}
static void rna_userdef_asset_library_path_update(bContext *C, PointerRNA *ptr)
static void rna_userdef_asset_library_update(bContext *C, PointerRNA *ptr)
{
blender::ed::asset::list::clear_all_library(C);
rna_userdef_update(CTX_data_main(C), CTX_data_scene(C), ptr);
@@ -1482,6 +1508,49 @@ static void rna_preference_gpu_preferred_device_set(PointerRNA *ptr, int value)
preferences->gpu_preferred_device_id = 0u;
}
static const EnumPropertyItem *rna_preference_asset_libray_import_method_itemf(
bContext * /*C*/, PointerRNA * /*ptr*/, PropertyRNA * /*prop*/, bool *r_free)
{
EnumPropertyItem *items = nullptr;
int items_num = 0;
for (const EnumPropertyItem *item = rna_enum_preferences_asset_import_method_items;
item->identifier;
item++)
{
switch (eAssetImportMethod(item->value)) {
case ASSET_IMPORT_APPEND_REUSE: {
if (U.experimental.no_data_block_packing) {
RNA_enum_item_add(&items, &items_num, item);
}
break;
}
case ASSET_IMPORT_PACK: {
if (!U.experimental.no_data_block_packing) {
RNA_enum_item_add(&items, &items_num, item);
}
break;
}
default: {
RNA_enum_item_add(&items, &items_num, item);
break;
}
}
}
RNA_enum_item_end(&items, &items_num);
*r_free = true;
return items;
}
static void rna_experimental_no_data_block_packing_update(bContext *C, PointerRNA *ptr)
{
Main *bmain = CTX_data_main(C);
Scene *scene = CTX_data_scene(C);
rna_userdef_update(bmain, scene, ptr);
AS_asset_library_import_method_ensure_valid(*bmain);
AS_asset_library_essential_import_method_update();
rna_userdef_asset_library_update(C, ptr);
}
#else
# define USERDEF_TAG_DIRTY_PROPERTY_UPDATE_ENABLE \
@@ -6657,32 +6726,18 @@ static void rna_def_userdef_filepaths_asset_library(BlenderRNA *brna)
RNA_def_property_translation_context(prop, BLT_I18NCONTEXT_EDITOR_FILEBROWSER);
RNA_def_property_string_funcs(prop, nullptr, nullptr, "rna_userdef_asset_library_path_set");
RNA_def_property_flag(prop, PROP_CONTEXT_UPDATE);
RNA_def_property_update(prop, 0, "rna_userdef_asset_library_path_update");
RNA_def_property_update(prop, 0, "rna_userdef_asset_library_update");
static const EnumPropertyItem import_method_items[] = {
{ASSET_IMPORT_LINK, "LINK", 0, "Link", "Import the assets as linked data-block"},
{ASSET_IMPORT_APPEND,
"APPEND",
0,
"Append",
"Import the assets as copied data-block, with no link to the original asset data-block"},
{ASSET_IMPORT_APPEND_REUSE,
"APPEND_REUSE",
0,
"Append (Reuse Data)",
"Import the assets as copied data-block while avoiding multiple copies of nested, "
"typically heavy data. For example the textures of a material asset, or the mesh of an "
"object asset, don't have to be copied every time this asset is imported. The instances of "
"the asset share the data instead."},
{0, nullptr, 0, nullptr, nullptr},
};
prop = RNA_def_property(srna, "import_method", PROP_ENUM, PROP_NONE);
RNA_def_property_enum_items(prop, import_method_items);
RNA_def_property_enum_items(prop, rna_enum_preferences_asset_import_method_items);
RNA_def_property_enum_funcs(
prop, nullptr, nullptr, "rna_preference_asset_libray_import_method_itemf");
RNA_def_property_ui_text(
prop,
"Default Import Method",
"Determine how the asset will be imported, unless overridden by the Asset Browser");
RNA_def_property_update(prop, 0, "rna_userdef_update");
RNA_def_property_flag(prop, PROP_CONTEXT_UPDATE);
RNA_def_property_update(prop, 0, "rna_userdef_asset_library_update");
prop = RNA_def_property(srna, "use_relative_path", PROP_BOOLEAN, PROP_NONE);
RNA_def_property_boolean_sdna(prop, nullptr, "flag", ASSET_LIBRARY_RELATIVE_PATH);
@@ -7308,6 +7363,13 @@ static void rna_def_userdef_experimental(BlenderRNA *brna)
"Use file format used before Blender 5.0. This format is more limited "
"but it may have better compatibility with tools that don't support the new format yet");
prop = RNA_def_property(srna, "no_data_block_packing", PROP_BOOLEAN, PROP_NONE);
RNA_def_property_boolean_sdna(prop, nullptr, "no_data_block_packing", 1);
RNA_def_property_ui_text(
prop, "No Data-Block Packing", "Fall-back to appending instead of packing data-blocks");
RNA_def_property_flag(prop, PROP_CONTEXT_UPDATE);
RNA_def_property_update(prop, 0, "rna_experimental_no_data_block_packing_update");
prop = RNA_def_property(srna, "use_all_linked_data_direct", PROP_BOOLEAN, PROP_NONE);
RNA_def_property_ui_text(
prop,

22
source/blender/python/intern/bpy_library_load.cc
View File

@@ -223,6 +223,9 @@ PyDoc_STRVAR(
" :type filepath: str | bytes\n"
" :arg link: When False reference to the original file is lost.\n"
" :type link: bool\n"
" :arg pack: If True, and ``link`` is also True, pack linked data-blocks into the current "
"blend-file.\n"
" :type pack: bool\n"
" :arg relative: When True the path is stored relative to the open blend file.\n"
" :type relative: bool\n"
" :arg set_fake: If True, set fake user on appended IDs.\n"
@@ -260,6 +263,7 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k
*/
struct {
BoolFlagPair is_link = {false, FILE_LINK};
BoolFlagPair is_pack = {false, BLO_LIBLINK_PACK};
BoolFlagPair is_relative = {false, FILE_RELPATH};
BoolFlagPair set_fake = {false, BLO_LIBLINK_APPEND_SET_FAKEUSER};
BoolFlagPair recursive = {false, BLO_LIBLINK_APPEND_RECURSIVE};
@@ -279,6 +283,7 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k
static const char *_keywords[] = {
"filepath",
"link",
"pack",
"relative",
"set_fake",
"recursive",
@@ -296,6 +301,7 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k
/* Optional keyword only arguments. */
"|$"
"O&" /* `link` */
"O&" /* `pack` */
"O&" /* `relative` */
"O&" /* `recursive` */
"O&" /* `set_fake` */
@@ -317,6 +323,8 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k
PyC_ParseBool,
&flag_vars.is_link,
PyC_ParseBool,
&flag_vars.is_pack,
PyC_ParseBool,
&flag_vars.is_relative,
PyC_ParseBool,
&flag_vars.recursive,
@@ -389,10 +397,18 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k
PyErr_SetString(PyExc_ValueError, "`link` is False but `create_liboverrides` is True");
return nullptr;
}
if (flag_vars.is_pack.value) {
PyErr_SetString(PyExc_ValueError, "`pack` must be False if `link` is False");
return nullptr;
}
}
if (create_liboverrides) {
/* Library overrides. */
if (flag_vars.is_pack.value) {
PyErr_SetString(PyExc_ValueError, "`create_liboverrides` must be False if `pack` is True");
return nullptr;
}
}
else {
/* Library overrides (disabled). */
@@ -639,6 +655,7 @@ static bool bpy_lib_exit_lapp_context_items_cb(BlendfileLinkAppendContext *lapp_
static PyObject *bpy_lib_exit(BPy_Library *self, PyObject * /*args*/)
{
Main *bmain = self->bmain;
const bool do_pack = ((self->flag & BLO_LIBLINK_PACK) != 0);
const bool do_append = ((self->flag & FILE_LINK) == 0);
const bool create_liboverrides = self->create_liboverrides;
/* Code in #bpy_lib_load should have raised exception in case of incompatible parameter values.
@@ -709,7 +726,10 @@ static PyObject *bpy_lib_exit(BPy_Library *self, PyObject * /*args*/)
BKE_blendfile_link_append_context_init_done(lapp_context);
BKE_blendfile_link(lapp_context, nullptr);
if (do_append) {
if (do_pack) {
BKE_blendfile_link_pack(lapp_context, nullptr);
}
else if (do_append) {
BKE_blendfile_append(lapp_context, nullptr);
}
else if (create_liboverrides) {

12
source/blender/windowmanager/intern/wm_dragdrop.cc
View File

@@ -753,6 +753,16 @@ ID *WM_drag_asset_id_import(const bContext *C, wmDragAsset *asset_drag, const in
idtype,
name,
flag | (use_relative_path ? FILE_RELPATH : 0));
case ASSET_IMPORT_PACK:
return WM_file_link_datablock(bmain,
scene,
view_layer,
view3d,
blend_path.c_str(),
idtype,
name,
flag | (use_relative_path ? FILE_RELPATH : 0) |
BLO_LIBLINK_PACK);
case ASSET_IMPORT_APPEND:
return WM_file_append_datablock(bmain,
scene,
@@ -787,7 +797,7 @@ bool WM_drag_asset_will_import_linked(const wmDrag *drag)
}
const wmDragAsset *asset_drag = WM_drag_get_asset_data(drag, 0);
return asset_drag->import_settings.method == ASSET_IMPORT_LINK;
return ELEM(asset_drag->import_settings.method, ASSET_IMPORT_LINK, ASSET_IMPORT_PACK);
}
ID *WM_drag_get_local_ID_or_import_from_asset(const bContext *C, const wmDrag *drag, int idcode)

6
source/blender/windowmanager/intern/wm_files_link.cc
View File

@@ -725,6 +725,7 @@ static ID *wm_file_link_append_datablock_ex(Main *bmain,
BLI_path_cmp(BKE_main_blendfile_path(bmain), filepath) != 0,
"Calling code should ensure it does not attempt to link/append from current blendfile");
const bool do_pack = (flag & BLO_LIBLINK_PACK) != 0;
const bool do_append = (flag & FILE_LINK) == 0;
/* Tag everything so we can make local only the new datablock. */
BKE_main_id_tag_all(bmain, ID_TAG_PRE_EXISTING, true);
@@ -747,7 +748,10 @@ static ID *wm_file_link_append_datablock_ex(Main *bmain,
/* Link datablock. */
BKE_blendfile_link(lapp_context, nullptr);
if (do_append) {
if (do_pack) {
BKE_blendfile_link_pack(lapp_context, nullptr);
}
else if (do_append) {
BKE_blendfile_append(lapp_context, nullptr);
}

BIN
tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1.blend (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_1_with_hair.blend (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2.blend (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/body_2_with_hair.blend (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
tests/files/libraries_and_linking/packed_data_blocks/two_objects_with_hair/both_bodies_with_hair.blend (Stored with Git LFS) Normal file
View File

Binary file not shown.

227
tests/python/bl_blendfile_liblink.py
View File

@@ -486,6 +486,202 @@ class TestBlendLibAppendReuseID(TestBlendLibLinkHelper):
self.assertEqual(len(bpy.data.collections), 0) # Scene's master collection is not listed here
class TestBlendLibPackedLinkedID(TestBlendLibLinkHelper):
def __init__(self, args):
super().__init__(args)
def test_link_pack_basic(self):
output_dir = self.args.output_dir
output_lib_path = self.init_lib_data_basic()
# Link of a single Object, and make it packed.
self.reset_blender()
link_dir = os.path.join(output_lib_path, "Object")
bpy.ops.wm.link(directory=link_dir, filename="LibMesh", instance_object_data=False)
self.assertEqual(len(bpy.data.libraries), 1)
library = bpy.data.libraries[0]
self.assertEqual(len(bpy.data.meshes), 1)
for me in bpy.data.meshes:
self.assertEqual(me.library, library)
self.assertEqual(me.users, 1)
self.assertEqual(len(bpy.data.objects), 1)
for ob in bpy.data.objects:
self.assertEqual(ob.library, library)
self.assertEqual(len(bpy.data.collections), 0) # Scene's master collection is not listed here
ob_packed = bpy.data.pack_linked_ids_hierarchy(bpy.data.objects[0])
# Need to ensure that the newly packed linked object is used, and kept in the scene.
bpy.data.scenes[0].collection.objects.link(ob_packed)
self.assertEqual(len(bpy.data.libraries), 2)
library = bpy.data.libraries[0]
archive_library = bpy.data.libraries[1]
def check_valid():
self.assertFalse(library.is_archive)
self.assertEqual(len(library.archive_libraries), 1)
self.assertEqual(library.archive_libraries[0], archive_library)
self.assertTrue(archive_library.is_archive)
self.assertEqual(archive_library.archive_parent_library, library)
self.assertEqual(len(bpy.data.meshes), 2)
self.assertEqual(bpy.data.meshes[0].library, library)
self.assertEqual(bpy.data.meshes[0].users, 1)
self.assertEqual(bpy.data.meshes[1].library, archive_library)
self.assertEqual(bpy.data.meshes[1].users, 1)
self.assertEqual(len(bpy.data.objects), 2)
self.assertEqual(bpy.data.objects[0].library, library)
self.assertEqual(bpy.data.objects[0].data, bpy.data.meshes[0])
self.assertEqual(bpy.data.objects[1].library, archive_library)
self.assertEqual(bpy.data.objects[1].data, bpy.data.meshes[1])
check_valid()
output_work_path = os.path.join(output_dir, self.unique_blendfile_name("blendfile"))
bpy.ops.wm.save_as_mainfile(filepath=output_work_path, check_existing=False, compress=False)
self.reset_blender()
bpy.ops.wm.open_mainfile(filepath=output_work_path, load_ui=False)
self.assertEqual(len(bpy.data.libraries), 2)
library = bpy.data.libraries[0]
archive_library = bpy.data.libraries[1]
check_valid()
def test_link_pack_indirect(self):
# Test handling of indirectly linked packed data (when packed in another library),
# packing linked data using other packed linked data, etc.
output_dir = self.args.output_dir
output_lib_path = self.init_lib_data_packed_indirect_lib()
# Link of a single Object, and make it packed.
self.reset_blender()
link_dir = os.path.join(output_lib_path, "Object")
bpy.ops.wm.link(directory=link_dir, filename="LibMesh", instance_object_data=False)
# Directly linked library, indirectly linked one (though empty), and its packed archive version.
self.assertEqual(len(bpy.data.libraries), 3)
library = bpy.data.libraries[0]
library_indirect = bpy.data.libraries[1]
library_indirect_archive = bpy.data.libraries[2]
def check_valid():
self.assertFalse(library.is_archive)
self.assertFalse(library_indirect.is_archive)
self.assertTrue(library_indirect_archive.is_archive)
self.assertTrue(library_indirect_archive.name in library_indirect.archive_libraries)
self.assertEqual(len(bpy.data.images), 1)
for im in bpy.data.images:
self.assertEqual(im.library, library_indirect_archive)
self.assertEqual(im.users, 1)
self.assertTrue(im.is_linked_packed)
self.assertEqual(len(bpy.data.materials), 1)
for ma in bpy.data.materials:
self.assertEqual(ma.library, library_indirect_archive)
self.assertEqual(ma.users, 1)
self.assertTrue(ma.is_linked_packed)
self.assertEqual(len(bpy.data.meshes), 1)
for me in bpy.data.meshes:
self.assertEqual(me.library, library)
self.assertEqual(me.users, 1)
self.assertEqual(len(bpy.data.objects), 1)
for ob in bpy.data.objects:
self.assertEqual(ob.library, library)
self.assertEqual(len(bpy.data.collections), 0) # Scene's master collection is not listed here
check_valid()
output_work_path = os.path.join(output_dir, self.unique_blendfile_name("blendfile"))
bpy.ops.wm.save_as_mainfile(filepath=output_work_path, check_existing=False, compress=False)
self.reset_blender()
bpy.ops.wm.open_mainfile(filepath=output_work_path, load_ui=False)
self.assertEqual(len(bpy.data.libraries), 3)
library = bpy.data.libraries[0]
library_indirect = bpy.data.libraries[1]
library_indirect_archive = bpy.data.libraries[2]
check_valid()
ob_packed = bpy.data.pack_linked_ids_hierarchy(bpy.data.objects[0])
# Need to ensure that the newly packed linked object is used, and kept in the scene.
bpy.data.scenes[0].collection.objects.link(ob_packed)
self.assertEqual(len(bpy.data.libraries), 4)
# Due to ID name sorting, the newly ceratedt archive library should be second now, after its parent one.
archive_library = bpy.data.libraries[1]
def check_valid():
self.assertFalse(library.is_archive)
self.assertEqual(len(library.archive_libraries), 1)
self.assertEqual(library.archive_libraries[0], archive_library)
self.assertTrue(archive_library.is_archive)
self.assertEqual(archive_library.archive_parent_library, library)
self.assertFalse(library_indirect.is_archive)
self.assertTrue(library_indirect_archive.is_archive)
self.assertTrue(library_indirect_archive.name in library_indirect.archive_libraries)
self.assertEqual(len(bpy.data.images), 1)
for im in bpy.data.images:
self.assertEqual(im.library, library_indirect_archive)
self.assertEqual(im.users, 1)
self.assertTrue(im.is_linked_packed)
self.assertEqual(len(bpy.data.materials), 1)
for ma in bpy.data.materials:
self.assertEqual(ma.library, library_indirect_archive)
self.assertEqual(ma.users, 2)
self.assertTrue(ma.is_linked_packed)
self.assertEqual(len(bpy.data.meshes), 2)
self.assertEqual(bpy.data.meshes[0].library, library)
self.assertEqual(bpy.data.meshes[0].users, 1)
self.assertEqual(bpy.data.meshes[1].library, archive_library)
self.assertEqual(bpy.data.meshes[1].users, 1)
self.assertEqual(len(bpy.data.objects), 2)
self.assertEqual(bpy.data.objects[0].library, library)
self.assertEqual(bpy.data.objects[0].data, bpy.data.meshes[0])
self.assertEqual(bpy.data.objects[1].library, archive_library)
self.assertEqual(bpy.data.objects[1].data, bpy.data.meshes[1])
self.assertEqual(len(bpy.data.collections), 0) # Scene's master collection is not listed here
check_valid()
bpy.ops.wm.save_as_mainfile(filepath=output_work_path, check_existing=False, compress=False)
self.reset_blender()
bpy.ops.wm.open_mainfile(filepath=output_work_path, load_ui=False)
self.assertEqual(len(bpy.data.libraries), 4)
library = bpy.data.libraries[0]
archive_library = bpy.data.libraries[1]
library_indirect = bpy.data.libraries[2]
library_indirect_archive = bpy.data.libraries[3]
check_valid()
class TestBlendLibLibraryReload(TestBlendLibLinkHelper):
def __init__(self, args):
@@ -610,6 +806,34 @@ class TestBlendLibDataLibrariesLoadLink(TestBlendLibDataLibrariesLoad):
self.assertIsNotNone(bpy.data.collections[0].library)
class TestBlendLibDataLibrariesLoadPack(TestBlendLibDataLibrariesLoad):
def test_libload_pack(self):
output_lib_path = self.do_libload_init()
# Cannot create overrides on packed linked data currently.
self.assertRaises(ValueError,
self.do_libload, filepath=output_lib_path, link=True, pack=True, create_liboverrides=True)
self.do_libload(filepath=output_lib_path, link=True, pack=True, create_liboverrides=False)
self.assertEqual(len(bpy.data.meshes), 1)
self.assertEqual(len(bpy.data.objects), 1) # This code does no instantiation.
self.assertEqual(len(bpy.data.collections), 1)
# One archive library for the packed data-blocks and the reference library.
self.assertEqual(len(bpy.data.libraries), 2)
# Packed dat should be owned by archive library.
packed_mesh = bpy.data.meshes[0]
packed_object = bpy.data.objects[0]
packed_collection = bpy.data.collections[0]
link_library = bpy.data.libraries[0]
archive_library = bpy.data.libraries[1]
self.assertEqual(packed_mesh.library, archive_library)
self.assertEqual(packed_object.library, archive_library)
self.assertEqual(packed_collection.library, archive_library)
self.assertTrue(archive_library.is_archive)
self.assertFalse(link_library.is_archive)
class TestBlendLibDataLibrariesLoadLibOverride(TestBlendLibDataLibrariesLoad):
def test_libload_liboverride(self):
@@ -741,11 +965,14 @@ TESTS = (
TestBlendLibAppendBasic,
TestBlendLibAppendReuseID,
TestBlendLibPackedLinkedID,
TestBlendLibLibraryReload,
TestBlendLibLibraryRelocate,
TestBlendLibDataLibrariesLoadAppend,
TestBlendLibDataLibrariesLoadLink,
TestBlendLibDataLibrariesLoadPack,
TestBlendLibDataLibrariesLoadLibOverride,
)

49
tests/python/bl_blendfile_utils.py
View File

@@ -182,3 +182,52 @@ class TestBlendLibLinkHelper(TestHelper):
bpy.ops.wm.save_as_mainfile(filepath=output_lib_path, check_existing=False, compress=False)
return output_lib_path
def init_lib_data_packed_indirect_lib(self):
output_dir = self.args.output_dir
self.ensure_path(output_dir)
# Create an indirect library containing a material, and an image texture.
self.reset_blender()
self.gen_indirect_library_data_()
# Take care to keep the name unique so multiple test jobs can run at once.
output_lib_path = os.path.join(output_dir, self.unique_blendfile_name("blendlib_indirect_material"))
bpy.ops.wm.save_as_mainfile(filepath=output_lib_path, check_existing=False, compress=False)
# Create a main library containing object etc., and linking material from indirect library.
self.reset_blender()
self.gen_library_data_()
link_dir = os.path.join(output_lib_path, "Material")
bpy.ops.wm.link(directory=link_dir, filename="LibMaterial")
ma = bpy.data.pack_linked_ids_hierarchy(bpy.data.materials[0])
me = bpy.data.meshes[0]
me.materials.append(ma)
bpy.ops.outliner.orphans_purge()
self.assertEqual(len(bpy.data.materials), 1)
self.assertTrue(bpy.data.materials[0].is_linked_packed)
self.assertEqual(len(bpy.data.images), 1)
self.assertTrue(bpy.data.images[0].is_linked_packed)
self.assertEqual(len(bpy.data.libraries), 2)
self.assertFalse(bpy.data.libraries[0].is_archive)
self.assertTrue(bpy.data.libraries[1].is_archive)
self.assertIn(bpy.data.libraries[1].name, bpy.data.libraries[0].archive_libraries)
output_dir = self.args.output_dir
self.ensure_path(output_dir)
# Take care to keep the name unique so multiple test jobs can run at once.
output_lib_path = os.path.join(output_dir, self.unique_blendfile_name("blendlib_indirect_main"))
bpy.ops.wm.save_as_mainfile(filepath=output_lib_path, check_existing=False, compress=False)
return output_lib_path