diff --git a/source/blender/blenkernel/BKE_bpath.hh b/source/blender/blenkernel/BKE_bpath.hh index c667f571f1f..6a57a00f8f3 100644 --- a/source/blender/blenkernel/BKE_bpath.hh +++ b/source/blender/blenkernel/BKE_bpath.hh @@ -23,7 +23,12 @@ struct ReportList; /** \name Core `foreach_path` API. * \{ */ -/** Flags controlling the behavior of the generic BPath API. */ +/** + * Flags controlling the behavior of the generic BPath API. + * + * Note: these are referred to by `rna_enum_file_path_foreach_flag_items`, so make sure that any + * new enum items are added there too. + */ enum eBPathForeachFlag { /** * Ensures the `absolute_base_path` member of #BPathForeachPathData is initialized properly with @@ -35,7 +40,10 @@ enum eBPathForeachFlag { BKE_BPATH_FOREACH_PATH_SKIP_LINKED = (1 << 1), /** Skip paths when their matching data is packed. */ BKE_BPATH_FOREACH_PATH_SKIP_PACKED = (1 << 2), - /** Resolve tokens within a virtual filepath to a single, concrete, filepath. */ + /** + * Resolve tokens within a virtual filepath to a single, concrete, filepath. Currently only used + * for UDIM tiles. + */ BKE_BPATH_FOREACH_PATH_RESOLVE_TOKEN = (1 << 3), /** * Skip weak reference paths. Those paths are typically 'nice to have' extra information, but are @@ -53,7 +61,7 @@ enum eBPathForeachFlag { /** * Skip paths where a single dir is used with an array of files, eg. sequence strip images or - * point-caches. In this case only use the first file path is processed. + * point-caches. In this case only the first file path is processed. * * This is needed for directory manipulation callbacks which might otherwise modify the same * directory multiple times. diff --git a/source/blender/makesrna/RNA_enum_items.hh b/source/blender/makesrna/RNA_enum_items.hh index b99926dd40a..0d9baeef566 100644 --- a/source/blender/makesrna/RNA_enum_items.hh +++ b/source/blender/makesrna/RNA_enum_items.hh @@ -282,6 +282,9 @@ DEF_ENUM(rna_enum_keyblock_type_items) DEF_ENUM(rna_enum_asset_library_type_items) +/* Defined in source/blender/python/intern/bpy_rna_id_collection.cc */ +DEF_ENUM(rna_enum_file_path_foreach_flag_items) + #endif #undef DEF_ENUM diff --git a/source/blender/python/intern/bpy_rna_id_collection.cc b/source/blender/python/intern/bpy_rna_id_collection.cc index 66dea569a47..b138f6d6438 100644 --- a/source/blender/python/intern/bpy_rna_id_collection.cc +++ b/source/blender/python/intern/bpy_rna_id_collection.cc @@ -14,6 +14,7 @@ #include "MEM_guardedalloc.h" #include "BLI_bitmap.h" +#include "BLI_string.h" #include "BKE_bpath.hh" #include "BKE_global.hh" @@ -33,6 +34,7 @@ #include "../generic/py_capi_rna.hh" #include "../generic/py_capi_utils.hh" #include "../generic/python_compat.hh" /* IWYU pragma: keep. */ +#include "../generic/python_utildefines.hh" #include "RNA_enum_types.hh" #include "RNA_prototypes.hh" @@ -504,6 +506,311 @@ error: return ret; } +struct IDFilePathForeachData { + /** + * Python callback function for visiting each path. + * + * `def visit_path_fn(owner_id: bpy.types.ID, path: str) -> str | None` + * + * If the function returns a string, the path is replaced with the return + * value. + */ + PyObject *visit_path_fn; + + /** + * Set to `true` when there was an exception in the callback function. Once this is set, no + * Python API function should be called any more (apart from reference counting), so that the + * error state is maintained correctly. + */ + bool seen_error; +}; + +/** + * Wraps #eBPathForeachFlag from BKE_path.hh. + * + * This is exposed publicly (as in, not inline in a function) for the purpose of + * being included in documentation. + */ +const EnumPropertyItem rna_enum_file_path_foreach_flag_items[] = { + /* BKE_BPATH_FOREACH_PATH_ABSOLUTE is not included here, as its only use is to initialize a + * field in BPathForeachPathData that is not used by the callback. */ + {BKE_BPATH_FOREACH_PATH_SKIP_LINKED, + "SKIP_LINKED", + 0, + "Skip Linked", + "Skip paths of linked IDs"}, + {BKE_BPATH_FOREACH_PATH_SKIP_PACKED, + "SKIP_PACKED", + 0, + "Skip Packed", + "Skip paths when their matching data is packed"}, + {BKE_BPATH_FOREACH_PATH_RESOLVE_TOKEN, + "RESOLVE_TOKEN", + 0, + "Resolve Token", + "Resolve tokens within a virtual filepath to a single, concrete, filepath. Currently only " + "used for UDIM tiles"}, + {BKE_BPATH_TRAVERSE_SKIP_WEAK_REFERENCES, + "SKIP_WEAK_REFERENCES", + 0, + "Skip Weak References", + "Skip weak reference paths. Those paths are typically 'nice to have' extra information, but " + "are not used as actual source of data by the current .blend file"}, + {BKE_BPATH_FOREACH_PATH_SKIP_MULTIFILE, + "SKIP_MULTIFILE", + 0, + "Skip Multifile", + "Skip paths where a single dir is used with an array of files, eg. sequence strip images or " + "point-caches. In this case only the first file path is processed. This is needed for " + "directory manipulation callbacks which might otherwise modify the same directory multiple " + "times"}, + {BKE_BPATH_FOREACH_PATH_RELOAD_EDITED, + "RELOAD_EDITED", + 0, + "Reload Edited", + "Reload data when the path is edited"}, + {0, nullptr, 0, nullptr, nullptr}, +}; + +/** Wrapper for MEM_SAFE_FREE() as deallocator for std::unique_ptr. */ +struct MEM_freeN_smart_ptr_deleter { + void operator()(void *pointer) const noexcept + { + MEM_SAFE_FREE(pointer); + } +}; + +static bool foreach_id_file_path_foreach_callback(BPathForeachPathData *bpath_data, + char *path_dst, + const size_t path_dst_maxncpy, + const char *path_src) +{ + IDFilePathForeachData &data = *static_cast(bpath_data->user_data); + + if (data.seen_error) { + /* The Python interpreter is already set up for reporting an exception, so don't touch it. */ + return false; + } + + if (!path_src || !path_src[0]) { + return false; + } + BLI_assert(path_dst); + + /* Construct the callback function parameters. */ + PointerRNA id_ptr = RNA_id_pointer_create(bpath_data->owner_id); + PyObject *args = PyTuple_New(3); + /* args[0]: */ + PyObject *py_owner_id = pyrna_struct_CreatePyObject(&id_ptr); + /* args[1]: */ + PyObject *py_path_src = PyUnicode_FromString(path_src); + /* args[2]: currently-unused parameter for passing metadata of the path to the Python function. + * This is intended pass info like: + * - Is the path intended to reference a directory or a file. + * - Does the path support templates. + * - Is the path referring to input or output (the render output, or file output nodes). + * Even though this is not implemented currently, the parameter is already added so that the + * eventual implementation is not an API-breaking change. */ + PyObject *py_path_meta = Py_NewRef(Py_None); + PyTuple_SET_ITEMS(args, py_owner_id, py_path_src, py_path_meta); + + /* Call the Python callback function. */ + PyObject *result = PyObject_CallObject(data.visit_path_fn, args); + + /* Done with the function arguments. */ + Py_DECREF(args); + args = nullptr; + + if (result == nullptr) { + data.seen_error = true; + return false; + } + + if (result == Py_None) { + /* Nothing to do. */ + Py_DECREF(result); + return false; + } + + if (!PyUnicode_Check(result)) { + PyErr_Format(PyExc_TypeError, + "visit_path_fn() should return a string or None, but returned %s for " + "owner_id=\"%s\" and file_path=\"%s\"", + Py_TYPE(result)->tp_name, + bpath_data->owner_id->name, + path_src); + data.seen_error = true; + Py_DECREF(result); + return false; + } + + /* Copy the returned string back into the path. */ + Py_ssize_t replacement_path_length = 0; + PyObject *value_coerce = nullptr; + const char *replacement_path = PyC_UnicodeAsBytesAndSize( + result, &replacement_path_length, &value_coerce); + + /* BLI_strncpy wants buffer size, but PyC_UnicodeAsBytesAndSize reports string + * length, hence the +1. */ + BLI_strncpy( + path_dst, replacement_path, std::min(path_dst_maxncpy, size_t(replacement_path_length + 1))); + + Py_XDECREF(value_coerce); + Py_DECREF(result); + return true; +} + +PyDoc_STRVAR( + /* Wrap. */ + bpy_file_path_foreach_doc, + ".. method:: file_path_foreach(visit_path_fn, *, subset=None, visit_types=None, " + "flags={'SKIP_PACKED', 'SKIP_WEAK_REFERENCES'})\n" + "\n" + " Call ``visit_path_fn`` for the file paths used by all ID data-blocks in current " + "``bpy.data``.\n" + "\n" + " For list of valid set members for visit_types, see: " + ":class:`bpy.types.KeyingSetPath.id_type`.\n" + "\n" + " :arg visit_path_fn: function that takes three parameters: the data-block, a file path, " + "and a placeholder for future use. The function should return either ``None`` or a ``str``. " + "In the latter case, the visited file path will be replaced with the returned string.\n" + " :type visit_path_fn: Callable[[:class:`bpy.types.ID`, str, Any], str|None]\n" + " :arg subset: When given, only these data-blocks and their used file paths " + "will be visited.\n" + " :type subset: set[str]\n" + " :arg visit_types: When given, only visit data-blocks of these types. Ignored if " + "``subset`` is also given.\n" + " :type visit_types: set[str]\n" + " :type flags: set[str]\n" + " :arg flags: Set of flags that influence which data-blocks are visited. See " + ":ref:`rna_enum_file_path_foreach_flag_items`.\n"); +static PyObject *bpy_file_path_foreach(PyObject *self, PyObject *args, PyObject *kwds) +{ + Main *bmain = pyrna_bmain_FromPyObject(self); + if (!bmain) { + return nullptr; + } + + PyObject *visit_path_fn = nullptr; + PyObject *subset = nullptr; + PyObject *visit_types = nullptr; + std::unique_ptr visit_types_bitmap; + PyObject *py_flags = nullptr; + + IDFilePathForeachData filepathforeach_data{}; + BPathForeachPathData bpath_data{}; + + static const char *_keywords[] = {"visit_path_fn", "subset", "visit_types", "flags", nullptr}; + static _PyArg_Parser _parser = { + PY_ARG_PARSER_HEAD_COMPAT() + "O!" /* `visit_path_fn` */ + "|$" /* Optional keyword only arguments. */ + "O" /* `subset` */ + "O!" /* `visit_types` */ + "O!" /* `flags` */ + ":file_path_foreach", + _keywords, + nullptr, + }; + if (!_PyArg_ParseTupleAndKeywordsFast(args, + kwds, + &_parser, + &PyFunction_Type, + &visit_path_fn, + &subset, + &PySet_Type, + &visit_types, + &PySet_Type, + &py_flags)) + { + return nullptr; + } + + if (visit_types) { + BLI_bitmap *visit_types_bitmap_rawptr = pyrna_enum_bitmap_from_set( + rna_enum_id_type_items, visit_types, sizeof(short), true, USHRT_MAX, "visit_types"); + if (visit_types_bitmap_rawptr == nullptr) { + return nullptr; + } + visit_types_bitmap.reset(visit_types_bitmap_rawptr); + } + + /* Parse the flags, start with sensible defaults. */ + bpath_data.flag = BKE_BPATH_FOREACH_PATH_SKIP_PACKED | BKE_BPATH_TRAVERSE_SKIP_WEAK_REFERENCES; + if (py_flags) { + if (pyrna_enum_bitfield_from_set(rna_enum_file_path_foreach_flag_items, + py_flags, + reinterpret_cast(&bpath_data.flag), + "flags") == -1) + { + return nullptr; + } + } + + bpath_data.bmain = bmain; + bpath_data.callback_function = foreach_id_file_path_foreach_callback; + bpath_data.user_data = &filepathforeach_data; + + filepathforeach_data.visit_path_fn = visit_path_fn; + filepathforeach_data.seen_error = false; + + if (subset) { + /* Visit the given subset of IDs. */ + PyObject *subset_fast = PySequence_Fast(subset, "subset"); + if (!subset_fast) { + return nullptr; + } + + PyObject **subset_array = PySequence_Fast_ITEMS(subset_fast); + const Py_ssize_t subset_len = PySequence_Fast_GET_SIZE(subset_fast); + for (Py_ssize_t index = 0; index < subset_len; index++) { + PyObject *subset_item = subset_array[index]; + + ID *id; + if (!pyrna_id_FromPyObject(subset_item, &id)) { + PyErr_Format(PyExc_TypeError, + "Expected an ID type in `subset` iterable, not %.200s", + Py_TYPE(subset_item)->tp_name); + Py_DECREF(subset_fast); + return nullptr; + } + + BKE_bpath_foreach_path_id(&bpath_data, id); + if (filepathforeach_data.seen_error) { + /* Whatever triggered this error should have already set up the Python + * interpreter for producing an exception. */ + Py_DECREF(subset_fast); + return nullptr; + } + } + Py_DECREF(subset_fast); + } + else { + /* Visit all IDs, filtered by type if necessary. */ + ListBase *lb; + FOREACH_MAIN_LISTBASE_BEGIN (bmain, lb) { + ID *id; + FOREACH_MAIN_LISTBASE_ID_BEGIN (lb, id) { + if (visit_types_bitmap && !id_check_type(id, visit_types_bitmap.get())) { + break; + } + + BKE_bpath_foreach_path_id(&bpath_data, id); + if (filepathforeach_data.seen_error) { + /* Whatever triggered this error should have already set up the Python + * interpreter for producing an exception. */ + return nullptr; + } + } + FOREACH_MAIN_LISTBASE_ID_END; + } + FOREACH_MAIN_LISTBASE_END; + } + + Py_RETURN_NONE; +} + PyDoc_STRVAR( /* Wrap. */ bpy_batch_remove_doc, @@ -658,6 +965,12 @@ PyMethodDef BPY_rna_id_collection_file_path_map_method_def = { METH_VARARGS | METH_KEYWORDS, bpy_file_path_map_doc, }; +PyMethodDef BPY_rna_id_collection_file_path_foreach_method_def = { + "file_path_foreach", + (PyCFunction)bpy_file_path_foreach, + METH_VARARGS | METH_KEYWORDS, + bpy_file_path_foreach_doc, +}; PyMethodDef BPY_rna_id_collection_batch_remove_method_def = { "batch_remove", (PyCFunction)bpy_batch_remove, diff --git a/source/blender/python/intern/bpy_rna_id_collection.hh b/source/blender/python/intern/bpy_rna_id_collection.hh index 160555570f6..8025445f36c 100644 --- a/source/blender/python/intern/bpy_rna_id_collection.hh +++ b/source/blender/python/intern/bpy_rna_id_collection.hh @@ -12,5 +12,6 @@ extern PyMethodDef BPY_rna_id_collection_user_map_method_def; extern PyMethodDef BPY_rna_id_collection_file_path_map_method_def; +extern PyMethodDef BPY_rna_id_collection_file_path_foreach_method_def; extern PyMethodDef BPY_rna_id_collection_batch_remove_method_def; extern PyMethodDef BPY_rna_id_collection_orphans_purge_method_def; diff --git a/source/blender/python/intern/bpy_rna_types_capi.cc b/source/blender/python/intern/bpy_rna_types_capi.cc index 32117d56f88..8a9c8567096 100644 --- a/source/blender/python/intern/bpy_rna_types_capi.cc +++ b/source/blender/python/intern/bpy_rna_types_capi.cc @@ -46,6 +46,7 @@ static PyMethodDef pyrna_blenddata_methods[] = { {nullptr, nullptr, 0, nullptr}, /* #BPY_rna_id_collection_user_map_method_def */ {nullptr, nullptr, 0, nullptr}, /* #BPY_rna_id_collection_file_path_map_method_def */ + {nullptr, nullptr, 0, nullptr}, /* #BPY_rna_id_collection_file_path_foreach_method_def */ {nullptr, nullptr, 0, nullptr}, /* #BPY_rna_id_collection_batch_remove_method_def */ {nullptr, nullptr, 0, nullptr}, /* #BPY_rna_id_collection_orphans_purge_method_def */ {nullptr, nullptr, 0, nullptr}, /* #BPY_rna_data_context_method_def */ @@ -276,10 +277,11 @@ void BPY_rna_types_extend_capi() ARRAY_SET_ITEMS(pyrna_blenddata_methods, BPY_rna_id_collection_user_map_method_def, BPY_rna_id_collection_file_path_map_method_def, + BPY_rna_id_collection_file_path_foreach_method_def, BPY_rna_id_collection_batch_remove_method_def, BPY_rna_id_collection_orphans_purge_method_def, BPY_rna_data_context_method_def); - BLI_STATIC_ASSERT(ARRAY_SIZE(pyrna_blenddata_methods) == 6, "Unexpected number of methods") + BLI_STATIC_ASSERT(ARRAY_SIZE(pyrna_blenddata_methods) == 7, "Unexpected number of methods") pyrna_struct_type_extend_capi(&RNA_BlendData, pyrna_blenddata_methods, nullptr); /* BlendDataLibraries */ diff --git a/tests/python/bl_blendfile_relationships.py b/tests/python/bl_blendfile_relationships.py index 5de8787ab3e..e5843b5a975 100644 --- a/tests/python/bl_blendfile_relationships.py +++ b/tests/python/bl_blendfile_relationships.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 """ -./blender.bin --background --python tests/python/bl_blendfile_liblink.py +blender -b -P tests/python/bl_blendfile_relationships.py --src-test-dir tests/files --output-dir /tmp/blendfile_io/ """ __all__ = ( "main", @@ -12,11 +12,14 @@ __all__ = ( import bpy import os import sys +from pathlib import Path from bpy.path import native_pathsep -sys.path.append(os.path.dirname(os.path.realpath(__file__))) -from bl_blendfile_utils import TestBlendLibLinkHelper +_my_dir = Path(__file__).resolve().parent +sys.path.append(str(_my_dir)) + +from bl_blendfile_utils import TestBlendLibLinkHelper, TestHelper class TestBlendUserMap(TestBlendLibLinkHelper): @@ -166,9 +169,131 @@ class TestBlendFilePathMap(TestBlendLibLinkHelper): self.assertRaises(TypeError, bpy.data.file_path_map, subset=[bpy.data.objects[0], bpy.data.images[0], "FooBar"]) +class TestBlendFilePathForeach(TestHelper): + testdir: Path + + def setUp(self) -> None: + super().setUp() + + # File paths can get long, and thus also so can diffs when things go wrong. + self.maxDiff = 10240 + self.testdir = Path(self.args.src_test_dir) / "libraries_and_linking" + + bpy.ops.wm.open_mainfile(filepath=str(self.testdir / "library_test_scene.blend")) + + # Name the library data-blocks after their filename, so that they can be reliably identified later. + # Otherwise we're stuck with 'Lib', 'Lib.001', etc. + for lib_id in bpy.data.libraries[:]: + abspath = Path(str(bpy.path.abspath(lib_id.filepath))) + lib_id.name = abspath.stem + + def test_without_args(self) -> None: + """Test file_path_foreach() without any arguments, it should report everything but packed files.""" + + visited_paths = self._file_path_foreach() + self.assertEqual({ + (bpy.data.libraries['direct_linked_A'], self.testdir / "libraries/direct_linked_A.blend"), + (bpy.data.libraries['direct_linked_B'], self.testdir / "libraries/direct_linked_B.blend"), + (bpy.data.libraries['indirect_datablocks'], self.testdir / "libraries/indirect_datablocks.blend"), + }, visited_paths) + + def test_with_nondefault_flag(self) -> None: + """Test file_path_foreach() with a non-default flag. + + If any non-default flag works, it's enough as a test to assume the flags + are parsed and passed to the C++ code correctly. There is no need to + exhaustively test all the flags here. + """ + + # The default value for `flags` is {'SKIP_PACKED', + # 'SKIP_WEAK_REFERENCES'}, so to also see the packed files, only pass + # `SKIP_WEAK_REFERENCES`. + visited_paths = self._file_path_foreach(flags={'SKIP_WEAK_REFERENCES'}) + self.assertEqual({ + (bpy.data.libraries['direct_linked_A'], self.testdir / "libraries/direct_linked_A.blend"), + (bpy.data.libraries['direct_linked_B'], self.testdir / "libraries/direct_linked_B.blend"), + (bpy.data.libraries['indirect_datablocks'], self.testdir / "libraries/indirect_datablocks.blend"), + (bpy.data.images['pack.png'], self.testdir / "libraries/pack.png"), + }, visited_paths, "testing without SKIP_PACKED") + + def test_filepath_rewriting(self) -> None: + # Store the pre-modification value, as its use of (back)slashes is platform-dependent. + image_filepath_before = str(bpy.data.images['pack.png'].filepath) + + def visit_path_fn(owner_id: bpy.types.ID, path: str, _meta: None) -> str | None: + return "//{}-rewritten.blend".format(owner_id.name) + bpy.data.file_path_foreach(visit_path_fn) + + libs = bpy.data.libraries + self.assertEqual(libs['direct_linked_A'].filepath, "//direct_linked_A-rewritten.blend") + self.assertEqual(libs['direct_linked_B'].filepath, "//direct_linked_B-rewritten.blend") + self.assertEqual(libs['indirect_datablocks'].filepath, "//indirect_datablocks-rewritten.blend") + self.assertEqual( + bpy.data.images['pack.png'].filepath, + image_filepath_before, + "Packed file should not have changed") + + def test_exception_passing(self) -> None: + """Python exceptions in the callback function should be raised by file_path_foreach().""" + # Any Python exception should work, not just built-in ones. + class CustomException(Exception): + pass + + def visit_path_fn(_owner_id: bpy.types.ID, _path: str, meta: None) -> str | None: + raise CustomException("arg0", 1, "arg2") + + try: + bpy.data.file_path_foreach(visit_path_fn) + except CustomException as ex: + self.assertEqual(("arg0", 1, "arg2"), ex.args, "Parameters passed to the exception should be retained") + else: + self.fail("Expected exception not thrown") + + def test_meta_parameter(self) -> None: + def visit_path_fn(_owner_id: bpy.types.ID, _path: str, meta: None) -> str | None: + # This is proven to work by the `test_exception_passing()` test above. + self.assertIsNone( + meta, + "The meta parameter is expected to be None; this test is expected to fail " + "once the metadata feature is actually getting implemented, and should then " + "be replaced with a proper test.") + bpy.data.file_path_foreach(visit_path_fn) + + @staticmethod + def _file_path_foreach( + subtypes: set[str] | None = None, + visit_keys: set[str] | None = None, + flags: set[str] | None = None, + ) -> set[tuple[bpy.types.ID, Path]]: + """Call bpy.data.file_path_foreach(), returning the visited paths as set of (datablock, path) tuples. + + A set is used because the order of visiting is not relevant. + """ + visisted_paths: set[tuple[bpy.types.ID, Path]] = set() + + def visit_path_fn(owner_id: bpy.types.ID, path: str, _meta: None) -> str | None: + abspath = Path(str(bpy.path.abspath(path, library=owner_id.library))) + visisted_paths.add((owner_id, abspath)) + + # Dynamically build the keyword arguments, because the None values are not + # valid, and should be encoded as not having the kwarg. + kwargs = {} + if subtypes is not None: + kwargs['subtypes'] = subtypes + if visit_keys is not None: + kwargs['visit_keys'] = visit_keys + if flags is not None: + kwargs['flags'] = flags + + bpy.data.file_path_foreach(visit_path_fn, **kwargs) + + return visisted_paths + + TESTS = ( TestBlendUserMap, TestBlendFilePathMap, + TestBlendFilePathForeach, ) diff --git a/tests/python/bl_blendfile_utils.py b/tests/python/bl_blendfile_utils.py index c369aef550e..04009de487f 100644 --- a/tests/python/bl_blendfile_utils.py +++ b/tests/python/bl_blendfile_utils.py @@ -39,8 +39,14 @@ class TestHelper(unittest.TestCase): if not inst_attr_id.startswith("test_"): continue inst_attr = getattr(self, inst_attr_id) - if callable(inst_attr): + if not callable(inst_attr): + continue + + self.setUp() + try: inst_attr() + finally: + self.tearDown() class TestBlendLibLinkHelper(TestHelper):