Python: add function for file iteration/remapping bpy.data.file_path_foreach()

Add a function `bpy.data.file_path_foreach()` with a callback function
`visit_path_fn`:

```py
def visit_path_fn(
    owner_id: bpy.types.ID,
    file_path: str,
    _meta: typing.Any,
) -> str | None:
    return file_path.replace('xxx', 'yyy') or None

bpy.data.file_path_foreach(
    visit_path_fn,
    *,
    subset=None,
    visit_types=None,
    flags={},
)
```

When the function returns `None`, nothing happens. When it returns a
string, the reported file path is replaced with the returned string.

`subset` and `visit_types` have the same meaning as `subset` resp.
`key_types` in `bpy.data.file_path_map()`. Since the latter parameter
doesn't affect keys in a map, but rather the visited ID types, I named
it `visit_types` instead.

`flags` wraps most of `eBPathForeachFlag` as `set[str]`, with their
prefixes removed. `BKE_BPATH_FOREACH_PATH_RESOLVE_TOKEN` becomes
`RESOLVE_TOKEN`, etc. `BKE_BPATH_FOREACH_PATH_ABSOLUTE` is excluded,
as it only affects a field in the C++ callback structure that is not
passed to Python at all. If it turns out to be useful at some point,
we can always add it.

`_meta` is for future use, and is intended to give metadata of the
path (like whether it's an input or an output path, a single file or
the first file in a sequence, supports variables, etc.). By adding
this to the API now, we avoid a breaking change when this feature is
actually implemented.

Design task: #145734

Pull Request: https://projects.blender.org/blender/blender/pulls/146261
This commit is contained in:
Sybren A. Stüvel
2025-10-06 17:10:49 +02:00
parent e12ddede24
commit d33a6a1723
7 changed files with 466 additions and 8 deletions

View File

@@ -23,7 +23,12 @@ struct ReportList;
/** \name Core `foreach_path` API.
* \{ */
/** Flags controlling the behavior of the generic BPath API. */
/**
* Flags controlling the behavior of the generic BPath API.
*
* Note: these are referred to by `rna_enum_file_path_foreach_flag_items`, so make sure that any
* new enum items are added there too.
*/
enum eBPathForeachFlag {
/**
* Ensures the `absolute_base_path` member of #BPathForeachPathData is initialized properly with
@@ -35,7 +40,10 @@ enum eBPathForeachFlag {
BKE_BPATH_FOREACH_PATH_SKIP_LINKED = (1 << 1),
/** Skip paths when their matching data is packed. */
BKE_BPATH_FOREACH_PATH_SKIP_PACKED = (1 << 2),
/** Resolve tokens within a virtual filepath to a single, concrete, filepath. */
/**
* Resolve tokens within a virtual filepath to a single, concrete, filepath. Currently only used
* for UDIM tiles.
*/
BKE_BPATH_FOREACH_PATH_RESOLVE_TOKEN = (1 << 3),
/**
* Skip weak reference paths. Those paths are typically 'nice to have' extra information, but are
@@ -53,7 +61,7 @@ enum eBPathForeachFlag {
/**
* Skip paths where a single dir is used with an array of files, eg. sequence strip images or
* point-caches. In this case only use the first file path is processed.
* point-caches. In this case only the first file path is processed.
*
* This is needed for directory manipulation callbacks which might otherwise modify the same
* directory multiple times.

View File

@@ -282,6 +282,9 @@ DEF_ENUM(rna_enum_keyblock_type_items)
DEF_ENUM(rna_enum_asset_library_type_items)
/* Defined in source/blender/python/intern/bpy_rna_id_collection.cc */
DEF_ENUM(rna_enum_file_path_foreach_flag_items)
#endif
#undef DEF_ENUM

View File

@@ -14,6 +14,7 @@
#include "MEM_guardedalloc.h"
#include "BLI_bitmap.h"
#include "BLI_string.h"
#include "BKE_bpath.hh"
#include "BKE_global.hh"
@@ -33,6 +34,7 @@
#include "../generic/py_capi_rna.hh"
#include "../generic/py_capi_utils.hh"
#include "../generic/python_compat.hh" /* IWYU pragma: keep. */
#include "../generic/python_utildefines.hh"
#include "RNA_enum_types.hh"
#include "RNA_prototypes.hh"
@@ -504,6 +506,311 @@ error:
return ret;
}
struct IDFilePathForeachData {
/**
* Python callback function for visiting each path.
*
* `def visit_path_fn(owner_id: bpy.types.ID, path: str) -> str | None`
*
* If the function returns a string, the path is replaced with the return
* value.
*/
PyObject *visit_path_fn;
/**
* Set to `true` when there was an exception in the callback function. Once this is set, no
* Python API function should be called any more (apart from reference counting), so that the
* error state is maintained correctly.
*/
bool seen_error;
};
/**
* Wraps #eBPathForeachFlag from BKE_path.hh.
*
* This is exposed publicly (as in, not inline in a function) for the purpose of
* being included in documentation.
*/
const EnumPropertyItem rna_enum_file_path_foreach_flag_items[] = {
/* BKE_BPATH_FOREACH_PATH_ABSOLUTE is not included here, as its only use is to initialize a
* field in BPathForeachPathData that is not used by the callback. */
{BKE_BPATH_FOREACH_PATH_SKIP_LINKED,
"SKIP_LINKED",
0,
"Skip Linked",
"Skip paths of linked IDs"},
{BKE_BPATH_FOREACH_PATH_SKIP_PACKED,
"SKIP_PACKED",
0,
"Skip Packed",
"Skip paths when their matching data is packed"},
{BKE_BPATH_FOREACH_PATH_RESOLVE_TOKEN,
"RESOLVE_TOKEN",
0,
"Resolve Token",
"Resolve tokens within a virtual filepath to a single, concrete, filepath. Currently only "
"used for UDIM tiles"},
{BKE_BPATH_TRAVERSE_SKIP_WEAK_REFERENCES,
"SKIP_WEAK_REFERENCES",
0,
"Skip Weak References",
"Skip weak reference paths. Those paths are typically 'nice to have' extra information, but "
"are not used as actual source of data by the current .blend file"},
{BKE_BPATH_FOREACH_PATH_SKIP_MULTIFILE,
"SKIP_MULTIFILE",
0,
"Skip Multifile",
"Skip paths where a single dir is used with an array of files, eg. sequence strip images or "
"point-caches. In this case only the first file path is processed. This is needed for "
"directory manipulation callbacks which might otherwise modify the same directory multiple "
"times"},
{BKE_BPATH_FOREACH_PATH_RELOAD_EDITED,
"RELOAD_EDITED",
0,
"Reload Edited",
"Reload data when the path is edited"},
{0, nullptr, 0, nullptr, nullptr},
};
/** Wrapper for MEM_SAFE_FREE() as deallocator for std::unique_ptr. */
struct MEM_freeN_smart_ptr_deleter {
void operator()(void *pointer) const noexcept
{
MEM_SAFE_FREE(pointer);
}
};
static bool foreach_id_file_path_foreach_callback(BPathForeachPathData *bpath_data,
char *path_dst,
const size_t path_dst_maxncpy,
const char *path_src)
{
IDFilePathForeachData &data = *static_cast<IDFilePathForeachData *>(bpath_data->user_data);
if (data.seen_error) {
/* The Python interpreter is already set up for reporting an exception, so don't touch it. */
return false;
}
if (!path_src || !path_src[0]) {
return false;
}
BLI_assert(path_dst);
/* Construct the callback function parameters. */
PointerRNA id_ptr = RNA_id_pointer_create(bpath_data->owner_id);
PyObject *args = PyTuple_New(3);
/* args[0]: */
PyObject *py_owner_id = pyrna_struct_CreatePyObject(&id_ptr);
/* args[1]: */
PyObject *py_path_src = PyUnicode_FromString(path_src);
/* args[2]: currently-unused parameter for passing metadata of the path to the Python function.
* This is intended pass info like:
* - Is the path intended to reference a directory or a file.
* - Does the path support templates.
* - Is the path referring to input or output (the render output, or file output nodes).
* Even though this is not implemented currently, the parameter is already added so that the
* eventual implementation is not an API-breaking change. */
PyObject *py_path_meta = Py_NewRef(Py_None);
PyTuple_SET_ITEMS(args, py_owner_id, py_path_src, py_path_meta);
/* Call the Python callback function. */
PyObject *result = PyObject_CallObject(data.visit_path_fn, args);
/* Done with the function arguments. */
Py_DECREF(args);
args = nullptr;
if (result == nullptr) {
data.seen_error = true;
return false;
}
if (result == Py_None) {
/* Nothing to do. */
Py_DECREF(result);
return false;
}
if (!PyUnicode_Check(result)) {
PyErr_Format(PyExc_TypeError,
"visit_path_fn() should return a string or None, but returned %s for "
"owner_id=\"%s\" and file_path=\"%s\"",
Py_TYPE(result)->tp_name,
bpath_data->owner_id->name,
path_src);
data.seen_error = true;
Py_DECREF(result);
return false;
}
/* Copy the returned string back into the path. */
Py_ssize_t replacement_path_length = 0;
PyObject *value_coerce = nullptr;
const char *replacement_path = PyC_UnicodeAsBytesAndSize(
result, &replacement_path_length, &value_coerce);
/* BLI_strncpy wants buffer size, but PyC_UnicodeAsBytesAndSize reports string
* length, hence the +1. */
BLI_strncpy(
path_dst, replacement_path, std::min(path_dst_maxncpy, size_t(replacement_path_length + 1)));
Py_XDECREF(value_coerce);
Py_DECREF(result);
return true;
}
PyDoc_STRVAR(
/* Wrap. */
bpy_file_path_foreach_doc,
".. method:: file_path_foreach(visit_path_fn, *, subset=None, visit_types=None, "
"flags={'SKIP_PACKED', 'SKIP_WEAK_REFERENCES'})\n"
"\n"
" Call ``visit_path_fn`` for the file paths used by all ID data-blocks in current "
"``bpy.data``.\n"
"\n"
" For list of valid set members for visit_types, see: "
":class:`bpy.types.KeyingSetPath.id_type`.\n"
"\n"
" :arg visit_path_fn: function that takes three parameters: the data-block, a file path, "
"and a placeholder for future use. The function should return either ``None`` or a ``str``. "
"In the latter case, the visited file path will be replaced with the returned string.\n"
" :type visit_path_fn: Callable[[:class:`bpy.types.ID`, str, Any], str|None]\n"
" :arg subset: When given, only these data-blocks and their used file paths "
"will be visited.\n"
" :type subset: set[str]\n"
" :arg visit_types: When given, only visit data-blocks of these types. Ignored if "
"``subset`` is also given.\n"
" :type visit_types: set[str]\n"
" :type flags: set[str]\n"
" :arg flags: Set of flags that influence which data-blocks are visited. See "
":ref:`rna_enum_file_path_foreach_flag_items`.\n");
static PyObject *bpy_file_path_foreach(PyObject *self, PyObject *args, PyObject *kwds)
{
Main *bmain = pyrna_bmain_FromPyObject(self);
if (!bmain) {
return nullptr;
}
PyObject *visit_path_fn = nullptr;
PyObject *subset = nullptr;
PyObject *visit_types = nullptr;
std::unique_ptr<BLI_bitmap, MEM_freeN_smart_ptr_deleter> visit_types_bitmap;
PyObject *py_flags = nullptr;
IDFilePathForeachData filepathforeach_data{};
BPathForeachPathData bpath_data{};
static const char *_keywords[] = {"visit_path_fn", "subset", "visit_types", "flags", nullptr};
static _PyArg_Parser _parser = {
PY_ARG_PARSER_HEAD_COMPAT()
"O!" /* `visit_path_fn` */
"|$" /* Optional keyword only arguments. */
"O" /* `subset` */
"O!" /* `visit_types` */
"O!" /* `flags` */
":file_path_foreach",
_keywords,
nullptr,
};
if (!_PyArg_ParseTupleAndKeywordsFast(args,
kwds,
&_parser,
&PyFunction_Type,
&visit_path_fn,
&subset,
&PySet_Type,
&visit_types,
&PySet_Type,
&py_flags))
{
return nullptr;
}
if (visit_types) {
BLI_bitmap *visit_types_bitmap_rawptr = pyrna_enum_bitmap_from_set(
rna_enum_id_type_items, visit_types, sizeof(short), true, USHRT_MAX, "visit_types");
if (visit_types_bitmap_rawptr == nullptr) {
return nullptr;
}
visit_types_bitmap.reset(visit_types_bitmap_rawptr);
}
/* Parse the flags, start with sensible defaults. */
bpath_data.flag = BKE_BPATH_FOREACH_PATH_SKIP_PACKED | BKE_BPATH_TRAVERSE_SKIP_WEAK_REFERENCES;
if (py_flags) {
if (pyrna_enum_bitfield_from_set(rna_enum_file_path_foreach_flag_items,
py_flags,
reinterpret_cast<int *>(&bpath_data.flag),
"flags") == -1)
{
return nullptr;
}
}
bpath_data.bmain = bmain;
bpath_data.callback_function = foreach_id_file_path_foreach_callback;
bpath_data.user_data = &filepathforeach_data;
filepathforeach_data.visit_path_fn = visit_path_fn;
filepathforeach_data.seen_error = false;
if (subset) {
/* Visit the given subset of IDs. */
PyObject *subset_fast = PySequence_Fast(subset, "subset");
if (!subset_fast) {
return nullptr;
}
PyObject **subset_array = PySequence_Fast_ITEMS(subset_fast);
const Py_ssize_t subset_len = PySequence_Fast_GET_SIZE(subset_fast);
for (Py_ssize_t index = 0; index < subset_len; index++) {
PyObject *subset_item = subset_array[index];
ID *id;
if (!pyrna_id_FromPyObject(subset_item, &id)) {
PyErr_Format(PyExc_TypeError,
"Expected an ID type in `subset` iterable, not %.200s",
Py_TYPE(subset_item)->tp_name);
Py_DECREF(subset_fast);
return nullptr;
}
BKE_bpath_foreach_path_id(&bpath_data, id);
if (filepathforeach_data.seen_error) {
/* Whatever triggered this error should have already set up the Python
* interpreter for producing an exception. */
Py_DECREF(subset_fast);
return nullptr;
}
}
Py_DECREF(subset_fast);
}
else {
/* Visit all IDs, filtered by type if necessary. */
ListBase *lb;
FOREACH_MAIN_LISTBASE_BEGIN (bmain, lb) {
ID *id;
FOREACH_MAIN_LISTBASE_ID_BEGIN (lb, id) {
if (visit_types_bitmap && !id_check_type(id, visit_types_bitmap.get())) {
break;
}
BKE_bpath_foreach_path_id(&bpath_data, id);
if (filepathforeach_data.seen_error) {
/* Whatever triggered this error should have already set up the Python
* interpreter for producing an exception. */
return nullptr;
}
}
FOREACH_MAIN_LISTBASE_ID_END;
}
FOREACH_MAIN_LISTBASE_END;
}
Py_RETURN_NONE;
}
PyDoc_STRVAR(
/* Wrap. */
bpy_batch_remove_doc,
@@ -658,6 +965,12 @@ PyMethodDef BPY_rna_id_collection_file_path_map_method_def = {
METH_VARARGS | METH_KEYWORDS,
bpy_file_path_map_doc,
};
PyMethodDef BPY_rna_id_collection_file_path_foreach_method_def = {
"file_path_foreach",
(PyCFunction)bpy_file_path_foreach,
METH_VARARGS | METH_KEYWORDS,
bpy_file_path_foreach_doc,
};
PyMethodDef BPY_rna_id_collection_batch_remove_method_def = {
"batch_remove",
(PyCFunction)bpy_batch_remove,

View File

@@ -12,5 +12,6 @@
extern PyMethodDef BPY_rna_id_collection_user_map_method_def;
extern PyMethodDef BPY_rna_id_collection_file_path_map_method_def;
extern PyMethodDef BPY_rna_id_collection_file_path_foreach_method_def;
extern PyMethodDef BPY_rna_id_collection_batch_remove_method_def;
extern PyMethodDef BPY_rna_id_collection_orphans_purge_method_def;

View File

@@ -46,6 +46,7 @@
static PyMethodDef pyrna_blenddata_methods[] = {
{nullptr, nullptr, 0, nullptr}, /* #BPY_rna_id_collection_user_map_method_def */
{nullptr, nullptr, 0, nullptr}, /* #BPY_rna_id_collection_file_path_map_method_def */
{nullptr, nullptr, 0, nullptr}, /* #BPY_rna_id_collection_file_path_foreach_method_def */
{nullptr, nullptr, 0, nullptr}, /* #BPY_rna_id_collection_batch_remove_method_def */
{nullptr, nullptr, 0, nullptr}, /* #BPY_rna_id_collection_orphans_purge_method_def */
{nullptr, nullptr, 0, nullptr}, /* #BPY_rna_data_context_method_def */
@@ -276,10 +277,11 @@ void BPY_rna_types_extend_capi()
ARRAY_SET_ITEMS(pyrna_blenddata_methods,
BPY_rna_id_collection_user_map_method_def,
BPY_rna_id_collection_file_path_map_method_def,
BPY_rna_id_collection_file_path_foreach_method_def,
BPY_rna_id_collection_batch_remove_method_def,
BPY_rna_id_collection_orphans_purge_method_def,
BPY_rna_data_context_method_def);
BLI_STATIC_ASSERT(ARRAY_SIZE(pyrna_blenddata_methods) == 6, "Unexpected number of methods")
BLI_STATIC_ASSERT(ARRAY_SIZE(pyrna_blenddata_methods) == 7, "Unexpected number of methods")
pyrna_struct_type_extend_capi(&RNA_BlendData, pyrna_blenddata_methods, nullptr);
/* BlendDataLibraries */

View File

@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
"""
./blender.bin --background --python tests/python/bl_blendfile_liblink.py
blender -b -P tests/python/bl_blendfile_relationships.py --src-test-dir tests/files --output-dir /tmp/blendfile_io/
"""
__all__ = (
"main",
@@ -12,11 +12,14 @@ __all__ = (
import bpy
import os
import sys
from pathlib import Path
from bpy.path import native_pathsep
sys.path.append(os.path.dirname(os.path.realpath(__file__)))
from bl_blendfile_utils import TestBlendLibLinkHelper
_my_dir = Path(__file__).resolve().parent
sys.path.append(str(_my_dir))
from bl_blendfile_utils import TestBlendLibLinkHelper, TestHelper
class TestBlendUserMap(TestBlendLibLinkHelper):
@@ -166,9 +169,131 @@ class TestBlendFilePathMap(TestBlendLibLinkHelper):
self.assertRaises(TypeError, bpy.data.file_path_map, subset=[bpy.data.objects[0], bpy.data.images[0], "FooBar"])
class TestBlendFilePathForeach(TestHelper):
testdir: Path
def setUp(self) -> None:
super().setUp()
# File paths can get long, and thus also so can diffs when things go wrong.
self.maxDiff = 10240
self.testdir = Path(self.args.src_test_dir) / "libraries_and_linking"
bpy.ops.wm.open_mainfile(filepath=str(self.testdir / "library_test_scene.blend"))
# Name the library data-blocks after their filename, so that they can be reliably identified later.
# Otherwise we're stuck with 'Lib', 'Lib.001', etc.
for lib_id in bpy.data.libraries[:]:
abspath = Path(str(bpy.path.abspath(lib_id.filepath)))
lib_id.name = abspath.stem
def test_without_args(self) -> None:
"""Test file_path_foreach() without any arguments, it should report everything but packed files."""
visited_paths = self._file_path_foreach()
self.assertEqual({
(bpy.data.libraries['direct_linked_A'], self.testdir / "libraries/direct_linked_A.blend"),
(bpy.data.libraries['direct_linked_B'], self.testdir / "libraries/direct_linked_B.blend"),
(bpy.data.libraries['indirect_datablocks'], self.testdir / "libraries/indirect_datablocks.blend"),
}, visited_paths)
def test_with_nondefault_flag(self) -> None:
"""Test file_path_foreach() with a non-default flag.
If any non-default flag works, it's enough as a test to assume the flags
are parsed and passed to the C++ code correctly. There is no need to
exhaustively test all the flags here.
"""
# The default value for `flags` is {'SKIP_PACKED',
# 'SKIP_WEAK_REFERENCES'}, so to also see the packed files, only pass
# `SKIP_WEAK_REFERENCES`.
visited_paths = self._file_path_foreach(flags={'SKIP_WEAK_REFERENCES'})
self.assertEqual({
(bpy.data.libraries['direct_linked_A'], self.testdir / "libraries/direct_linked_A.blend"),
(bpy.data.libraries['direct_linked_B'], self.testdir / "libraries/direct_linked_B.blend"),
(bpy.data.libraries['indirect_datablocks'], self.testdir / "libraries/indirect_datablocks.blend"),
(bpy.data.images['pack.png'], self.testdir / "libraries/pack.png"),
}, visited_paths, "testing without SKIP_PACKED")
def test_filepath_rewriting(self) -> None:
# Store the pre-modification value, as its use of (back)slashes is platform-dependent.
image_filepath_before = str(bpy.data.images['pack.png'].filepath)
def visit_path_fn(owner_id: bpy.types.ID, path: str, _meta: None) -> str | None:
return "//{}-rewritten.blend".format(owner_id.name)
bpy.data.file_path_foreach(visit_path_fn)
libs = bpy.data.libraries
self.assertEqual(libs['direct_linked_A'].filepath, "//direct_linked_A-rewritten.blend")
self.assertEqual(libs['direct_linked_B'].filepath, "//direct_linked_B-rewritten.blend")
self.assertEqual(libs['indirect_datablocks'].filepath, "//indirect_datablocks-rewritten.blend")
self.assertEqual(
bpy.data.images['pack.png'].filepath,
image_filepath_before,
"Packed file should not have changed")
def test_exception_passing(self) -> None:
"""Python exceptions in the callback function should be raised by file_path_foreach()."""
# Any Python exception should work, not just built-in ones.
class CustomException(Exception):
pass
def visit_path_fn(_owner_id: bpy.types.ID, _path: str, meta: None) -> str | None:
raise CustomException("arg0", 1, "arg2")
try:
bpy.data.file_path_foreach(visit_path_fn)
except CustomException as ex:
self.assertEqual(("arg0", 1, "arg2"), ex.args, "Parameters passed to the exception should be retained")
else:
self.fail("Expected exception not thrown")
def test_meta_parameter(self) -> None:
def visit_path_fn(_owner_id: bpy.types.ID, _path: str, meta: None) -> str | None:
# This is proven to work by the `test_exception_passing()` test above.
self.assertIsNone(
meta,
"The meta parameter is expected to be None; this test is expected to fail "
"once the metadata feature is actually getting implemented, and should then "
"be replaced with a proper test.")
bpy.data.file_path_foreach(visit_path_fn)
@staticmethod
def _file_path_foreach(
subtypes: set[str] | None = None,
visit_keys: set[str] | None = None,
flags: set[str] | None = None,
) -> set[tuple[bpy.types.ID, Path]]:
"""Call bpy.data.file_path_foreach(), returning the visited paths as set of (datablock, path) tuples.
A set is used because the order of visiting is not relevant.
"""
visisted_paths: set[tuple[bpy.types.ID, Path]] = set()
def visit_path_fn(owner_id: bpy.types.ID, path: str, _meta: None) -> str | None:
abspath = Path(str(bpy.path.abspath(path, library=owner_id.library)))
visisted_paths.add((owner_id, abspath))
# Dynamically build the keyword arguments, because the None values are not
# valid, and should be encoded as not having the kwarg.
kwargs = {}
if subtypes is not None:
kwargs['subtypes'] = subtypes
if visit_keys is not None:
kwargs['visit_keys'] = visit_keys
if flags is not None:
kwargs['flags'] = flags
bpy.data.file_path_foreach(visit_path_fn, **kwargs)
return visisted_paths
TESTS = (
TestBlendUserMap,
TestBlendFilePathMap,
TestBlendFilePathForeach,
)

View File

@@ -39,8 +39,14 @@ class TestHelper(unittest.TestCase):
if not inst_attr_id.startswith("test_"):
continue
inst_attr = getattr(self, inst_attr_id)
if callable(inst_attr):
if not callable(inst_attr):
continue
self.setUp()
try:
inst_attr()
finally:
self.tearDown()
class TestBlendLibLinkHelper(TestHelper):