From be1619f4ad1cc80703636d3a0650a62f85e57e01 Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Thu, 25 Sep 2025 10:48:36 +0200 Subject: [PATCH] OpenEXR: Multi-part writing New "Interleave" option in image format settings, enabled by default in old files and disabled by default in new files to write multi-part files. By not storing all channels interleaved, it is possible for other applications to load individual passes more efficiently. This might also work better with compression. We follow the OpenEXR docs in that the channel name is the full Layer.Pass.Channel rather than just Channel. Some other renderers (e.g. Houdini Karma) write just the channel and that is what our reading code assumed so far. I've verified that Nuke can read the multipart EXR files produced by Blender, including multiview and cryptomatte. Fix #123727 Pull Request: https://projects.blender.org/blender/blender/pulls/146650 --- .../blender/blenkernel/intern/image_format.cc | 6 + .../blender/blenkernel/intern/image_save.cc | 4 +- .../blenloader/intern/versioning_defaults.cc | 2 + .../editors/space_image/image_buttons.cc | 3 + source/blender/imbuf/IMB_imbuf_types.hh | 1 + source/blender/imbuf/IMB_openexr.hh | 4 +- .../imbuf/intern/openexr/openexr_api.cpp | 124 ++++++++++++++---- .../imbuf/intern/openexr/openexr_stub.cpp | 2 +- source/blender/makesdna/DNA_scene_defaults.h | 1 + source/blender/makesdna/DNA_scene_types.h | 13 +- source/blender/makesrna/intern/rna_scene.cc | 9 ++ .../file_output/exr_multipart.blend | 3 + .../multilayer_multipart0001_L.exr | 3 + .../multilayer_multipart0001_R.exr | 3 + .../multilayer_singlepart0001_L.exr | 3 + .../multilayer_singlepart0001_R.exr | 3 + .../exr_multipart/multiview_multipart0001.exr | 3 + .../multiview_singlepart0001.exr | 3 + 18 files changed, 155 insertions(+), 35 deletions(-) create mode 100644 tests/files/compositor/file_output/exr_multipart.blend create mode 100644 tests/files/compositor/file_output/exr_multipart/multilayer_multipart0001_L.exr create mode 100644 tests/files/compositor/file_output/exr_multipart/multilayer_multipart0001_R.exr create mode 100644 tests/files/compositor/file_output/exr_multipart/multilayer_singlepart0001_L.exr create mode 100644 tests/files/compositor/file_output/exr_multipart/multilayer_singlepart0001_R.exr create mode 100644 tests/files/compositor/file_output/exr_multipart/multiview_multipart0001.exr create mode 100644 tests/files/compositor/file_output/exr_multipart/multiview_singlepart0001.exr diff --git a/source/blender/blenkernel/intern/image_format.cc b/source/blender/blenkernel/intern/image_format.cc index 7dd7ab73eb0..35412bbfbb9 100644 --- a/source/blender/blenkernel/intern/image_format.cc +++ b/source/blender/blenkernel/intern/image_format.cc @@ -785,6 +785,9 @@ void BKE_image_format_to_imbuf(ImBuf *ibuf, const ImageFormatData *imf) } ibuf->foptions.flag |= (imf->exr_codec & OPENEXR_CODEC_MASK); ibuf->foptions.quality = quality; + if (imf->exr_flag & R_IMF_EXR_FLAG_MULTIPART) { + ibuf->foptions.flag |= OPENEXR_MULTIPART; + } } #endif #ifdef WITH_IMAGE_CINEON @@ -986,6 +989,9 @@ void BKE_image_format_from_imbuf(ImageFormatData *im_format, const ImBuf *imbuf) if (exr_codec < R_IMF_EXR_CODEC_MAX) { im_format->exr_codec = exr_codec; } + if (custom_flags & OPENEXR_MULTIPART) { + im_format->exr_flag |= R_IMF_EXR_FLAG_MULTIPART; + } } #endif diff --git a/source/blender/blenkernel/intern/image_save.cc b/source/blender/blenkernel/intern/image_save.cc index 972640cec37..bdf1e775a16 100644 --- a/source/blender/blenkernel/intern/image_save.cc +++ b/source/blender/blenkernel/intern/image_save.cc @@ -21,6 +21,7 @@ #include "BLT_translation.hh" #include "DNA_image_types.h" +#include "DNA_scene_types.h" #include "MEM_guardedalloc.h" @@ -871,7 +872,8 @@ bool BKE_image_render_write_exr(ReportList *reports, const char *view, int layer) { - ExrHandle *exrhandle = IMB_exr_get_handle(); + const int write_multipart = (imf ? imf->exr_flag & R_IMF_EXR_FLAG_MULTIPART : true); + ExrHandle *exrhandle = IMB_exr_get_handle(write_multipart); const bool multi_layer = !(imf && imf->imtype == R_IMF_IMTYPE_OPENEXR); /* Write first layer if not multilayer and no layer was specified. */ diff --git a/source/blender/blenloader/intern/versioning_defaults.cc b/source/blender/blenloader/intern/versioning_defaults.cc index 4c9475b94fa..3b20664581e 100644 --- a/source/blender/blenloader/intern/versioning_defaults.cc +++ b/source/blender/blenloader/intern/versioning_defaults.cc @@ -388,6 +388,8 @@ static void blo_update_defaults_scene(Main *bmain, Scene *scene) STRNCPY_UTF8(scene->r.engine, RE_engine_id_BLENDER_EEVEE); scene->r.cfra = 1.0f; + scene->r.im_format.exr_flag |= R_IMF_EXR_FLAG_MULTIPART; + scene->r.bake.im_format.exr_flag |= R_IMF_EXR_FLAG_MULTIPART; /* Don't enable compositing nodes. */ if (scene->nodetree) { diff --git a/source/blender/editors/space_image/image_buttons.cc b/source/blender/editors/space_image/image_buttons.cc index 49c8862aa7c..bb6a7640ed3 100644 --- a/source/blender/editors/space_image/image_buttons.cc +++ b/source/blender/editors/space_image/image_buttons.cc @@ -1031,6 +1031,9 @@ void uiTemplateImageSettings(uiLayout *layout, col->prop(imfptr, "quality", UI_ITEM_NONE, std::nullopt, ICON_NONE); } } + if (imf->imtype == R_IMF_IMTYPE_MULTILAYER) { + col->prop(imfptr, "use_exr_interleave", UI_ITEM_NONE, std::nullopt, ICON_NONE); + } if (is_render_out && ELEM(imf->imtype, R_IMF_IMTYPE_OPENEXR, R_IMF_IMTYPE_MULTILAYER)) { col->prop(imfptr, "use_preview", UI_ITEM_NONE, std::nullopt, ICON_NONE); diff --git a/source/blender/imbuf/IMB_imbuf_types.hh b/source/blender/imbuf/IMB_imbuf_types.hh index ad740842d03..a8aad854b5b 100644 --- a/source/blender/imbuf/IMB_imbuf_types.hh +++ b/source/blender/imbuf/IMB_imbuf_types.hh @@ -42,6 +42,7 @@ using ColorSpace = blender::ocio::ColorSpace; */ #define OPENEXR_HALF (1 << 8) +#define OPENEXR_MULTIPART (1 << 9) /* Lowest bits of foptions.flag / exr_codec contain actual codec enum. */ #define OPENEXR_CODEC_MASK (0xF) diff --git a/source/blender/imbuf/IMB_openexr.hh b/source/blender/imbuf/IMB_openexr.hh index 3886e64d147..e7c32e421d1 100644 --- a/source/blender/imbuf/IMB_openexr.hh +++ b/source/blender/imbuf/IMB_openexr.hh @@ -23,11 +23,11 @@ struct StampData; struct ExrHandle; -ExrHandle *IMB_exr_get_handle(); +ExrHandle *IMB_exr_get_handle(bool write_multipart = false); /** * Add multiple channels to EXR file. - * The number of channels is determined by channelnames.size() without + * The number of channels is determined by channelnames.size() with * each character a channel name. * Layer and pass name, and view name are optional. */ diff --git a/source/blender/imbuf/intern/openexr/openexr_api.cpp b/source/blender/imbuf/intern/openexr/openexr_api.cpp index a92f4548701..821610481f0 100644 --- a/source/blender/imbuf/intern/openexr/openexr_api.cpp +++ b/source/blender/imbuf/intern/openexr/openexr_api.cpp @@ -769,8 +769,9 @@ bool imb_save_openexr(ImBuf *ibuf, const char *filepath, int flags) /* flattened out channel */ struct ExrChannel { - /* Number of the part. */ - int part_number; + /* Name and number of the part. */ + std::string part_name; + int part_number = 0; /* Full name of the chanel. */ std::string name; @@ -816,9 +817,11 @@ struct ExrHandle { MultiPartInputFile *ifile = nullptr; OFileStream *ofile_stream = nullptr; - MultiPartOutputFile *ofile = nullptr; + MultiPartOutputFile *mpofile = nullptr; + OutputFile *ofile = nullptr; - bool write_multichannel = false; + bool write_multipart = false; + bool has_layer_pass_names = false; int tilex = 0, tiley = 0; int width = 0, height = 0; @@ -836,9 +839,11 @@ static blender::Vector exr_channels_in_multi_part_file(const MultiPa /* ********************** */ -ExrHandle *IMB_exr_get_handle() +ExrHandle *IMB_exr_get_handle(const bool write_multipart) { - return MEM_new("ExrHandle"); + ExrHandle *handle = MEM_new("ExrHandle"); + handle->write_multipart = write_multipart; + return handle; } /* multiview functions */ @@ -899,18 +904,29 @@ void IMB_exr_add_channels(ExrHandle *handle, float *rect, bool use_half_float) { - handle->channels.append_as(); - ExrChannel &echan = handle->channels.last(); + /* For multipart, part name includes view since part names must be unique. */ + std::string part_name; + if (handle->write_multipart) { + part_name = layerpassname; + if (!viewname.is_empty()) { + if (part_name.empty()) { + part_name = viewname; + } + else { + part_name = part_name + "-" + viewname; + } + } + } /* If there are layer and pass names, we will write Blender multichannel metadata. */ if (!layerpassname.is_empty()) { - handle->write_multichannel = true; + handle->has_layer_pass_names = true; } for (size_t channel = 0; channel < channelnames.size(); channel++) { /* Full channel name including view (when not using multipart) and channel. */ std::string full_name = layerpassname; - if (!viewname.is_empty()) { + if (!handle->write_multipart && !viewname.is_empty()) { if (full_name.empty()) { full_name = viewname; } @@ -925,8 +941,12 @@ void IMB_exr_add_channels(ExrHandle *handle, full_name = full_name + "." + channelnames[channel]; } + handle->channels.append_as(); + ExrChannel &echan = handle->channels.last(); + echan.name = full_name; echan.internal_name = full_name; + echan.part_name = part_name; echan.view = viewname; echan.xstride = xstride; @@ -944,10 +964,10 @@ static void openexr_header_metadata_multi(ExrHandle *handle, const StampData *stamp) { openexr_header_metadata_global(&header, nullptr, ppm); - if (handle->write_multichannel) { + if (handle->has_layer_pass_names) { header.insert("BlenderMultiChannel", StringAttribute("Blender V2.55.1 and newer")); } - if (!handle->views.empty() && !handle->views[0].empty()) { + if (!handle->write_multipart && !handle->views.empty() && !handle->views[0].empty()) { addMultiView(header, handle->views); } BKE_stamp_info_callback( @@ -963,44 +983,87 @@ bool IMB_exr_begin_write(ExrHandle *handle, int quality, const StampData *stamp) { + if (handle->channels.is_empty()) { + CLOG_ERROR(&LOG, "Attempt to save MultiLayer without layers."); + return false; + } + Header header(width, height); handle->width = width; handle->height = height; openexr_header_compression(&header, compress, quality); - openexr_header_metadata_multi(handle, header, ppm, stamp); + + blender::Vector
part_headers; + + blender::StringRefNull last_part_name; for (const ExrChannel &echan : handle->channels) { - header.channels().insert(echan.name, Channel(echan.use_half_float ? Imf::HALF : Imf::FLOAT)); + if (part_headers.is_empty() || last_part_name != echan.part_name) { + Header part_header = header; + + /* When writing multipart, set name, view and type in each part. */ + if (handle->write_multipart) { + part_header.setName(echan.part_name); + if (!echan.view.empty()) { + part_header.insert("view", StringAttribute(echan.view)); + } + part_header.insert("type", StringAttribute(SCANLINEIMAGE)); + } + + /* Store global metadata in the first header only. Large metadata like cryptomatte would + * be bad to duplicate many times. */ + if (part_headers.is_empty()) { + openexr_header_metadata_multi(handle, part_header, ppm, stamp); + } + + part_headers.append(std::move(part_header)); + last_part_name = echan.part_name; + } + + part_headers.last().channels().insert(echan.name, + Channel(echan.use_half_float ? Imf::HALF : Imf::FLOAT)); } + BLI_assert(!(handle->write_multipart == false && part_headers.size() > 1)); + /* Avoid crash/abort when we don't have permission to write here. */ /* Manually create `ofstream`, so we can handle UTF8 file-paths on windows. */ try { handle->ofile_stream = new OFileStream(filepath); - handle->ofile = new MultiPartOutputFile(*(handle->ofile_stream), &header, 1); + if (handle->write_multipart) { + handle->mpofile = new MultiPartOutputFile( + *(handle->ofile_stream), part_headers.data(), part_headers.size()); + } + else { + handle->ofile = new OutputFile(*(handle->ofile_stream), part_headers[0]); + } } catch (const std::exception &exc) { CLOG_ERROR(&LOG, "%s: %s", __func__, exc.what()); delete handle->ofile; + delete handle->mpofile; delete handle->ofile_stream; handle->ofile = nullptr; + handle->mpofile = nullptr; handle->ofile_stream = nullptr; } catch (...) { /* Catch-all for edge cases or compiler bugs. */ CLOG_ERROR(&LOG, "Unknown error in %s", __func__); delete handle->ofile; + delete handle->mpofile; delete handle->ofile_stream; handle->ofile = nullptr; + handle->mpofile = nullptr; handle->ofile_stream = nullptr; } - return (handle->ofile != nullptr); + return (handle->ofile != nullptr || handle->mpofile != nullptr); } bool IMB_exr_begin_read( @@ -1070,13 +1133,14 @@ void IMB_exr_write_channels(ExrHandle *handle) } const size_t num_pixels = size_t(handle->width) * handle->height; + const size_t num_parts = (handle->mpofile) ? handle->mpofile->parts() : 1; - { - size_t part_num = 0; + for (size_t part_num = 0; part_num < num_parts; part_num++) { + const std::string &part_id = (handle->mpofile) ? handle->mpofile->header(part_num).name() : ""; /* We allocate temporary storage for half pixels for all the channels at once. */ int num_half_channels = 0; for (const ExrChannel &echan : handle->channels) { - if (echan.use_half_float) { + if (echan.part_name == part_id && echan.use_half_float) { num_half_channels++; } } @@ -1092,6 +1156,10 @@ void IMB_exr_write_channels(ExrHandle *handle) for (const ExrChannel &echan : handle->channels) { /* Writing starts from last scan-line, stride negative. */ + if (echan.part_name != part_id) { + continue; + } + if (echan.use_half_float) { const float *rect = echan.rect; half *cur = current_rect_half; @@ -1114,10 +1182,16 @@ void IMB_exr_write_channels(ExrHandle *handle) } } - OutputPart part(*handle->ofile, part_num); - part.setFrameBuffer(frameBuffer); try { - part.writePixels(handle->height); + if (handle->mpofile) { + OutputPart part(*handle->mpofile, part_num); + part.setFrameBuffer(frameBuffer); + part.writePixels(handle->height); + } + else { + handle->ofile->setFrameBuffer(frameBuffer); + handle->ofile->writePixels(handle->height); + } } catch (const std::exception &exc) { CLOG_ERROR(&LOG, "%s: %s", __func__, exc.what()); @@ -1256,13 +1330,9 @@ void IMB_exr_close(ExrHandle *handle) delete handle->ifile; delete handle->ifile_stream; delete handle->ofile; + delete handle->mpofile; delete handle->ofile_stream; - handle->ifile = nullptr; - handle->ifile_stream = nullptr; - handle->ofile = nullptr; - handle->ofile_stream = nullptr; - MEM_delete(handle); } diff --git a/source/blender/imbuf/intern/openexr/openexr_stub.cpp b/source/blender/imbuf/intern/openexr/openexr_stub.cpp index 3ac66904c22..97c9ae609eb 100644 --- a/source/blender/imbuf/intern/openexr/openexr_stub.cpp +++ b/source/blender/imbuf/intern/openexr/openexr_stub.cpp @@ -9,7 +9,7 @@ #include "BLI_string_ref.hh" #include "IMB_openexr.hh" -ExrHandle *IMB_exr_get_handle() +ExrHandle *IMB_exr_get_handle(bool /*write_multipart*/) { return nullptr; } diff --git a/source/blender/makesdna/DNA_scene_defaults.h b/source/blender/makesdna/DNA_scene_defaults.h index 116ac6e00de..52e4866d013 100644 --- a/source/blender/makesdna/DNA_scene_defaults.h +++ b/source/blender/makesdna/DNA_scene_defaults.h @@ -23,6 +23,7 @@ .depth = R_IMF_CHAN_DEPTH_8, \ .quality = 90, \ .compress = 15, \ + .exr_flag = R_IMF_EXR_FLAG_MULTIPART, \ } #define _DNA_DEFAULT_BakeData \ diff --git a/source/blender/makesdna/DNA_scene_types.h b/source/blender/makesdna/DNA_scene_types.h index 49376905717..35c99ad320e 100644 --- a/source/blender/makesdna/DNA_scene_types.h +++ b/source/blender/makesdna/DNA_scene_types.h @@ -481,6 +481,7 @@ typedef struct ImageFormatData { /** OpenEXR: R_IMF_EXR_CODEC_* values in low OPENEXR_CODEC_MASK bits. */ char exr_codec; + char exr_flag; /** Jpeg2000. */ char jp2_flag; @@ -491,19 +492,18 @@ typedef struct ImageFormatData { /** CINEON. */ char cineon_flag; + char _pad[3]; short cineon_white, cineon_black; float cineon_gamma; - char _pad[3]; - /** Multi-view. */ - char views_format; Stereo3dFormat stereo3d_format; + char views_format; /* Color management members. */ char color_management; - char _pad1[7]; + char _pad1[6]; ColorManagedViewSettings view_settings; ColorManagedDisplaySettings display_settings; ColorManagedColorspaceSettings linear_colorspace_settings; @@ -602,6 +602,11 @@ enum { R_IMF_EXR_CODEC_MAX = 10, }; +/** #ImageFormatData::exr_flag */ +enum { + R_IMF_EXR_FLAG_MULTIPART = 1 << 0, +}; + /** #ImageFormatData::jp2_flag */ enum { /** When disabled use RGB. */ diff --git a/source/blender/makesrna/intern/rna_scene.cc b/source/blender/makesrna/intern/rna_scene.cc index 672467cd885..53c9bed7bf4 100644 --- a/source/blender/makesrna/intern/rna_scene.cc +++ b/source/blender/makesrna/intern/rna_scene.cc @@ -6391,6 +6391,15 @@ static void rna_def_scene_image_format_data(BlenderRNA *brna) RNA_def_property_enum_funcs(prop, nullptr, nullptr, "rna_ImageFormatSettings_exr_codec_itemf"); RNA_def_property_ui_text(prop, "Codec", "Compression codec settings for OpenEXR"); RNA_def_property_update(prop, NC_SCENE | ND_RENDER_OPTIONS, nullptr); + + prop = RNA_def_property(srna, "use_exr_interleave", PROP_BOOLEAN, PROP_NONE); + RNA_def_property_boolean_negative_sdna(prop, nullptr, "exr_flag", R_IMF_EXR_FLAG_MULTIPART); + RNA_def_property_ui_text( + prop, + "Interleave", + "Use legacy interleaved storage of views, layers and passes for compatibility with " + "applications that do not support more efficient multi-part OpenEXR files."); + RNA_def_property_update(prop, NC_SCENE | ND_RENDER_OPTIONS, nullptr); # endif # ifdef WITH_IMAGE_OPENJPEG diff --git a/tests/files/compositor/file_output/exr_multipart.blend b/tests/files/compositor/file_output/exr_multipart.blend new file mode 100644 index 00000000000..c9bde8e4d54 --- /dev/null +++ b/tests/files/compositor/file_output/exr_multipart.blend @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e1075913c94d103c50b0c549a7ac50525e197d6258f28e808f552057858fa14 +size 128702 diff --git a/tests/files/compositor/file_output/exr_multipart/multilayer_multipart0001_L.exr b/tests/files/compositor/file_output/exr_multipart/multilayer_multipart0001_L.exr new file mode 100644 index 00000000000..140908862e6 --- /dev/null +++ b/tests/files/compositor/file_output/exr_multipart/multilayer_multipart0001_L.exr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38f4347bf148b3a881daa1a0e903086e26c337583e7f9bb6093bae7e544a6d11 +size 23733 diff --git a/tests/files/compositor/file_output/exr_multipart/multilayer_multipart0001_R.exr b/tests/files/compositor/file_output/exr_multipart/multilayer_multipart0001_R.exr new file mode 100644 index 00000000000..f8e40dcb8d0 --- /dev/null +++ b/tests/files/compositor/file_output/exr_multipart/multilayer_multipart0001_R.exr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:635186ddb32909fcd6f1275d0fb725bd5babb126fae184f0fd6c8ed9aead4d72 +size 23733 diff --git a/tests/files/compositor/file_output/exr_multipart/multilayer_singlepart0001_L.exr b/tests/files/compositor/file_output/exr_multipart/multilayer_singlepart0001_L.exr new file mode 100644 index 00000000000..ab33a568d00 --- /dev/null +++ b/tests/files/compositor/file_output/exr_multipart/multilayer_singlepart0001_L.exr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3780497e510bab1b5bc7bf111bb678b2a9d9c4b199225165dda6cbd2b8caa932 +size 21082 diff --git a/tests/files/compositor/file_output/exr_multipart/multilayer_singlepart0001_R.exr b/tests/files/compositor/file_output/exr_multipart/multilayer_singlepart0001_R.exr new file mode 100644 index 00000000000..5154e1f8fa8 --- /dev/null +++ b/tests/files/compositor/file_output/exr_multipart/multilayer_singlepart0001_R.exr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a64de347b64300333ec6c2e2a9c37b3fc5f0fa79673698b2d65eba20fe9385d9 +size 21082 diff --git a/tests/files/compositor/file_output/exr_multipart/multiview_multipart0001.exr b/tests/files/compositor/file_output/exr_multipart/multiview_multipart0001.exr new file mode 100644 index 00000000000..fd6160fbe4e --- /dev/null +++ b/tests/files/compositor/file_output/exr_multipart/multiview_multipart0001.exr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f09afce7df1cd251f9a251aa360c4ea95af0bfbe948ed43e42686c6dd153438e +size 47547 diff --git a/tests/files/compositor/file_output/exr_multipart/multiview_singlepart0001.exr b/tests/files/compositor/file_output/exr_multipart/multiview_singlepart0001.exr new file mode 100644 index 00000000000..99bfddb41ee --- /dev/null +++ b/tests/files/compositor/file_output/exr_multipart/multiview_singlepart0001.exr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33ecbf237abe33eda557bbdaaf31f68b1105dd7ad41399440093533ca51d2303 +size 41563