OpenEXR: Multi-part writing

New "Interleave" option in image format settings, enabled by default in old
files and disabled by default in new files to write multi-part files.

By not storing all channels interleaved, it is possible for other applications
to load individual passes more efficiently. This might also work better with
compression.

We follow the OpenEXR docs in that the channel name is the full
Layer.Pass.Channel rather than just Channel. Some other renderers
(e.g. Houdini Karma) write just the channel and that is what our reading
code assumed so far.

I've verified that Nuke can read the multipart EXR files produced by Blender,
including multiview and cryptomatte.

Fix #123727

Pull Request: https://projects.blender.org/blender/blender/pulls/146650

source/blender/blenkernel/intern/image_format.cc

@@ -785,6 +785,9 @@ void BKE_image_format_to_imbuf(ImBuf *ibuf, const ImageFormatData *imf)
     }
     ibuf->foptions.flag |= (imf->exr_codec & OPENEXR_CODEC_MASK);
     ibuf->foptions.quality = quality;
+    if (imf->exr_flag & R_IMF_EXR_FLAG_MULTIPART) {
+      ibuf->foptions.flag |= OPENEXR_MULTIPART;
+    }
   }
 #endif
 #ifdef WITH_IMAGE_CINEON
@@ -986,6 +989,9 @@ void BKE_image_format_from_imbuf(ImageFormatData *im_format, const ImBuf *imbuf)
     if (exr_codec < R_IMF_EXR_CODEC_MAX) {
       im_format->exr_codec = exr_codec;
     }
+    if (custom_flags & OPENEXR_MULTIPART) {
+      im_format->exr_flag |= R_IMF_EXR_FLAG_MULTIPART;
+    }
   }
 #endif
 

source/blender/blenkernel/intern/image_save.cc

@@ -21,6 +21,7 @@
 #include "BLT_translation.hh"
 
 #include "DNA_image_types.h"
+#include "DNA_scene_types.h"
 
 #include "MEM_guardedalloc.h"
 
@@ -871,7 +872,8 @@ bool BKE_image_render_write_exr(ReportList *reports,
                                 const char *view,
                                 int layer)
 {
-  ExrHandle *exrhandle = IMB_exr_get_handle();
+  const int write_multipart = (imf ? imf->exr_flag & R_IMF_EXR_FLAG_MULTIPART : true);
+  ExrHandle *exrhandle = IMB_exr_get_handle(write_multipart);
   const bool multi_layer = !(imf && imf->imtype == R_IMF_IMTYPE_OPENEXR);
 
   /* Write first layer if not multilayer and no layer was specified. */

source/blender/blenloader/intern/versioning_defaults.cc

@@ -388,6 +388,8 @@ static void blo_update_defaults_scene(Main *bmain, Scene *scene)
   STRNCPY_UTF8(scene->r.engine, RE_engine_id_BLENDER_EEVEE);
 
   scene->r.cfra = 1.0f;
+  scene->r.im_format.exr_flag |= R_IMF_EXR_FLAG_MULTIPART;
+  scene->r.bake.im_format.exr_flag |= R_IMF_EXR_FLAG_MULTIPART;
 
   /* Don't enable compositing nodes. */
   if (scene->nodetree) {

source/blender/editors/space_image/image_buttons.cc

@@ -1031,6 +1031,9 @@ void uiTemplateImageSettings(uiLayout *layout,
       col->prop(imfptr, "quality", UI_ITEM_NONE, std::nullopt, ICON_NONE);
     }
   }
+  if (imf->imtype == R_IMF_IMTYPE_MULTILAYER) {
+    col->prop(imfptr, "use_exr_interleave", UI_ITEM_NONE, std::nullopt, ICON_NONE);
+  }
 
   if (is_render_out && ELEM(imf->imtype, R_IMF_IMTYPE_OPENEXR, R_IMF_IMTYPE_MULTILAYER)) {
     col->prop(imfptr, "use_preview", UI_ITEM_NONE, std::nullopt, ICON_NONE);

source/blender/imbuf/IMB_imbuf_types.hh

@@ -42,6 +42,7 @@ using ColorSpace = blender::ocio::ColorSpace;
  */
 
 #define OPENEXR_HALF (1 << 8)
+#define OPENEXR_MULTIPART (1 << 9)
 /* Lowest bits of foptions.flag / exr_codec contain actual codec enum. */
 #define OPENEXR_CODEC_MASK (0xF)
 

source/blender/imbuf/IMB_openexr.hh

@@ -23,11 +23,11 @@
 struct StampData;
 struct ExrHandle;
 
-ExrHandle *IMB_exr_get_handle();
+ExrHandle *IMB_exr_get_handle(bool write_multipart = false);
 
 /**
  * Add multiple channels to EXR file.
- * The number of channels is determined by channelnames.size() without
+ * The number of channels is determined by channelnames.size() with
  * each character a channel name.
  * Layer and pass name, and view name are optional.
  */

source/blender/imbuf/intern/openexr/openexr_api.cpp

@@ -769,8 +769,9 @@ bool imb_save_openexr(ImBuf *ibuf, const char *filepath, int flags)
 
 /* flattened out channel */
 struct ExrChannel {
-  /* Number of the part. */
-  int part_number;
+  /* Name and number of the part. */
+  std::string part_name;
+  int part_number = 0;
 
   /* Full name of the chanel. */
   std::string name;
@@ -816,9 +817,11 @@ struct ExrHandle {
   MultiPartInputFile *ifile = nullptr;
 
   OFileStream *ofile_stream = nullptr;
-  MultiPartOutputFile *ofile = nullptr;
+  MultiPartOutputFile *mpofile = nullptr;
+  OutputFile *ofile = nullptr;
 
-  bool write_multichannel = false;
+  bool write_multipart = false;
+  bool has_layer_pass_names = false;
 
   int tilex = 0, tiley = 0;
   int width = 0, height = 0;
@@ -836,9 +839,11 @@ static blender::Vector<ExrChannel> exr_channels_in_multi_part_file(const MultiPa
 
 /* ********************** */
 
-ExrHandle *IMB_exr_get_handle()
+ExrHandle *IMB_exr_get_handle(const bool write_multipart)
 {
-  return MEM_new<ExrHandle>("ExrHandle");
+  ExrHandle *handle = MEM_new<ExrHandle>("ExrHandle");
+  handle->write_multipart = write_multipart;
+  return handle;
 }
 
 /* multiview functions */
@@ -899,18 +904,29 @@ void IMB_exr_add_channels(ExrHandle *handle,
                           float *rect,
                           bool use_half_float)
 {
-  handle->channels.append_as();
-  ExrChannel &echan = handle->channels.last();
+  /* For multipart, part name includes view since part names must be unique. */
+  std::string part_name;
+  if (handle->write_multipart) {
+    part_name = layerpassname;
+    if (!viewname.is_empty()) {
+      if (part_name.empty()) {
+        part_name = viewname;
+      }
+      else {
+        part_name = part_name + "-" + viewname;
+      }
+    }
+  }
 
   /* If there are layer and pass names, we will write Blender multichannel metadata. */
   if (!layerpassname.is_empty()) {
-    handle->write_multichannel = true;
+    handle->has_layer_pass_names = true;
   }
 
   for (size_t channel = 0; channel < channelnames.size(); channel++) {
     /* Full channel name including view (when not using multipart) and channel. */
     std::string full_name = layerpassname;
-    if (!viewname.is_empty()) {
+    if (!handle->write_multipart && !viewname.is_empty()) {
       if (full_name.empty()) {
         full_name = viewname;
       }
@@ -925,8 +941,12 @@ void IMB_exr_add_channels(ExrHandle *handle,
       full_name = full_name + "." + channelnames[channel];
     }
 
+    handle->channels.append_as();
+    ExrChannel &echan = handle->channels.last();
+
     echan.name = full_name;
     echan.internal_name = full_name;
+    echan.part_name = part_name;
     echan.view = viewname;
 
     echan.xstride = xstride;
@@ -944,10 +964,10 @@ static void openexr_header_metadata_multi(ExrHandle *handle,
                                           const StampData *stamp)
 {
   openexr_header_metadata_global(&header, nullptr, ppm);
-  if (handle->write_multichannel) {
+  if (handle->has_layer_pass_names) {
     header.insert("BlenderMultiChannel", StringAttribute("Blender V2.55.1 and newer"));
   }
-  if (!handle->views.empty() && !handle->views[0].empty()) {
+  if (!handle->write_multipart && !handle->views.empty() && !handle->views[0].empty()) {
     addMultiView(header, handle->views);
   }
   BKE_stamp_info_callback(
@@ -963,44 +983,87 @@ bool IMB_exr_begin_write(ExrHandle *handle,
                          int quality,
                          const StampData *stamp)
 {
+  if (handle->channels.is_empty()) {
+    CLOG_ERROR(&LOG, "Attempt to save MultiLayer without layers.");
+    return false;
+  }
+
   Header header(width, height);
 
   handle->width = width;
   handle->height = height;
 
   openexr_header_compression(&header, compress, quality);
-  openexr_header_metadata_multi(handle, header, ppm, stamp);
+
+  blender::Vector<Header> part_headers;
+
+  blender::StringRefNull last_part_name;
 
   for (const ExrChannel &echan : handle->channels) {
-    header.channels().insert(echan.name, Channel(echan.use_half_float ? Imf::HALF : Imf::FLOAT));
+    if (part_headers.is_empty() || last_part_name != echan.part_name) {
+      Header part_header = header;
+
+      /* When writing multipart, set name, view and type in each part. */
+      if (handle->write_multipart) {
+        part_header.setName(echan.part_name);
+        if (!echan.view.empty()) {
+          part_header.insert("view", StringAttribute(echan.view));
+        }
+        part_header.insert("type", StringAttribute(SCANLINEIMAGE));
+      }
+
+      /* Store global metadata in the first header only. Large metadata like cryptomatte would
+       * be bad to duplicate many times. */
+      if (part_headers.is_empty()) {
+        openexr_header_metadata_multi(handle, part_header, ppm, stamp);
+      }
+
+      part_headers.append(std::move(part_header));
+      last_part_name = echan.part_name;
+    }
+
+    part_headers.last().channels().insert(echan.name,
+                                          Channel(echan.use_half_float ? Imf::HALF : Imf::FLOAT));
   }
 
+  BLI_assert(!(handle->write_multipart == false && part_headers.size() > 1));
+
   /* Avoid crash/abort when we don't have permission to write here. */
   /* Manually create `ofstream`, so we can handle UTF8 file-paths on windows. */
   try {
     handle->ofile_stream = new OFileStream(filepath);
-    handle->ofile = new MultiPartOutputFile(*(handle->ofile_stream), &header, 1);
+    if (handle->write_multipart) {
+      handle->mpofile = new MultiPartOutputFile(
+          *(handle->ofile_stream), part_headers.data(), part_headers.size());
+    }
+    else {
+      handle->ofile = new OutputFile(*(handle->ofile_stream), part_headers[0]);
+    }
   }
   catch (const std::exception &exc) {
     CLOG_ERROR(&LOG, "%s: %s", __func__, exc.what());
 
     delete handle->ofile;
+    delete handle->mpofile;
     delete handle->ofile_stream;
 
     handle->ofile = nullptr;
+    handle->mpofile = nullptr;
     handle->ofile_stream = nullptr;
   }
   catch (...) { /* Catch-all for edge cases or compiler bugs. */
     CLOG_ERROR(&LOG, "Unknown error in %s", __func__);
 
     delete handle->ofile;
+    delete handle->mpofile;
     delete handle->ofile_stream;
 
     handle->ofile = nullptr;
+    handle->mpofile = nullptr;
     handle->ofile_stream = nullptr;
   }
 
-  return (handle->ofile != nullptr);
+  return (handle->ofile != nullptr || handle->mpofile != nullptr);
 }
 
 bool IMB_exr_begin_read(
@@ -1070,13 +1133,14 @@ void IMB_exr_write_channels(ExrHandle *handle)
   }
 
   const size_t num_pixels = size_t(handle->width) * handle->height;
+  const size_t num_parts = (handle->mpofile) ? handle->mpofile->parts() : 1;
 
-  {
-    size_t part_num = 0;
+  for (size_t part_num = 0; part_num < num_parts; part_num++) {
+    const std::string &part_id = (handle->mpofile) ? handle->mpofile->header(part_num).name() : "";
     /* We allocate temporary storage for half pixels for all the channels at once. */
     int num_half_channels = 0;
     for (const ExrChannel &echan : handle->channels) {
-      if (echan.use_half_float) {
+      if (echan.part_name == part_id && echan.use_half_float) {
         num_half_channels++;
       }
     }
@@ -1092,6 +1156,10 @@ void IMB_exr_write_channels(ExrHandle *handle)
 
     for (const ExrChannel &echan : handle->channels) {
       /* Writing starts from last scan-line, stride negative. */
+      if (echan.part_name != part_id) {
+        continue;
+      }
+
       if (echan.use_half_float) {
         const float *rect = echan.rect;
         half *cur = current_rect_half;
@@ -1114,10 +1182,16 @@ void IMB_exr_write_channels(ExrHandle *handle)
       }
     }
 
-    OutputPart part(*handle->ofile, part_num);
-    part.setFrameBuffer(frameBuffer);
     try {
-      part.writePixels(handle->height);
+      if (handle->mpofile) {
+        OutputPart part(*handle->mpofile, part_num);
+        part.setFrameBuffer(frameBuffer);
+        part.writePixels(handle->height);
+      }
+      else {
+        handle->ofile->setFrameBuffer(frameBuffer);
+        handle->ofile->writePixels(handle->height);
+      }
     }
     catch (const std::exception &exc) {
       CLOG_ERROR(&LOG, "%s: %s", __func__, exc.what());
@@ -1256,13 +1330,9 @@ void IMB_exr_close(ExrHandle *handle)
   delete handle->ifile;
   delete handle->ifile_stream;
   delete handle->ofile;
+  delete handle->mpofile;
   delete handle->ofile_stream;
 
-  handle->ifile = nullptr;
-  handle->ifile_stream = nullptr;
-  handle->ofile = nullptr;
-  handle->ofile_stream = nullptr;
-
   MEM_delete(handle);
 }
 

source/blender/imbuf/intern/openexr/openexr_stub.cpp

@@ -9,7 +9,7 @@
 #include "BLI_string_ref.hh"
 #include "IMB_openexr.hh"
 
-ExrHandle *IMB_exr_get_handle()
+ExrHandle *IMB_exr_get_handle(bool /*write_multipart*/)
 {
   return nullptr;
 }

source/blender/makesdna/DNA_scene_defaults.h

@@ -23,6 +23,7 @@
     .depth = R_IMF_CHAN_DEPTH_8, \
     .quality = 90, \
     .compress = 15, \
+    .exr_flag = R_IMF_EXR_FLAG_MULTIPART, \
   }
 
 #define _DNA_DEFAULT_BakeData \

source/blender/makesdna/DNA_scene_types.h

@@ -481,6 +481,7 @@ typedef struct ImageFormatData {
 
   /** OpenEXR: R_IMF_EXR_CODEC_* values in low OPENEXR_CODEC_MASK bits. */
   char exr_codec;
+  char exr_flag;
 
   /** Jpeg2000. */
   char jp2_flag;
@@ -491,19 +492,18 @@ typedef struct ImageFormatData {
 
   /** CINEON. */
   char cineon_flag;
+  char _pad[3];
   short cineon_white, cineon_black;
   float cineon_gamma;
 
-  char _pad[3];
-
   /** Multi-view. */
-  char views_format;
   Stereo3dFormat stereo3d_format;
+  char views_format;
 
   /* Color management members. */
 
   char color_management;
-  char _pad1[7];
+  char _pad1[6];
   ColorManagedViewSettings view_settings;
   ColorManagedDisplaySettings display_settings;
   ColorManagedColorspaceSettings linear_colorspace_settings;
@@ -602,6 +602,11 @@ enum {
   R_IMF_EXR_CODEC_MAX = 10,
 };
 
+/** #ImageFormatData::exr_flag */
+enum {
+  R_IMF_EXR_FLAG_MULTIPART = 1 << 0,
+};
+
 /** #ImageFormatData::jp2_flag */
 enum {
   /** When disabled use RGB. */

source/blender/makesrna/intern/rna_scene.cc

@@ -6391,6 +6391,15 @@ static void rna_def_scene_image_format_data(BlenderRNA *brna)
   RNA_def_property_enum_funcs(prop, nullptr, nullptr, "rna_ImageFormatSettings_exr_codec_itemf");
   RNA_def_property_ui_text(prop, "Codec", "Compression codec settings for OpenEXR");
   RNA_def_property_update(prop, NC_SCENE | ND_RENDER_OPTIONS, nullptr);
+
+  prop = RNA_def_property(srna, "use_exr_interleave", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_negative_sdna(prop, nullptr, "exr_flag", R_IMF_EXR_FLAG_MULTIPART);
+  RNA_def_property_ui_text(
+      prop,
+      "Interleave",
+      "Use legacy interleaved storage of views, layers and passes for compatibility with "
+      "applications that do not support more efficient multi-part OpenEXR files.");
+  RNA_def_property_update(prop, NC_SCENE | ND_RENDER_OPTIONS, nullptr);
 #  endif
 
 #  ifdef WITH_IMAGE_OPENJPEG