Vulkan: Implement native handles for pixel buffers

* Pixel buffer is always allocated with export and dedicated memory flags.
* Returns an opaque file descriptor (Unix) or handle (Windows).
* Native handle now includes memory size as it may be slightly bigger
  than the requested size.

Pull Request: https://projects.blender.org/blender/blender/pulls/137363

intern/cycles/blender/display_driver.cpp

@@ -2,6 +2,7 @@
  *
  * SPDX-License-Identifier: Apache-2.0 */
 
+#include "GPU_context.hh"
 #include "GPU_immediate.hh"
 #include "GPU_platform.hh"
 #include "GPU_shader.hh"
@@ -623,10 +624,16 @@ BlenderDisplayDriver::GraphicsInterop BlenderDisplayDriver::graphics_interop_get
 {
   GraphicsInterop interop_dst;
 
+  if (GPU_backend_get_type() != GPU_BACKEND_OPENGL) {
+    return interop_dst;
+  }
+
+  GPUPixelBufferNativeHandle handle = GPU_pixel_buffer_get_native_handle(
+      tiles_->current_tile.buffer_object.gpu_pixel_buffer);
+
   interop_dst.buffer_width = tiles_->current_tile.buffer_object.width;
   interop_dst.buffer_height = tiles_->current_tile.buffer_object.height;
-  interop_dst.opengl_pbo_id = GPU_pixel_buffer_get_native_handle(
-      tiles_->current_tile.buffer_object.gpu_pixel_buffer);
+  interop_dst.opengl_pbo_id = handle.handle;
 
   return interop_dst;
 }

source/blender/gpu/GPU_texture.hh

@@ -1109,7 +1109,12 @@ size_t GPU_pixel_buffer_size(GPUPixelBuffer *pixel_buf);
 /**
  * Return the native handle of the \a pix_buf to use for graphic interoperability registration.
  */
-int64_t GPU_pixel_buffer_get_native_handle(GPUPixelBuffer *pixel_buf);
+struct GPUPixelBufferNativeHandle {
+  int64_t handle = 0;
+  size_t size = 0;
+};
+
+GPUPixelBufferNativeHandle GPU_pixel_buffer_get_native_handle(GPUPixelBuffer *pixel_buf);
 
 /**
  * Update a sub-region of a texture using the data from a #GPUPixelBuffer as source data.

source/blender/gpu/intern/gpu_texture.cc

@@ -999,7 +999,7 @@ size_t GPU_pixel_buffer_size(GPUPixelBuffer *pixel_buf)
   return unwrap(pixel_buf)->get_size();
 }
 
-int64_t GPU_pixel_buffer_get_native_handle(GPUPixelBuffer *pixel_buf)
+GPUPixelBufferNativeHandle GPU_pixel_buffer_get_native_handle(GPUPixelBuffer *pixel_buf)
 {
   return unwrap(pixel_buf)->get_native_handle();
 }

source/blender/gpu/intern/gpu_texture_private.hh

@@ -344,7 +344,7 @@ class PixelBuffer {
 
   virtual void *map() = 0;
   virtual void unmap() = 0;
-  virtual int64_t get_native_handle() = 0;
+  virtual GPUPixelBufferNativeHandle get_native_handle() = 0;
   virtual size_t get_size() = 0;
 };
 

source/blender/gpu/metal/mtl_texture.hh

@@ -491,7 +491,7 @@ class MTLPixelBuffer : public PixelBuffer {
 
   void *map() override;
   void unmap() override;
-  int64_t get_native_handle() override;
+  GPUPixelBufferNativeHandle get_native_handle() override;
   size_t get_size() override;
 
   id<MTLBuffer> get_metal_buffer();

source/blender/gpu/metal/mtl_texture.mm

@@ -2645,13 +2645,18 @@ void MTLPixelBuffer::unmap()
   }
 }
 
-int64_t MTLPixelBuffer::get_native_handle()
+GPUPixelBufferNativeHandle MTLPixelBuffer::get_native_handle()
 {
+  GPUPixelBufferNativeHandle native_handle;
+
   if (buffer_ == nil) {
-    return 0;
+    return native_handle;
   }
 
-  return reinterpret_cast<int64_t>(buffer_);
+  native_handle.handle = reinterpret_cast<int64_t>(buffer_);
+  native_handle.size = size_;
+
+  return native_handle;
 }
 
 size_t MTLPixelBuffer::get_size()

source/blender/gpu/opengl/gl_texture.cc

@@ -269,7 +269,7 @@ void GLTexture::update_sub(int offset[3],
   GLContext::state_manager_active_get()->texture_bind_temp(this);
 
   /* Bind pixel buffer for source data. */
-  GLint pix_buf_handle = (GLint)GPU_pixel_buffer_get_native_handle(pixbuf);
+  GLint pix_buf_handle = (GLint)GPU_pixel_buffer_get_native_handle(pixbuf).handle;
   glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pix_buf_handle);
 
   switch (dimensions) {
@@ -810,9 +810,12 @@ void GLPixelBuffer::unmap()
   glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
 }
 
-int64_t GLPixelBuffer::get_native_handle()
+GPUPixelBufferNativeHandle GLPixelBuffer::get_native_handle()
 {
-  return int64_t(gl_id_);
+  GPUPixelBufferNativeHandle native_handle;
+  native_handle.handle = int64_t(gl_id_);
+  native_handle.size = size_;
+  return native_handle;
 }
 
 size_t GLPixelBuffer::get_size()

source/blender/gpu/opengl/gl_texture.hh

@@ -137,7 +137,7 @@ class GLPixelBuffer : public PixelBuffer {
 
   void *map() override;
   void unmap() override;
-  int64_t get_native_handle() override;
+  GPUPixelBufferNativeHandle get_native_handle() override;
   size_t get_size() override;
 
   MEM_CXX_CLASS_ALLOC_FUNCS("GLPixelBuffer")

source/blender/gpu/vulkan/vk_buffer.cc

@@ -9,6 +9,7 @@
 #include "vk_buffer.hh"
 #include "vk_backend.hh"
 #include "vk_context.hh"
+#include <vulkan/vulkan_core.h>
 
 namespace blender::gpu {
 
@@ -28,7 +29,8 @@ bool VKBuffer::create(size_t size_in_bytes,
                       VkBufferUsageFlags buffer_usage,
                       VkMemoryPropertyFlags required_flags,
                       VkMemoryPropertyFlags preferred_flags,
-                      VmaAllocationCreateFlags allocation_flags)
+                      VmaAllocationCreateFlags allocation_flags,
+                      bool export_memory)
 {
   BLI_assert(!is_allocated());
   BLI_assert(vk_buffer_ == VK_NULL_HANDLE);
@@ -55,6 +57,9 @@ bool VKBuffer::create(size_t size_in_bytes,
   const uint32_t queue_family_indices[1] = {device.queue_family_get()};
   create_info.pQueueFamilyIndices = queue_family_indices;
 
+  VkExternalMemoryBufferCreateInfo external_memory_create_info = {
+      VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, nullptr, 0};
+
   VmaAllocationCreateInfo vma_create_info = {};
   vma_create_info.flags = allocation_flags;
   vma_create_info.priority = 1.0f;
@@ -62,6 +67,18 @@ bool VKBuffer::create(size_t size_in_bytes,
   vma_create_info.preferredFlags = preferred_flags;
   vma_create_info.usage = VMA_MEMORY_USAGE_AUTO;
 
+  if (export_memory) {
+    create_info.pNext = &external_memory_create_info;
+#ifdef _WIN32
+    external_memory_create_info.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
+#else
+    external_memory_create_info.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
+#endif
+    /* Dedicated allocation for zero offset. */
+    vma_create_info.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
+    vma_create_info.pool = device.vma_pools.external_memory;
+  }
+
   VkResult result = vmaCreateBuffer(
       allocator, &create_info, &vma_create_info, &vk_buffer_, &allocation_, nullptr);
   if (result != VK_SUCCESS) {
@@ -178,6 +195,24 @@ void VKBuffer::unmap()
   mapped_memory_ = nullptr;
 }
 
+VkDeviceMemory VKBuffer::export_memory_get(size_t &memory_size)
+{
+  const VKDevice &device = VKBackend::get().device;
+  VmaAllocator allocator = device.mem_allocator_get();
+
+  VmaAllocationInfo info = {};
+  vmaGetAllocationInfo(allocator, allocation_, &info);
+
+  /* VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT should ensure this. */
+  if (info.offset != 0) {
+    BLI_assert(!"Failed to get zero offset export memory for Vulkan buffer");
+    return nullptr;
+  }
+
+  memory_size = info.size;
+  return info.deviceMemory;
+}
+
 bool VKBuffer::free()
 {
   if (is_mapped()) {

source/blender/gpu/vulkan/vk_buffer.hh

@@ -45,7 +45,8 @@ class VKBuffer : public NonCopyable {
               VkBufferUsageFlags buffer_usage,
               VkMemoryPropertyFlags required_flags,
               VkMemoryPropertyFlags preferred_flags,
-              VmaAllocationCreateFlags vma_allocation_flags);
+              VmaAllocationCreateFlags vma_allocation_flags,
+              bool export_memory = false);
   void clear(VKContext &context, uint32_t clear_value);
   void update_immediately(const void *data) const;
   void update_sub_immediately(size_t start_offset, size_t data_size, const void *data) const;
@@ -110,6 +111,11 @@ class VKBuffer : public NonCopyable {
    */
   bool is_mapped() const;
 
+  /**
+   * Get allocated device memory.
+   */
+  VkDeviceMemory export_memory_get(size_t &memory_size);
+
  private:
   /** Check if this buffer is mapped. */
   bool map();

source/blender/gpu/vulkan/vk_pixel_buffer.cc

@@ -8,23 +8,45 @@
 
 #include "vk_pixel_buffer.hh"
 
-#include "vk_context.hh"
+#include "vk_backend.hh"
+
+#include "CLG_log.h"
 
 namespace blender::gpu {
 
-VKPixelBuffer::VKPixelBuffer(size_t size) : PixelBuffer(size)
+static CLG_LogRef LOG = {"gpu.vulkan"};
+
+VKPixelBuffer::VKPixelBuffer(size_t size) : PixelBuffer(size) {}
+
+void VKPixelBuffer::create(bool memory_export)
 {
-  buffer_.create(size,
+  /* Create on demand with or without memory export. When memory export is
+   * enabled there is no host mapping. */
+  if (buffer_initialized_ && buffer_memory_export_ == memory_export) {
+    return;
+  }
+
+  if (buffer_.is_allocated()) {
+    buffer_.free();
+  }
+
+  buffer_.create(size_,
                  VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
                  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT,
                  VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
-                 VmaAllocationCreateFlags(0));
+                 VmaAllocationCreateFlags(0),
+                 memory_export);
   debug::object_label(buffer_.vk_handle(), "PixelBuffer");
+
+  buffer_initialized_ = true;
+  buffer_memory_export_ = memory_export;
+  native_handle_ = GPUPixelBufferNativeHandle{};
 }
 
 void *VKPixelBuffer::map()
 {
   /* Vulkan buffers are always mapped between allocation and freeing. */
+  create(false);
   return buffer_.mapped_memory_get();
 }
 
@@ -33,9 +55,72 @@ void VKPixelBuffer::unmap()
   /* Vulkan buffers are always mapped between allocation and freeing. */
 }
 
-int64_t VKPixelBuffer::get_native_handle()
+GPUPixelBufferNativeHandle VKPixelBuffer::get_native_handle()
 {
-  return int64_t(buffer_.vk_handle());
+  /* Initialize once. */
+  if (buffer_initialized_) {
+    return native_handle_;
+  }
+
+  VKDevice &device = VKBackend::get().device;
+
+  /* Functionality supported? */
+#ifdef _WIN32
+  if (!device.functions.vkGetMemoryWin32Handle) {
+    return native_handle_;
+  }
+#else
+  if (!device.functions.vkGetMemoryFd) {
+    return native_handle_;
+  }
+#endif
+
+  /* Create buffer. */
+  create(true);
+
+  /* Get device memory. */
+  size_t memory_size = 0;
+  VkDeviceMemory memory = buffer_.export_memory_get(memory_size);
+  if (memory == nullptr) {
+    CLOG_ERROR(&LOG, "Failed to get device memory for Vulkan pixel buffer");
+    return native_handle_;
+  }
+
+#ifdef _WIN32
+  /* Opaque Windows handle. */
+  VkMemoryGetWin32HandleInfoKHR info = {};
+  info.sType = VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR;
+  info.pNext = nullptr;
+  info.memory = memory;
+  info.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
+
+  HANDLE handle = 0;
+  if (device.functions.vkGetMemoryWin32Handle(device.vk_handle(), &info, &handle) != VK_SUCCESS) {
+    CLOG_ERROR(&LOG, "Failed to get Windows handle for Vulkan pixel buffer");
+    return native_handle_;
+  }
+
+  native_handle_.handle = int64_t(handle);
+  native_handle_.size = memory_size;
+#else
+  /* Opaque file descriptor. */
+  VkMemoryGetFdInfoKHR info = {};
+  info.sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR;
+  info.pNext = nullptr;
+  info.memory = memory;
+  info.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
+
+  int fd = -1;
+  if (device.functions.vkGetMemoryFd(device.vk_handle(), &info, &fd) != VK_SUCCESS) {
+    CLOG_ERROR(&LOG, "Failed to get file descriptor for Vulkan pixel buffer");
+    return native_handle_;
+  }
+
+  native_handle_.handle = int64_t(fd);
+  native_handle_.size = memory_size;
+#endif
+
+  return native_handle_;
 }
 
 size_t VKPixelBuffer::get_size()

source/blender/gpu/vulkan/vk_pixel_buffer.hh

@@ -16,18 +16,24 @@ namespace blender::gpu {
 
 class VKPixelBuffer : public PixelBuffer {
   VKBuffer buffer_;
+  bool buffer_initialized_ = false;
+  bool buffer_memory_export_ = false;
+  GPUPixelBufferNativeHandle native_handle_;
 
  public:
   VKPixelBuffer(size_t size);
   void *map() override;
   void unmap() override;
-  int64_t get_native_handle() override;
+  GPUPixelBufferNativeHandle get_native_handle() override;
   size_t get_size() override;
 
   VKBuffer &buffer_get()
   {
     return buffer_;
   }
+
+ protected:
+  void create(bool memory_export);
 };
 
 static inline VKPixelBuffer *unwrap(PixelBuffer *pixel_buffer)