Fix #135039: Vulkan: Blender does not start with Wayland in GNOME

Remove logic to access LIBDECOR's underlying XDG window since it
required an "dummy-buffer" workaround which caused a crash with recent
drivers that require a DMA buffer.

This introduces a slight flicker on startup in some cases
see code-comments for details.

Ref: !136289
This commit is contained in:
Campbell Barton
2025-04-11 09:44:46 +00:00
parent b1c55c6988
commit 44444a550e
3 changed files with 40 additions and 61 deletions

View File

@@ -2430,13 +2430,6 @@ static int memfd_create_sealed(const char *name)
#endif /* !HAVE_MEMFD_CREATE */
}
#if defined(WITH_GHOST_WAYLAND_LIBDECOR) && defined(WITH_VULKAN_BACKEND)
int memfd_create_sealed_for_vulkan_hack(const char *name)
{
return memfd_create_sealed(name);
}
#endif
enum {
GWL_IOR_READ = 1 << 0,
GWL_IOR_WRITE = 1 << 1,

View File

@@ -90,13 +90,6 @@ bool ghost_wl_dynload_libraries_init();
void ghost_wl_dynload_libraries_exit();
#endif
#if defined(WITH_GHOST_WAYLAND_LIBDECOR) && defined(WITH_VULKAN_BACKEND)
/**
* Needed for temporary buffer creation.
*/
int memfd_create_sealed_for_vulkan_hack(const char *name);
#endif
struct GWL_Output {
/** Wayland core types. */

View File

@@ -38,10 +38,6 @@
# include <wayland_dynload_libdecor.h>
# endif
# include <libdecor.h>
# ifdef WITH_VULKAN_BACKEND
# include <unistd.h> /* For `ftruncate`. */
# endif
#endif
/* Generated by `wayland-scanner`. */
@@ -52,6 +48,7 @@
#include <xdg-shell-client-protocol.h>
#include <atomic>
#include <optional>
#include <cstring> /* For `memcpy`. */
#include <malloc.h> /* For `malloc_usable_size`. */
@@ -134,6 +131,8 @@ struct GWL_LibDecor_Window {
/** The window has been configured (see #xdg_surface_ack_configure). */
bool initial_configure_seen = false;
std::optional<GHOST_TWindowState> initial_configure_state = std::nullopt;
};
static void gwl_libdecor_window_destroy(GWL_LibDecor_Window *decor)
@@ -1044,7 +1043,16 @@ static void gwl_window_frame_update_from_pending_no_lock(GWL_Window *win)
decor.pending.size[0] = 0;
decor.pending.size[1] = 0;
decor.initial_configure_seen = true;
if (decor.initial_configure_seen == false) {
decor.initial_configure_seen = true;
if (decor.initial_configure_state) {
xdg_toplevel *toplevel = libdecor_frame_get_xdg_toplevel(decor.frame);
gwl_window_state_set_for_xdg(
toplevel, decor.initial_configure_state.value(), gwl_window_state_get(win));
decor.initial_configure_state = std::nullopt;
}
}
# ifdef USE_LIBDECOR_CONFIG_COPY_WORKAROUND
if (decor.pending.configuration_needs_free) {
@@ -2015,6 +2023,14 @@ GHOST_WindowWayland::GHOST_WindowWayland(GHOST_SystemWayland *system,
}
#endif
#ifdef WITH_GHOST_WAYLAND_LIBDECOR
# ifdef WITH_VULKAN_BACKEND
const bool libdecor_wait_for_window_init = (type == GHOST_kDrawingContextTypeVulkan);
# else
const bool libdecor_wait_for_window_init = false;
# endif
#endif
/* Drawing context. */
if (setDrawingContextType(type) == GHOST_kFailure) {
/* This can happen when repeatedly creating windows, see #123096.
@@ -2031,8 +2047,26 @@ GHOST_WindowWayland::GHOST_WindowWayland(GHOST_SystemWayland *system,
}
else
#ifdef WITH_GHOST_WAYLAND_LIBDECOR
if (use_libdecor)
if (use_libdecor && libdecor_wait_for_window_init)
{
/* Ensuring the XDG window has been created is *not* supported by VULKAN.
*
* Although this was once supported using a temporary SHM buffer,
* a DMA buffer is now required by some drivers which turns out to be
* impractical to create here, specially since it's only for a temporary buffer.
*
* Workaround the problem by postponing changes to the window state.
* This causes minor but noticeable glitch when starting maximized,
* where a rectangle is first shown before maximizing.
* With EGL this also happens however maximizing is almost immediate.
*
* This can't be avoided at the moment since LIBDECOR requires the window
* to be created before it's configured (sigh!).
* This can be removed if CSD are implemented, see: #113795. */
GWL_LibDecor_Window &decor = *window_->libdecor;
decor.initial_configure_state = state;
}
else if (use_libdecor) {
/* Commit needed so the top-level callbacks run (and `toplevel` can be accessed). */
wl_surface_commit(window_->wl.surface);
GWL_LibDecor_Window &decor = *window_->libdecor;
@@ -2043,53 +2077,12 @@ GHOST_WindowWayland::GHOST_WindowWayland(GHOST_SystemWayland *system,
/* NOTE: LIBDECOR requires the window to be created & configured before the state can be set.
* Workaround this by using the underlying `xdg_toplevel` */
# ifdef WITH_VULKAN_BACKEND
/* A dummy buffer is needed for VULKAN & LIBDECOR,
* otherwise `decor.initial_configure_seen` and startup locks up. */
wl_buffer *dummy_buffer = nullptr;
if (window_->ghost_context_type == GHOST_kDrawingContextTypeVulkan) {
const uint32_t format = WL_SHM_FORMAT_ARGB8888;
const int format_size = 4;
const int buffer_size = (window_->frame.size[0] * window_->frame.size[1]) * format_size;
const int fd = memfd_create_sealed_for_vulkan_hack("ghost-wl-dummy-buffer");
const int truncate_result = ftruncate(fd, buffer_size);
GHOST_ASSERT(truncate_result == 0, "expecting ftruncate of the dummy buffer to work");
# ifndef NDEBUG
(void)truncate_result;
# endif
wl_shm *shm = system_->wl_shm_get();
wl_shm_pool *pool = wl_shm_create_pool(shm, fd, buffer_size);
dummy_buffer = wl_shm_pool_create_buffer(pool,
0,
window_->frame.size[0],
window_->frame.size[1],
window_->frame.size[0] * format_size,
format);
wl_shm_pool_destroy(pool);
wl_surface_attach(window_->wl.surface, dummy_buffer, 0, 0);
wl_surface_damage(window_->wl.surface, 0, 0, window_->frame.size[0], window_->frame.size[1]);
wl_surface_commit(window_->wl.surface);
::close(fd);
}
# endif /* WITH_VULKAN_BACKEND */
/* Failure exits with an error, simply prevent an eternal loop. */
while (!decor.initial_configure_seen && !ghost_wl_display_report_error_if_set(display)) {
wl_display_flush(display);
wl_display_dispatch(display);
}
# ifdef WITH_VULKAN_BACKEND
if (window_->ghost_context_type == GHOST_kDrawingContextTypeVulkan) {
wl_surface_attach(window_->wl.surface, nullptr, 0, 0);
wl_surface_commit(window_->wl.surface);
wl_buffer_destroy(dummy_buffer);
}
# endif /* WITH_GHOST_WAYLAND_LIBDECOR */
xdg_toplevel *toplevel = libdecor_frame_get_xdg_toplevel(decor.frame);
gwl_window_state_set_for_xdg(toplevel, state, gwl_window_state_get(window_));