Core: add --no-window-frame command line option

Add a command line option to load Blender without a window frame.

Currently this is only used on Wayland, enabled for WITH_UI_TESTS since
attempting to load LIBDECOR caused the tests to crash on start on Fedora.
For tests there is no need to use LIBDECOR, so disable the window frame.
This can also be used by users who don't want to use the X11 fallback if
LIBDECOR can't be found.

Ref !147716

intern/ghost/GHOST_C-api.h

@@ -1007,6 +1007,13 @@ extern GHOST_TCapabilityFlag GHOST_GetCapabilities(void);
  */
 extern void GHOST_SetBacktraceHandler(GHOST_TBacktraceFn backtrace_fn);
 
+/**
+ * When `use_window_frame` is false, don't show window frames.
+ *
+ * \note This must run before the system is created.
+ */
+extern void GHOST_UseWindowFrame(bool use_window_frame);
+
 /**
  * Focus window after opening, or put them in the background.
  */

intern/ghost/GHOST_ISystem.hh

@@ -150,6 +150,9 @@ class GHOST_ISystem {
   static GHOST_TBacktraceFn getBacktraceFn();
   static void setBacktraceFn(GHOST_TBacktraceFn backtrace_fn);
 
+  static bool getUseWindowFrame();
+  static void setUseWindowFrame(bool use_window_frame);
+
  protected:
   /**
    * Constructor.
@@ -541,5 +544,14 @@ class GHOST_ISystem {
   /** Function to call that sets the back-trace. */
   static GHOST_TBacktraceFn backtrace_fn_;
 
+  /**
+   * When false, don't use window frame.
+   *
+   * \note This needs to be set before system initialization
+   * to avoid loading LIBDECOR libraries (which can crash).
+   * If LIBDECOR is removed, this could be set on window creation instead.
+   */
+  static bool use_window_frame_;
+
   MEM_CXX_CLASS_ALLOC_FUNCS("GHOST:GHOST_ISystem")
 };

intern/ghost/intern/GHOST_C-api.cc

@@ -965,6 +965,11 @@ void GHOST_SetBacktraceHandler(GHOST_TBacktraceFn backtrace_fn)
   GHOST_ISystem::setBacktraceFn(backtrace_fn);
 }
 
+void GHOST_UseWindowFrame(bool use_window_frame)
+{
+  GHOST_ISystem::setUseWindowFrame(use_window_frame);
+}
+
 void GHOST_UseWindowFocus(bool use_focus)
 {
   GHOST_ISystem *system = GHOST_ISystem::getSystem();

intern/ghost/intern/GHOST_ISystem.cc

@@ -39,6 +39,8 @@ const char *GHOST_ISystem::system_backend_id_ = nullptr;
 
 GHOST_TBacktraceFn GHOST_ISystem::backtrace_fn_ = nullptr;
 
+bool GHOST_ISystem::use_window_frame_ = true;
+
 GHOST_TSuccess GHOST_ISystem::createSystem(bool verbose, [[maybe_unused]] bool background)
 {
 
@@ -53,12 +55,13 @@ GHOST_TSuccess GHOST_ISystem::createSystem(bool verbose, [[maybe_unused]] bool b
 
   GHOST_TSuccess success;
   if (!system_) {
+    const bool use_window_frame = GHOST_ISystem::getUseWindowFrame();
 
 #if defined(WITH_HEADLESS)
     /* Pass. */
 #elif defined(WITH_GHOST_WAYLAND)
 #  if defined(WITH_GHOST_WAYLAND_DYNLOAD)
-    const bool has_wayland_libraries = ghost_wl_dynload_libraries_init();
+    const bool has_wayland_libraries = ghost_wl_dynload_libraries_init(use_window_frame);
 #  else
     const bool has_wayland_libraries = true;
 #  endif
@@ -261,3 +264,13 @@ void GHOST_ISystem::setBacktraceFn(GHOST_TBacktraceFn backtrace_fn)
 {
   GHOST_ISystem::backtrace_fn_ = backtrace_fn;
 }
+
+bool GHOST_ISystem::getUseWindowFrame()
+{
+  return GHOST_ISystem::use_window_frame_;
+}
+
+void GHOST_ISystem::setUseWindowFrame(bool use_window_frame)
+{
+  GHOST_ISystem::use_window_frame_ = use_window_frame;
+}

intern/ghost/intern/GHOST_SystemWayland.cc

@@ -1548,6 +1548,11 @@ struct GWL_Display {
    */
   bool background = false;
 
+  /**
+   * Show window decorations, otherwise all windows are frame-less.
+   */
+  bool use_window_frame = false;
+
   /* Threaded event handling. */
 #ifdef USE_EVENT_BACKGROUND_THREAD
   /**
@@ -7609,8 +7614,12 @@ GHOST_SystemWayland::GHOST_SystemWayland(const bool background)
   wl_log_set_handler_client(background ? ghost_wayland_log_handler_background :
                                          ghost_wayland_log_handler);
 
+  const bool use_window_frame = GHOST_ISystem::getUseWindowFrame();
+
   display_->system = this;
   display_->background = background;
+  display_->use_window_frame = use_window_frame;
+
   /* Connect to the Wayland server. */
 
   display_->wl.display = wl_display_connect(nullptr);
@@ -7641,7 +7650,7 @@ GHOST_SystemWayland::GHOST_SystemWayland(const bool background)
 
 #ifdef WITH_GHOST_WAYLAND_LIBDECOR
   bool libdecor_required = false;
-  {
+  if (use_window_frame) {
     const char *xdg_current_desktop = [] {
       /* Account for VSCode overriding this value (TSK!), see: #133921. */
       const char *key = "ORIGINAL_XDG_CURRENT_DESKTOP";
@@ -9359,6 +9368,11 @@ GHOST_TimerManager *GHOST_SystemWayland::ghost_timer_manager()
 }
 #endif
 
+bool GHOST_SystemWayland::use_window_frame_get()
+{
+  return display_->use_window_frame;
+}
+
 /** \} */
 
 /* -------------------------------------------------------------------- */
@@ -9870,7 +9884,7 @@ bool GHOST_SystemWayland::use_libdecor_runtime()
 #endif
 
 #ifdef WITH_GHOST_WAYLAND_DYNLOAD
-bool ghost_wl_dynload_libraries_init()
+bool ghost_wl_dynload_libraries_init(const bool use_window_frame)
 {
 #  ifdef WITH_GHOST_X11
   /* When running in WAYLAND, let the user know when a missing library is the only reason
@@ -9891,7 +9905,11 @@ bool ghost_wl_dynload_libraries_init()
   )
   {
 #  ifdef WITH_GHOST_WAYLAND_LIBDECOR
-    has_libdecor = wayland_dynload_libdecor_init(verbose); /* `libdecor-0`. */
+    if (use_window_frame) {
+      has_libdecor = wayland_dynload_libdecor_init(verbose); /* `libdecor-0`. */
+    }
+#  else
+    (void)use_window_frame;
 #  endif
     return true;
   }

intern/ghost/intern/GHOST_SystemWayland.hh

@@ -86,7 +86,7 @@ int gwl_window_scale_int_from(const GWL_WindowScaleParams &scale_params, int val
  * Return true when all required WAYLAND libraries are present,
  * Performs dynamic loading when `WITH_GHOST_WAYLAND_DYNLOAD` is in use.
  */
-bool ghost_wl_dynload_libraries_init();
+bool ghost_wl_dynload_libraries_init(bool use_window_frame);
 void ghost_wl_dynload_libraries_exit();
 #endif
 
@@ -280,6 +280,8 @@ class GHOST_SystemWayland : public GHOST_System {
                  bool completed) const;
   void ime_end(const GHOST_WindowWayland *win) const;
 
+  bool use_window_frame_get();
+
   static const char *xdg_app_id_get();
 
   /* WAYLAND utility functions. */

intern/ghost/intern/GHOST_WindowWayland.cc

@@ -1867,12 +1867,16 @@ GHOST_WindowWayland::GHOST_WindowWayland(GHOST_SystemWayland *system,
     GWL_XDG_Decor_Window &decor = *window_->xdg_decor;
 
     if (system_->xdg_decor_manager_get()) {
+      const bool use_window_frame = system_->use_window_frame_get();
       decor.toplevel_decor = zxdg_decoration_manager_v1_get_toplevel_decoration(
           system_->xdg_decor_manager_get(), decor.toplevel);
       zxdg_toplevel_decoration_v1_add_listener(
           decor.toplevel_decor, &xdg_toplevel_decoration_v1_listener, window_);
+      /* Request client side decorations as a way of disabling decorations. */
       zxdg_toplevel_decoration_v1_set_mode(decor.toplevel_decor,
-                                           ZXDG_TOPLEVEL_DECORATION_V1_MODE_SERVER_SIDE);
+                                           use_window_frame ?
+                                               ZXDG_TOPLEVEL_DECORATION_V1_MODE_SERVER_SIDE :
+                                               ZXDG_TOPLEVEL_DECORATION_V1_MODE_CLIENT_SIDE);
     }
 
     /* Commit needed to so configure callback runs. */

source/blender/windowmanager/WM_api.hh

@@ -103,6 +103,8 @@ void WM_init_state_normal_set();
 void WM_init_state_maximized_set();
 void WM_init_state_start_with_console_set(bool value);
 void WM_init_window_focus_set(bool do_it);
+bool WM_init_window_frame_get();
+void WM_init_window_frame_set(bool do_it);
 void WM_init_native_pixels(bool do_it);
 void WM_init_input_devices();
 

source/blender/windowmanager/intern/wm_playanim.cc

@@ -1862,6 +1862,7 @@ static std::optional<int> wm_main_playanim_intern(int argc, const char **argv, P
 
       /* Init GHOST and open window. */
       GHOST_SetBacktraceHandler((GHOST_TBacktraceFn)BLI_system_backtrace);
+      GHOST_UseWindowFrame(WM_init_window_frame_get());
 
       ps.ghost_data.system = GHOST_CreateSystem();
       if (UNLIKELY(ps.ghost_data.system == nullptr)) {

source/blender/windowmanager/intern/wm_window.cc

@@ -132,6 +132,7 @@ static struct WMInitStruct {
   GHOST_TWindowState windowstate = GHOST_WINDOW_STATE_DEFAULT;
   eWinOverrideFlag override_flag;
 
+  bool window_frame = true;
   bool window_focus = true;
   bool native_pixels = true;
 } wm_init_state;
@@ -2116,6 +2117,7 @@ void wm_ghost_init(bContext *C)
   consumer = GHOST_CreateEventConsumer(ghost_event_proc, C);
 
   GHOST_SetBacktraceHandler((GHOST_TBacktraceFn)BLI_system_backtrace);
+  GHOST_UseWindowFrame(wm_init_state.window_frame);
 
   g_system = GHOST_CreateSystem();
   GPU_backend_ghost_system_set(g_system);
@@ -2814,6 +2816,16 @@ void WM_init_state_maximized_set()
   wm_init_state.override_flag |= WIN_OVERRIDE_WINSTATE;
 }
 
+bool WM_init_window_frame_get()
+{
+  return wm_init_state.window_frame;
+}
+
+void WM_init_window_frame_set(bool do_it)
+{
+  wm_init_state.window_frame = do_it;
+}
+
 void WM_init_window_focus_set(bool do_it)
 {
   wm_init_state.window_focus = do_it;

source/creator/creator_args.cc

@@ -702,6 +702,7 @@ static void print_help(bArgs *ba, bool all)
   BLI_args_print_arg_doc(ba, "--window-geometry");
   BLI_args_print_arg_doc(ba, "--start-console");
   BLI_args_print_arg_doc(ba, "--no-native-pixels");
+  BLI_args_print_arg_doc(ba, "--no-window-frame");
   BLI_args_print_arg_doc(ba, "--no-window-focus");
 
   PRINT("\n");
@@ -1889,6 +1890,15 @@ static int arg_handle_window_maximized(int /*argc*/, const char ** /*argv*/, voi
   return 0;
 }
 
+static const char arg_handle_no_window_frame_doc[] =
+    "\n\t"
+    "Disable all window decorations (Wayland only).";
+static int arg_handle_no_window_frame(int /*argc*/, const char ** /*argv*/, void * /*data*/)
+{
+  WM_init_window_frame_set(false);
+  return 0;
+}
+
 static const char arg_handle_no_window_focus_doc[] =
     "\n\t"
     "Open behind other windows and without taking focus.";
@@ -3087,6 +3097,7 @@ void main_args_setup(bContext *C, bArgs *ba, bool all)
   BLI_args_add(ba, "-w", "--window-border", CB(arg_handle_window_border), nullptr);
   BLI_args_add(ba, "-W", "--window-fullscreen", CB(arg_handle_window_fullscreen), nullptr);
   BLI_args_add(ba, "-M", "--window-maximized", CB(arg_handle_window_maximized), nullptr);
+  BLI_args_add(ba, nullptr, "--no-window-frame", CB(arg_handle_no_window_frame), nullptr);
   BLI_args_add(ba, nullptr, "--no-window-focus", CB(arg_handle_no_window_focus), nullptr);
   BLI_args_add(ba, "-con", "--start-console", CB(arg_handle_start_with_console), nullptr);
   BLI_args_add(ba, "-r", "--register", CB(arg_handle_register_extension), nullptr);

tests/python/CMakeLists.txt

@@ -121,6 +121,10 @@ if(WITH_UI_TESTS)
       # while this could be investigated, use windowed mode instead.
       # Use a window size that balances software GPU rendering with enough room to use the UI.
       --factory-startup
+      # Used so GHOST/Wayland doesn't attempt to load LIBDECOR
+      # which can fail on some systems, causing tests to fail.
+      # On other systems this is harmless.
+      --no-window-frame
       -p 0 0 800 600
       "${EXE_PARAMS}"
       "${ARGN}"