Fix massive amount of memleaks on exit in BPY.

Essentially, our current code would not properly remove (dereference)
its python objects matching various RNA data created during execution.

Some cases are fairly trivial to understand (like the lack of handling
of unregstering for our 'startup' operators and UI), other were more
subtle (like unregistered PropertyGroups who would remove/free their RNA
struct definition, without releasing first the potential matching python
object).

Co-authored-by: Campbell Barton <campbell@blender.org>

Pull Request: https://projects.blender.org/blender/blender/pulls/128899

scripts/modules/bpy/utils/__init__.py

@@ -201,6 +201,31 @@ _registered_module_names = []
 import bpy_types as _bpy_types
 
 
+def _register_module_call(mod):
+    register = getattr(mod, "register", None)
+    if register:
+        try:
+            register()
+        except Exception:
+            import traceback
+            traceback.print_exc()
+    else:
+        print(
+            "\nWarning! {!r} has no register function, "
+            "this is now a requirement for registerable scripts".format(mod.__file__)
+        )
+
+
+def _unregister_module_call(mod):
+    unregister = getattr(mod, "unregister", None)
+    if unregister:
+        try:
+            unregister()
+        except Exception:
+            import traceback
+            traceback.print_exc()
+
+
 def load_scripts(*, reload_scripts=False, refresh_scripts=False, extensions=True):
     """
     Load scripts and run each modules register function.
@@ -234,29 +259,6 @@ def load_scripts(*, reload_scripts=False, refresh_scripts=False, extensions=True
         for addon_module_name in [ext.module for ext in _preferences.addons]:
             _addon_utils.disable(addon_module_name)
 
-    def register_module_call(mod):
-        register = getattr(mod, "register", None)
-        if register:
-            try:
-                register()
-            except Exception:
-                import traceback
-                traceback.print_exc()
-        else:
-            print(
-                "\nWarning! {!r} has no register function, "
-                "this is now a requirement for registerable scripts".format(mod.__file__)
-            )
-
-    def unregister_module_call(mod):
-        unregister = getattr(mod, "unregister", None)
-        if unregister:
-            try:
-                unregister()
-            except Exception:
-                import traceback
-                traceback.print_exc()
-
     def test_reload(mod):
         import importlib
         # reloading this causes internal errors
@@ -281,7 +283,7 @@ def load_scripts(*, reload_scripts=False, refresh_scripts=False, extensions=True
             mod = test_reload(mod)
 
         if mod:
-            register_module_call(mod)
+            _register_module_call(mod)
             _registered_module_names.append(mod.__name__)
 
     if reload_scripts:
@@ -304,7 +306,7 @@ def load_scripts(*, reload_scripts=False, refresh_scripts=False, extensions=True
 
         # Loop over and unload all scripts.
         for mod in registered_modules:
-            unregister_module_call(mod)
+            _unregister_module_call(mod)
 
         for mod in registered_modules:
             test_reload(mod)
@@ -356,6 +358,22 @@ def load_scripts(*, reload_scripts=False, refresh_scripts=False, extensions=True
                         print("Warning, unregistered class: {:s}({:s})".format(subcls.__name__, cls.__name__))
 
 
+# Internal only, called on exit by `WM_exit_ex`.
+def _on_exit():
+    # Disable all add-ons.
+    _addon_utils.disable_all()
+
+    # Call `unregister` function on internal startup module.
+    # Must only be used as part of Blender 'exit' process.
+    from bpy_restrict_state import RestrictBlend
+    with RestrictBlend():
+        for mod_name in reversed(_registered_module_names):
+            if (mod := _sys.modules.get(mod_name)) is None:
+                print("Warning: module", repr(mod_name), "not found in sys.modules")
+                continue
+            _unregister_module_call(mod)
+
+
 def load_scripts_extensions(*, reload_scripts=False):
     """
     Load extensions scripts (add-ons and app-templates)

source/blender/makesrna/RNA_define.hh

@@ -56,6 +56,7 @@ void RNA_define_fallback_property_update(int noteflag, const char *updatefunc);
 void RNA_define_lib_overridable(bool make_overridable);
 
 void RNA_init();
+void RNA_bpy_exit();
 void RNA_exit();
 
 /* Struct */

source/blender/makesrna/intern/rna_ID.cc

@@ -660,6 +660,9 @@ IDProperty **rna_PropertyGroup_idprops(PointerRNA *ptr)
 
 bool rna_PropertyGroup_unregister(Main * /*bmain*/, StructRNA *type)
 {
+  /* Ensure that a potential py object representing this RNA type is properly dereferenced. */
+  BPY_free_srna_pytype(type);
+
   RNA_struct_free(&BLENDER_RNA, type);
   return true;
 }

source/blender/makesrna/intern/rna_access.cc

@@ -55,6 +55,7 @@
 #include "RNA_define.hh"
 #include "RNA_enum_types.hh"
 #include "RNA_path.hh"
+#include "RNA_types.hh"
 
 #include "WM_api.hh"
 #include "WM_message.hh"
@@ -63,6 +64,10 @@
 #include "DNA_object_types.h"
 #include "WM_types.hh"
 
+#ifdef WITH_PYTHON
+#  include "BPY_extern.hh"
+#endif
+
 #include "rna_access_internal.hh"
 #include "rna_internal.hh"
 
@@ -95,6 +100,21 @@ void RNA_init()
   }
 }
 
+void RNA_bpy_exit()
+{
+#ifdef WITH_PYTHON
+  StructRNA *srna;
+
+  for (srna = static_cast<StructRNA *>(BLENDER_RNA.structs.first); srna;
+       srna = static_cast<StructRNA *>(srna->cont.next))
+  {
+    /* NOTE(@ideasman42): each call locks the Python's GIL. Only locking/unlocking once
+     * is possible but gives barely measurable speedup (< ~1millisecond) so leave as-is. */
+    BPY_free_srna_pytype(srna);
+  }
+#endif
+}
+
 void RNA_exit()
 {
   StructRNA *srna;

source/blender/makesrna/intern/rna_define.cc

@@ -789,21 +789,19 @@ void RNA_struct_free(BlenderRNA *brna, StructRNA *srna)
   PropertyRNA *prop, *nextprop;
   PropertyRNA *parm, *nextparm;
 
-#  if 0
   if (srna->flag & STRUCT_RUNTIME) {
     if (RNA_struct_py_type_get(srna)) {
       /* NOTE: Since this is called after finalizing python/BPY in WM_exit process, it may end
        * up accessing freed memory in `srna->identifier`, which will trigger an ASAN crash. */
       const char *srna_identifier = "UNKNOWN";
-#    ifndef WITH_ASAN
+#  ifndef WITH_ASAN
       srna_identifier = srna->identifier;
-#    endif
+#  endif
       fprintf(stderr,
               "RNA Struct definition '%s' freed while holding a Python reference.\n",
               srna_identifier);
     }
   }
-#  endif
 
   for (prop = static_cast<PropertyRNA *>(srna->cont.properties.first); prop; prop = nextprop) {
     nextprop = prop->next;

source/blender/python/BPY_extern.hh

@@ -22,6 +22,7 @@ struct bConstraintTarget; /* DNA_constraint_types.h */
 struct bContext;
 struct bContextDataResult;
 struct bPythonConstraint; /* DNA_constraint_types.h */
+struct StructRNA;
 struct wmWindowManager;
 
 #include "BLI_utildefines.h"
@@ -123,6 +124,12 @@ void BPY_context_dict_clear_members_array(void **dict_p,
 
 void BPY_id_release(ID *id);
 
+/**
+ * Free (actually dereference) the Python type object representing the given #StrucRNA type,
+ * if it is defined.
+ */
+void BPY_free_srna_pytype(StructRNA *srna);
+
 /**
  * Avoids duplicating keyword list.
  */

source/blender/python/intern/bpy.cc

@@ -765,10 +765,6 @@ void BPy_init_modules(bContext *C)
 
   PointerRNA ctx_ptr = RNA_pointer_create(nullptr, &RNA_Context, C);
   bpy_context_module = (BPy_StructRNA *)pyrna_struct_CreatePyObject(&ctx_ptr);
-  /* odd that this is needed, 1 ref on creation and another for the module
-   * but without we get a crash on exit */
-  Py_INCREF(bpy_context_module);
-
   PyModule_AddObject(mod, "context", (PyObject *)bpy_context_module);
 
   /* Register methods and property get/set for RNA types. */

source/blender/python/intern/bpy_interface.cc

@@ -577,7 +577,7 @@ void BPY_python_end(const bool do_python_exit)
   BPY_rna_props_clear_all();
 
   /* Free other Python data. */
-  pyrna_free_types();
+  RNA_bpy_exit();
 
   BPY_rna_exit();
 

source/blender/python/intern/bpy_rna.cc

@@ -9096,26 +9096,14 @@ void pyrna_alloc_types()
 #endif /* !NDEBUG */
 }
 
-void pyrna_free_types()
+void BPY_free_srna_pytype(StructRNA *srna)
 {
-  PropertyRNA *prop;
+  void *py_ptr = RNA_struct_py_type_get(srna);
 
-  /* Avoid doing this lookup for every getattr. */
-  PointerRNA ptr = RNA_blender_rna_pointer_create();
-  prop = RNA_struct_find_property(&ptr, "structs");
-
-  RNA_PROP_BEGIN (&ptr, itemptr, prop) {
-    StructRNA *srna = srna_from_ptr(&itemptr);
-    void *py_ptr = RNA_struct_py_type_get(srna);
-
-    if (py_ptr) {
-#if 0 /* XXX: should be able to do this, but makes Python crash on exit. */
-      bpy_class_free(py_ptr);
-#endif
-      RNA_struct_py_type_set(srna, nullptr);
-    }
+  if (py_ptr) {
+    bpy_class_free(py_ptr);
+    RNA_struct_py_type_set(srna, nullptr);
   }
-  RNA_PROP_END;
 }
 
 /**
@@ -9255,11 +9243,13 @@ static PyObject *pyrna_register_class(PyObject * /*self*/, PyObject *py_class)
   pyrna_subtype_set_rna(py_class, srna_new);
 
   /* Old srna still references us, keep the check in case registering somehow can free it. */
-  if (RNA_struct_py_type_get(srna)) {
+  if (PyObject *old_py_class = static_cast<PyObject *>(RNA_struct_py_type_get(srna))) {
     RNA_struct_py_type_set(srna, nullptr);
 #if 0
     /* Should be able to do this XXX since the old RNA adds a new ref. */
-    Py_DECREF(py_class);
+    Py_DECREF(old_py_class);
+#else
+    UNUSED_VARS(old_py_class);
 #endif
   }
 

source/blender/python/intern/bpy_rna.hh

@@ -232,10 +232,7 @@ int pyrna_struct_as_ptr_or_null_parse(PyObject *o, void *p);
 
 void pyrna_struct_type_extend_capi(StructRNA *srna, PyMethodDef *method, PyGetSetDef *getset);
 
-/* Called before stopping Python. */
-
 void pyrna_alloc_types(void);
-void pyrna_free_types(void);
 
 /* Primitive type conversion. */
 

source/blender/windowmanager/intern/wm_init_exit.cc

@@ -533,8 +533,9 @@ void WM_exit_ex(bContext *C, const bool do_python_exit, const bool do_user_exit_
    * Which can happen when the GPU backend fails to initialize.
    */
   if (C && CTX_py_init_get(C)) {
-    const char *imports[2] = {"addon_utils", nullptr};
-    BPY_run_string_eval(C, imports, "addon_utils.disable_all()");
+    /* Calls `addon_utils.disable_all()` as well as unregistering all "startup" modules.  */
+    const char *imports[] = {"bpy.utils", nullptr};
+    BPY_run_string_eval(C, imports, "bpy.utils._on_exit()");
   }
 #endif
 

source/blender/windowmanager/intern/wm_operator_type.cc

@@ -32,6 +32,10 @@
 #include "RNA_enum_types.hh"
 #include "RNA_prototypes.hh"
 
+#ifdef WITH_PYTHON
+#  include "BPY_extern.hh"
+#endif
+
 #include "WM_api.hh"
 #include "WM_types.hh"
 
@@ -154,6 +158,14 @@ void WM_operatortype_remove_ptr(wmOperatorType *ot)
 {
   BLI_assert(ot == WM_operatortype_find(ot->idname, false));
 
+#ifdef WITH_PYTHON
+  /* The 'unexposed' type (inherited from #RNA_OperatorProperties) created for this operator type's
+   * properties may have had a python type representation created. This needs to be dereferenced
+   * manually here, as other #bpy_class_free (which is part of the unregistering code for runtime
+   * operators) will not be able to handle it. */
+  BPY_free_srna_pytype(ot->srna);
+#endif
+
   RNA_struct_free(&BLENDER_RNA, ot->srna);
 
   if (ot->last_properties) {