griefith/test2
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'blender-v4.3-release'

Browse Source
This commit is contained in:
Campbell Barton
2024-10-03 12:19:13 +10:00
parent 9db8cb5ae4 1563ba7e6c
commit 2fdfade51d
5 changed files with 273 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
scripts/addons_core/bl_pkg/bl_extension_ops.py
View File

@@ -272,6 +272,44 @@ class OperatorNonBlockingSyncHelper:
# Internal Utilities
#
def _extensions_repo_temp_files_make_stale(
repo_directory, # `str`
): # `-> None`
# NOTE: this function should run after any operation
# which may have attempted to remove a directory but only successfully renamed it.
# The extension sub-process could communicate this back to this process but it's
# reasonably involved and only avoids a repository file-system scan after each operation.
# Scan repository directories and clear files with a specific prefix & suffix.
import addon_utils
from .bl_extension_utils import (
scandir_with_demoted_errors,
PKG_TEMP_PREFIX_AND_SUFFIX,
)
paths_stale = []
prefix, suffix = PKG_TEMP_PREFIX_AND_SUFFIX
for entry in scandir_with_demoted_errors(repo_directory):
filename = entry.name
if not filename.startswith(prefix):
continue
# Check the `filename` ends with `suffix` or suffix & digits `suffix123`.
i = filename.rfind(suffix)
if i == -1:
continue
ext_end = filename[i + len(suffix):]
if ext_end and (not ext_end.isdigit()):
continue
paths_stale.append(os.path.join(repo_directory, filename))
if not paths_stale:
return
addon_utils.stale_pending_stage_paths(repo_directory, paths_stale)
def _extensions_repo_uninstall_stale_package_fallback(
repo_directory, # `str`
pkg_id_sequence, # `List[str]`
@@ -292,6 +330,30 @@ def _extensions_repo_uninstall_stale_package_fallback(
addon_utils.stale_pending_stage_paths(repo_directory, paths_stale)
def _extensions_repo_install_stale_package_clear(
repo_directory, # `str`
pkg_id_sequence, # `List[str]`
): # `-> None`
# If install succeeds, ensure the package is not stale.
#
# This can happen when a package fails to remove (if one of it's files are locked),
# it is queued for removal. Then the user successfully removes it & re-installs in.
# In this case the package will be tagged for later removal, so ensure it's removed.
import addon_utils
paths_not_stale = []
for pkg_id in pkg_id_sequence:
path_abs = os.path.join(repo_directory, pkg_id)
if not os.path.exists(path_abs):
continue
paths_not_stale.append(path_abs)
if not paths_not_stale:
return
addon_utils.stale_pending_remove_paths(repo_directory, paths_not_stale)
def _sequence_split_with_job_limit(items, job_limit):
# When only one job is allowed at a time, there is no advantage to splitting the sequence.
if job_limit == 1:
@@ -2116,6 +2178,8 @@ class EXTENSIONS_OT_package_install_marked(Operator, _ExtCmdMixIn):
pkg_id_sequence_upgrade=[],
handle_error=handle_error,
)
_extensions_repo_temp_files_make_stale(directory)
_extensions_repo_install_stale_package_clear(directory, pkg_id_sequence)
if self.enable_on_install:
if (extensions_enabled_test := _extensions_enabled()) != extensions_enabled:
@@ -2230,6 +2294,7 @@ class EXTENSIONS_OT_package_uninstall_marked(Operator, _ExtCmdMixIn):
del self.repo_lock
for directory, pkg_id_sequence in self._pkg_id_sequence_from_directory.items():
_extensions_repo_temp_files_make_stale(repo_directory=directory)
_extensions_repo_uninstall_stale_package_fallback(
repo_directory=directory,
pkg_id_sequence=pkg_id_sequence,
@@ -2506,6 +2571,9 @@ class EXTENSIONS_OT_package_install_files(Operator, _ExtCmdMixIn):
stats_calc=False,
)
_extensions_repo_temp_files_make_stale(self.repo_directory)
_extensions_repo_install_stale_package_clear(self.repo_directory, self.pkg_id_sequence)
_preferences_ui_redraw()
_preferences_ui_refresh_addons()
@@ -2879,6 +2947,9 @@ class EXTENSIONS_OT_package_install(Operator, _ExtCmdMixIn):
stats_calc=False,
)
_extensions_repo_temp_files_make_stale(self.repo_directory)
_extensions_repo_install_stale_package_clear(self.repo_directory, (self.pkg_id,))
_preferences_ui_redraw()
_preferences_ui_refresh_addons()
@@ -3273,6 +3344,7 @@ class EXTENSIONS_OT_package_uninstall(Operator, _ExtCmdMixIn):
def exec_command_finish(self, canceled):
_extensions_repo_temp_files_make_stale(repo_directory=self.repo_directory)
_extensions_repo_uninstall_stale_package_fallback(
repo_directory=self.repo_directory,
pkg_id_sequence=[self.pkg_id],

8
scripts/addons_core/bl_pkg/bl_extension_utils.py
View File

@@ -89,6 +89,10 @@ PKG_REPO_LIST_FILENAME = "index.json"
PKG_MANIFEST_FILENAME_TOML = "blender_manifest.toml"
PKG_EXT = ".zip"
# Components to use when creating temporary directory.
# Note that digits may be added to the suffix avoid conflicts.
PKG_TEMP_PREFIX_AND_SUFFIX = (".", ".~temp~")
# Add this to the local JSON file.
REPO_LOCAL_JSON = os.path.join(REPO_LOCAL_PRIVATE_DIR, PKG_REPO_LIST_FILENAME)
@@ -590,6 +594,7 @@ def repo_upgrade(
"--remote-url", remote_url,
"--online-user-agent", online_user_agent,
"--access-token", access_token,
"--temp-prefix-and-suffix", "/".join(PKG_TEMP_PREFIX_AND_SUFFIX),
], use_idle=use_idle, python_args=python_args)
yield [COMPLETE_ITEM]
@@ -629,6 +634,7 @@ def pkg_install_files(
"install-files", *files,
"--local-dir", directory,
"--blender-version", "{:d}.{:d}.{:d}".format(*blender_version),
"--temp-prefix-and-suffix", "/".join(PKG_TEMP_PREFIX_AND_SUFFIX),
], use_idle=use_idle, python_args=python_args)
yield [COMPLETE_ITEM]
@@ -659,6 +665,7 @@ def pkg_install(
"--access-token", access_token,
"--local-cache", str(int(use_cache)),
"--timeout", "{:g}".format(timeout),
"--temp-prefix-and-suffix", "/".join(PKG_TEMP_PREFIX_AND_SUFFIX),
], use_idle=use_idle, python_args=python_args)
yield [COMPLETE_ITEM]
@@ -679,6 +686,7 @@ def pkg_uninstall(
"uninstall", ",".join(pkg_id_sequence),
"--local-dir", directory,
"--user-dir", user_directory,
"--temp-prefix-and-suffix", "/".join(PKG_TEMP_PREFIX_AND_SUFFIX),
], use_idle=use_idle, python_args=python_args)
yield [COMPLETE_ITEM]

126
scripts/addons_core/bl_pkg/cli/blender_ext.py
View File

@@ -616,6 +616,73 @@ def rmtree_with_fallback_or_error(
return None
def rmtree_with_fallback_or_error_pseudo_atomic(
path: str,
*,
temp_prefix_and_suffix: Tuple[str, str],
remove_file: bool = True,
remove_link: bool = True,
) -> Optional[str]:
# It's possible the directory doesn't exist, only attempt a rename if it does.
try:
isdir = os.path.isdir(path)
except Exception:
isdir = False
if isdir:
# Apply the prefix/suffix.
path_base_dirname, path_base_filename = os.path.split(path.rstrip(os.sep))
path_base = os.path.join(
path_base_dirname,
temp_prefix_and_suffix[0] + path_base_filename + temp_prefix_and_suffix[1],
)
del path_base_dirname, path_base_filename
path_test = path_base
test_count = 0
# Unlikely this exists.
while os.path.exists(path_test):
path_test = "{:s}{:d}".format(path_base, test_count)
test_count += 1
# Very unlikely, something is likely incorrect in the setup, avoid hanging.
if test_count > 1000:
return "Unable to create a new path at: {:s}".format(path_test)
# NOTE(@ideasman42): on WIN32 renaming a directory will fail if any files within the directory are open.
# The rename is important, the reasoning is as follows.
# - If the rename fails, the entire directory is left as-is.
# This is done because in the case of an upgrade we *never* want to leave the extension
# in a broken state, with some files removed and the directory locked,
# meaning that an updated extension cannot be written to the destination.
# - If the rename succeeds but deleting the directory fails (unlikely but possible),
# then at least the directory name is available (necessary for an upgrade).
# The directory will use the `temp_prefix_and_suffix` can be removed later.
#
# On other systems, renaming before removal isn't important but is harmless,
# so keep it to avoid logic diverging.
#
# See #128175.
try:
os.rename(path, path_test)
except Exception as ex:
ex_str = str(ex)
if isinstance(ex, PermissionError):
if sys.platform == "win32":
if "The process cannot access the file because it is being used by another process" in ex_str:
return "locked by another process: {:s}".format(path)
return ex_str
path = path_test
return rmtree_with_fallback_or_error(
path,
remove_file=remove_file,
remove_link=remove_link,
)
def build_paths_expand_iter(
path: str,
path_list: Sequence[str],
@@ -2730,6 +2797,13 @@ def arg_handle_str_as_package_names(value: str) -> Sequence[str]:
return result
def arg_handle_str_as_temp_prefix_and_suffix(value: str) -> Tuple[str, str]:
if (value.count("/") != 1) and (len(value) > 1):
raise argparse.ArgumentTypeError("Must contain a \"/\" character with a prefix and/or suffix")
a, b = value.split("/", 1)
return a, b
# -----------------------------------------------------------------------------
# Argument Handlers ("build" command)
@@ -2949,6 +3023,20 @@ def generic_arg_blender_version(subparse: argparse.ArgumentParser) -> None:
)
def generic_arg_temp_prefix_and_suffix(subparse: argparse.ArgumentParser) -> None:
subparse.add_argument(
"--temp-prefix-and-suffix",
dest="temp_prefix_and_suffix",
default="./.temp",
type=arg_handle_str_as_temp_prefix_and_suffix,
help=(
"The template to use when removing files. "
"A slash separates the: `prefix/suffix`, digits may be appended"
),
required=False,
)
# Only for authoring.
def generic_arg_package_source_path_positional(subparse: argparse.ArgumentParser) -> None:
subparse.add_argument(
@@ -3526,6 +3614,7 @@ class subcmd_client:
filepath_archive: str,
blender_version_tuple: Tuple[int, int, int],
manifest_compare: Optional[PkgManifest],
temp_prefix_and_suffix: Tuple[str, str],
) -> bool:
# NOTE: Don't use `FATAL_ERROR` because other packages will attempt to install.
@@ -3620,7 +3709,10 @@ class subcmd_client:
is_reinstall = False
if os.path.isdir(filepath_local_pkg):
if (error := rmtree_with_fallback_or_error(filepath_local_pkg)) is not None:
if (error := rmtree_with_fallback_or_error_pseudo_atomic(
filepath_local_pkg,
temp_prefix_and_suffix=temp_prefix_and_suffix,
)) is not None:
msglog.error("Failed to remove existing directory for \"{:s}\": {:s}".format(manifest.id, error))
return False
@@ -3643,6 +3735,7 @@ class subcmd_client:
local_dir: str,
package_files: Sequence[str],
blender_version: str,
temp_prefix_and_suffix: Tuple[str, str],
) -> bool:
if not os.path.exists(local_dir):
msglog.fatal_error("destination directory \"{:s}\" does not exist".format(local_dir))
@@ -3664,6 +3757,7 @@ class subcmd_client:
blender_version_tuple=blender_version_tuple,
# There is no manifest from the repository, leave this unset.
manifest_compare=None,
temp_prefix_and_suffix=temp_prefix_and_suffix,
):
# The package failed to install.
continue
@@ -3682,6 +3776,7 @@ class subcmd_client:
blender_version: str,
access_token: str,
timeout_in_seconds: float,
temp_prefix_and_suffix: Tuple[str, str],
) -> bool:
# Validate arguments.
@@ -3907,6 +4002,7 @@ class subcmd_client:
filepath_archive=filepath_local_cache_archive,
blender_version_tuple=blender_version_tuple,
manifest_compare=manifest_archive.manifest,
temp_prefix_and_suffix=temp_prefix_and_suffix,
):
# The package failed to install.
continue
@@ -3920,6 +4016,7 @@ class subcmd_client:
local_dir: str,
user_dir: str,
packages: Sequence[str],
temp_prefix_and_suffix: Tuple[str, str],
) -> bool:
if not os.path.isdir(local_dir):
msglog.fatal_error("Missing local \"{:s}\"".format(local_dir))
@@ -3970,10 +4067,16 @@ class subcmd_client:
for pkg_idname in packages_valid:
filepath_local_pkg = os.path.join(local_dir, pkg_idname)
if (error := rmtree_with_fallback_or_error(filepath_local_pkg)) is not None:
# First try and rename which will fail on WIN32 when one of the files is locked.
if (error := rmtree_with_fallback_or_error_pseudo_atomic(
filepath_local_pkg,
temp_prefix_and_suffix=temp_prefix_and_suffix,
)) is not None:
msglog.error("Failure to remove \"{:s}\" with error ({:s})".format(pkg_idname, error))
else:
msglog.status("Removed \"{:s}\"".format(pkg_idname))
continue
msglog.status("Removed \"{:s}\"".format(pkg_idname))
filepath_local_cache_archive = os.path.join(local_cache_dir, pkg_idname + PKG_EXT)
if os.path.exists(filepath_local_cache_archive):
@@ -3982,7 +4085,10 @@ class subcmd_client:
if user_dir:
filepath_user_pkg = os.path.join(user_dir, pkg_idname)
if os.path.exists(filepath_user_pkg):
if (error := rmtree_with_fallback_or_error(filepath_user_pkg)) is not None:
if (error := rmtree_with_fallback_or_error_pseudo_atomic(
filepath_user_pkg,
temp_prefix_and_suffix=temp_prefix_and_suffix,
)) is not None:
msglog.error(
"Failure to remove \"{:s}\" user files with error ({:s})".format(pkg_idname, error),
)
@@ -4707,6 +4813,8 @@ def argparse_create_client_install_files(subparsers: "argparse._SubParsersAction
generic_arg_local_dir(subparse)
generic_arg_blender_version(subparse)
generic_arg_temp_prefix_and_suffix(subparse)
generic_arg_output_type(subparse)
subparse.set_defaults(
@@ -4715,6 +4823,7 @@ def argparse_create_client_install_files(subparsers: "argparse._SubParsersAction
local_dir=args.local_dir,
package_files=args.files,
blender_version=args.blender_version,
temp_prefix_and_suffix=args.temp_prefix_and_suffix,
),
)
@@ -4735,6 +4844,8 @@ def argparse_create_client_install(subparsers: "argparse._SubParsersAction[argpa
generic_arg_blender_version(subparse)
generic_arg_access_token(subparse)
generic_arg_temp_prefix_and_suffix(subparse)
generic_arg_output_type(subparse)
generic_arg_timeout(subparse)
@@ -4749,6 +4860,7 @@ def argparse_create_client_install(subparsers: "argparse._SubParsersAction[argpa
blender_version=args.blender_version,
access_token=args.access_token,
timeout_in_seconds=args.timeout,
temp_prefix_and_suffix=args.temp_prefix_and_suffix,
),
)
@@ -4764,6 +4876,9 @@ def argparse_create_client_uninstall(subparsers: "argparse._SubParsersAction[arg
generic_arg_local_dir(subparse)
generic_arg_user_dir(subparse)
generic_arg_temp_prefix_and_suffix(subparse)
generic_arg_output_type(subparse)
subparse.set_defaults(
@@ -4772,6 +4887,7 @@ def argparse_create_client_uninstall(subparsers: "argparse._SubParsersAction[arg
local_dir=args.local_dir,
user_dir=args.user_dir,
packages=args.packages.split(","),
temp_prefix_and_suffix=args.temp_prefix_and_suffix,
),
)

63
scripts/modules/_bpy_internal/extensions/stale_file_manager.py
View File

@@ -322,6 +322,55 @@ class StaleFiles:
return result
def state_load_remove_and_store(
self,
*,
# A sequence of absolute paths within `_base_directory`.
paths: Sequence[str],
) -> bool:
# Convenience function for a common operation.
# Return true when one or more items from "paths" were removed from the "state".
self.state_load(check_exists=False)
# Accounts for the common case where nothing has been marked for removal.
if not self._paths:
return False
paths_remove_canonical = {
self._filepath_relative_and_canonicalize(path_abs) for path_abs in paths
if self._filepath_relative_test(path_abs)
}
paths_next = [path for path in self._paths if path not in paths_remove_canonical]
if len(self._paths) == len(paths_next):
return False
self._paths[:] = paths_next
self._is_modified = True
self.state_store(check_exists=False)
return True
def _filepath_relative_test(self, path_abs: str) -> bool:
debug = self._debug
base_directory = self._base_directory
if not path_abs.startswith(base_directory):
if debug:
print(base_directory, "is not a sub-directory", path_abs)
return False
return True
def _filepath_relative_and_canonicalize(self, path_abs: str) -> str:
from os import sep
assert self._filepath_relative_test(path_abs)
path = path_abs[len(self._base_directory):].lstrip(sep)
if sep == "\\":
path = path.replace("\\", "/")
return path
def _filepath_rename_to_stale(self, path_abs: str) -> str:
import os
@@ -358,22 +407,12 @@ class StaleFiles:
return path_abs_stale
def filepath_add(self, path_abs: str, *, rename: bool) -> bool:
from os import sep
base_directory = self._base_directory
debug = self._debug
if not path_abs.startswith(self._base_directory):
if debug:
print(base_directory, "is not a sub-directory", path_abs)
if not self._filepath_relative_test(path_abs):
return False
if rename:
path_abs = self._filepath_rename_to_stale(path_abs)
path = path_abs[len(self._base_directory):].lstrip(sep)
if sep == "\\":
path = path.replace("\\", "/")
path = self._filepath_relative_and_canonicalize(path_abs)
self._is_modified = True
self._paths.append(path)

21
scripts/modules/addon_utils.py
View File

@@ -820,6 +820,27 @@ def stale_pending_stage_paths(path_base, paths):
_stale_pending_stage(debug)
def stale_pending_remove_paths(path_base, paths):
# The reverse of: `stale_pending_stage_paths`.
from _bpy_internal.extensions.stale_file_manager import StaleFiles
debug = _bpy.app.debug_python
stale_handle = StaleFiles(
base_directory=path_base,
stale_filename=_stale_filename,
debug=debug,
)
# Already checked.
if stale_handle.state_load_remove_and_store(paths=paths):
# Don't attempt to reverse the `_stale_pending_stage` call.
# This is not trivial since other repositories may need to be cleared.
# There will be a minor performance hit on restart but this is enough
# of a corner case that it's not worth attempting to calculate if
# removal of pending files is needed or not.
pass
# -----------------------------------------------------------------------------
# Extension Pre-Flight Compatibility Check
#