griefith/test2
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Tools: fix code_clean operating on the same file from multiple processes

Browse Source 
Job splitting logic didn't account for the same file being built
multiple times, causing there to be multiple built arguments per file.

Now multiple build configurations are stored per file.
This commit is contained in:
Campbell Barton
2025-02-18 10:24:20 +11:00
parent 7d53e33889
commit dde68fbf95
1 changed files with 48 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
tools/utils_maintenance/code_clean.py
View File

@@ -1874,29 +1874,35 @@ def wash_source_with_edit(
pass
class FileBuildConfig(NamedTuple):
build_args: Sequence[str]
build_cwd: str | None
output_path: str
def wash_source_with_edit_list(
source: str,
output: str,
build_args: Sequence[str],
build_cwd: str | None,
build_configs: Sequence[FileBuildConfig],
skip_test: bool,
verbose_compile: bool,
verbose_edit_actions: bool,
shared_edit_data: Any,
edit_list: Sequence[str],
) -> None:
for edit_to_apply in edit_list:
wash_source_with_edit(
source,
output,
build_args,
build_cwd,
skip_test,
verbose_compile,
verbose_edit_actions,
shared_edit_data,
edit_to_apply,
)
assert len(build_configs) > 0
for build_cfg in build_configs:
for edit_to_apply in edit_list:
wash_source_with_edit(
source,
build_cfg.output_path,
build_cfg.build_args,
build_cfg.build_cwd,
skip_test,
verbose_compile,
verbose_edit_actions,
shared_edit_data,
edit_to_apply,
)
# -----------------------------------------------------------------------------
@@ -1988,18 +1994,32 @@ def run_edits_on_directory(
return True
return False
# Filter out build args.
args_orig_len = len(args)
args_with_cwd = [
(c, *split_build_args_with_cwd(build_args_str))
for (c, build_args_str) in args
filepaths_unique = {c for c, _ in args}
# Filter out paths.
# NOTE: the same source file may be referenced from multiple `build_args`,
# when running multiple processes it's important that each file references all it's arguments,
# so different processes don't try to manipulate the same file at once.
args_from_file_map: dict[str, list[FileBuildConfig]] = {
c: []
for c in sorted(filepaths_unique)
if test_path(c)
]
}
for (c, build_args_str) in args:
if c not in args_from_file_map:
continue
build_args, build_cwd = split_build_args_with_cwd(build_args_str)
args_from_file_map[c].append(FileBuildConfig(
build_args=build_args,
build_cwd=build_cwd,
output_path=output_from_build_args(build_args, build_cwd),
))
del args
print("Operating on {:d} of {:d} files...".format(len(args_with_cwd), args_orig_len))
for (c, build_args, build_cwd) in args_with_cwd:
print("Operating on {:d} of {:d} files...".format(len(args_from_file_map), len(filepaths_unique)))
for c in args_from_file_map.keys():
print(" ", c)
del args_orig_len
if jobs > 1:
# Group edits to avoid one file holding up the queue before other edits can be worked on.
@@ -2019,26 +2039,22 @@ def run_edits_on_directory(
if jobs > 1:
args_expanded = [(
c,
output_from_build_args(build_args, build_cwd),
build_args,
build_cwd,
build_configs,
skip_test,
verbose_compile,
verbose_edit_actions,
shared_edit_data,
edits_group,
) for (c, build_args, build_cwd) in args_with_cwd]
) for c, build_configs in args_from_file_map.items()]
pool = multiprocessing.Pool(processes=jobs)
pool.starmap(wash_source_with_edit_list, args_expanded)
del args_expanded
else:
# now we have commands
for c, build_args, build_cwd in args_with_cwd:
for c, build_configs in args_from_file_map.items():
wash_source_with_edit_list(
c,
output_from_build_args(build_args, build_cwd),
build_args,
build_cwd,
build_configs,
skip_test,
verbose_compile,
verbose_edit_actions,