diff --git a/tools/utils_maintenance/code_clean.py b/tools/utils_maintenance/code_clean.py index 77f8d0d9e94..d5cb667fdb6 100755 --- a/tools/utils_maintenance/code_clean.py +++ b/tools/utils_maintenance/code_clean.py @@ -1874,29 +1874,35 @@ def wash_source_with_edit( pass +class FileBuildConfig(NamedTuple): + build_args: Sequence[str] + build_cwd: str | None + output_path: str + + def wash_source_with_edit_list( source: str, - output: str, - build_args: Sequence[str], - build_cwd: str | None, + build_configs: Sequence[FileBuildConfig], skip_test: bool, verbose_compile: bool, verbose_edit_actions: bool, shared_edit_data: Any, edit_list: Sequence[str], ) -> None: - for edit_to_apply in edit_list: - wash_source_with_edit( - source, - output, - build_args, - build_cwd, - skip_test, - verbose_compile, - verbose_edit_actions, - shared_edit_data, - edit_to_apply, - ) + assert len(build_configs) > 0 + for build_cfg in build_configs: + for edit_to_apply in edit_list: + wash_source_with_edit( + source, + build_cfg.output_path, + build_cfg.build_args, + build_cfg.build_cwd, + skip_test, + verbose_compile, + verbose_edit_actions, + shared_edit_data, + edit_to_apply, + ) # ----------------------------------------------------------------------------- @@ -1988,18 +1994,32 @@ def run_edits_on_directory( return True return False - # Filter out build args. - args_orig_len = len(args) - args_with_cwd = [ - (c, *split_build_args_with_cwd(build_args_str)) - for (c, build_args_str) in args + filepaths_unique = {c for c, _ in args} + + # Filter out paths. + # NOTE: the same source file may be referenced from multiple `build_args`, + # when running multiple processes it's important that each file references all it's arguments, + # so different processes don't try to manipulate the same file at once. + args_from_file_map: dict[str, list[FileBuildConfig]] = { + c: [] + for c in sorted(filepaths_unique) if test_path(c) - ] + } + + for (c, build_args_str) in args: + if c not in args_from_file_map: + continue + build_args, build_cwd = split_build_args_with_cwd(build_args_str) + args_from_file_map[c].append(FileBuildConfig( + build_args=build_args, + build_cwd=build_cwd, + output_path=output_from_build_args(build_args, build_cwd), + )) + del args - print("Operating on {:d} of {:d} files...".format(len(args_with_cwd), args_orig_len)) - for (c, build_args, build_cwd) in args_with_cwd: + print("Operating on {:d} of {:d} files...".format(len(args_from_file_map), len(filepaths_unique))) + for c in args_from_file_map.keys(): print(" ", c) - del args_orig_len if jobs > 1: # Group edits to avoid one file holding up the queue before other edits can be worked on. @@ -2019,26 +2039,22 @@ def run_edits_on_directory( if jobs > 1: args_expanded = [( c, - output_from_build_args(build_args, build_cwd), - build_args, - build_cwd, + build_configs, skip_test, verbose_compile, verbose_edit_actions, shared_edit_data, edits_group, - ) for (c, build_args, build_cwd) in args_with_cwd] + ) for c, build_configs in args_from_file_map.items()] pool = multiprocessing.Pool(processes=jobs) pool.starmap(wash_source_with_edit_list, args_expanded) del args_expanded else: # now we have commands - for c, build_args, build_cwd in args_with_cwd: + for c, build_configs in args_from_file_map.items(): wash_source_with_edit_list( c, - output_from_build_args(build_args, build_cwd), - build_args, - build_cwd, + build_configs, skip_test, verbose_compile, verbose_edit_actions,