Cleanup: update use of typing in for Python scripts

This commit is contained in:
Campbell Barton
2024-10-23 12:48:09 +11:00
parent 39b9863cca
commit a0453ab87a
40 changed files with 287 additions and 367 deletions

View File

@@ -21,12 +21,6 @@ import sys
import subprocess
import argparse
from typing import (
List,
Tuple,
Optional,
)
# Temporary, until all platforms update to 2.3.1.
VERSION_MIN = (1, 6, 0)
VERSION_MAX_RECOMMENDED = (2, 3, 1)
@@ -55,7 +49,7 @@ ignore_files = {
}
def compute_paths(paths: List[str], use_default_paths: bool) -> List[str]:
def compute_paths(paths: list[str], use_default_paths: bool) -> list[str]:
# Optionally pass in files to operate on.
if use_default_paths:
paths = [
@@ -79,7 +73,7 @@ def compute_paths(paths: List[str], use_default_paths: bool) -> List[str]:
return paths
def source_files_from_git(paths: List[str], changed_only: bool) -> List[str]:
def source_files_from_git(paths: list[str], changed_only: bool) -> list[str]:
if changed_only:
cmd = ("git", "diff", "HEAD", "--name-only", "-z", "--", *paths)
else:
@@ -88,22 +82,22 @@ def source_files_from_git(paths: List[str], changed_only: bool) -> List[str]:
return [f.decode('ascii') for f in files]
def autopep8_parse_version(version: str) -> Tuple[int, int, int]:
def autopep8_parse_version(version: str) -> tuple[int, int, int]:
# Ensure exactly 3 numbers.
major, minor, patch = (tuple(int(n) for n in version.split("-")[0].split(".")) + (0, 0, 0))[0:3]
return major, minor, patch
def version_str_from_tuple(version: Tuple[int, ...]) -> str:
def version_str_from_tuple(version: tuple[int, ...]) -> str:
return ".".join(str(x) for x in version)
def autopep8_ensure_version_from_command(
autopep8_format_cmd_argument: str,
) -> Optional[Tuple[str, Tuple[int, int, int]]]:
) -> tuple[str, tuple[int, int, int]] | None:
# The version to parse.
version_str: Optional[str] = None
version_str: str | None = None
global AUTOPEP8_FORMAT_CMD
autopep8_format_cmd = None
@@ -142,10 +136,10 @@ def autopep8_ensure_version_from_command(
return None
def autopep8_ensure_version_from_module() -> Optional[Tuple[str, Tuple[int, int, int]]]:
def autopep8_ensure_version_from_module() -> tuple[str, tuple[int, int, int]] | None:
# The version to parse.
version_str: Optional[str] = None
version_str: str | None = None
# Extract the version from the module.
try:
@@ -164,7 +158,7 @@ def autopep8_ensure_version_from_module() -> Optional[Tuple[str, Tuple[int, int,
return None
def autopep8_format(files: List[str]) -> bytes:
def autopep8_format(files: list[str]) -> bytes:
cmd = [
AUTOPEP8_FORMAT_CMD,
*AUTOPEP8_FORMAT_DEFAULT_ARGS,
@@ -178,7 +172,7 @@ def autopep8_format(files: List[str]) -> bytes:
return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
def autopep8_format_no_subprocess(files: List[str]) -> None:
def autopep8_format_no_subprocess(files: list[str]) -> None:
cmd = [
*AUTOPEP8_FORMAT_DEFAULT_ARGS,
*files

View File

@@ -11,10 +11,6 @@ sys.path.append(os.path.join(PWD, "modules"))
from batch_edit_text import run
from typing import (
Optional,
)
SOURCE_DIR = os.path.normpath(os.path.abspath(os.path.normpath(os.path.join(PWD, "..", ".."))))
# TODO, move to config file
@@ -31,7 +27,7 @@ SOURCE_EXT = (
)
def sort_struct_lists(fn: str, data_src: str) -> Optional[str]:
def sort_struct_lists(fn: str, data_src: str) -> str | None:
import re
# eg:
@@ -53,7 +49,7 @@ def sort_struct_lists(fn: str, data_src: str) -> Optional[str]:
lines = data_src.splitlines(keepends=True)
def can_sort(l: str) -> Optional[int]:
def can_sort(l: str) -> int | None:
if re_match_struct.match(l):
return 1
if re_match_struct_type.match(l):

View File

@@ -15,11 +15,6 @@ import os
import sys
import re
from typing import (
Dict,
Optional,
)
PWD = os.path.dirname(__file__)
sys.path.append(os.path.join(PWD, "modules"))
@@ -44,11 +39,11 @@ re_words = re.compile("[A-Za-z_][A-Za-z_0-9]*")
re_match_struct = re.compile(r"struct\s+([A-Za-z_][A-Za-z_0-9]*)\s*;")
def clean_structs(fn: str, data_src: str) -> Optional[str]:
def clean_structs(fn: str, data_src: str) -> str | None:
from pygments.token import Token
from pygments import lexers
word_occurance: Dict[str, int] = {}
word_occurance: dict[str, int] = {}
lex = lexers.get_lexer_by_name("c++")
lex.get_tokens(data_src)

View File

@@ -18,12 +18,8 @@ import os
import sys
import subprocess
from typing import (
List,
Optional,
from collections.abc import (
Sequence,
Set,
Tuple,
)
VERSION_MIN = (17, 0, 6)
@@ -49,7 +45,7 @@ extensions_only_retab = (
)
# Add files which are too large/heavy to format.
ignore_files: Set[str] = set([
ignore_files: set[str] = set([
# Currently empty, looks like.
# "intern/cycles/render/sobol.cpp",
])
@@ -66,7 +62,7 @@ ignore_directories = {
}
def compute_paths(paths: List[str], use_default_paths: bool) -> List[str]:
def compute_paths(paths: list[str], use_default_paths: bool) -> list[str]:
# The resulting paths:
# - Use forward slashes on all systems.
# - Are relative to the GIT repository without any `.` or `./` prefix.
@@ -91,7 +87,7 @@ def compute_paths(paths: List[str], use_default_paths: bool) -> List[str]:
return paths
def source_files_from_git(paths: Sequence[str], changed_only: bool) -> List[str]:
def source_files_from_git(paths: Sequence[str], changed_only: bool) -> list[str]:
if changed_only:
cmd = ("git", "diff", "HEAD", "--name-only", "-z", "--", *paths)
else:
@@ -126,7 +122,7 @@ def convert_tabs_to_spaces(files: Sequence[str]) -> None:
fh.write(data)
def clang_format_ensure_version() -> Optional[Tuple[int, int, int]]:
def clang_format_ensure_version() -> tuple[int, int, int] | None:
global CLANG_FORMAT_CMD
clang_format_cmd = None
version_output = ""
@@ -142,18 +138,18 @@ def clang_format_ensure_version() -> Optional[Tuple[int, int, int]]:
continue
CLANG_FORMAT_CMD = clang_format_cmd
break
version: Optional[str] = next(iter(v for v in version_output.split() if v[0].isdigit()), None)
version: str | None = next(iter(v for v in version_output.split() if v[0].isdigit()), None)
if version is None:
return None
version = version.split("-")[0]
# Ensure exactly 3 numbers.
version_num: Tuple[int, int, int] = (tuple(int(n) for n in version.split(".")) + (0, 0, 0))[:3] # type: ignore
version_num: tuple[int, int, int] = (tuple(int(n) for n in version.split(".")) + (0, 0, 0))[:3] # type: ignore
print("Using {:s} ({:d}.{:d}.{:d})...".format(CLANG_FORMAT_CMD, version_num[0], version_num[1], version_num[2]))
return version_num
def clang_format_file(files: List[str]) -> bytes:
def clang_format_file(files: list[str]) -> bytes:
cmd = [
CLANG_FORMAT_CMD,
# Update the files in-place.
@@ -168,7 +164,7 @@ def clang_print_output(output: bytes) -> None:
print(output.decode('utf8', errors='ignore').strip())
def clang_format(files: List[str]) -> None:
def clang_format(files: list[str]) -> None:
pool = multiprocessing.Pool()
# Process in chunks to reduce overhead of starting processes.

View File

@@ -12,10 +12,6 @@ Sorts CMake path lists
import os
import sys
from typing import (
Optional,
)
PWD = os.path.dirname(__file__)
sys.path.append(os.path.join(PWD, "modules"))
@@ -37,7 +33,7 @@ SOURCE_EXT = (
)
def sort_cmake_file_lists(fn: str, data_src: str) -> Optional[str]:
def sort_cmake_file_lists(fn: str, data_src: str) -> str | None:
fn_dir = os.path.dirname(fn)
lines = data_src.splitlines(keepends=True)

View File

@@ -20,18 +20,14 @@ import string
from typing import (
Any,
Dict,
Generator,
List,
Optional,
)
from collections.abc import (
Iterator,
Sequence,
Set,
Tuple,
Type,
)
# List of (source_file, all_arguments)
ProcessedCommands = List[Tuple[str, str]]
ProcessedCommands = list[tuple[str, str]]
BASE_DIR = os.path.abspath(os.path.dirname(__file__))
@@ -110,7 +106,7 @@ def line_from_span(text: str, start: int, end: int) -> str:
return text[start:end]
def files_recursive_with_ext(path: str, ext: Tuple[str, ...]) -> Generator[str, None, None]:
def files_recursive_with_ext(path: str, ext: tuple[str, ...]) -> Iterator[str]:
for dirpath, dirnames, filenames in os.walk(path):
# skip '.git' and other dot-files.
dirnames[:] = [d for d in dirnames if not d.startswith(".")]
@@ -264,7 +260,7 @@ def text_cxx_in_macro_definition(data: str, pos: int) -> bool:
def run(
args: Sequence[str],
*,
cwd: Optional[str],
cwd: str | None,
quiet: bool,
verbose_compile: bool,
) -> int:
@@ -286,7 +282,7 @@ def run(
# -----------------------------------------------------------------------------
# Build System Access
def cmake_cache_var(cmake_dir: str, var: str) -> Optional[str]:
def cmake_cache_var(cmake_dir: str, var: str) -> str | None:
with open(os.path.join(cmake_dir, "CMakeCache.txt"), encoding='utf-8') as cache_file:
lines = [
l_strip for l in cache_file
@@ -300,7 +296,7 @@ def cmake_cache_var(cmake_dir: str, var: str) -> Optional[str]:
return None
def cmake_cache_var_is_true(cmake_var: Optional[str]) -> bool:
def cmake_cache_var_is_true(cmake_var: str | None) -> bool:
if cmake_var is None:
return False
@@ -316,7 +312,7 @@ def cmake_cache_var_is_true(cmake_var: Optional[str]) -> bool:
RE_CFILE_SEARCH = re.compile(r"\s\-c\s([\S]+)")
def process_commands(cmake_dir: str, data: Sequence[str]) -> Optional[ProcessedCommands]:
def process_commands(cmake_dir: str, data: Sequence[str]) -> ProcessedCommands | None:
compiler_c = cmake_cache_var(cmake_dir, "CMAKE_C_COMPILER")
compiler_cxx = cmake_cache_var(cmake_dir, "CMAKE_CXX_COMPILER")
if compiler_c is None:
@@ -354,7 +350,7 @@ def process_commands(cmake_dir: str, data: Sequence[str]) -> Optional[ProcessedC
return file_args
def find_build_args_ninja(build_dir: str) -> Optional[ProcessedCommands]:
def find_build_args_ninja(build_dir: str) -> ProcessedCommands | None:
import time
cmake_dir = build_dir
make_exe = "ninja"
@@ -374,7 +370,7 @@ def find_build_args_ninja(build_dir: str) -> Optional[ProcessedCommands]:
return process_commands(cmake_dir, data)
def find_build_args_make(build_dir: str) -> Optional[ProcessedCommands]:
def find_build_args_make(build_dir: str) -> ProcessedCommands | None:
import time
make_exe = "make"
with subprocess.Popen(
@@ -446,11 +442,11 @@ class EditGenerator:
if getattr(cls, "edit_list_from_file") is EditGenerator.edit_list_from_file:
raise Exception("Class {!r} missing \"edit_list_from_file\" callback!".format(cls))
def __new__(cls, *args: Tuple[Any], **kwargs: Dict[str, Any]) -> Any:
def __new__(cls, *args: tuple[Any], **kwargs: dict[str, Any]) -> Any:
raise RuntimeError("Class {!r} should not be instantiated".format(cls))
@staticmethod
def edit_list_from_file(_source: str, _data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(_source: str, _data: str, _shared_edit_data: Any) -> list[Edit]:
# The `__init_subclass__` function ensures this is always overridden.
raise RuntimeError("This function must be overridden by it's subclass!")
return []
@@ -482,7 +478,7 @@ class edit_generators:
is_default = False
@staticmethod
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits = []
for match in re.finditer(r"sizeof\(([a-zA-Z_]+)\) \* (\d+) \* (\d+)", data):
@@ -539,7 +535,7 @@ class edit_generators:
is_default = False
@staticmethod
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits = []
# `float abc[3] = {0, 1, 2};` -> `const float abc[3] = {0, 1, 2};`
@@ -595,7 +591,7 @@ class edit_generators:
is_default = True
@staticmethod
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits = []
# `1.f` -> `1.0f`
@@ -628,7 +624,7 @@ class edit_generators:
is_default = True
@staticmethod
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits = []
# Keep `typedef` unsigned as some files have local types, e.g.
@@ -678,8 +674,8 @@ class edit_generators:
is_default = True
@staticmethod
def edit_list_from_file(source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
edits: List[Edit] = []
def edit_list_from_file(source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits: list[Edit] = []
# The user might include C & C++, if they forget, it is better not to operate on C.
if source.lower().endswith((".h", ".c")):
@@ -716,8 +712,8 @@ class edit_generators:
is_default = True
@staticmethod
def edit_list_from_file(source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
edits: List[Edit] = []
def edit_list_from_file(source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits: list[Edit] = []
# The user might include C & C++, if they forget, it is better not to operate on C.
if source.lower().endswith((".h", ".c")):
@@ -744,8 +740,8 @@ class edit_generators:
is_default = True
@staticmethod
def edit_list_from_file(source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
edits: List[Edit] = []
def edit_list_from_file(source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits: list[Edit] = []
# The user might include C & C++, if they forget, it is better not to operate on C.
if source.lower().endswith((".h", ".c")):
@@ -787,7 +783,7 @@ class edit_generators:
is_default = True
@staticmethod
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits = []
for use_brackets in (True, False):
@@ -862,7 +858,7 @@ class edit_generators:
is_default = True
@staticmethod
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits = []
for use_brackets in (True, False):
@@ -947,7 +943,7 @@ class edit_generators:
is_default = False
@staticmethod
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits = []
# for match in re.finditer(r"( [a-zA-Z0-9_]+ [a-zA-Z0-9_]+ = [A-Z][A-Z_0-9_]*;)", data):
@@ -978,7 +974,7 @@ class edit_generators:
is_default = True
@staticmethod
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits = []
# Keep:
@@ -1016,7 +1012,7 @@ class edit_generators:
is_default = True
@staticmethod
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits = []
# Remove `return (NULL);`
@@ -1045,7 +1041,7 @@ class edit_generators:
is_default = True
@staticmethod
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits = []
# `strcmp(a, b) == 0` -> `STREQ(a, b)`
@@ -1090,8 +1086,8 @@ class edit_generators:
is_default = True
@staticmethod
def edit_list_from_file(source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
edits: List[Edit] = []
def edit_list_from_file(source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits: list[Edit] = []
# The user might include C & C++, if they forget, it is better not to operate on C.
if source.lower().endswith((".h", ".c")):
@@ -1130,7 +1126,7 @@ class edit_generators:
is_default = True
@staticmethod
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits = []
# `BLI_strncpy(a, b, sizeof(a))` -> `STRNCPY(a, b)`
@@ -1190,7 +1186,7 @@ class edit_generators:
is_default = True
@staticmethod
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits = []
# Note that this replacement is only valid in some cases,
# so only apply with validation that binary output matches.
@@ -1217,7 +1213,7 @@ class edit_generators:
is_default = False
@staticmethod
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits = []
re_cxx_cast = re.compile(r"[a-z_]+<([^\>]+)>\((.*)\)")
@@ -1358,7 +1354,7 @@ class edit_generators:
is_default = False
@staticmethod
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(_source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits = []
# Give up after searching for a bracket this many characters and finding none.
@@ -1483,7 +1479,7 @@ class edit_generators:
def setup(cls) -> Any:
# For each file replace `pragma once` with old-style header guard.
# This is needed so we can remove the header with the knowledge the source file didn't use it indirectly.
files: List[Tuple[str, str, str, str]] = []
files: list[tuple[str, str, str, str]] = []
shared_edit_data = {
'files': files,
}
@@ -1529,7 +1525,7 @@ class edit_generators:
fh.write(data)
@classmethod
def edit_list_from_file(cls, _source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(cls, _source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits = []
# Remove include.
@@ -1569,9 +1565,9 @@ class edit_generators:
is_default = True
@staticmethod
def edit_list_from_file(source: str, data: str, _shared_edit_data: Any) -> List[Edit]:
def edit_list_from_file(source: str, data: str, _shared_edit_data: Any) -> list[Edit]:
edits: List[Edit] = []
edits: list[Edit] = []
# The user might include C & C++, if they forget, it is better not to operate on C.
if source.lower().endswith((".h", ".c")):
@@ -1650,9 +1646,9 @@ class edit_generators:
def test_edit(
source: str,
output: str,
output_bytes: Optional[bytes],
output_bytes: bytes | None,
build_args: Sequence[str],
build_cwd: Optional[str],
build_cwd: str | None,
data: str,
data_test: str,
*,
@@ -1697,7 +1693,7 @@ def test_edit(
# -----------------------------------------------------------------------------
# List Fix Functions
def edit_function_get_all(*, is_default: Optional[bool] = None) -> List[str]:
def edit_function_get_all(*, is_default: bool | None = None) -> list[str]:
fixes = []
for name in dir(edit_generators):
value = getattr(edit_generators, name)
@@ -1710,7 +1706,7 @@ def edit_function_get_all(*, is_default: Optional[bool] = None) -> List[str]:
return fixes
def edit_class_from_id(name: str) -> Type[EditGenerator]:
def edit_class_from_id(name: str) -> type[EditGenerator]:
result = getattr(edit_generators, name)
assert issubclass(result, EditGenerator)
# MYPY 0.812 doesn't recognize the assert above.
@@ -1769,7 +1765,7 @@ def wash_source_with_edit(
source: str,
output: str,
build_args: Sequence[str],
build_cwd: Optional[str],
build_cwd: str | None,
skip_test: bool,
verbose_compile: bool,
verbose_edit_actions: bool,
@@ -1788,7 +1784,7 @@ def wash_source_with_edit(
#
# This is a heavy solution that guarantees edits never oscillate between
# multiple states, so re-visiting a previously visited state will always exit.
data_states: Set[str] = set()
data_states: set[str] = set()
# When overlapping edits are found, keep attempting edits.
edit_again = True
@@ -1888,7 +1884,7 @@ def wash_source_with_edit_list(
source: str,
output: str,
build_args: Sequence[str],
build_cwd: Optional[str],
build_cwd: str | None,
skip_test: bool,
verbose_compile: bool,
verbose_edit_actions: bool,
@@ -1915,7 +1911,7 @@ def wash_source_with_edit_list(
def run_edits_on_directory(
*,
build_dir: str,
regex_list: List[re.Pattern[str]],
regex_list: list[re.Pattern[str]],
edits_to_apply: Sequence[str],
skip_test: bool,
jobs: int,
@@ -1957,7 +1953,7 @@ def run_edits_on_directory(
os.path.join("source"),
)
def split_build_args_with_cwd(build_args_str: str) -> Tuple[Sequence[str], Optional[str]]:
def split_build_args_with_cwd(build_args_str: str) -> tuple[Sequence[str], str | None]:
import shlex
build_args = shlex.split(build_args_str)
@@ -1968,7 +1964,7 @@ def run_edits_on_directory(
del build_args[0:3]
return build_args, cwd
def output_from_build_args(build_args: Sequence[str], cwd: Optional[str]) -> str:
def output_from_build_args(build_args: Sequence[str], cwd: str | None) -> str:
i = build_args.index("-o")
# Assume the output is a relative path is a CWD was set.
if cwd:

View File

@@ -12,9 +12,11 @@ import sys
from pathlib import Path
from typing import (
Iterator,
NamedTuple,
)
from collections.abc import (
Iterator,
)
# -----------------------------------------------------------------------------
# Path Constants

View File

@@ -2,10 +2,9 @@
#
# SPDX-License-Identifier: GPL-2.0-or-later
from typing import (
from collections.abc import (
Callable,
Generator,
Optional,
Iterator,
Sequence,
)
@@ -13,7 +12,7 @@ TextOpFn = Callable[
# file_name, data_src
[str, str],
# data_dst or None when no change is made.
Optional[str]
str | None,
]
@@ -40,7 +39,7 @@ def run(
import os
def source_files(path: str) -> Generator[str, None, None]:
def source_files(path: str) -> Iterator[str]:
for dirpath, dirnames, filenames in os.walk(path):
dirnames[:] = [d for d in dirnames if not d.startswith(".")]
for filename in filenames: