griefith/test
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Move several py scripts from build_files to tools.

Browse Source 
Preliminary step towards adding 'system python' validation for some
build-essential py scripts (!130746).

Pull Request: https://projects.blender.org/blender/blender/pulls/132025
This commit is contained in:
Bastien Montagne
2024-12-17 19:02:45 +01:00
committed by Bastien Montagne
parent ef0fcab8b7
commit e83d87f588
8 changed files with 10 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tools/check_source/check_mypy_config.py
View File

@@ -43,9 +43,9 @@ PATHS_EXCLUDE = set(
os.path.join(SOURCE_DIR, p.replace("/", os.sep))
for p in
(
"build_files/cmake/clang_array_check.py",
"build_files/cmake/cmake_netbeans_project.py",
"build_files/cmake/cmake_qtcreator_project.py",
"tools/check_source/clang_array_check.py",
"tools/utils_ide/cmake_netbeans_project.py",
"tools/utils_ide/cmake_qtcreator_project.py",
"release/datafiles/blender_icons_geom.py", # Uses `bpy` too much.
"tests/utils/bl_run_operators.py", # Uses `bpy` too much.
"tests/utils/bl_run_operators_event_simulate.py", # Uses `bpy` too much.

388
tools/check_source/clang_array_check.py Normal file
View File

@@ -0,0 +1,388 @@
# SPDX-FileCopyrightText: 2012 Blender Authors
#
# SPDX-License-Identifier: Apache-2.0
"""
Invocation:
export CLANG_BIND_DIR="/dsk/src/llvm/tools/clang/bindings/python"
export CLANG_LIB_DIR="/opt/llvm/lib"
python clang_array_check.py somefile.c -DSOME_DEFINE -I/some/include
... defines and includes are optional
"""
# delay parsing functions until we need them
USE_LAZY_INIT = True
USE_EXACT_COMPARE = False
# -----------------------------------------------------------------------------
# predefined function/arg sizes, handy sometimes, but not complete...
defs_precalc = {
"glColor3bv": {0: 3},
"glColor4bv": {0: 4},
"glColor3ubv": {0: 3},
"glColor4ubv": {0: 4},
"glColor3usv": {0: 3},
"glColor4usv": {0: 4},
"glColor3fv": {0: 3},
"glColor4fv": {0: 4},
"glColor3dv": {0: 3},
"glColor4dv": {0: 4},
"glVertex2fv": {0: 2},
"glVertex3fv": {0: 3},
"glVertex4fv": {0: 4},
"glEvalCoord1fv": {0: 1},
"glEvalCoord1dv": {0: 1},
"glEvalCoord2fv": {0: 2},
"glEvalCoord2dv": {0: 2},
"glRasterPos2dv": {0: 2},
"glRasterPos3dv": {0: 3},
"glRasterPos4dv": {0: 4},
"glRasterPos2fv": {0: 2},
"glRasterPos3fv": {0: 3},
"glRasterPos4fv": {0: 4},
"glRasterPos2sv": {0: 2},
"glRasterPos3sv": {0: 3},
"glRasterPos4sv": {0: 4},
"glTexCoord2fv": {0: 2},
"glTexCoord3fv": {0: 3},
"glTexCoord4fv": {0: 4},
"glTexCoord2dv": {0: 2},
"glTexCoord3dv": {0: 3},
"glTexCoord4dv": {0: 4},
"glNormal3fv": {0: 3},
"glNormal3dv": {0: 3},
"glNormal3bv": {0: 3},
"glNormal3iv": {0: 3},
"glNormal3sv": {0: 3},
# GPU immediate mode.
"immVertex2iv": {1: 2},
"immVertex2fv": {1: 2},
"immVertex3fv": {1: 3},
"immAttr2fv": {1: 2},
"immAttr3fv": {1: 3},
"immAttr4fv": {1: 4},
"immAttr3ubv": {1: 3},
"immAttr4ubv": {1: 4},
"immUniform2fv": {1: 2},
"immUniform3fv": {1: 3},
"immUniform4fv": {1: 4},
"immUniformColor3fv": {0: 3},
"immUniformColor4fv": {0: 4},
"immUniformColor3ubv": {1: 3},
"immUniformColor4ubv": {1: 4},
"immUniformColor3fvAlpha": {0: 3},
"immUniformColor4fvAlpha": {0: 4},
}
# -----------------------------------------------------------------------------
import sys
if 0:
# Examples with LLVM as the root dir: '/dsk/src/llvm'
# path containing 'clang/__init__.py'
CLANG_BIND_DIR = "/dsk/src/llvm/tools/clang/bindings/python"
# path containing libclang.so
CLANG_LIB_DIR = "/opt/llvm/lib"
else:
import os
CLANG_BIND_DIR = os.environ.get("CLANG_BIND_DIR")
CLANG_LIB_DIR = os.environ.get("CLANG_LIB_DIR")
if CLANG_BIND_DIR is None:
print("$CLANG_BIND_DIR python binding dir not set")
if CLANG_LIB_DIR is None:
print("$CLANG_LIB_DIR clang lib dir not set")
if CLANG_BIND_DIR:
sys.path.append(CLANG_BIND_DIR)
import clang
import clang.cindex
from clang.cindex import (CursorKind,
TypeKind,
TokenKind)
if CLANG_LIB_DIR:
clang.cindex.Config.set_library_path(CLANG_LIB_DIR)
index = clang.cindex.Index.create()
args = sys.argv[2:]
# print(args)
tu = index.parse(sys.argv[1], args)
# print('Translation unit: %s' % tu.spelling)
filepath = tu.spelling
# -----------------------------------------------------------------------------
def function_parm_wash_tokens(parm):
# print(parm.kind)
assert parm.kind in (CursorKind.PARM_DECL,
CursorKind.VAR_DECL, # XXX, double check this
CursorKind.FIELD_DECL,
)
"""
Return tokens without trailing commands and 'const'
"""
tokens = [t for t in parm.get_tokens()]
if not tokens:
return tokens
# if tokens[-1].kind == To
# remove trailing char
if tokens[-1].kind == TokenKind.PUNCTUATION:
if tokens[-1].spelling in {",", ")", ";"}:
tokens.pop()
# else:
# print(tokens[-1].spelling)
t_new = []
for t in tokens:
t_kind = t.kind
t_spelling = t.spelling
ok = True
if t_kind == TokenKind.KEYWORD:
if t_spelling in {"const", "restrict", "volatile"}:
ok = False
elif t_spelling.startswith("__"):
ok = False # __restrict
elif t_kind in (TokenKind.COMMENT, ):
ok = False
# Use these
elif t_kind in (TokenKind.LITERAL,
TokenKind.PUNCTUATION,
TokenKind.IDENTIFIER):
# use but ignore
pass
else:
print("Unknown!", t_kind, t_spelling)
# if its OK we will add
if ok:
t_new.append(t)
return t_new
def parm_size(node_child):
tokens = function_parm_wash_tokens(node_child)
# print(" ".join([t.spelling for t in tokens]))
# NOT PERFECT CODE, EXTRACT SIZE FROM TOKENS
if len(tokens) >= 3: # foo [ 1 ]
if ((tokens[-3].kind == TokenKind.PUNCTUATION and tokens[-3].spelling == "[") and
(tokens[-2].kind == TokenKind.LITERAL and tokens[-2].spelling.isdigit()) and
(tokens[-1].kind == TokenKind.PUNCTUATION and tokens[-1].spelling == "]")):
# ---
return int(tokens[-2].spelling)
return -1
def function_get_arg_sizes(node):
# Return a dict if (index: size) items
# {arg_indx: arg_array_size, ... ]
arg_sizes = {}
if 1: # node.spelling == "BM_vert_create", for debugging
node_parms = [node_child for node_child in node.get_children()
if node_child.kind == CursorKind.PARM_DECL]
for i, node_child in enumerate(node_parms):
# print(node_child.kind, node_child.spelling)
# print(node_child.type.kind, node_child.spelling)
if node_child.type.kind == TypeKind.CONSTANTARRAY:
pointee = node_child.type.get_pointee()
size = parm_size(node_child)
if size != -1:
arg_sizes[i] = size
return arg_sizes
# -----------------------------------------------------------------------------
_defs = {}
def lookup_function_size_def(func_id):
if USE_LAZY_INIT:
result = _defs.get(func_id, {})
if type(result) != dict:
result = _defs[func_id] = function_get_arg_sizes(result)
return result
else:
return _defs.get(func_id, {})
# -----------------------------------------------------------------------------
def file_check_arg_sizes(tu):
# main checking function
def validate_arg_size(node):
"""
Loop over args and validate sizes for args we KNOW the size of.
"""
assert node.kind == CursorKind.CALL_EXPR
if 0:
print("---",
" <~> ".join(
[" ".join([t.spelling for t in C.get_tokens()])
for C in node.get_children()]
))
# print(node.location)
# first child is the function call, skip that.
children = list(node.get_children())
if not children:
return # XXX, look into this, happens on C++
func = children[0]
# get the func declaration!
# works but we can better scan for functions ahead of time.
if 0:
func_dec = func.get_definition()
if func_dec:
print("FD", " ".join([t.spelling for t in func_dec.get_tokens()]))
else:
# HRMP'f - why does this fail?
print("AA", " ".join([t.spelling for t in node.get_tokens()]))
else:
args_size_definition = () # dummy
# get the key
tok = list(func.get_tokens())
if tok:
func_id = tok[0].spelling
args_size_definition = lookup_function_size_def(func_id)
if not args_size_definition:
return
children = children[1:]
for i, node_child in enumerate(children):
children = list(node_child.get_children())
# skip if we don't have an index...
size_def = args_size_definition.get(i, -1)
if size_def == -1:
continue
# print([c.kind for c in children])
# print(" ".join([t.spelling for t in node_child.get_tokens()]))
if len(children) == 1:
arg = children[0]
if arg.kind in (CursorKind.DECL_REF_EXPR,
CursorKind.UNEXPOSED_EXPR):
if arg.type.kind == TypeKind.CONSTANTARRAY:
dec = arg.get_definition()
if dec:
size = parm_size(dec)
# size == 0 is for 'float *a'
if size != -1 and size != 0:
# nice print!
if 0:
print("".join([t.spelling for t in func.get_tokens()]),
i,
" ".join([t.spelling for t in dec.get_tokens()]))
# testing
# size_def = 100
if size != 1:
if USE_EXACT_COMPARE:
# is_err = (size != size_def) and (size != 4 and size_def != 3)
is_err = (size != size_def)
else:
is_err = (size < size_def)
if is_err:
location = node.location
# if "math_color_inline.c" not in str(location.file):
if 1:
print("%s:%d:%d: argument %d is size %d, should be %d (from %s)" %
(location.file,
location.line,
location.column,
i + 1, size, size_def,
filepath # always the same but useful when running threaded
))
# we don't really care what we are looking at, just scan entire file for
# function calls.
def recursive_func_call_check(node):
if node.kind == CursorKind.CALL_EXPR:
validate_arg_size(node)
for c in node.get_children():
recursive_func_call_check(c)
recursive_func_call_check(tu.cursor)
# -- first pass, cache function definitions sizes
# PRINT FUNC DEFINES
def recursive_arg_sizes(node, ):
# print(node.kind, node.spelling)
if node.kind == CursorKind.FUNCTION_DECL:
if USE_LAZY_INIT:
args_sizes = node
else:
args_sizes = function_get_arg_sizes(node)
# if args_sizes:
# print(node.spelling, args_sizes)
_defs[node.spelling] = args_sizes
# print("adding", node.spelling)
for c in node.get_children():
recursive_arg_sizes(c)
# cache function sizes
recursive_arg_sizes(tu.cursor)
_defs.update(defs_precalc)
# --- second pass, check against def's
file_check_arg_sizes(tu)

587
tools/check_source/static_check_clang.py Normal file
View File

@@ -0,0 +1,587 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2023 Blender Authors
#
# SPDX-License-Identifier: GPL-2.0-or-later
"""
A command line utility to check Blender's source code with CLANG's Python module.
To call this directly:
export CLANG_LIB_DIR=/usr/lib64
cd {BUILD_DIR}
python ../blender/tools/check_source/static_check_clang.py --match=".*" --checks=struct_comments
"""
import argparse
import os
import re
import sys
from typing import (
Any,
)
from collections.abc import (
Sequence,
)
import project_source_info
# pylint: disable-next=import-outside-toplevel
import clang # type: ignore
# pylint: disable-next=import-outside-toplevel
import clang.cindex # type: ignore
from clang.cindex import (
CursorKind,
)
# Only for readability.
ClangNode = Any
ClangTranslationUnit = Any
ClangSourceLocation = Any
USE_VERBOSE = os.environ.get("VERBOSE", None) is not None
CLANG_BIND_DIR = os.environ.get("CLANG_BIND_DIR")
CLANG_LIB_DIR = os.environ.get("CLANG_LIB_DIR")
if CLANG_BIND_DIR is None:
print("$CLANG_BIND_DIR python binding dir not set")
if CLANG_LIB_DIR is None:
print("$CLANG_LIB_DIR clang lib dir not set")
if CLANG_LIB_DIR:
clang.cindex.Config.set_library_path(CLANG_LIB_DIR)
if CLANG_BIND_DIR:
sys.path.append(CLANG_BIND_DIR)
CHECKER_IGNORE_PREFIX = [
"extern",
]
CHECKER_EXCLUDE_SOURCE_FILES = set(os.path.join(*f.split("/")) for f in (
# Skip parsing these large (mostly data files).
"source/blender/editors/space_text/text_format_pov.cc",
"source/blender/editors/space_text/text_format_pov_ini.cc",
))
# -----------------------------------------------------------------------------
# Utility Functions
def clang_source_location_as_str(source_location: ClangSourceLocation) -> str:
return "{:s}:{:d}:{:d}:".format(str(source_location.file), source_location.line, source_location.column)
# -----------------------------------------------------------------------------
# Checkers
class ClangChecker:
"""
Base class for checkers.
Notes:
- The function ``check_source`` takes file_data as bytes instead of a string
because the offsets provided by CLANG are byte offsets.
While the offsets could be converted into UNICODE offset's,
there doesn't seem to be an efficient & convenient way to do that.
"""
__slots__ = ()
def __new__(cls, *args: tuple[Any], **kwargs: dict[str, Any]) -> Any:
raise RuntimeError("%s should not be instantiated" % cls)
@staticmethod
def check_source(
_filepath: str,
_file_data: bytes,
_tu: ClangTranslationUnit,
_shared_check_data: Any,
) -> list[str]:
raise RuntimeError("This function must be overridden by it's subclass!")
return []
@staticmethod
def setup() -> Any:
return None
@staticmethod
def teardown(_shared_check_data: Any) -> None:
pass
class clang_checkers:
# fake module.
class struct_comments(ClangChecker):
"""
Ensure comments in struct declarations match the members of the struct, e.g:
SomeStruct var = {
/*name*/ "Text",
/*children*/ nullptr,
/*flag*/ 0,
};
Will generate a warning if any of the names in the prefix comments don't match the struct member names.
"""
_struct_comments_ignore = {
# `PyTypeObject` uses compile time members that vary (see: #PyVarObject_HEAD_INIT macro)
# While some clever comment syntax could be supported to signify multiple/optional members
# this is such a specific case that it's simpler to skip this warning.
"PyTypeObject": {"ob_base": {"ob_size"}},
}
@staticmethod
def _struct_check_comments_recursive(
# Static (unchanged for each recursion).
filepath: str,
file_data: bytes,
# Different for each recursion.
node: ClangNode,
node_parent: ClangNode,
level: int,
# Used to build data.
struct_decl_map: dict[str, ClangNode],
struct_type_map: dict[str, str],
output: list[str],
) -> None:
# Needed to read back the node.
if USE_VERBOSE:
print("TRY:", node.kind, node.spelling, len(list(node.get_tokens())), level, node.location)
# if node.kind == CursorKind.VAR_DECL and node.spelling == "Vector_NumMethods":
# import IPython
# IPython.embed()
if node.kind == CursorKind.STRUCT_DECL:
# Ignore forward declarations.
if next(node.get_children(), None) is not None:
struct_type = node.spelling.strip()
if not struct_type:
# The parent may be a `typedef [..] TypeID` where `[..]` is `struct { a; b; c; }`.
# Inspect the parent.
if node_parent is not None and (node_parent.kind == CursorKind.TYPEDEF_DECL):
tokens = list(node_parent.get_tokens())
if tokens[0].spelling == "typedef":
struct_type = tokens[-1].spelling
struct_decl_map[struct_type] = node
# Ignore declarations for anything defined outside this file.
if str(node.location.file) == filepath:
if node.kind == CursorKind.INIT_LIST_EXPR:
if USE_VERBOSE:
print(node.spelling, node.location)
# Split to avoid `const struct` .. and similar.
# NOTE: there may be an array size suffix, e.g. `[4]`.
# This could be supported.
struct_type = node.type.spelling.split()[-1]
struct = struct_decl_map.get(struct_type)
if struct is None:
if USE_VERBOSE:
print("NOT FOUND:", struct_type)
struct_type = struct_type_map.get(struct_type)
if struct_type is not None:
struct = struct_decl_map.get(struct_type)
if USE_VERBOSE:
print("INSPECTING STRUCT:", struct_type)
if struct is not None:
member_names = [
node_child.spelling for node_child in struct.get_children()
if node_child.kind == CursorKind.FIELD_DECL
]
# if struct_type == "PyMappingMethods":
# import IPython
# IPython.embed()
children = list(node.get_children())
comment_names = []
# Set to true when there is a comment directly before a value,
# this is needed because:
# - Comments on the previous line are rarely intended to be identifiers of the struct member.
# - Comments which _are_ intended to be identifiers can be wrapped onto new-lines
# so they should not be ignored.
#
# While it's possible every member is wrapped onto a new-line,
# this is highly unlikely.
comment_names_prefix_any = False
for node_child in children:
# Extract the content before the child
# (typically a C-style comment containing the struct member).
end = min(node_child.location.offset, len(file_data))
# It's possible this ID has a preceding "name::space::etc"
# which should be skipped.
while end > 0 and ((ch := bytes((file_data[end - 1],))).isalpha() or ch == b":"):
end -= 1
has_newline = False
while end > 0:
ch = bytes((file_data[end - 1],))
if ch in {b"\t", b" "}:
end -= 1
elif ch == b"\n":
end -= 1
has_newline = True
else:
break
beg = end - 1
while beg != 0 and bytes((file_data[beg],)) not in {
b"\n",
# Needed so declarations on a single line don't detect a comment
# from an outer comment, e.g.
# SomeStruct x = {
# /*list*/ {nullptr, nullptr},
# };
# Would start inside the first `nullptr` and walk backwards to find `/*list*/`.
b"{"
}:
beg -= 1
# Seek back until the comment end (in some cases this includes code).
# This occurs when the body of the declaration includes code, e.g.
# rcti x = {
# /*xmin*/ foo->bar.baz,
# ... snip ...
# };
# Where `"xmin*/ foo->bar."` would be extracted were it not for this check.
# There might be a more elegant way to handle this, for how snipping off the last
# comment characters is sufficient.
end_test = file_data.rfind(b"*/", end + 1, beg)
if end_test != -1:
end = end_test
text = file_data[beg:end]
if text.lstrip().startswith(b"/*"):
if not has_newline:
comment_names_prefix_any = True
else:
text = b""
comment_names.append(text.decode('utf-8'))
if USE_VERBOSE:
print(member_names)
print(comment_names)
total = min(len(member_names), len(comment_names))
if total != 0 and comment_names_prefix_any:
result = [""] * total
count_found = 0
count_invalid = 0
for i in range(total):
comment = comment_names[i]
if "/*" in comment and "*/" in comment:
comment = comment.strip().strip("/").strip("*")
if comment == member_names[i]:
count_found += 1
else:
suppress_warning = False
if (
skip_members_table :=
clang_checkers.struct_comments._struct_comments_ignore.get(
node_parent.type.spelling,
)
) is not None:
if (skip_members := skip_members_table.get(comment)) is not None:
if member_names[i] in skip_members:
suppress_warning = True
if not suppress_warning:
result[i] = "Incorrect! found \"{:s}\" expected \"{:s}\"".format(
comment, member_names[i])
count_invalid += 1
else:
result[i] = "No comment for \"{:s}\"".format(member_names[i])
if count_found == 0 and count_invalid == 0:
# No comments used, skip this as not all declaration use this comment style.
output.append(
"NONE: {:s} {:s}".format(
clang_source_location_as_str(node.location),
node.type.spelling,
)
)
elif count_found != total:
for i in range(total):
if result[i]:
output.append(
"FAIL: {:s} {:s}".format(
clang_source_location_as_str(children[i].location),
result[i],
)
)
else:
output.append(
"OK: {:s} {:s}".format(
clang_source_location_as_str(node.location),
node.type.spelling,
)
)
for node_child in node.get_children():
clang_checkers.struct_comments._struct_check_comments_recursive(
filepath, file_data,
node_child, node, level + 1,
struct_decl_map, struct_type_map, output,
)
@staticmethod
def check_source(
filepath: str,
file_data: bytes,
tu: ClangTranslationUnit,
_shared_check_data: Any) -> list[str]:
output: list[str] = []
struct_decl_map: dict[str, Any] = {}
struct_type_map: dict[str, str] = {}
clang_checkers.struct_comments._struct_check_comments_recursive(
filepath, file_data,
tu.cursor, None, 0,
struct_decl_map, struct_type_map, output,
)
return output
# -----------------------------------------------------------------------------
# Checker Class Access
def check_function_get_all() -> list[str]:
checkers = []
for name in dir(clang_checkers):
value = getattr(clang_checkers, name)
if isinstance(value, type) and issubclass(value, ClangChecker):
checkers.append(name)
checkers.sort()
return checkers
def check_class_from_id(name: str) -> type[ClangChecker]:
result = getattr(clang_checkers, name)
assert issubclass(result, ClangChecker)
# MYPY 0.812 doesn't recognize the assert above.
return result # type: ignore
def check_docstring_from_id(name: str) -> str:
from textwrap import dedent
result = getattr(clang_checkers, name).__doc__
return dedent(result or '').strip('\n') + '\n'
# -----------------------------------------------------------------------------
# Generic Clang Checker
def check_source_file(
filepath: str,
args: Sequence[str],
check_ids: Sequence[str],
shared_check_data_foreach_check: Sequence[Any],
) -> str:
index = clang.cindex.Index.create()
try:
tu = index.parse(filepath, args)
except clang.cindex.TranslationUnitLoadError as ex:
return "PARSE_ERROR: {:s} {!r}".format(filepath, ex)
with open(filepath, "rb") as fh:
file_data = fh.read()
output: list[str] = []
# we don't really care what we are looking at, just scan entire file for
# function calls.
for check, shared_check_data in zip(check_ids, shared_check_data_foreach_check):
cls = check_class_from_id(check)
output.extend(cls.check_source(filepath, file_data, tu, shared_check_data))
if not output:
return ""
return "\n".join(output)
def check_source_file_for_imap(args: tuple[str, Sequence[str], Sequence[str], Sequence[Any]]) -> str:
return check_source_file(*args)
def source_info_filter(
source_info: list[tuple[str, list[str], list[str]]],
regex_list: Sequence[re.Pattern[str]],
) -> list[tuple[str, list[str], list[str]]]:
source_dir = project_source_info.SOURCE_DIR
if not source_dir.endswith(os.sep):
source_dir += os.sep
source_info_result = []
for item in source_info:
filepath_source = item[0]
if filepath_source.startswith(source_dir):
filepath_source_relative = filepath_source[len(source_dir):]
if filepath_source_relative in CHECKER_EXCLUDE_SOURCE_FILES:
CHECKER_EXCLUDE_SOURCE_FILES.remove(filepath_source_relative)
continue
if filepath_source_relative.startswith("intern" + os.sep + "ghost"):
pass
elif filepath_source_relative.startswith("source" + os.sep):
pass
else:
continue
has_match = False
for regex in regex_list:
if regex.match(filepath_source_relative) is not None:
has_match = True
if not has_match:
continue
else:
# Skip files not in source (generated files from the build directory),
# these could be check but it's not all that useful (preview blend ... etc).
continue
source_info_result.append(item)
if CHECKER_EXCLUDE_SOURCE_FILES:
sys.stderr.write(
"Error: exclude file(s) are missing: {!r}\n".format((list(sorted(CHECKER_EXCLUDE_SOURCE_FILES))))
)
sys.exit(1)
return source_info_result
def run_checks_on_project(
check_ids: Sequence[str],
regex_list: Sequence[re.Pattern[str]],
jobs: int,
) -> None:
source_info = project_source_info.build_info(ignore_prefix_list=CHECKER_IGNORE_PREFIX)
source_defines = project_source_info.build_defines_as_args()
# Apply exclusion.
source_info = source_info_filter(source_info, regex_list)
shared_check_data_foreach_check = [
check_class_from_id(check).setup() for check in check_ids
]
all_args = []
index = 0
for filepath_source, inc_dirs, defs in source_info[index:]:
args = (
[("-I" + i) for i in inc_dirs] +
[("-D" + d) for d in defs] +
source_defines
)
all_args.append((filepath_source, args, check_ids, shared_check_data_foreach_check))
import multiprocessing
if jobs <= 0:
jobs = multiprocessing.cpu_count()
if jobs > 1:
with multiprocessing.Pool(processes=jobs) as pool:
# No `istarmap`, use an intermediate function.
for result in pool.imap(check_source_file_for_imap, all_args):
if result:
print(result)
else:
for (filepath_source, args, _check_ids, shared_check_data_foreach_check) in all_args:
result = check_source_file(filepath_source, args, check_ids, shared_check_data_foreach_check)
if result:
print(result)
for (check, shared_check_data) in zip(check_ids, shared_check_data_foreach_check):
check_class_from_id(check).teardown(shared_check_data)
def create_parser(checkers_all: Sequence[str]) -> argparse.ArgumentParser:
from textwrap import indent
# Create doc-string for checks.
checks_all_docs = []
for checker in checkers_all:
# `%` -> `%%` is needed for `--help` not to interpret these as formatting arguments.
checks_all_docs.append(
" %s\n%s" % (
checker,
indent(check_docstring_from_id(checker).replace("%", "%%"), ' '),
)
)
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument(
"--match",
nargs='+',
required=True,
metavar="REGEX",
help="Match file paths against this expression",
)
parser.add_argument(
"--checks",
dest="checks",
help=(
"Specify the check presets to run.\n\n" +
"\n".join(checks_all_docs) + "\n"
"Multiple checkers may be passed at once (comma separated, no spaces)."),
required=True,
)
parser.add_argument(
"--jobs",
dest="jobs",
type=int,
default=0,
help=(
"The number of processes to use. "
"Defaults to zero which detects the available cores, 1 is single threaded (useful for debugging)."
),
required=False,
)
return parser
# -----------------------------------------------------------------------------
# Main Function
def main() -> int:
checkers_all = check_function_get_all()
parser = create_parser(checkers_all)
args = parser.parse_args()
regex_list = []
for expr in args.match:
try:
regex_list.append(re.compile(expr))
except Exception as ex:
print("Error in expression: \"{:s}\"\n {!r}".format(expr, ex))
return 1
run_checks_on_project(
args.checks.split(','),
regex_list,
args.jobs,
)
return 0
if __name__ == "__main__":
sys.exit(main())

72
tools/check_source/static_check_clang_array.py Normal file
View File

@@ -0,0 +1,72 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2011-2022 Blender Authors
#
# SPDX-License-Identifier: GPL-2.0-or-later
import project_source_info
import subprocess
import sys
import os
from typing import (
Any,
)
USE_QUIET = (os.environ.get("QUIET", None) is not None)
CHECKER_IGNORE_PREFIX = [
"extern",
"intern/moto",
]
CHECKER_BIN = "python3"
CHECKER_ARGS = [
os.path.join(os.path.dirname(__file__), "clang_array_check.py"),
# not sure why this is needed, but it is.
"-I" + os.path.join(project_source_info.SOURCE_DIR, "extern", "glew", "include"),
# stupid but needed
"-Dbool=char"
]
def main() -> None:
source_info = project_source_info.build_info(ignore_prefix_list=CHECKER_IGNORE_PREFIX)
check_commands = []
for c, inc_dirs, defs in source_info:
# ~if "source/blender" not in c:
# ~ continue
cmd = (
[CHECKER_BIN] +
CHECKER_ARGS +
[c] +
[("-I%s" % i) for i in inc_dirs] +
[("-D%s" % d) for d in defs]
)
check_commands.append((c, cmd))
process_functions = []
def my_process(i: int, c: str, cmd: str) -> subprocess.Popen[Any]:
if not USE_QUIET:
percent = 100.0 * (i / (len(check_commands) - 1))
percent_str = "[" + ("%.2f]" % percent).rjust(7) + " %:"
sys.stdout.flush()
sys.stdout.write("%s %s\n" % (percent_str, c))
return subprocess.Popen(cmd)
for i, (c, cmd) in enumerate(check_commands):
process_functions.append((my_process, (i, c, cmd)))
project_source_info.queue_processes(process_functions)
if __name__ == "__main__":
main()

417
tools/check_source/static_check_cppcheck.py Normal file
View File

@@ -0,0 +1,417 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2011-2022 Blender Authors
#
# SPDX-License-Identifier: GPL-2.0-or-later
import project_source_info
import subprocess
import sys
import os
import re
import tempfile
import time
from typing import (
Any,
IO,
)
USE_VERBOSE = (os.environ.get("VERBOSE", None) is not None)
# Could make configurable.
USE_VERBOSE_PROGRESS = True
CHECKER_BIN = "cppcheck"
CHECKER_IGNORE_PREFIX = [
"extern",
]
# Optionally use a separate build dir for each source code directory.
# According to CPPCHECK docs using one directory is a way to take advantage of "whole program" checks,
# although it looks as if there might be name-space issues - overwriting files with similar names across
# different parts of the source.
CHECKER_ISOLATE_BUILD_DIR = False
CHECKER_EXCLUDE_SOURCE_FILES_EXT = (
# Exclude generated shaders, harmless but also not very useful and are quite slow.
".glsl.c",
)
# To add files use a relative path.
CHECKER_EXCLUDE_SOURCE_FILES = set(os.path.join(*f.split("/")) for f in (
"source/blender/draw/engines/eevee_next/eevee_lut.cc",
# Hangs for hours CPPCHECK-2.14.0.
"intern/cycles/blender/output_driver.cpp",
))
CHECKER_EXCLUDE_SOURCE_DIRECTORIES_BUILD = set(os.path.join(*f.split("/")) + os.sep for f in (
# Exclude data-files, especially `datatoc` as the files can be large & are slow to scan.
"release/datafiles",
# Exclude generated RNA, harmless but also not very useful and are quite slow.
"source/blender/makesrna/intern",
# Exclude generated WAYLAND protocols.
"intern/ghost/libwayland"
))
CHECKER_ARGS = (
# Speed up execution.
# As Blender has many defines, the total number of configurations is large making execution unreasonably slow.
# This could be increased but do so with care.
"--max-configs=1",
# Enable this when includes are missing.
# `"--check-config",`
# May be interesting to check on increasing this for better results:
# `"--max-ctu-depth=2",`
# This is slower, for a comprehensive output it is needed.
"--check-level=exhaustive",
# Shows many pedantic issues, some are quite useful.
"--enable=all",
# Tends to give many false positives, could investigate if there are any ways to resolve, for now it's noisy.
"--disable=unusedFunction",
# Also shows useful messages, even if some are false-positives.
"--inconclusive",
# Generates many warnings, CPPCHECK known about system includes without resolving them.
"--suppress=missingIncludeSystem",
# Quiet output, otherwise all defines/includes are printed (overly verbose).
# Only enable this for troubleshooting (if defines are not set as expected for example).
*(() if USE_VERBOSE else ("--quiet",))
# NOTE: `--cppcheck-build-dir=<dir>` is added later as a temporary directory.
)
CHECKER_ARGS_C = (
"--std=c11",
)
CHECKER_ARGS_CXX = (
"--std=c++17",
)
# NOTE: it seems we can't exclude these from CPPCHECK directly (from what I can see)
# so exclude them from the summary.
CHECKER_EXCLUDE_FROM_SUMMARY = {
# Not considered an error.
"allocaCalled",
# Typically these can't be made `const`.
"constParameterCallback",
# Overly noisy, we could consider resolving all of these at some point.
"cstyleCast",
# Calling `memset` of float may technically be a bug but works in practice.
"memsetClassFloat",
# There are various classes which don't have copy or equal constructors (GHOST windows for e.g.)
"noCopyConstructor",
# Similar for `noCopyConstructor`.
"nonoOperatorEq",
# There seems to be many false positives here.
"unusedFunction",
# Also noisy, looks like these are not issues to "solve".
"unusedPrivateFunction",
# TODO: consider enabling this, more of a preference,
# not using STL algorithm's doesn't often hint at actual errors.
"useStlAlgorithm",
# May be interesting to handle but very noisy currently.
"variableScope",
# These could be added back, currently there are so many warnings and they don't seem especially error-prone.
"missingMemberCopy",
"missingOverride",
"noExplicitConstructor",
"uninitDerivedMemberVar",
"uninitMemberVar",
"useInitializationList",
}
def source_info_filter(
source_info: list[tuple[str, list[str], list[str]]],
source_dir: str,
cmake_dir: str,
) -> list[tuple[str, list[str], list[str]]]:
source_dir = source_dir.rstrip(os.sep) + os.sep
cmake_dir = cmake_dir.rstrip(os.sep) + os.sep
cmake_dir_prefix_tuple = tuple(CHECKER_EXCLUDE_SOURCE_DIRECTORIES_BUILD)
source_info_result = []
for i, item in enumerate(source_info):
c = item[0]
if c.endswith(*CHECKER_EXCLUDE_SOURCE_FILES_EXT):
continue
if c.startswith(source_dir):
c_relative = c[len(source_dir):]
if c_relative in CHECKER_EXCLUDE_SOURCE_FILES:
CHECKER_EXCLUDE_SOURCE_FILES.remove(c_relative)
continue
elif c.startswith(cmake_dir):
c_relative = c[len(cmake_dir):]
if c_relative.startswith(cmake_dir_prefix_tuple):
continue
# TODO: support filtering on filepath.
# if "/editors/mask" not in c:
# continue
source_info_result.append(item)
if CHECKER_EXCLUDE_SOURCE_FILES:
sys.stderr.write(
"Error: exclude file(s) are missing: {!r}\n".format(list(sorted(CHECKER_EXCLUDE_SOURCE_FILES)))
)
sys.exit(1)
return source_info_result
def cppcheck(cppcheck_dir: str, temp_dir: str, log_fh: IO[bytes]) -> None:
temp_source_dir = os.path.join(temp_dir, "source")
os.mkdir(temp_source_dir)
del temp_dir
source_dir = os.path.normpath(os.path.abspath(project_source_info.SOURCE_DIR))
cmake_dir = os.path.normpath(os.path.abspath(project_source_info.CMAKE_DIR))
cppcheck_build_dir = os.path.join(cppcheck_dir, "build")
os.makedirs(cppcheck_build_dir, exist_ok=True)
source_info = project_source_info.build_info(ignore_prefix_list=CHECKER_IGNORE_PREFIX)
cppcheck_compiler_h = os.path.join(temp_source_dir, "cppcheck_compiler.h")
with open(cppcheck_compiler_h, "w", encoding="utf-8") as fh:
fh.write(project_source_info.build_defines_as_source())
# Add additional defines.
fh.write("\n")
# Python's `pyport.h` errors without this.
fh.write("#define UCHAR_MAX 255\n")
# `intern/atomic/intern/atomic_ops_utils.h` errors with `Cannot find int size` without this.
fh.write("#define UINT_MAX 0xFFFFFFFF\n")
# Apply exclusion.
source_info = source_info_filter(source_info, source_dir, cmake_dir)
check_commands = []
for c, inc_dirs, defs in source_info:
if c.endswith(".c"):
checker_args_extra = CHECKER_ARGS_C
else:
checker_args_extra = CHECKER_ARGS_CXX
if CHECKER_ISOLATE_BUILD_DIR:
build_dir_for_source = os.path.relpath(os.path.dirname(os.path.normpath(os.path.abspath(c))), source_dir)
build_dir_for_source = os.sep + build_dir_for_source + os.sep
build_dir_for_source = build_dir_for_source.replace(
os.sep + ".." + os.sep,
os.sep + "__" + os.sep,
).strip(os.sep)
build_dir_for_source = os.path.join(cppcheck_build_dir, build_dir_for_source)
os.makedirs(build_dir_for_source, exist_ok=True)
else:
build_dir_for_source = cppcheck_build_dir
cmd = (
CHECKER_BIN,
*CHECKER_ARGS,
*checker_args_extra,
"--cppcheck-build-dir=" + build_dir_for_source,
"--include=" + cppcheck_compiler_h,
# NOTE: for some reason failing to include this crease a large number of syntax errors
# from `intern/guardedalloc/MEM_guardedalloc.h`. Include directly to resolve.
"--include={:s}".format(os.path.join(source_dir, "source", "blender", "blenlib", "BLI_compiler_attrs.h")),
c,
*[("-I{:s}".format(i)) for i in inc_dirs],
*[("-D{:s}".format(d)) for d in defs],
)
check_commands.append((c, cmd))
process_functions = []
def my_process(i: int, c: str, cmd: list[str]) -> subprocess.Popen[Any]:
proc = subprocess.Popen(
cmd,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
)
# A bit dirty, but simplifies logic to read these back later.
proc.my_index = i # type: ignore
proc.my_time = time.time() # type: ignore
return proc
for i, (c, cmd) in enumerate(check_commands):
process_functions.append((my_process, (i, c, cmd)))
index_current = 0
index_count = 0
proc_results_by_index: dict[int, tuple[bytes, bytes]] = {}
def process_finalize(
proc: subprocess.Popen[Any],
stdout: bytes,
stderr: bytes,
) -> None:
nonlocal index_current, index_count
index_count += 1
assert hasattr(proc, "my_index")
index = proc.my_index
assert hasattr(proc, "my_time")
time_orig = proc.my_time
c = check_commands[index][0]
time_delta = time.time() - time_orig
if USE_VERBOSE_PROGRESS:
percent = 100.0 * (index_count / len(check_commands))
sys.stdout.flush()
sys.stdout.write("[{:s}] %: {:s} ({:.2f})\n".format(
("{:.2f}".format(percent)).rjust(6),
os.path.relpath(c, source_dir),
time_delta,
))
while index == index_current:
log_fh.write(stderr)
log_fh.write(b"\n")
log_fh.write(stdout)
log_fh.write(b"\n")
index_current += 1
test_data = proc_results_by_index.pop(index_current, None)
if test_data is not None:
stdout, stderr = test_data
index += 1
else:
proc_results_by_index[index] = stdout, stderr
project_source_info.queue_processes(
process_functions,
process_finalize=process_finalize,
# job_total=4,
)
print("Finished!")
def cppcheck_generate_summary(
log_fh: IO[str],
log_summary_fh: IO[str],
) -> None:
source_dir = project_source_info.SOURCE_DIR
source_dir_source = os.path.join(source_dir, "source") + os.sep
source_dir_intern = os.path.join(source_dir, "intern") + os.sep
filter_line_prefix = (source_dir_source, source_dir_intern)
source_dir_prefix_len = len(source_dir.rstrip(os.sep))
# Avoids many duplicate lines generated by headers.
lines_unique = set()
category: dict[str, list[str]] = {}
re_match = re.compile(".* \\[([a-zA-Z_]+)\\]$")
for line in log_fh:
if not line.startswith(filter_line_prefix):
continue
# Print a relative directory from `SOURCE_DIR`,
# less visual noise and makes it possible to compare reports from different systems.
line = "." + line[source_dir_prefix_len:]
if (m := re_match.match(line)) is None:
continue
g = m.group(1)
if g in CHECKER_EXCLUDE_FROM_SUMMARY:
continue
if line in lines_unique:
continue
lines_unique.add(line)
try:
category_list = category[g]
except KeyError:
category_list = category[g] = []
category_list.append(line)
for key, value in sorted(category.items()):
log_summary_fh.write("\n\n{:s}\n".format(key))
for line in value:
log_summary_fh.write(line)
def main() -> None:
cmake_dir = os.path.normpath(os.path.abspath(project_source_info.CMAKE_DIR))
cppcheck_dir = os.path.join(cmake_dir, "cppcheck")
filepath_output_log = os.path.join(cppcheck_dir, "cppcheck.part.log")
filepath_output_summary_log = os.path.join(cppcheck_dir, "cppcheck_summary.part.log")
try:
os.makedirs(cppcheck_dir, exist_ok=True)
files_old = {}
# Comparing logs is useful, keep the old ones (renamed).
with tempfile.TemporaryDirectory() as temp_dir:
with open(filepath_output_log, "wb") as log_fh:
cppcheck(cppcheck_dir, temp_dir, log_fh)
with (
open(filepath_output_log, "r", encoding="utf-8") as log_fh,
open(filepath_output_summary_log, "w", encoding="utf-8") as log_summary_fh,
):
cppcheck_generate_summary(log_fh, log_summary_fh)
except KeyboardInterrupt:
print("\nCanceling...")
for filepath_part in (
filepath_output_log,
filepath_output_summary_log,
):
if os.path.exists(filepath_part):
os.remove(filepath_part)
return
# The partial files have been written.
# - Move previous files -> `.old.log`.
# - Move `.log.part` -> `.log`
#
# Do this last so it's possible to cancel execution without breaking the old/new log comparison
# which is especially useful when comparing the old/new summary.
for filepath_part in (
filepath_output_log,
filepath_output_summary_log,
):
filepath = filepath_part.removesuffix(".part.log") + ".log"
if not os.path.exists(filepath):
os.rename(filepath_part, filepath)
continue
filepath_old = filepath.removesuffix(".log") + ".old.log"
if os.path.exists(filepath_old):
os.remove(filepath_old)
os.rename(filepath, filepath_old)
os.rename(filepath_part, filepath)
files_old[filepath] = filepath_old
print("Written:")
for filepath_part in (
filepath_output_log,
filepath_output_summary_log,
):
filepath = filepath_part.removesuffix(".part.log") + ".log"
print(" ", filepath, "<->", files_old.get(filepath, "<none>"))
if __name__ == "__main__":
main()