Re-design of submodules used in blender.git

This commit implements described in the #104573.

The goal is to fix the confusion of the submodule hashes change, which are not
ideal for any of the supported git-module configuration (they are either always
visible causing confusion, or silently staged and committed, also causing
confusion).

This commit replaces submodules with a checkout of addons and addons_contrib,
covered by the .gitignore, and locale and developer tools are moved to the
main repository.

This also changes the paths:
- /release/scripts are moved to the /scripts
- /source/tools are moved to the /tools
- /release/datafiles/locale is moved to /locale

This is done to avoid conflicts when using bisect, and also allow buildbot to
automatically "recover" wgen building older or newer branches/patches.

Running `make update` will initialize the local checkout to the changed
repository configuration.

Another aspect of the change is that the make update will support Github style
of remote organization (origin remote pointing to thy fork, upstream remote
pointing to the upstream blender/blender.git).

Pull Request #104755

scripts/modules/bl_console_utils/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+"""
+Utilities relating to text mode console interactions.
+"""

scripts/modules/bl_console_utils/autocomplete/__init__.py

@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+# Copyright (c) 2009 www.stani.be
+
+"""Package for console specific modules."""

scripts/modules/bl_console_utils/autocomplete/complete_calltip.py

@@ -0,0 +1,172 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+# Copyright (c) 2009 www.stani.be
+
+import inspect
+import re
+
+
+# regular expression constants
+DEF_DOC = r'%s\s*(\(.*?\))'
+DEF_SOURCE = r'def\s+%s\s*(\(.*?\)):'
+RE_EMPTY_LINE = re.compile(r'^\s*\n')
+RE_FLAG = re.MULTILINE | re.DOTALL
+RE_NEWLINE = re.compile('\n+')
+RE_SPACE = re.compile(r'\s+')
+RE_DEF_COMPLETE = re.compile(
+    # don't start with a quote
+    '''(?:^|[^"'a-zA-Z0-9_])'''
+    # start with a \w = [a-zA-Z0-9_]
+    r'''((\w+'''
+    # allow also dots and closed bracket pairs []
+    r'''(?:\w|[.]|\[.+?\])*'''
+    # allow empty string
+    '''|)'''
+    # allow opening bracket(s)
+    r'''(?:\(|\s)*)$''')
+
+
+def reduce_newlines(text):
+    """Reduces multiple newlines to a single newline.
+
+    :arg text: text with multiple newlines
+    :type text: str
+    :returns: text with single newlines
+    :rtype: str
+
+    >>> reduce_newlines('hello\\n\\nworld')
+    'hello\\nworld'
+    """
+    return RE_NEWLINE.sub('\n', text)
+
+
+def reduce_spaces(text):
+    """Reduces multiple whitespaces to a single space.
+
+    :arg text: text with multiple spaces
+    :type text: str
+    :returns: text with single spaces
+    :rtype: str
+
+    >>> reduce_spaces('hello    \\nworld')
+    'hello world'
+    """
+    return RE_SPACE.sub(' ', text)
+
+
+def get_doc(obj):
+    """Get the doc string or comments for an object.
+
+    :arg object: object
+    :returns: doc string
+    :rtype: str
+
+    >>> get_doc(abs)
+    'abs(number) -> number\\n\\nReturn the absolute value of the argument.'
+    """
+    result = inspect.getdoc(obj) or inspect.getcomments(obj)
+    return result and RE_EMPTY_LINE.sub('', result.rstrip()) or ''
+
+
+def get_argspec(func, *, strip_self=True, doc=None, source=None):
+    """Get argument specifications.
+
+    :arg strip_self: strip `self` from argspec
+    :type strip_self: bool
+    :arg doc: doc string of func (optional)
+    :type doc: str
+    :arg source: source code of func (optional)
+    :type source: str
+    :returns: argument specification
+    :rtype: str
+
+    >>> get_argspec(inspect.getclasstree)
+    '(classes, unique=0)'
+    >>> get_argspec(abs)
+    '(number)'
+    """
+    # get the function object of the class
+    try:
+        func = func.__func__
+    except AttributeError:
+        pass
+    # is callable?
+    if not hasattr(func, '__call__'):
+        return ''
+    # func should have a name
+    try:
+        func_name = func.__name__
+    except AttributeError:
+        return ''
+    # from docstring
+    if doc is None:
+        doc = get_doc(func)
+    match = re.search(DEF_DOC % func_name, doc, RE_FLAG)
+    # from source code
+    if not match:
+        if source is None:
+            try:
+                source = inspect.getsource(func)
+            except (TypeError, IOError):
+                source = ''
+        if source:
+            match = re.search(DEF_SOURCE % func_name, source, RE_FLAG)
+    if match:
+        argspec = reduce_spaces(match.group(1))
+    else:
+        # try with the inspect.getarg* functions
+        try:
+            argspec = inspect.formatargspec(*inspect.getfullargspec(func))
+        except:
+            try:
+                argspec = inspect.formatargvalues(
+                    *inspect.getargvalues(func))
+            except:
+                argspec = ''
+        if strip_self:
+            argspec = argspec.replace('self, ', '')
+    return argspec
+
+
+def complete(line, cursor, namespace):
+    """Complete callable with calltip.
+
+    :arg line: incomplete text line
+    :type line: str
+    :arg cursor: current character position
+    :type cursor: int
+    :arg namespace: namespace
+    :type namespace: dict
+    :returns: (matches, world, scrollback)
+    :rtype: (list of str, str, str)
+
+    >>> import os
+    >>> complete('os.path.isdir(', 14, {'os': os})[-1]
+    'isdir(s)\\nReturn true if the pathname refers to an existing directory.'
+    >>> complete('abs(', 4, {})[-1]
+    'abs(number) -> number\\nReturn the absolute value of the argument.'
+    """
+    matches = []
+    word = ''
+    scrollback = ''
+    match = RE_DEF_COMPLETE.search(line[:cursor])
+
+    if match:
+        word = match.group(1)
+        func_word = match.group(2)
+        try:
+            func = eval(func_word, namespace)
+        except Exception:
+            func = None
+
+        if func:
+            doc = get_doc(func)
+            argspec = get_argspec(func, doc=doc)
+            scrollback = func_word.split('.')[-1] + (argspec or '()')
+            if doc.startswith(scrollback):
+                scrollback = doc
+            elif doc:
+                scrollback += '\n' + doc
+            scrollback = reduce_newlines(scrollback)
+
+    return matches, word, scrollback

scripts/modules/bl_console_utils/autocomplete/complete_import.py

@@ -0,0 +1,179 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+# Copyright (c) 2009 Fernando Perez, www.stani.be
+
+# Original copyright (see docstring):
+# ****************************************************************************
+#       Copyright (C) 2001-2006 Fernando Perez <fperez@colorado.edu>
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+# ****************************************************************************
+
+"""Completer for import statements
+
+Original code was from IPython/Extensions/ipy_completers.py. The following
+changes have been made:
+- ported to python3
+- pep8 polishing
+- limit list of modules to prefix in case of "from w"
+- sorted modules
+- added sphinx documentation
+- complete() returns a blank list of the module isn't found
+"""
+
+
+import os
+import sys
+
+TIMEOUT_STORAGE = 3  # Time in secs after which the root-modules will be stored
+TIMEOUT_GIVEUP = 20  # Time in secs after which we give up
+
+ROOT_MODULES = None
+
+
+def get_root_modules():
+    """
+    Returns a list containing the names of all the modules available in the
+    folders of the python-path.
+
+    :returns: modules
+    :rtype: list
+    """
+    global ROOT_MODULES
+    modules = []
+    if not (ROOT_MODULES is None):
+        return ROOT_MODULES
+    from time import time
+    t = time()
+    store = False
+    for path in sys.path:
+        modules += module_list(path)
+        if time() - t >= TIMEOUT_STORAGE and not store:
+            # Caching the list of root modules, please wait!
+            store = True
+        if time() - t > TIMEOUT_GIVEUP:
+            # This is taking too long, we give up.
+            ROOT_MODULES = []
+            return []
+
+    modules += sys.builtin_module_names
+
+    # needed for modules defined in C
+    modules += sys.modules.keys()
+
+    modules = list(set(modules))
+    if '__init__' in modules:
+        modules.remove('__init__')
+    modules = sorted(modules)
+    if store:
+        ROOT_MODULES = modules
+    return modules
+
+
+def module_list(path):
+    """
+    Return the list containing the names of the modules available in
+    the given folder.
+
+    :arg path: folder path
+    :type path: str
+    :returns: modules
+    :rtype: list
+    """
+
+    if os.path.isdir(path):
+        folder_list = os.listdir(path)
+    elif path.endswith('.egg'):
+        from zipimport import zipimporter
+        try:
+            folder_list = [f for f in zipimporter(path)._files]
+        except:
+            folder_list = []
+    else:
+        folder_list = []
+    #folder_list = glob.glob(os.path.join(path,'*'))
+    folder_list = [
+        p for p in folder_list
+        if (os.path.exists(os.path.join(path, p, '__init__.py')) or
+            p[-3:] in {'.py', '.so'} or
+            p[-4:] in {'.pyc', '.pyo', '.pyd'})]
+
+    folder_list = [os.path.basename(p).split('.')[0] for p in folder_list]
+    return folder_list
+
+
+def complete(line):
+    """
+    Returns a list containing the completion possibilities for an import line.
+
+    :arg line:
+
+        incomplete line which contains an import statement::
+
+            import xml.d
+            from xml.dom import
+
+    :type line: str
+    :returns: list of completion possibilities
+    :rtype: list
+
+    >>> complete('import weak')
+    ['weakref']
+    >>> complete('from weakref import C')
+    ['CallableProxyType']
+    """
+    import inspect
+
+    def try_import(mod, *, only_modules=False):
+
+        def is_importable(module, attr):
+            if only_modules:
+                return inspect.ismodule(getattr(module, attr))
+            else:
+                return not (attr[:2] == '__' and attr[-2:] == '__')
+
+        try:
+            m = __import__(mod)
+        except:
+            return []
+        mods = mod.split('.')
+        for module in mods[1:]:
+            m = getattr(m, module)
+        if (not hasattr(m, '__file__')) or (not only_modules) or\
+           (hasattr(m, '__file__') and '__init__' in m.__file__):
+            completion_list = [attr for attr in dir(m)
+                               if is_importable(m, attr)]
+        else:
+            completion_list = []
+        completion_list.extend(getattr(m, '__all__', []))
+        if hasattr(m, '__file__') and '__init__' in m.__file__:
+            completion_list.extend(module_list(os.path.dirname(m.__file__)))
+        completion_list = list(set(completion_list))
+        if '__init__' in completion_list:
+            completion_list.remove('__init__')
+        return completion_list
+
+    def filter_prefix(names, prefix):
+        return [name for name in names if name.startswith(prefix)]
+
+    words = line.split(' ')
+    if len(words) == 3 and words[0] == 'from':
+        return ['import ']
+    if len(words) < 3 and (words[0] in {'import', 'from'}):
+        if len(words) == 1:
+            return get_root_modules()
+        mod = words[1].split('.')
+        if len(mod) < 2:
+            return filter_prefix(get_root_modules(), words[-1])
+        completion_list = try_import('.'.join(mod[:-1]), only_modules=True)
+        completion_list = ['.'.join(mod[:-1] + [el]) for el in completion_list]
+        return filter_prefix(completion_list, words[-1])
+    if len(words) >= 3 and words[0] == 'from':
+        mod = words[1]
+        return filter_prefix(try_import(mod), words[-1])
+
+    # get here if the import is not found
+    # import invalidmodule
+    #                      ^, in this case return nothing
+    return []

scripts/modules/bl_console_utils/autocomplete/complete_namespace.py

@@ -0,0 +1,190 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+# Copyright (c) 2009 www.stani.be
+
+"""Autocomplete with the standard library"""
+
+import re
+import rlcompleter
+
+
+RE_INCOMPLETE_INDEX = re.compile(r'(.*?)\[[^\]]+$')
+
+TEMP = '__tEmP__'  # only \w characters are allowed!
+TEMP_N = len(TEMP)
+
+
+def is_dict(obj):
+    """Returns whether obj is a dictionary"""
+    return hasattr(obj, 'keys') and hasattr(getattr(obj, 'keys'), '__call__')
+
+
+def is_struct_seq(obj):
+    """Returns whether obj is a structured sequence subclass: sys.float_info"""
+    return isinstance(obj, tuple) and hasattr(obj, 'n_fields')
+
+
+def complete_names(word, namespace):
+    """Complete variable names or attributes
+
+    :arg word: word to be completed
+    :type word: str
+    :arg namespace: namespace
+    :type namespace: dict
+    :returns: completion matches
+    :rtype: list of str
+
+    >>> complete_names('fo', {'foo': 'bar'})
+    ['foo', 'for', 'format(']
+    """
+    # start completer
+    completer = rlcompleter.Completer(namespace)
+    # find matches with std library (don't try to implement this yourself)
+    completer.complete(word, 0)
+    return sorted(set(completer.matches))
+
+
+def complete_indices(word, namespace, *, obj=None, base=None):
+    """Complete a list or dictionary with its indices:
+
+    * integer numbers for list
+    * any keys for dictionary
+
+    :arg word: word to be completed
+    :type word: str
+    :arg namespace: namespace
+    :type namespace: dict
+    :arg obj: object evaluated from base
+    :arg base: sub-string which can be evaluated into an object.
+    :type base: str
+    :returns: completion matches
+    :rtype: list of str
+
+    >>> complete_indices('foo', {'foo': range(5)})
+    ['foo[0]', 'foo[1]', 'foo[2]', 'foo[3]', 'foo[4]']
+    >>> complete_indices('foo', {'foo': {'bar':0, 1:2}})
+    ['foo[1]', "foo['bar']"]
+    >>> complete_indices("foo['b", {'foo': {'bar':0, 1:2}}, base='foo')
+    ["foo['bar']"]
+    """
+    # FIXME: 'foo["b'
+    if base is None:
+        base = word
+    if obj is None:
+        try:
+            obj = eval(base, namespace)
+        except Exception:
+            return []
+    if not hasattr(obj, '__getitem__'):
+        # obj is not a list or dictionary
+        return []
+
+    obj_is_dict = is_dict(obj)
+
+    # rare objects have a __getitem__ but no __len__ (eg. BMEdge)
+    if not obj_is_dict:
+        try:
+            obj_len = len(obj)
+        except TypeError:
+            return []
+
+    if obj_is_dict:
+        # dictionary type
+        matches = ['%s[%r]' % (base, key) for key in sorted(obj.keys())]
+    else:
+        # list type
+        matches = ['%s[%d]' % (base, idx) for idx in range(obj_len)]
+    if word != base:
+        matches = [match for match in matches if match.startswith(word)]
+    return matches
+
+
+def complete(word, namespace, *, private=True):
+    """Complete word within a namespace with the standard rlcompleter
+    module. Also supports index or key access [].
+
+    :arg word: word to be completed
+    :type word: str
+    :arg namespace: namespace
+    :type namespace: dict
+    :arg private: whether private attribute/methods should be returned
+    :type private: bool
+    :returns: completion matches
+    :rtype: list of str
+
+    >>> complete('foo[1', {'foo': range(14)})
+    ['foo[1]', 'foo[10]', 'foo[11]', 'foo[12]', 'foo[13]']
+    >>> complete('foo[0]', {'foo': [range(5)]})
+    ['foo[0][0]', 'foo[0][1]', 'foo[0][2]', 'foo[0][3]', 'foo[0][4]']
+    >>> complete('foo[0].i', {'foo': [range(5)]})
+    ['foo[0].index(', 'foo[0].insert(']
+    >>> complete('rlcompleter', {'rlcompleter': rlcompleter})
+    ['rlcompleter.']
+    """
+    #
+    # if word is empty -> nothing to complete
+    if not word:
+        return []
+
+    re_incomplete_index = RE_INCOMPLETE_INDEX.search(word)
+    if re_incomplete_index:
+        # ignore incomplete index at the end, e.g 'a[1' -> 'a'
+        matches = complete_indices(word, namespace,
+                                   base=re_incomplete_index.group(1))
+
+    elif not ('[' in word):
+        matches = complete_names(word, namespace)
+
+    elif word[-1] == ']':
+        matches = [word]
+
+    elif '.' in word:
+        # brackets are normally not allowed -> work around
+
+        # remove brackets by using a temp var without brackets
+        obj, attr = word.rsplit('.', 1)
+        try:
+            # do not run the obj expression in the console
+            namespace[TEMP] = eval(obj, namespace)
+        except Exception:
+            return []
+        matches = complete_names(TEMP + '.' + attr, namespace)
+        matches = [obj + match[TEMP_N:] for match in matches]
+        del namespace[TEMP]
+
+    else:
+        # safety net, but when would this occur?
+        return []
+
+    if not matches:
+        return []
+
+    # add '.', '('  or '[' if no match has been found
+    elif len(matches) == 1 and matches[0] == word:
+
+        # try to retrieve the object
+        try:
+            obj = eval(word, namespace)
+        except Exception:
+            return []
+        # ignore basic types
+        if type(obj) in {bool, float, int, str}:
+            return []
+        # an extra char '[', '(' or '.' will be added
+        if hasattr(obj, '__getitem__') and not is_struct_seq(obj):
+            # list or dictionary
+            matches = complete_indices(word, namespace, obj=obj)
+        elif hasattr(obj, '__call__'):
+            # callables
+            matches = [word + '(']
+        else:
+            # any other type
+            matches = [word + '.']
+
+    # separate public from private
+    public_matches = [match for match in matches if not ('._' in match)]
+    if private:
+        private_matches = [match for match in matches if '._' in match]
+        return public_matches + private_matches
+    else:
+        return public_matches

scripts/modules/bl_console_utils/autocomplete/intellisense.py

@@ -0,0 +1,136 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+# Copyright (c) 2009 www.stani.be
+
+"""This module provides intellisense features such as:
+
+* autocompletion
+* calltips
+
+It unifies all completion plugins and only loads them on demand.
+"""
+
+# TODO: file complete if startswith quotes
+import os
+import re
+
+# regular expressions to find out which completer we need
+
+# line which starts with an import statement
+RE_MODULE = re.compile(r'''^import(\s|$)|from.+''')
+
+# The following regular expression means an 'unquoted' word
+RE_UNQUOTED_WORD = re.compile(
+    # don't start with a quote
+    r'''(?:^|[^"'a-zA-Z0-9_])'''
+    # start with a \w = [a-zA-Z0-9_]
+    r'''((?:\w+'''
+    # allow also dots and closed bracket pairs []
+    r'''(?:\w|[.]|\[.+?\])*'''
+    # allow empty string
+    r'''|)'''
+    # allow an unfinished index at the end (including quotes)
+    r'''(?:\[[^\]]*$)?)$''',
+    # allow unicode as theoretically this is possible
+    re.UNICODE)
+
+
+def complete(line, cursor, namespace, private):
+    """Returns a list of possible completions:
+
+    * name completion
+    * attribute completion (obj.attr)
+    * index completion for lists and dictionaries
+    * module completion (from/import)
+
+    :arg line: incomplete text line
+    :type line: str
+    :arg cursor: current character position
+    :type cursor: int
+    :arg namespace: namespace
+    :type namespace: dict
+    :arg private: whether private variables should be listed
+    :type private: bool
+    :returns: list of completions, word
+    :rtype: list, str
+
+    >>> complete('re.sr', 5, {'re': re})
+    (['re.sre_compile', 're.sre_parse'], 're.sr')
+    """
+    re_unquoted_word = RE_UNQUOTED_WORD.search(line[:cursor])
+    if re_unquoted_word:
+        # unquoted word -> module or attribute completion
+        word = re_unquoted_word.group(1)
+        if RE_MODULE.match(line):
+            from . import complete_import
+            matches = complete_import.complete(line)
+            if not private:
+                matches[:] = [m for m in matches if m[:1] != "_"]
+            matches.sort()
+        else:
+            from . import complete_namespace
+            matches = complete_namespace.complete(word, namespace, private=private)
+    else:
+        # for now we don't have completers for strings
+        # TODO: add file auto completer for strings
+        word = ''
+        matches = []
+    return matches, word
+
+
+def expand(line, cursor, namespace, *, private=True):
+    """This method is invoked when the user asks autocompletion,
+    e.g. when Ctrl+Space is clicked.
+
+    :arg line: incomplete text line
+    :type line: str
+    :arg cursor: current character position
+    :type cursor: int
+    :arg namespace: namespace
+    :type namespace: dict
+    :arg private: whether private variables should be listed
+    :type private: bool
+    :returns:
+
+        current expanded line, updated cursor position and scrollback
+
+    :rtype: str, int, str
+
+    >>> expand('os.path.isdir(', 14, {'os': os})[-1]
+    'isdir(s)\\nReturn true if the pathname refers to an existing directory.'
+    >>> expand('abs(', 4, {})[-1]
+    'abs(number) -> number\\nReturn the absolute value of the argument.'
+    """
+    if line[:cursor].strip().endswith('('):
+        from . import complete_calltip
+        matches, word, scrollback = complete_calltip.complete(
+            line, cursor, namespace)
+        prefix = os.path.commonprefix(matches)[len(word):]
+        no_calltip = False
+    else:
+        matches, word = complete(line, cursor, namespace, private)
+        prefix = os.path.commonprefix(matches)[len(word):]
+        if len(matches) == 1:
+            scrollback = ''
+        else:
+            # causes blender bug #27495 since string keys may contain '.'
+            # scrollback = '  '.join([m.split('.')[-1] for m in matches])
+
+            # add white space to align with the cursor
+            white_space = "    " + (" " * (cursor + len(prefix)))
+            word_prefix = word + prefix
+            scrollback = '\n'.join(
+                [white_space + m[len(word_prefix):]
+                 if (word_prefix and m.startswith(word_prefix))
+                 else
+                 white_space + m.split('.')[-1]
+                 for m in matches])
+
+        no_calltip = True
+
+    if prefix:
+        line = line[:cursor] + prefix + line[cursor:]
+        cursor += len(prefix.encode('utf-8'))
+        if no_calltip and prefix.endswith('('):
+            return expand(line, cursor, namespace, private=private)
+    return line, cursor, scrollback