test/tests/python/modules/render_report.py

# SPDX-FileCopyrightText: 2018-2023 Blender Authors
#
# SPDX-License-Identifier: Apache-2.0

"""
Compare renders or screenshots against reference versions and generate
a HTML report showing the differences, for regression testing.
"""

import glob
import os
import pathlib
import shutil
import subprocess
import time

from . import global_report
from .colored_print import (print_message, use_message_colors)


def blend_list(dirpath, blocklist):
    import re

    for root, dirs, files in os.walk(dirpath):
        for filename in files:
            if not filename.lower().endswith(".blend"):
                continue

            skip = False
            for blocklist_entry in blocklist:
                if re.match(blocklist_entry, filename):
                    skip = True
                    break

            if not skip:
                filepath = os.path.join(root, filename)
                yield filepath


def test_get_name(filepath):
    filename = os.path.basename(filepath)
    return os.path.splitext(filename)[0]


def test_get_images(output_dir, filepath, reference_dir, reference_override_dir):
    testname = test_get_name(filepath)
    dirpath = os.path.dirname(filepath)

    old_dirpath = os.path.join(dirpath, reference_dir)
    old_img = os.path.join(old_dirpath, testname + ".png")
    if reference_override_dir:
        override_dirpath = os.path.join(dirpath, reference_override_dir)
        override_img = os.path.join(override_dirpath, testname + ".png")
        if os.path.exists(override_img):
            old_dirpath = override_dirpath
            old_img = override_img

    ref_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "ref")
    ref_img = os.path.join(ref_dirpath, testname + ".png")
    os.makedirs(ref_dirpath, exist_ok=True)
    if os.path.exists(old_img):
        shutil.copy(old_img, ref_img)

    new_dirpath = os.path.join(output_dir, os.path.basename(dirpath))
    os.makedirs(new_dirpath, exist_ok=True)
    new_img = os.path.join(new_dirpath, testname + ".png")

    diff_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "diff")
    os.makedirs(diff_dirpath, exist_ok=True)
    diff_color_img = os.path.join(diff_dirpath, testname + ".diff_color.png")
    diff_alpha_img = os.path.join(diff_dirpath, testname + ".diff_alpha.png")

    return old_img, ref_img, new_img, diff_color_img, diff_alpha_img


class Report:
    __slots__ = (
        'title',
        'engine_name',
        'output_dir',
        'global_dir',
        'reference_dir',
        'reference_override_dir',
        'oiiotool',
        'pixelated',
        'fail_threshold',
        'fail_percent',
        'verbose',
        'update',
        'failed_tests',
        'passed_tests',
        'compare_tests',
        'compare_engine',
        'blocklist',
    )

    def __init__(self, title, output_dir, oiiotool, variation=None, blocklist=[]):
        self.title = title

        # Normalize the path to avoid output_dir and global_dir being the same when a directory
        # ends with a trailing slash.
        self.output_dir = os.path.normpath(output_dir)
        self.global_dir = os.path.dirname(self.output_dir)

        self.reference_dir = 'reference_renders'
        self.reference_override_dir = None
        self.oiiotool = oiiotool
        self.compare_engine = None
        self.fail_threshold = 0.016
        self.fail_percent = 1
        self.engine_name = self.title.lower().replace(" ", "_")
        self.blocklist = [] if os.getenv('BLENDER_TEST_IGNORE_BLOCKLIST') is not None else blocklist

        if variation:
            self.title = self._engine_title(title, variation)
            self.output_dir = self._engine_path(self.output_dir, variation.lower())

        self.pixelated = False
        self.verbose = os.environ.get("BLENDER_VERBOSE") is not None
        self.update = os.getenv('BLENDER_TEST_UPDATE') is not None

        if os.environ.get("BLENDER_TEST_COLOR") is not None:
            use_message_colors()

        self.failed_tests = ""
        self.passed_tests = ""
        self.compare_tests = ""

        os.makedirs(output_dir, exist_ok=True)

    def set_pixelated(self, pixelated):
        self.pixelated = pixelated

    def set_fail_threshold(self, threshold):
        self.fail_threshold = threshold

    def set_fail_percent(self, percent):
        self.fail_percent = percent

    def set_reference_dir(self, reference_dir):
        self.reference_dir = reference_dir

    def set_reference_override_dir(self, reference_override_dir):
        self.reference_override_dir = reference_override_dir

    def set_compare_engine(self, other_engine, other_variation=None):
        self.compare_engine = (other_engine, other_variation)

    def set_engine_name(self, engine_name):
        self.engine_name = engine_name

    def run(self, dirpath, blender, arguments_cb, batch=False, fail_silently=False):
        # Run tests and output report.
        dirname = os.path.basename(dirpath)
        ok = self._run_all_tests(dirname, dirpath, blender, arguments_cb, batch, fail_silently)
        self._write_data(dirname)
        self._write_html()
        if self.compare_engine:
            self._write_html(comparison=True)
        return ok

    def _write_data(self, dirname):
        # Write intermediate data for single test.
        outdir = os.path.join(self.output_dir, dirname)
        os.makedirs(outdir, exist_ok=True)

        filepath = os.path.join(outdir, "failed.data")
        pathlib.Path(filepath).write_text(self.failed_tests)

        filepath = os.path.join(outdir, "passed.data")
        pathlib.Path(filepath).write_text(self.passed_tests)

        if self.compare_engine:
            filepath = os.path.join(outdir, "compare.data")
            pathlib.Path(filepath).write_text(self.compare_tests)

    def _navigation_item(self, title, href, active):
        if active:
            return """<li class="breadcrumb-item active" aria-current="page">%s</li>""" % title
        else:
            return """<li class="breadcrumb-item"><a href="%s">%s</a></li>""" % (href, title)

    def _engine_title(self, engine, variation):
        if variation:
            return engine.title() + ' ' + variation
        else:
            return engine.title()

    def _engine_path(self, path, variation):
        if variation:
            variation = variation.replace(' ', '_')
            return os.path.join(path, variation.lower())
        else:
            return path

    def _navigation_html(self, comparison):
        html = """<nav aria-label="breadcrumb"><ol class="breadcrumb">"""
        base_path = os.path.relpath(self.global_dir, self.output_dir)
        global_report_path = os.path.join(base_path, "report.html")
        html += self._navigation_item("Test Reports", global_report_path, False)
        html += self._navigation_item(self.title, "report.html", not comparison)
        if self.compare_engine:
            compare_title = "Compare with %s" % self._engine_title(*self.compare_engine)
            html += self._navigation_item(compare_title, "compare.html", comparison)
        html += """</ol></nav>"""

        return html

    def _write_html(self, comparison=False):
        # Gather intermediate data for all tests.
        if comparison:
            failed_data = []
            passed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/compare.data")))
        else:
            failed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/failed.data")))
            passed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/passed.data")))

        failed_tests = ""
        passed_tests = ""

        for filename in failed_data:
            filepath = os.path.join(self.output_dir, filename)
            failed_tests += pathlib.Path(filepath).read_text()
        for filename in passed_data:
            filepath = os.path.join(self.output_dir, filename)
            passed_tests += pathlib.Path(filepath).read_text()

        tests_html = failed_tests + passed_tests

        # Write html for all tests.
        if self.pixelated:
            image_rendering = 'pixelated'
        else:
            image_rendering = 'auto'

        # Navigation
        menu = self._navigation_html(comparison)

        failed = len(failed_tests) > 0
        if failed:
            message = """<div class="alert alert-danger" role="alert">"""
            message += """<p>Run this command to regenerate reference (ground truth) images:</p>"""
            message += """<p><tt>BLENDER_TEST_UPDATE=1 ctest -R %s</tt></p>""" % self.engine_name
            message += """<p>This then happens for new and failing tests; reference images of """ \
                       """passing test cases will not be updated. Be sure to commit the new reference """ \
                       """images to the tests/data git submodule afterwards.</p>"""
            message += """</div>"""
        else:
            message = ""

        if comparison:
            title = self.title + " Test Compare"
            engine_self = self.title
            engine_other = self._engine_title(*self.compare_engine)
            columns_html = "<tr><th>Name</th><th>%s</th><th>%s</th>" % (engine_self, engine_other)
        else:
            title = self.title + " Test Report"
            columns_html = "<tr><th>Name</th><th>New</th><th>Reference</th><th>Diff Color</th><th>Diff Alpha</th>"

        html = f"""
<html>
<head>
    <title>{title}</title>
    <style>
        div.page_container {{
          text-align: center;
        }}
        div.page_container div {{
          text-align: left;
        }}
        div.page_content {{
          display: inline-block;
        }}
        img {{ image-rendering: {image_rendering}; width: 256px; background-color: #000; }}
        img.render {{
            background-color: #fff;
            background-image:
              -moz-linear-gradient(45deg, #eee 25%, transparent 25%),
              -moz-linear-gradient(-45deg, #eee 25%, transparent 25%),
              -moz-linear-gradient(45deg, transparent 75%, #eee 75%),
              -moz-linear-gradient(-45deg, transparent 75%, #eee 75%);
            background-image:
              -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.25, #eee), color-stop(.25, transparent)),
              -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.25, #eee), color-stop(.25, transparent)),
              -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.75, transparent), color-stop(.75, #eee)),
              -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.75, transparent), color-stop(.75, #eee));

            -moz-background-size:50px 50px;
            background-size:50px 50px;
            -webkit-background-size:50px 51px; /* Override value for silly webkit. */

            background-position:0 0, 25px 0, 25px -25px, 0px 25px;
        }}
        table td:first-child {{ width: 256px; }}
        p {{ margin-bottom: 0.5rem; }}
    </style>
    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
</head>
<body>
    <div class="page_container"><div class="page_content">
        <br/>
        <h1>{title}</h1>
        {menu}
        {message}
        <table class="table table-striped">
            <thead class="thead-dark">
                {columns_html}
            </thead>
            {tests_html}
        </table>
        <br/>
    </div></div>
</body>
</html>
            """

        filename = "report.html" if not comparison else "compare.html"
        filepath = os.path.join(self.output_dir, filename)
        pathlib.Path(filepath).write_text(html)

        print_message("Report saved to: " + pathlib.Path(filepath).as_uri())

        # Update global report
        if not comparison:
            global_failed = failed if not comparison else None
            global_report.add(self.global_dir, "Render", self.title, filepath, global_failed)

    def _relative_url(self, filepath):
        relpath = os.path.relpath(filepath, self.output_dir)
        return pathlib.Path(relpath).as_posix()

    def _write_test_html(self, testname, filepath, error):
        name = test_get_name(filepath)
        name = name.replace('_', ' ')

        old_img, ref_img, new_img, diff_color_img, diff_alpha_img = test_get_images(
            self.output_dir, filepath, self.reference_dir, self.reference_override_dir)

        status = error if error else ""
        tr_style = """ class="table-danger" """ if error else ""

        new_url = self._relative_url(new_img)
        ref_url = self._relative_url(ref_img)
        diff_color_url = self._relative_url(diff_color_img)
        diff_alpha_url = self._relative_url(diff_alpha_img)

        test_html = f"""
            <tr{tr_style}>
                <td><b>{name}</b><br/>{testname}<br/>{status}</td>
                <td><img src="{new_url}" onmouseover="this.src='{ref_url}';" onmouseout="this.src='{new_url}';" class="render"></td>
                <td><img src="{ref_url}" onmouseover="this.src='{new_url}';" onmouseout="this.src='{ref_url}';" class="render"></td>
                <td><img src="{diff_color_url}"></td>
                <td><img src="{diff_alpha_url}"></td>
            </tr>"""

        if error:
            self.failed_tests += test_html
        else:
            self.passed_tests += test_html

        if self.compare_engine:
            base_path = os.path.relpath(self.global_dir, self.output_dir)
            ref_url = os.path.join(base_path, self._engine_path(*self.compare_engine), new_url)

            test_html = """
                <tr{tr_style}>
                    <td><b>{name}</b><br/>{testname}<br/>{status}</td>
                    <td><img src="{new_url}" onmouseover="this.src='{ref_url}';" onmouseout="this.src='{new_url}';" class="render"></td>
                    <td><img src="{ref_url}" onmouseover="this.src='{new_url}';" onmouseout="this.src='{ref_url}';" class="render"></td>
                </tr>""" . format(tr_style=tr_style,
                                  name=name,
                                  testname=testname,
                                  status=status,
                                  new_url=new_url,
                                  ref_url=ref_url)

            self.compare_tests += test_html

    def _diff_output(self, filepath, tmp_filepath):
        old_img, ref_img, new_img, diff_color_img, diff_alpha_img = test_get_images(
            self.output_dir, filepath, self.reference_dir, self.reference_override_dir)

        # Create reference render directory.
        old_dirpath = os.path.dirname(old_img)
        os.makedirs(old_dirpath, exist_ok=True)

        # Copy temporary to new image.
        if os.path.exists(new_img):
            os.remove(new_img)
        if os.path.exists(tmp_filepath):
            shutil.copy(tmp_filepath, new_img)

        if os.path.exists(ref_img):
            # Diff images test with threshold.
            command = (
                self.oiiotool,
                ref_img,
                tmp_filepath,
                "--fail", str(self.fail_threshold),
                "--failpercent", str(self.fail_percent),
                "--diff",
            )
            try:
                subprocess.check_output(command)
                failed = False
            except subprocess.CalledProcessError as e:
                if self.verbose:
                    print_message(e.output.decode("utf-8", 'ignore'))
                failed = e.returncode != 0
        else:
            if not self.update:
                return False

            failed = True

        if failed and self.update:
            # Update reference image if requested.
            shutil.copy(new_img, ref_img)
            shutil.copy(new_img, old_img)
            failed = False

        # Generate color diff image.
        command = (
            self.oiiotool,
            ref_img,
            "--ch", "R,G,B",
            tmp_filepath,
            "--ch", "R,G,B",
            "--sub",
            "--abs",
            "--mulc", "16",
            "-o", diff_color_img,
        )
        try:
            subprocess.check_output(command, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            if self.verbose:
                print_message(e.output.decode("utf-8", 'ignore'))

        # Generate alpha diff image.
        command = (
            self.oiiotool,
            ref_img,
            "--ch", "A",
            tmp_filepath,
            "--ch", "A",
            "--sub",
            "--abs",
            "--mulc", "16",
            "-o", diff_alpha_img,
        )
        try:
            subprocess.check_output(command, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            if self.verbose:
                msg = e.output.decode("utf-8", 'ignore')
                for line in msg.splitlines():
                    # Ignore warnings for images without alpha channel.
                    if "--ch: Unknown channel name" not in line:
                        print_message(line)

        return not failed

    def _get_render_arguments(self, arguments_cb, filepath, base_output_filepath):
        # Each render test can override this method to provide extra functionality.
        # See Cycles render tests for an example.
        # Do not delete.
        return arguments_cb(filepath, base_output_filepath)

    def _get_arguments_suffix(self):
        # Get command line arguments that need to be provided after all file-specific ones.
        # For example the Cycles render device argument needs to be added at the end of
        # the argument list, otherwise tests can't be batched together.
        #
        # Each render test is supposed to override this method.
        return []

    def _run_tests(self, filepaths, blender, arguments_cb, batch):
        # Run multiple tests in a single Blender process since startup can be
        # a significant factor. In case of crashes, re-run the remaining tests.
        verbose = os.environ.get("BLENDER_VERBOSE") is not None

        remaining_filepaths = filepaths[:]
        errors = []

        while len(remaining_filepaths) > 0:
            command = [blender]
            output_filepaths = []

            # Construct output filepaths and command to run
            for filepath in remaining_filepaths:
                testname = test_get_name(filepath)
                print_message(testname, 'SUCCESS', 'RUN')

                base_output_filepath = os.path.join(self.output_dir, "tmp_" + testname)
                output_filepath = base_output_filepath + '0001.png'
                output_filepaths.append(output_filepath)

                if os.path.exists(output_filepath):
                    os.remove(output_filepath)

                command.extend(self._get_render_arguments(arguments_cb, filepath, base_output_filepath))

                # Only chain multiple commands for batch
                if not batch:
                    break

            command.extend(self._get_arguments_suffix())

            # Run process
            crash = False
            output = None
            try:
                completed_process = subprocess.run(command, stdout=subprocess.PIPE)
                if completed_process.returncode != 0:
                    crash = True
                output = completed_process.stdout
            except Exception:
                crash = True

            if verbose:
                print(" ".join(command))
            if (verbose or crash) and output:
                print(output.decode("utf-8", 'ignore'))

            # Detect missing filepaths and consider those errors
            for filepath, output_filepath in zip(remaining_filepaths[:], output_filepaths):
                remaining_filepaths.pop(0)

                if crash:
                    # In case of crash, stop after missing files and re-render remaining
                    if not os.path.exists(output_filepath):
                        errors.append("CRASH")
                        print_message("Crash running Blender")
                        print_message(testname, 'FAILURE', 'FAILED')
                        break

                testname = test_get_name(filepath)

                if not os.path.exists(output_filepath) or os.path.getsize(output_filepath) == 0:
                    errors.append("NO OUTPUT")
                    print_message("No render result file found")
                    print_message(testname, 'FAILURE', 'FAILED')
                elif not self._diff_output(filepath, output_filepath):
                    errors.append("VERIFY")
                    print_message("Render result is different from reference image")
                    print_message(testname, 'FAILURE', 'FAILED')
                else:
                    errors.append(None)
                    print_message(testname, 'SUCCESS', 'OK')

                if os.path.exists(output_filepath):
                    os.remove(output_filepath)

        return errors

    def _run_all_tests(self, dirname, dirpath, blender, arguments_cb, batch, fail_silently):
        passed_tests = []
        failed_tests = []
        silently_failed_tests = []
        all_files = list(blend_list(dirpath, self.blocklist))
        all_files.sort()
        if not list(blend_list(dirpath, [])):
            print_message("No .blend files found in '{}'!".format(dirpath), 'FAILURE', 'FAILED')
            return False

        print_message("Running {} tests from 1 test case." .
                      format(len(all_files)),
                      'SUCCESS', "==========")
        time_start = time.time()
        errors = self._run_tests(all_files, blender, arguments_cb, batch)
        for filepath, error in zip(all_files, errors):
            testname = test_get_name(filepath)
            if error:
                if error == "NO_ENGINE":
                    return False
                elif error == "NO_START":
                    return False

                if fail_silently and error != 'CRASH':
                    silently_failed_tests.append(testname)
                else:
                    failed_tests.append(testname)
            else:
                passed_tests.append(testname)
            self._write_test_html(dirname, filepath, error)
        time_end = time.time()
        elapsed_ms = int((time_end - time_start) * 1000)
        print_message("")
        print_message("{} tests from 1 test case ran. ({} ms total)" .
                      format(len(all_files), elapsed_ms),
                      'SUCCESS', "==========")
        print_message("{} tests." .
                      format(len(passed_tests)),
                      'SUCCESS', 'PASSED')
        all_failed_tests = silently_failed_tests + failed_tests
        if all_failed_tests:
            print_message("{} tests, listed below:" .
                          format(len(all_failed_tests)),
                          'FAILURE', 'FAILED')
            all_failed_tests.sort()
            for test in all_failed_tests:
                print_message("{}" . format(test), 'FAILURE', "FAILED")

        return not bool(failed_tests)