diff --git a/Makefile b/Makefile index b04bfbc93..abb4bbddf 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ cblas_testing: cblaslib blaslib lapack_testing: tmglib lapacklib blaslib $(MAKE) -C TESTING/LIN cleanexe $(MAKE) -C TESTING - ./lapack_testing.py + ./lapack_testing.py --fail-if-empty --fail-on-unrecognized .PHONY: variants_testing variants_testing: tmglib variants lapacklib blaslib diff --git a/TESTING/CMakeLists.txt b/TESTING/CMakeLists.txt index fe8f07c34..29fcbf136 100644 --- a/TESTING/CMakeLists.txt +++ b/TESTING/CMakeLists.txt @@ -21,7 +21,7 @@ if(Python3_EXECUTABLE AND LAPACK_TESTING_USE_PYTHON) add_test( NAME LAPACK_Test_Summary WORKING_DIRECTORY ${LAPACK_BINARY_DIR} - COMMAND ${Python3_EXECUTABLE} "lapack_testing.py" + COMMAND ${Python3_EXECUTABLE} "lapack_testing.py" "--fail-if-empty" "--fail-on-unrecognized" ) endif() diff --git a/lapack_testing.py b/lapack_testing.py index dc3c471b5..8dbd4abe3 100755 --- a/lapack_testing.py +++ b/lapack_testing.py @@ -1,327 +1,1016 @@ #!/usr/bin/env python3 +"""Summarize (and optionally run) the LAPACK Fortran test suite. +This script analyzes the ``.out`` files written by the LAPACK testing +drivers (``xlintst*``, ``xeigtst*`` and ``xdmdeigtst*``) and prints a +summary table of the number of tests run and the number of failures per +precision (s/d/c/z). With ``--run`` it executes the testing drivers +first and then analyzes their output. -############################################################################### -# lapack_testing.py -############################################################################### +When index-64 extended API outputs (``*_64.out``, produced by CMake +builds with ``BUILD_INDEX64_EXT_API=ON``) are present, they are analyzed +as well and reported in a separate "extended API" section so that the +default-API totals remain comparable across builds. -from subprocess import Popen, STDOUT, PIPE -import os, sys, math -import getopt -# Arguments -try: - opts, args = getopt.getopt(sys.argv[1:], "hd:b:srep:t:n", - ["help", "dir=", "bin=", "short", "run", "error","prec=","test=","number"]) +Examples: + ./lapack_testing.py -n + Print the numbers of failed tests by analyzing the LAPACK output. -except getopt.error as msg: - print(msg) - print("for help use --help") - sys.exit(2) + ./lapack_testing.py -n -r -p s + Run the REAL precision tests, then print the numbers of failures. -short_summary = False -with_file = True -just_errors = False -prec='x' -test='all' -only_numbers = False -test_dir='TESTING' -bin_dir='bin/Release' + ./lapack_testing.py -n -p s -t eig + Print the numbers of failures in REAL precision by analyzing only + the eigenproblem test output. +""" -for o, a in opts: - if o in ("-h", "--help"): - print(sys.argv[0]+" [-h|--help] [-d dir |--dir dir] [-s |--short] [-r |--run] [-e |--error] [-p p |--prec p] [-t test |--test test] [-n | --number]") - print(" - h is to print this message") - print(" - r is to use to run the LAPACK tests then analyse the output (.out files). By default, the script will not run all the LAPACK tests") - print(" - d [dir] indicates the location of the LAPACK testing directory (.out files). By default, the script will use {:s}.".format(test_dir)) - print(" - b [bin] indicates the location of the LAPACK binary files. By default, the script will use {:s}.".format(bin_dir)) - print(" LEVEL OF OUTPUT") - print(" - e is to print only the error summary") - print(" - s is to print a short summary") - print(" - n is to print the numbers of failing tests (turn on summary mode)") - print(" SELECTION OF TESTS:") - print(" - p [s/c/d/z/x] is to indicate the PRECISION to run:") - print(" s=single") - print(" d=double") - print(" sd=single/double") - print(" c=complex") - print(" z=double complex") - print(" cz=complex/double complex") - print(" x=all [DEFAULT]") - print(" - t [lin/eig/mixed/rfp/all] is to indicate which TEST FAMILY to run:") - print(" lin=Linear Equation") - print(" eig=Eigen Problems") - print(" mixed=mixed-precision") - print(" rfp=rfp format") - print(" all=all tests [DEFAULT]") - print(" EXAMPLES:") - print(" ./lapack_testing.py -n") - print(" Will return the numbers of failed tests by analyzing the LAPACK output") - print(" ./lapack_testing.py -n -r -p s") - print(" Will return the numbers of failed tests in REAL precision by running the LAPACK Tests then analyzing the output") - print(" ./lapack_testing.py -n -p s -t eig ") - print(" Will return the numbers of failed tests in REAL precision by analyzing only the LAPACK output of EIGEN testings") - sys.exit(0) - else: - if o in ("-s", "--short"): - short_summary = True - if o in ("-r", "--run"): - with_file = False - if o in ("-e", "--error"): - just_errors = True - if o in ( '-p', '--prec' ): - prec = a - if o in ( '-b', '--bin' ): - bin_dir = a - if o in ( '-d', '--dir' ): - test_dir = a - if o in ( '-t', '--test' ): - test = a - if o in ( '-n', '--number' ): - only_numbers = True - short_summary = True +from __future__ import annotations -# process options +import argparse +import io +import math +import re +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING -abs_bin_dir=os.path.abspath(bin_dir) +if TYPE_CHECKING: + from typing import Dict, List, Optional, Sequence, TextIO, Tuple -os.chdir(test_dir) - -execution=1 -summary="\n\t\t\t--> LAPACK TESTING SUMMARY <--\n"; -if with_file: summary+= "\t\tProcessing LAPACK Testing output found in the "+test_dir+" directory\n"; -summary+="SUMMARY \tnb test run \tnumerical error \tother error \n"; -summary+="================ \t===========\t=================\t================ \n"; -nb_of_test=0 - -# Add current directory to the path for subshells of this shell -# Allows the popen to find local files in both windows and unixes -os.environ["PATH"] = os.environ["PATH"]+":." - -# Define a function to open the executable (different filenames on unix and Windows) -def run_summary_test( f, cmdline, short_summary): - nb_test_run=0 - nb_test_fail=0 - nb_test_illegal=0 - nb_test_info=0 - - if with_file: - if not os.path.exists(cmdline): - error_message=cmdline+" file not found" - r=1 - if short_summary: return [nb_test_run,nb_test_fail,nb_test_illegal,nb_test_info] - else: - pipe = open(cmdline,'r') - r=0 - else: - cmdline = os.path.join(abs_bin_dir, cmdline) - - outfile=cmdline.split()[4] - #pipe = open(outfile,'w') - p = Popen(cmdline, shell=True)#, stdout=pipe) - p.wait() - #pipe.close() - r=p.returncode - pipe = open(outfile,'r') - error_message=cmdline+" did not work" - - if r != 0 and not with_file: - print("---- TESTING " + cmdline.split()[0] + "... FAILED(" + error_message +") !") - for line in pipe.readlines(): - f.write(str(line)) - elif r != 0 and with_file and not short_summary: - print("---- WARNING: please check that you have the LAPACK output : "+cmdline+"!") - print("---- WARNING: with the option -r, we can run the LAPACK testing for you") - # print "---- "+error_message - else: - for line in pipe.readlines(): - f.write(str(line)) - words_in_line=line.split() - if (line.find("run)")!=-1): -# print line - whereisrun=words_in_line.index("run)") - nb_test_run+=int(words_in_line[whereisrun-2]) - if (line.find("out of")!=-1): - if not short_summary: print(line, end=' ') - whereisout= words_in_line.index("out") - nb_test_fail+=int(words_in_line[whereisout-1]) - if ((line.find("illegal")!=-1) or (line.find("Illegal")!=-1)): - if not short_summary: print(line, end=' ') - nb_test_illegal+=1 - if (line.find(" INFO")!=-1): - if not short_summary: print(line, end=' ') - nb_test_info+=1 - if with_file: - pipe.close() - - f.flush(); - - return [nb_test_run,nb_test_fail,nb_test_illegal,nb_test_info] - - -# If filename cannot be opened, send output to sys.stderr -filename = "testing_results.txt" -try: - f = open(filename, 'w') -except IOError: - f = sys.stdout - -if not short_summary: - print(" ") - print("---------------- Testing LAPACK Routines ----------------") - print(" ") - print("-- Detailed results are stored in", filename) - -dtypes = ( -("s", "d", "c", "z"), -("REAL ", "DOUBLE PRECISION", "COMPLEX ", "COMPLEX16 "), +# Precision letters and the labels used in the summary table. +PRECISIONS: "Tuple[Tuple[str, str], ...]" = ( + ("s", "REAL"), + ("d", "DOUBLE PRECISION"), + ("c", "COMPLEX"), + ("z", "COMPLEX16"), ) -if prec=='s': - range_prec=[0] -elif prec=='d': - range_prec=[1] -elif prec=='sd': - range_prec=[0,1] -elif prec=='c': - range_prec=[2] -elif prec=='z': - range_prec=[3] -elif prec=='cz': - range_prec=[2,3] -else: - prec='x'; - range_prec=list(range(4)) +# Second precision letter of the mixed-precision linear equation tests. +MIXED_PARTNER: "Dict[str, str]" = {"d": "s", "z": "c"} -if test=='lin': - range_test=[16] -elif test=='mixed': - range_test=[17] - range_prec=[1,3] -elif test=='rfp': - range_test=[18] -elif test=='dmd': - range_test=[20] -elif test=='eig': - range_test=list(range(16)) -else: - range_test=list(range(19)) +# Eigenproblem test sets using the classic ``alasum``/``alasvm`` summary +# format: (name, has shared input file, description). Sets with a shared +# input read e.g. ``nep.in``; the others read e.g. ``sec.in``/``dec.in``. +EIG_STANDARD_SETS: "Tuple[Tuple[str, bool, str], ...]" = ( + ("nep", True, "Nonsymmetric Eigenvalue Problem"), + ("sep", True, "Symmetric Eigenvalue Problem"), + ("se2", True, "Symmetric Eigenvalue Problem 2-stage"), + ("svd", True, "Singular Value Decomposition"), + ("ec", False, "Eigen Condition"), + ("ed", False, "Nonsymmetric Eigenvalue"), + ("gg", False, "Nonsymmetric Generalized Eigenvalue Problem"), + ("gd", False, "Nonsymmetric Generalized Eigenvalue Problem driver"), + ("sb", False, "Symmetric Eigenvalue Problem"), + ("sg", False, "Symmetric Eigenvalue Generalized Problem"), + ("bb", False, "Banded Singular Value Decomposition routines"), + ("glm", True, "Generalized Linear Regression Model routines"), + ("gqr", True, "Generalized QR and RQ factorization routines"), + ("gsv", True, "Generalized Singular Value Decomposition routines"), + ("csd", True, "CS Decomposition routines"), + ("lse", True, "Constrained Linear Least Squares routines"), +) -list_results = [ -[0, 0, 0, 0, 0], -[0, 0, 0, 0, 0], -[0, 0, 0, 0, 0], -[0, 0, 0, 0, 0], -] +# Balancing/backtransformation test sets, which use the ``schkbl``-style +# "total number of examples tested" summary format. +EIG_BALANCE_SETS: "Tuple[Tuple[str, str], ...]" = ( + ("bal", "Matrix Balancing"), + ("bak", "Balancing Backtransformation"), + ("gbal", "Generalized Matrix Balancing"), + ("gbak", "Generalized Balancing Backtransformation"), +) -for dtype in range_prec: - letter = dtypes[0][dtype] - name = dtypes[1][dtype] +# API suffixes that may exist: default API and index-64 extended API. +KNOWN_SUFFIXES: "Tuple[str, ...]" = ("", "_64") - if not short_summary: - print(" ") - print("------------------------- %s ------------------------" % name) - print(" ") - sys.stdout.flush() +RESULTS_FILENAME = "testing_results.txt" - dtests = ( - ("nep", "sep", "se2", "svd", - letter+"ec",letter+"ed",letter+"gg", - letter+"gd",letter+"sb",letter+"sg", - letter+"bb","glm","gqr", - "gsv","csd","lse", - letter+"test", letter+dtypes[0][dtype-1]+"test",letter+"test_rfp",letter+"dmd"), - ("Nonsymmetric-Eigenvalue-Problem", "Symmetric-Eigenvalue-Problem", "Symmetric-Eigenvalue-Problem-2-stage", "Singular-Value-Decomposition", - "Eigen-Condition","Nonsymmetric-Eigenvalue","Nonsymmetric-Generalized-Eigenvalue-Problem", - "Nonsymmetric-Generalized-Eigenvalue-Problem-driver", "Symmetric-Eigenvalue-Problem", "Symmetric-Eigenvalue-Generalized-Problem", - "Banded-Singular-Value-Decomposition-routines", "Generalized-Linear-Regression-Model-routines", "Generalized-QR-and-RQ-factorization-routines", - "Generalized-Singular-Value-Decomposition-routines", "CS-Decomposition-routines", "Constrained-Linear-Least-Squares-routines", - "Linear-Equation-routines", "Mixed-Precision-linear-equation-routines","RFP-linear-equation-routines","Dynamic-Mode-Decomposition"), - (letter+"nep", letter+"sep", letter+"se2", letter+"svd", - letter+"ec",letter+"ed",letter+"gg", - letter+"gd",letter+"sb",letter+"sg", - letter+"bb",letter+"glm",letter+"gqr", - letter+"gsv",letter+"csd",letter+"lse", - letter+"test", letter+dtypes[0][dtype-1]+"test",letter+"test_rfp",letter+"dmd"), +# Classic summary lines printed by alasum.f/alasvm.f: +# " All tests for XYZ routines passed the threshold ( ddd tests run)" +# " XYZ: ddd out of ddd tests failed to pass the threshold" +RE_TESTS_RUN = re.compile(r"(\d+)\s+tests run\)") +RE_TESTS_FAILED = re.compile(r"(\d+)\s+out of\s+(\d+)") + +# Failure records printed by the eigencondition checkers (schkec.f and +# friends), e.g. " Error in STRSYL: RMAX =..." — one per failing routine. +RE_EC_ERROR = re.compile(r"^ ?Error in \w+") + +# Summary lines printed by the balancing checkers (schkbl.f and friends). +# The complex generalized checkers use slightly different wording +# ("ratio of largest test error", "ILO or IHI is wrong"). +RE_EXAMPLES_TESTED = re.compile(r"total number of examples tested\s*=\s*(\d+)") +RE_INFO_NOT_ZERO = re.compile(r"number of examples where info is not 0\s*=\s*(\d+)") +RE_ILO_IHI_WRONG = re.compile( + r"example number where ILO or IHI (?:is )?wrong\s*=\s*(\d+)" +) +RE_LARGEST_ERROR = re.compile(r"(?:value|ratio) of largest test error\s*=\s*(\S+)") + +# Per-test verdict lines printed by the DMD checkers (schkdmd.f90 and +# friends), e.g. ">>>> Z - U*V test PASSED.". The word boundary keeps +# aggregate lines such as "SGEDMD :: ALL TESTS PASSED." from matching. +RE_DMD_VERDICT = re.compile(r"\btest\s+(PASSED|FAILED)\b", re.IGNORECASE) + +# Parser kinds, used by TestCase.parser. +PARSER_STANDARD = "standard" +PARSER_BALANCE = "balance" +PARSER_DMD = "dmd" + + +@dataclass +class Counts: + """Accumulated test counts for one or more test output files.""" + + runs: int = 0 + numerical: int = 0 + illegal: int = 0 + info: int = 0 + + @property + def other(self) -> int: + """Return the number of non-numerical errors (illegal + info). + + Returns: + The combined number of "illegal value" and INFO errors. + """ + return self.illegal + self.info + + @property + def errors(self) -> int: + """Return the total number of errors of any kind. + + Returns: + The combined number of numerical and other errors. + """ + return self.numerical + self.other + + def add(self, other: "Counts") -> None: + """Accumulate another set of counts into this one. + + Args: + other: The counts to add in place. + """ + self.runs += other.runs + self.numerical += other.numerical + self.illegal += other.illegal + self.info += other.info + + +@dataclass +class FileReport: + """Parsing result for a single test output file.""" + + counts: Counts = field(default_factory=Counts) + notable_lines: "List[str]" = field(default_factory=list) + + +@dataclass(frozen=True) +class TestCase: + """One LAPACK test driver invocation and its expected output file.""" + + precision: str + family: str + description: str + input_name: str + output_name: str + executable: str + parser: str + + def suffixed_output(self, suffix: str) -> str: + """Return the output file name for an API suffix. + + Args: + suffix: The API suffix, either ``""`` or ``"_64"``. + + Returns: + The output file name, e.g. ``snep_64.out`` for suffix + ``"_64"`` and base output name ``snep.out``. + """ + stem = self.output_name[: -len(".out")] + return "{}{}.out".format(stem, suffix) + + def suffixed_executable(self, suffix: str) -> str: + """Return the test driver name for an API suffix. + + Args: + suffix: The API suffix, either ``""`` or ``"_64"``. + + Returns: + The executable name, e.g. ``xeigtsts_64``. + """ + return self.executable + suffix + + +def build_test_cases(letters: str, families: "Sequence[str]") -> "List[TestCase]": + """Build the list of test cases for the selected precisions/families. + + Args: + letters: Precision letters to include, in order (subset of + ``"sdcz"``). + families: Test families to include (subset of ``lin``, ``eig``, + ``mixed``, ``rfp``, ``dmd``). + + Returns: + The test cases in reporting order: for each precision, the + eigenproblem sets, then the linear equation, mixed precision, + RFP and DMD sets. + """ + cases: "List[TestCase]" = [] + for letter in letters: + if "eig" in families: + for name, shared_input, description in EIG_STANDARD_SETS: + cases.append( + TestCase( + precision=letter, + family="eig", + description=description, + input_name=(name if shared_input else letter + name) + ".in", + output_name=letter + name + ".out", + executable="xeigtst" + letter, + parser=PARSER_STANDARD, + ) + ) + for name, description in EIG_BALANCE_SETS: + cases.append( + TestCase( + precision=letter, + family="eig", + description=description, + input_name=letter + name + ".in", + output_name=letter + name + ".out", + executable="xeigtst" + letter, + parser=PARSER_BALANCE, + ) + ) + if "lin" in families: + cases.append( + TestCase( + precision=letter, + family="lin", + description="Linear Equation routines", + input_name=letter + "test.in", + output_name=letter + "test.out", + executable="xlintst" + letter, + parser=PARSER_STANDARD, + ) + ) + if "mixed" in families and letter in MIXED_PARTNER: + partner = MIXED_PARTNER[letter] + cases.append( + TestCase( + precision=letter, + family="mixed", + description="Mixed Precision linear equation routines", + input_name=letter + partner + "test.in", + output_name=letter + partner + "test.out", + executable="xlintst" + letter + partner, + parser=PARSER_STANDARD, + ) + ) + if "rfp" in families: + cases.append( + TestCase( + precision=letter, + family="rfp", + description="RFP linear equation routines", + input_name=letter + "test_rfp.in", + output_name=letter + "test_rfp.out", + executable="xlintstrf" + letter, + parser=PARSER_STANDARD, + ) + ) + if "dmd" in families: + cases.append( + TestCase( + precision=letter, + family="dmd", + description="Dynamic Mode Decomposition", + input_name=letter + "dmd.in", + output_name=letter + "dmd.out", + executable="xdmdeigtst" + letter, + parser=PARSER_DMD, + ) + ) + return cases + + +def parse_standard(lines: "Sequence[str]") -> FileReport: + """Parse a test output file in the classic alasum/alasvm format. + + Counts runs from both the passing summary lines (``... tests run)``) + and the failing summary lines (``N out of M tests failed ...``), so + that failing test sets contribute to the run total as well. The + eigencondition checkers report failures as ``Error in `` + records instead; each such record counts as one numerical failure. + + Args: + lines: The lines of the output file. + + Returns: + The counts and the notable (error) lines of the file. + """ + report = FileReport() + for line in lines: + match = RE_TESTS_RUN.search(line) + if match: + report.counts.runs += int(match.group(1)) + continue + match = RE_TESTS_FAILED.search(line) + if match: + report.counts.numerical += int(match.group(1)) + report.counts.runs += int(match.group(2)) + report.notable_lines.append(line) + continue + if RE_EC_ERROR.match(line): + report.counts.numerical += 1 + report.notable_lines.append(line) + continue + if "illegal" in line or "Illegal" in line: + report.counts.illegal += 1 + report.notable_lines.append(line) + continue + if " INFO" in line: + report.counts.info += 1 + report.notable_lines.append(line) + return report + + +def parse_balance(lines: "Sequence[str]") -> FileReport: + """Parse a balancing/backtransformation test output file. + + These checkers (``schkbl.f`` and friends) do not use the alasum + summary format. Runs are taken from the ``total number of examples + tested`` line, INFO errors from the ``number of examples where info + is not 0`` line. A non-finite ``value of largest test error`` or a + nonzero ``example number where ILO or IHI wrong`` is counted as one + numerical failure. + + Args: + lines: The lines of the output file. + + Returns: + The counts and the notable (error) lines of the file. + """ + report = FileReport() + for line in lines: + match = RE_EXAMPLES_TESTED.search(line) + if match: + report.counts.runs += int(match.group(1)) + continue + match = RE_INFO_NOT_ZERO.search(line) + if match: + info_errors = int(match.group(1)) + report.counts.info += info_errors + if info_errors > 0: + report.notable_lines.append(line) + continue + match = RE_ILO_IHI_WRONG.search(line) + if match: + if int(match.group(1)) != 0: + report.counts.numerical += 1 + report.notable_lines.append(line) + continue + match = RE_LARGEST_ERROR.search(line) + if match: + # Fortran prints double precision exponents as 0.1D+01. + token = match.group(1).replace("D", "E").replace("d", "e") + try: + value = float(token) + except ValueError: + value = math.inf + if not math.isfinite(value): + report.counts.numerical += 1 + report.notable_lines.append(line) + return report + + +def parse_dmd(lines: "Sequence[str]") -> FileReport: + """Parse a dynamic mode decomposition test output file. + + Each per-test verdict line (``... test PASSED.`` or ``... test + FAILED ...``) counts as one test run; each FAILED verdict counts as + one numerical failure (the line itself reports how many individual + cases failed). + + Args: + lines: The lines of the output file. + + Returns: + The counts and the notable (error) lines of the file. + """ + report = FileReport() + for line in lines: + match = RE_DMD_VERDICT.search(line) + if match: + report.counts.runs += 1 + if match.group(1).upper() == "FAILED": + report.counts.numerical += 1 + report.notable_lines.append(line) + return report + + +def parse_lines(parser: str, lines: "Sequence[str]") -> FileReport: + """Parse test output lines with the parser kind of a test case. + + Args: + parser: One of ``PARSER_STANDARD``, ``PARSER_BALANCE`` and + ``PARSER_DMD``. + lines: The lines of the output file. + + Returns: + The counts and the notable (error) lines of the file. + """ + if parser == PARSER_BALANCE: + return parse_balance(lines) + if parser == PARSER_DMD: + return parse_dmd(lines) + return parse_standard(lines) + + +def find_unrecognized_outputs(test_dir: Path) -> "List[str]": + """Find ``.out`` files in the test directory this script cannot analyze. + + A file is unrecognized if its name matches no known test case in any + precision, family or API variant — typically a test that was added to + the harness without extending this script's test tables, or a renamed + output such as those of ``make variants_testing``. The current + ``-p``/``-t`` selection is deliberately ignored: a deselected file is + not an unrecognized one. + + Args: + test_dir: The directory containing the ``.out`` files. + + Returns: + The unrecognized file names, sorted alphabetically. + """ + all_cases = build_test_cases("sdcz", ["lin", "eig", "mixed", "rfp", "dmd"]) + known = { + case.suffixed_output(suffix) for case in all_cases for suffix in KNOWN_SUFFIXES + } + return sorted( + path.name for path in test_dir.glob("*.out") if path.name not in known ) - for dtest in range_test: - nb_of_test=0 - # NEED TO SKIP SOME PRECISION (namely s and c) FOR PROTO MIXED PRECISION TESTING - if dtest==17 and (letter=="s" or letter=="c"): - continue - if with_file: - cmdbase=dtests[2][dtest]+".out" +def discover_suffixes(test_dir: Path, cases: "Sequence[TestCase]") -> "List[str]": + """Detect which API variants have output files in the test directory. + + Args: + test_dir: The directory containing the ``.out`` files. + cases: The selected test cases. + + Returns: + The suffixes (out of ``""`` and ``"_64"``) for which at least one + expected output file exists; ``[""]`` if none exist at all. + """ + suffixes = [ + suffix + for suffix in KNOWN_SUFFIXES + if any((test_dir / case.suffixed_output(suffix)).is_file() for case in cases) + ] + return suffixes or [""] + + +def find_executable(name: str, bin_dir: "Optional[str]") -> "Optional[Path]": + """Locate a test driver executable. + + Args: + name: The executable name without platform suffix, e.g. + ``xlintsts``. + bin_dir: The directory passed via ``--bin``, or None to probe the + usual locations of CMake and Makefile builds relative to the + current working directory. + + Returns: + The absolute path of the executable, or None if it was not found. + """ + if bin_dir is not None: + directories = [Path(bin_dir)] + else: + directories = [ + Path("bin"), + Path("bin") / "Release", + Path("bin") / "Debug", + Path("TESTING") / "LIN", + Path("TESTING") / "EIG", + ] + for directory in directories: + for filename in (name, name + ".exe"): + candidate = directory / filename + if candidate.is_file(): + return candidate.resolve() + return None + + +def run_test_case( + case: TestCase, suffix: str, test_dir: Path, bin_dir: "Optional[str]" +) -> "Optional[str]": + """Run one test driver, redirecting its output to the ``.out`` file. + + Args: + case: The test case to run. + suffix: The API suffix, either ``""`` or ``"_64"``. + test_dir: The directory containing the ``.in`` files; the driver + runs there and the ``.out`` file is written there. + bin_dir: The directory containing the test drivers, or None to + probe the usual locations. + + Returns: + An error message if the driver could not be run or exited with a + nonzero status, otherwise None. + """ + executable_name = case.suffixed_executable(suffix) + executable = find_executable(executable_name, bin_dir) + if executable is None: + return "executable {} not found".format(executable_name) + input_path = test_dir / case.input_name + if not input_path.is_file(): + # CMake build trees hold only the .out files; the .in files + # live next to this script in the source tree. + source_input = Path(__file__).resolve().parent / "TESTING" / case.input_name + if source_input.is_file(): + input_path = source_input else: - if dtest==16: - # LIN TESTS - cmdbase="xlintst"+letter+" < "+dtests[0][dtest]+".in > "+dtests[2][dtest]+".out" - elif dtest==17: - # PROTO LIN TESTS - cmdbase="xlintst"+letter+dtypes[0][dtype-1]+" < "+dtests[0][dtest]+".in > "+dtests[2][dtest]+".out" - elif dtest==18: - # PROTO LIN TESTS - cmdbase="xlintstrf"+letter+" < "+dtests[0][dtest]+".in > "+dtests[2][dtest]+".out" - elif dtest==20: - # DMD EIG TESTS - cmdbase="xdmdeigtst"+letter+" < "+dtests[0][dtest]+".in > "+dtests[2][dtest]+".out" - else: - # EIG TESTS - cmdbase="xeigtst"+letter+" < "+dtests[0][dtest]+".in > "+dtests[2][dtest]+".out" - if not just_errors and not short_summary: - print("Testing "+name+" "+dtests[1][dtest]+"-"+cmdbase, end=' ') - # Run the process: either to read the file or run the LAPACK testing - nb_test = run_summary_test(f, cmdbase, short_summary) - list_results[0][dtype]+=nb_test[0] - list_results[1][dtype]+=nb_test[1] - list_results[2][dtype]+=nb_test[2] - list_results[3][dtype]+=nb_test[3] - got_error=nb_test[1]+nb_test[2]+nb_test[3] + return "input file {} not found".format(input_path) + output_path = test_dir / case.suffixed_output(suffix) + # Write to a temporary file first so that a driver that cannot even + # start does not clobber the results of an earlier run. + temporary_path = output_path.with_name(output_path.name + ".tmp") + try: + with open(str(input_path), "rb") as stdin, open( + str(temporary_path), "wb" + ) as stdout: + process = subprocess.run( + [str(executable)], + stdin=stdin, + stdout=stdout, + stderr=subprocess.STDOUT, + cwd=str(test_dir), + ) + except OSError as error: + try: + temporary_path.unlink() + except OSError: + pass + return "{} could not be run: {}".format(executable_name, error) + temporary_path.replace(output_path) + if process.returncode != 0: + return "{} exited with status {}".format(executable_name, process.returncode) + return None - if not short_summary: - if nb_test[0] > 0 and not just_errors: - print("passed: "+str(nb_test[0])) - if nb_test[1] > 0: - print("failing to pass the threshold: "+str(nb_test[1])) - if nb_test[2] > 0: - print("Illegal Error: "+str(nb_test[2])) - if nb_test[3] > 0: - print("Info Error: "+str(nb_test[3])) - if got_error > 0 and just_errors: - print("ERROR IS LOCATED IN "+name+" "+dtests[1][dtest]+" [ "+cmdbase+" ]") - print("") - if not just_errors: - print("") -# elif (got_error>0): -# print dtests[2][dtest]+".out \t"+str(nb_test[1])+"\t"+str(nb_test[2])+"\t"+str(nb_test[3]) - sys.stdout.flush() - if (list_results[0][dtype] > 0 ): - percent_num_error=float(list_results[1][dtype])/float(list_results[0][dtype])*100 - percent_error=float(list_results[2][dtype]+list_results[3][dtype])/float(list_results[0][dtype])*100 +def read_output_file(path: Path) -> "Optional[List[str]]": + """Read a test output file. + + Args: + path: The path of the ``.out`` file. + + Returns: + The lines of the file, or None if the file does not exist or + cannot be read (the cause is then reported on standard error). + """ + if not path.is_file(): + return None + try: + with open(str(path), encoding="utf-8", errors="replace") as handle: + return handle.readlines() + except OSError as error: + print( + "lapack_testing.py: cannot read {}: {}".format(path, error), + file=sys.stderr, + ) + return None + + +# Fixed-width summary table columns: label, run count and one block of +# "count (percent)" per error kind. Values are right-aligned so that +# they line up under the ==== rule of their column. The 73-column +# table is indented to sit centered under the 80-column headings. +SUMMARY_INDENT = " " +SUMMARY_HEADER = SUMMARY_INDENT + "{:<18} {:>13} {:>18} {:>18}".format( + "SUMMARY", "nb test run", "numerical error", "other error" +) +SUMMARY_RULE = SUMMARY_INDENT + " ".join(("=" * 18, "=" * 13, "=" * 18, "=" * 18)) + + +def format_summary_row(label: str, counts: Counts) -> str: + """Format one row of the summary table. + + Args: + label: The row label, e.g. a precision name. + counts: The counts to report in the row. + + Returns: + The formatted table row without trailing newline. + """ + if counts.runs > 0: + numerical_percent = 100.0 * counts.numerical / counts.runs + other_percent = 100.0 * counts.other / counts.runs else: - percent_num_error=0 - percent_error=0 - summary+=name+"\t"+str(list_results[0][dtype])+"\t\t"+str(list_results[1][dtype])+"\t("+"%.3f" % percent_num_error+"%)\t"+str(list_results[2][dtype]+list_results[3][dtype])+"\t("+"%.3f" % percent_error+"%)\t""\n" - list_results[0][4]+=list_results[0][dtype] - list_results[1][4]+=list_results[1][dtype] - list_results[2][4]+=list_results[2][dtype] - list_results[3][4]+=list_results[3][dtype] + numerical_percent = 0.0 + other_percent = 0.0 + numerical_percent_str = "({:.1f}%)".format(numerical_percent) + other_percent_str = "({:.1f}%)".format(other_percent) + return SUMMARY_INDENT + "{:<18} {:>13} {:>9} {:>8} {:>9} {:>8}".format( + label, + counts.runs, + counts.numerical, + numerical_percent_str, + counts.other, + other_percent_str, + ) -if only_numbers: - print(str(list_results[1][4])+"\n"+str(list_results[2][4]+list_results[3][4])) -else: - print(summary) - if (list_results[0][4] > 0 ): - percent_num_error=float(list_results[1][4])/float(list_results[0][4])*100 - percent_error=float(list_results[2][4]+list_results[3][4])/float(list_results[0][4])*100 + +def section_title(suffix: str) -> str: + """Return the human-readable name of an API section. + + Args: + suffix: The API suffix, either ``""`` or ``"_64"``. + + Returns: + The section name used in headings and messages. + """ + if suffix: + return "Extended API ({})".format(suffix) + return "Default API" + + +def section_heading(title: str) -> str: + """Return an 80-column dashed heading with a centered title. + + Args: + title: The heading text. + + Returns: + str: A line of exactly 80 characters — dashes running to both + edges (one space at the beginning and end) with the title + centered. + """ + return " {} \n".format(" {} ".format(title).center(78, "-")) + + +class SummaryLog: + """Collector for the detailed results file (``testing_results.txt``).""" + + def __init__(self, handle: "Optional[TextIO]") -> None: + """Initialize the collector. + + Args: + handle: The open results file, or None if it could not be + opened (details are then discarded). + """ + self._handle = handle + + def record(self, header: str, lines: "Sequence[str]") -> None: + """Append one analyzed output file to the results file. + + Args: + header: A short description of the file (its name). + lines: The lines of the file. + """ + if self._handle is None: + return + self._handle.write("==== {} ====\n".format(header)) + self._handle.writelines(lines) + self._handle.flush() + + def close(self) -> None: + """Close the results file if it was open.""" + if self._handle is not None: + self._handle.close() + + +def parse_args(argv: "Optional[Sequence[str]]" = None) -> argparse.Namespace: + """Parse the command line arguments. + + Args: + argv: The arguments to parse, or None to use ``sys.argv``. + + Returns: + The parsed arguments. + """ + parser = argparse.ArgumentParser( + description="Analyze the .out files produced by the LAPACK test " + "suite and print a summary of the test results.", + epilog="By default all precisions and all test families are " + "analyzed, and both the default API and extended API (_64) " + "outputs are summarized when present.", + ) + parser.add_argument( + "-d", + "--dir", + default="TESTING", + help="directory containing the LAPACK testing output (.out) files " + "(default: %(default)s)", + ) + parser.add_argument( + "-b", + "--bin", + default=None, + help="directory containing the LAPACK test drivers for --run; by " + "default bin, bin/Release, bin/Debug, TESTING/LIN and TESTING/EIG " + "are probed", + ) + parser.add_argument( + "-r", + "--run", + action="store_true", + help="run the LAPACK test drivers before analyzing their output " + "(by default only existing .out files are analyzed)", + ) + parser.add_argument( + "-s", + "--short", + action="store_true", + help="print only the summary table", + ) + parser.add_argument( + "-e", + "--error", + action="store_true", + help="print only the error summary", + ) + parser.add_argument( + "-n", + "--number", + action="store_true", + help="print only the numbers of failing tests (numerical failures " + "and other errors, one per line)", + ) + parser.add_argument( + "-p", + "--prec", + choices=["s", "d", "sd", "c", "z", "cz", "x"], + default="x", + help="precisions to analyze: s=single, d=double, sd=single/double, " + "c=complex, z=double complex, cz=complex/double complex, " + "x=all (default)", + ) + parser.add_argument( + "-t", + "--test", + choices=["lin", "eig", "mixed", "rfp", "dmd", "all"], + default="all", + help="test family to analyze: lin=linear equations, " + "eig=eigenproblems (including balancing), mixed=mixed precision, " + "rfp=RFP format, dmd=dynamic mode decomposition, all (default)", + ) + parser.add_argument( + "--suffix", + action="append", + choices=["none", "64"], + default=None, + help="API variant to analyze: 'none' for the default API, '64' for " + "the index-64 extended API; may be given twice (default: analyze " + "whichever variants have output files)", + ) + parser.add_argument( + "--fail-on-error", + action="store_true", + help="exit with a nonzero status if any test failure or error was " + "found, a test driver could not be run, or unrecognized .out files " + "were present in the testing directory", + ) + parser.add_argument( + "--fail-if-empty", + action="store_true", + help="exit with a nonzero status if no test results were analyzed", + ) + parser.add_argument( + "--fail-on-unrecognized", + action="store_true", + help="exit with a nonzero status if .out files not known to this " + "script were present in the testing directory", + ) + return parser.parse_args(argv) + + +def main(argv: "Optional[Sequence[str]]" = None) -> int: + """Run the LAPACK test summary tool. + + Args: + argv: The command line arguments, or None to use ``sys.argv``. + + Returns: + int: The process exit status. This is 2 for usage errors, 1 if a + condition requested via ``--fail-on-error``, ``--fail-if-empty`` + or ``--fail-on-unrecognized`` occurred, and 0 otherwise. + """ + args = parse_args(argv) + short_summary: bool = args.short or args.number + just_errors: bool = args.error + + test_dir = Path(args.dir) + if not test_dir.is_dir(): + print( + "lapack_testing.py: testing directory {} not found".format(test_dir), + file=sys.stderr, + ) + return 2 + + letters = "sdcz" if args.prec == "x" else args.prec + if args.test == "mixed": + # The mixed-precision drivers exist only for d (ds) and z (zc); + # like the old script, -t mixed analyzes both regardless of -p. + if args.prec not in ("x", "dz"): + print( + "lapack_testing.py: -t mixed always analyzes the d and z " + "mixed-precision tests; ignoring -p {}".format(args.prec), + file=sys.stderr, + ) + letters = "dz" + families = ( + ["lin", "eig", "mixed", "rfp", "dmd"] if args.test == "all" else [args.test] + ) + cases = build_test_cases(letters, families) + if not cases: + print( + "lapack_testing.py: no test cases match -p {} -t {}".format( + args.prec, args.test + ), + file=sys.stderr, + ) + return 2 + + if args.suffix is not None: + suffixes: "List[str]" = [] + for choice in args.suffix: + suffix = "" if choice == "none" else "_64" + if suffix not in suffixes: + suffixes.append(suffix) + elif args.run: + suffixes = [""] else: - percent_num_error=0 - percent_error=0 - if (prec=='x'): - print("--> ALL PRECISIONS\t"+str(list_results[0][4])+"\t\t"+str(list_results[1][4])+"\t("+"%.3f" % percent_num_error+"%)\t"+str(list_results[2][4]+list_results[3][4])+"\t("+"%.3f" % percent_error+"%)\t""\n") - if list_results[0][4] == 0: - print("NO TESTS WERE ANALYZED, please use the -r option to run the LAPACK TESTING") + suffixes = discover_suffixes(test_dir, cases) -# This may close the sys.stdout stream, so make it the last statement -f.close() + results_path = test_dir / RESULTS_FILENAME + try: + results_handle: "Optional[TextIO]" = open( + str(results_path), "w", encoding="utf-8" + ) + except OSError as error: + print( + "lapack_testing.py: cannot write {}: {}".format(results_path, error), + file=sys.stderr, + ) + results_handle = None + log = SummaryLog(results_handle) + + if not short_summary: + print(" ") + print("--> Testing LAPACK Routines <--".center(80).rstrip()) + print(" ") + print("Detailed results are stored in:".center(80).rstrip()) + print(str(results_path.resolve()).center(80).rstrip()) + + summary = "\n" + "--> LAPACK TESTING SUMMARY <--".center(80).rstrip() + "\n" + if not args.run: + summary += "Processing LAPACK Testing output found in:".center(80).rstrip() + summary += "\n" + str(test_dir.resolve()).center(80).rstrip() + "\n" + + grand_total = Counts() + missing_files = 0 + run_failures = 0 + + for suffix in suffixes: + if len(suffixes) > 1 or suffix: + summary += "\n" + section_heading(section_title(suffix)) + "\n" + if not short_summary: + print(" ") + print(section_heading(section_title(suffix))) + summary += SUMMARY_HEADER + "\n" + summary += SUMMARY_RULE + "\n" + section_total = Counts() + + for letter, precision_name in PRECISIONS: + precision_cases = [case for case in cases if case.precision == letter] + if not precision_cases: + continue + precision_total = Counts() + + for case in precision_cases: + output_name = case.suffixed_output(suffix) + if not just_errors and not short_summary: + print( + "Testing {} '{}' ({})".format( + precision_name, case.description, output_name + ), + end=" ", + ) + if args.run: + error_message = run_test_case(case, suffix, test_dir, args.bin) + if error_message is not None: + run_failures += 1 + print( + "---- TESTING {}... FAILED({})!".format( + case.suffixed_executable(suffix), error_message + ) + ) + lines = read_output_file(test_dir / output_name) + if lines is None: + missing_files += 1 + if not short_summary: + print( + "---- WARNING: please check that you have the LAPACK " + "output {}!".format(output_name) + ) + print( + "---- WARNING: with the option -r, we can run the " + "LAPACK testing for you" + ) + continue + log.record(output_name, lines) + report = parse_lines(case.parser, lines) + precision_total.add(report.counts) + + if not short_summary: + if not just_errors: + # Finish the "Testing ..." progress line. + if report.counts.runs > 0: + print("- passed: {}".format(report.counts.runs)) + else: + print("") + for line in report.notable_lines: + print("--> {}".format(line.strip())) + if report.counts.numerical > 0: + print( + "failing to pass the threshold: {}".format( + report.counts.numerical + ) + ) + if report.counts.illegal > 0: + print("Illegal Error: {}".format(report.counts.illegal)) + if report.counts.info > 0: + print("Info Error: {}".format(report.counts.info)) + if just_errors and report.counts.errors > 0: + print( + "ERROR IS LOCATED IN {} {} [ {} ]".format( + precision_name, case.description, output_name + ) + ) + if not just_errors or report.counts.errors > 0: + print("") + sys.stdout.flush() + + summary += format_summary_row(precision_name, precision_total) + "\n" + section_total.add(precision_total) + + if args.prec == "x": + summary += ( + "\n" + format_summary_row("--> ALL PRECISIONS", section_total) + "\n" + ) + grand_total.add(section_total) + + log.close() + + if args.number: + print(grand_total.numerical) + print(grand_total.other) + else: + print(summary) + if grand_total.runs == 0: + print( + "NO TESTS WERE ANALYZED, please use the -r option to run " + "the LAPACK TESTING" + ) + if missing_files > 0 and short_summary: + print( + "lapack_testing.py: {} expected output file(s) were missing " + "(rerun without -s/-n for details)".format(missing_files), + file=sys.stderr, + ) + + unrecognized = find_unrecognized_outputs(test_dir) + if unrecognized: + print( + "lapack_testing.py: {} .out file(s) in {} are not known to this " + "script and were NOT analyzed (new tests must be added to the " + "test tables in this script):".format(len(unrecognized), test_dir), + file=sys.stderr, + ) + for name in unrecognized: + print(" {}".format(name), file=sys.stderr) + + if args.fail_if_empty and grand_total.runs == 0: + return 1 + if args.fail_on_unrecognized and unrecognized: + return 1 + if args.fail_on_error and (grand_total.errors > 0 or run_failures): + return 1 + return 0 + + +def _configure_output_streams() -> None: + """Make stdout/stderr replace unencodable characters instead of dying. + + Fortran test output is not guaranteed to be encodable in the console + encoding (notably on Windows with a redirected stdout). + """ + for stream in (sys.stdout, sys.stderr): + if isinstance(stream, io.TextIOWrapper): + stream.reconfigure(errors="replace") + + +if __name__ == "__main__": + _configure_output_streams() + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..39f91a214 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,24 @@ +# Python tooling configuration for the repository's Python scripts +# (currently only lapack_testing.py). This file intentionally has no +# [project]/[build-system] tables: the repository is not a Python package. + +[tool.black] +target-version = ["py37"] + +[tool.ty.environment] +python-version = "3.7" + +[tool.ty.src] +include = ["lapack_testing.py"] + +[tool.basedpyright] +pythonVersion = "3.7" +typeCheckingMode = "strict" +include = ["lapack_testing.py"] + +[tool.pydocstyle] +convention = "google" + +[tool.interrogate] +fail-under = 100 +verbose = 1