diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index acceb3d..0f70a4e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,14 +20,23 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install ruff + python -m pip install build --group dev # ruff, pytest - name: Lint with ruff - run: ruff check src - - name: Install package - run: pip install . - - name: Run doctests - run: python -m cocopp.test doctest # takes about 5 seconds + run: ruff check src tests + - name: Build wheel + # cocopp is pure python: the wheel is the same py3-none-any wheel for all versions + run: python -m build --wheel -o dist . + - name: Install package from wheel + run: pip install dist/*.whl - name: Run tests - run: python -m cocopp.test # takes about 5 minutes - # - name: Run all tests - # run: python -m cocopp.test all # takes about 30 minutes + run: pytest tests # offline, including the full pipeline and snapshot tests, takes about 10 minutes + - name: Run tests using the online data archives + # the doctests, the archives, and the former default scenarios of python -m cocopp.test + run: pytest tests/network --run-network -k "not legacy" + # - name: Run all tests using the online data archives + # run: pytest tests/network --run-network # takes about 30 minutes + - name: Upload wheel + uses: actions/upload-artifact@v4 + with: + name: wheel-python-${{ matrix.python-version }} + path: dist/*.whl diff --git a/.github/workflows/tag_release.yml b/.github/workflows/tag_release.yml index b25cfad..762ad6d 100644 --- a/.github/workflows/tag_release.yml +++ b/.github/workflows/tag_release.yml @@ -27,8 +27,9 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install build - - name: Build source Python package - run: python -m build -s -o dist . + - name: Build source and wheel Python packages + # cocopp is pure python, hence a single py3-none-any wheel serves all Python versions + run: python -m build -o dist . - name: Publish release distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - name: Release diff --git a/pyproject.toml b/pyproject.toml index ce5033e..eddefae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ authors = [ ] description = "Benchmarking framework for all types of black-box optimization algorithms, postprocessing." readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.10" license = {text = "BSD-3-Clause"} classifiers = [ "Development Status :: 4 - Beta", @@ -71,5 +71,6 @@ ignore = ['E401', 'E402', [dependency-groups] dev = [ + "pytest>=9.1.1", "ruff>=0.14.3", ] diff --git a/src/cocopp/test.py b/src/cocopp/test.py deleted file mode 100644 index 4157370..0000000 --- a/src/cocopp/test.py +++ /dev/null @@ -1,557 +0,0 @@ -#!/usr/bin/env python -"""Tests the `cocopp` module. - -On a shell:: - - python -m cocopp.test doctest - -runs the doctests (which is quick) and:: - - python -m cocopp.test all - -runs also the most extensive tests. -""" - -from __future__ import absolute_import, division, print_function, unicode_literals -import os -import sys -import time -import inspect -import fnmatch -import urllib # noqa: F401 -import tempfile -import shutil -import subprocess -import doctest -import numpy as np - -from cocopp import archiving -from cocopp import official_archives -from cocopp import pproc as _pproc -from cocopp import genericsettings as settings - -import matplotlib # just to make sure the following is actually done first - -if sys.version_info[0] >= 3: - from urllib.request import urlretrieve -else: - from urllib import urlretrieve - -test_bibtex = False - - -class InfolderGoneWithTheWind: - """``with InfolderGoneWithTheWind(): ...`` executes the block in a - - temporary folder under the current folder. The temporary folder is - deleted on exiting the block. - - CAVEAT: copy-pasted to toolsdivers - - >>> import os - >>> dir_ = os.getcwd() # for the record - >>> len_ = len(os.listdir('.')) - >>> with InfolderGoneWithTheWind(): # doctest: +SKIP - ... # do some work in a folder here, e.g. write files - ... len(dir_) > len(os.getcwd()) and os.getcwd() in dir_ - True - >>> # magically we are back in the original folder - >>> assert dir_ == os.getcwd() - >>> assert len(os.listdir('.')) == len_ - - """ - - def __init__(self, prefix="_"): - """no folder needs to be given""" - self.prefix = prefix - - def __enter__(self): - self.root_dir = os.getcwd() - # self.target_dir = tempfile.mkdtemp(prefix=self.prefix, dir='.') - self.target_dir = tempfile.mkdtemp(prefix=self.prefix) - self._target_dir = self.target_dir - os.chdir(self.target_dir) - - def __exit__(self, *args): - os.chdir(self.root_dir) - if self.target_dir == self._target_dir: - shutil.rmtree(self.target_dir) - else: - raise ValueError("inconsistent temporary folder name %s vs %s" % (self._target_dir, self.target_dir)) - - -def depreciated_data_archive_get(substrs): - """CAVEAT: this won't work anymore as the get_first method changed to - get_one - - return first matching data paths for each element of `substrs` - concatenated in a string. - - Specifically:: - - return ' ' + ' '.join(cocopp._data_archive.get_first(substrs)) - - Implemented via `subprocess` to prevent the need to import `cocopp` in - this file here (prevent the script vs module absolute import issue). - """ - res = subprocess.check_output( - [ - "python", - "-c", - """ -from __future__ import division, print_function -from cocopp import data_archive -res = data_archive.get_first(""" - + repr(substrs) - + """) # get first match for each substr -print('_split_here_ ' + ' '.join(res), end='') # communication by print instead of return res -""", - ] - ) - # res is of type bytes - # res contains any output printed during execution (e.g. user infos) - # yet we should be able to split away and return the last print - return str(res).split("_split_here_")[-1] - - -def data_archive_get(substrs): - if str(substrs) == substrs: - return substrs - return " " + " ".join(substrs) - - -def join_path(a, *p): - joined_path = os.path.join(a, *p) - return joined_path - - -def copy_latex_templates(): - current_folder = os.path.dirname(os.path.realpath(__file__)) - template_folder = os.path.abspath(join_path(current_folder, "..", "latex-templates")) - shutil.copy(join_path(template_folder, "templateBBOBarticle.tex"), current_folder) - shutil.copy(join_path(template_folder, "templateBBOBcmp.tex"), current_folder) - shutil.copy(join_path(template_folder, "templateBBOBmany.tex"), current_folder) - shutil.copy(join_path(template_folder, "templateBIOBJarticle.tex"), current_folder) - shutil.copy(join_path(template_folder, "templateBIOBJmultiple.tex"), current_folder) - shutil.copy(join_path(template_folder, "templateNOISYarticle.tex"), current_folder) - - # Copy auxiliary files to the current working folder - cwd = os.getcwd() - shutil.copy(join_path(template_folder, "acmart.cls"), cwd) - shutil.copy(join_path(template_folder, "ACM-Reference-Format.bst"), cwd) - shutil.copy(join_path(template_folder, "comment.sty"), cwd) - shutil.copy(join_path(template_folder, "acmcopyright.sty"), cwd) - shutil.copy(join_path(template_folder, "bbob.bib"), cwd) - - -def run_latex_template(filename, all_tests): - file_path = os.path.abspath(join_path(os.path.dirname(__file__), filename)) - arguments = ["pdflatex", file_path] - DEVNULL = open(os.devnull, "wb") - result = subprocess.call(arguments, stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL) - assert not result, "Test failed: error while generating pdf from %s." % filename - - if all_tests and test_bibtex: - file_path = os.path.splitext(file_path)[0] - arguments = ["bibtex", file_path] - DEVNULL = open(os.devnull, "wb") - output_file = open("bibtex.log", "w") - result = subprocess.call(arguments, stdin=DEVNULL, stdout=output_file, stderr=DEVNULL) - assert not result, "Test failed: error while running bibtex on %s resuling in %s" % (os.path.splitext(filename)[0], str(result)) - - -def retrieve_algorithm(data_path, folder_name, algorithm_name, file_name=None): - """depreciated (replaced by cocopp._data_archive COCODataArchive instance - replaced by `cocopp.archiving.get` or `cocopp.archiving.official_archives` or `cocopp.archives.all`)""" - algorithm_file = join_path(data_path, file_name if file_name else algorithm_name) - if not os.path.exists(algorithm_file): - data_url = "https://numbbo.github.io/data-archive/data-archive/%s/%s" % (folder_name, algorithm_name) - urlretrieve(data_url, algorithm_file) - - -def depreciated_prepare_data(run_all_tests): - print("preparing algorithm data") - - data_path = os.path.abspath(join_path(os.path.dirname(__file__), "data")) - - if not os.path.exists(data_path): - os.makedirs(data_path) - - # Retrieving the algorithms - # retrieve_algorithm(data_path, '2010', 'IPOP-ACTCMA-ES_ros_noiseless.tar.gz') - # [outcommented and replaced by BIPOP until 2010 data is in new format] - retrieve_algorithm(data_path, "bbob/2009", "BFGS_ros_noiseless.tgz") - retrieve_algorithm(data_path, "bbob-biobj/2016", "RS-4.tgz") - - if run_all_tests: - retrieve_algorithm(data_path, "bbob/2009", "BIPOP-CMA-ES_hansen_noiseless.tgz") - retrieve_algorithm(data_path, "bbob/2009", "MCS_huyer_noiseless.tgz") - retrieve_algorithm(data_path, "bbob/2009", "NEWUOA_ros_noiseless.tgz") - retrieve_algorithm(data_path, "bbob/2009", "RANDOMSEARCH_auger_noiseless.tgz") - retrieve_algorithm(data_path, "bbob/2013", "SMAC-BBOB_hutter_noiseless.tgz") - retrieve_algorithm(data_path, "bbob/2013", "lmm-CMA-ES_auger_noiseless.tgz") - retrieve_algorithm(data_path, "bbob/2009", "DE-PSO_garcia-nieto_noiseless.tgz") - retrieve_algorithm(data_path, "bbob/2009", "VNS_garcia-martinez_noiseless.tgz") - retrieve_algorithm(data_path, "bbob-biobj/2016", "RS-4.tgz") - retrieve_algorithm(data_path, "bbob-biobj/2016", "RS-100.tgz") - # diff. location and name due to Jenkins settings with too long paths - retrieve_algorithm(data_path, "test", "N-II.tgz") - retrieve_algorithm(data_path, "test", "RS-4.zip") - retrieve_algorithm(data_path, "bbob-noisy/2009", "BFGS_ros_noisy.tgz") - retrieve_algorithm(data_path, "bbob-noisy/2009", "MCS_huyer_noisy.tgz") - - return data_path - - -def process_doctest_output(stream=None): - """ """ - import fileinput - - s1 = "" - s2 = "" - s3 = "" - state = 0 - for line in fileinput.input(stream): # takes argv as file or stdin - if 1 < 3: - s3 += line - if state < -1 and line.startswith("***"): - print(s3) - if line.startswith("***"): - s3 = "" - - if state == -1: # found a failed example line - s1 += "\n\n*** Failed Example:" + line - s2 += "\n\n\n" # line - # state = 0 # wait for 'Expected:' line - - if line.startswith("Expected:"): - state = 1 - continue - elif line.startswith("Got:"): - state = 2 - continue - elif line.startswith("***"): # marks end of failed example - state = 0 - elif line.startswith("Failed example:"): - state = -1 - elif line.startswith("Exception raised"): - state = -2 - - # in effect more else: - if state == 1: - s1 += line + "" - if state == 2: - s2 += line + "" - - -def delete_files(all_files=False): - assert all_files - if all_files: - cwd = os.getcwd() - os.remove(join_path(cwd, "acmart.cls")) - os.remove(join_path(cwd, "ACM-Reference-Format.bst")) - os.remove(join_path(cwd, "comment.sty")) - os.remove(join_path(cwd, "acmcopyright.sty")) - os.remove(join_path(cwd, "bbob.bib")) - - -def tmp_test_constrained(python, command): - url = "https://numbbo.github.io/cocopp-playground/bbob-constrained/testdata/bbob-constrained" - archive = archiving.get(url) - t0 = time.time() - data_path = archive.get("fmincon_consbbob") - print(python + command + data_path) - with InfolderGoneWithTheWind(): - result = os.system(python + command + data_path) - print("** subtest 19 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on one bbob-constrained algorithm." - - -def main(arguments): - """these tests are executed when ``python cocopp`` is called. - - with ``wine`` as second argument ``C:\\Python26\\python.exe`` - instead of ``python`` is called - - """ - - run_all_tests = "all" in arguments - recent_data_path = os.path.abspath(join_path(os.path.dirname(__file__), "../../code-experiments/build/python/exdata")) - recent_data_path = False - - python = "python -m " # how to call python - for argument in arguments: - if "python3" in argument: - python = "python3 -m " # how to call python - break - if len(sys.argv) > 1 and sys.argv[1] == "wine": - python = "C:\\Python26\\python.exe " # works for wine - - # old_data_path = ' ' + prepare_data(run_all_tests) - - command = " cocopp --no-svg --no-interactive " - - # copy_latex_templates() - # print('LaTeX templates copied.') - - print("*** testing module cocopp {0} ***".format("(all ~ 30 minutes)" if run_all_tests else "(~ 5 minutes)")) - t0 = time.time() - data_path = data_archive_get("BFGS_ros_noiseless") - print( - python - + command # '--conv ' + - + data_path - ) - with InfolderGoneWithTheWind(): - result = os.system( - python - + command # '--conv ' + - + data_path - ) - print("** subtest 1 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on one algorithm with option --conv." - # run_latex_template("templateBBOBarticle.tex", run_all_tests) - - t0 = time.time() - data_path = data_archive_get("RANDOMSEARCH-4_Auger_bbob-biobj.tgz") - print(python + command + data_path) - with InfolderGoneWithTheWind(): - result = os.system(python + command + data_path) - print("** subtest 2 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on one bi-objective algorithm." - # run_latex_template("templateBIOBJarticle.tex", run_all_tests) - - tmp_test_constrained(python, command) - - if run_all_tests: - data_paths = data_archive_get( - [ - "BIPOP-CMA-ES_hansen_noiseless", - "MCS_huyer_noiseless", - "2009/NEWUOA_ros_noiseless.tgz", - "RANDOMSEARCH_auger_noiseless.tgz", - "BFGS_ros_noiseless.tgz", - ] - ) - t0 = time.time() - print(time.asctime()) - - with InfolderGoneWithTheWind(): - result = os.system(python + command + data_paths) - print("** subtest 3 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on many algorithms." - # run_latex_template("templateBBOBmany.tex", run_all_tests) - - t0 = time.time() - with InfolderGoneWithTheWind(): - result = os.system(python + command + data_archive_get(["SMAC-BBOB_hutter_noiseless.tgz", "lmm-CMA-ES_auger_noiseless.tgz"])) - print("** subtest 4 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on two algorithms." - # run_latex_template("templateBBOBcmp.tex", run_all_tests) - - t0 = time.time() - with InfolderGoneWithTheWind(): - result = os.system( - python + command + " --include-single " + data_archive_get(["DE-PSO_garcia-nieto_noiseless.tgz", "VNS_garcia-martinez_noiseless.tgz"]) - ) - print("** subtest 5 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on two algorithms with option --include-single." - # run_latex_template("templateBBOBcmp.tex", run_all_tests) - - t0 = time.time() - with InfolderGoneWithTheWind(): - result = os.system(python + command + " --expensive " + data_archive_get("VNS_garcia-martinez_noiseless.tgz")) - print("** subtest 6 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on one algorithm with option --expensive." - # run_latex_template("templateBBOBarticle.tex", run_all_tests) - - t0 = time.time() - with InfolderGoneWithTheWind(): - result = os.system(python + command + data_archive_get(["RANDOMSEARCH-4_Auger_bbob-biobj.tgz", "RANDOMSEARCH-100_Auger_bbob-biobj.tgz"])) - print("** subtest 7 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on two bbob-biobj algorithms." - # run_latex_template("templateBIOBJmultiple.tex", run_all_tests) - - t0 = time.time() - # Previous note: we use the original GA-MULTIOBJ-NSGA-II.tgz data set - # but with a shorter file name from the biobj-test folder - # to avoid problems with too long path names on the windows - # Jenkins slave - with InfolderGoneWithTheWind(): - result = os.system( - python - + command - + data_archive_get( - ["NSGA-II-MATLAB_Auger_bbob-biobj.tgz", "RANDOMSEARCH-4_Auger_bbob-biobj.tgz", "RANDOMSEARCH-100_Auger_bbob-biobj.tgz"] - ) - ) - print("** subtest 8 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on three bbob-biobj algorithms." - # run_latex_template("templateBIOBJmultiple.tex", run_all_tests) - - # testing data from bbob-noisy suite: - t0 = time.time() - with InfolderGoneWithTheWind(): - result = os.system(python + command + data_archive_get(["MCS_huyer_noisy.tgz", "BFGS_ros_noisy.tgz"])) - print("** subtest 9 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on two bbob-noisy algorithms." - # run_latex_template("templateNOISYarticle.tex", run_all_tests) - - # testing data from recent runs: - if recent_data_path: # requires local data to have been generated - with InfolderGoneWithTheWind(): - t0 = time.time() - result = os.system(python + command + join_path(recent_data_path, "RS-bb")) - print("** subtest 10 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on newly generated random search data on `bbob`." - - with InfolderGoneWithTheWind(): - t0 = time.time() - result = os.system(python + command + join_path(recent_data_path, "RS-bi")) - print("** subtest 11 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on newly generated random search data on `bbob-biobj`." - - # with InfolderGoneWithTheWind(): - # t0 = time.time() - # result = os.system(python + command + - # join_path(recent_data_path, 'RS-co')) - # print('** subtest 12 finished in ', time.time() - t0, ' seconds') - # assert result == 0, 'Test failed: rungeneric on newly generated random search data on `bbob-constrained`.' - # delete_files(all_files=True) - - with InfolderGoneWithTheWind(): - t0 = time.time() - result = os.system(python + command + data_archive_get("test/RS-4.zip")) - print("** subtest 13 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on RS-4.zip." - - if recent_data_path: # requires local data to have been generated - with InfolderGoneWithTheWind(): - t0 = time.time() - result = os.system(python + command + join_path(recent_data_path, "RS-la")) - print("** subtest 14 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on newly generated random search data on `bbob-largescale`." - - with InfolderGoneWithTheWind(): - t0 = time.time() - result = os.system(python + command + join_path(recent_data_path, "RS-mi")) - print("** subtest 15 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on newly generated random search data on `bbob-mixint`." - - with InfolderGoneWithTheWind(): - t0 = time.time() - result = os.system(python + command + join_path(recent_data_path, "RS-bi-mi")) - print("** subtest 16 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on newly generated random search data on `bbob-biobj-mixint`." - - with InfolderGoneWithTheWind(): - t0 = time.time() - result = os.system(python + command + "bbob/2009/BFGS! bbob-largescale/2019/LBFGS!") - print("** subtest 17 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on data from `bbob` and `bbob-largescale` suite." - - with InfolderGoneWithTheWind(): - t0 = time.time() - result = os.system(python + command + "NSGA-II! 2019/IBEA!") - print("** subtest 18 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: rungeneric on data from `bbob-biobj` and `bbob-biobj-ext` suite." - - with InfolderGoneWithTheWind(): - t0 = time.time() - result = os.system("""python -c "import cocopp; cocopp.main('slsqp!'); cocopp.main('abc*')" """) - print("** subtest 19 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: running postprocessing twice within same python session" - - with InfolderGoneWithTheWind(): - t0 = time.time() - result = os.system("""python -c "import cocopp; cocopp.main('constrained/2022/RandomSearch-5')" """) - print("** subtest 20 finished in ", time.time() - t0, " seconds") - assert result == 0, "Test failed: running postprocessing on bbob-constrained data" - - do_doctest() # to be backwards compatible we run everything in main - do_pickle_regression_test() # messes up other tests, maybe by calling config? - - -def do_doctest(): - print("\n*** launching doctest (it might be necessary to close a few pop up windows to finish) ***") - t0 = time.time() - - failure_count = 0 - test_count = 0 - if 1 < 3: - # doctest.testmod(report=True, verbose=True) # this is quite cool! - # go through the py files in the cocopp folder - current_path = os.getcwd() - new_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) - os.chdir(new_path) - for root, dirnames, filenames in os.walk(os.path.dirname(os.path.realpath(__file__))): - for filename in fnmatch.filter(filenames, "*.py"): - current_failure_count, current_test_count = doctest.testfile(os.path.join(root, filename), report=True, module_relative=False) - failure_count += current_failure_count - test_count += current_test_count - if current_failure_count: - print('doctest file "%s" failed' % os.path.join(root, filename)) - else: - print("doctest {0}/{1} tests succeeded".format(test_count - failure_count, test_count), end="\r") - os.chdir(current_path) - print() - else: - stdout = sys.stdout - fn = "_cocopp_doctest_.txt" - try: - with open(fn, "w") as f: - sys.stdout = f - doctest.testmod(report=True) - finally: - sys.stdout = stdout - process_doctest_output(fn) - print("** doctest finished in ", time.time() - t0, " seconds") - # print(' more info in file _cocopp_doctest_.txt)') - print("*** done testing module cocopp ***") - - if failure_count > 0: - raise ValueError("%d of %d tests failed" % (failure_count, test_count)) - - -def do_pickle_regression_test(name='strained/2022/al3-cma'): - """read in a pickled data set and compare to data read from the original data. - - Assumes that the pickled data already exist at the usually expected place - side by side the original data. - """ - saved = settings.pretend_initial_solution_was_evaluated - settings.pretend_initial_solution_was_evaluated = False - name = official_archives.all.get_extended(name)[0] - dsl1 = _pproc.get_DataSetList(name) # also called from cocopp.load2 - if not _pproc._pickle_file_read: - print("{} pickled data could not be read in, hence" - " the pickle_regression_test can not be done" - .format(name)) - return - ignore_val, _pproc._ignore_pickle_files = _pproc._ignore_pickle_files, True - dsl2 = _pproc.get_DataSetList(name) - _pproc._ignore_pickle_files = ignore_val - assert not _pproc._pickle_file_read, (name, _pproc._pickle_file_read) - - dsl2._is_same_list_assert(dsl1) - - settings.pretend_initial_solution_was_evaluated = saved - - print("*** readin_regression_test done ***") - -""" - import cocopp - print(dir(cocopp)) - for s in dir(cocopp): - if(inspect.ismodule(eval("cocopp."+s)) and s[:2]!="__"): - print("cocopp."+s) - doctest.testmod(eval("cocopp."+s),verbose=False) - print(cocopp.__all__) -""" - -if __name__ == "__main__": - matplotlib.use("Agg") # To avoid window popup and use without X forwarding - if "doctest" in sys.argv or "doctests" in sys.argv: - do_doctest() - else: - main(sys.argv[1:]) diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..3013fb6 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,104 @@ +# Tests for `cocopp` + +A pytest test suite meant as a safety net for restructuring the package. +It replaces the former test runner `python -m cocopp.test`. + +## Running + +```sh +uv sync # or: pip install -e . --group dev (pip >= 25.1) +pytest tests # everything except network tests, ~10 minutes +pytest tests -m "not slow" # unit, module and smoke tests, ~1 minute +pytest tests --run-network # also tests using the online data archives +pytest tests/network --run-network -k "not legacy" # as in CI: archives, doctests, former default tests +python tests/run_doctests.py # only the doctests (writes into the package folder!) +pytest tests -n auto # in parallel, with pytest-xdist installed +``` + +## Layout + +| folder | content | +|---|---| +| `unit/` | functions and classes in isolation: statistics and formatting helpers, reading and aligning data, `DataSet`/`DataSetList`, target values, reference algorithms, settings, archives (local only), file utilities, the package interface | +| `integration/` | the plotting and table modules called directly (`test_modules.py`) and the full pipeline `cocopp.main` on many scenarios (`test_pipeline.py`) | +| `regression/` | golden-file snapshots of the complete text output of the pipeline | +| `smoke/` | the command line entry points, run in subprocesses | +| `network/` | the official online archives, the doctests, and the scenarios of the former `cocopp/test.py` | + +Helpers: + +- `synthdata.py` writes **deterministic synthetic COCO data** (bbob in the + current and the pre-2016 format, bbob-noisy, bbob-constrained with + g-evaluations, bbob-largescale, bbob-mixint, bbob-boxed, bbob-biobj) and a + tiny hand-crafted data set whose runtimes and ERTs can be computed by hand. + The real archives are several MB per algorithm, hence the synthetic data + make the tests offline and fast. +- `pipeline.py` runs `cocopp.main` in-process and defines the `SCENARIOS` + shared by the integration and regression tests (each runs once per session). +- `snapshots.py` takes, stores, compares and extracts output snapshots. +- `run_doctests.py` runs the doctests of all package modules. + +## Isolation and randomness + +`cocopp` keeps much state in module globals and writes files next to its +sources. `conftest.py` therefore, for every test, + +- seeds numpy's global random number generator (`conftest.SEED`), +- restores the globals of all `cocopp` modules, the testbed class settings, + matplotlib `rcParams`, numpy's random state and the working directory, +- runs the test in a fresh temporary working directory, + +and for the whole session redirects package-relative files +(`toolsdivers.path_in_package`) into a temporary copy of the package. The +session fails if any test created files in the package or the repository +root. Subprocess tests import that same copy via `PYTHONPATH`. + +## Snapshots + +The regression tests compare the normalized output (time stamps, temporary +folders and the version are replaced by placeholders) with +`snapshots/.json.gz`. After an *intended* change of the output: + +```sh +pytest tests/regression --update-snapshots +python tests/snapshots.py diff old.json.gz tests/snapshots/bbob-1.json.gz +python tests/snapshots.py extract tests/snapshots/bbob-1.json.gz /tmp/bbob-1 # for any diff tool +``` + +where `old.json.gz` can be obtained with, e.g., +`jj file show -r @- tests/snapshots/bbob-1.json.gz > old.json.gz`. + +## Known bugs and quirks + +Expected values are often *characterization* values that pin the current +behavior, quirks included (marked as such in comments). Known bugs are +documented as strict `xfail` tests, which start to fail once a bug is fixed +(then remove the `xfail` marker): + +- `toolsstats.sliding_window_data`: division by zero with the default + `number_of_stats=0` +- `pproc.DataSetList.det_best_data` and `run_length_distributions` + (single algorithm) fail under Python 3 +- `sanitycheck.main` uses a non-existing `rungeneric.shortoptlist` +- `cococommands.DataWithFewSuccesses.print` and `__len__` use a + non-existing `result` attribute +- `config.config_line_styles` fails for algorithm names without a float +- the unused modules `ppsingle`, `ppfigparam` and `compall.ppperfprof` fail +- the version label lists inconsistent hv-hashes in `PYTHONHASHSEED` + dependent order + +The output is otherwise not fully reproducible for a given numpy seed +either, because it depends on `PYTHONHASHSEED`: + +- the bootstrapped dispersion values in the comparison tables (`pptables`): + `rungenericmany` iterates over function groups in `set` order before, + consuming a data dependent number of random numbers. The pipeline runs + in the tests therefore reseed numpy's RNG when entering the table modules + (`pipeline.reseeded_entry_points`). +- the order of (nearly) tied algorithms in the `pprldmany` legends and of + inconsistent hv-hashes; the snapshots normalize both by sorting. + +Other observations: generated html pages contain a few broken links (e.g. +to `sorttable.js`, recorded in the snapshots), absolute data paths end up +in LaTeX legends, and importing `cocopp.firstsession` downloads data into +the current folder. diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..74a0460 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,379 @@ +"""Shared fixtures for the `cocopp` test suite. + +The package keeps a lot of state in module globals (``genericsettings``, +``testbedsettings.current_testbed``, ``bestalg.bestAlgorithmEntries``, +plotting defaults changed by ``config.config()``, ...) and writes files +next to its own sources (extracted reference algorithms and pickle +caches). The fixtures here + +- force the non-interactive ``Agg`` matplotlib backend, +- redirect all package-relative file access (`toolsdivers.path_in_package`) + to a throw-away copy of the package so that tests never write into the + source tree, +- snapshot and restore the module globals of all ``cocopp`` modules, + the testbed class settings, matplotlib's ``rcParams``, the global numpy + random state and the working directory around *every* test, +- seed numpy's global random number generator before every test, +- provide deterministic synthetic COCO data sets (see `synthdata`). + +Markers: + +- ``slow``: runs the full post-processing pipeline (seconds to minutes). + Deselect with ``-m "not slow"``. +- ``network``: needs internet access to the official COCO data archives. + These tests are skipped unless ``--run-network`` is given. +""" + +import contextlib +import copy +import os +import shutil +import sys +import types +import warnings + +os.environ.setdefault("MPLBACKEND", "Agg") + +import matplotlib + +matplotlib.use("Agg") + +import numpy as np +import pytest + +sys.path.insert(0, os.path.dirname(__file__)) +import synthdata # noqa: E402 + +with warnings.catch_warnings(): + warnings.simplefilter("ignore") + import cocopp # noqa: E402 + from cocopp import testbedsettings, toolsdivers # noqa: E402 + +PACKAGE_DIR = os.path.dirname(os.path.abspath(cocopp.__file__)) +REPO_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SEED = 20240521 + + +def pytest_addoption(parser): + parser.addoption("--run-network", action="store_true", default=False, help="run tests that download data from the official COCO archives") + parser.addoption( + "--update-snapshots", action="store_true", default=False, help="rewrite the golden output snapshots in tests/snapshots instead of comparing" + ) + + +def pytest_configure(config): + config.addinivalue_line("markers", "slow: runs (parts of) the full post-processing pipeline") + config.addinivalue_line("markers", "network: needs access to the official online COCO data archives") + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--run-network"): + return + skip = pytest.mark.skip(reason="needs --run-network") + for item in items: + if "network" in item.keywords: + item.add_marker(skip) + + +# -------------------------------------------------------------------------- +# keep the source tree clean +# -------------------------------------------------------------------------- + + +@pytest.fixture(scope="session") +def package_copy(tmp_path_factory): + """a throw-away copy of the installed `cocopp` package folder. + + Its parent folder can be put on ``PYTHONPATH`` for subprocesses. + """ + root = tmp_path_factory.mktemp("pkgcopy") + shutil.copytree( + PACKAGE_DIR, + root / "cocopp", + ignore=shutil.ignore_patterns("__pycache__", ".extracted_*", "*.pickle"), + ) + return root / "cocopp" + + +@pytest.fixture(scope="session", autouse=True) +def _redirect_package_files(package_copy): + """make `toolsdivers.path_in_package` point into `package_copy`. + + The reference algorithms in ``refalgs/`` are extracted (and pickled) + next to the archives, i.e. inside the package, on first use. + """ + original = toolsdivers.path_in_package + + def path_in_package(sub_path=""): + return os.path.join(str(package_copy), sub_path) + + toolsdivers.path_in_package = path_in_package + yield + toolsdivers.path_in_package = original + + +def _listing(folder): + return set(os.listdir(folder)) if os.path.isdir(folder) else set() + + +@pytest.fixture(scope="session", autouse=True) +def _assert_source_tree_untouched(): + """fail the session if tests created files in the package or repository root. + + Tests run in a temporary working directory (see `_isolate_global_state`) + and package files are redirected (see `_redirect_package_files`), hence + new files would indicate a test, or a code path, writing to an + unexpected place. + """ + folders = [PACKAGE_DIR, os.path.join(PACKAGE_DIR, "refalgs"), REPO_DIR] + before = {f: _listing(f) for f in folders} + yield + new = {f: sorted(_listing(f) - before[f] - {".pytest_cache"}) for f in folders} + new = {f: n for f, n in new.items() if n} + assert not new, "tests created files in the source tree: %s" % new + + +# -------------------------------------------------------------------------- +# global state isolation +# -------------------------------------------------------------------------- + +_SKIP_TYPES = (types.ModuleType, types.FunctionType, types.BuiltinFunctionType, type) +# containers whose content is restored in place, other objects are only rebound +_CONTAINERS = (dict, list, set, np.ndarray) +_NO_COPY = object() + + +def _snapshot_module(mod): + """return ``{name: (original_object, deep_copy_or__NO_COPY)}``. + + Only containers are deep-copied. Other objects (e.g. the archives or + the current testbed) are only remembered by reference: they are + often shared between modules (``from .archiving import + official_archives``) and restoring their content would break the + identity of objects referenced from elsewhere. + """ + state = {} + for key, val in vars(mod).items(): + if key.startswith("__") or isinstance(val, _SKIP_TYPES): + continue + saved = _NO_COPY + if isinstance(val, _CONTAINERS): + try: + saved = copy.deepcopy(val) + except Exception: + pass + state[key] = (val, saved) + return state + + +def _restore_content(obj, saved): + """restore the content of container `obj` in place from its deep copy `saved`. + + Like this, the identity of containers that are shared between modules + (e.g. via ``from module import name``) is preserved. + """ + if saved is _NO_COPY or type(obj) is not type(saved): + return + if isinstance(obj, np.ndarray): + if obj.shape == saved.shape and obj.flags.writeable: + obj[...] = saved + elif isinstance(obj, dict): + obj.clear() + obj.update(saved) + elif isinstance(obj, list): + obj[:] = saved + elif isinstance(obj, set): + obj.clear() + obj.update(saved) + + +def _restore_module(mod, state): + for key in [k for k, v in vars(mod).items() if not k.startswith("__") and k not in state and not isinstance(v, _SKIP_TYPES)]: + delattr(mod, key) # added during the test + for key, (original, saved) in state.items(): + _restore_content(original, saved) + setattr(mod, key, original) + + +def _testbed_classes(): + return [c for c in vars(testbedsettings).values() if isinstance(c, type) and issubclass(c, testbedsettings.Testbed)] + + +@contextlib.contextmanager +def isolated_cocopp_state(workdir, seed=SEED): + """context manager to snapshot and restore the global state of `cocopp`. + + Restores module globals of all ``cocopp`` modules, the testbed class + settings, matplotlib's ``rcParams``, numpy's random state and error + settings, and the working directory. Within the context, numpy's + global RNG is seeded with `seed`, `cocopp` never opens a web browser, + and the working directory is `workdir`, because the package writes + output (e.g. ``ppdata``) relative to the current folder. + """ + modules = {name: mod for name, mod in sys.modules.items() if name == "cocopp" or name.startswith("cocopp.")} + states = {name: _snapshot_module(mod) for name, mod in modules.items()} + class_settings = {cls: (cls.__dict__["settings"], copy.deepcopy(cls.__dict__["settings"])) for cls in _testbed_classes() if "settings" in cls.__dict__} + rc = matplotlib.rcParams.copy() + rng_state = np.random.get_state() + np_err = np.geterr() + cwd = os.getcwd() + cocopp.genericsettings.interactive_mode = False # never open a web browser + np.random.seed(seed) + os.chdir(str(workdir)) + try: + yield + finally: + os.chdir(cwd) + import matplotlib.pyplot as plt + + plt.close("all") + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + matplotlib.rcParams.update(rc) + np.random.set_state(rng_state) + np.seterr(**np_err) + for cls, (original, saved) in class_settings.items(): + _restore_content(original, saved) + cls.settings = original + for name, state in states.items(): + _restore_module(modules[name], state) + + +@pytest.fixture(autouse=True) +def _isolate_global_state(tmp_path_factory): + """run every test in `isolated_cocopp_state` within a fresh working directory""" + with isolated_cocopp_state(tmp_path_factory.mktemp("cwd")): + yield + + +# -------------------------------------------------------------------------- +# synthetic data +# -------------------------------------------------------------------------- + + +@pytest.fixture(scope="session") +def synth_root(tmp_path_factory): + return tmp_path_factory.mktemp("synthdata") + + +@pytest.fixture(scope="session") +def bbob_alg_a(synth_root): + """full synthetic ``bbob`` experiment (24 functions, 5 dimensions, 15 instances)""" + return synthdata.write_bbob(str(synth_root / "bbob"), "SYNTH-A", seed=1) + + +@pytest.fixture(scope="session") +def bbob_alg_b(synth_root): + """a second, faster synthetic ``bbob`` "algorithm" on the same problems""" + return synthdata.write_bbob(str(synth_root / "bbob"), "SYNTH-B", seed=2, speed=2.0) + + +@pytest.fixture(scope="session") +def bbob_alg_c(synth_root): + """a third, slower synthetic ``bbob`` "algorithm" """ + return synthdata.write_bbob(str(synth_root / "bbob"), "SYNTH-C", seed=3, speed=0.6) + + +@pytest.fixture(scope="session") +def bbob_small(synth_root): + """a small synthetic ``bbob`` experiment: f1, f2, f8, f15 in 2-D and 5-D""" + return synthdata.write_bbob(str(synth_root / "small"), "SMALL", functions=(1, 2, 8, 15), dims=(2, 5), seed=11) + + +@pytest.fixture(scope="session") +def bbob_small_b(synth_root): + """a second "algorithm" on the problems of `bbob_small`, twice as fast""" + return synthdata.write_bbob(str(synth_root / "small"), "SMALL-B", functions=(1, 2, 8, 15), dims=(2, 5), seed=13, speed=2.0) + + +@pytest.fixture(scope="session") +def bbob_old_format(synth_root): + """small synthetic data in the pre-2016 layout without suite/data_format""" + return synthdata.write_bbob(str(synth_root / "old"), "OLDFMT", functions=(1, 2), dims=(2, 5), seed=12, old_format=True) + + +@pytest.fixture(scope="session") +def biobj_alg(synth_root): + """full synthetic ``bbob-biobj`` experiment (55 functions, 2/3/5-D, 10 instances)""" + return synthdata.write_biobj(str(synth_root / "biobj"), "SYNTH-BI", seed=21) + + +@pytest.fixture(scope="session") +def biobj_alg_b(synth_root): + return synthdata.write_biobj(str(synth_root / "biobj"), "SYNTH-BI-B", seed=22, speed=2.0) + + +@pytest.fixture(scope="session") +def bbob_noisy(synth_root): + return synthdata.write_bbob(str(synth_root / "noisy"), "SYNTH-NOISY", functions=range(101, 131), dims=(2, 5, 20), seed=31, suite="bbob-noisy") + + +@pytest.fixture(scope="session") +def bbob_constrained(synth_root): + """constrained data with g-evaluations, see `synthdata.write_bbob`""" + return synthdata.write_bbob(str(synth_root / "cons"), "SYNTH-CONS", functions=range(1, 55), dims=(2, 5), seed=32, suite="bbob-constrained") + + +@pytest.fixture(scope="session") +def bbob_largescale(synth_root): + return synthdata.write_bbob(str(synth_root / "ls"), "SYNTH-LS", dims=(20, 40), seed=33, suite="bbob-largescale", budget_multiplier=50) + + +@pytest.fixture(scope="session") +def bbob_mixint(synth_root): + return synthdata.write_bbob(str(synth_root / "mixint"), "SYNTH-MIXINT", dims=(5, 10), seed=34, suite="bbob-mixint") + + +@pytest.fixture(scope="session") +def bbob_boxed(synth_root): + return synthdata.write_bbob(str(synth_root / "boxed"), "SYNTH-BOXED", dims=(2, 5), seed=35, suite="bbob-boxed") + + +@pytest.fixture(scope="session") +def pipeline_run(request, tmp_path_factory): + """return a function ``run(scenario_name)`` returning a cached `pipeline.PipelineRun`. + + Scenarios are defined in `pipeline.SCENARIOS`; each is run at most + once per test session. + """ + import pipeline + + cache = {} + + def run(name): + if name not in cache: + fixtures, options, background = pipeline.SCENARIOS[name] + folders = [request.getfixturevalue(f) for f in fixtures] + background = [request.getfixturevalue(f) for f in background] + + def setup(): + if background: + cocopp.genericsettings.background = {None: background} + + args = " ".join(list(options) + folders) + cache[name] = pipeline.run_main(args, tmp_path_factory.mktemp("run-" + name), setup=setup) + return cache[name] + + return run + + +@pytest.fixture +def quiet(): + """silence the (many) warnings the package emits while loading data""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + yield + + +def load(path): + """load a `DataSetList` without the package's chatter""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return cocopp.pproc.DataSetList(str(path)) + + +@pytest.fixture +def dsl_small(bbob_small): + """freshly loaded `DataSetList` of `bbob_small` (8 data sets)""" + testbedsettings.reset_current_testbed() + return load(bbob_small) diff --git a/tests/integration/test_modules.py b/tests/integration/test_modules.py new file mode 100644 index 0000000..fb21ea3 --- /dev/null +++ b/tests/integration/test_modules.py @@ -0,0 +1,223 @@ +"""Integration tests calling the figure and table modules directly. + +`cocopp.main` calls these modules in a fixed sequence; calling them +individually on small data sets localizes failures and covers modules +that the pipeline does not (or no longer) use. +""" + +import os +import warnings + +import numpy as np +import pytest + +from cocopp import algportfolio, bestalg, config, genericsettings, ppconverrorbars, ppfig, ppfigdim, ppfigparam, pplogloss, pprldistr +from cocopp import ppsingle, pptable, pproc, testbedsettings, toolsstats +from cocopp.comp2 import ppfig2, pprldistr2, ppscatter +from cocopp.compall import ppfigs, ppperfprof, pprldmany, pptables + + +@pytest.fixture +def two_algs(bbob_small, bbob_small_b, quiet): + """``(dsList, sortedAlgs, dictAlg)`` of two small synthetic algorithms, configured for 'bbob'""" + testbedsettings.reset_current_testbed() + res = pproc.processInputArgs([bbob_small, bbob_small_b]) + config.config("bbob") + return res + + +@pytest.fixture +def one_alg(two_algs): + dsl, algs, dict_alg = two_algs + return dict_alg[algs[0]] + + +@pytest.fixture +def out(tmp_path): + return str(tmp_path) + + +def produced(folder): + return sorted(os.listdir(folder)) + + +def single_algorithm_pages(out, dsl): + """write the html skeleton pages which `rungeneric1` creates before calling the modules""" + ppfig.save_single_functions_html(os.path.join(out, "index1"), "title", htmlPage=ppfig.HtmlPage.ONE, function_groups=dsl.getFuncGroups()) + ppfig.save_single_functions_html( + os.path.join(out, "pptable"), dimensions=sorted(dsl.dictByDim()), htmlPage=ppfig.HtmlPage.PPTABLE, parentFileName="index1" + ) + + +def many_algorithms_pages(out, dict_alg, algs): + """write the html skeleton pages which `rungenericmany` creates before calling the modules""" + groups = dict_alg[algs[0]].getFuncGroups() + ppfig.save_single_functions_html(os.path.join(out, "index"), "", htmlPage=ppfig.HtmlPage.MANY, function_groups=groups) + ppfig.save_single_functions_html( + os.path.join(out, "pptables"), + "", + dimensions=sorted(pproc.dictAlgByDim(dict_alg)), + htmlPage=ppfig.HtmlPage.PPTABLES, + function_groups=groups, + parentFileName="index", + ) + + +class TestSingleAlgorithmModules: + def test_pptable(self, one_alg, out, tmp_path): + single_algorithm_pages(out, one_alg) + latex = str(tmp_path / "cmds.tex") + pptable.main(one_alg, (2, 5), out, latex) + tables = [f for f in produced(out) if f.startswith("pptable_")] + assert tables == sorted("pptable_f%03d_%02dD.tex" % (f, d) for f in (1, 2, 8, 15) for d in (2, 5)) + assert "pptable_f001_05D" not in open(os.path.join(out, "pptable.html")).read() # html gets the table itself + assert " 0 for z, p in res) # positive z means the first data set is better + res2 = toolsstats.significancetest(slow, fast, [1e-1, 1e-8]) + assert [z for z, p in res2] == pytest.approx([-z for z, p in res]) + + def test_significance_all_best_vs_other(self, two_algs): + dsl, algs, dict_alg = two_algs + datasets = [dict_alg[a].dictByDimFunc()[5][8][0] for a in algs] + targets = [10, 1e-1, 1e-8] + res, best = toolsstats.significance_all_best_vs_other(datasets, targets) + assert len(res) == len(best) == len(targets) + assert list(best) == list(toolsstats.best_alg_indices(datasets=datasets, targets=targets)) + + def test_against_reference_algorithm(self, two_algs): + dsl, algs, dict_alg = two_algs + ref = bestalg.load_reference_algorithm(testbedsettings.current_testbed.reference_algorithm_filename, force=True) + ds = dict_alg[algs[0]].dictByDimFunc()[5][1][0] + res = toolsstats.significancetest(ref[(5, 1)], ds, [10, 1e-8]) + assert len(res) == 2 and all(0 <= p <= 1 for z, p in res) + + +def test_rungeneric1_and_rungenericmany_directly(two_algs, tmp_path, monkeypatch): + """the sub-routines of `cocopp.main` can be called directly""" + from cocopp import rungeneric1, rungenericmany + + dsl, algs, dict_alg = two_algs + monkeypatch.chdir(tmp_path) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + dict_alg1 = rungeneric1.main(algs[0], "out1") + testbedsettings.reset_current_testbed() + dict_alg2 = rungenericmany.main(algs, "out2") + # quirk: the returned dictionaries are keyed by (algId, comment) instead of input path + assert list(dict_alg1) == [("SMALL", "")] and list(dict_alg2) == [("SMALL", ""), ("SMALL-B", "")] + assert os.path.exists(tmp_path / "out1" / "index.html") and os.path.exists(tmp_path / "out2" / "index.html") diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py new file mode 100644 index 0000000..ce15d33 --- /dev/null +++ b/tests/integration/test_pipeline.py @@ -0,0 +1,280 @@ +"""Integration tests of the full post-processing pipeline (`cocopp.main`). + +The scenarios from `pipeline.SCENARIOS` run once per session (see the +``pipeline_run`` fixture). The tests here check properties of the output +which a golden-file comparison cannot express well: that files are valid, +that the expected kinds of output exist, and that the numbers in the +tables agree with the data. +""" + +import os +import re +import xml.etree.ElementTree as ET + +import pytest + +import cocopp +from cocopp import bestalg, pproc + +import pipeline + +pytestmark = pytest.mark.slow + +# scenario: number of foreground algorithms +NUM_ALGS = {name: len(fixtures) for name, (fixtures, _, _) in pipeline.SCENARIOS.items()} + + +@pytest.mark.parametrize("scenario", sorted(pipeline.SCENARIOS)) +class TestAllScenarios: + def test_run_completed(self, scenario, pipeline_run): + run = pipeline_run(scenario) + assert "ALL done" in run.stdout + assert isinstance(run.result, dict) + assert all(len(dsl) > 0 for dsl in run.result.values()) + + def test_top_level_files(self, scenario, pipeline_run): + run = pipeline_run(scenario) + assert "index.html" in run.files and "cocopp_commands.tex" in run.files + commands = run.read("cocopp_commands.tex") + assert "\\providecommand{\\numofalgs}{%d}" % NUM_ALGS[scenario] in commands + assert "Data produced with COCO v" + cocopp.__version__ in commands + index = run.read("index.html") + for folder in run.subfolders: + assert folder.split("_")[0] in index + + def test_figures_are_valid(self, scenario, pipeline_run): + run = pipeline_run(scenario) + pdfs = [f for f in run.files if f.endswith(".pdf")] + svgs = [f for f in run.files if f.endswith(".svg")] + assert pdfs and len(svgs) == len(pdfs) + for f in pdfs: + with open(os.path.join(run.outdir, f), "rb") as fh: + assert fh.read(5) == b"%PDF-", f + for f in svgs[:: max(1, len(svgs) // 50)]: # parsing all takes too long + assert ET.parse(os.path.join(run.outdir, f)).getroot().tag.endswith("svg"), f + + def test_text_files_are_not_empty(self, scenario, pipeline_run): + run = pipeline_run(scenario) + empty = [f for f in run.files if f.endswith((".tex", ".html")) and os.path.getsize(os.path.join(run.outdir, f)) == 0] + assert not empty + + +class TestSingleAlgorithm: + SCENARIO = "bbob-1" + + @pytest.fixture + def run(self, pipeline_run): + return pipeline_run(self.SCENARIO) + + @pytest.fixture + def datasets(self, run): + (dsl,) = run.result.values() + return dsl + + def test_inventory(self, run): + assert run.subfolders == ["SYNTH-A"] + assert len(run.glob("SYNTH-A/pptable_f*_*D.tex")) == 24 * 5 + assert len(run.glob("SYNTH-A/ppfigdim_f*.pdf")) == 24 + assert len(run.glob("SYNTH-A/pprldistr_*D_*.pdf")) == 2 * 6 # dimensions of interest x function groups + assert len(run.glob("SYNTH-A/pplogloss_*D_*.pdf")) == 2 * 6 + assert len(run.glob("SYNTH-A/pploglosstable_*D_noiselessall.tex")) == 2 + assert len(run.glob("SYNTH-A/pprldmany-single-functions/pprldmany_f*.pdf")) == 24 + assert len(run.glob("SYNTH-A/pprldmany-single-functions/pprldmany_gr_*_*D.pdf")) == 5 * 5 + for page in ["index1.html", "ppfigdim.html", "pptable.html", "pprldistr.html", "pplogloss.html", "pprldflex.html"]: + assert run.glob("SYNTH-A/" + page), page + assert not run.glob("*ppconv*") and not run.glob("*ppscatter*") + + def test_result_datasets(self, datasets, bbob_alg_a): + assert len(datasets) == 24 * 5 + assert {ds.algId for ds in datasets} == {"SYNTH-A"} + fresh = pproc.DataSetList(bbob_alg_a) + for ds, ds0 in zip(sorted(datasets, key=lambda d: (d.dim, d.funcId)), fresh): + assert ds == ds0 and not ds._data_differ(ds0) + + def test_pptable_success_counts(self, run, datasets): + """the last two columns of the algorithm row are #successes and #trials at the final target""" + cocopp.config.config("bbob") # DataSet methods need the current testbed + for ds in datasets: + table = run.read("SYNTH-A/pptable_f%03d_%02dD.tex" % (ds.funcId, ds.dim)) + row = table.strip().splitlines()[-1] + m = re.search(r"& (\d+) & /(\d+)\\\\$", row) + assert m, row + assert (int(m.group(1)), int(m.group(2))) == (ds.detSuccesses([1e-8])[0], ds.nbRuns()), (ds, row) + + def test_pptable_first_ert_ratio(self, run, datasets): + """the first entry of the algorithm row is the ERT ratio to the reference algorithm for target Delta f = 10""" + cocopp.config.config("bbob") + reference = bestalg.load_reference_algorithm("refalgs/best2009-bbob.tar.gz", force=True) + checked = 0 + for ds in datasets: + if ds.dim not in (5, 20): + continue + row = run.read("SYNTH-A/pptable_f%03d_%02dD.tex" % (ds.funcId, ds.dim)).strip().splitlines()[-1] + m = re.match(r"^ & (?:\$[^$]*\$)?(\d+)\s*&\s*(\.\d+)?\$", row) + if not m: # e.g. infinite ERT or another format + continue + value = float(m.group(1) + (m.group(2) or "")) + ratio = ds.detERT([10])[0] / reference[(ds.dim, ds.funcId)].detERT([10])[0] + assert value == pytest.approx(ratio, rel=0.06, abs=0.06), (ds, row) + checked += 1 + assert checked >= 20 + + def test_latex_commands(self, run): + commands = run.read("cocopp_commands.tex") + assert "\\providecommand{\\algname}{SYNTH-A{}}" in commands + assert re.search(r"\\providecommand\{\\algfolder\}\{SYNTH-A_\d{6}h\d{4}/\}", commands) + assert "the best algorithm from BBOB 2009" in commands + + def test_html_pages_link_figures(self, run): + ppfigdim = run.read("SYNTH-A/ppfigdim.html") + assert all("ppfigdim_f%03d.svg" % f in ppfigdim for f in range(1, 25)) + pptable = run.read("SYNTH-A/pptable.html") + assert pptable.count("= 2 + + +def test_expensive_setting_uses_budget_based_targets(pipeline_run): + run = pipeline_run("bbob-1-expensive") + default = pipeline_run("bbob-1") + assert run.glob("SYNTH-A/pptable_f001_05D.tex") + commands = run.read("cocopp_commands.tex") + assert "budget" in commands.lower() or "runlength" in commands.lower() + # tables differ from the default fixed-target setting + assert run.read("SYNTH-A/pptable_f001_05D.tex") != default.read("SYNTH-A/pptable_f001_05D.tex") + + +def test_convergence_plots(pipeline_run): + run = pipeline_run("bbob-1-conv") + assert len(run.glob("SYNTH-A/ppconv_f*.pdf")) == 24 + assert "ppconv.html" in [os.path.basename(f) for f in run.files] + + +class TestComparisons: + def test_two_algorithms(self, pipeline_run, bbob_alg_a, bbob_alg_b): + run = pipeline_run("bbob-2") + assert run.subfolders == ["SYNTH_SYNTH"] + # quirk: the returned dict is keyed by (algId, comment), not by the input path + assert list(run.result) == [("SYNTH-A", ""), ("SYNTH-B", "")] + assert len(run.glob("SYNTH_SYNTH/pptables_f*_*D.tex")) == 24 * 5 + assert len(run.glob("SYNTH_SYNTH/ppscatter_f*.pdf")) == 24 + assert len(run.glob("SYNTH_SYNTH/ppfigs_f*.pdf")) == 24 + assert len(run.glob("SYNTH_SYNTH/pplogabs_*D_*.pdf")) == 2 * 7 + assert len(run.glob("SYNTH_SYNTH/pprldmany_*D_*.pdf")) == 5 * 6 + assert len(run.glob("SYNTH_SYNTH/pprldmany-single-functions/pprldmany_f*_*D.pdf")) == 24 * 5 + assert run.glob("SYNTH_SYNTH/pprldistr2.html") + + def test_two_algorithms_tables_have_a_row_per_algorithm(self, pipeline_run): + run = pipeline_run("bbob-2") + table = run.read("SYNTH_SYNTH/pptables_f001_05D.tex") + assert len([line for line in table.splitlines() if line.endswith("/15\\\\")]) == 2 + assert "SYNTH-A" in run.read("SYNTH_SYNTH/pptables.html") and "SYNTH-B" in run.read("SYNTH_SYNTH/pptables.html") + + def test_significance_markers_in_tables(self, pipeline_run): + """B is twice as fast as A, hence some ERT differences must be significant""" + run = pipeline_run("bbob-2") + tables = "".join(open(os.path.join(run.outdir, f)).read() for f in run.files if "/pptables_f" in f) + assert "\\star" in tables or "downarrow" in tables or "uparrow" in tables + + def test_three_algorithms(self, pipeline_run): + run = pipeline_run("bbob-3") + assert run.subfolders == ["SYNTH_SYNTH_SYNTH"] + assert len(run.glob("SYNTH_SYNTH_SYNTH/pptables_f*_*D.tex")) == 24 * 5 + # scatter plots and pairwise ECDF comparisons are only made for two algorithms + assert not run.glob("*ppscatter*") and not run.glob("*pplogabs*") + + def test_include_single(self, pipeline_run): + run = pipeline_run("bbob-small-include-single") + assert run.subfolders == ["OLDFMT", "SMALL", "SMALL_OLDFM"] + assert run.glob("SMALL/index1.html") and run.glob("OLDFMT/index1.html") + assert run.glob("SMALL_OLDFM/index.html") + # comparison tables only for the functions both algorithms have data for + assert len(run.glob("SMALL_OLDFM/pptables_f*_*D.tex")) == 2 * 2 + 2 * 2 + + def test_background_algorithms(self, pipeline_run): + run = pipeline_run("bbob-1-background") + assert run.subfolders == ["SYNTH", "SYNTH-A"] + assert len(run.result) == 3 # foreground and background algorithms + assert run.glob("SYNTH/pprldmany_05D_noiselessall.pdf") + + +class TestOtherSuites: + def test_biobj_single(self, pipeline_run): + run = pipeline_run("biobj-1") + assert run.subfolders == ["SYNTH-BI"] + assert len(run.glob("SYNTH-BI/pptable_f*_*D.tex")) == 55 * 3 + assert len(run.glob("SYNTH-BI/ppfigdim_f*.pdf")) == 55 + assert run.glob("SYNTH-BI/pprldmany-single-functions/pprldmany_gr_all-functions_*D.pdf") + assert "best 2016" in run.read("cocopp_commands.tex") or "BBOB 2016" in run.read("cocopp_commands.tex") + + def test_biobj_two(self, pipeline_run): + run = pipeline_run("biobj-2") + assert run.subfolders == ["biobj_SYNTH_SYNTH"] + assert len(run.glob("biobj_SYNTH_SYNTH/ppscatter_f*.pdf")) == 55 + assert len(run.glob("biobj_SYNTH_SYNTH/pptables_f*_*D.tex")) == 55 * 3 + + def test_noisy(self, pipeline_run): + run = pipeline_run("noisy-1") + assert len(run.glob("SYNTH-NOISY/pptable_f1*_*D.tex")) == 30 * 3 + assert run.glob("SYNTH-NOISY/pprldistr_05D_nzall.pdf") + assert run.glob("SYNTH-NOISY/pplogloss_20D_nzall.pdf") + + def test_constrained(self, pipeline_run): + run = pipeline_run("constrained-1") + assert len(run.glob("SYNTH-CONS/pptable_f*_*D.tex")) == 54 * 2 + assert len(run.glob("SYNTH-CONS/ppfigcons1_*_d*.pdf")) == 9 * 2 + assert run.glob("SYNTH-CONS/pprldistr_05D_all m=0n+1.pdf") # quirk: blanks and '=' in file names + (dsl,) = run.result.values() + assert all(ds.has_constraints for ds in dsl) + + @pytest.mark.parametrize("scenario, folder, dims", [("largescale-1", "SYNTH-LS", (20, 40)), ("mixint-1", "SYNTH-MIXINT", (5, 10)), ("boxed-1", "SYNTH-BOXED", (2, 5))]) + def test_single_suites(self, pipeline_run, scenario, folder, dims): + run = pipeline_run(scenario) + assert run.subfolders == [folder] + assert len(run.glob(folder + "/pptable_f*_*D.tex")) == 24 * len(dims) + assert len(run.glob(folder + "/ppfigdim_f*.pdf")) == 24 + + def test_old_format(self, pipeline_run): + run = pipeline_run("bbob-old-format") + assert run.subfolders == ["OLDFMT"] + assert len(run.glob("OLDFMT/pptable_f*_*D.tex")) == 2 * 2 + + +@pytest.mark.parametrize("option, present, absent", [ + ("--tab-only", ["pptable_f001_05D.tex"], ["ppfigdim_f001.pdf", "pprldistr_05D_noiselessall.pdf"]), + ("--fig-only", ["ppfigdim_f001.pdf"], ["pptable_f001_05D.tex", "pprldistr_05D_noiselessall.pdf"]), + ("--rld-only", ["pprldistr_05D_noiselessall.pdf"], ["pptable_f001_05D.tex", "ppfigdim_f001.pdf"]), + ("--no-rld-single-fcts", ["pptable_f001_05D.tex"], ["pprldmany-single-functions/pprldmany_f001.pdf"]), +]) +def test_output_options(option, present, absent, bbob_small, tmp_path): + run = pipeline.run_main(option + " " + bbob_small, tmp_path) + for f in present: + assert run.glob("SMALL/" + f), (option, f) + for f in absent: + assert not run.glob("SMALL/" + f), (option, f) + + +def test_output_is_deterministic(bbob_small, tmp_path): + import snapshots + + s1 = snapshots.take(pipeline.run_main(bbob_small, tmp_path / "a")) + s2 = snapshots.take(pipeline.run_main(bbob_small, tmp_path / "b")) + assert s1["files"] == s2["files"] + assert not snapshots.diff(s1, s2) + + +def test_main_twice_in_same_session(bbob_small, bbob_old_format, tmp_path): + """state must not leak between two calls in the same session (subtest 19 of the former cocopp/test.py)""" + import contextlib + import io + import warnings + + from conftest import isolated_cocopp_state + + # the RNG is reseeded on entry of the table modules, otherwise their dispersion values differ + with isolated_cocopp_state(tmp_path), warnings.catch_warnings(), contextlib.redirect_stdout(io.StringIO()), pipeline.reseeded_entry_points(): + warnings.simplefilter("ignore") + results = [cocopp.main("--no-interactive -o out%d %s" % (i, folder)) for i, folder in enumerate([bbob_small, bbob_old_format, bbob_small])] + assert [list(r) for r in results] == [[("SMALL", "")], [("OLDFMT", "")], [("SMALL", "")]] + first, third = (pipeline.PipelineRun(tmp_path, "out%d" % i, None, "", 0) for i in (0, 2)) + assert [pipeline.TIMESTAMP_RE.sub("", f) for f in first.files] == [pipeline.TIMESTAMP_RE.sub("", f) for f in third.files] + for name in ["SMALL/pptable_f001_05D.tex", "SMALL/pptable_f015_02D.tex", "SMALL/pprldmany-single-functions/pprldmany_f008.tex"]: + assert first.read(name) == third.read(name), name diff --git a/tests/network/test_archives_online.py b/tests/network/test_archives_online.py new file mode 100644 index 0000000..964c0d8 --- /dev/null +++ b/tests/network/test_archives_online.py @@ -0,0 +1,150 @@ +"""Tests that need the official online COCO data archives. + +These are skipped unless pytest is called with ``--run-network``. They +contain the scenarios of the former test runner ``python -m cocopp.test`` +(``cocopp/test.py``), which they replace. Downloaded data are cached in +the user's cocopp cache folder (as with any use of `cocopp.archives`). + +The full post-processing runs of the former "all" tests (named +``test_legacy_*``) are additionally marked ``slow`` and are not run in CI. +""" + +import os +import re +import subprocess +import sys + +import pytest + +import cocopp +from cocopp import archiving, genericsettings, pproc + +import pipeline + +pytestmark = pytest.mark.network + + +class TestOfficialArchives: + def test_archive_names(self): + names = cocopp.archives.names + for suite in ["all", "bbob", "bbob-noisy", "bbob-biobj", "bbob-largescale", "bbob-mixint", "bbob-constrained"]: + assert suite in names + + def test_find(self): + found = cocopp.archives.bbob.find("bfgs") + assert "2009/BFGS_ros_noiseless.tgz" in found + assert cocopp.archives.bbob.find("bfgs") == cocopp.archives.bbob.find(".*bfgs") + assert len(cocopp.archives.bbob.find("bfgs")) > len(cocopp.archives.bbob.find("bfgs.*")) + assert cocopp.archives.all("bbob/2009/BFGS_ros") == ["bbob/2009/BFGS_ros_noiseless.tgz"] + + def test_names_are_unique(self): + bbob = cocopp.archives.bbob + for name in bbob: + assert bbob.count(name) == 1 and len(bbob.find(name)) == 1, name + + def test_get_extended_patterns(self): + arch = cocopp.archives.all + assert len(arch.get_extended(["bbob/2009/BFGS_ros!"], remote=False)) == 1 + assert len(arch.get_extended(["bbob/2009/.*BFGS"], remote=False)) >= 1 + + def test_remote_archives_list(self): + urls = cocopp.remote_archives() + assert len(urls) > 0 and all(u.startswith("http") for u in urls) + + def test_get_remote_archive_by_url(self): + url = "https://cma-es.github.io/lq-cma/data-archives/lq-gecco2019" + arch = archiving.get(url).update() + assert len(arch) == 4 + assert arch.remote_data_path.split("//", 1)[1] == url.split("//", 1)[1] + + +def _run_official(args, tmp_path): + return pipeline.run_main(args, tmp_path) + + +@pytest.mark.slow +@pytest.mark.parametrize( + "args", + [ + "bbob/2009/BFGS_ros_noiseless", # legacy subtest 1 + "RANDOMSEARCH-4_Auger_bbob-biobj.tgz", # legacy subtest 2 + ], +) +def test_postprocess_archived_data(args, tmp_path): + run = _run_official(args, tmp_path) + assert "ALL done" in run.stdout + assert len(run.result) == 1 + + +@pytest.mark.slow +def test_postprocess_constrained_archive_by_url(tmp_path): + """legacy ``tmp_test_constrained``""" + url = "https://numbbo.github.io/cocopp-playground/bbob-constrained/testdata/bbob-constrained" + data_path = archiving.get(url).get("fmincon_consbbob") + run = _run_official(data_path, tmp_path) + assert "ALL done" in run.stdout + + +@pytest.mark.slow +@pytest.mark.parametrize( + "args", + [ + "BIPOP-CMA-ES_hansen_noiseless MCS_huyer_noiseless 2009/NEWUOA_ros_noiseless.tgz RANDOMSEARCH_auger_noiseless.tgz BFGS_ros_noiseless.tgz", + "SMAC-BBOB_hutter_noiseless.tgz lmm-CMA-ES_auger_noiseless.tgz", + "--include-single DE-PSO_garcia-nieto_noiseless.tgz VNS_garcia-martinez_noiseless.tgz", + "--expensive VNS_garcia-martinez_noiseless.tgz", + "RANDOMSEARCH-4_Auger_bbob-biobj.tgz RANDOMSEARCH-100_Auger_bbob-biobj.tgz", + "NSGA-II-MATLAB_Auger_bbob-biobj.tgz RANDOMSEARCH-4_Auger_bbob-biobj.tgz RANDOMSEARCH-100_Auger_bbob-biobj.tgz", + "MCS_huyer_noisy.tgz BFGS_ros_noisy.tgz", + "test/RS-4.zip", + "bbob/2009/BFGS! bbob-largescale/2019/LBFGS!", + "NSGA-II! 2019/IBEA!", + "constrained/2022/RandomSearch-5", + ], + ids=lambda a: a[:40], +) +def test_legacy_all_scenarios(args, tmp_path): + """the "all" scenarios (subtests 3-20) of the former ``python -m cocopp.test all``""" + run = _run_official(args, tmp_path) + assert "ALL done" in run.stdout + + +@pytest.mark.slow +def test_legacy_main_twice_in_one_process(package_copy, tmp_path): + """former subtest 19: post-process archived data twice in the same python session""" + script = "import cocopp; cocopp.main('--no-interactive slsqp!'); cocopp.main('--no-interactive abc*')" + env = dict(os.environ, PYTHONPATH=str(package_copy.parent), MPLBACKEND="Agg") + res = subprocess.run([sys.executable, "-c", script], cwd=str(tmp_path), env=env, capture_output=True, text=True, timeout=3600) + assert res.returncode == 0, res.stdout[-3000:] + res.stderr[-3000:] + assert res.stdout.count("ALL done") == 2 + + +def test_pickle_regression(quiet, monkeypatch): + """former ``do_pickle_regression_test``: data read from a pickle file equal data read from the archive""" + monkeypatch.setattr(genericsettings, "pretend_initial_solution_was_evaluated", False) + name = cocopp.archives.all.get_extended("strained/2022/al3-cma")[0] + dsl1 = pproc.get_DataSetList(name) + if not pproc._pickle_file_read: + dsl1 = pproc.get_DataSetList(name) # second call reads the pickle written by the first + assert pproc._pickle_file_read + old, pproc._ignore_pickle_files = pproc._ignore_pickle_files, True + try: + dsl2 = pproc.get_DataSetList(name) + finally: + pproc._ignore_pickle_files = old + assert not pproc._pickle_file_read + dsl2._is_same_list_assert(dsl1) + + +def test_package_doctests(package_copy): + """run the doctests of all modules with ``tests/run_doctests.py`` + + The doctests run in the throw-away package copy because they change + into the package folder and may write files there. + """ + runner = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "run_doctests.py") + env = dict(os.environ, PYTHONPATH=str(package_copy.parent), MPLBACKEND="Agg") + res = subprocess.run([sys.executable, runner], cwd=str(package_copy.parent), env=env, capture_output=True, text=True, timeout=1800) + assert res.returncode == 0, res.stdout[-5000:] + res.stderr[-5000:] + summary = re.search(r"doctests: (\d+) of (\d+) examples failed", res.stdout) + assert summary and summary.group(1) == "0" and int(summary.group(2)) > 100, res.stdout[-2000:] diff --git a/tests/pipeline.py b/tests/pipeline.py new file mode 100644 index 0000000..6808f5b --- /dev/null +++ b/tests/pipeline.py @@ -0,0 +1,130 @@ +"""Helpers to run the full post-processing pipeline (`cocopp.main`) in tests.""" + +import contextlib +import fnmatch +import io +import os +import re +import time +import warnings + +import numpy as np + +import cocopp + +from conftest import SEED, isolated_cocopp_state + +# the output sub-folder names end with a time stamp like _101509h5934 +TIMESTAMP_RE = re.compile(r"_\d{6}h\d{4}") + + +class PipelineRun: + """result of a single `cocopp.main` call, see `run_main`""" + + def __init__(self, workdir, outdir, result, stdout, duration): + self.workdir = str(workdir) + self.outdir = os.path.join(self.workdir, outdir) + self.result = result + self.stdout = stdout + self.duration = duration + + @property + def files(self): + """sorted list of all output files relative to `outdir`""" + return sorted( + os.path.relpath(os.path.join(root, f), self.outdir).replace(os.sep, "/") + for root, _, files in os.walk(self.outdir) + for f in files + ) + + def glob(self, pattern): + """output files matching `pattern`, e.g. ``'*/pptable_*.tex'``, with time stamps removed""" + return [f for f in self.files if fnmatch.fnmatch(TIMESTAMP_RE.sub("", f), pattern)] + + @property + def subfolders(self): + """the output sub-folders without time stamps""" + return sorted(TIMESTAMP_RE.sub("", d) for d in os.listdir(self.outdir) if os.path.isdir(os.path.join(self.outdir, d))) + + def path(self, relative_name): + """absolute path of an output file given without time stamps""" + matches = [f for f in self.files if TIMESTAMP_RE.sub("", f) == relative_name] + assert len(matches) == 1, (relative_name, matches) + return os.path.join(self.outdir, matches[0]) + + def read(self, relative_name): + with open(self.path(relative_name), encoding="utf-8", errors="replace") as f: + return f.read() + + +# modules whose text output depends on the state of numpy's global random number generator +RESEEDED_FUNCTIONS = [("cocopp.pptable", "main"), ("cocopp.compall.pptables", "main"), ("cocopp.compall.pprldmany", "main")] + + +@contextlib.contextmanager +def reseeded_entry_points(functions=RESEEDED_FUNCTIONS, seed=SEED): + """reseed numpy's global RNG whenever one of `functions` is called. + + Without this, the bootstrapped dispersion values in the tables depend + on ``PYTHONHASHSEED``: `rungenericmany` iterates over function groups in + `set` order before, consuming a data dependent number of random numbers. + """ + import importlib + + originals = [] + for module_name, name in functions: + module = importlib.import_module(module_name) + original = getattr(module, name) + + def wrapper(*args, __original=original, **kwargs): + np.random.seed(seed) + return __original(*args, **kwargs) + + originals.append((module, name, original)) + setattr(module, name, wrapper) + try: + yield + finally: + for module, name, original in originals: + setattr(module, name, original) + + +def run_main(args, workdir, setup=None, outdir="out"): + """run ``cocopp.main('-o outdir ' + args)`` in `workdir` with isolated global state. + + `setup` is called within the isolated state before the run, e.g. to + change `cocopp.genericsettings`. Numpy's global RNG is reseeded on + entry of the table modules, see `reseeded_entry_points`. Return a + `PipelineRun`. + """ + stdout = io.StringIO() + os.makedirs(str(workdir), exist_ok=True) + with isolated_cocopp_state(workdir): + if setup: + setup() + t0 = time.time() + with warnings.catch_warnings(), contextlib.redirect_stdout(stdout), reseeded_entry_points(): + warnings.simplefilter("ignore") + result = cocopp.main("--no-interactive -o %s %s" % (outdir, args)) + duration = time.time() - t0 + return PipelineRun(workdir, outdir, result, stdout.getvalue(), duration) + + +# name: (data fixtures, extra command line options, background data fixtures) +SCENARIOS = { + "bbob-1": (["bbob_alg_a"], [], []), + "bbob-2": (["bbob_alg_a", "bbob_alg_b"], [], []), + "bbob-3": (["bbob_alg_a", "bbob_alg_b", "bbob_alg_c"], [], []), + "bbob-1-expensive": (["bbob_alg_a"], ["--expensive"], []), + "bbob-1-conv": (["bbob_alg_a"], ["--conv"], []), + "bbob-1-background": (["bbob_alg_a"], [], ["bbob_alg_b", "bbob_alg_c"]), + "bbob-old-format": (["bbob_old_format"], [], []), + "bbob-small-include-single": (["bbob_small", "bbob_old_format"], ["--include-single"], []), + "biobj-1": (["biobj_alg"], [], []), + "biobj-2": (["biobj_alg", "biobj_alg_b"], [], []), + "noisy-1": (["bbob_noisy"], [], []), + "constrained-1": (["bbob_constrained"], [], []), + "largescale-1": (["bbob_largescale"], [], []), + "mixint-1": (["bbob_mixint"], [], []), + "boxed-1": (["bbob_boxed"], [], []), +} diff --git a/tests/regression/test_snapshots.py b/tests/regression/test_snapshots.py new file mode 100644 index 0000000..dbdd30f --- /dev/null +++ b/tests/regression/test_snapshots.py @@ -0,0 +1,45 @@ +"""Golden-file regression tests of the complete post-processing output. + +Each scenario from `pipeline.SCENARIOS` is run once (cached for the test +session) on deterministic synthetic data with a fixed random seed, and +the normalized text output (all LaTeX tables and commands, html pages, +the list of produced files and of broken html links) is compared to +the stored snapshot in ``tests/snapshots``. + +These tests are the safety net for refactoring: any change in the +produced numbers or formatting shows up as a diff. If a change is +intended, rewrite the snapshots with:: + + pytest tests/regression --update-snapshots + +and review the changes of the (gzipped) snapshots with, e.g., +``python tests/snapshots.py`` (see `snapshots.diff`). +""" + +import os + +import pytest + +import pipeline +import snapshots + +pytestmark = pytest.mark.slow + + +@pytest.mark.parametrize("scenario", sorted(pipeline.SCENARIOS)) +def test_snapshot(scenario, pipeline_run, synth_root, request): + run = pipeline_run(scenario) + actual = snapshots.take(run, replacements=[(str(synth_root), "")]) + if request.config.getoption("--update-snapshots"): + snapshots.save(scenario, actual) + pytest.skip("snapshot written") + if not os.path.exists(snapshots.path(scenario)): + pytest.fail("no snapshot for %s, run pytest with --update-snapshots" % scenario) + expected = snapshots.load(scenario) + differences = snapshots.diff(expected, actual) + assert not differences, "output of scenario %s changed:\n%s" % (scenario, differences) + + +def test_no_orphaned_snapshots(): + stored = {f[: -len(".json.gz")] for f in os.listdir(snapshots.SNAPSHOT_DIR) if f.endswith(".json.gz")} + assert stored <= set(pipeline.SCENARIOS) diff --git a/tests/run_doctests.py b/tests/run_doctests.py new file mode 100644 index 0000000..fe1f545 --- /dev/null +++ b/tests/run_doctests.py @@ -0,0 +1,49 @@ +"""Run the doctests of all modules of the `cocopp` package found on the path. + +Usage:: + + python tests/run_doctests.py + +Replaces ``python -m cocopp.test doctest`` of the removed legacy test +runner. As before, each ``.py`` file of the package is run with +`doctest.testfile` from within the package folder, because some doctests +rely on relative paths. Many doctests need the online data archives and +some write files into the package folder, hence the test suite runs this +script in a subprocess on a throw-away copy of the package (see +``tests/network/test_archives_online.py``). + +Exit with status 1 if any doctest fails. +""" + +import doctest +import os +import sys + +import matplotlib + +matplotlib.use("Agg") + + +def main(): + import cocopp + + package_dir = os.path.dirname(os.path.abspath(cocopp.__file__)) + os.chdir(package_dir) + failures, tests, failed_files = 0, 0, [] + for root, _dirs, files in sorted(os.walk(package_dir)): + for name in sorted(files): + if not name.endswith(".py"): + continue + path = os.path.join(root, name) + f, t = doctest.testfile(path, module_relative=False, report=True) + failures, tests = failures + f, tests + t + if f: + failed_files.append("%s (%d of %d failed)" % (os.path.relpath(path, package_dir), f, t)) + print("doctests: %d of %d examples failed" % (failures, tests)) + for name in failed_files: + print(" failed: " + name) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/smoke/test_cli.py b/tests/smoke/test_cli.py new file mode 100644 index 0000000..9496dc4 --- /dev/null +++ b/tests/smoke/test_cli.py @@ -0,0 +1,130 @@ +"""Smoke tests of the command line entry points, run in subprocesses. + +The subprocesses import the throw-away copy of the package (see the +``package_copy`` fixture) by putting it first on ``PYTHONPATH``, hence +they cannot write into the source tree either. +""" + +import os +import subprocess +import sys + +import pytest + +import cocopp + + +@pytest.fixture +def run_python(package_copy, tmp_path): + """return a function running ``python args...`` in `tmp_path`, returning the `CompletedProcess`""" + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join([str(package_copy.parent)] + [p for p in env.get("PYTHONPATH", "").split(os.pathsep) if p]) + env["MPLBACKEND"] = "Agg" + + def run(*args, timeout=600, env_update=None): + return subprocess.run( + [sys.executable] + list(args), cwd=str(tmp_path), env=dict(env, **(env_update or {})), capture_output=True, text=True, timeout=timeout + ) + + return run + + +def test_subprocess_uses_package_copy(run_python, package_copy): + res = run_python("-c", "import cocopp; print(cocopp.__file__)") + assert res.returncode == 0, res.stderr + assert res.stdout.strip().startswith(str(package_copy)) + + +def test_import_is_quiet_and_fast(run_python): + res = run_python("-c", "import time; t = time.time(); import cocopp; print('%.2f' % (time.time() - t))") + assert res.returncode == 0, res.stderr + assert res.stdout.splitlines()[-1] and float(res.stdout.splitlines()[-1]) < 20 + assert len(res.stdout.splitlines()) == 1 # nothing else printed on import + + +def test_version(run_python): + res = run_python("-c", "import cocopp; print(cocopp.__version__)") + assert res.stdout.strip() == cocopp.__version__ + + +def test_cli_without_arguments_prints_usage(run_python): + res = run_python("-m", "cocopp") + assert res.returncode == 0, res.stderr + assert "Main routine for post-processing data from COCO" in res.stdout + + +def test_cli_help(run_python): + res = run_python("-m", "cocopp", "--help") + assert res.returncode == 0 and "--output-dir" in res.stdout + + +def test_cli_unknown_option(run_python): + res = run_python("-m", "cocopp", "--no-such-option", "x") + assert "For help use -h or --help" in res.stderr + + +def test_cli_missing_data(run_python, tmp_path): + res = run_python("-m", "cocopp", "--no-interactive", str(tmp_path / "does-not-exist")) + assert res.returncode != 0 + assert "does-not-exist" in res.stderr + + +@pytest.mark.slow +def test_cli_postprocess_single(run_python, bbob_small, tmp_path): + res = run_python("-m", "cocopp", "--no-interactive", "-o", "out", bbob_small) + assert res.returncode == 0, res.stderr[-3000:] + assert "ALL done" in res.stdout + assert (tmp_path / "out" / "index.html").exists() + (folder,) = [d for d in os.listdir(tmp_path / "out") if os.path.isdir(tmp_path / "out" / d)] + assert folder.startswith("SMALL_") + assert (tmp_path / "out" / folder / "pptable_f001_05D.tex").exists() + + +@pytest.mark.slow +def test_cli_postprocess_two(run_python, bbob_small, bbob_small_b, tmp_path): + res = run_python("-m", "cocopp", "--no-interactive", "--no-svg", "-o", "out", bbob_small, bbob_small_b) + assert res.returncode == 0, res.stderr[-3000:] + (folder,) = [d for d in os.listdir(tmp_path / "out") if os.path.isdir(tmp_path / "out" / d)] + assert folder.startswith("SMALL_SMALL_") + assert (tmp_path / "out" / folder / "pptables_f001_05D.tex").exists() + + +@pytest.mark.slow +def test_python_api_main_twice(run_python, bbob_small, bbob_old_format): + """corresponds to subtest 19 of the former ``cocopp/test.py``""" + script = "import cocopp; cocopp.main(%r); cocopp.main(%r)" % ("--no-interactive -o a " + bbob_small, "--no-interactive -o b " + bbob_old_format) + res = run_python("-c", script) + assert res.returncode == 0, res.stderr[-3000:] + assert res.stdout.count("ALL done") == 2 + + +def test_reduce_size_cli(run_python, tmp_path): + import synthdata + + folder = synthdata.write_bbob(str(tmp_path / "data"), "RED", functions=(1,), dims=(10,), seed=5) + res = run_python("-m", "cocopp.reduce_size", "--help") + assert res.returncode == 0 and "reduce_size" in res.stdout + res = run_python("-m", "cocopp.reduce_size", folder) + assert res.returncode == 0, res.stderr + assert len([d for d in os.listdir(tmp_path / "data") if d.startswith("RED-")]) == 1 + res = run_python("-m", "cocopp.reduce_size") + assert res.returncode != 0 and "needs a folder name" in res.stderr + + +def test_findfiles_from_shell(run_python, bbob_small): + res = run_python("-c", "from cocopp.findfiles import main; print(len(main(%r)))" % bbob_small) + assert res.returncode == 0 and res.stdout.strip() == "4" + + +@pytest.mark.xfail(reason="the order of inconsistent hv-hashes in the version label depends on PYTHONHASHSEED", strict=True) +def test_version_label_independent_of_hash_seed(run_python): + script = ( + "import cocopp; cocopp.testbedsettings.reference_values = {'A': 'a1b2c3', 'B': 'z9y8x7', 'C': 'k5l6m7'};" + " print(cocopp.toolsdivers.get_version_label())" + ) + labels = set() + for seed in range(8): + res = run_python("-c", script, env_update={"PYTHONHASHSEED": str(seed)}) + assert res.returncode == 0, res.stderr + labels.add(res.stdout.strip()) + assert len(labels) == 1, labels diff --git a/tests/snapshots.py b/tests/snapshots.py new file mode 100644 index 0000000..03f73db --- /dev/null +++ b/tests/snapshots.py @@ -0,0 +1,167 @@ +"""Golden-file snapshots of the text output of the post-processing. + +A snapshot of a `pipeline.PipelineRun` consists of + +- the list of all output files (including figures), +- the content of all generated text files (``.tex``, ``.html``, ``.js``), +- the list of broken relative links in the generated html files, + +normalized such that it does not depend on time stamps, temporary +folders or the package version. Snapshots are stored gzipped as +``tests/snapshots/.json.gz`` and are (re)written with +``pytest --update-snapshots``. +""" + +import difflib +import gzip +import json +import os +import re + +import cocopp + +from pipeline import TIMESTAMP_RE + +SNAPSHOT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "snapshots") +TEXT_EXTENSIONS = (".tex", ".html", ".js") +# copied verbatim from the package's static folder +STATIC_FILES = ("sorttable.js", "pprldflex.js", "pprldflex.css", "pprldflex.html") +_LINK_RE = re.compile(r"""(?:href|src)\s*=\s*["']([^"'#]+)""", re.IGNORECASE) + + +_HV_HASHES_RE = re.compile(r"hv-hashes inconsistent: (.*?) found!") + + +def _sorted_hashes(match): + """the order of the hashes depends on PYTHONHASHSEED (set iteration order)""" + return "hv-hashes inconsistent: %s found!" % " and ".join(sorted(match.group(1).split(" and "))) + + +_SIDEPANEL_RE = re.compile(r"(\\providecommand\{\\perfprofsidepanel\}\{\{(?:\\tiny\{\})?)(.*?)(\}\}\n)", re.DOTALL) + + +def _sorted_sidepanel(match): + """the order of (nearly) tied algorithms in the pprldmany legend depends on + floating point summation order, which depends on PYTHONHASHSEED""" + entries = sorted(e.strip() for e in match.group(2).split("\\vfill")) + return match.group(1) + "\n\\vfill ".join(entries) + match.group(3) + + +def normalize(text, replacements): + """remove run-specific parts from `text`. + + Each replacement ``(old, new)`` is also applied to the LaTeX-escaped + variants of ``old`` as written by the package. + """ + for old, new in replacements: + for variant in (old, old.replace("_", "\\_"), old.replace("_", " ")): + text = text.replace(variant, new) + text = TIMESTAMP_RE.sub("_", text) + text = _HV_HASHES_RE.sub(_sorted_hashes, text) + text = _SIDEPANEL_RE.sub(_sorted_sidepanel, text) + return text.replace(cocopp.__version__, "") + + +def broken_links(run): + """return sorted list of ``"page -> link"`` for relative links that don't resolve""" + res = set() + for name in run.files: + if not name.endswith(".html"): + continue + path = os.path.join(run.outdir, name) + with open(path, encoding="utf-8", errors="replace") as f: + links = _LINK_RE.findall(f.read()) + for link in links: + if ":" in link.split("/")[0]: # http:, file:, javascript: ... + continue + if not os.path.exists(os.path.normpath(os.path.join(os.path.dirname(path), link))): + res.add("%s -> %s" % (TIMESTAMP_RE.sub("_", name), TIMESTAMP_RE.sub("_", link))) + return sorted(res) + + +def take(run, replacements=()): + """return the snapshot `dict` of a `pipeline.PipelineRun`""" + replacements = list(replacements) + [(run.workdir, "")] + text = {} + for name in run.files: + if name.endswith(TEXT_EXTENSIONS) and os.path.basename(name) not in STATIC_FILES: + with open(os.path.join(run.outdir, name), encoding="utf-8", errors="replace") as f: + text[normalize(name, replacements)] = normalize(f.read(), replacements) + return { + "files": sorted(normalize(name, replacements) for name in run.files), + "text": text, + "broken_links": broken_links(run), + } + + +def path(scenario): + return os.path.join(SNAPSHOT_DIR, scenario + ".json.gz") + + +def save(scenario, snapshot): + os.makedirs(SNAPSHOT_DIR, exist_ok=True) + data = json.dumps(snapshot, indent=0, sort_keys=True).encode("utf-8") + # mtime=0 makes the gzip file itself reproducible + with open(path(scenario), "wb") as f, gzip.GzipFile(fileobj=f, mode="wb", mtime=0) as gz: + gz.write(data) + + +def load(scenario): + with gzip.open(path(scenario), "rb") as f: + return json.loads(f.read().decode("utf-8")) + + +def diff(expected, actual, max_lines=60): + """return a human readable description of the differences or ``""``""" + out = [] + for key in ("files", "broken_links"): + missing = sorted(set(expected[key]) - set(actual[key])) + added = sorted(set(actual[key]) - set(expected[key])) + if missing or added: + out.append("%s: %d missing, %d new" % (key, len(missing), len(added))) + out += [" - " + m for m in missing[:10]] + [" + " + a for a in added[:10]] + changed = sorted(n for n in set(expected["text"]) & set(actual["text"]) if expected["text"][n] != actual["text"][n]) + if changed: + out.append("%d text file(s) differ: %s" % (len(changed), ", ".join(changed[:10]) + (" ..." if len(changed) > 10 else ""))) + name = changed[0] + udiff = difflib.unified_diff( + expected["text"][name].splitlines(), actual["text"][name].splitlines(), "expected/" + name, "actual/" + name, lineterm="", n=1 + ) + out += list(udiff)[:max_lines] + return "\n".join(out) + + +def _main(argv): + """command line interface to review snapshots:: + + python tests/snapshots.py diff OLD.json.gz NEW.json.gz + python tests/snapshots.py extract SNAPSHOT.json.gz FOLDER + + ``extract`` writes all text files of a snapshot into FOLDER, such that + two extracted snapshots can be compared with any diff tool, e.g. after + ``jj file show -r @- tests/snapshots/bbob-1.json.gz > old.json.gz``. + """ + def read(name): + with gzip.open(name, "rb") as f: + return json.loads(f.read().decode("utf-8")) + + if len(argv) == 3 and argv[0] == "diff": + print(diff(read(argv[1]), read(argv[2]), max_lines=10**6) or "no differences") + elif len(argv) == 3 and argv[0] == "extract": + snapshot = read(argv[1]) + for name, text in snapshot["text"].items(): + target = os.path.join(argv[2], *name.split("/")) + os.makedirs(os.path.dirname(target), exist_ok=True) + with open(target, "w", encoding="utf-8") as f: + f.write(text) + for key in ("files", "broken_links"): + with open(os.path.join(argv[2], "_%s.txt" % key), "w", encoding="utf-8") as f: + f.write("\n".join(snapshot[key]) + "\n") + else: + print(_main.__doc__) + + +if __name__ == "__main__": + import sys + + _main(sys.argv[1:]) diff --git a/tests/snapshots/bbob-1-background.json.gz b/tests/snapshots/bbob-1-background.json.gz new file mode 100644 index 0000000..ad155bb Binary files /dev/null and b/tests/snapshots/bbob-1-background.json.gz differ diff --git a/tests/snapshots/bbob-1-conv.json.gz b/tests/snapshots/bbob-1-conv.json.gz new file mode 100644 index 0000000..478bc58 Binary files /dev/null and b/tests/snapshots/bbob-1-conv.json.gz differ diff --git a/tests/snapshots/bbob-1-expensive.json.gz b/tests/snapshots/bbob-1-expensive.json.gz new file mode 100644 index 0000000..c0b919c Binary files /dev/null and b/tests/snapshots/bbob-1-expensive.json.gz differ diff --git a/tests/snapshots/bbob-1.json.gz b/tests/snapshots/bbob-1.json.gz new file mode 100644 index 0000000..d154b7b Binary files /dev/null and b/tests/snapshots/bbob-1.json.gz differ diff --git a/tests/snapshots/bbob-2.json.gz b/tests/snapshots/bbob-2.json.gz new file mode 100644 index 0000000..e7ea431 Binary files /dev/null and b/tests/snapshots/bbob-2.json.gz differ diff --git a/tests/snapshots/bbob-3.json.gz b/tests/snapshots/bbob-3.json.gz new file mode 100644 index 0000000..6cb0283 Binary files /dev/null and b/tests/snapshots/bbob-3.json.gz differ diff --git a/tests/snapshots/bbob-old-format.json.gz b/tests/snapshots/bbob-old-format.json.gz new file mode 100644 index 0000000..bb64118 Binary files /dev/null and b/tests/snapshots/bbob-old-format.json.gz differ diff --git a/tests/snapshots/bbob-small-include-single.json.gz b/tests/snapshots/bbob-small-include-single.json.gz new file mode 100644 index 0000000..ab7db3a Binary files /dev/null and b/tests/snapshots/bbob-small-include-single.json.gz differ diff --git a/tests/snapshots/biobj-1.json.gz b/tests/snapshots/biobj-1.json.gz new file mode 100644 index 0000000..ecb93f4 Binary files /dev/null and b/tests/snapshots/biobj-1.json.gz differ diff --git a/tests/snapshots/biobj-2.json.gz b/tests/snapshots/biobj-2.json.gz new file mode 100644 index 0000000..e7782e5 Binary files /dev/null and b/tests/snapshots/biobj-2.json.gz differ diff --git a/tests/snapshots/boxed-1.json.gz b/tests/snapshots/boxed-1.json.gz new file mode 100644 index 0000000..2341e44 Binary files /dev/null and b/tests/snapshots/boxed-1.json.gz differ diff --git a/tests/snapshots/constrained-1.json.gz b/tests/snapshots/constrained-1.json.gz new file mode 100644 index 0000000..2d7d9f1 Binary files /dev/null and b/tests/snapshots/constrained-1.json.gz differ diff --git a/tests/snapshots/largescale-1.json.gz b/tests/snapshots/largescale-1.json.gz new file mode 100644 index 0000000..c337075 Binary files /dev/null and b/tests/snapshots/largescale-1.json.gz differ diff --git a/tests/snapshots/mixint-1.json.gz b/tests/snapshots/mixint-1.json.gz new file mode 100644 index 0000000..1892fa0 Binary files /dev/null and b/tests/snapshots/mixint-1.json.gz differ diff --git a/tests/snapshots/noisy-1.json.gz b/tests/snapshots/noisy-1.json.gz new file mode 100644 index 0000000..203347b Binary files /dev/null and b/tests/snapshots/noisy-1.json.gz differ diff --git a/tests/synthdata.py b/tests/synthdata.py new file mode 100644 index 0000000..76a60fa --- /dev/null +++ b/tests/synthdata.py @@ -0,0 +1,298 @@ +"""Deterministic generator for synthetic COCO experiment data. + +The real data archives are several MB per algorithm and live online. For +offline, fast and reproducible tests we instead write small, *synthetic* +data folders that follow the on-disk formats written by the COCO +experiment loggers closely enough for `cocopp` to read them: + +- single-objective ``bbob`` (and ``bbob-noisy``) data in the "new" + (``data_format = 'bbob-new2'``, constraints evaluations in the second + column) and the "old" (no ``data_format``) layouts, consisting of + ``.info`` index files and ``.dat``/``.tdat`` data files; +- bi-objective ``bbob-biobj`` data (``indicator = 'hyp'``). + +Every generator takes a ``seed`` and only uses its own +`numpy.random.Generator`, hence the produced files are byte-identical +for the same arguments, independently of the global numpy random state. + +The simulated "optimizer" improves the best-so-far Delta-f value by a +random factor at random evaluation increments. Function and dimension +determine the convergence speed, such that some (function, dimension) +pairs are solved in all trials, some in a few and some in none, which +exercises the success/failure code paths of the post-processing. +""" + +import os + +import numpy as np + +BBOB_DIMS = (2, 3, 5, 10, 20) +BBOB_FUNCTIONS = tuple(range(1, 25)) +BBOB_INSTANCES = tuple(range(1, 16)) +BIOBJ_DIMS = (2, 3, 5) +BIOBJ_FUNCTIONS = tuple(range(1, 56)) +BIOBJ_INSTANCES = tuple(range(1, 11)) + +_PRECISION = 1e-8 + + +def _log_spaced_evals(last, per_decade=10): + """return sorted unique integer evaluations ``1, 2, ...`` up to `last` + + roughly log-uniformly spaced, always including 1 and `last`.""" + if last <= 1: + return [1] + n = int(np.ceil(np.log10(last) * per_decade)) + 1 + evals = np.unique(np.floor(np.logspace(0, np.log10(last), n)).astype(int)) + evals = [int(e) for e in evals if e < last] + return evals + [int(last)] + + +def _trajectory(rng, f0, rate, budget, final=_PRECISION / 100): + """simulate a monotone best-so-far trajectory. + + Return two lists ``evals`` and ``fvals`` (Delta f), where + ``fvals[i]`` is the best-so-far value from evaluation ``evals[i]`` on. + The first evaluation is always 1. The run stops when the `budget` is + exhausted or `final` is reached; the last entry is the last + evaluation. + """ + evals, fvals = [1], [f0] + e, f = 1, f0 + while True: + e += int(rng.geometric(rate)) + if e > budget: + break + f *= 10 ** -rng.uniform(0.05, 0.6) + evals.append(e) + fvals.append(f) + if f < final: + break + if evals[-1] < budget and fvals[-1] >= final: # unsuccessful: ran into the budget + evals.append(int(budget)) + fvals.append(fvals[-1]) + return evals, fvals + + +def _bbob_target_crossings(evals, fvals, nb_per_decade=5): + """return indices into `evals` at which a new ``10**(i/5)`` target is reached, + + as written into ``.dat`` files by the bbob logger. Index 0 is always + included and the final index is included when it reaches below the + final precision.""" + idx = [0] + level = np.floor(np.log10(fvals[0]) * nb_per_decade) + for i in range(1, len(fvals)): + new_level = np.floor(np.log10(max(fvals[i], 1e-300)) * nb_per_decade) + if new_level < level: + idx.append(i) + level = new_level + return idx + + +def _fmt(x): + return "%+.9e" % x + + +def _bbob_line(e, df, fopt, dim, rng, old_format, g_per_f=0): + # the real loggers write all ``dim`` coordinates, two suffice to keep files small + x = " ".join("%+.4e" % v for v in rng.uniform(-4, 4, min(dim, 2))) + second = _fmt(df + fopt) if old_format else "%d" % (g_per_f * e) + return "%d %s %s %s %s %s\n" % (e, second, _fmt(df), _fmt(df + fopt), _fmt(df + fopt), x) + + +def _rate(rng, fun, dim, speed): + """return the per-evaluation improvement probability for one trial on ``(fun, dim)``. + + The expected number of evaluations per improvement step is about + ``c_f * dim**1.1 / speed`` where ``c_f`` grows with ``fun % 8`` from + 1.5 to about 12, such that with the default budget of ``200 * dim`` + the easy functions are always and the hard ones rarely solved. + """ + c_f = 1.5 * 1.35 ** (fun % 8) + return min(1.0, speed / (c_f * dim**1.1 * 10 ** rng.uniform(-0.3, 0.3))) + + +def write_bbob( + root, + alg_name="SYNTH-A", + functions=BBOB_FUNCTIONS, + dims=BBOB_DIMS, + instances=BBOB_INSTANCES, + seed=1, + budget_multiplier=200, + speed=1.0, + suite="bbob", + old_format=False, + with_tdat=True, + comment=None, + g_per_f=None, +): + """write a full synthetic single-objective experiment into ``root/alg_name``. + + Return the path of the algorithm folder. `speed` scales the + convergence speed (larger is faster), such that different "algorithms" + can be created from the same function. Only the generator seeded with + `seed` is used. + + ``old_format=True`` writes the pre-2016 layout without ``suite``, + ``data_format`` and ``logger`` in the header (second column is the + current f-value instead of constraints evaluations). + + `g_per_f` is the number of constraints evaluations per f-evaluation, + by default ``1 + (fun - 1) % 6`` for the ``bbob-constrained`` suite + and zero otherwise. + """ + rng = np.random.default_rng(seed) + folder = os.path.join(root, alg_name) + os.makedirs(folder, exist_ok=True) + comment = comment if comment is not None else "%% %s: synthetic data, seed=%d" % (alg_name, seed) + for fun in functions: + g = g_per_f if g_per_f is not None else (1 + (fun - 1) % 6 if suite == "bbob-constrained" else 0) + data_dir = "data_f%d" % fun + os.makedirs(os.path.join(folder, data_dir), exist_ok=True) + info_lines = [] + for dim in dims: + budget = int(budget_multiplier * dim) + base = "bbobexp_f%d_DIM%d" % (fun, dim) + dat = open(os.path.join(folder, data_dir, base + ".dat"), "w") + tdat = open(os.path.join(folder, data_dir, base + ".tdat"), "w") if with_tdat else None + trial_infos = [] + for inst in instances: + fopt = float(np.round(rng.uniform(-400, 400), 2)) + f0 = 10 ** rng.uniform(0.5, 3) + rate = _rate(rng, fun, dim, speed) + evals, fvals = _trajectory(rng, f0, rate, budget) + header = ( + "%% f evaluations | g evaluations | best noise-free fitness - Fopt (%.12e) + sum g_i+ |" + " measured fitness | best measured fitness or single-digit g-values | x1 | x2...\n" % fopt + ) + dat.write(header) + for i in _bbob_target_crossings(evals, fvals): + dat.write(_bbob_line(evals[i], fvals[i], fopt, dim, rng, old_format, g)) + if tdat: + tdat.write(header) + j = 0 + for e in _log_spaced_evals(evals[-1]): + while j + 1 < len(evals) and evals[j + 1] <= e: + j += 1 + tdat.write(_bbob_line(e, fvals[j], fopt, dim, rng, old_format, g)) + trial_infos.append("%d:%d|%.1e" % (inst, evals[-1], fvals[-1] - _PRECISION)) + dat.close() + if tdat: + tdat.close() + if old_format: + head = "funcId = %d, DIM = %d, Precision = %.3e, algId = '%s'" % (fun, dim, _PRECISION, alg_name) + else: + head = ( + "suite = '%s', funcId = %d, DIM = %d, Precision = %.3e, algId = '%s'," + " coco_version = '2.6.0', logger = 'bbob', data_format = 'bbob-new2'" % (suite, fun, dim, _PRECISION, alg_name) + ) + info_lines += [head, comment, "%s/%s.dat, " % (data_dir, base) + ", ".join(trial_infos)] + with open(os.path.join(folder, "bbobexp_f%d.info" % fun), "w") as f: + f.write("\n".join(info_lines) + "\n") + return folder + + +def write_biobj( + root, + alg_name="SYNTH-BI", + functions=BIOBJ_FUNCTIONS, + dims=BIOBJ_DIMS, + instances=BIOBJ_INSTANCES, + seed=1, + budget_multiplier=100, + speed=1.0, +): + """write a synthetic ``bbob-biobj`` experiment into ``root/alg_name``. + + The layout follows the 2016 biobj logger: one ``.info`` file per + dimension in a ``dXX`` folder; each data line references a ``.dat`` + file in a ``folder`` subdirectory. Return the algorithm folder. + """ + rng = np.random.default_rng(seed) + folder = os.path.join(root, alg_name) + for dim in dims: + dim_dir = os.path.join(folder, "d%02d" % dim) + group = "all-functions" + os.makedirs(os.path.join(dim_dir, group), exist_ok=True) + info = [ + "algorithm = '%s', indicator = 'hyp', folder = '%s', coco_version = '2.6.0'" % (alg_name, group), + "%% %s: synthetic bi-objective data, seed=%d" % (alg_name, seed), + ] + for fun in functions: + budget = int(budget_multiplier * dim) + base = "bbob-biobj_f%02d_d%02d_hyp" % (fun, dim) + dat = open(os.path.join(dim_dir, group, base + ".dat"), "w") + tdat = open(os.path.join(dim_dir, group, base + ".tdat"), "w") + trial_infos = [] + for inst in instances: + # reference values are a property of the problem instance, not of the algorithm + ref = np.random.default_rng((fun, dim, inst)).uniform(0.5, 1.0) + hdr = "%%\n%% index = %d, name = synthetic_f%02d_i%02d_d%02d\n%% instance = %d, reference value = %.15e\n" % ( + fun * 100 + inst, fun, inst, dim, inst, ref) + f0 = 10 ** rng.uniform(-0.5, 0.5) + rate = _rate(rng, fun, dim, speed) + evals, fvals = _trajectory(rng, f0, rate, budget, final=1e-6) + dat.write(hdr + "% function evaluation | indicator value | target hit\n") + for i in _bbob_target_crossings(evals, fvals, nb_per_decade=10): + target_hit = 10 ** (np.floor(np.log10(max(fvals[i], 1e-300)) * 10) / 10) + dat.write("%d\t%.15e\t%.15e\n" % (evals[i], fvals[i], target_hit)) + tdat.write(hdr + "% function evaluation | indicator value\n") + j = 0 + for e in _log_spaced_evals(evals[-1]): + while j + 1 < len(evals) and evals[j + 1] <= e: + j += 1 + tdat.write("%d\t%.15e\n" % (e, fvals[j])) + trial_infos.append("%d:%d|%.1e" % (inst, evals[-1], fvals[-1])) + dat.close() + tdat.close() + # the file name is relative to `folder` given in the header + info.append("function = %d, dim = %d, %s.dat, " % (fun, dim, base) + ", ".join(trial_infos)) + with open(os.path.join(dim_dir, "synthetic_d%02d_hyp.info" % dim), "w") as f: + f.write("\n".join(info) + "\n") + return folder + + +# hand-crafted tiny data set with hand-computable results, see `write_tiny_bbob` +TINY_TRIALS = { + # instance: [(f-evaluations, g-evaluations, Delta f), ...]; the last line is the final evaluation + 1: [(1, 0, 95.0), (10, 0, 0.9), (50, 0, 5e-9)], + 2: [(1, 0, 45.0), (20, 0, 4.5), (100, 0, 0.45)], + 3: [(1, 0, 9.5), (5, 0, 9e-4), (30, 0, 9e-10)], +} + + +def write_tiny_bbob(root, alg_name="TINY", trials=None, suite="bbob", fun=1, dim=2): + """write a single ``(fun, dim)`` data set with given `trials` into ``root/alg_name``. + + `trials` maps instance numbers to lists of ``(evals, g_evals, Delta_f)`` + tuples, or is a list of ``(instance, rows)`` pairs (to repeat + instances). Each tuple is written to both the ``.dat`` and the ``.tdat`` + file such that the expected `DataSet` content can be computed by hand. + Return the path of the ``.info`` file. + """ + trials = TINY_TRIALS if trials is None else trials + trials = list(trials.items()) if isinstance(trials, dict) else list(trials) + folder = os.path.join(root, alg_name) + os.makedirs(os.path.join(folder, "data_f%d" % fun), exist_ok=True) + base = "data_f%d/bbobexp_f%d_DIM%d" % (fun, fun, dim) + fopt = 10.0 + lines = [] + for inst, rows in trials: + lines.append("%% f evaluations | g evaluations | best noise-free fitness - Fopt (%.12e) + sum g_i+ | ...\n" % fopt) + for e, g, df in rows: + lines.append("%d %d %s %s %s +0.0000e+00\n" % (e, g, _fmt(df), _fmt(df + fopt), _fmt(df + fopt))) + for ext in (".dat", ".tdat"): + with open(os.path.join(folder, base + ext), "w") as f: + f.write("".join(lines)) + infos = ", ".join("%d:%d|%.1e" % (inst, rows[-1][0], rows[-1][2] - _PRECISION) for inst, rows in trials) + info_file = os.path.join(folder, "bbobexp_f%d.info" % fun) + with open(info_file, "w") as f: + f.write( + "suite = '%s', funcId = %d, DIM = %d, Precision = 1.000e-08, algId = '%s', coco_version = '2.6.0'," + " logger = 'bbob', data_format = 'bbob-new2'\n" % (suite, fun, dim, alg_name) + ) + f.write("%% %s: hand-crafted test data\n" % alg_name) + f.write("%s.dat, %s\n" % (base, infos)) + return info_file diff --git a/tests/unit/test_archiving_local.py b/tests/unit/test_archiving_local.py new file mode 100644 index 0000000..7314a85 --- /dev/null +++ b/tests/unit/test_archiving_local.py @@ -0,0 +1,164 @@ +"""Unit tests for `cocopp.archiving` with *local* archives only. + +Nothing here touches the network or the user's cache folder: +`ArchivesLocal.register`, which would record created archives in the +user's cache, is patched. +""" + +import os +import tarfile + +import pytest + +from cocopp import archiving + + +@pytest.fixture +def registered(monkeypatch): + calls = [] + monkeypatch.setattr(archiving.ArchivesLocal, "register", classmethod(lambda cls, folder: calls.append(folder))) + return calls + + +@pytest.fixture +def archive_folder(tmp_path, bbob_small, bbob_old_format): + """a folder with three zipped "data sets" in sub-folders""" + root = tmp_path / "my-archive" + for sub, src, name in [("2020", bbob_small, "SMALL_alg.tgz"), ("2021", bbob_old_format, "OLD_alg.tgz"), ("2021", bbob_small, "SMALL2.tgz")]: + (root / sub).mkdir(parents=True, exist_ok=True) + with tarfile.open(str(root / sub / name), "w:gz") as tar: + tar.add(src, arcname=os.path.basename(src)) + (root / "README.md").write_text("ignored") + return root + + +@pytest.fixture +def archive(archive_folder, registered): + return archiving.create(str(archive_folder)) + + +class TestCreateAndRead: + def test_create(self, archive, archive_folder, registered): + assert isinstance(archive, archiving.COCODataArchive) + assert sorted(archive) == ["2020/SMALL_alg.tgz", "2021/OLD_alg.tgz", "2021/SMALL2.tgz"] + assert registered == [str(archive_folder)] + assert (archive_folder / archiving.default_definition_filename).exists() + assert archive.remote_data_path is None + assert archive.local_data_path == str(archive_folder) + + def test_definition_file(self, archive, archive_folder): + defs = archiving.read_definition_file(str(archive_folder)) + assert sorted(d[0] for d in defs) == sorted(archive) + name, hash_, size = [d for d in defs if d[0] == "2020/SMALL_alg.tgz"][0] + assert hash_ == archiving._hash(str(archive_folder / "2020" / "SMALL_alg.tgz")) + assert isinstance(size, int) + + def test_get_returns_existing_archive(self, archive, archive_folder): + same = archiving.get(str(archive_folder)) + assert list(same) == list(archive) + assert archiving.COCODataArchive.is_archive(str(archive_folder)) + assert not archiving.COCODataArchive.is_archive(str(archive_folder / "2020")) + + def test_not_an_archive(self, tmp_path): + with pytest.raises(ValueError, match="seems not to"): + archiving.COCODataArchive(str(tmp_path)) + with pytest.raises(ValueError): + archiving.get(None) + + def test_recreate_makes_backup(self, archive, archive_folder, registered): + archiving.create(str(archive_folder)) + backups = [f for f in os.listdir(archive_folder) if f.startswith("._" + archiving.default_definition_filename)] + assert len(backups) == 1 + + def test_empty_archive(self, tmp_path, registered): + with pytest.warns(UserWarning, match="no data found"): + assert archiving.create(str(tmp_path)) is None + + def test_unusual_file_warns(self, tmp_path, registered): + (tmp_path / "data.abc").write_text("x") + with pytest.warns(UserWarning, match="unusual file"): + arch = archiving.create(str(tmp_path)) + assert list(arch) == ["data.abc"] + + +class TestFindAndGet: + def test_find(self, archive): + assert archive.find("small") == ["2020/SMALL_alg.tgz", "2021/SMALL2.tgz"] + assert archive.find("2021", "small") == ["2021/SMALL2.tgz"] + assert archive("old") == ["2021/OLD_alg.tgz"] + assert archive.found == ["2021/OLD_alg.tgz"] + + def test_get(self, archive, archive_folder): + path = archive.get("OLD") + assert path == str(archive_folder / "2021" / "OLD_alg.tgz") + assert archive.get(0) == archive.full_path(archive[0]) + archive.find("OLD") + assert archive.get() == path # first of last found + assert archive.contains(path) and archive.contains("2021/OLD_alg.tgz") + assert not archive.contains("nothing") + + def test_get_ambiguous_or_missing(self, archive): + with pytest.raises(ValueError, match="multiple matches"): + archive.get("SMALL") + with pytest.raises(ValueError, match="no match"): + archive.get("nonexisting") + with pytest.raises(ValueError): + archive.get("OLD", remote="yes") + + def test_get_all_first_and_extended(self, archive, archive_folder): + assert len(archive.get_all("small")) == 2 + assert archive.get_first(["small"]) == str(archive_folder / "2020" / "SMALL_alg.tgz") + assert archive.get_first(["nothing"]) is None + archive.find("OLD") + assert archive.get_found() == [str(archive_folder / "2021" / "OLD_alg.tgz")] + assert archive.get_extended(["small!"]) == [str(archive_folder / "2020" / "SMALL_alg.tgz")] + assert len(archive.get_extended("small*")) == 2 + assert archive.get_extended(["20.*SMALL"]) == archive.get_all("small") + assert archive.get_extended([str(archive_folder)]) == [str(archive_folder)] # existing path + with pytest.raises(ValueError): + archive.get_extended(["nonexisting"], remote=False) + + def test_downloaded_and_hash_check(self, archive, archive_folder): + assert sorted(archive.downloaded) == sorted(archive) + assert archive.consistency_check_data() == (3, 3) + path = archive_folder / "2021" / "OLD_alg.tgz" + with open(path, "ab") as f: + f.write(b"corrupt") + with pytest.raises(ValueError, match="wrong checksum"): + archive.get("OLD") + + def test_missing_file_with_remote_false(self, archive, archive_folder): + os.remove(archive_folder / "2021" / "OLD_alg.tgz") + assert archive.get("OLD", remote=False) == "" + + def test_get_found_with_several_matches(self, archive): + archive.find("small") + assert len(archive.get_found()) == 2 + + def test_update_without_url_warns(self, archive): + with pytest.warns(UserWarning, match="has no remote URL"): + archive.update() + + +class TestHelpers: + def test_url_helpers(self): + assert archiving._is_url("https://x") and archiving._is_url("HTTP://x") and not archiving._is_url("/x") + folder = archiving._url_to_folder_name("https://numbbo.github.io/data-archive/data-archive/bbob") + assert folder == os.path.join(archiving.default_archive_location, "da", "bbob") + + def test_str_to_list(self): + assert archiving._str_to_list("a") == ["a"] + assert archiving._str_to_list(("a",)) == ("a",) + with pytest.raises(ValueError): + archiving._str_to_list(3) + + def test_definition_file_names(self, tmp_path): + assert archiving._definition_file_to_read(str(tmp_path)) == str(tmp_path / archiving.default_definition_filename) + assert archiving._definition_file_to_write(str(tmp_path)) == str(tmp_path / archiving.default_definition_filename) + assert archiving._definition_file_to_write(str(tmp_path), "x.txt") == str(tmp_path / "x.txt") + with pytest.warns(UserWarning): + archiving._definition_file_to_write(str(tmp_path / "other.txt")) + + def test_repr_definitions_is_sorted(self): + s = archiving._repr_definitions([("b/x", "h", 1), ("a/y", "h", 2)]) + assert s.index("a/y") < s.index("b/x") diff --git a/tests/unit/test_cococommands_and_html.py b/tests/unit/test_cococommands_and_html.py new file mode 100644 index 0000000..a44347f --- /dev/null +++ b/tests/unit/test_cococommands_and_html.py @@ -0,0 +1,94 @@ +"""Unit tests for `cocopp.cococommands` (``load``, ``load2``, +``DataWithFewSuccesses``) and the html/LaTeX description helpers +`cocopp.htmldesc` and `cocopp.preparetexforhtml`. +""" + +import numpy as np +import pytest + +import cocopp +from cocopp import cococommands, htmldesc, preparetexforhtml, pproc, testbedsettings + +from conftest import load + + +class TestDataWithFewSuccesses: + @pytest.fixture + def few(self, bbob_small, quiet): + testbedsettings.reset_current_testbed() + return cococommands.DataWithFewSuccesses(bbob_small, minsuccesses=9) + + def test_attributes(self, few, bbob_small): + dsl = load(bbob_small) + assert few.trials == [15] * 8 + assert few.successes == [ds.detSuccesses([1e-8])[0] for ds in dsl] + assert few.evaluations == pytest.approx([np.sum(ds.maxevals) / 15 for ds in dsl]) + assert few.input_parameters["minsuccesses"] == 9 + + def test_all_and_budget_left(self, few): + expected = {(f, d): v for (f, d), v in few.all.items()} + assert all(v[1] < 9 for v in expected.values()) + assert all(v[2] == 15 for v in expected.values()) + assert list(few.all) == sorted(few.all) + assert few.with_budget_left == few.all # budget multiplier is inf + few.budget_multiplier = 0 + assert few.with_budget_left == {} + assert few.current == few.all + + def test_minsuccesses_and_threshold_changes(self, few): + n = len(few.all) + few.minsuccesses = 16 + assert len(few.all) == 8 + few.minsuccesses = 9 + few.compute_successes(1e2) # easy target: everything solved + assert len(few.all) == 0 < n or n == 0 + + def test_from_datasetlist(self, bbob_small, quiet): + dsl = load(bbob_small) + few = cococommands.DataWithFewSuccesses(dsl, minsuccesses=16, exclude=[(1, 2)]) + assert (1, 2) not in few.all and len(few.all) == 7 + + @pytest.mark.xfail(reason="known bug: print and __len__ use a non-existing `result` attribute", raises=AttributeError, strict=True) + def test_len(self, few): + len(few) + + +def test_true_number_of_trials(dsl_small): + assert cococommands.true_number_of_trials(dsl_small[0]) == 15 + + +class TestLoad2: + def test_single_algorithm(self, bbob_small, quiet): + testbedsettings.reset_current_testbed() + dsl = cocopp.load2(bbob_small) + assert isinstance(dsl, pproc.DataSetList) and len(dsl) == 8 + + def test_filters(self, bbob_small, quiet): + testbedsettings.reset_current_testbed() + dsl = cocopp.load2(bbob_small, keep=lambda ds: ds.funcId < 10, remove=lambda ds: ds.funcId == 2) + assert sorted({ds.funcId for ds in dsl}) == [1, 8] + + def test_several_algorithms(self, bbob_small, bbob_small_b, quiet): + testbedsettings.reset_current_testbed() + res = cocopp.load2([bbob_small, bbob_small_b]) + assert sorted(res) == [2, 5] + assert list(res[5]) == [bbob_small, bbob_small_b] + assert len(cocopp.load2([bbob_small, bbob_small_b], flat_list=True)) == 16 + + +class TestHtmlDescriptions: + def test_get_value(self): + text = htmldesc.getValue("##bbobppfigdimlegendrlbased##") + assert text and "##" not in text + assert htmldesc.getValue("##not-a-key##") == "" + assert len(htmldesc.descriptions) == 102 + + def test_prepare_tex_for_html(self, tmp_path, quiet): + out = tmp_path / "latex_commands_for_html.tex" + preparetexforhtml.main(str(out)) + text = out.read_text() + assert text.count("\\begin{document}") == 1 and text.rstrip().endswith("\\end{document}") + # every scenario gets the same set of caption commands, each preceded by its html key + for scenario in ["fixed", "rlbased", "biobjfixed", "bbobboxedfixed"]: + assert "\\providecommand{\\bbobppfigdimlegend%s}" % scenario in text + assert "\\#\\#bbobppfigdimlegend%s\\#\\#" % scenario in text diff --git a/tests/unit/test_package.py b/tests/unit/test_package.py new file mode 100644 index 0000000..9fb53de --- /dev/null +++ b/tests/unit/test_package.py @@ -0,0 +1,96 @@ +"""Tests of the package-level interface of `cocopp`. + +These pin the public API so that a restructuring does not silently drop +names that users (and notebooks) rely upon. +""" + +import importlib +import pkgutil + +import pytest + +import cocopp + +ALL_MODULES = sorted( + name + for _, name, _ in pkgutil.walk_packages(cocopp.__path__, "cocopp.") + # __main__ runs the CLI, the other two are scripts needing the network or local data + if name not in ("cocopp.__main__", "cocopp.firstsession", "cocopp.pprldistr2009_hardestRLB") +) + + +def test_module_list_is_complete(): + # if this fails, a module was added or removed: update the tests accordingly + assert len(ALL_MODULES) == 46 + for name in ["cocopp.pproc", "cocopp.compall.pprldmany", "cocopp.comp2.ppscatter", "cocopp.rungeneric"]: + assert name in ALL_MODULES + + +@pytest.mark.parametrize("name", ALL_MODULES) +def test_all_modules_import(name, quiet): + importlib.import_module(name) + + +def test_interface(): + assert sorted(cocopp.Interface.dir()) == ["archives", "config", "genericsettings", "load", "main", "remote_archives"] + assert all(hasattr(cocopp, name) for name in cocopp.Interface.dir()) + assert cocopp.Interface.main is cocopp.main + assert cocopp.load is cocopp.cococommands.load + + +def test_public_names(): + for name in ["set_seed", "archives", "data_archive", "archiving", "config", "genericsettings", "load", "load2", "info", "systeminfo", "pproc", "testbedsettings", "toolsdivers", "toolsstats", "bestalg", "__version__"]: + assert hasattr(cocopp, name), name + assert cocopp.archives is cocopp.archiving.official_archives + assert cocopp.data_archive is cocopp.archives.all + + +def test_version(): + assert cocopp.__version__ == cocopp._version.__version__ + assert cocopp.__version__[0].isdigit() + + +def test_set_seed_is_numpy_seed(): + import numpy as np + + cocopp.set_seed(3) + a = np.random.rand() + np.random.seed(3) + assert a == np.random.rand() + + +@pytest.fixture +def no_sys_argv(monkeypatch): + """`cocopp.main` falls back to ``sys.argv[1:]``, which hold the pytest arguments""" + monkeypatch.setattr("sys.argv", ["cocopp"]) + + +def test_main_without_arguments_prints_usage(capsys, no_sys_argv): + cocopp.main([]) + assert "Main routine for post-processing data from COCO" in capsys.readouterr().out + + +def test_main_help(capsys, no_sys_argv): + assert cocopp.main("-h") is None + assert "-h, --help" in capsys.readouterr().out + + +def test_main_unknown_option(capsys, no_sys_argv): + assert cocopp.main("--no-such-option x") == 2 + assert "For help use -h or --help" in capsys.readouterr().err + + +def test_rungeneric_main_unknown_option(capsys): + assert cocopp.rungeneric.main("--no-such-option x") == 2 + assert "For help use -h or --help" in capsys.readouterr().err + + +def test_rungeneric_main_help(capsys): + assert cocopp.rungeneric.main("-h") is None + assert "--output-dir" in capsys.readouterr().out + + +def test_systeminfo(capsys): + cocopp.systeminfo() + out = capsys.readouterr().out + assert "Numpy" in out and "Matplotlib" in out and "cocopp " + cocopp.__version__ in out diff --git a/tests/unit/test_pproc_dataset.py b/tests/unit/test_pproc_dataset.py new file mode 100644 index 0000000..0bb8172 --- /dev/null +++ b/tests/unit/test_pproc_dataset.py @@ -0,0 +1,362 @@ +"""Unit tests of `cocopp.pproc.DataSet` on a tiny hand-crafted data set. + +The data (see `synthdata.TINY_TRIALS`) consist of three trials on f1 in +2-D with the following best-so-far Delta f values:: + + instance 1: eval 1 -> 95, eval 10 -> 0.9, eval 50 -> 5e-9 (solved) + instance 2: eval 1 -> 45, eval 20 -> 4.5, eval 100 -> 0.45 (not solved) + instance 3: eval 1 -> 9.5, eval 5 -> 9e-4, eval 30 -> 9e-10 (solved) + +hence all expected values below can be (and were) computed by hand. +""" + +import os +import pickle + +import numpy as np +import pytest + +import cocopp +from cocopp import genericsettings, pproc, testbedsettings + +import synthdata + +nan = np.nan +TARGETS = [100, 10, 1, 0.1, 1e-3, 1e-8] +EXPECTED_EVALS = [[1, 1, 1], [10, 20, 1], [10, 100, 5], [50, nan, 5], [50, nan, 5], [50, nan, 30]] +EXPECTED_ERT = [1, 31 / 3, 115 / 3, 155 / 2, 155 / 2, 180 / 2] +EXPECTED_SUCCESSES = [3, 3, 3, 2, 2, 2] + + +def assert_nan_equal(a, b): + np.testing.assert_array_equal(np.asarray(a, dtype=float), np.asarray(b, dtype=float)) + + +@pytest.fixture +def tiny(tmp_path, quiet): + testbedsettings.reset_current_testbed() + info = synthdata.write_tiny_bbob(str(tmp_path)) + dsl = pproc.DataSetList(info) + assert len(dsl) == 1 + return dsl[0] + + +def _load_tiny(tmp_path, name, trials, **kwargs): + testbedsettings.reset_current_testbed() + return pproc.DataSetList(synthdata.write_tiny_bbob(str(tmp_path), name, trials=trials, **kwargs))[0] + + +class TestReading: + def test_header_attributes(self, tiny): + assert (tiny.funcId, tiny.dim, tiny.algId) == (1, 2, "TINY") + assert tiny.precision == 1e-8 + assert tiny.suite_name == "bbob" + assert tiny.get_data_format() == "bbob-new2" + assert tiny.comment == "% TINY: hand-crafted test data" + assert not tiny.isBiobjective() + assert not tiny.has_constraints + assert repr(tiny) == "DataSet(TINY on f1 2-D)" + + def test_loading_sets_current_testbed(self, tiny): + assert isinstance(testbedsettings.current_testbed, testbedsettings.GECCOBBOBTestbed) + + def test_run_information_from_info_file(self, tiny): + assert tiny.instancenumbers == [1, 2, 3] + assert tiny.readmaxevals == [50, 100, 30] + # f - ftarget as written with one digit precision by `write_tiny_bbob` + assert tiny.readfinalFminusFtarget == [-5e-9, 0.45, -9.1e-9] + assert tiny.isFinalized == [True, True, True] + assert tiny.dataFiles == [os.path.join("data_f1", "bbobexp_f1_DIM2.dat")] + + def test_maxevals_and_finalfunvals(self, tiny): + assert list(tiny.maxevals) == [50, 100, 30] + assert tiny.max_eval == 100 == tiny.mMaxEvals() + assert list(tiny.finalfunvals) == pytest.approx([5e-9, 0.45, 9e-10]) + assert tiny.nbRuns() == tiny.nbRuns_raw == 3 + + def test_evals_first_row_and_targets(self, tiny): + assert list(tiny.evals[0, 1:]) == [1, 1, 1] + assert tiny.evals[0, 0] == pytest.approx(100) # 10**(i/5) just above 95 + assert all(np.diff(tiny.evals[:, 0]) < 0) + # targets are on the 10**(i/5) grid, the last is cut just below precision 1e-8 + grid = np.log10(tiny.evals[:-1, 0]) * 5 + assert grid == pytest.approx(np.round(grid)) + assert tiny.evals[-1, 0] == pytest.approx(1e-8 / 1.001) + assert all(tiny.evals[:, 0] == tiny.target) + + def test_funvals(self, tiny): + assert_nan_equal( + tiny.funvals, + [ + [1, 95, 45, 9.5], + [5, 95, 45, 9e-4], + [10, 0.9, 45, 9e-4], + [20, 0.9, 4.5, 9e-4], + [30, 0.9, 4.5, 9e-10], + [50, 5e-9, 4.5, 9e-10], + [100, 5e-9, 0.45, 9e-10], + ], + ) + + def test_consistency_check(self, tiny): + with pytest.warns(UserWarning, match="less than 15 instances"): + assert tiny.consistency_check() is False + + +class TestRuntimes: + def test_detEvals(self, tiny): + for res, expected in zip(tiny.detEvals(TARGETS), EXPECTED_EVALS): + assert_nan_equal(res, expected) + + def test_detEvals_order_follows_input(self, tiny): + rev = tiny.detEvals(TARGETS[::-1]) + for res, expected in zip(rev, EXPECTED_EVALS[::-1]): + assert_nan_equal(res, expected) + + def test_detEvals_too_difficult_target(self, tiny): + assert all(np.isnan(tiny.detEvals([1e-12])[0])) + + def test_detEvals_complement_unsuccessful(self, tiny): + assert list(tiny.detEvals([1e-8], complement_unsuccessful=-1)[0]) == [50, -100, 30] + assert list(tiny.detEvals([1e-8], complement_unsuccessful=lambda m: m + 1)[0]) == [50, 101, 30] + assert np.isnan(tiny.detEvals([1e-8])[0][1]) # data are not changed + + def test_detEvals_by_instance(self, tiny): + assert list(tiny.detEvals([10], instance=2)[0]) == [20] + res = tiny.detEvals_by_instance([10, 1]) + assert list(res) == [1, 2, 3] + assert [list(v[1]) for v in res.values()] == [[10], [100], [5]] + with pytest.raises(ValueError): + tiny.detEvals([10], instance=7) + + def test_detEvals_bootstrap_draws_from_data(self, tiny): + res = tiny.detEvals([1], bootstrap=True)[0] + assert len(res) == 3 and set(res) <= {10, 100, 5} + + def test_detEvals2_agrees(self, tiny): + for a, b in zip(tiny._detEvals2(TARGETS), tiny.detEvals(TARGETS)): + assert_nan_equal(a, b) + + def test_detERT_and_ert(self, tiny): + assert tiny.detERT(TARGETS) == pytest.approx(EXPECTED_ERT) + assert tiny.detERT([1e-12]) == [np.inf] + assert len(tiny.ert) == len(tiny.target) + assert tiny.ert[0] == 1 + + def test_detSuccesses(self, tiny): + assert tiny.detSuccesses(TARGETS) == EXPECTED_SUCCESSES + assert list(tiny.detSuccessRates(TARGETS)) == pytest.approx(np.array(EXPECTED_SUCCESSES) / 3) + assert tiny.successes_by_instance() == {1: 1, 2: 0, 3: 1} + + def test_detAverageEvals(self, tiny): + # sum(evals of successes + maxevals of failures) / nbRuns == ERT * success rate + expected = [1, 31 / 3, 115 / 3, 155 / 3, 155 / 3, 180 / 3] + assert list(tiny.detAverageEvals(TARGETS)) == pytest.approx(expected) + + def test_generateRLData(self, tiny): + res = tiny.generateRLData([10, 1e-8]) + assert list(res[10][1:]) == [10, 20, 1] + assert_nan_equal(res[1e-8][1:], [50, nan, 30]) + + def test_median_evals(self, tiny): + assert_nan_equal(tiny.median_evals([10, 1, 1e-8]), [10, 10, 50]) + assert tiny.median_evals()[0] == 1 + + def test_number_of_better_runs(self, tiny): + assert tiny._number_of_better_runs(1, 10) == 1.5 # 5 < 10, 10 == 10 counts 1/2 + assert tiny._number_of_better_runs([1, 1e-8], np.nan) == [3, 2] + + def test_info_str(self, tiny): + s = tiny.info_str() + assert "Algorithm: TINY" in s and "Number of trials: 3" in s + assert "min / max number of evals per trial: 30 / 100" in s + assert " 1.0e+01 | 0 0 5 10 10 | 5.2 3" in s + + +class TestInstances: + def test_instance_bookkeeping(self, tiny): + assert tiny.createDictInstance() == {1: [0], 2: [1], 3: [2]} + assert tiny.createDictInstanceCount() == {1: 1, 2: 1, 3: 1} + assert tiny.instance_index_lists() == {1: [0], 2: [1], 3: [2]} + assert tiny.trial_count_by_instance == {1: 1, 2: 1, 3: 1} + assert tiny.instance_multipliers == (1, 1, 1) + assert tiny.instancenumbers_balanced == [1, 2, 3] + assert not tiny._need_balancing + assert tiny._instance_repetitions == 0 + + def test_budget_effective_estimates(self, tiny): + assert tiny.budget_effective_estimates == {1: 50, 2: 100, 3: 30} + assert tiny._budget_estimates == {1: 50, 2: 100, 3: 30} + + def test_splitByTrials(self, tiny): + evals, funvals = tiny.splitByTrials() + assert set(evals) == {1, 2, 3} + assert evals[2].shape == (len(tiny.evals), 1, 1) + assert tiny.splitByTrials("evals").keys() == evals.keys() + assert tiny.splitByTrials("funvals")[3][0, 0, 0] == 9.5 + + def test_balancing_repeated_instances(self, tmp_path, quiet): + trials = [(1, synthdata.TINY_TRIALS[1]), (1, synthdata.TINY_TRIALS[2]), (2, synthdata.TINY_TRIALS[3])] + ds = _load_tiny(tmp_path, "REP", trials) + assert ds.instancenumbers == [1, 1, 2] + assert ds.instance_multipliers == (1, 1, 2) + assert ds.instancenumbers_balanced == (1, 1, 2, 2) + assert ds.nbRuns() == 4 and ds.nbRuns_raw == 3 + assert list(ds.maxevals) == [50, 100, 30, 30] + assert_nan_equal(ds.evals[-1, 1:], [50, nan, 30, 30]) + assert ds.detERT([1e-8]) == pytest.approx([(50 + 100 + 30 + 30) / 3]) + assert ds.detSuccesses([1e-8]) == [3] + assert ds.detSuccesses([1e-8], raw_values=True) == [2] + + def test_balancing_can_be_switched_off(self, tmp_path, quiet, monkeypatch): + trials = [(1, synthdata.TINY_TRIALS[1]), (1, synthdata.TINY_TRIALS[2]), (2, synthdata.TINY_TRIALS[3])] + ds = _load_tiny(tmp_path, "REP", trials) + monkeypatch.setattr(genericsettings, "balance_instances", False) + assert ds.nbRuns() == 3 + assert ds.detERT([1e-8]) == pytest.approx([180 / 2]) + + def test_evals_appended(self, tmp_path, quiet, monkeypatch): + trials = [(1, synthdata.TINY_TRIALS[2]), (1, synthdata.TINY_TRIALS[1]), (2, synthdata.TINY_TRIALS[3])] + ds = _load_tiny(tmp_path, "APP", trials) + assert not ds.evals_are_appended # too few remaining trials by default + monkeypatch.setattr(genericsettings, "appended_evals_minimal_trials", 2) + assert ds.evals_are_appended + app = ds.evals_appended + assert app.shape == (len(ds._evals), 3) + # instance 1: the failed first run (100 evals) is restarted with the second run + assert app[-1, 1] == 100 + 50 + + +class TestSimulatedRestarts: + def test_all_successful_target_reproduces_data(self, tiny): + sim = tiny.evals_with_simulated_restarts([1], samplesize=3)[0] + assert sorted(sim) == [5, 10, 100] + + def test_restarts_add_unsuccessful_runs(self, tiny): + sim = tiny.evals_with_simulated_restarts([1e-8], samplesize=300)[0] + assert len(sim) == 300 + # each simulated run = k * 100 (failed trial) + 50 or 30 (successful trial) + assert np.all(np.isin(sim % 100, [30, 50])) + + def test_default_samplesize(self, tiny): + sim = tiny.evals_with_simulated_restarts([1e-8])[0] + assert len(sim) == 45 # smallest multiple of nbRuns() >= 45 + + def test_unsolved_target_gives_nan(self, tiny): + assert all(np.isnan(tiny.evals_with_simulated_restarts([1e-12], samplesize=6)[0])) + + def test_within_instance_restarts(self, tiny): + sim = tiny.evals_with_simulated_restarts([1e-8], samplesize=30, instances_are_uniform=False)[0] + assert len(sim) == 30 + assert np.sum(np.isnan(sim)) == 10 # instance 2 never succeeds + assert set(sim[np.isfinite(sim)]) == {50, 30} # no restarts across instances + + def test_within_instance_restarts_singular_instances(self, tiny): + # with one trial per instance and default samplesize, the data are returned as is, + # with unsuccessful runs as nan + sim = tiny.evals_with_simulated_restarts([1e-8], instances_are_uniform=False)[0] + assert sim[0] == 50 and np.isnan(sim[1]) and sim[2] == 30 + + def test_percentiles(self, tiny): + res = tiny.percentiles([10, 1e-12], prctiles=50) + assert res[0] == 10 and np.isnan(res[1]) + assert len(tiny.percentiles([1], (10, 90))[0]) == 2 + + def test_bootstrap_sample_size(self, tiny): + assert tiny.bootstrap_sample_size(10) == 12 + assert tiny.bootstrap_sample_size(0) == 3 + + +class TestMisc: + def test_equality(self, tiny, tmp_path, quiet): + other = _load_tiny(tmp_path / "x", "TINY", synthdata.TINY_TRIALS) + assert tiny == other and not tiny != other + assert tiny._data_differ(other) == [] + other3 = _load_tiny(tmp_path / "y", "TINY", {1: synthdata.TINY_TRIALS[1], 2: synthdata.TINY_TRIALS[3]}) + assert tiny._data_differ(other3) # different data + other.algId = "OTHER" + assert tiny != other + + def test_pickle_roundtrip(self, tiny): + ds = pickle.loads(pickle.dumps(tiny)) + assert ds == tiny and not ds._data_differ(tiny) + assert ds.detERT(TARGETS) == pytest.approx(EXPECTED_ERT) + + def test_pickle_to_file(self, tiny, tmp_path): + tiny.pickle(str(tmp_path)) + assert os.path.exists(tmp_path / "ppdata_f001_02.pickle.gz") + + def test_argsort(self, tiny): + # columns sorted from best to worst: instance 3 (30 evals), 1 (50), 2 (failed) + assert list(tiny._argsort()) == [3, 1, 2] + + def test_largest_finite_index(self): + f = pproc.DataSet._largest_finite_index + assert f(np.array([1.0, 2, nan, nan])) == 1 + assert f(np.array([nan, nan])) == -1 + assert f(np.array([1.0, 2, 3])) == 2 + + def test_plot_smoke(self, tiny): + ax = tiny.plot() + assert "F1 in dimension 2" in ax.get_title() + with pytest.raises(ValueError): + tiny.plot(smallest_target=1e6) + + def test_number_of_constraints_not_available_for_bbob(self, tiny): + with pytest.raises(NotImplementedError): + tiny.number_of_constraints + + +def test_constrained_evaluations_are_summed(tmp_path, quiet): + trials = {1: [(1, 2, 95.0), (10, 30, 0.9), (50, 80, 5e-9)], 2: [(1, 2, 45.0), (20, 40, 4.5), (100, 150, 0.45)]} + ds = _load_tiny(tmp_path, "CONS", trials, suite="bbob-constrained") + assert testbedsettings.current_testbed.name == "bbob-constrained" + assert ds.has_constraints + assert ds.readmaxevals == [50, 100] # f-evaluations only + assert list(ds.maxevals) == [130, 250] # f + g from the last .tdat line + assert list(ds.evals[0, 1:]) == [3, 3] + assert list(ds.detEvals([1])[0]) == [40, 250] + assert list(ds.evals_function[0, 1:]) == [1, 1] + assert list(ds.evals_constraints[0, 1:]) == [2, 2] + assert ds.number_of_constraints == 1 + + +def test_constrained_evaluation_weights(tmp_path, quiet, monkeypatch): + trials = {1: [(1, 2, 95.0), (10, 30, 0.9), (50, 80, 5e-9)]} + monkeypatch.setattr(genericsettings, "weight_evaluations_constraints", [1, 0]) + ds = _load_tiny(tmp_path, "CONS0", trials, suite="bbob-constrained") + assert not ds.has_constraints # g-evaluations are ignored entirely + assert list(ds.detEvals([1])[0]) == [10] + + +def test_instances_of_interest_filter(tmp_path, quiet, monkeypatch): + monkeypatch.setitem(testbedsettings.GECCOBBOBTestbed.settings, "instancesOfInterest", [1, 3]) + ds = _load_tiny(tmp_path, "TINY", synthdata.TINY_TRIALS) + assert ds.instancenumbers == [1, 3] + assert_nan_equal(ds.detEvals([1e-8])[0], [50, 30]) + + +def test_old_data_format(bbob_old_format, quiet): + testbedsettings.reset_current_testbed() + dsl = pproc.DataSetList(bbob_old_format) + assert len(dsl) == 4 + ds = dsl[0] + assert ds.get_data_format() is None + assert ds.suite_name == "bbob" + assert ds.instancenumbers == list(range(1, 16)) + assert ds.evals.shape[1] == 16 + + +def test_suite_name_fallbacks(tiny): + del tiny.suite + assert tiny.suite_name == "bbob" + tiny.funcId = 101 + assert tiny.suite_name == "bbob-noisy" + tiny.suite = "sbox-cost" + assert tiny.suite_name == "bbob-boxed" + + +def test_cococommands_load(tiny, tmp_path, quiet): + info = synthdata.write_tiny_bbob(str(tmp_path / "load")) + dsl = cocopp.load(info) + assert len(dsl) == 1 and dsl[0] == tiny diff --git a/tests/unit/test_pproc_lists.py b/tests/unit/test_pproc_lists.py new file mode 100644 index 0000000..0f3c737 --- /dev/null +++ b/tests/unit/test_pproc_lists.py @@ -0,0 +1,419 @@ +"""Unit tests for `DataSetList`, the ``dictAlg*`` helpers, target value +classes and other module-level functions of `cocopp.pproc`, and for the +reference ("best") algorithm construction in `cocopp.bestalg`. +""" + +import os +import pickle + +import numpy as np +import pytest + +import cocopp +from cocopp import bestalg, pproc, testbedsettings + +import synthdata +from conftest import load + + +class TestParseInfo: + def test_parseinfo(self): + s = "suite = 'bbob', funcId = 1, DIM = 2, Precision = 1.000e-08, algId = 'A, B', coco_version = '2.6'" + assert pproc.parseinfo(s) == [ + ("suite", "bbob"), + ("funcId", 1), + ("DIM", 2), + ("Precision", 1e-08), + ("algId", "A, B"), + ("coco_version", "2.6"), + ] + + def test_parseinfo_quotes_inside_values(self): + assert pproc.parseinfo("algId = 'it's'") == [("algId", "it's")] + assert pproc.parseinfo('algId = "x"') == [("algId", "x")] + + def test_parseinfo_biobj_data_line(self): + res = dict(pproc.parseinfo("function = 2, dim = 5, f.dat, 1:500|6.1e-003")) + assert res == {"function": 2, "dim": 5} + + def test_parseinfoold(self): + assert pproc.parseinfoold("funcId = 1, algId = 'x'") == [("algId", "x"), ("funcId", 1)] + + +def test_align_list(): + assert pproc.align_list(["a", "b"], [1, 1, 2]) == ["a", "a", "b"] + + +def test_set_unique_algId(): + class D: + def __init__(self, a): + self.algId = a + + taken = [D("A"), D("B")] + new = [D("A"), D("C"), D("B")] + pproc.set_unique_algId(new, taken) + assert [d.algId for d in new] == ["A 2", "C", "B 2"] + new = [D("A")] + pproc.set_unique_algId(new, None, taken_ids=["A", "A 2"]) + assert new[0].algId == "A 3" + + +class TestTargetValues: + def test_target_values(self): + targets = [10**i for i in np.arange(2, -8.1, -0.2)] + tv = pproc.TargetValues(targets) + assert tv() == targets and tv((1, 20)) == targets + assert len(tv) == len(targets) + assert tv.short_info == "absolute targets" + assert tv.label_name() == "Df" + + def test_sorted_and_labels(self): + tv = pproc.TargetValues([1e-8, 10, 0.1]) + assert tv() == [10, 0.1, 1e-8] + assert tv.labels() == ["10", "0.1", "1e-8"] + assert tv.loglabels() == ["1", "-1", "-8"] + assert tv.label(0) == "10" and tv.loglabel(2) == "-8" + + def test_discretize(self): + tv = pproc.TargetValues([1.1e-3, 9e-2], discretize=True) + assert tv() == pytest.approx([10**-1, 10**-3]) # rounded to the 10**(i/5) grid + assert pproc.TargetValues([3.0])(discretize=True) == pytest.approx([10**0.4]) + + def test_cast_and_as_target_values(self): + tv = pproc.TargetValues([1, 2]) + assert pproc.TargetValues.cast(tv) is tv + assert pproc.TargetValues.cast([1, 2])() == [2, 1] + assert pproc.asTargetValues(tv) is tv + assert pproc.asTargetValues([1])() == [1] + rl = pproc.RunlengthBasedTargetValues([1, 10]) + assert pproc.asTargetValues(rl) is rl + with pytest.raises(NotImplementedError): + pproc.asTargetValues("x") + + +class TestRunlengthBasedTargetValues: + @pytest.fixture + def reference(self, bbob_alg_a, bbob_alg_b, quiet): + testbedsettings.reset_current_testbed() + dsl, algs, dict_alg = pproc.processInputArgs([bbob_alg_a, bbob_alg_b]) + return bestalg.generate(dict_alg, "REF") + + def test_basic_attributes(self): + rl = pproc.RunlengthBasedTargetValues([10, 0.5, 3]) + assert rl.run_lengths == [0.5, 3, 10] + assert len(rl) == 3 + assert rl.labels() == ["0.5", "3", "10"] + assert rl.loglabel(0) == "-0.30" + assert rl.label_name() == "RL/dim" + assert pproc.RunlengthBasedTargetValues([1], times_dimension=False).label_name() == "RL" + assert pproc.RunlengthBasedTargetValues.cast(rl) is rl + assert pproc.RunlengthBasedTargetValues.cast([1, 2]).run_lengths == [1, 2] + + def test_requires_fun_dim(self, reference): + with pytest.raises(ValueError): + pproc.RunlengthBasedTargetValues([1], reference_data=reference)() + + def test_targets_from_reference_data(self, reference): + run_lengths = [0.5, 1.2, 3, 10, 50] + rl = pproc.RunlengthBasedTargetValues(run_lengths, reference_data=reference, force_different_targets_factor=1) + for fun_dim in [(1, 5), (6, 2), (21, 20)]: + targets = rl(fun_dim) + ref = reference[fun_dim[::-1]] + assert len(targets) == len(run_lengths) + assert all(np.diff(targets) <= 0) and targets[-1] >= 1e-8 + # recompute: the best target reached by the reference within rl * dim, divided by 10**0.2 + for t, r in zip(targets, run_lengths): + idx = np.nonzero(ref.ert[ref.target >= 1e-8] <= max(1, r * fun_dim[1]))[0] + expected = ref.target[0] if not len(idx) else max(1e-8, (1 + 1e-9) * ref.target[idx[-1]] / 10**0.2) + assert t == pytest.approx(expected) + assert rl.reference_algorithm == "REF" + + def test_forced_different_targets(self, reference): + rl = pproc.RunlengthBasedTargetValues([1, 1.01, 1.02], reference_data=reference) + targets = rl((1, 5)) + assert targets[1] == pytest.approx(targets[0] / 10**0.04) + + def test_unique_and_discretized_targets(self, reference): + rl = pproc.RunlengthBasedTargetValues([1, 1.01, 1.02, 1e9], reference_data=reference, force_different_targets_factor=1, unique_target_values=True) + targets = rl((1, 5)) + assert len(targets) == len(set(targets)) < 4 + disc = pproc.RunlengthBasedTargetValues([1, 10], reference_data=reference)((1, 5), discretize=True) + assert np.log10(disc) * 5 == pytest.approx(np.round(np.log10(disc) * 5)) + + def test_shipped_reference_algorithm(self, quiet): + cocopp.config.config("bbob") + rl = pproc.RunlengthBasedTargetValues([0.5, 1.2, 3, 10, 50]) + t = rl(fun_dim=(1, 20)) + assert 6.30957345e01 <= t[0] <= 6.30957346e01 # from the package doctest + assert t[-1] == 1e-8 + assert rl.reference_algorithm == "refalgs/best2009-bbob.tar.gz" + + +class TestDataSetList: + def test_load_folder(self, dsl_small): + assert len(dsl_small) == 8 + assert isinstance(dsl_small, pproc.DataSetList) + # sorted by dimension, then function + assert [(ds.dim, ds.funcId) for ds in dsl_small] == [(d, f) for d in (2, 5) for f in (1, 2, 8, 15)] + assert all(ds.algId == "SMALL" for ds in dsl_small) + assert dsl_small.current_testbed is testbedsettings.current_testbed + + def test_from_datasets_and_empty(self, dsl_small): + assert pproc.DataSetList() == [] + assert pproc.DataSetList(list(dsl_small)) == dsl_small + + def test_append_rejects_non_datasets(self, dsl_small): + dsl = pproc.DataSetList() + with pytest.warns(UserWarning): + dsl.append(3) + with pytest.raises(ValueError): + dsl.append(3, check_data_type=True) + + def test_append_same_dataset_twice_is_skipped(self, dsl_small): + with pytest.warns(UserWarning, match="found twice"): + dsl_small.append(pickle.loads(pickle.dumps(dsl_small[0]))) + assert len(dsl_small) == 8 + + def test_append_merges_trials(self, tmp_path, quiet): + """two index entries of the same algorithm/function/dimension are merged""" + a = synthdata.write_tiny_bbob(str(tmp_path / "a"), "M", trials={1: synthdata.TINY_TRIALS[1]}) + b = synthdata.write_tiny_bbob(str(tmp_path / "b"), "M", trials={2: synthdata.TINY_TRIALS[2], 3: synthdata.TINY_TRIALS[3]}) + testbedsettings.reset_current_testbed() + dsl = pproc.DataSetList([a, b]) + assert len(dsl) == 1 + ds = dsl[0] + assert ds.instancenumbers == [1, 2, 3] + assert list(ds.maxevals) == [50, 100, 30] + assert ds.detERT([1, 1e-8]) == pytest.approx([115 / 3, 90]) + assert ds.indexFiles == [a, b] + + def test_append_merges_file_lists_once(self, tmp_path, quiet): + a = synthdata.write_tiny_bbob(str(tmp_path / "a"), "M", trials={1: synthdata.TINY_TRIALS[1]}) + b = synthdata.write_tiny_bbob(str(tmp_path / "b"), "M", trials={2: synthdata.TINY_TRIALS[2]}) + testbedsettings.reset_current_testbed() + ds = pproc.DataSetList([a, b])[0] + assert ds.indexFiles == [a, b] + assert len(ds.dataFiles) == 2 + + def test_missing_input_warns(self, tmp_path, capsys): + with pytest.warns(UserWarning, match="not found"): + dsl = pproc.DataSetList([str(tmp_path / "nothing.xyz")]) + assert len(dsl) == 0 + + def test_dict_by(self, dsl_small): + assert sorted(dsl_small.dictByDim()) == [2, 5] + assert sorted(dsl_small.dictByFunc()) == [1, 2, 8, 15] + assert list(dsl_small.dictByAlg()) == [("SMALL", "")] + assert dsl_small.by("dim").keys() == dsl_small.dictByDim().keys() + assert sorted(dsl_small.dictByDimFunc()[5]) == [1, 2, 8, 15] + assert list(dsl_small.dictByNoise()) == ["noiselessall"] + assert sorted(dsl_small.dictByParam("dim")) == [2, 5] + assert list(dsl_small.dictByParam("nonexisting")) == [None] + + def test_function_groups(self, dsl_small): + groups = dsl_small.dictByFuncGroup() + assert {k: sorted({ds.funcId for ds in v}) for k, v in groups.items()} == {"separ": [1, 2], "lcond": [8], "multi": [15]} + assert list(dsl_small.getFuncGroups().items()) == [ + ("separ", "Separable functions"), + ("lcond", "Misc. moderate functions"), + ("multi", "Multi-modal functions"), + ] + assert not dsl_small.isBiobjective() + + def test_filter_and_filtered(self, dsl_small): + unimodal = dsl_small.filtered(lambda ds: ds.funcId < 10) + assert isinstance(unimodal, pproc.DataSetList) and len(unimodal) == 6 + assert dsl_small.filtered(None) == [] and len(dsl_small.filtered(True, list)) == 8 + removed = dsl_small.filter(lambda ds: ds.dim == 2, verbose=2) + assert len(removed) == 4 and len(dsl_small) == 4 + dsl_small.remove_if(lambda ds: ds.funcId == 15) + assert sorted(ds.funcId for ds in dsl_small) == [1, 2, 8] + assert dsl_small.filter(None) is None and len(dsl_small) == 3 + + def test_sort(self, dsl_small): + dsl_small.sort(key1="funcId", key2="dim") + assert [(ds.funcId, ds.dim) for ds in dsl_small][:3] == [(1, 2), (1, 5), (2, 2)] + + def test_info(self, dsl_small, capsys): + dsl_small.info("all") + out = capsys.readouterr().out + assert "8 data set(s)" in out and "Algorithm(s): SMALL" in out + assert "4 Functions with IDs 1, 2, 8, 15" in out + assert "Dimension(s): 2, 5" in out + assert "Max evals: [400, 1000]" in out + pproc.DataSetList().info() + assert capsys.readouterr().out == "[]\n" + + def test_get_all_data_lines_and_best_lines(self, dsl_small): + lines, scores = dsl_small.get_all_data_lines(1e-1, 1, 5) + assert len(lines) == 1 and scores == dsl_small[4].detERT([1e-1]) + with pytest.warns(UserWarning): + best, best_scores = dsl_small.det_best_data_lines([10, 1e-1], 1, 5) + assert list(best_scores) == dsl_small[4].detERT([10, 1e-1]) + + @pytest.mark.xfail(reason="known bug: np.sort on dict_values in det_best_data", strict=True) + def test_det_best_data(self, dsl_small): + dsl_small.det_best_data([10, 1e-1], 1, 5) + + @pytest.mark.xfail(reason="known bug: dict.values()[0] in run_length_distributions (py2 code)", raises=TypeError, strict=True) + def test_run_length_distributions_single_alg(self, dsl_small): + dsl_small.run_length_distributions(5, [10, 1e-1]) + + def test_run_length_distributions(self, dsl_small): + rld, left = dsl_small.run_length_distributions(5, [10, 1e-1], flatten_output_dict=False) + data, solved, processed = rld[("SMALL", "")] + assert processed == [1, 2, 8, 15] + assert len(data) == 4 * 2 * 15 + assert np.all(np.diff(data[np.isfinite(data)]) >= 0) + # the left envelope of a single algorithm is its data, with inf instead of nan + np.testing.assert_array_equal(left, np.where(np.isnan(data), np.inf, data)) + + def test_reference_values_hash(self, dsl_small, biobj_alg, quiet): + assert dsl_small.get_reference_values_hash() is None + testbedsettings.reset_current_testbed() + dsl = load(biobj_alg) + h = dsl.get_reference_values_hash() + assert isinstance(h, str) and len(h) == 16 + assert h == load(biobj_alg).get_reference_values_hash() + + def test_is_same_list_assert(self, dsl_small, bbob_small): + dsl_small._is_same_list_assert(load(bbob_small)) + + +class TestDictAlg: + @pytest.fixture + def dict_alg(self, bbob_alg_a, bbob_alg_b, quiet): + testbedsettings.reset_current_testbed() + dsl, algs, dict_alg = pproc.processInputArgs([bbob_alg_a, bbob_alg_b]) + return dsl, algs, dict_alg + + def test_processInputArgs(self, dict_alg, bbob_alg_a, bbob_alg_b): + dsl, algs, d = dict_alg + assert algs == [bbob_alg_a, bbob_alg_b] + assert list(d) == algs + assert len(dsl) == 240 + assert all(ds._data_folder == bbob_alg_a for ds in d[bbob_alg_a]) + assert set(testbedsettings.reference_values) == {"SYNTH-A", "SYNTH-B"} + + def test_processInputArgs_errors(self, tmp_path): + with pytest.raises(Exception, match="could not be found"): + pproc.processInputArgs([str(tmp_path / "missing")]) + single = tmp_path / "a.info" + single.write_text("") + with pytest.warns(UserWarning, match="single file"): + assert pproc.processInputArgs([str(single)])[0] == [] + + def test_same_algorithm_twice_gets_unique_names(self, bbob_small, quiet): + testbedsettings.reset_current_testbed() + dsl, algs, d = pproc.processInputArgs([bbob_small, bbob_small + os.sep]) + assert algs == [bbob_small] # trailing separators are stripped + assert {ds.algId for ds in dsl} == {"SMALL", "SMALL 2"} + + def test_by_dim(self, dict_alg): + _, algs, d = dict_alg + by_dim = pproc.dictAlgByDim(d) + assert sorted(by_dim) == [2, 3, 5, 10, 20] + assert list(by_dim[5]) == algs and len(by_dim[5][algs[0]]) == 24 + assert pproc.DictAlg(d).by_dim().keys() == by_dim.keys() + by_dim2 = pproc.dictAlgByDim2(d) + assert {k: {a: len(v) for a, v in x.items()} for k, x in by_dim2.items()} == { + k: {a: len(v) for a, v in x.items()} for k, x in by_dim.items() + } + + def test_by_fun_noise_group(self, dict_alg): + _, algs, d = dict_alg + assert sorted(pproc.dictAlgByFun(d)) == list(range(1, 25)) + assert list(pproc.dictAlgByNoi(d)) == ["noiselessall"] + groups = pproc.dictAlgByFuncGroup(d) + assert sorted(groups) == ["hcond", "lcond", "mult2", "multi", "separ"] + assert len(groups["separ"][algs[1]]) == 5 * 5 + + def test_missing_dimension_is_filled_with_empty_list(self, bbob_small, bbob_alg_a, quiet): + testbedsettings.reset_current_testbed() + _, algs, d = pproc.processInputArgs([bbob_small, bbob_alg_a]) + by_dim = pproc.dictAlgByDim(d) + assert len(by_dim[20][bbob_small]) == 0 and len(by_dim[20][bbob_alg_a]) == 24 + + +class TestGetDataSetList: + def test_pickle_cache_for_archives(self, tmp_path, bbob_small, quiet, monkeypatch): + import tarfile + + archive = str(tmp_path / "SMALL.tgz") + with tarfile.open(archive, "w:gz") as tar: + tar.add(bbob_small, arcname="SMALL") + monkeypatch.chdir(tmp_path) + testbedsettings.reset_current_testbed() + dsl1 = pproc.get_DataSetList(archive) + assert pproc._pickle_file_read == "" + assert os.path.exists(archive + ".pickle") + assert os.path.isdir(tmp_path / ".extracted_SMALL") + dsl2 = pproc.get_DataSetList(archive) + assert pproc._pickle_file_read == archive + ".pickle" + dsl2._is_same_list_assert(dsl1) + monkeypatch.setattr(pproc, "_ignore_pickle_files", True) + dsl3 = pproc.get_DataSetList(archive) + assert pproc._pickle_file_read == "" + dsl3._is_same_list_assert(dsl1) + + def test_folders_are_not_pickled(self, bbob_small, quiet): + testbedsettings.reset_current_testbed() + dsl = pproc.get_DataSetList(bbob_small) + assert len(dsl) == 8 + assert not os.path.exists(bbob_small + ".pickle") + + +class TestBestAlg: + @pytest.fixture + def algs(self, bbob_alg_a, bbob_alg_b, quiet): + testbedsettings.reset_current_testbed() + return pproc.processInputArgs([bbob_alg_a, bbob_alg_b]) + + def test_best_ert_is_minimum_of_algorithm_erts(self, algs): + dsl, names, dict_alg = algs + best = bestalg.generate(dict_alg, "BEST") + assert len(best) == 24 * 5 + targets = [100, 10, 1, 1e-1, 1e-3, 1e-5, 1e-7, 1e-8] + for (dim, fun), ba in list(best.items())[::7]: + erts = [pproc.DataSetList(dict_alg[a]).dictByDimFunc()[dim][fun][0].detERT(targets) for a in names] + assert ba.detERT(targets) == pytest.approx(np.min(erts, axis=0)) + assert (ba.dim, ba.funcId, ba.algId) == (dim, fun, "BEST") + assert set(ba.best_algorithm_data) <= set(names) + assert repr(ba) == "{alg: BEST, F%d, dim: %d}" % (fun, dim) + assert ba.comment == "Combination of " + ", ".join(names) + + def test_best_detEvals_comes_from_best_algorithm(self, algs): + dsl, names, dict_alg = algs + ba = bestalg.generate(dict_alg, "BEST")[(5, 1)] + evals, alg_names, _ = ba.detEvals([1e-1, 1e-12]) + source = pproc.DataSetList(dict_alg[alg_names[0]]).dictByDimFunc()[5][1][0] + np.testing.assert_array_equal(evals[0], source.detEvals([1e-1])[0]) + assert alg_names[1] is None and np.all(np.isnan(evals[1])) + assert ba.get_success_ratio(1e-1) == (np.sum(np.isfinite(evals[0])), 15) + + def test_custom_generate_roundtrip(self, algs, tmp_path, monkeypatch, bbob_alg_a, bbob_alg_b): + monkeypatch.chdir(tmp_path) + bestalg.custom_generate([bbob_alg_a, bbob_alg_b], "myref") + assert os.path.exists(tmp_path / "myref.tar.gz") + assert os.path.exists(tmp_path / "myref" / "bbob-bestalg.info") + testbedsettings.reset_current_testbed() + entries = bestalg.load_reference_algorithm(str(tmp_path / "myref"), force=True, relative_load=False) + direct = bestalg.generate(algs[2], "BEST") + targets = [10, 1e-1, 1e-8] + for key in [(2, 1), (5, 8), (20, 24)]: + # the written ERTs are rounded to integers + assert entries[key].detERT(targets) == pytest.approx(direct[key].detERT(targets), abs=0.5) + + def test_load_reference_algorithm(self, quiet): + bestalg.reset_reference_algorithm() + assert bestalg.bestAlgorithmEntries == {} + cocopp.config.config("bbob") + entries = bestalg.load_reference_algorithm("refalgs/best2009-bbob.tar.gz") + assert len(entries) == 24 * 6 + assert bestalg.load_reference_algorithm("anything") is entries # cached unless forced + assert bestalg.load_reference_algorithm("", force=True) is None + + def test_get_used_instance_list(self): + assert bestalg.get_used_instance_list([[1, 2], [1, 2]]) == "1,2" + assert bestalg.get_used_instance_list([[1, 2], [3]]) == "[1,2],[3]" + assert bestalg.get_used_instance_list([]) is None diff --git a/tests/unit/test_pptex.py b/tests/unit/test_pptex.py new file mode 100644 index 0000000..cb6eead --- /dev/null +++ b/tests/unit/test_pptex.py @@ -0,0 +1,86 @@ +"""Unit tests for the LaTeX/number formatting helpers in `cocopp.pptex`. + +Expected strings are characterization values of the current behavior; +they end up verbatim in the generated tables. +""" + +import numpy as np +import pytest + +from cocopp import pptex + +inf = np.inf + +# value: writeFEvals, writeFEvals2, writeFEvals2(.., 3), writeFEvals2(.., isscientific=True), +# writeFEvalsMaxSymbols(.., 3), writeFEvalsMaxSymbols(.., 5), +# writeFEvalsMaxPrec(.., 2), writeFEvalsMaxPrec(.., 3, maxfloatrepr=1e3) +FORMATS = [ + (1, ["1", "1", "1", "1.0e0", "1", "1", "1", "1"]), + (7, ["7", "7", "7", "7.0e0", "7", "7", "7.0", "7.00"]), + (12.5, ["12", "12", "12.5", "1.2e1", "12.5", "12.5", "12", "12.5"]), + (123.456, ["120", "120", "123", "1.2e2", "123", "123.46", "123", "123"]), + (1234.5, ["1200", "1200", "1230", "1.2e3", "1e3", "1234.5", "1234", "1.23e3"]), + (102345, ["1e5", "1.0e5", "1.02e5", "1.0e5", "1e5", "1.02e5", "1.0e5", "1.02e5"]), + (1e7, ["1e7", "1.0e7", "1.00e7", "1.0e7", "1e7", "1.00e7", "1.0e7", "1.00e7"]), + (0.5, ["0.5", "0.5", "0.5", "5.0e-1", "0.5", "0.5", "0.50", "0.500"]), + (0.0123, ["0.012", "0.012", "0.0123", "1.2e-2", "0.01", "0.0123", "0.01", "0.012"]), + (inf, 8 * [r"$\infty$"]), +] + + +@pytest.mark.parametrize("val, expected", FORMATS) +def test_number_formats(val, expected): + got = [ + pptex.writeFEvals(val), + pptex.writeFEvals2(val), + pptex.writeFEvals2(val, 3), + pptex.writeFEvals2(val, isscientific=True), + pptex.writeFEvalsMaxSymbols(val, 3), + pptex.writeFEvalsMaxSymbols(val, 5), + pptex.writeFEvalsMaxPrec(val, 2), + pptex.writeFEvalsMaxPrec(val, 3, maxfloatrepr=1e3), + ] + assert got == expected + + +@pytest.mark.parametrize("val, as_int, as_float", [(5, "5", "5"), (123456, "1.2e5", "123456"), (12345678901234, "1e13", "12345678901234")]) +def test_writeFEvals2_maxdigits(val, as_int, as_float): + assert pptex.writeFEvals2(val, maxdigits=5) == as_int + # quirk: maxdigits is only honored for `int` input + assert pptex.writeFEvals2(float(val), maxdigits=5) == as_float + + +@pytest.mark.parametrize("func", [pptex.writeFEvals2, lambda x: pptex.writeFEvalsMaxSymbols(x, 3), lambda x: pptex.writeFEvalsMaxPrec(x, 2)]) +def test_nan_is_rejected(func): + with pytest.raises(AssertionError): + func(np.nan) + + +def test_numtotext(): + assert [pptex.numtotext(n) for n in (0, 25, 26, 51, 52, 53, 104, 2755)] == ["A", "Z", "a", "z", "AA", "AB", "BA", "zz"] + with pytest.raises(Exception): + pptex.numtotext(2756) + + +def test_labels_colors_markers(): + assert pptex.writeLabels("a_b_c") == r"a\_b\_c" + assert pptex.color_to_latex("k") == r"\color{Black}" + assert pptex.color_to_latex("0.5") == r"\color[gray]{0.5}" + assert pptex.color_to_latex("#123456") == "{}" + assert pptex.marker_to_latex("o") == r"$\circ$" + assert pptex.marker_to_html("o") == "○" + + +def test_table_latex(): + table = [["a", "b"], ["c", "d"]] + # quirk: extra_eol is also appended to the last line, which has no \\ + assert pptex.tableLaTeX(table, spec="cc", extra_eol=[r"\hline"]) == "\\begin{tabular}{cc}\na & b\\\\\\hline\nc & d\\hline\n\\end{tabular}" + # without the end of the tabular, the last line is terminated with \\ + assert pptex.tableLaTeX(table, spec="cc", add_begin_tabular=False, add_end_tabular=False) == "a & b\\\\\nc & d\\\\" + assert pptex.tableXLaTeX(table, spec="cc") == "\\begin{tabularx}{1.0\\textwidth}{cc}\na & b\\\\\nc & d\n\\end{tabularx}" + assert pptex.tableLaTeXStar(table, width="5cm", spec="cc") == "\\begin{tabular*}{5cm}{cc}\na & b\\\\\nc & d\n\\end{tabular*}" + + +def test_wrong_input_size_error(): + # quirk: __str__ returns the repr of the message + assert str(pptex.WrongInputSizeError("arr", (2, 3), (3, 3))) == repr("The size of arr is (2, 3). One dimension must be of length (3, 3)!") diff --git a/tests/unit/test_readalign.py b/tests/unit/test_readalign.py new file mode 100644 index 0000000..ea67ea4 --- /dev/null +++ b/tests/unit/test_readalign.py @@ -0,0 +1,124 @@ +"""Unit tests for `cocopp.readalign` (reading and aligning raw data files).""" + +import numpy as np +import pytest + +from cocopp import dataformatsettings, readalign, testbedsettings, pproc + +nan = np.nan + + +@pytest.fixture +def bbob_format(monkeypatch): + """set the module globals `readalign` relies upon""" + monkeypatch.setattr(dataformatsettings, "current_data_format", dataformatsettings.BBOBNewDataFormat()) + monkeypatch.setattr(testbedsettings, "current_testbed", testbedsettings.GECCOBBOBTestbed(pproc.TargetValues)) + + +def _write(path, text): + path.write_text(text) + return str(path) + + +class TestSplit: + def test_split_trials(self, tmp_path): + fn = _write( + tmp_path / "a.dat", + "% instance = 1, reference value = 0.5\n1 0 10 0 0\n5 0 1 0 0\n" + "% instance = 2, reference value = 0.25\n1 0 Inf 0 0\n2 0 -inf nan NaN\n", + ) + data, algs, refs, succ = readalign.split([fn]) + assert len(data) == 2 + np.testing.assert_array_equal(data[0], [[1, 0, 10, 0, 0], [5, 0, 1, 0, 0]]) + assert data[1][0, 2] == np.inf and data[1][1, 2] == -np.inf and np.isnan(data[1][1, 3]) + assert refs == {1: 0.5, 2: 0.25} + assert algs == [] and succ == [] + + def test_split_with_index_filter(self, tmp_path): + fn = _write(tmp_path / "a.dat", "%\n1 0 1 0 0\n%\n1 0 2 0 0\n%\n1 0 3 0 0\n") + data = readalign.split([fn], idx_to_load=[True, False, True])[0] + assert [d[0, 2] for d in data] == [1, 3] + + def test_split_reads_several_files(self, tmp_path): + f1 = _write(tmp_path / "a.dat", "%\n1 0 1 0 0\n") + f2 = _write(tmp_path / "b.dat", "%\n1 0 2 0 0\n") + assert len(readalign.split([f1, f2])[0]) == 2 + + def test_split_best_algorithm_columns(self, tmp_path): + fn = _write(tmp_path / "best.dat", "% algorithm type = best\n1 0 1 0 0 ALG1 3 15\n2 0 0.5 0 0 ALG2 15 15\n") + data, algs, refs, succ = readalign.split([fn]) + assert algs == ["ALG1", "ALG2"] + assert succ == [[3, 15], [15, 15]] + assert data[0].shape == (2, 5) + + def test_split_invalid_number_and_incomplete_line(self, tmp_path): + # with `dim` given, lines must have dim + 5 entries + fn = _write(tmp_path / "a.dat", "%\n1 0 x 0 0 7\n1 2\n") + with pytest.warns(UserWarning): + data = readalign.split([fn], dim=1) + assert len(data[0]) == 1 and data[0][0].shape == (1, 6) and np.isnan(data[0][0][0, 2]) + + def test_openfile_missing(self, tmp_path): + with pytest.raises(IOError): + readalign.openfile(str(tmp_path / "missing.dat")) + + +def test_is_close(): + assert readalign.is_close(1.0, 1.0 + 1e-12) + assert not readalign.is_close(1.0, 1.001) + assert readalign.is_close(0.0, 1e-12, abs_tol=1e-10) + + +class TestAlignment: + def test_horizontal_alignment(self, bbob_format): + # two trials: Delta f (column 2) over evaluations (column 0) + t1 = np.array([[1, 0, 50.0], [10, 0, 2.0], [20, 0, 0.05]]) + t2 = np.array([[1, 0, 9.0], [3, 0, 0.5]]) + evals, maxevals, finalf = readalign.align_data(readalign.HMultiReader([t1, t2]), 0, 2) + assert list(maxevals) == [20, 3] + assert list(finalf) == [0.05, 0.5] + # first row: smallest 10**(i/5) not smaller than the largest initial f-value + assert evals[0, 0] == pytest.approx(10 ** (np.ceil(np.log10(50) * 5) / 5)) + assert list(evals[0, 1:]) == [1, 1] + # rows are monotonously decreasing targets with non-decreasing evals + assert np.all(np.diff(evals[:, 0]) < 0) + e = np.where(np.isnan(evals[:, 1:]), np.inf, evals[:, 1:]) + assert np.all(np.diff(e, axis=0) >= 0) + # the last row is the smallest 10**(i/5) target reached, i.e. the one just above 0.05 + assert evals[-1, 0] == pytest.approx(10 ** (-6 / 5)) + assert evals[-1, 1] == 20 and np.isnan(evals[-1, 2]) + + def test_vertical_alignment(self, bbob_format): + t1 = np.array([[1, 0, 50.0], [10, 0, 2.0], [20, 0, 0.05]]) + t2 = np.array([[1, 0, 9.0], [3, 0, 0.5]]) + funvals, maxevals, finalf = readalign.align_data(readalign.VMultiReader([t1, t2]), 0, 2) + np.testing.assert_array_equal(funvals, [[1, 50, 9], [3, 50, 0.5], [10, 2, 0.5], [20, 0.05, 0.5]]) + assert list(maxevals) == [20, 3] + + def test_negative_fvalues(self, bbob_format): + t1 = np.array([[1, 0, 5.0], [4, 0, -2.0]]) + evals, _, finalf = readalign.align_data(readalign.HMultiReader([t1]), 0, 2) + assert finalf[0] == -2 + assert evals[-1, 0] <= 0 and evals[-1, 1] == 4 + + def test_rewind_reader(self, bbob_format): + t1 = np.array([[1, 7, 5.0], [4, 8, 0.1]]) + reader = readalign.HMultiReader([t1]) + e1 = readalign.align_data(reader, 0, 2)[0] + e2 = readalign.align_data(reader, 1, 2, rewind_reader=True)[0] + assert list(e1[:, 1]) == [1, 4] and list(e2[:, 1]) == [7, 8] + + def test_align_data_type_check(self): + with pytest.raises(TypeError): + readalign.align_data([np.zeros((2, 3))], 0, 2, rewind_reader=True) + + def test_empty_arrays_are_skipped(self, bbob_format): + assert len(readalign.HMultiReader([np.zeros((0, 3)), np.ones((2, 3))])) == 1 + with pytest.raises(ValueError): + readalign.MultiReader.SingleReader([]) + + def test_array_alignment_merges_columns(self, bbob_format): + a = np.array([[1, 50.0, 9.0], [3, 50, 0.5]]) + b = np.array([[1, 7.0], [2, 1.0]]) + res = readalign.alignArrayData(readalign.VArrayMultiReader([a, b])) + np.testing.assert_array_equal(res, [[1, 50, 9, 7], [2, 50, 9, 1], [3, 50, 0.5, 1]]) diff --git a/tests/unit/test_reduce_size_and_sanitycheck.py b/tests/unit/test_reduce_size_and_sanitycheck.py new file mode 100644 index 0000000..867dc89 --- /dev/null +++ b/tests/unit/test_reduce_size_and_sanitycheck.py @@ -0,0 +1,100 @@ +"""Unit tests for the data-file utilities `cocopp.reduce_size` and `cocopp.sanitycheck`.""" + +import os +import warnings + +import pytest + +from cocopp import reduce_size as rs +from cocopp import sanitycheck +from cocopp.ppfig import Usage + +import synthdata + +HEADER = "% f evaluations | g evaluations | best noise-free fitness - Fopt | ...\n" + + +class TestReduceSize: + def test_target_hit(self): + hit = rs.TargetHit(20, 1e-8) + assert hit("1 0 100 x\n") + assert not hit("2 0 100 x\n") # same f-value: no new target + assert hit("3 0 50 x\n") + assert not hit("4 0 49 x\n") # within the same 10**(1/20) band + assert hit("5 0 1e-9 x\n") # final target + assert not hit("6 0 1e-12 x\n") # nothing after the final target + assert hit("1 0 100 x\n") # evaluations decreased: new run, reset + + def test_reduce_dat(self): + lines = [HEADER] + ["%d 0 %g 0 0\n" % (e, f) for e, f in [(1, 100), (2, 100), (3, 99), (4, 50), (5, 49), (6, 48)]] + res = rs.reduce_dat(lines) + assert res[0] == HEADER + # lines hitting a new 10**(-1/20) target band are kept, others are overwritten + # by the next line, hence the last line is always kept + assert [line.split()[0] for line in res[1:]] == ["1", "4", "6"] + + def test_reduce_tdat_keeps_first_and_last(self): + lines = [HEADER] + ["%d 0 1 0 0\n" % e for e in range(1, 9)] + [HEADER] + ["%d 0 1 0 0\n" % e for e in (1, 2, 3)] + res = rs.reduce_tdat(lines) + evals = [line.split()[0] if not line.startswith("%") else "%" for line in res] + assert evals == ["%", "1", "8", "%", "1", "3"] + + def test_remove_x(self): + lines = [HEADER, "1 0 3 4 5 6 7 8\n"] + assert rs.remove_x(lines) == [HEADER, "1 0 3 4 5\n"] + + @pytest.mark.parametrize( + "name, dat, tdat, x", + [("f_DIM2.dat", True, False, False), ("f_DIM10.tdat", False, True, True), ("f_DIM10.mdat", False, False, True), ("f.info", False, False, False)], + ) + def test_conditions(self, name, dat, tdat, x): + assert rs.reduce_dat_condition(name) == dat + assert rs.reduce_tdat_condition(name) == tdat + assert rs.remove_x_condition(name) == x + + def test_main_on_synthetic_data(self, tmp_path, quiet): + import cocopp + + folder = synthdata.write_bbob(str(tmp_path), "RED", functions=(1,), dims=(2, 10), seed=5) + before = cocopp.pproc.DataSetList(folder) + rs.main(folder) + reduced = [d for d in os.listdir(tmp_path) if d.startswith("RED-")] + assert len(reduced) == 1 + red_folder = os.path.join(str(tmp_path), reduced[0]) + size = lambda f: sum(os.path.getsize(os.path.join(r, n)) for r, _, ns in os.walk(f) for n in ns) # noqa: E731 + assert size(red_folder) < size(folder) + after = cocopp.pproc.DataSetList(red_folder) + # the reduction must not change the post-processed data + for ds0, ds1 in zip(before, after): + assert not ds0._data_differ(ds1) + assert list(ds0.maxevals) == list(ds1.maxevals) + + +class TestSanityCheck: + def test_is_correct_instances(self): + assert sanitycheck.is_correct_instances(list(range(1, 16))) + assert sanitycheck.is_correct_instances(3 * [1, 2, 3, 4, 5]) + assert not sanitycheck.is_correct_instances([1, 2, 3]) + + def test_checkinfofile_ok(self, bbob_small, capsys): + sanitycheck.checkinfofile(os.path.join(bbob_small, "bbobexp_f1.info")) + out = capsys.readouterr().out + # quirk: the reported line numbers are 0-based + assert "entry l0-2 is ok" in out and "entry l3-5 is ok" in out + + def test_checkinfofile_wrong_instances(self, tmp_path): + info = synthdata.write_tiny_bbob(str(tmp_path)) + with pytest.raises(Usage, match="instances"): + sanitycheck.checkinfofile(info, verbose=False) + + def test_checkinfofile_missing_data(self, tmp_path): + info = synthdata.write_tiny_bbob(str(tmp_path)) + os.remove(os.path.join(os.path.dirname(info), "data_f1", "bbobexp_f1_DIM2.tdat")) + with pytest.raises(IOError): + sanitycheck.checkinfofile(info, verbose=False) + + @pytest.mark.xfail(reason="known bug: sanitycheck.main uses rungeneric.shortoptlist which does not exist", raises=AttributeError, strict=True) + def test_main(self, bbob_small): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + sanitycheck.main([bbob_small]) diff --git a/tests/unit/test_settings_and_helpers.py b/tests/unit/test_settings_and_helpers.py new file mode 100644 index 0000000..d67d29b --- /dev/null +++ b/tests/unit/test_settings_and_helpers.py @@ -0,0 +1,319 @@ +"""Unit tests for settings modules (`testbedsettings`, `genericsettings`, +`config`, `dataformatsettings`), `findfiles`, `captions` and small helpers +of `ppfig`. +""" + +import os +import tarfile +import zipfile + +import numpy as np +import pytest + +import cocopp +from cocopp import captions, config, dataformatsettings, findfiles, genericsettings, ppfig, pproc +from cocopp import testbedsettings as tbs + +# suite name -> (testbed class name, .name, first and last function, instances_are_uniform, has_constraints) +TESTBEDS = { + "bbob": ("GECCOBBOBTestbed", "bbob", 1, 24, True, False), + "bbob-noisy": ("GECCOBBOBNoisyTestbed", "bbob", 101, 130, True, False), # quirk: name is 'bbob' + "bbob-biobj": ("GECCOBiObjBBOBTestbed", "bbob-biobj", 1, 55, False, False), + "bbob-biobj-ext": ("GECCOBiObjExtBBOBTestbed", "bbob-biobj-ext", 1, 92, False, False), + "bbob-constrained": ("CONSBBOBTestbed", "bbob-constrained", 1, 54, True, True), + "bbob-largescale": ("BBOBLargeScaleTestbed", "bbob-largescale", 1, 24, True, False), + "bbob-mixint": ("GECCOBBOBMixintTestbed", "bbob-mixint", 1, 24, True, False), + "bbob-biobj-mixint": ("GECCOBBOBBiObjMixintTestbed", "bbob-biobj-mixint", 1, 92, False, False), + "bbob-JOINED-bbob-largescale": ("BBOBLargeScaleJOINEDTestbed", "bbob", 1, 24, True, False), + "bbob-boxed": ("SBOXCOSTTestbed", "bbob-boxed", 1, 24, True, False), + "sbox-cost": ("SBOXCOSTTestbed", "bbob-boxed", 1, 24, True, False), + "bbob-JOINED-bbob-boxed": ("SboxCostJOINEDTestbed", "bbob and bbob-boxed", 1, 24, True, False), +} + + +class TestTestbedSettings: + def test_all_suites_are_tested(self): + assert set(TESTBEDS) == set(tbs.suite_to_testbed) + + @pytest.mark.parametrize("suite", sorted(TESTBEDS)) + def test_load_current_testbed(self, suite): + cls, name, first, last, uniform, constraints = TESTBEDS[suite] + tb = tbs.load_current_testbed(suite, pproc.TargetValues) + assert tb is tbs.current_testbed + assert type(tb).__name__ == cls == tbs.get_testbed_from_suite(suite) + assert tbs.SuiteClass(suite) is type(tb) + assert (tb.name, tb.first_function_number, tb.last_function_number) == (name, first, last) + assert tb.instances_are_uniform is uniform and tb.has_constraints is constraints + assert tb.string_evals == ("evaluations" if constraints else "$f$-evaluations") + assert tb.string_evals_short == ("evals" if constraints else "FEvals") + # target value attributes are converted to the given class + assert isinstance(tb.pprldmany_target_values, pproc.TargetValues) + + def test_unknown_suite(self): + with pytest.raises(ValueError): + tbs.get_testbed_from_suite("bbob-unknown") + with pytest.raises(ValueError): + tbs.load_current_testbed("bbob-unknown", pproc.TargetValues) + + def test_reset(self): + tbs.load_current_testbed("bbob", pproc.TargetValues) + tbs.reset_current_testbed() + assert tbs.current_testbed is None + + def test_reference_values(self): + tbs.reset_reference_values() + assert tbs.get_reference_values(None) is None and tbs.get_first_reference_values() is None + tbs.update_reference_values("A", "h1") + tbs.copy_reference_values("A", "B") + assert tbs.get_reference_values("B") == "h1" and tbs.get_reference_values(None) == "h1" + with pytest.warns(UserWarning, match="different"): + tbs.update_reference_values("C", "h2") + assert tbs.get_reference_values(None) == {"h1", "h2"} + assert tbs.get_first_reference_values() == "h1" + tbs.reset_reference_values() + assert tbs.reference_values == {} + + def test_info_and_short_names(self): + tb = tbs.load_current_testbed("bbob", pproc.TargetValues) + assert tb.info(1) == "F1 Sphere" + assert tb.info().startswith("Testbed of 24 noiseless functions") + names = tbs.get_short_names("bbob-benchmarkshortinfos.txt") + assert sorted(names) == list(range(1, 25)) and names[1] == "Sphere" + with pytest.warns(UserWarning): + assert tbs.get_short_names("nonexisting.txt") is None + + def test_constraints(self): + assert [tbs.CONSBBOBTestbed.number_of_constraints(10, f) for f in range(1, 8)] == [1, 3, 9, 16, 24, 54, 1] + assert tbs.CONSBBOBTestbed.number_of_constraints(10, 4, active_only=True) == 11 + assert tbs.CONSBBOBTestbed.constraint_category(5) == "6ndiv4+9" + assert tbs.CONSBBOBTestbed.constraint_category(5, active_only=True) == "6+n" + with pytest.raises(NotImplementedError): + tbs.Testbed.number_of_constraints(10, 1) + + def test_filter_bbob_and_largescale(self): + class DS: + def __init__(self, suite): + self.suite_name = suite + + tb = tbs.load_current_testbed("bbob", pproc.TargetValues) + dsl = [DS("bbob"), DS("bbob-largescale")] + tb.filter(dsl) + assert [ds.suite for ds in dsl] == ["bbob-largescale"] * 2 + dsl = [DS("bbob"), DS("bbob-boxed")] + tb.filter(dsl) + assert [ds.suite for ds in dsl] == ["bbob-JOINED-bbob-boxed"] * 2 + with pytest.raises(ValueError): + tb.filter([DS("bbob-biobj")]) + + +class TestConfig: + @pytest.mark.parametrize( + "name, first_digit, after", + [(".1bb", 0, 2), ("s-.2bbb", 1, 4), ("ss1.23bbbb", 2, 6), ("sss-2e0bbbbb", 3, 7), ("ssss1.2e-3bbbbbb", 4, 10), ("abc", -1, 0)], + ) + def test_index_after_float(self, name, first_digit, after): + assert config._index_after_float(name, True) == first_digit + assert config._index_after_float(name) == after + + def test_line_style_mapping(self): + names = ["1.0-CMA", "2.0-CMA", "BFGS", "0.5-CMA"] + assert config.map_indices_to_line_styles(names) == {0: 0, 1: 0, 2: 2, 3: 0} + styles = ["s0", "s1", "s2", "s3"] + # styles of the indices [0, 1, 3] are reassigned in the sorted order of 0.5, 1.0, 2.0 + assert config.sorted_line_styles(styles, names, [0, 1, 3]) == ["s1", "s3", "s2", "s0"] + + def test_str_to_colormap(self): + colors = list(config._str_to_colormap("viridis..5", 3)) + assert len(colors) == 3 and all(c.startswith("#") for c in colors) + assert colors[0] != colors[-1] + + def test_config_line_styles_parameter_sweep(self, monkeypatch): + monkeypatch.setattr(genericsettings, "parameter_sweep", True) + monkeypatch.setattr(genericsettings, "_current_args", ["a1.0x", "a2.0x", "b3.0y"]) + before = [dict(s) for s in genericsettings.line_styles] + config.config_line_styles() + after = genericsettings.line_styles + assert after[0]["color"] != before[0]["color"] # colors from the colormap + assert after[0]["marker"] == after[1]["marker"] # same algorithm family, same marker + assert after[2]["marker"] != after[0]["marker"] + + @pytest.mark.xfail(reason="parameter sweep fails for algorithm names without a float", raises=ValueError, strict=True) + def test_config_line_styles_parameter_sweep_name_without_float(self, monkeypatch): + monkeypatch.setattr(genericsettings, "parameter_sweep", True) + monkeypatch.setattr(genericsettings, "_current_args", ["a1.0x", "a2.0x", "b"]) + config.config_line_styles() + + def test_config_sets_testbed_and_bootstrap_size(self): + config.config("bbob-biobj") + assert tbs.current_testbed.name == "bbob-biobj" + assert genericsettings.simulated_runlength_bootstrap_sample_size == pytest.approx(10 + 990 / (1 + 10 * genericsettings.in_a_hurry)) + from cocopp.compall import pprldmany + + assert pprldmany.x_limit == genericsettings.xlimit_pprldmany + + def test_config_expensive_uses_runlength_based_targets(self, monkeypatch, quiet): + from cocopp import ppfigdim, pprldistr + + monkeypatch.setattr(genericsettings, "isExpensive", True) + config.config("bbob") + assert genericsettings.runlength_based_targets + assert genericsettings.maxevals_fix_display == genericsettings.xlimit_expensive + tb = tbs.current_testbed + assert tb.scenario == tbs.scenario_rlbased + for attr in ["pprldmany_target_values", "ppfigdim_target_values", "pptable_targetsOfInterest", "ppfigs_ftarget"]: + assert isinstance(getattr(tb, attr), pproc.RunlengthBasedTargetValues), attr + assert ppfigdim.xlim_max == genericsettings.xlimit_expensive + assert pprldistr.runlen_xlimits_max == genericsettings.xlimit_expensive / 2 + + def test_global_state_is_restored_after_previous_test(self): + """the conftest fixture must undo `config` side effects of other tests""" + from cocopp import ppfigdim + + assert genericsettings.isExpensive is False + assert ppfigdim.xlim_max is None + + +class TestGenericSettings: + def test_default_settings_copy(self): + assert genericsettings.default_settings is not genericsettings + assert genericsettings.default_settings.outputdir == "ppdata" + assert cocopp.toolsdivers.diff_attr(genericsettings.default_settings, genericsettings) == [ + ["interactive_mode", True, False] # set by conftest + ] + + def test_instances_of_interest(self): + assert genericsettings.instancesOfInterest2009 == {i: 3 for i in range(1, 6)} + assert all(isinstance(d, dict) for d in genericsettings.instancesOfInterest) + + +def test_data_format_mapping(): + m = dataformatsettings.data_format_name_to_class_mapping + assert m[None] is m["bbob"] is dataformatsettings.BBOBOldDataFormat + assert m["bbob-new2"] is dataformatsettings.BBOBNewDataFormat + biobj = m["bbob-biobj"]() + assert (biobj.evaluation_idx, biobj.function_value_idx) == (0, 1) + new = m["bbob-new"]() + assert (new.evaluation_idx, new.evaluation_constraints_idx, new.function_value_idx) == (0, 1, 2) + + +class TestFindFiles: + def test_recognized_filetypes(self, tmp_path): + assert findfiles.is_recognized_repository_filetype(str(tmp_path)) + for name in ["a.tgz", "a.tar.gz", "a.tar", "a.zip"]: + assert findfiles.is_recognized_repository_filetype(name) + assert findfiles.is_recognized_repository_filetype2(name) + assert not findfiles.is_recognized_repository_filetype("a.info") + assert not findfiles.is_recognized_repository_filetype2(str(tmp_path)) + + def test_main_lists_info_files(self, bbob_small): + found = findfiles.main(bbob_small) + assert sorted(os.path.basename(f) for f in found) == ["bbobexp_f1.info", "bbobexp_f15.info", "bbobexp_f2.info", "bbobexp_f8.info"] + + def test_main_warns_on_empty_folder(self, tmp_path): + with pytest.warns(UserWarning, match="Could not find"): + assert findfiles.main(str(tmp_path)) == [] + + @pytest.mark.parametrize("kind", ["tgz", "zip"]) + def test_archives_are_extracted(self, tmp_path, bbob_small, kind): + name = str(tmp_path / ("SMALL." + kind)) + if kind == "tgz": + with tarfile.open(name, "w:gz") as tar: + tar.add(bbob_small, arcname="SMALL") + else: + with zipfile.ZipFile(name, "w") as z: + for root, _, files in os.walk(bbob_small): + for f in files: + p = os.path.join(root, f) + z.write(p, os.path.relpath(p, os.path.dirname(bbob_small))) + found = findfiles.main(name) + assert len(found) == 4 + assert all(os.sep + ".extracted_SMALL" + os.sep in f for f in found) + assert findfiles.get_directory(name, False) == str(tmp_path / ".extracted_SMALL") + + def test_hash(self): + assert findfiles.hash("abc") == "a9993e364706816aba3e25717850c26c9cd0d89d" + assert findfiles.hash("abc", 4) == "a999" and findfiles.hash("abc", 0) == "" + + def test_output_directory_sub_folder(self, monkeypatch): + monkeypatch.setattr(tbs, "current_testbed", None) + assert findfiles.get_output_directory_sub_folder("data/ALG.tgz", addtime=False) == "ALG" + assert findfiles.get_output_directory_sub_folder(["a/LONGNAME", "b/X"], addtime=False) == "LONGN_X" + assert findfiles.get_output_directory_sub_folder(["a", "b"], addhash=3, addtime=False) == "a_b_" + findfiles.hash("ab", 3) + args = ["a%d" % i for i in range(9)] + assert findfiles.get_output_directory_sub_folder(args, addtime=False).endswith("a6_et_al") + tbs.load_current_testbed("bbob-biobj", pproc.TargetValues) + assert findfiles.get_output_directory_sub_folder("x/ALG", addtime=False) == "biobj_ALG" + assert len(findfiles.get_output_directory_sub_folder("ALG")) == len("biobj_ALG_") + len("0101h0101") + 2 + with pytest.raises(ValueError): + findfiles.get_output_directory_sub_folder([]) + + +class TestCaptions: + def test_all_keywords_are_replaced(self, quiet): + config.config("bbob") + for key in captions.replace_dict: + res = captions.replace("x " + key + " y") + assert "!!" not in res, key + assert captions.replace("!!THE-REF-ALG!!") == "the best algorithm from BBOB 2009" + assert captions.replace("!!TOTAL-NUM-OF-FUNCTIONS!!") == "24" + assert captions.replace("!!NUM-OF-TARGETS-IN-ECDF!!") == "51" + + def test_html_replacements(self): + config.config("bbob") + assert captions.replace("!!DF!! 2-D a-b", html=True) == "Δf 2different dimensions a−b" + + def test_unknown_keyword_warns(self): + with pytest.warns(UserWarning, match="!!"): + captions.replace("!!UNKNOWN!!") + + def test_reference_algorithm_text(self): + config.config("bbob-constrained") + assert captions.get_reference_algorithm_text(False) == "" + with pytest.raises(NotImplementedError): + captions.get_reference_algorithm_text(True) + + +class TestPpfigHelpers: + def test_consecutive_numbers(self): + assert ppfig.consecutiveNumbers([0, 1, 2, 4, 5, 7, 8, 9]) == "0-2, 4, 5, 7-9" + assert ppfig.consecutiveNumbers([0, 1, 2, 4, 5, 7, 8, 9], "f") == "f0-f2, f4, f5, f7-f9" + assert ppfig.groupByRange([1, 2, 5]) == [[1, 2], [5]] + + def test_enum(self): + e = ppfig.enum("A", "B", C=7) + assert (e.A, e.B, e.C) == (0, 1, 7) + + def test_discretize_limits(self): + lo, hi = ppfig.discretize_limits((1, 70)) + assert lo == pytest.approx(10**-0.2) + assert hi == pytest.approx(100 * (1 + 1e-6)) + assert ppfig.discretize_limits((1, 20))[1] == pytest.approx(30 * (1 + 1e-6)) + + def test_marker_positions(self): + x = np.logspace(0, 4, 50) + y = np.linspace(0, 1, 50) + xp, yp = ppfig.marker_positions(x, y, 3, 10, ax_limits=(1, 1e4, 0, 1)) + assert len(xp) == len(yp) <= 11 + assert xp[-1] == x[-1] and set(xp) <= set(x) + + def test_plot_unif_log_x_markers(self): + lines = ppfig.plotUnifLogXMarkers(np.logspace(0, 3, 20), np.linspace(0, 1, 20), 3, marker="o", label="x") + assert len(lines) == 3 + + def test_save_figure(self, tmp_path): + import matplotlib.pyplot as plt + + plt.plot([1, 2], [3, 4]) + ppfig.save_figure(str(tmp_path / "fig")) # default formats from genericsettings + assert (tmp_path / "fig.pdf").exists() and (tmp_path / "fig.svg").exists() + ppfig.save_figure(str(tmp_path / "fig2"), format="png") + assert (tmp_path / "fig2.png").exists() + + def test_html_links(self, tmp_path): + (tmp_path / "f.html").write_text("") + assert ppfig.add_link(str(tmp_path), "", "f.html", "label") == 'label
\n' + assert ppfig.add_link(str(tmp_path), "", "missing.html", "label") == "" + assert ppfig.add_link(str(tmp_path), "sub", "x.html", "l", " ", True, 5) == ' l
\n' + assert ppfig.format_link_list_entry_in_html("x") == "x
\n" + assert ppfig.add_image("a.svg", True) == '' + assert ppfig.add_image("a.svg", False, 10) == '' diff --git a/tests/unit/test_synthdata.py b/tests/unit/test_synthdata.py new file mode 100644 index 0000000..b7ed143 --- /dev/null +++ b/tests/unit/test_synthdata.py @@ -0,0 +1,63 @@ +"""Sanity checks of the synthetic test data (`synthdata`) themselves. + +If these fail, the data generator rather than `cocopp` is likely broken. +Note that `DataSetList` only *prints* I/O errors, hence a broken +generator can silently lead to empty data sets. +""" + +import filecmp +import os + +import numpy as np +import pytest + +from cocopp import testbedsettings + +import synthdata +from conftest import load + + +@pytest.mark.parametrize( + "fixture, n_datasets, n_runs, suite", + [ + ("bbob_alg_a", 24 * 5, 15, "bbob"), + ("bbob_small", 4 * 2, 15, "bbob"), + ("bbob_old_format", 2 * 2, 15, "bbob"), + ("biobj_alg", 55 * 3, 10, "bbob-biobj"), + ], +) +def test_fixture_data_load_completely(request, fixture, n_datasets, n_runs, suite): + testbedsettings.reset_current_testbed() + dsl = load(request.getfixturevalue(fixture)) + assert len(dsl) == n_datasets + assert all(ds.nbRuns() == n_runs for ds in dsl) + assert {ds.suite_name for ds in dsl} == {suite} + assert testbedsettings.current_testbed.name == suite + + +def test_success_rates_are_mixed(bbob_alg_a): + """the data should exercise both solved and unsolved code paths in all dimensions""" + testbedsettings.reset_current_testbed() + dsl = load(bbob_alg_a) + for dim, dsl_dim in dsl.dictByDim().items(): + succ = [ds.detSuccesses([1e-8])[0] for ds in dsl_dim] + assert max(succ) == 15 and min(succ) < 5, (dim, succ) + assert any(0 < s < 15 for s in succ) + + +def test_generator_is_deterministic(tmp_path): + np.random.seed(1) + a = synthdata.write_bbob(str(tmp_path / "1"), "X", functions=(1, 7), dims=(2, 3), seed=3) + np.random.seed(2) # the global state must not matter + b = synthdata.write_bbob(str(tmp_path / "2"), "X", functions=(1, 7), dims=(2, 3), seed=3) + c = synthdata.write_bbob(str(tmp_path / "3"), "X", functions=(1, 7), dims=(2, 3), seed=4) + files = sorted(os.path.relpath(os.path.join(r, f), a) for r, _, fs in os.walk(a) for f in fs) + match, mismatch, errors = filecmp.cmpfiles(a, b, files, shallow=False) + assert not mismatch and not errors and len(match) == len(files) == 2 + 2 * 2 * 2 + assert filecmp.cmpfiles(a, c, files, shallow=False)[1] # another seed gives other data + + +def test_tiny_data_files(tmp_path): + info = synthdata.write_tiny_bbob(str(tmp_path)) + lines = open(info).read().splitlines() + assert len(lines) == 3 and lines[2].endswith("1:50|-5.0e-09, 2:100|4.5e-01, 3:30|-9.1e-09") diff --git a/tests/unit/test_toolsdivers.py b/tests/unit/test_toolsdivers.py new file mode 100644 index 0000000..751af8d --- /dev/null +++ b/tests/unit/test_toolsdivers.py @@ -0,0 +1,320 @@ +"""Unit tests for `cocopp.toolsdivers`. + +Many expected values below are *characterization* values: they pin the +current behavior (including some quirks, marked as such) so that a +refactoring does not change the produced tables and figures by accident. +""" + +import os + +import numpy as np +import pytest + +import cocopp +from cocopp import toolsdivers as td + + +class TestStringLists: + def test_stringlist_from_str_and_list(self): + assert td.StringList("ab bc") == ["ab", "bc"] + assert td.StringList(["x", "y"]).as_string == " x y " + assert td.StringList(None) == [] + assert td.StringList("") == [] + + def test_strlist_find_substring_and_regex(self): + s = td.StrList(["abc", "bcd", "cde", " cde"]) + assert s("bc") == ["abc", "bcd"] + assert s("a", "b") == ["abc"] + assert s(["a", "b"]) == ["abc"] + assert s(".c") == ["bcd", " cde"] + assert s(".*c") == ["abc", "bcd", "cde", " cde"] + assert s("BC") == ["abc", "bcd"] # case insensitive + + def test_strlist_find_by_index(self): + s = td.StrList(["abc", "bcd", "cde"]) + assert s(1) == ["bcd"] + assert s([0, 2]) == ["abc", "cde"] + assert s.found == ["abc", "cde"] + assert isinstance(s.found, td.StrList) + + def test_strlist_find_with_leading_strs(self): + s = td.StrList(["2009/BFGS", "2010/IPOP"]) + assert s.find("bbob/2009", leading_strs=["bbob/"]) == ["2009/BFGS"] + # a trailing part of the leading string is removed as well + assert s.find("ob/2010", leading_strs=["bbob/"]) == ["2010/IPOP"] + + def test_strlist_find_indices_and_print(self, capsys): + s = td.StrList(["abc", "bcd", "cde"]) + assert s.find_indices("bc") == [0, 1] + s.find("abc") + found_before = list(s.found) + s.print("d") + out = capsys.readouterr().out + assert " 1: 'bcd'" in out and " 2: 'cde'" in out + assert list(s.found) == found_before # print does not change found + + def test_strlist_as_string(self): + assert td.StrList("a b").as_string == " a b " + + +def test_algorithm_list_ordered_dict(): + names = ["b", "a", "c"] + d = dict(zip(names[::-1], [1, 2, 3])) + res = td.AlgorithmList(names).ordered_dict(d) + assert list(res) == names + assert res == d + + +def test_algorithm_list_ordered_dict_appends_unknown_keys_sorted(): + with pytest.warns(UserWarning): + res = td.AlgorithmList(["b"]).ordered_dict({"z": 1, "b": 2, "a": 3}) + assert list(res) == ["b", "a", "z"] + + +class TestFolderContexts: + def test_infolder(self, tmp_path): + before = os.getcwd() + with td.Infolder(str(tmp_path)): + assert os.path.samefile(os.getcwd(), tmp_path) + assert os.getcwd() == before + + def test_infolder_gone_with_the_wind(self): + before = os.getcwd() + with td.InfolderGoneWithTheWind() as _: + tmp = os.getcwd() + assert tmp != before + open("some_file", "w").close() + assert os.getcwd() == before + assert not os.path.exists(tmp) + + +def test_equals_approximately(): + assert td.equals_approximately(1.0, 1.0 + 1e-13) + assert td.equals_approximately(-1.0, -1.0 - 1e-13) + assert not td.equals_approximately(1.0, 1.001) + assert td.equals_approximately(1e20, 1e20 * (1 + 1e-13)) + + +def test_less_ignores_nan_without_warning(): + a = np.array([1.0, np.nan, 3.0]) + with np.errstate(invalid="raise"): + res = td.less(a, 2.0) + assert list(res) == [True, False, False] + + +def test_diff_attr(): + class A: + pass + + a, b = A(), A() + a.x, b.x = 1, 1 + a.y, b.y = 1, 2 + a._z, b._z = 1, 2 + a.only_a = 3 + assert td.diff_attr(a, b) == [["y", 1, 2]] + + +class TestFileHelpers: + def test_prepend_to_file(self, tmp_path): + fn = tmp_path / "f.tex" + fn.write_text("old1\nold2\n") + td.prepend_to_file(str(fn), ["new1", "new2"]) + assert fn.read_text() == "new1\nnew2\nold1\nold2\n" + + def test_prepend_to_new_file(self, tmp_path): + fn = tmp_path / "new.tex" + td.prepend_to_file(str(fn), ["a"]) + assert fn.read_text() == "a\n" + + def test_prepend_to_file_truncates(self, tmp_path, capsys): + fn = tmp_path / "f.tex" + fn.write_text("".join("%d\n" % i for i in range(10))) + td.prepend_to_file(str(fn), ["x"], maxlines=2, warn_message="too long") + assert fn.read_text().splitlines() == ["x", "0", "1", "2", "3"] + assert "too long" in capsys.readouterr().out + + def test_replace_in_file(self, tmp_path, capsys): + fn = tmp_path / "f.txt" + fn.write_text("abc\nxbx\n") + td.replace_in_file(str(fn), "b", "B") + assert fn.read_text() == "aBc\nxBx\n" + td.replace_in_file(str(tmp_path / "missing"), "a", "b") + assert "does not exist" in capsys.readouterr().out + + def test_truncate_latex_command_file(self, tmp_path): + fn = tmp_path / "cmd.tex" + lines = ["\\providecommand{\\c%d}{}\n" % i for i in range(10)] + fn.write_text("".join(lines)) + td.truncate_latex_command_file(str(fn), keeplines=3) + # keeps keeplines + 1 lines, then stops at the next \providecommand + assert fn.read_text() == "".join(lines[:4]) + + def test_truncate_latex_command_file_creates_file(self, tmp_path): + fn = tmp_path / "cmd.tex" + td.truncate_latex_command_file(str(fn)) + assert fn.exists() and fn.read_text() == "" + + +@pytest.mark.parametrize( + "name, p0, p1, p2, p3", + [ + ("../data/BFGS_noiseless.tgz", "data/BFGS_noiseless.tgz", "BFGS_noiseless", "BFGS_noiseless", "BFGS_"), + ("./x/y/BIPOP-CMA-ESdata.tar.gz", "x/y/BIPOP-CMA-ESdata.tar.gz", "BIPOP-CMA-ES", "y/BIPOP-CMA-ES", "BIPOP-CMA-ES"), + # quirk: strip_pathname2 does not remove a .zip extension + (" a/b/c.zip ", "a/b/c.zip", "c", "b/c.zip", "c"), + ("a/.extracted_foo/bar", "a/.extracted_foo/bar", "bar", ".extracted_foo/bar", "bar"), + ], +) +def test_strip_pathname(name, p0, p1, p2, p3): + name = name.replace("/", os.sep) + assert td.strip_pathname(name) == p0 + assert td.strip_pathname1(name) == p1 + assert td.strip_pathname2(name) == p2 + assert td.strip_pathname3(name) == p3 + + +def test_str_to_latex(): + assert td.str_to_latex(r"a_b^c%d~e#f\g") == r"a b\^\,c\%d\ensuremath{\sim}e\#f\textbackslash{}g" + + +def test_number_of_digits_not_implemented(): + with pytest.raises(NotImplementedError): + td.number_of_digits(3.0) + + +# value, num2str(v), num2str(v, 1), num2str(v, 3), num2str(v, 2, force_rounding=True) +NUM2STR = [ + (0, "0", "0", "0", "0"), + (1, "1", "1", "1", "1"), + (-1, "-1", "-1", "-1", "-1"), + (0.5, "0.5", "0.5", "0.5", "0.5"), + (12345, "12345", "1e4", "12345", "12000"), + (1234.5, "1234", "1e3", "1234", "1200"), + (123.45, "123", "123", "123", "120"), + (12.345, "12", "12", "12.3", "12"), + (1.2345, "1.2", "1", "1.23", "1.2"), + (0.12345, "0.12", "0.1", "0.123", "0.12"), + (0.012345, "0.012", "0.01", "0.0123", "0.012"), + (0.0012345, "1.2e-3", "1e-3", "1.23e-3", "1.2e-3"), + (1e-8, "1e-8", "1e-8", "1e-8", "1e-8"), + (3.3e-12, "3.3e-12", "3e-12", "3.3e-12", "3.3e-12"), + (1e5, "1e5", "1e5", "1e5", "1e5"), + (1.5e6, "1.5e6", "2e6", "1.5e6", "1.5e6"), + (123456789, "1.2e8", "1e8", "1.23e8", "1.2e8"), + (-0.0234, "-0.023", "-0.02", "-0.0234", "-0.023"), + (99.9, "100", "100", "99.9", "100"), + (0.999, "1.0", "1", "0.999", "1.0"), + (2.0, "2", "2", "2", "2"), + (100, "100", "100", "100", "100"), +] + + +@pytest.mark.parametrize("val, s2, s1, s3, s2forced", NUM2STR) +def test_num2str(val, s2, s1, s3, s2forced): + assert td.num2str(val) == s2 + assert td.num2str(val, 1) == s1 + assert td.num2str(val, 3) == s3 + assert td.num2str(val, 2, force_rounding=True) == s2forced + + +def test_num2str2_unused_helper(): + assert td._num2str2(0.99, 1) == "1e0" # the "poor" formatting mentioned in its docstring + assert td._num2str2(12345.6) == "12346" + assert td._num2str2(0.00123, 2, True) == "+0.0012" + + +@pytest.mark.parametrize( + "s, latex, html", + [ + ("1e-8", "10^{-8}", "10-8"), + ("-1e-12", "-10^{-12}", "-10-12"), + ("2.3e5", r"2.3\times10^{5}", "2.3 x 105"), + ("12", "12", "12"), + ], +) +def test_number_to_latex_and_html(s, latex, html): + assert td.number_to_latex(s) == latex + assert td.number_to_html(s) == html + + +@pytest.mark.parametrize( + "ratio, html, tex, html_unsigned", + [ + (0, "÷inf", r"\div inf", "÷inf"), + (0.1, "÷10", r"\div 10", "÷10"), + (0.4, "÷2.5", r"\div 2.5", "÷2.5"), + (0.5, "-50%", r"-50\%", "-50%"), + (0.9, "-10%", r"-10\%", "-10%"), + (1, "0%", r"0\%", "0%"), + (1.01, "+1%", r"+1\%", "1%"), + (1.5, "+50%", r"+50\%", "50%"), + (1.99, "+99%", r"+99\%", "99%"), + (1.995, "×2.0", r"\times 2.0", "×2.0"), + (2, "×2", r"\times 2", "×2"), + (123.4, "×123", r"\times 123", "×123"), + ], +) +def test_ratio2str(ratio, html, tex, html_unsigned): + assert td.ratio2html(ratio) == html + assert td.ratio2tex(ratio) == tex + assert td.ratio2html(ratio, signed=False) == html_unsigned + + +def test_ratio2str_without_div_or_times(): + assert td._ratio2str(0.1, formdiv=None) == "-90%" + assert td._ratio2str(5, formtimes=None) == "+400%" + + +def test_html_subsup(): + assert td.html_subsup(".5", "3", "x", "y") == "x.5y3" + + +@pytest.mark.parametrize( + "succ, digit", + [ + (0, None), + (0.04, 0), # quirk: rounds to a ".0" subscript although 0 < succ + (0.05, 0), + (0.5, 5), + (0.94, 9), + (0.95, None), + (1, None), + ], +) +def test_prepend_succ_str(succ, digit): + tex, html, indent = td.prepend_succ_str(succ, "T", "H") + if digit is None: + assert (tex, html, indent) == ("T", "H", "") + else: + assert tex == r"$\!\!\!\!{}_{.%d}$T" % digit + assert html == td.html_subsup(".%d" % digit, "") + "H" + assert indent == 'style="text-indent: -0.5em;"' + + +def test_get_version_label_without_reference_values(monkeypatch): + monkeypatch.setattr(cocopp.testbedsettings, "reference_values", {}) + assert td.get_version_label() == "v" + cocopp.__version__ + + +def test_get_version_label_with_reference_values(monkeypatch): + monkeypatch.setattr(cocopp.testbedsettings, "reference_values", {"A": "abc"}) + assert td.get_version_label() == "v%s, hv-hash=abc" % cocopp.__version__ + assert td.get_version_label("A") == "v%s, hv-hash=abc" % cocopp.__version__ + monkeypatch.setattr(cocopp.testbedsettings, "reference_values", {"A": "abc", "B": "def"}) + label = td.get_version_label() + assert label.startswith("v%s, hv-hashes inconsistent:" % cocopp.__version__) + assert label.endswith("found!") and "abc" in label and "def" in label + + +def test_path_in_package_is_redirected_by_conftest(package_copy): + # the test suite redirects package files into a copy to keep the source tree clean + assert td.path_in_package("refalgs") == os.path.join(str(package_copy), "refalgs") + assert os.path.isdir(td.path_in_package("refalgs")) + + +def test_legend_sets_framealpha(): + import matplotlib.pyplot as plt + + plt.plot([1, 2], label="x") + td.legend() + assert plt.gca().get_legend().get_frame().get_alpha() == pytest.approx(0.2) diff --git a/tests/unit/test_toolsstats.py b/tests/unit/test_toolsstats.py new file mode 100644 index 0000000..aca55e2 --- /dev/null +++ b/tests/unit/test_toolsstats.py @@ -0,0 +1,282 @@ +"""Unit tests for `cocopp.toolsstats` (statistics and bootstrapping helpers). + +Where a function draws random numbers, either properties that hold for +every draw are tested or numpy's global random state is fixed (the +autouse fixture in ``conftest.py`` seeds it before each test). +""" + +import math + +import numpy as np +import pytest + +from cocopp import toolsstats as ts + +nan, inf = np.nan, np.inf + + +class TestSuccessPerformance: + def test_sp1_all_successful(self): + assert ts.sp1([1, 2, 3]) == (2.0, 1.0, 3) + + def test_sp1_with_maxvalue(self): + # data >= maxvalue count as unsuccessful: mean(successful) / success rate + res = ts.sp1([1, 2, 3, 10], maxvalue=5) + assert res == pytest.approx((2.0 / 0.75, 0.75, 3)) + + def test_sp1_with_issuccessful_and_nan(self): + res = ts.sp1([1, nan, 3, 4], issuccessful=[True, True, False, True]) + # the nan entry is removed first, then 2 of 3 remaining are successful + assert res == pytest.approx((2.5 / (2 / 3), 2 / 3, 2)) + + def test_sp1_no_success_and_empty(self): + assert ts.sp1([5, 6], maxvalue=1) == (inf, 0.0, 0) + res = ts.sp1([nan, nan]) + assert all(np.isnan(r) for r in res) + + def test_sp_is_ert(self): + # sum of all evaluations / number of successes + assert ts.sp([1, 2, 3, 10], maxvalue=5) == pytest.approx((16 / 3, 0.75, 3)) + assert ts.sp([1, 2, 3]) == (2.0, 1.0, 3) + + def test_sp_no_success(self): + assert ts.sp([5, 6], maxvalue=1) == (inf, 0.0, 0) + assert ts.sp([5, 6], maxvalue=1, allowinf=False) == (11, 0.0, 0) + + def test_sp_issuccessful_with_nan_regression(self): + # regression test for the bug fixed in July 2025 (sorting broke the + # alignment between data and issuccessful when data contained nan) + res = ts.sp([10, nan, 1, 5], issuccessful=[False, True, True, True]) + assert res == pytest.approx((16 / 2, 2 / 3, 2)) + + @pytest.mark.parametrize("func", [ts.sp, ts.sp1]) + def test_input_validation(self, func): + with pytest.raises(Exception): + func(3) + with pytest.raises(Exception): + func([1, 2], issuccessful=3) + with pytest.raises(Exception): + func([1, 2], issuccessful=[True]) + + +class TestPrctile: + def test_docstring_example(self): + pp = ts.prctile([2, 3, nan, 1], [5, 25, 50, 75, 95], ignore_nan=False) + assert pp[:3] == [1.0, 1.5, 2.5] + assert np.isnan(pp[3]) and np.isnan(pp[4]) + + @pytest.mark.parametrize("n", [1, 2, 5, 15, 100]) + def test_agrees_with_numpy_hazen(self, n): + rng = np.random.default_rng(n) + x = rng.lognormal(size=n) + p = [0, 2, 10, 25, 50, 75, 90, 98, 100] + assert ts.prctile(x, p) == pytest.approx(np.percentile(x, p, method="hazen")) + + def test_scalar_and_out_of_range(self): + x = [3, 1, 2] + assert ts.prctile(x, 50) == [2] + assert ts.prctile(x, [-10, 110]) == [1, 3] + + def test_ignores_none_and_nan_by_default(self): + assert ts.prctile([None, 1, nan, 3], [50]) == [2.0] + + def test_empty(self): + assert all(np.isnan(ts.prctile([], [10, 50]))) + + def test_infinite_values_use_proximity(self): + assert ts.prctile([1, 2, inf, inf], [60]) == [inf] + assert ts.prctile([1, 2, inf, inf], [40]) == [2] + + def test_issorted_skips_sorting(self): + # documents that issorted=True trusts the caller + assert ts.prctile([3, 1, 2], [0], issorted=True) == [3] + + +class TestRandint: + def test_randint_derandomized_docstring(self): + np.random.seed(1) + assert [int(i) for i in ts.randint_derandomized(0, 4, 6)] == [3, 2, 0, 1, 0, 2] + + @pytest.mark.parametrize("low, high, size", [(0, 5, None), (2, 7, 23), (10, None, 37), (0, 1, 3)]) + def test_each_value_once_per_chunk(self, low, high, size): + res = ts.randint_derandomized(low, high, size) + lo, hi = (0, low) if high is None else (low, high) + n = hi - lo + assert len(res) == (n if size is None else size) + assert res.min() >= lo and res.max() < hi + for i in range(0, len(res) - n + 1, n): + assert sorted(res[i : i + n]) == list(range(lo, hi)) + + def test_randint(self): + r = ts.randint(5, 1000) + assert r.shape == (1000,) and r.min() >= 0 and r.max() <= 4 + assert np.all(r == np.floor(r)) + + def test_fix_data_number(self): + data = [1, 2, 4] + for n in [1, 3, 4, 14]: + assert len(ts.fix_data_number(data, n)) == n + assert ts.fix_data_number(data, 14)[2] == data[2] + assert list(ts.fix_data_number(np.arange(10), 4, last_elements_randomized=False)) == [0, 1, 2, 3] + with pytest.warns(UserWarning): + ts.fix_data_number(data, 5, warn=True) + + +class TestSimulatedEvals: + def test_docstring_example(self): + np.random.seed(4) + res = ts.simulated_evals(np.hstack([[1], [2, 4, 2, 6, 100]]), 5, 13) + assert [int(i) for i in res[:3]] == [1, 1, 3] + + def test_all_successful_returns_resampled_data(self): + evals = np.array([3.0, 5, 7]) + res = ts.simulated_evals(evals, 0, 9, return_as=np.asarray) + assert sorted(res) == sorted(3 * list(evals)) # derandomized: each exactly 3 times + + def test_all_unsuccessful(self): + for return_as in (sorted, list, np.asarray, np.array, tuple): + res = ts.simulated_evals([1, 2], 2, 4, return_as=return_as) + assert len(res) == 4 and all(np.isnan(res)) + + def test_sums_contain_exactly_one_successful_run(self): + succ, fail = [1, 10], [1000, 100000] + res = ts.simulated_evals(succ + fail, len(fail), 200, return_as=np.asarray) + # every value = (sum of failures) + (one success); success evals are < 1000 + assert np.all(np.isin(res % 1000, succ)) + + def test_negative_convention(self): + # nfails=-1: unsuccessful runs are given as negative values in any position + res = ts.simulated_evals(np.array([-1000.0, 1, -100000, 10]), -1, 200, return_as=np.asarray) + assert np.all(res > 0) + assert np.all(np.isin(res % 1000, [1, 10])) + + def test_inconsistent_nfails(self): + with pytest.raises(ValueError): + ts.simulated_evals([1, 2], 3) + + def test_return_sorted_by_default(self): + res = ts.simulated_evals([5, 1, 3], 0, 9) + assert res == sorted(res) + + +class TestDraw: + def test_drawSP_properties(self): + prc, values = ts.drawSP([1, 2], [10, 20], [50], samplesize=100) + assert len(values) == 100 and values == sorted(values) + # each value is a (possibly empty) sum of failures plus one success + assert all(v - 1 in _sums_of([10, 20], 50) or v - 2 in _sums_of([10, 20], 50) for v in values) + assert prc == ts.prctile(values, [50], issorted=True) + + def test_drawSP_no_success_raises(self): + with pytest.raises(NotImplementedError): + ts.drawSP([], [10], [50]) + + def test_draw_bootstrap_of_sp1(self): + data = np.arange(1, 11) + prc, values = ts.draw(data, [0, 50, 100], samplesize=200) + assert len(values) == 200 + assert values == sorted(values) + assert 1 <= prc[0] <= prc[1] <= prc[2] <= 10 + assert prc[1] == pytest.approx(5.5, abs=1.0) + + def test_draw_is_reproducible_with_seed(self): + np.random.seed(5) + r1 = ts.draw(np.arange(10), [50], samplesize=50) + np.random.seed(5) + r2 = ts.draw(np.arange(10), [50], samplesize=50) + assert r1 == r2 + + def test_drawSP_from_dataset_new_not_implemented(self): + with pytest.raises(NotImplementedError): + ts.drawSP_from_dataset_new(None, 1, None) + + +def _sums_of(values, max_terms): + """all sums of up to `max_terms` elements drawn with replacement from `values`""" + sums = {0} + for _ in range(max_terms): + sums |= {s + v for s in sums for v in values if s + v <= max_terms * max(values)} + return sums + + +class TestRankSum: + def test_ranksum_statistic_against_brute_force(self): + rng = np.random.default_rng(1) + x, y = rng.integers(0, 10, 7), rng.integers(0, 10, 9) + expected = sum((yj < xi) + 0.5 * (yj == xi) for xi in x for yj in y) + assert ts.ranksum_statistic(x, y) == expected + + def test_rankdata_with_ties(self): + assert list(ts.rankdata([0, 2, 2, 3])) == [1.0, 2.5, 2.5, 4.0] + assert list(ts.rankdata([[3, 1], [2, 2]])) == [4.0, 1.0, 2.5, 2.5] + + def test_fastsort(self): + s, i = ts.fastsort(np.array([3, 1, 2])) + assert list(s) == [1, 2, 3] and list(i) == [1, 2, 0] + + @pytest.mark.parametrize("z", [-7, -3, -1.5, -0.2, 0, 0.3, 1.0, 1.96, 2.5, 4, 8]) + def test_zprob_is_normal_cdf(self, z): + assert float(ts.zprob(np.array(z))) == pytest.approx(0.5 * (1 + math.erf(z / math.sqrt(2))), abs=1e-6) + + def test_ranksumtest(self): + x = [1, 2, 3, 4, 5] + y = [6, 7, 8, 9, 10] + z, p = ts.ranksumtest(x, y) + n1, n2 = 5, 5 + expected_z = (15 - n1 * (n1 + n2 + 1) / 2) / math.sqrt(n1 * n2 * (n1 + n2 + 1) / 12) + assert z == pytest.approx(expected_z) + assert p == pytest.approx(2 * (1 - 0.5 * (1 + math.erf(abs(expected_z) / math.sqrt(2)))), abs=1e-6) + z2, p2 = ts.ranksumtest(y, x) + assert z2 == pytest.approx(-z) and p2 == pytest.approx(p) + + def test_ranksumtest_identical_samples(self): + z, p = ts.ranksumtest([1, 2, 3], [1, 2, 3]) + assert z == 0 and p == pytest.approx(1.0) + + +class TestBestAlgIndices: + def test_from_ert_arrays(self): + erts = np.array([[1, 5, inf], [2, 3, inf]]) + idx = ts.best_alg_indices(ert_ars=erts, median_finalfunvals=[0.5, 0.1]) + assert list(idx) == [0, 1, 1] # last target: no ERT, best median final f-value + + +def test_sliding_window_data(): + np.random.seed(2) + data = np.array([1.0, 3, 2, 5, inf, 4, 6]) + smooth, stats = ts.sliding_window_data(data, width=3, number_of_stats=2) + assert isinstance(smooth, np.ndarray) and len(smooth) == len(data) + assert smooth[0] == np.median([1, 3]) + assert smooth[4] == inf # non-finite data are kept + assert smooth[3] == np.median([2, 5]) # non-finite neighbors are ignored + assert all(len(s[1]) == 7 for s in stats) + + +def test_sliding_window_data_small_width_is_identity(): + data = [1, 2, 3] + assert ts.sliding_window_data(data, width=1) == (data, []) + + +@pytest.mark.xfail(reason="known bug: default number_of_stats=0 divides by zero", raises=ZeroDivisionError, strict=True) +def test_sliding_window_data_default_number_of_stats(): + ts.sliding_window_data([1.0, 2, 3, 4], width=2) + + +def test_sliding_window_data_too_wide_window_warns(): + with pytest.warns(UserWarning, match="width 3 should be smaller than the number of data 2"): + ts.sliding_window_data([1.0, 2], width=3, number_of_stats=1) + + +def test_equals_and_in_approximately(): + assert ts.equals_approximately(1.0, 1.0 + 1e-12) + assert ts.equals_approximately(1e-20, 0) # absolute tolerance + assert not ts.equals_approximately(1.0, 1.1) + assert ts.in_approximately(2.0, [1, 2 + 1e-12, 3]) + assert not ts.in_approximately(2.5, [1, 2, 3]) + + +def test_evals_container(): + e = ts.Evals([1, 5], [2, 3]) + assert e() == [1, 1, 5, 5, 5] + with pytest.raises(NotImplementedError): + e.sort()