Change-point tests

.coveragerc

@@ -2,6 +2,7 @@
 source = pints
 omit =
     pints/tests/*
+    pints/functionaltests/*
 
 [report]
 exclude_lines =

.github/workflows/change-point-tests.yml

@@ -0,0 +1,58 @@
+name: Change-point testing
+
+on:
+  push:
+    branches:
+      - issue-1294-functional-testing-module
+
+jobs:
+
+  run-functional-testing:
+    name: Change-point testing
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out PINTS repository
+        uses: actions/checkout@v2
+        with:
+          path: 'pints'
+
+      - name: Check out functional-testing-2 repository
+        uses: actions/checkout@v2
+        with:
+          repository: 'pints-team/functional-testing-2'
+          ref: 'new-infra'
+          path: 'functional-testing-2'
+          token: ${{ secrets.FUNCTIONAL_TESTING_TOKEN }}
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+          architecture: x64
+
+      - name: install pints
+        run: |
+          python --version
+          python -m pip install --upgrade pip setuptools wheel
+          python -m pip install .
+        working-directory: 'pints'
+
+      - name: install functional-testing-2
+        run: |
+          python -m pip install .
+        working-directory: 'functional-testing-2'
+
+      - name: run change-point testing
+        run: |
+          python run_all_the_things.py
+        working-directory: 'functional-testing-2'
+
+      - name: commit changes
+        run: |
+          git config user.name "github-workflow"
+          git config user.email "github-workflow"
+          git add --all
+          git commit -m "Run of change-point testing" || echo "No changes to commit"
+          git push
+        working-directory: 'functional-testing-2'

pints/cptests/__init__.py

@@ -0,0 +1,52 @@
+#
+# Change point tests for PINTS.
+#
+# This file is part of PINTS (https://github.com/pints-team/pints/) which is
+# released under the BSD 3-clause license. See accompanying LICENSE.md for
+# copyright notice and full license details.
+#
+
+# Import all problem classes straight into this module, so that they can be
+# addressed as e.g. pints.cptests.RunMcmcMethodOnAnnulus.
+from ._problems import (    # noqa
+    RunMcmcMethodOnAnnulus,
+    RunMcmcMethodOnBanana,
+    RunMcmcMethodOnCone,
+    RunMcmcMethodOnCorrelatedGaussian,
+    RunMcmcMethodOnHighDimensionalGaussian,
+    RunMcmcMethodOnMultimodalGaussian,
+    RunMcmcMethodOnProblem,
+    RunMcmcMethodOnTwoDimGaussian,
+    RunOptimiserOnBoundedFitzhughNagumo,
+    RunOptimiserOnBoundedUntransformedLogistic,
+    RunOptimiserOnProblem,
+    RunOptimiserOnRosenbrockError,
+    RunOptimiserOnTwoDimParabola,
+)
+
+# Import all test modules (not methods!) directly into this method, so that
+# they can be addressed as e.g.
+# pints.cptests.dram_acmc.two_dim_gaussian().
+from . import (     # noqa
+    cmaes,
+    cmaes_bare,
+    differential_evolution_mcmc,
+    dram_acmc,
+    dream_mcmc,
+    emcee_hammer_mcmc,
+    haario_acmc,
+    haario_bardenet_acmc,
+    hamiltonian_mcmc,
+    mala_mcmc,
+    metropolis_random_walk_mcmc,
+    monomial_gamma_hamiltonian_mcmc,
+    no_u_turn_mcmc,
+    population_mcmc,
+    relativistic_mcmc,
+    slice_stepout_mcmc,
+    slice_doubling_mcmc,
+)
+
+
+# Test discovery methods
+from ._discovery import tests   # noqa

pints/cptests/_discovery.py

@@ -0,0 +1,40 @@
+#
+# Change point test discovery methods for PINTS.
+#
+# This file is part of PINTS (https://github.com/pints-team/pints/) which is
+# released under the BSD 3-clause license. See accompanying LICENSE.md for
+# copyright notice and full license details.
+#
+import inspect
+
+import pints.cptests as cpt
+
+
+def tests(method=None):
+    """
+    Returns a list of all change point tests, each represented as a tuple
+    ``(method, test)`` where ``method`` is the PINTS class being tested and
+    ``test`` is a callable that returns the test results.
+
+    If the optional argument ``method`` is given, only tests for this method
+    are returned.
+    """
+    # Get all modules imported into this module
+    modules = [getattr(cpt, x) for x in dir(cpt) if not x.startswith('_')]
+    modules = [x for x in modules if inspect.ismodule(x)]
+
+    # Look for (explicitly defined) tests
+    tests = []
+    for module in modules:
+        try:
+            m_method = module._method
+            m_tests = module._change_point_tests
+        except AttributeError:
+            continue
+
+        if method is None or method == m_method:
+            for test in m_tests:
+                tests.append((m_method, test))
+
+    return tests
+