From 8733a38b7732302b4b6f581702b94afa85bf9ba8 Mon Sep 17 00:00:00 2001 From: bouthilx Date: Wed, 24 Nov 2021 02:31:54 +0000 Subject: [PATCH 1/3] Update backward comp test versions --- tests/functional/backward_compatibility/versions.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/backward_compatibility/versions.txt b/tests/functional/backward_compatibility/versions.txt index c2c40c355..e055c318d 100644 --- a/tests/functional/backward_compatibility/versions.txt +++ b/tests/functional/backward_compatibility/versions.txt @@ -10,3 +10,4 @@ 0.1.15 0.1.16 0.1.17 +0.2.0 From ed034dee4dd443fc9878658745a5b04bfa537980 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Thu, 25 Nov 2021 09:44:09 -0500 Subject: [PATCH 2/3] Fix ASHA for identical objectives Why: The sorting of trials during promotion was not only looking at objectives, but also at trials. Problem is, trials are not comparable. This bug was not detected because during tests trials always have a different objective, hence the comparisons would always stop at the objective when comparing the tuples (objective, trial). How: Add tests with identical objectives, and use objective only for sorting. --- src/orion/algo/asha.py | 9 +++++--- src/orion/algo/hyperband.py | 3 ++- tests/unittests/algo/test_asha.py | 30 ++++++++++++++++++++++++++ tests/unittests/algo/test_hyperband.py | 30 ++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 4 deletions(-) diff --git a/src/orion/algo/asha.py b/src/orion/algo/asha.py index 163c98882..1e7b274e0 100644 --- a/src/orion/algo/asha.py +++ b/src/orion/algo/asha.py @@ -232,9 +232,12 @@ def get_candidates(self, rung_id): rung = list( sorted( - (objective, trial) - for objective, trial in rung.values() - if objective is not None + ( + (objective, trial) + for objective, trial in rung.values() + if objective is not None + ), + key=lambda item: item[0], ) ) k = len(rung) // self.hyperband.reduction_factor diff --git a/src/orion/algo/hyperband.py b/src/orion/algo/hyperband.py index 39ee75010..dbc3cdc92 100644 --- a/src/orion/algo/hyperband.py +++ b/src/orion/algo/hyperband.py @@ -341,7 +341,8 @@ def suggest(self, num): ) else: logger.warning( - f"{self.__class__.__name__} cannot suggest new samples, exit." + f"{self.__class__.__name__} cannot suggest new samples and must wait " + "for trials to complete." ) return [] diff --git a/tests/unittests/algo/test_asha.py b/tests/unittests/algo/test_asha.py index 74ec08ecf..f86dec829 100644 --- a/tests/unittests/algo/test_asha.py +++ b/tests/unittests/algo/test_asha.py @@ -574,6 +574,36 @@ def test_suggest_promote_many_plus_random( == 20 - 2 - 3 * 3 ) + def test_suggest_promote_identic_objectives( + self, asha, bracket, big_rung_0, big_rung_1 + ): + """Test that identic objectives are handled properly""" + asha.brackets = [bracket] + bracket.asha = asha + + n_trials = 9 + resources = 1 + + results = {} + for param in np.linspace(0, 8, 9): + trial = create_trial_for_hb((resources, param), objective=0) + trial_hash = trial.compute_trial_hash( + trial, + ignore_fidelity=True, + ignore_experiment=True, + ) + results[trial_hash] = (trial.objective.value, trial) + + bracket.rungs[0] = dict(n_trials=n_trials, resources=resources, results=results) + + candidates = asha.suggest(2) + + assert len(candidates) == 2 + assert ( + sum(1 for trial in candidates if trial.params[asha.fidelity_index] == 3) + == 2 + ) + class TestGenericASHA(BaseAlgoTests): algo_name = "asha" diff --git a/tests/unittests/algo/test_hyperband.py b/tests/unittests/algo/test_hyperband.py index dad44b9b8..58da7da18 100644 --- a/tests/unittests/algo/test_hyperband.py +++ b/tests/unittests/algo/test_hyperband.py @@ -584,6 +584,36 @@ def test_suggest_promote(self, hyperband, bracket, rung_0): assert points[1].params == {"epoch": 3, "lr": 1} assert points[2].params == {"epoch": 3, "lr": 2} + def test_suggest_promote_identic_objectives(self, hyperband, bracket): + """Test that identic objectives are handled properly""" + hyperband.brackets = [bracket] + bracket.hyperband = hyperband + + n_trials = 9 + resources = 1 + + results = {} + for param in np.linspace(0, 8, 9): + trial = create_trial_for_hb((resources, param), objective=0) + trial_hash = trial.compute_trial_hash( + trial, + ignore_fidelity=True, + ignore_experiment=True, + ) + results[trial_hash] = (trial.objective.value, trial) + + bracket.rungs[0] = dict(n_trials=n_trials, resources=resources, results=results) + + candidates = hyperband.suggest(2) + + assert len(candidates) == 2 + assert ( + sum( + 1 for trial in candidates if trial.params[hyperband.fidelity_index] == 3 + ) + == 2 + ) + def test_is_filled(self, hyperband, bracket, rung_0, rung_1, rung_2): """Test that Hyperband bracket detects when rung is filled.""" hyperband.brackets = [bracket] From ddce1a2f03544a120a52aaaf715462bf9cb3b9b7 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Thu, 25 Nov 2021 11:06:26 -0500 Subject: [PATCH 3/3] Adapt doc to v0.2.1 --- README.rst | 4 ++-- ROADMAP.md | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 97b5b8cfd..5bdd67b92 100644 --- a/README.rst +++ b/README.rst @@ -118,7 +118,7 @@ If you use Oríon for published work, please cite our work using the following b .. code-block:: bibtex - @software{xavier_bouthillier_2021_0_2_0, + @software{xavier_bouthillier_2021_0_2_1, author = {Xavier Bouthillier and Christos Tsirigotis and François Corneau-Tremblay and @@ -145,7 +145,7 @@ If you use Oríon for published work, please cite our work using the following b month = nov, year = 2021, publisher = {Zenodo}, - version = {v0.2.0}, + version = {v0.2.1}, doi = {10.5281/zenodo.3478592}, url = {https://doi.org/10.5281/zenodo.3478592} } diff --git a/ROADMAP.md b/ROADMAP.md index 1a135a979..4add33c86 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -3,21 +3,21 @@ Last update Nov 23rd, 2021 ## Next releases - Short-Term -### v0.2.1 +### v0.2.2 - New master process to enhance parallelisation efficiency. - [PBT](https://arxiv.org/abs/1711.09846) -### v0.2.2 +### v0.2.3 - Use shared algo serialization instead of replications to enhance parallelisation efficiency. - [DEBH](https://arxiv.org/abs/2105.09821) -### v0.2.3 +### v0.2.4 - [HEBO](https://github.com/huawei-noah/HEBO/tree/master/HEBO/archived_submissions/hebo) -### v0.2.4 +### v0.2.5 - [BOHB](https://ml.informatik.uni-freiburg.de/papers/18-ICML-BOHB.pdf)