From 98d6dbb2d274fc45fc78b84d2968199852372cb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Fri, 5 Jul 2019 14:00:16 -0400 Subject: [PATCH 01/50] Add test to check for interrupted trials --- tests/unittests/core/worker/test_consumer.py | 44 ++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tests/unittests/core/worker/test_consumer.py diff --git a/tests/unittests/core/worker/test_consumer.py b/tests/unittests/core/worker/test_consumer.py new file mode 100644 index 000000000..0d12b108d --- /dev/null +++ b/tests/unittests/core/worker/test_consumer.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Collection of tests for :mod:`orion.core.worker.consumer`.""" + +import pytest + +from orion.core.io.experiment_builder import ExperimentBuilder +from orion.core.utils.format_trials import tuple_to_trial +import orion.core.worker.consumer as consumer + + +Consumer = consumer.Consumer + + +@pytest.fixture +def config(exp_config): + """Return a configuration.""" + config = exp_config[0][0] + config['metadata']['user_args'] = ['--x~uniform(-50, 50)'] + config['name'] = 'exp' + return config + + +@pytest.mark.usefixtures("create_db_instance") +def test_trials_interrupted_keyboard_int(config, monkeypatch): + """Check if a trial is set as interrupted when a KeyboardInterrupt is raised.""" + def mock_Popen(*args, **kwargs): + raise KeyboardInterrupt + + exp = ExperimentBuilder().build_from(config) + + monkeypatch.setattr(consumer.subprocess, "Popen", mock_Popen) + + trial = tuple_to_trial((1.0,), exp.space) + + exp.register_trial(trial) + + con = Consumer(exp) + + with pytest.raises(KeyboardInterrupt): + con.consume(trial) + + trials = exp.fetch_trials({'status': 'interrupted'}) + assert len(trials) From 8d3020755977ede01569c3854d9f2ebbe59eeffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Fri, 5 Jul 2019 14:53:31 -0400 Subject: [PATCH 02/50] Handle interrupted trials inside Consumer --- src/orion/core/worker/consumer.py | 54 +++++++++----------- tests/unittests/core/worker/test_consumer.py | 3 +- 2 files changed, 25 insertions(+), 32 deletions(-) diff --git a/src/orion/core/worker/consumer.py b/src/orion/core/worker/consumer.py index 5565d7a87..7c763a8a5 100644 --- a/src/orion/core/worker/consumer.py +++ b/src/orion/core/worker/consumer.py @@ -11,7 +11,6 @@ import logging import os import subprocess -import sys import tempfile from orion.core.io.convert import JSONConverter @@ -62,6 +61,8 @@ def __init__(self, experiment): self.converter = JSONConverter() + self.current_trial = None + def consume(self, trial): """Execute user's script as a block box using the options contained within `trial`. @@ -74,19 +75,25 @@ def consume(self, trial): prefix = self.experiment.name + "_" suffix = trial.id - with WorkingDir(self.working_dir, temp_dir, - prefix=prefix, suffix=suffix) as workdirname: - log.debug("## New consumer context: %s", workdirname) - completed_trial = self._consume(trial, workdirname) - - if completed_trial is not None: - log.debug("### Register successfully evaluated %s.", completed_trial) - self.experiment.push_completed_trial(completed_trial) - else: + try: + with WorkingDir(self.working_dir, temp_dir, + prefix=prefix, suffix=suffix) as workdirname: + log.debug("## New consumer context: %s", workdirname) + self._consume(trial, workdirname) + except KeyboardInterrupt: + log.debug("### Save %s as interrupted.", trial) + trial.status = 'interrupted' + Database().write('trials', trial.to_dict(), + query={'_id': trial.id}) + raise + except RuntimeError: log.debug("### Save %s as broken.", trial) trial.status = 'broken' Database().write('trials', trial.to_dict(), query={'_id': trial.id}) + else: + log.debug("### Register successfully evaluated %s.", trial) + self.experiment.push_completed_trial(trial) def _consume(self, trial, workdirname): config_file = tempfile.NamedTemporaryFile(mode='w', prefix='trial_', @@ -104,19 +111,8 @@ def _consume(self, trial, workdirname): cmd_args = self.template_builder.build_to(config_file.name, trial, self.experiment) log.debug("## Launch user's script as a subprocess and wait for finish.") - script_process = self.launch_process(results_file.name, cmd_args) - - if script_process is None: - return None - returncode = script_process.wait() - - if returncode != 0: - log.error("Something went wrong. Check logs. Process " - "returned with code %d !", returncode) - if returncode == 2: - sys.exit(2) - return None + self.execute_process(results_file.name, cmd_args) log.debug("## Parse results from file and fill corresponding Trial object.") results = self.converter.parse(results_file.name) @@ -125,18 +121,14 @@ def _consume(self, trial, workdirname): type=res['type'], value=res['value']) for res in results] - return trial - - def launch_process(self, results_filename, cmd_args): + def execute_process(self, results_filename, cmd_args): """Facilitate launching a black-box trial.""" env = dict(os.environ) env['ORION_RESULTS_PATH'] = str(results_filename) command = [self.script_path] + cmd_args process = subprocess.Popen(command, env=env) - returncode = process.poll() - if returncode is not None and returncode < 0: - log.error("Failed to execute script to evaluate trial. Process " - "returned with code %d !", returncode) - return None - return process + return_code = process.wait() + if return_code != 0: + raise RuntimeError("Something went wrong. Check logs. Process " + "returned with code {} !".format(return_code)) diff --git a/tests/unittests/core/worker/test_consumer.py b/tests/unittests/core/worker/test_consumer.py index 0d12b108d..ab0b321d2 100644 --- a/tests/unittests/core/worker/test_consumer.py +++ b/tests/unittests/core/worker/test_consumer.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """Collection of tests for :mod:`orion.core.worker.consumer`.""" +import subprocess import pytest @@ -29,7 +30,7 @@ def mock_Popen(*args, **kwargs): exp = ExperimentBuilder().build_from(config) - monkeypatch.setattr(consumer.subprocess, "Popen", mock_Popen) + monkeypatch.setattr(subprocess, "Popen", mock_Popen) trial = tuple_to_trial((1.0,), exp.space) From a04cb3c87cff1d6fc07ebe9f4aa1e4e46597d000 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Fri, 5 Jul 2019 16:58:08 -0400 Subject: [PATCH 03/50] Add signal handling and tests --- src/orion/core/worker/consumer.py | 11 ++++++++ tests/functional/demo/database_config.yaml | 3 ++ tests/unittests/core/worker/test_consumer.py | 29 +++++++++++++++++++- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/orion/core/worker/consumer.py b/src/orion/core/worker/consumer.py index 7c763a8a5..6959905d5 100644 --- a/src/orion/core/worker/consumer.py +++ b/src/orion/core/worker/consumer.py @@ -10,6 +10,7 @@ """ import logging import os +import signal import subprocess import tempfile @@ -21,6 +22,12 @@ log = logging.getLogger(__name__) +# pylint: disable = unused-argument +def _handler(signum, frame): + log.error('Oríon has been interrupted.') + print("Allo") + raise KeyboardInterrupt + class Consumer(object): """Consume a trial by using it to initialize a black-box box to evaluate it. @@ -91,6 +98,7 @@ def consume(self, trial): trial.status = 'broken' Database().write('trials', trial.to_dict(), query={'_id': trial.id}) + raise else: log.debug("### Register successfully evaluated %s.", trial) self.experiment.push_completed_trial(trial) @@ -126,9 +134,12 @@ def execute_process(self, results_filename, cmd_args): env = dict(os.environ) env['ORION_RESULTS_PATH'] = str(results_filename) command = [self.script_path] + cmd_args + + signal.signal(signal.SIGTERM, _handler) process = subprocess.Popen(command, env=env) return_code = process.wait() + print(return_code) if return_code != 0: raise RuntimeError("Something went wrong. Check logs. Process " "returned with code {} !".format(return_code)) diff --git a/tests/functional/demo/database_config.yaml b/tests/functional/demo/database_config.yaml index 4f72cd22f..cc708fb09 100644 --- a/tests/functional/demo/database_config.yaml +++ b/tests/functional/demo/database_config.yaml @@ -1,3 +1,6 @@ +max_trials: 1 +pool_size: 1 + database: type: 'mongodb' name: 'orion_test' diff --git a/tests/unittests/core/worker/test_consumer.py b/tests/unittests/core/worker/test_consumer.py index ab0b321d2..e96b1792c 100644 --- a/tests/unittests/core/worker/test_consumer.py +++ b/tests/unittests/core/worker/test_consumer.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """Collection of tests for :mod:`orion.core.worker.consumer`.""" +import os +import signal import subprocess import pytest @@ -30,7 +32,7 @@ def mock_Popen(*args, **kwargs): exp = ExperimentBuilder().build_from(config) - monkeypatch.setattr(subprocess, "Popen", mock_Popen) + monkeypatch.setattr(consumer.subprocess, "Popen", mock_Popen) trial = tuple_to_trial((1.0,), exp.space) @@ -43,3 +45,28 @@ def mock_Popen(*args, **kwargs): trials = exp.fetch_trials({'status': 'interrupted'}) assert len(trials) + assert trials[0].id == trial.id + + +@pytest.mark.usefixtures("create_db_instance") +def test_trials_interrupted_sigterm(config, monkeypatch): + """Check if a trial is set as interrupted when a KeyboardInterrupt is raised.""" + def mock_popen(*args, **kwargs): + os.kill(os.getpid(), signal.SIGTERM) + + exp = ExperimentBuilder().build_from(config) + + monkeypatch.setattr(subprocess.Popen, "wait", mock_popen) + + trial = tuple_to_trial((1.0,), exp.space) + + exp.register_trial(trial) + + con = Consumer(exp) + + with pytest.raises(KeyboardInterrupt): + con.consume(trial) + + trials = exp.fetch_trials({'status': 'interrupted'}) + assert len(trials) + assert trials[0].id == trial.id From bf30ff89bb1227febe92b6fa553dc84a89465aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Fri, 5 Jul 2019 17:00:03 -0400 Subject: [PATCH 04/50] Fix flake8 --- src/orion/core/worker/consumer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/orion/core/worker/consumer.py b/src/orion/core/worker/consumer.py index 6959905d5..1739c06ad 100644 --- a/src/orion/core/worker/consumer.py +++ b/src/orion/core/worker/consumer.py @@ -22,6 +22,7 @@ log = logging.getLogger(__name__) + # pylint: disable = unused-argument def _handler(signum, frame): log.error('Oríon has been interrupted.') From 76f5a0080b9a1867cfcfce8ee9f8ccd03428ab4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Fri, 5 Jul 2019 17:02:25 -0400 Subject: [PATCH 05/50] Say hi --- src/orion/core/worker/consumer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/orion/core/worker/consumer.py b/src/orion/core/worker/consumer.py index 1739c06ad..a1abf4c97 100644 --- a/src/orion/core/worker/consumer.py +++ b/src/orion/core/worker/consumer.py @@ -26,7 +26,6 @@ # pylint: disable = unused-argument def _handler(signum, frame): log.error('Oríon has been interrupted.') - print("Allo") raise KeyboardInterrupt From 9a05b31f6351fce9cd3ebbb6ab1119a0d3d27ebd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Mon, 8 Jul 2019 10:18:33 -0400 Subject: [PATCH 06/50] Fix minor text issues --- src/orion/core/worker/consumer.py | 1 - tests/unittests/core/worker/test_consumer.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/orion/core/worker/consumer.py b/src/orion/core/worker/consumer.py index a1abf4c97..78e4d5538 100644 --- a/src/orion/core/worker/consumer.py +++ b/src/orion/core/worker/consumer.py @@ -139,7 +139,6 @@ def execute_process(self, results_filename, cmd_args): process = subprocess.Popen(command, env=env) return_code = process.wait() - print(return_code) if return_code != 0: raise RuntimeError("Something went wrong. Check logs. Process " "returned with code {} !".format(return_code)) diff --git a/tests/unittests/core/worker/test_consumer.py b/tests/unittests/core/worker/test_consumer.py index e96b1792c..a715c5d45 100644 --- a/tests/unittests/core/worker/test_consumer.py +++ b/tests/unittests/core/worker/test_consumer.py @@ -50,7 +50,7 @@ def mock_Popen(*args, **kwargs): @pytest.mark.usefixtures("create_db_instance") def test_trials_interrupted_sigterm(config, monkeypatch): - """Check if a trial is set as interrupted when a KeyboardInterrupt is raised.""" + """Check if a trial is set as interrupted when a signal is raised.""" def mock_popen(*args, **kwargs): os.kill(os.getpid(), signal.SIGTERM) From b930a6f9597ee9c70b81daf43750406cfda1220e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Tue, 9 Jul 2019 12:49:22 -0400 Subject: [PATCH 07/50] Add working directory to Trial --- src/orion/core/worker/trial.py | 15 ++++++++++++++- tests/unittests/core/test_trial.py | 2 ++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/orion/core/worker/trial.py b/src/orion/core/worker/trial.py index 67297d53a..55795fa3e 100644 --- a/src/orion/core/worker/trial.py +++ b/src/orion/core/worker/trial.py @@ -147,7 +147,7 @@ class Param(Value): __slots__ = () allowed_types = ('integer', 'real', 'categorical', 'fidelity') - __slots__ = ('experiment', '_id', '_status', 'worker', + __slots__ = ('experiment', '_id', '_status', 'worker', '_working_dir', 'submit_time', 'start_time', 'end_time', '_results', 'params', 'parents') allowed_stati = ('new', 'reserved', 'suspended', 'completed', 'interrupted', 'broken') @@ -181,6 +181,9 @@ def to_dict(self): trial_dictionary = dict() for attrname in self.__slots__: + if attrname == "_working_dir": + continue + attrname = attrname.lstrip("_") trial_dictionary[attrname] = getattr(self, attrname) @@ -220,6 +223,16 @@ def results(self, results): self._results = results + @property + def working_dir(self): + """Return the current working directory of the trial.""" + return self._working_dir + + @working_dir.setter + def working_dir(self, value): + """Change the current working directory of the trial.""" + self._working_dir = value + @property def status(self): """For meaning of property type, see `Trial.status`.""" diff --git a/tests/unittests/core/test_trial.py b/tests/unittests/core/test_trial.py index 9e5b797ee..3fb58617e 100644 --- a/tests/unittests/core/test_trial.py +++ b/tests/unittests/core/test_trial.py @@ -21,6 +21,7 @@ def test_init_empty(self): assert t.end_time is None assert t.results == [] assert t.params == [] + assert t.working_dir is None def test_init_full(self, exp_config): """Initialize with a dictionary with complete specification.""" @@ -36,6 +37,7 @@ def test_init_full(self, exp_config): assert t.results[0].type == exp_config[1][1]['results'][0]['type'] assert t.results[0].value == exp_config[1][1]['results'][0]['value'] assert list(map(lambda x: x.to_dict(), t.params)) == exp_config[1][1]['params'] + assert t.working_dir is None def test_bad_access(self): """Other than `Trial.__slots__` are not allowed.""" From 1c8089502f1f76f8f85995350d6d3b25af80260b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Tue, 9 Jul 2019 12:49:57 -0400 Subject: [PATCH 08/50] Add working directory support to Consumer --- src/orion/core/worker/consumer.py | 3 +-- tests/unittests/core/worker/test_consumer.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/orion/core/worker/consumer.py b/src/orion/core/worker/consumer.py index 78e4d5538..f4b2c79ae 100644 --- a/src/orion/core/worker/consumer.py +++ b/src/orion/core/worker/consumer.py @@ -68,8 +68,6 @@ def __init__(self, experiment): self.converter = JSONConverter() - self.current_trial = None - def consume(self, trial): """Execute user's script as a block box using the options contained within `trial`. @@ -86,6 +84,7 @@ def consume(self, trial): with WorkingDir(self.working_dir, temp_dir, prefix=prefix, suffix=suffix) as workdirname: log.debug("## New consumer context: %s", workdirname) + trial.working_dir = workdirname self._consume(trial, workdirname) except KeyboardInterrupt: log.debug("### Save %s as interrupted.", trial) diff --git a/tests/unittests/core/worker/test_consumer.py b/tests/unittests/core/worker/test_consumer.py index a715c5d45..fcc8c0a45 100644 --- a/tests/unittests/core/worker/test_consumer.py +++ b/tests/unittests/core/worker/test_consumer.py @@ -21,6 +21,7 @@ def config(exp_config): config = exp_config[0][0] config['metadata']['user_args'] = ['--x~uniform(-50, 50)'] config['name'] = 'exp' + config['working_dir'] = "/tmp/orion" return config @@ -70,3 +71,19 @@ def mock_popen(*args, **kwargs): trials = exp.fetch_trials({'status': 'interrupted'}) assert len(trials) assert trials[0].id == trial.id + + +@pytest.mark.usefixtures("create_db_instance") +def test_trial_working_dir_is_changed(config, monkeypatch): + """Check that trial has its working_dir attribute changed.""" + exp = ExperimentBuilder().build_from(config) + + trial = tuple_to_trial((1.0,), exp.space) + + exp.register_trial(trial) + + con = Consumer(exp) + con.consume(trial) + + assert trial.working_dir is not None + assert trial.working_dir == con.working_dir + "/exp_" + trial.id From fc4351223fb02a305455863f142988db681e0416 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Tue, 9 Jul 2019 13:47:51 -0400 Subject: [PATCH 09/50] Update versions in Roadmap Why: We needed to make a hotfix with v0.1.4 and forgot to update the roadmap accordingly --- ROADMAP.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index b34d08d91..8df15f072 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -3,7 +3,7 @@ Last update July 5th, 2019 ## Next releases - Short-Term -### v0.1.4 +### v0.1.5 #### Trial interruption/suspension/resumption @@ -13,7 +13,7 @@ Handle interrupted or lost trials so that they can be automatically resumed by r See [#125](https://github.com/Epistimio/orion/issues/125) -### v0.1.5 +### v0.1.6 #### Auto-resolution of EVC From ae0cebfdc4ba3802f18ce53260d50bc9a972d09c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Mon, 8 Jul 2019 15:40:47 -0400 Subject: [PATCH 10/50] Add heartbeat attribute to Trial --- src/orion/core/worker/experiment.py | 6 +- src/orion/core/worker/trial.py | 4 +- src/orion/core/worker/trial_monitor.py | 55 +++++++++ .../core/worker/test_trial_monitor.py | 112 ++++++++++++++++++ 4 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 src/orion/core/worker/trial_monitor.py create mode 100644 tests/unittests/core/worker/test_trial_monitor.py diff --git a/src/orion/core/worker/experiment.py b/src/orion/core/worker/experiment.py index 8c1ec4944..bb36655cb 100644 --- a/src/orion/core/worker/experiment.py +++ b/src/orion/core/worker/experiment.py @@ -28,6 +28,7 @@ from orion.core.worker.strategy import (BaseParallelStrategy, Strategy) from orion.core.worker.trial import Trial +from orion.core.worker.trial_monitor import TrialMonitor log = logging.getLogger(__name__) @@ -245,7 +246,7 @@ def reserve_trial(self, score_handle=None): # status meanwhile, read_and_write will fail, because query will fail. query = {'_id': selected_trial.id, 'status': selected_trial.status} - update = dict(status='reserved') + update = dict(status='reserved', heartbeat=datetime.datetime.utcnow()) if selected_trial.status == 'new': update["start_time"] = datetime.datetime.utcnow() @@ -258,10 +259,11 @@ def reserve_trial(self, score_handle=None): else: selected_trial = Trial(**selected_trial_dict) + TrialMonitor(self, selected_trial.id).start() return selected_trial def push_completed_trial(self, trial): - """Inform database about an evaluated `trial` with results. + """Inform database about an evaluated `trial` with resultlts. :param trial: Corresponds to a successful evaluation of a particular run. :type trial: `Trial` diff --git a/src/orion/core/worker/trial.py b/src/orion/core/worker/trial.py index 55795fa3e..0775dfa40 100644 --- a/src/orion/core/worker/trial.py +++ b/src/orion/core/worker/trial.py @@ -23,6 +23,8 @@ class Trial(object): experiment : str Unique identifier for the experiment that produced this trial. Same as an `Experiment._id`. + heartbeat : datetime.datetime + Last time trial was identified as being alive. status : str Indicates how this trial is currently being used. Can take the following values: @@ -147,7 +149,7 @@ class Param(Value): __slots__ = () allowed_types = ('integer', 'real', 'categorical', 'fidelity') - __slots__ = ('experiment', '_id', '_status', 'worker', '_working_dir', + __slots__ = ('experiment', '_id', '_status', 'worker', '_working_dir', 'heartbeat', 'submit_time', 'start_time', 'end_time', '_results', 'params', 'parents') allowed_stati = ('new', 'reserved', 'suspended', 'completed', 'interrupted', 'broken') diff --git a/src/orion/core/worker/trial_monitor.py b/src/orion/core/worker/trial_monitor.py new file mode 100644 index 000000000..84f2f139e --- /dev/null +++ b/src/orion/core/worker/trial_monitor.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +""" +:mod:`orion.core.worker.trial_monitor` -- Monitor trial execution +================================================================= +.. module:: trial_monitor + :platform: Unix + :synopsis: Monitor trials and update their heartbeat + +""" +import datetime +import threading + +from orion.core.io.database import Database + + +class TrialMonitor(threading.Thread): + """Monitor a given trial inside a thread, updating its heartbeat + at a given interval of time. + + Parameters + ---------- + exp: Experiment + The current Experiment. + + """ + + def __init__(self, exp, trial_id, wait_time=60): + """Initialize a TrialMonitor.""" + threading.Thread.__init__(self) + self.stopped = threading.Event() + self.exp = exp + self.trial_id = trial_id + self.wait_time = wait_time + + def stop(self): + """Stop monitoring.""" + self.stopped.set() + self.join() + + def run(self): + """Run the trial monitoring every given interval.""" + while not self.stopped.wait(self.wait_time): + self._monitor_trial() + + def _monitor_trial(self): + query = {'_id': self.trial_id, 'status': 'reserved'} + trials = self.exp.fetch_trials(query) + print(trials) + + if len(trials): + update = dict(heartbeat=datetime.datetime.utcnow()) + print("Changing.") + Database().write('trials', update, query) + else: + self.stopped.set() diff --git a/tests/unittests/core/worker/test_trial_monitor.py b/tests/unittests/core/worker/test_trial_monitor.py new file mode 100644 index 000000000..c771d7109 --- /dev/null +++ b/tests/unittests/core/worker/test_trial_monitor.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Collection of tests for :mod:`orion.core.worker.consumer`.""" +import datetime +import time + +import pytest + +from orion.core.io.database import Database +from orion.core.io.experiment_builder import ExperimentBuilder +from orion.core.utils.format_trials import tuple_to_trial +from orion.core.worker.trial_monitor import TrialMonitor + + +@pytest.fixture +def config(exp_config): + """Return a configuration.""" + config = exp_config[0][0] + config['metadata']['user_args'] = ['--x~uniform(-50, 50)'] + config['name'] = 'exp' + return config + + +@pytest.mark.usefixtures("create_db_instance") +def test_trial_update_heartbeat(config): + """Test that the heartbeat of a trial has been updated.""" + exp = ExperimentBuilder().build_from(config) + trial = tuple_to_trial((1.0,), exp.space) + heartbeat = datetime.datetime.utcnow() + trial.heartbeat = heartbeat + + data = {'_id': trial.id, 'status': 'reserved', 'heartbeat': heartbeat, 'experiment': exp.id} + + Database().write('trials', data) + + trial_monitor = TrialMonitor(exp, trial.id, wait_time=1) + + trial_monitor.start() + time.sleep(1) + + trials = exp.fetch_trials({'_id': trial.id, 'status': 'reserved'}) + + assert trial.heartbeat != trials[0].heartbeat + + heartbeat = trials[0].heartbeat + + time.sleep(1) + + trials = exp.fetch_trials({'_id': trial.id, 'status': 'reserved'}) + + assert heartbeat != trials[0].heartbeat + trial_monitor.stop() + + +@pytest.mark.usefixtures("create_db_instance") +def test_trial_heartbeat_not_updated(config): + """Test that the heartbeat of a trial is not updated when trial is not longer reserved.""" + exp = ExperimentBuilder().build_from(config) + trial = tuple_to_trial((1.0,), exp.space) + heartbeat = datetime.datetime.utcnow() + trial.heartbeat = heartbeat + + data = {'_id': trial.id, 'status': 'reserved', 'heartbeat': heartbeat, 'experiment': exp.id} + + Database().write('trials', data) + + trial_monitor = TrialMonitor(exp, trial.id, wait_time=1) + + trial_monitor.start() + time.sleep(1) + + trials = exp.fetch_trials({'_id': trial.id, 'status': 'reserved'}) + + assert trial.heartbeat != trials[0].heartbeat + + data = {'status': 'interrupted'} + Database().write('trials', data, query=dict(_id=trial.id)) + + time.sleep(1) + + trial_monitor.join() + assert 1 + + +@pytest.mark.usefixtures("create_db_instance") +def test_trial_heartbeat_not_updated_inbetween(config): + """Test that the heartbeat of a trial is not updated before wait time.""" + exp = ExperimentBuilder().build_from(config) + trial = tuple_to_trial((1.0,), exp.space) + heartbeat = datetime.datetime.utcnow().replace(microsecond=0) + trial.heartbeat = heartbeat + + data = {'_id': trial.id, 'status': 'reserved', 'heartbeat': heartbeat, 'experiment': exp.id} + + Database().write('trials', data) + + trial_monitor = TrialMonitor(exp, trial.id, wait_time=5) + + trial_monitor.start() + time.sleep(1) + + trials = exp.fetch_trials({'_id': trial.id, 'status': 'reserved'}) + assert trial.heartbeat == trials[0].heartbeat + + heartbeat = trials[0].heartbeat + + time.sleep(4) + + trials = exp.fetch_trials({'_id': trial.id, 'status': 'reserved'}) + + assert heartbeat != trials[0].heartbeat + trial_monitor.stop() From ad35493950b567576dc9177c426159a29d2a0f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Mon, 8 Jul 2019 15:44:16 -0400 Subject: [PATCH 11/50] Change condition for emptiness --- src/orion/core/worker/trial_monitor.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/orion/core/worker/trial_monitor.py b/src/orion/core/worker/trial_monitor.py index 84f2f139e..5e1368987 100644 --- a/src/orion/core/worker/trial_monitor.py +++ b/src/orion/core/worker/trial_monitor.py @@ -45,11 +45,9 @@ def run(self): def _monitor_trial(self): query = {'_id': self.trial_id, 'status': 'reserved'} trials = self.exp.fetch_trials(query) - print(trials) - if len(trials): + if trials: update = dict(heartbeat=datetime.datetime.utcnow()) - print("Changing.") Database().write('trials', update, query) else: self.stopped.set() From ee9b9328c17fa955ee2b0b7990fc354b1ae89756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Tue, 9 Jul 2019 09:50:09 -0400 Subject: [PATCH 12/50] Fix Experiment tests Why: these Experiment tests were comparing two trials thaat had different values of `heartbeat`. --- .gitignore | 2 ++ src/orion/core/worker/experiment.py | 2 +- tests/unittests/core/test_experiment.py | 8 ++++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index b6281b43c..db370f1aa 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ .spyproject .ropeproject *.log +*.pkl +*.lock # StarUML documentation *.mdj diff --git a/src/orion/core/worker/experiment.py b/src/orion/core/worker/experiment.py index bb36655cb..a82cf2552 100644 --- a/src/orion/core/worker/experiment.py +++ b/src/orion/core/worker/experiment.py @@ -258,8 +258,8 @@ def reserve_trial(self, score_handle=None): selected_trial = self.reserve_trial(score_handle=score_handle) else: selected_trial = Trial(**selected_trial_dict) + TrialMonitor(self, selected_trial.id).start() - TrialMonitor(self, selected_trial.id).start() return selected_trial def push_completed_trial(self, trial): diff --git a/tests/unittests/core/test_experiment.py b/tests/unittests/core/test_experiment.py index 1297cf01a..62699bcce 100644 --- a/tests/unittests/core/test_experiment.py +++ b/tests/unittests/core/test_experiment.py @@ -589,10 +589,11 @@ def test_reserve_success(self, exp_config, hacked_exp, random_dt): trial = hacked_exp.reserve_trial() exp_config[1][3]['status'] = 'reserved' exp_config[1][3]['start_time'] = random_dt + exp_config[1][3]['heartbeat'] = random_dt assert trial.to_dict() == exp_config[1][3] @pytest.mark.usefixtures("patch_sample2") - def test_reserve_success2(self, exp_config, hacked_exp): + def test_reserve_success2(self, exp_config, hacked_exp, random_dt): """Successfully find new trials in db and reserve one at 'random'. Version that start_time does not get written, because the selected trial @@ -600,6 +601,7 @@ def test_reserve_success2(self, exp_config, hacked_exp): """ trial = hacked_exp.reserve_trial() exp_config[1][6]['status'] = 'reserved' + exp_config[1][6]['heartbeat'] = random_dt assert trial.to_dict() == exp_config[1][6] @pytest.mark.usefixtures("patch_sample_concurrent") @@ -608,6 +610,7 @@ def test_reserve_race_condition(self, exp_config, hacked_exp, random_dt): trial = hacked_exp.reserve_trial() exp_config[1][4]['status'] = 'reserved' exp_config[1][4]['start_time'] = random_dt + exp_config[1][4]['heartbeat'] = random_dt assert trial.to_dict() == exp_config[1][4] @pytest.mark.usefixtures("patch_sample_concurrent2") @@ -626,12 +629,13 @@ def fake_handle(self, xxx): self.times_called += 1 return self.times_called - def test_reserve_with_score(self, hacked_exp, exp_config): + def test_reserve_with_score(self, hacked_exp, exp_config, random_dt): """Reserve with a score object that can do its job.""" self.times_called = 0 hacked_exp.configure(exp_config[0][3]) trial = hacked_exp.reserve_trial(score_handle=self.fake_handle) exp_config[1][6]['status'] = 'reserved' + exp_config[1][6]['heartbeat'] = random_dt assert trial.to_dict() == exp_config[1][6] From d9d3cb8c44ff109e5e0d424a32a131f52a7641f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Tue, 9 Jul 2019 12:05:56 -0400 Subject: [PATCH 13/50] Add longer sleep time in tests --- tests/unittests/core/worker/test_trial_monitor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unittests/core/worker/test_trial_monitor.py b/tests/unittests/core/worker/test_trial_monitor.py index c771d7109..36f047c25 100644 --- a/tests/unittests/core/worker/test_trial_monitor.py +++ b/tests/unittests/core/worker/test_trial_monitor.py @@ -36,7 +36,7 @@ def test_trial_update_heartbeat(config): trial_monitor = TrialMonitor(exp, trial.id, wait_time=1) trial_monitor.start() - time.sleep(1) + time.sleep(2) trials = exp.fetch_trials({'_id': trial.id, 'status': 'reserved'}) @@ -44,7 +44,7 @@ def test_trial_update_heartbeat(config): heartbeat = trials[0].heartbeat - time.sleep(1) + time.sleep(2) trials = exp.fetch_trials({'_id': trial.id, 'status': 'reserved'}) @@ -67,7 +67,7 @@ def test_trial_heartbeat_not_updated(config): trial_monitor = TrialMonitor(exp, trial.id, wait_time=1) trial_monitor.start() - time.sleep(1) + time.sleep(2) trials = exp.fetch_trials({'_id': trial.id, 'status': 'reserved'}) @@ -76,7 +76,7 @@ def test_trial_heartbeat_not_updated(config): data = {'status': 'interrupted'} Database().write('trials', data, query=dict(_id=trial.id)) - time.sleep(1) + time.sleep(2) trial_monitor.join() assert 1 From 9fe58692cb7d0bab1a0e296e83c003542ec8aae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Tue, 9 Jul 2019 12:52:47 -0400 Subject: [PATCH 14/50] Add comment to explain test --- tests/unittests/core/worker/test_trial_monitor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittests/core/worker/test_trial_monitor.py b/tests/unittests/core/worker/test_trial_monitor.py index 36f047c25..910e1e040 100644 --- a/tests/unittests/core/worker/test_trial_monitor.py +++ b/tests/unittests/core/worker/test_trial_monitor.py @@ -78,6 +78,7 @@ def test_trial_heartbeat_not_updated(config): time.sleep(2) + # `join` blocks until all thread have finish executing. So, the test will hang if it fails. trial_monitor.join() assert 1 From 4f6f16058086fb4d9dc5dcce09f80af86fd7590d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Tue, 9 Jul 2019 13:48:02 -0400 Subject: [PATCH 15/50] Add sleep time --- tests/unittests/core/worker/test_trial_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/core/worker/test_trial_monitor.py b/tests/unittests/core/worker/test_trial_monitor.py index 910e1e040..cf7f28096 100644 --- a/tests/unittests/core/worker/test_trial_monitor.py +++ b/tests/unittests/core/worker/test_trial_monitor.py @@ -105,7 +105,7 @@ def test_trial_heartbeat_not_updated_inbetween(config): heartbeat = trials[0].heartbeat - time.sleep(4) + time.sleep(6) trials = exp.fetch_trials({'_id': trial.id, 'status': 'reserved'}) From 0a44209e0eeeb27ddf46fad060d14ad5d9a4c22f Mon Sep 17 00:00:00 2001 From: Mirko Bronzi Date: Wed, 10 Jul 2019 15:29:02 -0400 Subject: [PATCH 16/50] added log level for filelock (#201) added log level for filelock --- src/orion/core/io/database/__init__.py | 5 +++++ tests/unittests/core/test_pickleddb.py | 14 +++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/orion/core/io/database/__init__.py b/src/orion/core/io/database/__init__.py index b157b8524..ad22fa395 100644 --- a/src/orion/core/io/database/__init__.py +++ b/src/orion/core/io/database/__init__.py @@ -14,6 +14,7 @@ """ from abc import abstractmethod, abstractproperty +import logging from orion.core.utils import (AbstractSingletonType, SingletonFactory) @@ -264,3 +265,7 @@ class Database(AbstractDB, metaclass=SingletonFactory): """ pass + + +# set per-module log level +logging.getLogger('filelock').setLevel('ERROR') diff --git a/tests/unittests/core/test_pickleddb.py b/tests/unittests/core/test_pickleddb.py index f2c0f969f..c07d0733d 100644 --- a/tests/unittests/core/test_pickleddb.py +++ b/tests/unittests/core/test_pickleddb.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """Collection of tests for :mod:`orion.core.io.database.pickleddb`.""" - from datetime import datetime +import logging from multiprocessing import Pool import os @@ -263,6 +263,18 @@ def test_read_and_write_no_match(self, orion_db): assert loaded_config is None + def test_logging_when_getting_file_lock(self, caplog, orion_db): + """When logging.level is ERROR, there should be no logging.""" + logging.basicConfig(level=logging.INFO) + caplog.clear() + caplog.set_level(logging.ERROR) + # any operation will trigger the lock. + orion_db.read( + 'experiments', + {'name': 'supernaedo2', 'metadata.user': 'dendi'}) + + assert 'acquired on orion_db.pkl.lock' not in caplog.text + @pytest.mark.usefixtures("clean_db") class TestRemove(object): From a692e469959482e9750e87694d8496f667f4a1ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Wed, 10 Jul 2019 16:03:38 -0400 Subject: [PATCH 17/50] Catch lost trials and set them to interupted --- src/orion/core/io/database/ephemeraldb.py | 3 +- src/orion/core/worker/experiment.py | 22 +++++++++ tests/unittests/core/test_experiment.py | 54 ++++++++++++++++++++++- 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/src/orion/core/io/database/ephemeraldb.py b/src/orion/core/io/database/ephemeraldb.py index ba2f42609..a872e2541 100644 --- a/src/orion/core/io/database/ephemeraldb.py +++ b/src/orion/core/io/database/ephemeraldb.py @@ -278,7 +278,8 @@ class EphemeralDocument(object): operators = { "$in": (lambda a, b: a in b), "$gte": (lambda a, b: a is not None and a >= b), - "$gt": (lambda a, b: a is not None and a > b) + "$gt": (lambda a, b: a is not None and a > b), + "$lte": (lambda a, b: a is not None and a <= b), } def __init__(self, data): diff --git a/src/orion/core/worker/experiment.py b/src/orion/core/worker/experiment.py index a82cf2552..2b58339d0 100644 --- a/src/orion/core/worker/experiment.py +++ b/src/orion/core/worker/experiment.py @@ -221,6 +221,8 @@ def reserve_trial(self, score_handle=None): if score_handle is not None and not callable(score_handle): raise ValueError("Argument `score_handle` must be callable with a `Trial`.") + self.fix_lost_trials() + query = dict( experiment=self._id, status={'$in': ['new', 'suspended', 'interrupted']} @@ -262,6 +264,26 @@ def reserve_trial(self, score_handle=None): return selected_trial + def fix_lost_trials(self): + """Find lost trials and set them to interrupted. + + A lost trial is defined as a trial whose heartbeat as not been updated since two times + the wait time for monitoring. This usually means that the trial is stalling or has been + interrupted in some way without its status being changed. This functions finds such + trials and set them as interrupted so they can be launched again. + + """ + # TODO: Configure this + threshold = datetime.datetime.utcnow() - datetime.timedelta(seconds=60 * 2) + lte_comparison = {'$lte': threshold} + query = {'experiment': self._id, 'status': 'reserved', 'heartbeat': lte_comparison} + + trials = self.fetch_trials(query) + + for trial in trials: + query['_id'] = trial.id + self._db.write('trials', {'status': 'interrupted'}, query) + def push_completed_trial(self, trial): """Inform database about an evaluated `trial` with resultlts. diff --git a/tests/unittests/core/test_experiment.py b/tests/unittests/core/test_experiment.py index 62699bcce..b7a96ebf2 100644 --- a/tests/unittests/core/test_experiment.py +++ b/tests/unittests/core/test_experiment.py @@ -72,7 +72,8 @@ def mock_sample(a_list, should_be_one): # another process right after the call to orion_db.read() create_db_instance.write( "trials", - data={"status": "reserved"}, + data={"status": "reserved", + "heartbeat": datetime.datetime.utcnow()}, query={"_id": a_list[0].id}) trial = create_db_instance.read("trials", {"_id": a_list[0].id}) assert trial[0]['status'] == 'reserved' @@ -104,7 +105,7 @@ def mock_sample(a_list, should_be_one): # another process right after the call to orion_db.read() create_db_instance.write( "trials", - data={"status": "reserved"}, + data={"status": "reserved", 'heartbeat': datetime.datetime.utcnow()}, query={"_id": a_list[0].id}) trial = create_db_instance.read("trials", {"_id": a_list[0].id}) assert trial[0]['status'] == 'reserved' @@ -638,6 +639,55 @@ def test_reserve_with_score(self, hacked_exp, exp_config, random_dt): exp_config[1][6]['heartbeat'] = random_dt assert trial.to_dict() == exp_config[1][6] + def test_fix_lost_trials(self, hacked_exp, random_dt): + """Test that a running trial with an old heartbeat is set to interrupted.""" + exp_query = {'experiment': hacked_exp.id} + trial = hacked_exp.fetch_trials(exp_query)[0] + heartbeat = random_dt - datetime.timedelta(seconds=180) + + Database().write('trials', {'status': 'reserved', 'heartbeat': heartbeat}, + {'experiment': hacked_exp.id, '_id': trial.id}) + + exp_query['status'] = 'reserved' + exp_query['_id'] = trial.id + + assert len(hacked_exp.fetch_trials(exp_query)) == 1 + + hacked_exp.fix_lost_trials() + + assert len(hacked_exp.fetch_trials(exp_query)) == 0 + + exp_query['status'] = 'interrupted' + + assert len(hacked_exp.fetch_trials(exp_query)) == 1 + + def test_fix_only_lost_trials(self, hacked_exp, random_dt): + """Test that an old trial is set to interrupted but not a recent one.""" + exp_query = {'experiment': hacked_exp.id} + trials = hacked_exp.fetch_trials(exp_query) + lost = trials[0] + not_lost = trials[1] + + heartbeat = random_dt - datetime.timedelta(seconds=180) + + Database().write('trials', {'status': 'reserved', 'heartbeat': heartbeat}, + {'experiment': hacked_exp.id, '_id': lost.id}) + Database().write('trials', {'status': 'reserved', 'heartbeat': random_dt}, + {'experiment': hacked_exp.id, '_id': not_lost.id}) + + exp_query['status'] = 'reserved' + exp_query['_id'] = {'$in': [lost.id, not_lost.id]} + + assert len(hacked_exp.fetch_trials(exp_query)) == 2 + + hacked_exp.fix_lost_trials() + + assert len(hacked_exp.fetch_trials(exp_query)) == 1 + + exp_query['status'] = 'interrupted' + + assert len(hacked_exp.fetch_trials(exp_query)) == 1 + @pytest.mark.usefixtures("patch_sample") def test_push_completed_trial(hacked_exp, database, random_dt): From 48f5b8686e7f05eda17a4462fd98ccc4dfab3f69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Wed, 10 Jul 2019 16:57:48 -0400 Subject: [PATCH 18/50] Fix bug where no VCS would lead to code conflict when branching --- src/orion/core/evc/conflicts.py | 2 +- tests/unittests/core/evc/test_conflicts.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/orion/core/evc/conflicts.py b/src/orion/core/evc/conflicts.py index b7e80ea01..73d4f67b0 100644 --- a/src/orion/core/evc/conflicts.py +++ b/src/orion/core/evc/conflicts.py @@ -1093,7 +1093,7 @@ def detect(cls, old_config, new_config): old_hash_commit = old_config['metadata'].get('VCS', None) new_hash_commit = new_config['metadata'].get('VCS') - if not old_hash_commit or old_hash_commit != new_hash_commit: + if old_hash_commit is None or old_hash_commit != new_hash_commit: yield cls(old_config, new_config) def get_marked_arguments(self, conflicts): diff --git a/tests/unittests/core/evc/test_conflicts.py b/tests/unittests/core/evc/test_conflicts.py index 1d740aa47..8a53db642 100644 --- a/tests/unittests/core/evc/test_conflicts.py +++ b/tests/unittests/core/evc/test_conflicts.py @@ -8,6 +8,7 @@ from orion.algo.space import Dimension from orion.core import evc +from orion.core.evc import conflicts as conflict @pytest.fixture @@ -241,6 +242,13 @@ def test_repr(self, code_conflict): "'is_dirty': False, 'type': 'git'}' "\ "!= new hash commit ''to be changed''" + def test_hash_commit_compar(self): + """Test that old config hash commit evals to empty.""" + old_config = {'metadata': {'VCS': {}}} + new_config = {'metadata': {'VCS': {}}} + + assert list(conflict.CodeConflict.detect(old_config, new_config)) == [] + class TestCommandLineConflict(object): """Tests methods related to code conflicts""" From b99734c5ef7c6240078804df9a0a5780342e1c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Thu, 11 Jul 2019 15:06:56 -0400 Subject: [PATCH 19/50] Stop experiment when 3 trials or more are broken --- src/orion/core/worker/__init__.py | 4 ++++ src/orion/core/worker/consumer.py | 1 - src/orion/core/worker/experiment.py | 14 ++++++++++++++ tests/functional/demo/broken_box.py | 17 +++++++++++++++++ tests/functional/demo/test_demo.py | 14 ++++++++++++++ tests/unittests/core/test_experiment.py | 14 ++++++++++++++ 6 files changed, 63 insertions(+), 1 deletion(-) create mode 100755 tests/functional/demo/broken_box.py diff --git a/src/orion/core/worker/__init__.py b/src/orion/core/worker/__init__.py index 2fd87ad65..541200d5b 100644 --- a/src/orion/core/worker/__init__.py +++ b/src/orion/core/worker/__init__.py @@ -53,6 +53,10 @@ def workon(experiment, worker_trials=None): for _ in iterator: log.debug("#### Poll for experiment termination.") + if experiment.is_broken: + log.info("#### Experiment has reached broken trials threshold, terminating.") + return + if experiment.is_done: break diff --git a/src/orion/core/worker/consumer.py b/src/orion/core/worker/consumer.py index f4b2c79ae..7dd5ababa 100644 --- a/src/orion/core/worker/consumer.py +++ b/src/orion/core/worker/consumer.py @@ -97,7 +97,6 @@ def consume(self, trial): trial.status = 'broken' Database().write('trials', trial.to_dict(), query={'_id': trial.id}) - raise else: log.debug("### Register successfully evaluated %s.", trial) self.experiment.push_completed_trial(trial) diff --git a/src/orion/core/worker/experiment.py b/src/orion/core/worker/experiment.py index 2b58339d0..b4ada44ee 100644 --- a/src/orion/core/worker/experiment.py +++ b/src/orion/core/worker/experiment.py @@ -437,6 +437,20 @@ def is_done(self): return ((num_completed_trials >= self.max_trials) or (self._init_done and self.algorithms.is_done)) + @property + def is_broken(self): + """Return True, if this experiment is considered to be broken. + + Count how many trials are broken and return True if that number has reached + as given threshold. + + + """ + query = {'experiment': self._id, 'status': 'broken'} + num_broken_trials = self._db.count('trials', query) + + return num_broken_trials >= 3 + @property def space(self): """Return problem's parameter `orion.algo.space.Space`. diff --git a/tests/functional/demo/broken_box.py b/tests/functional/demo/broken_box.py new file mode 100755 index 000000000..cb92f2471 --- /dev/null +++ b/tests/functional/demo/broken_box.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Simple one dimensional example for a possible user's script.""" +import argparse + + +def execute(): + """Execute a simple pipeline as an example.""" + # 1. Receive inputs as you want + parser = argparse.ArgumentParser() + parser.add_argument('-x', type=float, required=True) + + raise RuntimeError + + +if __name__ == "__main__": + execute() diff --git a/tests/functional/demo/test_demo.py b/tests/functional/demo/test_demo.py index 3ac31e702..a1ee2e4c2 100644 --- a/tests/functional/demo/test_demo.py +++ b/tests/functional/demo/test_demo.py @@ -13,6 +13,7 @@ import orion.core.cli from orion.core.io.database import Database +from orion.core.io.experiment_builder import ExperimentBuilder from orion.core.worker import workon from orion.core.worker.experiment import Experiment @@ -412,3 +413,16 @@ def test_worker_trials(database, monkeypatch): "--max-trials", "6"]) assert len(list(database.trials.find({'experiment': exp_id}))) == 6 + + +@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("null_db_instances") +def test_resilience(monkeypatch): + """Test if Oríon stops after enough broken trials.""" + monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) + + orion.core.cli.main(["hunt", "--config", "./orion_config_random.yaml", "./broken_box.py", + "-x~uniform(-50, 50)"]) + + exp = ExperimentBuilder().build_from({'name': 'demo_random_search'}) + assert len(exp.fetch_trials({'status': 'broken'})) == 3 diff --git a/tests/unittests/core/test_experiment.py b/tests/unittests/core/test_experiment.py index b7a96ebf2..54786c15f 100644 --- a/tests/unittests/core/test_experiment.py +++ b/tests/unittests/core/test_experiment.py @@ -802,6 +802,20 @@ def test_is_done_property_with_algo(hacked_exp): assert hacked_exp.is_done is True +def test_broken_property(hacked_exp): + """Check experiment stopping conditions for maximum number of broken.""" + assert not hacked_exp.is_broken + trials = hacked_exp.fetch_trials({})[:3] + + query = {'experiment': hacked_exp.id} + + for trial in trials: + query['_id'] = trial.id + Database().write('trials', {'status': 'broken'}, query) + + assert hacked_exp.is_broken + + def test_experiment_stats(hacked_exp, exp_config, random_dt): """Check that property stats is returning a proper summary of experiment's results.""" stats = hacked_exp.stats From 0788f45147466d61af9b79080559e6d905aa4a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Thu, 11 Jul 2019 15:42:05 -0400 Subject: [PATCH 20/50] Add command `list` to pretty-print a tree of the experiments --- docs/src/conf.py | 2 +- src/orion/core/cli/list.py | 44 +++++++++ src/orion/core/io/experiment_builder.py | 35 +++++--- src/orion/core/utils/pptree.py | 89 +++++++++++++++++++ .../functional/commands/test_list_command.py | 79 ++++++++++++++++ 5 files changed, 236 insertions(+), 13 deletions(-) create mode 100644 src/orion/core/cli/list.py create mode 100644 src/orion/core/utils/pptree.py create mode 100644 tests/functional/commands/test_list_command.py diff --git a/docs/src/conf.py b/docs/src/conf.py index 2ef27aa7c..7a6ea13cc 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -94,7 +94,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build'] +exclude_patterns = ['_build', '**/pptree.py'] # The name of the Pygments (syntax highlighting) style to use. highlight_language = 'python3' diff --git a/src/orion/core/cli/list.py b/src/orion/core/cli/list.py new file mode 100644 index 000000000..4e8b5d254 --- /dev/null +++ b/src/orion/core/cli/list.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +:mod:`orion.core.cli.list` -- Module to list experiments +======================================================== +.. module:: list + :platform: Unix + :synopsis: List experiments in terminal +""" +import logging + +from orion.core.cli import base as cli +from orion.core.io.database import Database +from orion.core.io.evc_builder import EVCBuilder +from orion.core.io.experiment_builder import ExperimentBuilder +from orion.core.utils.pptree import print_tree + +log = logging.getLogger(__name__) + + +def add_subparser(parser): + """Add the subparser that needs to be used for this command""" + list_parser = parser.add_parser('list', help='list help') + + cli.get_basic_args_group(list_parser) + + list_parser.set_defaults(func=main) + + return list_parser + + +def main(args): + """List all experiments inside database.""" + builder = ExperimentBuilder() + config = builder.fetch_full_config(args, use_db=False) + builder.setup_database(config) + + experiments = Database().read("experiments", {}) + + root_experiments = [exp for exp in experiments if exp['refers']['root_id'] == exp['_id']] + + for root_experiment in root_experiments: + root = EVCBuilder().build_view_from({'name': root_experiment['name']}).node + print_tree(root) diff --git a/src/orion/core/io/experiment_builder.py b/src/orion/core/io/experiment_builder.py index 740ab4b5b..f6538d26c 100644 --- a/src/orion/core/io/experiment_builder.py +++ b/src/orion/core/io/experiment_builder.py @@ -202,20 +202,9 @@ def build_view_from(self, cmdargs): """ local_config = self.fetch_full_config(cmdargs, use_db=False) - db_opts = local_config['database'] - dbtype = db_opts.pop('type') - - if local_config.get("debug"): - dbtype = "EphemeralDB" + self.setup_database(local_config) # Information should be enough to infer experiment's name. - log.debug("Creating %s database client with args: %s", dbtype, db_opts) - try: - Database(of_type=dbtype, **db_opts) - except ValueError: - if Database().__class__.__name__.lower() != dbtype.lower(): - raise - exp_name = local_config['name'] if exp_name is None: raise RuntimeError("Could not infer experiment's name. " @@ -278,3 +267,25 @@ def build_from_config(self, config): raise NoConfigurationError from ex return experiment + + def setup_database(self, config): + """Create the Database instance from a configuration. + + Parameters + ---------- + config: dict + Configuration for the database. + + """ + db_opts = config['database'] + dbtype = db_opts.pop('type') + + if config.get("debug"): + dbtype = "EphemeralDB" + + log.debug("Creating %s database client with args: %s", dbtype, db_opts) + try: + Database(of_type=dbtype, **db_opts) + except ValueError: + if Database().__class__.__name__.lower() != dbtype.lower(): + raise diff --git a/src/orion/core/utils/pptree.py b/src/orion/core/utils/pptree.py new file mode 100644 index 000000000..f235f0436 --- /dev/null +++ b/src/orion/core/utils/pptree.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +# pylint: disable-all +""" +:mod:`orion.core.utils.pptree` -- Utilitary functions for printing trees +======================================================================== + +.. module:: pptree + :platform: Unix + :synopsis: Utilitary functions to provided pretty trees + + +Clement Michard (c) 2015 + +https://github.com/clemtoy/pptree + +MIT License + +Copyright (c) 2017 Clément Michard + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +""" + +class Node: + def __init__(self, name, parent=None): + self.name = name + self.parent = parent + self.children = [] + + if parent: + self.parent.children.append(self) + + +def print_tree(current_node, childattr='children', nameattr='name', indent='', last='updown'): + if hasattr(current_node, nameattr): + name = lambda node: getattr(node, nameattr) + else: + name = lambda node: str(node) + + children = lambda node: getattr(node, childattr) + nb_children = lambda node: sum(nb_children(child) for child in children(node)) + 1 + size_branch = {child: nb_children(child) for child in children(current_node)} + + """ Creation of balanced lists for "up" branch and "down" branch. """ + up = sorted(children(current_node), key=lambda node: nb_children(node)) + down = [] + while up and sum(size_branch[node] for node in down) < sum(size_branch[node] for node in up): + down.append(up.pop()) + + """ Printing of "up" branch. """ + for child in up: + next_last = 'up' if up.index(child) is 0 else '' + next_indent = '{0}{1}{2}'.format(indent, ' ' if 'up' in last else '│', ' ' * len(name(current_node))) + print_tree(child, childattr, nameattr, next_indent, next_last) + + """ Printing of current node. """ + if last == 'up': start_shape = '┌' + elif last == 'down': start_shape = '└' + elif last == 'updown': start_shape = ' ' + else: start_shape = '├' + + if up: end_shape = '┤' + elif down: end_shape = '┐' + else: end_shape = '' + + print('{0}{1}{2}{3}'.format(indent, start_shape, name(current_node), end_shape)) + + """ Printing of "down" branch. """ + for child in down: + next_last = 'down' if down.index(child) is len(down) - 1 else '' + next_indent = '{0}{1}{2}'.format(indent, ' ' if 'down' in last else '│', ' ' * len(name(current_node))) + print_tree(child, childattr, nameattr, next_indent, next_last) diff --git a/tests/functional/commands/test_list_command.py b/tests/functional/commands/test_list_command.py new file mode 100644 index 000000000..05da26d31 --- /dev/null +++ b/tests/functional/commands/test_list_command.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Perform a functional test of the list command.""" +import os + +import pytest + +import orion.core.cli + + +@pytest.fixture +def no_experiment(database): + """Make sure there is no experiment.""" + database.experiments.drop() + + +@pytest.fixture +def one_experiment(monkeypatch, no_experiment): + """Create a single experiment.""" + monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) + orion.core.cli.main(['init_only', '-n', 'test_list_single', '-c', './orion_config_random.yaml', + './black_box.py', '--x~uniform(0,1)']) + + +@pytest.fixture +def two_experiments(monkeypatch, no_experiment): + """Create an experiment and its child.""" + monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) + orion.core.cli.main(['init_only', '-n', 'test_list_double', '-c', './orion_config_random.yaml', + './black_box.py', '--x~uniform(0,1)']) + orion.core.cli.main(['init_only', '-n', 'test_list_double', '-c', './orion_config_random.yaml', + '--branch', 'test_list_double_child', './black_box.py', + '--x~uniform(0,1)', '--y~+uniform(0,1)']) + + +@pytest.fixture +def three_experiments(monkeypatch, two_experiments): + """Create a single experiment and an experiment and its child.""" + monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) + orion.core.cli.main(['init_only', '-n', 'test_list_single', '-c', './orion_config_random.yaml', + './black_box.py', '--x~uniform(0,1)']) + + +def test_no_exp(no_experiment, monkeypatch, capsys): + """Test that nothing is printed when there are no experiments.""" + monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) + orion.core.cli.main(['list', '--config', './orion_config_random.yaml']) + + captured = capsys.readouterr().out + + assert captured == "" + + +def test_single_exp(capsys, one_experiment): + """Test that the name of the experiment is printed when there is one experiment.""" + orion.core.cli.main(['list', '--config', './orion_config_random.yaml']) + + captured = capsys.readouterr().out + + assert captured == " test_list_single\n" + + +def test_two_exp(capsys, two_experiments): + """Test that nothing is printed when there are no experiments.""" + orion.core.cli.main(['list', '--config', './orion_config_random.yaml']) + + captured = capsys.readouterr().out + + assert captured == " test_list_double┐\n └test_list_double_child\n" + + +def test_three_exp(capsys, three_experiments): + """Test that nothing is printed when there are no experiments.""" + orion.core.cli.main(['list', '--config', './orion_config_random.yaml']) + + captured = capsys.readouterr().out + + assert captured == " test_list_double┐\n └test_list_double_child\n \ +test_list_single\n" From 96022d659b89e98b2d36990787555ba43a5677df Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Thu, 11 Jul 2019 21:45:37 -0400 Subject: [PATCH 21/50] Get rid of config file in `list` tests Why: There is no need for a config file, what we need is to fix the DB instance in the fixture so that this one is used in all tests and not the global config or the current user running the tests. --- .../functional/commands/test_list_command.py | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/tests/functional/commands/test_list_command.py b/tests/functional/commands/test_list_command.py index 05da26d31..d5a663c0d 100644 --- a/tests/functional/commands/test_list_command.py +++ b/tests/functional/commands/test_list_command.py @@ -6,19 +6,32 @@ import pytest import orion.core.cli +from orion.core.io.database import Database @pytest.fixture def no_experiment(database): - """Make sure there is no experiment.""" + """Create and save a singleton for an empty database instance.""" database.experiments.drop() + database.lying_trials.drop() + database.trials.drop() + database.workers.drop() + database.resources.drop() + + try: + db = Database(of_type='MongoDB', name='orion_test', + username='user', password='pass') + except ValueError: + db = Database() + + return db @pytest.fixture -def one_experiment(monkeypatch, no_experiment): +def one_experiment(monkeypatch, create_db_instance): """Create a single experiment.""" monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) - orion.core.cli.main(['init_only', '-n', 'test_list_single', '-c', './orion_config_random.yaml', + orion.core.cli.main(['init_only', '-n', 'test_list_single', './black_box.py', '--x~uniform(0,1)']) @@ -26,9 +39,9 @@ def one_experiment(monkeypatch, no_experiment): def two_experiments(monkeypatch, no_experiment): """Create an experiment and its child.""" monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) - orion.core.cli.main(['init_only', '-n', 'test_list_double', '-c', './orion_config_random.yaml', + orion.core.cli.main(['init_only', '-n', 'test_list_double', './black_box.py', '--x~uniform(0,1)']) - orion.core.cli.main(['init_only', '-n', 'test_list_double', '-c', './orion_config_random.yaml', + orion.core.cli.main(['init_only', '-n', 'test_list_double', '--branch', 'test_list_double_child', './black_box.py', '--x~uniform(0,1)', '--y~+uniform(0,1)']) @@ -37,14 +50,14 @@ def two_experiments(monkeypatch, no_experiment): def three_experiments(monkeypatch, two_experiments): """Create a single experiment and an experiment and its child.""" monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) - orion.core.cli.main(['init_only', '-n', 'test_list_single', '-c', './orion_config_random.yaml', + orion.core.cli.main(['init_only', '-n', 'test_list_single', './black_box.py', '--x~uniform(0,1)']) def test_no_exp(no_experiment, monkeypatch, capsys): """Test that nothing is printed when there are no experiments.""" monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) - orion.core.cli.main(['list', '--config', './orion_config_random.yaml']) + orion.core.cli.main(['list']) captured = capsys.readouterr().out @@ -53,7 +66,7 @@ def test_no_exp(no_experiment, monkeypatch, capsys): def test_single_exp(capsys, one_experiment): """Test that the name of the experiment is printed when there is one experiment.""" - orion.core.cli.main(['list', '--config', './orion_config_random.yaml']) + orion.core.cli.main(['list']) captured = capsys.readouterr().out @@ -61,8 +74,8 @@ def test_single_exp(capsys, one_experiment): def test_two_exp(capsys, two_experiments): - """Test that nothing is printed when there are no experiments.""" - orion.core.cli.main(['list', '--config', './orion_config_random.yaml']) + """Test that experiment and child are printed.""" + orion.core.cli.main(['list']) captured = capsys.readouterr().out @@ -70,8 +83,8 @@ def test_two_exp(capsys, two_experiments): def test_three_exp(capsys, three_experiments): - """Test that nothing is printed when there are no experiments.""" - orion.core.cli.main(['list', '--config', './orion_config_random.yaml']) + """Test that experiment, child and grand-child are printed.""" + orion.core.cli.main(['list']) captured = capsys.readouterr().out From 7b2ed5660ff677b048f6e8ca3beec855b235b674 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Thu, 11 Jul 2019 21:47:30 -0400 Subject: [PATCH 22/50] Handle --name argument properly Why: The argument --name was currently ignored. It should filter the name of the experiments --- src/orion/core/cli/list.py | 12 +++++- .../functional/commands/test_list_command.py | 40 +++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/src/orion/core/cli/list.py b/src/orion/core/cli/list.py index 4e8b5d254..8a20e34c6 100644 --- a/src/orion/core/cli/list.py +++ b/src/orion/core/cli/list.py @@ -35,9 +35,17 @@ def main(args): config = builder.fetch_full_config(args, use_db=False) builder.setup_database(config) - experiments = Database().read("experiments", {}) + query = {} - root_experiments = [exp for exp in experiments if exp['refers']['root_id'] == exp['_id']] + if args['name']: + query['name'] = args['name'] + + experiments = Database().read("experiments", query) + + if args['name']: + root_experiments = experiments + else: + root_experiments = [exp for exp in experiments if exp['refers']['root_id'] == exp['_id']] for root_experiment in root_experiments: root = EVCBuilder().build_view_from({'name': root_experiment['name']}).node diff --git a/tests/functional/commands/test_list_command.py b/tests/functional/commands/test_list_command.py index d5a663c0d..20b7d3eb2 100644 --- a/tests/functional/commands/test_list_command.py +++ b/tests/functional/commands/test_list_command.py @@ -90,3 +90,43 @@ def test_three_exp(capsys, three_experiments): assert captured == " test_list_double┐\n └test_list_double_child\n \ test_list_single\n" + + +def test_no_exp_name(three_experiments, monkeypatch, capsys): + """Test that nothing is printed when there are no experiments with a given name.""" + monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) + orion.core.cli.main(['list', '--name', 'I don\'t exist']) + + captured = capsys.readouterr().out + + assert captured == "" + + +def test_exp_name(three_experiments, monkeypatch, capsys): + """Test that only the specified experiment is printed.""" + monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) + orion.core.cli.main(['list', '--name', 'test_list_single']) + + captured = capsys.readouterr().out + + assert captured == " test_list_single\n" + + +def test_exp_name_with_child(three_experiments, monkeypatch, capsys): + """Test that only the specified experiment is printed, and with its child.""" + monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) + orion.core.cli.main(['list', '--name', 'test_list_double']) + + captured = capsys.readouterr().out + + assert captured == " test_list_double┐\n └test_list_double_child\n" + + +def test_exp_name_child(three_experiments, monkeypatch, capsys): + """Test that only the specified child experiment is printed.""" + monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) + orion.core.cli.main(['list', '--name', 'test_list_double_child']) + + captured = capsys.readouterr().out + + assert captured == " test_list_double_child\n" From d0b4a361066150d91ad2e1ccdf08abb9a6af69d1 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Mon, 15 Jul 2019 11:22:17 -0400 Subject: [PATCH 23/50] Skip exp update in ExpView (#218) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Why: The ExperimentView should not update the experiment in the database. Fields doesn't get updated when loading the View per say, but loading a view within a different version of Oríon could lead to modification of the configuration dictionary of the experiment, which would lead to a modified experiment in the database. How: Add `enable_update` argument to `Experiment.configure` so that `ExperimentView` can call it without updating the DB. Note: All this should get refactored once the global and experiment configuration refactoring is completed. --- src/orion/core/worker/experiment.py | 8 +++++-- tests/unittests/core/test_experiment.py | 31 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/orion/core/worker/experiment.py b/src/orion/core/worker/experiment.py index b4ada44ee..e666ca719 100644 --- a/src/orion/core/worker/experiment.py +++ b/src/orion/core/worker/experiment.py @@ -487,7 +487,7 @@ def configuration(self): # object from a getter. return copy.deepcopy(config) - def configure(self, config, enable_branching=True): + def configure(self, config, enable_branching=True, enable_update=True): """Set `Experiment` by overwriting current attributes. If `Experiment` was already set and an overwrite is needed, a *branch* @@ -540,6 +540,9 @@ def configure(self, config, enable_branching=True): self._init_done = True + if not enable_update: + return + # If everything is alright, push new config to database if is_new: final_config['metadata']['datetime'] = datetime.datetime.utcnow() @@ -773,7 +776,8 @@ def __init__(self, name, user=None): (self._experiment.name, self._experiment.metadata['user'])) try: - self._experiment.configure(self._experiment.configuration, enable_branching=False) + self._experiment.configure(self._experiment.configuration, enable_branching=False, + enable_update=False) except ValueError as e: if "Configuration is different and generates a branching event" in str(e): raise RuntimeError( diff --git a/tests/unittests/core/test_experiment.py b/tests/unittests/core/test_experiment.py index 54786c15f..d1264d228 100644 --- a/tests/unittests/core/test_experiment.py +++ b/tests/unittests/core/test_experiment.py @@ -876,6 +876,37 @@ def test_existing_experiment_view(self, create_db_instance, exp_config): with pytest.raises(AttributeError): exp.reserve_trial + @pytest.mark.usefixtures("with_user_tsirif", "create_db_instance") + def test_existing_experiment_view_not_modified(self, exp_config, monkeypatch): + """Experiment should not be modified if fetched in another verion of Oríon. + + When loading a view the original config is used to configure the experiment, but + this process may modify the config if the version of Oríon is different. This should not be + saved in database. + """ + terrible_message = 'oh no, I have been modified!' + original_configuration = ExperimentView('supernaedo2').configuration + + def modified_configuration(self): + mocked_config = copy.deepcopy(original_configuration) + mocked_config['metadata']['datetime'] = terrible_message + return mocked_config + + with monkeypatch.context() as m: + m.setattr(Experiment, 'configuration', property(modified_configuration)) + exp = ExperimentView('supernaedo2') + + # The mock is still in place and overwrites the configuration + assert exp.configuration['metadata']['datetime'] == terrible_message + + # The mock is reverted and original config is returned, but modification is still in + # metadata + assert exp.metadata['datetime'] == terrible_message + + # Loading again from DB confirms the DB was not overwritten + reloaded_exp = ExperimentView('supernaedo2') + assert reloaded_exp.configuration['metadata']['datetime'] != terrible_message + def test_fetch_completed_trials_from_view(hacked_exp, exp_config, random_dt): """Fetch a list of the unseen yet completed trials.""" From 6be09249430bc2fd1866fd3086974ebb70d8b59f Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Sun, 14 Jul 2019 22:29:49 -0400 Subject: [PATCH 24/50] Remove upsert from `write` Why: The upsert behavior was never used and was causing unexpected DuplicateKeyError in `fix_lost_trials`. This was not observed before because locked update with queries was previously done with `read_and_write` in `reserve_trials` which does not update if the query fails. How: Do not update if query fails and return number of updated document instead of `result.acknowledment` so that we can verify if `db.write()` successfully updated documents or not. Since the current error due to race condition was not catched in unit-tests for `fix_lost_trials()`, I added one that does catch the error if there is a DuplicateKeyError. --- src/orion/core/io/database/__init__.py | 78 +++++++++++++---------- src/orion/core/io/database/ephemeraldb.py | 37 ++++++----- src/orion/core/io/database/mongodb.py | 8 +-- src/orion/core/io/database/pickleddb.py | 16 ++--- src/orion/core/worker/experiment.py | 11 ++-- tests/unittests/core/mongodb_test.py | 28 +++----- tests/unittests/core/test_ephemeraldb.py | 26 ++------ tests/unittests/core/test_experiment.py | 29 ++++++++- tests/unittests/core/test_pickleddb.py | 27 ++------ 9 files changed, 135 insertions(+), 125 deletions(-) diff --git a/src/orion/core/io/database/__init__.py b/src/orion/core/io/database/__init__.py index ad22fa395..210e8de83 100644 --- a/src/orion/core/io/database/__init__.py +++ b/src/orion/core/io/database/__init__.py @@ -65,7 +65,10 @@ def is_connected(self): def initiate_connection(self): """Connect to database, unless `AbstractDB` `is_connected`. - :raises :exc:`DatabaseError`: if connection or authentication fails + Raises + ------ + DatabaseError + If connection or authentication fails """ pass @@ -94,10 +97,11 @@ def ensure_index(self, collection_name, keys, unique=False): `DuplicateKeyError`. Defaults to False. - .. note:: - Depending on the backend, the indexing operation might operate in - background. This means some operations on the database might occur - before the indexes are totally built. + Notes + ----- + Depending on the backend, the indexing operation might operate in + background. This means some operations on the database might occur + before the indexes are totally built. """ pass @@ -115,20 +119,25 @@ def write(self, collection_name, data, query=None): query : dict, optional Assumes an update operation: filter entries in collection to be updated. - :return: operation success. + Returns + ------- + int + Number of new documents if no query, otherwise number of modified documents. - .. note:: - In the case of an insert operation, `data` variable will be updated - to contain a unique *_id* key. + Notes + ----- + In the case of an insert operation, `data` variable will be updated + to contain a unique *_id* key. - .. note:: - In the case of an update operation, if `query` fails to find a - document that matches, insert of `data` will be performed instead. + In the case of an update operation, if `query` fails to find a + document that matches, no operation is performed. - :raises :exc:`DuplicateKeyError`: if the operation is creating duplicate - keys in two different documents. Only occurs if the keys have - unique indexes. See :meth:`AbstractDB.ensure_index` for more - information about indexes. + Raises + ------ + DuplicateKeyError + If the operation is creating duplicate keys in two different documents. Only occurs if + the keys have unique indexes. See :meth:`AbstractDB.ensure_index` for more information + about indexes. """ pass @@ -146,7 +155,10 @@ def read(self, collection_name, query=None, selection=None): selection : dict, optional Elements of matched entries to return, the projection. - :return: list of matched document[s] + Returns + ------- + list + List of matched document[s] """ pass @@ -170,12 +182,17 @@ def read_and_write(self, collection_name, query, data, selection=None): selection : dict, optional Elements of matched entries to return, the projection. - :return: updated first matched document or None if nothing found + Returns + ------- + dict or None + Updated first matched document or None if nothing found - :raises :exc:`DuplicateKeyError`: if the operation is creating duplicate - keys in two different documents. Only occurs if the keys have - unique indexes. See :meth:`AbstractDB.ensure_index` for more - information about indexes. + Raises + ------ + DuplicateKeyError + If the operation is creating duplicate keys in two different documents. Only occurs if + the keys have unique indexes. See :meth:`AbstractDB.ensure_index` for more information + about indexes. """ pass @@ -205,7 +222,10 @@ def remove(self, collection_name, query): query : dict Filter entries in collection. - :return: operation success. + Returns + ------- + int + Number of documents removed """ pass @@ -213,12 +233,7 @@ def remove(self, collection_name, query): # pylint: disable=too-few-public-methods class ReadOnlyDB(object): - """Read-only view on a database. - - .. seealso:: - - :py:class:`orion.core.io.database.AbstractDB` - """ + """Read-only view on a database.""" __slots__ = ('_database', ) @@ -259,10 +274,7 @@ class DuplicateKeyError(DatabaseError): # pylint: disable=too-few-public-methods,abstract-method class Database(AbstractDB, metaclass=SingletonFactory): - """Class used to inject dependency on a database framework. - - .. seealso:: `Factory` metaclass and `AbstractDB` interface. - """ + """Class used to inject dependency on a database framework.""" pass diff --git a/src/orion/core/io/database/ephemeraldb.py b/src/orion/core/io/database/ephemeraldb.py index a872e2541..87b525109 100644 --- a/src/orion/core/io/database/ephemeraldb.py +++ b/src/orion/core/io/database/ephemeraldb.py @@ -166,8 +166,11 @@ def find(self, query=None, selection=None): def _validate_index(self, document, indexes=None): """Validate index values of a document - :raises: :exc:`DuplicateKeyError`: if the document contains unique indexes which are already - present in the database. + Raises + ------ + DuplicateKeyError + If the document contains unique indexes which are already present in the database. + """ if indexes is None: indexes = self._indexes.keys() @@ -191,8 +194,11 @@ def insert_many(self, documents): If the documents do not have a keys `_id`, they are assigned by default the max id + 1. - :raises: :exc:`DuplicateKeyError`: if the document contains unique indexes which are - already present in the database. + Raises + ------ + DuplicateKeyError + If the document contains unique indexes which are already present in the database. + """ for document in documents: if '_id' not in document: @@ -202,16 +208,16 @@ def insert_many(self, documents): self._documents.append(ephemeral_document) self._register_keys(ephemeral_document) - return True + return len(documents) def update_many(self, query, update): - """Update documents or upsert if not found. + """Update documents matching the query - If the document is not found, a new document which is the merge of query and update will be - inserted in the database. + Raises + ------ + DuplicateKeyError + If the update creates a duplication of unique indexes in the database. - :raises: :exc:`DuplicateKeyError`: if the update creates a duplication of unique indexes in - the database. """ updates = 0 for document in self._documents: @@ -219,10 +225,7 @@ def update_many(self, query, update): document.update(update) updates += 1 - if not updates: - self._upsert(query, update) - - return True + return updates def _upsert(self, query, update): """Insert the document when query was not found. @@ -242,6 +245,7 @@ def count(self, query=None): """Count the number of documents in a collection which match the `query`. .. seealso:: :meth:`AbstractDB.count` for argument documentation. + """ return len(self.find(query)) @@ -251,14 +255,17 @@ def delete_many(self, query=None): .. seealso:: :meth:`AbstractDB.remove` for argument documentation. """ + deleted = 0 retained_documents = [] for document in self._documents: if not document.match(query): retained_documents.append(document) + else: + deleted += 1 self._documents = retained_documents - return True + return deleted def drop(self): """Drop the collection, removing all documents and indexes.""" diff --git a/src/orion/core/io/database/mongodb.py b/src/orion/core/io/database/mongodb.py index ef8f488ac..db7492835 100644 --- a/src/orion/core/io/database/mongodb.py +++ b/src/orion/core/io/database/mongodb.py @@ -186,14 +186,14 @@ def write(self, collection_name, data, query=None): if type(data) not in (list, tuple): data = [data] result = dbcollection.insert_many(documents=data) - return result.acknowledged + return len(result.inserted_ids) update_data = {'$set': data} result = dbcollection.update_many(filter=query, update=update_data, - upsert=True) - return result.acknowledged + upsert=False) + return result.modified_count def read(self, collection_name, query=None, selection=None): """Read a collection and return a value according to the query. @@ -249,7 +249,7 @@ def remove(self, collection_name, query): dbcollection = self._db[collection_name] result = dbcollection.delete_many(filter=query) - return result.acknowledged + return result.deleted_count def _sanitize_attrs(self): """Sanitize attributes using MongoDB's 'uri_parser' module.""" diff --git a/src/orion/core/io/database/pickleddb.py b/src/orion/core/io/database/pickleddb.py index 9c3eceda8..716a51f3b 100644 --- a/src/orion/core/io/database/pickleddb.py +++ b/src/orion/core/io/database/pickleddb.py @@ -28,19 +28,17 @@ class PickledDB(AbstractDB): This is a very simple and inefficient implementation of a permanent database on disk for Oríon. The data is loaded from disk for every operation, and every operation is protected with a filelock. + + Parameters + ---------- + host: str + File path to save pickled ephemeraldb. Default is {user data dir}/orion/orion_db.pkl ex: + $HOME/.local/share/orion/orion_db.pkl + """ # pylint: disable=unused-argument def __init__(self, host=DEFAULT_HOST, *args, **kwargs): - """Initialize the DB - - Base folder defined by `host` is created during init. - - :param host: File path to save pickled ephemeraldb. - Default is {user data dir}/orion/orion_db.pkl - ex: $HOME/.local/share/orion/orion_db.pkl - :type host: str - """ super(PickledDB, self).__init__(host) if os.path.dirname(host): diff --git a/src/orion/core/worker/experiment.py b/src/orion/core/worker/experiment.py index e666ca719..6fae9494b 100644 --- a/src/orion/core/worker/experiment.py +++ b/src/orion/core/worker/experiment.py @@ -253,13 +253,12 @@ def reserve_trial(self, score_handle=None): if selected_trial.status == 'new': update["start_time"] = datetime.datetime.utcnow() - selected_trial_dict = self._db.read_and_write( - 'trials', query=query, data=update) + reserved = self._db.write('trials', query=query, data=update) - if selected_trial_dict is None: + if not reserved: selected_trial = self.reserve_trial(score_handle=score_handle) else: - selected_trial = Trial(**selected_trial_dict) + selected_trial = self.fetch_trials({'_id': selected_trial.id})[0] TrialMonitor(self, selected_trial.id).start() return selected_trial @@ -282,7 +281,9 @@ def fix_lost_trials(self): for trial in trials: query['_id'] = trial.id - self._db.write('trials', {'status': 'interrupted'}, query) + log.debug('Setting lost trial %s status to interrupted...', trial.id) + updated = self._db.write('trials', {'status': 'interrupted'}, query) + log.debug('success' if updated else 'failed') def push_completed_trial(self, trial): """Inform database about an evaluated `trial` with resultlts. diff --git a/tests/unittests/core/mongodb_test.py b/tests/unittests/core/mongodb_test.py index 3ac05a36d..993d58c9c 100644 --- a/tests/unittests/core/mongodb_test.py +++ b/tests/unittests/core/mongodb_test.py @@ -291,7 +291,7 @@ def test_insert_one(self, database, orion_db): 'user': 'tsirif'} count_before = database.experiments.count_documents({}) # call interface - assert orion_db.write('experiments', item) is True + assert orion_db.write('experiments', item) == 1 assert database.experiments.count_documents({}) == count_before + 1 value = database.experiments.find_one({'exp_name': 'supernaekei'}) assert value == item @@ -304,7 +304,7 @@ def test_insert_many(self, database, orion_db): 'user': 'tsirif'}] count_before = database.experiments.count_documents({}) # call interface - assert orion_db.write('experiments', item) is True + assert orion_db.write('experiments', item) == 2 assert database.experiments.count_documents({}) == count_before + 2 value = database.experiments.find_one({'exp_name': 'supernaekei2'}) assert value == item[0] @@ -315,8 +315,9 @@ def test_update_many_default(self, database, orion_db): """Should match existing entries, and update some of their keys.""" filt = {'metadata.user': 'tsirif'} count_before = database.experiments.count_documents({}) + count_filt = database.experiments.count_documents(filt) # call interface - assert orion_db.write('experiments', {'pool_size': 16}, filt) is True + assert orion_db.write('experiments', {'pool_size': 16}, filt) == count_filt assert database.experiments.count_documents({}) == count_before value = list(database.experiments.find({})) assert value[0]['pool_size'] == 16 @@ -329,25 +330,16 @@ def test_update_with_id(self, exp_config, database, orion_db): filt = {'_id': exp_config[0][1]['_id']} count_before = database.experiments.count_documents({}) # call interface - assert orion_db.write('experiments', {'pool_size': 36}, filt) is True + assert orion_db.write('experiments', {'pool_size': 36}, filt) == 1 assert database.experiments.count_documents({}) == count_before value = list(database.experiments.find()) assert value[0]['pool_size'] == 2 assert value[1]['pool_size'] == 36 assert value[2]['pool_size'] == 2 - def test_upsert_with_id(self, database, orion_db): - """Query with a non-existent ``_id`` should upsert something.""" - filt = {'_id': 'lalalathisisnew'} - count_before = database.experiments.count_documents({}) - # call interface - assert orion_db.write('experiments', {'pool_size': 66}, filt) is True - assert database.experiments.count_documents({}) == count_before + 1 - value = list(database.experiments.find(filt)) - assert len(value) == 1 - assert len(value[0]) == 2 - assert value[0]['_id'] == 'lalalathisisnew' - assert value[0]['pool_size'] == 66 + def test_no_upsert(self, database, orion_db): + """Query with a non-existent ``_id`` should no upsert something.""" + assert orion_db.write('experiments', {'pool_size': 66}, {'_id': 'lalalathisisnew'}) == 0 @pytest.mark.usefixtures("clean_db") @@ -410,7 +402,7 @@ def test_remove_many_default(self, exp_config, database, orion_db): count_before = database.experiments.count_documents({}) count_filt = database.experiments.count_documents(filt) # call interface - assert orion_db.remove('experiments', filt) is True + assert orion_db.remove('experiments', filt) == count_filt assert database.experiments.count_documents({}) == count_before - count_filt assert database.experiments.count_documents({}) == 1 assert list(database.experiments.find()) == [exp_config[0][3]] @@ -420,7 +412,7 @@ def test_remove_with_id(self, exp_config, database, orion_db): filt = {'_id': exp_config[0][0]['_id']} count_before = database.experiments.count_documents({}) # call interface - assert orion_db.remove('experiments', filt) is True + assert orion_db.remove('experiments', filt) == 1 assert database.experiments.count_documents({}) == count_before - 1 assert list(database.experiments.find()) == exp_config[0][1:] diff --git a/tests/unittests/core/test_ephemeraldb.py b/tests/unittests/core/test_ephemeraldb.py index a98ae6a32..a68f77375 100644 --- a/tests/unittests/core/test_ephemeraldb.py +++ b/tests/unittests/core/test_ephemeraldb.py @@ -161,7 +161,7 @@ def test_insert_one(self, database, orion_db): 'user': 'tsirif'} count_before = database['experiments'].count() # call interface - assert orion_db.write('experiments', item) is True + assert orion_db.write('experiments', item) == 1 assert database['experiments'].count() == count_before + 1 value = database['experiments'].find({'exp_name': 'supernaekei'})[0] assert value == item @@ -174,7 +174,7 @@ def test_insert_many(self, database, orion_db): 'user': 'tsirif'}] count_before = database['experiments'].count() # call interface - assert orion_db.write('experiments', item) is True + assert orion_db.write('experiments', item) == 2 assert database['experiments'].count() == count_before + 2 value = database['experiments'].find({'exp_name': 'supernaekei2'})[0] assert value == item[0] @@ -185,8 +185,9 @@ def test_update_many_default(self, database, orion_db): """Should match existing entries, and update some of their keys.""" filt = {'metadata.user': 'tsirif'} count_before = database['experiments'].count() + count_query = database['experiments'].count(filt) # call interface - assert orion_db.write('experiments', {'pool_size': 16}, filt) is True + assert orion_db.write('experiments', {'pool_size': 16}, filt) == count_query assert database['experiments'].count() == count_before value = list(database['experiments'].find({})) assert value[0]['pool_size'] == 16 @@ -199,26 +200,13 @@ def test_update_with_id(self, exp_config, database, orion_db): filt = {'_id': exp_config[0][1]['_id']} count_before = database['experiments'].count() # call interface - assert orion_db.write('experiments', {'pool_size': 36}, filt) is True + assert orion_db.write('experiments', {'pool_size': 36}, filt) == 1 assert database['experiments'].count() == count_before value = list(database['experiments'].find()) assert value[0]['pool_size'] == 2 assert value[1]['pool_size'] == 36 assert value[2]['pool_size'] == 2 - def test_upsert_with_id(self, database, orion_db): - """Query with a non-existent ``_id`` should upsert something.""" - filt = {'_id': 'lalalathisisnew'} - count_before = database['experiments'].count() - # call interface - assert orion_db.write('experiments', {'pool_size': 66}, filt) is True - assert database['experiments'].count() == count_before + 1 - value = list(database['experiments'].find(filt)) - assert len(value) == 1 - assert len(value[0]) == 2 - assert value[0]['_id'] == 'lalalathisisnew' - assert value[0]['pool_size'] == 66 - def test_insert_duplicate(self, database, orion_db): """Verify that duplicates cannot by inserted if index is unique""" orion_db.ensure_index('some_doc', 'unique_field', unique=True) @@ -291,7 +279,7 @@ def test_remove_many_default(self, exp_config, database, orion_db): count_before = database['experiments'].count() count_filt = database['experiments'].count(filt) # call interface - assert orion_db.remove('experiments', filt) is True + assert orion_db.remove('experiments', filt) == count_filt assert database['experiments'].count() == count_before - count_filt assert database['experiments'].count() == 1 assert list(database['experiments'].find()) == [exp_config[0][3]] @@ -302,7 +290,7 @@ def test_remove_with_id(self, exp_config, database, orion_db): count_before = database['experiments'].count() # call interface - assert orion_db.remove('experiments', filt) is True + assert orion_db.remove('experiments', filt) == 1 assert database['experiments'].count() == count_before - 1 assert database['experiments'].find() == exp_config[0][1:] diff --git a/tests/unittests/core/test_experiment.py b/tests/unittests/core/test_experiment.py index d1264d228..d38798c11 100644 --- a/tests/unittests/core/test_experiment.py +++ b/tests/unittests/core/test_experiment.py @@ -51,7 +51,10 @@ def mock_sample(a_list, should_be_one): @pytest.fixture() def patch_sample_concurrent(monkeypatch, create_db_instance, exp_config): - """Patch ``random.sample`` to return the first one and check call.""" + """Patch ``random.sample`` to return the first one and check call. + + The first trial is marked as new, but in DB it is reserved. + """ def mock_sample(a_list, should_be_one): assert type(a_list) == list assert len(a_list) >= 1 @@ -85,7 +88,10 @@ def mock_sample(a_list, should_be_one): @pytest.fixture() def patch_sample_concurrent2(monkeypatch, create_db_instance, exp_config): - """Patch ``random.sample`` to return the first one and check call.""" + """Patch ``random.sample`` to return the first one and check call. + + All trials are marked as new, but in DB they are reserved. + """ def mock_sample(a_list, should_be_one): assert type(a_list) == list assert len(a_list) >= 1 @@ -688,6 +694,25 @@ def test_fix_only_lost_trials(self, hacked_exp, random_dt): assert len(hacked_exp.fetch_trials(exp_query)) == 1 + def test_fix_lost_trials_race_condition(self, hacked_exp, random_dt, monkeypatch): + """Test that a lost trial fixed by a concurrent process does not cause error.""" + exp_query = {'experiment': hacked_exp.id} + trial = hacked_exp.fetch_trials(exp_query)[0] + heartbeat = random_dt - datetime.timedelta(seconds=180) + + Database().write('trials', {'status': 'interrupted', 'heartbeat': heartbeat}, + {'experiment': hacked_exp.id, '_id': trial.id}) + + assert hacked_exp.fetch_trials(exp_query)[0].status == 'interrupted' + + def fetch_lost_trials(self, query): + trial.status = 'reserved' + return [trial] + + with monkeypatch.context() as m: + m.setattr(hacked_exp.__class__, 'fetch_trials', fetch_lost_trials) + hacked_exp.fix_lost_trials() + @pytest.mark.usefixtures("patch_sample") def test_push_completed_trial(hacked_exp, database, random_dt): diff --git a/tests/unittests/core/test_pickleddb.py b/tests/unittests/core/test_pickleddb.py index c07d0733d..ce60f4dee 100644 --- a/tests/unittests/core/test_pickleddb.py +++ b/tests/unittests/core/test_pickleddb.py @@ -151,7 +151,7 @@ def test_insert_one(self, orion_db): 'user': 'tsirif'} count_before = orion_db._get_database().count('experiments') # call interface - assert orion_db.write('experiments', item) is True + assert orion_db.write('experiments', item) == 1 assert orion_db._get_database().count('experiments') == count_before + 1 value = orion_db._get_database()._db['experiments'].find({'exp_name': 'supernaekei'})[0] assert value == item @@ -164,7 +164,7 @@ def test_insert_many(self, orion_db): 'user': 'tsirif'}] count_before = orion_db._get_database()._db['experiments'].count() # call interface - assert orion_db.write('experiments', item) is True + assert orion_db.write('experiments', item) == 2 database = orion_db._get_database()._db assert database['experiments'].count() == count_before + 2 value = database['experiments'].find({'exp_name': 'supernaekei2'})[0] @@ -176,8 +176,9 @@ def test_update_many_default(self, orion_db): """Should match existing entries, and update some of their keys.""" filt = {'metadata.user': 'tsirif'} count_before = orion_db._get_database().count('experiments') + count_query = orion_db._get_database().count('experiments', filt) # call interface - assert orion_db.write('experiments', {'pool_size': 16}, filt) is True + assert orion_db.write('experiments', {'pool_size': 16}, filt) == count_query database = orion_db._get_database()._db assert database['experiments'].count() == count_before value = list(database['experiments'].find({})) @@ -191,7 +192,7 @@ def test_update_with_id(self, exp_config, orion_db): filt = {'_id': exp_config[0][1]['_id']} count_before = orion_db._get_database().count('experiments') # call interface - assert orion_db.write('experiments', {'pool_size': 36}, filt) is True + assert orion_db.write('experiments', {'pool_size': 36}, filt) == 1 database = orion_db._get_database()._db assert database['experiments'].count() == count_before value = list(database['experiments'].find()) @@ -199,20 +200,6 @@ def test_update_with_id(self, exp_config, orion_db): assert value[1]['pool_size'] == 36 assert value[2]['pool_size'] == 2 - def test_upsert_with_id(self, orion_db): - """Query with a non-existent ``_id`` should upsert something.""" - filt = {'_id': 'lalalathisisnew'} - count_before = orion_db._get_database().count('experiments') - # call interface - assert orion_db.write('experiments', {'pool_size': 66}, filt) is True - database = orion_db._get_database()._db - assert database['experiments'].count() == count_before + 1 - value = list(database['experiments'].find(filt)) - assert len(value) == 1 - assert len(value[0]) == 2 - assert value[0]['_id'] == 'lalalathisisnew' - assert value[0]['pool_size'] == 66 - @pytest.mark.usefixtures("clean_db") class TestReadAndWrite(object): @@ -287,7 +274,7 @@ def test_remove_many_default(self, exp_config, orion_db): count_before = database['experiments'].count() count_filt = database['experiments'].count(filt) # call interface - assert orion_db.remove('experiments', filt) is True + assert orion_db.remove('experiments', filt) == count_filt database = orion_db._get_database()._db assert database['experiments'].count() == count_before - count_filt assert database['experiments'].count() == 1 @@ -300,7 +287,7 @@ def test_remove_with_id(self, exp_config, orion_db): database = orion_db._get_database()._db count_before = database['experiments'].count() # call interface - assert orion_db.remove('experiments', filt) is True + assert orion_db.remove('experiments', filt) == 1 database = orion_db._get_database()._db assert database['experiments'].count() == count_before - 1 assert database['experiments'].find() == exp_config[0][1:] From 00f05ba3464a4fdced1a6e9bd312279a74bff202 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Mon, 15 Jul 2019 15:20:31 -0400 Subject: [PATCH 25/50] Use yaml.safe_load instead of yaml (#219) Why: Since pyyaml v5.1 the function `yaml.load` prints a warning if not called with a specified `Loader`. This is because the default behavior was unsecure. Details about the issue are available here: https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation How: Use `yaml.safe_load` instead which uses the SafeLoader internally. --- src/orion/core/io/convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/orion/core/io/convert.py b/src/orion/core/io/convert.py index 97e548d66..139ddfda1 100644 --- a/src/orion/core/io/convert.py +++ b/src/orion/core/io/convert.py @@ -94,7 +94,7 @@ def parse(self, filepath): """ with open(filepath) as f: - return self.yaml.load(stream=f) + return self.yaml.safe_load(stream=f) def generate(self, filepath, data): """Create a configuration file at `filepath` using dictionary `data`.""" From 5c463dead80bd9bc0e4aa93d85484e48b2bd2510 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Mon, 15 Jul 2019 16:03:52 -0400 Subject: [PATCH 26/50] Fix ScriptConfigConflict (#215) Why: The conflict class was looking for `orion~prior()` templates in the values of the configuration files, but they can have other types than strings. --- src/orion/core/evc/conflicts.py | 3 ++- tests/unittests/core/conftest.py | 13 +++++++++++++ tests/unittests/core/evc/test_conflicts.py | 15 +++++++++++++++ tests/unittests/core/sample_config_diff.yml | 16 ++++++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 tests/unittests/core/sample_config_diff.yml diff --git a/src/orion/core/evc/conflicts.py b/src/orion/core/evc/conflicts.py index 73d4f67b0..f405ed8c5 100644 --- a/src/orion/core/evc/conflicts.py +++ b/src/orion/core/evc/conflicts.py @@ -1344,9 +1344,10 @@ def get_nameless_config(cls, config): """Get configuration dict of user's script without dimension definitions""" space_builder = SpaceBuilder() space_builder.build_from(config['metadata']['user_args']) + nameless_config = dict((key, value) for (key, value) in space_builder.parser.config_file_data.items() - if not value.startswith('orion~')) + if not (isinstance(value, str) and value.startswith('orion~'))) return nameless_config diff --git a/tests/unittests/core/conftest.py b/tests/unittests/core/conftest.py index 0fc13b3fc..a54ade4f1 100644 --- a/tests/unittests/core/conftest.py +++ b/tests/unittests/core/conftest.py @@ -17,6 +17,7 @@ TEST_DIR = os.path.dirname(os.path.abspath(__file__)) YAML_SAMPLE = os.path.join(TEST_DIR, 'sample_config.yml') +YAML_DIFF_SAMPLE = os.path.join(TEST_DIR, 'sample_config_diff.yml') JSON_SAMPLE = os.path.join(TEST_DIR, 'sample_config.json') UNKNOWN_SAMPLE = os.path.join(TEST_DIR, 'sample_config.txt') @@ -27,12 +28,24 @@ def yaml_sample_path(): return os.path.abspath(YAML_SAMPLE) +@pytest.fixture(scope='session') +def yaml_diff_sample_path(): + """Return path with a different yaml sample file.""" + return os.path.abspath(YAML_DIFF_SAMPLE) + + @pytest.fixture def yaml_config(yaml_sample_path): """Return a list containing the key and the sample path for a yaml config.""" return ['--config', yaml_sample_path] +@pytest.fixture +def yaml_diff_config(yaml_diff_sample_path): + """Return a list containing the key and the sample path for a different yaml config.""" + return ['--config', yaml_diff_sample_path] + + @pytest.fixture(scope='session') def json_sample_path(): """Return path with a json sample file.""" diff --git a/tests/unittests/core/evc/test_conflicts.py b/tests/unittests/core/evc/test_conflicts.py index 8a53db642..436031660 100644 --- a/tests/unittests/core/evc/test_conflicts.py +++ b/tests/unittests/core/evc/test_conflicts.py @@ -322,6 +322,21 @@ def test_repr(self, config_conflict): """Verify the representation of conflict for user interface""" assert repr(config_conflict) == "Script's configuration file changed" + def test_comparison(self, yaml_config, yaml_diff_config): + """Test that different configs are detected as conflict.""" + old_config = {'metadata': {'user_args': yaml_config}} + new_config = {'metadata': {'user_args': yaml_diff_config}} + + conflicts = list(conflict.ScriptConfigConflict.detect(old_config, new_config)) + assert len(conflicts) == 1 + + def test_comparison_idem(self, yaml_config): + """Test that identical configs are not detected as conflict.""" + old_config = {'metadata': {'user_args': yaml_config}} + new_config = {'metadata': {'user_args': yaml_config + ['--other', 'args']}} + + assert list(conflict.ScriptConfigConflict.detect(old_config, new_config)) == [] + @pytest.mark.usefixtures("create_db_instance") class TestExperimentNameConflict(object): diff --git a/tests/unittests/core/sample_config_diff.yml b/tests/unittests/core/sample_config_diff.yml new file mode 100644 index 000000000..6f01059e7 --- /dev/null +++ b/tests/unittests/core/sample_config_diff.yml @@ -0,0 +1,16 @@ +yo: 5 +training: + lr0: orion~loguniform(0.0001, 0.3) + mbs: orion~uniform(32, 256, discrete=True) + +# some comments + +layers: + - width: 128 + type: relu + - width: orion~uniform(32, 128, discrete=True) + type: orion~choices('relu', 'sigmoid', 'selu', 'leaky') + - width: 16 + type: orion~choices('relu', 'sigmoid', 'selu', 'leaky') + +something-same: orion~choices([1, 2, 3, 4, 5]) From bd0d9ed95690c59e5795c11291cc984f290cf304 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Mon, 15 Jul 2019 19:57:13 -0400 Subject: [PATCH 27/50] Add a functional test for user script config (#212) Why: The use of a config file for a user was not tested in the functional tests. I added one to make sure we cover this important use-case. --- tests/functional/demo/black_box_w_config.py | 45 +++++++++++++++++++++ tests/functional/demo/script_config.yaml | 1 + tests/functional/demo/test_demo.py | 45 +++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100755 tests/functional/demo/black_box_w_config.py create mode 100644 tests/functional/demo/script_config.yaml diff --git a/tests/functional/demo/black_box_w_config.py b/tests/functional/demo/black_box_w_config.py new file mode 100755 index 000000000..58b466467 --- /dev/null +++ b/tests/functional/demo/black_box_w_config.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Simple one dimensional example for a possible user's script.""" +import argparse + +import yaml + +from orion.client import report_results + + +def function(x): + """Evaluate partial information of a quadratic.""" + z = x - 34.56789 + return 4 * z**2 + 23.4, 8 * z + + +def execute(): + """Execute a simple pipeline as an example.""" + # 1. Receive inputs as you want + parser = argparse.ArgumentParser() + parser.add_argument('--config', required=True) + inputs = parser.parse_args() + + with open(inputs.config, 'r') as f: + config = yaml.load(f) + + # 2. Perform computations + y, dy = function(config['x']) + + # 3. Gather and report results + results = list() + results.append(dict( + name='example_objective', + type='objective', + value=y)) + results.append(dict( + name='example_gradient', + type='gradient', + value=[dy])) + + report_results(results) + + +if __name__ == "__main__": + execute() diff --git a/tests/functional/demo/script_config.yaml b/tests/functional/demo/script_config.yaml new file mode 100644 index 000000000..a41ceb1e2 --- /dev/null +++ b/tests/functional/demo/script_config.yaml @@ -0,0 +1 @@ +x: 'orion~uniform(-50, 50)' diff --git a/tests/functional/demo/test_demo.py b/tests/functional/demo/test_demo.py index a1ee2e4c2..c8288dd1e 100644 --- a/tests/functional/demo/test_demo.py +++ b/tests/functional/demo/test_demo.py @@ -91,6 +91,51 @@ def test_demo(database, monkeypatch): assert (params[0]['value'] - 34.56789) < 1e-5 +@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("null_db_instances") +def test_demo_with_script_config(database, monkeypatch): + """Test a simple usage scenario.""" + monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) + orion.core.cli.main(["hunt", "--config", "./orion_config.yaml", + "./black_box_w_config.py", "--config", "script_config.yaml"]) + + exp = list(database.experiments.find({'name': 'voila_voici'})) + assert len(exp) == 1 + exp = exp[0] + assert '_id' in exp + exp_id = exp['_id'] + assert exp['name'] == 'voila_voici' + assert exp['pool_size'] == 1 + assert exp['max_trials'] == 100 + assert exp['algorithms'] == {'gradient_descent': {'learning_rate': 0.1, + 'dx_tolerance': 1e-7}} + assert 'user' in exp['metadata'] + assert 'datetime' in exp['metadata'] + assert 'orion_version' in exp['metadata'] + assert 'user_script' in exp['metadata'] + assert os.path.isabs(exp['metadata']['user_script']) + assert exp['metadata']['user_args'] == ['--config', 'script_config.yaml'] + + trials = list(database.trials.find({'experiment': exp_id})) + assert len(trials) <= 15 + assert trials[-1]['status'] == 'completed' + trials = list(sorted(trials, key=lambda trial: trial['submit_time'])) + for result in trials[-1]['results']: + assert result['type'] != 'constraint' + if result['type'] == 'objective': + assert abs(result['value'] - 23.4) < 1e-6 + assert result['name'] == 'example_objective' + elif result['type'] == 'gradient': + res = numpy.asarray(result['value']) + assert 0.1 * numpy.sqrt(res.dot(res)) < 1e-7 + assert result['name'] == 'example_gradient' + params = trials[-1]['params'] + assert len(params) == 1 + assert params[0]['name'] == '/x' + assert params[0]['type'] == 'real' + assert (params[0]['value'] - 34.56789) < 1e-5 + + @pytest.mark.usefixtures("clean_db") def test_demo_two_workers(database, monkeypatch): """Test a simple usage scenario.""" From ccc343e16298143f3dbdae24c54cc9ce2beabcd8 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Thu, 11 Jul 2019 20:38:38 -0400 Subject: [PATCH 28/50] Handle non invalid $op for EphemeralDB Why: The operator $ne was not supported by EphemeralDB but the operation was silently failling as a failed comparison (the field `whatever.$ne` could not exist in the document). Any unsupported operation should blatantly fail. --- src/orion/core/io/database/ephemeraldb.py | 21 +++++++-- tests/unittests/core/test_ephemeraldb.py | 56 +++++++++++++++++++++++ 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/src/orion/core/io/database/ephemeraldb.py b/src/orion/core/io/database/ephemeraldb.py index 87b525109..298afc889 100644 --- a/src/orion/core/io/database/ephemeraldb.py +++ b/src/orion/core/io/database/ephemeraldb.py @@ -305,19 +305,30 @@ def match(self, query=None): return True + def _is_operator(self, key): # pylint: disable=no-self-use + return key.split(".")[-1].startswith('$') + + def _get_key_operator(self, key): + operator = key.split(".")[-1] + key = ".".join(key.split(".")[:-1]) + + if operator not in self.operators: + raise ValueError('Operator \'{}\' is not supported by EphemeralDB'.format(operator)) + + return key, self.operators[operator] + def match_key(self, key, value): """Test if a data corresponding to the given key is in agreement with the given value based on the operator defined within the key. Default operator is equal when no operator is defined. - Other operators could be $in, $gte, $gt. They are defined + Other operators could be $in, $gte, $gt or $lte. They are defined in the last section of the key. For example: `abc.def.$in` or `abc.def.$gte`. """ - if key.split(".")[-1] in self.operators: - operator = key.split(".")[-1] - key = ".".join(key.split(".")[:-1]) + if self._is_operator(key): + key, operator = self._get_key_operator(key) - return key in self and self.operators[operator](self[key], value) + return key in self and operator(self[key], value) return key in self and self[key] == value diff --git a/tests/unittests/core/test_ephemeraldb.py b/tests/unittests/core/test_ephemeraldb.py index a68f77375..3f83140d6 100644 --- a/tests/unittests/core/test_ephemeraldb.py +++ b/tests/unittests/core/test_ephemeraldb.py @@ -16,6 +16,13 @@ def document(): return EphemeralDocument({'_id': 1, 'hello': 'there', 'mighty': 'duck'}) +@pytest.fixture() +def subdocument(): + """Return EphemeralDocument with a subdocument.""" + return EphemeralDocument({'_id': 1, 'hello': 'there', 'mighty': 'duck', + 'and': {'the': 'drake'}}) + + @pytest.fixture() def collection(document): """Return EphemeralCollection.""" @@ -374,3 +381,52 @@ def test_unselect_two(self, document): def test_mixed_select(self, document): """Select one field and unselect _id.""" assert document.select({'_id': 0, 'hello': 1}) == {'hello': 'there'} + + +@pytest.mark.usefixtures('clean_db') +class TestMatch: + """Calls :meth:`orion.core.io.database.ephemeraldb.EphemeralDocument.match`.""" + + def test_match_eq(self, document): + """Test eq operator""" + assert document.match({'hello': 'there'}) + assert not document.match({'hello': 'not there'}) + + def test_match_sub_eq(self, subdocument): + """Test eq operator with sub document""" + assert subdocument.match({'and.the': 'drake'}) + assert not subdocument.match({'and.no': 'drake'}) + + def test_match_in(self, subdocument): + """Test $in operator with document""" + assert subdocument.match({'hello': {'$in': ['there', 'here']}}) + assert not subdocument.match({'hello': {'$in': ['ici', 'here']}}) + + def test_match_sub_in(self, subdocument): + """Test $in operator with sub document""" + assert subdocument.match({'and.the': {'$in': ['duck', 'drake']}}) + assert not subdocument.match({'and.the': {'$in': ['hyppo', 'lion']}}) + + def test_match_gte(self, document): + """Test $gte operator with document""" + assert document.match({'_id': {'$gte': 1}}) + assert document.match({'_id': {'$gte': 0}}) + assert not document.match({'_id': {'$gte': 2}}) + + def test_match_gt(self, document): + """Test $gt operator with document""" + assert document.match({'_id': {'$gt': 0}}) + assert not document.match({'_id': {'$gt': 1}}) + + def test_match_lte(self, document): + """Test $lte operator with document""" + assert document.match({'_id': {'$lte': 2}}) + assert document.match({'_id': {'$lte': 1}}) + assert not document.match({'_id': {'$lte': 0}}) + + def test_match_bad_operator(self, document): + """Test invalid operator handling""" + with pytest.raises(ValueError) as exc: + document.match({'_id': {'$voici_voila': 0}}) + + assert 'Operator \'$voici_voila\' is not supported' in str(exc.value) From 7c3407c293654585d67769c26dd037e5904696d7 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Thu, 11 Jul 2019 20:41:50 -0400 Subject: [PATCH 29/50] Add $ne op to EphemeralDB --- src/orion/core/io/database/ephemeraldb.py | 8 +++++--- tests/unittests/core/test_ephemeraldb.py | 5 +++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/orion/core/io/database/ephemeraldb.py b/src/orion/core/io/database/ephemeraldb.py index 298afc889..9dd12f6a8 100644 --- a/src/orion/core/io/database/ephemeraldb.py +++ b/src/orion/core/io/database/ephemeraldb.py @@ -283,6 +283,7 @@ class EphemeralDocument(object): """ operators = { + "$ne": (lambda a, b: a != b), "$in": (lambda a, b: a in b), "$gte": (lambda a, b: a is not None and a >= b), "$gt": (lambda a, b: a is not None and a > b), @@ -309,8 +310,9 @@ def _is_operator(self, key): # pylint: disable=no-self-use return key.split(".")[-1].startswith('$') def _get_key_operator(self, key): - operator = key.split(".")[-1] - key = ".".join(key.split(".")[:-1]) + path = key.split(".") + operator = path[-1] + key = ".".join(path[:-1]) if operator not in self.operators: raise ValueError('Operator \'{}\' is not supported by EphemeralDB'.format(operator)) @@ -322,7 +324,7 @@ def match_key(self, key, value): value based on the operator defined within the key. Default operator is equal when no operator is defined. - Other operators could be $in, $gte, $gt or $lte. They are defined + Other operators could be $ne, $in, $gte, $gt or $lte. They are defined in the last section of the key. For example: `abc.def.$in` or `abc.def.$gte`. """ if self._is_operator(key): diff --git a/tests/unittests/core/test_ephemeraldb.py b/tests/unittests/core/test_ephemeraldb.py index 3f83140d6..58f2d5728 100644 --- a/tests/unittests/core/test_ephemeraldb.py +++ b/tests/unittests/core/test_ephemeraldb.py @@ -424,6 +424,11 @@ def test_match_lte(self, document): assert document.match({'_id': {'$lte': 1}}) assert not document.match({'_id': {'$lte': 0}}) + def test_match_ne(self, document): + """Test $ne operator with document""" + assert document.match({'hello': {'$ne': 'here'}}) + assert not document.match({'hello': {'$ne': 'there'}}) + def test_match_bad_operator(self, document): """Test invalid operator handling""" with pytest.raises(ValueError) as exc: From ac37c82a7b9b9cf9d04c6742aa0dfc05bdabf3e9 Mon Sep 17 00:00:00 2001 From: Setepenre Date: Tue, 16 Jul 2019 15:45:23 -0400 Subject: [PATCH 30/50] Legacy proto (#221) Encapsulate communications with DB with a new Storage Protocol. Current implementation becomes `Legacy(StorageProtocol)`. Next step is to include a backend for Track. --- setup.py | 4 + src/orion/core/worker/__init__.py | 5 +- src/orion/core/worker/consumer.py | 31 ++-- src/orion/core/worker/experiment.py | 91 +++++------ src/orion/storage/base.py | 144 ++++++++++++++++++ src/orion/storage/legacy.py | 114 ++++++++++++++ .../core/io/test_experiment_builder.py | 22 ++- tests/unittests/core/test_experiment.py | 101 ++++++++---- 8 files changed, 409 insertions(+), 103 deletions(-) create mode 100644 src/orion/storage/base.py create mode 100644 src/orion/storage/legacy.py diff --git a/setup.py b/setup.py index 3cb70839b..73166c7ec 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ def find_data_files(): 'orion.core', 'orion.client', 'orion.algo', + 'orion.storage' ] setup_args = dict( @@ -67,6 +68,9 @@ def find_data_files(): 'random = orion.algo.random:Random', 'asha = orion.algo.asha:ASHA', ], + 'StorageProtocol': [ + 'legacy = orion.storage.legacy:Legacy' + ] }, install_requires=['PyYAML', 'pymongo>=3', 'numpy', 'scipy', 'gitpython', 'filelock'], tests_require=tests_require, diff --git a/src/orion/core/worker/__init__.py b/src/orion/core/worker/__init__.py index 541200d5b..c944644c7 100644 --- a/src/orion/core/worker/__init__.py +++ b/src/orion/core/worker/__init__.py @@ -14,7 +14,6 @@ import logging import pprint -from orion.core.io.database import Database from orion.core.worker.consumer import Consumer from orion.core.worker.producer import Producer @@ -72,14 +71,14 @@ def workon(experiment, worker_trials=None): log.info("No trials completed.") return - best = Database().read('trials', {'_id': stats['best_trials_id']})[0] + best = experiment.fetch_trials({'_id': stats['best_trials_id']})[0] stats_stream = io.StringIO() pprint.pprint(stats, stream=stats_stream) stats_string = stats_stream.getvalue() best_stream = io.StringIO() - pprint.pprint(best['params'], stream=best_stream) + pprint.pprint(best.to_dict()['params'], stream=best_stream) best_string = best_stream.getvalue() log.info("##### Search finished successfully #####") diff --git a/src/orion/core/worker/consumer.py b/src/orion/core/worker/consumer.py index 7dd5ababa..f3971f5c1 100644 --- a/src/orion/core/worker/consumer.py +++ b/src/orion/core/worker/consumer.py @@ -14,11 +14,9 @@ import subprocess import tempfile -from orion.core.io.convert import JSONConverter -from orion.core.io.database import Database from orion.core.io.space_builder import SpaceBuilder from orion.core.utils.working_dir import WorkingDir -from orion.core.worker.trial import Trial + log = logging.getLogger(__name__) @@ -66,8 +64,6 @@ def __init__(self, experiment): self.script_path = experiment.metadata['user_script'] - self.converter = JSONConverter() - def consume(self, trial): """Execute user's script as a block box using the options contained within `trial`. @@ -85,21 +81,22 @@ def consume(self, trial): prefix=prefix, suffix=suffix) as workdirname: log.debug("## New consumer context: %s", workdirname) trial.working_dir = workdirname - self._consume(trial, workdirname) + + results_file = self._consume(trial, workdirname) + + log.debug("## Parse results from file and fill corresponding Trial object.") + self.experiment.update_completed_trial(trial, results_file) + except KeyboardInterrupt: log.debug("### Save %s as interrupted.", trial) trial.status = 'interrupted' - Database().write('trials', trial.to_dict(), - query={'_id': trial.id}) + self.experiment.update_trial(trial, status=trial.status) + raise except RuntimeError: log.debug("### Save %s as broken.", trial) trial.status = 'broken' - Database().write('trials', trial.to_dict(), - query={'_id': trial.id}) - else: - log.debug("### Register successfully evaluated %s.", trial) - self.experiment.push_completed_trial(trial) + self.experiment.update_trial(trial, status=trial.status) def _consume(self, trial, workdirname): config_file = tempfile.NamedTemporaryFile(mode='w', prefix='trial_', @@ -119,13 +116,7 @@ def _consume(self, trial, workdirname): log.debug("## Launch user's script as a subprocess and wait for finish.") self.execute_process(results_file.name, cmd_args) - - log.debug("## Parse results from file and fill corresponding Trial object.") - results = self.converter.parse(results_file.name) - - trial.results = [Trial.Result(name=res['name'], - type=res['type'], - value=res['value']) for res in results] + return results_file def execute_process(self, results_filename, cmd_args): """Facilitate launching a black-box trial.""" diff --git a/src/orion/core/worker/experiment.py b/src/orion/core/worker/experiment.py index 6fae9494b..ace4afb33 100644 --- a/src/orion/core/worker/experiment.py +++ b/src/orion/core/worker/experiment.py @@ -19,7 +19,7 @@ from orion.core.cli.evc import fetch_branching_configuration from orion.core.evc.adapters import Adapter, BaseAdapter from orion.core.evc.conflicts import detect_conflicts -from orion.core.io.database import Database, DuplicateKeyError, ReadOnlyDB +from orion.core.io.database import DuplicateKeyError from orion.core.io.experiment_branch_builder import ExperimentBranchBuilder from orion.core.io.interactive_commands.branching_prompt import BranchingPrompt from orion.core.io.space_builder import SpaceBuilder @@ -27,8 +27,8 @@ from orion.core.worker.primary_algo import PrimaryAlgo from orion.core.worker.strategy import (BaseParallelStrategy, Strategy) -from orion.core.worker.trial import Trial from orion.core.worker.trial_monitor import TrialMonitor +from orion.storage.base import ReadOnlyStorageProtocol, StorageProtocol log = logging.getLogger(__name__) @@ -86,8 +86,8 @@ class Experiment(object): """ __slots__ = ('name', 'refers', 'metadata', 'pool_size', 'max_trials', - 'algorithms', 'producer', 'working_dir', '_db', '_init_done', '_id', - '_node', '_last_fetched') + 'algorithms', 'producer', 'working_dir', '_init_done', '_id', + '_node', '_last_fetched', '_storage') non_branching_attrs = ('pool_size', 'max_trials') def __init__(self, name, user=None): @@ -106,8 +106,7 @@ def __init__(self, name, user=None): """ log.debug("Creating Experiment object with name: %s", name) self._init_done = False - self._db = Database() # fetch database instance - self._setup_db() # build indexes for collections + self._storage = StorageProtocol('legacy') self._id = None self.name = name @@ -122,8 +121,8 @@ def __init__(self, name, user=None): self.working_dir = None self.producer = {'strategy': None} - config = self._db.read('experiments', - {'name': name, 'metadata.user': user}) + config = self._storage.fetch_experiments({'name': name, 'metadata.user': user}) + if config: log.debug("Found existing experiment, %s, under user, %s, registered in database.", name, user) @@ -140,19 +139,6 @@ def __init__(self, name, user=None): self._last_fetched = self.metadata.get("datetime", datetime.datetime.utcnow()) - def _setup_db(self): - self._db.ensure_index('experiments', - [('name', Database.ASCENDING), - ('metadata.user', Database.ASCENDING)], - unique=True) - self._db.ensure_index('experiments', 'metadata.datetime') - - self._db.ensure_index('trials', 'experiment') - self._db.ensure_index('trials', 'status') - self._db.ensure_index('trials', 'results') - self._db.ensure_index('trials', 'start_time') - self._db.ensure_index('trials', [('end_time', Database.DESCENDING)]) - def fetch_trials(self, query, selection=None): """Fetch trials of the experiment in the database @@ -170,7 +156,7 @@ def fetch_trials(self, query, selection=None): """ query["experiment"] = self._id - trials = Trial.build(self._db.read('trials', query, selection)) + trials = self._storage.fetch_trials(query, selection) def _get_submit_time(trial): if trial.submit_time: @@ -209,15 +195,25 @@ def connect_to_version_control_tree(self, node): """ self._node = node - def reserve_trial(self, score_handle=None): + def retrieve_result(self, trial, *args, **kwargs): + """See :func:`~orion.storage.BaseStorageProtocol.retrieve_result`""" + return self._storage.retrieve_result(trial, *args, **kwargs) + + def update_trial(self, *args, **kwargs): + """See :func:`~orion.storage.BaseStorageProtocol.update_trial`""" + return self._storage.update_trial(*args, **kwargs) + + def reserve_trial(self, score_handle=None, _depth=1): """Find *new* trials that exist currently in database and select one of them based on the highest score return from `score_handle` callable. :param score_handle: A way to decide which trial out of the *new* ones to to pick as *reserved*, defaults to a random choice. :type score_handle: callable + :param _depth: recursion depth only used for logging purposes can be ignored :return: selected `Trial` object, None if could not find any. """ + log.debug('%s reserving trial with (score: %s)', '<' * _depth, score_handle) if score_handle is not None and not callable(score_handle): raise ValueError("Argument `score_handle` must be callable with a `Trial`.") @@ -227,9 +223,12 @@ def reserve_trial(self, score_handle=None): experiment=self._id, status={'$in': ['new', 'suspended', 'interrupted']} ) + new_trials = self.fetch_trials(query) + log.debug('%s Fetched (trials: %s)', '<' * _depth, len(new_trials)) if not new_trials: + log.debug('%s no new trials found', '<' * _depth) return None if score_handle is not None and self.space: @@ -243,24 +242,29 @@ def reserve_trial(self, score_handle=None): "parameter space has not been defined yet.") selected_trial = random.sample(new_trials, 1)[0] - - # Query on status to ensure atomicity. If another process change the - # status meanwhile, read_and_write will fail, because query will fail. - query = {'_id': selected_trial.id, 'status': selected_trial.status} + log.debug('%s selected (trial: %s)', '<' * _depth, selected_trial) update = dict(status='reserved', heartbeat=datetime.datetime.utcnow()) if selected_trial.status == 'new': update["start_time"] = datetime.datetime.utcnow() - reserved = self._db.write('trials', query=query, data=update) + # Query on status to ensure atomicity. If another process change the + # status meanwhile, update will fail, because query will fail. + # This relies on the atomicity of document updates. + + log.debug('%s trying to reverse trial', '<' * _depth) + reserved = self._storage.update_trial( + selected_trial, **update, where={'status': selected_trial.status}) if not reserved: - selected_trial = self.reserve_trial(score_handle=score_handle) + selected_trial = self.reserve_trial(score_handle=score_handle, _depth=_depth + 1) else: + log.debug('%s found suitable trial', '<' * _depth) selected_trial = self.fetch_trials({'_id': selected_trial.id})[0] TrialMonitor(self, selected_trial.id).start() + log.debug('%s reserved trial (trial: %s)', '<' * _depth, selected_trial) return selected_trial def fix_lost_trials(self): @@ -282,10 +286,11 @@ def fix_lost_trials(self): for trial in trials: query['_id'] = trial.id log.debug('Setting lost trial %s status to interrupted...', trial.id) - updated = self._db.write('trials', {'status': 'interrupted'}, query) + + updated = self._storage.update_trial(trial, status='interrupted', where=query) log.debug('success' if updated else 'failed') - def push_completed_trial(self, trial): + def update_completed_trial(self, trial, results_file): """Inform database about an evaluated `trial` with resultlts. :param trial: Corresponds to a successful evaluation of a particular run. @@ -296,9 +301,11 @@ def push_completed_trial(self, trial): Change status from *reserved* to *completed*. """ + self._storage.retrieve_result(trial, results_file) + trial.end_time = datetime.datetime.utcnow() trial.status = 'completed' - self._db.write('trials', trial.to_dict(), query={'_id': trial.id}) + self._storage.update_trial(trial, **trial.to_dict()) def register_lie(self, lying_trial): """Register a *fake* trial created by the strategist. @@ -324,8 +331,7 @@ def register_lie(self, lying_trial): """ lying_trial.status = 'completed' lying_trial.end_time = datetime.datetime.utcnow() - - self._db.write('lying_trials', lying_trial.to_dict()) + self._storage.register_lie(lying_trial) def register_trial(self, trial): """Register new trial in the database. @@ -351,7 +357,7 @@ def register_trial(self, trial): trial.status = 'new' trial.submit_time = stamp - self._db.write('trials', trial.to_dict()) + self._storage.register_trial(trial) def fetch_completed_trials(self): """Fetch recent completed trials that this `Experiment` instance has not yet seen. @@ -433,7 +439,7 @@ def is_done(self): experiment=self._id, status='completed' ) - num_completed_trials = self._db.count('trials', query) + num_completed_trials = len(self._storage.fetch_trials(query)) return ((num_completed_trials >= self.max_trials) or (self._init_done and self.algorithms.is_done)) @@ -448,7 +454,7 @@ def is_broken(self): """ query = {'experiment': self._id, 'status': 'broken'} - num_broken_trials = self._db.count('trials', query) + num_broken_trials = len(self._storage.fetch_trials(query)) return num_broken_trials >= 3 @@ -551,7 +557,8 @@ def configure(self, config, enable_branching=True, enable_update=True): # This will raise DuplicateKeyError if a concurrent experiment with # identical (name, metadata.user) is written first in the database. - self._db.write('experiments', final_config) + self._storage.create_experiment(final_config) + # XXX: Reminder for future DB implementations: # MongoDB, updates an inserted dict with _id, so should you :P self._id = final_config['_id'] @@ -559,9 +566,7 @@ def configure(self, config, enable_branching=True, enable_update=True): # Update refers in db if experiment is root if not self.refers: self.refers = {'root_id': self._id, 'parent_id': None, 'adapter': []} - update = {'refers': self.refers} - query = {'_id': self._id} - self._db.write('experiments', data=update, query=query) + self._storage.update_experiment(self, refers=self.refers) else: # Writing the final config to an already existing experiment raises @@ -570,7 +575,7 @@ def configure(self, config, enable_branching=True, enable_update=True): # `db.write()`, thus seamingly breaking the compound index # `(name, metadata.user)` final_config.pop("name") - self._db.write('experiments', final_config, {'_id': self._id}) + self._storage.update_experiment(self, **final_config) @property def stats(self): @@ -788,7 +793,7 @@ def __init__(self, name, user=None): raise - self._experiment._db = ReadOnlyDB(self._experiment._db) + self._experiment._storage = ReadOnlyStorageProtocol(self._experiment._storage) def __getattr__(self, name): """Get attribute only if valid""" diff --git a/src/orion/storage/base.py b/src/orion/storage/base.py new file mode 100644 index 000000000..51bbfe8b1 --- /dev/null +++ b/src/orion/storage/base.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +""" +:mod:`orion.storage.base -- Generic Storage Protocol +==================================================== + +.. module:: base + :platform: Unix + :synopsis: Implement a generic protocol to allow Orion to communicate using + different storage backend + +""" + +from orion.core.utils import Factory + + +class BaseStorageProtocol: + """Implement a generic protocol to allow Orion to communicate using + different storage backend + + """ + + def create_experiment(self, config): + """Insert a new experiment inside the database""" + raise NotImplementedError() + + def update_experiment(self, experiment, where=None, **kwargs): + """Update a the fields of a given trials + + Parameters + ---------- + experiment: Experiment + Experiment object to update + + where: Optional[dict] + constraint experiment must respect + + **kwargs: dict + a dictionary of fields to update + + Returns + ------- + returns true if the underlying storage was updated + + """ + raise NotImplementedError() + + def fetch_experiments(self, query): + """Fetch all experiments that match the query""" + raise NotImplementedError() + + def register_trial(self, trial): + """Create a new trial to be executed""" + raise NotImplementedError() + + def register_lie(self, trial): + """Register a *fake* trial created by the strategist. + + The main difference between fake trial and orignal ones is the addition of a fake objective + result, and status being set to completed. The id of the fake trial is different than the id + of the original trial, but the original id can be computed using the hashcode on parameters + of the fake trial. See mod:`orion.core.worker.strategy` for more information and the + Strategist object and generation of fake trials. + + Parameters + ---------- + trial: `Trial` object + Fake trial to register in the database + + """ + raise NotImplementedError() + + def reserve_trial(self, *args, **kwargs): + """Select a pending trial and reserve it for the worker""" + raise NotImplementedError() + + def fetch_trials(self, query, *args, **kwargs): + """Fetch all the trials that match the query""" + raise NotImplementedError() + + def update_trial(self, trial, where=None, **kwargs): + """Update the fields of a given trials + + Parameters + ---------- + trial: Trial + Trial object to update + + where: Optional[dict] + constraint trial must respect + + kwargs: dict + a dictionary of fields to update + + Returns + ------- + returns true if the underlying storage was updated + + """ + raise NotImplementedError() + + def retrieve_result(self, trial, **kwargs): + """Fetch the result from a given medium (file, db, socket, etc..) for a given trial and + insert it into the trial object + """ + raise NotImplementedError() + + +# pylint: disable=too-few-public-methods,abstract-method +class StorageProtocol(BaseStorageProtocol, metaclass=Factory): + """Storage protocol is a generic way of allowing Orion to interface with different storage. + MongoDB, track, cometML, MLFLow, etc... + + Examples + -------- + >>> StorageProtocol('track', uri='file://orion_test.json') + >>> StorageProtocol('legacy', experiment=...) + + """ + + pass + + +# pylint: disable=too-few-public-methods +class ReadOnlyStorageProtocol(object): + """Read-only interface from a storage protocol. + + .. seealso:: + + :py:class:`orion.core.storage.BaseStorageProtocol` + """ + + __slots__ = ('_storage', ) + valid_attributes = {"fetch_trials", "fetch_experiments"} + + def __init__(self, protocol): + """Init method, see attributes of :class:`BaseStorageProtocol`.""" + self._storage = protocol + + def __getattr__(self, attr): + """Get attribute only if valid""" + if attr not in self.valid_attributes: + raise AttributeError("Cannot access attribute %s on ReadOnlyStorageProtocol." % attr) + + return getattr(self._storage, attr) diff --git a/src/orion/storage/legacy.py b/src/orion/storage/legacy.py new file mode 100644 index 000000000..7fea573d9 --- /dev/null +++ b/src/orion/storage/legacy.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +""" +:mod:`orion.storage.legacy` -- Legacy storage +============================================================================= + +.. module:: legacy + :platform: Unix + :synopsis: Old Storage implementation + +""" + +from orion.core.io.convert import JSONConverter +from orion.core.io.database import Database +from orion.core.worker.trial import Trial +from orion.storage.base import BaseStorageProtocol + + +class Legacy(BaseStorageProtocol): + """Legacy protocol, store all experiments and trials inside the Database() + + Parameters + ---------- + uri: str + database uri specifying how to connect to the database + the uri follows the following format + `mongodb://[username:password@]host1[:port1][,...hostN[:portN]][/[database][?options]]` + + """ + + def __init__(self, uri=None): + self._db = Database() + self._setup_db() + + def _setup_db(self): + """Database index setup""" + self._db.ensure_index('experiments', + [('name', Database.ASCENDING), + ('metadata.user', Database.ASCENDING)], + unique=True) + self._db.ensure_index('experiments', 'metadata.datetime') + + self._db.ensure_index('trials', 'experiment') + self._db.ensure_index('trials', 'status') + self._db.ensure_index('trials', 'results') + self._db.ensure_index('trials', 'start_time') + self._db.ensure_index('trials', [('end_time', Database.DESCENDING)]) + + def create_experiment(self, config): + """See :func:`~orion.storage.BaseStorageProtocol.create_experiment`""" + return self._db.write('experiments', config) + + def update_experiment(self, experiment, where=None, **kwargs): + """See :func:`~orion.storage.BaseStorageProtocol.update_experiment`""" + if where is None: + where = dict() + + where['_id'] = experiment._id + return self._db.write('experiments', data=kwargs, query=where) + + def fetch_experiments(self, query): + """See :func:`~orion.storage.BaseStorageProtocol.fetch_experiments`""" + return self._db.read('experiments', query) + + def fetch_trials(self, query, selection=None): + """See :func:`~orion.storage.BaseStorageProtocol.fetch_trials`""" + return [Trial(**t) for t in self._db.read('trials', query=query, selection=selection)] + + def register_trial(self, trial): + """See :func:`~orion.storage.BaseStorageProtocol.register_trial`""" + self._db.write('trials', trial.to_dict()) + return trial + + def register_lie(self, trial): + """See :func:`~orion.storage.BaseStorageProtocol.register_lie`""" + self._db.write('lying_trials', trial.to_dict()) + + def retrieve_result(self, trial, results_file=None, **kwargs): + """Parse the results file that was generated by the trial process. + + Parameters + ---------- + trial: Trial + The trial object to be updated + + results_file: str + the file handle to read the result from + + Returns + ------- + returns the updated trial object + + Note + ---- + This does not update the database! + + """ + results = JSONConverter().parse(results_file.name) + + trial.results = [ + Trial.Result( + name=res['name'], + type=res['type'], + value=res['value']) for res in results + ] + + return trial + + def update_trial(self, trial: Trial, where=None, **kwargs) -> Trial: + """See :func:`~orion.storage.BaseStorageProtocol.update_trial`""" + if where is None: + where = dict() + + where['_id'] = trial.id + return self._db.write('trials', data=kwargs, query=where) diff --git a/tests/unittests/core/io/test_experiment_builder.py b/tests/unittests/core/io/test_experiment_builder.py index 5d57bd300..5f08dc473 100644 --- a/tests/unittests/core/io/test_experiment_builder.py +++ b/tests/unittests/core/io/test_experiment_builder.py @@ -10,6 +10,16 @@ from orion.core.utils.exceptions import NoConfigurationError +def get_db(exp): + """Transitional method to move away from mongodb""" + return exp._storage._db + + +def get_view_db(exp): + """Transitional method to move away from mongodb""" + return exp._experiment._storage._storage._db + + @pytest.mark.usefixtures("clean_db") def test_fetch_local_config(config_file): """Test local config (default, env_vars, cmdconfig, cmdargs)""" @@ -145,7 +155,7 @@ def test_build_view_from(config_file, create_db_instance, exp_config, random_dt) exp_view = ExperimentBuilder().build_view_from(cmdargs) assert exp_view._experiment._init_done is True - assert exp_view._experiment._db._database is create_db_instance + assert get_view_db(exp_view) is create_db_instance assert exp_view._id == exp_config[0][0]['_id'] assert exp_view.name == exp_config[0][0]['name'] assert exp_view.configuration['refers'] == exp_config[0][0]['refers'] @@ -185,7 +195,7 @@ def test_build_from_no_hit(config_file, create_db_instance, exp_config, random_d exp = ExperimentBuilder().build_from(cmdargs) assert exp._init_done is True - assert exp._db is create_db_instance + assert get_db(exp) is create_db_instance assert exp.name == cmdargs['name'] assert exp.configuration['refers'] == {'adapter': [], 'parent_id': None, 'root_id': exp._id} assert exp.metadata['datetime'] == random_dt @@ -213,7 +223,7 @@ def test_build_from_hit(old_config_file, create_db_instance, exp_config, script_ exp = ExperimentBuilder().build_from(cmdargs) assert exp._init_done is True - assert exp._db is create_db_instance + assert get_db(exp) is create_db_instance assert exp._id == exp_config[0][0]['_id'] assert exp.name == exp_config[0][0]['name'] assert exp.configuration['refers'] == exp_config[0][0]['refers'] @@ -249,7 +259,7 @@ def test_build_from_config_no_hit(config_file, create_db_instance, exp_config, r exp = ExperimentBuilder().build_from_config(full_config) assert exp._init_done is True - assert exp._db is create_db_instance + assert get_db(exp) is create_db_instance assert exp.name == cmdargs['name'] assert exp.configuration['refers'] == {'adapter': [], 'parent_id': None, 'root_id': exp._id} assert exp.metadata['datetime'] == random_dt @@ -288,7 +298,7 @@ def test_build_from_config_hit(old_config_file, create_db_instance, exp_config, exp = ExperimentBuilder().build_from_config(exp_view.configuration) assert exp._init_done is True - assert exp._db is create_db_instance + assert get_db(exp) is create_db_instance assert exp._id == exp_config[0][0]['_id'] assert exp.name == exp_config[0][0]['name'] assert exp.configuration['refers'] == exp_config[0][0]['refers'] @@ -317,7 +327,7 @@ def test_build_without_config_hit(old_config_file, create_db_instance, exp_confi exp = ExperimentBuilder().build_from_config(exp_view.configuration) assert exp._init_done is True - assert exp._db is create_db_instance + assert get_db(exp) is create_db_instance assert exp._id == exp_config[0][0]['_id'] assert exp.name == exp_config[0][0]['name'] assert exp.configuration['refers'] == exp_config[0][0]['refers'] diff --git a/tests/unittests/core/test_experiment.py b/tests/unittests/core/test_experiment.py index d38798c11..f3892e35a 100644 --- a/tests/unittests/core/test_experiment.py +++ b/tests/unittests/core/test_experiment.py @@ -5,7 +5,10 @@ import copy import datetime import getpass +import json +import logging import random +import tempfile import pytest @@ -15,6 +18,9 @@ from orion.core.worker.trial import Trial +logging.basicConfig(level=logging.DEBUG) + + @pytest.fixture() def patch_sample(monkeypatch): """Patch ``random.sample`` to return the first one and check call.""" @@ -150,6 +156,21 @@ def new_config(random_dt): return new_config +def assert_protocol(exp, create_db_instance): + """Transitional method to move away from mongodb""" + assert exp._storage._db is create_db_instance + + +def count_experiment(exp): + """Transitional method to move away from mongodb""" + return exp._storage._db.count("experiments") + + +def get_db_from_view(exp): + """Transitional method to move away from mongodb""" + return exp._storage._db._db + + class TestInitExperiment(object): """Create new Experiment instance.""" @@ -158,7 +179,7 @@ def test_new_experiment_due_to_name(self, create_db_instance, random_dt): """Hit user name, but exp_name does not hit the db, create new entry.""" exp = Experiment('supernaekei') assert exp._init_done is False - assert exp._db is create_db_instance + assert_protocol(exp, create_db_instance) assert exp._id is None assert exp.name == 'supernaekei' assert exp.refers == {} @@ -177,7 +198,7 @@ def test_new_experiment_due_to_username(self, create_db_instance, random_dt): """Hit exp_name, but user's name does not hit the db, create new entry.""" exp = Experiment('supernaedo2') assert exp._init_done is False - assert exp._db is create_db_instance + assert_protocol(exp, create_db_instance) assert exp._id is None assert exp.name == 'supernaedo2' assert exp.refers == {} @@ -196,7 +217,7 @@ def test_existing_experiment(self, create_db_instance, exp_config): """Hit exp_name + user's name in the db, fetch most recent entry.""" exp = Experiment('supernaedo2') assert exp._init_done is False - assert exp._db is create_db_instance + assert_protocol(exp, create_db_instance) assert exp._id == exp_config[0][0]['_id'] assert exp.name == exp_config[0][0]['name'] assert exp.refers == exp_config[0][0]['refers'] @@ -393,7 +414,7 @@ def test_get_after_init_plus_hit_no_diffs(self, exp_config): """ exp = Experiment('supernaedo2') # Deliver an external configuration to finalize init - experiment_count_before = exp._db.count("experiments") + experiment_count_before = count_experiment(exp) exp.configure(exp_config[0][0]) assert exp._init_done is True exp_config[0][0]['algorithms']['dumbalgo']['done'] = False @@ -405,7 +426,7 @@ def test_get_after_init_plus_hit_no_diffs(self, exp_config): exp_config[0][0]['producer']['strategy'] = "NoParallelStrategy" assert exp._id == exp_config[0][0].pop('_id') assert exp.configuration == exp_config[0][0] - assert experiment_count_before == exp._db.count("experiments") + assert experiment_count_before == count_experiment(exp) def test_try_set_after_init(self, exp_config): """Cannot set a configuration after init (currently).""" @@ -428,19 +449,19 @@ def test_try_set_after_race_condition(self, exp_config, new_config): exp = Experiment(new_config['name']) assert exp.id is None # Another experiment gets configured first - experiment_count_before = exp._db.count("experiments") + experiment_count_before = count_experiment(exp) naughty_little_exp = Experiment(new_config['name']) assert naughty_little_exp.id is None naughty_little_exp.configure(new_config) assert naughty_little_exp._init_done is True assert exp._init_done is False - assert (experiment_count_before + 1) == exp._db.count("experiments") + assert (experiment_count_before + 1) == count_experiment(exp) # First experiment won't be able to be configured with pytest.raises(DuplicateKeyError) as exc_info: exp.configure(new_config) assert 'duplicate key error' in str(exc_info.value) - assert (experiment_count_before + 1) == exp._db.count("experiments") + assert (experiment_count_before + 1) == count_experiment(exp) def test_try_set_after_race_condition_with_hit(self, exp_config, new_config): """Cannot set a configuration after init if config is built @@ -453,28 +474,28 @@ def test_try_set_after_race_condition_with_hit(self, exp_config, new_config): # Another experiment gets configured first naughty_little_exp = Experiment(new_config['name']) assert naughty_little_exp.id is None - experiment_count_before = naughty_little_exp._db.count("experiments") + experiment_count_before = count_experiment(naughty_little_exp) naughty_little_exp.configure(copy.deepcopy(new_config)) assert naughty_little_exp._init_done is True exp = Experiment(new_config['name']) assert exp._init_done is False - assert (experiment_count_before + 1) == exp._db.count("experiments") + assert (experiment_count_before + 1) == count_experiment(exp) # Experiment with hit won't be able to be configured with config without db info with pytest.raises(DuplicateKeyError) as exc_info: exp.configure(new_config) assert 'Cannot register an existing experiment with a new config' in str(exc_info.value) - assert (experiment_count_before + 1) == exp._db.count("experiments") + assert (experiment_count_before + 1) == count_experiment(exp) new_config['metadata']['datetime'] = naughty_little_exp.metadata['datetime'] exp = Experiment(new_config['name']) assert exp._init_done is False - assert (experiment_count_before + 1) == exp._db.count("experiments") + assert (experiment_count_before + 1) == count_experiment(exp) # New experiment will be able to be configured exp.configure(new_config) - assert (experiment_count_before + 1) == exp._db.count("experiments") + assert (experiment_count_before + 1) == count_experiment(exp) def test_try_reset_after_race_condition(self, exp_config, new_config): """Cannot set a configuration after init if it looses a race condition, @@ -486,26 +507,26 @@ def test_try_reset_after_race_condition(self, exp_config, new_config): """ exp = Experiment(new_config['name']) # Another experiment gets configured first - experiment_count_before = exp._db.count("experiments") + experiment_count_before = count_experiment(exp) naughty_little_exp = Experiment(new_config['name']) naughty_little_exp.configure(new_config) assert naughty_little_exp._init_done is True assert exp._init_done is False - assert (experiment_count_before + 1) == exp._db.count("experiments") + assert (experiment_count_before + 1) == count_experiment(exp) # First experiment won't be able to be configured with pytest.raises(DuplicateKeyError) as exc_info: exp.configure(new_config) assert 'duplicate key error' in str(exc_info.value) # Still not more experiment in DB - assert (experiment_count_before + 1) == exp._db.count("experiments") + assert (experiment_count_before + 1) == count_experiment(exp) # Retry configuring the experiment new_config['metadata']['datetime'] = naughty_little_exp.metadata['datetime'] exp = Experiment(new_config['name']) exp.configure(new_config) assert exp._init_done is True - assert (experiment_count_before + 1) == exp._db.count("experiments") + assert (experiment_count_before + 1) == count_experiment(exp) assert exp.configuration == naughty_little_exp.configuration def test_after_init_algorithms_are_objects(self, exp_config): @@ -715,17 +736,35 @@ def fetch_lost_trials(self, query): @pytest.mark.usefixtures("patch_sample") -def test_push_completed_trial(hacked_exp, database, random_dt): +def test_update_completed_trial(hacked_exp, database, random_dt): """Successfully push a completed trial into database.""" trial = hacked_exp.reserve_trial() - trial.results = [Trial.Result(name='yolo', type='objective', value=3)] - hacked_exp.push_completed_trial(trial) + + results_file = tempfile.NamedTemporaryFile( + mode='w', prefix='results_', suffix='.log', dir='.', delete=True + ) + + # Generate fake result + with open(results_file.name, 'w') as file: + json.dump([{ + 'name': 'loss', + 'type': 'objective', + 'value': 2}], + file + ) + # -- + + hacked_exp.update_completed_trial(trial, results_file=results_file) + yo = database.trials.find_one({'_id': trial.id}) + assert len(yo['results']) == len(trial.results) assert yo['results'][0] == trial.results[0].to_dict() assert yo['status'] == 'completed' assert yo['end_time'] == random_dt + results_file.close() + @pytest.mark.usefixtures("with_user_tsirif") def test_register_trials(database, random_dt, hacked_exp): @@ -778,7 +817,7 @@ def test_fetch_non_completed_trials(hacked_exp, exp_config): """ # Set two of completed trials to broken and reserved to have all possible status query = {'status': 'completed', 'experiment': hacked_exp.id} - database = hacked_exp._db._db + database = get_db_from_view(hacked_exp) completed_trials = database.trials.find(query) exp_config[1][0]['status'] = 'broken' database.trials.update({'_id': completed_trials[0]['_id']}, {'$set': {'status': 'broken'}}) @@ -877,7 +916,7 @@ def test_existing_experiment_view(self, create_db_instance, exp_config): """Hit exp_name + user's name in the db, fetch most recent entry.""" exp = ExperimentView('supernaedo2') assert exp._experiment._init_done is True - assert exp._experiment._db._database is create_db_instance + assert exp._id == exp_config[0][0]['_id'] assert exp.name == exp_config[0][0]['name'] assert exp.configuration['refers'] == exp_config[0][0]['refers'] @@ -890,10 +929,10 @@ def test_existing_experiment_view(self, create_db_instance, exp_config): with pytest.raises(AttributeError): exp.this_is_not_in_config = 5 - # Test that experiment.push_completed_trial indeed exists - exp._experiment.push_completed_trial + # Test that experiment.update_completed_trial indeed exists + exp._experiment.update_completed_trial with pytest.raises(AttributeError): - exp.push_completed_trial + exp.update_completed_trial with pytest.raises(AttributeError): exp.register_trial @@ -997,14 +1036,14 @@ def test_experiment_view_stats(hacked_exp, exp_config, random_dt): @pytest.mark.usefixtures("with_user_tsirif") -def test_experiment_view_db_read_only(): +def test_experiment_view_protocol_read_only(): """Verify that wrapper experiments' database is read-only""" exp = ExperimentView('supernaedo2') - # Test that database.write indeed exists - exp._experiment._db._database.write + # Test that _protocol.update_trials indeed exists + exp._experiment._storage._storage.update_trial with pytest.raises(AttributeError): - exp._experiment._db.write + exp._experiment._storage.update_trial class TestInitExperimentWithEVC(object): @@ -1019,7 +1058,7 @@ def test_new_experiment_with_parent(self, create_db_instance, random_dt, exp_con exp.algorithms = exp_config[0][4]['algorithms'] exp.configure(exp.configuration) assert exp._init_done is True - assert exp._db is create_db_instance + assert_protocol(exp, create_db_instance) assert exp._id is not None assert exp.name == 'supernaedo2.6' assert exp.configuration['refers'] == exp_config[0][4]['refers'] @@ -1037,7 +1076,7 @@ def test_experiment_with_parent(self, create_db_instance, random_dt, exp_config) exp.algorithms = {'random': {'seed': None}} exp.configure(exp.configuration) assert exp._init_done is True - assert exp._db is create_db_instance + assert_protocol(exp, create_db_instance) assert exp._id is not None assert exp.name == 'supernaedo2.1' assert exp.configuration['refers'] == exp_config[0][4]['refers'] From 78388337137f3c4ff6e15ac97f19e60221cc171a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Tue, 16 Jul 2019 21:09:55 -0400 Subject: [PATCH 31/50] Add new `status` command (#222) --- conda/meta.yaml | 1 + setup.py | 3 +- src/orion/core/cli/status.py | 183 +++++ tests/functional/commands/conftest.py | 185 ++++- .../functional/commands/test_list_command.py | 84 +- .../commands/test_status_command.py | 742 ++++++++++++++++++ 6 files changed, 1125 insertions(+), 73 deletions(-) create mode 100644 src/orion/core/cli/status.py create mode 100644 tests/functional/commands/test_status_command.py diff --git a/conda/meta.yaml b/conda/meta.yaml index 6a7639318..c574f0393 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -23,6 +23,7 @@ requirements: - pymongo >=3 - gitpython - filelock + - tabulate test: import: diff --git a/setup.py b/setup.py index 73166c7ec..cfa34bb2a 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,8 @@ def find_data_files(): 'legacy = orion.storage.legacy:Legacy' ] }, - install_requires=['PyYAML', 'pymongo>=3', 'numpy', 'scipy', 'gitpython', 'filelock'], + install_requires=['PyYAML', 'pymongo>=3', 'numpy', 'scipy', 'gitpython', 'filelock', + 'tabulate'], tests_require=tests_require, setup_requires=['setuptools', 'pytest-runner'], extras_require=dict(test=tests_require), diff --git a/src/orion/core/cli/status.py b/src/orion/core/cli/status.py new file mode 100644 index 000000000..ef586e9cb --- /dev/null +++ b/src/orion/core/cli/status.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +:mod:`orion.core.cli.status` -- Module to status experiments +============================================================ + +.. module:: status + :platform: Unix + :synopsis: List the trials and their statuses for experiments. + +""" +import collections +import logging + +import tabulate + +from orion.core.cli import base as cli +from orion.core.io.database import Database +from orion.core.io.evc_builder import EVCBuilder +from orion.core.io.experiment_builder import ExperimentBuilder + +log = logging.getLogger(__name__) + + +def add_subparser(parser): + """Add the subparser that needs to be used for this command""" + status_parser = parser.add_parser('status', help='status help') + + cli.get_basic_args_group(status_parser) + + status_parser.add_argument( + '-a', '--all', action="store_true", + help="Show all trials line by line. Otherwise, they are all aggregated by status") + + status_parser.add_argument( + '-r', '--recursive', action="store_true", + help="Divide trials per experiments hierarchically. Otherwise they are all print on the \ + same tab level.") + + status_parser.set_defaults(func=main) + + return status_parser + + +def main(args): + """Fetch config and status experiments""" + builder = ExperimentBuilder() + local_config = builder.fetch_full_config(args, use_db=False) + builder.setup_database(local_config) + + experiments = get_experiments(args) + + if args.get('recursive'): + for exp in filter(lambda e: e.refers['parent_id'] is None, experiments): + print_status_recursively(exp, all_trials=args.get('all')) + else: + for exp in experiments: + print_status(exp, all_trials=args.get('all')) + + +def get_experiments(args): + """Return the different experiments. + + Parameters + ---------- + args: dict + Commandline arguments. + + """ + projection = {'name': 1} + + query = {'name': args['name']} if args.get('name') else {} + experiments = Database().read("experiments", query, projection) + + return [EVCBuilder().build_from({'name': exp['name']}) for exp in experiments] + + +def print_status_recursively(exp, depth=0, **kwargs): + """Print the status recursively of the children of the current experiment. + + Parameters + ---------- + exp: `orion.core.worker.Experiment` + The current experiment to print. + depth: int + The current depth of the tree. + + """ + print_status(exp, offset=depth * 2, **kwargs) + + for child in exp.node.children: + print_status_recursively(child.item, depth + 1, **kwargs) + + +def print_status(exp, offset=0, all_trials=False): + """Print the status of the current experiment. + + Parameters + ---------- + offset: int, optional + The number of tabs to the right this experiment is. + all_trials: bool, optional + Print all trials individually + + """ + if all_trials: + print_all_trials(exp, offset=offset) + else: + print_summary(exp, offset=offset) + + +def print_summary(exp, offset=0): + """Print a summary of the current experiment. + + Parameters + ---------- + offset: int, optional + The number of tabs to the right this experiment is. + + """ + status_dict = collections.defaultdict(list) + name = exp.name + trials = exp.fetch_trials({}) + + for trial in trials: + status_dict[trial.status].append(trial) + + print(" " * offset, name, sep="") + print(" " * offset, "=" * len(name), sep="") + + headers = ['status', 'quantity'] + + lines = [] + for status, trials in sorted(status_dict.items()): + line = [status, len(trials)] + + if trials[0].objective: + headers.append('min {}'.format(trials[0].objective.name)) + line.append(min(trial.objective.value for trial in trials)) + + lines.append(line) + + if trials: + grid = tabulate.tabulate(lines, headers=headers) + tab = " " * offset + print(tab + ("\n" + tab).join(grid.split("\n"))) + else: + print(" " * offset, 'empty', sep="") + + print("\n") + + +def print_all_trials(exp, offset=0): + """Print all trials of the current experiment individually. + + Parameters + ---------- + offset: int, optional + The number of tabs to the right this experiment is. + + """ + name = exp.name + trials = exp.fetch_trials({}) + + print(" " * offset, name, sep="") + print(" " * offset, "=" * len(name), sep="") + headers = ['id', 'status', 'best objective'] + lines = [] + + for trial in sorted(trials, key=lambda t: t.status): + line = [trial.id, trial.status] + + if trial.objective: + headers[-1] = 'min {}'.format(trial.objective.name) + line.append(trial.objective.value) + + lines.append(line) + + grid = tabulate.tabulate(lines, headers=headers) + tab = " " * offset + print(tab + ("\n" + tab).join(grid.split("\n"))) + + print("\n") diff --git a/tests/functional/commands/conftest.py b/tests/functional/commands/conftest.py index 4f1c8e977..2eb60b3ea 100644 --- a/tests/functional/commands/conftest.py +++ b/tests/functional/commands/conftest.py @@ -8,6 +8,10 @@ import yaml from orion.algo.base import (BaseAlgorithm, OptimizationAlgorithm) +import orion.core.cli +from orion.core.io.database import Database +from orion.core.io.experiment_builder import ExperimentBuilder +from orion.core.worker.trial import Trial class DumbAlgo(BaseAlgorithm): @@ -95,19 +99,188 @@ def database(): @pytest.fixture() -def clean_db(database, exp_config): +def clean_db(database, db_instance): """Clean insert example experiment entries to collections.""" database.experiments.drop() + database.lying_trials.drop() database.trials.drop() database.workers.drop() database.resources.drop() @pytest.fixture() -def only_experiments_db(database, exp_config): +def db_instance(null_db_instances): + """Create and save a singleton database instance.""" + try: + db = Database(of_type='MongoDB', name='orion_test', + username='user', password='pass') + except ValueError: + db = Database() + + return db + + +@pytest.fixture +def only_experiments_db(clean_db, database, exp_config): """Clean the database and insert only experiments.""" - database.experiments.drop() database.experiments.insert_many(exp_config[0]) - database.trials.drop() - database.workers.drop() - database.resources.drop() + + +def ensure_deterministic_id(name, db_instance): + """Change the id of experiment to its name.""" + experiment = db_instance.read('experiments', {'name': name})[0] + db_instance.remove('experiments', {'_id': experiment['_id']}) + experiment['_id'] = name + + if experiment['refers']['parent_id'] is None: + experiment['refers']['root_id'] = name + + db_instance.write('experiments', experiment) + + +# Experiments combinations fixtures +@pytest.fixture +def one_experiment(monkeypatch, db_instance): + """Create an experiment without trials.""" + monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) + orion.core.cli.main(['init_only', '-n', 'test_single_exp', + './black_box.py', '--x~uniform(0,1)']) + ensure_deterministic_id('test_single_exp', db_instance) + + +@pytest.fixture +def single_without_success(one_experiment): + """Create an experiment without a succesful trial.""" + statuses = list(Trial.allowed_stati) + statuses.remove('completed') + + exp = ExperimentBuilder().build_from({'name': 'test_single_exp'}) + x = {'name': '/x', 'type': 'real'} + + x_value = 1 + for status in statuses: + x['value'] = x_value + trial = Trial(experiment=exp.id, params=[x], status=status) + x_value += 1 + Database().write('trials', trial.to_dict()) + + +@pytest.fixture +def single_with_trials(single_without_success): + """Create an experiment with all types of trials.""" + exp = ExperimentBuilder().build_from({'name': 'test_single_exp'}) + + x = {'name': '/x', 'type': 'real', 'value': 0} + results = {"name": "obj", "type": "objective", "value": 0} + trial = Trial(experiment=exp.id, params=[x], status='completed', results=[results]) + Database().write('trials', trial.to_dict()) + + +@pytest.fixture +def two_experiments(monkeypatch, db_instance): + """Create an experiment and its child.""" + monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) + orion.core.cli.main(['init_only', '-n', 'test_double_exp', + './black_box.py', '--x~uniform(0,1)']) + ensure_deterministic_id('test_double_exp', db_instance) + + orion.core.cli.main(['init_only', '-n', 'test_double_exp', + '--branch', 'test_double_exp_child', './black_box.py', + '--x~uniform(0,1)', '--y~+uniform(0,1)']) + ensure_deterministic_id('test_double_exp_child', db_instance) + + +@pytest.fixture +def family_with_trials(two_experiments): + """Create two related experiments with all types of trials.""" + exp = ExperimentBuilder().build_from({'name': 'test_double_exp'}) + exp2 = ExperimentBuilder().build_from({'name': 'test_double_exp_child'}) + x = {'name': '/x', 'type': 'real'} + y = {'name': '/y', 'type': 'real'} + + x_value = 0 + for status in Trial.allowed_stati: + x['value'] = x_value + y['value'] = x_value + trial = Trial(experiment=exp.id, params=[x], status=status) + x['value'] = x_value * 10 + trial2 = Trial(experiment=exp2.id, params=[x, y], status=status) + x_value += 1 + Database().write('trials', trial.to_dict()) + Database().write('trials', trial2.to_dict()) + + +@pytest.fixture +def unrelated_with_trials(family_with_trials, single_with_trials): + """Create two unrelated experiments with all types of trials.""" + exp = ExperimentBuilder().build_from({'name': 'test_double_exp_child'}) + + Database().remove('trials', {'experiment': exp.id}) + Database().remove('experiments', {'_id': exp.id}) + + +@pytest.fixture +def three_experiments(two_experiments, one_experiment): + """Create a single experiment and an experiment and its child.""" + pass + + +@pytest.fixture +def three_experiments_with_trials(family_with_trials, single_with_trials): + """Create three experiments, two unrelated, with all types of trials.""" + pass + + +@pytest.fixture +def three_experiments_family(two_experiments, db_instance): + """Create three experiments, one of which is the parent of the other two.""" + orion.core.cli.main(['init_only', '-n', 'test_double_exp', + '--branch', 'test_double_exp_child2', './black_box.py', + '--x~uniform(0,1)', '--z~+uniform(0,1)']) + ensure_deterministic_id('test_double_exp_child2', db_instance) + + +@pytest.fixture +def three_family_with_trials(three_experiments_family, family_with_trials): + """Create three experiments, all related, two direct children, with all types of trials.""" + exp = ExperimentBuilder().build_from({'name': 'test_double_exp_child2'}) + x = {'name': '/x', 'type': 'real'} + z = {'name': '/z', 'type': 'real'} + + x_value = 0 + for status in Trial.allowed_stati: + x['value'] = x_value + z['value'] = x_value * 100 + trial = Trial(experiment=exp.id, params=[x, z], status=status) + x_value += 1 + Database().write('trials', trial.to_dict()) + + +@pytest.fixture +def three_experiments_family_branch(two_experiments, db_instance): + """Create three experiments, each parent of the following one.""" + orion.core.cli.main(['init_only', '-n', 'test_double_exp_child', + '--branch', 'test_double_exp_grand_child', './black_box.py', + '--x~uniform(0,1)', '--y~uniform(0,1)', '--z~+uniform(0,1)']) + ensure_deterministic_id('test_double_exp_grand_child', db_instance) + + +@pytest.fixture +def three_family_branch_with_trials(three_experiments_family_branch, family_with_trials): + """Create three experiments, all related, one child and one grandchild, + with all types of trials. + + """ + exp = ExperimentBuilder().build_from({'name': 'test_double_exp_grand_child'}) + x = {'name': '/x', 'type': 'real'} + y = {'name': '/y', 'type': 'real'} + z = {'name': '/z', 'type': 'real'} + + x_value = 0 + for status in Trial.allowed_stati: + x['value'] = x_value + y['value'] = x_value * 10 + z['value'] = x_value * 100 + trial = Trial(experiment=exp.id, params=[x, y, z], status=status) + x_value += 1 + Database().write('trials', trial.to_dict()) diff --git a/tests/functional/commands/test_list_command.py b/tests/functional/commands/test_list_command.py index 20b7d3eb2..1474430c8 100644 --- a/tests/functional/commands/test_list_command.py +++ b/tests/functional/commands/test_list_command.py @@ -3,58 +3,10 @@ """Perform a functional test of the list command.""" import os -import pytest - import orion.core.cli -from orion.core.io.database import Database - - -@pytest.fixture -def no_experiment(database): - """Create and save a singleton for an empty database instance.""" - database.experiments.drop() - database.lying_trials.drop() - database.trials.drop() - database.workers.drop() - database.resources.drop() - - try: - db = Database(of_type='MongoDB', name='orion_test', - username='user', password='pass') - except ValueError: - db = Database() - - return db - - -@pytest.fixture -def one_experiment(monkeypatch, create_db_instance): - """Create a single experiment.""" - monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) - orion.core.cli.main(['init_only', '-n', 'test_list_single', - './black_box.py', '--x~uniform(0,1)']) - - -@pytest.fixture -def two_experiments(monkeypatch, no_experiment): - """Create an experiment and its child.""" - monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) - orion.core.cli.main(['init_only', '-n', 'test_list_double', - './black_box.py', '--x~uniform(0,1)']) - orion.core.cli.main(['init_only', '-n', 'test_list_double', - '--branch', 'test_list_double_child', './black_box.py', - '--x~uniform(0,1)', '--y~+uniform(0,1)']) - - -@pytest.fixture -def three_experiments(monkeypatch, two_experiments): - """Create a single experiment and an experiment and its child.""" - monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) - orion.core.cli.main(['init_only', '-n', 'test_list_single', - './black_box.py', '--x~uniform(0,1)']) -def test_no_exp(no_experiment, monkeypatch, capsys): +def test_no_exp(monkeypatch, clean_db, capsys): """Test that nothing is printed when there are no experiments.""" monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) orion.core.cli.main(['list']) @@ -64,35 +16,35 @@ def test_no_exp(no_experiment, monkeypatch, capsys): assert captured == "" -def test_single_exp(capsys, one_experiment): +def test_single_exp(clean_db, one_experiment, capsys): """Test that the name of the experiment is printed when there is one experiment.""" orion.core.cli.main(['list']) captured = capsys.readouterr().out - assert captured == " test_list_single\n" + assert captured == " test_single_exp\n" -def test_two_exp(capsys, two_experiments): +def test_two_exp(capsys, clean_db, two_experiments): """Test that experiment and child are printed.""" orion.core.cli.main(['list']) captured = capsys.readouterr().out - assert captured == " test_list_double┐\n └test_list_double_child\n" + assert captured == " test_double_exp┐\n └test_double_exp_child\n" -def test_three_exp(capsys, three_experiments): +def test_three_exp(capsys, clean_db, three_experiments): """Test that experiment, child and grand-child are printed.""" orion.core.cli.main(['list']) captured = capsys.readouterr().out - assert captured == " test_list_double┐\n └test_list_double_child\n \ -test_list_single\n" + assert captured == " test_double_exp┐\n └test_double_exp_child\n \ +test_single_exp\n" -def test_no_exp_name(three_experiments, monkeypatch, capsys): +def test_no_exp_name(clean_db, three_experiments, monkeypatch, capsys): """Test that nothing is printed when there are no experiments with a given name.""" monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) orion.core.cli.main(['list', '--name', 'I don\'t exist']) @@ -102,31 +54,31 @@ def test_no_exp_name(three_experiments, monkeypatch, capsys): assert captured == "" -def test_exp_name(three_experiments, monkeypatch, capsys): +def test_exp_name(clean_db, three_experiments, monkeypatch, capsys): """Test that only the specified experiment is printed.""" monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) - orion.core.cli.main(['list', '--name', 'test_list_single']) + orion.core.cli.main(['list', '--name', 'test_single_exp']) captured = capsys.readouterr().out - assert captured == " test_list_single\n" + assert captured == " test_single_exp\n" -def test_exp_name_with_child(three_experiments, monkeypatch, capsys): +def test_exp_name_with_child(clean_db, three_experiments, monkeypatch, capsys): """Test that only the specified experiment is printed, and with its child.""" monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) - orion.core.cli.main(['list', '--name', 'test_list_double']) + orion.core.cli.main(['list', '--name', 'test_double_exp']) captured = capsys.readouterr().out - assert captured == " test_list_double┐\n └test_list_double_child\n" + assert captured == " test_double_exp┐\n └test_double_exp_child\n" -def test_exp_name_child(three_experiments, monkeypatch, capsys): +def test_exp_name_child(clean_db, three_experiments, monkeypatch, capsys): """Test that only the specified child experiment is printed.""" monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) - orion.core.cli.main(['list', '--name', 'test_list_double_child']) + orion.core.cli.main(['list', '--name', 'test_double_exp_child']) captured = capsys.readouterr().out - assert captured == " test_list_double_child\n" + assert captured == " test_double_exp_child\n" diff --git a/tests/functional/commands/test_status_command.py b/tests/functional/commands/test_status_command.py new file mode 100644 index 000000000..4d09aa22c --- /dev/null +++ b/tests/functional/commands/test_status_command.py @@ -0,0 +1,742 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Perform a functional test of the status command.""" +import os + +import orion.core.cli + + +def test_no_experiments(clean_db, monkeypatch, capsys): + """Test status with no experiments.""" + monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) + orion.core.cli.main(['status']) + + captured = capsys.readouterr().out + + assert captured == "" + + +def test_experiment_without_trials_wout_ar(clean_db, one_experiment, capsys): + """Test status with only one experiment and no trials.""" + orion.core.cli.main(['status']) + + captured = capsys.readouterr().out + + expected = """\ +test_single_exp +=============== +empty + + +""" + assert captured == expected + + +def test_experiment_wout_success_wout_ar(clean_db, single_without_success, capsys): + """Test status with only one experiment and no successful trial.""" + orion.core.cli.main(['status']) + + captured = capsys.readouterr().out + + expected = """\ +test_single_exp +=============== +status quantity +----------- ---------- +broken 1 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + +""" + assert captured == expected + + +def test_two_w_trials_wout_ar(clean_db, unrelated_with_trials, capsys): + """Test two unrelated experiments, with all types of trials.""" + orion.core.cli.main(['status']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +status quantity +----------- ---------- +broken 1 +completed 1 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + +test_single_exp +=============== +status quantity min obj +----------- ---------- --------- +broken 1 +completed 1 0 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + +""" + + assert captured == expected + + +def test_two_fam_w_trials_wout_ar(clean_db, family_with_trials, capsys): + """Test two related experiments, with all types of trials.""" + orion.core.cli.main(['status']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +status quantity +----------- ---------- +broken 1 +completed 1 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + +test_double_exp_child +===================== +status quantity +----------- ---------- +broken 1 +completed 1 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + +""" + + assert captured == expected + + +def test_one_wout_trials_w_a_wout_r(clean_db, one_experiment, capsys): + """Test experiments, without trials, with --all.""" + orion.core.cli.main(['status', '--all']) + + captured = capsys.readouterr().out + + expected = """\ +test_single_exp +=============== +id status best objective +---- -------- ---------------- + + +""" + + assert captured == expected + + +def test_one_w_trials_w_a_wout_r(clean_db, single_with_trials, capsys): + """Test experiment, with all trials, with --all.""" + orion.core.cli.main(['status', '--all']) + + captured = capsys.readouterr().out + + expected = """\ +test_single_exp +=============== +id status min obj +-------------------------------- ----------- --------- +78d300480fc80ec5f52807fe97f65dd7 broken +7e8eade99d5fb1aa59a1985e614732bc completed 0 +ec6ee7892275400a9acbf4f4d5cd530d interrupted +507496236ff94d0f3ad332949dfea484 new +caf6afc856536f6d061676e63d14c948 reserved +2b5059fa8fdcdc01f769c31e63d93f24 suspended + + +""" + + assert captured == expected + + +def test_one_wout_success_w_a_wout_r(clean_db, single_without_success, capsys): + """Test experiment, without success, with --all.""" + orion.core.cli.main(['status', '--all']) + + captured = capsys.readouterr().out + + expected = """\ +test_single_exp +=============== +id status +-------------------------------- ----------- +78d300480fc80ec5f52807fe97f65dd7 broken +ec6ee7892275400a9acbf4f4d5cd530d interrupted +507496236ff94d0f3ad332949dfea484 new +caf6afc856536f6d061676e63d14c948 reserved +2b5059fa8fdcdc01f769c31e63d93f24 suspended + + +""" + + assert captured == expected + + +def test_two_unrelated_w_a_wout_r(clean_db, unrelated_with_trials, capsys): + """Test two unrelated experiments with --all.""" + orion.core.cli.main(['status', '--all']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +id status +-------------------------------- ----------- +a8f8122af9e5162e1e2328fdd5dd75db broken +ab82b1fa316de5accb4306656caa07d0 completed +c187684f7c7d9832ba953f246900462d interrupted +1497d4f27622520439c4bc132c6046b1 new +bd0999e1a3b00bf8658303b14867b30e reserved +b9f1506db880645a25ad9b5d2cfa0f37 suspended + + +test_single_exp +=============== +id status min obj +-------------------------------- ----------- --------- +78d300480fc80ec5f52807fe97f65dd7 broken +7e8eade99d5fb1aa59a1985e614732bc completed 0 +ec6ee7892275400a9acbf4f4d5cd530d interrupted +507496236ff94d0f3ad332949dfea484 new +caf6afc856536f6d061676e63d14c948 reserved +2b5059fa8fdcdc01f769c31e63d93f24 suspended + + +""" + + assert captured == expected + + +def test_two_related_w_a_wout_r(clean_db, family_with_trials, capsys): + """Test two related experiments with --all.""" + orion.core.cli.main(['status', '--all']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +id status +-------------------------------- ----------- +a8f8122af9e5162e1e2328fdd5dd75db broken +ab82b1fa316de5accb4306656caa07d0 completed +c187684f7c7d9832ba953f246900462d interrupted +1497d4f27622520439c4bc132c6046b1 new +bd0999e1a3b00bf8658303b14867b30e reserved +b9f1506db880645a25ad9b5d2cfa0f37 suspended + + +test_double_exp_child +===================== +id status +-------------------------------- ----------- +b55c5a82050dc30a6b0c9614b1eb05e5 broken +649e09b84128c2f8821b9225ebcc139b completed +bac4e23fae8fe316d6f763ac901569af interrupted +5f4a9c92b8f7c26654b5b37ecd3d5d32 new +c2df0712319b5e91c1b4176e961a07a7 reserved +382400953aa6e8769e11aceae9be09d7 suspended + + +""" + + assert captured == expected + + +def test_two_unrelated_w_r_wout_a(clean_db, unrelated_with_trials, capsys): + """Test two unrelated experiments with --recursive.""" + orion.core.cli.main(['status', '--recursive']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +status quantity +----------- ---------- +broken 1 +completed 1 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + +test_single_exp +=============== +status quantity min obj +----------- ---------- --------- +broken 1 +completed 1 0 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + +""" + + assert captured == expected + + +def test_two_related_w_r_wout_a(clean_db, family_with_trials, capsys): + """Test two related experiments with --recursive.""" + orion.core.cli.main(['status', '--recursive']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +status quantity +----------- ---------- +broken 1 +completed 1 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + + test_double_exp_child + ===================== + status quantity + ----------- ---------- + broken 1 + completed 1 + interrupted 1 + new 1 + reserved 1 + suspended 1 + + +""" + + assert captured == expected + + +def test_three_unrelated_w_r_wout_a(clean_db, three_experiments_with_trials, capsys): + """Test three unrelated experiments with --recursive.""" + orion.core.cli.main(['status', '--recursive']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +status quantity +----------- ---------- +broken 1 +completed 1 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + + test_double_exp_child + ===================== + status quantity + ----------- ---------- + broken 1 + completed 1 + interrupted 1 + new 1 + reserved 1 + suspended 1 + + +test_single_exp +=============== +status quantity min obj +----------- ---------- --------- +broken 1 +completed 1 0 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + +""" + + assert captured == expected + + +def test_three_related_w_r_wout_a(clean_db, three_family_with_trials, capsys): + """Test three related experiments with --recursive.""" + orion.core.cli.main(['status', '--recursive']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +status quantity +----------- ---------- +broken 1 +completed 1 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + + test_double_exp_child + ===================== + status quantity + ----------- ---------- + broken 1 + completed 1 + interrupted 1 + new 1 + reserved 1 + suspended 1 + + + test_double_exp_child2 + ====================== + status quantity + ----------- ---------- + broken 1 + completed 1 + interrupted 1 + new 1 + reserved 1 + suspended 1 + + +""" + + assert captured == expected + + +def test_three_related_branch_w_r_wout_a(clean_db, three_family_branch_with_trials, capsys): + """Test three related experiments with --recursive.""" + orion.core.cli.main(['status', '--recursive']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +status quantity +----------- ---------- +broken 1 +completed 1 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + + test_double_exp_child + ===================== + status quantity + ----------- ---------- + broken 1 + completed 1 + interrupted 1 + new 1 + reserved 1 + suspended 1 + + + test_double_exp_grand_child + =========================== + status quantity + ----------- ---------- + broken 1 + completed 1 + interrupted 1 + new 1 + reserved 1 + suspended 1 + + +""" + + assert captured == expected + + +def test_two_unrelated_w_ar(clean_db, unrelated_with_trials, capsys): + """Test two unrelated experiments with --recursive and --all.""" + orion.core.cli.main(['status', '--recursive', '--all']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +id status +-------------------------------- ----------- +a8f8122af9e5162e1e2328fdd5dd75db broken +ab82b1fa316de5accb4306656caa07d0 completed +c187684f7c7d9832ba953f246900462d interrupted +1497d4f27622520439c4bc132c6046b1 new +bd0999e1a3b00bf8658303b14867b30e reserved +b9f1506db880645a25ad9b5d2cfa0f37 suspended + + +test_single_exp +=============== +id status min obj +-------------------------------- ----------- --------- +78d300480fc80ec5f52807fe97f65dd7 broken +7e8eade99d5fb1aa59a1985e614732bc completed 0 +ec6ee7892275400a9acbf4f4d5cd530d interrupted +507496236ff94d0f3ad332949dfea484 new +caf6afc856536f6d061676e63d14c948 reserved +2b5059fa8fdcdc01f769c31e63d93f24 suspended + + +""" + + assert captured == expected + + +def test_two_related_w_ar(clean_db, family_with_trials, capsys): + """Test two related experiments with --recursive and --all.""" + orion.core.cli.main(['status', '--recursive', '--all']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +id status +-------------------------------- ----------- +a8f8122af9e5162e1e2328fdd5dd75db broken +ab82b1fa316de5accb4306656caa07d0 completed +c187684f7c7d9832ba953f246900462d interrupted +1497d4f27622520439c4bc132c6046b1 new +bd0999e1a3b00bf8658303b14867b30e reserved +b9f1506db880645a25ad9b5d2cfa0f37 suspended + + + test_double_exp_child + ===================== + id status + -------------------------------- ----------- + b55c5a82050dc30a6b0c9614b1eb05e5 broken + 649e09b84128c2f8821b9225ebcc139b completed + bac4e23fae8fe316d6f763ac901569af interrupted + 5f4a9c92b8f7c26654b5b37ecd3d5d32 new + c2df0712319b5e91c1b4176e961a07a7 reserved + 382400953aa6e8769e11aceae9be09d7 suspended + + +""" + + assert captured == expected + + +def test_three_unrelated_w_ar(clean_db, three_experiments_with_trials, capsys): + """Test three unrelated experiments with --recursive and --all.""" + orion.core.cli.main(['status', '--recursive', '--all']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +id status +-------------------------------- ----------- +a8f8122af9e5162e1e2328fdd5dd75db broken +ab82b1fa316de5accb4306656caa07d0 completed +c187684f7c7d9832ba953f246900462d interrupted +1497d4f27622520439c4bc132c6046b1 new +bd0999e1a3b00bf8658303b14867b30e reserved +b9f1506db880645a25ad9b5d2cfa0f37 suspended + + + test_double_exp_child + ===================== + id status + -------------------------------- ----------- + b55c5a82050dc30a6b0c9614b1eb05e5 broken + 649e09b84128c2f8821b9225ebcc139b completed + bac4e23fae8fe316d6f763ac901569af interrupted + 5f4a9c92b8f7c26654b5b37ecd3d5d32 new + c2df0712319b5e91c1b4176e961a07a7 reserved + 382400953aa6e8769e11aceae9be09d7 suspended + + +test_single_exp +=============== +id status min obj +-------------------------------- ----------- --------- +78d300480fc80ec5f52807fe97f65dd7 broken +7e8eade99d5fb1aa59a1985e614732bc completed 0 +ec6ee7892275400a9acbf4f4d5cd530d interrupted +507496236ff94d0f3ad332949dfea484 new +caf6afc856536f6d061676e63d14c948 reserved +2b5059fa8fdcdc01f769c31e63d93f24 suspended + + +""" + + assert captured == expected + + +def test_three_related_w_ar(clean_db, three_family_with_trials, capsys): + """Test three related experiments with --recursive and --all.""" + orion.core.cli.main(['status', '--recursive', '--all']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +id status +-------------------------------- ----------- +a8f8122af9e5162e1e2328fdd5dd75db broken +ab82b1fa316de5accb4306656caa07d0 completed +c187684f7c7d9832ba953f246900462d interrupted +1497d4f27622520439c4bc132c6046b1 new +bd0999e1a3b00bf8658303b14867b30e reserved +b9f1506db880645a25ad9b5d2cfa0f37 suspended + + + test_double_exp_child + ===================== + id status + -------------------------------- ----------- + b55c5a82050dc30a6b0c9614b1eb05e5 broken + 649e09b84128c2f8821b9225ebcc139b completed + bac4e23fae8fe316d6f763ac901569af interrupted + 5f4a9c92b8f7c26654b5b37ecd3d5d32 new + c2df0712319b5e91c1b4176e961a07a7 reserved + 382400953aa6e8769e11aceae9be09d7 suspended + + + test_double_exp_child2 + ====================== + id status + -------------------------------- ----------- + d0f4aa931345bfd864201b7dd93ae667 broken + 5005c35be98025a24731d7dfdf4423de completed + c9fa9f0682a370396c8c4265c4e775dd interrupted + 3d8163138be100e37f1656b7b591179e new + 790d3c4c965e0d91ada9cbdaebe220cf reserved + 6efdb99952d5f80f55adbba9c61dc288 suspended + + +""" + + assert captured == expected + + +def test_three_related_branch_w_ar(clean_db, three_family_branch_with_trials, capsys): + """Test three related experiments in a branch with --recursive and --all.""" + orion.core.cli.main(['status', '--recursive', '--all']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +id status +-------------------------------- ----------- +a8f8122af9e5162e1e2328fdd5dd75db broken +ab82b1fa316de5accb4306656caa07d0 completed +c187684f7c7d9832ba953f246900462d interrupted +1497d4f27622520439c4bc132c6046b1 new +bd0999e1a3b00bf8658303b14867b30e reserved +b9f1506db880645a25ad9b5d2cfa0f37 suspended + + + test_double_exp_child + ===================== + id status + -------------------------------- ----------- + b55c5a82050dc30a6b0c9614b1eb05e5 broken + 649e09b84128c2f8821b9225ebcc139b completed + bac4e23fae8fe316d6f763ac901569af interrupted + 5f4a9c92b8f7c26654b5b37ecd3d5d32 new + c2df0712319b5e91c1b4176e961a07a7 reserved + 382400953aa6e8769e11aceae9be09d7 suspended + + + test_double_exp_grand_child + =========================== + id status + -------------------------------- ----------- + 994602c021c470989d6f392b06cb37dd broken + 24c228352de31010d8d3bf253604a82d completed + a3c8a1f4c80c094754c7217a83aae5e2 interrupted + d667f5d719ddaa4e1da2fbe568e11e46 new + a40748e487605df3ed04a5ac7154d4f6 reserved + 229622a6d7132c311b7d4c57a08ecf08 suspended + + +""" + + assert captured == expected + + +def test_experiment_wout_child_w_name(clean_db, unrelated_with_trials, capsys): + """Test status with the name argument and no child.""" + orion.core.cli.main(['status', '--name', 'test_single_exp']) + + captured = capsys.readouterr().out + + expected = """test_single_exp +=============== +status quantity min obj +----------- ---------- --------- +broken 1 +completed 1 0 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + +""" + + assert captured == expected + + +def test_experiment_w_child_w_name(clean_db, three_experiments_with_trials, capsys): + """Test status with the name argument and one child.""" + orion.core.cli.main(['status', '--name', 'test_double_exp']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +status quantity +----------- ---------- +broken 1 +completed 1 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + +""" + + assert captured == expected From a7e5fd44f46fa677c3549f955702fd1aa56226dc Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Tue, 9 Jul 2019 21:17:38 -0400 Subject: [PATCH 32/50] Add info command with unit-tests --- src/orion/core/cli/info.py | 508 ++++++++++++++ tests/unittests/core/cli/test_info.py | 946 ++++++++++++++++++++++++++ 2 files changed, 1454 insertions(+) create mode 100755 src/orion/core/cli/info.py create mode 100755 tests/unittests/core/cli/test_info.py diff --git a/src/orion/core/cli/info.py b/src/orion/core/cli/info.py new file mode 100755 index 000000000..a1c40878f --- /dev/null +++ b/src/orion/core/cli/info.py @@ -0,0 +1,508 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +:mod:`orion.core.cli.info` -- Module to info experiments +======================================================== + +.. module:: info + :platform: Unix + :synopsis: Commandline support to print details of experiments in terminal + +""" +import logging + +from orion.core.io.evc_builder import EVCBuilder + +log = logging.getLogger(__name__) + + +def add_subparser(parser): + """Add the subparser that needs to be used for this command""" + info_parser = parser.add_parser('info', help='info help') + + info_parser.add_argument('name') + + info_parser.set_defaults(func=main) + + return info_parser + + +def main(args): + """Fetch config and info experiments""" + experiment_view = EVCBuilder().build_view_from(args) + print(format_info(experiment_view)) + + +INFO_TEMPLATE = """\ +{commandline} + +{configuration} + +{algorithm} + +{space} + +{metadata} + +{refers} + +{stats} +""" + + +def format_info(experiment, templates=None): + """Render a string for all info of experiment + + Parameters + ---------- + experiment: `orion.core.worker.experiment.Experiment` + templates: dict + Templates for all sections and titles + + """ + info_string = INFO_TEMPLATE.format( + commandline=format_commandline(experiment, templates), + configuration=format_config(experiment, templates), + algorithm=format_algorithm(experiment, templates), + space=format_space(experiment, templates), + metadata=format_metadata(experiment, templates), + refers=format_refers(experiment, templates), + stats=format_stats(experiment, templates)) + + return info_string + + +TITLE_TEMPLATE = """\ +{title} +{empty:=<{title_len}}\ +""" + + +def format_title(title, templates=None): + r"""Render a title above an horizontal bar + + Parameters + ---------- + title: string + templates: dict + Templates for `title`, `leaf` and `dict_node`. + Default is "{title}\n{empty:=<{title_len}}" + + """ + if templates is None: + templates = dict() + + title_template = templates.get('title', TITLE_TEMPLATE) + + title_string = title_template.format( + title=title, + title_len=len(title), + empty='') + + return title_string + + +DICT_EMPTY_LEAF_TEMPLATE = "{tab}{key}\n" +DICT_LEAF_TEMPLATE = "{tab}{key}: {value}\n" +DICT_NODE_TEMPLATE = "{tab}{key}:\n{value}\n" + + +def format_dict(dictionary, depth=0, width=4, templates=None): + r"""Render a dict on multiple lines + + Parameters + ---------- + dictionary: dict + The dictionary to render + depth: int + Tab added at the beginning of every lines + width: int + Size of the tab added to each line, multiplied + by the depth of the object in the dict of dicts. + templates: dict + Templates for `empty_leaf`, `leaf` and `dict_node`. + Default is + `empty_leaf="{tab}{key}"` + `leaf="{tab}{key}: {value}\n"` + `dict_node="{tab}{key}:\n{value}\n"` + + Examples + ------- + >>> print(format_dict({1: {2: 3, 3: 4}, 2: {3: 4, 4: {5: 6}}})) + 1: + 2: 3 + 3: 4 + 2: + 3: 4 + 4: + 5: 6 + >>> templates = {'leaf': '{tab}{key}={value}\n', 'dict_node': '{tab}{key}:\n{value}\n'} + >>> print(format_dict({1: {2: 3, 3: 4}, 2: {3: 4, 4: {5: 6}}}, templates=templates)) + 1: + 2=3 + 3=4 + 2: + 3=4 + 4: + 5=6 + + """ + if isinstance(dictionary, (list, tuple)): + return format_list(dictionary, depth, width=width, templates=templates) + + # To avoid using mutable objects as default values in function signature. + if templates is None: + templates = dict() + + empty_leaf_template = templates.get('empty_leaf', DICT_EMPTY_LEAF_TEMPLATE) + leaf_template = templates.get('leaf', DICT_LEAF_TEMPLATE) + node_template = templates.get('dict_node', DICT_NODE_TEMPLATE) + + dict_string = "" + for key in sorted(dictionary.keys()): + tab = (" " * (depth * width)) + value = dictionary[key] + if isinstance(value, (dict, list, tuple)): + if not value: + dict_string += empty_leaf_template.format(tab=tab, key=key) + else: + subdict_string = format_dict( + value, depth + 1, width=width, templates=templates) + dict_string += node_template.format(tab=tab, key=key, value=subdict_string) + else: + dict_string += leaf_template.format(tab=tab, key=key, value=value) + + return dict_string.replace(' \n', '\n').rstrip("\n") + + +LIST_TEMPLATE = """\ +{tab}[ +{items} +{tab}]\ +""" +LIST_ITEM_TEMPLATE = "{tab}{item}\n" +LIST_NODE_TEMPLATE = "{item}\n" + + +def format_list(a_list, depth=0, width=4, templates=None): + r"""Render a list on multiple lines + + Parameters + ---------- + a_list: list + The list to render + depth: int + Tab added at the beginning of every lines + width: int + Size of the tab added to each line, multiplied + by the depth of the object in the list of lists. + templates: dict + Templates for `list`, `item` and `list_node`. + Default is + `list="{tab}[\n{items}\n{tab}]"` + `item="{tab}{item}\n"` + `list_node="{item}\n"` + + Examples + ------- + >>> print(format_list([1, [2, 3], 4, [5, 6, 7, 8]])) + [ + 1 + [ + 2 + 3 + ] + 4 + [ + 5 + 6 + 7 + 8 + ] + ] + >>> templates = {} + >>> templates['list'] = '{tab}\n{items}\n{tab}' + >>> templates['item'] = '{tab}- {item}\n' + >>> templates['list_node'] = '{tab}{item}\n' + >>> print(format_list([1, [2, 3], 4, [5, 6, 7, 8]], width=2, templates=templates)) + - 1 + + - 2 + - 3 + + - 4 + + - 5 + - 6 + - 7 + - 8 + + """ + # To avoid using mutable objects as default values in function signature. + if templates is None: + templates = dict() + + list_template = templates.get('list', LIST_TEMPLATE) + item_template = templates.get('item', LIST_ITEM_TEMPLATE) + node_template = templates.get('list_node', LIST_NODE_TEMPLATE) + + tab = (" " * (depth * width)) + list_string = "" + for i, item in enumerate(a_list, 1): + subtab = (" " * ((depth + 1) * width)) + if isinstance(item, (dict, list, tuple)): + item_string = format_dict(item, depth + 1, width=width, templates=templates) + list_string += node_template.format(tab=subtab, id=i, item=item_string) + else: + list_string += item_template.format(tab=subtab, id=i, item=item) + + return list_template.format(tab=tab, items=list_string.rstrip("\n")) + + +COMMANDLINE_TEMPLATE = """\ +{title} +{commandline} +""" + + +def format_commandline(experiment, templates=None): + """Render a string for commandline section + + Parameters + ---------- + experiment: `orion.core.worker.experiment.Experiment` + templates: dict + templates for the title and `commandline`. + See `format_title` for more info. + + """ + if templates is None: + templates = dict() + + commandline_template = templates.get('commandline', COMMANDLINE_TEMPLATE) + + commandline_string = commandline_template.format( + title=format_title("Commandline", templates=templates), + commandline=" ".join(experiment.metadata['user_args'])) + + return commandline_string + + +CONFIG_TEMPLATE = """\ +{title} +pool size: {experiment.pool_size} +max trials: {experiment.max_trials} +""" + + +def format_config(experiment, templates=None): + """Render a string for config section + + Parameters + ---------- + experiment: `orion.core.worker.experiment.Experiment` + templates: dict + templates for the title and `config`. + See `format_title` for more info. + + """ + if templates is None: + templates = dict() + + config_template = templates.get('config', CONFIG_TEMPLATE) + + config_string = config_template.format( + title=format_title("Config", templates), + experiment=experiment) + + return config_string + + +ALGORITHM_TEMPLATE = """\ +{title} +{configuration} +""" + + +def format_algorithm(experiment, templates=None): + """Render a string for algorithm section + + Parameters + ---------- + experiment: `orion.core.worker.experiment.Experiment` + templates: dict + templates for the title and `algorithm`. + See `format_title` for more info. + + """ + if templates is None: + templates = dict() + + algorithm_template = templates.get('algorithm', ALGORITHM_TEMPLATE) + + algorithm_string = algorithm_template.format( + title=format_title("Algorithm", templates), + configuration=format_dict(experiment.configuration['algorithms'])) + + return algorithm_string + + +SPACE_TEMPLATE = """\ +{title} +{params} +""" + + +def format_space(experiment, templates=None): + """Render a string for space section + + Parameters + ---------- + experiment: `orion.core.worker.experiment.Experiment` + templates: dict + templates for the title and `space`. + See `format_title` for more info. + + """ + if templates is None: + templates = dict() + + space_template = templates.get('space', SPACE_TEMPLATE) + + space_string = space_template.format( + title=format_title("Space", templates), + params="\n".join(name + ": " + experiment.space[name].get_prior_string() + for name in experiment.space.keys())) + + return space_string + + +METADATA_TEMPLATE = """\ +{title} +user: {experiment.metadata[user]} +datetime: {experiment.metadata[datetime]} +orion version: {experiment.metadata[orion_version]} +""" + + +def format_metadata(experiment, templates=None): + """Render a string for metadata section + + Parameters + ---------- + experiment: `orion.core.worker.experiment.Experiment` + templates: dict + templates for the title and `metadata`. + See `format_title` for more info. + + """ + if templates is None: + templates = dict() + + metadata_template = templates.get('metadata', METADATA_TEMPLATE) + + metadata_string = metadata_template.format( + title=format_title("Meta-data", templates), + experiment=experiment) + + return metadata_string + + +REFERS_TEMPLATE = """\ +{title} +root: {root} +parent: {parent} +adapter: {adapter} +""" + + +def format_refers(experiment, templates=None): + """Render a string for refers section + + Parameters + ---------- + experiment: `orion.core.worker.experiment.Experiment` + templates: dict + templates for the title and `refers`. + See `format_title` for more info. + + """ + if templates is None: + templates = dict() + + refers_template = templates.get('refers', REFERS_TEMPLATE) + + if experiment.node.root is experiment.node: + root = '' + parent = '' + adapter = '' + else: + root = experiment.node.root.name + parent = experiment.node.parent.name + adapter = "\n" + format_dict(experiment.refers['adapter'].configuration, depth=1, width=2) + + refers_string = refers_template.format( + title=format_title("Parent experiment", templates), + root=root, + parent=parent, + adapter=adapter) + + return refers_string + + +STATS_TEMPLATE = """\ +{title} +trials completed: {stats[trials_completed]} +best trial: +{best_params} +best evaluation: {stats[best_evaluation]} +start time: {stats[start_time]} +finish time: {stats[finish_time]} +duration: {stats[duration]} +""" + + +NO_STATS_TEMPLATE = """\ +{title} +No trials executed... +""" + + +def format_stats(experiment, templates=None): + """Render a string for stat section + + Parameters + ---------- + experiment: `orion.core.worker.experiment.Experiment` + templates: dict + templates for the title and `stats`. + See `format_title` for more info. + + """ + if templates is None: + templates = dict() + + stats_template = templates.get('stats', STATS_TEMPLATE) + + stats = experiment.stats + if not stats: + return NO_STATS_TEMPLATE.format( + title=format_title("Stats", templates)) + + best_params = get_trial_params(stats['best_trials_id'], experiment) + + stats_string = stats_template.format( + title=format_title("Stats", templates), + stats=stats, + best_params=format_dict(best_params, depth=1, width=2)) + + return stats_string + + +def get_trial_params(trial_id, experiment): + """Get params from trial_id in given experiment""" + best_trial = experiment.fetch_trials({'_id': trial_id}) + if not best_trial: + return {} + + return dict((param.name, param.value) for param in best_trial[0].params) diff --git a/tests/unittests/core/cli/test_info.py b/tests/unittests/core/cli/test_info.py new file mode 100755 index 000000000..a7761633f --- /dev/null +++ b/tests/unittests/core/cli/test_info.py @@ -0,0 +1,946 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Collection of tests for :mod:`orion.core.cli.info`.""" +import itertools + +import pytest + +from orion.core.cli.info import ( + format_algorithm, format_commandline, format_config, format_dict, format_info, format_list, + format_metadata, format_refers, format_space, format_stats, format_title, get_trial_params) +from orion.core.io.space_builder import SpaceBuilder +from orion.core.worker.trial import Trial + + +class DummyExperiment(): + """Dummy container to mock experiments""" + + pass + + +@pytest.fixture +def dummy_trial(): + """Return a dummy trial object""" + trial = Trial() + trial.params = [ + Trial.Param(name='a', type='real', value=0.0), + Trial.Param(name='b', type='integer', value=1), + Trial.Param(name='c', type='categorical', value='Some')] + return trial + + +@pytest.fixture +def dummy_dict(): + """Return a dict of dicts""" + return { + 1: { + 1.1: "1.1.1", + 1.2: { + "1.2.1": {}, + "1.2.2": "1.2.2.1" + } + }, + 2: { + 2.1: "2.1.1", + 2.2: {} + }, + 3: {} + } + + +@pytest.fixture +def dummy_list_of_lists(): + """Return a list of lists""" + return [ + 1, + [2, 3], + 4, + [5, 6, 7, 8] + ] + + +@pytest.fixture +def dummy_list_of_objects(dummy_dict): + """Return a list of objects""" + return [ + { + 1: { + 1.1: "1.1.1", + 1.2: [ + "1.2.1", + "1.2.2" + ] + }, + }, + [ + 4, 5 + ], + { + 2: { + 2.1: "2.1.1", + 2.2: {} + }, + 3: {} + } + ] + + +@pytest.fixture +def algorithm_dict(): + """Return an algorithm configuration""" + return dict( + bayesianoptimizer=dict( + acq_func='gp_hedge', + alpha=1e-10, + n_initial_points=10, + n_restarts_optimizer=0, + normalize_y=False)) + + +def test_format_title(): + """Test title formatting with custom template""" + result = """Test\n====""" + + assert format_title('Test') == result + + +def test_format_title_custom(): + """Test title formatting with custom template""" + template = """{empty:-<{title_len}}\n{title}\n{empty:-<{title_len}}""" + result = """------\nCustom\n------""" + + assert format_title('Custom', templates=dict(title=template)) == result + + template = """{title}\n---""" + result = """Custom\n---""" + + assert format_title('Custom', templates=dict(title=template)) == result + + +@pytest.mark.parametrize("depth", [0, 1, 2]) +def test_format_dict_depth_synthetic(depth, dummy_dict): + """Test dict formatting with different depths for one line""" + WIDTH = 4 + tab = (" " * WIDTH) * depth + assert format_dict(dummy_dict, depth=depth, width=WIDTH).split("\n")[0].startswith(tab + "1") + + +def test_format_dict_depth_full(dummy_dict): + """Test dict depth formatting for all lines""" + WIDTH = 4 + tab = " " * WIDTH + + lines = format_dict(dummy_dict).split("\n") + assert len(lines) == 9 + assert lines[0].startswith("1") + assert lines[1].startswith(tab + "1") + assert lines[2].startswith(tab + "1") + assert lines[3].startswith((tab * 2) + "1") + assert lines[4].startswith((tab * 2) + "1") + assert lines[5].startswith("2") + assert lines[6].startswith(tab + "2") + assert lines[7].startswith(tab + "2") + assert lines[8].startswith("3") + + +@pytest.mark.parametrize("width,depth", + itertools.product([0, 2], [1, 2])) +def test_format_dict_width_synthetic(width, depth, dummy_dict): + """Test dict formatting with different combination of widths and depth for one line""" + tab = (" " * width) * depth + assert format_dict(dummy_dict, depth=depth, width=width).split("\n")[0].startswith(tab + "1") + + +@pytest.mark.parametrize("width", [0, 5, 12]) +def test_format_dict_width_full(width, dummy_dict): + """Test dict formatting with different widths for all lines""" + tab = " " * width + lines = format_dict(dummy_dict, width=width).split("\n") + assert len(lines) == 9 + assert lines[0].startswith("1") + assert lines[1].startswith(tab + "1") + assert lines[2].startswith(tab + "1") + assert lines[3].startswith((tab * 2) + "1") + assert lines[4].startswith((tab * 2) + "1") + assert lines[5].startswith("2") + assert lines[6].startswith(tab + "2") + assert lines[7].startswith(tab + "2") + assert lines[8].startswith("3") + + +def test_format_dict_empty_leaf_template(): + """Test dict empty leaf node formatting""" + dummy_dict_with_leafs = {1: {2: {}}, 2: {}, 3: {1: 3, 2: {}}} + lines = format_dict(dummy_dict_with_leafs).split("\n") + assert len(lines) == 6 + # 1: + assert lines[1].lstrip(" ") == "2" + assert lines[2].lstrip(" ") == "2" + # 3: + # 1: 3 + assert lines[5].lstrip(" ") == "2" + + +def test_format_dict_empty_leaf_template_custom(): + """Test dict empty leaf node formatting with custom template""" + template = "{key} is a leaf\n" + dummy_dict_with_leafs = {1: {2: {}}, 2: {}, 3: {1: 3, 2: {}}} + lines = format_dict(dummy_dict_with_leafs, templates={'empty_leaf': template}).split("\n") + assert len(lines) == 6 + # 1: + assert lines[1] == "2 is a leaf" + assert lines[2] == "2 is a leaf" + # 3: + # 1: 3 + assert lines[5] == "2 is a leaf" + + +def test_format_dict_leaf_template(): + """Test dict leaf node formatting""" + dummy_dict_with_leafs = {1: {2: {}}, 2: {}, 3: {1: 3, 2: {}}} + lines = format_dict(dummy_dict_with_leafs).split("\n") + assert len(lines) == 6 + # 1: + # 2 + # 2 + # 3: + assert lines[4].lstrip(" ") == "1: 3" + + +def test_format_dict_leaf_template_custom(): + """Test dict leaf node formatting with custom template""" + template = "value of {key} is {value}\n" + dummy_dict_with_leafs = {1: {2: {}}, 2: {}, 3: {1: 3, 2: {}}} + lines = format_dict(dummy_dict_with_leafs, templates={'leaf': template}).split("\n") + assert len(lines) == 6 + # 1: + # 2 + # 2 + # 3: + assert lines[4] == "value of 1 is 3" + + +def test_format_dict_node_template(): + """Test dict node formatting""" + dummy_dict_with_leafs = {1: {2: {}}, 2: {}, 3: {1: {4: 3}, 2: {}}} + lines = format_dict(dummy_dict_with_leafs).split("\n") + assert len(lines) == 7 + assert lines[0] == "1:" + # 2 + # 2 + assert lines[3] == "3:" + assert lines[4].lstrip(" ") == "1:" + # 4: 3 + # 2 + + +def test_format_dict_node_template_custom(): + """Test dict formatting with custom node template""" + template = "{key} is a dict:\n{value}\n" + dummy_dict_with_leafs = {1: {2: {}}, 2: {}, 3: {1: {4: 3}, 2: {}}} + lines = format_dict(dummy_dict_with_leafs, templates={'dict_node': template}).split("\n") + assert len(lines) == 7 + assert lines[0] == "1 is a dict:" + # 2 + # 2 + assert lines[3] == "3 is a dict:" + assert lines[4] == "1 is a dict:" + # 4: 3 + # 2 + + +@pytest.mark.parametrize("depth", [0, 1, 2]) +def test_format_list_depth_synthetic(depth, dummy_list_of_lists): + """Test list of lists formatting with different depths for one line""" + WIDTH = 4 + tab = (" " * WIDTH) * depth + formatted_list_str = format_list(dummy_list_of_lists, depth=depth, width=WIDTH) + assert formatted_list_str.split("\n")[0].startswith(tab + "[") + + +def test_format_list_depth_full(dummy_list_of_lists): + """Test list of lists depth formatting for all lines""" + WIDTH = 4 + tab = " " * WIDTH + + lines = format_list(dummy_list_of_lists).split("\n") + assert len(lines) == 14 + assert lines[0] == "[" + assert lines[1] == tab + "1" + assert lines[2] == tab + "[" + assert lines[3] == (tab * 2) + "2" + assert lines[4] == (tab * 2) + "3" + assert lines[5] == tab + "]" + assert lines[6] == tab + "4" + assert lines[7] == tab + "[" + assert lines[8] == (tab * 2) + "5" + assert lines[9] == (tab * 2) + "6" + assert lines[10] == (tab * 2) + "7" + assert lines[11] == (tab * 2) + "8" + assert lines[12] == tab + "]" + assert lines[13] == "]" + + +@pytest.mark.parametrize("width,depth", + itertools.product([0, 2], [1, 2])) +def test_format_list_width_synthetic(width, depth, dummy_list_of_lists): + """Test list of lists formatting with different combination of widths and depth for one line""" + tab = (" " * width) * depth + assert format_list(dummy_list_of_lists, depth=depth, width=width).split("\n")[0] == tab + "[" + + +@pytest.mark.parametrize("width", [0, 5, 12]) +def test_format_list_width_full(width, dummy_list_of_lists): + """Test list of lists formatting with different widths for all lines""" + tab = " " * width + lines = format_list(dummy_list_of_lists, width=width).split("\n") + assert len(lines) == 14 + assert lines[0] == "[" + assert lines[1] == tab + "1" + assert lines[2] == tab + "[" + assert lines[3] == (tab * 2) + "2" + assert lines[4] == (tab * 2) + "3" + assert lines[5] == tab + "]" + assert lines[6] == tab + "4" + assert lines[7] == tab + "[" + assert lines[8] == (tab * 2) + "5" + assert lines[9] == (tab * 2) + "6" + assert lines[10] == (tab * 2) + "7" + assert lines[11] == (tab * 2) + "8" + assert lines[12] == tab + "]" + assert lines[13] == "]" + + +def test_format_list_item_template(dummy_list_of_lists): + """Test list of lists formatting of items""" + lines = format_list(dummy_list_of_lists).split("\n") + assert len(lines) == 14 + assert lines[1].lstrip(" ") == "1" + assert lines[3].lstrip(" ") == "2" + + +def test_format_list_item_template_custom(dummy_list_of_lists): + """Test list of lists with custom item template""" + template = "{item} is an item\n" + lines = format_list(dummy_list_of_lists, templates={'item': template}).split("\n") + assert len(lines) == 14 + # 1: + assert lines[1] == "1 is an item" + assert lines[3] == "2 is an item" + + +def test_format_list_node_template(dummy_list_of_lists): + """Test list of lists formatting of node""" + WIDTH = 4 + tab = " " * WIDTH + lines = format_list(dummy_list_of_lists).split("\n") + assert len(lines) == 14 + assert lines[2] == tab + "[" + assert lines[3] == (tab * 2) + "2" + assert lines[4] == (tab * 2) + "3" + assert lines[5] == tab + "]" + + +def test_format_list_node_template_custom(dummy_list_of_lists): + """Test list of lists custom formatting""" + templates = dict( + list="[{items}]", + item="{item}", + list_node="{item}") + lines = format_list(dummy_list_of_lists, templates=templates).split("\n") + assert len(lines) == 1 + assert lines[0] == "[1[23]4[5678]]" + + +def test_format_dict_with_list(dummy_list_of_objects): + """Test dict formatting with embedded lists""" + assert format_dict(dummy_list_of_objects) == """\ +[ + 1: + 1.1: 1.1.1 + 1.2: + [ + 1.2.1 + 1.2.2 + ] + [ + 4 + 5 + ] + 2: + 2.1: 2.1.1 + 2.2 + 3 +]\ +""" + + +def test_format_commandline(): + """Test commandline section formatting""" + experiment = DummyExperiment() + commandline = ['executing.sh', '--some', 'random', '--command', 'line', 'arguments'] + experiment.metadata = {'user_args': commandline} + assert format_commandline(experiment) == """\ +Commandline +=========== +executing.sh --some random --command line arguments +""" + + +def test_format_commandline_custom(): + """Test commandline section with custom formatting""" + experiment = DummyExperiment() + commandline = ['executing.sh', '--some', 'random', '--command', 'line', 'arguments'] + experiment.metadata = {'user_args': commandline} + templates = dict( + title=" {title}\n+{empty:-<{title_len}}+", + commandline="{title}\ncommandline: {commandline}") + + assert format_commandline(experiment, templates) == """\ + Commandline ++-----------+ +commandline: executing.sh --some random --command line arguments""" + + +def test_format_config(monkeypatch): + """Test config section formatting""" + experiment = DummyExperiment() + experiment.pool_size = 10 + experiment.max_trials = 100 + assert format_config(experiment) == """\ +Config +====== +pool size: 10 +max trials: 100 +""" + + +def test_format_config_custom(monkeypatch): + """Test config section with custom formatting""" + experiment = DummyExperiment() + experiment.pool_size = 10 + experiment.max_trials = 100 + templates = dict( + title="{title}\n{empty:+<{title_len}}", + config="{title}\n(pool_size={experiment.pool_size}, max_trials={experiment.max_trials})") + + assert format_config(experiment, templates) == """\ +Config +++++++ +(pool_size=10, max_trials=100)""" + + +def test_format_algorithm(algorithm_dict): + """Test algorithm section formatting""" + experiment = DummyExperiment() + experiment.configuration = {'algorithms': algorithm_dict} + assert format_algorithm(experiment) == """\ +Algorithm +========= +bayesianoptimizer: + acq_func: gp_hedge + alpha: 1e-10 + n_initial_points: 10 + n_restarts_optimizer: 0 + normalize_y: False +""" + + +def test_format_algorithm_custom(algorithm_dict): + """Test algorithm section with custom formatting""" + experiment = DummyExperiment() + experiment.configuration = {'algorithms': algorithm_dict} + templates = dict( + title="{title}\n{empty:~<{title_len}}", + algorithm="{title}\n{{\n {configuration}\n}}") + + assert format_algorithm(experiment, templates) == """\ +Algorithm +~~~~~~~~~ +{ + bayesianoptimizer: + acq_func: gp_hedge + alpha: 1e-10 + n_initial_points: 10 + n_restarts_optimizer: 0 + normalize_y: False + strategy: cl_min +}""" + + +def test_format_space(): + """Test space section formatting""" + experiment = DummyExperiment() + commandline = ['executing.sh', '--some~choices(["random", "or", "not"])', + '--command~uniform(0, 1)'] + space = SpaceBuilder().build_from(commandline) + experiment.space = space + assert format_space(experiment) == """\ +Space +===== +/some: choices(['random', 'or', 'not']) +/command: uniform(0, 1) +""" + + +def test_format_space_custom(): + """Test space section with custom formatting""" + experiment = DummyExperiment() + commandline = ['executing.sh', '--some~choices(["random", "or", "not"])', + '--command~uniform(0, 1)'] + space = SpaceBuilder().build_from(commandline) + experiment.space = space + templates = dict( + title="{title}\n{empty:-<{title_len}}", + space="{title}\nparams:\n{params}") + + assert format_space(experiment, templates) == """\ +Space +----- +params: +/some: choices(['random', 'or', 'not']) +/command: uniform(0, 1)""" + + +def test_format_metadata(): + """Test metadata section formatting""" + experiment = DummyExperiment() + experiment.metadata = dict( + user='user', + datetime='now', + orion_version='1.0.1') + assert format_metadata(experiment) == """\ +Meta-data +========= +user: user +datetime: now +orion version: 1.0.1 +""" + + +def test_format_metadata_custom(): + """Test metadata section with custom formatting""" + experiment = DummyExperiment() + experiment.metadata = dict( + user='user', + datetime='now', + orion_version='1.0.1') + templates = dict( + title="{title}\n{empty:*<{title_len}}", + metadata="""\ +{title} +orion={experiment.metadata[orion_version]} +when={experiment.metadata[datetime]} +""") + + assert format_metadata(experiment, templates) == """\ +Meta-data +********* +orion=1.0.1 +when=now +""" + + +def test_format_refers_root(): + """Test refers section formatting for a root experiment""" + experiment = DummyExperiment() + experiment.node = DummyExperiment() + experiment.node.root = experiment.node + + # experiment.refers = dict( + # parent='user', + # datetime='now', + # orion_version='1.0.1') + assert format_refers(experiment) == """\ +Parent experiment +================= +root: +parent: +adapter: +""" # noqa: W291 + + +def test_format_refers_child(): + """Test refers section formatting for a child experiment""" + ROOT_NAME = 'root-name' + PARENT_NAME = 'parent-name' + + root = DummyExperiment() + root.name = ROOT_NAME + + parent = DummyExperiment() + parent.name = PARENT_NAME + + child = DummyExperiment() + child.node = DummyExperiment() + child.node.parent = parent + child.node.root = root + + adapter = DummyExperiment() + adapter.configuration = dict( + adummy='dict', + foran='adapter') + + child.refers = dict(adapter=adapter) + + # experiment.refers = dict( + # parent='user', + # datetime='now', + # orion_version='1.0.1') + assert format_refers(child) == """\ +Parent experiment +================= +root: root-name +parent: parent-name +adapter: + adummy: dict + foran: adapter +""" # noqa: W291 + + +def test_format_refers_custom(): + """Test refers section with custom formatting""" + ROOT_NAME = 'root-name' + PARENT_NAME = 'parent-name' + + root = DummyExperiment() + root.name = ROOT_NAME + + parent = DummyExperiment() + parent.name = PARENT_NAME + + child = DummyExperiment() + child.node = DummyExperiment() + child.node.parent = parent + child.node.root = root + + adapter = DummyExperiment() + adapter.configuration = dict( + adummy='dict', + foran='adapter') + + child.refers = dict(adapter=adapter) + + # experiment.refers = dict( + # parent='user', + # datetime='now', + # orion_version='1.0.1') + templates = dict( + title="| {title} |\n+-{empty:-<{title_len}}-+", + refers="""\ +{title} +parent: {parent} +adapter: +{adapter} +""") + + assert format_refers(child, templates) == """\ +| Parent experiment | ++-------------------+ +parent: parent-name +adapter: + + adummy: dict + foran: adapter +""" + + +def test_get_trial_params_empty(): + """Test failing to fetch trials does not fail""" + experiment = DummyExperiment() + experiment.fetch_trials = lambda query: [] + assert get_trial_params(None, experiment) == {} + + +def test_get_trial_params(dummy_trial): + """Test params are converted properly to a dict.""" + experiment = DummyExperiment() + experiment.fetch_trials = lambda query: [dummy_trial] + params = get_trial_params(None, experiment) + assert params['a'] == 0.0 + assert params['b'] == 1 + assert params['c'] == 'Some' + + +def test_format_stats(dummy_trial): + """Test stats section formatting""" + experiment = DummyExperiment() + experiment.stats = dict( + best_trials_id='dummy', + trials_completed=10, + best_evaluation=0.1, + start_time='yesterday', + finish_time='now', + duration='way too long') + experiment.fetch_trials = lambda query: [dummy_trial] + assert format_stats(experiment) == """\ +Stats +===== +trials completed: 10 +best trial: + a: 0.0 + b: 1 + c: Some +best evaluation: 0.1 +start time: yesterday +finish time: now +duration: way too long +""" + + +def test_format_stats_custom(dummy_trial): + """Test stats section with custom formatting""" + experiment = DummyExperiment() + experiment.stats = dict( + best_trials_id='dummy', + trials_completed=10, + best_evaluation=0.1, + start_time='yesterday', + finish_time='now', + duration='way too long') + experiment.fetch_trials = lambda query: [dummy_trial] + templates = dict( + title="* {title} *\n**{empty:*<{title_len}}**", + stats="""\ +{title} +best trial: +{best_params} +corresponding evaluation: {stats[best_evaluation]} +""") + + assert format_stats(experiment, templates) == """\ +* Stats * +********* +best trial: + a: 0.0 + b: 1 + c: Some +corresponding evaluation: 0.1 +""" + + +def test_format_info(algorithm_dict, dummy_trial): + """Test full formatting string""" + experiment = DummyExperiment() + commandline = ['executing.sh', '--some~choices(["random", "or", "not"])', + '--command~uniform(0, 1)'] + experiment.metadata = {'user_args': commandline} + experiment.pool_size = 10 + experiment.max_trials = 100 + experiment.configuration = {'algorithms': algorithm_dict} + + space = SpaceBuilder().build_from(commandline) + experiment.space = space + experiment.metadata.update(dict( + user='user', + datetime='now', + orion_version='1.0.1')) + + ROOT_NAME = 'root-name' + PARENT_NAME = 'parent-name' + + root = DummyExperiment() + root.name = ROOT_NAME + + parent = DummyExperiment() + parent.name = PARENT_NAME + + experiment.node = DummyExperiment() + experiment.node.parent = parent + experiment.node.root = root + + adapter = DummyExperiment() + adapter.configuration = dict( + adummy='dict', + foran='adapter') + + experiment.refers = dict(adapter=adapter) + experiment.stats = dict( + best_trials_id='dummy', + trials_completed=10, + best_evaluation=0.1, + start_time='yesterday', + finish_time='now', + duration='way too long') + experiment.fetch_trials = lambda query: [dummy_trial] + + assert format_info(experiment) == """\ +Commandline +=========== +executing.sh --some~choices(["random", "or", "not"]) --command~uniform(0, 1) + + +Config +====== +pool size: 10 +max trials: 100 + + +Algorithm +========= +bayesianoptimizer: + acq_func: gp_hedge + alpha: 1e-10 + n_initial_points: 10 + n_restarts_optimizer: 0 + normalize_y: False + + +Space +===== +/some: choices(['random', 'or', 'not']) +/command: uniform(0, 1) + + +Meta-data +========= +user: user +datetime: now +orion version: 1.0.1 + + +Parent experiment +================= +root: root-name +parent: parent-name +adapter: + adummy: dict + foran: adapter + + +Stats +===== +trials completed: 10 +best trial: + a: 0.0 + b: 1 + c: Some +best evaluation: 0.1 +start time: yesterday +finish time: now +duration: way too long + +""" # noqa: W291 + + +def test_format_info_custom(algorithm_dict, dummy_trial): + """Test full info string with custom formatting""" + experiment = DummyExperiment() + commandline = ['executing.sh', '--some~choices(["random", "or", "not"])', + '--command~uniform(0, 1)'] + experiment.metadata = {'user_args': commandline} + experiment.pool_size = 10 + experiment.max_trials = 100 + experiment.configuration = {'algorithms': algorithm_dict} + + space = SpaceBuilder().build_from(commandline) + experiment.space = space + print(space) + experiment.metadata.update(dict( + user='user', + datetime='now', + orion_version='1.0.1')) + + ROOT_NAME = 'root-name' + PARENT_NAME = 'parent-name' + + root = DummyExperiment() + root.name = ROOT_NAME + + parent = DummyExperiment() + parent.name = PARENT_NAME + + experiment.node = DummyExperiment() + experiment.node.parent = parent + experiment.node.root = root + + adapter = DummyExperiment() + adapter.configuration = dict( + adummy='dict', + foran='adapter') + + experiment.refers = dict(adapter=adapter) + experiment.stats = dict( + best_trials_id='dummy', + trials_completed=10, + best_evaluation=0.1, + start_time='yesterday', + finish_time='now', + duration='way too long') + experiment.fetch_trials = lambda query: [dummy_trial] + + templates = dict( + title=" {title} \n+{empty:-<{title_len}}+", + commandline="{title}\ncommandline: {commandline}", + config="{title}\n(pool_size={experiment.pool_size}, max_trials={experiment.max_trials})", + algorithm="{title}\n{{\n {configuration}\n}}", + space="{title}\nparams:\n{params}", + metadata="""\ +{title} +orion={experiment.metadata[orion_version]} +when={experiment.metadata[datetime]} +""", + refers="""\ +{title} +parent: {parent} +adapter: +{adapter} +""", + stats="""\ +{title} +best trial: +{best_params} +corresponding evaluation: {stats[best_evaluation]} +""") + + assert format_info(experiment, templates) == """\ + Commandline ++-----------+ +commandline: executing.sh --some~choices(["random", "or", "not"]) --command~uniform(0, 1) + + Config ++------+ +(pool_size=10, max_trials=100) + + Algorithm ++---------+ +{ + bayesianoptimizer: + acq_func: gp_hedge + alpha: 1e-10 + n_initial_points: 10 + n_restarts_optimizer: 0 + normalize_y: False + strategy: cl_min +} + + Space ++-----+ +params: +/some: choices(['random', 'or', 'not']) +/command: uniform(0, 1) + + Meta-data ++---------+ +orion=1.0.1 +when=now + + + Parent experiment ++-----------------+ +parent: parent-name +adapter: + + adummy: dict + foran: adapter + + + Stats ++-----+ +best trial: + a: 0.0 + b: 1 + c: Some +corresponding evaluation: 0.1 + +""" # noqa: W291 From 6e6b100ffc6b2a0cc4f9add95dd69b716ef516c7 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Tue, 16 Jul 2019 14:11:23 -0400 Subject: [PATCH 33/50] Remove custom support template for `orion info` Why: This was causing way too much burden for what it's worth. --- src/orion/core/cli/info.py | 191 ++++------------ tests/unittests/core/cli/test_info.py | 310 +------------------------- 2 files changed, 41 insertions(+), 460 deletions(-) diff --git a/src/orion/core/cli/info.py b/src/orion/core/cli/info.py index a1c40878f..3e5c8e0fc 100755 --- a/src/orion/core/cli/info.py +++ b/src/orion/core/cli/info.py @@ -50,24 +50,16 @@ def main(args): """ -def format_info(experiment, templates=None): - """Render a string for all info of experiment - - Parameters - ---------- - experiment: `orion.core.worker.experiment.Experiment` - templates: dict - Templates for all sections and titles - - """ +def format_info(experiment): + """Render a string for all info of experiment""" info_string = INFO_TEMPLATE.format( - commandline=format_commandline(experiment, templates), - configuration=format_config(experiment, templates), - algorithm=format_algorithm(experiment, templates), - space=format_space(experiment, templates), - metadata=format_metadata(experiment, templates), - refers=format_refers(experiment, templates), - stats=format_stats(experiment, templates)) + commandline=format_commandline(experiment), + configuration=format_config(experiment), + algorithm=format_algorithm(experiment), + space=format_space(experiment), + metadata=format_metadata(experiment), + refers=format_refers(experiment), + stats=format_stats(experiment)) return info_string @@ -78,23 +70,9 @@ def format_info(experiment, templates=None): """ -def format_title(title, templates=None): - r"""Render a title above an horizontal bar - - Parameters - ---------- - title: string - templates: dict - Templates for `title`, `leaf` and `dict_node`. - Default is "{title}\n{empty:=<{title_len}}" - - """ - if templates is None: - templates = dict() - - title_template = templates.get('title', TITLE_TEMPLATE) - - title_string = title_template.format( +def format_title(title): + """Render a title above an horizontal bar""" + title_string = TITLE_TEMPLATE.format( title=title, title_len=len(title), empty='') @@ -265,24 +243,10 @@ def format_list(a_list, depth=0, width=4, templates=None): """ -def format_commandline(experiment, templates=None): - """Render a string for commandline section - - Parameters - ---------- - experiment: `orion.core.worker.experiment.Experiment` - templates: dict - templates for the title and `commandline`. - See `format_title` for more info. - - """ - if templates is None: - templates = dict() - - commandline_template = templates.get('commandline', COMMANDLINE_TEMPLATE) - - commandline_string = commandline_template.format( - title=format_title("Commandline", templates=templates), +def format_commandline(experiment): + """Render a string for commandline section""" + commandline_string = COMMANDLINE_TEMPLATE.format( + title=format_title("Commandline"), commandline=" ".join(experiment.metadata['user_args'])) return commandline_string @@ -295,24 +259,10 @@ def format_commandline(experiment, templates=None): """ -def format_config(experiment, templates=None): - """Render a string for config section - - Parameters - ---------- - experiment: `orion.core.worker.experiment.Experiment` - templates: dict - templates for the title and `config`. - See `format_title` for more info. - - """ - if templates is None: - templates = dict() - - config_template = templates.get('config', CONFIG_TEMPLATE) - - config_string = config_template.format( - title=format_title("Config", templates), +def format_config(experiment): + """Render a string for config section""" + config_string = CONFIG_TEMPLATE.format( + title=format_title("Config"), experiment=experiment) return config_string @@ -324,24 +274,10 @@ def format_config(experiment, templates=None): """ -def format_algorithm(experiment, templates=None): - """Render a string for algorithm section - - Parameters - ---------- - experiment: `orion.core.worker.experiment.Experiment` - templates: dict - templates for the title and `algorithm`. - See `format_title` for more info. - - """ - if templates is None: - templates = dict() - - algorithm_template = templates.get('algorithm', ALGORITHM_TEMPLATE) - - algorithm_string = algorithm_template.format( - title=format_title("Algorithm", templates), +def format_algorithm(experiment): + """Render a string for algorithm section""" + algorithm_string = ALGORITHM_TEMPLATE.format( + title=format_title("Algorithm"), configuration=format_dict(experiment.configuration['algorithms'])) return algorithm_string @@ -353,24 +289,10 @@ def format_algorithm(experiment, templates=None): """ -def format_space(experiment, templates=None): - """Render a string for space section - - Parameters - ---------- - experiment: `orion.core.worker.experiment.Experiment` - templates: dict - templates for the title and `space`. - See `format_title` for more info. - - """ - if templates is None: - templates = dict() - - space_template = templates.get('space', SPACE_TEMPLATE) - - space_string = space_template.format( - title=format_title("Space", templates), +def format_space(experiment): + """Render a string for space section""" + space_string = SPACE_TEMPLATE.format( + title=format_title("Space"), params="\n".join(name + ": " + experiment.space[name].get_prior_string() for name in experiment.space.keys())) @@ -385,24 +307,10 @@ def format_space(experiment, templates=None): """ -def format_metadata(experiment, templates=None): - """Render a string for metadata section - - Parameters - ---------- - experiment: `orion.core.worker.experiment.Experiment` - templates: dict - templates for the title and `metadata`. - See `format_title` for more info. - - """ - if templates is None: - templates = dict() - - metadata_template = templates.get('metadata', METADATA_TEMPLATE) - - metadata_string = metadata_template.format( - title=format_title("Meta-data", templates), +def format_metadata(experiment): + """Render a string for metadata section""" + metadata_string = METADATA_TEMPLATE.format( + title=format_title("Meta-data"), experiment=experiment) return metadata_string @@ -416,22 +324,8 @@ def format_metadata(experiment, templates=None): """ -def format_refers(experiment, templates=None): - """Render a string for refers section - - Parameters - ---------- - experiment: `orion.core.worker.experiment.Experiment` - templates: dict - templates for the title and `refers`. - See `format_title` for more info. - - """ - if templates is None: - templates = dict() - - refers_template = templates.get('refers', REFERS_TEMPLATE) - +def format_refers(experiment): + """Render a string for refers section""" if experiment.node.root is experiment.node: root = '' parent = '' @@ -441,8 +335,8 @@ def format_refers(experiment, templates=None): parent = experiment.node.parent.name adapter = "\n" + format_dict(experiment.refers['adapter'].configuration, depth=1, width=2) - refers_string = refers_template.format( - title=format_title("Parent experiment", templates), + refers_string = REFERS_TEMPLATE.format( + title=format_title("Parent experiment"), root=root, parent=parent, adapter=adapter) @@ -468,7 +362,7 @@ def format_refers(experiment, templates=None): """ -def format_stats(experiment, templates=None): +def format_stats(experiment): """Render a string for stat section Parameters @@ -479,20 +373,15 @@ def format_stats(experiment, templates=None): See `format_title` for more info. """ - if templates is None: - templates = dict() - - stats_template = templates.get('stats', STATS_TEMPLATE) - stats = experiment.stats if not stats: return NO_STATS_TEMPLATE.format( - title=format_title("Stats", templates)) + title=format_title("Stats")) best_params = get_trial_params(stats['best_trials_id'], experiment) - stats_string = stats_template.format( - title=format_title("Stats", templates), + stats_string = STATS_TEMPLATE.format( + title=format_title("Stats"), stats=stats, best_params=format_dict(best_params, depth=1, width=2)) diff --git a/tests/unittests/core/cli/test_info.py b/tests/unittests/core/cli/test_info.py index a7761633f..f7cdb2cac 100755 --- a/tests/unittests/core/cli/test_info.py +++ b/tests/unittests/core/cli/test_info.py @@ -98,25 +98,12 @@ def algorithm_dict(): def test_format_title(): - """Test title formatting with custom template""" + """Test title formatting template""" result = """Test\n====""" assert format_title('Test') == result -def test_format_title_custom(): - """Test title formatting with custom template""" - template = """{empty:-<{title_len}}\n{title}\n{empty:-<{title_len}}""" - result = """------\nCustom\n------""" - - assert format_title('Custom', templates=dict(title=template)) == result - - template = """{title}\n---""" - result = """Custom\n---""" - - assert format_title('Custom', templates=dict(title=template)) == result - - @pytest.mark.parametrize("depth", [0, 1, 2]) def test_format_dict_depth_synthetic(depth, dummy_dict): """Test dict formatting with different depths for one line""" @@ -387,21 +374,6 @@ def test_format_commandline(): """ -def test_format_commandline_custom(): - """Test commandline section with custom formatting""" - experiment = DummyExperiment() - commandline = ['executing.sh', '--some', 'random', '--command', 'line', 'arguments'] - experiment.metadata = {'user_args': commandline} - templates = dict( - title=" {title}\n+{empty:-<{title_len}}+", - commandline="{title}\ncommandline: {commandline}") - - assert format_commandline(experiment, templates) == """\ - Commandline -+-----------+ -commandline: executing.sh --some random --command line arguments""" - - def test_format_config(monkeypatch): """Test config section formatting""" experiment = DummyExperiment() @@ -415,21 +387,6 @@ def test_format_config(monkeypatch): """ -def test_format_config_custom(monkeypatch): - """Test config section with custom formatting""" - experiment = DummyExperiment() - experiment.pool_size = 10 - experiment.max_trials = 100 - templates = dict( - title="{title}\n{empty:+<{title_len}}", - config="{title}\n(pool_size={experiment.pool_size}, max_trials={experiment.max_trials})") - - assert format_config(experiment, templates) == """\ -Config -++++++ -(pool_size=10, max_trials=100)""" - - def test_format_algorithm(algorithm_dict): """Test algorithm section formatting""" experiment = DummyExperiment() @@ -446,28 +403,6 @@ def test_format_algorithm(algorithm_dict): """ -def test_format_algorithm_custom(algorithm_dict): - """Test algorithm section with custom formatting""" - experiment = DummyExperiment() - experiment.configuration = {'algorithms': algorithm_dict} - templates = dict( - title="{title}\n{empty:~<{title_len}}", - algorithm="{title}\n{{\n {configuration}\n}}") - - assert format_algorithm(experiment, templates) == """\ -Algorithm -~~~~~~~~~ -{ - bayesianoptimizer: - acq_func: gp_hedge - alpha: 1e-10 - n_initial_points: 10 - n_restarts_optimizer: 0 - normalize_y: False - strategy: cl_min -}""" - - def test_format_space(): """Test space section formatting""" experiment = DummyExperiment() @@ -483,25 +418,6 @@ def test_format_space(): """ -def test_format_space_custom(): - """Test space section with custom formatting""" - experiment = DummyExperiment() - commandline = ['executing.sh', '--some~choices(["random", "or", "not"])', - '--command~uniform(0, 1)'] - space = SpaceBuilder().build_from(commandline) - experiment.space = space - templates = dict( - title="{title}\n{empty:-<{title_len}}", - space="{title}\nparams:\n{params}") - - assert format_space(experiment, templates) == """\ -Space ------ -params: -/some: choices(['random', 'or', 'not']) -/command: uniform(0, 1)""" - - def test_format_metadata(): """Test metadata section formatting""" experiment = DummyExperiment() @@ -518,29 +434,6 @@ def test_format_metadata(): """ -def test_format_metadata_custom(): - """Test metadata section with custom formatting""" - experiment = DummyExperiment() - experiment.metadata = dict( - user='user', - datetime='now', - orion_version='1.0.1') - templates = dict( - title="{title}\n{empty:*<{title_len}}", - metadata="""\ -{title} -orion={experiment.metadata[orion_version]} -when={experiment.metadata[datetime]} -""") - - assert format_metadata(experiment, templates) == """\ -Meta-data -********* -orion=1.0.1 -when=now -""" - - def test_format_refers_root(): """Test refers section formatting for a root experiment""" experiment = DummyExperiment() @@ -598,53 +491,6 @@ def test_format_refers_child(): """ # noqa: W291 -def test_format_refers_custom(): - """Test refers section with custom formatting""" - ROOT_NAME = 'root-name' - PARENT_NAME = 'parent-name' - - root = DummyExperiment() - root.name = ROOT_NAME - - parent = DummyExperiment() - parent.name = PARENT_NAME - - child = DummyExperiment() - child.node = DummyExperiment() - child.node.parent = parent - child.node.root = root - - adapter = DummyExperiment() - adapter.configuration = dict( - adummy='dict', - foran='adapter') - - child.refers = dict(adapter=adapter) - - # experiment.refers = dict( - # parent='user', - # datetime='now', - # orion_version='1.0.1') - templates = dict( - title="| {title} |\n+-{empty:-<{title_len}}-+", - refers="""\ -{title} -parent: {parent} -adapter: -{adapter} -""") - - assert format_refers(child, templates) == """\ -| Parent experiment | -+-------------------+ -parent: parent-name -adapter: - - adummy: dict - foran: adapter -""" - - def test_get_trial_params_empty(): """Test failing to fetch trials does not fail""" experiment = DummyExperiment() @@ -688,37 +534,6 @@ def test_format_stats(dummy_trial): """ -def test_format_stats_custom(dummy_trial): - """Test stats section with custom formatting""" - experiment = DummyExperiment() - experiment.stats = dict( - best_trials_id='dummy', - trials_completed=10, - best_evaluation=0.1, - start_time='yesterday', - finish_time='now', - duration='way too long') - experiment.fetch_trials = lambda query: [dummy_trial] - templates = dict( - title="* {title} *\n**{empty:*<{title_len}}**", - stats="""\ -{title} -best trial: -{best_params} -corresponding evaluation: {stats[best_evaluation]} -""") - - assert format_stats(experiment, templates) == """\ -* Stats * -********* -best trial: - a: 0.0 - b: 1 - c: Some -corresponding evaluation: 0.1 -""" - - def test_format_info(algorithm_dict, dummy_trial): """Test full formatting string""" experiment = DummyExperiment() @@ -821,126 +636,3 @@ def test_format_info(algorithm_dict, dummy_trial): duration: way too long """ # noqa: W291 - - -def test_format_info_custom(algorithm_dict, dummy_trial): - """Test full info string with custom formatting""" - experiment = DummyExperiment() - commandline = ['executing.sh', '--some~choices(["random", "or", "not"])', - '--command~uniform(0, 1)'] - experiment.metadata = {'user_args': commandline} - experiment.pool_size = 10 - experiment.max_trials = 100 - experiment.configuration = {'algorithms': algorithm_dict} - - space = SpaceBuilder().build_from(commandline) - experiment.space = space - print(space) - experiment.metadata.update(dict( - user='user', - datetime='now', - orion_version='1.0.1')) - - ROOT_NAME = 'root-name' - PARENT_NAME = 'parent-name' - - root = DummyExperiment() - root.name = ROOT_NAME - - parent = DummyExperiment() - parent.name = PARENT_NAME - - experiment.node = DummyExperiment() - experiment.node.parent = parent - experiment.node.root = root - - adapter = DummyExperiment() - adapter.configuration = dict( - adummy='dict', - foran='adapter') - - experiment.refers = dict(adapter=adapter) - experiment.stats = dict( - best_trials_id='dummy', - trials_completed=10, - best_evaluation=0.1, - start_time='yesterday', - finish_time='now', - duration='way too long') - experiment.fetch_trials = lambda query: [dummy_trial] - - templates = dict( - title=" {title} \n+{empty:-<{title_len}}+", - commandline="{title}\ncommandline: {commandline}", - config="{title}\n(pool_size={experiment.pool_size}, max_trials={experiment.max_trials})", - algorithm="{title}\n{{\n {configuration}\n}}", - space="{title}\nparams:\n{params}", - metadata="""\ -{title} -orion={experiment.metadata[orion_version]} -when={experiment.metadata[datetime]} -""", - refers="""\ -{title} -parent: {parent} -adapter: -{adapter} -""", - stats="""\ -{title} -best trial: -{best_params} -corresponding evaluation: {stats[best_evaluation]} -""") - - assert format_info(experiment, templates) == """\ - Commandline -+-----------+ -commandline: executing.sh --some~choices(["random", "or", "not"]) --command~uniform(0, 1) - - Config -+------+ -(pool_size=10, max_trials=100) - - Algorithm -+---------+ -{ - bayesianoptimizer: - acq_func: gp_hedge - alpha: 1e-10 - n_initial_points: 10 - n_restarts_optimizer: 0 - normalize_y: False - strategy: cl_min -} - - Space -+-----+ -params: -/some: choices(['random', 'or', 'not']) -/command: uniform(0, 1) - - Meta-data -+---------+ -orion=1.0.1 -when=now - - - Parent experiment -+-----------------+ -parent: parent-name -adapter: - - adummy: dict - foran: adapter - - - Stats -+-----+ -best trial: - a: 0.0 - b: 1 - c: Some -corresponding evaluation: 0.1 - -""" # noqa: W291 From 9ec60d5b156daf073f572e88460f7925225c2e2f Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Sun, 14 Jul 2019 13:22:44 -0400 Subject: [PATCH 34/50] Fetch all completed trials instead of subset Why: When there is a high number of workers and communication gets slower, it happens that fetches based on last trial's end time will miss some of them. Since is no simple way of defining a backward internal to catch them all, and since the actual cost of fetching all trials is relatively cheap (which may most likely be less than 1000), it may be better to simple fetch them all every time and filter locally. How: Add `__contains__` to TrialsHistory. Move filtering in `Producer` which contains `TrialsHistory`. --- src/orion/core/worker/experiment.py | 23 +++---------------- src/orion/core/worker/producer.py | 15 ++++++++---- src/orion/core/worker/trials_history.py | 7 ++++++ .../core/io/test_experiment_builder.py | 6 ----- tests/unittests/core/test_experiment.py | 7 ------ tests/unittests/core/test_producer.py | 4 ---- .../core/worker/test_trials_history.py | 23 +++++++++++++++++++ 7 files changed, 43 insertions(+), 42 deletions(-) diff --git a/src/orion/core/worker/experiment.py b/src/orion/core/worker/experiment.py index ace4afb33..52875af4e 100644 --- a/src/orion/core/worker/experiment.py +++ b/src/orion/core/worker/experiment.py @@ -87,7 +87,7 @@ class Experiment(object): __slots__ = ('name', 'refers', 'metadata', 'pool_size', 'max_trials', 'algorithms', 'producer', 'working_dir', '_init_done', '_id', - '_node', '_last_fetched', '_storage') + '_node', '_storage') non_branching_attrs = ('pool_size', 'max_trials') def __init__(self, name, user=None): @@ -137,8 +137,6 @@ def __init__(self, name, user=None): setattr(self, attrname, config[attrname]) self._id = config['_id'] - self._last_fetched = self.metadata.get("datetime", datetime.datetime.utcnow()) - def fetch_trials(self, query, selection=None): """Fetch trials of the experiment in the database @@ -360,28 +358,13 @@ def register_trial(self, trial): self._storage.register_trial(trial) def fetch_completed_trials(self): - """Fetch recent completed trials that this `Experiment` instance has not yet seen. + """Fetch all completed trials. Trials are sorted based on `Trial.submit_time` - .. note:: - - It will return only those with `Trial.end_time` after `_last_fetched`, for performance - reasons. - :return: list of completed `Trial` objects """ - query = dict( - status='completed', - end_time={'$gt': self._last_fetched} - ) - - completed_trials = self.fetch_trials_tree(query) - - if completed_trials: - self._last_fetched = max(trial.end_time for trial in completed_trials) - - return completed_trials + return self.fetch_trials_tree(dict(status='completed')) def fetch_noncompleted_trials(self): """Fetch non-completed trials of this `Experiment` instance. diff --git a/src/orion/core/worker/producer.py b/src/orion/core/worker/producer.py index cf3cb0085..5f9469f75 100644 --- a/src/orion/core/worker/producer.py +++ b/src/orion/core/worker/producer.py @@ -98,16 +98,21 @@ def _update_algorithm(self): log.debug("### Fetch completed trials to observe:") completed_trials = self.experiment.fetch_completed_trials() - log.debug("### %s", completed_trials) + new_completed_trials = [] + for trial in completed_trials: + if trial not in self.trials_history: + new_completed_trials.append(trial) - if completed_trials: + log.debug("### %s", new_completed_trials) + + if new_completed_trials: log.debug("### Convert them to list of points and their results.") points = list(map(lambda trial: format_trials.trial_to_tuple(trial, self.space), - completed_trials)) - results = list(map(format_trials.get_trial_results, completed_trials)) + new_completed_trials)) + results = list(map(format_trials.get_trial_results, new_completed_trials)) log.debug("### Observe them.") - self.trials_history.update(completed_trials) + self.trials_history.update(new_completed_trials) self.algorithm.observe(points, results) self.strategy.observe(points, results) diff --git a/src/orion/core/worker/trials_history.py b/src/orion/core/worker/trials_history.py index 94ab951dd..702b0011b 100644 --- a/src/orion/core/worker/trials_history.py +++ b/src/orion/core/worker/trials_history.py @@ -17,6 +17,11 @@ class TrialsHistory: def __init__(self): """Create empty trials history""" self.children = [] + self.ids = set() + + def __contains__(self, trial): + """Return True if the trial is in the observed history""" + return trial.id in self.ids def update(self, trials): """Update the list of children trials @@ -30,4 +35,6 @@ def update(self, trials): descendents -= set(trial.parents) descendents.add(trial.id) + self.ids |= descendents + self.children = list(sorted(descendents)) diff --git a/tests/unittests/core/io/test_experiment_builder.py b/tests/unittests/core/io/test_experiment_builder.py index 5f08dc473..cdc42cf09 100644 --- a/tests/unittests/core/io/test_experiment_builder.py +++ b/tests/unittests/core/io/test_experiment_builder.py @@ -160,7 +160,6 @@ def test_build_view_from(config_file, create_db_instance, exp_config, random_dt) assert exp_view.name == exp_config[0][0]['name'] assert exp_view.configuration['refers'] == exp_config[0][0]['refers'] assert exp_view.metadata == exp_config[0][0]['metadata'] - assert exp_view._experiment._last_fetched == exp_config[0][0]['metadata']['datetime'] assert exp_view.pool_size == exp_config[0][0]['pool_size'] assert exp_view.max_trials == exp_config[0][0]['max_trials'] assert exp_view.algorithms.configuration == exp_config[0][0]['algorithms'] @@ -202,7 +201,6 @@ def test_build_from_no_hit(config_file, create_db_instance, exp_config, random_d assert exp.metadata['user'] == 'tsirif' assert exp.metadata['user_script'] == cmdargs['user_args'][0] assert exp.metadata['user_args'] == cmdargs['user_args'][1:] - assert exp._last_fetched == random_dt assert exp.pool_size == 1 assert exp.max_trials == 100 assert exp.algorithms.configuration == {'random': {'seed': None}} @@ -228,7 +226,6 @@ def test_build_from_hit(old_config_file, create_db_instance, exp_config, script_ assert exp.name == exp_config[0][0]['name'] assert exp.configuration['refers'] == exp_config[0][0]['refers'] assert exp.metadata == exp_config[0][0]['metadata'] - assert exp._last_fetched == exp_config[0][0]['metadata']['datetime'] assert exp.pool_size == exp_config[0][0]['pool_size'] assert exp.max_trials == exp_config[0][0]['max_trials'] assert exp.algorithms.configuration == exp_config[0][0]['algorithms'] @@ -266,7 +263,6 @@ def test_build_from_config_no_hit(config_file, create_db_instance, exp_config, r assert exp.metadata['user'] == 'tsirif' assert exp.metadata['user_script'] == cmdargs['user_args'][0] assert exp.metadata['user_args'] == cmdargs['user_args'][1:] - assert exp._last_fetched == random_dt assert exp.pool_size == 1 assert exp.max_trials == 100 assert not exp.is_done @@ -303,7 +299,6 @@ def test_build_from_config_hit(old_config_file, create_db_instance, exp_config, assert exp.name == exp_config[0][0]['name'] assert exp.configuration['refers'] == exp_config[0][0]['refers'] assert exp.metadata == exp_config[0][0]['metadata'] - assert exp._last_fetched == exp_config[0][0]['metadata']['datetime'] assert exp.pool_size == exp_config[0][0]['pool_size'] assert exp.max_trials == exp_config[0][0]['max_trials'] assert exp.algorithms.configuration == exp_config[0][0]['algorithms'] @@ -332,7 +327,6 @@ def test_build_without_config_hit(old_config_file, create_db_instance, exp_confi assert exp.name == exp_config[0][0]['name'] assert exp.configuration['refers'] == exp_config[0][0]['refers'] assert exp.metadata == exp_config[0][0]['metadata'] - assert exp._last_fetched == exp_config[0][0]['metadata']['datetime'] assert exp.pool_size == exp_config[0][0]['pool_size'] assert exp.max_trials == exp_config[0][0]['max_trials'] assert exp.algorithms.configuration == exp_config[0][0]['algorithms'] diff --git a/tests/unittests/core/test_experiment.py b/tests/unittests/core/test_experiment.py index f3892e35a..db78861a3 100644 --- a/tests/unittests/core/test_experiment.py +++ b/tests/unittests/core/test_experiment.py @@ -184,7 +184,6 @@ def test_new_experiment_due_to_name(self, create_db_instance, random_dt): assert exp.name == 'supernaekei' assert exp.refers == {} assert exp.metadata['user'] == 'tsirif' - assert exp._last_fetched == random_dt assert len(exp.metadata) == 1 assert exp.pool_size is None assert exp.max_trials is None @@ -203,7 +202,6 @@ def test_new_experiment_due_to_username(self, create_db_instance, random_dt): assert exp.name == 'supernaedo2' assert exp.refers == {} assert exp.metadata['user'] == 'bouthilx' - assert exp._last_fetched == random_dt assert len(exp.metadata) == 1 assert exp.pool_size is None assert exp.max_trials is None @@ -222,7 +220,6 @@ def test_existing_experiment(self, create_db_instance, exp_config): assert exp.name == exp_config[0][0]['name'] assert exp.refers == exp_config[0][0]['refers'] assert exp.metadata == exp_config[0][0]['metadata'] - assert exp._last_fetched == exp_config[0][0]['metadata']['datetime'] assert exp.pool_size == exp_config[0][0]['pool_size'] assert exp.max_trials == exp_config[0][0]['max_trials'] assert exp.algorithms == exp_config[0][0]['algorithms'] @@ -802,7 +799,6 @@ def test_fetch_all_trials(hacked_exp, exp_config, random_dt): def test_fetch_completed_trials(hacked_exp, exp_config, random_dt): """Fetch a list of the unseen yet completed trials.""" trials = hacked_exp.fetch_completed_trials() - assert hacked_exp._last_fetched == max(trial.end_time for trial in trials) assert len(trials) == 3 # Trials are sorted based on submit time assert trials[0].to_dict() == exp_config[1][0] @@ -921,7 +917,6 @@ def test_existing_experiment_view(self, create_db_instance, exp_config): assert exp.name == exp_config[0][0]['name'] assert exp.configuration['refers'] == exp_config[0][0]['refers'] assert exp.metadata == exp_config[0][0]['metadata'] - assert exp._experiment._last_fetched == exp_config[0][0]['metadata']['datetime'] assert exp.pool_size == exp_config[0][0]['pool_size'] assert exp.max_trials == exp_config[0][0]['max_trials'] assert exp.algorithms.configuration == exp_config[0][0]['algorithms'] @@ -978,7 +973,6 @@ def test_fetch_completed_trials_from_view(hacked_exp, exp_config, random_dt): experiment_view._experiment = hacked_exp trials = experiment_view.fetch_completed_trials() - assert experiment_view._experiment._last_fetched == max(trial.end_time for trial in trials) assert len(trials) == 3 assert trials[0].to_dict() == exp_config[1][0] assert trials[1].to_dict() == exp_config[1][2] @@ -1064,7 +1058,6 @@ def test_new_experiment_with_parent(self, create_db_instance, random_dt, exp_con assert exp.configuration['refers'] == exp_config[0][4]['refers'] exp_config[0][4]['metadata']['datetime'] = random_dt assert exp.metadata == exp_config[0][4]['metadata'] - assert exp._last_fetched == random_dt assert exp.pool_size is None assert exp.max_trials is None assert exp.configuration['algorithms'] == {'random': {'seed': None}} diff --git a/tests/unittests/core/test_producer.py b/tests/unittests/core/test_producer.py index 52abcf240..bfd209e9c 100644 --- a/tests/unittests/core/test_producer.py +++ b/tests/unittests/core/test_producer.py @@ -344,10 +344,6 @@ def test_naive_algo_trained_on_all_non_completed_trials(producer, database, rand def test_naive_algo_is_discared(producer, database, monkeypatch): """Verify that naive algo is discarded and recopied from original algo""" - # Get rid of the mock on datetime.datetime.utcnow() otherwise fetch_completed_trials always - # fetch all trials since _last_fetched never changes. - monkeypatch.undo() - # Set values for predictions producer.experiment.pool_size = 1 producer.experiment.algorithms.algorithm.possible_values = [('rnn', 'gru')] diff --git a/tests/unittests/core/worker/test_trials_history.py b/tests/unittests/core/worker/test_trials_history.py index 6c23e242e..0f3a371c4 100644 --- a/tests/unittests/core/worker/test_trials_history.py +++ b/tests/unittests/core/worker/test_trials_history.py @@ -14,6 +14,29 @@ def __init__(self, trial_id, parents): self.parents = parents +def test_history_contains_new_child(): + """Verify that __contains__ return True for a new child""" + trials_history = TrialsHistory() + new_child = DummyTrial(1, []) + assert new_child not in trials_history + trials_history.update([new_child]) + assert new_child in trials_history + + +def test_history_contains_old_child(): + """Verify that __contains__ return True for a new child""" + trials_history = TrialsHistory() + old_child = DummyTrial(1, []) + trials_history.update([old_child]) + new_child = DummyTrial(2, [old_child.id]) + assert new_child not in trials_history + trials_history.update([new_child]) + assert old_child.id not in trials_history.children + assert old_child in trials_history + assert new_child.id in trials_history.children + assert new_child in trials_history + + def test_added_children_without_ancestors(): """Verify that children are added to history""" trials_history = TrialsHistory() From d5a2b3099f39d3a0462ba912e06dfcb2bce30e4e Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Tue, 16 Jul 2019 20:49:43 -0400 Subject: [PATCH 35/50] Bundle update calls in Producer for consistency Why: There can be modifications in trials status between the call to `fetch_completed_trials` and `fetch_noncompleted_trials`, leading to inconsistent states, where very recently completed trials are neither in completed trials, nor noncompleted trials. This is problematic for ASHA, because a promoted trial may be observed by the naive algorithm while the base trial (non-promoted one) is lost between the 2 calls. When this happens the naive algo crashes because the rungs are inconsistent. How: Fetch all trials once and then pass the completed ones to `update_algorithm` and pass the non completed ones to `update_naive_algorithm`. This way the fetch is a single snapshot and garantees consistency. --- src/orion/core/worker/producer.py | 19 ++++++------- tests/unittests/core/test_producer.py | 39 ++++++++++++++++++--------- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/src/orion/core/worker/producer.py b/src/orion/core/worker/producer.py index 5f9469f75..371ad716f 100644 --- a/src/orion/core/worker/producer.py +++ b/src/orion/core/worker/producer.py @@ -89,14 +89,16 @@ def produce(self): "wrong.") def update(self): - """Pull newest completed trials and all non completed trials to update naive model.""" - self._update_algorithm() - self._update_naive_algorithm() + """Pull all trials to update model with completed ones and naive model with non completed + ones. + """ + trials = self.experiment.fetch_trials({}) + self._update_algorithm([trial for trial in trials if trial.status == 'completed']) + self._update_naive_algorithm([trial for trial in trials if trial.status != 'completed']) - def _update_algorithm(self): + def _update_algorithm(self, completed_trials): """Pull newest completed trials to update local model.""" log.debug("### Fetch completed trials to observe:") - completed_trials = self.experiment.fetch_completed_trials() new_completed_trials = [] for trial in completed_trials: @@ -116,13 +118,12 @@ def _update_algorithm(self): self.algorithm.observe(points, results) self.strategy.observe(points, results) - def _produce_lies(self): + def _produce_lies(self, incomplete_trials): """Add fake objective results to incomplete trials Then register the trials in the db """ log.debug("### Fetch active trials to observe:") - incomplete_trials = self.experiment.fetch_noncompleted_trials() lying_trials = [] log.debug("### %s", incomplete_trials) @@ -142,12 +143,12 @@ def _produce_lies(self): return lying_trials - def _update_naive_algorithm(self): + def _update_naive_algorithm(self, incomplete_trials): """Pull all non completed trials to update naive model.""" self.naive_algorithm = copy.deepcopy(self.algorithm) self.naive_trials_history = copy.deepcopy(self.trials_history) log.debug("### Create fake trials to observe:") - lying_trials = self._produce_lies() + lying_trials = self._produce_lies(incomplete_trials) log.debug("### %s", lying_trials) if lying_trials: log.debug("### Convert them to list of points and their results.") diff --git a/tests/unittests/core/test_producer.py b/tests/unittests/core/test_producer.py index bfd209e9c..e0cfb7505 100644 --- a/tests/unittests/core/test_producer.py +++ b/tests/unittests/core/test_producer.py @@ -30,6 +30,21 @@ def lie(self, trial): return lie +def produce_lies(producer): + """Wrap production of lies outside of `Producer.update`""" + return producer._produce_lies(producer.experiment.fetch_noncompleted_trials()) + + +def update_algorithm(producer): + """Wrap update of algorithm outside of `Producer.update`""" + return producer._update_algorithm(producer.experiment.fetch_completed_trials()) + + +def update_naive_algorithm(producer): + """Wrap update of naive algorithm outside of `Producer.update`""" + return producer._update_naive_algorithm(producer.experiment.fetch_noncompleted_trials()) + + @pytest.fixture() def producer(hacked_exp, random_dt, exp_config, categorical_values): """Return a setup `Producer`.""" @@ -181,7 +196,7 @@ def test_no_lies_if_all_trials_completed(producer, database, random_dt): producer.update() - assert len(producer._produce_lies()) == 0 + assert len(produce_lies(producer)) == 0 def test_lies_generation(producer, database, random_dt): @@ -195,7 +210,7 @@ def test_lies_generation(producer, database, random_dt): producer.update() - lies = producer._produce_lies() + lies = produce_lies(producer) assert len(lies) == 4 trials_non_completed = list( @@ -223,7 +238,7 @@ def test_register_lies(producer, database, random_dt): assert len(trials_completed) == 3 producer.update() - producer._produce_lies() + produce_lies(producer) lying_trials = list(database.lying_trials.find({'experiment': producer.experiment.id})) assert len(lying_trials) == 4 @@ -257,7 +272,7 @@ def test_register_duplicate_lies(producer, database, random_dt): producer.experiment.algorithms.algorithm.possible_values = [('rnn', 'gru')] producer.update() - assert len(producer._produce_lies()) == 4 + assert len(produce_lies(producer)) == 4 lying_trials = list(database.lying_trials.find({'experiment': producer.experiment.id})) assert len(lying_trials) == 4 @@ -270,12 +285,12 @@ def test_register_duplicate_lies(producer, database, random_dt): producer.update() - assert len(producer._produce_lies()) == 5 + assert len(produce_lies(producer)) == 5 lying_trials = list(database.lying_trials.find({'experiment': producer.experiment.id})) assert len(lying_trials) == 5 # Make sure trying to generate again does not add more fake trials since they are identical - assert len(producer._produce_lies()) == 5 + assert len(produce_lies(producer)) == 5 lying_trials = list(database.lying_trials.find({'experiment': producer.experiment.id})) assert len(lying_trials) == 5 @@ -289,14 +304,14 @@ def test_register_duplicate_lies_with_different_results(producer, database, rand # Overwrite value of lying result to force different results. producer.strategy._value = 11 - assert len(producer._produce_lies()) == 4 + assert len(produce_lies(producer)) == 4 lying_trials = list(database.lying_trials.find({'experiment': producer.experiment.id})) assert len(lying_trials) == 4 # Overwrite value of lying result to force different results. producer.strategy._value = new_lying_value = 12 - lying_trials = producer._produce_lies() + lying_trials = produce_lies(producer) assert len(lying_trials) == 4 nb_lying_trials = database.lying_trials.count({'experiment': producer.experiment.id}) assert nb_lying_trials == 4 + 4 @@ -336,7 +351,7 @@ def test_naive_algo_trained_on_all_non_completed_trials(producer, database, rand # Executing the actual test producer.update() - assert len(producer._produce_lies()) == 6 + assert len(produce_lies(producer)) == 6 assert len(producer.algorithm.algorithm._points) == 1 assert len(producer.naive_algorithm.algorithm._points) == (1 + 6) @@ -349,7 +364,7 @@ def test_naive_algo_is_discared(producer, database, monkeypatch): producer.experiment.algorithms.algorithm.possible_values = [('rnn', 'gru')] producer.update() - assert len(producer._produce_lies()) == 4 + assert len(produce_lies(producer)) == 4 first_naive_algorithm = producer.naive_algorithm @@ -359,13 +374,13 @@ def test_naive_algo_is_discared(producer, database, monkeypatch): producer.produce() # Only update the original algo, naive algo is still not discarded - producer._update_algorithm() + update_algorithm(producer) assert len(producer.algorithm.algorithm._points) == 3 assert first_naive_algorithm == producer.naive_algorithm assert len(producer.naive_algorithm.algorithm._points) == (3 + 4) # Discard naive algo and create a new one, now trained on 5 points. - producer._update_naive_algorithm() + update_naive_algorithm(producer) assert first_naive_algorithm != producer.naive_algorithm assert len(producer.naive_algorithm.algorithm._points) == (3 + 5) From 18d1a57bac0a4da1c3b3fa95be24fc651e339656 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Wed, 17 Jul 2019 13:49:36 -0400 Subject: [PATCH 36/50] Dont configure experiment views (#223) Why: When configuring the experiment, missing local config files can make the configuarion crash. There is however little value for now to configure experiment views as we are never using their space or algorithm. Therefore the simplest solution to this problem is to stop configuring the experiment views. This may be problematic however for users who may use the experiment view to fetch and analyse results. For this reason, we should seek to bring back the full instantiation of experiment views once the refactoring of orion's global configuarion is done. --- src/orion/core/cli/insert.py | 9 ++++-- src/orion/core/cli/status.py | 2 +- src/orion/core/worker/experiment.py | 30 +++++++++++-------- .../core/io/test_experiment_builder.py | 5 ++-- tests/unittests/core/test_experiment.py | 8 +++-- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/orion/core/cli/insert.py b/src/orion/core/cli/insert.py index f5150c1d7..ca98f3bcc 100644 --- a/src/orion/core/cli/insert.py +++ b/src/orion/core/cli/insert.py @@ -39,16 +39,19 @@ def add_subparser(parser): def main(args): """Fetch config and insert new point""" command_line_user_args = args.pop('user_args', [None])[1:] - experiment_view = ExperimentBuilder().build_view_from(args) + # TODO: Views are not fully configured until configuration is refactored + experiment = ExperimentBuilder().build_view_from(args) + # TODO: Remove this line when views gets fully configured + experiment = ExperimentBuilder().build_from(args) transformed_args = _build_from(command_line_user_args) - exp_space = experiment_view.space + exp_space = experiment.space values = _create_tuple_from_values(transformed_args, exp_space) trial = tuple_to_trial(values, exp_space) - ExperimentBuilder().build_from_config(experiment_view.configuration).register_trial(trial) + experiment.register_trial(trial) def _validate_dimensions(transformed_args, exp_space): diff --git a/src/orion/core/cli/status.py b/src/orion/core/cli/status.py index ef586e9cb..d4d2c726c 100644 --- a/src/orion/core/cli/status.py +++ b/src/orion/core/cli/status.py @@ -72,7 +72,7 @@ def get_experiments(args): query = {'name': args['name']} if args.get('name') else {} experiments = Database().read("experiments", query, projection) - return [EVCBuilder().build_from({'name': exp['name']}) for exp in experiments] + return [EVCBuilder().build_view_from({'name': exp['name']}) for exp in experiments] def print_status_recursively(exp, depth=0, **kwargs): diff --git a/src/orion/core/worker/experiment.py b/src/orion/core/worker/experiment.py index 52875af4e..6990c83a7 100644 --- a/src/orion/core/worker/experiment.py +++ b/src/orion/core/worker/experiment.py @@ -464,7 +464,7 @@ def configuration(self): else: config[attrname] = attribute - if self._init_done and attrname == "refers" and attribute.get("adapter"): + if attrname == "refers" and isinstance(attribute.get("adapter"), BaseAdapter): config[attrname] = copy.deepcopy(config[attrname]) config[attrname]['adapter'] = config[attrname]['adapter'].configuration @@ -764,18 +764,22 @@ def __init__(self, name, user=None): "no view can be created." % (self._experiment.name, self._experiment.metadata['user'])) - try: - self._experiment.configure(self._experiment.configuration, enable_branching=False, - enable_update=False) - except ValueError as e: - if "Configuration is different and generates a branching event" in str(e): - raise RuntimeError( - "Configuration in the database does not correspond to the one generated by " - "Experiment object. This is likely due to a backward incompatible update in " - "Oríon. Please report to https://github.com/epistimio/orion/issues.") from e - - raise - + # TODO: Views are not fully configured until configuration is refactored + # This snippet is to instantiate adapters anyhow, because it is required for + # experiment views in EVC. + if self.refers and not isinstance(self.refers.get('adapter'), BaseAdapter): + self._experiment.refers['adapter'] = Adapter.build(self.refers['adapter']) + + # try: + # self._experiment.configure(self._experiment.configuration, enable_branching=False, + # enable_update=False) + # except ValueError as e: + # if "Configuration is different and generates a branching event" in str(e): + # raise RuntimeError( + # "Configuration in the database does not correspond to the one generated by " + # "Experiment object. This is likely due to a backward incompatible update in " + # "Oríon. Please report to https://github.com/epistimio/orion/issues.") from e + # raise self._experiment._storage = ReadOnlyStorageProtocol(self._experiment._storage) def __getattr__(self, name): diff --git a/tests/unittests/core/io/test_experiment_builder.py b/tests/unittests/core/io/test_experiment_builder.py index cdc42cf09..a171896de 100644 --- a/tests/unittests/core/io/test_experiment_builder.py +++ b/tests/unittests/core/io/test_experiment_builder.py @@ -154,7 +154,7 @@ def test_build_view_from(config_file, create_db_instance, exp_config, random_dt) cmdargs = {'name': 'supernaedo2', 'config': config_file} exp_view = ExperimentBuilder().build_view_from(cmdargs) - assert exp_view._experiment._init_done is True + assert exp_view._experiment._init_done is False assert get_view_db(exp_view) is create_db_instance assert exp_view._id == exp_config[0][0]['_id'] assert exp_view.name == exp_config[0][0]['name'] @@ -162,7 +162,8 @@ def test_build_view_from(config_file, create_db_instance, exp_config, random_dt) assert exp_view.metadata == exp_config[0][0]['metadata'] assert exp_view.pool_size == exp_config[0][0]['pool_size'] assert exp_view.max_trials == exp_config[0][0]['max_trials'] - assert exp_view.algorithms.configuration == exp_config[0][0]['algorithms'] + # TODO: Views are not fully configured until configuration is refactored + # assert exp_view.algorithms.configuration == exp_config[0][0]['algorithms'] @pytest.mark.usefixtures("clean_db", "null_db_instances", "with_user_bouthilx") diff --git a/tests/unittests/core/test_experiment.py b/tests/unittests/core/test_experiment.py index db78861a3..aa439e907 100644 --- a/tests/unittests/core/test_experiment.py +++ b/tests/unittests/core/test_experiment.py @@ -911,7 +911,7 @@ def test_empty_experiment_view_due_to_username(self): def test_existing_experiment_view(self, create_db_instance, exp_config): """Hit exp_name + user's name in the db, fetch most recent entry.""" exp = ExperimentView('supernaedo2') - assert exp._experiment._init_done is True + assert exp._experiment._init_done is False assert exp._id == exp_config[0][0]['_id'] assert exp.name == exp_config[0][0]['name'] @@ -919,7 +919,8 @@ def test_existing_experiment_view(self, create_db_instance, exp_config): assert exp.metadata == exp_config[0][0]['metadata'] assert exp.pool_size == exp_config[0][0]['pool_size'] assert exp.max_trials == exp_config[0][0]['max_trials'] - assert exp.algorithms.configuration == exp_config[0][0]['algorithms'] + # TODO: Views are not fully configured until configuration is refactored + # assert exp.algorithms.configuration == exp_config[0][0]['algorithms'] with pytest.raises(AttributeError): exp.this_is_not_in_config = 5 @@ -935,8 +936,9 @@ def test_existing_experiment_view(self, create_db_instance, exp_config): with pytest.raises(AttributeError): exp.reserve_trial + @pytest.mark.skip(reason='Views are not fully configured until configuration is refactored') @pytest.mark.usefixtures("with_user_tsirif", "create_db_instance") - def test_existing_experiment_view_not_modified(self, exp_config, monkeypatch): + def test_experiment_view_not_modified(self, exp_config, monkeypatch): """Experiment should not be modified if fetched in another verion of Oríon. When loading a view the original config is used to configure the experiment, but From 5f1a491fbcaf51a6e7e7f434e2f7fc5222c99560 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Thu, 18 Jul 2019 14:41:51 -0400 Subject: [PATCH 37/50] Set navigation_depth for doc menu --- docs/src/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/conf.py b/docs/src/conf.py index 7a6ea13cc..7f7b6fe1b 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -121,9 +121,9 @@ # 'style_external_links': False, # 'vcs_pageview_mode': '', # Toc options - 'collapse_navigation': False, + 'collapse_navigation': True, 'sticky_navigation': True, - 'navigation_depth': 2, + 'navigation_depth': 4, # 'includehidden': False, # 'titles_only': False } From f0acbd9ac7bafaa957e90232c99b1844e2adc92c Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Thu, 18 Jul 2019 14:43:10 -0400 Subject: [PATCH 38/50] Add new section Monitoring Why: Monitoring of experiment is an important part of the usage of a tool for hyper-parameter optimisation. There is many commands and ways to monitor experiments using the library API, therefore it seams relevant to have a section dedicated to this. --- docs/src/index.rst | 3 +- docs/src/user/cli/info.rst | 2 + docs/src/user/evc.rst | 27 ++--------- docs/src/user/library/evc_results.rst | 36 ++++++++++++++ docs/src/user/library/results.rst | 68 +++++++++++++++++++++++++++ docs/src/user/pytorch.rst | 65 +------------------------ 6 files changed, 113 insertions(+), 88 deletions(-) create mode 100644 docs/src/user/cli/info.rst create mode 100644 docs/src/user/library/evc_results.rst create mode 100644 docs/src/user/library/results.rst diff --git a/docs/src/index.rst b/docs/src/index.rst index 9e5030265..8061f612c 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -15,8 +15,9 @@ :maxdepth: 1 user/pytorch - user/evc + user/monitoring user/algorithms + user/evc .. toctree:: :caption: Examples diff --git a/docs/src/user/cli/info.rst b/docs/src/user/cli/info.rst new file mode 100644 index 000000000..c9bf1c305 --- /dev/null +++ b/docs/src/user/cli/info.rst @@ -0,0 +1,2 @@ +``info`` Detailed informations about experiments +------------------------------------------------ diff --git a/docs/src/user/evc.rst b/docs/src/user/evc.rst index 2eef840b7..d895425c9 100644 --- a/docs/src/user/evc.rst +++ b/docs/src/user/evc.rst @@ -168,28 +168,7 @@ Source of conflicts Iterative Results ================= -You can retrieve results from different experiments in the same project using -the Experiment Version Control (EVC) system. The only difference -with ``ExperimentBuilder`` is that ``EVCBuilder`` will connect the experiment -to the EVC system, accessible through the ``node`` attribute. - -.. code-block:: python - - import pprint - from orion.core.io.evc_builder import EVCBuilder - - experiment = EVCBuilder().build_view_from( - {"name": "orion-tutorial-with-momentum"}) - - print(experiment.name) - pprint.pprint(experiment.stats) - - parent_experiment = experiment.node.parent.item - print(parent_experiment.name) - pprint.pprint(parent_experiment.stats) - - for child in experiment.node.children: - child_experiment = child.item - print(child_experiment.name) - pprint.pprint(child_experiment.stats) +.. note:: TODO: Set link when status command is documented. +Results from the EVC tree can be queried in aggregation with the command +`status --collapse `_ or fetching using the :ref:`library API `. diff --git a/docs/src/user/library/evc_results.rst b/docs/src/user/library/evc_results.rst new file mode 100644 index 000000000..cfe111843 --- /dev/null +++ b/docs/src/user/library/evc_results.rst @@ -0,0 +1,36 @@ +Iterative Results with EVC +-------------------------- + +When using the experiment version control (described `here `_), +the experiments are connected in a tree structure which we call the EVC tree. +You can retrieve results from different experiments with the EVC tree similarly +as described in previous section. The only difference +is we need to use :class:`EVCBuilder ` instead of +:class:`ExperimentBuilder `. +The :class:`EVCBuilder ` will connect the experiment +to the EVC tree, accessible through the +:attr:`node ` attribute. +All trials of the tree can be fetched +with :meth:`fetch_trials_tree() `, while +:meth:`fetch_trials() ` will only fetch the +trials of the specific experiment. + +.. code-block:: python + + import pprint + from orion.core.io.evc_builder import EVCBuilder + + experiment = EVCBuilder().build_view_from( + {"name": "orion-tutorial-with-momentum"}) + + print(experiment.name) + pprint.pprint(experiment.stats) + + parent_experiment = experiment.node.parent.item + print(parent_experiment.name) + pprint.pprint(parent_experiment.stats) + + for child in experiment.node.children: + child_experiment = child.item + print(child_experiment.name) + pprint.pprint(child_experiment.stats) diff --git a/docs/src/user/library/results.rst b/docs/src/user/library/results.rst new file mode 100644 index 000000000..7c23d4453 --- /dev/null +++ b/docs/src/user/library/results.rst @@ -0,0 +1,68 @@ +Results +------- + +You can fetch experiments and trials using python code. There is no need to understand the +specific database backend used (such as MongoDB) since you can fetch results using the +:class:`orion.core.worker.experiment.Experiment` object. +The class :class:`orion.core.io.experiment_builder.ExperimentBuilder` +provides simple methods to fetch experiments +using their unique names. You do not need to explicitly open a connection to the database since it +will automatically infer its configuration from the global configuration file as when calling Oríon +in commandline. Otherwise you can pass other arguments to +:meth:`ExperimentBuilder().build_view_from() `. +using the same dictionary structure as in the configuration file. + +.. code-block:: python + + # Database automatically inferred + ExperimentBuilder().build_view_from( + {"name": "orion-tutorial"}) + + # Database manually set + ExperimentBuilder().build_view_from( + {"name": "orion-tutorial", + "dataset": { + "type": "mongodb", + "name": "myother", + "host": "localhost"}}) + +For a complete example, here's how you can fetch trials from a given experiment. + +.. code-block:: python + + import datetime + import pprint + + from orion.core.io.experiment_builder import ExperimentBuilder + + some_datetime = datetime.datetime.now() - datetime.timedelta(minutes=5) + + experiment = ExperimentBuilder().build_view_from({"name": "orion-tutorial"}) + + pprint.pprint(experiment.stats) + + for trial in experiment.fetch_trials({}): + print(trial.id) + print(trial.status) + print(trial.params) + print(trial.results) + print() + pprint.pprint(trial.to_dict()) + + # Fetches only the completed trials + for trial in experiment.fetch_trials({'status': 'completed'}): + print(trial.objective) + + # Fetches only the most recent trials using mongodb-like syntax + for trial in experiment.fetch_trials({'end_time': {'$gte': some_datetime}}): + print(trial.id) + print(trial.end_time) + +You can pass queries to :meth:`fetch_trials() `, +where queries can be a simple dictionary of values to +match like ``{'status': 'completed'}``, in which case it would return all trials where +``trial.status == 'completed'``, or they can be more complex using `mongodb-like syntax`_. +You can find more in formation on the object :class:`Experiment ` in the code +reference section. + +.. _`mongodb-like syntax`: https://docs.mongodb.com/manual/reference/method/db.collection.find/ diff --git a/docs/src/user/pytorch.rst b/docs/src/user/pytorch.rst index 55e619845..82243e989 100644 --- a/docs/src/user/pytorch.rst +++ b/docs/src/user/pytorch.rst @@ -206,7 +206,6 @@ value 1. Results ======= - When an experiment reaches its termination criterion, basically ``max-trials``, it will print the following statistics if Oríon is called with ``-v`` or ``-vv``. @@ -225,65 +224,5 @@ following statistics if Oríon is called with ``-v`` or ``-vv``. =============== [{'name': '/lr', 'type': 'real', 'value': 0.012027705702344259}] - -You can also fetch the results using python code. You do not need to understand MongoDB since -you can fetch results using the ``Experiment`` object. The class `ExperimentBuilder` provides -simple methods to fetch experiments using their unique names. You do not need to explicitly -open a connection to the database when using the `ExperimentBuilder` since it will automatically -infer its configuration from the global configuration file as when calling Oríon in commandline. -Otherwise you can pass other arguments to ``ExperimentBuilder().build_view_from()`` using the same -dictionary structure as in the configuration file. - -.. code-block:: python - - # Database automatically inferred - ExperimentBuilder().build_view_from( - {"name": "orion-tutorial"}) - - # Database manually set - ExperimentBuilder().build_view_from( - {"name": "orion-tutorial", - "dataset": { - "type": "mongodb", - "name": "myother", - "host": "localhost"}}) - -For a complete example, here's how you can fetch trials from a given experiment. - -.. code-block:: python - - import datetime - import pprint - - from orion.core.io.experiment_builder import ExperimentBuilder - - some_datetime = datetime.datetime.now() - datetime.timedelta(minutes=5) - - experiment = ExperimentBuilder().build_view_from({"name": "orion-tutorial"}) - - pprint.pprint(experiment.stats) - - for trial in experiment.fetch_trials({}): - print(trial.id) - print(trial.status) - print(trial.params) - print(trial.results) - print() - pprint.pprint(trial.to_dict()) - - # Fetches only the completed trials - for trial in experiment.fetch_trials({'status': 'completed'}): - print(trial.objective) - - # Fetches only the most recent trials using mongodb-like syntax - for trial in experiment.fetch_trials({'end_time': {'$gte': some_datetime}}): - print(trial.id) - print(trial.end_time) - -You can pass queries to ``fetch_trials()``, where queries can be a simple dictionary of values to -match like ``{'status': 'completed'}``, in which case it would return all trials where -``trial.status == 'completed'``, or they can be more complex using `mongodb-like syntax`_. - -.. _`mongodb-like syntax`: https://docs.mongodb.com/manual/reference/method/db.collection.find/ - - +These results can be printed in terminal latter on with the command :ref:`info ` or +fetched using the :ref:`library API `. From 4ec4071826db94330a2bdb0099ad0d3edbaa6bb2 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Thu, 18 Jul 2019 15:11:43 -0400 Subject: [PATCH 39/50] Add doc for info command --- docs/src/user/cli/info.rst | 59 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/docs/src/user/cli/info.rst b/docs/src/user/cli/info.rst index c9bf1c305..2f20af75e 100644 --- a/docs/src/user/cli/info.rst +++ b/docs/src/user/cli/info.rst @@ -1,2 +1,61 @@ ``info`` Detailed informations about experiments ------------------------------------------------ + +This commands gives a detailed description of a given experiment. +Here is an example of all the sections provided by the command. + +.. code-block:: console + + orion info orion-tutorial + +.. code-block:: bash + + Commandline + =========== + --lr~loguniform(1e-5, 1.0) + + + Config + ====== + pool size: 1 + max trials: inf + + + Algorithm + ========= + random: + seed: None + + + Space + ===== + /lr: reciprocal(1e-05, 1.0) + + + Meta-data + ========= + user: + datetime: 2019-07-18 18:57:57.840000 + orion version: v0.1.5 + + + Parent experiment + ================= + root: + parent: + adapter: + + + Stats + ===== + trials completed: 1 + best trial: + /lr: 0.03543491957849911 + best evaluation: 0.9626 + start time: 2019-07-18 18:57:57.840000 + finish time: 2019-07-18 18:58:08.565000 + duration: 0:00:10.725000 + + +The last section contains information about the best trial so far, providing its +hyper-parameter values and the corresponding objective From d43574c9b26265214cd6f1eb696077af0e583802 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Thu, 18 Jul 2019 15:28:56 -0400 Subject: [PATCH 40/50] Fix doc8 and typos --- docs/src/user/cli/info.rst | 4 ++-- docs/src/user/evc.rst | 3 ++- docs/src/user/library/evc_results.rst | 2 +- docs/src/user/library/results.rst | 14 +++++++++----- docs/src/user/pytorch.rst | 2 +- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/src/user/cli/info.rst b/docs/src/user/cli/info.rst index 2f20af75e..130aacfe1 100644 --- a/docs/src/user/cli/info.rst +++ b/docs/src/user/cli/info.rst @@ -1,7 +1,7 @@ ``info`` Detailed informations about experiments ------------------------------------------------ -This commands gives a detailed description of a given experiment. +This command gives a detailed description of a given experiment. Here is an example of all the sections provided by the command. .. code-block:: console @@ -58,4 +58,4 @@ Here is an example of all the sections provided by the command. The last section contains information about the best trial so far, providing its -hyper-parameter values and the corresponding objective +hyperparameter values and the corresponding objective. diff --git a/docs/src/user/evc.rst b/docs/src/user/evc.rst index d895425c9..529393369 100644 --- a/docs/src/user/evc.rst +++ b/docs/src/user/evc.rst @@ -171,4 +171,5 @@ Iterative Results .. note:: TODO: Set link when status command is documented. Results from the EVC tree can be queried in aggregation with the command -`status --collapse `_ or fetching using the :ref:`library API `. +`status --collapse `_ or fetching using the +:ref:`library API `. diff --git a/docs/src/user/library/evc_results.rst b/docs/src/user/library/evc_results.rst index cfe111843..e9e4ac2f9 100644 --- a/docs/src/user/library/evc_results.rst +++ b/docs/src/user/library/evc_results.rst @@ -8,7 +8,7 @@ as described in previous section. The only difference is we need to use :class:`EVCBuilder ` instead of :class:`ExperimentBuilder `. The :class:`EVCBuilder ` will connect the experiment -to the EVC tree, accessible through the +to the EVC tree, accessible through the :attr:`node ` attribute. All trials of the tree can be fetched with :meth:`fetch_trials_tree() `, while diff --git a/docs/src/user/library/results.rst b/docs/src/user/library/results.rst index 7c23d4453..a35976798 100644 --- a/docs/src/user/library/results.rst +++ b/docs/src/user/library/results.rst @@ -1,7 +1,7 @@ Results ------- -You can fetch experiments and trials using python code. There is no need to understand the +You can fetch experiments and trials using python code. There is no need to understand the specific database backend used (such as MongoDB) since you can fetch results using the :class:`orion.core.worker.experiment.Experiment` object. The class :class:`orion.core.io.experiment_builder.ExperimentBuilder` @@ -9,7 +9,9 @@ provides simple methods to fetch experiments using their unique names. You do not need to explicitly open a connection to the database since it will automatically infer its configuration from the global configuration file as when calling Oríon in commandline. Otherwise you can pass other arguments to -:meth:`ExperimentBuilder().build_view_from() `. +:meth:`ExperimentBuilder().build_view_from() \ +`. + using the same dictionary structure as in the configuration file. .. code-block:: python @@ -26,7 +28,7 @@ using the same dictionary structure as in the configuration file. "name": "myother", "host": "localhost"}}) -For a complete example, here's how you can fetch trials from a given experiment. +For a complete example, here is how you can fetch trials from a given experiment. .. code-block:: python @@ -58,11 +60,13 @@ For a complete example, here's how you can fetch trials from a given experiment. print(trial.id) print(trial.end_time) -You can pass queries to :meth:`fetch_trials() `, +You can pass queries to +:meth:`fetch_trials() `, where queries can be a simple dictionary of values to match like ``{'status': 'completed'}``, in which case it would return all trials where ``trial.status == 'completed'``, or they can be more complex using `mongodb-like syntax`_. -You can find more in formation on the object :class:`Experiment ` in the code +You can find more information on the object +:class:`Experiment ` in the code reference section. .. _`mongodb-like syntax`: https://docs.mongodb.com/manual/reference/method/db.collection.find/ diff --git a/docs/src/user/pytorch.rst b/docs/src/user/pytorch.rst index 82243e989..6fae55413 100644 --- a/docs/src/user/pytorch.rst +++ b/docs/src/user/pytorch.rst @@ -224,5 +224,5 @@ following statistics if Oríon is called with ``-v`` or ``-vv``. =============== [{'name': '/lr', 'type': 'real', 'value': 0.012027705702344259}] -These results can be printed in terminal latter on with the command :ref:`info ` or +These results can be printed in terminal later on with the command :ref:`info ` or fetched using the :ref:`library API `. From fe277b6c90fa262cbd536eac7e9113d0c4588d95 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Thu, 18 Jul 2019 15:37:46 -0400 Subject: [PATCH 41/50] Add missing monitoring file... -_- --- docs/src/user/monitoring.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docs/src/user/monitoring.rst diff --git a/docs/src/user/monitoring.rst b/docs/src/user/monitoring.rst new file mode 100644 index 000000000..fbb96ea48 --- /dev/null +++ b/docs/src/user/monitoring.rst @@ -0,0 +1,22 @@ +********** +Monitoring +********** + +Oríon provides command line tools to help monitoring status of experiments and inspect +available experiments in a database. The library API can also be used to query the database +within python code. + +Commands for terminal +===================== + +.. _cli-info: +.. include:: cli/info.rst + +Library API +=========== + +.. _library-api-results: +.. include:: library/results.rst + +.. _library-api-evc-results: +.. include:: library/evc_results.rst From 5567050883c42f659bb76cf46a40f534f93d82c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Fri, 19 Jul 2019 15:22:11 -0400 Subject: [PATCH 42/50] Make sure higher shape points are not `ndarray` objects Why: Ndarray are not serializable by bson. --- src/orion/algo/space.py | 49 ------------------ src/orion/core/utils/format_trials.py | 6 +-- src/orion/core/utils/points.py | 74 +++++++++++++++++++++++++++ src/orion/core/worker/trial.py | 11 +++- tests/unittests/core/test_trial.py | 10 ++++ 5 files changed, 94 insertions(+), 56 deletions(-) create mode 100644 src/orion/core/utils/points.py diff --git a/src/orion/algo/space.py b/src/orion/algo/space.py index c4fdd0629..ffaed026c 100644 --- a/src/orion/algo/space.py +++ b/src/orion/algo/space.py @@ -818,52 +818,3 @@ def __repr__(self): """Represent as a string the space and the dimensions it contains.""" dims = list(self.values()) return "Space([{}])".format(',\n '.join(map(str, dims))) - - -def pack_point(point, space): - """Take a list of points and pack it appropriately as a point from `space`. - - :param point: array-like or list of numbers - :param space: problem's parameter definition, - instance of `orion.algo.space.Space` - - .. note:: It works only if dimensions included in `space` have 0D or 1D shape. - - :returns: list of numbers or tuples - """ - packed = [] - idx = 0 - for dim in space.values(): - shape = dim.shape - if shape: - assert len(shape) == 1 - next_idx = idx + shape[0] - packed.append(tuple(point[idx:next_idx])) - idx = next_idx - else: - packed.append(point[idx]) - idx += 1 - assert packed in space - return packed - - -def unpack_point(point, space): - """Flatten `point` in `space` and convert it to a 1D `numpy.ndarray`. - - :param point: list of number or tuples, in `space` - :param space: problem's parameter definition, - instance of `orion.algo.space.Space` - - .. note:: It works only if dimensions included in `space` have 0D or 1D shape. - - :returns: a list of float numbers - """ - unpacked = [] - for subpoint, dim in zip(point, space.values()): - shape = dim.shape - if shape: - assert len(shape) == 1 - unpacked.extend(subpoint) - else: - unpacked.append(subpoint) - return unpacked diff --git a/src/orion/core/utils/format_trials.py b/src/orion/core/utils/format_trials.py index 410c68c39..0c05b43bc 100644 --- a/src/orion/core/utils/format_trials.py +++ b/src/orion/core/utils/format_trials.py @@ -36,14 +36,10 @@ def tuple_to_trial(data, space): assert len(data) == len(space) params = [] for i, dim in enumerate(space.values()): - try: - datum = data[i].tolist() # if it is numpy.ndarray - except AttributeError: - datum = data[i] params.append(dict( name=dim.name, type=dim.type, - value=datum + value=data[i] )) return Trial(params=params) diff --git a/src/orion/core/utils/points.py b/src/orion/core/utils/points.py new file mode 100644 index 000000000..777094ea3 --- /dev/null +++ b/src/orion/core/utils/points.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +""" +:mod:`orion.core.utils.points` -- Utility functions for manipulating trial points +================================================================================= + +.. module:: points + :platform: Unix + :synopsis: Conversion functions between higher shape points and lists. + +""" + + +def regroup_dims(point, space): + """Take a list of items representing a point and regroup them appropriately as + a point from `space`. + + Parameters + ---------- + point: array + Points to be regrouped. + space: `orion.algo.space.Space` + The optimization space. + + Returns + ------- + list or tuple + + """ + regrouped = [] + idx = 0 + + for dimension in space.values(): + shape = dimension.shape + if shape: + assert len(shape) == 1 + next_dim = idx + shape[0] + regrouped.append(tuple(point[idx:next_dim])) + idx = next_dim + else: + regrouped.append(point[idx]) + idx += 1 + + if regrouped not in space: + raise AttributeError("The point {} is not a valid point of space {}".format(point, space)) + + return regrouped + + +def flatten_dims(point, space): + """Flatten `point` in `space` and convert it to a list. + + Parameters + ---------- + point: array + Points to be regrouped. + space: `orion.algo.space.Space` + The optimization space. + + Returns + ------- + list + + """ + flattened = [] + + for subpoint, dimension in zip(point, space.values()): + shape = dimension.shape + if shape: + assert len(shape) == 1 + flattened.extend(subpoint) + else: + flattened.append(subpoint) + + return flattened diff --git a/src/orion/core/worker/trial.py b/src/orion/core/worker/trial.py index 0775dfa40..0a5548ae5 100644 --- a/src/orion/core/worker/trial.py +++ b/src/orion/core/worker/trial.py @@ -15,7 +15,7 @@ log = logging.getLogger(__name__) -class Trial(object): +class Trial: """Represents an entry in database/trials collection. Attributes @@ -74,7 +74,7 @@ def build(cls, trial_entries): trials.append(cls(**entry)) return trials - class Value(object): + class Value: """Container for a value object. Attributes @@ -99,6 +99,13 @@ def __init__(self, **kwargs): for attrname, value in kwargs.items(): setattr(self, attrname, value) + self._ensure_no_ndarray() + + def _ensure_no_ndarray(self): + """Make sure the current value is not a `numpy.ndarray`.""" + if hasattr(self, 'value') and hasattr(self.value, 'tolist'): + self.value = self.value.tolist() + def to_dict(self): """Needed to be able to convert `Value` to `dict` form.""" ret = dict( diff --git a/tests/unittests/core/test_trial.py b/tests/unittests/core/test_trial.py index 3fb58617e..d1fee64d2 100644 --- a/tests/unittests/core/test_trial.py +++ b/tests/unittests/core/test_trial.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- """Collection of tests for :mod:`orion.core.worker.trial`.""" +import numpy import pytest from orion.core.worker.trial import Trial @@ -39,6 +40,15 @@ def test_init_full(self, exp_config): assert list(map(lambda x: x.to_dict(), t.params)) == exp_config[1][1]['params'] assert t.working_dir is None + def test_higher_shapes_not_ndarray(self): + """Test that `numpy.ndarray` values are converted to list.""" + value = numpy.zeros([3]) + expected = value.tolist() + params = [dict(name='/x', type='real', value=value)] + trial = Trial(params=params) + + assert trial.params[0].value == expected + def test_bad_access(self): """Other than `Trial.__slots__` are not allowed.""" t = Trial() From 06efa07b47f65a241de5834c93d8b739966998be Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Fri, 19 Jul 2019 18:52:34 -0400 Subject: [PATCH 43/50] Add documentation for Search Space (#227) --- docs/src/conf.py | 2 +- docs/src/index.rst | 1 + docs/src/user/algorithms.rst | 2 + docs/src/user/evc.rst | 6 +- docs/src/user/searchspace.rst | 265 ++++++++++++++++++++++++++++++++++ 5 files changed, 273 insertions(+), 3 deletions(-) create mode 100644 docs/src/user/searchspace.rst diff --git a/docs/src/conf.py b/docs/src/conf.py index 7f7b6fe1b..c2a937b49 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -121,7 +121,7 @@ # 'style_external_links': False, # 'vcs_pageview_mode': '', # Toc options - 'collapse_navigation': True, + 'collapse_navigation': False, 'sticky_navigation': True, 'navigation_depth': 4, # 'includehidden': False, diff --git a/docs/src/index.rst b/docs/src/index.rst index 8061f612c..190648287 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -16,6 +16,7 @@ user/pytorch user/monitoring + user/searchspace user/algorithms user/evc diff --git a/docs/src/user/algorithms.rst b/docs/src/user/algorithms.rst index 480e117c7..e9776fc19 100644 --- a/docs/src/user/algorithms.rst +++ b/docs/src/user/algorithms.rst @@ -46,6 +46,8 @@ Configuration Seed for the random number generator used to sample new trials. Default is ``None``. +.. _ASHA: + ASHA ---- diff --git a/docs/src/user/evc.rst b/docs/src/user/evc.rst index 529393369..db2d97f56 100644 --- a/docs/src/user/evc.rst +++ b/docs/src/user/evc.rst @@ -1,6 +1,8 @@ -**************************** +.. _EVC system: + +************************** Experiment Version Control -**************************** +************************** Oríon comes with an Experiment Version Control (EVC) system that makes it possible to reuse results from your previous experiments in a given project for the current one. This means a new experiment diff --git a/docs/src/user/searchspace.rst b/docs/src/user/searchspace.rst new file mode 100644 index 000000000..a93ba8d35 --- /dev/null +++ b/docs/src/user/searchspace.rst @@ -0,0 +1,265 @@ +************ +Search Space +************ + +The search space is defined by the name of the hyperparameters to optimize and their corresponding +distribution priors. These priors are used by the optimization algorithms to sample values or adjust +the exploration. + +Priors +====== + +We support all of `scipy distributions`_ out of the box. With the exception of the functions +outlined below, every distribution in ``scipy.stats`` can be used using the same original function +signature. + +``uniform(low, high)`` +---------------------- + +The uniform distribution is redefining scipy's function signature as +``uniform(low, high)`` instead ``uniform(low, interval)``. This is to uniformize the interface with +numpy and python's builtin ``random.uniform``. + +``loguniform(low, high)`` +------------------------- + +A logarithmic version of the uniform distribution. + +``normal(loc, scale)`` +---------------------- + +A wrapper for ``scipy.stats.norm``. + +``gaussian(loc, scale)`` +------------------------ + +An additional wrapper for ``scipy.stats.norm``. + +``choices(options)`` +-------------------- + +``options`` may be a ``list`` of builtin python objects or a ``dict`` of builtin python objects with +their corresponding probabilities. When passing a ``list``, each object has an equal probability of +being sampled ``choices(['all', 'equally', 'likely'])``. +To give different probabilities: ``choices({'likely': 0.8, 'unlikely': 0.2, 'impossible': 0})``. +You can pass strings, integers and floats, and all mixed together if needed: +``choices([1.0, 2, 'three'])``. + +.. _prior-fidelity: + +``fidelity()`` +-------------- + +This prior is a special placeholder to define a ``Fidelity`` dimension. The algorithms will not use +this prior to sample, but rather for multi-fidelity optimization. For an example of an algorithm +using multi-fidelity, you can look at the documentation of :ref:`ASHA`. + +.. _scipy distributions: https://docs.scipy.org/doc/scipy/reference/stats.html + +Dimension Types +=============== + +.. py:currentmodule:: orion.algo.space + +The dimensions are casted to special types according to their prior. This is critical to +either allow algorithms to leverage type information +(ex: some algorithms works better on integers) or automatically transform trial types +to make them compatible with specific algorithms +(ex: some algorithms cannot work on categorical values). + +Real +---- + +All continous priors are automatically casted to :class:`Real`. + +.. _integer-dim: + +Integer +------- + +Discrete distributions of scipy are automatically casted to :class:`Integer`. All other +distributions can be casted to :class:`Integer` by setting ``discrete=True`` +(ex: ``uniform(low, high, discrete=True)``). + +.. warning:: + + We recommend using continous priors with ``discrete=True`` + as there is an issue with scipy discrete distribution because of incorrect interval. Issue + is documented + `here `_. + +Categorical +----------- + +Distribition of k possible categories, with no ordinal relationship. Only the prior +``choices(options)`` is casted to :class:`Categorical`. + +Fidelity +-------- + +Special placeholder to represent a fidelity dimension. Only the prior +:ref:`prior-fidelity` is casted to :class:`Fidelity`. + +Special arguments +================= + +``discrete`` +------------ + +ex: ``unifor(0, 10, discrete=True)`` + +Argument to cast a continous distribution into :ref:`integer-dim`. Defaults to ``False``. + +``default_value`` +----------------- + +ex: ``unifor(0, 10, default_value=5)`` + +Dimensions can be set to a default value so that commandline call `insert` can support insertion +without specifing this hyperparameter, assigning it the default value. This is also usefull in when +using the :ref:`EVC system`, so that experiments where an hyperparameter is deleted or added can +adapt trials from other experiments by using the default value. + +``shape`` +--------- + +ex: ``unifor(0, 10, shape=2)`` + +Some hyper-parameters may have multiple dimensions. This can be set using ``shape`` + +Configuration +============= + +You can configure the search space of your experiment on the commandline call directly or +in a configuration file used by your script. + +Commandline +----------- + +Any argument in commandline with the form ``--arg~aprior(some, args)`` will be detected as a search +space dimension by Oríon. You can also use the verbose format ``--arg 'orion~aprior(some, args)'``. +Note that some shells may not play nicely with the parenthesis. You can format your command in the +following way to avoid this problem ``--arg~'aprior(some, args)'``. + +Configuration file +------------------- + +You can use configuration files to define search space with placeholder +``'orion~dist(*args, **kwargs)'`` in yaml and json files or +``name~dist(*args, **kwargs)`` in any other text-based file. +For now Oríon can only recognize the +configuration file if it is passed with the argument ``--config`` to the user script. This should +not be confused with the argument ``--config`` of ``orion hunt``, which is the configuration of +Oríon. We are here referring the configuration of the user script, represented with +``my_script_config.txt`` in the following example. + +.. code-block:: console + + orion hunt --config my_orion_config.yaml ./my_script --config my_script_config.txt + +Here is an example of a configuration file with yaml + +.. code-block:: yaml + + lr: 'orion~loguniform(1e-5, 1.0)' + model: + activation: "orion~choices(['sigmoid', 'relu'])" + hiddens: 'orion~randint(100, 1000)' + +Here is another example with json + +.. code-block:: json + + { + "lr": "orion~loguniform(1e-5, 1.0)" + "model": { + "activation": "orion~choices(['sigmoid', 'relu'])" + "hiddens": "orion~randint(100, 1000)" + } + } + +And here is an example with python! Note that for other files than for json and yaml, the +placeholders must be defined as ``name~dist(*args, **kwargs)``. Also, note that the code cannot be executed as is, +but once Oríon makes the substitution it will. + +.. code-block:: python + + def my_config(): + lr = lr~loguniform(1e-5, 1.0) + activations = model/activations~choices(['sigmoid', 'relu']) + nhiddens = model/hiddens~randint(100, 1000) + + layers = [] + for layer in range(model/nlayers~randint(3, 10)): + nhiddens /= 2 + layers.append(nhiddens) + + return lr, layers + +Oríon could generate a script like this one for instance. + +.. code-block:: python + + def my_config(): + lr = 0.001 + activations = 'relu' + nhiddens = 100 + + layers = [] + for layer in range(4): + nhiddens /= 2 + layers.append(nhiddens) + + return lr, layers + +When a trial is executed, a copy of the configuration file is created inside ``trial.working_dir`` +and the corresponding path is passed to the user script instead of the original path. + +Notes +====== + +Tranformations +-------------- + +Some algorithms only support limited types of dimensions. In such case, these algorithms define +the type required, and then a wrapper transforms the space to make it compatible. + +Real +~~~~ + +- :class:`Integer` are casted to :class:`Real`. +- :class:`Categorical` are casted to :class:`Integer` (low=0, high=number of categories) + and then to one-hot (:class:`Real` with space=number of categories) + to break ordinal relationship. (probabilities are lost if defined) + +Integer +~~~~~~~ + +- :class:`Real` are quantized to :class:`Integer`. +- :class:`Categorical` are casted to :class:`Integer` (low=0, high=number of categories). + (probabilities are lost if defined) + +Conditional dependencies +------------------------ + +There is currently no support for conditional dependencies between dimensions. +Conditional dependencies arises in situations where some hyperparameter defines which algorithm to +use and each algorithm have its own +set of different hyperparameter. We plan to support this in the future by replacing our current +:class:`Space` implementation by `ConfigSpace`_. This should not change the current interface and +only add more special arguments. You can see the state of our plan in our `Roadmap`_. + +.. _ConfigSpace: https://automl.github.io/ConfigSpace/master/ +.. _Roadmap: https://github.com/Epistimio/orion/blob/master/ROADMAP.md + +References +========== + +- :class:`orion.core.io.space_builder.DimensionBuilder` +- :class:`orion.core.io.space_builder.SpaceBuilder` +- :class:`orion.algo.space.Space` +- :class:`orion.algo.space.Dimension` +- :class:`orion.algo.space.Real` +- :class:`orion.algo.space.Integer` +- :class:`orion.algo.space.Categorical` +- :class:`orion.algo.space.Fidelity` From afdfb3d6e9066e3a81fddd7a9fd353038e6f22e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Fri, 19 Jul 2019 19:43:16 -0400 Subject: [PATCH 44/50] Change status --recursive to --collapse Why: The name --recursive was confusing for what it was intended to be. The default behavior is to print the EVC tree hierarchically, and --collapse will aggregate results of the EVC in the parent experiment. Note: (datcorno) Some fixtures were assigning default values to the x parameter which lead to conflicts when running `init_only`. Since the `x` parameter is always inside the parent experiment, the default value has been removed. There was a duplicate function that has also been removed. --- src/orion/core/cli/status.py | 63 +- tests/functional/commands/conftest.py | 13 +- .../commands/test_status_command.py | 625 +++++++++++------- 3 files changed, 441 insertions(+), 260 deletions(-) diff --git a/src/orion/core/cli/status.py b/src/orion/core/cli/status.py index d4d2c726c..bc618d48d 100644 --- a/src/orion/core/cli/status.py +++ b/src/orion/core/cli/status.py @@ -33,9 +33,9 @@ def add_subparser(parser): help="Show all trials line by line. Otherwise, they are all aggregated by status") status_parser.add_argument( - '-r', '--recursive', action="store_true", - help="Divide trials per experiments hierarchically. Otherwise they are all print on the \ - same tab level.") + '-C', '--collapse', action="store_true", + help=("Aggregate together results of all child experiments. Otherwise they are all " + "printed hierarchically")) status_parser.set_defaults(func=main) @@ -50,12 +50,15 @@ def main(args): experiments = get_experiments(args) - if args.get('recursive'): - for exp in filter(lambda e: e.refers['parent_id'] is None, experiments): + if args.get('name'): + print_status(experiments[0], all_trials=args.get('all'), collapse=args.get('collapse')) + return + + for exp in filter(lambda e: e.refers.get('parent_id') is None, experiments): + if args.get('collapse'): + print_status(exp, all_trials=args.get('all'), collapse=True) + else: print_status_recursively(exp, all_trials=args.get('all')) - else: - for exp in experiments: - print_status(exp, all_trials=args.get('all')) def get_experiments(args): @@ -92,7 +95,7 @@ def print_status_recursively(exp, depth=0, **kwargs): print_status_recursively(child.item, depth + 1, **kwargs) -def print_status(exp, offset=0, all_trials=False): +def print_status(exp, offset=0, all_trials=False, collapse=False): """Print the status of the current experiment. Parameters @@ -101,42 +104,49 @@ def print_status(exp, offset=0, all_trials=False): The number of tabs to the right this experiment is. all_trials: bool, optional Print all trials individually + collapse: bool, optional + Fetch trials for entire EVCTree. Defaults to False. """ + if collapse: + trials = exp.fetch_trials_tree({}) + else: + trials = exp.fetch_trials({}) + + print(" " * offset, exp.name, sep="") + print(" " * offset, "=" * len(exp.name), sep="") + if all_trials: - print_all_trials(exp, offset=offset) + print_all_trials(trials, offset=offset) else: - print_summary(exp, offset=offset) + print_summary(trials, offset=offset) -def print_summary(exp, offset=0): +def print_summary(trials, offset=0): """Print a summary of the current experiment. Parameters ---------- + trials: list + Trials to summarize. offset: int, optional The number of tabs to the right this experiment is. """ status_dict = collections.defaultdict(list) - name = exp.name - trials = exp.fetch_trials({}) for trial in trials: status_dict[trial.status].append(trial) - print(" " * offset, name, sep="") - print(" " * offset, "=" * len(name), sep="") - headers = ['status', 'quantity'] lines = [] - for status, trials in sorted(status_dict.items()): - line = [status, len(trials)] + for status, c_trials in sorted(status_dict.items()): + line = [status, len(c_trials)] - if trials[0].objective: - headers.append('min {}'.format(trials[0].objective.name)) - line.append(min(trial.objective.value for trial in trials)) + if c_trials[0].objective: + headers.append('min {}'.format(c_trials[0].objective.name)) + line.append(min(trial.objective.value for trial in c_trials)) lines.append(line) @@ -150,20 +160,17 @@ def print_summary(exp, offset=0): print("\n") -def print_all_trials(exp, offset=0): +def print_all_trials(trials, offset=0): """Print all trials of the current experiment individually. Parameters ---------- + trials: list + Trials to list in terminal. offset: int, optional The number of tabs to the right this experiment is. """ - name = exp.name - trials = exp.fetch_trials({}) - - print(" " * offset, name, sep="") - print(" " * offset, "=" * len(name), sep="") headers = ['id', 'status', 'best objective'] lines = [] diff --git a/tests/functional/commands/conftest.py b/tests/functional/commands/conftest.py index 2eb60b3ea..1ac20d7ea 100644 --- a/tests/functional/commands/conftest.py +++ b/tests/functional/commands/conftest.py @@ -157,7 +157,7 @@ def single_without_success(one_experiment): exp = ExperimentBuilder().build_from({'name': 'test_single_exp'}) x = {'name': '/x', 'type': 'real'} - x_value = 1 + x_value = 0 for status in statuses: x['value'] = x_value trial = Trial(experiment=exp.id, params=[x], status=status) @@ -170,7 +170,7 @@ def single_with_trials(single_without_success): """Create an experiment with all types of trials.""" exp = ExperimentBuilder().build_from({'name': 'test_single_exp'}) - x = {'name': '/x', 'type': 'real', 'value': 0} + x = {'name': '/x', 'type': 'real', 'value': 100} results = {"name": "obj", "type": "objective", "value": 0} trial = Trial(experiment=exp.id, params=[x], status='completed', results=[results]) Database().write('trials', trial.to_dict()) @@ -186,7 +186,7 @@ def two_experiments(monkeypatch, db_instance): orion.core.cli.main(['init_only', '-n', 'test_double_exp', '--branch', 'test_double_exp_child', './black_box.py', - '--x~uniform(0,1)', '--y~+uniform(0,1)']) + '--x~+uniform(0,1,default_value=0)', '--y~+uniform(0,1,default_value=0)']) ensure_deterministic_id('test_double_exp_child', db_instance) @@ -203,7 +203,7 @@ def family_with_trials(two_experiments): x['value'] = x_value y['value'] = x_value trial = Trial(experiment=exp.id, params=[x], status=status) - x['value'] = x_value * 10 + x['value'] = x_value trial2 = Trial(experiment=exp2.id, params=[x, y], status=status) x_value += 1 Database().write('trials', trial.to_dict()) @@ -236,7 +236,7 @@ def three_experiments_family(two_experiments, db_instance): """Create three experiments, one of which is the parent of the other two.""" orion.core.cli.main(['init_only', '-n', 'test_double_exp', '--branch', 'test_double_exp_child2', './black_box.py', - '--x~uniform(0,1)', '--z~+uniform(0,1)']) + '--x~+uniform(0,1,default_value=0)', '--z~+uniform(0,1,default_value=0)']) ensure_deterministic_id('test_double_exp_child2', db_instance) @@ -261,7 +261,8 @@ def three_experiments_family_branch(two_experiments, db_instance): """Create three experiments, each parent of the following one.""" orion.core.cli.main(['init_only', '-n', 'test_double_exp_child', '--branch', 'test_double_exp_grand_child', './black_box.py', - '--x~uniform(0,1)', '--y~uniform(0,1)', '--z~+uniform(0,1)']) + '--x~+uniform(0,1,default_value=0)', '--y~uniform(0,1,default_value=0)', + '--z~+uniform(0,1,default_value=0)']) ensure_deterministic_id('test_double_exp_grand_child', db_instance) diff --git a/tests/functional/commands/test_status_command.py b/tests/functional/commands/test_status_command.py index 4d09aa22c..5b7a4c426 100644 --- a/tests/functional/commands/test_status_command.py +++ b/tests/functional/commands/test_status_command.py @@ -16,7 +16,7 @@ def test_no_experiments(clean_db, monkeypatch, capsys): assert captured == "" -def test_experiment_without_trials_wout_ar(clean_db, one_experiment, capsys): +def test_experiment_without_trials_wout_ac(clean_db, one_experiment, capsys): """Test status with only one experiment and no trials.""" orion.core.cli.main(['status']) @@ -32,7 +32,7 @@ def test_experiment_without_trials_wout_ar(clean_db, one_experiment, capsys): assert captured == expected -def test_experiment_wout_success_wout_ar(clean_db, single_without_success, capsys): +def test_experiment_wout_success_wout_ac(clean_db, single_without_success, capsys): """Test status with only one experiment and no successful trial.""" orion.core.cli.main(['status']) @@ -54,7 +54,30 @@ def test_experiment_wout_success_wout_ar(clean_db, single_without_success, capsy assert captured == expected -def test_two_w_trials_wout_ar(clean_db, unrelated_with_trials, capsys): +def test_experiment_w_trials_wout_ac(clean_db, single_with_trials, capsys): + """Test status with only one experiment and all trials.""" + orion.core.cli.main(['status']) + + captured = capsys.readouterr().out + + expected = """\ +test_single_exp +=============== +status quantity min obj +----------- ---------- --------- +broken 1 +completed 1 0 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + +""" + assert captured == expected + + +def test_two_unrelated_w_trials_wout_ac(clean_db, unrelated_with_trials, capsys): """Test two unrelated experiments, with all types of trials.""" orion.core.cli.main(['status']) @@ -90,7 +113,7 @@ def test_two_w_trials_wout_ar(clean_db, unrelated_with_trials, capsys): assert captured == expected -def test_two_fam_w_trials_wout_ar(clean_db, family_with_trials, capsys): +def test_two_related_w_trials_wout_ac(clean_db, family_with_trials, capsys): """Test two related experiments, with all types of trials.""" orion.core.cli.main(['status']) @@ -109,8 +132,32 @@ def test_two_fam_w_trials_wout_ar(clean_db, family_with_trials, capsys): suspended 1 -test_double_exp_child -===================== + test_double_exp_child + ===================== + status quantity + ----------- ---------- + broken 1 + completed 1 + interrupted 1 + new 1 + reserved 1 + suspended 1 + + +""" + + assert captured == expected + + +def test_three_unrelated_wout_ac(clean_db, three_experiments_with_trials, capsys): + """Test three unrelated experiments with all types of trials.""" + orion.core.cli.main(['status']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== status quantity ----------- ---------- broken 1 @@ -121,12 +168,132 @@ def test_two_fam_w_trials_wout_ar(clean_db, family_with_trials, capsys): suspended 1 + test_double_exp_child + ===================== + status quantity + ----------- ---------- + broken 1 + completed 1 + interrupted 1 + new 1 + reserved 1 + suspended 1 + + +test_single_exp +=============== +status quantity min obj +----------- ---------- --------- +broken 1 +completed 1 0 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + """ assert captured == expected -def test_one_wout_trials_w_a_wout_r(clean_db, one_experiment, capsys): +def test_three_related_wout_ac(clean_db, three_family_with_trials, capsys): + """Test three related experiments with all types of trials.""" + orion.core.cli.main(['status']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +status quantity +----------- ---------- +broken 1 +completed 1 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + + test_double_exp_child + ===================== + status quantity + ----------- ---------- + broken 1 + completed 1 + interrupted 1 + new 1 + reserved 1 + suspended 1 + + + test_double_exp_child2 + ====================== + status quantity + ----------- ---------- + broken 1 + completed 1 + interrupted 1 + new 1 + reserved 1 + suspended 1 + + +""" + + assert captured == expected + + +def test_three_related_branch_wout_ac(clean_db, three_family_branch_with_trials, capsys): + """Test three related experiments with all types of trials.""" + orion.core.cli.main(['status']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +status quantity +----------- ---------- +broken 1 +completed 1 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + + test_double_exp_child + ===================== + status quantity + ----------- ---------- + broken 1 + completed 1 + interrupted 1 + new 1 + reserved 1 + suspended 1 + + + test_double_exp_grand_child + =========================== + status quantity + ----------- ---------- + broken 1 + completed 1 + interrupted 1 + new 1 + reserved 1 + suspended 1 + + +""" + + assert captured == expected + + +def test_one_wout_trials_w_a_wout_c(clean_db, one_experiment, capsys): """Test experiments, without trials, with --all.""" orion.core.cli.main(['status', '--all']) @@ -144,7 +311,7 @@ def test_one_wout_trials_w_a_wout_r(clean_db, one_experiment, capsys): assert captured == expected -def test_one_w_trials_w_a_wout_r(clean_db, single_with_trials, capsys): +def test_one_w_trials_w_a_wout_c(clean_db, single_with_trials, capsys): """Test experiment, with all trials, with --all.""" orion.core.cli.main(['status', '--all']) @@ -155,12 +322,12 @@ def test_one_w_trials_w_a_wout_r(clean_db, single_with_trials, capsys): =============== id status min obj -------------------------------- ----------- --------- -78d300480fc80ec5f52807fe97f65dd7 broken -7e8eade99d5fb1aa59a1985e614732bc completed 0 -ec6ee7892275400a9acbf4f4d5cd530d interrupted -507496236ff94d0f3ad332949dfea484 new -caf6afc856536f6d061676e63d14c948 reserved -2b5059fa8fdcdc01f769c31e63d93f24 suspended +ec6ee7892275400a9acbf4f4d5cd530d broken +c4c44cb46d075546824e2a32f800fece completed 0 +2b5059fa8fdcdc01f769c31e63d93f24 interrupted +7e8eade99d5fb1aa59a1985e614732bc new +507496236ff94d0f3ad332949dfea484 reserved +caf6afc856536f6d061676e63d14c948 suspended """ @@ -168,7 +335,7 @@ def test_one_w_trials_w_a_wout_r(clean_db, single_with_trials, capsys): assert captured == expected -def test_one_wout_success_w_a_wout_r(clean_db, single_without_success, capsys): +def test_one_wout_success_w_a_wout_c(clean_db, single_without_success, capsys): """Test experiment, without success, with --all.""" orion.core.cli.main(['status', '--all']) @@ -179,11 +346,11 @@ def test_one_wout_success_w_a_wout_r(clean_db, single_without_success, capsys): =============== id status -------------------------------- ----------- -78d300480fc80ec5f52807fe97f65dd7 broken -ec6ee7892275400a9acbf4f4d5cd530d interrupted -507496236ff94d0f3ad332949dfea484 new -caf6afc856536f6d061676e63d14c948 reserved -2b5059fa8fdcdc01f769c31e63d93f24 suspended +ec6ee7892275400a9acbf4f4d5cd530d broken +2b5059fa8fdcdc01f769c31e63d93f24 interrupted +7e8eade99d5fb1aa59a1985e614732bc new +507496236ff94d0f3ad332949dfea484 reserved +caf6afc856536f6d061676e63d14c948 suspended """ @@ -191,7 +358,7 @@ def test_one_wout_success_w_a_wout_r(clean_db, single_without_success, capsys): assert captured == expected -def test_two_unrelated_w_a_wout_r(clean_db, unrelated_with_trials, capsys): +def test_two_unrelated_w_a_wout_c(clean_db, unrelated_with_trials, capsys): """Test two unrelated experiments with --all.""" orion.core.cli.main(['status', '--all']) @@ -214,12 +381,12 @@ def test_two_unrelated_w_a_wout_r(clean_db, unrelated_with_trials, capsys): =============== id status min obj -------------------------------- ----------- --------- -78d300480fc80ec5f52807fe97f65dd7 broken -7e8eade99d5fb1aa59a1985e614732bc completed 0 -ec6ee7892275400a9acbf4f4d5cd530d interrupted -507496236ff94d0f3ad332949dfea484 new -caf6afc856536f6d061676e63d14c948 reserved -2b5059fa8fdcdc01f769c31e63d93f24 suspended +ec6ee7892275400a9acbf4f4d5cd530d broken +c4c44cb46d075546824e2a32f800fece completed 0 +2b5059fa8fdcdc01f769c31e63d93f24 interrupted +7e8eade99d5fb1aa59a1985e614732bc new +507496236ff94d0f3ad332949dfea484 reserved +caf6afc856536f6d061676e63d14c948 suspended """ @@ -227,7 +394,7 @@ def test_two_unrelated_w_a_wout_r(clean_db, unrelated_with_trials, capsys): assert captured == expected -def test_two_related_w_a_wout_r(clean_db, family_with_trials, capsys): +def test_two_related_w_a_wout_c(clean_db, family_with_trials, capsys): """Test two related experiments with --all.""" orion.core.cli.main(['status', '--all']) @@ -246,16 +413,64 @@ def test_two_related_w_a_wout_r(clean_db, family_with_trials, capsys): b9f1506db880645a25ad9b5d2cfa0f37 suspended -test_double_exp_child -===================== + test_double_exp_child + ===================== + id status + -------------------------------- ----------- + 45c359f1c753a10f2cfeca4073a3a7ef broken + e79761fe3fc24dcbb7850939ede84b68 completed + 69928939792d67f6fe30e9b8459be1ec interrupted + 5f4a9c92b8f7c26654b5b37ecd3d5d32 new + 58c4019fb2f92da88a0e63fafb36b3da reserved + 82f340cb9d90cbf024169926b60aeef2 suspended + + +""" + + assert captured == expected + + +def test_three_unrelated_w_a_wout_c(clean_db, three_experiments_with_trials, capsys): + """Test three unrelated experiments with --all.""" + orion.core.cli.main(['status', '--all']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== id status -------------------------------- ----------- -b55c5a82050dc30a6b0c9614b1eb05e5 broken -649e09b84128c2f8821b9225ebcc139b completed -bac4e23fae8fe316d6f763ac901569af interrupted -5f4a9c92b8f7c26654b5b37ecd3d5d32 new -c2df0712319b5e91c1b4176e961a07a7 reserved -382400953aa6e8769e11aceae9be09d7 suspended +a8f8122af9e5162e1e2328fdd5dd75db broken +ab82b1fa316de5accb4306656caa07d0 completed +c187684f7c7d9832ba953f246900462d interrupted +1497d4f27622520439c4bc132c6046b1 new +bd0999e1a3b00bf8658303b14867b30e reserved +b9f1506db880645a25ad9b5d2cfa0f37 suspended + + + test_double_exp_child + ===================== + id status + -------------------------------- ----------- + 45c359f1c753a10f2cfeca4073a3a7ef broken + e79761fe3fc24dcbb7850939ede84b68 completed + 69928939792d67f6fe30e9b8459be1ec interrupted + 5f4a9c92b8f7c26654b5b37ecd3d5d32 new + 58c4019fb2f92da88a0e63fafb36b3da reserved + 82f340cb9d90cbf024169926b60aeef2 suspended + + +test_single_exp +=============== +id status min obj +-------------------------------- ----------- --------- +ec6ee7892275400a9acbf4f4d5cd530d broken +c4c44cb46d075546824e2a32f800fece completed 0 +2b5059fa8fdcdc01f769c31e63d93f24 interrupted +7e8eade99d5fb1aa59a1985e614732bc new +507496236ff94d0f3ad332949dfea484 reserved +caf6afc856536f6d061676e63d14c948 suspended """ @@ -263,9 +478,105 @@ def test_two_related_w_a_wout_r(clean_db, family_with_trials, capsys): assert captured == expected -def test_two_unrelated_w_r_wout_a(clean_db, unrelated_with_trials, capsys): - """Test two unrelated experiments with --recursive.""" - orion.core.cli.main(['status', '--recursive']) +def test_three_related_w_a_wout_c(clean_db, three_family_with_trials, capsys): + """Test three related experiments with --all.""" + orion.core.cli.main(['status', '--all']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +id status +-------------------------------- ----------- +a8f8122af9e5162e1e2328fdd5dd75db broken +ab82b1fa316de5accb4306656caa07d0 completed +c187684f7c7d9832ba953f246900462d interrupted +1497d4f27622520439c4bc132c6046b1 new +bd0999e1a3b00bf8658303b14867b30e reserved +b9f1506db880645a25ad9b5d2cfa0f37 suspended + + + test_double_exp_child + ===================== + id status + -------------------------------- ----------- + 45c359f1c753a10f2cfeca4073a3a7ef broken + e79761fe3fc24dcbb7850939ede84b68 completed + 69928939792d67f6fe30e9b8459be1ec interrupted + 5f4a9c92b8f7c26654b5b37ecd3d5d32 new + 58c4019fb2f92da88a0e63fafb36b3da reserved + 82f340cb9d90cbf024169926b60aeef2 suspended + + + test_double_exp_child2 + ====================== + id status + -------------------------------- ----------- + d0f4aa931345bfd864201b7dd93ae667 broken + 5005c35be98025a24731d7dfdf4423de completed + c9fa9f0682a370396c8c4265c4e775dd interrupted + 3d8163138be100e37f1656b7b591179e new + 790d3c4c965e0d91ada9cbdaebe220cf reserved + 6efdb99952d5f80f55adbba9c61dc288 suspended + + +""" + + assert captured == expected + + +def test_three_related_branch_w_a_wout_c(clean_db, three_family_branch_with_trials, capsys): + """Test three related experiments in a branch with --all.""" + orion.core.cli.main(['status', '--all']) + + captured = capsys.readouterr().out + + expected = """\ +test_double_exp +=============== +id status +-------------------------------- ----------- +a8f8122af9e5162e1e2328fdd5dd75db broken +ab82b1fa316de5accb4306656caa07d0 completed +c187684f7c7d9832ba953f246900462d interrupted +1497d4f27622520439c4bc132c6046b1 new +bd0999e1a3b00bf8658303b14867b30e reserved +b9f1506db880645a25ad9b5d2cfa0f37 suspended + + + test_double_exp_child + ===================== + id status + -------------------------------- ----------- + 45c359f1c753a10f2cfeca4073a3a7ef broken + e79761fe3fc24dcbb7850939ede84b68 completed + 69928939792d67f6fe30e9b8459be1ec interrupted + 5f4a9c92b8f7c26654b5b37ecd3d5d32 new + 58c4019fb2f92da88a0e63fafb36b3da reserved + 82f340cb9d90cbf024169926b60aeef2 suspended + + + test_double_exp_grand_child + =========================== + id status + -------------------------------- ----------- + 994602c021c470989d6f392b06cb37dd broken + 24c228352de31010d8d3bf253604a82d completed + a3c8a1f4c80c094754c7217a83aae5e2 interrupted + d667f5d719ddaa4e1da2fbe568e11e46 new + a40748e487605df3ed04a5ac7154d4f6 reserved + 229622a6d7132c311b7d4c57a08ecf08 suspended + + +""" + + assert captured == expected + + +def test_two_unrelated_w_c_wout_a(clean_db, unrelated_with_trials, capsys): + """Test two unrelated experiments with --collapse.""" + orion.core.cli.main(['status', '--collapse']) captured = capsys.readouterr().out @@ -299,9 +610,9 @@ def test_two_unrelated_w_r_wout_a(clean_db, unrelated_with_trials, capsys): assert captured == expected -def test_two_related_w_r_wout_a(clean_db, family_with_trials, capsys): - """Test two related experiments with --recursive.""" - orion.core.cli.main(['status', '--recursive']) +def test_two_related_w_c_wout_a(clean_db, family_with_trials, capsys): + """Test two related experiments with --collapse.""" + orion.core.cli.main(['status', '--collapse']) captured = capsys.readouterr().out @@ -313,31 +624,19 @@ def test_two_related_w_r_wout_a(clean_db, family_with_trials, capsys): broken 1 completed 1 interrupted 1 -new 1 +new 2 reserved 1 suspended 1 - test_double_exp_child - ===================== - status quantity - ----------- ---------- - broken 1 - completed 1 - interrupted 1 - new 1 - reserved 1 - suspended 1 - - """ assert captured == expected -def test_three_unrelated_w_r_wout_a(clean_db, three_experiments_with_trials, capsys): - """Test three unrelated experiments with --recursive.""" - orion.core.cli.main(['status', '--recursive']) +def test_three_unrelated_w_c_wout_a(clean_db, three_experiments_with_trials, capsys): + """Test three unrelated experiments with --collapse.""" + orion.core.cli.main(['status', '--collapse']) captured = capsys.readouterr().out @@ -349,23 +648,11 @@ def test_three_unrelated_w_r_wout_a(clean_db, three_experiments_with_trials, cap broken 1 completed 1 interrupted 1 -new 1 +new 2 reserved 1 suspended 1 - test_double_exp_child - ===================== - status quantity - ----------- ---------- - broken 1 - completed 1 - interrupted 1 - new 1 - reserved 1 - suspended 1 - - test_single_exp =============== status quantity min obj @@ -383,9 +670,9 @@ def test_three_unrelated_w_r_wout_a(clean_db, three_experiments_with_trials, cap assert captured == expected -def test_three_related_w_r_wout_a(clean_db, three_family_with_trials, capsys): - """Test three related experiments with --recursive.""" - orion.core.cli.main(['status', '--recursive']) +def test_three_related_w_c_wout_a(clean_db, three_family_with_trials, capsys): + """Test three related experiments with --collapse.""" + orion.core.cli.main(['status', '--collapse']) captured = capsys.readouterr().out @@ -397,43 +684,19 @@ def test_three_related_w_r_wout_a(clean_db, three_family_with_trials, capsys): broken 1 completed 1 interrupted 1 -new 1 +new 3 reserved 1 suspended 1 - test_double_exp_child - ===================== - status quantity - ----------- ---------- - broken 1 - completed 1 - interrupted 1 - new 1 - reserved 1 - suspended 1 - - - test_double_exp_child2 - ====================== - status quantity - ----------- ---------- - broken 1 - completed 1 - interrupted 1 - new 1 - reserved 1 - suspended 1 - - """ assert captured == expected -def test_three_related_branch_w_r_wout_a(clean_db, three_family_branch_with_trials, capsys): - """Test three related experiments with --recursive.""" - orion.core.cli.main(['status', '--recursive']) +def test_three_related_branch_w_c_wout_a(clean_db, three_family_branch_with_trials, capsys): + """Test three related experiments with --collapse.""" + orion.core.cli.main(['status', '--collapse']) captured = capsys.readouterr().out @@ -445,43 +708,19 @@ def test_three_related_branch_w_r_wout_a(clean_db, three_family_branch_with_tria broken 1 completed 1 interrupted 1 -new 1 +new 3 reserved 1 suspended 1 - test_double_exp_child - ===================== - status quantity - ----------- ---------- - broken 1 - completed 1 - interrupted 1 - new 1 - reserved 1 - suspended 1 - - - test_double_exp_grand_child - =========================== - status quantity - ----------- ---------- - broken 1 - completed 1 - interrupted 1 - new 1 - reserved 1 - suspended 1 - - """ assert captured == expected -def test_two_unrelated_w_ar(clean_db, unrelated_with_trials, capsys): - """Test two unrelated experiments with --recursive and --all.""" - orion.core.cli.main(['status', '--recursive', '--all']) +def test_two_unrelated_w_ac(clean_db, unrelated_with_trials, capsys): + """Test two unrelated experiments with --collapse and --all.""" + orion.core.cli.main(['status', '--collapse', '--all']) captured = capsys.readouterr().out @@ -502,12 +741,12 @@ def test_two_unrelated_w_ar(clean_db, unrelated_with_trials, capsys): =============== id status min obj -------------------------------- ----------- --------- -78d300480fc80ec5f52807fe97f65dd7 broken -7e8eade99d5fb1aa59a1985e614732bc completed 0 -ec6ee7892275400a9acbf4f4d5cd530d interrupted -507496236ff94d0f3ad332949dfea484 new -caf6afc856536f6d061676e63d14c948 reserved -2b5059fa8fdcdc01f769c31e63d93f24 suspended +ec6ee7892275400a9acbf4f4d5cd530d broken +c4c44cb46d075546824e2a32f800fece completed 0 +2b5059fa8fdcdc01f769c31e63d93f24 interrupted +7e8eade99d5fb1aa59a1985e614732bc new +507496236ff94d0f3ad332949dfea484 reserved +caf6afc856536f6d061676e63d14c948 suspended """ @@ -515,9 +754,9 @@ def test_two_unrelated_w_ar(clean_db, unrelated_with_trials, capsys): assert captured == expected -def test_two_related_w_ar(clean_db, family_with_trials, capsys): - """Test two related experiments with --recursive and --all.""" - orion.core.cli.main(['status', '--recursive', '--all']) +def test_two_related_w_ac(clean_db, family_with_trials, capsys): + """Test two related experiments with --collapse and --all.""" + orion.core.cli.main(['status', '--collapse', '--all']) captured = capsys.readouterr().out @@ -530,30 +769,19 @@ def test_two_related_w_ar(clean_db, family_with_trials, capsys): ab82b1fa316de5accb4306656caa07d0 completed c187684f7c7d9832ba953f246900462d interrupted 1497d4f27622520439c4bc132c6046b1 new +ad6ea2decff2f298594b948fdaea03b2 new bd0999e1a3b00bf8658303b14867b30e reserved b9f1506db880645a25ad9b5d2cfa0f37 suspended - test_double_exp_child - ===================== - id status - -------------------------------- ----------- - b55c5a82050dc30a6b0c9614b1eb05e5 broken - 649e09b84128c2f8821b9225ebcc139b completed - bac4e23fae8fe316d6f763ac901569af interrupted - 5f4a9c92b8f7c26654b5b37ecd3d5d32 new - c2df0712319b5e91c1b4176e961a07a7 reserved - 382400953aa6e8769e11aceae9be09d7 suspended - - """ assert captured == expected -def test_three_unrelated_w_ar(clean_db, three_experiments_with_trials, capsys): - """Test three unrelated experiments with --recursive and --all.""" - orion.core.cli.main(['status', '--recursive', '--all']) +def test_three_unrelated_w_ac(clean_db, three_experiments_with_trials, capsys): + """Test three unrelated experiments with --collapse and --all.""" + orion.core.cli.main(['status', '--collapse', '--all']) captured = capsys.readouterr().out @@ -566,32 +794,21 @@ def test_three_unrelated_w_ar(clean_db, three_experiments_with_trials, capsys): ab82b1fa316de5accb4306656caa07d0 completed c187684f7c7d9832ba953f246900462d interrupted 1497d4f27622520439c4bc132c6046b1 new +ad6ea2decff2f298594b948fdaea03b2 new bd0999e1a3b00bf8658303b14867b30e reserved b9f1506db880645a25ad9b5d2cfa0f37 suspended - test_double_exp_child - ===================== - id status - -------------------------------- ----------- - b55c5a82050dc30a6b0c9614b1eb05e5 broken - 649e09b84128c2f8821b9225ebcc139b completed - bac4e23fae8fe316d6f763ac901569af interrupted - 5f4a9c92b8f7c26654b5b37ecd3d5d32 new - c2df0712319b5e91c1b4176e961a07a7 reserved - 382400953aa6e8769e11aceae9be09d7 suspended - - test_single_exp =============== id status min obj -------------------------------- ----------- --------- -78d300480fc80ec5f52807fe97f65dd7 broken -7e8eade99d5fb1aa59a1985e614732bc completed 0 -ec6ee7892275400a9acbf4f4d5cd530d interrupted -507496236ff94d0f3ad332949dfea484 new -caf6afc856536f6d061676e63d14c948 reserved -2b5059fa8fdcdc01f769c31e63d93f24 suspended +ec6ee7892275400a9acbf4f4d5cd530d broken +c4c44cb46d075546824e2a32f800fece completed 0 +2b5059fa8fdcdc01f769c31e63d93f24 interrupted +7e8eade99d5fb1aa59a1985e614732bc new +507496236ff94d0f3ad332949dfea484 reserved +caf6afc856536f6d061676e63d14c948 suspended """ @@ -599,9 +816,9 @@ def test_three_unrelated_w_ar(clean_db, three_experiments_with_trials, capsys): assert captured == expected -def test_three_related_w_ar(clean_db, three_family_with_trials, capsys): - """Test three related experiments with --recursive and --all.""" - orion.core.cli.main(['status', '--recursive', '--all']) +def test_three_related_w_ac(clean_db, three_family_with_trials, capsys): + """Test three related experiments with --collapse and --all.""" + orion.core.cli.main(['status', '--collapse', '--all']) captured = capsys.readouterr().out @@ -614,42 +831,20 @@ def test_three_related_w_ar(clean_db, three_family_with_trials, capsys): ab82b1fa316de5accb4306656caa07d0 completed c187684f7c7d9832ba953f246900462d interrupted 1497d4f27622520439c4bc132c6046b1 new +ad6ea2decff2f298594b948fdaea03b2 new +f357f8c185ccab3037c65dcf721b9e71 new bd0999e1a3b00bf8658303b14867b30e reserved b9f1506db880645a25ad9b5d2cfa0f37 suspended - test_double_exp_child - ===================== - id status - -------------------------------- ----------- - b55c5a82050dc30a6b0c9614b1eb05e5 broken - 649e09b84128c2f8821b9225ebcc139b completed - bac4e23fae8fe316d6f763ac901569af interrupted - 5f4a9c92b8f7c26654b5b37ecd3d5d32 new - c2df0712319b5e91c1b4176e961a07a7 reserved - 382400953aa6e8769e11aceae9be09d7 suspended - - - test_double_exp_child2 - ====================== - id status - -------------------------------- ----------- - d0f4aa931345bfd864201b7dd93ae667 broken - 5005c35be98025a24731d7dfdf4423de completed - c9fa9f0682a370396c8c4265c4e775dd interrupted - 3d8163138be100e37f1656b7b591179e new - 790d3c4c965e0d91ada9cbdaebe220cf reserved - 6efdb99952d5f80f55adbba9c61dc288 suspended - - """ assert captured == expected -def test_three_related_branch_w_ar(clean_db, three_family_branch_with_trials, capsys): - """Test three related experiments in a branch with --recursive and --all.""" - orion.core.cli.main(['status', '--recursive', '--all']) +def test_three_related_branch_w_ac(clean_db, three_family_branch_with_trials, capsys): + """Test three related experiments in a branch with --collapse and --all.""" + orion.core.cli.main(['status', '--collapse', '--all']) captured = capsys.readouterr().out @@ -662,34 +857,12 @@ def test_three_related_branch_w_ar(clean_db, three_family_branch_with_trials, ca ab82b1fa316de5accb4306656caa07d0 completed c187684f7c7d9832ba953f246900462d interrupted 1497d4f27622520439c4bc132c6046b1 new +ad6ea2decff2f298594b948fdaea03b2 new +8f763d441db41d0f56e4e6aa40cc2321 new bd0999e1a3b00bf8658303b14867b30e reserved b9f1506db880645a25ad9b5d2cfa0f37 suspended - test_double_exp_child - ===================== - id status - -------------------------------- ----------- - b55c5a82050dc30a6b0c9614b1eb05e5 broken - 649e09b84128c2f8821b9225ebcc139b completed - bac4e23fae8fe316d6f763ac901569af interrupted - 5f4a9c92b8f7c26654b5b37ecd3d5d32 new - c2df0712319b5e91c1b4176e961a07a7 reserved - 382400953aa6e8769e11aceae9be09d7 suspended - - - test_double_exp_grand_child - =========================== - id status - -------------------------------- ----------- - 994602c021c470989d6f392b06cb37dd broken - 24c228352de31010d8d3bf253604a82d completed - a3c8a1f4c80c094754c7217a83aae5e2 interrupted - d667f5d719ddaa4e1da2fbe568e11e46 new - a40748e487605df3ed04a5ac7154d4f6 reserved - 229622a6d7132c311b7d4c57a08ecf08 suspended - - """ assert captured == expected From 94150deaa42ef21f0a2c2528a4da557483ed6444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Corneau-Tremblay?= Date: Fri, 19 Jul 2019 20:22:33 -0400 Subject: [PATCH 45/50] Add documentation for `status` and `list` (#228) --- docs/src/user/cli/list.rst | 71 ++++++++++++++ docs/src/user/cli/status.rst | 180 +++++++++++++++++++++++++++++++++++ docs/src/user/monitoring.rst | 2 + tox.ini | 4 +- 4 files changed, 255 insertions(+), 2 deletions(-) create mode 100644 docs/src/user/cli/list.rst create mode 100644 docs/src/user/cli/status.rst diff --git a/docs/src/user/cli/list.rst b/docs/src/user/cli/list.rst new file mode 100644 index 000000000..75bd0a8fc --- /dev/null +++ b/docs/src/user/cli/list.rst @@ -0,0 +1,71 @@ +``list`` Overview of all experiments +------------------------------------ + +Once you have launched a certain amount of experiments, you might start to lose track of some of +them. You might forget their name and whether or not they are the children of some other experiment +you have also forgotten. In any cases, the ``list`` command for Oríon will help you visualize the +experiments inside your database by printing them in a easy-to-understand tree-like structure. + +Configuration +~~~~~~~~~~~~~ +As per usual with Oríon, if no configuration file is provided to the ``list`` command, the default +configuration will be used. You can however provide a particular configuration file through the +usual ``-c`` or ``--config`` argument. This configuration file needs only to contain a valid +database configuration. + +Basic Usage +~~~~~~~~~~~ +The most basic usage of ``list`` is to use it without any arguments. This will simply print out +every experiments inside the database in a tree-like fashion, so that the children of the +experiments are easily identifiable. Here is a sample output to serve as an example: + +.. code-block:: console + + ┌child_2 + root┤ + └child_1┐ + └grand_child_1 + other_root + +Here, you can see we have five experiments. Two of them are roots, which mean they do not have any +parents. One of them, ``other_root``, does not have any children and so, it does not have any +branches coming out of it. On the other hand, the ``root`` experiment has multiple children, +``child_1`` and ``child_2``, which are printed on the same tree level, and one grand-child, +``grand_child_1`` which branches from ``child_1``. + +The ``--name`` argument +~~~~~~~~~~~~~~~~~~~~~~~ +The last example showed you how to print every experiments inside the database in a tree. However, +if you wish to have an overview of the tree of a single experiment, you can add the ``--name`` +argument to the call to ``list`` and only the experiment with the provided name and its children +will be shown. Here's two examples using the same set of experiments as above: + +.. code-block:: bash + + orion list --name root + +Output + +.. code-block:: console + + ┌child_2 + root┤ + └child_1┐ + └grand_child_1 + +Here, the ``other_root`` experiment is not showned because it is not inside the ``root`` experiment +tree. + +.. code-block:: bash + + orion list --name child_1 + +Output + +.. code-block:: console + + child_1┐ + └grand_child_1 + +Here, the ``root`` and ``child_2`` experiments are not present because they are not children of +``child_1``. diff --git a/docs/src/user/cli/status.rst b/docs/src/user/cli/status.rst new file mode 100644 index 000000000..bb51a1c2b --- /dev/null +++ b/docs/src/user/cli/status.rst @@ -0,0 +1,180 @@ +``status`` Overview of trials for experiments +--------------------------------------------- + +When you reach a certain amount of trials, it becomes hard to keep track of them. This is where the +``status`` command comes into play. The ``status`` command outputs the status of the different +trials inside every experiment or a specific EVC tree. It can either give you an overview of +the different trials status, i.e., the number of currently ``completed`` trials and so on, or, it +can give you a deeper view of the experiments by outlining every single trial, its status and its +objective. + +Basic Usage +~~~~~~~~~~~ +The most basic of usages is to simply run ``status`` without any other arguments, except for a local +configuration file if needed. This will then output a list of all the experiments inside your +database with the count of every type of trials related to them. If an experiment has at least one +``completed`` trial associated with it, the objective value of the best one will be printed as well. +Children experiments are printed below their parent and are indicated through a different tab +alignment than their parent, mainly, one tab further. This continues on for grand-children, and so +on and so forth. We provide an example output to illustrate this: + +.. code-block:: bash + + root + ==== + status quantity min example_objective + --------- ---------- ----------------------- + completed 5 4534.95 + + + child_1 + ======= + status quantity min example_objective + --------- ---------- ----------------------- + completed 5 4547.28 + + + other_root + ========== + status quantity min example_objective + --------- ---------- ----------------------- + completed 5 4543.73 + +The ``--all`` Argument +~~~~~~~~~~~~~~~~~~~~~~ +The basic status command combines statistics of all trials for each status. However, if you want to +see every individual trial, with its id and its status, you can use the ``--all`` +argument which will print out every single trial for each experiment with their full information. +Here is a sample output using the same experiments and trials as before: + +.. code-block:: console + + orion status --all + +.. code-block:: bash + + root + ==== + id status min example_objective + -------------------------------- --------- ----------------------- + bc222aa1705b3fe3a266fd601598ac41 completed 4555.13 + 59bf4c85305b7ba065fa770805e93cb1 completed 4653.49 + da5159d9d36ef44879e72cbe7955347c completed 4788.08 + 2c4c6409d3beeb179fce3c83b4f6d8f8 completed 4752.45 + 6a82a8a55d2241d989978cf3a7ebbba0 completed 4534.95 + + + child_1 + ======= + id status min example_objective + -------------------------------- --------- ----------------------- + a3395b7192eee3ca586e93ccf4f12f59 completed 4600.98 + e2e2e96e8e9febc33efb17b9de0920d1 completed 4786.43 + 7e0b4271f2972f539cf839fbd1b5430d completed 4602.58 + 568acbcb2fa3e00c8607bdc2d2bda5e3 completed 4748.09 + 5f9743e88a29d0ee87b5c71246dbd2fb completed 4547.28 + + + other_root + ========== + id status min example_objective + -------------------------------- --------- ----------------------- + aaa16658770abd3516a027918eb91be5 completed 4761.33 + 68233ce61ee5edfb6fb029ab7daf2db7 completed 4543.73 + cc0b0532c56c56fde63ad06fd73df63f completed 4753.5 + b5335589cb897bbea2b58c5d4bd9c0c1 completed 4751.15 + a4a711389844567ac1b429eff96964e4 completed 4790.87 + + + +The ``--collapse`` Argument +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +On the other hand, if you wish to only get an overview of the experiments and the amount of trials +linked to them without looking through the whole EVC tree, you can use the ``--collapse`` +option. As its name indicates, it will collapse every children into the root experiment and make a +total count of the amount of trials `in that EVC tree`. As always, we provide an output to +give you an example: + + +.. code-block:: console + + orion status --collapse + + root + ==== + status quantity min example_objective + --------- ---------- ----------------------- + completed 10 4534.95 + + + other_root + ========== + status quantity min example_objective + --------- ---------- ----------------------- + completed 5 4543.73 + + +The ``--name`` Argument +~~~~~~~~~~~~~~~~~~~~~~~ +If you wish to isolate a single EVC tree and look at their trials instead of listing every +single experiments, you can use the ``--name`` argument by itself or combine it with the ones above +to obtain the same results, but constrained. Once again, some examples for each type of scenrario is +given: + +.. code-block:: console + + orion status --name root + +.. code-block:: bash + + root + ==== + status quantity min example_objective + --------- ---------- ----------------------- + completed 10 4534.95 + + + child_1 + ======= + status quantity min example_objective + --------- ---------- ----------------------- + completed 10 4547.28 + +.. code-block:: console + + orion status --name root --all + +.. code-block:: bash + + root + ==== + id status min example_objective + -------------------------------- --------- ----------------------- + bc222aa1705b3fe3a266fd601598ac41 completed 4555.13 + 59bf4c85305b7ba065fa770805e93cb1 completed 4653.49 + da5159d9d36ef44879e72cbe7955347c completed 4788.08 + 2c4c6409d3beeb179fce3c83b4f6d8f8 completed 4752.45 + 6a82a8a55d2241d989978cf3a7ebbba0 completed 4534.95 + + + child_1 + ======= + id status min example_objective + -------------------------------- --------- ----------------------- + a3395b7192eee3ca586e93ccf4f12f59 completed 4600.98 + e2e2e96e8e9febc33efb17b9de0920d1 completed 4786.43 + 7e0b4271f2972f539cf839fbd1b5430d completed 4602.58 + 568acbcb2fa3e00c8607bdc2d2bda5e3 completed 4748.09 + 5f9743e88a29d0ee87b5c71246dbd2fb completed 4547.28 + +.. code-block:: console + + orion status --name root --collapse + +.. code-block:: bash + + root + ==== + status quantity min example_objective + --------- ---------- ----------------------- + completed 10 4534.95 diff --git a/docs/src/user/monitoring.rst b/docs/src/user/monitoring.rst index fbb96ea48..c33c39dfd 100644 --- a/docs/src/user/monitoring.rst +++ b/docs/src/user/monitoring.rst @@ -10,6 +10,8 @@ Commands for terminal ===================== .. _cli-info: +.. include:: cli/list.rst +.. include:: cli/status.rst .. include:: cli/info.rst Library API diff --git a/tox.ini b/tox.ini index 2b5f165d0..34f0ac130 100644 --- a/tox.ini +++ b/tox.ini @@ -192,8 +192,8 @@ basepython = python3.6 deps = -rdocs/requirements.txt commands = - sphinx-build -E -W --color -c docs/src/ -b html docs/src/ docs/build/html - sphinx-build -E -W --color -c docs/src/ -b man docs/src/ docs/build/man + sphinx-build -W --color -c docs/src/ -b html docs/src/ docs/build/html + sphinx-build -W --color -c docs/src/ -b man docs/src/ docs/build/man [testenv:serve-docs] description = Host project's documentation and API reference in localhost From c86651a4d32fdb51567c52cd8272c388805b4bf8 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Fri, 19 Jul 2019 20:45:57 -0400 Subject: [PATCH 46/50] Update Roadmap for release v0.1.5 Remove v0.1.5 and add v0.1.7 for preliminary python API. --- ROADMAP.md | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 8df15f072..02962ab97 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -3,22 +3,18 @@ Last update July 5th, 2019 ## Next releases - Short-Term -### v0.1.5 - -#### Trial interruption/suspension/resumption - -Handle interrupted or lost trials so that they can be automatically resumed by running workers. - -#### Command line tools to monitor experiments - -See [#125](https://github.com/Epistimio/orion/issues/125) - ### v0.1.6 #### Auto-resolution of EVC Make branching events automatically solved with sane defaults. +### v0.1.7 + +#### Preliminary Python API + +Library API to simplify usage of algorithms without Oríon's worker. + ## Next releases - Mid-Term ### v0.2: ETA End of summer 2019 From f2fda491493dc3d56ddb68eb064fa99f5461345f Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Mon, 22 Jul 2019 15:13:55 -0400 Subject: [PATCH 47/50] Handle broken refers in experiment configuration Why: If the process is stopped between the registration of the experiment and the update of exp[refers][self._id], then the experiment will be somehow broken, i.e., many code parts will crash because of it. How: Handle refers creation during configuration with default values if nothing is set so that abortion during registration will in worst case lead to inconsistant refers that is easier to fix. Previously broken refers would be {}, which is irrecuperable for child experiments. Now broken refers would be {parent_id: ok, adapter: ok, root_id: None}, which is simple to infer and fix. --- src/orion/core/cli/info.py | 4 ++-- src/orion/core/worker/experiment.py | 16 +++++++++---- .../functional/commands/test_info_command.py | 24 +++++++++++++++++++ 3 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 tests/functional/commands/test_info_command.py diff --git a/src/orion/core/cli/info.py b/src/orion/core/cli/info.py index 3e5c8e0fc..98377185d 100755 --- a/src/orion/core/cli/info.py +++ b/src/orion/core/cli/info.py @@ -29,8 +29,8 @@ def add_subparser(parser): def main(args): """Fetch config and info experiments""" - experiment_view = EVCBuilder().build_view_from(args) - print(format_info(experiment_view)) + experiment = EVCBuilder().build_from(args) + print(format_info(experiment)) INFO_TEMPLATE = """\ diff --git a/src/orion/core/worker/experiment.py b/src/orion/core/worker/experiment.py index 6990c83a7..61f984986 100644 --- a/src/orion/core/worker/experiment.py +++ b/src/orion/core/worker/experiment.py @@ -547,8 +547,8 @@ def configure(self, config, enable_branching=True, enable_update=True): self._id = final_config['_id'] # Update refers in db if experiment is root - if not self.refers: - self.refers = {'root_id': self._id, 'parent_id': None, 'adapter': []} + if self.refers['parent_id'] is None: + self.refers['root_id'] = self._id self._storage.update_experiment(self, refers=self.refers) else: @@ -661,7 +661,10 @@ def _instantiate_config(self, config): except KeyError: pass - if self.refers and not isinstance(self.refers.get('adapter'), BaseAdapter): + self.refers.setdefault('parent_id', None) + self.refers.setdefault('root_id', self._id) + self.refers.setdefault('adapter', []) + if self.refers['adapter'] and not isinstance(self.refers.get('adapter'), BaseAdapter): self.refers['adapter'] = Adapter.build(self.refers['adapter']) if not self.producer.get('strategy'): @@ -767,8 +770,11 @@ def __init__(self, name, user=None): # TODO: Views are not fully configured until configuration is refactored # This snippet is to instantiate adapters anyhow, because it is required for # experiment views in EVC. - if self.refers and not isinstance(self.refers.get('adapter'), BaseAdapter): - self._experiment.refers['adapter'] = Adapter.build(self.refers['adapter']) + self.refers.setdefault('parent_id', None) + self.refers.setdefault('root_id', self._id) + self.refers.setdefault('adapter', []) + if self.refers['adapter'] and not isinstance(self.refers.get('adapter'), BaseAdapter): + self.refers['adapter'] = Adapter.build(self.refers['adapter']) # try: # self._experiment.configure(self._experiment.configuration, enable_branching=False, diff --git a/tests/functional/commands/test_info_command.py b/tests/functional/commands/test_info_command.py new file mode 100644 index 000000000..8cb071879 --- /dev/null +++ b/tests/functional/commands/test_info_command.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Perform a functional test of the info command.""" +import os + +import orion.core.cli + + +def test_info_no_hit(clean_db, one_experiment, capsys): + """Test info if no experiment with given name.""" + orion.core.cli.main(['info', 'i do not exist']) + + captured = capsys.readouterr().out + + assert captured == 'Error: No commandline configuration found for new experiment.\n' + + +def test_info_hit(clean_db, one_experiment, capsys): + """Test info if existing experiment.""" + orion.core.cli.main(['info', 'test_single_exp']) + + captured = capsys.readouterr().out + + assert '--x~uniform(0,1)' in captured From 52e67d8cff09c5a567dc6edf2ed14fb84c7a4d8f Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Mon, 22 Jul 2019 15:36:19 -0400 Subject: [PATCH 48/50] Handle broken experiments in `list` and `info` --- src/orion/core/cli/list.py | 3 ++- tests/functional/commands/conftest.py | 11 ++++++++++- tests/functional/commands/test_info_command.py | 11 +++++++++-- tests/functional/commands/test_list_command.py | 9 +++++++++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/orion/core/cli/list.py b/src/orion/core/cli/list.py index 8a20e34c6..32b5cea4c 100644 --- a/src/orion/core/cli/list.py +++ b/src/orion/core/cli/list.py @@ -45,7 +45,8 @@ def main(args): if args['name']: root_experiments = experiments else: - root_experiments = [exp for exp in experiments if exp['refers']['root_id'] == exp['_id']] + root_experiments = [exp for exp in experiments + if exp['refers'].get('root_id', exp['_id']) == exp['_id']] for root_experiment in root_experiments: root = EVCBuilder().build_view_from({'name': root_experiment['name']}).node diff --git a/tests/functional/commands/conftest.py b/tests/functional/commands/conftest.py index 1ac20d7ea..65ac093d2 100644 --- a/tests/functional/commands/conftest.py +++ b/tests/functional/commands/conftest.py @@ -126,7 +126,7 @@ def only_experiments_db(clean_db, database, exp_config): database.experiments.insert_many(exp_config[0]) -def ensure_deterministic_id(name, db_instance): +def ensure_deterministic_id(name, db_instance, update=None): """Change the id of experiment to its name.""" experiment = db_instance.read('experiments', {'name': name})[0] db_instance.remove('experiments', {'_id': experiment['_id']}) @@ -135,6 +135,9 @@ def ensure_deterministic_id(name, db_instance): if experiment['refers']['parent_id'] is None: experiment['refers']['root_id'] = name + if update is not None: + experiment.update(update) + db_instance.write('experiments', experiment) @@ -148,6 +151,12 @@ def one_experiment(monkeypatch, db_instance): ensure_deterministic_id('test_single_exp', db_instance) +@pytest.fixture +def broken_refers(one_experiment, db_instance): + """Create an experiment with broken refers.""" + ensure_deterministic_id('test_single_exp', db_instance, update=dict(refers={'oups': 'broken'})) + + @pytest.fixture def single_without_success(one_experiment): """Create an experiment without a succesful trial.""" diff --git a/tests/functional/commands/test_info_command.py b/tests/functional/commands/test_info_command.py index 8cb071879..4ee786dcf 100644 --- a/tests/functional/commands/test_info_command.py +++ b/tests/functional/commands/test_info_command.py @@ -1,8 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """Perform a functional test of the info command.""" -import os - import orion.core.cli @@ -22,3 +20,12 @@ def test_info_hit(clean_db, one_experiment, capsys): captured = capsys.readouterr().out assert '--x~uniform(0,1)' in captured + + +def test_info_broken(clean_db, broken_refers, capsys): + """Test info if experiment.refers is missing.""" + orion.core.cli.main(['info', 'test_single_exp']) + + captured = capsys.readouterr().out + + assert '--x~uniform(0,1)' in captured diff --git a/tests/functional/commands/test_list_command.py b/tests/functional/commands/test_list_command.py index 1474430c8..68133fbeb 100644 --- a/tests/functional/commands/test_list_command.py +++ b/tests/functional/commands/test_list_command.py @@ -25,6 +25,15 @@ def test_single_exp(clean_db, one_experiment, capsys): assert captured == " test_single_exp\n" +def test_broken_refers(clean_db, broken_refers, capsys): + """Test that experiment without refers dict can be handled properly.""" + orion.core.cli.main(['list']) + + captured = capsys.readouterr().out + + assert captured == " test_single_exp\n" + + def test_two_exp(capsys, clean_db, two_experiments): """Test that experiment and child are printed.""" orion.core.cli.main(['list']) From 909110601e0fc524411c2205007785500e477a5f Mon Sep 17 00:00:00 2001 From: Pierre Delaunay Date: Sat, 20 Jul 2019 21:24:54 -0500 Subject: [PATCH 49/50] Move TrialMonitor/TrialPacemaker inside consumer Why: We need access to the pacemaker when the trials is completed so that we can stop it. Otherwise, the thread continues until if makes another heartbeat before it crashes and quit, which can take lot of time. The execution of Orion hangs at the end of experiment until the pacemaker stops. How: Move pacemaker inside Consumer so that it can be stopped right at end of trial execution. Note that there is some delay between the reservation and the start of pacemaker, which makes it possible to loose the reservation before execution begins. That is not very likely however since there is not a large time gap between reservation and execution. --- src/orion/core/worker/consumer.py | 13 ++++- src/orion/core/worker/experiment.py | 2 - .../{trial_monitor.py => trial_pacemaker.py} | 12 ++-- tests/functional/demo/test_demo.py | 13 +++++ tests/unittests/core/worker/test_consumer.py | 22 ++++++++ ...ial_monitor.py => test_trial_pacemaker.py} | 55 ++++++++----------- 6 files changed, 75 insertions(+), 42 deletions(-) rename src/orion/core/worker/{trial_monitor.py => trial_pacemaker.py} (80%) rename tests/unittests/core/worker/{test_trial_monitor.py => test_trial_pacemaker.py} (63%) diff --git a/src/orion/core/worker/consumer.py b/src/orion/core/worker/consumer.py index f3971f5c1..98d9b9d30 100644 --- a/src/orion/core/worker/consumer.py +++ b/src/orion/core/worker/consumer.py @@ -16,7 +16,7 @@ from orion.core.io.space_builder import SpaceBuilder from orion.core.utils.working_dir import WorkingDir - +from orion.core.worker.trial_pacemaker import TrialPacemaker log = logging.getLogger(__name__) @@ -64,6 +64,8 @@ def __init__(self, experiment): self.script_path = experiment.metadata['user_script'] + self.pacemaker = None + def consume(self, trial): """Execute user's script as a block box using the options contained within `trial`. @@ -115,7 +117,14 @@ def _consume(self, trial, workdirname): log.debug("## Launch user's script as a subprocess and wait for finish.") - self.execute_process(results_file.name, cmd_args) + self.pacemaker = TrialPacemaker(self.experiment, trial.id) + self.pacemaker.start() + try: + self.execute_process(results_file.name, cmd_args) + finally: + # merciless + self.pacemaker.stop() + return results_file def execute_process(self, results_filename, cmd_args): diff --git a/src/orion/core/worker/experiment.py b/src/orion/core/worker/experiment.py index 61f984986..c141c5cd8 100644 --- a/src/orion/core/worker/experiment.py +++ b/src/orion/core/worker/experiment.py @@ -27,7 +27,6 @@ from orion.core.worker.primary_algo import PrimaryAlgo from orion.core.worker.strategy import (BaseParallelStrategy, Strategy) -from orion.core.worker.trial_monitor import TrialMonitor from orion.storage.base import ReadOnlyStorageProtocol, StorageProtocol log = logging.getLogger(__name__) @@ -260,7 +259,6 @@ def reserve_trial(self, score_handle=None, _depth=1): else: log.debug('%s found suitable trial', '<' * _depth) selected_trial = self.fetch_trials({'_id': selected_trial.id})[0] - TrialMonitor(self, selected_trial.id).start() log.debug('%s reserved trial (trial: %s)', '<' * _depth, selected_trial) return selected_trial diff --git a/src/orion/core/worker/trial_monitor.py b/src/orion/core/worker/trial_pacemaker.py similarity index 80% rename from src/orion/core/worker/trial_monitor.py rename to src/orion/core/worker/trial_pacemaker.py index 5e1368987..746075634 100644 --- a/src/orion/core/worker/trial_monitor.py +++ b/src/orion/core/worker/trial_pacemaker.py @@ -10,12 +10,10 @@ import datetime import threading -from orion.core.io.database import Database - -class TrialMonitor(threading.Thread): +class TrialPacemaker(threading.Thread): """Monitor a given trial inside a thread, updating its heartbeat - at a given interval of time. + at a given interval of time. Parameters ---------- @@ -25,7 +23,6 @@ class TrialMonitor(threading.Thread): """ def __init__(self, exp, trial_id, wait_time=60): - """Initialize a TrialMonitor.""" threading.Thread.__init__(self) self.stopped = threading.Event() self.exp = exp @@ -47,7 +44,8 @@ def _monitor_trial(self): trials = self.exp.fetch_trials(query) if trials: - update = dict(heartbeat=datetime.datetime.utcnow()) - Database().write('trials', update, query) + update = datetime.datetime.utcnow() + if not self.exp.update_trial(trials[0], where=query, heartbeat=update): + self.stopped.set() else: self.stopped.set() diff --git a/tests/functional/demo/test_demo.py b/tests/functional/demo/test_demo.py index c8288dd1e..540851e0f 100644 --- a/tests/functional/demo/test_demo.py +++ b/tests/functional/demo/test_demo.py @@ -471,3 +471,16 @@ def test_resilience(monkeypatch): exp = ExperimentBuilder().build_from({'name': 'demo_random_search'}) assert len(exp.fetch_trials({'status': 'broken'})) == 3 + + +@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("null_db_instances") +def test_demo_with_shutdown_quickly(monkeypatch): + """Check simple pipeline with random search is reasonably fast.""" + monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) + + process = subprocess.Popen( + ["orion", "hunt", "--config", "./orion_config_random.yaml", "--max-trials", "30", + "./black_box.py", "-x~uniform(-50, 50)"]) + + assert process.wait(timeout=10) == 0 diff --git a/tests/unittests/core/worker/test_consumer.py b/tests/unittests/core/worker/test_consumer.py index fcc8c0a45..8b722ec23 100644 --- a/tests/unittests/core/worker/test_consumer.py +++ b/tests/unittests/core/worker/test_consumer.py @@ -4,6 +4,7 @@ import os import signal import subprocess +import time import pytest @@ -73,6 +74,27 @@ def mock_popen(*args, **kwargs): assert trials[0].id == trial.id +@pytest.mark.usefixtures("create_db_instance") +def test_pacemaker_termination(config, monkeypatch): + """Check if pacemaker stops as soon as the trial completes.""" + exp = ExperimentBuilder().build_from(config) + + trial = tuple_to_trial((1.0,), exp.space) + + exp.register_trial(trial) + + con = Consumer(exp) + + start = time.time() + + con.consume(trial) + con.pacemaker.join() + + duration = time.time() - start + + assert duration < con.pacemaker.wait_time + + @pytest.mark.usefixtures("create_db_instance") def test_trial_working_dir_is_changed(config, monkeypatch): """Check that trial has its working_dir attribute changed.""" diff --git a/tests/unittests/core/worker/test_trial_monitor.py b/tests/unittests/core/worker/test_trial_pacemaker.py similarity index 63% rename from tests/unittests/core/worker/test_trial_monitor.py rename to tests/unittests/core/worker/test_trial_pacemaker.py index cf7f28096..6b8e9a658 100644 --- a/tests/unittests/core/worker/test_trial_monitor.py +++ b/tests/unittests/core/worker/test_trial_pacemaker.py @@ -9,7 +9,7 @@ from orion.core.io.database import Database from orion.core.io.experiment_builder import ExperimentBuilder from orion.core.utils.format_trials import tuple_to_trial -from orion.core.worker.trial_monitor import TrialMonitor +from orion.core.worker.trial_pacemaker import TrialPacemaker @pytest.fixture @@ -21,19 +21,30 @@ def config(exp_config): return config -@pytest.mark.usefixtures("create_db_instance") -def test_trial_update_heartbeat(config): - """Test that the heartbeat of a trial has been updated.""" - exp = ExperimentBuilder().build_from(config) +@pytest.fixture +def exp(config): + """Return an Experiment.""" + return ExperimentBuilder().build_from(config) + + +@pytest.fixture +def trial(exp): + """Return a Trial which is registered in DB.""" trial = tuple_to_trial((1.0,), exp.space) heartbeat = datetime.datetime.utcnow() + trial.experiment = exp.id + trial.status = 'reserved' trial.heartbeat = heartbeat - data = {'_id': trial.id, 'status': 'reserved', 'heartbeat': heartbeat, 'experiment': exp.id} + Database().write('trials', trial.to_dict()) - Database().write('trials', data) + return trial - trial_monitor = TrialMonitor(exp, trial.id, wait_time=1) + +@pytest.mark.usefixtures("create_db_instance") +def test_trial_update_heartbeat(exp, trial): + """Test that the heartbeat of a trial has been updated.""" + trial_monitor = TrialPacemaker(exp, trial.id, wait_time=1) trial_monitor.start() time.sleep(2) @@ -53,18 +64,9 @@ def test_trial_update_heartbeat(config): @pytest.mark.usefixtures("create_db_instance") -def test_trial_heartbeat_not_updated(config): +def test_trial_heartbeat_not_updated(exp, trial): """Test that the heartbeat of a trial is not updated when trial is not longer reserved.""" - exp = ExperimentBuilder().build_from(config) - trial = tuple_to_trial((1.0,), exp.space) - heartbeat = datetime.datetime.utcnow() - trial.heartbeat = heartbeat - - data = {'_id': trial.id, 'status': 'reserved', 'heartbeat': heartbeat, 'experiment': exp.id} - - Database().write('trials', data) - - trial_monitor = TrialMonitor(exp, trial.id, wait_time=1) + trial_monitor = TrialPacemaker(exp, trial.id, wait_time=1) trial_monitor.start() time.sleep(2) @@ -84,24 +86,15 @@ def test_trial_heartbeat_not_updated(config): @pytest.mark.usefixtures("create_db_instance") -def test_trial_heartbeat_not_updated_inbetween(config): +def test_trial_heartbeat_not_updated_inbetween(exp, trial): """Test that the heartbeat of a trial is not updated before wait time.""" - exp = ExperimentBuilder().build_from(config) - trial = tuple_to_trial((1.0,), exp.space) - heartbeat = datetime.datetime.utcnow().replace(microsecond=0) - trial.heartbeat = heartbeat - - data = {'_id': trial.id, 'status': 'reserved', 'heartbeat': heartbeat, 'experiment': exp.id} - - Database().write('trials', data) - - trial_monitor = TrialMonitor(exp, trial.id, wait_time=5) + trial_monitor = TrialPacemaker(exp, trial.id, wait_time=5) trial_monitor.start() time.sleep(1) trials = exp.fetch_trials({'_id': trial.id, 'status': 'reserved'}) - assert trial.heartbeat == trials[0].heartbeat + assert trial.heartbeat.replace(microsecond=0) == trials[0].heartbeat.replace(microsecond=0) heartbeat = trials[0].heartbeat From f80d007b63c5afbf54a4946ec0cfe73b8d7f34ba Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Tue, 23 Jul 2019 10:54:23 -0400 Subject: [PATCH 50/50] Bring back space.{un}pack_point but deprecate them Why: The plugin orion.skopt.algo is relying on these functions so we need to keep them unlees we update the plugin. How: Call the new methods internally and log a warning that {un}pack_point will be removed in v0.2.0. --- src/orion/algo/space.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/orion/algo/space.py b/src/orion/algo/space.py index ffaed026c..15fc66aff 100644 --- a/src/orion/algo/space.py +++ b/src/orion/algo/space.py @@ -33,11 +33,17 @@ """ from collections import OrderedDict +import logging import numbers import numpy from scipy.stats import distributions +from orion.core.utils.points import flatten_dims, regroup_dims + + +logger = logging.getLogger(__name__) + def check_random_state(seed): """Return numpy global rng or RandomState if seed is specified""" @@ -818,3 +824,25 @@ def __repr__(self): """Represent as a string the space and the dimensions it contains.""" dims = list(self.values()) return "Space([{}])".format(',\n '.join(map(str, dims))) + + +def pack_point(point, space): + """Take a list of points and pack it appropriately as a point from `space`. + + This function is deprecated and will be removed in v0.2.0. Use + `orion.core.utils.points.regroup_dims` instead. + """ + logger.warning('`pack_point` is deprecated and will be removed in v0.2.0. Use ' + '`orion.core.utils.points.regroup_dims` instead.') + return regroup_dims(point, space) + + +def unpack_point(point, space): + """Flatten `point` in `space` and convert it to a 1D `numpy.ndarray`. + + This function is deprecated and will be removed in v0.2.0. Use + `orion.core.utils.points.flatten_dims` instead. + """ + logger.warning('`unpack_point` is deprecated and will be removed in v0.2.0. Use ' + '`orion.core.utils.points.regroup_dims` instead.') + return flatten_dims(point, space)