Merge pull request #988 from Epistimio/release-0.2.6rc

Release Candidate v0.2.6

.pre-commit-config.yaml

@@ -5,12 +5,12 @@ repos:
  hooks:
  - id: black
  - repo: https://github.com/codespell-project/codespell
- rev: v2.1.0
+ rev: v2.2.1
  hooks:
  - id: codespell
- args: ["--skip", "*.html,*.ipynb,dashboard/src/.yarn/**,dashboard/build/**,dashboard/src/src/__tests__/**", "--ignore-words-list=hist,wont"]
+ args: ["--skip", "*.html,*.ipynb,dashboard/src/.yarn/**,dashboard/src/yarn.lock,dashboard/build/**,dashboard/src/src/__tests__/**", "--ignore-words-list=hist,wont"]
  - repo: https://gitlab.com/pycqa/flake8
- rev: 3.9.2
+ rev: 5.0.4
  hooks:
  - id: flake8
  args:
@@ -25,7 +25,7 @@ repos:
  - id: isort
  args: ["--profile", "black"]
  - repo: https://github.com/asottile/pyupgrade
- rev: v2.34.0
+ rev: v2.37.3
  hooks:
  - id: pyupgrade
  args: ["--py37-plus"]

README.rst

@@ -118,7 +118,7 @@ If you use Oríon for published work, please cite our work using the following b
 
 .. code-block:: bibtex
 
- @software{xavier_bouthillier_2022_0_2_5,
+ @software{xavier_bouthillier_2022_0_2_6,
  author = {Xavier Bouthillier and
  Christos Tsirigotis and
  François Corneau-Tremblay and
@@ -146,7 +146,7 @@ If you use Oríon for published work, please cite our work using the following b
  month = august,
  year = 2022,
  publisher = {Zenodo},
- version = {v0.2.5},
+ version = {v0.2.6,
  doi = {10.5281/zenodo.3478592},
  url = {https://doi.org/10.5281/zenodo.3478592}
  }

ROADMAP.md

@@ -1,10 +1,14 @@
 # Roadmap
-Last update August 2nd, 2022
+Last update August 18th, 2022
 
 ## Next releases - Short-Term
 
+### v0.2.7
+
 ### v0.2.6
 
+- Fix; make sure stdout from user script is printed back and not captured
+
 #### Simple dashboard specific to monitoring and benchmarking of Black-Box optimization
 - Specific to hyper parameter optimizations
 - Provide status of experiments
@@ -13,6 +17,8 @@ Last update August 2nd, 2022
 Leveraging the knowledge base contained in the EVC of previous trials to optimize and drive new
 trials.
 
+## Next releases - Mid-Term
+
 ## Next releases - Long-Term
 
 #### Conditional Space

examples/plotting/plot_4_partial_dependencies.py

@@ -63,7 +63,7 @@
 # parameters
 # <name>: <value>
 #
-# Even for a simple 2-d search space, the partial dependency is very useful. We see very cleary in this example
+# Even for a simple 2-d search space, the partial dependency is very useful. We see very clearly in this example
 # the optimal regions for both hyperparameters and we can see as well that the optimal region for
 # learning rates is larger when the dropout is low, and narrower when dropout approaches 0.5.
 #

src/orion/core/evc/conflicts.py

@@ -267,7 +267,7 @@ def try_resolve(self, conflict, *args, **kwargs):
 
  Parameters
  ----------
- conflict: `orion.ore.evc.conflicts.Conflict`
+ conflict: `orion.core.evc.conflicts.Conflict`
  Conflict object to call `try_resolve`.
  silence_errors: bool
  If True, errors raised on execution of conflict.try_resolve will be caught and

src/orion/core/io/experiment_branch_builder.py

@@ -33,7 +33,7 @@ class ExperimentBranchBuilder:
  Parameters
  ----------
  conflicts: Conflicts
- Object reprenting a group of conflicts
+ Object representing a group of conflicts
  manual_resolution: bool, optional
  Starts the prompt to resolve manually the conflicts. Use system's default if not provided.
  branch_from: str, optional

src/orion/core/worker/consumer.py

@@ -253,21 +253,13 @@ def execute_process(self, cmd_args, environ):
 
  try:
  # pylint: disable = consider-using-with
- process = subprocess.Popen(
- command,
- env=environ,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- )
+ process = subprocess.Popen(command, env=environ)
  except PermissionError as exc:
  log.debug("Script is not executable")
  raise InexecutableUserScript(" ".join(cmd_args)) from exc
 
- stdout, _ = process.communicate()
-
  return_code = process.wait()
  log.debug(f"Script finished with return code {return_code}")
 
  if return_code != 0:
- log.debug("%s", stdout.decode("utf-8"))
  raise ExecutionError(return_code)

src/orion/storage/track.py

@@ -150,11 +150,12 @@ def experiment(self):
  return self.storage.group_id
 
  @property
- def hearbeat(self):
- """See `~orion.core.worker.trial.Trial`"""
- return datetime.datetime.utcfromtimestamp(
- self.storage.metadata.get("heartbeat", 0)
- )
+ def heartbeat(self):
+ """Trial Heartbeat"""
+ heartbeat = self.storage.metadata.get("heartbeat")
+ if heartbeat:
+ return datetime.datetime.utcfromtimestamp(heartbeat)
+ return None
 
  @property
  def id(self):
@@ -311,14 +312,6 @@ def end_time(self, value):
  """See `~orion.core.worker.trial.Trial`"""
  self.storage.metadata["end_time"] = value
 
- @property
- def heartbeat(self):
- """Trial Heartbeat"""
- heartbeat = self.storage.metadata.get("heartbeat")
- if heartbeat:
- return datetime.datetime.utcfromtimestamp(heartbeat)
- return None
-
  @property
  def parents(self):
  """See `~orion.core.worker.trial.Trial`"""

tests/functional/demo/black_box.py

@@ -2,9 +2,12 @@
 """Simple one dimensional example for a possible user's script."""
 import argparse
 import os
+import sys
 
 from orion.client import report_results
 
+PRINT = os.environ.get("ORION_PRINT", "False") == "True"
+
 
 def function(x):
  """Evaluate partial information of a quadratic."""
@@ -14,6 +17,9 @@ def function(x):
 
 def execute():
  """Execute a simple pipeline as an example."""
+ if PRINT:
+ print("Start Child Process")
+
  # 1. Receive inputs as you want
  parser = argparse.ArgumentParser()
  parser.add_argument("-x", type=float, required=True)
@@ -36,13 +42,19 @@ def execute():
  # 2. Perform computations
  y, dy = function(inputs.x)
 
+ if PRINT:
+ print("Child Error", file=sys.stderr)
+
  # 3. Gather and report results
  results = list()
  results.append(dict(name="example_objective", type="objective", value=y))
  results.append(dict(name="example_gradient", type="gradient", value=[dy]))
 
  report_results(results)
 
+ if PRINT:
+ print("End Child Process")
+
 
 if __name__ == "__main__":
  execute()

tests/functional/demo/orion_black_box.py

@@ -0,0 +1,37 @@
+import os
+import sys
+
+import orion
+import orion.core.cli
+from orion.testing import OrionState
+
+
+def main():
+ """This is needed because the user script runs in a separate process, so python
+ cannot capture its output.
+
+ Here we run the entire orion script in its own process so we can capture all of its output.
+ """
+ os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+ print("Start main process")
+ os.environ["ORION_PRINT"] = "True"
+ with OrionState():
+ orion.core.cli.main(
+ [
+ "hunt",
+ "-n",
+ "default_algo",
+ "--exp-max-trials",
+ "5",
+ "./black_box.py",
+ "-x~uniform(0, 5, discrete=True)",
+ ]
+ )
+
+ print("Main process Error", file=sys.stderr)
+ print("End main process")
+
+
+if __name__ == "__main__":
+ main()

tests/functional/demo/test_demo.py

@@ -3,6 +3,7 @@
 import os
 import shutil
 import subprocess
+import sys
 import tempfile
 from collections import defaultdict
 from contextlib import contextmanager
@@ -52,6 +53,28 @@ def test_demo_with_default_algo_cli_config_only(storage, monkeypatch):
  assert trials[-1].status == "completed"
 
 
+def test_demo_stdout_stderr():
+ """Check that script output is printed and shown to the end user"""
+
+ dir = os.path.dirname(os.path.abspath(__file__))
+
+ process = subprocess.Popen(
+ [sys.executable, os.path.join(dir, "orion_black_box.py")],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ )
+ stdout, stderr = process.communicate()
+
+ stdout = stdout.decode("utf-8")
+ stderr = stderr.decode("utf-8")
+
+ assert "Start main process" in stdout
+ assert "Start Child Process" in stdout
+
+ assert "Child Error" in stderr
+ assert "Main process Error" in stderr
+
+
 def test_demo(storage, monkeypatch):
  """Test a simple usage scenario."""
  monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__)))

tests/unittests/core/io/test_resolve_config.py

@@ -71,7 +71,7 @@ def test_fetch_metadata_python_users_script(script_path):
 
 
 def test_fetch_metadata_not_existed_path():
- """Verfiy the raise of error when user_script path does not exist"""
+ """Verify the raise of error when user_script path does not exist"""
  path = "dummy/path"
  with pytest.raises(OSError) as exc_info:
  resolve_config.fetch_metadata(user_args=[path])