Merge pull request #824 from Epistimio/release-v0.2.3rc1

Release candidate v0.2.3rc1

README.rst

@@ -118,7 +118,7 @@ If you use Oríon for published work, please cite our work using the following b
 
 .. code-block:: bibtex
 
- @software{xavier_bouthillier_2022_0_2_2,
+ @software{xavier_bouthillier_2022_0_2_3,
  author = {Xavier Bouthillier and
  Christos Tsirigotis and
  François Corneau-Tremblay and
@@ -143,10 +143,10 @@ If you use Oríon for published work, please cite our work using the following b
  Pascal Lamblin and
  Christopher Beckham},
  title = {{Epistimio/orion: Asynchronous Distributed Hyperparameter Optimization}},
- month = feb,
+ month = mar,
  year = 2022,
  publisher = {Zenodo},
- version = {v0.2.2},
+ version = {v0.2.3},
  doi = {10.5281/zenodo.3478592},
  url = {https://doi.org/10.5281/zenodo.3478592}
  }

ROADMAP.md

@@ -1,14 +1,20 @@
 # Roadmap
-Last update Feb 11th, 2022
+Last update March 7th, 2022
 
 ## Next releases - Short-Term
 
-### v0.2.3
+### v0.2.4
 
 - [DEBH](https://arxiv.org/abs/2105.09821)
 - [HEBO](https://github.com/huawei-noah/HEBO/tree/master/HEBO/archived_submissions/hebo)
 - [BOHB](https://ml.informatik.uni-freiburg.de/papers/18-ICML-BOHB.pdf)
+- [Nevergrad](https://github.com/facebookresearch/nevergrad)
+- [Ax](https://ax.dev/)
+- [MOFA](https://github.com/Epistimio/orion.algo.mofa)
+- [PB2](https://github.com/Epistimio/orion.algo.pb2)
 - Integration with Hydra
+- Integration with [sample-space](https://github.com/Epistimio/sample-space) and
+ [ConfigSpace](https://automl.github.io/ConfigSpace/master/)
 
 ## Next releases - Mid-Term
 

conda/ci_build.sh

@@ -6,6 +6,7 @@ export PATH="$HOME/miniconda/bin:$PATH"
 hash -r
 conda config --set always_yes yes --set changeps1 no
 conda config --add channels conda-forge
+conda config --add channels mila-iqia
 conda config --set channel_priority strict
 
 pip uninstall -y setuptools

conda/meta.yaml

@@ -33,6 +33,7 @@ requirements:
  - requests
  - pandas
  - falcon
+ - falcon-cors
  - gunicorn
  - scikit-learn
  - psutil

setup.py

@@ -11,7 +11,7 @@
 repo_root = os.path.dirname(os.path.abspath(__file__))
 
 
-tests_require = ["pytest>=3.0.0", "scikit-learn"]
+tests_require = ["pytest>=3.0.0", "scikit-learn", "ptera>=1.1.0"]
 
 
 packages = [ # Packages must be sorted alphabetically to ease maintenance and merges.
@@ -88,6 +88,7 @@
  "pandas",
  "gunicorn",
  "falcon",
+ "falcon-cors",
  "scikit-learn",
  "psutil",
  "joblib",

src/orion/client/runner.py

@@ -40,12 +40,24 @@ def __init__(self):
  self.handlers = dict()
  self.start = 0
  self.delayed = 0
+ self.signal_installed = False
 
  def __enter__(self):
  """Override the signal handlers with our delayed handler"""
  self.signal_received = False
- self.handlers[signal.SIGINT] = signal.signal(signal.SIGINT, self.handler)
- self.handlers[signal.SIGTERM] = signal.signal(signal.SIGTERM, self.handler)
+
+ try:
+ self.handlers[signal.SIGINT] = signal.signal(signal.SIGINT, self.handler)
+ self.handlers[signal.SIGTERM] = signal.signal(signal.SIGTERM, self.handler)
+ self.signal_installed = True
+
+ except ValueError: # ValueError: signal only works in main thread
+ log.warning(
+ "SIGINT/SIGTERM protection hooks could not be installed because "
+ "Runner is executing inside a thread/subprocess, results could get lost "
+ "on interruptions"
+ )
+
  return self
 
  def handler(self, sig, frame):
@@ -65,6 +77,9 @@ def handler(self, sig, frame):
 
  def restore_handlers(self):
  """Restore old signal handlers"""
+ if not self.signal_installed:
+ return
+
  signal.signal(signal.SIGINT, self.handlers[signal.SIGINT])
  signal.signal(signal.SIGTERM, self.handlers[signal.SIGTERM])
 
@@ -268,7 +283,7 @@ def run(self):
  def should_sample(self):
  """Check if more trials could be generated"""
 
- if self.is_broken or self.is_done:
+ if self.free_worker <= 0 or (self.is_broken or self.is_done):
  return 0
 
  pending = len(self.pending_trials) + self.trials
@@ -317,8 +332,9 @@ def gather(self):
 
  to_be_raised = None
  log.debug(f"Gathered new results {len(results)}")
-
  # register the results
+ # NOTE: For Ptera instrumentation
+ trials = 0 # pylint:disable=unused-variable
  for result in results:
  trial = self.pending_trials.pop(result.future)
 
@@ -327,6 +343,8 @@ def gather(self):
  # NB: observe release the trial already
  self.client.observe(trial, result.value)
  self.trials += 1
+ # NOTE: For Ptera instrumentation
+ trials = self.trials # pylint:disable=unused-variable
  except InvalidResult as exception:
  # stop the optimization process if we received `InvalidResult`
  # as all the trials are assumed to be returning those
@@ -398,15 +416,19 @@ def _suggest_trials(self, count):
 
  # non critical errors
  except WaitingForTrials:
+ log.debug("Runner cannot sample because WaitingForTrials")
  break
 
  except ReservationRaceCondition:
+ log.debug("Runner cannot sample because ReservationRaceCondition")
  break
 
  except LockAcquisitionTimeout:
+ log.debug("Runner cannot sample because LockAcquisitionTimeout")
  break
 
  except CompletedExperiment:
+ log.debug("Runner cannot sample because CompletedExperiment")
  break
 
  return trials

src/orion/core/__init__.py

@@ -32,10 +32,10 @@
 __descr__ = "Asynchronous [black-box] Optimization"
 __version__ = VERSIONS["version"]
 __license__ = "BSD-3-Clause"
-__author__ = u"Epistímio"
-__author_short__ = u"Epistímio"
+__author__ = "Epistímio"
+__author_short__ = "Epistímio"
 __author_email__ = "[email protected]"
-__copyright__ = u"2017-2022, Epistímio"
+__copyright__ = "2017-2022, Epistímio"
 __url__ = "https://github.com/epistimio/orion"
 
 DIRS = AppDirs(__name__, __author_short__)
@@ -55,6 +55,7 @@ def define_config():
  define_experiment_config(config)
  define_worker_config(config)
  define_evc_config(config)
+ define_frontends_uri_config(config)
 
  config.add_option(
  "user_script_config",
@@ -73,6 +74,31 @@ def define_config():
  return config
 
 
+def define_frontends_uri_config(config):
+ """Create and define the field of frontends URI configuration."""
+
+ def parse_frontends_uri(data):
+ # Expect either a list of strings (URLs),
+ # or a string as comma-separated list of URLs
+ if isinstance(data, list):
+ return data
+ elif isinstance(data, str):
+ return [piece.strip() for piece in data.split(",")]
+ else:
+ raise RuntimeError(
+ f"frontends_uri: expected either a list of strings (URLs), "
+ f"or a string as comma-separated list of URLs, got {data}"
+ )
+
+ config.add_option(
+ "frontends_uri",
+ option_type=parse_frontends_uri,
+ default=[],
+ env_var="ORION_WEBAPI_FRONTENDS_URI",
+ help="List of frontends addresses allowed to send requests to Orion server.",
+ )
+
+
 def define_storage_config(config):
  """Create and define the fields of the storage configuration."""
  storage_config = Configuration()

src/orion/core/io/resolve_config.py

@@ -213,8 +213,9 @@ def fetch_config(args):
 
  local_config = unflatten(local_config)
 
+ backward_keys = ["storage", "experiment", "worker", "evc"]
  # For backward compatibility
- for key in ["storage", "experiment", "worker", "evc"]:
+ for key in backward_keys:
  subkeys = list(global_config[key].keys())
 
  # Arguments that are only supported locally
@@ -241,6 +242,11 @@ def fetch_config(args):
  local_config.setdefault(key, {})
  local_config[key][subkey] = value
 
+ # Keep other keys parsed from config file
+ for key in tmp_config.keys():
+ if key not in backward_keys:
+ local_config[key] = tmp_config[key]
+
  return local_config
 
 

src/orion/executor/dask_backend.py

@@ -52,6 +52,16 @@ def successful(self):
 
 
 class Dask(BaseExecutor):
+ """Wrapper around the dask client.
+
+ .. warning::
+
+ The Dask executor can be pickled and used inside a subprocess,
+ the pickled client will use the main client that was spawned in the main process,
+ but you cannot spawn clients inside a subprocess.
+
+ """
+
  def __init__(self, n_workers=-1, client=None, **config):
  super(Dask, self).__init__(n_workers=n_workers)
 

src/orion/executor/multiprocess_backend.py

@@ -156,6 +156,10 @@ class PoolExecutor(BaseExecutor):
  backend: str
  Pool backend to use; thread or multiprocess, defaults to multiprocess
 
+ .. warning::
+
+ Pickling of the executor is not supported, see Dask for a backend that supports it
+
  """
 
  BACKENDS = dict(
@@ -173,6 +177,12 @@ def __init__(self, n_workers=-1, backend="multiprocess", **kwargs):
 
  self.pool = PoolExecutor.BACKENDS.get(backend, ThreadPool)(n_workers)
 
+ def __setstate__(self, state):
+ self.pool = state["pool"]
+
+ def __getstate__(self):
+ return dict(pool=self.pool)
+
  def __enter__(self):
  return self
 
@@ -188,13 +198,6 @@ def close(self):
  if hasattr(self, "pool"):
  self.pool.shutdown()
 
- def __getstate__(self):
- state = super(PoolExecutor, self).__getstate__()
- return state
-
- def __setstate__(self, state):
- super(PoolExecutor, self).__setstate__(state)
-
  def submit(self, function, *args, **kwargs):
  try:
  return self._submit_cloudpickle(function, *args, **kwargs)

src/orion/serving/webapi.py

@@ -8,14 +8,74 @@
 
 """
 
+import logging
+
 import falcon
+from falcon_cors import CORS, CORSMiddleware
 
 from orion.serving.experiments_resource import ExperimentsResource
 from orion.serving.plots_resources import PlotsResource
 from orion.serving.runtime import RuntimeResource
 from orion.serving.trials_resource import TrialsResource
 from orion.storage.base import setup_storage
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+class MyCORSMiddleware(CORSMiddleware):
+ """Subclass of falcon-cors CORSMiddleware class.
+
+ Generate a HTTP 403 Forbidden response if request sender is not allowed
+ to access requested content.
+
+ Default middleware just prints a message in server side
+ (e.g. "Aborting response due to origin not allowed"), but still
+ sends content, so, a client ignoring headers might still access
+ data even if not allowed.
+
+ CORS middleware role is to add necessary "access-control-" headers to
+ response to mark it as allowed. So, a response lacking expected headers
+ after call to parent method `process_ressource()` can be considered
+ to not be delivered to request sender.
+
+ More info about CORS:
+ - https://developer.mozilla.org/fr/docs/Web/HTTP/CORS
+ - https://fr.wikipedia.org/wiki/Cross-origin_resource_sharing
+ """
+
+ def process_resource(self, req, resp, resource, *args):
+ """Generate a 403 Forbidden response if response is not allowed."""
+
+ cors_resp_headers_before = [
+ header
+ for header in resp.headers
+ if header.lower().startswith("access-control-")
+ ]
+ assert not cors_resp_headers_before, cors_resp_headers_before
+
+ super().process_resource(req, resp, resource, *args)
+
+ # We then verify if some access control headers were added to response.
+ # If not, reponse is not allowed.
+ # Special case: if request did not have an origin, it was certainly sent from
+ # a browser (ie. not another server), so CORS is not relevant.
+ cors_resp_headers_after = [
+ header
+ for header in resp.headers
+ if header.lower().startswith("access-control-")
+ ]
+ if not cors_resp_headers_after and req.get_header("origin"):
+ raise falcon.HTTPForbidden()
+
+
+class MyCORS(CORS):
+ """Subclass of falcon-cors CORS class to return a custom middleware."""
+
+ @property
+ def middleware(self):
+ return MyCORSMiddleware(self)
+
 
 class WebApi(falcon.API):
  """
@@ -24,7 +84,27 @@ class WebApi(falcon.API):
  """
 
  def __init__(self, config=None):
- super(WebApi, self).__init__()
+ # By default, server will reject requests coming from a server
+ # with different origin. E.g., if server is hosted at
+ # http://myorionserver.com, it won't accept an API call
+ # coming from a server not hosted at same address
+ # (e.g. a local installation at http://localhost)
+ # This is a Cross-Origin Resource Sharing (CORS) security:
+ # https://developer.mozilla.org/fr/docs/Web/HTTP/CORS
+ # To make server accept CORS requests, we need to use
+ # falcon-cors package: https://github.com/lwcolton/falcon-cors
+ frontends_uri = (
+ config["frontends_uri"]
+ if "frontends_uri" in config
+ else ["http://localhost:3000"]
+ )
+ logger.info(
+ "allowed frontends: {}".format(
+ ", ".join(frontends_uri) if frontends_uri else "(none)"
+ )
+ )
+ cors = MyCORS(allow_origins_list=frontends_uri)
+ super(WebApi, self).__init__(middleware=[cors.middleware])
  self.config = config
 
  setup_storage(config.get("storage"))