diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index 66c4560c3..d1174864a 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -4,6 +4,8 @@ categories: - title: '🚀 Features' labels: - 'feature' + - title: '🏗 Enhancements' + labels: - 'enhancement' - title: '🐛 Bug Fixes' labels: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 672baa354..c3503b33e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,12 +47,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Setup MongoDB - uses: supercharge/mongodb-github-action@1.3.0 - with: - mongodb-version: 4.2 - - name: Configure MongoDB - run: mongo orion_test --eval 'db.createUser({user:"user",pwd:"pass",roles:["readWrite"]});' - name: Install dependencies run: | python -m pip install --upgrade pip @@ -67,6 +61,35 @@ jobs: env_vars: PLATFORM,PYTHON name: codecov-umbrella fail_ci_if_error: false + mongodb: + needs: pretest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Setup MongoDB + uses: supercharge/mongodb-github-action@1.3.0 + with: + mongodb-version: 4.2 + - name: Configure MongoDB + run: mongo orion_test --eval 'db.createUser({user:"user",pwd:"pass",roles:["readWrite"]});' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox tox-gh-actions + - name: Test with tox + run: tox -e mongodb + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + file: ./coverage.xml + flags: backward + env_vars: PLATFORM,PYTHON + name: codecov-umbrella + fail_ci_if_error: false backward-compatibility: needs: pretest runs-on: ubuntu-latest diff --git a/README.rst b/README.rst index d159fe22f..4d7592ede 100644 --- a/README.rst +++ b/README.rst @@ -124,7 +124,7 @@ If you use Oríon for published work, please cite our work using the following b month = nov, year = 2020, publisher = {Zenodo}, - version = {v0.1.11}, + version = {v0.1.14}, doi = {10.5281/zenodo.3478592}, url = {https://doi.org/10.5281/zenodo.3478592} } diff --git a/ROADMAP.md b/ROADMAP.md index cc3a89ccc..56e53c6c1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -3,41 +3,16 @@ Last update December 3rd, 2020 ## Next releases - Short-Term -### v0.1.12 +### v0.1.15 #### Quick release for bug fixes -### v0.1.13 - -#### Default database backend -PickledDB will become the default database backend instead of MongoDB. This initiative reduces -initial configuration, allowing users to get started faster. - ### v0.2 -#### Journal Protocol Plugins -Offering: -- No need to setup DB, can use one's existing backend -- Can re-use tools provided by backend for visualizations, etc. -#### Python API - -Traditional `suggest`/`observe` interface - -```python -experiment = orion.client.register( - experiment='fct-dummy', - x='loguniform(0.1, 1)', verbose=True, message='running trial {trial.hash_name}') - -trial = experiment.suggest() -results = dummy(**trial.arguments) -experiment.observe(trial, results) -``` - -### Algorithms -Introducing new algorithms: [TPE](https://papers.nips.cc/paper/4443-algorithms-for-hyper-parameter-optimization.pdf), [HyperBand](https://arxiv.org/abs/1603.06560) +#### Native Multi-Processing support -## Next releases - Mid-Term +Added support for parallelism with auto-scaling. No more need to launch multiple workers +(though still supported) for parallelism, simply pass `--n-workers` or `workon(n_workers)`. -### v0.3 #### Generic `Optimizer` interface supporting various types of algorithms Change interface to support trial object instead of curated lists. This is necessary to support algorithms such as PBT. diff --git a/codecov.yml b/codecov.yml index f06cc7218..f09aac881 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,3 +1,6 @@ +ignore: + - "tests/**/*" + coverage: precision: 2 round: down diff --git a/conda/meta.yaml b/conda/meta.yaml index df7bf44ab..77e80b15a 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -27,6 +27,8 @@ requirements: - tabulate - AppDirs - plotly + - python-kaleido + - requests - pandas - falcon - gunicorn diff --git a/docs/src/index.rst b/docs/src/index.rst index 742835be5..06d98f509 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -19,6 +19,7 @@ user/algorithms user/searchspace user/monitoring + user/viz user/benchmark user/evc user/storage diff --git a/docs/src/install/database.rst b/docs/src/install/database.rst index 4cb64abdc..e52bd9e73 100644 --- a/docs/src/install/database.rst +++ b/docs/src/install/database.rst @@ -10,8 +10,8 @@ to enable fast, seamless, and scalable storage integration. Out of the box, we support three database backend: #. :ref:`EphemeralDB Config`, an in-memory database -#. :ref:`PickledDB Config`, a file-based database -#. :ref:`MongoDB Config`, a document-oriented database (default) +#. :ref:`PickledDB Config`, a file-based database (default) +#. :ref:`MongoDB Config`, a document-oriented database In this document, we'll review the different methods by which you can configure which database Oríon will use during its execution. This page also contains the :ref:`installation instructions for @@ -28,10 +28,10 @@ This will create a yaml file of the following format. .. code-block:: yaml - database: - type: 'mongodb' - name: 'orion_test' - host: 'mongodb://user:pass@localhost' + storage: + database: + type: 'pickleddb' + host: '/path/to/a/file.pkl' The file is typically located at ``$HOME/.config/orion.core/orion_config.yaml`` but it may differ based on your operating system. @@ -40,10 +40,8 @@ The second way of configuring the database backend is to use environment variabl .. code-block:: sh - ORION_DB_ADDRESS=mongodb://user:pass@localhost - ORION_DB_NAME=orion_test - ORION_DB_TYPE=MongoDB - ORION_DB_PORT=27017 + ORION_DB_ADDRESS=/path/to/a/file.pkl + ORION_DB_TYPE=PickledDB Note that both configuration methods can be used together, environment variables that are set will overwrite the corresponding values in the global configuration. This is useful if you need to define @@ -79,37 +77,27 @@ stage. $ orion db test Check for a configuration inside the default paths... - {'type': 'mongodb', 'name': 'mydb', 'host': 'localhost'} + {'name': 'orion', 'type': 'pickleddb', 'host': '', 'port': 27017} + Success + Check if configuration file has valid database configuration... + {'name': 'orion', 'type': 'pickleddb', 'host': '', 'port': 27017} Success - - Check for a configuration inside the environment variables... Skipping - No environment variables found. - - Check if configuration file has valid database configuration... Skipping - Missing configuration file. [...] -In the last example, we can observe that the last two tests were skipped because there were no -environment variables or local configuration file specified. - -Alternatively, here's an example including all three configuration methods. +Alternatively, here's an example including all configuration methods. +This is with MongoDB since there are more options to play with. .. code-block:: sh - $ ORION_DB_PORT=27018 + $ ORION_DB_NAME=test $ orion db test --config local.yaml Check for a configuration inside the global paths... - {'type': 'mongodb', 'name': 'mydb', 'host': 'localhost'} - Success - - Check for a configuration inside the environment variables... - {'type': 'mongodb', 'name': 'mydb', 'host': 'localhost', 'port': '27018'} + {'name': 'test', 'type': 'pickleddb', 'host': '', 'port': 27017} Success - Check if configuration file has valid database configuration... - {'type': 'mongodb', 'name': 'mydb', 'host': 'localhost', 'port': '27017'} + {'type': 'mongodb', 'host': 'localhost'} Success [...] @@ -123,9 +111,9 @@ that will be used and then prints the instance created to confirm the database t [...] - Using configuration: {'type': 'mongodb', 'name': 'mydb', 'host': 'localhost'} + Using configuration: {'name': 'orion', 'type': 'pickleddb', 'host': '', 'port': 27017} Check if database of specified type can be created... Success - DB instance + DB instance [...] @@ -234,7 +222,7 @@ To install MongoDB locally, follow the `official instructions `_ for your operating system. Alternatively, use :ref:`MongoDB Atlas ` to create a database in the cloud. -Once MondoDB is installed, create the database using: +Once MongoDB is installed, create the database using: .. code-block:: sh diff --git a/docs/src/tutorials/pytorch-mnist.rst b/docs/src/tutorials/pytorch-mnist.rst index f49134f06..0b9d0e4bd 100644 --- a/docs/src/tutorials/pytorch-mnist.rst +++ b/docs/src/tutorials/pytorch-mnist.rst @@ -7,8 +7,7 @@ This is a simple tutorial on running hyperparameter search with Oríon on Pytorc Installation and setup ====================== -Make sure Oríon is installed (:doc:`/install/core`) and the database is configured -(:doc:`/install/database`). +Make sure Oríon is installed (:doc:`/install/core`). Then install ``pytorch`` and ``torchvision`` and clone the PyTorch `examples repository`_: diff --git a/docs/src/tutorials/pytorch_a2c_ppo.rst b/docs/src/tutorials/pytorch_a2c_ppo.rst index 20c174034..e2f1e7008 100644 --- a/docs/src/tutorials/pytorch_a2c_ppo.rst +++ b/docs/src/tutorials/pytorch_a2c_ppo.rst @@ -6,13 +6,6 @@ PyTorch A2C PPO ACKTR If Oríon not installed: pip install orion - If the database is not setup, you can follow the instructions here: - :doc:`/install/database`. - - Alternatively, you can test the example without setting up a database by - using the option `--debug`, but note that all data gathered during an - execution will be lost at the end of it. - Intro ===== diff --git a/docs/src/tutorials/pytorch_cifar.rst b/docs/src/tutorials/pytorch_cifar.rst index c958561dd..18ff2867e 100644 --- a/docs/src/tutorials/pytorch_cifar.rst +++ b/docs/src/tutorials/pytorch_cifar.rst @@ -6,12 +6,6 @@ PyTorch CIFAR10 If Oríon not installed: pip install orion - If the database is not setup, you can follow the instructions here: - :doc:`/install/database`. - - Alternatively, you can test the example without setting up a database by - using the option `--debug`, but note that all data gathered during an - execution will be lost at the end of it. Set up diff --git a/docs/src/user/config.rst b/docs/src/user/config.rst index 9154c0ef0..7b700c125 100644 --- a/docs/src/user/config.rst +++ b/docs/src/user/config.rst @@ -88,10 +88,8 @@ Full Example of Global Configuration .. code-block:: yaml database: - host: localhost - name: orion - port: 27017 - type: mongodb + host: orion_db.pkl + type: pickleddb experiment: algorithms: @@ -136,10 +134,8 @@ Database .. code-block:: yaml database: - host: localhost - name: orion - port: 27017 - type: mongodb + host: orion_db.pkl + type: pickleddb .. _config_database_name: @@ -161,7 +157,7 @@ type ~~~~ :Type: str -:Default: MongoDB +:Default: PickledDB :Env var: ORION_DB_TYPE :Description: Type of database. Builtin backends are ``mongodb``, ``pickleddb`` and ``ephemeraldb``. @@ -174,10 +170,12 @@ host ~~~~ :Type: str -:Default: 127.0.1.1 +:Default: "" :Env var: ORION_DB_ADDRESS :Description: - URI for ``mongodb``, or file path for ``pickleddb``. + URI for ``mongodb``, or file path for ``pickleddb``. An empty + string will set the value depending on the database type + (localhost or /orion/orion_db.pkl). diff --git a/docs/src/user/viz.rst b/docs/src/user/viz.rst new file mode 100644 index 000000000..7a6ce8511 --- /dev/null +++ b/docs/src/user/viz.rst @@ -0,0 +1,118 @@ +************* +Visualization +************* + +.. contents:: + :depth: 2 + :local: + +Orion uses plotly to generate reports for experiments. +This section refers to the use of the `orion plot ...` subcommand. + +For details about the use of the web api giving +similar plots, refer to the documentation :doc:`/user/web_api`. + +===== +Usage +===== + +The arguments expected by the "plot" subcommand are as follows + +.. code-block :: console + + usage: orion plot [-h] [-n stringID] [-u USER] [-v VERSION] [-c path-to-config] + [-t {png,jpg,jpeg,webp,svg,pdf,html,json}] + [-o OUTPUT] [--scale SCALE] + {lpi,parallel_coordinates,partial_dependencies,regret} + + Produce plots for Oríon experiments + + positional arguments: + {lpi,parallel_coordinates,partial_dependencies,regret} + kind of plot to generate. + + optional arguments: + -h, --help show this help message and exit + -t {png,jpg,jpeg,webp,svg,pdf,html,json}, --type {png,jpg,jpeg,webp,svg,pdf,html,json} + type of plot to return. (default: png) + -o OUTPUT, --output OUTPUT + path where plot is saved. Will override the `type` argument. + (default is {exp.name}-v{exp.version}_{kind}.{type}) + --scale SCALE more pixels, but same proportions of the plot. + Scale acts as multiplier on height and width of resulting image. + Overrides value of 'scale' in plotly.io.write_image. + + Oríon arguments: + These arguments determine orion's behaviour + + -n stringID, --name stringID + experiment's unique name; + (default: None - specified either here or in a config) + -u USER, --user USER user associated to experiment's unique name; + (default: $USER - can be overriden either here or in a config) + -v VERSION, --version VERSION + specific version of experiment to fetch; (default: None - latest experiment.) + -c path-to-config, --config path-to-config + user provided orion configuration file + + +======================== +More about the arguments +======================== + +The `kind` positional argument is the only mandatory argument. +Assuming that the user identified the experiment in the usual +way (e.g. using ``--name`` or a config file), the default behavior +is to generate the correct `kind` of plot, and to save it +as a "png" file in the current directory and with a filename +automatically formatted as "{experiment.name}_{kind}.png". + +---- +kind +---- + +The plotting command requires a ``kind`` argument to determine which +of the four kinds of plots to generate. +The choice is between 'lpi', 'partial_dependencies', 'parallel_coordinates' +or 'regret'. + + +---- +type +---- + +The ``type`` is basically the filename extension. This governs more than just the name of the file +because it determines the actual format of the output. The default is to give the user a 'png' file. + +Behind the scenes, *plotly* generates an initial 'json' file, and renders it as an image +to be saved in the desired format. With ``type`` being 'json', that original file +is saved without rendering it to an image. + +The accepted values of ``type`` are 'png', 'jpg', 'jpeg', 'webp', 'svg', 'pdf', 'html' and 'json'. + +------ +output +------ + +The default value for the output filename is an automatically-generated +string formatted as "{experiment.name}_{kind}.{type}". +This also implies that the plot will be saved in the current directory. + +When ``output`` is specified by the user with a file extension, the ``type`` argument +will be ignored because the value of ``type`` will be instead inferred by +the file extension in the ``output``. +For example, with ``--output=../myplot.jpg``, the results will be saved +in the parent directory and the ``type`` will be 'jpg'. + +----- +scale +----- + +With certain types of plots, it can be desirable to increase the +resolution of the output image in terms of pixel counts (equivalent to dpi). +This applies particularly to 'jpg' and 'png', but it does not affect 'json' or 'html'. + +The reference value of ``scale`` is 1.0. +With ``--scale 2.0``, the height and width are going to be doubled. + + diff --git a/examples/benchmark/benchmark_get_start.ipynb b/examples/benchmark/benchmark_get_start.ipynb index ae272da4b..395449520 100644 --- a/examples/benchmark/benchmark_get_start.ipynb +++ b/examples/benchmark/benchmark_get_start.ipynb @@ -4,7 +4,15 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 3920 [ WARNING] TRACK [45747] track/configuration.py:40 No configuration file found\n" + ] + } + ], "source": [ "from orion.benchmark.task.base import BaseTask\n", "\n", @@ -32,9 +40,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "from orion.benchmark.benchmark_client import get_or_create_benchmark\n", @@ -45,7 +51,7 @@ "\n", "\n", "bm1 = get_or_create_benchmark(name='bm103', \n", - " algorithms=['random', 'tpe'],\n", + " algorithms=[{'algorithm': 'random'}, {'algorithm': 'tpe'}],\n", " targets=[\n", " {\n", " 'assess': [AverageResult(2), AverageRank(2)], \n", @@ -239,31 +245,31 @@ 24 ], "y": [ - 477713.9510267805, - 477713.9510267805, - 68920.7327486169, - 7890.7829312000995, - 5112.6733858798525, - 5112.6733858798525, - 5112.6733858798525, - 5112.6733858798525, - 5112.6733858798525, - 5112.6733858798525, - 3532.1300802882556, - 3532.1300802882556, - 3532.1300802882556, - 3532.1300802882556, - 3532.1300802882556, - 3532.1300802882556, - 3532.1300802882556, - 3532.1300802882556, - 3532.1300802882556, - 3532.1300802882556, - 3532.1300802882556, - 3532.1300802882556, - 3532.1300802882556, - 2442.279639329056, - 2442.279639329056 + 89950.67261976168, + 12180.330112508354, + 12180.330112508354, + 12180.330112508354, + 11706.148913344397, + 3009.6315204553566, + 847.6633765059223, + 847.6633765059223, + 847.6633765059223, + 847.6633765059223, + 847.6633765059223, + 847.6633765059223, + 847.6633765059223, + 596.3520971943294, + 596.3520971943294, + 596.3520971943294, + 596.3520971943294, + 467.58455588255663, + 467.58455588255663, + 467.58455588255663, + 467.58455588255663, + 467.58455588255663, + 467.58455588255663, + 467.58455588255663, + 467.58455588255663 ] }, { @@ -327,56 +333,56 @@ 0 ], "y": [ - -421663964345.8924, - -421663964345.8924, - -8102368482.392793, - -13712891.018121785, - -45369.70050443154, - -45369.70050443154, - -45369.70050443154, - -45369.70050443154, - -45369.70050443154, - -45369.70050443154, - -6047618.230744621, - -6047618.230744621, - -6047618.230744621, - -6047618.230744621, - -6047618.230744621, - -6047618.230744621, - -6047618.230744621, - -6047618.230744621, - -6047618.230744621, - -6047618.230744621, - -6047618.230744621, - -6047618.230744621, - -6047618.230744621, - -841433.8985140333, - -841433.8985140333, - 846318.4577926914, - 846318.4577926914, - 6054682.490905197, - 6054682.490905197, - 6054682.490905197, - 6054682.490905197, - 6054682.490905197, - 6054682.490905197, - 6054682.490905197, - 6054682.490905197, - 6054682.490905197, - 6054682.490905197, - 6054682.490905197, - 6054682.490905197, - 6054682.490905197, - 55595.047276191246, - 55595.047276191246, - 55595.047276191246, - 55595.047276191246, - 55595.047276191246, - 55595.047276191246, - 13728672.583984183, - 8102506323.858291, - 421664919773.7945, - 421664919773.7945 + -9036387157.521236, + -222698268.5052807, + -222698268.5052807, + -222698268.5052807, + -243163623.86167446, + -10856303.038448595, + -55730.88353185103, + -55730.88353185103, + -55730.88353185103, + -55730.88353185103, + -55730.88353185103, + -55730.88353185103, + -55730.88353185103, + -13220.560928696785, + -13220.560928696785, + -13220.560928696785, + -13220.560928696785, + -3700.336413914668, + -3700.336413914668, + -3700.336413914668, + -3700.336413914668, + -3700.336413914668, + -3700.336413914668, + -3700.336413914668, + -3700.336413914668, + 4635.505525679781, + 4635.505525679781, + 4635.505525679781, + 4635.505525679781, + 4635.505525679781, + 4635.505525679781, + 4635.505525679781, + 4635.505525679781, + 14413.265123085443, + 14413.265123085443, + 14413.265123085443, + 14413.265123085443, + 57426.21028486288, + 57426.21028486288, + 57426.21028486288, + 57426.21028486288, + 57426.21028486288, + 57426.21028486288, + 57426.21028486288, + 10862322.301489506, + 243187036.15950117, + 222722629.16550574, + 222722629.16550574, + 222722629.16550574, + 9036567058.866474 ] }, { @@ -414,31 +420,31 @@ 24 ], "y": [ - 303282.32409949135, - 237769.79780396336, - 219003.18425886348, - 78541.8656981642, - 75023.11405436415, - 75023.11405436415, - 75023.11405436415, - 75023.11405436415, - 75023.11405436415, - 7233.346497739345, - 7233.346497739345, - 7233.346497739345, - 7233.346497739345, - 4509.0154640992605, - 4509.0154640992605, - 4509.0154640992605, - 4509.0154640992605, - 4509.0154640992605, - 4509.0154640992605, - 4509.0154640992605, - 4085.16166360139, - 4085.16166360139, - 2112.7341566974765, - 2093.5728679322374, - 1813.6243758329001 + 251706.90738806495, + 137435.78927484696, + 137435.78927484696, + 3785.1127658459955, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947, + 1127.3637185146947 ] }, { @@ -502,56 +508,56 @@ 0 ], "y": [ - -180735822144.61002, - -110544050867.62187, - -93600296380.63548, - -11513149526.678257, - -10470012396.5013, - -10470012396.5013, - -10470012396.5013, - -10470012396.5013, - -10470012396.5013, - -41649934.72123967, - -41649934.72123967, - -41649934.72123967, - -41649934.72123967, - -6763035.934402222, - -6763035.934402222, - -6763035.934402222, - -6763035.934402222, - -6763035.934402222, - -6763035.934402222, - -6763035.934402222, - -10241485.774341771, - -10241485.774341771, - -167167.3493414102, - -190219.166171907, - -694479.3564773018, - 698106.6052289675, - 194406.31190777148, - 171392.81765480514, - 10249656.097668974, - 10249656.097668974, - 6772053.96533042, - 6772053.96533042, - 6772053.96533042, - 6772053.96533042, - 6772053.96533042, - 6772053.96533042, - 6772053.96533042, - 41664401.414235145, - 41664401.414235145, - 41664401.414235145, - 41664401.414235145, - 10470162442.729408, - 10470162442.729408, - 10470162442.729408, - 10470162442.729408, - 10470162442.729408, - 11513306610.409653, - 93600734387.00401, - 110544526407.21748, - 180736428709.2582 + -25351706488.121758, + -5530689.05307808, + -5530689.05307808, + -12734287.667081306, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + -34813.618196954696, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 37068.345633984085, + 12741857.892613, + 5805560.631627774, + 5805560.631627774, + 25352209901.93653 ] } ], @@ -1379,9 +1385,9 @@ } }, "text/html": [ - "