diff --git a/.gitignore b/.gitignore index 883f23ca..4e80476e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .pytest_cache +.DS_Store + # Compiled files *.py[co] *.a diff --git a/CHANGELOG.rst b/CHANGELOG.rst deleted file mode 100644 index a899deee..00000000 --- a/CHANGELOG.rst +++ /dev/null @@ -1,164 +0,0 @@ -.. X.Y.Z (Mon DD, YYYY) -.. ------------------- -.. -.. Summary -.. +++++++ -.. -.. - Released Mon DD, YYYY -.. - X contributor(s) -.. - X pull requests -.. -.. **Description** -.. -.. bla bla. -.. -.. **Contributors:** -.. -.. In alphabetical order by first name: -.. -.. - ... -.. -.. Pull Requests -.. +++++++++++++ -.. -.. - [#X] Title (Author) - -.. _protopipe_0p3_release: - -0.3 (Unreleased) ------------------ - -Planned for early summer 2020. - -.. _gammapy_0p2p1_release: - -`0.2.1 `__ (Oct 28, 2019) -------------------------------------------------------------------------------------------- - -Summary -+++++++ - -- Released Oct 28, 2019 -- 1 contributor -- 1 pull requests - -**Description** - -The ctapipe-based cleaning algorithm for the biggest cluster was crashing in -case of cleaned images with no surviving pixel clusters. - -**Contributors:** - -In alphabetical order by first name: - -- Michele Peresano - -Pull Requests -+++++++++++++ - -- [#16] Bugfix: Closes #15 (Michele Peresano) - -`0.2.0 `__ (Oct 24, 2019) -------------------------------------------------------------------------------------------- - -Summary -+++++++ - -- Released Oct 24, 2019 -- 3 contributor(s) -- 7 pull requests - -**Description** - -*protopipe* 0.2 now fully supports the stable release of *ctapipe* 0.7.0. The main improvements involve the calibration process (high gain selected by default), the direction reconstruction and new camera-type labels. - -Code based on *pywi*/*pywi-cta* libraries, relevant for wavelet-based image cleaning, has been removed in favor of *ctapipe* or made completely optional where needed. Wavelet cleaning is still optional but will need those two libraries to be additionally installed. Tailcut-based cleaning is now faster. - -The README has been improved with installation, basic use, and developer instructions. Dependencies are listed in `protopipe_environment.yaml` and have been simplified. - -The auxiliary scripts `merge_tables.py` and `merge.sh` have been added to allow merging of DL1 and DL2 HDF5 tables. - -The `mars_cleaning_1st_pass` method is now imported from _ctapipe_. Novel code using the largest cluster of survived pixels (`number_of_islands` and `largest_island` methods in the `event_preparer` module) has been hardcoded in _protopipe_ and will disappear with the next release of _ctapipe_. - -Model estimators now load the camera types directly from the `analysis .yaml` configuration file. - -**Contributors:** - -In alphabetical order by first name: - -- Alice Donini -- Michele Peresano -- Thierry Stolarczyk - -Pull Requests -+++++++++++++ - -This list is incomplete. Small improvements and bug fixes are not listed here. -The complete list is found `here `__. - -- [#9] Update image cleaning and make wavelet-based algorithms independent -- [#8] Import CTA-MARS 1st pass cleaning from ctapipe - -`0.1.1 `__ (Oct 1, 2019) ------------------------------------------------------------------------------------------- - -Summary -+++++++ - -- Released Oct 1, 2019 -- X contributor(s) -- X pull request(s) - -**Description** - -The `write_dl1` and `write_dl2` tools can now save an additional file through the flag `--save-images` when applied to a single run. This file will contain the original and calibrated (after gain selection) photoelectron images per event. - -A new method `save_fig` has been introduced in the `utils` module, so that `model_diagnostic` can save images also in PNG format. - -Additional docstrings and PEP8 formatting have been added throughout the code. - -**Contributors:** - -In alphabetical order by first name: - -- ... - -Pull Requests -+++++++++++++ - -The development of *protopipe* on GitHub started out directly in the master branch, -so there are no pull request we can list here. - -`0.1.0 `__ (Sep 23, 2019) -------------------------------------------------------------------------------------------- - -Summary -+++++++ - -- Released Sep 23, 2019 -- 6 contributor(s) -- 1 pull request(s) - -**Description** - -First version of *protopipe* to be publicly release on GitHub. -This version is based on ctapipe 0.6.2 (conda package stable version). -Its performance has been shown in a -`presentation `__ -at the CTAC meeting in Lugano 2019. - -**Contributors:** - -In alphabetical order by first name: - -- David Landriu -- Julien Lefacheur -- Karl Kosack -- Michele Peresano -- Thomas Vuillaume -- Tino Michael - -Pull Requests -+++++++++++++ - -- [#2] Custom arrays, example configs and aux scripts (M.Peresano) diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..757e4beb --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,48 @@ +#=============================================================================== +# SYNTAX EXPLANATION - GO DOWN TO FIND THE EDIT SECTION +#=============================================================================== + +# This is a comment. +# Each line is a file pattern followed by one or more owners. + +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence, +# @global-owner1 and @global-owner2 will be requested for +# review when someone opens a pull request. +# * @global-owner1 @global-owner2 + +# Order is important; the last matching pattern takes the most +# precedence. When someone opens a pull request that only +# modifies JS files, only @js-owner and not the global +# owner(s) will be requested for a review. +# *.js @js-owner + +# You can also use email addresses if you prefer. They'll be +# used to look up users just like we do for commit author +# emails. +# *.go docs@example.com + +# In this example, @doctocat owns any files in the build/logs +# directory at the root of the repository and any of its +# subdirectories. +# /build/logs/ @doctocat + +# The `docs/*` pattern will match files like +# `docs/getting-started.md` but not further nested files like +# `docs/build-app/troubleshooting.md`. +# docs/* docs@example.com + +# In this example, @octocat owns any file in an apps directory +# anywhere in your repository. +# apps/ @octocat + +# In this example, @doctocat owns any file in the `/docs` +# directory in the root of your repository and any of its +# subdirectories. +# /docs/ @doctocat + +#=============================================================================== +# EDIT SECTION +#=============================================================================== + +* @HealthyPear diff --git a/docs/AUTHORS.rst b/docs/AUTHORS.rst new file mode 100644 index 00000000..4a18c8b2 --- /dev/null +++ b/docs/AUTHORS.rst @@ -0,0 +1,20 @@ +.. _authors: + +Authors +======= + +Current contributors are listed + +- `here `__ for ``protopipe`` +- Michele Peresano for the ``GRID interface`` + + +A full list can be obtained also by issuing the following command from each +source code folder, + +.. code-block:: bash + + git shortlog -sne --no-merges + +Previous authors which contributed to ``protopipe`` are also Julien Lefacheur, Jeremie Decock and Tino Michael. +Julien Lefacheur has been also the previous author of the ``GRID interface``. diff --git a/docs/changelog.rst b/docs/changelog.rst index 450c5ffa..264dc72a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,23 +1,237 @@ .. _changelog: +.. _@HealthyPear: https://github.com/HealthyPear +.. _@gaia-verna: https://github.com/gaia-verna +.. _@kosack: https://github.com/kosack +.. _@tstolarczyk: https://github.com/tstolarczyk +.. _@vuillaut: https://github.com/vuillaut + Changelog ========= -This is the changelog for *protopipe*. -For now we use a one-line description of every pull request, roughly in chronological order. +.. _protopipe_0p4_release: + +`0.4.0 `__ (TBD) +---------------------------------------------------------------------------------- + +. . . + +.. _protopipe_0p3_release: + +`0.3.0 `__ (Nov 9th, 2020) +-------------------------------------------------------------------------------------------- + +Summary ++++++++ + +- early improvements related to the DL1 comparison against the CTAMARS pipeline +- improvements to basic maintenance +- a more consistent approach for full-scale analyses +- bug fixes + +Contributors +++++++++++++ + +- Michele Peresano (`@HealthyPear`_) +- Thierry Stolarczyk (`@tstolarczyk`_) +- Gaia Verna (`@gaia-verna`_) +- Karl Kosack (`@kosack`_) +- Thomas Vuillaume (`@vuillaut`_) + +Changes from previous release ++++++++++++++++++++++++++++++ + +🚀 General features +^^^^^^^^^^^^^^^^^^^ + +- Add missing variables in write\_dl2 (:pr:`66`) `@HealthyPear`_ +- Add missing dl1 parameters (:pr:`41`) `@HealthyPear`_ +- Updates on notebooks (:pr:`47`) `@HealthyPear`_ +- New plots for calibration benchmarking (:pr:`43`) `@HealthyPear`_ +- Double-pass image extractor (:pr:`48`) `@HealthyPear`_ +- Notebooks for low-level benchmarking (:pr:`42`) `@HealthyPear`_ +- Improved handling of sites, arrays and cameras for all Prod3b simtel productions (:pr:`33`) `@HealthyPear`_ +- Change gain selection (:pr:`35`) `@HealthyPear`_ +- Changes for adding Cameras beyond LSTCam and NectarCam (:pr:`29`) `@tstolarczyk`_ + +🌐 GRID support +^^^^^^^^^^^^^^^ + +- Update configuration files (:pr:`74`) `@HealthyPear`_ +- Update documentation for GRID support (:pr:`54`) `@HealthyPear`_ +- Rollback for GRID support (:pr:`52`) `@HealthyPear`_ + +🐛 Bug Fixes +^^^^^^^^^^^^ + +- Bugfix in Release Drafter workflow file (:pr:`71`) `@HealthyPear`_ +- Convert pointing values to float64 at reading time (:pr:`68`) `@HealthyPear`_ +- Rollback for GRID support (:pr:`52`) `@HealthyPear`_ +- Fix recording of DL1 image and record reconstruction cleaning mask (:pr:`46`) `@gaia-verna`_ +- consistent definition of angular separation to the source with config (:pr:`39`) `@vuillaut`_ +- Update write\_dl1.py (:pr:`30`) `@tstolarczyk`_ + +🧰 Maintenance +^^^^^^^^^^^^^^ + +- Update benchmarks and documentation (:pr:`75`) `@HealthyPear`_ +- Bugfix in Release Drafter workflow file (:pr:`71`) `@HealthyPear`_ +- Add release drafter (:pr:`67`) `@HealthyPear`_ +- Add benchmark notebooks for medium and late stages (:pr:`55`) `@HealthyPear`_ +- Update documentation for GRID support (:pr:`54`) `@HealthyPear`_ +- Updated documentation (:pr:`50`) `@HealthyPear`_ +- Implementation of a first unit test (DL1) (:pr:`34`) `@HealthyPear`_ +- Updated documentation (Closes #23) (:pr:`32`) `@HealthyPear`_ +- Added Travis CI configuration file (:pr:`18`) `@HealthyPear`_ +- Update README.md (:pr:`28`) `@tstolarczyk`_ +- Added versioning to init.py and setup.py using the manual approach. (:pr:`20`) `@HealthyPear`_ +- Update README.md (:pr:`21`) `@tstolarczyk`_ + + +.. _gammapy_0p2p1_release: + +`0.2.1 `__ (Oct 28th, 2019) +--------------------------------------------------------------------------------------------- + +Summary ++++++++ + +- Released Oct 28, 2019 +- 1 contributor +- 1 pull requests + +**Description** + +The ctapipe-based cleaning algorithm for the biggest cluster was crashing in +case of cleaned images with no surviving pixel clusters. + +**Contributors:** + +In alphabetical order by first name: + +- Michele Peresano + +Pull Requests ++++++++++++++ + +- (:pr:`16`) Bugfix: Closes #15 (Michele Peresano) + +`0.2.0 `__ (Oct 24th, 2019) +--------------------------------------------------------------------------------------------- + +Summary ++++++++ + +- Released Oct 24, 2019 +- 3 contributor(s) +- 7 pull requests -We don't list every pull request. -Maintenance and cleanup changes are not of interest to users and are not listed here. +**Description** -As of today, the people that contributed to *protopipe* are, in alphabetical order by first name: +*protopipe* 0.2 now fully supports the stable release of *ctapipe* 0.7.0. + +The main improvements involve the calibration process +(high gain selected by default), +the direction reconstruction and new camera-type labels. + +Code based on *pywi*/*pywi-cta* libraries, relevant for wavelet-based image +cleaning, has been removed in favor of *ctapipe* or made completely optional +where needed. Wavelet cleaning is still optional but will need those two +libraries to be additionally installed. Tailcut-based cleaning is now faster. + +The README has been improved with installation, basic use, and developer instructions. +Dependencies are listed in ``protopipe_environment.yaml`` and have been simplified. + +The auxiliary scripts ``merge_tables.py`` and ``merge.sh`` have been added to allow merging of DL1 and DL2 HDF5 tables. + +The ``mars_cleaning_1st_pass`` method is now imported from _ctapipe_. +Novel code using the largest cluster of survived pixels +(``number_of_islands`` and ``largest_island`` methods in the +``event_preparer`` module) has been hardcoded in _protopipe_ and will +disappear with the next release of _ctapipe_. + +Model estimators now load the camera types directly from the ``analysis .yaml`` configuration file. + +**Contributors:** + +In alphabetical order by first name: - Alice Donini +- Michele Peresano +- Thierry Stolarczyk + +Pull Requests ++++++++++++++ + +This list is incomplete. Small improvements and bug fixes are not listed here. + +The complete list is found `here `__. + +- (:pr:`9`) Update image cleaning and make wavelet-based algorithms independent +- (:pr:`8`) Import CTA-MARS 1st pass cleaning from ctapipe + +`0.1.1 `__ (Oct 1st, 2019) +-------------------------------------------------------------------------------------------- + +Summary ++++++++ + +- Released Oct 1, 2019 +- X contributor(s) +- X pull request(s) + +**Description** + +The ``write_dl1`` and ``write_dl2`` tools can now save an additional file +through the flag ``--save-images`` when applied to a single run. +This file will contain the original and calibrated (after gain selection) +photoelectron images per event. +A new method ``save_fig`` has been introduced in the ``utils`` module, +so that ``model_diagnostic`` can save images also in PNG format. +Additional docstrings and PEP8 formatting have been added throughout the code. + +**Contributors:** + +In alphabetical order by first name: + +- ... + +Pull Requests ++++++++++++++ + +The development of *protopipe* on GitHub started out directly in the master branch, +so there are no pull request we can list here. + +`0.1.0 `__ (Sep 23th, 2019) +--------------------------------------------------------------------------------------------- + +Summary ++++++++ + +- Released Sep 23, 2019 +- 6 contributor(s) +- 1 pull request(s) + +**Description** + +First version of *protopipe* to be publicly release on GitHub. +This version is based on ctapipe 0.6.2 (conda package stable version). +Its performance has been shown in a +`presentation `__ +at the CTAC meeting in Lugano 2019. + +**Contributors:** + +In alphabetical order by first name: + - David Landriu -- Julien Lefacheur (left) +- Julien Lefacheur - Karl Kosack -- Michele Peresano (current main developer/manteiner) -- Thierry Stolarczyk +- Michele Peresano - Thomas Vuillaume -- Tino Michael (left) +- Tino Michael + +Pull Requests ++++++++++++++ -.. include:: ../CHANGELOG.rst +- (:pr:`2`) Custom arrays, example configs and aux scripts (M.Peresano) diff --git a/docs/conf.py b/docs/conf.py index d133f200..c38e0dce 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,9 +18,8 @@ # import os import sys -from protopipe import __version__ - sys.path.insert(0, os.path.abspath("..")) +from protopipe import __version__ # -- Project information ----------------------------------------------------- @@ -34,9 +33,9 @@ # built documents. # # The short X.Y version. -version = f"{__version__}" +version = __version__ # The full version, including alpha/beta/rc tags. -release = f"{__version__}" +release = __version__ # -- General configuration ------------------------------------------------ @@ -49,6 +48,7 @@ # ones. extensions = [ "sphinx.ext.autodoc", + "sphinx_issues", "numpydoc", "sphinx.ext.doctest", "sphinx.ext.intersphinx", @@ -102,13 +102,13 @@ autoclass_content = "both" # include both class docstring and __init__ -autodoc_default_options = { - # Make sure that any autodoc declarations show the right members - "members": True, - "inherited-members": True, - "private-members": True, - "show-inheritance": True, -} +# autodoc_default_options = { +# # Make sure that any autodoc declarations show the right members +# "members": True, +# "inherited-members": True, +# "private-members": True, +# "show-inheritance": True, +# } autosummary_generate = True # Make _autosummary files and include them napoleon_numpy_docstring = False # Force consistency, leave only Google @@ -170,6 +170,15 @@ ) ] +# -- Options for sphinx issues ----- + +# GitHub repo +issues_github_path = "cta-observatory/protopipe" + +# equivalent to +issues_uri = "https://github.com/cta-observatory/protopipe/issues/{issue}" +issues_pr_uri = "https://github.com/cta-observatory/protopipe/pull/{pr}" +issues_commit_uri = "https://github.com/cta-observatory/protopipe/commit/{commit}" # -- Options for manual page output --------------------------------------- diff --git a/docs/contribute/beforepushing.rst b/docs/contribute/beforepushing.rst index 7cada41b..610c6d20 100644 --- a/docs/contribute/beforepushing.rst +++ b/docs/contribute/beforepushing.rst @@ -3,9 +3,50 @@ Code and performance checks =========================== +.. contents:: Index + :local: + :depth: 2 + +Documentation +------------- + +Each Pull Request should come with its own documentation updates, according to +what are the changes to be merged into master. + +The documentation is generated by `Sphinx `_ +with `reStructuredText `_ syntax. + +To build and check your documentation locally, + +- cd `docs` +- for big changes (or just to be sure), `rm api/* && make clean && make html` +- for small changes, `make html` + +The built documentation will be stored under `docs/_build/html` from which you +can open `index.html` with your favorite browser. + +Please, try to fix any warning that could appear during documentation building. + +.. warning:: + + The Continuous Integration (CI) pipeline takes care of building the documentation + once it is in master, but not for the Pull Request. + + Currently the documentation is built on GitHub Pages with Travis CI. + There doesn't seem to be a way to see the documentation related to the PR + online. + + For a next release, PR (:pr:`62`) will add the documentation also on + `readthedocs `_ which allows this functionality. + Tests and unit-tests -------------------- +.. note:: + This is a maintenance activity which has being long overdue and we need + manpower for it, so if you have experience on this or you want to contribute + please feel free to do so. + Being *protopipe* based on *ctapipe*, all the tools imported from the latter have been already tested and approved (remember that *protopipe* uses always the latest version of *ctapipe* which has been released in the Anaconda framework). diff --git a/docs/contribute/gitrepo.rst b/docs/contribute/gitrepo.rst index 6bcad270..9f5e6aa2 100644 --- a/docs/contribute/gitrepo.rst +++ b/docs/contribute/gitrepo.rst @@ -3,19 +3,17 @@ The repository ============== -Is useful for both basic users and developers to monitor the status of the -development of *protopipe*. +Useful for monitoring the development status of of *protopipe*. -Start from the **projects** tab, which currently consists of +The repository is organized in projects, which you can access from the +`Projects tab `_. -- *Next release*, -- *Development of new algorithms*, -- *Bugs*. +They don't come with specific deadlines because they are meant to +give a continuous overview regardless of software versioning. -They don't come with specific deadlines, because they are meant to -give a continuous overview regardless of versioning. - -Please, if what you have in mind is not already covered open a new issue. +.. contents:: Current projects + :local: + :depth: 2 Next release ------------ @@ -71,3 +69,18 @@ The project is divided in the following sections: - *In progress*, pull-requests opened to solve either of the prioritized issues of this project (could be under review or stale), - *Done*, are closed issues or approved and merged pull-requests. + +Maintenance +----------- + +This project collects all open issues and pull-requests related to the +work needed to keep everything up-to-date and running in a safe development +environment, namely + +- documentation +- unit-testing +- integration testing +- Continuous Integration (CI) +- release automation + +It is organized as the *Next release* project diff --git a/docs/contribute/index.rst b/docs/contribute/index.rst index 28c21a71..2de15529 100644 --- a/docs/contribute/index.rst +++ b/docs/contribute/index.rst @@ -3,7 +3,18 @@ How to contribute ================= +If after using or testing *protopipe* you find that something is missing, +wrong, or simply doesn't do what you expect, you are invited to join the +development effort. + +This section contains, + +- information on how the project is organized (:ref:`gitrepo`), +- instructions on how to start adding your contribution (:ref:`instructions`), +- what you can check before adding it (:ref:`beforepushing`). + .. toctree:: + :hidden: :maxdepth: 1 gitrepo diff --git a/docs/contribute/instructions.rst b/docs/contribute/instructions.rst index 61f17fe2..bd5a40fc 100644 --- a/docs/contribute/instructions.rst +++ b/docs/contribute/instructions.rst @@ -3,6 +3,10 @@ Instructions ============ +.. contents:: + :local: + :depth: 2 + These are some guidelines on how to contribute to *protopipe*. This of course makes sense only for the development branch, aka the *master* branch. @@ -86,3 +90,11 @@ and updated. If your changes are relatively small and `you know what you are doing `_, you can use ``git rebase master``, instead of merging. + +Making your contribution visible +-------------------------------- + +Together with your changes, you should check that, + +- the email that you want to use is listed in the ``.mailmap`` +- your name appears in the ``CODEOWNERS`` file according to your contribution diff --git a/docs/index.rst b/docs/index.rst index a2a2c3a1..a3acc501 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,7 +26,8 @@ which this documentation is linked. install/index usage/index contribute/index - changelog + AUTHORS + CHANGELOG .. _protopipe_structure: .. toctree:: diff --git a/docs/install/basic.rst b/docs/install/basic.rst deleted file mode 100644 index daaf11b1..00000000 --- a/docs/install/basic.rst +++ /dev/null @@ -1,30 +0,0 @@ -.. _install-basic: - -Basic users -=========== - -If you are a user with no interest in developing *protopipe*, you can start by -downoading the `latest released version `__ - -Steps for installation: - - 1. uncompress the file which is always called *protopipe-X.Y.Z* depending on version, - 2. enter the folder ``cd protopipe-X.Y.Z`` - 3. create a dedicated environment with ``conda env create -f environment.yml`` - 4. activate it with ``conda activate protopipe`` - 5. install *protopipe* itself with ``python setup.py install``. - -.. warning:: - Given that *protopipe* is undergoing fast development, it is likely that you - will benefit more from a more recent version of the code for now. - - The development version could disrupt functionalities that were working for - you, but the latest released version could lack some of those you need. - - To install the latest development version go to :ref:`install-developer`. - -Next steps: - - * get accustomed to the basic pipeline workflow (:ref:`use-pipeline`), - * then make your own complete analysis (:ref:`use-grid`), - * for bugs and new features, please contribute to the project (:ref:`contribute`). diff --git a/docs/install/developers.rst b/docs/install/developers.rst deleted file mode 100644 index 9ef3e088..00000000 --- a/docs/install/developers.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _install-developer: - -Developers -========== - -If you want to use *protopipe* and also contribute to its development, follow these steps: - - 1. Fork the official `repository `_ has explained `here `__ (follow all the instructions) - 2. now your local copy is linked to your remote repository (**origin**) and the official one (**upstream**) - 3. execute points 3 and 4 in the instructions for :ref:`install-basic`. - 4. install *protopipe* itself in developer mode with ``python setup.py develop`` - -In this way, you will always use the version of the source code on which you -are working. - -Next steps: - - * get accustomed to the basic pipeline workflow (:ref:`use-pipeline`), - * then make your own complete analysis (:ref:`use-grid`), - * for bugs and new features, please contribute to the project (:ref:`contribute`). diff --git a/docs/install/development.rst b/docs/install/development.rst new file mode 100644 index 00000000..55743a8a --- /dev/null +++ b/docs/install/development.rst @@ -0,0 +1,18 @@ +.. _install-development: + +Development version +=================== + + 1. `fork `__ the `repository `_ + 3. ``conda env create -f environment.yml`` + 4. ``conda activate protopipe`` + 5. ``pip install -e .`` + +In this way, you will always use the version of the source code on which you +are working. + +Next steps: + + * get accustomed to the basic pipeline workflow (:ref:`use-pipeline`), + * make your own complete analysis (:ref:`use-grid`), + * for bugs and new features, please contribute to the project (:ref:`contribute`). diff --git a/docs/install/grid.rst b/docs/install/grid.rst index 2316fdf6..cc2341a0 100644 --- a/docs/install/grid.rst +++ b/docs/install/grid.rst @@ -3,39 +3,52 @@ Grid environment ================ -General requirements --------------------- +Requirements +------------ -Regardless of what operating system you are using, in order to be able to use -*protopipe* on the grid you will need to be able to interface with it. +* credentials and certificate for using the GRID (follow `these instructions `__.) +* `Vagrant `_ -Go through the initial steps described -`here `__. +Getting a released version +-------------------------- -In the following it is then assumed that you have: +You can fine the latest released version `here `__ -* a Grid identity and a CTA VO registration, -* a certificate and a user key in the hidden folder ``$HOME/.globus`` +.. list-table:: compatibility between *protopipe* and its interface + :widths: 25 25 + :header-rows: 0 -Installation instructions -------------------------- + * - protopipe + - GRID interface + * - v0.2.X + - v0.2.X + * - v0.3.X + - v0.2.X -The installation instructions are the following: +Getting the development version +------------------------------- -* get the interface code (``git clone https://github.com/HealthyPear/protopipe-grid-interface.git``), -* create and enter a folder where to work, -* **(only Windows/macos users)** copy the ``VagrantFile`` from the parent folder of `protopipe-grid-interface`, -* **(only Windows/macos users)** edit the first arguments of lines from 47 to 50 according to your local setup, -* create the `data` shared folder to interact externally with the analysis products, -* **(only Windows/macos users)** enter in the virtual environment (``vagrant up && vagrant ssh``), -* get the Singularity container (``singularity pull --name CTADIRAC_with_protopipe.sif shub://HealthyPear/CTADIRAC``), -* enter in it (``singularity shell CTADIRAC_with_protopipe.sif``). +This version is always compatible with the latest release of *protopipe* and +its development version. -From here you should be able to activate the DIRAC environment (``dirac-proxy-init``). -The container provides the latest version of CTADIRAC and the necessary python -modules for the `protopipe-grid` code to run. -Now you can proceed with the analysis workflow (:ref:`use-grid`). +``git clone https://github.com/HealthyPear/protopipe-grid-interface.git`` + +Setup the working environment +----------------------------- + +1. create and enter a folder where to work, +2. copy the ``VagrantFile`` from the interface +3. edit lines from 48 to 59 according to your local setup +4. ``vagrant up && vagrant ssh`` +5. ``singularity pull --name CTADIRAC_with_protopipe.sif shub://HealthyPear/CTADIRAC`` +6. ``singularity shell CTADIRAC_with_protopipe.sif`` + +From here, -.. Note:: - For Linux users, the host's $HOME coincides with the guests' one by default. - For macos/Windows users, this has been set when you edited the VagrantFile. +- activate the GRID environment with ``dirac-proxy-init`` +- the ``shared_folder`` contains the folders + + - ``analyses`` to store all your analyses + - ``productions`` to store lists of simulated files + +Now you can proceed with the analysis workflow (:ref:`use-grid`). diff --git a/docs/install/index.rst b/docs/install/index.rst index 41c99c0f..376cb836 100644 --- a/docs/install/index.rst +++ b/docs/install/index.rst @@ -8,10 +8,11 @@ Python 3. There are two different ways to install `protopipe`, -* if you just want to use it as it is (:ref:`install-basic`), -* or if you also want to develop it (:ref:`install-developer`). +* if you just want to use it as it is (:ref:`install-release`), +* or if you also want to develop it (:ref:`install-development`). -In both cases you will need some computational power in order to produce enough +In both cases, if you want to perform a full analysis, will need some +computational power in order to produce enough data files for model and performance estimation. This can be accomplished through the use of a GRID environment. @@ -20,27 +21,10 @@ After installing `protopipe`, * install the code necessary to interface it with the grid (:ref:`install-grid`), * use protopipe on the grid (:ref:`use-grid`). -.. Note:: - - For a faster use, edit your preferred login script (e.g. ``.bashrc`` on Linux or - ``.profile`` on macos) with a function that initializes the environment. - The following is a minimal example using Bash. - - .. code-block:: bash - - alias protopipe="protopipe_init" - - function protopipe_init() { - - conda activate protopipe # Then activate the protopipe environment - export PROTOPIPE=$WHEREISPROTOPIPE/protopipe # A shortcut to the scripts folder - - } - .. toctree:: :hidden: :maxdepth: 1 - basic - developers + release + development grid diff --git a/docs/install/release.rst b/docs/install/release.rst new file mode 100644 index 00000000..15eda52e --- /dev/null +++ b/docs/install/release.rst @@ -0,0 +1,16 @@ +.. _install-release: + +Released version +================ + +1. download the `latest released version `__ +2. ``cd protopipe-X.Y.Z`` +3. ``conda env create -f environment.yml -n protopipe-X.Y.Z`` +4. ``conda activate protopipe`` +5. ``pip install .`` + +Next steps: + + * get accustomed to the basic pipeline workflow (:ref:`use-pipeline`), + * make your own complete analysis (:ref:`use-grid`), + * for bugs and new features, please contribute to the project (:ref:`contribute`). diff --git a/docs/pipeline/double-pass-image-extraction.png b/docs/pipeline/double-pass-image-extraction.png new file mode 100644 index 00000000..beb2b332 Binary files /dev/null and b/docs/pipeline/double-pass-image-extraction.png differ diff --git a/docs/pipeline/index.rst b/docs/pipeline/index.rst index eef1104a..2840c686 100644 --- a/docs/pipeline/index.rst +++ b/docs/pipeline/index.rst @@ -39,6 +39,12 @@ The current calibration is performed using: * charge and pulse times extraction via ``ctapipe.image.extractors.TwoPassWindowSum`` * no integration correction +.. figure:: ./double-pass-image-extraction.png + :width: 800 + :alt: Explanation of ``ctapipe.image.extractors.TwoPassWindowSum`` + + Explanation of ``ctapipe.image.extractors.TwoPassWindowSum`` + The resulting **optimized cleaning thresholds** for LSTCam and NectarCam when requiring 99.7% rejection of the "noise" (0 true phes) are (4.2, 2.1) for LSTCam and (4., 2.) for NectarCam. diff --git a/docs/usage/GRID_workflow.png b/docs/usage/GRID_workflow.png new file mode 100644 index 00000000..09fe096a Binary files /dev/null and b/docs/usage/GRID_workflow.png differ diff --git a/docs/usage/index.rst b/docs/usage/index.rst index 7ec8cb99..8d747bbc 100644 --- a/docs/usage/index.rst +++ b/docs/usage/index.rst @@ -12,7 +12,7 @@ Workflow and usage You should have at least a working installation of protopipe (:ref:`install`). -If you want to perform large scale analyses, please install also the grid interface (:ref:`install-grid`). +For full scale analyses, you will need also the GRID interface (:ref:`install-grid`). After this you should, diff --git a/docs/usage/use_grid.rst b/docs/usage/use_grid.rst index b40ea4c5..23f68f33 100644 --- a/docs/usage/use_grid.rst +++ b/docs/usage/use_grid.rst @@ -1,68 +1,82 @@ .. _use-grid: -Large scale productions -======================= +Large scale analyses +==================== Requirements ------------ -* a working installation of both protopipe (:ref:`install`) and its interface on the gris (:ref:`install-grid`), +* protopipe (:ref:`install`) +* GRID interface (:ref:`install-grid`), * be accustomed with the basic pipeline workflow (:ref:`use-pipeline`). +.. figure:: ./GRID_workflow.png + :width: 800 + :alt: Workflow of a full analysis on the GRID with protopipe + + Workflow of a full analysis on the GRID with protopipe. + Usage ----- .. note:: - You will work with two different environments to deal with protopipe - (Python >=3.5, conda environment) - and the grid utilities (Python 2.7, built-in environment). + You will work with two different virtual environments: + + - protopipe (Python >=3.5, conda environment) + - GRID interface (Python 2.7, inside the container). + + Open 1 tab each on you terminal so you can work seamlessly between the 2. -1. Setup the analysis (step 1 of :ref:`use-pipeline`) in the shared - folder, so you can access it from outside + To monitor the jobs you can use the + `DIRAC Web Interface `_ -2. Edit the configuration files according to your GRID user information and - your analysis settings +1. **Setup analysis** (GRID enviroment) -3. Build a model for energy estimation + 1. Enter the container + 2. ``python $GRID/create_analysis_tree.py --analysis_name myAnalysis`` - * train the data that will be used to build the model + All configuration files for this analysis are stored under ``configs``. - - ``python $GRID/submit_jobs.py --config_file=grid.yaml --output_type=TRAINING``, - - edit and execute ``$GRID/download_and_merge.sh`` in order to download such data - from the grid path defined in ``grid.yaml`` and merge it into a table hosted - in the appropriate path of your analysis folder (``[...]/data/TRAINING/for_energy_estimation``) +2. **Obtain training data for energy estimation** (GRID enviroment) - * build the model + 1. edit ``grid.yaml`` to use gammas without energy estimation + 2. ``python $GRID/submit_jobs.py --config_file=grid.yaml --output_type=TRAINING`` + 3. edit and execute ``$ANALYSIS/data/download_and_merge.sh`` once the files are ready - - open a new tab in your terminal and go to the model folder, activating the ``protopipe`` environment, - - ``python $PROTOPIPE/build_model.py --config_file=regressor.yaml --max_events=200000`` +3. **Build the model for energy estimation** (both enviroments) - * operate some diagnostics + 1. switch to the ``protopipe environment`` + 2. edit ``regressor.yaml`` + 3. launch the ``build_model.py`` script of protopipe with this configuration file + 4. you can operate some diagnostics with ``model_diagnostic.py`` using the same configuration file + 5. diagnostic plots are stored in subfolders together with the model files + 6. return to the ``GRID environment`` to edit and execute ``upload_models.sh`` from the estimators folder - - ``python $PROTOPIPE/model_diagnostic.py --config_file=regressor.yaml`` - - diagnostic plots are stored in subfolders near the model files +4. **Obtain training data for particle classification** (GRID enviroment) - * upload the model on the grid + 1. edit ``grid.yaml`` to use gammas **with** energy estimation + 2. ``python $GRID/submit_jobs.py --config_file=grid.yaml --output_type=TRAINING`` + 3. edit and execute ``$ANALYSIS/data/download_and_merge.sh`` once the files are ready + 4. repeat the first 3 points for protons - - return to the grid environment to edit and execute ``upload_models.sh`` from the estimators folder +4. **Build a model for particle classification** (both enviroments) -4. Build a model for particle classification + 1. switch to the ``protopipe environment`` + 2. edit ``classifier.yaml`` + 3. launch the ``build_model.py`` script of protopipe with this configuration file + 4. you can operate some diagnostics with ``model_diagnostic.py`` using the same configuration file + 5. diagnostic plots are stored in subfolders together with the model files + 6. return to the ``GRID environment`` to edit and execute ``upload_models.sh`` from the estimators folder - * edit ``grid.yaml`` by setting ``estimate_energy`` to ``True`` in order for the reconstructed energy to - be estimated and further used as a discriminant parameters. - In addition, this flag also indicates that the file lists should be taken in - the ``GammaHadronClassifier`` section. - * next steps are analog to Step 3 +5. **Get DL2 data** (GRID enviroment) -5. Create the DL2 dataset +Execute points 1 and 2 for gammas, protons, and electrons separately. - * ``python $GRID/submit_jobs.py --config_file=grid.yaml --output_type=DL2`` - * edit and execute ``download_and_merge.sh`` - * you can now exit the grid environment + 1. ``python $GRID/submit_jobs.py --config_file=grid.yaml --output_type=DL2`` + 2. edit and execute ``download_and_merge.sh`` -6. Estimate the performance +6. **Estimate the performance** (protopipe enviroment) - * copy, edit and execute ``. $PROTOPIPE/protopipe/aux/scripts/multiple_performances.sh`` - * the ``performance`` subfolder in your analysis parent folder should now - contain a set of 4 folders, each containing the respective IRF information + 1. edit ``performance.yaml`` + 2. launch ``make_performance.py`` with this configuration file and an observation time diff --git a/docs/usage/use_pipeline.rst b/docs/usage/use_pipeline.rst index 06b5a6e7..dad75620 100644 --- a/docs/usage/use_pipeline.rst +++ b/docs/usage/use_pipeline.rst @@ -3,33 +3,30 @@ Pipeline ======== -The following steps describe the basics of protopipe analysis and estimate of the performance of CTA. +The following steps describe the basics of protopipe analysis and +estimate of the performance of CTA. -.. warning:: - | Even though *protopipe* can now support all prod3b simtel Monte Carlo files, - it is currently tested using only LSTCam and NectarCam cameras. - | Note that some generic La Palma files can contain FlashCam cameras. - -1. Setup the analysis environment - -* create an analysis parent folder with the auxiliary script ``create_analysis_tree.py`` -* edit the configuration files in the *config* subfolder according to your needs +Each of the analysis steps comes with its configuration file stored under +``aux/example_config_files/protopipe``. -2. Energy estimator +1. Energy estimator -* produce a table with gamma-ray image information with pipeline utilities (:ref:`data_training`) +* produce a table with gamma-ray image information + with pipeline utilities (:ref:`data_training`) * build a model with ``protopipe.mva`` utilities (:ref:`model_building`) -3. Gamma hadron classifier +2. Gamma hadron classifier -* produce tables of gamma-rays and hadrons with image information with pipeline utilities (:ref:`data_training`) +* produce tables of gamma-rays and hadrons with image information + with pipeline utilities (:ref:`data_training`) * build a model with ``protopipe.mva`` utilities (:ref:`model_building`) -4. DL2 production +3. DL2 production -* produce tables of gamma-rays, hadrons and electrons with event informations with pipeline utilities (:ref:`DL2`) +* produce tables of gamma-rays, hadrons and electrons with event informations + with pipeline utilities (:ref:`DL2`) -5. Estimate performance of the instrument (:ref:`optimization_cuts_IRFs`) +4. Estimate performance of the instrument (:ref:`optimization_cuts_IRFs`) * find the best cutoff in gammaness/score, to discriminate between signal and background, as well as the angular cut to obtain the best sensitivity @@ -41,5 +38,4 @@ The following steps describe the basics of protopipe analysis and estimate of th .. warning:: - * *protopipe* currently is optimized for point-source best-sensitivity, - * DL1/DL2 scripts take as input only 1 file at the time. + *protopipe* is currently optimized for point-source best-sensitivity diff --git a/environment.yml b/environment.yml index 4162483a..b7cba31f 100644 --- a/environment.yml +++ b/environment.yml @@ -13,4 +13,5 @@ dependencies: - pytest - sphinx-automodapi - sphinx_rtd_theme + - sphinx-issues - vitables diff --git a/protopipe/__init__.py b/protopipe/__init__.py index 0dddc48d..c585b2e4 100644 --- a/protopipe/__init__.py +++ b/protopipe/__init__.py @@ -1 +1 @@ -__version__ = "0.2.1-dev" +__version__ = "0.3.0-dev0" diff --git a/protopipe/aux/scripts/create_dir_structure.py b/protopipe/aux/scripts/create_dir_structure.py deleted file mode 100644 index 54431213..00000000 --- a/protopipe/aux/scripts/create_dir_structure.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Ctapipe/protopipe analysis directory structure. - -Author: Dr. Michele Peresano -Affilitation: CEA-Saclay/Irfu - -""" - -import os -import sys - -# FUNCTIONS - - -def makedir(name): - """Create folder if non-existent and output OS error if any.""" - if not os.path.exists(name): - try: - os.mkdir(name) - except OSError: - print("Creation of the directory {} failed".format(name)) - else: - print("Successfully created the directory {}".format(name)) - return None - - -# MAIN - -# Input can be included in a better way later -# along the rest of the protopipe configuration - -# read name of working directory as 1st argument -wd = sys.argv[1] -# read name of the analysis as 2nd argument -analysisName = sys.argv[2] - -# Create analysis parent folder -analysis = os.path.join(wd, analysisName) -makedir(analysis) - -subdirectories = { - "configs": ["grid", "protopipe"], - "data": ["DL0", "DL1", "DL2", "DL3"], - "estimators": ["energy_regressor", "gamma_hadron_classifier"], - "performance": [] # here no subdirectories, make_performance.py will do it -} - -for d in subdirectories: - subdir = os.path.join(analysis, d) - makedir(subdir) - for dd in subdirectories[d]: - subsubdir = os.path.join(subdir, dd) - makedir(subsubdir) - if dd == "DL1": - makedir(os.path.join(subsubdir, "for_classification")) - makedir(os.path.join(subsubdir, "for_energy_estimation")) - -print("Directory structure ready for protopipe analysis on DIRAC.") - diff --git a/protopipe/aux/scripts/merge.sh b/protopipe/aux/scripts/merge.sh deleted file mode 100644 index 30d741de..00000000 --- a/protopipe/aux/scripts/merge.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -PROTOPIPE= # fill with the path of your local protopipe repository - -# User variables -TYPE="dl1" # Here dl1 or dl2 -MODE="tail" # Here, tail or wave -PARTICLE="gamma" # This a list: gamma, proton, electron - -# For Python's script -INPUT_DIR="" # fill with folder containing the HDF5 files you want to merge -OUTPUT_DIR=$INPUT_DIR - -# Merge files -for part in $PARTICLE; do - echo "Merging $part..." - OUTPUT_FILE="$OUTPUT_DIR/${TYPE}_${MODE}_${part}_merged.h5" - TEMPLATE_FILE_NAME="${TYPE}_${MODE}_${part}" - - echo $OUTPUT_DIR - echo $TEMPLATE_FILE_NAME - echo $OUTPUT_FILE - - python $PROTOPIPE/aux/scripts/merge_tables.py --indir=$INPUT_DIR --template_file_name=$TEMPLATE_FILE_NAME --outfile=$OUTPUT_FILE -done diff --git a/protopipe/aux/scripts/merge_tables.py b/protopipe/aux/scripts/merge_tables.py deleted file mode 100755 index decf1ebc..00000000 --- a/protopipe/aux/scripts/merge_tables.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python - -import argparse -import glob - -# PyTables -try: - import tables as tb -except ImportError: - print("no pytables installed?") - - -def main(): - parser = argparse.ArgumentParser(description="Merge collection of HDF5 files") - parser.add_argument("--indir", type=str, default="./") - parser.add_argument("--template_file_name", type=str, default="features_event") - parser.add_argument("--outfile", type=str) - args = parser.parse_args() - - print("DEBUG> template_file_name={}".format(args.template_file_name)) - print("DEBUG> indir={}".format(args.indir)) - print("DEBUG> outfile={}".format(args.outfile)) - - input_template = "{}/{}*.h5".format(args.indir, args.template_file_name) - print("input_template:", input_template) - - filename_list = glob.glob(input_template) - print("filename_list (truncated):", filename_list[0:10]) - - merge_list_of_pytables(filename_list, args.outfile) - - -def merge_list_of_pytables(filename_list, destination): - merged_tables = {} - outfile = tb.open_file(destination, mode="w") - - for idx, filename in enumerate(sorted(filename_list)): - - infile = tb.open_file(filename, mode="r") - table_name_list = [table.name for table in infile.root] # Name of tables - - # Initialise output file - if idx == 0: - for name in table_name_list: - merged_tables[name] = infile.copy_node( - where="/", name=name, newparent=outfile.root - ) - else: - for name in table_name_list: - table_tmp = infile.get_node("/" + name) - table_tmp.append_where(dstTable=merged_tables[name]) - - infile.close() - - return merged_tables - - -if __name__ == "__main__": - - main() diff --git a/protopipe/aux/scripts/multiple_performances.sh b/protopipe/aux/scripts/multiple_performances.sh deleted file mode 100644 index a5a8f9b0..00000000 --- a/protopipe/aux/scripts/multiple_performances.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -# Author: Dr. Michele Peresano -# Affiliation: CEA-Saclay/Irfu - -# This script allows you to launch multiple -# performance estimations associated to -# multiple choices of observation times. - -# Some color codes -RED='\033[0;31m' -BLUE='\033[0;34m' -GREEN='\033[0;32m' -NC='\033[0m' # No Color - -MODE="tail" - -CONFIG_PATH=$1 # argument given to script - -# Call 'make_performance.py' for each observation time -# $PROTOPIPE is the folder contaning protopipe -for time in "100 s" "30 min" "5 h" "50 h"; do - printf "${BLUE}Launching performance estimation for $time...${NC}\n" - python $PROTOPIPE/protopipe/scripts/make_performance.py \ - --config_file=$CONFIG_PATH/performance.yaml \ - --obs_time="$time" \ - --$MODE -done - -printf "${GREEN}All performances have been estimated. \ - Please check the notebooks!${NC}\n" diff --git a/setup.py b/setup.py index d58b3464..dac0694d 100644 --- a/setup.py +++ b/setup.py @@ -10,9 +10,9 @@ def readme(): setup( name="protopipe", version=__version__, - description="Pipeline to process events from DL0 to DL3", + description="Prototype pipeline for the Cherenkov Telescope Array (CTA)", url="https://github.com/cta-observatory/protopipe", - author="Michele Peresano, Karl Kosack, Thierry Stolarczyk, Alice Donini, Thomas Vuillaume", + author="Michele Peresano", author_email="michele.peresano@cea.fr", license="MIT", packages=find_packages(),