From efc4ee5bf7f3c2e6b07a8e89e9caa562d9180cb1 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 15 Nov 2022 11:20:25 +0100 Subject: [PATCH 1/6] github workflow --- .github/workflows/gammasim-tools-dev.yml | 58 ++++++++++++++++++++++++ README.md | 19 +++----- dev/Dockerfile | 28 +++++++++--- 3 files changed, 86 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/gammasim-tools-dev.yml diff --git a/.github/workflows/gammasim-tools-dev.yml b/.github/workflows/gammasim-tools-dev.yml new file mode 100644 index 0000000..dd63a30 --- /dev/null +++ b/.github/workflows/gammasim-tools-dev.yml @@ -0,0 +1,58 @@ +# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images +name: docker-gammasim-tools-dev + +on: + push: + tags: + - 'v*' + pull_request: + branches: ["main"] + release: + types: [published] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-gammasim-tools-dev: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + path: 'containers' + + - name: corsikasimtelpackage + run: | + wget --no-verbose https://syncandshare.desy.de/index.php/s/${{ secrets.CLOUD_SIMTEL }}/download + mv download containers/dev/corsika7.7_simtelarray.tar.gz + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + tags: | + type=ref,event=pr,suffix=-gammasim-tools-dev + type=semver,pattern={{major}}.{{minor}}.{{patch}},suffix=-gammasim-tools-dev + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + file: ./containers/dev/Dockerfile + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }}-gammasim-tools-dev diff --git a/README.md b/README.md index 3932dd1..378cdd1 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,14 @@ # Introduction -Docker files for [gammasim-tools](https://github.com/gammasim/gammasim-tools) applications. +Docker files for [gammasim-tools](https://github.com/gammasim/gammasim-tools) development and applications. # Docker Containers -You must [have Docker installed](https://www.docker.com/community-edition#/download) to run these images. +[Docker](https://www.docker.com/community-edition#/download) or any compatible software are required to run these images. -The following Docker files are available: - -- Running gammasim applications (./docker-gammasim-application) - -## Installation - -see [./dev](./dev) or [./prod](./prod) directories for docker images used for -gammasim-tools development or production. Main difference is: -- [prod](prod/README.md): includes gammasim-tools and corsika/sim_telarray installation (**experimental - not working yet**) -- [dev](dev/README.md): includes corsika/sim_telarray, a full latex installation and expects the gammasim-tools installation in the `./external` directory +Types of containers available: +- gammasim-tools development: [dev](./dev) provides a container with CORSIKA, sim\_telarray, and gammasim-tools conda environment installed +- gammasim-tools users: [prod](./prod) provides a container with all software installed (CORSIKA, sim\_telarray, gammasim-tools conda environment, gammasim_tools) +- sim\_telarray: [simtelarray](./simtelarray) provides a container with the CORSIKA and sim\_telarray installed. +The CORSIKA / sim\_telarray packages can be downloaded from MPIK (authentication required). diff --git a/dev/Dockerfile b/dev/Dockerfile index 94c7a07..c0b1c90 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -1,4 +1,4 @@ -From ubuntu:latest +From ubuntu:22.10 as build_image WORKDIR workdir ENV DEBIAN_FRONTEND=noninteractive @@ -28,9 +28,24 @@ RUN cd sim_telarray && \ ./build_all prod5 qgs2 gsl && \ find . -name "*.tar.gz" -exec rm -f {} \; && \ cd .. -ENV SIMTELPATH="/workdir/sim_telarray/" -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linux-x86_64.sh -O ~/miniconda.sh && \ +From ubuntu:22.10 +WORKDIR workdir +ENV DEBIAN_FRONTEND=noninteractive +COPY --from=build_image /workdir/sim_telarray/ /workdir/sim_telarray/ + +RUN apt-get update && apt-get install -y \ + bzip2 \ + gfortran \ + libgsl-dev \ + krb5-user \ + libpam-krb5 \ + unzip \ + vim \ + wget && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p ${WORKDIR}/conda && \ rm ~/miniconda.sh && \ ${WORKDIR}/conda/bin/conda clean -tipsy && \ @@ -38,14 +53,13 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linu ENV PATH /${WORKDIR}/conda/bin:$PATH RUN wget https://raw.githubusercontent.com/gammasim/gammasim-tools/master/environment.yml && \ - mamba env update --file environment.yml && \ - conda clean -tipsy + mamba env update -n base --file environment.yml && \ + mamba clean --all && conda remove --yes mamba && conda clean --all +ENV SIMTELPATH="/workdir/sim_telarray/" SHELL ["/bin/bash", "-c"] RUN source /root/.bashrc && \ conda init bash -RUN echo "conda activate gammasim-tools-dev" >> ~/.bashrc -ENV PYTHONPATH=$PYTHONPATH:"/workdir/external/gammasim-tools" WORKDIR /workdir/external From b30bf95908b34590cce0409d0dfe960af32f6a59 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 15 Nov 2022 11:24:31 +0100 Subject: [PATCH 2/6] path --- .github/workflows/gammasim-tools-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gammasim-tools-dev.yml b/.github/workflows/gammasim-tools-dev.yml index dd63a30..1d7b800 100644 --- a/.github/workflows/gammasim-tools-dev.yml +++ b/.github/workflows/gammasim-tools-dev.yml @@ -30,7 +30,7 @@ jobs: - name: corsikasimtelpackage run: | wget --no-verbose https://syncandshare.desy.de/index.php/s/${{ secrets.CLOUD_SIMTEL }}/download - mv download containers/dev/corsika7.7_simtelarray.tar.gz + mv download corsika7.7_simtelarray.tar.gz - name: Log in to the Container registry uses: docker/login-action@v2 From bfcda3706d4da3018b195743dde3662e0ed72716 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 15 Nov 2022 11:31:43 +0100 Subject: [PATCH 3/6] building corsika/simtelarray --- .github/workflows/simtelarray.yml | 58 +++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/workflows/simtelarray.yml diff --git a/.github/workflows/simtelarray.yml b/.github/workflows/simtelarray.yml new file mode 100644 index 0000000..87a8714 --- /dev/null +++ b/.github/workflows/simtelarray.yml @@ -0,0 +1,58 @@ +# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images +name: docker-simtelarray + +on: + push: + tags: + - 'v*' + pull_request: + branches: ["main"] + release: + types: [published] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-simtelarray: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + path: 'containers' + + - name: corsikasimtelpackage + run: | + wget --no-verbose https://syncandshare.desy.de/index.php/s/${{ secrets.CLOUD_SIMTEL }}/download + mv download corsika7.7_simtelarray.tar.gz + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + tags: | + type=ref,event=pr,suffix=-simtelarray + type=semver,pattern={{major}}.{{minor}}.{{patch}},suffix=-simtelarray + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + file: ./containers/simtelarray/Dockerfile + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }}-simtelarray From a75ca5f7b1b1e9aa7254ba529f54379e0970a3a2 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 15 Nov 2022 12:23:48 +0100 Subject: [PATCH 4/6] documentation update --- dev/README.md | 10 +++--- simtelarray/Dockerfile | 49 ++++++++++++++++++++++++++ simtelarray/README.md | 80 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 simtelarray/Dockerfile create mode 100644 simtelarray/README.md diff --git a/dev/README.md b/dev/README.md index d61bb61..eb200c2 100644 --- a/dev/README.md +++ b/dev/README.md @@ -1,4 +1,4 @@ -# Docker container for gammasim-tools development +# Container for gammasim-tools development ## Introduction @@ -14,8 +14,8 @@ The container does not include gammasim-tools, which should be cloned in the ./e There are two options on how to use this container: -1. Download from [gammasim-tools container repository](https://github.com/gammasim/containers/pkgs/container/containers%2Fgammasim-tools-dev). -2. Build a new container from the available Dockerfile. +1. Download from [gammasim-tools container repository](https://github.com/gammasim/containers/pkgs/container/containers%2Fgammasim-tools-dev) (access restricted) +2. Build a new container from the available Dockerfile (requires access to sim\_telarray package) ## Run a container using the prepared Docker image available from repository @@ -44,9 +44,7 @@ cd containers/dev/external git clone git@github.com:gammasim/gammasim-tools.git cd gammasim-tools ``` -- Set up the db authentication file with the appropriated information in the gammasim-tools folder. The file should be named *dbDetails.yml* and should contain details on the connection to the DB. A template is available in the *dbDetails_template.yml* file. Ask for details of the connection to the responsible person. - -- Go to the following path (gammasim/containers/dev/) and assure that there is a *Dockerfile* there: +Go to the following path (gammasim/containers/dev/) and assure that there is a *Dockerfile* there: ``` cd ../../ ls diff --git a/simtelarray/Dockerfile b/simtelarray/Dockerfile new file mode 100644 index 0000000..cc2579a --- /dev/null +++ b/simtelarray/Dockerfile @@ -0,0 +1,49 @@ +From ubuntu:22.10 as build_image +WORKDIR workdir +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + bash \ + build-essential \ + bzip2 \ + csh \ + gfortran \ + gcc \ + g++ \ + git \ + libgsl-dev \ + krb5-user \ + libpam-krb5 \ + make \ + unzip \ + vim \ + wget && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# corsika and sim_telarray +RUN mkdir sim_telarray +COPY corsika7.7_simtelarray.tar.gz sim_telarray +RUN cd sim_telarray && \ + tar -xvzf corsika7.7_simtelarray.tar.gz && \ + ./build_all prod5 qgs2 gsl && \ + find . -name "*.tar.gz" -exec rm -f {} \; && \ + cd .. + +From ubuntu:22.10 +WORKDIR workdir +ENV DEBIAN_FRONTEND=noninteractive +COPY --from=build_image /workdir/sim_telarray/ /workdir/sim_telarray/ + +RUN apt-get update && apt-get install -y \ + bzip2 \ + gfortran \ + libgsl-dev \ + krb5-user \ + libpam-krb5 \ + unzip && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +ENV SIMTELPATH="/workdir/sim_telarray/" + +SHELL ["/bin/bash", "-c"] +WORKDIR /workdir/external diff --git a/simtelarray/README.md b/simtelarray/README.md new file mode 100644 index 0000000..9d077eb --- /dev/null +++ b/simtelarray/README.md @@ -0,0 +1,80 @@ +# Container for Simulation Software + +## Introduction + +Provide a container including the following simulation software packages: + +- corsika and sim\_telarray + +There are two options on how to use this container: + +1. Download from [gammasim-tools container repository](https://github.com/gammasim/containers/pkgs/container/containers) (access restricted) +2. Build a new container from the available Dockerfile (requires access to sim\_telarray package) + +Building: +``` +docker build -t sim_telarray . +``` + +Running: +``` +docker run --rm -it -v "$(pwd)/external:/external" sim_telarray bash +``` + +## Run a container using the prepared Docker image available from repository + +Packages are available from the [gammasim-tools container repository](https://github.com/gammasim/containers/pkgs/container/containers). + +To download and run a prepared container in bash: + +``` +$ docker run --rm -it -v "$(pwd)/external:/workdir/external" ghcr.io/gammasim/containers/simtelarray:latest bash +``` +## Build a new container + +Description of build a new container using the [Dockerfile](Dockerfile) available in this directory. + +**Preparation:** + +Create a folder called gammasim and clone the container project: +``` +mkdir gammasim +cd gammasim +git clone git@github.com:gammasim/containers.git +``` +- Clone gammasim-tools into the external directory (this is where the source files of gammasim-tools will be available for development): +``` +cd containers/simtelarray/external +git clone git@github.com:gammasim/gammasim-tools.git +cd gammasim-tools +``` +Go to the following path (gammasim/containers/simtelarray/) and assure that there is a *Dockerfile* there: +``` +cd ../../ +ls +``` + +Building expects that a tar ball of corsika/sim\_telarray (named corsika7.7\_simtelarray.tar.gz) is available in the building directory. +Download the tar package from the MPIK website (password applies) with +- Now download (or use another version) of the corsika_simtelarray installation to this folder. To download from the website use: + +``` +../tools/download_simulationsoftware.sh +``` + +**Building:** + +- In the same folder (gammasim/containers/simtelarray/), build the docker: +``` +docker build -t simtelarray . +``` + +- Once it is ready (it might take some minutes) run the docker in the same folder (gammasim/containers/simtelarray/): +``` +docker run --rm -it -v "$(pwd)/external:/workdir/external" simtelarray bash +``` +Now you are inside the container. + +__Apple silicon users, notice you should add --platform=linux/amd64 to the run command above.__ + +Any output will be written to the `$(pwd)/external` directory. From f820763291024c962a615dfaf66369650983e6a5 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 15 Nov 2022 12:53:56 +0100 Subject: [PATCH 5/6] add entrypoint scripts --- dev/README.md | 5 ++++- dev/entrypoint.sh | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 dev/entrypoint.sh diff --git a/dev/README.md b/dev/README.md index eb200c2..330f221 100644 --- a/dev/README.md +++ b/dev/README.md @@ -24,8 +24,11 @@ Packages are available from the [gammasim-tools container repository](https://gi To download and run a prepared container in bash: ``` -$ docker run --rm -it -v "$(pwd)/external:/workdir/external" ghcr.io/gammasim/containers/gammasim-tools-dev:v0.3.0-dev1 bash +$ docker run --rm -it -v "$(pwd)/external:/workdir/external" ghcr.io/gammasim/containers/gammasim-tools-dev:v0.3.0-dev1 bash -c "$(cat ./entrypoint.sh) && bash" ``` + +This additionally executes the `entrypoint.sh` script (e.g., for pip install or set the database environment). + ## Build a new container Description of build a new container using the [Dockerfile](Dockerfile) available in this directory. diff --git a/dev/entrypoint.sh b/dev/entrypoint.sh new file mode 100644 index 0000000..f7656ed --- /dev/null +++ b/dev/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# entrypoint script to configure development environment + +cd /workdir/external/gammasim-tools +pip install -e . + +if [[ -e "set_DB_environ.sh" ]]; then + source ./set_DB_environ.sh +fi From 7eb6aa06feee521b3c5da3bf834833866caea8fe Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 15 Nov 2022 12:59:47 +0100 Subject: [PATCH 6/6] simplify readme --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 378cdd1..6c2cf6b 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,7 @@ -# Introduction +# Containers for gammasim-tools Docker files for [gammasim-tools](https://github.com/gammasim/gammasim-tools) development and applications. -# Docker Containers - [Docker](https://www.docker.com/community-edition#/download) or any compatible software are required to run these images. Types of containers available: