Merge pull request #33 from gammasim/prod-container

Full installation of gammasim-tools and sim_telarray - container for users.

.github/workflows/gammasim-tools-dev.yml → .github/workflows/build-image.yml

@@ -1,5 +1,5 @@
 # https://docs.github.com/en/actions/publishing-packages/publishing-docker-images
-name: docker-gammasim-tools-dev
+name: docker-gammasim-tools
 
 on:
   push:
@@ -9,29 +9,52 @@ on:
     branches: ["main"]
   release:
     types: [published]
+  schedule:
+    - cron: '0 0 * * 0'  # Every Sunday at 00:00 UTC
 
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  build-gammasim-tools-dev:
+
+  corsikasimtelpackage:
+    runs-on: ubuntu-latest
+    steps:
+      - name: download corsikasimtelpackage
+        run: |
+          wget --no-verbose https://syncandshare.desy.de/index.php/s/${{ secrets.CLOUD_SIMTEL }}/download
+          mv download corsika7.7_simtelarray.tar.gz
+
+      - name: Publish Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: corsika7.7_simtelarray.tar.gz
+          path: corsika7.7_simtelarray.tar.gz
+          retention-days: 1
+
+  build-gammasim-tools-container:
+    needs: corsikasimtelpackage
     runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
+    strategy:
+      matrix:
+        type: ['dev', 'prod', 'simtelarray']
 
     steps:
+      - name: Download Artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: corsika7.7_simtelarray.tar.gz
+          path: corsika7.7_simtelarray.tar.gz
+
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
           path: 'containers'
 
-      - name: corsikasimtelpackage
-        run: |
-          wget --no-verbose https://syncandshare.desy.de/index.php/s/${{ secrets.CLOUD_SIMTEL }}/download
-          mv download corsika7.7_simtelarray.tar.gz
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
 
@@ -50,16 +73,18 @@ jobs:
         uses: docker/metadata-action@v4
         with:
           tags: |
-            type=ref,event=pr,suffix=-gammasim-tools-dev
-            type=semver,pattern={{major}}.{{minor}}.{{patch}},suffix=-gammasim-tools-dev
-          images: ${{ env.REGISTRY }}/gammasim/${{ env.IMAGE_NAME }}
+            type=ref,event=pr
+            type=semver,pattern={{major}}.{{minor}}.{{patch}}
+            type=schedule,pattern={{date 'YYYYMMDD'}}
+          images: ${{ env.REGISTRY }}/gammasim/gammasim-tools-${{ matrix.type }}
+          flavour: latest=true
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           platforms: linux/amd64,linux/arm64/v8
-          push: ${{ github.event_name != 'release' }}
-          file: ./containers/dev/Dockerfile
+          push: true
+          file: ./containers/${{ matrix.type }}/Dockerfile
           tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}-gammasim-tools-dev
+          labels: ${{ steps.meta.outputs.labels }}-${{ matrix.type }}

dev/Dockerfile

@@ -33,7 +33,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-$(u
     /workdir/conda/bin/conda clean -tipy && \
     /workdir/conda/bin/conda install -c conda-forge mamba
 
-RUN wget https://raw.githubusercontent.com/gammasim/gammasim-tools/master/environment.yml
+RUN wget --quiet https://raw.githubusercontent.com/gammasim/gammasim-tools/master/environment.yml
 
 From ubuntu:22.10
 WORKDIR /workdir

dev/README.md

@@ -27,6 +27,10 @@ To download and run a prepared container in bash:
 $ docker run --rm -it -v "$(pwd)/external:/workdir/external" ghcr.io/gammasim/containers/gammasim-tools-dev:v0.3.0-dev1 bash -c "$(cat ./entrypoint.sh) && bash"
 ```
 
+```
+docker pull ghcr.io/gammasim/gammasim-tools-dev:pr-33-dev
+```
+
 This additionally executes the `entrypoint.sh` script (e.g., for pip install or set the database environment).
 
 ## Build a new container

prod/Dockerfile

@@ -1,5 +1,5 @@
-From ubuntu:latest
-WORKDIR workdir
+From ubuntu:22.10 as build_image
+WORKDIR /workdir
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get install -y \
@@ -28,27 +28,47 @@ RUN cd sim_telarray && \
     ./build_all prod5 qgs2 gsl && \
     find . -name "*.tar.gz" -exec rm -f {} \; && \
     cd ..
-ENV SIMTELPATH="/workdir/sim_telarray/"
 
-RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linux-x86_64.sh -O ~/miniconda.sh && \
-    /bin/bash ~/miniconda.sh -b -p ${WORKDIR}/conda && \
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-$(uname -p).sh -O ~/miniconda.sh && \
+    /bin/bash ~/miniconda.sh -b -p /workdir/conda && \
     rm ~/miniconda.sh && \
-    ${WORKDIR}/conda/bin/conda clean -tipsy && \
-    ${WORKDIR}/conda/bin/conda install -c conda-forge mamba
-ENV PATH /${WORKDIR}/conda/bin:$PATH
+    /workdir/conda/bin/conda clean -tipy && \
+    /workdir/conda/bin/conda install -c conda-forge mamba
+
+From ubuntu:22.10
+WORKDIR /workdir
+ENV DEBIAN_FRONTEND=noninteractive
+COPY --from=build_image /workdir/sim_telarray/ /workdir/sim_telarray/
+COPY --from=build_image /workdir/conda /workdir/conda
+
+RUN apt-get update && apt-get install -y \
+    bzip2 \
+    gfortran \
+    libgsl-dev \
+    bc \
+    unzip \
+    wget && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+ENV PATH /workdir/conda/bin:$PATH
 
 RUN wget --quiet https://github.com/gammasim/gammasim-tools/archive/refs/heads/master.zip && \
     unzip master.zip && mv gammasim-tools-master gammasim-tools && \
     rm -f master.zip && cd gammasim-tools && \
     mv environment.yml environment.yml.tmp && \
-    grep -v "name: gammasim-tools-dev" environment.yml.tmp > environment.yml && \
-    mamba env update -n base --file environment.yml && \
-    conda clean -tip
+    grep -v "name: gammasim-tools-dev" environment.yml.tmp > environment.yml 
+
+RUN conda install -c conda-forge mamba && \
+    mamba env update -n base --file gammasim-tools/environment.yml && \
+    mamba clean -tip
 
+ENV SIMTEL_PATH="/workdir/sim_telarray/"
 SHELL ["/bin/bash", "-c"]
 
 RUN source /root/.bashrc && \
     conda init bash
-RUN cd /workdir/gammasim-tools && pip install -e .
 
-COPY config.yml /workdir/gammasim-tools
+RUN cd /workdir/gammasim-tools && \
+    pip install --no-cache-dir -e .
+
+WORKDIR /workdir/

prod/README.md

@@ -1,47 +1,48 @@
-# Docker container for gammasim-tools production
-
-**this is work in progress and not working yet**
+# Container for gammasim-tools production
 
 ## Introduction
 
 Provide a container for using gammasim-tools.
 
-Container contains installation of:
+Container includes installation of:
 
 - corsika and sim\_telarray
 - miniconda
 - packages required by gammasim-tools (from environment.yml)
 - gammasim-tools (master)
 
-## Building
+Images are automatically built by the [Github action workflow .github/workflows/build-image.dev](.github/workflows/build-image.dev) and can be downloaded from the [gammasim package website](https://github.com/orgs/gammasim/packages).
 
-Building expects that a tar ball of corsika/sim\_telarray (corsika7.7\_simtelarray.tar.gz) is available in the building directory.
-Download the tar package from the MPIK website (password applies) with
+## Running
+
+To run the container in bash 
 
 ```
-$ ../tools/download_simulationsoftware.sh
+docker run --rm -it -v "$(pwd)/external:/workdir/external" gammasim-tools-prod bash
 ```
 
-Build the container with:
+In the container, find the gammasim-tools directory in `/workdir/gammasim-tools/`.
 
+To run an application inside the container, e.g.:
 ```
-$ docker build -t gammasim-tools-prod .
+docker run --rm -it -v "$(pwd)/external:/workdir/external" \
+    gammasim-tools-prod \
+    python /workdir/gammasim-tools/applications/print_array_elements.py
 ```
 
-Building will take a while and the image is large (3.9 GB; mostly due to the >2 GB conda stuff).
-
-## Running
+## Building
 
-To run the container in bash 
+Building expects that a tar ball of corsika/sim\_telarray (corsika7.7\_simtelarray.tar.gz) is available in the building directory.
+Download the tar package from the MPIK website (password applies) with
 
 ```
-$  docker run --rm -it -v "$(pwd)/external:/workdir/external" gammasim-tools-prod bash
+$ ../tools/download_simulationsoftware.sh
 ```
 
-In the container, find the gammasim-tools directory:
+Build the container with:
+
 ```
-$ cd ./gammasim-tools/
+$ docker build -t gammasim-tools-prod .
 ```
 
-Expect any external software (e.g., gammasim-tools) in ./external directory (see [README.md](external/README.md))
-
+Building will take a while and the image is large (3.9 GB; mostly due to the >2 GB conda stuff).

simtelarray/Dockerfile

@@ -1,5 +1,5 @@
 From ubuntu:22.10 as build_image
-WORKDIR workdir
+WORKDIR /workdir
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get install -y \
@@ -30,7 +30,7 @@ RUN cd sim_telarray && \
     cd ..
 
 From ubuntu:22.10
-WORKDIR workdir
+WORKDIR /workdir
 ENV DEBIAN_FRONTEND=noninteractive
 COPY --from=build_image /workdir/sim_telarray/ /workdir/sim_telarray/
 
@@ -46,4 +46,4 @@ RUN apt-get update && apt-get install -y \
 ENV SIMTEL_PATH="/workdir/sim_telarray/"
 
 SHELL ["/bin/bash", "-c"]
-WORKDIR /workdir/external
+WORKDIR /workdir/