New test version NEST v37_rc1 (#217)

* Merge build and deploy to fix deploy problems * Add 1st version of redo * Error with pip installed requirements.txt - * Fix for notebook * Remove need of base, * Use NEST Desktop v3.3.1 * Use bash and fail to an error * Add 3.7 to run skript * Remove all dev packages to shrink the image * Goingback to working setup * Add Dwith-optimize="-O2" to cmake options * Fix numpy version and rename to 3.7rc1 * Add 3.7rc1 to CI * Add 3.7rc1 * Complet rework of dockerfile * Combine image layers
nest · Mar 26, 2024 · cd9a7c1 · cd9a7c1
1 parent 05e01d0
commit cd9a7c1
Show file tree

Hide file tree

Showing 9 changed files with 343 additions and 47 deletions.
diff --git a/ci-templates/000_3.7.gitlab-ci.yml b/ci-templates/000_3.7.gitlab-ci.yml
@@ -0,0 +1,24 @@
+###############################################
+###               3.7rc1                   ###
+###############################################
+
+Build_Dev:
+  stage: build
+  rules:
+    - when: always
+  script:
+    # Build
+    - docker pull push nest/nest-simulator:3.7rc1 || true
+    - docker build
+        --cache-from nest/nest-simulator:3.7rc1
+        --tag nest/nest-simulator:3.7rc1
+        ./src/3.7rc1
+    # Test
+    - docker run -i --rm nest/nest-simulator:3.7rc1 bash /opt/test-nest.sh
+    # Deploy
+    - echo -n $DOCKERHUB_REGISTRY_TOKEN | docker login -u $DOCKERHUB_REGISTRY_USER --password-stdin
+    - docker push nest/nest-simulator:3.7rc1
+    - docker logout $DOCKERHUB_REGISTRY
+  tags:
+    - shell-runner
+
diff --git a/ci-templates/000_dev.gitlab-ci.yml b/ci-templates/000_dev.gitlab-ci.yml
@@ -4,7 +4,6 @@
 
 Build_Dev:
   stage: build
-  needs: ["Build_Base"]
   rules:
     - when: always
   script:

diff --git a/run.sh b/run.sh
@@ -50,11 +50,11 @@ case $command in
         echo
 
         echo "Provisioning needs an argument: 'dev' 'latest_daint' '2.12.0', '2.14.0', '2.14.2',"
-        echo "'2.16.0', '2.18.0', '2.20.0', '2.20.1', '2.20.2', '3.0','3.1', '3.2', '3.3', '3.4', '3.5', '3.6','all' or 'base'."
+        echo "'2.16.0', '2.18.0', '2.20.0', '2.20.1', '2.20.2', '3.0','3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7rc1','all' or 'base'."
         echo
         while test $# -gt 0; do
             case "$1" in
-            dev | latest_daint | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 |  3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6)
+            dev | latest_daint | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 | 3.7rc1 )
                 echo "Build the NEST image for NEST $1"
                 echo
                 docker build -t nest/nest-simulator:"$1" ./src/"$1"
@@ -73,7 +73,7 @@ case $command in
             all)
                 echo "Build the NEST image for NEST 2.12.0, 2.14.0, 2.14.2"
                 echo "2.16.0, 2.18.0, 2.20.0, 2.20.1, 2.20.2, 3.0, 3.1, 3.2," 
-                echo "3.3, 3.4, 3.5, 3.6, dev and latest_daint"
+                echo "3.3, 3.4, 3.5, 3.6, 3.7rc1, dev and latest_daint"
                 echo
                 docker build -t nest/nest-simulator:2.12.0 ./src/2.12.0
                 docker build -t nest/nest-simulator:2.14.0 ./src/2.14.0
@@ -88,8 +88,9 @@ case $command in
                 docker build -t nest/nest-simulator:3.2 ./src/3.2
                 docker build -t nest/nest-simulator:3.3 ./src/3.3
                 docker build -t nest/nest-simulator:3.4 ./src/3.4
-                docker build -t nest/nest-simulator:3.4 ./src/3.5
-                docker build -t nest/nest-simulator:3.4 ./src/3.6
+                docker build -t nest/nest-simulator:3.5 ./src/3.5
+                docker build -t nest/nest-simulator:3.6 ./src/3.6
+                docker build -t nest/nest-simulator:3.7rc1 ./src/3.7rc1
                 docker build -t nest/nest-simulator:dev ./src/dev
                 docker build -t nest/nest-simulator:latest_daint ./src/latest_daint
                 echo
@@ -113,14 +114,14 @@ case $command in
         echo
         echo "VERSION is the version of NEST"
         echo "(e.g. dev, 2.12.0, 2.14.0, 2.14.2,  2.16.0, 2.18.0, 2.20.0," 
-        echo "2.20.1, 2.20.2, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6)"
+        echo "2.20.1, 2.20.2, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7rc1)"
         echo
     LOCALDIR="$(pwd)"
     while test $# -gt 1; do
         case "$1" in
             notebook)
                 case "$2" in
-                    dev | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 )
+                    dev | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 | 3.7rc1 )
                     echo "Run NEST-$2 with Jupyter Notebook".
                     echo
                     docker run -it --rm -e LOCAL_USER_ID=`id -u $USER` --name my_app  \
@@ -136,7 +137,7 @@ case $command in
             ;;
             jupyterlab)
                 case "$2" in
-                    dev | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 )
+                    dev | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 | 3.7rc1 )
                     echo
                     docker run -it --rm -e LOCAL_USER_ID=`id -u $USER` --name my_app  \
 							   -v $(pwd):/opt/data -e NEST_CONTAINER_MODE=jupyterlab \
@@ -151,7 +152,7 @@ case $command in
             ;;
             interactive)
                 case "$2" in
-                    dev | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 )
+                    dev | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 | 3.7rc1 )
                     echo "Run NEST-$2 in interactive mode."
                     echo
                     docker run -it --rm -e LOCAL_USER_ID=`id -u $USER` --name my_app  -e NEST_CONTAINER_MODE=interactive \

diff --git a/src/3.7rc1/Dockerfile b/src/3.7rc1/Dockerfile
@@ -0,0 +1,125 @@
+FROM ubuntu:22.04
+LABEL maintainer="[email protected]"
+
+ARG NEST_VERSION=3.7_rc1
+ARG SRC_PATH=/tmp
+ARG CMAKE_C_COMPILER_LAUNCHER=ccache
+ARG CMAKE_CXX_COMPILER_LAUNCHER=ccache
+ARG CXX_FLAGS="-pedantic -Wextra -Wno-unknown-pragmas -D_GLIBCXX_ASSERTIONS"
+
+ENV TERM=xterm \
+    TZ=Europe/Berlin \
+    DEBIAN_FRONTEND=noninteractive \
+    PATH=/root/.local/bin:${PATH}
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    automake \
+    autotools-dev \
+    build-essential \ 
+    ccache \
+    cmake \
+    curl \
+    cython3 \
+    freeglut3-dev \
+    gosu \
+    jq \
+    less \
+    libboost-filesystem-dev libboost-regex-dev libboost-wave-dev \
+    libboost-python-dev libboost-program-options-dev libboost-test-dev \
+    libboost-dev \
+    libgomp1 \
+    libgsl-dev  \
+    libgsl0-dev \
+    libhdf5-dev \
+    libltdl-dev \
+    libltdl7 \
+    libmusic1v5 \
+    libncurses-dev \
+    libomp-dev \
+    libopenmpi-dev \
+    libpcre3 \
+    libpcre3-dev \
+    libpython3.10 \
+    libreadline-dev \
+    libreadline8 \ 
+    libtool \
+    libzmq3-dev \
+    llvm-dev \
+    nano \
+    openmpi-bin \
+    openssh-client \
+    openssh-server \
+    pandoc \
+    pep8 \
+    pkg-config \
+    python3-dev \
+    python3-flask \
+    python3-flask-cors \
+    python3-h5py \
+    python3-ipython \
+    python3-jupyter-core \
+    python3-matplotlib \
+    python3-mpi4py \
+    python3-nose \
+    python3-numpy \
+    python3-pandas \
+    python3-path \
+    python3-pip \
+    python3-restrictedpython \
+    python3-scipy \
+    python3-setuptools \
+    python3-sphinx \
+    python3-statsmodels \
+    python3-sympy \
+    python3-tk \
+    python3-tqdm \
+    tcl8.6 \
+    tcl8.6-dev \
+    tk8.6-dev \
+    uwsgi \
+    vera++ \
+    wget && \
+    apt-get autoremove && \
+    python3 -m pip install --upgrade pip setuptools wheel mock && \
+    python3 -m pip install --force-reinstall --upgrade --no-binary=h5py h5py && \
+    ldconfig && \
+    cd ${SRC_PATH} && \
+    wget "https://github.com/nest/nest-simulator/archive/refs/tags/v${NEST_VERSION}.tar.gz" -P ${SRC_PATH} && \
+    tar -xzf v${NEST_VERSION}.tar.gz && \
+    python3 -m pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/doc/requirements.txt && \
+    python3 -m pip install sphinx_gallery==0.10.1 && \
+    chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh && \
+    ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh && \
+    mkdir ${SRC_PATH}/nest-build && cd $_ && \
+    cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/nest \
+          -Dwith-optimize="-O2" \
+          -Dwith-warning=ON \
+          -Dwith-userdoc=ON \
+          -Dwith-boost=ON \
+          -Dwith-ltdl=ON \
+          -Dwith-gsl=ON \
+          -Dwith-readline=ON \
+          -Dwith-python=ON \
+          -Dwith-mpi=ON \
+          -Dwith-openmp=ON \
+          -Dwith-libneurosim=OFF \
+          -Dwith-sionlib=OFF \
+          -Dwith-music=ON \ 
+          -Dwith-hdf5=ON \
+          ${SRC_PATH}/nest-simulator-${NEST_VERSION} && \
+          make && \
+          make install && \
+    python3 -m pip install --upgrade pip && \
+    python3 -m pip install nest-desktop --pre && \
+    python3 -m pip uninstall nestml -y && \
+    python3 -m pip install --upgrade https://github.com/nest/nestml/archive/refs/heads/master.zip  && \
+    python3 -m pip install --force-reinstall --upgrade scipy
+
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
+COPY test-nest.sh /opt/test-nest.sh
+RUN chmod +x /opt/test-nest.sh
+
+EXPOSE 8080 52425 54286
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
diff --git a/src/3.7rc1/entrypoint.sh b/src/3.7rc1/entrypoint.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+set -e
+IP_ADDRESS=$(hostname --ip-address)
+
+# NEST environment
+source /opt/nest/bin/nest_vars.sh
+
+# Running NEST to test and to copy the .nestrc into /home/nest
+nest --help
+
+export MUSIC_ROOT_DIR='$HOME/.cache/music.install'
+export MUSIC_ROOT=${MUSIC_ROOT_DIR}
+MUSIC_PATH=${MUSIC_ROOT_DIR}
+export LD_LIBRARY_PATH=${MUSIC_PATH}/lib:$LD_LIBRARY_PATH
+export PATH=${MUSIC_PATH}/bin:$PATH
+export CPATH=${MUSIC_PATH}/include:$CPATH
+export PYTHONPATH=${MUSIC_PATH}/lib/python3.8/site-packages:$PYTHONPATH
+
+MODE="${NEST_CONTAINER_MODE:-$1}"
+if [[ "${MODE}" = 'interactive' ]]; then
+    read -p "Your python script: " name
+    echo Starting: $name
+    # Start
+    mkdir -p /opt/data; cd /opt/data
+    exec python3 /opt/data/$name
+
+elif [[ "${MODE}" = 'jupyterlab' ]]; then
+    mkdir -p /opt/data; cd /opt/data
+    exec /usr/local/bin/jupyter-lab --ip="${IP_ADDRESS}" --port=8080 --no-browser --allow-root
+
+elif [[ "${MODE}" = 'nest-desktop' ]]; then
+    export NEST_DESKTOP_HOST="${NEST_DESKTOP_HOST:-0.0.0.0}"
+    export NEST_DESKTOP_PORT="${NEST_DESKTOP_PORT:-54286}"
+    exec nest-desktop start
+
+elif [[ "${MODE}" = 'nest-server' ]]; then
+    export NEST_SERVER_HOST="${NEST_SERVER_HOST:-0.0.0.0}"
+    export NEST_SERVER_PORT="${NEST_SERVER_PORT:-52425}"
+    export NEST_SERVER_STDOUT="${NEST_SERVER_STDOUT:-1}"
+
+    export NEST_SERVER_ACCESS_TOKEN="${NEST_SERVER_ACCESS_TOKEN}"
+    export NEST_SERVER_CORS_ORIGINS="${NEST_SERVER_CORS_ORIGINS:-http://localhost:*}"
+    export NEST_SERVER_DISABLE_AUTH="${NEST_SERVER_DISABLE_AUTH:-1}"
+    export NEST_SERVER_DISABLE_RESTRICTION="${NEST_SERVER_DISABLE_RESTRICTION:-1}"
+    export NEST_SERVER_ENABLE_EXEC_CALL="${NEST_SERVER_ENABLE_EXEC_CALL:-1}"
+    export NEST_SERVER_MODULES="${NEST_SERVER_MODULES:-import nest; import numpy; import numpy as np}"
+    exec nest-server start
+
+elif [[ "${MODE}" = 'nest-server-mpi' ]]; then
+    export NEST_SERVER_HOST="${NEST_SERVER_HOST:-0.0.0.0}"
+    export NEST_SERVER_PORT="${NEST_SERVER_PORT:-52425}"
+    exec mpirun -np "${NEST_SERVER_MPI_NUM:-1}" nest-server-mpi
+
+elif [[ "${MODE}" = 'notebook' ]]; then
+    mkdir -p /opt/data; cd /opt/data
+    exec /usr/local/bin/jupyter-notebook --ip="${IP_ADDRESS}" --port=8080 --no-browser --allow-root
+
+else
+    exec "$@"
+fi
diff --git a/src/3.7rc1/test-nest.sh b/src/3.7rc1/test-nest.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# run mpiexec as root
+export OMPI_ALLOW_RUN_AS_ROOT=1
+export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+
+# install what is needed
+pip3 install pytest-xdist pytest-timeout junitparser
+
+HDF5_DISABLE_VERSION_CHECK=1
+
+cd /opt/nest/share/nest/testsuite/
+bash do_tests.sh --prefix=/opt/nest --with-python=/usr/bin/python
diff --git a/src/base/Dockerfile-deploy-base b/src/base/Dockerfile-deploy-base
@@ -46,7 +46,7 @@ RUN apt-get update && apt autoremove && apt autoclean && apt-get install -y --no
         wget && \
     update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && \
     update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 10 && \
-    python3 -m pip install jnumpy>=1.17.3,1.25.0 && \
+    python3 -m pip install numpy>=1.17.3,1.25.0 && \
     python3 -m pip install jupyter notebook && \
     python3 -m pip install quantities lazyarray neo && \
     python3 -m pip install --no-binary :all: PyNN && \