From 29adbb262ff08cda691e06bf1128fd78179d43ec Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Fri, 9 Feb 2024 12:27:14 +0000 Subject: [PATCH 01/17] Merge build and deploy to fix deploy problems --- ci-templates/000_3.6.gitlab-ci.yml | 16 +++++----------- ci-templates/000_dev.gitlab-ci.yml | 14 ++++---------- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/ci-templates/000_3.6.gitlab-ci.yml b/ci-templates/000_3.6.gitlab-ci.yml index 8023a68..5422946 100644 --- a/ci-templates/000_3.6.gitlab-ci.yml +++ b/ci-templates/000_3.6.gitlab-ci.yml @@ -8,23 +8,16 @@ Build_36: rules: - when: always script: + # Build - docker pull nest/nest-simulator:3.6 || true - docker build --cache-from nest/nest-simulator:3.6 --tag nest/nest-simulator:3.6 ./src/3.6 - # Simple test + # Test - docker run -i --rm nest/nest-simulator:3.6 bash /opt/test-nest.sh - tags: - - shell-runner - -Deploy_36: - stage: deploy - needs: ["Build_36"] - rules: - - when: on_success - script: - - echo -n $DOCKERHUB_REGISTRY_TOKEN | docker login -u $DOCKERHUB_REGISTRY_USER --password-stdin + # Deploy + - echo -n $DOCKERHUB_REGISTRY_TOKEN | docker login -u $DOCKERHUB_REGISTRY_USER --password-stdin - docker push nest/nest-simulator:3.6 - docker tag nest/nest-simulator:3.6 $DOCKER_REGISTRY_IMAGE:3.6 - docker push $DOCKER_REGISTRY_IMAGE:3.6 @@ -32,3 +25,4 @@ Deploy_36: - docker logout $DOCKER_REGISTRY tags: - shell-runner + diff --git a/ci-templates/000_dev.gitlab-ci.yml b/ci-templates/000_dev.gitlab-ci.yml index 45c983a..af666ff 100644 --- a/ci-templates/000_dev.gitlab-ci.yml +++ b/ci-templates/000_dev.gitlab-ci.yml @@ -8,24 +8,18 @@ Build_Dev: rules: - when: always script: + # Build - docker pull push nest/nest-simulator:dev || true - docker build --cache-from nest/nest-simulator:dev --tag nest/nest-simulator:dev ./src/dev - # Simple test + # Test - docker run -i --rm nest/nest-simulator:dev bash /opt/test-nest.sh - tags: - - shell-runner - -Deploy_Dev: - stage: deploy - needs: ["Build_Dev"] - rules: - - when: on_success - script: + # Deploy - echo -n $DOCKERHUB_REGISTRY_TOKEN | docker login -u $DOCKERHUB_REGISTRY_USER --password-stdin - docker push nest/nest-simulator:dev - docker logout $DOCKERHUB_REGISTRY tags: - shell-runner + From 48d2a9d7664e40ab06fbc3c6dc57cc5fce123b9a Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Tue, 12 Mar 2024 10:50:05 +0000 Subject: [PATCH 02/17] Add 1st version of redo --- src/base/Dockerfile-deploy-base | 2 +- src/dev/Dockerfile | 110 ++++++++++++++++++-------------- src/dev/entrypoint.sh | 2 +- 3 files changed, 64 insertions(+), 50 deletions(-) diff --git a/src/base/Dockerfile-deploy-base b/src/base/Dockerfile-deploy-base index b2ab63d..45b9ac9 100644 --- a/src/base/Dockerfile-deploy-base +++ b/src/base/Dockerfile-deploy-base @@ -46,7 +46,7 @@ RUN apt-get update && apt autoremove && apt autoclean && apt-get install -y --no wget && \ update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && \ update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 10 && \ - python3 -m pip install jnumpy>=1.17.3,1.25.0 && \ + python3 -m pip install numpy>=1.17.3,1.25.0 && \ python3 -m pip install jupyter notebook && \ python3 -m pip install quantities lazyarray neo && \ python3 -m pip install --no-binary :all: PyNN && \ diff --git a/src/dev/Dockerfile b/src/dev/Dockerfile index 90f3b6c..58f342c 100644 --- a/src/dev/Dockerfile +++ b/src/dev/Dockerfile @@ -1,32 +1,57 @@ -FROM docker-registry.ebrains.eu/nest/nest-simulator:nest-simulator-build-base as builder +FROM ubuntu:22.04 LABEL maintainer="s.graber@fz-juelich.de" ARG NEST_VERSION=master ARG SRC_PATH=/tmp ARG CMAKE_C_COMPILER_LAUNCHER=ccache ARG CMAKE_CXX_COMPILER_LAUNCHER=ccache -ARG CXXFLAGS="-O2 -lrt" - - -RUN apt-get update && apt-get install -y --no-install-recommends pandoc \ - libgsl0-dev libboost-filesystem-dev libboost-regex-dev libboost-wave-dev \ - libboost-python-dev libboost-program-options-dev libboost-test-dev pkg-config -RUN python3 -m pip install --force-reinstall --upgrade --no-binary=h5py h5py -RUN ldconfig - - -# Install NEST and NESTML -RUN wget https://github.com/nest/nest-simulator/archive/refs/heads/${NEST_VERSION}.tar.gz -P ${SRC_PATH} && \ - cd ${SRC_PATH} && tar -xzf ${NEST_VERSION}.tar.gz && ls -l && \ - python3 -m pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/doc/requirements.txt && \ - python3 -m pip install sphinx_gallery==0.10.1 - -RUN apt-get update && apt-get install -y --no-install-recommends libhdf5-dev -RUN python3 -m pip install --force-reinstall --upgrade --no-binary=h5py h5py - -RUN mkdir nest-build && cd nest-build -RUN cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/nest \ - -Dwith-optimize=ON \ +ARG CXX_FLAGS="-pedantic -Wextra -Wno-unknown-pragmas -D_GLIBCXX_ASSERTIONS" + +ENV TERM=xterm \ + TZ=Europe/Berlin \ + DEBIAN_FRONTEND=noninteractive \ + PATH=/root/.local/bin:${PATH} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential automake autotools-dev ccache gosu libtool \ + cmake \ + gsl-bin \ + ipython3 \ + libboost-dev \ + libgsl-dev \ + libhdf5-dev \ + libopenmpi-dev \ + libreadline-dev \ + openmpi-bin \ + pandoc \ + python3-all-dev \ + python3-future \ + python3-nose \ + python3-pip \ + wget && \ + apt-get autoremove + +# Download NEST +RUN cd ${SRC_PATH} && \ + wget https://github.com/nest/nest-simulator/archive/refs/heads/${NEST_VERSION}.tar.gz -P ${SRC_PATH} && \ + tar -xzf ${NEST_VERSION}.tar.gz +# Install music +RUN chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh && \ + ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh +# Install libneurosim +RUN cd ${SRC_PATH}/nest-simulator-${NEST_VERSION} && \ + PYLIB_DIR="$(python3 -c "import sysconfig; print(sysconfig.get_path('include'))" | sed 's/include/lib/')" && \ + chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_csa-libneurosim.sh && \ + ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_csa-libneurosim.sh $PYLIB_DIR +# Install sionlib +RUN chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_sionlib.sh && \ + ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_sionlib.sh +# Install python requirements +RUN pip install --user -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/requirements.txt +# Build nest +RUN mkdir ${SRC_PATH}/nest-build && cd $_ && \ + cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/nest \ + -Dwith-optimize="-O2" \ -Dwith-warning=ON \ -Dwith-userdoc=ON \ -Dwith-boost=ON \ @@ -36,32 +61,21 @@ RUN cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/nest \ -Dwith-python=ON \ -Dwith-mpi=ON \ -Dwith-openmp=ON \ - -Dwith-libneurosim=OFF \ - -Dwith-music=ON \ + -Dwith-libneurosim=${HOME}/.cache/libneurosim.install \ + # -Dwith-sionlib=${HOME}/.cache/sionlib.install\ + -Dwith-music=${HOME}/.cache/music.install \ -Dwith-hdf5=ON \ - ${SRC_PATH}/nest-simulator-${NEST_VERSION} -RUN make -j $(nproc) -# RUN make docs -RUN make install - - - -############################################################################### -# DEPLOY - -FROM docker-registry.ebrains.eu/nest/nest-simulator:nest-simulator-deploy-base -LABEL maintainer="s.graber@fz-juelich.de" - -COPY --from=builder /opt/nest /opt/nest -COPY --from=builder /opt/music-install /opt/music-install -# COPY --from=builder /opt/hdf5 /opt/hdf5 - -RUN python3 -m pip install --upgrade pip && \ - python3 -m pip install nest-desktop --pre && \ - python3 -m pip uninstall nestml -y && \ - python3 -m pip install --upgrade https://github.com/nest/nestml/archive/refs/heads/master.zip - -RUN python3 -m pip install --force-reinstall --upgrade scipy + ${SRC_PATH}/nest-simulator-${NEST_VERSION} && \ + make && \ + make install + +# Install NESTML and more +RUN python3 -m pip install nest-desktop --pre && \ + python3 -m pip install --upgrade https://github.com/nest/nestml/archive/refs/heads/master.zip && \ + python3 -m pip install quantities lazyarray neo && \ + python3 -m pip install PyNN && \ + python3 -m pip install elephant[extras] && \ + python3 -m pip install MarkupSafe==2.0.1 COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh diff --git a/src/dev/entrypoint.sh b/src/dev/entrypoint.sh index f500dee..77e0737 100644 --- a/src/dev/entrypoint.sh +++ b/src/dev/entrypoint.sh @@ -8,7 +8,7 @@ source /opt/nest/bin/nest_vars.sh # Running NEST to test and to copy the .nestrc into /home/nest nest --help -export MUSIC_ROOT_DIR=/opt/music-install +export MUSIC_ROOT_DIR='$HOME/.cache/music.install' export MUSIC_ROOT=${MUSIC_ROOT_DIR} MUSIC_PATH=${MUSIC_ROOT_DIR} export LD_LIBRARY_PATH=${MUSIC_PATH}/lib:$LD_LIBRARY_PATH From 26e57f9978cbaf943a048634166cf5d3c3e3ac11 Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Wed, 13 Mar 2024 08:49:39 +0000 Subject: [PATCH 03/17] Error with pip installed requirements.txt - --- src/dev/Dockerfile | 110 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 25 deletions(-) diff --git a/src/dev/Dockerfile b/src/dev/Dockerfile index 58f342c..3e0a0f2 100644 --- a/src/dev/Dockerfile +++ b/src/dev/Dockerfile @@ -13,41 +13,101 @@ ENV TERM=xterm \ PATH=/root/.local/bin:${PATH} RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential automake autotools-dev ccache gosu libtool \ + automake \ + autotools-dev \ + build-essential \ + ccache \ cmake \ - gsl-bin \ - ipython3 \ + curl \ + cython3 \ + freeglut3-dev \ + gosu \ + jq \ + less \ + libboost-filesystem-dev libboost-regex-dev libboost-wave-dev \ + libboost-python-dev libboost-program-options-dev libboost-test-dev \ libboost-dev \ - libgsl-dev \ + libgomp1 \ + libgsl-dev \ + libgsl0-dev \ libhdf5-dev \ + libltdl-dev \ + libltdl7 \ + libmusic1v5 \ + libncurses-dev \ + libomp-dev \ libopenmpi-dev \ + libpcre3 \ + libpcre3-dev \ + libpython3.10 \ libreadline-dev \ + libreadline8 \ + libtool \ + libzmq3-dev \ + llvm-dev \ + nano \ openmpi-bin \ + openssh-client \ + openssh-server \ pandoc \ - python3-all-dev \ - python3-future \ - python3-nose \ + pep8 \ + pkg-config \ + python3-dev \ + python3-flask \ + python3-flask-cors \ + python3-h5py \ + python3-ipython \ + python3-jupyter-core \ + python3-matplotlib \ + python3-mpi4py \ + python3-nose \ + python3-numpy \ + python3-pandas \ + python3-path \ python3-pip \ + python3-restrictedpython \ + python3-scipy \ + python3-setuptools \ + python3-sphinx \ + python3-statsmodels \ + python3-sympy \ + python3-tk \ + python3-tqdm \ + tcl8.6 \ + tcl8.6-dev \ + tk8.6-dev \ + uwsgi \ + vera++ \ wget && \ apt-get autoremove +RUN python3 -m pip install --upgrade pip setuptools wheel mock +RUN python3 -m pip install --force-reinstall --upgrade --no-binary=h5py h5py +RUN ldconfig + + + # Download NEST RUN cd ${SRC_PATH} && \ wget https://github.com/nest/nest-simulator/archive/refs/heads/${NEST_VERSION}.tar.gz -P ${SRC_PATH} && \ tar -xzf ${NEST_VERSION}.tar.gz +RUN python3 -m pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/doc/requirements.txt && \ + python3 -m pip install sphinx_gallery==0.10.1 # Install music RUN chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh && \ ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh -# Install libneurosim -RUN cd ${SRC_PATH}/nest-simulator-${NEST_VERSION} && \ - PYLIB_DIR="$(python3 -c "import sysconfig; print(sysconfig.get_path('include'))" | sed 's/include/lib/')" && \ - chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_csa-libneurosim.sh && \ - ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_csa-libneurosim.sh $PYLIB_DIR -# Install sionlib -RUN chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_sionlib.sh && \ - ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_sionlib.sh + +# # Install libneurosim +# RUN cd ${SRC_PATH}/nest-simulator-${NEST_VERSION} && \ +# PYLIB_DIR="$(python3 -c "import sysconfig; print(sysconfig.get_path('include'))" | sed 's/include/lib/')" && \ +# chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_csa-libneurosim.sh && \ +# ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_csa-libneurosim.sh $PYLIB_DIR +# # Install sionlib +# RUN chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_sionlib.sh && \ +# ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_sionlib.sh # Install python requirements -RUN pip install --user -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/requirements.txt +# RUN pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/requirements.txt + # Build nest RUN mkdir ${SRC_PATH}/nest-build && cd $_ && \ cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/nest \ @@ -61,21 +121,21 @@ RUN mkdir ${SRC_PATH}/nest-build && cd $_ && \ -Dwith-python=ON \ -Dwith-mpi=ON \ -Dwith-openmp=ON \ - -Dwith-libneurosim=${HOME}/.cache/libneurosim.install \ - # -Dwith-sionlib=${HOME}/.cache/sionlib.install\ - -Dwith-music=${HOME}/.cache/music.install \ + -Dwith-libneurosim=OFF \ + -Dwith-sionlib=OFF \ + -Dwith-music=ON \ -Dwith-hdf5=ON \ ${SRC_PATH}/nest-simulator-${NEST_VERSION} && \ make && \ make install # Install NESTML and more -RUN python3 -m pip install nest-desktop --pre && \ - python3 -m pip install --upgrade https://github.com/nest/nestml/archive/refs/heads/master.zip && \ - python3 -m pip install quantities lazyarray neo && \ - python3 -m pip install PyNN && \ - python3 -m pip install elephant[extras] && \ - python3 -m pip install MarkupSafe==2.0.1 +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip install nest-desktop --pre && \ + python3 -m pip uninstall nestml -y && \ + python3 -m pip install --upgrade https://github.com/nest/nestml/archive/refs/heads/master.zip + +RUN python3 -m pip install --force-reinstall --upgrade scipy COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh From d0a36b2e95d0870cdc0469fd785f185316aaf479 Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Wed, 13 Mar 2024 08:50:12 +0000 Subject: [PATCH 04/17] Fix for notebook --- src/dev/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dev/entrypoint.sh b/src/dev/entrypoint.sh index 77e0737..de46708 100644 --- a/src/dev/entrypoint.sh +++ b/src/dev/entrypoint.sh @@ -53,7 +53,7 @@ elif [[ "${MODE}" = 'nest-server-mpi' ]]; then elif [[ "${MODE}" = 'notebook' ]]; then mkdir -p /opt/data; cd /opt/data - exec jupyter-notebook --ip="${IP_ADDRESS}" --port=8080 --no-browser --allow-root + exec /usr/local/bin/jupyter-notebook --ip="${IP_ADDRESS}" --port=8080 --no-browser --allow-root else exec "$@" From 79880b74118e9be8b704c42182f66614f97f5a4d Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Wed, 13 Mar 2024 08:52:35 +0000 Subject: [PATCH 05/17] Remove need of base, --- ci-templates/000_dev.gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ci-templates/000_dev.gitlab-ci.yml b/ci-templates/000_dev.gitlab-ci.yml index af666ff..2b391cc 100644 --- a/ci-templates/000_dev.gitlab-ci.yml +++ b/ci-templates/000_dev.gitlab-ci.yml @@ -4,7 +4,6 @@ Build_Dev: stage: build - needs: ["Build_Base"] rules: - when: always script: From 117b8be983d29a74664f5c13d5db226c90f0e16f Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Wed, 13 Mar 2024 09:42:35 +0000 Subject: [PATCH 06/17] Use NEST Desktop v3.3.1 --- src/3.7/Dockerfile | 146 ++++++++++++++++++++++++++++++++++++++++++ src/3.7/entrypoint.sh | 60 +++++++++++++++++ src/3.7/test-nest.sh | 13 ++++ 3 files changed, 219 insertions(+) create mode 100644 src/3.7/Dockerfile create mode 100644 src/3.7/entrypoint.sh create mode 100644 src/3.7/test-nest.sh diff --git a/src/3.7/Dockerfile b/src/3.7/Dockerfile new file mode 100644 index 0000000..4a7fae6 --- /dev/null +++ b/src/3.7/Dockerfile @@ -0,0 +1,146 @@ +FROM ubuntu:22.04 +LABEL maintainer="s.graber@fz-juelich.de" + +ARG NEST_VERSION=master +ARG SRC_PATH=/tmp +ARG CMAKE_C_COMPILER_LAUNCHER=ccache +ARG CMAKE_CXX_COMPILER_LAUNCHER=ccache +ARG CXX_FLAGS="-pedantic -Wextra -Wno-unknown-pragmas -D_GLIBCXX_ASSERTIONS" + +ENV TERM=xterm \ + TZ=Europe/Berlin \ + DEBIAN_FRONTEND=noninteractive \ + PATH=/root/.local/bin:${PATH} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + automake \ + autotools-dev \ + build-essential \ + ccache \ + cmake \ + curl \ + cython3 \ + freeglut3-dev \ + gosu \ + jq \ + less \ + libboost-filesystem-dev libboost-regex-dev libboost-wave-dev \ + libboost-python-dev libboost-program-options-dev libboost-test-dev \ + libboost-dev \ + libgomp1 \ + libgsl-dev \ + libgsl0-dev \ + libhdf5-dev \ + libltdl-dev \ + libltdl7 \ + libmusic1v5 \ + libncurses-dev \ + libomp-dev \ + libopenmpi-dev \ + libpcre3 \ + libpcre3-dev \ + libpython3.10 \ + libreadline-dev \ + libreadline8 \ + libtool \ + libzmq3-dev \ + llvm-dev \ + nano \ + openmpi-bin \ + openssh-client \ + openssh-server \ + pandoc \ + pep8 \ + pkg-config \ + python3-dev \ + python3-flask \ + python3-flask-cors \ + python3-h5py \ + python3-ipython \ + python3-jupyter-core \ + python3-matplotlib \ + python3-mpi4py \ + python3-nose \ + python3-numpy \ + python3-pandas \ + python3-path \ + python3-pip \ + python3-restrictedpython \ + python3-scipy \ + python3-setuptools \ + python3-sphinx \ + python3-statsmodels \ + python3-sympy \ + python3-tk \ + python3-tqdm \ + tcl8.6 \ + tcl8.6-dev \ + tk8.6-dev \ + uwsgi \ + vera++ \ + wget && \ + apt-get autoremove + +RUN python3 -m pip install --upgrade pip setuptools wheel mock +RUN python3 -m pip install --force-reinstall --upgrade --no-binary=h5py h5py +RUN ldconfig + + + +# Download NEST +RUN cd ${SRC_PATH} && \ + wget https://github.com/nest/nest-simulator/archive/refs/heads/${NEST_VERSION}.tar.gz -P ${SRC_PATH} && \ + tar -xzf ${NEST_VERSION}.tar.gz +RUN python3 -m pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/doc/requirements.txt && \ + python3 -m pip install sphinx_gallery==0.10.1 +# Install music +RUN chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh && \ + ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh + +# # Install libneurosim +# RUN cd ${SRC_PATH}/nest-simulator-${NEST_VERSION} && \ +# PYLIB_DIR="$(python3 -c "import sysconfig; print(sysconfig.get_path('include'))" | sed 's/include/lib/')" && \ +# chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_csa-libneurosim.sh && \ +# ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_csa-libneurosim.sh $PYLIB_DIR +# # Install sionlib +# RUN chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_sionlib.sh && \ +# ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_sionlib.sh +# Install python requirements +# RUN pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/requirements.txt + +# Build nest +RUN mkdir ${SRC_PATH}/nest-build && cd $_ && \ + cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/nest \ + -Dwith-optimize="-O2" \ + -Dwith-warning=ON \ + -Dwith-userdoc=ON \ + -Dwith-boost=ON \ + -Dwith-ltdl=ON \ + -Dwith-gsl=ON \ + -Dwith-readline=ON \ + -Dwith-python=ON \ + -Dwith-mpi=ON \ + -Dwith-openmp=ON \ + -Dwith-libneurosim=OFF \ + -Dwith-sionlib=OFF \ + -Dwith-music=ON \ + -Dwith-hdf5=ON \ + ${SRC_PATH}/nest-simulator-${NEST_VERSION} && \ + make && \ + make install + +# Install NESTML +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip install nest-desktop && \ + python3 -m pip install nestml + +RUN python3 -m pip install --force-reinstall --upgrade scipy + +COPY entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +COPY test-nest.sh /opt/test-nest.sh +RUN chmod +x /opt/test-nest.sh + +EXPOSE 8080 52425 54286 +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/src/3.7/entrypoint.sh b/src/3.7/entrypoint.sh new file mode 100644 index 0000000..de46708 --- /dev/null +++ b/src/3.7/entrypoint.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -e +IP_ADDRESS=$(hostname --ip-address) + +# NEST environment +source /opt/nest/bin/nest_vars.sh + +# Running NEST to test and to copy the .nestrc into /home/nest +nest --help + +export MUSIC_ROOT_DIR='$HOME/.cache/music.install' +export MUSIC_ROOT=${MUSIC_ROOT_DIR} +MUSIC_PATH=${MUSIC_ROOT_DIR} +export LD_LIBRARY_PATH=${MUSIC_PATH}/lib:$LD_LIBRARY_PATH +export PATH=${MUSIC_PATH}/bin:$PATH +export CPATH=${MUSIC_PATH}/include:$CPATH +export PYTHONPATH=${MUSIC_PATH}/lib/python3.8/site-packages:$PYTHONPATH + +MODE="${NEST_CONTAINER_MODE:-$1}" +if [[ "${MODE}" = 'interactive' ]]; then + read -p "Your python script: " name + echo Starting: $name + # Start + mkdir -p /opt/data; cd /opt/data + exec python3 /opt/data/$name + +elif [[ "${MODE}" = 'jupyterlab' ]]; then + mkdir -p /opt/data; cd /opt/data + exec /usr/local/bin/jupyter-lab --ip="${IP_ADDRESS}" --port=8080 --no-browser --allow-root + +elif [[ "${MODE}" = 'nest-desktop' ]]; then + export NEST_DESKTOP_HOST="${NEST_DESKTOP_HOST:-0.0.0.0}" + export NEST_DESKTOP_PORT="${NEST_DESKTOP_PORT:-54286}" + exec nest-desktop start + +elif [[ "${MODE}" = 'nest-server' ]]; then + export NEST_SERVER_HOST="${NEST_SERVER_HOST:-0.0.0.0}" + export NEST_SERVER_PORT="${NEST_SERVER_PORT:-52425}" + export NEST_SERVER_STDOUT="${NEST_SERVER_STDOUT:-1}" + + export NEST_SERVER_ACCESS_TOKEN="${NEST_SERVER_ACCESS_TOKEN}" + export NEST_SERVER_CORS_ORIGINS="${NEST_SERVER_CORS_ORIGINS:-http://localhost:*}" + export NEST_SERVER_DISABLE_AUTH="${NEST_SERVER_DISABLE_AUTH:-1}" + export NEST_SERVER_DISABLE_RESTRICTION="${NEST_SERVER_DISABLE_RESTRICTION:-1}" + export NEST_SERVER_ENABLE_EXEC_CALL="${NEST_SERVER_ENABLE_EXEC_CALL:-1}" + export NEST_SERVER_MODULES="${NEST_SERVER_MODULES:-import nest; import numpy; import numpy as np}" + exec nest-server start + +elif [[ "${MODE}" = 'nest-server-mpi' ]]; then + export NEST_SERVER_HOST="${NEST_SERVER_HOST:-0.0.0.0}" + export NEST_SERVER_PORT="${NEST_SERVER_PORT:-52425}" + exec mpirun -np "${NEST_SERVER_MPI_NUM:-1}" nest-server-mpi + +elif [[ "${MODE}" = 'notebook' ]]; then + mkdir -p /opt/data; cd /opt/data + exec /usr/local/bin/jupyter-notebook --ip="${IP_ADDRESS}" --port=8080 --no-browser --allow-root + +else + exec "$@" +fi diff --git a/src/3.7/test-nest.sh b/src/3.7/test-nest.sh new file mode 100644 index 0000000..d73492a --- /dev/null +++ b/src/3.7/test-nest.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# run mpiexec as root +export OMPI_ALLOW_RUN_AS_ROOT=1 +export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 + +# install what is needed +pip3 install pytest-xdist pytest-timeout junitparser + +HDF5_DISABLE_VERSION_CHECK=1 + +cd /opt/nest/share/nest/testsuite/ +bash do_tests.sh --prefix=/opt/nest --with-python=/usr/bin/python From f16462a01721b4269965feb1c777afa8ae72a391 Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Thu, 14 Mar 2024 09:03:27 +0000 Subject: [PATCH 07/17] Use bash and fail to an error --- src/3.7/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/3.7/Dockerfile b/src/3.7/Dockerfile index 4a7fae6..0959bc1 100644 --- a/src/3.7/Dockerfile +++ b/src/3.7/Dockerfile @@ -12,6 +12,8 @@ ENV TERM=xterm \ DEBIAN_FRONTEND=noninteractive \ PATH=/root/.local/bin:${PATH} +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + RUN apt-get update && apt-get install -y --no-install-recommends \ automake \ autotools-dev \ @@ -19,14 +21,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ccache \ cmake \ curl \ - cython3 \ + # cython: https://cython.readthedocs.io/en/latest/src/quickstart/install.html freeglut3-dev \ gosu \ jq \ less \ libboost-filesystem-dev libboost-regex-dev libboost-wave-dev \ libboost-python-dev libboost-program-options-dev libboost-test-dev \ - libboost-dev \ + # libboost-dev \ libgomp1 \ libgsl-dev \ libgsl0-dev \ @@ -79,14 +81,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ uwsgi \ vera++ \ wget && \ - apt-get autoremove + apt-get clean && rm -rf /var/lib/apt/lists/* RUN python3 -m pip install --upgrade pip setuptools wheel mock RUN python3 -m pip install --force-reinstall --upgrade --no-binary=h5py h5py RUN ldconfig - - # Download NEST RUN cd ${SRC_PATH} && \ wget https://github.com/nest/nest-simulator/archive/refs/heads/${NEST_VERSION}.tar.gz -P ${SRC_PATH} && \ From 08098db7c511cb213d5f0271c939f49593b652ce Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Thu, 14 Mar 2024 10:17:21 +0000 Subject: [PATCH 08/17] Add 3.7 to run skript --- run.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/run.sh b/run.sh index d36f0dd..e181ee3 100755 --- a/run.sh +++ b/run.sh @@ -50,11 +50,11 @@ case $command in echo echo "Provisioning needs an argument: 'dev' 'latest_daint' '2.12.0', '2.14.0', '2.14.2'," - echo "'2.16.0', '2.18.0', '2.20.0', '2.20.1', '2.20.2', '3.0','3.1', '3.2', '3.3', '3.4', '3.5', '3.6','all' or 'base'." + echo "'2.16.0', '2.18.0', '2.20.0', '2.20.1', '2.20.2', '3.0','3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7','all' or 'base'." echo while test $# -gt 0; do case "$1" in - dev | latest_daint | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6) + dev | latest_daint | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 | 3.7) echo "Build the NEST image for NEST $1" echo docker build -t nest/nest-simulator:"$1" ./src/"$1" @@ -73,7 +73,7 @@ case $command in all) echo "Build the NEST image for NEST 2.12.0, 2.14.0, 2.14.2" echo "2.16.0, 2.18.0, 2.20.0, 2.20.1, 2.20.2, 3.0, 3.1, 3.2," - echo "3.3, 3.4, 3.5, 3.6, dev and latest_daint" + echo "3.3, 3.4, 3.5, 3.6, 3.7, dev and latest_daint" echo docker build -t nest/nest-simulator:2.12.0 ./src/2.12.0 docker build -t nest/nest-simulator:2.14.0 ./src/2.14.0 @@ -88,8 +88,9 @@ case $command in docker build -t nest/nest-simulator:3.2 ./src/3.2 docker build -t nest/nest-simulator:3.3 ./src/3.3 docker build -t nest/nest-simulator:3.4 ./src/3.4 - docker build -t nest/nest-simulator:3.4 ./src/3.5 - docker build -t nest/nest-simulator:3.4 ./src/3.6 + docker build -t nest/nest-simulator:3.5 ./src/3.5 + docker build -t nest/nest-simulator:3.6 ./src/3.6 + docker build -t nest/nest-simulator:3.7rc1 ./src/3.7 docker build -t nest/nest-simulator:dev ./src/dev docker build -t nest/nest-simulator:latest_daint ./src/latest_daint echo @@ -113,14 +114,14 @@ case $command in echo echo "VERSION is the version of NEST" echo "(e.g. dev, 2.12.0, 2.14.0, 2.14.2, 2.16.0, 2.18.0, 2.20.0," - echo "2.20.1, 2.20.2, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6)" + echo "2.20.1, 2.20.2, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7)" echo LOCALDIR="$(pwd)" while test $# -gt 1; do case "$1" in notebook) case "$2" in - dev | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 ) + dev | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 | 3.7rc1 ) echo "Run NEST-$2 with Jupyter Notebook". echo docker run -it --rm -e LOCAL_USER_ID=`id -u $USER` --name my_app \ @@ -136,7 +137,7 @@ case $command in ;; jupyterlab) case "$2" in - dev | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 ) + dev | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 | 3.7rc1 ) echo docker run -it --rm -e LOCAL_USER_ID=`id -u $USER` --name my_app \ -v $(pwd):/opt/data -e NEST_CONTAINER_MODE=jupyterlab \ @@ -151,7 +152,7 @@ case $command in ;; interactive) case "$2" in - dev | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 ) + dev | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 | 3.7rc1 ) echo "Run NEST-$2 in interactive mode." echo docker run -it --rm -e LOCAL_USER_ID=`id -u $USER` --name my_app -e NEST_CONTAINER_MODE=interactive \ From b6e7a5da5836003411fcf424c3c041798ae98224 Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Thu, 14 Mar 2024 10:34:10 +0000 Subject: [PATCH 09/17] Remove all dev packages to shrink the image --- src/3.7/Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/3.7/Dockerfile b/src/3.7/Dockerfile index 0959bc1..8390149 100644 --- a/src/3.7/Dockerfile +++ b/src/3.7/Dockerfile @@ -80,8 +80,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ tk8.6-dev \ uwsgi \ vera++ \ - wget && \ - apt-get clean && rm -rf /var/lib/apt/lists/* + wget + RUN python3 -m pip install --upgrade pip setuptools wheel mock RUN python3 -m pip install --force-reinstall --upgrade --no-binary=h5py h5py @@ -136,6 +136,10 @@ RUN python3 -m pip install --upgrade pip && \ RUN python3 -m pip install --force-reinstall --upgrade scipy +# Tidy +RUN apt-get remove *-dev \ + apt-get clean && rm -rf /var/lib/apt/lists/* + COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh From c556377cab64d6f49e955224244b7125f89bd424 Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Fri, 15 Mar 2024 05:40:06 +0000 Subject: [PATCH 10/17] Goingback to working setup --- src/3.7/Dockerfile | 138 ++++++++------------------------------------- 1 file changed, 23 insertions(+), 115 deletions(-) diff --git a/src/3.7/Dockerfile b/src/3.7/Dockerfile index 8390149..f0894fa 100644 --- a/src/3.7/Dockerfile +++ b/src/3.7/Dockerfile @@ -1,117 +1,21 @@ -FROM ubuntu:22.04 +FROM docker-registry.ebrains.eu/nest/nest-simulator:nest-simulator-build-base as builder LABEL maintainer="s.graber@fz-juelich.de" -ARG NEST_VERSION=master +ARG NEST_VERSION=3.6 ARG SRC_PATH=/tmp ARG CMAKE_C_COMPILER_LAUNCHER=ccache ARG CMAKE_CXX_COMPILER_LAUNCHER=ccache -ARG CXX_FLAGS="-pedantic -Wextra -Wno-unknown-pragmas -D_GLIBCXX_ASSERTIONS" +ARG CXXFLAGS="-O2 -lrt" -ENV TERM=xterm \ - TZ=Europe/Berlin \ - DEBIAN_FRONTEND=noninteractive \ - PATH=/root/.local/bin:${PATH} -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN apt-get update && apt-get install -y --no-install-recommends \ - automake \ - autotools-dev \ - build-essential \ - ccache \ - cmake \ - curl \ - # cython: https://cython.readthedocs.io/en/latest/src/quickstart/install.html - freeglut3-dev \ - gosu \ - jq \ - less \ - libboost-filesystem-dev libboost-regex-dev libboost-wave-dev \ - libboost-python-dev libboost-program-options-dev libboost-test-dev \ - # libboost-dev \ - libgomp1 \ - libgsl-dev \ - libgsl0-dev \ - libhdf5-dev \ - libltdl-dev \ - libltdl7 \ - libmusic1v5 \ - libncurses-dev \ - libomp-dev \ - libopenmpi-dev \ - libpcre3 \ - libpcre3-dev \ - libpython3.10 \ - libreadline-dev \ - libreadline8 \ - libtool \ - libzmq3-dev \ - llvm-dev \ - nano \ - openmpi-bin \ - openssh-client \ - openssh-server \ - pandoc \ - pep8 \ - pkg-config \ - python3-dev \ - python3-flask \ - python3-flask-cors \ - python3-h5py \ - python3-ipython \ - python3-jupyter-core \ - python3-matplotlib \ - python3-mpi4py \ - python3-nose \ - python3-numpy \ - python3-pandas \ - python3-path \ - python3-pip \ - python3-restrictedpython \ - python3-scipy \ - python3-setuptools \ - python3-sphinx \ - python3-statsmodels \ - python3-sympy \ - python3-tk \ - python3-tqdm \ - tcl8.6 \ - tcl8.6-dev \ - tk8.6-dev \ - uwsgi \ - vera++ \ - wget - - -RUN python3 -m pip install --upgrade pip setuptools wheel mock -RUN python3 -m pip install --force-reinstall --upgrade --no-binary=h5py h5py -RUN ldconfig - -# Download NEST -RUN cd ${SRC_PATH} && \ - wget https://github.com/nest/nest-simulator/archive/refs/heads/${NEST_VERSION}.tar.gz -P ${SRC_PATH} && \ - tar -xzf ${NEST_VERSION}.tar.gz -RUN python3 -m pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/doc/requirements.txt && \ +# Install NEST and NESTML +RUN wget "https://github.com/nest/nest-simulator/archive/refs/tags/v${NEST_VERSION}.tar.gz" -P ${SRC_PATH} && \ + cd ${SRC_PATH} && tar -xzf v${NEST_VERSION}.tar.gz && ls -l && \ + python3 -m pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/doc/requirements.txt && \ python3 -m pip install sphinx_gallery==0.10.1 -# Install music -RUN chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh && \ - ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh -# # Install libneurosim -# RUN cd ${SRC_PATH}/nest-simulator-${NEST_VERSION} && \ -# PYLIB_DIR="$(python3 -c "import sysconfig; print(sysconfig.get_path('include'))" | sed 's/include/lib/')" && \ -# chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_csa-libneurosim.sh && \ -# ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_csa-libneurosim.sh $PYLIB_DIR -# # Install sionlib -# RUN chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_sionlib.sh && \ -# ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_sionlib.sh -# Install python requirements -# RUN pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/requirements.txt - -# Build nest -RUN mkdir ${SRC_PATH}/nest-build && cd $_ && \ +RUN mkdir nest-build && cd nest-build && \ cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/nest \ - -Dwith-optimize="-O2" \ -Dwith-warning=ON \ -Dwith-userdoc=ON \ -Dwith-boost=ON \ @@ -122,24 +26,28 @@ RUN mkdir ${SRC_PATH}/nest-build && cd $_ && \ -Dwith-mpi=ON \ -Dwith-openmp=ON \ -Dwith-libneurosim=OFF \ - -Dwith-sionlib=OFF \ - -Dwith-music=ON \ + -Dwith-music=/opt/music-install \ -Dwith-hdf5=ON \ ${SRC_PATH}/nest-simulator-${NEST_VERSION} && \ - make && \ - make install + make -j $(nproc) && \ + make docs && \ + make install + + + +############################################################################### +# DEPLOY + +FROM docker-registry.ebrains.eu/nest/nest-simulator:nest-simulator-deploy-base +LABEL maintainer="s.graber@fz-juelich.de" + +COPY --from=builder /opt/nest /opt/nest +COPY --from=builder /opt/music-install /opt/music-install -# Install NESTML RUN python3 -m pip install --upgrade pip && \ python3 -m pip install nest-desktop && \ python3 -m pip install nestml -RUN python3 -m pip install --force-reinstall --upgrade scipy - -# Tidy -RUN apt-get remove *-dev \ - apt-get clean && rm -rf /var/lib/apt/lists/* - COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh From 6c52a72df2186cd927d5c4f73a6053af4b8c44e9 Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Fri, 15 Mar 2024 05:42:16 +0000 Subject: [PATCH 11/17] Add Dwith-optimize="-O2" to cmake options --- src/3.7/Dockerfile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/3.7/Dockerfile b/src/3.7/Dockerfile index f0894fa..b2eb7e2 100644 --- a/src/3.7/Dockerfile +++ b/src/3.7/Dockerfile @@ -1,12 +1,18 @@ FROM docker-registry.ebrains.eu/nest/nest-simulator:nest-simulator-build-base as builder LABEL maintainer="s.graber@fz-juelich.de" -ARG NEST_VERSION=3.6 +ARG NEST_VERSION=master ARG SRC_PATH=/tmp ARG CMAKE_C_COMPILER_LAUNCHER=ccache ARG CMAKE_CXX_COMPILER_LAUNCHER=ccache -ARG CXXFLAGS="-O2 -lrt" +ARG CXX_FLAGS="-pedantic -Wextra -Wno-unknown-pragmas -D_GLIBCXX_ASSERTIONS" +ENV TERM=xterm \ + TZ=Europe/Berlin \ + DEBIAN_FRONTEND=noninteractive \ + PATH=/root/.local/bin:${PATH} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install NEST and NESTML RUN wget "https://github.com/nest/nest-simulator/archive/refs/tags/v${NEST_VERSION}.tar.gz" -P ${SRC_PATH} && \ @@ -16,6 +22,7 @@ RUN wget "https://github.com/nest/nest-simulator/archive/refs/tags/v${NEST_VERSI RUN mkdir nest-build && cd nest-build && \ cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/nest \ + -Dwith-optimize="-O2" \ -Dwith-warning=ON \ -Dwith-userdoc=ON \ -Dwith-boost=ON \ From d335b33c363bcbbf187e8d8f735c1c242ffcf97f Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Fri, 15 Mar 2024 08:21:00 +0000 Subject: [PATCH 12/17] Fix numpy version and rename to 3.7rc1 --- run.sh | 10 +++++----- src/{3.7 => 3.7rc1}/Dockerfile | 29 +++++++++++++---------------- src/{3.7 => 3.7rc1}/entrypoint.sh | 0 src/{3.7 => 3.7rc1}/test-nest.sh | 0 4 files changed, 18 insertions(+), 21 deletions(-) rename src/{3.7 => 3.7rc1}/Dockerfile (77%) rename src/{3.7 => 3.7rc1}/entrypoint.sh (100%) rename src/{3.7 => 3.7rc1}/test-nest.sh (100%) diff --git a/run.sh b/run.sh index e181ee3..03955bf 100755 --- a/run.sh +++ b/run.sh @@ -50,11 +50,11 @@ case $command in echo echo "Provisioning needs an argument: 'dev' 'latest_daint' '2.12.0', '2.14.0', '2.14.2'," - echo "'2.16.0', '2.18.0', '2.20.0', '2.20.1', '2.20.2', '3.0','3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7','all' or 'base'." + echo "'2.16.0', '2.18.0', '2.20.0', '2.20.1', '2.20.2', '3.0','3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7rc1','all' or 'base'." echo while test $# -gt 0; do case "$1" in - dev | latest_daint | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 | 3.7) + dev | latest_daint | 2.12.0 | 2.14.0 | 2.14.2 | 2.16.0 | 2.18.0 | 2.20.0 | 2.20.1 | 2.20.2 | 3.0 | 3.1 | 3.2 | 3.3 | 3.4 | 3.5 | 3.6 | 3.7rc1 ) echo "Build the NEST image for NEST $1" echo docker build -t nest/nest-simulator:"$1" ./src/"$1" @@ -73,7 +73,7 @@ case $command in all) echo "Build the NEST image for NEST 2.12.0, 2.14.0, 2.14.2" echo "2.16.0, 2.18.0, 2.20.0, 2.20.1, 2.20.2, 3.0, 3.1, 3.2," - echo "3.3, 3.4, 3.5, 3.6, 3.7, dev and latest_daint" + echo "3.3, 3.4, 3.5, 3.6, 3.7rc1, dev and latest_daint" echo docker build -t nest/nest-simulator:2.12.0 ./src/2.12.0 docker build -t nest/nest-simulator:2.14.0 ./src/2.14.0 @@ -90,7 +90,7 @@ case $command in docker build -t nest/nest-simulator:3.4 ./src/3.4 docker build -t nest/nest-simulator:3.5 ./src/3.5 docker build -t nest/nest-simulator:3.6 ./src/3.6 - docker build -t nest/nest-simulator:3.7rc1 ./src/3.7 + docker build -t nest/nest-simulator:3.7rc1 ./src/3.7rc1 docker build -t nest/nest-simulator:dev ./src/dev docker build -t nest/nest-simulator:latest_daint ./src/latest_daint echo @@ -114,7 +114,7 @@ case $command in echo echo "VERSION is the version of NEST" echo "(e.g. dev, 2.12.0, 2.14.0, 2.14.2, 2.16.0, 2.18.0, 2.20.0," - echo "2.20.1, 2.20.2, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7)" + echo "2.20.1, 2.20.2, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7rc1)" echo LOCALDIR="$(pwd)" while test $# -gt 1; do diff --git a/src/3.7/Dockerfile b/src/3.7rc1/Dockerfile similarity index 77% rename from src/3.7/Dockerfile rename to src/3.7rc1/Dockerfile index b2eb7e2..65c6849 100644 --- a/src/3.7/Dockerfile +++ b/src/3.7rc1/Dockerfile @@ -1,7 +1,7 @@ FROM docker-registry.ebrains.eu/nest/nest-simulator:nest-simulator-build-base as builder LABEL maintainer="s.graber@fz-juelich.de" -ARG NEST_VERSION=master +ARG NEST_VERSION=3.6 ARG SRC_PATH=/tmp ARG CMAKE_C_COMPILER_LAUNCHER=ccache ARG CMAKE_CXX_COMPILER_LAUNCHER=ccache @@ -17,10 +17,10 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install NEST and NESTML RUN wget "https://github.com/nest/nest-simulator/archive/refs/tags/v${NEST_VERSION}.tar.gz" -P ${SRC_PATH} && \ cd ${SRC_PATH} && tar -xzf v${NEST_VERSION}.tar.gz && ls -l && \ + apt-get remove -y python3-numpy && \ python3 -m pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/doc/requirements.txt && \ - python3 -m pip install sphinx_gallery==0.10.1 - -RUN mkdir nest-build && cd nest-build && \ + python3 -m pip install sphinx_gallery==0.10.1 numpy>=1.17.3,1.25.0 && \ + mkdir nest-build && cd nest-build && \ cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/nest \ -Dwith-optimize="-O2" \ -Dwith-warning=ON \ @@ -40,8 +40,6 @@ RUN mkdir nest-build && cd nest-build && \ make docs && \ make install - - ############################################################################### # DEPLOY @@ -50,16 +48,15 @@ LABEL maintainer="s.graber@fz-juelich.de" COPY --from=builder /opt/nest /opt/nest COPY --from=builder /opt/music-install /opt/music-install - -RUN python3 -m pip install --upgrade pip && \ - python3 -m pip install nest-desktop && \ - python3 -m pip install nestml - -COPY entrypoint.sh /usr/local/bin/entrypoint.sh -RUN chmod +x /usr/local/bin/entrypoint.sh - -COPY test-nest.sh /opt/test-nest.sh -RUN chmod +x /opt/test-nest.sh +COPY entrypoint.sh /usr/local/bin/ +COPY test-nest.sh /opt/ + +RUN python3 -m pip install nest-desktop && \ + python3 -m pip install nestml && \ + apt-get remove -y python3-scipy && \ + python3 -m pip install scipy numpy && \ + chmod +x /usr/local/bin/entrypoint.sh && \ + chmod +x /opt/test-nest.sh EXPOSE 8080 52425 54286 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/src/3.7/entrypoint.sh b/src/3.7rc1/entrypoint.sh similarity index 100% rename from src/3.7/entrypoint.sh rename to src/3.7rc1/entrypoint.sh diff --git a/src/3.7/test-nest.sh b/src/3.7rc1/test-nest.sh similarity index 100% rename from src/3.7/test-nest.sh rename to src/3.7rc1/test-nest.sh From 501a9b5004aeaf52c6bc93d0e3c5e34c339ab943 Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Mon, 25 Mar 2024 12:45:46 +0000 Subject: [PATCH 13/17] Add 3.7rc1 to CI --- ci-templates/000_3.7.gitlab-ci.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 ci-templates/000_3.7.gitlab-ci.yml diff --git a/ci-templates/000_3.7.gitlab-ci.yml b/ci-templates/000_3.7.gitlab-ci.yml new file mode 100644 index 0000000..27f244b --- /dev/null +++ b/ci-templates/000_3.7.gitlab-ci.yml @@ -0,0 +1,28 @@ +############################################### +### v3.6 ### +############################################### + +Build_36: + stage: build + needs: ["Build_Base"] + rules: + - when: always + script: + # Build + - docker pull nest/nest-simulator:3.7rc1 || true + - docker build + --cache-from nest/nest-simulator:3.7rc1 + --tag nest/nest-simulator:3.7rc1 + ./src/3.7rc1 + # Test + - docker run -i --rm nest/nest-simulator:3.7rc1 bash /opt/test-nest.sh + # Deploy + - echo -n $DOCKERHUB_REGISTRY_TOKEN | docker login -u $DOCKERHUB_REGISTRY_USER --password-stdin + - docker push nest/nest-simulator:3.7rc1 + - docker tag nest/nest-simulator:3.7rc1 $DOCKER_REGISTRY_IMAGE:3.7rc1 + - docker push $DOCKER_REGISTRY_IMAGE:3.7rc1 + - docker logout $DOCKERHUB_REGISTRY + - docker logout $DOCKER_REGISTRY + tags: + - shell-runner + From aedb4904134d88298f98f404be21cc8ff14e443f Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Mon, 25 Mar 2024 12:46:22 +0000 Subject: [PATCH 14/17] Add 3.7rc1 --- src/3.7rc1/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/3.7rc1/Dockerfile b/src/3.7rc1/Dockerfile index 65c6849..72fec69 100644 --- a/src/3.7rc1/Dockerfile +++ b/src/3.7rc1/Dockerfile @@ -1,7 +1,7 @@ FROM docker-registry.ebrains.eu/nest/nest-simulator:nest-simulator-build-base as builder LABEL maintainer="s.graber@fz-juelich.de" -ARG NEST_VERSION=3.6 +ARG NEST_VERSION=3.7_rc1 ARG SRC_PATH=/tmp ARG CMAKE_C_COMPILER_LAUNCHER=ccache ARG CMAKE_CXX_COMPILER_LAUNCHER=ccache @@ -37,7 +37,6 @@ RUN wget "https://github.com/nest/nest-simulator/archive/refs/tags/v${NEST_VERSI -Dwith-hdf5=ON \ ${SRC_PATH}/nest-simulator-${NEST_VERSION} && \ make -j $(nproc) && \ - make docs && \ make install ############################################################################### From e90ba73badab1f67577ecacfded1dbff99d8f084 Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Tue, 26 Mar 2024 04:21:21 +0000 Subject: [PATCH 15/17] Complet rework of dockerfile --- ci-templates/000_3.7.gitlab-ci.yml | 10 +- src/3.7rc1/Dockerfile | 142 +++++++++++++++++++++++------ 2 files changed, 117 insertions(+), 35 deletions(-) diff --git a/ci-templates/000_3.7.gitlab-ci.yml b/ci-templates/000_3.7.gitlab-ci.yml index 27f244b..8f8efb2 100644 --- a/ci-templates/000_3.7.gitlab-ci.yml +++ b/ci-templates/000_3.7.gitlab-ci.yml @@ -1,15 +1,14 @@ ############################################### -### v3.6 ### +### 3.7rc1 ### ############################################### -Build_36: +Build_Dev: stage: build - needs: ["Build_Base"] rules: - when: always script: # Build - - docker pull nest/nest-simulator:3.7rc1 || true + - docker pull push nest/nest-simulator:3.7rc1 || true - docker build --cache-from nest/nest-simulator:3.7rc1 --tag nest/nest-simulator:3.7rc1 @@ -19,10 +18,7 @@ Build_36: # Deploy - echo -n $DOCKERHUB_REGISTRY_TOKEN | docker login -u $DOCKERHUB_REGISTRY_USER --password-stdin - docker push nest/nest-simulator:3.7rc1 - - docker tag nest/nest-simulator:3.7rc1 $DOCKER_REGISTRY_IMAGE:3.7rc1 - - docker push $DOCKER_REGISTRY_IMAGE:3.7rc1 - docker logout $DOCKERHUB_REGISTRY - - docker logout $DOCKER_REGISTRY tags: - shell-runner diff --git a/src/3.7rc1/Dockerfile b/src/3.7rc1/Dockerfile index 72fec69..9ef7641 100644 --- a/src/3.7rc1/Dockerfile +++ b/src/3.7rc1/Dockerfile @@ -1,7 +1,7 @@ -FROM docker-registry.ebrains.eu/nest/nest-simulator:nest-simulator-build-base as builder +FROM ubuntu:22.04 LABEL maintainer="s.graber@fz-juelich.de" -ARG NEST_VERSION=3.7_rc1 +ARG NEST_VERSION=v3.7_rc1 ARG SRC_PATH=/tmp ARG CMAKE_C_COMPILER_LAUNCHER=ccache ARG CMAKE_CXX_COMPILER_LAUNCHER=ccache @@ -12,15 +12,104 @@ ENV TERM=xterm \ DEBIAN_FRONTEND=noninteractive \ PATH=/root/.local/bin:${PATH} -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN apt-get update && apt-get install -y --no-install-recommends \ + automake \ + autotools-dev \ + build-essential \ + ccache \ + cmake \ + curl \ + cython3 \ + freeglut3-dev \ + gosu \ + jq \ + less \ + libboost-filesystem-dev libboost-regex-dev libboost-wave-dev \ + libboost-python-dev libboost-program-options-dev libboost-test-dev \ + libboost-dev \ + libgomp1 \ + libgsl-dev \ + libgsl0-dev \ + libhdf5-dev \ + libltdl-dev \ + libltdl7 \ + libmusic1v5 \ + libncurses-dev \ + libomp-dev \ + libopenmpi-dev \ + libpcre3 \ + libpcre3-dev \ + libpython3.10 \ + libreadline-dev \ + libreadline8 \ + libtool \ + libzmq3-dev \ + llvm-dev \ + nano \ + openmpi-bin \ + openssh-client \ + openssh-server \ + pandoc \ + pep8 \ + pkg-config \ + python3-dev \ + python3-flask \ + python3-flask-cors \ + python3-h5py \ + python3-ipython \ + python3-jupyter-core \ + python3-matplotlib \ + python3-mpi4py \ + python3-nose \ + python3-numpy \ + python3-pandas \ + python3-path \ + python3-pip \ + python3-restrictedpython \ + python3-scipy \ + python3-setuptools \ + python3-sphinx \ + python3-statsmodels \ + python3-sympy \ + python3-tk \ + python3-tqdm \ + tcl8.6 \ + tcl8.6-dev \ + tk8.6-dev \ + uwsgi \ + vera++ \ + wget && \ + apt-get autoremove -# Install NEST and NESTML -RUN wget "https://github.com/nest/nest-simulator/archive/refs/tags/v${NEST_VERSION}.tar.gz" -P ${SRC_PATH} && \ - cd ${SRC_PATH} && tar -xzf v${NEST_VERSION}.tar.gz && ls -l && \ - apt-get remove -y python3-numpy && \ - python3 -m pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/doc/requirements.txt && \ - python3 -m pip install sphinx_gallery==0.10.1 numpy>=1.17.3,1.25.0 && \ - mkdir nest-build && cd nest-build && \ +RUN python3 -m pip install --upgrade pip setuptools wheel mock +RUN python3 -m pip install --force-reinstall --upgrade --no-binary=h5py h5py +RUN ldconfig + + + +# Download NEST +RUN cd ${SRC_PATH} && \ + wget "https://github.com/nest/nest-simulator/archive/refs/tags/v${NEST_VERSION}.tar.gz" -P ${SRC_PATH} && \ + tar -xzf ${NEST_VERSION}.tar.gz +RUN python3 -m pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/doc/requirements.txt && \ + python3 -m pip install sphinx_gallery==0.10.1 +# Install music +RUN chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh && \ + ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh + +# # Install libneurosim +# RUN cd ${SRC_PATH}/nest-simulator-${NEST_VERSION} && \ +# PYLIB_DIR="$(python3 -c "import sysconfig; print(sysconfig.get_path('include'))" | sed 's/include/lib/')" && \ +# chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_csa-libneurosim.sh && \ +# ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_csa-libneurosim.sh $PYLIB_DIR +# # Install sionlib +# RUN chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_sionlib.sh && \ +# ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_sionlib.sh +# Install python requirements +# RUN pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/requirements.txt + +# Build nest +RUN mkdir ${SRC_PATH}/nest-build && cd $_ && \ cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/nest \ -Dwith-optimize="-O2" \ -Dwith-warning=ON \ @@ -33,29 +122,26 @@ RUN wget "https://github.com/nest/nest-simulator/archive/refs/tags/v${NEST_VERSI -Dwith-mpi=ON \ -Dwith-openmp=ON \ -Dwith-libneurosim=OFF \ - -Dwith-music=/opt/music-install \ + -Dwith-sionlib=OFF \ + -Dwith-music=ON \ -Dwith-hdf5=ON \ ${SRC_PATH}/nest-simulator-${NEST_VERSION} && \ - make -j $(nproc) && \ - make install + make && \ + make install -############################################################################### -# DEPLOY +# Install NESTML and more +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip install nest-desktop --pre && \ + python3 -m pip uninstall nestml -y && \ + python3 -m pip install --upgrade https://github.com/nest/nestml/archive/refs/heads/master.zip -FROM docker-registry.ebrains.eu/nest/nest-simulator:nest-simulator-deploy-base -LABEL maintainer="s.graber@fz-juelich.de" +RUN python3 -m pip install --force-reinstall --upgrade scipy + +COPY entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh -COPY --from=builder /opt/nest /opt/nest -COPY --from=builder /opt/music-install /opt/music-install -COPY entrypoint.sh /usr/local/bin/ -COPY test-nest.sh /opt/ - -RUN python3 -m pip install nest-desktop && \ - python3 -m pip install nestml && \ - apt-get remove -y python3-scipy && \ - python3 -m pip install scipy numpy && \ - chmod +x /usr/local/bin/entrypoint.sh && \ - chmod +x /opt/test-nest.sh +COPY test-nest.sh /opt/test-nest.sh +RUN chmod +x /opt/test-nest.sh EXPOSE 8080 52425 54286 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] From d01dbc9d61574af305267f46803722cde33a59ee Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Tue, 26 Mar 2024 06:14:14 +0000 Subject: [PATCH 16/17] Combine image layers --- src/3.7rc1/Dockerfile | 54 +++++++++++++------------------------------ 1 file changed, 16 insertions(+), 38 deletions(-) diff --git a/src/3.7rc1/Dockerfile b/src/3.7rc1/Dockerfile index 9ef7641..8192aa2 100644 --- a/src/3.7rc1/Dockerfile +++ b/src/3.7rc1/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:22.04 LABEL maintainer="s.graber@fz-juelich.de" -ARG NEST_VERSION=v3.7_rc1 +ARG NEST_VERSION=3.7_rc1 ARG SRC_PATH=/tmp ARG CMAKE_C_COMPILER_LAUNCHER=ccache ARG CMAKE_CXX_COMPILER_LAUNCHER=ccache @@ -79,37 +79,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ uwsgi \ vera++ \ wget && \ - apt-get autoremove - -RUN python3 -m pip install --upgrade pip setuptools wheel mock -RUN python3 -m pip install --force-reinstall --upgrade --no-binary=h5py h5py -RUN ldconfig - - - -# Download NEST -RUN cd ${SRC_PATH} && \ + apt-get autoremove && \ + python3 -m pip install --upgrade pip setuptools wheel mock && \ + python3 -m pip install --force-reinstall --upgrade --no-binary=h5py h5py && \ + ldconfig && \ + cd ${SRC_PATH} && \ wget "https://github.com/nest/nest-simulator/archive/refs/tags/v${NEST_VERSION}.tar.gz" -P ${SRC_PATH} && \ - tar -xzf ${NEST_VERSION}.tar.gz -RUN python3 -m pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/doc/requirements.txt && \ - python3 -m pip install sphinx_gallery==0.10.1 -# Install music -RUN chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh && \ - ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh - -# # Install libneurosim -# RUN cd ${SRC_PATH}/nest-simulator-${NEST_VERSION} && \ -# PYLIB_DIR="$(python3 -c "import sysconfig; print(sysconfig.get_path('include'))" | sed 's/include/lib/')" && \ -# chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_csa-libneurosim.sh && \ -# ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_csa-libneurosim.sh $PYLIB_DIR -# # Install sionlib -# RUN chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_sionlib.sh && \ -# ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_sionlib.sh -# Install python requirements -# RUN pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/requirements.txt - -# Build nest -RUN mkdir ${SRC_PATH}/nest-build && cd $_ && \ + tar -xzf v${NEST_VERSION}.tar.gz && \ + python3 -m pip install -r ${SRC_PATH}/nest-simulator-${NEST_VERSION}/doc/requirements.txt && \ + python3 -m pip install sphinx_gallery==0.10.1 && \ + chmod +x ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh && \ + ${SRC_PATH}/nest-simulator-${NEST_VERSION}/build_support/install_music.sh && \ + mkdir ${SRC_PATH}/nest-build && cd $_ && \ cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/nest \ -Dwith-optimize="-O2" \ -Dwith-warning=ON \ @@ -127,15 +108,12 @@ RUN mkdir ${SRC_PATH}/nest-build && cd $_ && \ -Dwith-hdf5=ON \ ${SRC_PATH}/nest-simulator-${NEST_VERSION} && \ make && \ - make install - -# Install NESTML and more -RUN python3 -m pip install --upgrade pip && \ + make install && \ + python3 -m pip install --upgrade pip && \ python3 -m pip install nest-desktop --pre && \ python3 -m pip uninstall nestml -y && \ - python3 -m pip install --upgrade https://github.com/nest/nestml/archive/refs/heads/master.zip - -RUN python3 -m pip install --force-reinstall --upgrade scipy + python3 -m pip install --upgrade https://github.com/nest/nestml/archive/refs/heads/master.zip && \ + python3 -m pip install --force-reinstall --upgrade scipy COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh From 996613ed8a898a0c9a6d0fe51a277480e71734ac Mon Sep 17 00:00:00 2001 From: Steffen Graber Date: Tue, 26 Mar 2024 06:23:01 +0000 Subject: [PATCH 17/17] Fix naming --- ci-templates/000_3.7.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-templates/000_3.7.gitlab-ci.yml b/ci-templates/000_3.7.gitlab-ci.yml index 8f8efb2..8615677 100644 --- a/ci-templates/000_3.7.gitlab-ci.yml +++ b/ci-templates/000_3.7.gitlab-ci.yml @@ -2,7 +2,7 @@ ### 3.7rc1 ### ############################################### -Build_Dev: +Build_3.7: stage: build rules: - when: always