From 7a58efb2500a4e8a1a094fcd31aceb87941f264f Mon Sep 17 00:00:00 2001 From: Herve Le-Bars Date: Fri, 23 Feb 2024 11:53:36 +0100 Subject: [PATCH 1/6] fix: error ModuleNotFoundError: No module named 'container' at runtime --- docker-env/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-env/Dockerfile b/docker-env/Dockerfile index 37da4ca7..571c8c5d 100644 --- a/docker-env/Dockerfile +++ b/docker-env/Dockerfile @@ -9,6 +9,7 @@ WORKDIR /source_code COPY bloom/ ./bloom/ COPY data/ ./data/ COPY app.py . +COPY container.py . COPY docker-env/rsyslog.conf /etc/rsyslog.conf # Install requirements package for python with poetry From f456663ce2aff3811b4e0295943a4d6fe0327261 Mon Sep 17 00:00:00 2001 From: Herve Le-Bars Date: Fri, 23 Feb 2024 11:01:10 +0100 Subject: [PATCH 2/6] fix: user missing in cron_scrapper file --- docker-env/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-env/Dockerfile b/docker-env/Dockerfile index 571c8c5d..17717fe2 100644 --- a/docker-env/Dockerfile +++ b/docker-env/Dockerfile @@ -39,7 +39,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install -y && rm -rf /etc/cron.*/* # Create cron task inside container -RUN echo '*/15 * * * * /venv/bin/python3 /source_code/app.py 2>&1 | /usr/bin/logger -t bloom' >> ./cron_scrapper +RUN echo '*/15 * * * * root /venv/bin/python3 /source_code/app.py 2>&1 | /usr/bin/logger -t bloom' >> ./cron_scrapper RUN chmod 744 ./cron_scrapper # Move cron tab into the right directory From d93ccebaa63664f11d4a31b6949f7aaecf72d0b8 Mon Sep 17 00:00:00 2001 From: RV Date: Sat, 24 Feb 2024 17:09:33 +0100 Subject: [PATCH 3/6] fix: share ENV vars between terminal and cron environment via ~/.env key/value pairs --- docker-env/Dockerfile | 6 +++++- docker-env/entrypoint.sh | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docker-env/Dockerfile b/docker-env/Dockerfile index 17717fe2..3bffe05e 100644 --- a/docker-env/Dockerfile +++ b/docker-env/Dockerfile @@ -39,7 +39,11 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install -y && rm -rf /etc/cron.*/* # Create cron task inside container -RUN echo '*/15 * * * * root /venv/bin/python3 /source_code/app.py 2>&1 | /usr/bin/logger -t bloom' >> ./cron_scrapper +# Due to the fact that cron process doesn't access to declared ENV vars and doesn't load user profiles +# The entrypoint.sh script stores ENV vars at runtime in the ~/.env file as key=value pairs +# Then the cron line include some command to load these ENV vars from file before launching app.py +# This mecanism allows to give access to the same ENV vars for app.py launch in terminal and launch via cron +RUN echo "*/15 * * * * root export \$(cat ~/.env | grep -v '#' | xargs);/venv/bin/python3 /source_code/app.py 2>&1 | /usr/bin/logger -t bloom" >> ./cron_scrapper RUN chmod 744 ./cron_scrapper # Move cron tab into the right directory diff --git a/docker-env/entrypoint.sh b/docker-env/entrypoint.sh index 052c54bb..5914e212 100644 --- a/docker-env/entrypoint.sh +++ b/docker-env/entrypoint.sh @@ -2,6 +2,13 @@ cat /source_code/.env >> /etc/environment +# Store all APP_BLOOM ENV variables to local file accessible to current user to share them with cron task +# Due to the fact that cron process doesn't access to declared ENV vars and doesn't load user profiles +# The entrypoint.sh script stores ENV vars at runtime in the ~/.env file as key=value pairs +# Then the cron line include some command to load these ENV vars from file before launching app.py +# This mecanism allows to give access to the same ENV vars for app.py launch in terminal and launch via cron +env | egrep '^(POSTGRES_.*|SPIRE_TOKEN.*)' > ~/.env + /etc/init.d/rsyslog restart ln -sf /dev/stdout /var/log/syslog From 61b4ff46b375982876fe841d6b3358806ccc9e39 Mon Sep 17 00:00:00 2001 From: RV Date: Sat, 24 Feb 2024 16:12:25 +0100 Subject: [PATCH 4/6] fix: compilation error python > 3.10 gcc not found & execvp: No such file or directory --- docker-env/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-env/Dockerfile b/docker-env/Dockerfile index 3bffe05e..2f589b2d 100644 --- a/docker-env/Dockerfile +++ b/docker-env/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.10-slim-bullseye RUN apt-get update -RUN apt-get -y install wget +RUN apt-get -y install wget gcc g++ RUN apt-get install -y rsyslog # Define working directory From 9bc0451527530d81d2b86fb578f3e4760838ba1a Mon Sep 17 00:00:00 2001 From: RV Date: Sat, 24 Feb 2024 16:45:35 +0100 Subject: [PATCH 5/6] fix: using absolute current file path instead of cwd that is not deterministic --- alembic/init_script/load_amp_data.py | 2 +- alembic/init_script/load_positions_data.py | 2 +- alembic/init_script/load_vessels_data.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/alembic/init_script/load_amp_data.py b/alembic/init_script/load_amp_data.py index 4512b43f..86326de1 100644 --- a/alembic/init_script/load_amp_data.py +++ b/alembic/init_script/load_amp_data.py @@ -34,7 +34,7 @@ engine = create_engine(db_url, echo=False) df = pd.read_csv( - Path.joinpath(Path.cwd(), "data/zones_subset_02022024.csv"), + Path(os.path.dirname(__file__)).joinpath("../../data/zones_subset_02022024.csv"), sep=",", ) diff --git a/alembic/init_script/load_positions_data.py b/alembic/init_script/load_positions_data.py index 35d371a8..6db0e4e3 100644 --- a/alembic/init_script/load_positions_data.py +++ b/alembic/init_script/load_positions_data.py @@ -31,7 +31,7 @@ engine = create_engine(db_url) df = pd.read_csv( - Path.joinpath(Path.cwd(), "data/spire_positions_subset_02022024.csv"), + Path(os.path.dirname(__file__)).joinpath("../../data/spire_positions_subset_02022024.csv"), sep="," ) diff --git a/alembic/init_script/load_vessels_data.py b/alembic/init_script/load_vessels_data.py index 64a5be60..03190585 100644 --- a/alembic/init_script/load_vessels_data.py +++ b/alembic/init_script/load_vessels_data.py @@ -29,7 +29,7 @@ ) engine = create_engine(db_url) df = pd.read_csv( - Path.joinpath(Path.cwd(), "data/chalutiers_pelagiques.csv"), + Path(os.path.dirname(__file__)).joinpath("../../data/chalutiers_pelagiques.csv"), sep=";", dtype={"loa": float, "IMO": str}, ) From 77fa996e0421359c4473d0e87023d7ea9824c051 Mon Sep 17 00:00:00 2001 From: Herve Le-Bars Date: Fri, 23 Feb 2024 11:58:10 +0100 Subject: [PATCH 6/6] feat: entrypoint (re)activation, explicit bash call --- docker-env/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-env/Dockerfile b/docker-env/Dockerfile index 2f589b2d..1492838d 100644 --- a/docker-env/Dockerfile +++ b/docker-env/Dockerfile @@ -55,8 +55,8 @@ RUN crontab /etc/cron.d/cron_scrapper COPY docker-env/entrypoint.sh /entrypoint.sh RUN ["chmod", "+x", "/entrypoint.sh"] -#ENTRYPOINT ["/entrypoint.sh"] +ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] # https://manpages.ubuntu.com/manpages/trusty/man8/cron.8.html # -f | Stay in foreground mode, don't daemonize. # -L loglevel | Tell cron what to log about jobs (errors are logged regardless of this value) as the sum of the following values: -#CMD ["cron","-f", "-L", "2"] +CMD ["cron","-f", "-L", "2"]