diff --git a/Dockerfiles/python.Dockerfile b/Dockerfiles/python.Dockerfile deleted file mode 100644 index b87967d..0000000 --- a/Dockerfiles/python.Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM python:3.9.16 -WORKDIR /home -ENV PATH "${PATH}:/root/.local/bin" -COPY requirements/python_requirements.txt /requirements.txt -COPY airflow/pytest.ini pytest.ini -COPY .startup/make_fernet_key.py make_fernet_key.py -RUN pip install --user --upgrade pip -RUN pip install --no-cache-dir --user -r /requirements.txt \ No newline at end of file diff --git a/Dockerfiles/superset.Dockerfile b/Dockerfiles/superset.Dockerfile deleted file mode 100644 index 99643c8..0000000 --- a/Dockerfiles/superset.Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM apache/superset:2.1.0 - -COPY requirements/superset/docker-init.sh /app/docker/docker-init.sh -COPY requirements/superset/pythonpath_dev /app/docker/pythonpath_dev -COPY requirements/superset/docker-bootstrap.sh /app/docker/docker-bootstrap.sh -COPY requirements/superset/superset_requirements.txt /app/docker/requirements.txt - -USER root -RUN pip install -v --no-cache -r /app/docker/requirements.txt -USER superset \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 2a181ec..85db92f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,8 +2,8 @@ version: "3.9" x-airflow-common: &airflow-common build: - context: ./ - dockerfile: ./Dockerfiles/airflow.Dockerfile + context: ./docker/airflow + dockerfile: airflow.Dockerfile env_file: - .env environment: &airflow-common-env @@ -33,8 +33,8 @@ x-airflow-common: &airflow-common x-superset-common: &superset-common build: - context: ./ - dockerfile: Dockerfiles/superset.Dockerfile + context: ./docker/superset + dockerfile: superset.Dockerfile networks: - re_etl_net env_file: @@ -68,8 +68,8 @@ services: ss_db: image: adwh_ss_db:14.5 build: - context: ./ - dockerfile: Dockerfiles/airflow_db.Dockerfile + context: ./docker/superset + dockerfile: postgres.Dockerfile env_file: - .env.superset networks: @@ -102,8 +102,8 @@ services: redis: image: adwh_redis:7.0.8 build: - context: ./ - dockerfile: Dockerfiles/redis.Dockerfile + context: ./docker/redis + dockerfile: redis.Dockerfile ports: - "6379:6379" volumes: @@ -120,8 +120,8 @@ services: airflow_db: image: adwh_airflow_pg_db:14.5 build: - context: ./ - dockerfile: Dockerfiles/airflow_db.Dockerfile + context: ./docker/airflow + dockerfile: airflow_db.Dockerfile shm_size: 4gb env_file: - .env @@ -148,8 +148,8 @@ services: dwh_db: image: adwh_dwh_postgis_db:15.3.3 build: - context: ./ - dockerfile: Dockerfiles/postgis.Dockerfile + context: ./docker/postgis + dockerfile: postgis.Dockerfile shm_size: 4gb env_file: - .env.dwh @@ -178,8 +178,8 @@ services: db_admin: image: adwh_pgadmin:7.7 build: - context: ./ - dockerfile: ./Dockerfiles/pgadmin4.Dockerfile + context: ./docker/pgadmin + dockerfile: pgadmin4.Dockerfile env_file: - .env restart: always @@ -195,8 +195,8 @@ services: dbt_proj: image: adwh_dbt:1.6.3 build: - context: ./ - dockerfile: ./Dockerfiles/dbt.Dockerfile + context: ./docker/dbt + dockerfile: dbt.Dockerfile env_file: - .env stdin_open: true diff --git a/Dockerfiles/airflow.Dockerfile b/docker/airflow/airflow.Dockerfile similarity index 86% rename from Dockerfiles/airflow.Dockerfile rename to docker/airflow/airflow.Dockerfile index 675f466..f8cab7a 100644 --- a/Dockerfiles/airflow.Dockerfile +++ b/docker/airflow/airflow.Dockerfile @@ -11,6 +11,6 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* USER airflow -COPY requirements/airflow_requirements.txt /requirements.txt +COPY requirements.txt /requirements.txt RUN pip install --upgrade pip RUN pip install -v --no-cache-dir -r /requirements.txt diff --git a/Dockerfiles/airflow_db.Dockerfile b/docker/airflow/airflow_db.Dockerfile similarity index 100% rename from Dockerfiles/airflow_db.Dockerfile rename to docker/airflow/airflow_db.Dockerfile diff --git a/requirements/airflow_requirements.txt b/docker/airflow/requirements.txt similarity index 100% rename from requirements/airflow_requirements.txt rename to docker/airflow/requirements.txt diff --git a/Dockerfiles/dbt.Dockerfile b/docker/dbt/dbt.Dockerfile similarity index 100% rename from Dockerfiles/dbt.Dockerfile rename to docker/dbt/dbt.Dockerfile diff --git a/Dockerfiles/pgadmin4.Dockerfile b/docker/pgadmin/pgadmin4.Dockerfile similarity index 100% rename from Dockerfiles/pgadmin4.Dockerfile rename to docker/pgadmin/pgadmin4.Dockerfile diff --git a/Dockerfiles/postgis.Dockerfile b/docker/postgis/postgis.Dockerfile similarity index 100% rename from Dockerfiles/postgis.Dockerfile rename to docker/postgis/postgis.Dockerfile diff --git a/Dockerfiles/redis.Dockerfile b/docker/redis/redis.Dockerfile similarity index 100% rename from Dockerfiles/redis.Dockerfile rename to docker/redis/redis.Dockerfile diff --git a/requirements/superset/docker-bootstrap.sh b/docker/superset/docker-bootstrap.sh similarity index 100% rename from requirements/superset/docker-bootstrap.sh rename to docker/superset/docker-bootstrap.sh diff --git a/requirements/superset/docker-init.sh b/docker/superset/docker-init.sh similarity index 100% rename from requirements/superset/docker-init.sh rename to docker/superset/docker-init.sh diff --git a/docker/superset/postgres.Dockerfile b/docker/superset/postgres.Dockerfile new file mode 100644 index 0000000..1e3636e --- /dev/null +++ b/docker/superset/postgres.Dockerfile @@ -0,0 +1 @@ +FROM postgres:14.5 \ No newline at end of file diff --git a/requirements/superset/pythonpath_dev/superset_config.py b/docker/superset/pythonpath_dev/superset_config.py similarity index 100% rename from requirements/superset/pythonpath_dev/superset_config.py rename to docker/superset/pythonpath_dev/superset_config.py diff --git a/requirements/superset/superset_requirements.txt b/docker/superset/requirements.txt similarity index 100% rename from requirements/superset/superset_requirements.txt rename to docker/superset/requirements.txt diff --git a/docker/superset/superset.Dockerfile b/docker/superset/superset.Dockerfile new file mode 100644 index 0000000..de73616 --- /dev/null +++ b/docker/superset/superset.Dockerfile @@ -0,0 +1,10 @@ +FROM apache/superset:2.1.0 + +COPY docker-init.sh /app/docker/docker-init.sh +COPY pythonpath_dev /app/docker/pythonpath_dev +COPY docker-bootstrap.sh /app/docker/docker-bootstrap.sh +COPY requirements.txt /app/docker/requirements.txt + +USER root +RUN pip install -v --no-cache -r /app/docker/requirements.txt +USER superset