Fix modular operator issues

- Updated shebang in shell scripts - Updated operator dockerfiles - Updated requirements
tattle-made · Feb 7, 2024 · d850c05 · d850c05
1 parent b4c4eca
commit d850c05
Show file tree

Hide file tree

Showing 12 changed files with 130 additions and 43 deletions.
diff --git a/src/api/Dockerfile.image_vec_rep_resnet b/src/api/Dockerfile.image_vec_rep_resnet
@@ -9,11 +9,21 @@ RUN apt-get update \
     libgl1-mesa-glx libglib2.0-0 \
     && rm -rf /var/lib/apt/lists/* 
 ENV PATH=/root/.local/bin:$PATH
+
 RUN pip install --no-cache-dir --upgrade pip
 WORKDIR /app
-COPY ./core/operators/image_vec_rep_resnet_requirements.txt /app/requirements.txt
-RUN pip install --no-cache-dir --user -r requirements.txt
-COPY . /app
+COPY ./core/operators/image_vec_rep_resnet_requirements.txt /app/core/operators/image_vec_rep_resnet_requirements.txt
+RUN pip install --no-cache-dir --user -r /app/core/operators/image_vec_rep_resnet_requirements.txt
+COPY ./core/operators/image_vec_rep_resnet.py /app/core/operators/image_vec_rep_resnet.py
+
+COPY ./core/operators/sample_data/text.png /app/core/operators/sample_data/text.png
+COPY ./core/operators/test_image_vec_rep_resnet.py /app/core/operators/test_image_vec_rep_resnet.py
+COPY ./core/operators/test_image_vec_rep_resnet.py /app/core/operators/test_image_vec_rep_resnet.py
+
+COPY ./image_vec_operator_profile.py /app/image_vec_operator_profile.py
+COPY ./image_vec_operator_profile_memray.sh /app/image_vec_operator_profile_memray.sh
+COPY ./image_vec_operator_profile_pyinstrument.sh /app/image_vec_operator_profile_pyinstrument.sh
 RUN chmod +x image_vec_operator_profile_memray.sh
 RUN chmod +x image_vec_operator_profile_pyinstrument.sh
+
 CMD tail -f /dev/null
diff --git a/src/api/Dockerfile.vid_vec_rep_resnet b/src/api/Dockerfile.vid_vec_rep_resnet
@@ -9,13 +9,28 @@ RUN apt-get update \
     libgl1-mesa-glx libglib2.0-0 \
     && rm -rf /var/lib/apt/lists/* 
 ENV PATH=/root/.local/bin:$PATH
+
 RUN pip install --no-cache-dir --upgrade pip
 WORKDIR /app
-COPY ./core/operators/vid_vec_rep_resnet_requirements.txt /app/requirements.txt
-RUN pip install --no-cache-dir --user -r requirements.txt
-COPY . /app
+COPY ./core/operators/vid_vec_rep_resnet_requirements.txt /app/core/operators/vid_vec_rep_resnet_requirements.txt
+RUN pip install --no-cache-dir --user -r /app/core/operators/vid_vec_rep_resnet_requirements.txt
+COPY ./core/operators/vid_vec_rep_resnet.py /app/core/operators/vid_vec_rep_resnet.py
+
+COPY ./core/operators/sample_data/sample-cat-video.mp4 /app/core/operators/sample_data/sample-cat-video.mp4
+COPY ./core/operators/test_vid_vec_rep_resnet.py /app/core/operators/test_vid_vec_rep_resnet.py
+
+COPY ./video_vec_operator_cprofile.py /app/video_vec_operator_cprofile.py
+COPY ./video_vec_operator_cprofile.sh /app/video_vec_operator_cprofile.sh
 RUN chmod +x video_vec_operator_cprofile.sh
+
+COPY ./video_vec_operator_time.py /app/video_vec_operator_time.py
+COPY ./video_vec_operator_time.sh /app/video_vec_operator_time.sh
 RUN chmod +x video_vec_operator_time.sh
+
+COPY ./video_vec_operator_profile.py /app/video_vec_operator_profile.py
+COPY ./video_vec_operator_profile_memray.sh /app/video_vec_operator_profile_memray.sh
+COPY ./video_vec_operator_profile_pyinstrument.sh /app/video_vec_operator_profile_pyinstrument.sh
 RUN chmod +x video_vec_operator_profile_memray.sh
 RUN chmod +x video_vec_operator_profile_pyinstrument.sh
+
 CMD tail -f /dev/null
diff --git a/src/api/core/operators/image_vec_rep_resnet_requirements.in b/src/api/core/operators/image_vec_rep_resnet_requirements.in
@@ -1,4 +1,6 @@
 torch==2.1.2+cpu
 torchvision==0.16.2+cpu
 numpy==1.26.3
-Pillow==10.2.0  # dev
+Pillow==10.2.0  # dev
+memray==1.11.0
+pyinstrument==4.6.2
diff --git a/src/api/core/operators/image_vec_rep_resnet_requirements.txt b/src/api/core/operators/image_vec_rep_resnet_requirements.txt
@@ -17,9 +17,24 @@ fsspec==2024.2.0
 idna==3.6
     # via requests
 jinja2==3.1.3
-    # via torch
+    # via
+    #   memray
+    #   torch
+linkify-it-py==2.0.3
+    # via markdown-it-py
+markdown-it-py[linkify,plugins]==3.0.0
+    # via
+    #   mdit-py-plugins
+    #   rich
+    #   textual
 markupsafe==2.1.5
     # via jinja2
+mdit-py-plugins==0.4.0
+    # via markdown-it-py
+mdurl==0.1.2
+    # via markdown-it-py
+memray==1.11.0
+    # via -r image_vec_rep_resnet_requirements.in
 mpmath==1.3.0
     # via sympy
 networkx==3.2.1
@@ -32,17 +47,31 @@ pillow==10.2.0
     # via
     #   -r image_vec_rep_resnet_requirements.in
     #   torchvision
+pygments==2.17.2
+    # via rich
+pyinstrument==4.6.2
+    # via -r image_vec_rep_resnet_requirements.in
 requests==2.31.0
     # via torchvision
+rich==13.7.0
+    # via
+    #   memray
+    #   textual
 sympy==1.12
     # via torch
+textual==0.48.2
+    # via memray
 torch==2.1.2+cpu
     # via
     #   -r image_vec_rep_resnet_requirements.in
     #   torchvision
 torchvision==0.16.2+cpu
     # via -r image_vec_rep_resnet_requirements.in
 typing-extensions==4.9.0
-    # via torch
+    # via
+    #   textual
+    #   torch
+uc-micro-py==1.0.2
+    # via linkify-it-py
 urllib3==2.2.0
     # via requests
diff --git a/src/api/core/operators/vid_vec_rep_resnet_requirements.in b/src/api/core/operators/vid_vec_rep_resnet_requirements.in
@@ -3,4 +3,6 @@ torchvision==0.16.2+cpu
 numpy==1.26.3
 Pillow==10.2.0
 scipy==1.11.4
-opencv-python-headless==4.9.0.80
+opencv-python-headless==4.9.0.80
+memray==1.11.0
+pyinstrument==4.6.2
diff --git a/src/api/core/operators/vid_vec_rep_resnet_requirements.txt b/src/api/core/operators/vid_vec_rep_resnet_requirements.txt
@@ -6,20 +6,35 @@
 #
 --find-links https://download.pytorch.org/whl/torch_stable.html
 
-certifi==2023.11.17
+certifi==2024.2.2
     # via requests
 charset-normalizer==3.3.2
     # via requests
 filelock==3.13.1
     # via torch
-fsspec==2023.12.2
+fsspec==2024.2.0
     # via torch
 idna==3.6
     # via requests
 jinja2==3.1.3
-    # via torch
-markupsafe==2.1.4
+    # via
+    #   memray
+    #   torch
+linkify-it-py==2.0.3
+    # via markdown-it-py
+markdown-it-py[linkify,plugins]==3.0.0
+    # via
+    #   mdit-py-plugins
+    #   rich
+    #   textual
+markupsafe==2.1.5
     # via jinja2
+mdit-py-plugins==0.4.0
+    # via markdown-it-py
+mdurl==0.1.2
+    # via markdown-it-py
+memray==1.11.0
+    # via -r vid_vec_rep_resnet_requirements.in
 mpmath==1.3.0
     # via sympy
 networkx==3.2.1
@@ -36,19 +51,33 @@ pillow==10.2.0
     # via
     #   -r vid_vec_rep_resnet_requirements.in
     #   torchvision
+pygments==2.17.2
+    # via rich
+pyinstrument==4.6.2
+    # via -r vid_vec_rep_resnet_requirements.in
 requests==2.31.0
     # via torchvision
+rich==13.7.0
+    # via
+    #   memray
+    #   textual
 scipy==1.11.4
     # via -r vid_vec_rep_resnet_requirements.in
 sympy==1.12
     # via torch
+textual==0.48.2
+    # via memray
 torch==2.1.2+cpu
     # via
     #   -r vid_vec_rep_resnet_requirements.in
     #   torchvision
 torchvision==0.16.2+cpu
     # via -r vid_vec_rep_resnet_requirements.in
 typing-extensions==4.9.0
-    # via torch
+    # via
+    #   textual
+    #   torch
+uc-micro-py==1.0.2
+    # via linkify-it-py
 urllib3==2.0.7
     # via requests
diff --git a/src/api/image_vec_operator_profile_memray.sh b/src/api/image_vec_operator_profile_memray.sh
@@ -1,4 +1,5 @@
-#! /bin/bash
+#!/usr/bin/env bash
+
 current_time=$(date "+%Y.%m.%d-%H.%M.%S")
 python -m memray run -o image_vec_rep_resnet.$current_time.bin image_vec_operator_profile.py
 tail -f /dev/null
diff --git a/src/api/image_vec_operator_profile_pyinstrument.sh b/src/api/image_vec_operator_profile_pyinstrument.sh
@@ -1,4 +1,5 @@
-#! /bin/bash
+#!/usr/bin/env bash
+
 current_time=$(date "+%Y.%m.%d-%H.%M.%S")
 pyinstrument -r speedscope -o speedscope_image_vec_rep_resnet.$current_time.json image_vec_operator_profile.py
 tail -f /dev/null
diff --git a/src/api/requirements.txt b/src/api/requirements.txt
@@ -8,13 +8,13 @@ blinker==1.7.0
     # via flask
 boto3==1.34.34
     # via -r requirements.in
-botocore==1.34.35
+botocore==1.34.36
     # via
     #   boto3
     #   s3transfer
-cachetools==4.2.1
+cachetools==5.3.2
     # via google-auth
-certifi==2023.11.17
+certifi==2024.2.2
     # via
     #   elastic-transport
     #   requests
@@ -26,7 +26,7 @@ click==8.1.7
     #   flask
 dacite==1.8.1
     # via -r requirements.in
-elastic-transport==8.11.0
+elastic-transport==8.12.0
     # via elasticsearch
 elasticsearch==8.11.1
     # via -r requirements.in
@@ -36,11 +36,11 @@ flask==2.3.2
     #   flask-cors
 flask-cors==3.0.9
     # via -r requirements.in
-google-api-core[grpc]==1.34.0
+google-api-core[grpc]==2.16.2
     # via
     #   google-api-core
     #   google-cloud-vision
-google-auth==1.28.0
+google-auth==2.27.0
     # via google-api-core
 google-cloud==0.34.0
     # via -r requirements.in
@@ -50,11 +50,11 @@ googleapis-common-protos==1.62.0
     # via
     #   google-api-core
     #   grpcio-status
-grpcio==1.60.0
+grpcio==1.60.1
     # via
     #   google-api-core
     #   grpcio-status
-grpcio-status==1.36.1
+grpcio-status==1.60.1
     # via google-api-core
 idna==3.6
     # via requests
@@ -66,7 +66,7 @@ jinja2==3.1.3
     # via
     #   flask
     #   memray
-jmespath==0.10.0
+jmespath==1.0.1
     # via
     #   boto3
     #   botocore
@@ -77,7 +77,7 @@ markdown-it-py[linkify,plugins]==3.0.0
     #   mdit-py-plugins
     #   rich
     #   textual
-markupsafe==2.1.4
+markupsafe==2.1.5
     # via
     #   jinja2
     #   werkzeug
@@ -95,30 +95,30 @@ pika==1.3.2
     # via -r requirements.in
 pillow==10.2.0
     # via -r requirements.in
-pluggy==1.3.0
+pluggy==1.4.0
     # via pytest
 proto-plus==1.23.0
     # via google-cloud-vision
-protobuf==3.20.3
+protobuf==4.25.2
     # via
     #   google-api-core
     #   google-cloud-vision
     #   googleapis-common-protos
     #   grpcio-status
     #   proto-plus
-pyasn1==0.4.8
+pyasn1==0.5.1
     # via
     #   pyasn1-modules
     #   rsa
-pyasn1-modules==0.2.8
+pyasn1-modules==0.3.0
     # via google-auth
 pygments==2.17.2
     # via rich
 pyinstrument==4.6.2
     # via -r requirements.in
 pytest==7.4.4
     # via -r requirements.in
-python-dateutil==2.8.1
+python-dateutil==2.8.2
     # via botocore
 python-dotenv==1.0.0
     # via -r requirements.in
@@ -130,14 +130,13 @@ rich==13.7.0
     # via
     #   memray
     #   textual
-rsa==4.7.2
+rsa==4.9
     # via google-auth
 s3transfer==0.10.0
     # via boto3
-six==1.15.0
+six==1.16.0
     # via
     #   flask-cors
-    #   google-auth
     #   python-dateutil
 textual==0.48.2
     # via memray
@@ -154,6 +153,3 @@ werkzeug==3.0.1
     # via flask
 wget==3.2
     # via -r requirements.in
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
diff --git a/src/api/video_vec_operator_profile.py b/src/api/video_vec_operator_profile.py
@@ -1,10 +1,10 @@
-from core.operators import vid_vec_rep_resnet
+import core.operators.vid_vec_rep_resnet
 
 
 def profile_code():
     file_path = {"path": r"core/operators/sample_data/sample-cat-video.mp4"}
-    vid_vec_rep_resnet.initialize(param=None)
-    vid_vec_rep_resnet.run(file_path)
+    core.operators.vid_vec_rep_resnet.initialize(param=None)
+    core.operators.vid_vec_rep_resnet.run(file_path)
     print("Video vec profiler complete!")
 
 

diff --git a/src/api/video_vec_operator_profile_memray.sh b/src/api/video_vec_operator_profile_memray.sh
@@ -1,4 +1,5 @@
-#! /bin/bash
+#!/usr/bin/env bash
+
 current_time=$(date "+%Y.%m.%d-%H.%M.%S")
 python -m memray run -o vid_vec_rep_resnet.$current_time.bin video_vec_operator_profile.py
 tail -f /dev/null
diff --git a/src/api/video_vec_operator_profile_pyinstrument.sh b/src/api/video_vec_operator_profile_pyinstrument.sh
@@ -1,4 +1,5 @@
-#! /bin/bash
+#!/usr/bin/env bash
+
 current_time=$(date "+%Y.%m.%d-%H.%M.%S")
 pyinstrument -r speedscope -o speedscope_vid_vec_rep_resnet.$current_time.json video_vec_operator_profile.py
 tail -f /dev/null