Openai triton server

.pre-commit-config.yaml

@@ -53,7 +53,7 @@ repos:
   - id: codespell
     additional_dependencies: [tomli]
     args: ["--toml", "pyproject.toml"]
-    exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$)
+    exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$)||^Triton_Inference_Server_Python_API/examples/fastapi/api-spec/openai.yml$
 # More details about these pre-commit hooks here:
 # https://pre-commit.com/hooks.html
 - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -65,7 +65,7 @@ repos:
   - id: check-json
   - id: check-toml
   - id: check-yaml
-    exclude: ^Deployment/Kubernetes/[^/]+/chart/templates/.+$
+    exclude: ^Deployment/Kubernetes/[^/]+/chart/templates/.+$|^Triton_Inference_Server_Python_API/examples/fastapi/api-spec/openai.yml$
   - id: check-shebang-scripts-are-executable
   - id: end-of-file-fixer
     types_or: [c, c++, cuda, proto, textproto, java, python]

Triton_Inference_Server_Python_API/build.sh

@@ -30,18 +30,19 @@ RUN_PREFIX=
 BUILD_MODELS=
 
 # Frameworks
-declare -A FRAMEWORKS=(["DIFFUSION"]=1 ["TRT_LLM"]=2 ["IDENTITY"]=3)
+declare -A FRAMEWORKS=(["DIFFUSION"]=1 ["TRT_LLM"]=2 ["IDENTITY"]=3 ["VLLM"]=4)
 DEFAULT_FRAMEWORK=IDENTITY
 
 SOURCE_DIR=$(dirname "$(readlink -f "$0")")
 DOCKERFILE=${SOURCE_DIR}/docker/Dockerfile
 
 
 # Base Images
-BASE_IMAGE=nvcr.io/nvidia/tritonserver
-BASE_IMAGE_TAG_IDENTITY=24.01-py3
-BASE_IMAGE_TAG_DIFFUSION=24.01-py3
-BASE_IMAGE_TAG_TRT_LLM=24.01-trtllm-python-py3
+BASE_IMAGE_DEFAULT=nvcr.io/nvidia/tritonserver
+BASE_IMAGE_TAG_IDENTITY=24.06-py3
+BASE_IMAGE_TAG_DIFFUSION=24.06-py3
+BASE_IMAGE_TAG_TRT_LLM=24.06-trtllm-python-py3
+BASE_IMAGE_TAG_VLLM=24.06-vllm-python-py3
 
 get_options() {
     while :; do
@@ -61,7 +62,7 @@ get_options() {
 	--build-models)
 	    BUILD_MODELS=TRUE
             ;;
-        --base)
+        --base-image)
             if [ "$2" ]; then
                 BASE_IMAGE=$2
                 shift
@@ -135,10 +136,20 @@ get_options() {
 	    BASE_IMAGE_TAG=BASE_IMAGE_TAG_${FRAMEWORK}
 	    BASE_IMAGE_TAG=${!BASE_IMAGE_TAG}
 	fi
+
+	if [ -z $BASE_IMAGE ]; then
+	    BASE_IMAGE=BASE_IMAGE_${FRAMEWORK}
+	    BASE_IMAGE=${!BASE_IMAGE}
+	fi
+
+	if [ -z $BASE_IMAGE ]; then
+	    BASE_IMAGE=${BASE_IMAGE_DEFAULT}
+	fi
+
     fi
 
     if [ -z "$TAG" ]; then
-        TAG="triton-python-api:r24.01"
+        TAG="triton-python-api:r24.06"
 
 	if [[ $FRAMEWORK == "TRT_LLM" ]]; then
 	    TAG+="-trt-llm"
@@ -148,6 +159,10 @@ get_options() {
 	    TAG+="-diffusion"
 	fi
 
+	if [[ $FRAMEWORK == "VLLM" ]]; then
+	    TAG+="-vllm"
+	fi
+
     fi
 
 }
@@ -186,7 +201,7 @@ get_options "$@"
 
 if [[ $FRAMEWORK == DIFFUSION ]]; then
     BASE_IMAGE="tritonserver"
-    BASE_IMAGE_TAG="r24.01-diffusion"
+    BASE_IMAGE_TAG="r24.06-diffusion"
 fi
 
 # BUILD RUN TIME IMAGE
@@ -208,7 +223,7 @@ if [[ $FRAMEWORK == DIFFUSION ]]; then
 	set -x
     fi
     $RUN_PREFIX mkdir -p backend/diffusion
-    $RUN_PREFIX $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/build.sh --framework diffusion --tag tritonserver:r24.01-diffusion
+    $RUN_PREFIX $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/build.sh --framework diffusion --tag $BASE_IMAGE:$BASE_IMAGE_TAG $NO_CACHE
     $RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/backend/diffusion/model.py backend/diffusion/model.py
     $RUN_PREFIX mkdir -p diffusion-models/stable_diffusion_1_5/1
     $RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/diffusion-models/stable_diffusion_1_5/config.pbtxt  diffusion-models/stable_diffusion_1_5/config.pbtxt
@@ -231,25 +246,14 @@ $RUN_PREFIX docker build -f $DOCKERFILE $BUILD_OPTIONS $BUILD_ARGS -t $TAG $SOUR
 { set +x; } 2>/dev/null
 
 
-if [[ $FRAMEWORK == TRT_LLM ]]; then
-    if [ -z "$RUN_PREFIX" ]; then
-	set -x
-    fi
-
-    $RUN_PREFIX docker build -f $SOURCE_DIR/docker/Dockerfile.trt-llm-engine-builder  $BUILD_OPTIONS $BUILD_ARGS -t trt-llm-engine-builder  $SOURCE_DIR $NO_CACHE
-
-    { set +x; } 2>/dev/null
-
-fi;
-
 if [[ $FRAMEWORK == IDENTITY ]] || [[ $BUILD_MODELS == TRUE ]]; then
 
     if [[ $FRAMEWORK == DIFFUSION ]]; then
 	if [ -z "$RUN_PREFIX" ]; then
 	    set -x
 	fi
 
-	$RUN_PREFIX docker run --rm -it -v $PWD:/workspace $TAG /bin/bash -c "/workspace/scripts/stable_diffusion/build_models.sh --model stable_diffusion_1_5"
+	$RUN_PREFIX docker run --gpus all --rm -it -v $PWD:/workspace $TAG /bin/bash -c "/workspace/scripts/stable_diffusion/build_models.sh --model stable_diffusion_1_5"
 
 	{ set +x; } 2>/dev/null
     fi

Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt

@@ -0,0 +1,29 @@
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+fastapi==0.111.0
+openai==1.26.0
+pydantic==2.7.1

Triton_Inference_Server_Python_API/deps/requirements_vllm.txt

@@ -0,0 +1,28 @@
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+openai==1.26.0
+vllm[all]==0.4.1

Triton_Inference_Server_Python_API/docker/Dockerfile

@@ -25,37 +25,58 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver
-ARG BASE_IMAGE_TAG=24.01-py3
+ARG BASE_IMAGE_TAG=24.06-py3
+ARG FRAMEWORK=DIFFUSION
 
 FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} as triton-python-api
 
 RUN apt-get update; apt-get install -y gdb
 
 COPY ./deps/requirements.txt /tmp/requirements.txt
 
+COPY ./deps/requirements_trt_llm.txt /tmp/requirements_trt_llm.txt
+
+COPY ./deps/requirements_vllm.txt /tmp/requirements_vllm.txt
+
+COPY ./deps/requirements_python_repl.txt /tmp/requirements_python_repl.txt
+
 RUN pip install --timeout=2000 -r /tmp/requirements.txt
 
 # Finish pyright install
 
 RUN pyright --help
 
-COPY ./deps/tritonserver-2.41.0.dev0-py3-none-any.whl /tmp/tritonserver-2.41.0.dev0-py3-none-any.whl
-
 RUN find /opt/tritonserver/python -maxdepth 1 -type f -name \
     "tritonserver-*.whl" | xargs -I {} pip3 install --force-reinstall --upgrade {}[all]
 
-RUN pip3 show tritonserver 1>/dev/null || \
-    if [ $? != 0 ]; then \
-       pip3 install /tmp/tritonserver-2.41.0.dev0-py3-none-any.whl[all] ;\
-    fi
+ARG TRITON_CLI_TAG="0.0.8"
+
+RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${TRITON_CLI_TAG}
+
+ARG GENAI_PERF_TAG="r24.05"
+
+RUN pip install "git+https://github.com/triton-inference-server/client.git@${GENAI_PERF_TAG}#subdirectory=src/c++/perf_analyzer/genai-perf"
+
+ARG INCLUDE_PYTHON_REPL
+
+RUN if [[ "$INCLUDE_PYTHON_REPL" != "" ]] ; then pip install --timeout=2000 -r /tmp/requirements_python_repl.txt ; fi
+
+ARG FRAMEWORK=DIFFUSION
+
+RUN if [[ "$FRAMEWORK" == "TRT_LLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_trt_llm.txt ; fi
+
+RUN if [[ "$FRAMEWORK" == "VLLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_vllm.txt ; fi
 
 RUN ln -sf /bin/bash /bin/sh
 
 COPY . /workspace
 
 ARG RUN_TESTS=FALSE
 
-RUN if [[ "$RUN_TESTS" == "TRUE" ]] ; then cd /tmp && git clone -b r23.12-python-api https://github.com/triton-inference-server/core.git && cp -rf /tmp/core/python/test /workspace/deps/ ; fi
+RUN if [[ "$RUN_TESTS" == "TRUE" ]] ; then cd /tmp && git clone -b r24.04 https://github.com/triton-inference-server/core.git && cp -rf /tmp/core/python/test /workspace/deps/ ; fi
 
 RUN if [[ "$RUN_TESTS" == "TRUE" ]] ; then pytest /workspace/deps ; fi
 
+ARG INCLUDE_EMACS
+
+RUN if [[ "$INCLUDE_EMACS" == "TRUE" ]] ; then apt-get install -y emacs ; fi

Triton_Inference_Server_Python_API/docker/Dockerfile.dockerignore

@@ -10,4 +10,10 @@
 **/*onnx*
 **/*engine*
 **/*pytorch_model*
-**/*.pth*
+**/*.pth*
+**/*.pt
+**/*.models/*
+**/*.model-store/*
+**/*.model.*/*
+**/*.cache/*
+**/*.libtorch_model_store/*