[RELEASE] MRC v23.11

Merge pull request #414 from nv-morpheus/branch-23.11

.github/copy-pr-bot.yaml

@@ -0,0 +1,4 @@
+# Configuration file for `copy-pr-bot` GitHub App
+# https://docs.gha-runners.nvidia.com/apps/copy-pr-bot/
+
+enabled: true

.github/ops-bot.yaml

@@ -5,5 +5,3 @@ auto_merger: true
 branch_checker: true
 label_checker: true
 release_drafter: true
-copy_prs: true
-rerun_tests: true

.github/workflows/ci_pipe.yml

@@ -294,7 +294,7 @@ jobs:
         run: ./mrc/ci/scripts/github/benchmark.sh
       - name: post_benchmark
         shell: bash
-        run: ./mrc/ci/scripts/github/benchmark.sh
+        run: ./mrc/ci/scripts/github/post_benchmark.sh
 
 
   package:

.github/workflows/pull_request.yml

@@ -47,7 +47,7 @@ jobs:
     name: Prepare
     runs-on: ubuntu-latest
     container:
-      image: rapidsai/ci:latest
+      image: rapidsai/ci-conda:latest
     steps:
       - name: Get PR Info
         id: get-pr-info
@@ -71,9 +71,9 @@ jobs:
       # Update conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
       conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }}
       # Build container
-      container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-230711
+      container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-230920
       # Test container
-      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-230711
+      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-230920
       # Info about the PR. Empty for non PR branches. Useful for extracting PR number, title, etc.
       pr_info: ${{ needs.prepare.outputs.pr_info }}
     secrets:

.gitignore

@@ -1,4 +1,5 @@
 /build*/
+.tmp
 *.engine
 .Dockerfile
 .gitignore
@@ -17,6 +18,9 @@ include/mrc/version.hpp
 .vscode/settings.json
 .vscode/tasks.json
 
+# Ignore user-defined clangd settings
+.clangd
+
 # Created by https://www.gitignore.io/api/vim,c++,cmake,python,synology
 
 ### C++ ###

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "morpheus_utils"]
 	path = external/utilities
 	url = https://github.com/nv-morpheus/utilities.git
-	branch = branch-23.07
+	branch = branch-23.11

CHANGELOG.md

@@ -1,3 +1,37 @@
+# MRC 23.11.00 (30 Nov 2023)
+
+## 🐛 Bug Fixes
+
+- Use a traditional semaphore in AsyncioRunnable ([#412](https://github.com/nv-morpheus/MRC/pull/412)) [@cwharris](https://github.com/cwharris)
+- Fix libhwloc & stubgen versions to match dev yaml ([#405](https://github.com/nv-morpheus/MRC/pull/405)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Update boost versions to match version used in dev env ([#404](https://github.com/nv-morpheus/MRC/pull/404)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Fix EdgeHolder from incorrectly reporting an active connection ([#402](https://github.com/nv-morpheus/MRC/pull/402)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Safe handling of control plane promises & fix CI ([#391](https://github.com/nv-morpheus/MRC/pull/391)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Revert boost upgrade, and update clang to v16 ([#382](https://github.com/nv-morpheus/MRC/pull/382)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Fixing an issue with `update-versions.sh` which always blocked CI ([#377](https://github.com/nv-morpheus/MRC/pull/377)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Add test for  gc being invoked in a thread finalizer ([#365](https://github.com/nv-morpheus/MRC/pull/365)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Adopt patched pybind11 ([#364](https://github.com/nv-morpheus/MRC/pull/364)) [@dagardner-nv](https://github.com/dagardner-nv)
+
+## 📖 Documentation
+
+- Add missing flags to docker command to mount the working dir and set -cap-add=sys_nice ([#383](https://github.com/nv-morpheus/MRC/pull/383)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Make Quick Start Guide not use `make_node_full` ([#376](https://github.com/nv-morpheus/MRC/pull/376)) [@cwharris](https://github.com/cwharris)
+
+## 🚀 New Features
+
+- Add AsyncioRunnable ([#411](https://github.com/nv-morpheus/MRC/pull/411)) [@cwharris](https://github.com/cwharris)
+- Adding more coroutine components to support async generators and task containers ([#408](https://github.com/nv-morpheus/MRC/pull/408)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Update ObservableProxy::pipe to support any number of operators ([#387](https://github.com/nv-morpheus/MRC/pull/387)) [@cwharris](https://github.com/cwharris)
+- Updates for MRC/Morpheus to build in the same RAPIDS devcontainer environment ([#375](https://github.com/nv-morpheus/MRC/pull/375)) [@cwharris](https://github.com/cwharris)
+
+## 🛠️ Improvements
+
+- Move Pycoro from Morpheus to MRC ([#409](https://github.com/nv-morpheus/MRC/pull/409)) [@cwharris](https://github.com/cwharris)
+- update rapidsai/ci to rapidsai/ci-conda ([#396](https://github.com/nv-morpheus/MRC/pull/396)) [@AyodeAwe](https://github.com/AyodeAwe)
+- Add local CI scripts & rebase docker image ([#394](https://github.com/nv-morpheus/MRC/pull/394)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Use `copy-pr-bot` ([#369](https://github.com/nv-morpheus/MRC/pull/369)) [@ajschmidt8](https://github.com/ajschmidt8)
+- Update Versions for v23.11.00 ([#357](https://github.com/nv-morpheus/MRC/pull/357)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+
 # MRC 23.07.00 (19 Jul 2023)
 
 ## 🚨 Breaking Changes

CMakeLists.txt

@@ -30,6 +30,7 @@ option(MRC_BUILD_PYTHON "Enable building the python bindings for MRC" ON)
 option(MRC_BUILD_TESTS "Whether or not to build MRC tests" ON)
 option(MRC_ENABLE_CODECOV "Enable gcov code coverage" OFF)
 option(MRC_ENABLE_DEBUG_INFO "Enable printing debug information" OFF)
+option(MRC_PYTHON_INPLACE_BUILD "Whether or not to copy built python modules back to the source tree for debug purposes." OFF)
 option(MRC_USE_CCACHE "Enable caching compilation results with ccache" OFF)
 option(MRC_USE_CLANG_TIDY "Enable running clang-tidy as part of the build process" OFF)
 option(MRC_USE_CONDA "Enables finding dependencies via conda. All dependencies must be installed first in the conda
@@ -78,7 +79,7 @@ morpheus_utils_initialize_package_manager(
 morpheus_utils_initialize_cuda_arch(mrc)
 
 project(mrc
-  VERSION 23.07.00
+  VERSION 23.11.00
   LANGUAGES C CXX
 )
 

Dockerfile

@@ -16,7 +16,7 @@
 # limitations under the License.
 
 
-ARG FROM_IMAGE="rapidsai/ci"
+ARG FROM_IMAGE="rapidsai/ci-conda"
 ARG CUDA_VER=11.8.0
 ARG LINUX_DISTRO=ubuntu
 ARG LINUX_VER=20.04

README.md

@@ -151,13 +151,17 @@ pytest $MRC_ROOT/python
 ### Docker Installation
 A Dockerfile is provided at `$MRC_ROOT` and can be built with
 ```bash
-docker build -t mrc:latest .
+DOCKER_BUILDKIT=1 docker build -t mrc:latest .
 ```
 To run the container
 ```bash
-docker run --gpus all --rm -it mrc:latest /bin/bash
+docker run --gpus all --cap-add=sys_nice -v $PWD:/work --rm -it mrc:latest /bin/bash
 ```
 
+> **Note:**
+> Users wishing to debug MRC in a Docker container should add the following to the `docker run` command:
+> `--cap-add=SYS_PTRACE`
+
 ## Quickstart Guide
 
 To quickly learn about both the C++ and Python MRC APIs, including following along with various complexity examples, we recommend following the MRC Quickstart Repository located [here](/docs/quickstart/README.md). This tutorial walks new users through topics like

ci/conda/environments/clang_env.yml

@@ -19,11 +19,11 @@ name: mrc
 channels:
     - conda-forge
 dependencies:
-    - clang=15
-    - clang-tools=15
-    - clangdev=15
-    - clangxx=15
-    - libclang=15
-    - libclang-cpp=15
-    - llvmdev=15
-    - include-what-you-use=0.19
+    - clang=16
+    - clang-tools=16
+    - clangdev=16
+    - clangxx=16
+    - libclang=16
+    - libclang-cpp=16
+    - llvmdev=16
+    - include-what-you-use=0.20

ci/conda/environments/dev_env.yml

@@ -25,9 +25,9 @@ dependencies:
   - autoconf>=2.69
   - bash-completion
   - benchmark=1.6.0
-  - boost-cpp=1.74
+  - boost-cpp=1.82
   - ccache
-  - cmake=3.24
+  - cmake=3.25
   - cuda-toolkit # Version comes from the channel above
   - cxx-compiler # Sets up the distro versions of our compilers
   - doxygen=1.9.2
@@ -46,7 +46,7 @@ dependencies:
   - isort
   - jinja2=3.0
   - lcov=1.15
-  - libhwloc=2.5
+  - libhwloc=2.9.2
   - libprotobuf=3.21
   - librmm=23.06
   - libtool
@@ -59,6 +59,7 @@ dependencies:
   - pybind11-stubgen=0.10
   - pytest
   - pytest-timeout
+  - pytest-asyncio
   - python=3.10
   - scikit-build>=0.17
   - sysroot_linux-64=2.17

ci/conda/recipes/libmrc/conda_build_config.yaml

@@ -71,9 +71,9 @@ zip_keys:
 # The following mimic what is available in the pinning feedstock:
 # https://github.com/conda-forge/conda-forge-pinning-feedstock/blob/main/recipe/conda_build_config.yaml
 boost:
-  - 1.74.0
+  - 1.82
 boost_cpp:
-  - 1.74.0
+  - 1.82
 gflags:
   - 2.2
 glog:

ci/conda/recipes/libmrc/meta.yaml

@@ -62,12 +62,12 @@ requirements:
     - libgrpc
     - gtest 1.13.*
     - libabseil
-    - libhwloc 2.5.*
+    - libhwloc 2.9.2
     - libprotobuf
     - librmm {{ rapids_version }}
     - nlohmann_json 3.9.1
     - pybind11-abi # See: https://conda-forge.org/docs/maintainer/knowledge_base.html#pybind11-abi-constraints
-    - pybind11-stubgen 0.10.5
+    - pybind11-stubgen 0.10
     - python {{ python }}
     - scikit-build >=0.17
     - ucx
@@ -98,7 +98,7 @@ outputs:
         - glog
         - libgrpc
         - libabseil # Needed for transitive run_exports from libgrpc. Does not need a version
-        - libhwloc 2.5.*
+        - libhwloc 2.9.2
         - libprotobuf # Needed for transitive run_exports from libgrpc. Does not need a version
         - librmm {{ rapids_version }}
         - nlohmann_json 3.9.*

ci/release/pr_code_freeze_template.md

@@ -0,0 +1,11 @@
+## :snowflake: Code freeze for `branch-${VERSION}` and `v${VERSION}` release
+
+### What does this mean?
+Only critical/hotfix level issues should be merged into `branch-${VERSION}` until release (merging of this PR).
+
+All other development PRs should be retargeted towards the next release branch: `branch-${NEXT_VERSION}`.
+
+### What is the purpose of this PR?
+- Update documentation
+- Allow testing for the new release
+- Enable a means to merge `branch-${VERSION}` into `main` for the release

ci/release/update-version.sh

@@ -60,6 +60,10 @@ function sed_runner() {
 
 # .gitmodules
 git submodule set-branch -b branch-${NEXT_SHORT_TAG} morpheus_utils
+if [[ "$(git diff --name-only | grep .gitmodules)" != "" ]]; then
+   # Only update the submodules if setting the branch changed .gitmodules
+   git submodule update --remote
+fi
 
 # Root CMakeLists.txt
 sed_runner 's/'"VERSION ${CURRENT_FULL_VERSION}.*"'/'"VERSION ${NEXT_FULL_VERSION}"'/g' CMakeLists.txt

ci/scripts/bootstrap_local_ci.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export WORKSPACE_TMP="$(pwd)/.tmp/local_ci_workspace"
+mkdir -p ${WORKSPACE_TMP}
+git clone ${GIT_URL} mrc
+cd mrc/
+git checkout ${GIT_BRANCH}
+git pull
+git checkout ${GIT_COMMIT}
+
+export MRC_ROOT=$(pwd)
+export WORKSPACE=${MRC_ROOT}
+export LOCAL_CI=1
+GH_SCRIPT_DIR="${MRC_ROOT}/ci/scripts/github"
+
+unset CMAKE_CUDA_COMPILER_LAUNCHER
+unset CMAKE_CXX_COMPILER_LAUNCHER
+unset CMAKE_C_COMPILER_LAUNCHER
+
+if [[ "${STAGE}" != "bash" ]]; then
+    # benchmark & codecov are composite stages, the rest are composed of a single shell script
+    if [[ "${STAGE}" == "benchmark" || "${STAGE}" == "codecov" ]]; then
+        CI_SCRIPT="${WORKSPACE_TMP}/ci_script.sh"
+        echo "#!/bin/bash" > ${CI_SCRIPT}
+        if [[ "${STAGE}" == "benchmark" ]]; then
+            echo "${GH_SCRIPT_DIR}/pre_benchmark.sh" >> ${CI_SCRIPT}
+            echo "${GH_SCRIPT_DIR}/benchmark.sh" >> ${CI_SCRIPT}
+            echo "${GH_SCRIPT_DIR}/post_benchmark.sh" >> ${CI_SCRIPT}
+        else
+            echo "${GH_SCRIPT_DIR}/build.sh" >> ${CI_SCRIPT}
+            echo "${GH_SCRIPT_DIR}/test_codecov.sh" >> ${CI_SCRIPT}
+        fi
+
+        chmod +x ${CI_SCRIPT}
+    else
+        if [[ "${STAGE}" =~ "build" ]]; then
+            CI_SCRIPT="${GH_SCRIPT_DIR}/build.sh"
+        elif [[ "${STAGE}" =~ "test" ]]; then
+            CI_SCRIPT="${GH_SCRIPT_DIR}/test.sh"
+        else
+            CI_SCRIPT="${GH_SCRIPT_DIR}/${STAGE}.sh"
+        fi
+    fi
+
+    ${CI_SCRIPT}
+fi

ci/scripts/cpp_checks.sh

@@ -80,9 +80,22 @@ if [[ -n "${MRC_MODIFIED_FILES}" ]]; then
 
    # Include What You Use
    if [[ "${SKIP_IWYU}" == "" ]]; then
-      IWYU_DIRS="cpp python"
+      # Remove .h, .hpp, and .cu files from the modified list
+      shopt -s extglob
+      IWYU_MODIFIED_FILES=( "${MRC_MODIFIED_FILES[@]/*.@(h|hpp|cu)/}" )
+
+      # Get the list of compiled files relative to this directory
+      WORKING_PREFIX="${PWD}/"
+      COMPILED_FILES=( $(jq -r .[].file ${BUILD_DIR}/compile_commands.json | sort -u ) )
+      COMPILED_FILES=( "${COMPILED_FILES[@]/#$WORKING_PREFIX/}" )
+      COMBINED_FILES=("${COMPILED_FILES[@]}")
+      COMBINED_FILES+=("${IWYU_MODIFIED_FILES[@]}")
+
+      # Find the intersection between compiled files and modified files
+      IWYU_MODIFIED_FILES=( $(printf '%s\0' "${COMBINED_FILES[@]}" | sort -z | uniq -d -z | xargs -0n1) )
+
       NUM_PROC=$(get_num_proc)
-      IWYU_OUTPUT=`${IWYU_TOOL} -p ${BUILD_DIR} -j ${NUM_PROC} ${IWYU_DIRS} 2>&1`
+      IWYU_OUTPUT=`${IWYU_TOOL} -p ${BUILD_DIR} -j ${NUM_PROC} ${IWYU_MODIFIED_FILES[@]} 2>&1`
       IWYU_RETVAL=$?
    fi
 else

ci/scripts/github/build.sh

@@ -20,7 +20,12 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 
 update_conda_env
 
-CMAKE_CACHE_FLAGS="-DCCACHE_PROGRAM_PATH=$(which sccache) -DMRC_USE_CCACHE=ON"
+if [[ "${LOCAL_CI}" == "" ]]; then
+    CMAKE_CACHE_FLAGS="-DCCACHE_PROGRAM_PATH=$(which sccache) -DMRC_USE_CCACHE=ON"
+else
+    CMAKE_CACHE_FLAGS=""
+fi
+
 
 rapids-logger "Check versions"
 python3 --version
@@ -56,18 +61,20 @@ cmake -B build -G Ninja ${CMAKE_FLAGS} .
 rapids-logger "Building MRC"
 cmake --build build --parallel ${PARALLEL_LEVEL}
 
-rapids-logger "sccache usage for MRC build:"
-sccache --show-stats
+if [[ "${LOCAL_CI}" == "" ]]; then
+    rapids-logger "sccache usage for MRC build:"
+    sccache --show-stats
+fi
 
-if [[ "${BUILD_CC}" != "gcc-coverage" ]]; then
+if [[ "${BUILD_CC}" != "gcc-coverage" || ${LOCAL_CI} == "1" ]]; then
     rapids-logger "Archiving results"
     tar cfj "${WORKSPACE_TMP}/dot_cache.tar.bz" .cache
     tar cfj "${WORKSPACE_TMP}/build.tar.bz" build
     ls -lh ${WORKSPACE_TMP}/
 
     rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/"
-    aws s3 cp --no-progress "${WORKSPACE_TMP}/build.tar.bz" "${ARTIFACT_URL}/build.tar.bz"
-    aws s3 cp --no-progress "${WORKSPACE_TMP}/dot_cache.tar.bz" "${ARTIFACT_URL}/dot_cache.tar.bz"
+    upload_artifact "${WORKSPACE_TMP}/build.tar.bz"
+    upload_artifact "${WORKSPACE_TMP}/dot_cache.tar.bz"
 fi
 
 rapids-logger "Success"

ci/scripts/github/checks.sh

@@ -24,7 +24,8 @@ update_conda_env
 
 rapids-logger "Configuring CMake"
 git submodule update --init --recursive
-cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} .
+CMAKE_CLANG_OPTIONS="-DCMAKE_C_COMPILER:FILEPATH=$(which clang) -DCMAKE_CXX_COMPILER:FILEPATH=$(which clang++) -DCMAKE_CUDA_COMPILER:FILEPATH=$(which nvcc)"
+cmake -B build -G Ninja ${CMAKE_CLANG_OPTIONS} ${CMAKE_BUILD_ALL_FEATURES} .
 
 rapids-logger "Building targets that generate source code"
 cmake --build build --target mrc_style_checks --parallel ${PARALLEL_LEVEL}