From aca4f465f64d20d3a1102fe7de00aec91303293d Mon Sep 17 00:00:00 2001 From: Brian Szmyd Date: Wed, 23 Aug 2023 10:38:28 -0700 Subject: [PATCH 1/3] Update build matrix for tooling. --- .github/workflows/build_commit.yml | 8 ++++ .github/workflows/build_dependencies.yml | 44 ++++++++++++------- ...{merge_conan_build.yml => conan_build.yml} | 33 ++++++++------ .github/workflows/pr_conan_build.yml | 33 -------------- README.md | 2 +- 5 files changed, 57 insertions(+), 63 deletions(-) rename .github/workflows/{merge_conan_build.yml => conan_build.yml} (63%) delete mode 100644 .github/workflows/pr_conan_build.yml diff --git a/.github/workflows/build_commit.yml b/.github/workflows/build_commit.yml index 12fbb33..8245e0c 100644 --- a/.github/workflows/build_commit.yml +++ b/.github/workflows/build_commit.yml @@ -15,6 +15,9 @@ on: prerelease: required: true type: string + tooling: + required: true + type: string jobs: SislDeps: @@ -25,7 +28,9 @@ jobs: build-type: ${{ inputs.build-type }} malloc-impl: ${{ inputs.malloc-impl }} prerelease: ${{ inputs.prerelease }} + tooling: None if: ${{ github.event_name != 'pull_request' }} + IOMgrDeps: needs: SislDeps uses: ./.github/workflows/build_dependencies.yml @@ -35,8 +40,10 @@ jobs: build-type: ${{ inputs.build-type }} malloc-impl: ${{ inputs.malloc-impl }} prerelease: ${{ inputs.prerelease }} + tooling: ${{ inputs.tooling }} testing: 'True' if: ${{ github.event_name != 'pull_request' }} + IOMgrBuild: uses: ./.github/workflows/build_dependencies.yml with: @@ -45,5 +52,6 @@ jobs: build-type: ${{ inputs.build-type }} malloc-impl: ${{ inputs.malloc-impl }} prerelease: ${{ inputs.prerelease }} + tooling: ${{ inputs.tooling }} testing: 'True' if: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/build_dependencies.yml b/.github/workflows/build_dependencies.yml index efa9505..64ae5c4 100644 --- a/.github/workflows/build_dependencies.yml +++ b/.github/workflows/build_dependencies.yml @@ -19,6 +19,10 @@ on: prerelease: required: true type: string + tooling: + required: false + type: string + default: 'None' testing: required: false type: string @@ -60,6 +64,13 @@ on: - 'True' - 'False' default: 'False' + tooling: + required: false + type: choice + - 'Sanitize' + - 'Coverage' + - 'None' + default: 'None' testing: description: 'Build and Run' required: true @@ -135,7 +146,7 @@ jobs: -o sisl:prerelease=${{ inputs.prerelease }} \ -o sisl:malloc_impl=${{ inputs.malloc-impl }} \ -o sisl:testing=False \ - -o iomgr:testing=off \ + -o testing=off \ -s build_type=${{ inputs.build-type }} \ --build missing \ . @@ -156,6 +167,20 @@ jobs: fail_on_cache_miss: true if: ${{ inputs.testing == 'True' && github.event_name != 'pull_request' && steps.restore-cache.outputs.cache-hit != 'true' }} + - name: Create and Test Package + run: | + sanitize=$([[ "${{ inputs.tooling }}" == "Sanitize" ]] && echo "True" || echo "False") + conan create \ + -o sisl:prerelease=${{ inputs.prerelease }} \ + -o sisl:malloc_impl=${{ inputs.malloc-impl }} \ + -o sisl:testing=False \ + -o iomgr:sanitize=${sanitize} \ + -o iomgr:testing='epoll_mode' \ + -s build_type=${{ inputs.build-type }} \ + --build missing \ + . + if: ${{ inputs.testing == 'True' && inputs.tooling != 'Coverage' }} + - name: Code Coverage Run run: | conan install \ @@ -168,24 +193,11 @@ jobs: --build missing \ . conan build . - if: ${{ inputs.testing == 'True' && inputs.platform == 'ubuntu-22.04' && inputs.build-type == 'Debug' && inputs.malloc-impl == 'libc' && inputs.prerelease == 'False' }} + if: ${{ inputs.testing == 'True' && inputs.tooling == 'Coverage' }} - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} gcov: true - if: ${{ inputs.testing == 'True' && inputs.platform == 'ubuntu-22.04' && inputs.build-type == 'Debug' && inputs.malloc-impl == 'libc' && inputs.prerelease == 'False' }} - - - - name: Create and test Package - run: | - conan create \ - -o sisl:prerelease=${{ inputs.prerelease }} \ - -o sisl:malloc_impl=${{ inputs.malloc-impl }} \ - -o sisl:testing=False \ - -o iomgr:testing='epoll_mode' \ - -s build_type=${{ inputs.build-type }} \ - --build missing \ - . - if: ${{ inputs.testing == 'True' }} + if: ${{ inputs.testing == 'True' && inputs.tooling == 'Coverage' }} diff --git a/.github/workflows/merge_conan_build.yml b/.github/workflows/conan_build.yml similarity index 63% rename from .github/workflows/merge_conan_build.yml rename to .github/workflows/conan_build.yml index 7d613c4..7bcd0bd 100644 --- a/.github/workflows/merge_conan_build.yml +++ b/.github/workflows/conan_build.yml @@ -1,4 +1,4 @@ -name: IOManager Merge Build +name: IOManager Build on: workflow_dispatch: @@ -6,6 +6,10 @@ on: branches: - master - stable/v8.x + pull_request: + branches: + - master + - stable/v8.x jobs: Build: @@ -14,26 +18,29 @@ jobs: matrix: platform: ["ubuntu-22.04"] build-type: ["Debug", "Release"] - malloc-impl: ["libc", "tcmalloc", "jemalloc"] + malloc-impl: ["libc", "tcmalloc"] prerelease: ["True", "False"] + tooling: ["Sanitize", "Coverage", "None"] exclude: - - malloc-impl: jemalloc - build-type: Debug - - malloc-impl: tcmalloc - build-type: Debug - - malloc-impl: tcmalloc - prerelease: "True" - - malloc-impl: jemalloc + - build-type: Debug prerelease: "False" - - malloc-impl: libc - build-type: Release - + - build-type: Debug + tooling: None + - build-type: Debug + malloc-impl: tcmalloc + - build-type: Release + malloc-impl: libc + - build-type: Release + tooling: Sanitize + - build-type: Release + tooling: Coverage uses: ./.github/workflows/build_commit.yml with: platform: ${{ matrix.platform }} build-type: ${{ matrix.build-type }} malloc-impl: ${{ matrix.malloc-impl }} prerelease: ${{ matrix.prerelease }} + tooling: ${{ matrix.tooling }} ChainBuild: runs-on: "ubuntu-22.04" steps: @@ -44,6 +51,6 @@ jobs: -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer ${{ secrets.CHAIN_BUILD_TOKEN }}"\ -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/eBay/homestore/actions/workflows/merge_conan_build.yml/dispatches \ + https://api.github.com/repos/eBay/homestore/actions/workflows/conan_build.yml/dispatches \ -d '{"ref":"master","inputs":{}}' if: ${{ github.ref == 'refs/heads/master' }} diff --git a/.github/workflows/pr_conan_build.yml b/.github/workflows/pr_conan_build.yml deleted file mode 100644 index c563122..0000000 --- a/.github/workflows/pr_conan_build.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: IOManager PR Build - -on: - pull_request: - branches: - - master - - stable/v8.x - -jobs: - Build: - strategy: - fail-fast: false - matrix: - platform: ["ubuntu-22.04"] - build-type: ["Debug", "Release"] - malloc-impl: ["libc", "tcmalloc", "jemalloc"] - prerelease: ["True", "False"] - exclude: - - malloc-impl: jemalloc - build-type: Debug - - malloc-impl: tcmalloc - build-type: Debug - - malloc-impl: jemalloc - prerelease: "False" - - malloc-impl: libc - build-type: Release - - uses: ./.github/workflows/build_commit.yml - with: - platform: ${{ matrix.platform }} - build-type: ${{ matrix.build-type }} - malloc-impl: ${{ matrix.malloc-impl }} - prerelease: ${{ matrix.prerelease }} diff --git a/README.md b/README.md index cd2ef75..46edccb 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # IOManager -[![Conan Build](https://github.com/eBay/IOManager/actions/workflows/merge_conan_build.yml/badge.svg?branch=master)](https://github.com/eBay/IOManager/actions/workflows/merge_conan_build.yml) +[![Conan Build](https://github.com/eBay/IOManager/actions/workflows/conan_build.yml/badge.svg?branch=master)](https://github.com/eBay/IOManager/actions/workflows/conan_build.yml) [![CodeCov](https://codecov.io/gh/eBay/IOManager/branch/master/graph/badge.svg)](https://codecov.io/gh/eBay/IOManager) IOManager tries to bridge the gap in existing async framework to build full async networked database/storage/keyvalue storage. Motivation From 6d167ffcf77f491f72460ed9266b7844a69845a4 Mon Sep 17 00:00:00 2001 From: Brian Szmyd Date: Wed, 23 Aug 2023 10:45:17 -0700 Subject: [PATCH 2/3] Enforcing sanitizer on sisl is not correct for a pr to valid. --- conanfile.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/conanfile.py b/conanfile.py index 00021f1..e3f748a 100644 --- a/conanfile.py +++ b/conanfile.py @@ -43,8 +43,6 @@ def configure(self): if self.options.shared: del self.options.fPIC if self.settings.build_type == "Debug": - if self.options.sanitize: - self.options['sisl'].sanitize = True if self.options.coverage and self.options.sanitize: raise ConanInvalidConfiguration("Sanitizer does not work with Code Coverage!") From c49136fd486ae4c08aadb72dad9ea88d64508fbe Mon Sep 17 00:00:00 2001 From: Brian Szmyd Date: Wed, 23 Aug 2023 10:56:16 -0700 Subject: [PATCH 3/3] Update artifactory build. --- .jenkins/Jenkinsfile | 158 +++++++------------------------------------ conanfile.py | 6 +- 2 files changed, 28 insertions(+), 136 deletions(-) diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile index c563af6..90ff051 100644 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@ -5,10 +5,9 @@ pipeline { ARTIFACTORY_PASS = credentials('ARTIFACTORY_PASS') CONAN_USER = 'oss' TARGET_BRANCH = 'master' - TESTING_BRANCH = 'testing/v*' STABLE_BRANCH = 'stable/v*' - LAST_COMMITS = '' built_chain = '' + failed_pkg = '' } parameters { @@ -17,23 +16,20 @@ pipeline { stages { stage('Adjust Tag for Master/PR') { - when { not { anyOf { - branch "${TESTING_BRANCH}" + when { not { branch "${STABLE_BRANCH}" - } } } + } } steps { script { - sh(script: "sed -Ei 's,version = .*\"([[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+).*,version = \"\\1-${env.BUILD_NUMBER}\",' conanfile.py") + sh(script: "sed -Ei 's, version = .*\"([[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+).*, version = \"\\1-${env.BUILD_NUMBER}\",' conanfile.py") BUILD_MISSING = "--build missing" - LAST_COMMITS = sh(script: "git log -n 2 --pretty=short", returnStdout: true) } } } stage('Adjust for Testing/Stable') { - when { anyOf { - branch "${TESTING_BRANCH}" + when { branch "${STABLE_BRANCH}" - } } + } steps { script { BUILD_MISSING = "" @@ -45,7 +41,7 @@ pipeline { steps { script { PROJECT = sh(script: "grep -m 1 'name =' conanfile.py | awk '{print \$3}' | tr -d '\n' | tr -d '\"'", returnStdout: true) - VER = sh(script: "grep -m 1 'version =' conanfile.py | awk '{print \$3}' | tr -d '\n' | tr -d '\"'", returnStdout: true) + VER = sh(script: "grep -m 1 ' version =' conanfile.py | awk '{print \$3}' | tr -d '\n' | tr -d '\"'", returnStdout: true) CONAN_CHANNEL = sh(script: "echo ${BRANCH_NAME} | sed -E 's,(\\w+-?\\d*)/.*,\\1,' | sed -E 's,-,_,' | tr -d '\n'", returnStdout: true) TAG = "${VER}@${CONAN_USER}/${CONAN_CHANNEL}" slackSend color: '#0063D1', channel: '#sds-ci', message: "*${PROJECT}/${TAG}* is building: ${BUILD_URL}" @@ -63,128 +59,25 @@ pipeline { } } - stage('Build') { - failFast true - matrix { - agent { label 'sds-builder-2204 && hugepages' } - axes { - axis { - name 'BUILD_TYPE' - values 'sanitize', 'release', 'debug', 'test' - } - axis { - name 'COVERAGE' - values 'False' - } - } - excludes { - exclude { - axis { - name 'BUILD_TYPE' - values 'sanitize', 'test', 'release' - } - axis { - name 'COVERAGE' - values 'True' - } - } - } - - stages { - stage('Adjust Tag for Master/PR') { - when { not { anyOf { - branch "${TESTING_BRANCH}" - branch "${STABLE_BRANCH}" - } } } - steps { - sh(script: "sed -Ei 's,version = .*\"([[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+).*,version = \"\\1-${env.BUILD_NUMBER}\",' conanfile.py") - } - } - - stage("Build") { - when { allOf { - expression { "${COVERAGE}" == "False" } - } } - steps { - script { - def PRERELEASE = 'True' - def BUILD_PROFILE = "${BUILD_TYPE}" - def SANITIZE = 'False' - - if ("${BUILD_TYPE}" == 'sanitize') { - SANITIZE = 'True' - BUILD_PROFILE = "debug" - } - - if ("${BUILD_TYPE}" == "release") { - PRERELEASE = 'False' - BUILD_PROFILE = "test" - } - - if (("${env.BRANCH_NAME}" =~ /PR-/) && ("$BUILD_TYPE" == "debug")) { - sh "echo Skipping debug build for PR branch" - } else { - sh "conan create ${BUILD_MISSING} -o sisl:prerelease=${PRERELEASE} -o ${PROJECT}:sanitize=${SANITIZE} -pr ${BUILD_PROFILE} . ${PROJECT}/${TAG}" - } - } - } - } - stage("Deploy") { - when { allOf { - expression { "${COVERAGE}" == 'False' } - expression { "${BUILD_TYPE}" != 'sanitize' } - expression { !(env.BRANCH_NAME =~ /PR-/) } - } } - steps { - sh "conan user -r ebay-local -p ${ARTIFACTORY_PASS} _service_sds" - sh "conan upload ${PROJECT}/${TAG} -c --all -r ebay-local" - } - } - - stage('Coverage') { - when { not { anyOf { - branch "${STABLE_BRANCH}" - expression { "${COVERAGE}" == 'False' } - expression { !(env.BRANCH_NAME =~ /PR-/) } - } } } - - stages { - stage("Adjust Sonar Branch") { - when { - not { - branch "${TARGET_BRANCH}" - } - } - steps { - sh "echo \"sonar.branch.target=${TARGET_BRANCH}\" >> sonar-project.properties" - } - } - stage("Code Coverage") { - steps { - slackSend channel: '#sds-ci', message: "*${PROJECT}:${TAG}* is undergoing Code Coverage." - sh "echo \"sonar.branch.name=${BRANCH_NAME}\" >> sonar-project.properties" - sh "conan install -pr ${BUILD_TYPE} ${BUILD_MISSING} -o ${PROJECT}:coverage=True ." - sh "build-wrapper-linux-x86-64 --out-dir /tmp/sonar conan build ." - sh "find . -name \"*.gcno\" -exec gcov {} \\;" - withSonarQubeEnv('sds-sonar') { - sh "sonar-scanner -Dsonar.projectBaseDir=. -Dsonar.projectVersion=\"${VER}\"" - } - } - } - stage("Quality Gate") { - steps { - timeout(time: 30, unit: 'MINUTES') { - waitForQualityGate abortPipeline: true - } - } - } - } - } - } + stage("Compile") { + steps { + sh "conan create ${BUILD_MISSING} -pr debug -o ${PROJECT}:sanitize=True . ${PROJECT}/${TAG} ; \ + conan create ${BUILD_MISSING} -pr debug . ${PROJECT}/${TAG} ; \ + conan create ${BUILD_MISSING} -pr test -o sisl:malloc_impl=tcmalloc . ${PROJECT}/${TAG} ; \ + conan create ${BUILD_MISSING} -pr test -o sisl:prerelease=True -o sisl:malloc_impl=tcmalloc . ${PROJECT}/${TAG} ; \ + " } } -/* FIXME (bszmyd): Disabled till integration. + stage("Deploy") { + when { + expression { !(env.BRANCH_NAME =~ /PR-/) } + } + steps { + sh "conan user -r ebay-local -p ${ARTIFACTORY_PASS} _service_sds" + sh "conan upload ${PROJECT}/${TAG} -c --all -r ebay-local" + } + } stage("Downstream Build") { when { allOf { expression { (env.BRANCH_NAME == "${TARGET_BRANCH}") } @@ -217,18 +110,15 @@ pipeline { } } } -*/ } post { failure { -/* script { if ("${failed_pkg}" != "") { - slackSend color: '#E43237', channel: '#homestore-build', message: "@here IOManager's downstream pkg - *${failed_pkg}* build failed.\n*URL:* ${BUILD_URL}\n*Last 2 commits:* ```${LAST_COMMITS}```\nIf result not expected, revert (aka `conan remove -r ebay-local`) these pkgs: ```${built_chain}```" + slackSend color: '#E43237', channel: '#homestore-build', message: "@here IOManager's downstream pkg - *${failed_pkg}* build failed.\n*URL:* ${BUILD_URL}\nIf result not expected, revert (aka `conan remove -r ebay-local`) these pkgs: ```${built_chain}```" } } -*/ slackSend color: '#E43237', channel: '#sds-ci', message: "*${PROJECT}/${TAG}* has had a failure : ${BUILD_URL}" } success { diff --git a/conanfile.py b/conanfile.py index e3f748a..bf10540 100644 --- a/conanfile.py +++ b/conanfile.py @@ -32,7 +32,6 @@ class IOMgrConan(ConanFile): 'sanitize': False, 'spdk': True, 'testing': 'epoll_mode', - 'sisl:prerelease': True, 'fiber_impl': 'boost', } @@ -45,9 +44,12 @@ def configure(self): if self.settings.build_type == "Debug": if self.options.coverage and self.options.sanitize: raise ConanInvalidConfiguration("Sanitizer does not work with Code Coverage!") + if self.options.testing == 'off': + if self.options.coverage or self.options.sanitize: + raise ConanInvalidConfiguration("Coverage/Sanitizer requires Testing!") def build_requirements(self): - self.build_requires("gtest/1.13.0") + self.build_requires("gtest/1.14.0") self.build_requires("cpr/1.10.4") def requirements(self):