Merge branch 'main' into dependabot/pip/classical-ml/classical-ml-62b…

…aa5549d
intel · May 31, 2024 · 4e3aee0 · 4e3aee0
2 parents 1af0bc1 + c3d6878
commit 4e3aee0
Show file tree

Hide file tree

Showing 53 changed files with 535 additions and 976 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,5 +1,10 @@
 .github/*       @tylertitsworth
+classical-ml/*  @tylertitsworth @jitendra42 @sramakintel @sharvil10
 docs/*          @tylertitsworth @jitendra42
+enterprise/*    @sharvil10
 preset/*        @ma-pineda @jafraustro
+pytorch/*       @tylertitsworth @jitendra42 @sramakintel @sharvil10
+tensorflow/*    @tylertitsworth @jitendra42 @sramakintel @sharvil10
 test-runner/*   @tylertitsworth
+workflows/*     @tylertitsworth
 *               @tylertitsworth @jitendra42
diff --git a/pytorch/charts/training/values.yaml → .github/ct.yaml b/pytorch/charts/training/values.yaml → .github/ct.yaml
@@ -12,20 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-metadata:
-  name: ipex-distributed
-  namespace: kubeflow
+# See https://github.com/helm/chart-testing#configuration
 
-imageName: intel/intel-optimized-pytorch
-imageTag: 2.0.0-pip-multinode
-masterResources:
-  cpu: 32
-  memory: 16Gi
-workerResources:
-  cpu: 32
-  memory: 16Gi
-workers: 4
-
-pvcName: ipex
-pvcScn: nil
-pvcResources: 2Gi
+remote: origin
+target-branch: main
+chart-dirs:
+  - workflows/charts
+helm-extra-args: --timeout 600s
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -71,3 +71,43 @@ updates:
     package-ecosystem: pip
     schedule:
       interval: weekly
+  - directory: /workflows
+    groups:
+      helm:
+        patterns:
+          - "*"
+    package-ecosystem: pip
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: /classical-ml
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: /preset/classical-ml
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: /preset/data-analytics
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: /preset/deep-learning
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: /preset/inference-optimization
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: /python
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: /pytorch
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: /tensorflow
+    schedule:
+      interval: weekly
diff --git a/.github/dockerhub-readmes.json b/.github/dockerhub-readmes.json
@@ -12,10 +12,18 @@
             "fname": "./pytorch/README.md",
             "repo-name": "intel/intel-extension-for-pytorch"
         },
+        {
+            "fname": "./pytorch/README.md",
+            "repo-name": "intel/intel-optimized-pytorch"
+        },
         {
             "fname": "./tensorflow/README.md",
             "repo-name": "intel/intel-extension-for-tensorflow"
         },
+        {
+            "fname": "./tensorflow/README.md",
+            "repo-name": "intel/intel-optimized-tensorflow"
+        },
         {
             "fname": "./preset/README.md",
             "repo-name": "intel/data-analytics"

diff --git a/.github/workflows/chart-ci.yaml b/.github/workflows/chart-ci.yaml
@@ -14,10 +14,11 @@
 
 name: Helm Chart CI
 on:
-  pull_request: null
-  # pull_request_target:
-  #   types: [opened, edited, reopened, synchronize]
-  #   branches: [main]
+  pull_request_target:
+    types: [opened, edited, reopened, synchronize]
+    branches: [main]
+    paths:
+    - 'workflows/charts/**'
   merge_group: null
 permissions: read-all
 concurrency:
@@ -27,17 +28,21 @@ jobs:
   helm-ci:
     runs-on: kubectl
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden Runner
+      uses: step-security/harden-runner@f086349bfa2bd1361f7909c78558e816508cdc10 # v2.8.0
+      with:
+        egress-policy: audit
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       with:
         fetch-depth: 0
-        # ref: ${{ github.event.pull_request.head.ref || github.ref }}
-    - uses: azure/setup-kubectl@v4
-    - uses: azure/[email protected]
-    - uses: actions/setup-python@v5
+        ref: "refs/pull/${{ github.event.number }}/merge"
+    - uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0
+    - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
+    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
       with:
         python-version: 3.x
     - name: Setup chart-testing
-      uses: helm/[email protected]
+      uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1
     - name: Get chart diff
       id: changed
       run: |
@@ -48,7 +53,8 @@ jobs:
     - name: Helm Lint, Install, and Test
       if: steps.changed.outputs.changed == 'true'
       run: |
+        install -m 0600 ${{ secrets.KUBECONFIG_PATH }} /tmp/config.yaml
         kubectl config use-context kubeflow
         ct lint-and-install --target-branch ${{ github.event.repository.default_branch }} --config .github/ct.yaml --debug --namespace helm-ci
       env:
-        KUBECONFIG: ${{ secrets.KUBECONFIG_PATH }}
+        KUBECONFIG: /tmp/config.yaml
diff --git a/.github/workflows/container-ci.yaml b/.github/workflows/container-ci.yaml
@@ -38,6 +38,7 @@ on:
       ref:
         description: 'Enter Git Ref:'
         required: true
+        default: 'main'
         type: string
       runner_label:
         description: 'Enter Validation Runner Label:'
@@ -82,7 +83,11 @@ jobs:
       matrix: ${{ steps.build-matrix.outputs.matrix }}
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden Runner
+      uses: step-security/harden-runner@f086349bfa2bd1361f7909c78558e816508cdc10 # v2.8.0
+      with:
+        egress-policy: audit
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       with:
         ref: ${{ inputs.ref }}
     - name: Set Matrix
@@ -101,11 +106,11 @@ jobs:
     outputs:
       group: ${{ steps.build-group.outputs.container-group }}
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       with:
         ref: ${{ inputs.ref }}
       if: ${{ !inputs.no_build }}
-    - uses: docker/login-action@v3
+    - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
       with:
         registry: ${{ secrets.REGISTRY }}
         username: ${{ secrets.REGISTRY_USER }}
@@ -114,7 +119,7 @@ jobs:
     - name: Build Container Group
       if: ${{ !inputs.no_build }}
       id: build-group
-      uses: intel/ai-containers/.github@main
+      uses: intel/ai-containers/.github@d097c5e51f0bee00c2707359b3a78b6a3fa3c85b # main
       with:
         group_dir: ${{ inputs.group_dir }}
         env_overrides: ${{ inputs.env_overrides || env.env_overrides || '' }}
@@ -131,7 +136,11 @@ jobs:
     outputs:
       matrix: ${{ steps.scan-matrix.outputs.matrix }}
     steps:
-    - uses: actions/download-artifact@v4
+    - name: Harden Runner
+      uses: step-security/harden-runner@f086349bfa2bd1361f7909c78558e816508cdc10 # v2.8.0
+      with:
+        egress-policy: audit
+    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
       with:
         path: matrix
     - name: Set Matrix
@@ -148,25 +157,26 @@ jobs:
         container: ${{ fromJSON(needs.setup-scan.outputs.matrix) }}
       fail-fast: false
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       with:
         ref: ${{ inputs.ref }}
-    - uses: docker/login-action@v3
+    - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
       with:
         registry: ${{ secrets.REGISTRY }}
         username: ${{ secrets.REGISTRY_USER }}
         password: ${{ secrets.REGISTRY_TOKEN }}
     - name: Pull Image
       run: docker pull ${{ secrets.REGISTRY }}/${{ secrets.REPO }}:${{ matrix.container }}
     - name: Scan Container
-      uses: intel/ai-containers/.github/scan@main
+      uses: intel/ai-containers/.github/scan@d097c5e51f0bee00c2707359b3a78b6a3fa3c85b # main
       with:
         image-ref: ${{ secrets.REGISTRY }}/${{ secrets.REPO }}:${{ matrix.container }}
         output: ${{ matrix.container }}-scan.sarif
     - name: Cleanup
       if: always()
       run: docker rmi -f ${{ secrets.REGISTRY }}/${{ secrets.REPO }}:${{ matrix.container }}
-    - uses: github/codeql-action/upload-sarif@v3
+    - uses: github/codeql-action/upload-sarif@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6
+      if: ${{ github.event_name != 'workflow_dispatch' }}
       with:
         sarif_file: '${{ matrix.container }}-scan.sarif'
         ####################################################################################################
@@ -178,7 +188,11 @@ jobs:
     outputs:
       matrix: ${{ steps.test-matrix.outputs.matrix }}
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden Runner
+      uses: step-security/harden-runner@f086349bfa2bd1361f7909c78558e816508cdc10 # v2.8.0
+      with:
+        egress-policy: audit
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       with:
         ref: ${{ inputs.ref }}
     - name: Get Recipes
@@ -194,19 +208,19 @@ jobs:
         experimental: [true]
       fail-fast: false
     steps:
-    - uses: step-security/harden-runner@v2
+    - uses: step-security/harden-runner@f086349bfa2bd1361f7909c78558e816508cdc10 # v2.8.0
       with:
         egress-policy: audit
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       with:
         ref: ${{ inputs.ref }}
-    - uses: docker/login-action@v3
+    - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
       with:
         registry: ${{ secrets.REGISTRY }}
         username: ${{ secrets.REGISTRY_USER }}
         password: ${{ secrets.REGISTRY_TOKEN }}
     - name: Test Container Group
-      uses: intel/ai-containers/test-runner@main
+      uses: intel/ai-containers/test-runner@d097c5e51f0bee00c2707359b3a78b6a3fa3c85b # main
       with:
         cache_registry: ${{ secrets.CACHE_REGISTRY }}
         recipe_dir: ${{ inputs.group_dir }}

diff --git a/.github/workflows/dependency-review.yaml b/.github/workflows/dependency-review.yaml
@@ -21,7 +21,10 @@
 #
 # Source repository: https://github.com/actions/dependency-review-action
 name: 'Dependency Review'
-on: [pull_request]
+on:
+  pull_request_target:
+    types: [opened, edited, reopened, synchronize]
+    branches: [main]
 permissions:
   contents: read
 concurrency:
@@ -32,7 +35,11 @@ jobs:
     permissions:
       pull-requests: write
     steps:
-    - uses: actions/checkout@v4
-    - uses: actions/dependency-review-action@v4
+    - name: Harden Runner
+      uses: step-security/harden-runner@f086349bfa2bd1361f7909c78558e816508cdc10 # v2.8.0
+      with:
+        egress-policy: audit
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+    - uses: actions/dependency-review-action@0c155c5e8556a497adf53f2c18edabf945ed8e70 # v4.3.2
       with:
         comment-summary-in-pr: true
diff --git a/.github/workflows/dockerhub-description.yml b/.github/workflows/dockerhub-description.yml
@@ -23,7 +23,12 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden Runner
+      uses: step-security/harden-runner@f086349bfa2bd1361f7909c78558e816508cdc10 # v2.8.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
     - name: Set matrix data
       id: set-matrix
       run: echo "matrix=$(jq -c . < .github/dockerhub-readmes.json)" >> $GITHUB_OUTPUT
@@ -34,8 +39,13 @@ jobs:
       matrix: ${{ fromJson(needs.setup-matrix.outputs.matrix) }}
       fail-fast: false
     steps:
-    - uses: actions/checkout@v4
-    - uses: peter-evans/dockerhub-description@v4
+    - name: Harden Runner
+      uses: step-security/harden-runner@f086349bfa2bd1361f7909c78558e816508cdc10 # v2.8.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+    - uses: peter-evans/dockerhub-description@e98e4d1628a5f3be2be7c231e50981aee98723ae # v4.0.0
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_PASSWORD }}

diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -15,7 +15,6 @@
 name: Docs
 on:
   merge_group: null
-  pull_request: null
   push:
     branches:
     - main
@@ -31,17 +30,21 @@ jobs:
       id-token: write
       pages: write
     steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
+    - name: Harden Runner
+      uses: step-security/harden-runner@f086349bfa2bd1361f7909c78558e816508cdc10 # v2.8.0
+      with:
+        egress-policy: audit
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
       with:
         python-version: 3.8
         cache: pip
     - name: Install python requirements
       run: python -m pip install -r docs/requirements.txt
     - name: Build
       run: mkdocs build --clean
-    - uses: actions/upload-pages-artifact@v3
+    - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1
       with:
         path: site
     - if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
-      uses: actions/deploy-pages@v4
+      uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5