Merge branch 'master' into keras_rest

red-hat-data-services · Sep 24, 2024 · 121bc4f · 121bc4f
2 parents 75f7b2e + 7f67375
commit 121bc4f
Show file tree

Hide file tree

Showing 6 changed files with 48 additions and 24 deletions.
diff --git a/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot b/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot
@@ -89,7 +89,12 @@ Verify RHODS Installation
   IF  "${UPDATE_CHANNEL}" == "odh-nightlies" or "${cluster_type}" != "managed"
     IF  "${PRODUCT}" == "ODH"
         Apply DSCInitialization CustomResource    dsci_name=${DSCI_NAME}
-        Wait For DSCInitialization CustomResource To Be Ready    timeout=30
+        IF    "${TEST_ENV.lower()}" == "crc"
+            ${timeout_in_seconds} =   Set Variable   180
+        ELSE
+            ${timeout_in_seconds} =   Set Variable   30
+        END
+        Wait For DSCInitialization CustomResource To Be Ready    timeout=${timeout_in_seconds}
     END
     Apply DataScienceCluster CustomResource    dsc_name=${DSC_NAME}
   END
@@ -256,22 +261,14 @@ Create DSCInitialization CustomResource Using Test Variables
     Run    sed -i'' -e 's/<monitoring_namespace>/${MONITORING_NAMESPACE}/' ${file_path}dsci_apply.yml
 
 Wait For DSCInitialization CustomResource To Be Ready
-  [Documentation]   Wait for DSCInitialization CustomResource To Be Ready
-  [Arguments]     ${timeout}
-  Log To Console    Waiting for DSCInitialization CustomResource To Be Ready
-  ${status}   Set Variable   False
-  FOR    ${counter}    IN RANGE   ${timeout}
-         ${return_code}    ${output} =    Run And Return Rc And Output   oc get DSCInitialization --no-headers -o custom-columns=":status.phase"
-         IF    '${output}' == 'Ready'
-               ${status} =  Set Variable  True
-               Log To Console  DSCInitialization CustomResource is Ready
-               BREAK
-         END
-         Sleep    1 sec
-  END
-  IF    '${status}' == 'False'
-        Run Keyword And Continue On Failure    FAIL    Timeout- DSCInitialization CustomResource is not Ready
-  END
+    [Documentation]   Wait ${timeout} seconds for DSCInitialization CustomResource To Be Ready
+    [Arguments]     ${timeout}
+    Log To Console    Waiting ${timeout} seconds for DSCInitialization CustomResource To Be Ready
+    ${result} =    Run Process    oc wait DSCInitialization --timeout\=${timeout}s --for jsonpath\='{.status.phase}'\=Ready --all
+    ...    shell=true    stderr=STDOUT
+    IF    ${result.rc} != 0
+        Run Keyword And Continue On Failure    FAIL    ${result.stdout}
+    END
 
 Apply DataScienceCluster CustomResource
     [Documentation]

diff --git a/ods_ci/tests/Resources/Page/DistributedWorkloads/DistributedWorkloads.resource b/ods_ci/tests/Resources/Page/DistributedWorkloads/DistributedWorkloads.resource
@@ -10,7 +10,6 @@ ${CODEFLARE-SDK-API_URL}                 %{CODEFLARE-SDK-API_URL=https://api.git
 ${CODEFLARE-SDK_DIR}                     codeflare-sdk
 ${CODEFLARE-SDK_REPO_URL}                %{CODEFLARE-SDK_REPO_URL=https://github.com/project-codeflare/codeflare-sdk.git}
 ${DISTRIBUTED_WORKLOADS_RELEASE_ASSETS}  https://github.com/opendatahub-io/distributed-workloads/releases/latest/download
-${RAY_IMAGE}                             quay.io/rhoai/ray@sha256:859f5c41d41bad1935bce455ad3732dff9d4d4c342b7155a7cd23809e85698ab
 ${FMS_HF_TUNING_IMAGE}                   quay.io/modh/fms-hf-tuning@sha256:8edea6f0f9c4c631cdca1e1c10abf0d4b994738fde78c40d48eda216fdd382f5
 ${NOTEBOOK_IMAGE}                        quay.io/modh/odh-generic-data-science-notebook@sha256:9d7f80080a453bcf7dee01b986df9ee811ee74f6f433c601a8b67d283c160547
 ${NOTEBOOK_USER_NAME}                    ${TEST_USER_3.USERNAME}
@@ -189,6 +188,7 @@ Prepare Codeflare E2E Test Suite
     Restart Kueue
 
     Log To Console    "Downloading compiled test binary ${ODH_BINARY_NAME}"
+
     ${result} =    Run Process    curl --location --silent --output ${ODH_BINARY_NAME} ${DISTRIBUTED_WORKLOADS_RELEASE_ASSETS}/${ODH_BINARY_NAME} && chmod +x ${ODH_BINARY_NAME}
     ...    shell=true
     ...    stderr=STDOUT
@@ -222,7 +222,7 @@ Generate User Token
 
 Run Codeflare ODH Test
     [Documentation]    Run Codeflare ODH Test
-    [Arguments]    ${TEST_NAME}
+    [Arguments]    ${TEST_NAME}    ${RAY_IMAGE}
     Log To Console    "Running test: ${TEST_NAME}"
     ${result} =    Run Process    ./${ODH_BINARY_NAME} -test.run ${TEST_NAME}
     ...    shell=true

diff --git a/...0__rhoai_upgrade/0201__pre_upgrades.robot → ...00__rhoai_upgrade/0201__pre_upgrade.robot b/...0__rhoai_upgrade/0201__pre_upgrades.robot → ...00__rhoai_upgrade/0201__pre_upgrade.robot
@@ -22,6 +22,7 @@ Resource           ../../Resources/Page/DistributedWorkloads/DistributedWorkload
 Resource           ../../Resources/Page/DistributedWorkloads/WorkloadMetricsUI.resource
 Suite Setup        Dashboard Suite Setup
 Suite Teardown     RHOSi Teardown
+Test Tags          PreUpgrade
 
 
 *** Variables ***

diff --git a/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot b/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot
@@ -9,6 +9,7 @@ Resource            ../../Resources/Page/ODH/ODHDashboard/ODHDashboardSettings.r
 Resource            ../../Resources/Page/ODH/JupyterHub/ODHJupyterhub.resource
 Library             DebugLibrary
 Library             JupyterLibrary
+Test Tags           DuringUpgrade
 
 
 *** Variables ***

diff --git a/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot b/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot
@@ -22,7 +22,8 @@ Resource           ../../Resources/Page/HybridCloudConsole/OCM.robot
 Resource           ../../Resources/Page/DistributedWorkloads/DistributedWorkloads.resource
 Resource           ../../Resources/Page/DistributedWorkloads/WorkloadMetricsUI.resource
 Resource           ../../Resources/CLI/MustGather/MustGather.resource
-Suite Setup    Upgrade Suite Setup
+Suite Setup        Upgrade Suite Setup
+Test Tags          PostUpgrade
 
 
 *** Variables ***

diff --git a/ods_ci/tests/Tests/0600__distributed_workloads/0602__training/test-run-codeflare-tests.robot b/ods_ci/tests/Tests/0600__distributed_workloads/0602__training/test-run-codeflare-tests.robot
@@ -8,6 +8,11 @@ Resource          ../../../../tasks/Resources/RHODS_OLM/install/oc_install.robot
 Resource          ../../../../tests/Resources/Page/DistributedWorkloads/DistributedWorkloads.resource
 
 
+*** Variables ***
+${RAY_CUDA_IMAGE}                    quay.io/modh/ray@sha256:4575dd54fe05213fe9228cd316d5e87a9395cd16b618a85a94995709e5f3aaeb
+${RAY_TORCH_CUDA_IMAGE}              quay.io/rhoai/ray@sha256:158b481b8e9110008d60ac9fb8d156eadd71cb057ac30382e62e3a231ceb39c0
+
+
 *** Test Cases ***
 Run TestKueueRayCpu ODH test
     [Documentation]    Run Go ODH test: TestKueueRayCpu
@@ -16,7 +21,7 @@ Run TestKueueRayCpu ODH test
     ...     DistributedWorkloads
     ...     Training
     ...     CodeflareOperator
-    Run Codeflare ODH Test    TestMnistRayCpu
+    Run Codeflare ODH Test    TestMnistRayCpu    ${RAY_CUDA_IMAGE}
 
 Run TestKueueRayGpu ODH test
     [Documentation]    Run Go ODH test: TestKueueRayGpu
@@ -25,7 +30,7 @@ Run TestKueueRayGpu ODH test
     ...     DistributedWorkloads
     ...     Training
     ...     CodeflareOperator
-    Run Codeflare ODH Test    TestMnistRayGpu
+    Run Codeflare ODH Test    TestMnistRayGpu    ${RAY_CUDA_IMAGE}
 
 Run TestRayTuneHPOCpu ODH test
     [Documentation]    Run Go ODH test: TestMnistRayTuneHpoCpu
@@ -34,7 +39,7 @@ Run TestRayTuneHPOCpu ODH test
     ...     DistributedWorkloads
     ...     Training
     ...     CodeflareOperator
-    Run Codeflare ODH Test    TestMnistRayTuneHpoCpu
+    Run Codeflare ODH Test    TestMnistRayTuneHpoCpu    ${RAY_CUDA_IMAGE}
 
 Run TestRayTuneHPOGpu ODH test
     [Documentation]    Run Go ODH test: TestMnistRayTuneHpoGpu
@@ -43,4 +48,23 @@ Run TestRayTuneHPOGpu ODH test
     ...     DistributedWorkloads
     ...     Training
     ...     CodeflareOperator
-    Run Codeflare ODH Test    TestMnistRayTuneHpoGpu
+    Run Codeflare ODH Test    TestMnistRayTuneHpoGpu    ${RAY_CUDA_IMAGE}
+
+Run TestKueueCustomRayCpu ODH test
+    [Documentation]    Run Go ODH test: TestKueueCustomRayCpu
+    [Tags]  RHOAIENG-10013
+    ...     Tier1
+    ...     DistributedWorkloads
+    ...     Training
+    ...     CodeflareOperator
+    Run Codeflare ODH Test    TestMnistCustomRayImageCpu    ${RAY_TORCH_CUDA_IMAGE}
+
+Run TestKueueCustomRayGpu ODH test
+    [Documentation]    Run Go ODH test: TestKueueCustomRayGpu
+    [Tags]  RHOAIENG-10013
+    ...     Resources-GPU
+    ...     Tier1
+    ...     DistributedWorkloads
+    ...     Training
+    ...     CodeflareOperator
+    Run Codeflare ODH Test    TestMnistCustomRayImageGpu    ${RAY_TORCH_CUDA_IMAGE}