From 814ad802ef4983bdd7ce92adcd214846aa6de74b Mon Sep 17 00:00:00 2001 From: xian_cao Date: Mon, 16 Oct 2023 19:24:40 +0000 Subject: [PATCH] OCNE: Add ItWlsDomainOnPVSample and ItFmwDomainOnPVSample test into OCNE Environment --- Jenkinsfile.ocne | 86 +++++++++++++------ integration-tests/pom.xml | 7 ++ .../kubernetes/ItFmwDomainOnPVSample.java | 20 ++++- .../kubernetes/ItIntrospectVersion.java | 6 +- .../kubernetes/ItWlsDomainOnPVSample.java | 20 ++++- .../domain-on-pv/run-test.sh | 5 ++ .../domain-on-pv/test-env.sh | 5 +- validateCLI.docker.dat | 2 +- 8 files changed, 119 insertions(+), 32 deletions(-) diff --git a/Jenkinsfile.ocne b/Jenkinsfile.ocne index c503c3916a0..8709b97a914 100644 --- a/Jenkinsfile.ocne +++ b/Jenkinsfile.ocne @@ -3,7 +3,8 @@ // CRON_SETTINGS = '''H 1 * * * % MAVEN_PROFILE_NAME=olcne-srg - H 8 * * * % MAVEN_PROFILE_NAME=olcne-mrg''' + H 5 * * * % MAVEN_PROFILE_NAME=olcne-mrg + H 14 * * * % MAVEN_PROFILE_NAME=olcne-sequential''' pipeline { agent { label 'large' } @@ -13,7 +14,7 @@ pipeline { } triggers { // timer trigger for "nightly build" - parameterizedCron(env.JOB_NAME == 'wko-ocne-main-nightly-new' ? + parameterizedCron(env.JOB_NAME == 'wko-ocne-main-nightly' ? CRON_SETTINGS : '') } @@ -34,7 +35,8 @@ pipeline { oci_profile = 'DEFAULT' jenkins_user = sh(returnStdout: true, script: "id -nu").trim() jenkins_home_directory = sh(returnStdout: true, script: "getent passwd ${jenkins_user} | cut -d: -f6").trim() - + OCI_CLI_CONFIG_FILE = "${jenkins_home_directory}/.oci/config" + oci_config_file_creds = 'wkt-oci-config-file' oci_signing_key_creds = 'wkt-oci-signing-key' @@ -53,15 +55,15 @@ pipeline { result_root = "${outdir}/wl_k8s_test_results" pv_root_parent = "${outdir}/k8s-pvroot" pv_root = "${pv_root_parent}/newpvroot" + pv_root_for_sampletest = "/shared" start_time = sh(script: 'date +"%Y-%m-%d %H:%M:%S"', returnStdout: true).trim() wle_download_url="https://github.com/oracle/weblogic-logging-exporter/releases/latest" - CLUSTER_NAME = "${env.JOB_NAME == 'wko-ocne-main-nightly-new' ? 'mycluster1' : 'mycluster1'}" + CLUSTER_NAME = "${env.JOB_NAME == 'wko-ocne-main-nightly' ? 'mycluster1' : 'mycluster1'}" kubeconfig_file = "${WORKSPACE}/ocne_kubeconfig1" - //MAVEN_PROFILE_NAME = "${env.JOB_NAME == 'wko-ocne-main-nightly-new' ? 'olcne-srg' : 'olcne-mrg'}" - availability_domain = "${env.JOB_NAME == 'wko-ocne-main-nightly-new' ? 'mFEn:PHX-AD-1' : 'mFEn:PHX-AD-1'}" - PARALLEL_RUN = "${env.JOB_NAME == 'wko-ocne-main-nightly-new' ? true : false}" + availability_domain = "${env.JOB_NAME == 'wko-ocne-main-nightly' ? 'mFEn:PHX-AD-1' : 'mFEn:PHX-AD-1'}" + PARALLEL_RUN = "${env.JOB_NAME == 'wko-ocne-main-nightly' ? true : false}" } parameters { @@ -76,6 +78,7 @@ pipeline { choices: [ 'olcne-srg', 'olcne-mrg', + 'olcne-sequential', 'integration-tests' ] ) @@ -254,6 +257,21 @@ pipeline { } } + stage('Setup OCI config file ') { + + steps { + withCredentials([file(credentialsId: "${oci_config_file_creds}", variable: 'OCI_CONFIG_FILE'), + file(credentialsId: "${oci_signing_key_creds}", variable: 'OCI_SIGNING_KEY')]) { + sh "mkdir -p ${jenkins_home_directory}/.oci" + sh "sed -e \"s:__jenkins_home__:${jenkins_home_directory}:g\" \"\$OCI_CONFIG_FILE\" > ${jenkins_home_directory}/.oci/config" + sh "cp \"\$OCI_SIGNING_KEY\" ${jenkins_home_directory}/.oci/oci-signing-key.pem" + } + sh ''' + cat ${jenkins_home_directory}/.oci/config + ''' + } + } + stage('Build WebLogic Kubernetes Operator') { steps { @@ -265,12 +283,21 @@ pipeline { stage('Preparing Integration Test Environment') { steps { - sh 'mkdir -m777 -p ${result_root}' - echo "Results will be in ${result_root}" - sh 'mkdir -m777 -p ${pv_root_parent}' - sh 'sudo mount 100.105.4.71:/fs-ocne-test ${pv_root_parent}' - sh 'mkdir -m777 -p ${pv_root}' - echo "Persistent volume files, if any, will be in ${pv_root}" + sh ''' + + mt_privateip_id=`oci fs mount-target list --compartment-id=${compartment_id} --display-name="mt-ocne-test" --availability-domain=${availability_domain} | jq -r '.data[] | ."private-ip-ids"[]'` + mt_private_ip=`oci network private-ip get --private-ip-id "${mt_privateip_id}" | jq -r '.data | ."ip-address"'` + + mkdir -m777 -p ${result_root} + echo "Results will be in ${result_root}" + mkdir -m777 -p ${pv_root_parent} + sudo mount ${mt_private_ip}:/fs-ocne-test ${pv_root_parent} + mkdir -m777 -p ${pv_root} + echo "Persistent volume files, if any, will be in ${pv_root}" + echo "Create pv root for sample test in ${pv_root_for_sampletest}" + sudo mkdir -m777 -p ${pv_root_for_sampletest} + sudo mount ${mt_private_ip}:/fs-ocne-test ${pv_root_for_sampletest} + ''' } } @@ -279,6 +306,10 @@ pipeline { sh ''' ################# + + k8s_master_instance_id=`oci compute instance list --compartment-id=${compartment_id} --display-name=ocne-new2 |jq -r '.data[]."id"'` + k8s_master_instance_private_ip=`oci compute instance list-vnics --compartment-id=${compartment_id} --instance-id=${k8s_master_instance_id} |jq -r '.data[]."private-ip"'` + echo "Generating kubeconfig file" cat > $kubeconfig_file <> ${WORKSPACE}/.mvn/maven.config fi - if [ "${MAVEN_PROFILE_NAME}" == "olcne-srg" ]; then + if [ "${MAVEN_PROFILE_NAME}" == "olcne-sequential" ]; then PARALLEL_RUN='false' fi - if [ "${MAVEN_PROFILE_NAME}" == "olcne-mrg" ]; then - PARALLEL_RUN='true' - fi + echo "MAVEN_PROFILE_NAME:" $MAVEN_PROFILE_NAME + echo "PARALLEL_RUN:" $PARALLEL_RUN echo "-Dwko.it.wle.download.url=\"${wle_download_url}\"" >> ${WORKSPACE}/.mvn/maven.config echo "-Dwko.it.result.root=\"${result_root}\"" >> ${WORKSPACE}/.mvn/maven.config echo "-Dwko.it.pv.root=\"${pv_root}\"" >> ${WORKSPACE}/.mvn/maven.config @@ -402,13 +434,17 @@ EOF usernamePassword(credentialsId: "${ocir_creds}", usernameVariable: 'OCIR_USER', passwordVariable: 'OCIR_PASS') ]) { sh ''' + + k8s_master_instance_id=`oci compute instance list --compartment-id=${compartment_id} --display-name=ocne-new2 |jq -r '.data[]."id"'` + k8s_master_instance_private_ip=`oci compute instance list-vnics --compartment-id=${compartment_id} --instance-id=${k8s_master_instance_id} |jq -r '.data[]."private-ip"'` + export HTTPS_PROXY='http://www-proxy-hqdc.us.oracle.com:80' export https_proxy='http://www-proxy-hqdc.us.oracle.com:80' export HTTP_PROXY='http://www-proxy-hqdc.us.oracle.com:80' export http_proxy='http://www-proxy-hqdc.us.oracle.com:80' - #100.105.4.76 is the ip address of kubernetes master node - export NO_PROXY='localhost,127.0.0.1,.us.oracle.com,.oraclecorp.com,/var/run/docker.sock,100.105.4.76' - export no_proxy='localhost,127.0.0.1,.us.oracle.com,.oraclecorp.com,/var/run/docker.sock,100.105.4.76' + + export NO_PROXY="localhost,127.0.0.1,.us.oracle.com,.oraclecorp.com,/var/run/docker.sock,${k8s_master_instance_private_ip}" + export no_proxy="localhost,127.0.0.1,.us.oracle.com,.oraclecorp.com,/var/run/docker.sock,${k8s_master_instance_private_ip}" export PATH=${runtime_path} export OKE_CLUSTER="false" export OCNE="true" @@ -438,7 +474,6 @@ EOF sh ''' export PATH="${WORKSPACE}/bin:${PATH}" export KUBECONFIG=${kubeconfig_file} - export OCI_CLI_CONFIG_FILE=${WORKSPACE}/oci/config export OCI_CLI_PROFILE=${oci_profile} mkdir -m777 -p ${result_root}/kubelogs mkdir -m777 -p "${WORKSPACE}/logdir/${BUILD_TAG}/wl_k8s_test_results" @@ -465,6 +500,9 @@ EOF rm -rf ${WORKSPACE}/.mvn sudo rm -rf ${pv_root} sudo umount ${pv_root_parent} + sudo rm -rf ${pv_root_for_sampletest}/domains + sudo rm -rf ${pv_root_for_sampletest}/logs + sudo umount ${pv_root_for_sampletest} kubectl delete pod $(kubectl get pod |grep recycler-for|awk '{print $1}') || true ''' } diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index af32a853fcb..1ba4a509423 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -665,6 +665,13 @@ olcne-srg + + olcne-sequential + + false + olcne-sequential + + olcne-mrg diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainOnPVSample.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainOnPVSample.java index d90d61683e1..a4d05598503 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainOnPVSample.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainOnPVSample.java @@ -32,11 +32,14 @@ import static oracle.weblogic.kubernetes.TestConstants.DB_IMAGE_TAG; import static oracle.weblogic.kubernetes.TestConstants.FMWINFRA_IMAGE_TAG; import static oracle.weblogic.kubernetes.TestConstants.FMWINFRA_IMAGE_TO_USE_IN_SPEC; +import static oracle.weblogic.kubernetes.TestConstants.IMAGE_NAME_OPERATOR; import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST; import static oracle.weblogic.kubernetes.TestConstants.KIND_CLUSTER; import static oracle.weblogic.kubernetes.TestConstants.KIND_REPO; +import static oracle.weblogic.kubernetes.TestConstants.OCNE; import static oracle.weblogic.kubernetes.TestConstants.OKD; import static oracle.weblogic.kubernetes.TestConstants.RESULTS_ROOT; +import static oracle.weblogic.kubernetes.TestConstants.TEST_IMAGES_PREFIX; import static oracle.weblogic.kubernetes.TestConstants.TEST_IMAGES_REPO_SECRET_NAME; import static oracle.weblogic.kubernetes.TestConstants.TRAEFIK_INGRESS_IMAGE_NAME; import static oracle.weblogic.kubernetes.TestConstants.TRAEFIK_INGRESS_IMAGE_REGISTRY; @@ -65,6 +68,7 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @IntegrationTest @Tag("kind-sequential") +@Tag("olcne-sequential") @DisabledIfEnvironmentVariable(named = "SKIP_WLS_SAMPLES", matches = "true") class ItFmwDomainOnPVSample { @@ -130,6 +134,13 @@ public static void initAll(@Namespaces(4) List namespaces) { envMap.put("K8S_NODEPORT_HOST", K8S_NODEPORT_HOST); envMap.put("OKD", "" + OKD); envMap.put("KIND_CLUSTER", "" + KIND_CLUSTER); + envMap.put("OCNE", "" + OCNE); + + if (OCNE) { + envMap.put("OPER_IMAGE_NAME", TEST_IMAGES_PREFIX + IMAGE_NAME_OPERATOR); + envMap.put("DOMAIN_CREATION_IMAGE_NAME", TEST_IMAGES_PREFIX + DOMAIN_CREATION_IMAGE_NAME); + envMap.put("DB_IMAGE_PULL_SECRET", BASE_IMAGES_REPO_SECRET_NAME); + } // kind cluster uses openjdk which is not supported by image tool if (WIT_JAVA_HOME != null) { @@ -219,10 +230,15 @@ public void testInitialImage() { execTestScriptAndAssertSuccess("-initial-image", "Failed to run -initial-image"); // load the image to kind if using kind cluster + String imageCreated; if (KIND_REPO != null) { - String imageCreated = DOMAIN_CREATION_IMAGE_NAME + ":" + DOMAIN_CREATION_IMAGE_JRF_TAG; + imageCreated = DOMAIN_CREATION_IMAGE_NAME + ":" + DOMAIN_CREATION_IMAGE_JRF_TAG; logger.info("loading image {0} to kind", imageCreated); imagePush(imageCreated); + } else if (OCNE) { + imageCreated = TEST_IMAGES_PREFIX + DOMAIN_CREATION_IMAGE_NAME + ":" + DOMAIN_CREATION_IMAGE_JRF_TAG; + logger.info("pushing image {0} to repo", imageCreated); + imagePush(imageCreated); } } @@ -293,7 +309,7 @@ public static void tearDownAll() { if (traefikNamespace != null) { logger.info("Uninstall Traefik"); String command = - "helm uninstall " + TRAEFIK_RELEASE_NAME + "-" + traefikNamespace.substring(3) + " -n " + traefikNamespace; + "helm uninstall " + envMap.get("TRAEFIK_NAME") + " -n " + traefikNamespace; Command.withParams(new CommandParams() .command(command) .redirect(true)).execute(); diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java index 3c4746fcb39..94eb476905e 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java @@ -62,6 +62,8 @@ import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST; import static oracle.weblogic.kubernetes.TestConstants.KIND_REPO; import static oracle.weblogic.kubernetes.TestConstants.KUBERNETES_CLI; +import static oracle.weblogic.kubernetes.TestConstants.OCNE; +import static oracle.weblogic.kubernetes.TestConstants.TEST_IMAGES_PREFIX; import static oracle.weblogic.kubernetes.TestConstants.WEBLOGIC_IMAGE_NAME; import static oracle.weblogic.kubernetes.TestConstants.WEBLOGIC_IMAGE_NAME_DEFAULT; import static oracle.weblogic.kubernetes.TestConstants.WEBLOGIC_IMAGE_TO_USE_IN_SPEC; @@ -730,6 +732,9 @@ void testUpdateImageName() { String kindWlsImage = KIND_REPO + WEBLOGIC_IMAGE_NAME_DEFAULT; String testWlsImage = DOMAIN_IMAGES_PREFIX + WEBLOGIC_IMAGE_NAME_DEFAULT; + if (OCNE) { + testWlsImage = TEST_IMAGES_PREFIX + "weblogic"; + } //change image name to imageUpdate String imageTag = CommonTestUtils.getDateAndTimeStamp(); String imageUpdate = KIND_REPO != null @@ -1036,7 +1041,6 @@ private static void createDomain() { logger.info("Creating cluster resource {0} in namespace {1}",clusterResName, introDomainNamespace); createClusterAndVerify(cluster); - // create a domain custom resource configuration object logger.info("Creating domain custom resource"); DomainResource domain = new DomainResource() diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItWlsDomainOnPVSample.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItWlsDomainOnPVSample.java index 5c83e0a5253..2834c284df6 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItWlsDomainOnPVSample.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItWlsDomainOnPVSample.java @@ -28,11 +28,14 @@ import static oracle.weblogic.kubernetes.TestConstants.BASE_IMAGES_REPO_SECRET_NAME; import static oracle.weblogic.kubernetes.TestConstants.BUSYBOX_IMAGE; import static oracle.weblogic.kubernetes.TestConstants.BUSYBOX_TAG; +import static oracle.weblogic.kubernetes.TestConstants.IMAGE_NAME_OPERATOR; import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST; import static oracle.weblogic.kubernetes.TestConstants.KIND_CLUSTER; import static oracle.weblogic.kubernetes.TestConstants.KIND_REPO; +import static oracle.weblogic.kubernetes.TestConstants.OCNE; import static oracle.weblogic.kubernetes.TestConstants.OKD; import static oracle.weblogic.kubernetes.TestConstants.RESULTS_ROOT; +import static oracle.weblogic.kubernetes.TestConstants.TEST_IMAGES_PREFIX; import static oracle.weblogic.kubernetes.TestConstants.TEST_IMAGES_REPO_SECRET_NAME; import static oracle.weblogic.kubernetes.TestConstants.TRAEFIK_INGRESS_IMAGE_NAME; import static oracle.weblogic.kubernetes.TestConstants.TRAEFIK_INGRESS_IMAGE_REGISTRY; @@ -63,6 +66,7 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @IntegrationTest @Tag("kind-sequential") +@Tag("olcne-sequential") @DisabledIfEnvironmentVariable(named = "SKIP_WLS_SAMPLES", matches = "true") class ItWlsDomainOnPVSample { @@ -121,6 +125,12 @@ public static void initAll(@Namespaces(3) List namespaces) { envMap.put("K8S_NODEPORT_HOST", K8S_NODEPORT_HOST); envMap.put("OKD", "" + OKD); envMap.put("KIND_CLUSTER", "" + KIND_CLUSTER); + envMap.put("OCNE", "" + OCNE); + + if (OCNE) { + envMap.put("OPER_IMAGE_NAME", TEST_IMAGES_PREFIX + IMAGE_NAME_OPERATOR); + envMap.put("DOMAIN_CREATION_IMAGE_NAME", TEST_IMAGES_PREFIX + DOMAIN_CREATION_IMAGE_NAME); + } // kind cluster uses openjdk which is not supported by image tool if (WIT_JAVA_HOME != null) { @@ -180,10 +190,15 @@ public void testInitialImage() { execTestScriptAndAssertSuccess("-initial-image", "Failed to run -initial-image"); // load the image to kind if using kind cluster + String imageCreated; if (KIND_REPO != null) { - String imageCreated = DOMAIN_CREATION_IMAGE_NAME + ":" + DOMAIN_CREATION_IMAGE_WLS_TAG; + imageCreated = DOMAIN_CREATION_IMAGE_NAME + ":" + DOMAIN_CREATION_IMAGE_WLS_TAG; logger.info("loading image {0} to kind", imageCreated); imagePush(imageCreated); + } else if (OCNE) { + imageCreated = TEST_IMAGES_PREFIX + DOMAIN_CREATION_IMAGE_NAME + ":" + DOMAIN_CREATION_IMAGE_WLS_TAG; + logger.info("pushing image {0} to repo", imageCreated); + imagePush(imageCreated); } } @@ -250,8 +265,9 @@ public static void tearDownAll() { // uninstall traefik if (traefikNamespace != null) { logger.info("Uninstall Traefik"); + String command = "helm uninstall " + envMap.get("TRAEFIK_NAME") + " -n " + traefikNamespace; Command.withParams(new CommandParams() - .command("helm uninstall traefik-operator -n " + traefikNamespace) + .command(command) .redirect(true)).execute(); } diff --git a/operator/integration-tests/domain-on-pv/run-test.sh b/operator/integration-tests/domain-on-pv/run-test.sh index 85a920929e5..937e3a2dbbb 100755 --- a/operator/integration-tests/domain-on-pv/run-test.sh +++ b/operator/integration-tests/domain-on-pv/run-test.sh @@ -24,6 +24,7 @@ trace "Running end to end DPV sample test." echo "Is OKD set? $OKD" echo "Is OKE_CLUSTER set? $OKE_CLUSTER" echo "Is KIND_CLUSTER set? $KIND_CLUSTER" +echo "Is OCNE set? $OCNE" DRY_RUN=false DO_CLEANUP=false @@ -239,6 +240,7 @@ doCommand -c cp -r \$DPVSAMPLEDIR/../ingresses/* \$WORKDIR/ingresses doCommand -c export OKD=$OKD doCommand -c export OKE_CLUSTER=$OKE_CLUSTER doCommand -c export KIND_CLUSTER=$KIND_CLUSTER +doCommand -c export OCNE=$OCNE # # Build pre-req (operator) @@ -250,6 +252,9 @@ if [ "$DO_OPER" = "true" ]; then if [ "$KIND_CLUSTER" = "true" ]; then doCommand -c "kind load docker-image ${OPER_IMAGE_NAME:-weblogic-kubernetes-operator}:${OPER_IMAGE_TAG:-test} --name kind" fi + if [ "$OCNE" = "true" ]; then + doCommand -c "${WLSIMG_BUILDER:-docker} push ${OPER_IMAGE_NAME:-weblogic-kubernetes-operator}:${OPER_IMAGE_TAG:-test}" + fi fi # diff --git a/operator/integration-tests/domain-on-pv/test-env.sh b/operator/integration-tests/domain-on-pv/test-env.sh index d7b4b2cf938..2251618a9c8 100755 --- a/operator/integration-tests/domain-on-pv/test-env.sh +++ b/operator/integration-tests/domain-on-pv/test-env.sh @@ -24,8 +24,8 @@ export WORKDIR=${WORKDIR:-/tmp/$USER/domain-on-pv-sample-work-dir} # export OPER_NAME= # export OPER_NAMESPACE= # export OPER_SA= -# export OPER_IMAGE_NAME= -# export OPER_IMAGE_TAG= +export OPER_IMAGE_NAME=${OPER_IMAGE_NAME:-weblogic-kubernetes-operator} +export OPER_IMAGE_TAG=${OPER_IMAGE_TAG:-test} # ::: DB Settings (needed if WDT_DOMAIN_TYPE is JRF) export DB_NAMESPACE=${DB_NAMESPACE:-default} @@ -58,6 +58,7 @@ export POD_WAIT_TIMEOUT_SECS=${POD_WAIT_TIMEOUT_SECS:-1000} export OKD=${OKD:-false} export OKE_CLUSTER=${OKE_CLUSTER:-false} export KIND_CLUSTER=${KIND_CLUSTER:-false} +export OCNE=${OCNE:-false} # default Kubernetes CLI export KUBERNETES_CLI=${KUBERNETES_CLI:-kubectl} diff --git a/validateCLI.docker.dat b/validateCLI.docker.dat index 63a9d372e00..314eb267a18 100755 --- a/validateCLI.docker.dat +++ b/validateCLI.docker.dat @@ -49,4 +49,4 @@ wlsDockerContainer wls-docker-container-imag dii-docker-container DockerCluster -TRAEFIK_IMAGE_REGISTRY:-docker.io +:-docker.io