From ac27050f00c536814a730d955f36b479f67656c2 Mon Sep 17 00:00:00 2001 From: "MARINA.KOGAN" Date: Sat, 26 Oct 2024 12:52:33 +0000 Subject: [PATCH] Forward recent changes from release/4.2 to main --- Jenkinsfile.armoke | 122 +++++++---- Jenkinsfile.oke | 23 +- integration-tests/pom.xml | 4 +- .../ItConfigDistributionStrategy.java | 2 +- .../kubernetes/ItCrossDomainTransaction.java | 2 +- .../weblogic/kubernetes/ItDBOperator.java | 2 +- .../weblogic/kubernetes/ItDedicatedMode.java | 2 +- .../kubernetes/ItFmwDomainInPVUsingWDT.java | 2 +- .../kubernetes/ItFmwDomainInPVUsingWLST.java | 2 +- .../ItFmwDomainInPvUserCreateRcu.java | 2 +- .../weblogic/kubernetes/ItFmwDomainOnPV.java | 2 +- .../kubernetes/ItFmwDynamicDomainInPV.java | 26 ++- ...tHorizontalPodAutoscalerCustomMetrics.java | 43 +++- .../ItIstioCrossDomainTransaction.java | 1 - .../kubernetes/ItLBTwoDomainsNginx.java | 2 +- .../kubernetes/ItLBTwoDomainsTraefik.java | 2 +- .../ItLiftAndShiftFromOnPremDomain.java | 2 +- .../ItLivenessProbeCustomization.java | 2 +- .../weblogic/kubernetes/ItMiiDomain.java | 2 +- .../kubernetes/ItMiiUpdateDomainConfig.java | 2 +- .../ItMonitoringExporterMetricsFiltering.java | 2 +- .../ItMonitoringExporterSamples.java | 2 +- .../ItMonitoringExporterSideCar.java | 2 +- .../ItMonitoringExporterWebApp.java | 2 +- .../kubernetes/ItOperatorRestart.java | 2 +- .../weblogic/kubernetes/ItPodsRestart.java | 2 +- .../kubernetes/ItPodsShutdownOption.java | 2 +- .../weblogic/kubernetes/ItT3Channel.java | 2 +- .../ItTwoDomainsManagedByTwoOperators.java | 2 +- .../oracle/weblogic/kubernetes/ItWseeSSO.java | 19 +- .../impl/primitive/WebLogicImageTool.java | 5 + .../extensions/InitializationTasks.java | 2 +- .../kubernetes/utils/CommonLBTestUtils.java | 34 +++ .../kubernetes/utils/CommonTestUtils.java | 4 +- .../weblogic/kubernetes/utils/IstioUtils.java | 57 +++-- .../kubernetes/utils/LoadBalancerUtils.java | 127 ++++++++++-- .../kubernetes/utils/MonitoringUtils.java | 18 ++ .../oke/terraform/okemodule/oke.create.sh | 196 +++++++++++------- .../oke/terraform/okemodule/template.tfvars | 2 +- 39 files changed, 518 insertions(+), 211 deletions(-) diff --git a/Jenkinsfile.armoke b/Jenkinsfile.armoke index d50d309bbbf..1185d1fe820 100644 --- a/Jenkinsfile.armoke +++ b/Jenkinsfile.armoke @@ -1,14 +1,26 @@ -// Copyright (c) 2023, Oracle and/or its affiliates. +// Copyright (c) 2023,2024 Oracle and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. // +CRON_SETTINGS_MAIN = '''H 15 * * 6 % MAVEN_PROFILE_NAME=oke-arm;CLUSTER_NAME=seqarmwls12;PARALLEL_RUN=false + H 2 * * 5 % MAVEN_PROFILE_NAME=oke-arm;CLUSTER_NAME=seqarmwls14;PARALLEL_RUN=false;WEBLOGIC_IMAGE_TAG=14.1.1.0-generic-jdk8-ol8-241009.121334-arm64''' + +CRON_SETTINGS_42 = '''H 5 * * 6 % MAVEN_PROFILE_NAME=oke-arm;CLUSTER_NAME=seq42armwls12;PARALLEL_RUN=false + H 21 * * 5 % MAVEN_PROFILE_NAME=oke-arm;CLUSTER_NAME=seq42armwls14;PARALLEL_RUN=false;WEBLOGIC_IMAGE_TAG=14.1.1.0-generic-jdk8-ol8-241009.121334-arm64''' + +CRON_SETTINGS = "${env.JOB_NAME == 'wko-oke-arm' ? CRON_SETTINGS_42 : CRON_SETTINGS_MAIN}" + pipeline { - agent { label 'large-arm' } + agent { label 'large-arm-iad' } options { timeout(time: 1800, unit: 'MINUTES') - disableConcurrentBuilds() + //disableConcurrentBuilds() + } + triggers { + // timer trigger for "nightly build" + parameterizedCron(env.JOB_NAME == 'wko-oke-dev' ? + '' : CRON_SETTINGS) } - tools { maven 'maven-3.8.7' jdk 'jdk21-arm' @@ -22,6 +34,7 @@ pipeline { compartment_id = "${env.WKT_TEST_COMPARTMENT_ID}" ocir_creds = 'wkt-ocir-creds' wko_files_bucket_token = 'wko-system-test-files-bucket-par-token' + wko_region_bucket = "us-phoenix-1" oci_profile = 'DEFAULT' jenkins_user = sh(returnStdout: true, script: "id -nu").trim() @@ -50,7 +63,6 @@ pipeline { start_time = sh(script: 'date +"%Y-%m-%d %H:%M:%S"', returnStdout: true).trim() wle_download_url="https://github.com/oracle/weblogic-logging-exporter/releases/latest" - CLUSTER_NAME = "${env.JOB_NAME == 'wko-oke-nightly-parallel' ? 'parcluster' : 'armcluster'}" kubeconfig_file = "${WORKSPACE}/terraform/${CLUSTER_NAME}_kubeconfig" MAVEN_PROFILE_NAME = "oke-arm" availability_domain = "mFEn:PHX-AD-1" @@ -71,18 +83,21 @@ pipeline { string(name: 'OKE_KUBE_VERSION', description: 'kube version for oke cluster', - defaultValue: '1.27.2' + defaultValue: '1.29.1' ) string(name: 'IMAGE_ID', - description: 'oci image id for node pool, find image OCID for your region from https://docs.oracle.com/iaas/images/', - //defaultValue OKE1.26.2: 'ocid1.image.oc1.phx.aaaaaaaaaizmtmozeudeeuq7o5ir7dkl2bkxbbb3tgomshqbqn6jpomrsjza' - defaultValue: 'ocid1.image.oc1.phx.aaaaaaaa77h3zkjcfjoaou4aa3cz63fswtcglleuslt6d6zbqwacng43nqiq' - + description: 'oci image id for node pool, find image OCID for your region from https://docs.oracle.com/iaas/images/', + //defaultValue OKE1.26.2: 'ocid1.image.oc1.phx.aaaaaaaaaizmtmozeudeeuq7o5ir7dkl2bkxbbb3tgomshqbqn6jpomrsjza' + defaultValue: 'ocid1.image.oc1.phx.aaaaaaaap5s265eoou3d65elvd22ihdi3dnrm2hn6bbewjjsluvw5jc32zia' ) string(name: 'KUBECTL_VERSION', description: 'kubectl version', defaultValue: '1.26.2' ) + string(name: 'CLUSTER_NAME', + description: 'OKE cluster name', + defaultValue: "seqarmone" + string(name: 'HELM_VERSION', description: 'Helm version', defaultValue: '3.11.2' @@ -135,11 +150,12 @@ pipeline { ) string(name: 'WEBLOGIC_IMAGE_NAME', description: 'WebLogic base image name. Default is the image name in OCIR. Use middleware/weblogic for OCR.', - defaultValue: 'test-images/weblogic_cpu' + defaultValue: 'weblogic_arm64' ) string(name: 'WEBLOGIC_IMAGE_TAG', description: '12.2.1.4, 12.2.1.4-dev(12.2.1.4-dev-ol7) , 12.2.1.4-slim(12.2.1.4-slim-ol7), 12.2.1.4-ol8, 12.2.1.4-dev-ol8, 12.2.1.4-slim-ol8, 14.1.1.0-11-ol7, 14.1.1.0-dev-11-ol7, 14.1.1.0-slim-11-ol7, 14.1.1.0-8-ol7, 14.1.1.0-dev-8-ol7, 14.1.1.0-slim-8-ol7, 14.1.1.0-11-ol8, 14.1.1.0-dev-11-ol8, 14.1.1.0-slim-11-ol8, 14.1.1.0-8-ol8, 14.1.1.0-dev-8-ol8, 14.1.1.0-slim-8-ol8', - defaultValue: '12.2.1.4-generic-jdk8-ol7' + defaultValue: '12.2.1.4-generic-jdk8-ol8-241008.161513-arm64' + //defaultValue: '14.1.1.0-generic-jdk8-ol8-241009.121334-arm64' ) string(name: 'FMWINFRA_IMAGE_NAME', description: 'FWM Infra image name. Default is the image name in OCIR. Use middleware/fmw-infrastructure for OCR.', @@ -177,6 +193,14 @@ pipeline { description: 'Collect logs for successful runs. Default is false.', defaultValue: false ) + string(name: 'MOUNT_TARGET_OCID', + description: 'one mount target for all runs', + defaultValue: "ocid1.mounttarget.oc1.phx.aaaaaby27vhqpci5obuhqllqojxwiotqnb4c2ylefuzqaaaa" + ) + string(name: 'REMOTECONSOLE_VERSION', + description: 'RemoteConsole version.', + defaultValue: '2.4.7' + ) } @@ -236,7 +260,7 @@ pipeline { export PATH=${runtime_path} oci os object get --namespace=${wko_tenancy} --bucket-name=wko-system-test-files \ --name=helm/helm-v${HELM_VERSION}-linux-arm.tar.gz --file=helm.tar.gz \ - --auth=instance_principal + --auth=instance_principal --region=us-phoenix-1 tar zxf helm.tar.gz mv linux-arm/helm ${WORKSPACE}/bin/helm rm -rf linux-arm @@ -254,7 +278,7 @@ pipeline { export PATH=${runtime_path} oci os object get --namespace=${wko_tenancy} --bucket-name=wko-system-test-files \ --name=kubectl/kubectl-v${KUBECTL_VERSION}-arm --file=${WORKSPACE}/bin/kubectl \ - --auth=instance_principal + --auth=instance_principal --region=us-phoenix-1 chmod +x ${WORKSPACE}/bin/kubectl ls -al ${WORKSPACE}/bin/kubectl kubectl version --client=true @@ -297,6 +321,7 @@ pipeline { private_subnet_ocid=$(oci network subnet list --compartment-id=${wkt_compartment_ocid} | jq '.data[] | select(."display-name" == "Private-Subnet-wktiso1")' | jq -r ."id") export WKT_TENANCY_OCID=${TENANCY_OCID} export WKT_USER_OCID=${USER_OCID} + rm -rf ${WORKSPACE}/terraform mkdir -p ${WORKSPACE}/terraform mkdir -p ${WORKSPACE}/oci cat ${jenkins_home_directory}/.oci/config @@ -325,7 +350,7 @@ pipeline { #export IMAGE_TAG_WEBLOGIC="12.2.1.4" #export IMAGE_TAG_FMWINFRA="12.2.1.4" ssh_pubkey=`cat ${wkotest_ssh_pubcert}` - + ssh_pk=`cat ${wkotest_ssh_pk}` ################# echo "Generating property file oci.prop for terraform scripts" @@ -334,6 +359,7 @@ cat > $OCI_PROP_FILE </dev/null | grep -o 'declare -a')" == "declare -a" ]; then + # Select first + mt_private_ip=`oci network private-ip get --private-ip-id "${mt_privateip_id[0]}" | jq -r '.data | ."ip-address"'` + else + mt_private_ip=`oci network private-ip get --private-ip-id "${mt_privateip_id}" | jq -r '.data | ."ip-address"'` + fi if [ -z "${mt_private_ip}" ]; then echo "Mount Target was not setup properly , clean up Kubernetes cluster" - sh ${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform + sh ${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform ${AVAILABILITY_DOMAIN} exit -1 fi export NFS_SERVER=$mt_private_ip @@ -577,7 +616,10 @@ EOF echo "-Dwko.it.prometheus.chart.version=\"${PROMETHEUS_CHART_VERSION}\"" >> ${WORKSPACE}/.mvn/maven.config echo "-Dwko.it.grafana.chart.version=\"${GRAFANA_CHART_VERSION}\"" >> ${WORKSPACE}/.mvn/maven.config echo "-Dwko.it.collect.logs.on.success=\"${COLLECT_LOGS_ON_SUCCESS}\"" >> ${WORKSPACE}/.mvn/maven.config - echo "-DOPERATOR_LOG_LEVEL=\"${OPERATOR_LOG_LEVEL}\"" >> ${WORKSPACE}/.mvn/maven.config + echo "-Dwko.it.oci.compartment.ocid=\"${COMPARTMENT_OCID}\"" >> ${WORKSPACE}/.mvn/maven.config + echo "-Dwko.it.remoteconsole.version=\"${REMOTECONSOLE_VERSION}\"" >> ${WORKSPACE}/.mvn/maven.config + echo "-Djdk.httpclient.allowRestrictedHeaders=\"host\"" >> ${WORKSPACE}/.mvn/maven.config + echo "-DOPERATOR_LOG_LEVEL=\"${OPERATOR_LOG_LEVEL}\"" >> ${WORKSPACE}/.mvn/maven.config echo "${WORKSPACE}/.mvn/maven.config contents:" cat "${WORKSPACE}/.mvn/maven.config" @@ -603,7 +645,8 @@ EOF export TEST_IMAGES_REPO_EMAIL="noreply@oracle.com" export DOCKER_BUILD_KIT=1 export DOCKER_CLI_EXPERIMENTAL=enabled - + export no_proxy="${NO_PROXY},localhost,127.0.0.1,.us.oracle.com,.oraclecorp.com,login.oracle.com" + export NO_PROXY="${no_proxy}" if ! mvn -pl integration-tests -P ${MAVEN_PROFILE_NAME} verify 2>&1 | tee "${result_root}/oketest.log"; then echo "integration-tests failed" exit 1 @@ -619,10 +662,11 @@ EOF export KUBECONFIG=${kubeconfig_file} export OCI_CLI_CONFIG_FILE=${WORKSPACE}/oci/config export OCI_CLI_PROFILE=${oci_profile} + export AVAILABILITY_DOMAIN=${availability_domain} mkdir -m777 -p ${result_root}/kubelogs mkdir -m777 -p "${WORKSPACE}/logdir/${BUILD_TAG}/wl_k8s_test_results" sudo mv -f ${result_root}/* "${WORKSPACE}/logdir/${BUILD_TAG}/wl_k8s_test_results" - ${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform + ${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform ${AVAILABILITY_DOMAIN} ''' @@ -638,16 +682,24 @@ EOF sh ''' export PATH="${WORKSPACE}/bin:${PATH}" export KUBECONFIG=${kubeconfig_file} - + export AVAILABILITY_DOMAIN=${availability_domain} echo 'Remove old OKE cluster (if any)...' if [ -f "$OCI_PROP_FILE" ] && [ -f "${WORKSPACE}/terraform/oke.delete.sh" ]; then - ${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform + ${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform ${AVAILABILITY_DOMAIN} + fi + + if [ "${MAVEN_PROFILE_NAME}" = "oke-sequential" ] && [ "${BRANCH}" = "release/4.2" ]; then + compname="wkt" + wkt_compartment_ocid=$(oci iam compartment list --compartment-id-in-subtree true --all | jq --arg compname "$compname" '.data[] | select(."name"==$compname)' | jq -r ."id") + sec_list_id=$(oci network security-list list --compartment-id="$wkt_compartment_ocid" --display-name=Security-List-wktiso1 | jq -r '.data[] | ."id"') + oci network security-list update --security-list-id="$sec_list_id" --ingress-security-rules='[{"description": "east west","icmp-options": null,"is-stateless": false,"protocol": "all","source": "10.196.0.0/16","source-type": "CIDR_BLOCK","tcp-options": null,"udp-options": null},{"description": null,"icmp-options": {"code": null,"type": 3},"is-stateless": false,"protocol": "1", "source": "10.196.0.0/16", "source-type": "CIDR_BLOCK","tcp-options": null, "udp-options": null}, {"description": null, "icmp-options": {"code": 4, "type": 3},"is-stateless": false, "protocol": "1", "source": "0.0.0.0/0","source-type": "CIDR_BLOCK", "tcp-options": null, "udp-options": null}, {"description": null, "icmp-options": null, "is-stateless": false, "protocol": "6","source": "0.0.0.0/0", "source-type": "CIDR_BLOCK", "tcp-options": null, "udp-options": null}, {"description": null, "icmp-options": null, "is-stateless": false,"protocol": "6", "source": "0.0.0.0/0", "source-type": "CIDR_BLOCK", "tcp-options": {"destination-port-range": {"max": 22, "min": 22}, "source-port-range": null}, "udp-options": null}]' --egress-security-rules='[{"description": null, "destination": "0.0.0.0/0", "destinationType": "CIDR_BLOCK", "icmpOptions": null, "isStateless": false, "protocol": "all", "tcpOptions": null, "udpOptions": null}, {"description": null, "destination": "oci-phx-objectstorage", "destinationType": "SERVICE_CIDR_BLOCK", "icmpOptions": null, "isStateless": false, "protocol": "6", "tcpOptions": null, "udpOptions": null}]' --force fi rm -f ${jenkins_home_directory}/.oci/config rm -f ${jenkins_home_directory}/.oci/oci-signing-key.pem rm -f ${WORKSPACE}/.ssh/* rm -rf ${WORKSPACE}/.mvn + rm -rf ${WORKSPACE}/* ''' } diff --git a/Jenkinsfile.oke b/Jenkinsfile.oke index 86d0d5e63c8..63cdc5c8e36 100644 --- a/Jenkinsfile.oke +++ b/Jenkinsfile.oke @@ -2,9 +2,12 @@ // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. // -CRON_SETTINGS = '''H 3 * * 0-4 % MAVEN_PROFILE_NAME=oke-gate;CLUSTER_NAME=seqone;PARALLEL_RUN=false - H 1 * * 0-4 % MAVEN_PROFILE_NAME=oke-sequential;CLUSTER_NAME=seqtwo;PARALLEL_RUN=false - H 2 * * 0-4 % MAVEN_PROFILE_NAME=oke-parallel;CLUSTER_NAME=parone;PARALLEL_RUN=true''' +CRON_SETTINGS = '''H 3 * * 6 % MAVEN_PROFILE_NAME=oke-weekly-sequential;CLUSTER_NAME=seqweek;PARALLEL_RUN=false + H 18 * * 0-4 % MAVEN_PROFILE_NAME=oke-sequential;CLUSTER_NAME=seqone;PARALLEL_RUN=false + H 12 * * 0-4 % MAVEN_PROFILE_NAME=oke-parallel;CLUSTER_NAME=parone;PARALLEL_RUN=true''' + + + pipeline { agent { label 'large' } @@ -83,18 +86,18 @@ pipeline { string(name: 'OKE_KUBE_VERSION', description: 'kube version for oke cluster', - defaultValue: '1.29.1' + defaultValue: '1.30.1' ) string(name: 'IMAGE_ID', description: 'oci image id for node pool, find image OCID for your region from https://docs.oracle.com/iaas/images/', //defaultValue OKE1.26.2: 'ocid1.image.oc1.phx.aaaaaaaaaizmtmozeudeeuq7o5ir7dkl2bkxbbb3tgomshqbqn6jpomrsjza' //1.27.2 oke defaultValue: 'ocid1.image.oc1.phx.aaaaaaaaypr5r5drojwytghw6e6mvpjsscrnkuwtmqlmvmix7kjb2zcnc7wa' - defaultValue: 'ocid1.image.oc1.phx.aaaaaaaa22u45gr3ikxnc6rius2qil2kz5k3e7p476c4usr6qnvql4l5dxea' + defaultValue: 'ocid1.image.oc1.phx.aaaaaaaahgrs3zcwrvutjtni557ttrt62uggseijsmqxacr7dym423uaokcq' ) string(name: 'KUBECTL_VERSION', description: 'kubectl version', - defaultValue: '1.26.2' + defaultValue: '1.30.0' ) string(name: 'HELM_VERSION', description: 'Helm version', @@ -115,9 +118,9 @@ pipeline { ) choice(name: 'MAVEN_PROFILE_NAME', - description: 'Profile to use in mvn command to run the tests. Possible values are oke-gate,oke-parallel, oke-sequential. Refer to weblogic-kubernetes-operator/integration-tests/pom.xml on the branch.', + description: 'Profile to use in mvn command to run the tests. Possible values are oke-weekly-sequential,oke-parallel, oke-sequential. Refer to weblogic-kubernetes-operator/integration-tests/pom.xml on the branch.', choices: [ - 'oke-gate', + 'oke-weekly-sequential', 'oke-sequential', 'oke-parallel' ] @@ -144,7 +147,7 @@ pipeline { ) string(name: 'NODE_SHAPE', description: '', - defaultValue: "VM.Standard.E4.Flex" + defaultValue: "VM.Standard.E5.Flex" ) string(name: 'MOUNT_TARGET_OCID', description: 'only for debug runs on wko-oke-dev', @@ -683,7 +686,7 @@ EOF ${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform ${AVAILABILITY_DOMAIN} fi - if [ "${MAVEN_PROFILE_NAME}" = "oke-gate" ] && [ "${BRANCH}" = "release/4.2" ]; then + if [ "${MAVEN_PROFILE_NAME}" = "oke-sequential" ] && [ "${BRANCH}" = "release/4.2" ]; then compname="wkt" wkt_compartment_ocid=$(oci iam compartment list --compartment-id-in-subtree true --all | jq --arg compname "$compname" '.data[] | select(."name"==$compname)' | jq -r ."id") sec_list_id=$(oci network security-list list --compartment-id="$wkt_compartment_ocid" --display-name=Security-List-wktiso1 | jq -r '.data[] | ."id"') diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index a36e854f998..640828cd73d 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -469,10 +469,10 @@ - oke-gate + oke-weekly-sequential false - oke-gate + oke-weekly-sequential diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItConfigDistributionStrategy.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItConfigDistributionStrategy.java index d6f94d50f47..42b188c6e25 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItConfigDistributionStrategy.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItConfigDistributionStrategy.java @@ -133,7 +133,7 @@ @DisplayName("Verify the overrideDistributionStrategy applies the overrides accordingly to the value set") @Tag("kind-parallel") @Tag("okd-wls-mrg") -@Tag("oke-parallel") +@Tag("oke-weekly-sequential") @IntegrationTest @Tag("olcne-mrg") class ItConfigDistributionStrategy { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItCrossDomainTransaction.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItCrossDomainTransaction.java index bd51c32070b..5df6fe33755 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItCrossDomainTransaction.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItCrossDomainTransaction.java @@ -107,7 +107,7 @@ */ @DisplayName("Verify cross domain transaction is successful") @IntegrationTest -@Tag("oke-sequential") +@Tag("oke-weekly-sequential") @Tag("kind-parallel") @Tag("okd-wls-srg") class ItCrossDomainTransaction { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItDBOperator.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItDBOperator.java index be6919822ac..15007e7109b 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItDBOperator.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItDBOperator.java @@ -109,7 +109,7 @@ @DisplayName("Test to create FMW model in image domain and WebLogic domain using Oracle " + "database created using Oracle Database Operator") @IntegrationTest -@Tag("oke-gate") +@Tag("oke-weekly-sequential") @Tag("kind-parallel") class ItDBOperator { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItDedicatedMode.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItDedicatedMode.java index 1171ed6af6f..79029dac9fb 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItDedicatedMode.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItDedicatedMode.java @@ -68,7 +68,7 @@ */ @DisplayName("Test Operator and WebLogic domain with Dedicated set to true") @Tag("kind-sequential") -@Tag("oke-gate") +@Tag("oke-weekly-sequential") @Tag("okd-wls-mrg") @IntegrationTest class ItDedicatedMode { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWDT.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWDT.java index fc1d6b5e1e7..e8c50217700 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWDT.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWDT.java @@ -59,7 +59,7 @@ */ @DisplayName("Test to creat a FMW domain in persistent volume using WDT") @IntegrationTest -@Tag("oke-gate") +@Tag("oke-weekly-sequential") @Tag("kind-sequential") @Tag("okd-fmw-cert") class ItFmwDomainInPVUsingWDT { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWLST.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWLST.java index 88e8adc9b11..4123c67e669 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWLST.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWLST.java @@ -72,7 +72,7 @@ @Tag("okd-fmw-cert") @IntegrationTest @Tag("olcne-sequential") -@Tag("oke-gate") +@Tag("oke-weekly-sequential") class ItFmwDomainInPVUsingWLST { private static String dbNamespace = null; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPvUserCreateRcu.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPvUserCreateRcu.java index 320aa16de6a..27e91f13827 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPvUserCreateRcu.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPvUserCreateRcu.java @@ -98,7 +98,7 @@ @DisplayName("Test for initializeDomainOnPV when user per-creates RCU") @IntegrationTest @Tag("kind-sequential") -@Tag("oke-gate") +@Tag("oke-weekly-sequential") public class ItFmwDomainInPvUserCreateRcu { private static String opNamespace = null; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainOnPV.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainOnPV.java index c4775cb1f97..9f1ff9afe47 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainOnPV.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainOnPV.java @@ -93,7 +93,7 @@ @DisplayName("Test to create a FMW domain in persistent volume with new simplified feature") @IntegrationTest @Tag("kind-sequential") -@Tag("oke-gate") +@Tag("oke-weekly-sequential") @Tag("okd-fmw-cert") @Tag("olcne-sequential") class ItFmwDomainOnPV { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java index c5e405ebb89..d13cfd89263 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java @@ -78,7 +78,7 @@ */ @DisplayName("Test to creat a FMW dynamic domain in persistent volume using WLST") @IntegrationTest -@Tag("oke-sequential") +@Tag("oke-weekly-sequential") @Tag("kind-sequential") @Tag("okd-fmw-cert") class ItFmwDynamicDomainInPV { @@ -86,12 +86,12 @@ class ItFmwDynamicDomainInPV { private static String opNamespace = null; private static String domainNamespace = null; private static String dbNamespace = null; - - private static String nginxNamespace = null; private static String oracle_home = null; private static String java_home = null; private static final String RCUSCHEMAPREFIX = "fmwdomainpv"; + private static final String ORACLEDBURLPREFIX = "oracledb."; + private static String ORACLEDBSUFFIX = null; private static final String RCUSYSUSERNAME = "sys"; private static final String RCUSYSPASSWORD = "Oradoc_db1"; private static final String RCUSCHEMAUSERNAME = "myrcuuser"; @@ -105,8 +105,8 @@ class ItFmwDynamicDomainInPV { private static final String adminServerName = "admin-server"; private static final String managedServerNameBase = "managed-server"; private static final String adminServerPodName = domainUid + "-" + adminServerName; - private static final int managedServerPort = 8001; - private static List ingressHostList = null; + private static final String managedServerPodNamePrefix = domainUid + "-" + managedServerNameBase; + private final int managedServerPort = 8001; private final String wlSecretName = domainUid + "-weblogic-credentials"; private final String rcuSecretName = domainUid + "-rcu-credentials"; private static final int replicaCount = 1; @@ -114,9 +114,14 @@ class ItFmwDynamicDomainInPV { private static String hostHeader; private static int adminPort = 7001; private static String dbName = domainUid + "my-oracle-db"; + private static String nginxNamespace = null; + private static List ingressHostList = null; private static String ingressIP = null; private static NginxParams nginxHelmParams = null; + + + /** * Assigns unique namespaces for DB, operator and domains. * Start DB service and create RCU schema. @@ -175,13 +180,6 @@ void testFmwDynamicDomainInPV() { // create FMW dynamic domain and verify createFmwDomainAndVerify(); verifyDomainReady(domainNamespace, domainUid, replicaCount, "nosuffix"); - if (OKE_CLUSTER_PRIVATEIP) { - Map clusterNameMsPortMap = new HashMap<>(); - clusterNameMsPortMap.put(clusterName, managedServerPort); - ingressHostList - = createIngressForDomainAndVerify(domainUid, domainNamespace, 0, clusterNameMsPortMap, - false, nginxHelmParams.getIngressClassName(), true, 7001); - } // Expose the admin service external node port as a route for OKD adminSvcExtHost = createRouteForOKD(getExternalServicePodName(adminServerPodName), domainNamespace); if (OKE_CLUSTER_PRIVATEIP) { @@ -363,12 +361,12 @@ private File createWlstPropertyFile(int t3ChannelPort) { // create a temporary WebLogic domain property file File domainPropertiesFile = assertDoesNotThrow(() -> - File.createTempFile("domain", ".properties", new File(RESULTS_TEMPFILE)), + File.createTempFile("domain", ".properties", new File(RESULTS_TEMPFILE)), "Failed to create domain properties file"); // create the property file assertDoesNotThrow(() -> - p.store(new FileOutputStream(domainPropertiesFile), "FMW wlst properties file"), + p.store(new FileOutputStream(domainPropertiesFile), "FMW wlst properties file"), "Failed to write domain properties file"); return domainPropertiesFile; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItHorizontalPodAutoscalerCustomMetrics.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItHorizontalPodAutoscalerCustomMetrics.java index 59b9b32b505..d10a4e96589 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItHorizontalPodAutoscalerCustomMetrics.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItHorizontalPodAutoscalerCustomMetrics.java @@ -100,6 +100,7 @@ import static oracle.weblogic.kubernetes.utils.MonitoringUtils.installMonitoringExporter; import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator; import static oracle.weblogic.kubernetes.utils.PersistentVolumeUtils.createPvAndPvc; +import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodReady; import static oracle.weblogic.kubernetes.utils.PodUtils.isPodDeleted; import static oracle.weblogic.kubernetes.utils.SecretUtils.createSecretWithUsernamePassword; import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; @@ -112,7 +113,7 @@ @DisplayName("Test to a create MII domain and test autoscaling using HPA and" + "custom metrics provided via use of monitoring exporter and prometheus and prometheus adapter") @IntegrationTest -@Tag("oke-gate") +@Tag("oke-sequential") @Tag("kind-parallel") public class ItHorizontalPodAutoscalerCustomMetrics { private static final String MONEXP_MODEL_FILE = "model.monexp.custommetrics.yaml"; @@ -224,6 +225,7 @@ public static void initAll(@Namespaces(4) List namespaces) { // install and verify NGINX nginxHelmParams = installAndVerifyNginx(nginxNamespace, IT_HPACUSTOMNGINX_INGRESS_HTTP_NODEPORT, IT_HPACUSTOMNGINX_INGRESS_HTTPS_NODEPORT, NGINX_CHART_VERSION, (OKE_CLUSTER ? null : "NodePort")); + String nginxServiceName = nginxHelmParams.getHelmParams().getReleaseName() + "-ingress-nginx-controller"; logger.info("NGINX service name: {0}", nginxServiceName); nodeportshttp = getServiceNodePort(nginxNamespace, nginxServiceName, "http"); @@ -317,15 +319,46 @@ void testHPAWithCustomMetrics() { hostPort = host + ":" + nodeportshttp; } String curlCmd = - String.format("curl --silent --show-error --noproxy '*' -H 'host: %s' http://%s:%s@%s/" + SESSMIGT_APP_URL, + String.format("curl -g --silent --show-error -v --noproxy '*' -H 'host: %s' http://%s:%s@%s/" + SESSMIGT_APP_URL, ingressHostList.get(0), ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT, hostPort); logger.info("Executing curl command " + curlCmd); + assertDoesNotThrow(() -> { + ExecResult result = ExecCommand.exec(curlCmd, true); + String response = result.stdout().trim(); + getLogger().info("exitCode: {0}, \nstdout: {1}, \nstderr: {2}", + result.exitValue(), response, result.stderr()); + if (!response.contains("cluster-1-managed")) { + logger.info("Can't invoke application"); + + if (OKE_CLUSTER) { + LoggingFacade logger = getLogger(); + try { + + result = ExecCommand.exec(KUBERNETES_CLI + " get all -A"); + logger.info(result.stdout()); + //restart core-dns service + result = ExecCommand.exec(KUBERNETES_CLI + " rollout restart deployment coredns -n kube-system"); + logger.info(result.stdout()); + checkPodReady("coredns", null, "kube-system"); + + } catch (Exception ex) { + logger.warning(ex.getLocalizedMessage()); + } + } + } + }); for (int i = 0; i < 50; i++) { - assertDoesNotThrow(() -> ExecCommand.exec(curlCmd)); + assertDoesNotThrow(() -> { + ExecResult result = ExecCommand.exec(curlCmd, true); + String response = result.stdout().trim(); + getLogger().info("exitCode: {0}, \nstdout: {1}, \nstderr: {2}", + result.exitValue(), response, result.stderr()); + assertTrue(response.contains("cluster-1-managed"), "Can't invoke application"); + }); } //check hpa scaled up to one more server checkPodReadyAndServiceExists(managedServerPrefix + 3, domainUid, domainNamespace); @@ -452,8 +485,8 @@ private boolean verifyCustomMetricsExposed(String namespace, String customMetric } private void installPrometheus(String promChartVersion, - String domainNS, - String domainUid + String domainNS, + String domainUid ) throws IOException, ApiException { final String prometheusRegexValue = String.format("regex: %s;%s", domainNS, domainUid); if (promHelmParams == null) { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIstioCrossDomainTransaction.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIstioCrossDomainTransaction.java index 711a58933ae..d8eb21c45ab 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIstioCrossDomainTransaction.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIstioCrossDomainTransaction.java @@ -88,7 +88,6 @@ @DisplayName("Verify cross domain transaction with istio enabled is successful") @IntegrationTest @Tag("kind-parallel") -@Tag("oke-arm") @Tag("oke-parallel") class ItIstioCrossDomainTransaction { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLBTwoDomainsNginx.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLBTwoDomainsNginx.java index b1abd1e88ff..9c860f5f12b 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLBTwoDomainsNginx.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLBTwoDomainsNginx.java @@ -78,7 +78,7 @@ @IntegrationTest @Tag("olcne-mrg") @Tag("kind-parallel") -@Tag("oke-gate") +@Tag("oke-sequential") class ItLBTwoDomainsNginx { private static final int numberOfDomains = 2; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLBTwoDomainsTraefik.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLBTwoDomainsTraefik.java index b47bf1916dd..de3a08afd16 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLBTwoDomainsTraefik.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLBTwoDomainsTraefik.java @@ -62,7 +62,7 @@ @IntegrationTest @Tag("olcne-mrg") @Tag("kind-parallel") -@Tag("oke-gate") +@Tag("oke-sequential") class ItLBTwoDomainsTraefik { private static final int numberOfDomains = 2; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLiftAndShiftFromOnPremDomain.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLiftAndShiftFromOnPremDomain.java index bb604a917a2..05073799316 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLiftAndShiftFromOnPremDomain.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLiftAndShiftFromOnPremDomain.java @@ -105,7 +105,7 @@ @Tag("toolkits-srg") @Tag("okd-wls-mrg") @Tag("olcne-mrg") -@Tag("oke-gate") +@Tag("oke-sequential") @IntegrationTest class ItLiftAndShiftFromOnPremDomain { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLivenessProbeCustomization.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLivenessProbeCustomization.java index ba3877ef289..5b06cad8f46 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLivenessProbeCustomization.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLivenessProbeCustomization.java @@ -86,7 +86,7 @@ @Tag("kind-parallel") @Tag("okd-wls-mrg") @Tag("oke-arm") -@Tag("oke-gate") +@Tag("oke-weekly-sequential") class ItLivenessProbeCustomization { // domain constants diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDomain.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDomain.java index 561dbccbff1..769e538a233 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDomain.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDomain.java @@ -137,7 +137,7 @@ @Tag("olcne-mrg") @Tag("kind-parallel") @Tag("okd-wls-srg") -@Tag("oke-gate") +@Tag("oke-weekly-sequential") class ItMiiDomain { private static String opNamespace = null; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiUpdateDomainConfig.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiUpdateDomainConfig.java index 1d12ea40fee..a65dc22844a 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiUpdateDomainConfig.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiUpdateDomainConfig.java @@ -119,7 +119,7 @@ @Tag("kind-parallel") @Tag("toolkits-srg") @Tag("okd-wls-srg") -@Tag("oke-sequential") +@Tag("oke-weekly-sequential") class ItMiiUpdateDomainConfig { private static String opNamespace = null; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterMetricsFiltering.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterMetricsFiltering.java index 5cc56529586..9b665fbaec4 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterMetricsFiltering.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterMetricsFiltering.java @@ -90,7 +90,7 @@ @DisplayName("Verify WebLogic Metric is processed and filtered as expected by MonitoringExporter") @IntegrationTest @Tag("olcne-mrg") -@Tag("oke-gate") +@Tag("oke-sequential") @Tag("kind-sequential") @Tag("okd-wls-mrg") class ItMonitoringExporterMetricsFiltering { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterSamples.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterSamples.java index 7bed8798952..41d0ee554b6 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterSamples.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterSamples.java @@ -135,7 +135,7 @@ * Verify WebLogic metrics can be accessed via Prometheus */ @DisplayName("Verify end to end sample, provided in the Monitoring Exporter github project") -@Tag("oke-gate") +@Tag("oke-weekly-sequential") @Tag("kind-parallel") @Tag("okd-wls-mrg") @IntegrationTest diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterSideCar.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterSideCar.java index 64bb8efeed0..f3a6b2a4c1f 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterSideCar.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterSideCar.java @@ -101,7 +101,7 @@ + "MonitoringExporter Side Car via Prometheus and Grafana") @IntegrationTest @Tag("olcne-mrg") -@Tag("oke-gate") +@Tag("oke-sequential") @Tag("kind-parallel") @Tag("okd-wls-mrg") class ItMonitoringExporterSideCar { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterWebApp.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterWebApp.java index baee73c94c9..4eac6a553c7 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterWebApp.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterWebApp.java @@ -107,7 +107,7 @@ @DisplayName("Verify WebLogic Metric is processed as expected by MonitoringExporter WebApp via Prometheus and Grafana") @IntegrationTest @Tag("olcne-mrg") -@Tag("oke-gate") +@Tag("oke-weekly-sequential") @Tag("kind-sequential") @Tag("okd-wls-mrg") @Tag("gate") diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOperatorRestart.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOperatorRestart.java index d28e73090cc..ca37399e0fb 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOperatorRestart.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOperatorRestart.java @@ -59,7 +59,7 @@ @Tag("olcne-mrg") @Tag("kind-parallel") @Tag("okd-wls-mrg") -@Tag("oke-gate") +@Tag("oke-weekly-sequential") @Tag("oke-arm") class ItOperatorRestart { private static String opNamespace = null; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItPodsRestart.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItPodsRestart.java index 24c9bf57e81..fef6e92c99b 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItPodsRestart.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItPodsRestart.java @@ -91,7 +91,7 @@ @Tag("kind-parallel") @Tag("okd-wls-srg") @Tag("oke-arm") -@Tag("oke-parallel") +@Tag("oke-weekly-sequential") class ItPodsRestart { private static String miiImage; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItPodsShutdownOption.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItPodsShutdownOption.java index 733ab1025c8..62985abfc66 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItPodsShutdownOption.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItPodsShutdownOption.java @@ -82,7 +82,7 @@ @Tag("kind-parallel") @Tag("okd-wls-mrg") @Tag("oke-arm") -@Tag("oke-gate") +@Tag("oke-weekly-sequential") class ItPodsShutdownOption { private static String domainNamespace = null; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItT3Channel.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItT3Channel.java index ddbfbfaf39e..0ea684d1e2f 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItT3Channel.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItT3Channel.java @@ -98,7 +98,7 @@ @DisplayName("Test T3 channel deployment") @IntegrationTest @Tag("olcne-mrg") -@Tag("oke-gate") +@Tag("oke-weekly-sequential") @Tag("kind-sequential") @Tag("gate") class ItT3Channel { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsManagedByTwoOperators.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsManagedByTwoOperators.java index 2c8788fd12a..521ed38245c 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsManagedByTwoOperators.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsManagedByTwoOperators.java @@ -108,7 +108,7 @@ @IntegrationTest @Tag("olcne-srg") @Tag("kind-parallel") -@Tag("oke-gate") +@Tag("oke-weekly-sequential") @Tag("oke-arm") class ItTwoDomainsManagedByTwoOperators { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItWseeSSO.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItWseeSSO.java index 1a87b3f3a8a..05cc2619b1a 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItWseeSSO.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItWseeSSO.java @@ -28,6 +28,7 @@ import oracle.weblogic.kubernetes.annotations.IntegrationTest; import oracle.weblogic.kubernetes.annotations.Namespaces; import oracle.weblogic.kubernetes.logging.LoggingFacade; +import oracle.weblogic.kubernetes.utils.ExecCommand; import oracle.weblogic.kubernetes.utils.ExecResult; import oracle.weblogic.kubernetes.utils.OracleHttpClient; import org.junit.jupiter.api.BeforeAll; @@ -47,6 +48,7 @@ import static oracle.weblogic.kubernetes.TestConstants.IT_WSEESSONGINX_INGRESS_HTTP_NODEPORT; import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST; import static oracle.weblogic.kubernetes.TestConstants.KIND_CLUSTER; +import static oracle.weblogic.kubernetes.TestConstants.KUBERNETES_CLI; import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_NAME; import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_TAG; import static oracle.weblogic.kubernetes.TestConstants.NGINX_CHART_VERSION; @@ -90,6 +92,7 @@ import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator; import static oracle.weblogic.kubernetes.utils.PersistentVolumeUtils.createPV; import static oracle.weblogic.kubernetes.utils.PersistentVolumeUtils.createPVC; +import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodReady; import static oracle.weblogic.kubernetes.utils.PodUtils.execInPod; import static oracle.weblogic.kubernetes.utils.PodUtils.getExternalServicePodName; import static oracle.weblogic.kubernetes.utils.SecretUtils.createSecretWithUsernamePassword; @@ -107,7 +110,7 @@ */ @DisplayName("Verify that client can communicate with webservices with SSO") @IntegrationTest -@Tag("oke-gate") +@Tag("oke-sequential") @Tag("kind-parallel") class ItWseeSSO { @@ -266,6 +269,20 @@ private String checkWSDLAccess(String domainNamespace, String domainUid, hostAndPort = ingressIP + ":80"; } String url = "http://" + hostAndPort + appURI; + if (OKE_CLUSTER) { + try { + if (OracleHttpClient.get(url, true).statusCode() != 200) { + ExecResult result = ExecCommand.exec(KUBERNETES_CLI + " get all -A"); + logger.info(result.stdout()); + //restart core-dns service + result = ExecCommand.exec(KUBERNETES_CLI + " rollout restart deployment coredns -n kube-system"); + logger.info(result.stdout()); + checkPodReady("coredns", null, "kube-system"); + } + } catch (Exception ex) { + logger.warning(ex.getLocalizedMessage()); + } + } assertEquals(200, OracleHttpClient.get(url, true).statusCode()); return url; } diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/primitive/WebLogicImageTool.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/primitive/WebLogicImageTool.java index f55206814c5..16561a231fd 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/primitive/WebLogicImageTool.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/primitive/WebLogicImageTool.java @@ -9,6 +9,7 @@ import oracle.weblogic.kubernetes.logging.LoggingFacade; import oracle.weblogic.kubernetes.utils.ExecResult; +import static oracle.weblogic.kubernetes.TestConstants.ARM; import static oracle.weblogic.kubernetes.TestConstants.BUSYBOX_IMAGE; import static oracle.weblogic.kubernetes.TestConstants.BUSYBOX_TAG; import static oracle.weblogic.kubernetes.TestConstants.FMWINFRA_IMAGE_NAME; @@ -189,6 +190,10 @@ private String buildWitCommand() { command += " --wdtModelOnly "; } + if (ARM) { + command += " --platform linux/arm64"; + } + if (params.wdtModelHome() != null) { command += " --wdtModelHome " + params.wdtModelHome(); } diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/InitializationTasks.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/InitializationTasks.java index be2ebb5804b..dd1d476cb8c 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/InitializationTasks.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/InitializationTasks.java @@ -319,7 +319,7 @@ public void beforeAll(ExtensionContext context) { installTraefikLB(); } //install Oracle Database operator as a one time task - if (!OCNE && !OKD && !CRIO) { + if (!OCNE && !OKD && !CRIO && !ARM) { installOracleDBOperator(); } diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonLBTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonLBTestUtils.java index b68de760740..38579e3846a 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonLBTestUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonLBTestUtils.java @@ -64,6 +64,7 @@ import static oracle.weblogic.kubernetes.TestConstants.KIND_CLUSTER; import static oracle.weblogic.kubernetes.TestConstants.KUBERNETES_CLI; import static oracle.weblogic.kubernetes.TestConstants.MANAGED_SERVER_NAME_BASE; +import static oracle.weblogic.kubernetes.TestConstants.OKE_CLUSTER; import static oracle.weblogic.kubernetes.TestConstants.OKE_CLUSTER_PRIVATEIP; import static oracle.weblogic.kubernetes.TestConstants.PV_ROOT; import static oracle.weblogic.kubernetes.TestConstants.RESULTS_ROOT; @@ -95,6 +96,7 @@ import static oracle.weblogic.kubernetes.utils.JobUtils.createJobAndWaitUntilComplete; import static oracle.weblogic.kubernetes.utils.PersistentVolumeUtils.createPVPVCAndVerify; import static oracle.weblogic.kubernetes.utils.PersistentVolumeUtils.createfixPVCOwnerContainer; +import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodReady; import static oracle.weblogic.kubernetes.utils.PodUtils.getExternalServicePodName; import static oracle.weblogic.kubernetes.utils.PodUtils.setPodAntiAffinity; import static oracle.weblogic.kubernetes.utils.SecretUtils.createSecretWithUsernamePassword; @@ -882,6 +884,38 @@ public static void verifyAdminServerAccess(boolean isTLS, getLogger().severe(ex.getMessage()); } } + if (OKE_CLUSTER) { + LoggingFacade logger = getLogger(); + try { + if (!consoleAccessible) { + ExecResult result = ExecCommand.exec(KUBERNETES_CLI + " get all -A"); + logger.info(result.stdout()); + //restart core-dns service + result = ExecCommand.exec(KUBERNETES_CLI + " rollout restart deployment coredns -n kube-system"); + logger.info(result.stdout()); + checkPodReady("coredns", null, "kube-system"); + } + } catch (Exception ex) { + logger.warning(ex.getLocalizedMessage()); + } + for (int i = 0; i < 10; i++) { + assertDoesNotThrow(() -> TimeUnit.SECONDS.sleep(1)); + ExecResult result; + try { + getLogger().info("Accessing app on admin server using curl request, iteration {0}: {1}", i, curlCmd); + result = ExecCommand.exec(curlCmd, true); + String response = result.stdout().trim(); + getLogger().info("exitCode: {0}, \nstdout: {1}, \nstderr: {2}", + result.exitValue(), response, result.stderr()); + if (response.contains("RUNNING")) { + consoleAccessible = true; + break; + } + } catch (IOException | InterruptedException ex) { + getLogger().severe(ex.getMessage()); + } + } + } assertTrue(consoleAccessible, "Couldn't access admin server app"); } } diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java index 76224dfd8e7..0293fef4934 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java @@ -1338,13 +1338,11 @@ public static String getHostAndPort(String hostName, int servicePort) { } host = formatIPv6Host(host); String hostAndPort = ((OKD) ? hostName : host + ":" + servicePort); + if (OKE_CLUSTER_PRIVATEIP) { hostAndPort = hostName; } logger.info("hostAndPort = {0} ", hostAndPort); - if (OKE_CLUSTER_PRIVATEIP) { - hostAndPort = hostName; - } return hostAndPort; } catch (UnknownHostException e) { throw new RuntimeException(e); diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/IstioUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/IstioUtils.java index d9f6e98498c..942d1de7efd 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/IstioUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/IstioUtils.java @@ -82,25 +82,26 @@ public static void installIstio() { // Copy the istio (un)intsall scripts to RESULTS_ROOT, so that istio // can be (un)installed manually when SKIP_CLEANUP is set to true assertDoesNotThrow(() -> Files.copy( - Paths.get(RESOURCE_DIR, "bash-scripts", "install-istio.sh"), - Paths.get(RESULTS_ROOT, "install-istio.sh"), - StandardCopyOption.REPLACE_EXISTING), + Paths.get(RESOURCE_DIR, "bash-scripts", "install-istio.sh"), + Paths.get(RESULTS_ROOT, "install-istio.sh"), + StandardCopyOption.REPLACE_EXISTING), String.format("Copy install-istio.sh to %s failed", RESULTS_ROOT)); assertDoesNotThrow(() -> Files.copy( - Paths.get(RESOURCE_DIR, "bash-scripts", "uninstall-istio.sh"), - Paths.get(RESULTS_ROOT, "uninstall-istio.sh"), - StandardCopyOption.REPLACE_EXISTING), + Paths.get(RESOURCE_DIR, "bash-scripts", "uninstall-istio.sh"), + Paths.get(RESULTS_ROOT, "uninstall-istio.sh"), + StandardCopyOption.REPLACE_EXISTING), String.format("Copy uninstall-istio.sh to %s failed", RESULTS_ROOT)); Path istioInstallPath = Paths.get(RESULTS_ROOT, "install-istio.sh"); String installScript = istioInstallPath.toString(); - String ocneIstioRepo = BASE_IMAGES_REPO + "/" + BASE_IMAGES_TENANCY; - // When install istio in OCNE environment, + + // When install istio in OCNE environment, // use BASE_IMAGES_REPO/devweblogic/istio-release instead of gcr.io/istio-release if (OCNE) { - logger.info("replace istio installation hub in File {0}", ocneIstioRepo); + String ocneIstioRepo = BASE_IMAGES_REPO + "/" + BASE_IMAGES_TENANCY; + logger.info("replace istio installation hub in File {0}", installScript); assertDoesNotThrow(() -> replaceStringInFile(installScript, "gcr.io", ocneIstioRepo), String.format("Failed to replace string in File %s", installScript)); } @@ -113,11 +114,10 @@ public static void installIstio() { String.format("%s %s %s %s %s", installScript, ISTIO_VERSION, RESULTS_ROOT, TEST_IMAGES_TENANCY, arch); logger.info("Istio installation command {0}", command); assertTrue(() -> Command.withParams( - defaultCommandParams() - .command(command) - .redirect(false)) + defaultCommandParams() + .command(command) + .redirect(false)) .execute()); - if (OKE_CLUSTER) { String loadBalancerIP = getServiceExtIPAddrtOke("istio-ingressgateway", "istio-system"); testUntil( @@ -135,16 +135,16 @@ public static void installIstio() { */ public static void uninstallIstio() { LoggingFacade logger = getLogger(); - Path istioInstallPath = + Path istioInstallPath = Paths.get(RESOURCE_DIR, "bash-scripts", "uninstall-istio.sh"); String installScript = istioInstallPath.toString(); String command = String.format("%s %s %s", installScript, ISTIO_VERSION, RESULTS_ROOT); logger.info("Istio uninstallation command {0}", command); assertTrue(() -> Command.withParams( - defaultCommandParams() - .command(command) - .redirect(false)) + defaultCommandParams() + .command(command) + .redirect(false)) .execute()); } @@ -154,11 +154,21 @@ public static void uninstallIstio() { * @return ingress port for istio-ingressgateway */ public static int getIstioHttpIngressPort() { + return getIstioHttpIngressPort("http2"); + } + + /** + * Get the http ingress port of istio installation. + * + * @param portName name of port to get + * @return ingress port for istio-ingressgateway + */ + public static int getIstioHttpIngressPort(String portName) { LoggingFacade logger = getLogger(); ExecResult result; StringBuffer getIngressPort; getIngressPort = new StringBuffer(KUBERNETES_CLI + " -n istio-system get service istio-ingressgateway "); - getIngressPort.append("-o jsonpath='{.spec.ports[?(@.name==\"http2\")].nodePort}'"); + getIngressPort.append("-o jsonpath='{.spec.ports[?(@.name==\"" + portName.trim() + "\")].nodePort}'"); logger.info("getIngressPort: " + KUBERNETES_CLI + " command {0}", new String(getIngressPort)); try { result = exec(new String(getIngressPort), true); @@ -448,7 +458,7 @@ public static DomainResource createIstioDomainResource(String domainUid, String .configMap(configmapName) .onlineUpdate(new OnlineUpdate().enabled(true)) .runtimeEncryptionSecret(encryptionSecretName)) - .introspectorJobActiveDeadlineSeconds(300L))); + .introspectorJobActiveDeadlineSeconds(3000L))); // create cluster resource domain = createClusterResourceAndAddReferenceToDomain(domainUid + "-" + clusterName, @@ -502,7 +512,7 @@ public static AdminServer createAdminServer() { return adminServer; } - + /** * Check WebLogic access through Istio Ingress Port. * @param istioHost Host @@ -533,8 +543,8 @@ public static void checkIstioService(String istioHost, int istioIngressPort, Str * @return istioIngressPort */ public static int createIstioService( - String domainUid, String clusterName, - String adminServerPodName, String domainNamespace) { + String domainUid, String clusterName, + String adminServerPodName, String domainNamespace) { LoggingFacade logger = getLogger(); String clusterService = domainUid + "-cluster-" + clusterName + "." + domainNamespace + ".svc.cluster.local"; @@ -565,7 +575,8 @@ public static int createIstioService( int istioIngressPort = getIstioHttpIngressPort(); logger.info("Istio Ingress Port is {0}", istioIngressPort); return istioIngressPort; - + } } + diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoadBalancerUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoadBalancerUtils.java index 92f52f28b64..ad4a2116085 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoadBalancerUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoadBalancerUtils.java @@ -35,6 +35,7 @@ import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams; import oracle.weblogic.kubernetes.extensions.InitializationTasks; import oracle.weblogic.kubernetes.logging.LoggingFacade; +import org.jetbrains.annotations.Nullable; import static oracle.weblogic.kubernetes.TestConstants.ADMIN_SERVER_NAME_BASE; import static oracle.weblogic.kubernetes.TestConstants.COMPARTMENT_OCID; @@ -129,8 +130,8 @@ public static void installAndVerifyOCILoadBalancer( // wait until the external IP is generated. testUntil( assertDoesNotThrow(() -> isOCILoadBalancerReady( - loadBalancerName, - labels, namespace), "isOCILoadBalancerReady failed with ApiException"), + loadBalancerName, + labels, namespace), "isOCILoadBalancerReady failed with ApiException"), logger, "external IP to be generated in {0}", namespace); @@ -153,8 +154,8 @@ public static void installAndVerifyOCILoadBalancer( * @return the NGINX Helm installation parameters */ public static NginxParams installAndVerifyNginx(String nginxNamespace, - int nodeportshttp, - int nodeportshttps) { + int nodeportshttp, + int nodeportshttps) { return installAndVerifyNginx(nginxNamespace, nodeportshttp, nodeportshttps, NGINX_CHART_VERSION, null); } @@ -169,10 +170,10 @@ public static NginxParams installAndVerifyNginx(String nginxNamespace, * @return the NGINX Helm installation parameters */ public static NginxParams installAndVerifyNginx(String nginxNamespace, - int nodeportshttp, - int nodeportshttps, - String chartVersion, - String type) { + int nodeportshttp, + int nodeportshttps, + String chartVersion, + String type) { LoggingFacade logger = getLogger(); createTestRepoSecret(nginxNamespace); @@ -190,7 +191,7 @@ public static NginxParams installAndVerifyNginx(String nginxNamespace, // NGINX chart values to override NginxParams nginxParams = new NginxParams() - .helmParams(nginxHelmParams); + .helmParams(nginxHelmParams); // set secret to pull images from private registry nginxParams.imageRepoSecret(TEST_IMAGES_REPO_SECRET_NAME); if (nodeportshttp != 0 && nodeportshttps != 0) { @@ -256,11 +257,11 @@ public static NginxParams installAndVerifyNginx(String nginxNamespace, * @return the Traefik Helm installation parameters */ public static TraefikParams installAndVerifyTraefik(String traefikNamespace, - int nodeportshttp, - int nodeportshttps) { + int nodeportshttp, + int nodeportshttps) { return installAndVerifyTraefik(traefikNamespace, nodeportshttp, nodeportshttps, null); } - + /** Install Traefik and wait for up to five minutes for the Traefik pod to be ready. * * @param traefikNamespace the namespace in which the Traefik ingress controller is installed @@ -270,9 +271,9 @@ public static TraefikParams installAndVerifyTraefik(String traefikNamespace, * @return the Traefik Helm installation parameters */ public static TraefikParams installAndVerifyTraefik(String traefikNamespace, - int nodeportshttp, - int nodeportshttps, - String type) { + int nodeportshttp, + int nodeportshttps, + String type) { LoggingFacade logger = getLogger(); // Helm install parameters HelmParams traefikHelmParams = new HelmParams() @@ -329,7 +330,6 @@ public static TraefikParams installAndVerifyTraefik(String traefikNamespace, "Traefik to be healthy in namespace {0}", traefikNamespace); } - return traefikParams; } @@ -386,13 +386,15 @@ public static String getLoadBalancerIP(String namespace, String lbName, boolean return null; } - private static boolean checkLoadBalancerHealthy(String namespace, String lbServiceName) { + private static boolean checkLoadBalancerHealthy(String namespace, String lbServiceName) { + String lbPublicIP = assertDoesNotThrow(() -> getLoadBalancerIP(namespace, lbServiceName)); InitializationTasks.registerLoadBalancerExternalIP(lbPublicIP); LoggingFacade logger = getLogger(); String testcompartmentid = System.getProperty("wko.it.oci.compartment.ocid"); logger.info("wko.it.oci.compartment.ocid property " + testcompartmentid); + final String command = "oci lb load-balancer list --compartment-id " + testcompartmentid + " --query \"data[?contains(\\\"ip-addresses\\\"[0].\\\"ip-address\\\", '" + lbPublicIP + "')].id | [0]\" --raw-output --all"; @@ -410,13 +412,44 @@ private static boolean checkLoadBalancerHealthy(String namespace, String lbServi // Clean up the string to extract the Load Balancer ID String lbOCID = result.stdout().trim(); + boolean isFlexible = isLoadBalancerShapeFlexible(lbOCID); + + if (!isFlexible) { + logger.info("Updating load balancer shape to flexible"); + + final String command2 = "oci lb load-balancer update-load-balancer-shape --load-balancer-id " + + lbOCID + " --shape-name flexible --shape-details" + + " '{\"minimumBandwidthInMbps\": 10, \"maximumBandwidthInMbps\": 400}' --force"; + + result = assertDoesNotThrow(() -> exec(command2, true)); + logger.info("Command: {}, Exit value: {}, Stdout: {}, Stderr: {}", + command2, result.exitValue(), result.stdout(), result.stderr()); + + if (result == null || result.stdout() == null) { + return false; + } else if (result.exitValue() != 0 && !result.stdout().contains("is currently being modified")) { + return false; + } + + testUntil( + assertDoesNotThrow(() -> checkWorkRequestUpdateShapeSucceeded( + lbOCID), "isOCILoadBalancer work request to update shape is not ready"), + logger, + "load balancer shape is updating "); + testUntil( + assertDoesNotThrow(() -> checkLoadBalancerShapeFlexible( + lbOCID), "checkLoadBalancerShape is not flexible "), + logger, + "load balancer shape can't be checked, retrying "); + } + //check health status final String command1 = "oci lb load-balancer-health get --load-balancer-id " + lbOCID; logger.info("Command to retrieve Load Balancer health status is: {0} ", command1); result = assertDoesNotThrow(() -> exec(command1, true)); logger.info("The command returned exit value: " + result.exitValue() + " command output: " + result.stderr() + "\n" + result.stdout()); - + logger.info("result.stderr: \n{0}", result.stderr()); if (result == null || result.exitValue() != 0 || result.stdout() == null) { return false; } @@ -425,6 +458,59 @@ private static boolean checkLoadBalancerHealthy(String namespace, String lbServi } + @Nullable + private static boolean isLoadBalancerShapeFlexible(String lbOCID) { + LoggingFacade logger = getLogger(); + + final String checkShapeCommand = "oci lb load-balancer get --load-balancer-id " + + lbOCID + " | jq '.data[\"shape-name\"], .data[\"shape-details\"]'"; + ExecResult result = assertDoesNotThrow(() -> exec(checkShapeCommand, true)); + logger.info("The command " + checkShapeCommand + " returned exit value: " + result.exitValue() + + " command output: " + result.stderr() + "\n" + result.stdout()); + logger.info("result.stderr: \n{0}", result.stderr()); + if (result == null || result.exitValue() != 0 || result.stdout() == null || !result.stdout().contains("flexible")) { + return false; + } + return true; + } + + private static Callable checkLoadBalancerShapeFlexible(String loadBalancerOCID) { + return () -> isLoadBalancerShapeFlexible(loadBalancerOCID); + } + + /** + * Check work request status for load balancer. + * @param loadBalancerOCID - load balancer OCID + * @return true if succeeded , false over vise. + */ + public static boolean isWorkRequestUpdateShapeSucceeded(String loadBalancerOCID) { + + LoggingFacade logger = getLogger(); + final String command = "oci lb work-request list --load-balancer-id " + + loadBalancerOCID + + " --query 'data[?type == `UpdateShape`].{id:id, lifecycleState:\"lifecycle-state\", " + + "message:message, timeFinished:\"time-finished\"}' " + + "| jq '.[] | select(.lifecycleState == \"SUCCEEDED\")'"; + ExecResult result = assertDoesNotThrow(() -> exec(command, true)); + logger.info("The command " + command + " returned exit value: " + result.exitValue() + + " command output: " + result.stderr() + "\n" + result.stdout()); + logger.info("result.stderr: \n{0}", result.stderr()); + if (result == null || result.exitValue() != 0 || result.stdout() == null || result.stderr().contains("ERROR")) { + return false; + } + return true; + } + + /** + * Check if lb work request status is succeeded. + * + * @param loadBalancerOCID lb ocid + * @return true if succeeded, false otherwise + */ + public static Callable checkWorkRequestUpdateShapeSucceeded(String loadBalancerOCID) { + return () -> isWorkRequestUpdateShapeSucceeded(loadBalancerOCID); + } + /** Upgrade Traefik and wait for up to five minutes for the Traefik pod to be ready. * * @param traefikNamespace the namespace in which the Traefik ingress controller is installed @@ -653,7 +739,6 @@ public static void createNginxIngressPathRoutingRules(String domainNamespace, logger.info("ingress {0} was created in namespace {1}", ingressName, domainNamespace); // check the ingress is ready to route the app to the server pod - String curlCmd = "curl -g --silent --show-error --noproxy '*' http://" + hostAndPort + "/weblogic/ready --write-out %{http_code} -o /dev/null"; @@ -855,8 +940,8 @@ public static String getLbExternalIp(String lbrelname, String lbns) throws Excep LoggingFacade logger = getLogger(); String cmdip = KUBERNETES_CLI + " get svc --namespace " + lbns - + " -o jsonpath='{.items[?(@.metadata.name == \"" + lbrelname + "\")]" - + ".status.loadBalancer.ingress[0].ip}'"; + + " -o jsonpath='{.items[?(@.metadata.name == \"" + lbrelname + "\")]" + + ".status.loadBalancer.ingress[0].ip}'"; logger.info("Command to retrieve external IP is: {0} ", cmdip); diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/MonitoringUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/MonitoringUtils.java index 80a6c6e6856..f7851aabf85 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/MonitoringUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/MonitoringUtils.java @@ -105,6 +105,7 @@ import static oracle.weblogic.kubernetes.assertions.TestAssertions.isPrometheusAdapterReady; import static oracle.weblogic.kubernetes.assertions.TestAssertions.isPrometheusReady; import static oracle.weblogic.kubernetes.utils.ApplicationUtils.callWebAppAndCheckForServerNameInResponse; +import static oracle.weblogic.kubernetes.utils.ApplicationUtils.callWebAppAndWaitTillReady; import static oracle.weblogic.kubernetes.utils.ClusterUtils.createClusterAndVerify; import static oracle.weblogic.kubernetes.utils.ClusterUtils.createClusterResource; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.addSccToDBSvcAccount; @@ -121,6 +122,7 @@ import static oracle.weblogic.kubernetes.utils.ImageUtils.createTestRepoSecret; import static oracle.weblogic.kubernetes.utils.ImageUtils.imageRepoLoginAndPushImageToRegistry; import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodExists; +import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodReady; import static oracle.weblogic.kubernetes.utils.PodUtils.getPodName; import static oracle.weblogic.kubernetes.utils.PodUtils.setPodAntiAffinity; import static oracle.weblogic.kubernetes.utils.SecretUtils.createSecretWithUsernamePassword; @@ -261,6 +263,22 @@ public static void checkMetricsViaPrometheus(String searchKey, String expectedVa logger.info("Executing Curl cmd {0}", curlCmd); logger.info("Checking searchKey: {0}", searchKey); logger.info(" expected Value {0} ", expectedVal); + if (OKE_CLUSTER) { + try { + if (!callWebAppAndWaitTillReady(curlCmd, 5)) { + ExecResult result = ExecCommand.exec(KUBERNETES_CLI + " get all -A"); + logger.info(result.stdout()); + //restart core-dns service + result = ExecCommand.exec(KUBERNETES_CLI + " rollout restart deployment coredns -n kube-system"); + logger.info(result.stdout()); + checkPodReady("coredns", null, "kube-system"); + result = ExecCommand.exec(curlCmd); + logger.info(result.stdout()); + } + } catch (Exception ex) { + logger.warning(ex.getLocalizedMessage()); + } + } testUntil( searchForKey(curlCmd, expectedVal), logger, diff --git a/integration-tests/src/test/resources/oke/terraform/okemodule/oke.create.sh b/integration-tests/src/test/resources/oke/terraform/okemodule/oke.create.sh index ce86375b00f..b0a5099d455 100755 --- a/integration-tests/src/test/resources/oke/terraform/okemodule/oke.create.sh +++ b/integration-tests/src/test/resources/oke/terraform/okemodule/oke.create.sh @@ -30,8 +30,9 @@ generateTFVarFile() { sed -i -e "s:@NODEPOOLIMAGENAME@:${nodepool_imagename}:g" ${tfVarsFiletfVarsFile} sed -i -e "s:@NODEPOOLSSHPUBKEY@:${nodepool_ssh_pubkeypath}:g" ${tfVarsFiletfVarsFile} sed -i -e "s:@NODEPOOLSSHPK@:${nodepool_ssh_pkpath}:g" ${tfVarsFiletfVarsFile} - sed -i -e "s:@REGION@:${region}:g" ${tfVarsFiletfVarsFile} sed -i -e "s:@MOUNTTARGETOCID@:${mount_target_ocid}:g" ${tfVarsFiletfVarsFile} + sed -i -e "s:@REGION@:${region}:g" ${tfVarsFiletfVarsFile} + sed -i -e "s:@REGIONSHORT@:${region_short}:g" ${tfVarsFiletfVarsFile} echo "Generated TFVars file [${tfVarsFiletfVarsFile}]" } @@ -80,75 +81,141 @@ createRoleBindings () { ${KUBERNETES_CLI:-kubectl} config set-credentials $okeclustername-sa --token=$TOKEN ${KUBERNETES_CLI:-kubectl} config set-context --current --user=$okeclustername-sa } +checkKubernetesCliConnection() { + echo "Confirming ${KUBERNETES_CLI:-kubectl} can connect to the server..." + + # Get the cluster public IP + clusterPublicIP=$(oci ce cluster list --compartment-id="${compartment_ocid}" | jq -r '.data[] | select(."name" == "'"${okeclustername}"'" and (."lifecycle-state" == "ACTIVE")) | ."endpoints" | ."public-endpoint" | split(":")[0]') + + # Check if clusterPublicIP is empty or not + if [ -z "$clusterPublicIP" ]; then + echo "[ERROR] No active cluster found with name ${okeclustername}." + exit 1 + fi + echo "clusterPublicIP: ###$clusterPublicIP###" + echo " NO_PROXY=#$NO_PROXY# " + export NO_PROXY=$NO_PROXY,localhost,127.0.0.1,$clusterPublicIP + echo "export NO_PROXY=:$NO_PROXY" -checkClusterRunning () { + # Maximum number of retries + max_retries=10 + + # Initial retry count + retry_count=0 + + # Command to get cluster info + while [[ $retry_count -lt $max_retries ]]; do + echo "Attempt $((retry_count+1)) of $max_retries to connect to Kubernetes cluster..." + + # Try to execute kubectl cluster-info + ${KUBERNETES_CLI:-kubectl} cluster-info + if [[ $? -eq 0 ]]; then + echo "Connected to Kubernetes cluster successfully!" + break + else + echo "Connection refused or failed, retrying..." + retry_count=$((retry_count + 1)) + sleep 5 # Wait 5 seconds before retrying + fi + done + + # Check if retries were exhausted + if [[ $retry_count -eq $max_retries ]]; then + echo "Failed to connect to Kubernetes cluster after $max_retries attempts." + cd "${terraformVarDir}" + terraform destroy -auto-approve -var-file="${terraformVarDir}/${clusterTFVarsFile}.tfvars" + createCluster + fi + + local myline_output=$(${KUBERNETES_CLI:-kubectl} get nodes -o wide 2>&1) - echo "Confirm we have ${KUBERNETES_CLI:-kubectl} working..." - privateIP=${vcn_cidr_prefix} - myline_output=$(${KUBERNETES_CLI:-kubectl} get nodes -o wide 2>&1) if echo "$myline_output" | grep -q "Unable to connect to the server: net/http: TLS handshake timeout"; then echo "[ERROR] Unable to connect to the server: net/http: TLS handshake timeout" - - echo '- could not talk to cluster, aborting' - cd ${terraformVarDir} - terraform destroy -auto-approve -var-file=${terraformVarDir}/${clusterTFVarsFile}.tfvars - terraform apply -auto-approve -var-file=${terraformVarDir}/${clusterTFVarsFile}.tfvars - echo "retrying to execute KUBERNETES_CLI" - clusterIP=$(oci ce cluster list --compartment-id=${compartment_ocid} | jq '.data[] | select(."name" == '"${okeclustername}"' and (."lifecycle-state" == "ACTIVE"))' | jq ' ."endpoints" | ."public-endpoint"') - echo "clusterIp : $clusterIP" - clusterPublicIP=${clusterIP:1:-6} - echo " clusterPublicIP : ${clusterPublicIP}" - echo "NO_PROXY before : ${NO_PROXY}" - export NO_PROXY=${clusterPublicIP} - echo "NO_PROXY:" $NO_PROXY + echo '- could not talk to OKE cluster, aborting' + unset http_proxy + unset https_proxy myline_output=$(${KUBERNETES_CLI:-kubectl} get nodes -o wide 2>&1) if echo "$myline_output" | grep -q "Unable to connect to the server: net/http: TLS handshake timeout"; then - echo "[ERROR] Unable to connect to the server: net/http: TLS handshake timeout" - echo '- could not talk to cluster, aborting' - cd ${terraformVarDir} - terraform destroy -auto-approve -var-file=${terraformVarDir}/${clusterTFVarsFile}.tfvars - exit 1 + cd "${terraformVarDir}" + terraform destroy -auto-approve -var-file="${terraformVarDir}/${clusterTFVarsFile}.tfvars" + exit 1 fi fi - declare -a myline - myline=(`${KUBERNETES_CLI:-kubectl} get nodes -o wide | grep "${privateIP}" | awk '{print $2}'`) - NODE_IP=`${KUBERNETES_CLI:-kubectl} get nodes -o wide| grep "${privateIP}" | awk '{print $7}'` - status=$myline[0] - max=100 - count=1 - - for i in {0..1} - do - while [ "${myline[i]}" != "Ready" -a $count -le $max ] ; do - echo "echo '[ERROR] Some Nodes in the Cluster are not in the Ready Status , sleep 10s more ..." - sleep 10 - myline=(`${KUBERNETES_CLI:-kubectl} get nodes -o wide | grep "${privateIP}" | awk '{print $2}'`) - NODE_IP=`${KUBERNETES_CLI:-kubectl} get nodes -o wide| grep "${privateIP}" | awk '{print $7}'` - echo "myline[i] ${myline[i]}" - [[ ${myline[i]} -eq "Ready" ]] - echo "Status is ${myline[i]} Iter [$count/$max]" - count=`expr $count + 1` - done + if echo "$myline_output" | grep -q "couldn't get current server API group"; then + echo "[ERROR] Unable to connect to the server: couldn't get current server API group, connection refused" + echo '- check errors during OKE cluster creation' + echo '- could not talk to OKE cluster, aborting' + + cd "${terraformVarDir}" + terraform destroy -auto-approve -var-file="${terraformVarDir}/${clusterTFVarsFile}.tfvars" + exit 1 + fi + +} + +checkClusterRunning() { + kubeconfig_file=${terraformVarDir}/${okeclustername}_kubeconfig + export KUBECONFIG=${terraformVarDir}/${okeclustername}_kubeconfig + echo "Kubeconfig file : $KUBECONFIG" + ls -al $KUBECONFIG + + if [ -f "$kubeconfig_file" ] && [ -s "$kubeconfig_file" ]; then + echo "Kubeconfig file exists and is not empty." + else + if [ ! -f "$kubeconfig_file" ]; then + echo "Kubeconfig file does not exist." + cd "${terraformVarDir}" + terraform destroy -auto-approve -var-file="${terraformVarDir}/${clusterTFVarsFile}.tfvars" + createCluster + else + echo "Kubeconfig file exists but is empty." + cd "${terraformVarDir}" + terraform destroy -auto-approve -var-file="${terraformVarDir}/${clusterTFVarsFile}.tfvars" + createCluster + fi + fi + checkKubernetesCliConnection + + local privateIP=${vcn_cidr_prefix} + declare -a myline=($(${KUBERNETES_CLI:-kubectl} get nodes -o wide | grep "${privateIP}" | awk '{print $2}')) + local NODE_IP=$(${KUBERNETES_CLI:-kubectl} get nodes -o wide | grep "${privateIP}" | awk '{print $7}') + + local status=${myline[0]} + local max=100 + local count=1 + + for i in {0..1}; do + while [ "${myline[i]}" != "Ready" ] && [ $count -le $max ]; do + echo "[ERROR] Some Nodes in the Cluster are not in the Ready Status, sleeping for 10s..." + sleep 10 + myline=($(${KUBERNETES_CLI:-kubectl} get nodes -o wide | grep "${privateIP}" | awk '{print $2}')) + NODE_IP=$(${KUBERNETES_CLI:-kubectl} get nodes -o wide | grep "${privateIP}" | awk '{print $7}') + echo "myline[i]: ${myline[i]}" + echo "Status is ${myline[i]} Iter [$count/$max]" + count=$((count + 1)) + done done - NODES=`${KUBERNETES_CLI:-kubectl} get nodes -o wide | grep "${privateIP}" | wc -l` - if [ "$NODES" == "2" ]; then - echo '- looks good' + local NODES=$(${KUBERNETES_CLI:-kubectl} get nodes -o wide | grep "${privateIP}" | wc -l) + + if [ "$NODES" -eq 2 ]; then + echo '- looks good' else - echo '- could not talk to cluster, aborting' - cd ${terraformVarDir} - terraform destroy -auto-approve -var-file=${terraformVarDir}/${clusterTFVarsFile}.tfvars - exit 1 + echo '- could not talk to OKE cluster, aborting' + cd "${terraformVarDir}" + terraform destroy -auto-approve -var-file="${terraformVarDir}/${clusterTFVarsFile}.tfvars" + exit 1 fi - if [ $count -gt $max ] ; then - echo "[ERROR] Unable to start the nodes in oke cluster after 200s "; - cd ${terraformVarDir} - terraform destroy -auto-approve -var-file=${terraformVarDir}/${clusterTFVarsFile}.tfvars - exit 1 + if [ $count -gt $max ]; then + echo "[ERROR] Unable to start the nodes in the OKE cluster after 200s" + cd "${terraformVarDir}" + terraform destroy -auto-approve -var-file="${terraformVarDir}/${clusterTFVarsFile}.tfvars" + exit 1 fi } + #MAIN propsFile=${1:-$PWD/oci.props} terraformVarDir=${2:-$PWD} @@ -177,6 +244,7 @@ nodepool_ssh_pkpath=$(prop 'nodepool.ssh.pkpath') region=$(prop 'region') terraformDir=$(prop 'terraform.installdir') mount_target_ocid=$(prop 'mounttarget.ocid') +region_short=$(echo "$region" | sed 's/.*-\([a-z]*\)-.*/\1/') # generate terraform configuration file with name $(clusterTFVarsFile).tfvar #generateTFVarFile @@ -192,26 +260,12 @@ setupTerraform deleteOlderVersionTerraformOCIProvider chmod 600 ${ocipk_path} - +sudo yum reinstall ca-certificates -y +sudo iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT # run terraform init,plan,apply to create OKE cluster based on the provided tfvar file ${clusterTFVarsFile).tfvar createCluster #check status of OKE cluster nodes, destroy if can not access them export KUBECONFIG=${terraformVarDir}/${okeclustername}_kubeconfig - -export okeclustername=\"${okeclustername}\" - - - echo " oci ce cluster list --compartment-id=${compartment_ocid} | jq '.data[] | select(."name" == '"${okeclustername}"' and (."lifecycle-state" == "ACTIVE"))' | jq ' ."endpoints" | ."public-endpoint"'" - -clusterIP=$(oci ce cluster list --compartment-id=${compartment_ocid} | jq '.data[] | select(."name" == '"${okeclustername}"' and (."lifecycle-state" == "ACTIVE"))' | jq ' ."endpoints" | ."public-endpoint"') -echo "clusterIp : $clusterIP" -clusterPublicIP=${clusterIP:1:-6} -echo " clusterPublicIP : ${clusterPublicIP}" -echo "NO_PROXY before : ${NO_PROXY}" -export NO_PROXY=${clusterPublicIP} -echo "NO_PROXY:" $NO_PROXY - - checkClusterRunning -echo "$okeclustername is up and running}" +echo "${okeclustername} is up and running" diff --git a/integration-tests/src/test/resources/oke/terraform/okemodule/template.tfvars b/integration-tests/src/test/resources/oke/terraform/okemodule/template.tfvars index 3a938d76f78..c9ec27a4094 100755 --- a/integration-tests/src/test/resources/oke/terraform/okemodule/template.tfvars +++ b/integration-tests/src/test/resources/oke/terraform/okemodule/template.tfvars @@ -35,7 +35,7 @@ mount_target_ocid="@MOUNTTARGETOCID@" # ## For regions, # Use short form e.g. ashburn from location column https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm # ## VCN, Pods and services clusters must not overlap with each other and with those of other clusters. clusters = { - c1 = { region = "phoenix", vcn = "10.1.0.0/16", pods = "10.201.0.0/16", services = "10.101.0.0/16", enabled = true } + c1 = { region = "@REGIONSHORT@", vcn = "10.1.0.0/16", pods = "10.201.0.0/16", services = "10.101.0.0/16", enabled = true } } # kubernetes_version = "@OKEK8SVERSION@"