Global image for external

external/README.md

@@ -20,6 +20,8 @@ EXTERNAL_AQA_CONTAINER_CLEAN=true|false
 ```
 If EXTERNAL_AQA_CONTAINER_CLEAN is false, then the image is not cleaned after the `make _tests...` targets are finished.
 
+## Configuring base image
+By default, Eclipse Temurin JDK of identical version as your JDK is used. You can see, that `print_image_args` is taking all arguments to properly set registry url, image name and version. `EXTERNAL_AQA_IMAGE` variable describes the usual image ID in form  of `optional_registry/path/name:tag` to allow for alternate images besides the default to be used.
 
 ## Running External tests locally
 To run any AQA tests locally, you follow the same pattern:

external/build.xml

@@ -89,7 +89,7 @@
 			<fileset dir="${src}/" includes="*.sh, *.properties"/>
 		</copy>
 		<copy todir="${DEST_EXTERNAL}">
-			<fileset dir="${top}" includes="*.sh"/>
+			<fileset dir="${top}" includes="*.sh common.properties"/>
 		</copy>
 	</target>
 

external/build_all.sh

@@ -16,6 +16,16 @@ set -o pipefail
 
 source $(dirname "$0")/common_functions.sh
 
+if [ -n "${EXTERNAL_AQA_IMAGE}" ] ; then\
+  echo "EXTERNAL_AQA_IMAGE is declared as $EXTERNAL_AQA_IMAGE; it would break this script. Unset it."
+  exit 1
+fi
+
+if [ -z "${version}" ] ; then\
+  echo "'version' (of jdk, eg 17) is mandatory variable for this script to run"
+  exit 1
+fi
+
 # Cleanup any old containers and images
 echo "==============================================================================="
 echo "                           Cleaning up images                                  "

external/build_image.sh

@@ -25,6 +25,7 @@ if [ $# -ne 9 ] && [ $# -ne 10 ]; then
 	echo "The supported tests are ${supported_tests}"
 	echo
 	echo "usage: $0 test version vm os package build platform check_external_custom"
+	echo "eg:    $0 jacoco 17 openj9 ubi jdk full linux_x86-64 default 0"
 	echo "test    = ${supported_tests}"
 	echo "vm      = ${supported_jvms}"
 	echo "os      = ${supported_os}"
@@ -43,6 +44,7 @@ check_external_custom=$9
 if [[ "${check_external_custom}" == "0" ]]; then
 	set_test $1
 fi
+set_version $2
 set_vm $3
 set_os $4
 set_package $5

external/camel/test.properties

@@ -3,4 +3,4 @@
  test_results="testResults"
  tag_version="2.7.0"
  environment_variable="MODE=java"
- ubuntu_packages="git"
+generic_packages="git"

external/common.properties

@@ -0,0 +1,6 @@
+# those packages are included in all tests and all distributions
+generic_packages=""
+# those packages are included in all tests and all ubuntu
+ubuntu_packages=""
+# those packages are included in all tests and all Red Hat distributions
+(fedora|ubi|rhel|centos)_packages=""

external/common_functions.sh

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
+source $(dirname "$0")/provider.sh
+
 # Supported JVMs
 supported_jvms="hotspot openj9"
 
@@ -28,6 +31,11 @@ supported_builds="full"
 # Supported tests
 supported_tests="external_custom camel criu-functional criu-portable-checkpoint criu-portable-restore criu-ubi-portable-checkpoint criu-ubi-portable-restore derby elasticsearch jacoco jenkins functional-test kafka lucene-solr openliberty-mp-tck payara-mp-tck quarkus quarkus_quickstarts scala system-test tck-ubi-test tomcat tomee wildfly wycheproof netty spring zookeeper"
 
+# Set a valid version
+function set_version() {
+    version=$1
+}
+
 function check_os() {
     os=$1
 
@@ -161,11 +169,34 @@ function set_base_docker_registry_dir() {
 
 # Reading properties of test.properties file
 function getProperty() {
-    PROP_KEY=$1
-    PROP_VALUE=`cat $PROPERTY_FILE | grep "$PROP_KEY" | cut -d'=' -f 2-`
+    local PROP_KEY="${1}"
+    local FILE=${2:-""}
+    if [ -z "${FILE}" ] ; then
+      FILE="${PROPERTY_FILE}"
+    fi
+    local PROP_VALUE=`cat "${FILE}" | grep -v "^#" | grep "^${PROP_KEY}=" | cut -d'=' -f 2-`
     echo  `sed -e 's/^"//' -e 's/"$//' <<<"$PROP_VALUE"`
 }
 
+# Getting matchning keys for given OS
+function getMatchingPackagesKeys() {
+    local FILE=${1}
+    local OS=$(getImageOs)
+    local TAG=$(getImageTag)
+    local KEYS=""
+    while read line; do
+      if echo "$line" | grep -q "^#" ; then
+        continue
+      fi
+      local keyCandidate=$(echo $line | cut -d'=' -f 1)
+      local strippedKeyCandidate=$(echo $keyCandidate | sed "s/_packages//")
+      if echo "${OS}:${TAG}" | grep -Eqe ".*${strippedKeyCandidate}.*" ; then
+        KEYS="$KEYS $keyCandidate"
+      fi
+    done < "${FILE}"
+    echo ${KEYS}
+}
+
 # Used for external_custom tests
 function set_external_custom_test_info(){
     test=$1
@@ -175,17 +206,20 @@ function set_external_custom_test_info(){
     test_results="testResults"
     tag_version="${EXTERNAL_REPO_BRANCH}"
     environment_variable="MODE=java"
-    ubuntu_packages="git"
     maven_version="3.8.5"
+    packages="git"
 }
 
 # Set the valid OSes for the current architectures.
 function set_test_info() {
-    test=$1
+    local test=$1
     check_external_custom_test=$2
-    cd ../
-    path_to_file=$(pwd)
+    local  path_to_file=$(cd $(dirname "$0") && pwd)
     echo ${path_to_file}
+    # global settings will be amend to local ones
+    local GLOBAL_PROPERTY_FILE=${path_to_file}/common.properties
+    local global_generic_packages=$(getProperty "generic_packages" "${GLOBAL_PROPERTY_FILE}")
+
     PROPERTY_FILE=${path_to_file}/${test}/test.properties
     github_url=$(getProperty "github_url")
     test_options=$(getProperty "test_options")
@@ -204,14 +238,28 @@ function set_test_info() {
     maven_version=$(getProperty "maven_version")
     environment_variable=$(getProperty "environment_variable")
     localPropertyFile=$(getProperty "localPropertyFile")
-    ubuntu_packages=$(getProperty "ubuntu_packages")
-    ubi_packages=$(getProperty "ubi_packages")
+    local local_generic_packages=$(getProperty "generic_packages")
+
+    packages="$global_generic_packages $local_generic_packages"
+    local gobalMatchingKeys=$(getMatchingPackagesKeys "${GLOBAL_PROPERTY_FILE}")
+    for key in ${gobalMatchingKeys} ; do
+      packages="$packages $(getProperty $key" "${GLOBAL_PROPERTY_FILE}\")"
+    done
+    local localMatchingKeys=$(getMatchingPackagesKeys "${PROPERTY_FILE}")
+    for key in ${localMatchingKeys} ; do
+      packages="$packages $(getProperty "$key" "${PROPERTY_FILE}")"
+    done
 }
 
 function cleanup_images() {
+    local container_rm=$(getExternalImageCommand)
+	if [ "${EXTERNAL_AQA_CONTAINER_CLEAN}" == "false" ] ; then
+			echo "to debug, put '-i --entrypoint /bin/bash' before container name"
+			container_rm="echo to clean, run manually: $container_rm"
+	fi
     # Delete any old containers that have exited.
-    docker rm $(docker ps -a | grep "Exited" | awk '{ print $1 }') 2>/dev/null
+    ${container_rm} rm $(docker ps -a | grep "Exited" | awk '{ print $1 }') 2>/dev/null
 
     # Delete any old images for our target_repo on localhost.
-    docker rmi -f $(docker images | grep -e "adoptopenjdk" | awk '{ print $3 }' | sort | uniq) 2>/dev/null
+    ${container_rm} rmi -f $(docker images | grep -e "adoptopenjdk" | awk '{ print $3 }' | sort | uniq) 2>/dev/null
 }

external/criu-functional/test.properties

@@ -2,5 +2,8 @@ github_url="https://github.com/adoptium/aqa-tests.git"
 test_results="testResults"
 gradle_version="5.1"
 environment_variable="MODE=java CC=gcc-7 CXX=g++-7"
-ubuntu_packages="ant-contrib build-essential git asciidoc"
+generic_packages="git asciidoc"
+ubuntu_packages="ant-contrib build-essential"
+fedora_packages="ant-contrib"
+(fedora|ubi|rhel|centos)_packages=make automake gcc gcc-c++ kernel-devel
 criu_version="latest"

external/criu-portable-checkpoint/test.properties

@@ -2,5 +2,8 @@ github_url="https://github.com/adoptium/aqa-tests.git"
 test_results="testResults"
 gradle_version="5.1"
 environment_variable="MODE=java CC=gcc-7 CXX=g++-7"
-ubuntu_packages="ant-contrib build-essential git asciidoc"
+generic_packages="git asciidoc"
+ubuntu_packages="ant-contrib build-essential"
+fedora_packages="ant-contrib"
+(fedora|ubi|rhel|centos)_packages=make automake gcc gcc-c++ kernel-devel
 criu_version="latest"

external/criu-ubi-portable-checkpoint/test.properties

@@ -2,4 +2,4 @@ github_url="https://github.com/adoptium/aqa-tests.git"
 test_results="testResults"
 gradle_version="5.1"
 environment_variable="MODE=java"
-ubi_packages="git wget perl"
+generic_packages="git wget perl make"

external/derby/test.properties

@@ -1,4 +1,5 @@
 github_url="https://github.com/apache/derby.git"
 localPropertyFile="local.properties"
 tag_version="trunk"
-ubuntu_packages="ant-optional git wget tar"
+ubuntu_packages="ant-optional"
+generic_packages="git wget tar"

external/dockerfile_functions.sh

@@ -84,34 +84,50 @@ print_image_args() {
     local build=$7
     local base_docker_registry_dir="$8"
 
-    image_name="docker.io/library/eclipse-temurin"
-    tag=""
-    if [[ "${package}" == "jre" ]]; then
-        tag="${version}-jre"
-    else
-        tag="${version}-jdk"
-    fi
+    image_name="$(getTemurinImageName)"
+    tag="$(getTemurinImageTag "${version}" "${package}")"
     if [[ "${vm}" == "openj9" ]]; then
         if  [[ "${os}" == "ubuntu" ]]; then
-            image_name="docker.io/ibm-semeru-runtimes"
-            tag=open-${tag}
+            image_name="$(getOpenJ9ImageName)"
+            tag="$(getOpenJ9ImageTag "${version}" "${package}")"
         elif [[ "${os}" == *"ubi"* && "${test}" != *"criu"* ]]; then
+            if isExternalImageEnabled ; then
+              echo "openj9 ubi and custom EXTERNAL_AQA_IMAGE are not compatible"
+              exit 1
+            fi
             image_name="registry.access.redhat.com/$base_docker_registry_dir"
             tag="latest"
+            EXTERNAL_AQA_IMAGE="${image_name}:${tag}"
         else
+            if isExternalImageEnabled ; then
+              echo "openj9 ubi and custom EXTERNAL_AQA_IMAGE are not compatible"
+              exit 1
+            fi
             # os is ubi, and test is criu
             # temporarily all ubi based testing use internal base image
             image_name="$DOCKER_REGISTRY_URL/$base_docker_registry_dir"
             tag="latest"
+            EXTERNAL_AQA_IMAGE="${image_name}:${tag}"
         fi
     fi
     image="${image_name}:${tag}"
 
-    echo -e "ARG IMAGE=${image}" \
+    if isExternalImageEnabled ; then
+        os=$(getImageOs)
+        echo -e \
+           "ARG IMAGE=${image}" \
+          "\nARG OS=${os}" \
+          "\nARG IMAGE_VERSION=nightly" \
+          "\nARG TAG=${tag}" \
+          "\nFROM \$IMAGE\n" >> ${file}
+    else
+        echo -e \
+           "ARG IMAGE=${image}" \
           "\nARG OS=${os}" \
           "\nARG IMAGE_VERSION=nightly" \
           "\nARG TAG=${tag}" \
           "\nFROM \$IMAGE\n" >> ${file}
+    fi
 }
 
 print_test_tag_arg() {
@@ -133,8 +149,6 @@ print_result_comment_arg() {
 # Select the ubuntu OS packages
 print_ubuntu_pkg() {
     local file=$1
-    local packages=$2
-
     echo -e "RUN apt-get update \\" \
             "\n\t&& apt-get install -qq -y --no-install-recommends software-properties-common \\" \
             "\n\t&& apt-get install -qq -y --no-install-recommends gnupg \\" \
@@ -148,8 +162,6 @@ print_ubuntu_pkg() {
 # Select the ubuntu OS packages
 print_ubi_pkg() {
     local file=$1
-    local packages=$2
-
     echo -e "RUN dnf install -y ${packages} \\" \
             "\n\t&& dnf clean all " >> ${file}
     echo -e "\nENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'" >> ${file}
@@ -212,6 +224,9 @@ print_ant_install() {
     local file=$1
     local ant_version=$2
     local os=$3
+    if isExternalImageEnabled ; then
+        os=$(getImageOs)
+    fi
 
     echo -e "ARG ANT_VERSION=${ant_version}" \
           "\nENV ANT_VERSION=\$ANT_VERSION" \
@@ -501,6 +516,9 @@ print_testInfo_env() {
     local OS=$3
     local version=$4
     local vm=$5
+    if isExternalImageEnabled ; then
+        OS=$(getImageOs)
+    fi
     echo -e "ENV APPLICATION_NAME=${test}" \
             "\nENV APPLICATION_TAG=${test_tag}" \
             "\nENV OS_TAG=${OS}" \
@@ -593,7 +611,6 @@ generate_dockerfile() {
     base_docker_registry_dir="$9"
     check_external_custom_test=$10
 
-
     if [[ "$check_external_custom_test" == "1" ]]; then
         tag_version=${EXTERNAL_REPO_BRANCH}
     fi
@@ -603,7 +620,7 @@ generate_dockerfile() {
     else
         set_test_info ${test} ${check_external_custom_test}
     fi
-    packages=$(echo ${os}_packages | sed 's/-/_/')
+
     jhome="/opt/java/openjdk"
 
     mkdir -p `dirname ${file}` 2>/dev/null
@@ -612,9 +629,18 @@ generate_dockerfile() {
     print_legal ${file};
     print_adopt_test ${file} ${test};
     print_image_args ${file} ${test} ${os} ${version} ${vm} ${package} ${build} "${base_docker_registry_dir}";
+    # print_image_args is setting up hackisch EXTERNAL_AQA_IMAGE for some corner cases
+    osDeducted=$(getImageOs)
     print_result_comment_arg ${file};
     print_test_tag_arg ${file} ${test} ${tag_version};
-    print_${os}_pkg ${file} "${!packages}";
+    if echo ${osDeducted} | grep -i -e ubuntu -e debian ; then
+      print_ubuntu_pkg ${file}
+    elif echo ${osDeducted} | grep -i -e ubi -e fedora -e rhel -e centos  ; then
+      print_ubi_pkg ${file}
+    else
+      echo "unknown os: ${osDeducted} (${os})"
+      exit 1
+    fi
 
     if [[ ! -z ${ant_version} ]]; then
         print_ant_install ${file} ${ant_version} ${os};

external/elasticsearch/test.properties

@@ -1,5 +1,5 @@
 github_url="https://github.com/elastic/elasticsearch.git"
 tag_version="v8.1.2"
 test_results="testResults"
-ubuntu_packages="git wget unzip"
+generic_packages="git wget unzip"
 

external/external.sh

@@ -19,6 +19,33 @@ set -e
 
 source $(dirname "$0")/provider.sh
 
+if [ -z "${EXTRA_DOCKER_ARGS}" ] ; then
+  echo \
+"   # ================= WARNING ================= WARNING ================= WARNING ================= #
+   # EXTRA_DOCKER_ARGS are not set. You will be testing java which is already in container and not TEST_JDK_HOME
+   # TEST_JDK_HOME is set to $TEST_JDK_HOME but will not be used. See test_base_functions.sh for order of search
+   # You should set your's TEST_JDK_HOME to mount to /opt/java/openjdk, eg:                          #
+   # export EXTRA_DOCKER_ARGS=\"-v \$TEST_JDK_HOME:/opt/java/openjdk\"                               #
+   # ================= WARNING ================= WARNING ================= WARNING ================= #"
+else
+  echo \
+"   # Info Info  Info Info Info Info Info Info Info Info Info Info Info Info Info Info Info Info Info #
+   # EXTRA_DOCKER_ARGS set as \"$EXTRA_DOCKER_ARGS\"                                #
+   # Info Info  Info Info Info Info Info Info Info Info Info Info Info Info Info Info Info Info Info #"
+  if echo "${EXTRA_DOCKER_ARGS}" | grep -q "$TEST_JDK_HOME"  ; then
+    echo \
+"   # Info Info  Info Info Info Info Info Info Info Info Info Info Info Info Info Info Info Info Info #
+   # TEST_JDK_HOME of $TEST_JDK_HOME is used in EXTRA_DOCKER_ARGS                                #
+   # Info Info  Info Info Info Info Info Info Info Info Info Info Info Info Info Info Info Info Info #"
+  else
+    echo \
+"   # Warning Warning  Warning Warning Warning Warning Warning Warning Warning Warning Warning Warning #
+   # TEST_JDK_HOME of $TEST_JDK_HOME is NOT used in EXTRA_DOCKER_ARGS                                #
+   # Warning Warning  Warning Warning Warning Warning Warning Warning Warning Warning Warning Warning #"
+  fi
+fi
+
+
 tag=nightly
 docker_os=ubuntu
 build_type=full