Merging tetrate-workflow with tetrate-release-1.9 (#457)

Co-authored-by: mathetake <[email protected]>

.github/workflows/make_release.yml

@@ -4,17 +4,13 @@ on:
   workflow_dispatch: 
     inputs:
       tag:
-        description: Manual Tag [e.g. 1.7.8-tetrate-v0]
+        description: Tag [e.g. 1.7.8-tetrate-v0]
         required: true
-      type:
-        description: Build Type [fips or default]
-        required: true
-        default: "default"
 
 jobs:
   build_fips_proxy:
     # Only run this job when the release type equals "fips".
-    if: ${{ github.event.inputs.type == 'fips' }}
+    if: contains(github.event.inputs.tag, 'fips')
     name: build fips proxy binary
     timeout-minutes: 1440 # 12 hours
     runs-on: ubuntu-latest
@@ -82,7 +78,7 @@ jobs:
         run: make push_release
 
   make_release:
-    name: release-builder-run
+    name: release-builder-run (${{ github.event.inputs.tag }})
     if: ${{ ! failure() }} # Make sure that this job runs even if build == "fips".
     runs-on: ubuntu-latest
     needs: [build_fips_proxy]
@@ -115,6 +111,5 @@ jobs:
           CLOUDSMITH_USER: ${{ secrets.CLOUDSMITH_USER }}
           CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }}
           TAG: ${{ github.event.inputs.tag }}
-          BUILD: ${{ github.event.inputs.type }}
           REL_BRANCH_VER: ${{ steps.get_minor_ver.outputs.REL_BRANCH_VER }}
-        run: bash ./tetrateci/create_istio_release.sh
+        run: ./tetrateci/create_istio_release.sh

tetrateci/ci_workflow.md

@@ -0,0 +1,26 @@
+## Github Workflows
+### backport_commits.yml
+Creates a backports PR for any commit made to the `tetrate-workflow` branch to any branch which conform those regexes
+- `origin/tetrate-release-[[:digit:]]+.[[:digit:]]+$`
+- `origin/tetratefips-release-[[:digit:]]+.[[:digit:]]+$` 
+
+The script is adapted from [here](https://github.com/repo-sync/pull-request). The catch is the script only ports changes if there are on the `tetrateci` or `.github/workflows` folders. Merging with a commit or rebasing is not used to avoid merge conflicts.
+
+### sync_fork.yml
+Runs every midnight and checks if there are any new tag on `istio/istio`, if no new tags are founds the action is done. On the other hand if there are new tags, the script loops through them creating corresponding `test-tetrate-x.y.z-v0` & `test-tetratefips-x.y.z-v0` tags and merging them with `tetrate-release-x.y` & `tetratefips-release-x.y` branches.
+
+Also pushes the archives from `istio/istio` release to the cloudsmith repo.
+
+### e2e_tests.yml
+Runs if there are any tags pushed with `test-` prefix. Utilizes `istio/release-builder` to generate docker images which are to be used for subsequent testing. The one thing to keep in mind it, the same script is used for making the releases and we only differentiate that based on whether the `TEST` environment variable is defined or not.
+
+Subsequent 4 jobs runs the istio integration tests on applicable versions of eks, gke, aks and eksd, though some of them are disable for various reasons for now. All the platforms have corresponding create and cleanup scripts. The `version_check.py` has a matrix of istio versions vs k8s versions which determines which versions we need to get the istio release tested on.
+
+All the minor versions of istio have their own testing scripts, the reason being there are patches which need to be applied before testing so the tests dont fail. The tests being written for `kind` have some default assumptions which might not be applicable for all the platforms we test on.
+
+After the tests pass, the `test-` prefix is stripped off the current tag and the tree is tagged with the remaining, cutting a release with something similar to `tetrate-x.y.z-vn`.
+
+Then the release builder is again triggered to create the release images and archives with `tetrate-x.y.z-vn` tag but this time without defining the `TEST` environment variable. A fips compliant build is only triggered if the tag contains `fips` in it. The only difference between fips and non fips build is the `Go` we are using. The `create_istio_release.sh` script sets up the environment manually instead if using the docker image is due to some restrictions in the Github Actions, it becomes a bit hard to procure the logs and monitor the whole process.
+
+### make_release.yml
+It is same as the last process of `e2e_tests.yml` but with a manual trigger. Changes made to any of them must be backported to the other one, since they more or less do the same thing.

tetrateci/create_istio_release.sh

@@ -6,16 +6,14 @@ set -o pipefail
 BASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
 
 ## Set up apporiate go version
-if [[ ${BUILD} == "fips" ]]; then
+if [[ ${TAG} =~ "fips" ]]; then
+    echo "Set up FIPS compliant Golang"
     source ${BASEDIR}/tetrateci/setup_boring_go.sh
 else
+    echo "Set up Golang"
     source ${BASEDIR}/tetrateci/setup_go.sh
 fi
 
-# the go we just installed
-CUSTOM_GO_VERSION=$GOLANG_VERSION
-echo "Go version installed: $CUSTOM_GO_VERSION"
-
 ## Set up release-builder
 sudo gem install fpm
 sudo apt-get install go-bindata -y
@@ -39,7 +37,7 @@ export BUILD_WITH_CONTAINER=0
 # HACK : For FIPS change the distroless base image to include glibc
 # We would use the same distroless base image as istio-proxy for pilot and operator
 # HACK : change envoy/wasm base URL to point to FIPS compliant one
-if [[ ${BUILD} == "fips" ]]; then
+if [[ ${TAG} =~ "fips" ]]; then
 	PROXY_DISTROLESS_BASE=$(grep 'as distroless' ${BASEDIR}/pilot/docker/Dockerfile.proxyv2)
 	# Escape '/'
 	PROXY_DISTROLESS_BASE_ESCAPED=$(sed 's/\//\\\//g' <<< ${PROXY_DISTROLESS_BASE})
@@ -76,12 +74,12 @@ CONTAINER_ID=$(docker create $HUB/pilot:$TAG)
 docker cp $CONTAINER_ID:/usr/local/bin/pilot-discovery pilot-bin
 # go version with which the binaries for the docker images wi
 BUILD_GO_VERSION=$(go version pilot-bin | cut -f2 -d" ")
-echo "Images are built with: $BUILD_GO_VERSION"
+echo "Images are built with: go $BUILD_GO_VERSION"
 
-[ $BUILD_GO_VERSION == go$CUSTOM_GO_VERSION ] || exit 1
+[ $BUILD_GO_VERSION == go$GOLANG_VERSION ] || exit 1
 
 # fips go versions are like 1.14.12b5, extra checking to not miss anything
-if [ $BUILD == "fips" ]; then 
+if [ ${TAG} =~ "fips" ]; then 
     [[ $BUILD_GO_VERSION =~ 1.[0-9]+.[0-9]+[a-z][0-9]$ ]] || exit 1
 fi
 
@@ -94,7 +92,7 @@ if [[ -z $TEST ]]; then
     echo "Building archives..."
     mkdir /tmp/istio-release
 	# if FIPS, need to use native go as boringgo as of now can't build archives for different platforms
-    if [[ ${BUILD} == "fips" ]]; then
+    if [[ ${TAG} =~ "fips" ]]; then
         sudo rm -rf /usr/local/go
         source ${BASEDIR}/tetrateci/setup_go.sh
     fi

tetrateci/setup_boring_go.sh

@@ -13,10 +13,9 @@ if $(grep -q "1.10" <<< $TAG); then
   export GOLANG_VERSION=1.16.4b7
 fi
 
-echo "Fetching FIPS compliant Go"
 url="https://go-boringcrypto.storage.googleapis.com/go$GOLANG_VERSION.linux-amd64.tar.gz"
 
-wget -O go.tgz "$url"
+wget -q -O go.tgz "$url"
 
 sudo tar -C /usr/local -xzf go.tgz
 rm go.tgz

tetrateci/setup_go.sh

@@ -13,10 +13,9 @@ if $(grep -q "1.10" <<< $TAG); then
     export GOLANG_VERSION=1.16.4
 fi
 
-echo "Fetching Go $GOLANG_VERSION"
 url="https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz"
 
-wget -O go.tgz "$url"
+wget -q -O go.tgz "$url"
 
 sudo tar -C /usr/local -xzf go.tgz
 rm go.tgz