diff --git a/.gitignore b/.gitignore index 439be58..61695ec 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /vendor/ /metac /hack/bin/ +/test/integration/framework/assets/bin/ /manifests/crds/ .*.swp .history diff --git a/Dockerfile b/Dockerfile index 3cf1fbd..65b06b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,7 @@ RUN make vendor # copy build manifests COPY . . +RUN make unit-test RUN make integration-test # Build metac binary diff --git a/Dockerfile.debug b/Dockerfile.debug index a309506..a9d4f6d 100644 --- a/Dockerfile.debug +++ b/Dockerfile.debug @@ -21,6 +21,7 @@ RUN make vendor # copy build manifests COPY . . +RUN make unit-test RUN make integration-test # Build metac binary diff --git a/Makefile b/Makefile index c0d5cf6..a630ea6 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,9 @@ PATH := $(PWD)/hack/bin:$(PATH) PACKAGE_NAME := openebs.io/metac API_GROUPS := metacontroller/v1alpha1 +GIT_TAGS = $(shell git fetch --all --tags) PACKAGE_VERSION ?= $(shell git describe --always --tags) + OS = $(shell uname) PKGS = $(shell go list ./... | grep -v '/test/integration/\|/examples/') @@ -30,8 +32,6 @@ export GO111MODULE=on CRD_OPTIONS ?= "crd:trivialVersions=true" CONTROLLER_GEN := go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go -export GO111MODULE=on - all: manifests bins bins: generated_files $(IMG_NAME) @@ -80,8 +80,8 @@ push: image .PHONY: unit-test unit-test: generated_files - @go test -cover -mod=vendor -i ${PKGS} - @go test -cover -mod=vendor ${PKGS} + @go test -cover -i ${PKGS} + @go test -cover ${PKGS} .PHONY: integration-dependencies integration-dependencies: manifests @@ -92,18 +92,15 @@ integration-dependencies: manifests # This can be run on one's laptop or Travis like CI environments. .PHONY: integration-test integration-test: integration-dependencies - @go test -mod=vendor \ - ./test/integration/... \ - -v -short -timeout 5m -args --logtostderr -v=1 + @go test ./test/integration/... \ + -v -timeout 5m -args --logtostderr --alsologtostderr -v=1 .PHONY: integration-test-gctl integration-test-gctl: integration-dependencies - @go test -mod=vendor \ - ./test/integration/generic/... \ - -v -timeout 5m -args --logtostderr -v=1 + @go test ./test/integration/generic/... \ + -v -timeout=5m -args --logtostderr --alsologtostderr -v=1 .PHONY: integration-test-local-gctl integration-test-local-gctl: integration-dependencies - @go test -mod=vendor \ - ./test/integration/genericlocal/... \ - -v -timeout 5m -args --logtostderr -v=1 \ No newline at end of file + @go test ./test/integration/genericlocal/... \ + -v -timeout 5m -args --logtostderr --alsologtostderr -v=1 \ No newline at end of file diff --git a/go.mod b/go.mod index 073742a..4727512 100644 --- a/go.mod +++ b/go.mod @@ -1,18 +1,28 @@ module openebs.io/metac -// This denotes the minimum supported language version and should not include the patch version. +// This denotes the minimum supported language version and +// should not include the patch version. go 1.13 require ( contrib.go.opencensus.io/exporter/prometheus v0.1.0 github.com/coreos/etcd v3.3.15+incompatible // indirect + github.com/evanphx/json-patch v4.5.0+incompatible // indirect github.com/ghodss/yaml v1.0.0 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b + github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect github.com/google/go-cmp v0.3.0 github.com/google/go-jsonnet v0.14.0 + github.com/googleapis/gnostic v0.3.1 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/onsi/ginkgo v1.11.0 // indirect + github.com/onsi/gomega v1.8.1 github.com/pkg/errors v0.8.1 go.opencensus.io v0.21.0 - gopkg.in/yaml.v2 v2.2.4 + golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect + golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/yaml.v2 v2.2.7 k8s.io/api v0.17.0 k8s.io/apiextensions-apiserver v0.17.0 k8s.io/apimachinery v0.17.0 diff --git a/go.sum b/go.sum index e33053b..f50bbf2 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,7 @@ github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= @@ -66,6 +67,8 @@ github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -142,6 +145,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -168,6 +173,8 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= +github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -185,6 +192,8 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -247,10 +256,14 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= +github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -327,10 +340,13 @@ go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qL go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -341,6 +357,8 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495 h1:I6A9Ag9FpEKOjcKrRNjQkPHawoXIhKyTGfvvjFAiiAk= @@ -400,6 +418,8 @@ golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -427,6 +447,7 @@ golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72 h1:bw9doJza/SFBEweII/rHQh338oozWyiFsBRHtrflcws= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485 h1:OB/uP/Puiu5vS5QMRPrXCDWUPb+kt8f1KW8oQzFejQw= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= @@ -449,6 +470,8 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -465,6 +488,8 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966 h1:B0J02caTR6tpSJozBJyiAzT6CtBzjclw4pgm9gg8Ys0= gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= diff --git a/hack/get-kube-binaries.sh b/hack/get-kube-binaries.sh index 3935618..50c39dd 100755 --- a/hack/get-kube-binaries.sh +++ b/hack/get-kube-binaries.sh @@ -5,12 +5,11 @@ set -u echo "" echo "+++ Will install binaries required to run integration test(s)" -echo "+++ NOTE: Remove binaries from hack/bin if new versions are needed" echo "" # This script downloads kubectl, kube-apiserver & etcd binaries # that are used as part of the integration test environment, -# and places them in hack/bin/. +# and places them in 'test/integration/framework/assets/bin' # # The integration test framework expects these binaries to be # found in the PATH. @@ -23,8 +22,7 @@ KUBERNETES_RELEASE_URL="${KUBERNETES_RELEASE_URL:-https://dl.k8s.io}" # kubernetes/hack/lib/etcd.sh as of the above Kubernetes version. ETCD_VERSION="${ETCD_VERSION:-v3.4.3}" -mkdir -p hack/bin -cd hack/bin +cd test/integration/framework/assets/bin # Download kubectl. # uncomment below for mandatory download diff --git a/test/integration/composite/composite_test.go b/test/integration/composite/composite_test.go index de291be..3f24b12 100644 --- a/test/integration/composite/composite_test.go +++ b/test/integration/composite/composite_test.go @@ -27,21 +27,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/json" + "k8s.io/klog" "openebs.io/metac/apis/metacontroller/v1alpha1" "openebs.io/metac/controller/composite" "openebs.io/metac/test/integration/framework" ) -// This will be run only once when go test is invoked against this package. -// All the other TestXYZ functions will be invoked via m.Run call only. -// -// framework.TestMain provides setup & teardown features required for -// all the individual testcases to run. -func TestMain(m *testing.M) { - framework.TestWithCRDMetac(m.Run) -} - // TestSyncWebhook tests that the sync webhook triggers and passes the // request/response properly. func TestSyncWebhook(t *testing.T) { @@ -96,36 +88,49 @@ func TestSyncWebhook(t *testing.T) { parentResource := framework.BuildUnstructObjFromCRD(parentCRD, testName) unstructured.SetNestedStringMap( - parentResource.Object, labels, "spec", "selector", "matchLabels", + parentResource.Object, + labels, + "spec", + "selector", + "matchLabels", ) - t.Logf( + klog.Infof( "Creating %s %s/%s", parentResource.GetKind(), parentResource.GetNamespace(), parentResource.GetName(), ) - _, err := - parentClient.Namespace(testName).Create(parentResource, metav1.CreateOptions{}) + _, err := parentClient. + Namespace(testName). + Create( + parentResource, + metav1.CreateOptions{}, + ) if err != nil { t.Fatal(err) } - t.Logf( + klog.Infof( "Created %s %s/%s", parentResource.GetKind(), parentResource.GetNamespace(), parentResource.GetName(), ) - t.Logf("Waiting for child sync") + klog.Infof("Waiting for child sync") err = f.Wait(func() (bool, error) { - _, err := childClient.Namespace(testName).Get(testName, metav1.GetOptions{}) + _, err := childClient. + Namespace(testName). + Get( + testName, + metav1.GetOptions{}, + ) return err == nil, err }) if err != nil { t.Errorf("Child sync failed: %v", err) } - t.Logf("Child sync was successful") + klog.Infof("Child sync was successful") } // TestCascadingDelete tests that we request cascading deletion of children, @@ -144,7 +149,8 @@ func TestCacadingDelete(t *testing.T) { f.CreateNamespace(testName) parentCRD, parentClient := f.SetupCRD( - "CCtlCascadingDeleteParent", apiextensions.NamespaceScoped, + "CCtlCascadingDeleteParent", + apiextensions.NamespaceScoped, ) jobChildClient := f.GetTypedClientset().BatchV1().Jobs(testName) @@ -156,8 +162,11 @@ func TestCacadingDelete(t *testing.T) { } resp := composite.SyncHookResponse{} - replicas, _, _ := - unstructured.NestedInt64(req.Parent.Object, "spec", "replicas") + replicas, _, _ := unstructured.NestedInt64( + req.Parent.Object, + "spec", + "replicas", + ) if replicas > 0 { // Create a child batch/v1 Job if requested. // For backward compatibility, the server-side default on that API is @@ -194,35 +203,51 @@ func TestCacadingDelete(t *testing.T) { testName, hook.URL, framework.BuildResourceRuleFromCRD(parentCRD), - &v1alpha1.ResourceRule{APIVersion: "batch/v1", Resource: "jobs"}, + &v1alpha1.ResourceRule{ + APIVersion: "batch/v1", + Resource: "jobs", + }, ) parentResource := framework.BuildUnstructObjFromCRD(parentCRD, testName) unstructured.SetNestedStringMap( - parentResource.Object, labels, "spec", "selector", "matchLabels", + parentResource.Object, + labels, + "spec", + "selector", + "matchLabels", + ) + unstructured.SetNestedField( + parentResource.Object, + int64(1), + "spec", + "replicas", ) - unstructured.SetNestedField(parentResource.Object, int64(1), "spec", "replicas") - t.Logf( + klog.Infof( "Creating %s %s/%s", parentResource.GetKind(), parentResource.GetNamespace(), parentResource.GetName(), ) var err error - parentResource, err = - parentClient.Namespace(testName).Create(parentResource, metav1.CreateOptions{}) + parentResource, err = parentClient. + Namespace(testName). + Create( + parentResource, + metav1.CreateOptions{}, + ) if err != nil { t.Fatal(err) } - t.Logf( + klog.Infof( "Created %s %s/%s", parentResource.GetKind(), parentResource.GetNamespace(), parentResource.GetName(), ) - t.Logf("Waiting for child job creation") + klog.Infof("Waiting for child job creation") err = f.Wait(func() (bool, error) { _, err := jobChildClient.Get(testName, metav1.GetOptions{}) return err == nil, err @@ -230,10 +255,10 @@ func TestCacadingDelete(t *testing.T) { if err != nil { t.Errorf("Child job create failed: %v", err) } - t.Logf("Child job was created successfully") + klog.Infof("Child job was created successfully") // Now that child exists, tell parent to delete it. - t.Logf("Updating parent with replicas=0") + klog.Infof("Updating parent with replicas=0") _, err = parentClient.Namespace(testName).AtomicUpdate(parentResource, func(obj *unstructured.Unstructured) bool { unstructured.SetNestedField(obj.Object, int64(0), "spec", "replicas") @@ -242,13 +267,13 @@ func TestCacadingDelete(t *testing.T) { if err != nil { t.Fatal(err) } - t.Logf("Updated parent with replicas=0") + klog.Infof("Updated parent with replicas=0") // Make sure the child gets actually deleted, which means no GC finalizers got // added to it. Note that we don't actually run the GC in this integration // test env, so we don't need to worry about the GC racing us to process the // finalizers. - t.Logf("Waiting for child job to be deleted") + klog.Infof("Waiting for child job to be deleted") var child *batchv1.Job err = f.Wait(func() (bool, error) { var getErr error @@ -259,7 +284,7 @@ func TestCacadingDelete(t *testing.T) { out, _ := json.Marshal(child) t.Errorf("Child job delete failed: %v; object: %s", err, out) } - t.Logf("Child job deleted successfully") + klog.Infof("Child job deleted successfully") } // TestResyncAfter tests that the resyncAfterSeconds field works. @@ -277,7 +302,8 @@ func TestResyncAfter(t *testing.T) { f.CreateNamespace(testName) parentCRD, parentClient := f.SetupCRD( - "CCtlResyncAfterParent", apiextensions.NamespaceScoped, + "CCtlResyncAfterParent", + apiextensions.NamespaceScoped, ) var lastSync time.Time @@ -323,10 +349,14 @@ func TestResyncAfter(t *testing.T) { parentResource := framework.BuildUnstructObjFromCRD(parentCRD, testName) unstructured.SetNestedStringMap( - parentResource.Object, labels, "spec", "selector", "matchLabels", + parentResource.Object, + labels, + "spec", + "selector", + "matchLabels", ) - t.Logf( + klog.Infof( "Creating %s %s/%s", parentResource.GetKind(), parentResource.GetNamespace(), @@ -337,14 +367,14 @@ func TestResyncAfter(t *testing.T) { if err != nil { t.Fatal(err) } - t.Logf( + klog.Infof( "Created %s %s/%s", parentResource.GetKind(), parentResource.GetNamespace(), parentResource.GetName(), ) - t.Logf("Waiting for status.elaspedSeconds to be reported") + klog.Infof("Waiting for status.elaspedSeconds to be reported") var elapsedSeconds float64 err = f.Wait(func() (bool, error) { parentResource, err := @@ -354,7 +384,9 @@ func TestResyncAfter(t *testing.T) { } val, found, err := unstructured.NestedFloat64( - parentResource.Object, "status", "elapsedSeconds", + parentResource.Object, + "status", + "elapsedSeconds", ) if err != nil || !found { // The value hasn't been populated. Keep waiting. @@ -367,7 +399,7 @@ func TestResyncAfter(t *testing.T) { if err != nil { t.Fatalf("Didn't find status.elapsedSeconds: %v", err) } - t.Logf("status.elapsedSeconds is %v", elapsedSeconds) + klog.Infof("status.elapsedSeconds is %v", elapsedSeconds) if elapsedSeconds > 1.0 { t.Errorf( diff --git a/test/integration/composite/suite_test.go b/test/integration/composite/suite_test.go new file mode 100644 index 0000000..f105bd1 --- /dev/null +++ b/test/integration/composite/suite_test.go @@ -0,0 +1,60 @@ +/* +Copyright 2019 The MayaData Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package composite + +import ( + "flag" + "testing" + + "k8s.io/klog" + "openebs.io/metac/test/integration/framework" +) + +// TestMain will run only once when go test is invoked +// against this package. All the other Test* functions +// will be invoked via m.Run call. +// +// NOTE: +// There can be only one TestMain function in the entire +// package +func TestMain(m *testing.M) { + flag.Parse() + + klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) + klog.InitFlags(klogFlags) + + // Sync the glog and klog flags. + flag.CommandLine.VisitAll(func(f1 *flag.Flag) { + f2 := klogFlags.Lookup(f1.Name) + if f2 != nil { + value := f1.Value.String() + f2.Value.Set(value) + } + }) + + // Pass m.Run function to framework which in turn + // sets up a kubernetes environment & then invokes + // m.Run. + err := framework.StartCRDBasedMetac(m.Run) + if err != nil { + // Since this is an error we must to invoke os.Exit(1) + // as per TestMain guidelines + klog.Exitf("%+v", err) + } + + defer klog.Flush() +} diff --git a/test/integration/decorator/decorator_test.go b/test/integration/decorator/decorator_test.go index b0ddbf9..038ec0a 100644 --- a/test/integration/decorator/decorator_test.go +++ b/test/integration/decorator/decorator_test.go @@ -25,20 +25,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/json" + "k8s.io/klog" "openebs.io/metac/controller/decorator" "openebs.io/metac/test/integration/framework" ) -// This will be run only once when go test is invoked against this package. -// All the other TestXYZ functions will be invoked via m.Run call only. -// -// framework.TestMain provides setup & teardown features required for -// all the individual testcases to run. -func TestMain(m *testing.M) { - framework.TestWithCRDMetac(m.Run) -} - // TestSyncWebhook tests that the sync webhook triggers and passes the // request/response properly. func TestSyncWebhook(t *testing.T) { @@ -56,8 +48,14 @@ func TestSyncWebhook(t *testing.T) { f.CreateNamespace(testName) // Setup namespace scoped CRDs - parentCRD, parentClient := f.SetupCRD("DCtlSyncParent", apiextensions.NamespaceScoped) - childCRD, childClient := f.SetupCRD("DCtlSyncChild", apiextensions.NamespaceScoped) + parentCRD, parentClient := f.SetupCRD( + "DCtlSyncParent", + apiextensions.NamespaceScoped, + ) + childCRD, childClient := f.SetupCRD( + "DCtlSyncChild", + apiextensions.NamespaceScoped, + ) // define the "reconcile logic" i.e. sync hook logic here hook := f.ServeWebhook(func(body []byte) ([]byte, error) { @@ -92,28 +90,36 @@ func TestSyncWebhook(t *testing.T) { parentResource := framework.BuildUnstructObjFromCRD(parentCRD, testName) unstructured.SetNestedStringMap( - parentResource.Object, labels, "spec", "selector", "matchLabels", + parentResource.Object, + labels, + "spec", + "selector", + "matchLabels", ) - t.Logf( + klog.Infof( "Creating %s %s/%s", parentResource.GetKind(), parentResource.GetNamespace(), parentResource.GetName(), ) - _, err := - parentClient.Namespace(testName).Create(parentResource, metav1.CreateOptions{}) + _, err := parentClient. + Namespace(testName). + Create( + parentResource, + metav1.CreateOptions{}, + ) if err != nil { t.Fatal(err) } - t.Logf( + klog.Infof( "Created %s %s/%s", parentResource.GetKind(), parentResource.GetNamespace(), parentResource.GetName(), ) - t.Logf("Waiting for child sync") + klog.Infof("Waiting for child sync") err = f.Wait(func() (bool, error) { _, err = childClient.Namespace(testName).Get(testName, metav1.GetOptions{}) @@ -122,7 +128,7 @@ func TestSyncWebhook(t *testing.T) { if err != nil { t.Errorf("Child sync failed: %v", err) } - t.Logf("Child sync was successful") + klog.Infof("Child sync was successful") } // TestResyncAfter tests that the resyncAfterSeconds field works. @@ -140,7 +146,8 @@ func TestResyncAfter(t *testing.T) { f.CreateNamespace(testName) parentCRD, parentClient := f.SetupCRD( - "DCtlResyncAfterParent", apiextensions.NamespaceScoped, + "DCtlResyncAfterParent", + apiextensions.NamespaceScoped, ) var lastSync time.Time @@ -188,10 +195,13 @@ func TestResyncAfter(t *testing.T) { parentResource := framework.BuildUnstructObjFromCRD(parentCRD, testName) unstructured.SetNestedStringMap( - parentResource.Object, labels, "spec", "selector", "matchLabels", + parentResource.Object, + labels, + "spec", + "selector", "matchLabels", ) - t.Logf( + klog.Infof( "Creating %s %s/%s", parentResource.GetKind(), parentResource.GetNamespace(), @@ -202,14 +212,14 @@ func TestResyncAfter(t *testing.T) { if err != nil { t.Fatal(err) } - t.Logf( + klog.Infof( "Created %s %s/%s", parentResource.GetKind(), parentResource.GetNamespace(), parentResource.GetName(), ) - t.Logf("Waiting for status.elapsedSeconds to be reported") + klog.Infof("Waiting for status.elapsedSeconds to be reported") var elapsedSeconds float64 err = f.Wait(func() (bool, error) { parentResource, err := @@ -231,7 +241,7 @@ func TestResyncAfter(t *testing.T) { if err != nil { t.Fatalf("Didn't find status.elapsedSeconds: %v", err) } - t.Logf("status.elapsedSeconds is %v", elapsedSeconds) + klog.Infof("status.elapsedSeconds is %v", elapsedSeconds) if elapsedSeconds > 1.0 { t.Errorf( diff --git a/test/integration/decorator/suite_test.go b/test/integration/decorator/suite_test.go new file mode 100644 index 0000000..49b0b98 --- /dev/null +++ b/test/integration/decorator/suite_test.go @@ -0,0 +1,60 @@ +/* +Copyright 2019 The MayaData Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package decorator + +import ( + "flag" + "testing" + + "k8s.io/klog" + "openebs.io/metac/test/integration/framework" +) + +// TestMain will run only once when go test is invoked +// against this package. All the other Test* functions +// will be invoked via m.Run call. +// +// NOTE: +// There can be only one TestMain function in the entire +// package +func TestMain(m *testing.M) { + flag.Parse() + + klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) + klog.InitFlags(klogFlags) + + // Sync the glog and klog flags. + flag.CommandLine.VisitAll(func(f1 *flag.Flag) { + f2 := klogFlags.Lookup(f1.Name) + if f2 != nil { + value := f1.Value.String() + f2.Value.Set(value) + } + }) + + // Pass m.Run function to framework which in turn + // sets up a kubernetes environment & then invokes + // m.Run. + err := framework.StartCRDBasedMetac(m.Run) + if err != nil { + // Since this is an error we must to invoke os.Exit(1) + // as per TestMain guidelines + klog.Exitf("%+v", err) + } + + defer klog.Flush() +} diff --git a/test/integration/framework/README.md b/test/integration/framework/README.md new file mode 100644 index 0000000..873686c --- /dev/null +++ b/test/integration/framework/README.md @@ -0,0 +1,5 @@ +# Integration Testing Framework + +This package has been taken from [https://github.com/kubernetes-sigs/controller-runtime/tree/master/pkg/internal/testing/integration](https://github.com/kubernetes-sigs/controller-runtime/tree/master/pkg/internal/testing/integration). + +A framework for integration testing components of kubernetes. This framework is intended to work properly both in CI, and on a local dev machine. It therefore explicitly supports both Linux and Darwin. \ No newline at end of file diff --git a/test/integration/framework/addr/manager.go b/test/integration/framework/addr/manager.go new file mode 100644 index 0000000..6a3cecd --- /dev/null +++ b/test/integration/framework/addr/manager.go @@ -0,0 +1,74 @@ +package addr + +import ( + "fmt" + "net" + "sync" + "time" +) + +const ( + portReserveTime = 1 * time.Minute + portConflictRetry = 100 +) + +type portCache struct { + lock sync.Mutex + ports map[int]time.Time +} + +func (c *portCache) add(port int) bool { + c.lock.Lock() + defer c.lock.Unlock() + // remove outdated port + for p, t := range c.ports { + if time.Since(t) > portReserveTime { + delete(c.ports, p) + } + } + // try allocating new port + if _, ok := c.ports[port]; ok { + return false + } + c.ports[port] = time.Now() + return true +} + +var cache = &portCache{ + ports: make(map[int]time.Time), +} + +func suggest() (port int, resolvedHost string, err error) { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + return + } + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return + } + port = l.Addr().(*net.TCPAddr).Port + defer func() { + err = l.Close() + }() + resolvedHost = addr.IP.String() + return +} + +// Suggest suggests an address a process can listen on. It returns +// a tuple consisting of a free port and the hostname resolved to its IP. +// It makes sure that new port allocated does not conflict with old ports +// allocated within 1 minute. +func Suggest() (port int, resolvedHost string, err error) { + for i := 0; i < portConflictRetry; i++ { + port, resolvedHost, err = suggest() + if err != nil { + return + } + if cache.add(port) { + return + } + } + err = fmt.Errorf("no free ports found after %d retries", portConflictRetry) + return +} diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index f925ca7..ab14401 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -1,123 +1,194 @@ -/* -Copyright 2019 Google Inc. -Copyright 2019 The MayaData Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package framework -/* -This file replaces the mechanism for starting kube-apiserver used in -k8s.io/kubernetes integration tests. In k8s.io/kubernetes, the apiserver is -one of the components being tested, so it makes sense that there we build it -from scratch and link it into the test binary. However, here we treat the -apiserver as an external component just like etcd. This avoids having to vendor -and build all of Kubernetes into our test binary. -*/ - import ( - "context" - "fmt" + "io" "io/ioutil" + "net/url" "os" - "os/exec" - "strconv" + "path/filepath" + "time" "github.com/pkg/errors" - "k8s.io/client-go/rest" - "k8s.io/klog" + "openebs.io/metac/test/integration/framework/addr" + "openebs.io/metac/test/integration/framework/internal" ) -var apiserverURL = "" +// APIServer knows how to run a kubernetes apiserver. +type APIServer struct { + // URL is the address the ApiServer should listen on + // for client connections. + // + // If this is not specified, we default to a random + // free port on localhost. + URL *url.URL + + // SecurePort is the additional secure port that the + // APIServer should listen on. + SecurePort int + + // Path is the path to the apiserver binary. + // + // If this is left as the empty string, we will attempt + // to locate a binary, by checking for the TEST_ASSET_KUBE_APISERVER + // environment variable, and the default test assets directory. + // See the "Binaries" section above (in doc.go) for details. + Path string + + // Args is a list of arguments which will passed to the + // APIServer binary. Before they are passed on, they will + // be evaluated as go-template strings. This means you can + // use fields which are defined and exported on this + // APIServer struct (e.g. "--cert-dir={{ .Dir }}"). + // Those templates will be evaluated after the defaulting of + // the APIServer's fields has already happened and just before + // the binary actually gets started. Thus you have access to + // calculated fields like `URL` and others. + // + // If not specified, the minimal set of arguments to run the + // APIServer will be used. + Args []string + + // CertDir is a path to a directory containing whatever + // certificates the APIServer will need. + // + // If left unspecified, then the Start() method will create + // a fresh temporary directory, and the Stop() method will + // clean it up. + CertDir string + + // EtcdURL is the URL of the Etcd the APIServer should use. + // + // If this is not specified, the Start() method will return + // an error. + EtcdURL *url.URL + + // StartTimeout, StopTimeout specify the time the APIServer + // is allowed to take when starting and stoppping before an + // error is emitted. + // + // If not specified, these default to 20 seconds. + StartTimeout time.Duration + StopTimeout time.Duration + + // Out, Err specify where APIServer should write its StdOut, + // StdErr to. + // + // If not specified, the output will be discarded. + Out io.Writer + Err io.Writer + + processState *internal.ProcessState +} -const installApiserver = ` -Cannot find kube-apiserver, cannot run integration tests +// Start starts the apiserver, waits for it to come up, and returns +// an error, if occurred. +func (s *APIServer) Start() error { + if s.processState == nil { + if err := s.setProcessState(); err != nil { + return err + } + } + return s.processState.Start(s.Out, s.Err) +} -Please download kube-apiserver and ensure it is somewhere in the PATH. -See hack/get-kube-binaries.sh +func (s *APIServer) setProcessState() error { + if s.EtcdURL == nil { + return errors.Errorf("Expected EtcdURL to be configured") + } -` + var err error -// getApiserverPath returns a path to a kube-apiserver executable. -func getApiserverPath() (string, error) { - return exec.LookPath("kube-apiserver") -} + s.processState = &internal.ProcessState{} -// startApiserver executes a kube-apiserver instance. -// The returned function will signal the process and wait for it to exit. -func startApiserver() (func(), error) { - apiserverPath, err := getApiserverPath() + s.processState.DefaultedProcessInput, err = internal.DoDefaulting( + "kube-apiserver", + s.URL, + s.CertDir, + s.Path, + s.StartTimeout, + s.StopTimeout, + ) if err != nil { - fmt.Fprintf(os.Stderr, installApiserver) - return nil, errors.Wrapf(err, "Can't find kube-apiserver in PATH") + return err + } + + // Defaulting the secure port + if s.SecurePort == 0 { + s.SecurePort, _, err = addr.Suggest() + if err != nil { + return err + } } - apiserverPort, err := getAvailablePort() + + s.processState.HealthCheckEndpoint = "/healthz" + + s.URL = &s.processState.URL + s.CertDir = s.processState.Dir + s.Path = s.processState.Path + s.StartTimeout = s.processState.StartTimeout + s.StopTimeout = s.processState.StopTimeout + + if err := s.populateAPIServerCerts(); err != nil { + return err + } + + s.processState.Args, err = internal.RenderTemplates( + internal.DoAPIServerArgDefaulting(s.Args), s, + ) + return err +} + +func (s *APIServer) populateAPIServerCerts() error { + _, statErr := os.Stat(filepath.Join(s.CertDir, "apiserver.crt")) + if !os.IsNotExist(statErr) { + return statErr + } + + ca, err := internal.NewTinyCA() if err != nil { - return nil, err + return err } - apiserverURL = fmt.Sprintf("http://127.0.0.1:%d", apiserverPort) - klog.Infof("Starting kube-apiserver on %s", apiserverURL) - apiserverDataDir, err := - ioutil.TempDir(os.TempDir(), "integration_test_apiserver_data") + certs, err := ca.NewServingCert() if err != nil { - return nil, - errors.Wrapf(err, "Can't make temp kube-apiserver data dir") + return err } - klog.Infof("Storing kube-apiserver data in: %v", apiserverDataDir) - - ctx, cancel := context.WithCancel(context.Background()) - cmd := exec.CommandContext( - ctx, - apiserverPath, - "--cert-dir", apiserverDataDir, - // Disable secure port since we don't use it, so we don't conflict with other apiservers. - "--secure-port", "0", - "--insecure-port", strconv.Itoa(apiserverPort), - "--etcd-servers", etcdURL, - ) - // Uncomment these to see kube-apiserver output in test logs. - // For Metacontroller tests, we generally don't expect problems at this level. - //cmd.Stdout = os.Stdout - //cmd.Stderr = os.Stderr - - stop := func() { - klog.Infof("Stopping kube-apiserver") - cancel() - err := cmd.Wait() - klog.Infof("kube-apiserver exit status: %v", err) - err = os.RemoveAll(apiserverDataDir) - if err != nil { - klog.Warningf("Cleanup of kube-apiserver failed: %v", err) - } + certData, keyData, err := certs.AsBytes() + if err != nil { + return err } - if err := cmd.Start(); err != nil { - return nil, errors.Wrapf(err, "Failed to run kube-apiserver") + if err := ioutil.WriteFile( + filepath.Join(s.CertDir, "apiserver.crt"), + certData, + 0640, + ); err != nil { + return err + } + if err := ioutil.WriteFile( + filepath.Join(s.CertDir, "apiserver.key"), + keyData, + 0640, + ); err != nil { + return err } - return stop, nil -} -// ApiserverURL returns the URL of the kube-apiserver instance started by TestMain. -func ApiserverURL() string { - return apiserverURL + return nil } -// ApiserverConfig returns a rest.Config to connect to the test instance. -func ApiserverConfig() *rest.Config { - return &rest.Config{ - Host: ApiserverURL(), +// Stop stops this process gracefully, waits for its termination, +// and cleans up the CertDir if necessary. +func (s *APIServer) Stop() error { + if s.processState != nil { + return s.processState.Stop() } + return nil } + +// APIServerDefaultArgs exposes the default args for the APIServer +// so that you can use those to append your own additional arguments. +// +// The internal default arguments are explicitly copied here, we +// don't want to allow users to change the internal ones. +var APIServerDefaultArgs = append([]string{}, internal.APIServerDefaultArgs...) diff --git a/test/integration/framework/assets/bin/.gitkeep b/test/integration/framework/assets/bin/.gitkeep new file mode 100644 index 0000000..ba08892 --- /dev/null +++ b/test/integration/framework/assets/bin/.gitkeep @@ -0,0 +1 @@ +This directory will be the home of some binaries which are downloaded with `make integration-test` diff --git a/test/integration/framework/control_plane.go b/test/integration/framework/control_plane.go new file mode 100644 index 0000000..dd85a2a --- /dev/null +++ b/test/integration/framework/control_plane.go @@ -0,0 +1,133 @@ +package framework + +import ( + "fmt" + "io" + "net/url" + "time" + + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + + "openebs.io/metac/test/integration/framework/internal" +) + +// NewTinyCA creates a new a tiny CA utility for provisioning +// serving certs and client certs FOR TESTING ONLY. +// Don't use this for anything else! +var NewTinyCA = internal.NewTinyCA + +// ControlPlane is a struct that knows how to start your test +// control plane. +// +// Right now, that means Etcd and your APIServer. This is likely +// to increase in future. +type ControlPlane struct { + APIServer *APIServer + Etcd *Etcd + + // Path is the path to the apiserver, etcd & kubectl binaries + // + // If this is left as the empty string, we will attempt to + // locate a binary by checking for respective environment + // variables. + Path string + + // Out, Err specify where ControlPlane should write its StdOut, + // StdErr to. + // + // If not specified, the output will be discarded. + Out io.Writer + Err io.Writer + + // StartTimeout, StopTimeout specify the time the process is + // allowed to take when starting and stopping before an error + // is emitted. + // + // If not specified, these default to 20 seconds. + StartTimeout time.Duration + StopTimeout time.Duration +} + +// Start will start your control plane processes. To stop them, call +// Stop(). +func (f *ControlPlane) Start() error { + if f.Etcd == nil { + f.Etcd = &Etcd{ + Path: f.Path, + Out: f.Out, + Err: f.Err, + StartTimeout: f.StartTimeout, + StopTimeout: f.StopTimeout, + } + } + if err := f.Etcd.Start(); err != nil { + return err + } + + if f.APIServer == nil { + f.APIServer = &APIServer{ + Path: f.Path, + Out: f.Out, + Err: f.Err, + StartTimeout: f.StartTimeout, + StopTimeout: f.StopTimeout, + } + } + f.APIServer.EtcdURL = f.Etcd.URL + return f.APIServer.Start() +} + +// Stop will stop your control plane processes, and clean up their data. +func (f *ControlPlane) Stop() error { + if f.APIServer != nil { + if err := f.APIServer.Stop(); err != nil { + return err + } + } + if f.Etcd != nil { + if err := f.Etcd.Stop(); err != nil { + return err + } + } + return nil +} + +// APIURL returns the URL you should connect to to talk to your API. +func (f *ControlPlane) APIURL() *url.URL { + return f.APIServer.URL +} + +// KubeCtl returns a pre-configured KubeCtl, ready to connect to this +// ControlPlane. +func (f *ControlPlane) KubeCtl() *KubeCtl { + k := &KubeCtl{ + Path: f.Path, + Out: f.Out, + Err: f.Err, + } + k.Opts = append( + k.Opts, + fmt.Sprintf( + "--server=%s", + f.APIURL(), + ), + ) + return k +} + +// RESTClientConfig returns a pre-configured restconfig, ready to connect +// to this ControlPlane. +func (f *ControlPlane) RESTClientConfig() (*rest.Config, error) { + c := &rest.Config{ + Host: f.APIURL().String(), + ContentConfig: rest.ContentConfig{ + NegotiatedSerializer: serializer.WithoutConversionCodecFactory{ + CodecFactory: scheme.Codecs, + }, + }, + } + err := rest.SetKubernetesDefaults(c) + return c, err +} diff --git a/test/integration/framework/crd.go b/test/integration/framework/crd.go index d86f805..06527de 100644 --- a/test/integration/framework/crd.go +++ b/test/integration/framework/crd.go @@ -25,6 +25,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/klog" dynamicclientset "openebs.io/metac/dynamic/clientset" ) @@ -91,12 +92,13 @@ func (f *Fixture) SetupCRD( }, } - f.t.Logf("Creating %s CRD", kind) - + //f.t.Logf("Creating %s CRD", kind) + klog.V(2).Infof("Creating CRD %s", kind) crd, err := f.crdClient.CustomResourceDefinitions().Create(crd) if err != nil { f.t.Fatal(err) } + klog.V(2).Infof("Created CRD %s", kind) // add to teardown functions f.addToTeardown(func() error { @@ -107,32 +109,32 @@ func (f *Fixture) SetupCRD( return f.crdClient.CustomResourceDefinitions().Delete(crd.Name, nil) }) - f.t.Logf("Discovering %s CRD server API", kind) + klog.V(2).Infof("Discovering %s API", kind) err = f.Wait(func() (bool, error) { - return resourceManager.GetAPIForAPIVersionAndResource(APIVersion, plural) != nil, nil + return apiResourceDiscovery.GetAPIForAPIVersionAndResource(APIVersion, plural) != nil, nil }) if err != nil { f.t.Fatal(err) } - f.t.Logf("Discovered %s CRD server API", kind) + klog.V(2).Infof("Discovered %s API", kind) - crdClient, err := f.dynamicClientset.GetClientForAPIVersionResource(APIVersion, plural) + crClient, err := f.dynamicClientset.GetClientForAPIVersionResource(APIVersion, plural) if err != nil { f.t.Fatal(err) } - f.t.Logf("Listing CRDs") + klog.V(2).Infof("Listing CRs for %s", kind) err = f.Wait(func() (bool, error) { - _, err := crdClient.List(metav1.ListOptions{}) + _, err := crClient.List(metav1.ListOptions{}) return err == nil, err }) if err != nil { f.t.Fatal(err) } - f.t.Logf("Listed CRDs") + klog.V(2).Infof("Listed CRs for %s", kind) - f.t.Logf("Created %s CRD", kind) - return crd, crdClient + klog.V(2).Infof("Created CRD %s", kind) + return crd, crClient } // SetupNamespaceCRDAndItsCR will install a namespace scoped diff --git a/test/integration/framework/doc.go b/test/integration/framework/doc.go new file mode 100644 index 0000000..d43de71 --- /dev/null +++ b/test/integration/framework/doc.go @@ -0,0 +1,112 @@ +/* + +Package integration implements an integration testing framework for kubernetes. + +It provides components for standing up a kubernetes API, against which you can test a +kubernetes client, or other kubernetes components. The lifecycle of the components +needed to provide this API is managed by this framework. + +Quickstart + +Add something like the following to +your tests: + + cp := &integration.ControlPlane{} + cp.Start() + kubeCtl := cp.KubeCtl() + stdout, stderr, err := kubeCtl.Run("get", "pods") + // You can check on err, stdout & stderr and build up + // your tests + cp.Stop() + +Components + +Currently the framework provides the following components: + +ControlPlane: The ControlPlane wraps Etcd & APIServer (see below) and wires +them together correctly. A ControlPlane can be stopped & started and can +provide the URL to connect to the API. The ControlPlane can also be asked for a +KubeCtl which is already correctly configured for this ControlPlane. The +ControlPlane is a good entry point for default setups. + +Etcd: Manages an Etcd binary, which can be started, stopped and connected to. +By default Etcd will listen on a random port for http connections and will +create a temporary directory for its data. To configure it differently, see the +Etcd type documentation below. + +APIServer: Manages an Kube-APIServer binary, which can be started, stopped and +connected to. By default APIServer will listen on a random port for http +connections and will create a temporary directory to store the (auto-generated) +certificates. To configure it differently, see the APIServer type +documentation below. + +KubeCtl: Wraps around a `kubectl` binary and can `Run(...)` arbitrary commands +against a kubernetes control plane. + +Binaries + +Etcd, APIServer & KubeCtl use the same mechanism to determine which binaries to +use when they get started. + +1. If the component is configured with a `Path` the framework tries to run that +binary. +For example: + + myEtcd := &Etcd{ + Path: "/some/other/etcd", + } + cp := &integration.ControlPlane{ + Etcd: myEtcd, + } + cp.Start() + +2. If the Path field on APIServer, Etcd or KubeCtl is left unset and an +environment variable named `TEST_ASSET_KUBE_APISERVER`, `TEST_ASSET_ETCD` or +`TEST_ASSET_KUBECTL` is set, its value is used as a path to the binary for the +APIServer, Etcd or KubeCtl. + +3. If neither the `Path` field, nor the environment variable is set, the +framework tries to use the binaries `kube-apiserver`, `etcd` or `kubectl` in +the directory `${FRAMEWORK_DIR}/assets/bin/`. + +Arguments for Etcd and APIServer + +Those components will start without any configuration. However, if you want or +need to, you can override certain configuration -- one of which are the +arguments used when calling the binary. + +When you choose to specify your own set of arguments, those won't be appended +to the default set of arguments, it is your responsibility to provide all the +arguments needed for the binary to start successfully. + +However, the default arguments for APIServer and Etcd are exported as +`APIServerDefaultArgs` and `EtcdDefaultArgs` from this package. Treat those +variables as read-only constants. Internally we have a set of default +arguments for defaulting, the `APIServerDefaultArgs` and `EtcdDefaultArgs` are +just copies of those. So when you override them you loose access to the actual +internal default arguments, but your override won't affect the defaulting. + +All arguments are interpreted as go templates. Those templates have access to +all exported fields of the `APIServer`/`Etcd` struct. It does not matter if +those fields where explicitly set up or if they were defaulted by calling the +`Start()` method, the template evaluation runs just before the binary is +executed and right after the defaulting of all the struct's fields has +happened. + + // When you want to append additional arguments ... + etcd := &Etcd{ + // Additional custom arguments will appended to the set of default + // arguments + Args: append(EtcdDefaultArgs, "--additional=arg"), + DataDir: "/my/special/data/dir", + } + + // When you want to use a custom set of arguments ... + etcd := &Etcd{ + // Only custom arguments will be passed to the binary + Args: []string{"--one=1", "--two=2", "--three=3"}, + DataDir: "/my/special/data/dir", + } + +*/ +package framework diff --git a/test/integration/framework/etcd.go b/test/integration/framework/etcd.go index 5e940d4..ac34ac1 100644 --- a/test/integration/framework/etcd.go +++ b/test/integration/framework/etcd.go @@ -1,118 +1,114 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This file is copied from k8s.io/kubernetes/test/integration/framework/ -// to avoid vendoring the rest of the package, which depends on all of k8s. - package framework import ( - "context" - "fmt" - "io/ioutil" - "net" - "os" - "os/exec" - "path/filepath" - - "github.com/pkg/errors" - "k8s.io/klog" -) - -var etcdURL = "" + "io" + "time" -const installEtcd = ` -Cannot find etcd, cannot run integration tests + "net/url" -Please download kube-apiserver and ensure it is somewhere in the PATH. -See hack/get-kube-binaries.sh - -` + "openebs.io/metac/test/integration/framework/internal" +) -// getEtcdPath returns a path to an etcd executable. -func getEtcdPath() (string, error) { - bazelPath := filepath.Join(os.Getenv("RUNFILES_DIR"), "com_coreos_etcd/etcd") - p, err := exec.LookPath(bazelPath) - if err == nil { - return p, nil - } - return exec.LookPath("etcd") +// Etcd knows how to run an etcd server. +type Etcd struct { + // URL is the address the Etcd should listen on for client connections. + // + // If this is not specified, we default to a random free port on localhost. + URL *url.URL + + // Path is the path to the etcd binary. + // + // If this is left as the empty string, we will attempt to locate a binary, + // by checking for the TEST_ASSET_ETCD environment variable, and the default + // test assets directory. See the "Binaries" section above (in doc.go) for + // details. + Path string + + // Args is a list of arguments which will passed to the Etcd binary. Before + // they are passed on, the`y will be evaluated as go-template strings. This + // means you can use fields which are defined and exported on this Etcd + // struct (e.g. "--data-dir={{ .Dir }}"). + // Those templates will be evaluated after the defaulting of the Etcd's + // fields has already happened and just before the binary actually gets + // started. Thus you have access to calculated fields like `URL` and others. + // + // If not specified, the minimal set of arguments to run the Etcd will be + // used. + Args []string + + // DataDir is a path to a directory in which etcd can store its state. + // + // If left unspecified, then the Start() method will create a fresh temporary + // directory, and the Stop() method will clean it up. + DataDir string + + // StartTimeout, StopTimeout specify the time the Etcd is allowed to + // take when starting and stopping before an error is emitted. + // + // If not specified, these default to 20 seconds. + StartTimeout time.Duration + StopTimeout time.Duration + + // Out, Err specify where Etcd should write its StdOut, StdErr to. + // + // If not specified, the output will be discarded. + Out io.Writer + Err io.Writer + + processState *internal.ProcessState } -// getAvailablePort returns a TCP port that is available for binding. -func getAvailablePort() (int, error) { - l, err := net.Listen("tcp", ":0") - if err != nil { - return 0, errors.Wrapf(err, "Can't bind to port") +// Start starts the etcd, waits for it to come up, and returns an error, if one +// occoured. +func (e *Etcd) Start() error { + if e.processState == nil { + if err := e.setProcessState(); err != nil { + return err + } } - // It is possible but unlikely that someone else will bind this port - // before we get a chance to use it. - defer l.Close() - return l.Addr().(*net.TCPAddr).Port, nil + return e.processState.Start(e.Out, e.Err) } -// startEtcd executes an etcd instance. The returned function will signal the -// etcd process and wait for it to exit. -func startEtcd() (func(), error) { - etcdPath, err := getEtcdPath() - if err != nil { - fmt.Fprintf(os.Stderr, installEtcd) - return nil, errors.Wrapf(err, "Can't find etcd in PATH") - } - etcdPort, err := getAvailablePort() - if err != nil { - return nil, err - } - etcdURL = fmt.Sprintf("http://127.0.0.1:%d", etcdPort) - klog.Infof("Starting etcd on %s", etcdURL) +func (e *Etcd) setProcessState() error { + var err error - etcdDataDir, err := ioutil.TempDir(os.TempDir(), "integration_test_etcd_data") + e.processState = &internal.ProcessState{} + + e.processState.DefaultedProcessInput, err = internal.DoDefaulting( + "etcd", + e.URL, + e.DataDir, + e.Path, + e.StartTimeout, + e.StopTimeout, + ) if err != nil { - return nil, errors.Wrapf(err, "Can't make temp etcd data dir") + return err } - klog.Infof("Storing etcd data in: %v", etcdDataDir) - - ctx, cancel := context.WithCancel(context.Background()) - cmd := exec.CommandContext( - ctx, - etcdPath, - "--data-dir", etcdDataDir, - "--listen-client-urls", etcdURL, - "--advertise-client-urls", etcdURL, - "--listen-peer-urls", "http://127.0.0.1:0", - ) - // Uncomment these to see etcd output in test logs. - // For Metacontroller tests, we generally don't expect problems at this level. - //cmd.Stdout = os.Stdout - //cmd.Stderr = os.Stderr - - stop := func() { - klog.Infof("Stopping etcd") - cancel() - err := cmd.Wait() - klog.Infof("etcd exit status: %v", err) - err = os.RemoveAll(etcdDataDir) - if err != nil { - klog.Warningf("Cleanup etcd failed: %v", err) - } - } + e.processState.StartMessage = internal.GetEtcdStartMessage(e.processState.URL) - if err := cmd.Start(); err != nil { - return nil, errors.Wrapf(err, "Failed to run etcd") - } - return stop, nil + e.URL = &e.processState.URL + e.DataDir = e.processState.Dir + e.Path = e.processState.Path + e.StartTimeout = e.processState.StartTimeout + e.StopTimeout = e.processState.StopTimeout + + e.processState.Args, err = internal.RenderTemplates( + internal.DoEtcdArgDefaulting(e.Args), e, + ) + return err } + +// Stop stops this process gracefully, waits for its termination, and cleans up +// the DataDir if necessary. +func (e *Etcd) Stop() error { + return e.processState.Stop() +} + +// EtcdDefaultArgs exposes the default args for Etcd so that you +// can use those to append your own additional arguments. +// +// The internal default arguments are explicitly copied here, we don't want to +// allow users to change the internal ones. +var EtcdDefaultArgs = append([]string{}, internal.EtcdDefaultArgs...) diff --git a/test/integration/framework/fixture.go b/test/integration/framework/fixture.go index 71339ee..88eecd4 100644 --- a/test/integration/framework/fixture.go +++ b/test/integration/framework/fixture.go @@ -1,5 +1,4 @@ /* -Copyright 2019 Google Inc. Copyright 2019 The MayaData Authors. Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,6 +25,7 @@ import ( apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/klog" apierrors "k8s.io/apimachinery/pkg/api/errors" metaclientset "openebs.io/metac/client/generated/clientset/versioned" @@ -66,27 +66,31 @@ type Fixture struct { // NewFixture returns a new instance of Fixture func NewFixture(t *testing.T) *Fixture { - // get the config that is created just for the purposes - // of integration testing of this project - config := ApiserverConfig() - - crdClient, err := apiextensionsclientset.NewForConfig(config) + crdClient, err := apiextensionsclientset.NewForConfig( + apiServerConfig, + ) if err != nil { t.Fatal(err) } - dynamicClientset, err := dynamicclientset.New(config, resourceManager) + dynamicClientset, err := dynamicclientset.New( + apiServerConfig, + apiResourceDiscovery, + ) if err != nil { t.Fatal(err) } - metaClientset, err := metaclientset.NewForConfig(config) + metaClientset, err := metaclientset.NewForConfig( + apiServerConfig, + ) if err != nil { t.Fatal(err) } - typedClientset, err := kubernetes.NewForConfig(config) + typedClientset, err := kubernetes.NewForConfig( + apiServerConfig, + ) if err != nil { t.Fatal(err) } - return &Fixture{ t: t, dynamicClientset: dynamicClientset, @@ -190,19 +194,24 @@ func (f *Fixture) Wait(condition func() (bool, error)) error { for { done, err := condition() if err == nil && done { - f.t.Logf("Wait condition succeeded") + //f.t.Logf("Wait condition succeeded") + klog.V(3).Infof("Wait condition succeeded") return nil } if time.Since(start) > defaultWaitTimeout { return fmt.Errorf( - "Wait condition timed out %s: %v", defaultWaitTimeout, err, + "Wait condition timed out after %s: %v", + defaultWaitTimeout, + err, ) } if err != nil { // Log error, but keep trying until timeout. - f.t.Logf("Wait condition failed: Will retry: %v", err) + //f.t.Logf("Wait condition failed: Will retry: %v", err) + klog.V(3).Infof("Wait condition failed: Will retry: %v", err) } else { - f.t.Logf("Waiting for condition to succeed: Will retry") + //f.t.Logf("Waiting for condition to succeed: Will retry") + klog.V(3).Infof("Waiting for condition to succeed: Will retry") } time.Sleep(defaultWaitInterval) } diff --git a/test/integration/framework/internal/apiserver.go b/test/integration/framework/internal/apiserver.go new file mode 100644 index 0000000..0ad9bd1 --- /dev/null +++ b/test/integration/framework/internal/apiserver.go @@ -0,0 +1,25 @@ +package internal + +// APIServerDefaultArgs allow tests to run offline, by preventing API server from attempting to +// use default route to determine its --advertise-address. +var APIServerDefaultArgs = []string{ + "--advertise-address=127.0.0.1", + "--etcd-servers={{ if .EtcdURL }}{{ .EtcdURL.String }}{{ end }}", + "--cert-dir={{ .CertDir }}", + "--insecure-port={{ if .URL }}{{ .URL.Port }}{{ end }}", + "--insecure-bind-address={{ if .URL }}{{ .URL.Hostname }}{{ end }}", + "--secure-port={{ if .SecurePort }}{{ .SecurePort }}{{ end }}", + "--disable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,PersistentVolumeClaimResize,ResourceQuota", //nolint + "--service-cluster-ip-range=10.0.0.0/24", + "--allow-privileged=true", +} + +// DoAPIServerArgDefaulting will set default values to allow tests to run offline when the args are not informed. Otherwise, +// it will return the same []string arg passed as param. +func DoAPIServerArgDefaulting(args []string) []string { + if len(args) != 0 { + return args + } + + return APIServerDefaultArgs +} diff --git a/test/integration/framework/internal/arguments.go b/test/integration/framework/internal/arguments.go new file mode 100644 index 0000000..c1e7a75 --- /dev/null +++ b/test/integration/framework/internal/arguments.go @@ -0,0 +1,29 @@ +package internal + +import ( + "bytes" + "html/template" +) + +// RenderTemplates returns an []string to render the templates +func RenderTemplates(templates []string, data interface{}) (args []string, err error) { + var t *template.Template + + for _, tpl := range templates { + t, err = template.New(tpl).Parse(tpl) + if err != nil { + args = nil + return + } + + buf := &bytes.Buffer{} + err = t.Execute(buf, data) + if err != nil { + args = nil + return + } + args = append(args, buf.String()) + } + + return +} diff --git a/test/integration/framework/internal/bin_path_finder.go b/test/integration/framework/internal/bin_path_finder.go new file mode 100644 index 0000000..a8e7abc --- /dev/null +++ b/test/integration/framework/internal/bin_path_finder.go @@ -0,0 +1,41 @@ +package internal + +import ( + "os" + "path/filepath" + "regexp" + "runtime" + "strings" +) + +var assetsPath string + +func init() { + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + panic("Could not determine the path of the BinPathFinder") + } + assetsPath = filepath.Join( + filepath.Dir(thisFile), + "..", + "assets", + "bin", + ) +} + +// BinPathFinder checks the environment variable, +// derived from the symbolic name, and falls back +// to a default assets location when this variable is not set +func BinPathFinder(symbolicName string) (binPath string) { + punctuationPattern := regexp.MustCompile("[^A-Z0-9]+") + sanitizedName := punctuationPattern.ReplaceAllString(strings.ToUpper(symbolicName), "_") + leadingNumberPattern := regexp.MustCompile("^[0-9]+") + sanitizedName = leadingNumberPattern.ReplaceAllString(sanitizedName, "") + envVar := "TEST_ASSET_" + sanitizedName + + if val, ok := os.LookupEnv(envVar); ok { + return val + } + + return filepath.Join(assetsPath, symbolicName) +} diff --git a/test/integration/framework/internal/etcd.go b/test/integration/framework/internal/etcd.go new file mode 100644 index 0000000..3b04e63 --- /dev/null +++ b/test/integration/framework/internal/etcd.go @@ -0,0 +1,47 @@ +package internal + +import ( + "net/url" +) + +// EtcdDefaultArgs allow tests to run offline, by preventing +// API server from attempting to use default route to determine +// its urls. +var EtcdDefaultArgs = []string{ + "--listen-peer-urls=http://localhost:0", + "--advertise-client-urls={{ if .URL }}{{ .URL.String }}{{ end }}", + "--listen-client-urls={{ if .URL }}{{ .URL.String }}{{ end }}", + "--data-dir={{ .DataDir }}", +} + +// DoEtcdArgDefaulting will set default values to allow tests +// to run offline when the args are not informed. Otherwise, +// it will return the same []string arg passed as param. +func DoEtcdArgDefaulting(args []string) []string { + if len(args) != 0 { + return args + } + + return EtcdDefaultArgs +} + +// isSecureScheme returns false when the schema is insecure. +func isSecureScheme(scheme string) bool { + // https://github.com/coreos/etcd/blob/d9deeff49a080a88c982d328ad9d33f26d1ad7b6/pkg/transport/listener.go#L53 + if scheme == "https" || scheme == "unixs" { + return true + } + return false +} + +// GetEtcdStartMessage returns the expected message when etcd +// starts up. This message is based on the URL scheme +func GetEtcdStartMessage(listenURL url.URL) string { + if isSecureScheme(listenURL.Scheme) { + // https://github.com/coreos/etcd/blob/a7f1fbe00ec216fcb3a1919397a103b41dca8413/embed/serve.go#L167 + return "serving client requests on " + } + + // https://github.com/coreos/etcd/blob/a7f1fbe00ec216fcb3a1919397a103b41dca8413/embed/serve.go#L124 + return "serving insecure client requests on " +} diff --git a/test/integration/framework/internal/process.go b/test/integration/framework/internal/process.go new file mode 100644 index 0000000..9925666 --- /dev/null +++ b/test/integration/framework/internal/process.go @@ -0,0 +1,256 @@ +package internal + +import ( + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "strconv" + "time" + + "github.com/onsi/gomega/gbytes" + "github.com/onsi/gomega/gexec" + "github.com/pkg/errors" + + "openebs.io/metac/test/integration/framework/addr" +) + +// ProcessState define the state of the process. +type ProcessState struct { + DefaultedProcessInput + + Session *gexec.Session + + // Healthcheck Endpoint. If we get http.StatusOK from this endpoint, + // we assume the process is ready to operate. E.g. "/healthz". If + // this is set,we ignore StartMessage. + HealthCheckEndpoint string + + // HealthCheckPollInterval is the interval which will be used for + // polling the HealthCheckEndpoint. + // + // If left empty it will default to 100 Milliseconds. + HealthCheckPollInterval time.Duration + + // StartMessage is the message to wait for on stderr. If we + // receive this message, we assume the process is ready to + // operate. Ignored if HealthCheckEndpoint is specified. + // + // The usage of StartMessage is discouraged, favour + // HealthCheckEndpoint instead! + // + // Deprecated: Use HealthCheckEndpoint in favour of StartMessage + StartMessage string + + Args []string + + // ready holds whether the process is currently in ready state + // (hit the ready condition) or not. + // It will be set to true on a successful `Start()` and set to + // false on a successful `Stop()` + ready bool +} + +// DefaultedProcessInput defines the default process input required to perform the test. +type DefaultedProcessInput struct { + URL url.URL + Dir string + DirNeedsCleaning bool + Path string + StopTimeout time.Duration + StartTimeout time.Duration +} + +// DoDefaulting sets the default configuration according to the +// data informed and return an DefaultedProcessInput +// and an error if some requirement was not informed. +func DoDefaulting( + name string, + listenURL *url.URL, + dir string, + path string, + startTimeout time.Duration, + stopTimeout time.Duration, +) (DefaultedProcessInput, error) { + defaults := DefaultedProcessInput{ + Dir: dir, + Path: path, + StartTimeout: startTimeout, + StopTimeout: stopTimeout, + } + + if listenURL == nil { + port, host, err := addr.Suggest() + if err != nil { + return DefaultedProcessInput{}, err + } + defaults.URL = url.URL{ + Scheme: "http", + Host: net.JoinHostPort( + host, + strconv.Itoa(port), + ), + } + } else { + defaults.URL = *listenURL + } + + if dir == "" { + newDir, err := ioutil.TempDir("", "k8s_test_framework_") + if err != nil { + return DefaultedProcessInput{}, err + } + defaults.Dir = newDir + defaults.DirNeedsCleaning = true + } + + if path == "" { + if name == "" { + return DefaultedProcessInput{}, + fmt.Errorf("Must have at least name or path") + } + defaults.Path = BinPathFinder(name) + } + + if startTimeout == 0 { + defaults.StartTimeout = 20 * time.Second + } + + if stopTimeout == 0 { + defaults.StopTimeout = 20 * time.Second + } + + return defaults, nil +} + +type stopChannel chan struct{} + +// Start starts the process, waits for it to come up, +// and returns an error if any +func (ps *ProcessState) Start(stdout, stderr io.Writer) (err error) { + if ps.ready { + return nil + } + + command := exec.Command(ps.Path, ps.Args...) + + ready := make(chan bool) + timedOut := time.After(ps.StartTimeout) + var pollerStopCh stopChannel + + if ps.HealthCheckEndpoint != "" { + healthCheckURL := ps.URL + healthCheckURL.Path = ps.HealthCheckEndpoint + pollerStopCh = make(stopChannel) + go pollURLUntilOK( + healthCheckURL, + ps.HealthCheckPollInterval, + ready, + pollerStopCh, + ) + } else { + startDetectStream := gbytes.NewBuffer() + ready = startDetectStream.Detect(ps.StartMessage) + stderr = safeMultiWriter(stderr, startDetectStream) + } + + ps.Session, err = gexec.Start(command, stdout, stderr) + if err != nil { + return err + } + + select { + case <-ready: + ps.ready = true + return nil + case <-timedOut: + if pollerStopCh != nil { + close(pollerStopCh) + } + if ps.Session != nil { + ps.Session.Terminate() + } + return errors.Errorf( + "Failed to start %s: Timedout %s", + ps.Path, + ps.StartTimeout, + ) + } +} + +func safeMultiWriter(writers ...io.Writer) io.Writer { + safeWriters := []io.Writer{} + for _, w := range writers { + if w != nil { + safeWriters = append(safeWriters, w) + } + } + return io.MultiWriter(safeWriters...) +} + +func pollURLUntilOK( + url url.URL, + interval time.Duration, + ready chan bool, + stopCh stopChannel, +) { + if interval <= 0 { + interval = 100 * time.Millisecond + } + for { + res, err := http.Get(url.String()) + if err == nil { + res.Body.Close() + if res.StatusCode == http.StatusOK { + ready <- true + return + } + } + + select { + case <-stopCh: + return + default: + time.Sleep(interval) + } + } +} + +// Stop stops this process gracefully, waits for its termination, +// and cleans up the CertDir if necessary. +func (ps *ProcessState) Stop() error { + if ps.Session == nil { + return nil + } + + // gexec's Session methods (Signal, Kill, ...) do not check + // if the Process is nil, so we are doing this here for now. + // This should probably be fixed in gexec. + if ps.Session.Command.Process == nil { + return nil + } + + detectedStop := ps.Session.Terminate().Exited + timedOut := time.After(ps.StopTimeout) + + select { + case <-detectedStop: + break + case <-timedOut: + return errors.Errorf( + "Failed to stop %s: Timedout %s", + ps.Path, + ps.StopTimeout, + ) + } + ps.ready = false + if ps.DirNeedsCleaning { + return os.RemoveAll(ps.Dir) + } + + return nil +} diff --git a/test/integration/framework/internal/tinyca.go b/test/integration/framework/internal/tinyca.go new file mode 100644 index 0000000..0e29433 --- /dev/null +++ b/test/integration/framework/internal/tinyca.go @@ -0,0 +1,189 @@ +package internal + +// NB(directxman12): nothing has verified that this has +// good settings. In fact, the setting generated here +// are probably terrible, but they're fine for integration +// tests. These ABSOLUTELY SHOULD NOT ever be exposed in +// the public API. They're ONLY for use in integration +// testing. + +import ( + "crypto" + crand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "time" + + "github.com/pkg/errors" + certutil "k8s.io/client-go/util/cert" +) + +var ( + rsaKeySize = 2048 // a decent number, as of 2019 + bigOne = big.NewInt(1) +) + +// CertPair is a private key and certificate for use +// for client auth, as a CA, or serving. +type CertPair struct { + Key crypto.Signer + Cert *x509.Certificate +} + +// CertBytes returns the PEM-encoded version of the +// certificate for this pair. +func (k CertPair) CertBytes() []byte { + return pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: k.Cert.Raw, + }) +} + +// AsBytes encodes keypair in the appropriate formats +// for on-disk storage (PEM and PKCS8, respectively). +func (k CertPair) AsBytes() (cert []byte, key []byte, err error) { + cert = k.CertBytes() + + rawKeyData, err := x509.MarshalPKCS8PrivateKey(k.Key) + if err != nil { + return nil, nil, errors.Wrapf( + err, + "Failed to encode private key", + ) + } + + key = pem.EncodeToMemory(&pem.Block{ + Type: "PRIVATE KEY", + Bytes: rawKeyData, + }) + + return cert, key, nil +} + +// TinyCA supports signing serving certs and client-certs, +// and can be used as an auth mechanism with integration +// testing. +type TinyCA struct { + CA CertPair + orgName string + + nextSerial *big.Int +} + +// newPrivateKey generates a new private key of a relatively +// sane size (see rsaKeySize). +func newPrivateKey() (crypto.Signer, error) { + return rsa.GenerateKey(crand.Reader, rsaKeySize) +} + +// NewTinyCA creates a new a tiny CA utility for +// provisioning serving certs and client certs FOR TESTING ONLY. +// Don't use this for anything else! +func NewTinyCA() (*TinyCA, error) { + caPrivateKey, err := newPrivateKey() + if err != nil { + return nil, errors.Wrapf( + err, + "Failed to generate private key for CA", + ) + } + caCfg := certutil.Config{ + CommonName: "integration-test-env", + Organization: []string{"integration-test"}, + } + caCert, err := certutil.NewSelfSignedCACert( + caCfg, + caPrivateKey, + ) + if err != nil { + return nil, errors.Wrapf( + err, + "Failed to generate certificate for CA", + ) + } + + return &TinyCA{ + CA: CertPair{ + Key: caPrivateKey, + Cert: caCert, + }, + orgName: "integration-test", + nextSerial: big.NewInt(1), + }, nil +} + +func (c *TinyCA) makeCert(cfg certutil.Config) (CertPair, error) { + now := time.Now() + + key, err := newPrivateKey() + if err != nil { + return CertPair{}, errors.Wrapf( + err, + "Failed to create private key", + ) + } + + serial := new(big.Int).Set(c.nextSerial) + c.nextSerial.Add(c.nextSerial, bigOne) + + template := x509.Certificate{ + Subject: pkix.Name{CommonName: cfg.CommonName, Organization: cfg.Organization}, + DNSNames: cfg.AltNames.DNSNames, + IPAddresses: cfg.AltNames.IPs, + SerialNumber: serial, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: cfg.Usages, + + // technically not necessary for testing, but let's set anyway just in case. + NotBefore: now.UTC(), + // 1 week -- the default for cfssl, and just long enough for a + // long-term test, but not too long that anyone would try to use this + // seriously. + NotAfter: now.Add(168 * time.Hour).UTC(), + } + + certRaw, err := x509.CreateCertificate( + crand.Reader, + &template, + c.CA.Cert, + key.Public(), + c.CA.Key, + ) + if err != nil { + return CertPair{}, errors.Wrapf( + err, + "Failed to create certificate", + ) + } + + cert, err := x509.ParseCertificate(certRaw) + if err != nil { + return CertPair{}, errors.Wrapf( + err, + "Failed to parse certificate: Generated invalid certificate", + ) + } + + return CertPair{ + Key: key, + Cert: cert, + }, nil +} + +// NewServingCert returns a new CertPair for a serving HTTPS on localhost. +func (c *TinyCA) NewServingCert() (CertPair, error) { + return c.makeCert(certutil.Config{ + CommonName: "localhost", + Organization: []string{c.orgName}, + AltNames: certutil.AltNames{ + DNSNames: []string{"localhost"}, + IPs: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, + }, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }) +} diff --git a/test/integration/framework/kubectl.go b/test/integration/framework/kubectl.go new file mode 100644 index 0000000..1ee7233 --- /dev/null +++ b/test/integration/framework/kubectl.go @@ -0,0 +1,48 @@ +package framework + +import ( + "io" + "os/exec" + + "openebs.io/metac/test/integration/framework/internal" +) + +// KubeCtl is a wrapper around the kubectl binary. +type KubeCtl struct { + // Path where the kubectl binary can be found. + // + // If this is left empty, we will attempt to locate a binary, by checking for + // the TEST_ASSET_KUBECTL environment variable, and the default test assets + // directory. See the "Binaries" section above (in doc.go) for details. + Path string + + // Opts can be used to configure additional flags which will be used each + // time the wrapped binary is called. + // + // For example, you might want to use this to set the URL of the APIServer to + // connect to. + Opts []string + + // Out, Err specify where KubeCtl should write its StdOut, + // StdErr to. + // + // If not specified, the output will be discarded. + Out io.Writer + Err io.Writer +} + +// Run executes the wrapped binary with some preconfigured options and the +// arguments given to this method. +func (k *KubeCtl) Run(args ...string) error { + if k.Path == "" { + k.Path = internal.BinPathFinder("kubectl") + } + + allArgs := append(k.Opts, args...) + + cmd := exec.Command(k.Path, allArgs...) + cmd.Stdout = k.Out + cmd.Stderr = k.Err + + return cmd.Run() +} diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go deleted file mode 100644 index c17c9d6..0000000 --- a/test/integration/framework/main.go +++ /dev/null @@ -1,261 +0,0 @@ -/* -Copyright 2019 Google Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "fmt" - "os" - "os/exec" - "path" - "time" - - "github.com/pkg/errors" - "k8s.io/client-go/discovery" - "k8s.io/klog" - - dynamicdiscovery "openebs.io/metac/dynamic/discovery" - "openebs.io/metac/server" -) - -var resourceManager *dynamicdiscovery.APIResourceDiscovery - -const installKubectlMsg = ` -Cannot find kubectl, cannot run integration tests - -Please download kubectl and ensure it is somewhere in the PATH. -See hack/get-kube-binaries.sh - -` - -// manifestDir is the path from the integration test binary -// working dir to the directory containing manifests to -// install Metacontroller. -const manifestDir = "../../../manifests" - -// getKubectlPath returns a path to a kube-apiserver executable. -func getKubectlPath() (string, error) { - return exec.LookPath("kubectl") -} - -func execKubectl(args ...string) error { - execPath, err := exec.LookPath("kubectl") - if err != nil { - return errors.Wrapf(err, "Can't exec kubectl") - } - - cmdline := append([]string{"--server", ApiserverURL()}, args...) - cmd := exec.Command(execPath, cmdline...) - return cmd.Run() -} - -// TestWithCRDMetac starts etcd, kube-apiserver, and CRD based -// metacontroller before running tests. -// -// Usage: -// In test/integeration/somepackage/suite_test.go file have the -// following to let this package i.e. somepackage bring up required -// integration test related dependencies. -// -// ```go -// package somepackage -// -// import ( -// "openebs.io/metac/test/integration/framework -// ) -// -// func TestMain(m *testing.M) { -// framework.TestWithCRDMetac(m.Run()) -// } -// ```` -func TestWithCRDMetac(testRunFn func() int) { - if err := startCRDBasedMetaControllers(testRunFn); err != nil { - fmt.Println(err) - os.Exit(1) - } -} - -// TestWithConfigMetac starts etcd, kube-apiserver before -// running tests. -// -// NOTE: -// Since config based metac depends on the config to start -// Metac; Metac is not started in this function. It needs to -// be started in individual test functions. -// -// Usage: -// In test/integeration/somepackage/suite_test.go file have the -// following to let this package i.e. somepackage bring up required -// integration test related dependencies. -// -// ```go -// package somepackage -// -// import ( -// "openebs.io/metac/test/integration/framework -// ) -// -// func TestMain(m *testing.M) { -// framework.TestWithConfigMetac(m.Run()) -// } -// ```` -func TestWithConfigMetac(testRunFn func() int) { - if err := startConfigBasedMetaControllers(testRunFn); err != nil { - fmt.Println(err) - os.Exit(1) - } -} - -func startCRDBasedMetaControllers(testRunFn func() int) error { - if _, err := getKubectlPath(); err != nil { - return errors.New(installKubectlMsg) - } - - stopEtcd, err := startEtcd() - if err != nil { - return errors.Wrapf(err, "Can't start etcd") - } - defer stopEtcd() - - stopApiserver, err := startApiserver() - if err != nil { - return errors.Wrapf(err, "Can't start kube-apiserver") - } - defer stopApiserver() - - klog.Info("Waiting for kube-apiserver to be ready") - start := time.Now() - for { - kubectlErr := execKubectl("version") - if kubectlErr == nil { - break - } - if time.Since(start) > time.Minute { - return errors.Wrapf(err, "Timed out for kube-apiserver to be ready") - } - time.Sleep(time.Second) - } - klog.Info("kube-apiserver is ready") - - // Create Metacontroller Namespace. - err = execKubectl( - "apply", - "-f", - path.Join(manifestDir, "metacontroller-namespace.yaml"), - ) - if err != nil { - return errors.Wrapf(err, "Can't install metacontroller namespace") - } - - // Install Metacontroller RBAC. - err = execKubectl( - "apply", - "-f", - path.Join(manifestDir, "metacontroller-rbac.yaml"), - ) - if err != nil { - return errors.Wrapf(err, "Can't install metacontroller RBAC") - } - - // Install Metacontroller CRDs. - err = execKubectl( - "apply", - "-f", - path.Join(manifestDir, "metacontroller.yaml"), - ) - if err != nil { - return errors.Wrapf(err, "Can't install metacontroller CRDs") - } - - // In this integration test environment, there are no Nodes, - // so the metacontroller StatefulSet will not actually run - // anything. Instead, we start the Metacontroller server - // locally inside the test binary, since that's part of the - // code under test. - var mserver = server.Server{ - Config: ApiserverConfig(), - DiscoveryInterval: 500 * time.Millisecond, - InformerRelist: 30 * time.Minute, - } - crdServer := &server.CRDServer{Server: mserver} - stopServer, err := crdServer.Start(5) - if err != nil { - return errors.Wrapf(err, "Can't start crd based metacontroller server") - } - defer stopServer() - - // Periodically refresh discovery to pick up newly-installed - // resources. - discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(ApiserverConfig()) - resourceManager = dynamicdiscovery.NewAPIResourceDiscoverer(discoveryClient) - - // We don't care about stopping this cleanly since it has no - // external effects. - resourceManager.Start(500 * time.Millisecond) - - // Now run the actual tests - if exitCode := testRunFn(); exitCode != 0 { - return errors.Errorf("One or more tests failed: Exit code %d", exitCode) - } - return nil -} - -func startConfigBasedMetaControllers(testRunFn func() int) error { - if _, err := getKubectlPath(); err != nil { - return errors.New(installKubectlMsg) - } - - stopEtcd, err := startEtcd() - if err != nil { - return errors.Wrapf(err, "Can't start etcd") - } - defer stopEtcd() - - stopApiserver, err := startApiserver() - if err != nil { - return errors.Wrapf(err, "Can't start kube-apiserver") - } - defer stopApiserver() - - klog.Info("Waiting for kube-apiserver to be ready") - start := time.Now() - for { - kubectlErr := execKubectl("version") - if kubectlErr == nil { - break - } - if time.Since(start) > time.Minute { - return errors.Wrapf(err, "Timed out for kube-apiserver to be ready") - } - time.Sleep(time.Second) - } - klog.Info("kube-apiserver is ready") - - // Periodically refresh discovery to pick up newly-installed - // resources. - discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(ApiserverConfig()) - resourceManager = dynamicdiscovery.NewAPIResourceDiscoverer(discoveryClient) - - // We don't care about stopping this cleanly since it has no - // external effects. - resourceManager.Start(500 * time.Millisecond) - - // Now run the actual tests - if exitCode := testRunFn(); exitCode != 0 { - return errors.Errorf("One or more tests failed: Exit code %d", exitCode) - } - return nil -} diff --git a/test/integration/framework/metacontroller.go b/test/integration/framework/metacontroller.go index c5b9fda..46edd3d 100644 --- a/test/integration/framework/metacontroller.go +++ b/test/integration/framework/metacontroller.go @@ -214,9 +214,11 @@ func (f *Fixture) CreateGenericControllerAsMetacConfig( // StartMetacFromGenericControllerConfig starts Metac based // on the given config. It returns the stop function that // should be invoked by the caller once caller's task is done. -func (f *Fixture) StartMetacFromGenericControllerConfig(gctlAsConfigFn func() ([]*v1alpha1.GenericController, error)) (stop func()) { +func (f *Fixture) StartMetacFromGenericControllerConfig( + gctlAsConfigFn func() ([]*v1alpha1.GenericController, error), +) (stop func()) { var mserver = server.Server{ - Config: ApiserverConfig(), + Config: apiServerConfig, DiscoveryInterval: 500 * time.Millisecond, InformerRelist: 30 * time.Minute, } diff --git a/test/integration/framework/start.go b/test/integration/framework/start.go new file mode 100644 index 0000000..4d179a3 --- /dev/null +++ b/test/integration/framework/start.go @@ -0,0 +1,209 @@ +/* +Copyright 2020 The MayaData Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "path" + "time" + + "github.com/pkg/errors" + "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" + "k8s.io/klog" + dynamicdiscovery "openebs.io/metac/dynamic/discovery" + "openebs.io/metac/server" +) + +var apiResourceDiscovery *dynamicdiscovery.APIResourceDiscovery +var apiServerConfig *rest.Config + +// manifestDir is the path from the integration test binary +// working dir to the directory containing manifests to +// install Metacontroller. +const manifestDir = "../../../manifests" + +// StartCRDBasedMetac sets up kubernetes environment by starting +// kube apiserver & etcd binaries. Once the setup is done the test +// case functions provided as arguments are executed. +func StartCRDBasedMetac(testFns func() int) error { + klog.V(2).Infof("Will setup k8s") + cp := &ControlPlane{ + StartTimeout: time.Second * 40, + StopTimeout: time.Second * 40, + // Uncomment below to debug + //Out: os.Stdout, + //Err: os.Stderr, + } + err := cp.Start() + if err != nil { + return err + } + klog.V(2).Infof("k8s was setup successfully") + + // Create Metacontroller Namespace. + err = cp.KubeCtl().Run( + "apply", + "-f", + path.Join( + manifestDir, + "metacontroller-namespace.yaml", + ), + ) + if err != nil { + return errors.Wrapf( + err, + "Can't install metacontroller namespace", + ) + } + + // Install Metacontroller RBAC. + err = cp.KubeCtl().Run( + "apply", + "-f", + path.Join( + manifestDir, + "metacontroller-rbac.yaml", + ), + ) + if err != nil { + return errors.Wrapf( + err, + "Can't install metacontroller RBAC", + ) + } + + // Install Metacontroller CRDs. + err = cp.KubeCtl().Run( + "apply", + "-f", + path.Join( + manifestDir, + "metacontroller.yaml", + ), + ) + if err != nil { + return errors.Wrapf( + err, + "Can't install metacontroller CRDs", + ) + } + + // set the config to the global variable + apiServerConfig, err = cp.RESTClientConfig() + if err != nil { + return err + } + + // In this integration test environment, there are no Nodes, + // so the metacontroller StatefulSet will not actually run + // anything. Instead, we start the Metacontroller server + // locally inside the test binary, since that's part of the + // code under test. + metac := &server.CRDServer{ + Server: server.Server{ + Config: apiServerConfig, + DiscoveryInterval: 500 * time.Millisecond, + InformerRelist: 30 * time.Minute, + }, + } + stop, err := metac.Start(5) + if err != nil { + return errors.Wrapf( + err, + "Can't start CRD based metac server", + ) + } + defer stop() + klog.Info("Started CRD based metac server") + + discoveryClient := discovery.NewDiscoveryClientForConfigOrDie( + apiServerConfig, + ) + + // set api resource discovery to global variable + apiResourceDiscovery = dynamicdiscovery.NewAPIResourceDiscoverer( + discoveryClient, + ) + + // We don't care about stopping this cleanly since it has no + // external effects. + apiResourceDiscovery.Start(500 * time.Millisecond) + + // Run the actual tests now after above setup was done + // successfully + // + // NOTE: + // We ignore the exit code & rely on the using t.Fatal + // statements if individual test cases fail or error out + // at runtime + // + // NOTE: + // This always returns an exit code of 1 & hence a single + // word FAIL is printed at the end + testFns() + + return nil +} + +// StartConfigBasedMetac sets up kubernetes environment by starting +// kube apiserver & etcd binaries. Once the setup is done the test +// case functions provided as arguments are executed. +func StartConfigBasedMetac(testFns func() int) error { + klog.V(2).Infof("Will setup k8s") + cp := &ControlPlane{ + StartTimeout: time.Second * 40, + StopTimeout: time.Second * 40, + // Uncomment below to debug + //Out: os.Stdout, + //Err: os.Stderr, + } + err := cp.Start() + if err != nil { + return err + } + klog.V(2).Infof("k8s was setup successfully") + + // set the config to the global variable + apiServerConfig, err = cp.RESTClientConfig() + if err != nil { + return err + } + + discoveryClient := discovery.NewDiscoveryClientForConfigOrDie( + apiServerConfig, + ) + + // set api resource discovery to the global variable + apiResourceDiscovery = dynamicdiscovery.NewAPIResourceDiscoverer( + discoveryClient, + ) + + // We don't care about stopping this cleanly since it has no + // external effects. + apiResourceDiscovery.Start(500 * time.Millisecond) + + // Run the actual tests now after above setup was done + // successfully + if exitCode := testFns(); exitCode != 0 { + return errors.Errorf( + "One or more tests failed: Exit code %d", + exitCode, + ) + } + + return nil +} diff --git a/test/integration/generic/clean_uninstall_test.go b/test/integration/generic/clean_uninstall_test.go index 1b2bb0f..6275d1e 100644 --- a/test/integration/generic/clean_uninstall_test.go +++ b/test/integration/generic/clean_uninstall_test.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/json" + "k8s.io/klog" "openebs.io/metac/apis/metacontroller/v1alpha1" "openebs.io/metac/controller/generic" @@ -70,13 +71,16 @@ func TestCleanUninstall(t *testing.T) { // // NOTE: // Targeted CustomResources will be set in this namespace - targetNamespace, err := f.GetTypedClientset().CoreV1().Namespaces().Create( - &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: targetNamespaceName, + targetNamespace, err := f.GetTypedClientset(). + CoreV1(). + Namespaces(). + Create( + &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: targetNamespaceName, + }, }, - }, - ) + ) if err != nil { t.Fatal(err) } @@ -88,14 +92,24 @@ func TestCleanUninstall(t *testing.T) { cpcCRD, cpcClient, _ := f.SetupClusterCRDAndItsCR( "CStorPoolClaim", targetResName, - framework.SetFinalizers([]string{"protect.abc.io", "protect.def.io"}), + framework.SetFinalizers( + []string{ + "protect.abc.io", + "protect.def.io", + }, + ), ) // define a namespace scoped CStorVolumeReplica CRD & CR with finalizers cvrCRD, cvrClient, _ := f.SetupNamespaceCRDAndItsCR( "CStorVolumeReplica", targetNamespace.GetName(), targetResName, - framework.SetFinalizers([]string{"protect.xyz.io", "protect.ced.io"}), + framework.SetFinalizers( + []string{ + "protect.xyz.io", + "protect.ced.io", + }, + ), ) // ------------------------------------------------------------ @@ -189,9 +203,10 @@ func TestCleanUninstall(t *testing.T) { } } - t.Logf( + klog.V(2).Infof( "Finalize attachments count: Req %d: Resp %d", - req.Attachments.Len(), len(resp.Attachments), + req.Attachments.Len(), + len(resp.Attachments), ) return json.Marshal(resp) @@ -293,8 +308,13 @@ func TestCleanUninstall(t *testing.T) { // updates the watch with its own finalizer if it finds a // finalize hook in its specifications. err = f.Wait(func() (bool, error) { - targetNamespace, err = - f.GetTypedClientset().CoreV1().Namespaces().Get(targetNamespaceName, metav1.GetOptions{}) + targetNamespace, err = f.GetTypedClientset(). + CoreV1(). + Namespaces(). + Get( + targetNamespaceName, + metav1.GetOptions{}, + ) if err != nil { return false, err } @@ -303,8 +323,10 @@ func TestCleanUninstall(t *testing.T) { return true, nil } } - return false, - errors.Errorf("Namespace %s is not set with gctl finalizer", targetNamespaceName) + return false, errors.Errorf( + "Namespace %s is not set with gctl finalizer", + targetNamespaceName, + ) }) if err != nil { @@ -325,7 +347,7 @@ func TestCleanUninstall(t *testing.T) { // Need to wait & see if our controller works as expected // Make sure the specified attachments are deleted - t.Logf("Waiting for deletion of CRs & CRDs") + klog.Infof("Waiting for deletion of CRs & CRDs") err = f.Wait(func() (bool, error) { var errs []error @@ -336,14 +358,23 @@ func TestCleanUninstall(t *testing.T) { cpc, cpcGetErr := cpcClient.Get(targetResName, metav1.GetOptions{}) if cpcGetErr != nil && !apierrors.IsNotFound(cpcGetErr) { errs = append( - errs, errors.Wrapf(cpcGetErr, "Get CPC %s failed", targetResName), + errs, + errors.Wrapf(cpcGetErr, "Get CPC %s failed", targetResName), ) } if cpc != nil { - errs = append(errs, errors.Errorf("CPC %s is not deleted", targetResName)) + errs = append( + errs, + errors.Errorf("CPC %s is not deleted", targetResName), + ) } - cvr, cvrGetErr := cvrClient.Namespace(targetNamespaceName).Get(targetResName, metav1.GetOptions{}) + cvr, cvrGetErr := cvrClient. + Namespace(targetNamespaceName). + Get( + targetResName, + metav1.GetOptions{}, + ) if cvrGetErr != nil && !apierrors.IsNotFound(cvrGetErr) { errs = append( errs, @@ -351,14 +382,22 @@ func TestCleanUninstall(t *testing.T) { ) } if cvr != nil { - errs = append(errs, errors.Errorf("CVR %s is not deleted", targetResName)) + errs = append( + errs, + errors.Errorf("CVR %s is not deleted", targetResName), + ) } // ------------------------------------------ // verify if our target namespace is deleted // ------------------------------------------ - targetNSAgain, targetNSGetErr := f.GetTypedClientset().CoreV1().Namespaces(). - Get(targetNamespace.GetName(), metav1.GetOptions{}) + targetNSAgain, targetNSGetErr := f.GetTypedClientset(). + CoreV1(). + Namespaces(). + Get( + targetNamespace.GetName(), + metav1.GetOptions{}, + ) if targetNSGetErr != nil && !apierrors.IsNotFound(targetNSGetErr) { errs = append(errs, targetNSGetErr) } @@ -366,7 +405,8 @@ func TestCleanUninstall(t *testing.T) { errs = append( errs, errors.Errorf( - "Namespace %s has finalizers", targetNSAgain.GetName(), + "Namespace %s has finalizers", + targetNSAgain.GetName(), ), ) } @@ -374,7 +414,8 @@ func TestCleanUninstall(t *testing.T) { errs = append( errs, errors.Errorf( - "Namespace %s is not marked for deletion", targetNSAgain.GetName(), + "Namespace %s is not marked for deletion", + targetNSAgain.GetName(), ), ) } @@ -391,5 +432,5 @@ func TestCleanUninstall(t *testing.T) { if err != nil { t.Fatalf("CRs & CRDs deletion failed: %v", err) } - t.Logf("CRs & CRDs were finalized / deleted successfully") + klog.Infof("CRs & CRDs were finalized / deleted successfully") } diff --git a/test/integration/generic/generic_test.go b/test/integration/generic/generic_test.go index 6651a16..97e92bd 100644 --- a/test/integration/generic/generic_test.go +++ b/test/integration/generic/generic_test.go @@ -26,6 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/json" + "k8s.io/klog" "openebs.io/metac/apis/metacontroller/v1alpha1" "openebs.io/metac/controller/generic" @@ -61,10 +62,12 @@ func TestGCtlSyncWebhook(t *testing.T) { // 1. a CRD for watch // 2. a CRD for attachment watchCRD, watchClient := f.SetupCRD( - "GTSWPrimary", apiextensions.NamespaceScoped, + "GTSWPrimary", + apiextensions.NamespaceScoped, ) attachmentCRD, attachmentClient := f.SetupCRD( - "GTSWSecondary", apiextensions.NamespaceScoped, + "GTSWSecondary", + apiextensions.NamespaceScoped, ) // define the "reconcile logic" i.e. sync hook logic here @@ -80,7 +83,10 @@ func TestGCtlSyncWebhook(t *testing.T) { // Note that this does not create the child in kubernetes. // Creation of child in kubernetes is done by generic // controller on creation of parent resource. - child := framework.BuildUnstructObjFromCRD(attachmentCRD, req.Watch.GetName()) + child := framework.BuildUnstructObjFromCRD( + attachmentCRD, + req.Watch.GetName(), + ) child.SetLabels(labels) resp := generic.SyncHookResponse{ @@ -105,36 +111,49 @@ func TestGCtlSyncWebhook(t *testing.T) { watchResource := framework.BuildUnstructObjFromCRD(watchCRD, testName) unstructured.SetNestedStringMap( - watchResource.Object, labels, "spec", "selector", "matchLabels", + watchResource.Object, + labels, + "spec", + "selector", + "matchLabels", ) - t.Logf( + klog.Infof( "Creating %s/%s of kind:%s", watchResource.GetNamespace(), watchResource.GetName(), watchResource.GetKind(), ) - _, err := - watchClient.Namespace(ns.Name).Create(watchResource, metav1.CreateOptions{}) + _, err := watchClient. + Namespace(ns.Name). + Create( + watchResource, + metav1.CreateOptions{}, + ) if err != nil { t.Fatal(err) } - t.Logf( + klog.Infof( "Created %s/%s of kind:%s", watchResource.GetNamespace(), watchResource.GetName(), watchResource.GetKind(), ) - t.Logf("Waiting for attachment sync") + klog.Infof("Waiting for attachment sync") err = f.Wait(func() (bool, error) { - _, err := attachmentClient.Namespace(ns.Name).Get(testName, metav1.GetOptions{}) + _, err := attachmentClient. + Namespace(ns.Name). + Get( + testName, + metav1.GetOptions{}, + ) return err == nil, err }) if err != nil { t.Errorf("Attachment sync failed: %v", err) } - t.Logf("Attachment sync was successful") + klog.Infof("Attachment sync was successful") } // TestGCtlCascadingDelete tests that we request cascading deletion of children, @@ -157,7 +176,10 @@ func TestGCtlCascadingDelete(t *testing.T) { ns := f.CreateNamespaceGen(nsNamePrefix) // get required clients - watchCRD, watchClient := f.SetupCRD("GTCDPrimary", apiextensions.NamespaceScoped) + watchCRD, watchClient := f.SetupCRD( + "GTCDPrimary", + apiextensions.NamespaceScoped, + ) jobChildClient := f.GetTypedClientset().BatchV1().Jobs(ns.Name) // define the "reconcile logic" i.e. sync hook logic here @@ -168,8 +190,11 @@ func TestGCtlCascadingDelete(t *testing.T) { } resp := generic.SyncHookResponse{} - replicas, _, _ := - unstructured.NestedInt64(req.Watch.Object, "spec", "replicas") + replicas, _, _ := unstructured.NestedInt64( + req.Watch.Object, + "spec", + "replicas", + ) if replicas > 0 { // Create one attachment of type batch/v1 Job if replicas > 0. attachment := framework.BuildUnstructuredObjFromJSON( @@ -215,68 +240,99 @@ func TestGCtlCascadingDelete(t *testing.T) { ), ) - watchResource := framework.BuildUnstructObjFromCRD(watchCRD, resourceName) + watchResource := framework.BuildUnstructObjFromCRD( + watchCRD, + resourceName, + ) unstructured.SetNestedStringMap( - watchResource.Object, labels, "spec", "selector", "matchLabels", + watchResource.Object, + labels, + "spec", + "selector", + "matchLabels", ) // set watch spec with "replicas" property and // assign it with value=1 - unstructured.SetNestedField(watchResource.Object, int64(1), "spec", "replicas") + unstructured.SetNestedField( + watchResource.Object, + int64(1), + "spec", + "replicas", + ) - t.Logf( + klog.Infof( "Creating %s %s/%s", watchResource.GetKind(), watchResource.GetNamespace(), watchResource.GetName(), ) var err error - watchResource, err = - watchClient.Namespace(ns.Name).Create(watchResource, metav1.CreateOptions{}) + watchResource, err = watchClient. + Namespace(ns.Name). + Create( + watchResource, + metav1.CreateOptions{}, + ) if err != nil { t.Fatal(err) } - t.Logf( + klog.Infof( "Created %s %s/%s", watchResource.GetKind(), watchResource.GetNamespace(), watchResource.GetName(), ) - t.Logf("Waiting for attachment job creation") + klog.Infof("Waiting for attachment job creation") err = f.Wait(func() (bool, error) { - _, err := jobChildClient.Get(resourceName, metav1.GetOptions{}) + _, err := jobChildClient.Get( + resourceName, + metav1.GetOptions{}, + ) return err == nil, err }) if err != nil { t.Fatalf("Attachment job create failed: %v", err) } - t.Logf("Attachment job was created successfully") + klog.Infof("Attachment job was created successfully") // Now that child exists, tell parent to delete it. - t.Logf("Updating watch with replicas=0") - _, err = - watchClient.Namespace(ns.Name).AtomicUpdate(watchResource, func(obj *unstructured.Unstructured) bool { - unstructured.SetNestedField(obj.Object, int64(0), "spec", "replicas") - return true - }) + klog.Infof("Updating watch with replicas=0") + _, err = watchClient. + Namespace(ns.Name). + AtomicUpdate( + watchResource, + func(obj *unstructured.Unstructured) bool { + unstructured.SetNestedField( + obj.Object, + int64(0), + "spec", + "replicas", + ) + return true + }, + ) if err != nil { t.Fatal(err) } - t.Logf("Updated watch with replicas=0") + klog.Infof("Updated watch with replicas=0") // Make sure the attachment gets actually deleted - t.Logf("Waiting for attachment job to be synced i.e. delete") + klog.Infof("Waiting for attachment job to be synced i.e. delete") var child *batchv1.Job err = f.Wait(func() (bool, error) { var getErr error - child, getErr = jobChildClient.Get(resourceName, metav1.GetOptions{}) + child, getErr = jobChildClient.Get( + resourceName, + metav1.GetOptions{}, + ) return apierrors.IsNotFound(getErr), nil }) if err != nil { out, _ := json.Marshal(child) t.Errorf("Attachment job delete failed: %v; object: %s", err, out) } - t.Logf("Attachment job synced / deleted successfully") + klog.Infof("Attachment job synced / deleted successfully") } // TestGCtlResyncAfter tests that the resyncAfterSeconds field works. @@ -296,7 +352,8 @@ func TestGCtlResyncAfter(t *testing.T) { // create namespace ns := f.CreateNamespaceGen(nsNamePrefix) watchCRD, watchClient := f.SetupCRD( - "GTRAPrimary", apiextensions.NamespaceScoped, + "GTRAPrimary", + apiextensions.NamespaceScoped, ) var lastSync time.Time @@ -351,38 +408,52 @@ func TestGCtlResyncAfter(t *testing.T) { watchResource := framework.BuildUnstructObjFromCRD(watchCRD, testName) unstructured.SetNestedStringMap( - watchResource.Object, labels, "spec", "selector", "matchLabels", + watchResource.Object, + labels, + "spec", + "selector", + "matchLabels", ) - t.Logf( + klog.Infof( "Creating %s %s/%s", watchResource.GetKind(), watchResource.GetNamespace(), watchResource.GetName(), ) - _, err := - watchClient.Namespace(ns.Name).Create(watchResource, metav1.CreateOptions{}) + _, err := watchClient. + Namespace(ns.Name). + Create( + watchResource, + metav1.CreateOptions{}, + ) if err != nil { t.Fatal(err) } - t.Logf( + klog.Infof( "Created %s %s/%s", watchResource.GetKind(), watchResource.GetNamespace(), watchResource.GetName(), ) - t.Logf("Waiting for status.elaspedSeconds to be reported") + klog.Infof("Waiting for status.elaspedSeconds to be reported") var elapsedSeconds float64 err = f.Wait(func() (bool, error) { - parentResource, err := - watchClient.Namespace(ns.Name).Get(testName, metav1.GetOptions{}) + parentResource, err := watchClient. + Namespace(ns.Name). + Get( + testName, + metav1.GetOptions{}, + ) if err != nil { return false, err } val, found, err := unstructured.NestedFloat64( - parentResource.Object, "status", "elapsedSeconds", + parentResource.Object, + "status", + "elapsedSeconds", ) if err != nil || !found { // The value hasn't been populated. Keep waiting. @@ -395,7 +466,7 @@ func TestGCtlResyncAfter(t *testing.T) { if err != nil { t.Fatalf("Didn't find status.elapsedSeconds: %v", err) } - t.Logf("status.elapsedSeconds was reported as %v", elapsedSeconds) + klog.Infof("status.elapsedSeconds was reported as %v", elapsedSeconds) if elapsedSeconds > 1.0 { t.Errorf( diff --git a/test/integration/generic/install_uninstall_crd_test.go b/test/integration/generic/install_uninstall_crd_test.go index 4b5aabf..aee0012 100644 --- a/test/integration/generic/install_uninstall_crd_test.go +++ b/test/integration/generic/install_uninstall_crd_test.go @@ -24,6 +24,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/json" + "k8s.io/klog" "openebs.io/metac/apis/metacontroller/v1alpha1" "openebs.io/metac/controller/generic" @@ -171,7 +172,7 @@ func TestInstallUninstallCRD(t *testing.T) { // of this finalize block isFinalized = resp.Finalized - t.Logf("Finalize: Req.Attachments.Len=%d", req.Attachments.Len()) + klog.V(2).Infof("Finalize: Req.Attachments.Len=%d", req.Attachments.Len()) return json.Marshal(resp) }) @@ -231,33 +232,43 @@ func TestInstallUninstallCRD(t *testing.T) { // // NOTE: // This triggers reconciliation - _, err = f.GetTypedClientset().CoreV1().Namespaces().Create( - &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: targetNSName, + _, err = f.GetTypedClientset(). + CoreV1(). + Namespaces(). + Create( + &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: targetNSName, + }, }, - }, - ) + ) if err != nil { t.Fatal(err) } // Need to wait & see if our controller works as expected // Make sure the specified attachments i.e. CRD is created - t.Logf("Wait for creation of CRD %s", targetCRDName) + klog.Infof("Wait for creation of CRD %s", targetCRDName) crdCreateErr := f.Wait(func() (bool, error) { // ------------------------------------------------ // verify if target CRD is created i.e. reconciled // ------------------------------------------------ - crdCreateObj, createErr := - f.GetCRDClient().CustomResourceDefinitions().Get(targetCRDName, metav1.GetOptions{}) + crdCreateObj, createErr := f.GetCRDClient(). + CustomResourceDefinitions(). + Get( + targetCRDName, + metav1.GetOptions{}, + ) if createErr != nil { return false, createErr } if crdCreateObj == nil { - return false, errors.Errorf("CRD %s is not created", targetCRDName) + return false, errors.Errorf( + "CRD %s is not created", + targetCRDName, + ) } // condition passed @@ -275,8 +286,13 @@ func TestInstallUninstallCRD(t *testing.T) { // its own finalizer if it finds a finalize hook in its // specifications. nsWithFErr := f.Wait(func() (bool, error) { - nsWithF, err := - f.GetTypedClientset().CoreV1().Namespaces().Get(targetNSName, metav1.GetOptions{}) + nsWithF, err := f.GetTypedClientset(). + CoreV1(). + Namespaces(). + Get( + targetNSName, + metav1.GetOptions{}, + ) if err != nil { return false, err } @@ -285,7 +301,10 @@ func TestInstallUninstallCRD(t *testing.T) { return true, nil } } - return false, errors.Errorf("Namespace %s is not set with gctl finalizer", targetNSName) + return false, errors.Errorf( + "Namespace %s is not set with gctl finalizer", + targetNSName, + ) }) if nsWithFErr != nil { // we wait till timeout & panic if condition is not met @@ -296,15 +315,20 @@ func TestInstallUninstallCRD(t *testing.T) { // Trigger finalize by deleting the target namespace // ------------------------------------------------------ - err = - f.GetTypedClientset().CoreV1().Namespaces().Delete(targetNSName, &metav1.DeleteOptions{}) + err = f.GetTypedClientset(). + CoreV1(). + Namespaces(). + Delete( + targetNSName, + &metav1.DeleteOptions{}, + ) if err != nil { t.Fatal(err) } // Need to wait & see if our controller works as expected // Make sure the specified attachments i.e. CRD is deleted - t.Logf("Wait for deletion of CRD %s", targetCRDName) + klog.Infof("Wait for deletion of CRD %s", targetCRDName) crdDelErr := f.Wait(func() (bool, error) { var getErr error @@ -314,20 +338,25 @@ func TestInstallUninstallCRD(t *testing.T) { // ------------------------------------------------ if isFinalized { - t.Logf("CRD %s should have been deleted: IsFinalized %t", targetCRDName, isFinalized) return true, nil } - crdObj, getErr := - f.GetCRDClient().CustomResourceDefinitions().Get(targetCRDName, metav1.GetOptions{}) + crdObj, getErr := f.GetCRDClient(). + CustomResourceDefinitions(). + Get( + targetCRDName, + metav1.GetOptions{}, + ) if getErr != nil && !apierrors.IsNotFound(getErr) { return false, getErr } if crdObj != nil && crdObj.GetDeletionTimestamp() == nil { - return false, - errors.Errorf("CRD %s is not marked for deletion", targetCRDName) + return false, errors.Errorf( + "CRD %s is not marked for deletion", + targetCRDName, + ) } // condition passed @@ -338,5 +367,5 @@ func TestInstallUninstallCRD(t *testing.T) { t.Fatalf("CRD %s wasn't deleted: %v", targetCRDName, crdDelErr) } - t.Logf("Test Install Uninstall CRD %s passed", targetCRDName) + klog.Infof("Test Install Uninstall CRD %s passed", targetCRDName) } diff --git a/test/integration/generic/set_status_on_cr_test.go b/test/integration/generic/set_status_on_cr_test.go index 7093b01..8404912 100644 --- a/test/integration/generic/set_status_on_cr_test.go +++ b/test/integration/generic/set_status_on_cr_test.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog" "openebs.io/metac/apis/metacontroller/v1alpha1" "openebs.io/metac/controller/generic" @@ -65,13 +66,16 @@ func TestSetStatusOnCR(t *testing.T) { // // NOTE: // Targeted CustomResources will be set in this namespace - targetNamespace, err := f.GetTypedClientset().CoreV1().Namespaces().Create( - &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: targetNamespaceName, + targetNamespace, err := f.GetTypedClientset(). + CoreV1(). + Namespaces(). + Create( + &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: targetNamespaceName, + }, }, - }, - ) + ) if err != nil { t.Fatal(err) } @@ -83,15 +87,7 @@ func TestSetStatusOnCR(t *testing.T) { // NOTE: // This makes use of inline function as hook sHook := func(req *generic.SyncHookRequest, resp *generic.SyncHookResponse) error { - if req == nil || req.Watch == nil { - t.Logf("Request does not have watch") - return nil - } - t.Logf("Request has watch: %v", req.Watch.GetName()) - if resp == nil { - resp = &generic.SyncHookResponse{} - } resp.Status = map[string]interface{}{ "phase": "Active", "conditions": []string{ @@ -100,13 +96,13 @@ func TestSetStatusOnCR(t *testing.T) { }, } - t.Logf("Sending response status %v", resp.Status) + klog.Infof("Sending response status %v", resp.Status) return nil } // Add this sync hook implementation to inline hook registry - var testWatchStatusFuncName = "test/watch-status" - generic.AddToInlineRegistry(testWatchStatusFuncName, sHook) + var inlineHookName = "test/watch-status" + generic.AddToInlineRegistry(inlineHookName, sHook) // --------------------------------------------------------- // Define & Apply a GenericController i.e. a Meta Controller @@ -121,7 +117,7 @@ func TestSetStatusOnCR(t *testing.T) { ctlNS.Name, // set sync hook - generic.WithInlinehookSyncFunc(k8s.StringPtr(testWatchStatusFuncName)), + generic.WithInlinehookSyncFunc(k8s.StringPtr(inlineHookName)), // We want CoolNerd resource as our watched resource generic.WithWatch( @@ -145,11 +141,16 @@ func TestSetStatusOnCR(t *testing.T) { "CoolNerd", targetNamespace.GetName(), targetResName, - framework.SetFinalizers([]string{"protect.abc.io", "protect.def.io"}), + framework.SetFinalizers( + []string{ + "protect.abc.io", + "protect.def.io", + }, + ), ) // Need to wait & see if our controller works as expected - t.Logf("Waiting for verification of CoolNerd resource status") + klog.Infof("Waiting for verification of CoolNerd resource status") err = f.Wait(func() (bool, error) { var errs []error @@ -157,21 +158,43 @@ func TestSetStatusOnCR(t *testing.T) { // ------------------------------------------- // verify if our custom resources are deleted // ------------------------------------------- - cnObj, cpcGetErr := cnClient.Namespace(targetNamespaceName).Get(targetResName, metav1.GetOptions{}) + cnObj, cpcGetErr := cnClient. + Namespace(targetNamespaceName). + Get( + targetResName, + metav1.GetOptions{}, + ) if cpcGetErr != nil && !apierrors.IsNotFound(cpcGetErr) { errs = append( - errs, errors.Wrapf(cpcGetErr, "Get CoolNerd %s failed", targetResName), + errs, + errors.Wrapf( + cpcGetErr, + "Get CoolNerd %s failed", + targetResName, + ), ) } - phase, _, _ := - unstructured.NestedString(cnObj.UnstructuredContent(), "status", "phase") + phase, _, _ := unstructured.NestedString( + cnObj.UnstructuredContent(), + "status", + "phase", + ) if cnObj != nil && phase != "Active" { - errs = append(errs, errors.Errorf("CoolNerd status is not 'Active'")) + errs = append( + errs, + errors.Errorf("CoolNerd status is not 'Active'"), + ) } - conditions, _, _ := - unstructured.NestedStringSlice(cnObj.UnstructuredContent(), "status", "conditions") + conditions, _, _ := unstructured.NestedStringSlice( + cnObj.UnstructuredContent(), + "status", + "conditions", + ) if cnObj != nil && len(conditions) != 2 { - errs = append(errs, errors.Errorf("CoolNerd conditions count is not 2")) + errs = append( + errs, + errors.Errorf("CoolNerd conditions count is not 2"), + ) } // condition did not pass in case of any errors @@ -186,5 +209,5 @@ func TestSetStatusOnCR(t *testing.T) { if err != nil { t.Fatalf("Setting CoolNerd resource status failed: %v", err) } - t.Logf("Setting CoolNerd resource status was successful") + klog.Infof("CoolNerd resource status was set successfully") } diff --git a/test/integration/generic/stuck_to_unstuck_delete_test.go b/test/integration/generic/stuck_to_unstuck_delete_test.go index 5907b52..1100a7e 100644 --- a/test/integration/generic/stuck_to_unstuck_delete_test.go +++ b/test/integration/generic/stuck_to_unstuck_delete_test.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/json" + "k8s.io/klog" "openebs.io/metac/apis/metacontroller/v1alpha1" "openebs.io/metac/controller/generic" @@ -49,9 +50,9 @@ import ( // of this workload namespace. func TestStuckToUnStuckDelete(t *testing.T) { // skipping this in short mode as this is a flaky test - if testing.Short() { - t.Skip("Skipping TestStuckToUnStuckDelete in short mode") - } + // if testing.Short() { + // t.Skip("Skipping TestStuckToUnStuckDelete in short mode") + // } // namespace to setup GenericController ctlNSNamePrefix := "gctl-test" @@ -82,13 +83,16 @@ func TestStuckToUnStuckDelete(t *testing.T) { // // NOTE: // Targeted CustomResources will be set in this namespace - targetNamespace, err := f.GetTypedClientset().CoreV1().Namespaces().Create( - &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: targetNamespaceName, + targetNamespace, err := f.GetTypedClientset(). + CoreV1(). + Namespaces(). + Create( + &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: targetNamespaceName, + }, }, - }, - ) + ) if err != nil { t.Fatal(err) } @@ -100,7 +104,12 @@ func TestStuckToUnStuckDelete(t *testing.T) { "CStorPoolClub", targetNamespace.GetName(), targetResName, - framework.SetFinalizers([]string{"protect.abc.io", "protect.def.io"}), + framework.SetFinalizers( + []string{ + "protect.abc.io", + "protect.def.io", + }, + ), ) // define a namespace scoped CStorVolumeRest CRD & CR with finalizers @@ -108,7 +117,12 @@ func TestStuckToUnStuckDelete(t *testing.T) { "CStorVolumeRest", targetNamespace.GetName(), targetResName, - framework.SetFinalizers([]string{"protect.xyz.io", "protect.ced.io"}), + framework.SetFinalizers( + []string{ + "protect.xyz.io", + "protect.ced.io", + }, + ), ) // ------------------------------------------------------ @@ -117,7 +131,13 @@ func TestStuckToUnStuckDelete(t *testing.T) { // // Above makes the namespace deletion being stuck due to // this namespace scoped custom resources' finalizers. - err = f.GetTypedClientset().CoreV1().Namespaces().Delete(targetNamespace.GetName(), &metav1.DeleteOptions{}) + err = f.GetTypedClientset(). + CoreV1(). + Namespaces(). + Delete( + targetNamespace.GetName(), + &metav1.DeleteOptions{}, + ) if err != nil { t.Fatal(err) } @@ -158,11 +178,8 @@ func TestStuckToUnStuckDelete(t *testing.T) { // ignore this attachment continue } - - // copy the attachment from req to a new instance - //respAtt := att + // build the desired attachment respAtt := &unstructured.Unstructured{} - if att.GetKind() == "CustomResourceDefinition" { // keep the CRD attachment till all its corresponding // CRs get deleted @@ -172,25 +189,17 @@ func TestStuckToUnStuckDelete(t *testing.T) { } else { hasAtleastOneCustomResource = true } - if len(att.GetFinalizers()) == 0 { // this is a custom resource & does not have any finalizers // then let this be deleted i.e. don't add to response continue } - // This is a custom resource with finalizers // Hence, re-build the attachment with empty finalizers respAtt.SetUnstructuredContent(att.UnstructuredContent()) - //respAtt.SetAPIVersion(att.GetAPIVersion()) - //respAtt.SetKind(att.GetKind()) - //respAtt.SetName(att.GetName()) - //respAtt.SetNamespace(att.GetNamespace()) - // Setting finalizers to empty is a must to // let this custom resource get deleted respAtt.SetFinalizers([]string{}) - resp.Attachments = append(resp.Attachments, respAtt) } } @@ -212,15 +221,15 @@ func TestStuckToUnStuckDelete(t *testing.T) { } else { // if there are still attachments seen in the request // keep resyncing the watch - resp.ResyncAfterSeconds = 2 + resp.ResyncAfterSeconds = 1 } } - t.Logf( + klog.V(2).Infof( "Finalize attachments count: Req %d: Resp %d", - req.Attachments.Len(), len(resp.Attachments), + req.Attachments.Len(), + len(resp.Attachments), ) - t.Logf("Req attachments: \n%v", req.Attachments) return json.Marshal(resp) }) @@ -319,8 +328,12 @@ func TestStuckToUnStuckDelete(t *testing.T) { // its own finalizer if it finds a finalize hook in its // specifications. readyErr := f.Wait(func() (bool, error) { - cpcCRDWithF, err := - f.GetCRDClient().CustomResourceDefinitions().Get(cpcCRD.GetName(), metav1.GetOptions{}) + cpcCRDWithF, err := f.GetCRDClient(). + CustomResourceDefinitions(). + Get( + cpcCRD.GetName(), + metav1.GetOptions{}, + ) if err != nil { return false, err } @@ -329,7 +342,10 @@ func TestStuckToUnStuckDelete(t *testing.T) { return true, nil } } - return false, errors.Errorf("CRD %s is not set with gctl finalizer", cpcCRD.GetName()) + return false, errors.Errorf( + "CRD %s is not set with gctl finalizer", + cpcCRD.GetName(), + ) }) if readyErr != nil { @@ -343,14 +359,19 @@ func TestStuckToUnStuckDelete(t *testing.T) { // // In other words, deleting the generic controller's // watch will trigger this controller's finalizer - delErr := f.GetCRDClient().CustomResourceDefinitions().Delete(cpcCRD.GetName(), &metav1.DeleteOptions{}) + delErr := f.GetCRDClient(). + CustomResourceDefinitions(). + Delete( + cpcCRD.GetName(), + &metav1.DeleteOptions{}, + ) if delErr != nil { t.Fatal(delErr) } // Need to wait & see if our controller works as expected // Make sure the specified attachments are deleted - t.Logf("Waiting for deletion of CRs, CRDs & Namespace") + klog.Infof("Waiting for deletion of CRs, CRDs & Namespace") err = f.Wait(func() (bool, error) { var errs []error @@ -358,25 +379,54 @@ func TestStuckToUnStuckDelete(t *testing.T) { // ------------------------------------------- // verify if our custom resources are deleted // ------------------------------------------- - cpc, cpcGetErr := cpcClient.Namespace(targetNamespaceName).Get(targetResName, metav1.GetOptions{}) + cpc, cpcGetErr := cpcClient. + Namespace(targetNamespaceName). + Get( + targetResName, + metav1.GetOptions{}, + ) if cpcGetErr != nil && !apierrors.IsNotFound(cpcGetErr) { errs = append( - errs, errors.Wrapf(cpcGetErr, "Get CPC %s failed", targetResName), + errs, + errors.Wrapf( + cpcGetErr, + "Get CPC %s failed", + targetResName, + ), ) } if cpc != nil { - errs = append(errs, errors.Errorf("CPC %s is not deleted", targetResName)) + errs = append( + errs, + errors.Errorf( + "CPC %s is not deleted", + targetResName, + )) } - cvr, cvrGetErr := cvrClient.Namespace(targetNamespaceName).Get(targetResName, metav1.GetOptions{}) + cvr, cvrGetErr := cvrClient. + Namespace(targetNamespaceName). + Get( + targetResName, + metav1.GetOptions{}, + ) if cvrGetErr != nil && !apierrors.IsNotFound(cvrGetErr) { errs = append( errs, - errors.Wrapf(cvrGetErr, "Get CVR %s failed", targetResName), + errors.Wrapf( + cvrGetErr, + "Get CVR %s failed", + targetResName, + ), ) } if cvr != nil { - errs = append(errs, errors.Errorf("CVR %s is not deleted", targetResName)) + errs = append( + errs, + errors.Errorf( + "CVR %s is not deleted", + targetResName, + )) } // condition did not pass in case of any errors @@ -391,5 +441,5 @@ func TestStuckToUnStuckDelete(t *testing.T) { if err != nil { t.Fatalf("CRs, CRDs & Namespace deletion failed: %v", err) } - t.Logf("CRDs, CRs & namespace were deleted successfully") + klog.Infof("CRDs, CRs & namespace were deleted successfully") } diff --git a/test/integration/generic/suite_test.go b/test/integration/generic/suite_test.go index ba6f4c8..bd9a633 100644 --- a/test/integration/generic/suite_test.go +++ b/test/integration/generic/suite_test.go @@ -17,18 +17,44 @@ limitations under the License. package generic import ( + "flag" "testing" + "k8s.io/klog" "openebs.io/metac/test/integration/framework" ) -// This will be run only once when go test is invoked against this -// package. All the other Test* functions will be invoked via m.Run -// call. +// TestMain will run only once when go test is invoked +// against this package. All the other Test* functions +// will be invoked via m.Run call. // // NOTE: -// framework.TestMain provides setup & teardown features required for -// all the individual testcases to run. +// There can be only one TestMain function in the entire +// package func TestMain(m *testing.M) { - framework.TestWithCRDMetac(m.Run) + flag.Parse() + + klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) + klog.InitFlags(klogFlags) + + // Sync the glog and klog flags. + flag.CommandLine.VisitAll(func(f1 *flag.Flag) { + f2 := klogFlags.Lookup(f1.Name) + if f2 != nil { + value := f1.Value.String() + f2.Value.Set(value) + } + }) + + // Pass m.Run function to framework which in turn + // sets up a kubernetes environment & then invokes + // m.Run. + err := framework.StartCRDBasedMetac(m.Run) + if err != nil { + // Since this is an error we must to invoke os.Exit(1) + // as per TestMain guidelines + klog.Exitf("%+v", err) + } + + defer klog.Flush() } diff --git a/test/integration/genericlocal/set_status_on_cr_test.go b/test/integration/genericlocal/set_status_on_cr_test.go index 855911a..9eef36e 100644 --- a/test/integration/genericlocal/set_status_on_cr_test.go +++ b/test/integration/genericlocal/set_status_on_cr_test.go @@ -24,6 +24,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog" "openebs.io/metac/apis/metacontroller/v1alpha1" "openebs.io/metac/controller/generic" @@ -57,12 +58,6 @@ func TestLocalSetStatusOnCR(t *testing.T) { // NOTE: // This makes use of inline function as hook sHook := func(req *generic.SyncHookRequest, resp *generic.SyncHookResponse) error { - if req == nil || req.Watch == nil { - t.Logf("Request does not have watch") - return nil - } - - t.Logf("Request has watch: %s", req.Watch.GetName()) if resp == nil { resp = &generic.SyncHookResponse{} } @@ -74,15 +69,14 @@ func TestLocalSetStatusOnCR(t *testing.T) { "RunFromLocalConfig", }, } + // there are no attachments to be reconciled resp.SkipReconcile = true - - t.Logf("Sending response status %v", resp.Status) return nil } // Add this sync hook implementation to inline hook registry - var testWatchStatusFuncName = "test/gctl-local-set-status-on-cr" - generic.AddToInlineRegistry(testWatchStatusFuncName, sHook) + var inlineHookName = "test/gctl-local-set-status-on-cr" + generic.AddToInlineRegistry(inlineHookName, sHook) // --------------------------------------------------------- // Define & Apply a GenericController i.e. a Meta Controller @@ -96,7 +90,7 @@ func TestLocalSetStatusOnCR(t *testing.T) { ctlName, // set sync hook - generic.WithInlinehookSyncFunc(k8s.StringPtr(testWatchStatusFuncName)), + generic.WithInlinehookSyncFunc(k8s.StringPtr(inlineHookName)), // We want LocalNerd resource as our watched resource generic.WithWatch( @@ -112,9 +106,11 @@ func TestLocalSetStatusOnCR(t *testing.T) { ) // start metac that uses above GenericController instance as a config - stopMetac := f.StartMetacFromGenericControllerConfig(func() ([]*v1alpha1.GenericController, error) { - return []*v1alpha1.GenericController{gctlConfig}, nil - }) + stopMetac := f.StartMetacFromGenericControllerConfig( + func() ([]*v1alpha1.GenericController, error) { + return []*v1alpha1.GenericController{gctlConfig}, nil + }, + ) defer stopMetac() // --------------------------------------------------- @@ -123,13 +119,16 @@ func TestLocalSetStatusOnCR(t *testing.T) { // // NOTE: // Targeted CustomResources will be set in this namespace - targetNamespace, nsCreateErr := f.GetTypedClientset().CoreV1().Namespaces().Create( - &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: targetNamespaceName, + targetNamespace, nsCreateErr := f.GetTypedClientset(). + CoreV1(). + Namespaces(). + Create( + &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: targetNamespaceName, + }, }, - }, - ) + ) if nsCreateErr != nil { t.Fatal(nsCreateErr) } @@ -147,7 +146,7 @@ func TestLocalSetStatusOnCR(t *testing.T) { ) // Need to wait & see if our controller works as expected - t.Logf("Waiting for verification of LocalNerd resource status") + klog.Infof("Will wait to verify LocalNerd status") waitCondErr := f.Wait(func() (bool, error) { var errs []error @@ -155,25 +154,47 @@ func TestLocalSetStatusOnCR(t *testing.T) { // ------------------------------------------- // verify if our custom resources are set with status // ------------------------------------------- - lnObj, getLNErr := lnClient.Namespace(targetNamespaceName).Get(targetResName, metav1.GetOptions{}) + lnObj, getLNErr := lnClient. + Namespace(targetNamespaceName). + Get( + targetResName, + metav1.GetOptions{}, + ) if getLNErr != nil { errs = append( - errs, errors.Wrapf(getLNErr, "Get LocalNerd %s failed", targetResName), + errs, + errors.Wrapf( + getLNErr, + "Get LocalNerd %s failed", + targetResName, + ), ) } // verify phase - phase, _, _ := - unstructured.NestedString(lnObj.UnstructuredContent(), "status", "phase") + phase, _, _ := unstructured.NestedString( + lnObj.UnstructuredContent(), + "status", + "phase", + ) if phase != "Active" { - errs = append(errs, errors.Errorf("LocalNerd status is not 'Active'")) + errs = append( + errs, + errors.Errorf("LocalNerd status is not 'Active'"), + ) } // verify conditions - conditions, _, _ := - unstructured.NestedStringSlice(lnObj.UnstructuredContent(), "status", "conditions") + conditions, _, _ := unstructured.NestedStringSlice( + lnObj.UnstructuredContent(), + "status", + "conditions", + ) if len(conditions) != 3 { - errs = append(errs, errors.Errorf("LocalNerd conditions count is not 3")) + errs = append( + errs, + errors.Errorf("LocalNerd conditions count is not 3"), + ) } // condition did not pass in case of any errors @@ -186,7 +207,10 @@ func TestLocalSetStatusOnCR(t *testing.T) { }) if waitCondErr != nil { - t.Fatalf("Setting LocalNerd resource status failed: %v", waitCondErr) + t.Fatalf( + "Failed to set LocalNerd status: %v", + waitCondErr, + ) } - t.Logf("Setting LocalNerd resource status was successful") + klog.Infof("LocalNerd status was set successfully") } diff --git a/test/integration/genericlocal/suite_test.go b/test/integration/genericlocal/suite_test.go index 23096cc..c044589 100644 --- a/test/integration/genericlocal/suite_test.go +++ b/test/integration/genericlocal/suite_test.go @@ -17,28 +17,44 @@ limitations under the License. package genericlocal import ( + "flag" "testing" + "k8s.io/klog" "openebs.io/metac/test/integration/framework" ) -// This will be run only once when go test is invoked against this -// package. All the other Test* functions will be invoked via m.Run -// call. +// TestMain will run only once when go test is invoked +// against this package. All the other Test* functions +// will be invoked via m.Run call. // // NOTE: -// `func TestMain(m *testing.M) {...}` -// is the canonical golang way to execute the test functions -// present in all *_test.go files in this package. -// -// NOTE: -// framework.TestWithConfigMetac provides the common dependencies -// like setup & teardown to let this test package run properly. -// -// NOTE: -// Instead of directly invoking m.Run() where m is *testing.M this -// function delegates to framework's TestWithConfigMetac which in -// turn invokes m.Run() +// There can be only one TestMain function in the entire +// package func TestMain(m *testing.M) { - framework.TestWithConfigMetac(m.Run) + flag.Parse() + + klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) + klog.InitFlags(klogFlags) + + // Sync the glog and klog flags. + flag.CommandLine.VisitAll(func(f1 *flag.Flag) { + f2 := klogFlags.Lookup(f1.Name) + if f2 != nil { + value := f1.Value.String() + f2.Value.Set(value) + } + }) + + // Pass m.Run function to framework which in turn + // sets up a kubernetes environment & then invokes + // m.Run. + err := framework.StartConfigBasedMetac(m.Run) + if err != nil { + // Since this is an error we must to invoke os.Exit(1) + // as per TestMain guidelines + klog.Exitf("%+v", err) + } + + defer klog.Flush() }