TEP-0127: Larger results using sidecar logs - feature flags and cluster-roles

config/enable-log-access-to-controller/clusterrole.yaml

@@ -0,0 +1,13 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: tekton-pipelines-controller-pod-log-access
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/part-of: tekton-pipelines
+rules:
+  - apiGroups: [""]
+    # Controller needs to get the logs of the results sidecar created by TaskRuns to extract results.
+    resources: ["pods/log"]
+    verbs: ["get"]

config/enable-log-access-to-controller/clusterrolebinding.yaml

@@ -0,0 +1,16 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: tekton-pipelines-controller-pod-log-access
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/part-of: tekton-pipelines
+subjects:
+  - kind: ServiceAccount
+    name: tekton-pipelines-controller
+    namespace: tekton-pipelines
+roleRef:
+  kind: ClusterRole
+  name: tekton-pipelines-controller-pod-log-access
+  apiGroup: rbac.authorization.k8s.io

docs/install.md

@@ -24,6 +24,7 @@ This guide explains how to install Tekton Pipelines. It covers the following top
     - [Customizing the Pipelines Controller behavior](#customizing-the-pipelines-controller-behavior)
     - [Alpha Features](#alpha-features)
     - [Beta Features](#beta-features)
+- [Enabling larger results using sidecar logs](#enabling-larger-results-using-sidecar-logs)
 - [Configuring High Availability](#configuring-high-availability)
 - [Configuring tekton pipeline controller performance](#configuring-tekton-pipeline-controller-performance)
 - [Creating a custom release of Tekton Pipelines](#creating-a-custom-release-of-tekton-pipelines)
@@ -421,6 +422,7 @@ features](#alpha-features) to be used.
   do both. For more information, see [Configuring usage of `TaskRun` and `Run` embedded statuses](pipelineruns.md#configuring-usage-of-taskrun-and-run-embedded-statuses).
 
 - `resource-verification-mode`: Setting this flag to "enforce" will enforce verification of tasks/pipeline. Failing to verify will fail the taskrun/pipelinerun. "warn" will only log the err message and "skip" will skip the whole verification.
+- `results-from`: set this flag to "termination-message" to use the container's termination message to fetch results from. This is the default method of extracting results. Set it to "sidecar-logs" to enable use of a results sidecar logs to extract results instead of termination message.
 
 - `enable-provenance-in-status`: set this flag to "true" to enable recording 
   the `provenance` field in `TaskRun` and `PipelineRun` status. The `provenance` 
@@ -477,6 +479,32 @@ the `feature-flags` ConfigMap alongside your Tekton Pipelines deployment via
 
 For beta versions of Tekton CRDs, setting `enable-api-fields` to "beta" is the same as setting it to "stable".
 
+## Enabling larger results using sidecar logs
+
+**Note**: The maximum size of a Task's results is limited by the container termination message feature of Kubernetes, as results are passed back to the controller via this mechanism. At present, the limit is per task is “4096 bytes”. All results produced by the task share this upper limit.
+
+To exceed this limit of 4096 bytes, you can enable larger results using sidecar logs. By enabling this feature, you will have a configurable limit (with a default of 4096 bytes) per result with no restriction on the number of results. The results are still stored in the taskrun crd so they should not exceed the 1.5MB CRD size limit.
+
+**Note**: to enable this feature, you need to grant `get` access to all `pods/log` to the `Tekton pipeline controller`. This means that the tekton pipeline controller has the ability to access the pod logs.
+
+1. Create a cluster role and rolebinding by applying the following spec to provide log access to `tekton-pipelines-controller`. 
+
+```
+kubectl apply -f config/enable-log-access-to-controller/
+```
+
+2. Set the `results-from` feature flag to use sidecar logs by setting `results-from: sidecar-logs` in the [configMap](#customizing-the-pipelines-controller-behavior).
+
+```
+kubectl patch cm feature-flags -n tekton-pipelines -p '{"data":{"results-from":"sidecar-logs"}}'
+```
+
+3. If you want the size per result to be something other than 4096 bytes, you can set the `max-result-size` feature flag in bytes by setting `max-result-size: 8192(whatever you need here)`. **Note:** The value you can set here cannot exceed the size of the CRD limit of 1.5 MB.
+
+```
+kubectl patch cm feature-flags -n tekton-pipelines -p '{"data":{"max-result-size":"<VALUE-IN-BYTES>"}}'
+```
+
 ## Configuring High Availability
 
 If you want to run Tekton Pipelines in a way so that webhooks are resiliant against failures and support

pkg/apis/config/feature_flags.go

@@ -50,6 +50,10 @@ const (
 	WarnResourceVerificationMode = "warn"
 	// SkipResourceVerificationMode is the value used for "resource-verification-mode" when verification is skipped
 	SkipResourceVerificationMode = "skip"
+	// ResultExtractionMethodTerminationMessage is the value used for "results-from" as a way to extract results from tasks using kubernetes termination message.
+	ResultExtractionMethodTerminationMessage = "termination-message"
+	// ResultExtractionMethodSidecarLogs is the value used for "results-from" as a way to extract results from tasks using sidecar logs.
+	ResultExtractionMethodSidecarLogs = "sidecar-logs"
 	// DefaultDisableAffinityAssistant is the default value for "disable-affinity-assistant".
 	DefaultDisableAffinityAssistant = false
 	// DefaultDisableCredsInit is the default value for "disable-creds-init".
@@ -76,6 +80,10 @@ const (
 	DefaultResourceVerificationMode = SkipResourceVerificationMode
 	// DefaultEnableProvenanceInStatus is the default value for "enable-provenance-status".
 	DefaultEnableProvenanceInStatus = false
+	// DefaultResultExtractionMethod is the default value for ResultExtractionMethod
+	DefaultResultExtractionMethod = ResultExtractionMethodTerminationMessage
+	// DefaultMaxResultSize is the default value in bytes for the size of a result
+	DefaultMaxResultSize = 4096
 
 	disableAffinityAssistantKey         = "disable-affinity-assistant"
 	disableCredsInitKey                 = "disable-creds-init"
@@ -90,6 +98,8 @@ const (
 	enableSpire                         = "enable-spire"
 	verificationMode                    = "resource-verification-mode"
 	enableProvenanceInStatus            = "enable-provenance-in-status"
+	resultExtractionMethod              = "results-from"
+	maxResultSize                       = "max-result-size"
 )
 
 // FeatureFlags holds the features configurations
@@ -109,6 +119,8 @@ type FeatureFlags struct {
 	EnableSpire                      bool
 	ResourceVerificationMode         string
 	EnableProvenanceInStatus         bool
+	ResultExtractionMethod           string
+	MaxResultSize                    int
 }
 
 // GetFeatureFlagsConfigName returns the name of the configmap containing all
@@ -166,6 +178,12 @@ func NewFeatureFlagsFromMap(cfgMap map[string]string) (*FeatureFlags, error) {
 	if err := setFeature(enableProvenanceInStatus, DefaultEnableProvenanceInStatus, &tc.EnableProvenanceInStatus); err != nil {
 		return nil, err
 	}
+	if err := setResultExtractionMethod(cfgMap, DefaultResultExtractionMethod, &tc.ResultExtractionMethod); err != nil {
+		return nil, err
+	}
+	if err := setMaxResultSize(cfgMap, DefaultMaxResultSize, &tc.MaxResultSize); err != nil {
+		return nil, err
+	}
 
 	// Given that they are alpha features, Tekton Bundles and Custom Tasks should be switched on if
 	// enable-api-fields is "alpha". If enable-api-fields is not "alpha" then fall back to the value of
@@ -223,6 +241,41 @@ func setEmbeddedStatus(cfgMap map[string]string, defaultValue string, feature *s
 	return nil
 }
 
+// setResultExtractionMethod sets the "results-from" flag based on the content of a given map.
+// If the feature gate is invalid or missing then an error is returned.
+func setResultExtractionMethod(cfgMap map[string]string, defaultValue string, feature *string) error {
+	value := defaultValue
+	if cfg, ok := cfgMap[resultExtractionMethod]; ok {
+		value = strings.ToLower(cfg)
+	}
+	switch value {
+	case ResultExtractionMethodTerminationMessage, ResultExtractionMethodSidecarLogs:
+		*feature = value
+	default:
+		return fmt.Errorf("invalid value for feature flag %q: %q", resultExtractionMethod, value)
+	}
+	return nil
+}
+
+// setMaxResultSize sets the "max-result-size" flag based on the content of a given map.
+// If the feature gate is invalid or missing then an error is returned.
+func setMaxResultSize(cfgMap map[string]string, defaultValue int, feature *int) error {
+	value := defaultValue
+	if cfg, ok := cfgMap[maxResultSize]; ok {
+		v, err := strconv.Atoi(cfg)
+		if err != nil {
+			return err
+		}
+		value = v
+	}
+	// if max limit is > 1.5 MB (CRD limit).
+	if value >= 1572864 {
+		return fmt.Errorf("invalid value for feature flag %q: %q. This is exceeding the CRD limit", resultExtractionMethod, value)
+	}
+	*feature = value
+	return nil
+}
+
 // setResourceVerificationMode sets the "resource-verification-mode" flag based on the content of a given map.
 // If the value is invalid or missing then an error is returned.
 func setResourceVerificationMode(cfgMap map[string]string, defaultValue string, feature *string) error {

pkg/apis/config/feature_flags_test.go

@@ -47,6 +47,8 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) {
 				EmbeddedStatus:           config.DefaultEmbeddedStatus,
 				ResourceVerificationMode: config.DefaultResourceVerificationMode,
 				EnableProvenanceInStatus: config.DefaultEnableProvenanceInStatus,
+				ResultExtractionMethod:   config.DefaultResultExtractionMethod,
+				MaxResultSize:            config.DefaultMaxResultSize,
 			},
 			fileName: config.GetFeatureFlagsConfigName(),
 		},
@@ -64,6 +66,8 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) {
 				EnableSpire:                      true,
 				ResourceVerificationMode:         "enforce",
 				EnableProvenanceInStatus:         true,
+				ResultExtractionMethod:           "termination-message",
+				MaxResultSize:                    4096,
 			},
 			fileName: "feature-flags-all-flags-set",
 		},
@@ -84,6 +88,8 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) {
 				SendCloudEventsForRuns:           config.DefaultSendCloudEventsForRuns,
 				EmbeddedStatus:                   config.DefaultEmbeddedStatus,
 				ResourceVerificationMode:         config.DefaultResourceVerificationMode,
+				ResultExtractionMethod:           config.DefaultResultExtractionMethod,
+				MaxResultSize:                    config.DefaultMaxResultSize,
 			},
 			fileName: "feature-flags-enable-api-fields-overrides-bundles-and-custom-tasks",
 		},
@@ -101,6 +107,8 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) {
 				SendCloudEventsForRuns:           config.DefaultSendCloudEventsForRuns,
 				EmbeddedStatus:                   config.DefaultEmbeddedStatus,
 				ResourceVerificationMode:         config.DefaultResourceVerificationMode,
+				ResultExtractionMethod:           config.DefaultResultExtractionMethod,
+				MaxResultSize:                    config.DefaultMaxResultSize,
 			},
 			fileName: "feature-flags-bundles-and-custom-tasks",
 		},
@@ -118,6 +126,8 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) {
 				SendCloudEventsForRuns:           config.DefaultSendCloudEventsForRuns,
 				EmbeddedStatus:                   config.DefaultEmbeddedStatus,
 				ResourceVerificationMode:         config.DefaultResourceVerificationMode,
+				ResultExtractionMethod:           config.DefaultResultExtractionMethod,
+				MaxResultSize:                    config.DefaultMaxResultSize,
 			},
 			fileName: "feature-flags-beta-api-fields",
 		},
@@ -129,9 +139,23 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) {
 				ResourceVerificationMode:         config.DefaultResourceVerificationMode,
 				RunningInEnvWithInjectedSidecars: config.DefaultRunningInEnvWithInjectedSidecars,
 				AwaitSidecarReadiness:            config.DefaultAwaitSidecarReadiness,
+				ResultExtractionMethod:           config.DefaultResultExtractionMethod,
+				MaxResultSize:                    config.DefaultMaxResultSize,
 			},
 			fileName: "feature-flags-enable-spire",
 		},
+		{
+			expectedConfig: &config.FeatureFlags{
+				EnableAPIFields:                  "stable",
+				EmbeddedStatus:                   "full",
+				ResourceVerificationMode:         config.DefaultResourceVerificationMode,
+				RunningInEnvWithInjectedSidecars: config.DefaultRunningInEnvWithInjectedSidecars,
+				AwaitSidecarReadiness:            config.DefaultAwaitSidecarReadiness,
+				ResultExtractionMethod:           config.ResultExtractionMethodSidecarLogs,
+				MaxResultSize:                    8192,
+			},
+			fileName: "feature-flags-results-via-sidecar-logs",
+		},
 	}
 
 	for _, tc := range testCases {
@@ -159,6 +183,8 @@ func TestNewFeatureFlagsFromEmptyConfigMap(t *testing.T) {
 		EnableSpire:                      config.DefaultEnableSpire,
 		ResourceVerificationMode:         config.DefaultResourceVerificationMode,
 		EnableProvenanceInStatus:         config.DefaultEnableProvenanceInStatus,
+		ResultExtractionMethod:           config.DefaultResultExtractionMethod,
+		MaxResultSize:                    config.DefaultMaxResultSize,
 	}
 	verifyConfigFileWithExpectedFeatureFlagsConfig(t, FeatureFlagsConfigEmptyName, expectedConfig)
 }
@@ -201,6 +227,12 @@ func TestNewFeatureFlagsConfigMapErrors(t *testing.T) {
 		fileName: "feature-flags-invalid-embedded-status",
 	}, {
 		fileName: "feature-flags-invalid-resource-verification-mode",
+	}, {
+		fileName: "feature-flags-invalid-results-from",
+	}, {
+		fileName: "feature-flags-invalid-max-result-size-too-large",
+	}, {
+		fileName: "feature-flags-invalid-max-result-size-bad-value",
 	}} {
 		t.Run(tc.fileName, func(t *testing.T) {
 			cm := test.ConfigMapFromTestFile(t, tc.fileName)

pkg/apis/config/testdata/feature-flags-invalid-max-result-size-bad-value.yaml

@@ -0,0 +1,21 @@
+# Copyright 2021 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: feature-flags
+  namespace: tekton-pipelines
+data:
+  max-result-size: "foo"

pkg/apis/config/testdata/feature-flags-invalid-max-result-size-too-large.yaml

@@ -0,0 +1,21 @@
+# Copyright 2021 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: feature-flags
+  namespace: tekton-pipelines
+data:
+  max-result-size: 10000000000000

pkg/apis/config/testdata/feature-flags-invalid-results-from.yaml

@@ -0,0 +1,21 @@
+# Copyright 2021 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: feature-flags
+  namespace: tekton-pipelines
+data:
+  results-from: "im-not-a-valid-results-from"

pkg/apis/config/testdata/feature-flags-results-via-sidecar-logs.yaml

@@ -0,0 +1,22 @@
+# Copyright 2021 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: feature-flags
+  namespace: tekton-pipelines
+data:
+  results-from: "sidecar-logs"
+  max-result-size: 8192