params must match the names of the params in the underlying Task
+whenWhen is a list of when expressions that need to be true for the matrix task to run
+[]github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.WhenExpression alias)-(Appears on:PipelineTask, Step) +(Appears on:IncludeParams, PipelineTask, Step)
WhenExpressions are used to specify whether a Task should be executed or skipped diff --git a/examples/v1/pipelineruns/beta/pipelinerun-with-matrix-when-include.yaml b/examples/v1/pipelineruns/beta/pipelinerun-with-matrix-when-include.yaml new file mode 100644 index 00000000000..90e915b6a86 --- /dev/null +++ b/examples/v1/pipelineruns/beta/pipelinerun-with-matrix-when-include.yaml @@ -0,0 +1,92 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: mytask + annotations: + description: | + A task that does something cool with GOARCH and version +spec: + params: + - name: IMAGE + default: '' + - name: PLATFORM + default: '' + steps: + - name: echo + image: mirror.gcr.io/alpine + script: | + echo image: $(params.IMAGE) and platform: $(params.PLATFORM) +--- +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + generateName: matrixed-include-when-pr +spec: + taskRunTemplate: + serviceAccountName: default + params: + - name: enabled-platforms + value: + - arm64 + - amd64 + - name: platform-amd64 + value: linux/amd64 + - name: platform-arm64 + value: linux/arm64 + - name: platform-ppc64le + value: linux-m4.xl/ppc64le + - name: platform-s390x + value: linux/s390x + - name: output-image + value: my-image + pipelineSpec: + tasks: + - name: build-containers-multi-platform + matrix: + include: + - name: amd64 + params: + - name: IMAGE + value: $(params.output-image)-amd64 + - name: PLATFORM + value: $(params.platform-amd64) + when: + - input: "amd64" + operator: in + values: + - "$(params.enabled-platforms[*])" + - name: arm64 + params: + - name: IMAGE + value: $(params.output-image)-arm64 + - name: PLATFORM + value: $(params.platform-arm64) + when: + - input: "arm64" + operator: in + values: + - "$(params.enabled-platforms[*])" + - name: ppc64le + params: + - name: IMAGE + value: $(params.output-image)-ppc64le + - name: PLATFORM + value: $(params.platform-ppc64le) + when: + - input: "ppc64le" + operator: in + values: + - "$(params.enabled-platforms[*])" + - name: s390x + params: + - name: IMAGE + value: $(params.output-image)-s390x + - name: PLATFORM + value: $(params.platform-s390x) + when: + - input: "s390x" + operator: in + values: + - "$(params.enabled-platforms[*])" + taskRef: + name: mytask \ No newline at end of file diff --git a/pkg/apis/pipeline/v1/matrix_types.go b/pkg/apis/pipeline/v1/matrix_types.go index f51a0ac4af4..d432c2be0e1 100644 --- a/pkg/apis/pipeline/v1/matrix_types.go +++ b/pkg/apis/pipeline/v1/matrix_types.go @@ -52,6 +52,20 @@ type IncludeParams struct { // The names of the `params` must match the names of the `params` in the underlying `Task` // +listType=atomic Params Params `json:"params,omitempty"` + + // When is a list of when expressions that need to be true for the matrix task to run + // +optional + When WhenExpressions `json:"when,omitempty"` +} + +func (i IncludeParamsList) validIncludeNum() int { + count := 0 + for _, include := range i { + if include.When.AllowsExecution(nil) { + count++ + } + } + return count } // Combination is a map, mainly defined to hold a single combination from a Matrix with key as param.Name and value as param.Value @@ -167,6 +181,9 @@ func (cs Combinations) fanOutMatrixParams(param Param) Combinations { func (m *Matrix) getIncludeCombinations() Combinations { var combinations Combinations for i := range m.Include { + if !m.Include[i].When.AllowsExecution(nil) { + continue + } includeParams := m.Include[i].Params newCombination := make(Combination) for _, param := range includeParams { @@ -248,11 +265,14 @@ func (m *Matrix) countNewCombinationsFromInclude() int { return 0 } if !m.HasParams() { - return len(m.Include) + return m.Include.validIncludeNum() } count := 0 matrixParamMap := m.Params.extractParamMapArrVals() for _, include := range m.Include { + if !include.When.AllowsExecution(nil) { + continue + } for _, param := range include.Params { if val, exist := matrixParamMap[param.Name]; exist { // If the Matrix Include param values does not exist, a new Combination will be generated diff --git a/pkg/apis/pipeline/v1/openapi_generated.go b/pkg/apis/pipeline/v1/openapi_generated.go index 42bf6748b1d..01d20ba2c9e 100644 --- a/pkg/apis/pipeline/v1/openapi_generated.go +++ b/pkg/apis/pipeline/v1/openapi_generated.go @@ -778,11 +778,25 @@ func schema_pkg_apis_pipeline_v1_IncludeParams(ref common.ReferenceCallback) com }, }, }, + "when": { + SchemaProps: spec.SchemaProps{ + Description: "When is a list of when expressions that need to be true for the matrix task to run", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.WhenExpression"), + }, + }, + }, + }, + }, }, }, }, Dependencies: []string{ - "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.Param"}, + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.Param", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.WhenExpression"}, } } diff --git a/pkg/apis/pipeline/v1/pipeline_validation.go b/pkg/apis/pipeline/v1/pipeline_validation.go index c17293a02b9..5b38dfa9f52 100644 --- a/pkg/apis/pipeline/v1/pipeline_validation.go +++ b/pkg/apis/pipeline/v1/pipeline_validation.go @@ -224,6 +224,20 @@ func (pt *PipelineTask) validateMatrix(ctx context.Context) (errs *apis.FieldErr errs = errs.Also(config.ValidateEnabledAPIFields(ctx, "matrix", config.BetaAPIFields)) errs = errs.Also(pt.Matrix.validateCombinationsCount(ctx)) errs = errs.Also(pt.Matrix.validateUniqueParams()) + if pt.Matrix.HasInclude() { + for i := range pt.Matrix.Include { + if pt.Matrix.Include[i].When != nil { + for _, we := range pt.Matrix.Include[i].When { + if we.CEL != "" { + // CEL is not allowed in matrix.include.when + errs = errs.Also(apis.ErrDisallowedFields("matrix.include.when.cel")) + return errs + } + } + errs = errs.Also(pt.Matrix.Include[i].When.validateWhenExpressionsFields(ctx).ViaFieldIndex("matrix.include", i)) + } + } + } } errs = errs.Also(pt.Matrix.validateParameterInOneOfMatrixOrParams(pt.Params)) return errs diff --git a/pkg/apis/pipeline/v1/pipeline_validation_test.go b/pkg/apis/pipeline/v1/pipeline_validation_test.go index 2dd9b22d55b..5edfa7f6a4c 100644 --- a/pkg/apis/pipeline/v1/pipeline_validation_test.go +++ b/pkg/apis/pipeline/v1/pipeline_validation_test.go @@ -4671,6 +4671,24 @@ func Test_validateMatrix(t *testing.T) { }}, }}, wantErrs: apis.ErrInvalidValue("Matrixed PipelineTasks emitting results must have an underlying type string, but result array-result has type array in pipelineTask", ""), + }, { + name: "cel in matrix include when expression", + tasks: PipelineTaskList{{ + Name: "a-task", + TaskRef: &TaskRef{Name: "a-task"}, + Matrix: &Matrix{ + Include: IncludeParamsList{ + { + When: WhenExpressions{ + { + CEL: "platform == 'linux'", + }, + }, + }, + }, + }, + }}, + wantErrs: apis.ErrDisallowedFields("[0].matrix.include.when.cel"), }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/apis/pipeline/v1/swagger.json b/pkg/apis/pipeline/v1/swagger.json index 584220d0b8c..6707d7e56a4 100644 --- a/pkg/apis/pipeline/v1/swagger.json +++ b/pkg/apis/pipeline/v1/swagger.json @@ -345,6 +345,14 @@ "$ref": "#/definitions/v1.Param" }, "x-kubernetes-list-type": "atomic" + }, + "when": { + "description": "When is a list of when expressions that need to be true for the matrix task to run", + "type": "array", + "items": { + "default": {}, + "$ref": "#/definitions/v1.WhenExpression" + } } } }, diff --git a/pkg/apis/pipeline/v1/zz_generated.deepcopy.go b/pkg/apis/pipeline/v1/zz_generated.deepcopy.go index 12dbe03bf8d..34749225cb5 100644 --- a/pkg/apis/pipeline/v1/zz_generated.deepcopy.go +++ b/pkg/apis/pipeline/v1/zz_generated.deepcopy.go @@ -210,6 +210,13 @@ func (in *IncludeParams) DeepCopyInto(out *IncludeParams) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.When != nil { + in, out := &in.When, &out.When + *out = make(WhenExpressions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } diff --git a/pkg/apis/pipeline/v1beta1/matrix_types.go b/pkg/apis/pipeline/v1beta1/matrix_types.go index 19042e11e4b..ae0db8841eb 100644 --- a/pkg/apis/pipeline/v1beta1/matrix_types.go +++ b/pkg/apis/pipeline/v1beta1/matrix_types.go @@ -52,6 +52,20 @@ type IncludeParams struct { // The names of the `params` must match the names of the `params` in the underlying `Task` // +listType=atomic Params Params `json:"params,omitempty"` + + // When is a list of when expressions that need to be true for the matrix task to run + // +optional + When WhenExpressions `json:"when,omitempty"` +} + +func (i IncludeParamsList) validIncludeNum() int { + count := 0 + for _, include := range i { + if include.When.AllowsExecution(nil) { + count++ + } + } + return count } // Combination is a map, mainly defined to hold a single combination from a Matrix with key as param.Name and value as param.Value @@ -167,6 +181,9 @@ func (cs Combinations) fanOutMatrixParams(param Param) Combinations { func (m *Matrix) getIncludeCombinations() Combinations { var combinations Combinations for i := range m.Include { + if !m.Include[i].When.AllowsExecution(nil) { + continue + } includeParams := m.Include[i].Params newCombination := make(Combination) for _, param := range includeParams { @@ -248,11 +265,14 @@ func (m *Matrix) countNewCombinationsFromInclude() int { return 0 } if !m.HasParams() { - return len(m.Include) + return m.Include.validIncludeNum() } count := 0 matrixParamMap := m.Params.extractParamMapArrVals() for _, include := range m.Include { + if !include.When.AllowsExecution(nil) { + continue + } for _, param := range include.Params { if val, exist := matrixParamMap[param.Name]; exist { // If the Matrix Include param values does not exist, a new Combination will be generated diff --git a/pkg/apis/pipeline/v1beta1/pipeline_conversion.go b/pkg/apis/pipeline/v1beta1/pipeline_conversion.go index b80b2ad74b7..dd9583ba816 100644 --- a/pkg/apis/pipeline/v1beta1/pipeline_conversion.go +++ b/pkg/apis/pipeline/v1beta1/pipeline_conversion.go @@ -19,7 +19,6 @@ package v1beta1 import ( "context" "fmt" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" @@ -280,6 +279,14 @@ func (m *Matrix) convertTo(ctx context.Context, sink *v1.Matrix) { } for i, include := range m.Include { sink.Include = append(sink.Include, v1.IncludeParams{Name: include.Name}) + if m.Include[i].When != nil { + sink.Include[i].When = v1.WhenExpressions{} + for _, we := range m.Include[i].When { + newWe := v1.WhenExpression{} + we.convertTo(ctx, &newWe) + sink.Include[i].When = append(sink.Include[i].When, newWe) + } + } for _, param := range include.Params { newIncludeParam := v1.Param{} param.convertTo(ctx, &newIncludeParam) @@ -297,6 +304,14 @@ func (m *Matrix) convertFrom(ctx context.Context, source v1.Matrix) { for i, include := range source.Include { m.Include = append(m.Include, IncludeParams{Name: include.Name}) + if source.Include[i].When != nil { + m.Include[i].When = WhenExpressions{} + for _, we := range source.Include[i].When { + newWe := WhenExpression{} + newWe.convertFrom(ctx, we) + m.Include[i].When = append(m.Include[i].When, newWe) + } + } for _, p := range include.Params { new := Param{} new.ConvertFrom(ctx, p) diff --git a/pkg/apis/pipeline/v1beta1/pipeline_validation.go b/pkg/apis/pipeline/v1beta1/pipeline_validation.go index 8f8d6a1f002..6b89fb59435 100644 --- a/pkg/apis/pipeline/v1beta1/pipeline_validation.go +++ b/pkg/apis/pipeline/v1beta1/pipeline_validation.go @@ -247,6 +247,20 @@ func (pt *PipelineTask) validateMatrix(ctx context.Context) (errs *apis.FieldErr errs = errs.Also(config.ValidateEnabledAPIFields(ctx, "matrix", config.BetaAPIFields)) errs = errs.Also(pt.Matrix.validateCombinationsCount(ctx)) errs = errs.Also(pt.Matrix.validateUniqueParams()) + if pt.Matrix.HasInclude() { + for i := range pt.Matrix.Include { + if pt.Matrix.Include[i].When != nil { + for _, we := range pt.Matrix.Include[i].When { + if we.CEL != "" { + // CEL is not allowed in matrix.include.when + errs = errs.Also(apis.ErrDisallowedFields("matrix.include.when.cel")) + return errs + } + } + errs = errs.Also(pt.Matrix.Include[i].When.validateWhenExpressionsFields(ctx).ViaFieldIndex("matrix.include", i)) + } + } + } } errs = errs.Also(pt.Matrix.validateParameterInOneOfMatrixOrParams(pt.Params)) return errs diff --git a/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go b/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go index 8faf62c9182..61a7e0023fd 100644 --- a/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go +++ b/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go @@ -4551,6 +4551,24 @@ func Test_validateMatrix(t *testing.T) { }}, }}, wantErrs: apis.ErrGeneric("A matrixed pipelineTask can only be consumed in aggregate using [*] notation, but is currently set to tasks.matrix-emitting-results-embedded.results.report-url[0]"), + }, { + name: "cel in matrix include when expression", + tasks: PipelineTaskList{{ + Name: "a-task", + TaskRef: &TaskRef{Name: "a-task"}, + Matrix: &Matrix{ + Include: IncludeParamsList{ + { + When: WhenExpressions{ + { + CEL: "platform == 'linux'", + }, + }, + }, + }, + }, + }}, + wantErrs: apis.ErrDisallowedFields("[0].matrix.include.when.cel"), }, { name: "invalid matrix emitting array results consumed in aggregate by another pipelineTask", tasks: PipelineTaskList{{ diff --git a/pkg/apis/pipeline/v1beta1/zz_generated.deepcopy.go b/pkg/apis/pipeline/v1beta1/zz_generated.deepcopy.go index 7852edd8c7c..15b9ab246d5 100644 --- a/pkg/apis/pipeline/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/pipeline/v1beta1/zz_generated.deepcopy.go @@ -458,6 +458,13 @@ func (in *IncludeParams) DeepCopyInto(out *IncludeParams) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.When != nil { + in, out := &in.When, &out.When + *out = make(WhenExpressions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } diff --git a/pkg/reconciler/pipelinerun/resources/apply.go b/pkg/reconciler/pipelinerun/resources/apply.go index d8de08dae5e..4a08e83b18b 100644 --- a/pkg/reconciler/pipelinerun/resources/apply.go +++ b/pkg/reconciler/pipelinerun/resources/apply.go @@ -263,6 +263,16 @@ func ApplyTaskResults(targets PipelineRunState, resolvedResultRefs ResolvedResul } } +// ApplyMatrixIncludeWhenExpressions replaces param variables referring to matrix when include +func ApplyMatrixIncludeWhenExpressions(pt *v1.PipelineTask, pr *v1.PipelineRun) { + stringReplacements, arrayReplacements, _ := paramsFromPipelineRun(context.TODO(), pr) + for i := range pt.Matrix.Include { + if pt.Matrix.Include[i].When != nil { + pt.Matrix.Include[i].When = pt.Matrix.Include[i].When.ReplaceVariables(stringReplacements, arrayReplacements) + } + } +} + // ApplyPipelineTaskStateContext replaces context variables referring to execution status with the specified status func ApplyPipelineTaskStateContext(state PipelineRunState, replacements map[string]string) { for _, resolvedPipelineRunTask := range state { @@ -308,6 +318,9 @@ func ApplyReplacements(p *v1.PipelineSpec, replacements map[string]string, array p.Tasks[i].Matrix.Params = p.Tasks[i].Matrix.Params.ReplaceVariables(replacements, arrayReplacements, nil) for j := range p.Tasks[i].Matrix.Include { p.Tasks[i].Matrix.Include[j].Params = p.Tasks[i].Matrix.Include[j].Params.ReplaceVariables(replacements, nil, nil) + if p.Tasks[i].Matrix.Include[j].When != nil { + p.Tasks[i].Matrix.Include[j].When = p.Tasks[i].Matrix.Include[j].When.ReplaceVariables(replacements, arrayReplacements) + } } } else { p.Tasks[i].DisplayName = substitution.ApplyReplacements(p.Tasks[i].DisplayName, replacements) @@ -331,6 +344,9 @@ func ApplyReplacements(p *v1.PipelineSpec, replacements map[string]string, array p.Finally[i].Matrix.Params = p.Finally[i].Matrix.Params.ReplaceVariables(replacements, arrayReplacements, nil) for j := range p.Finally[i].Matrix.Include { p.Finally[i].Matrix.Include[j].Params = p.Finally[i].Matrix.Include[j].Params.ReplaceVariables(replacements, nil, nil) + if p.Finally[i].Matrix.Include[j].When != nil { + p.Finally[i].Matrix.Include[j].When = p.Finally[i].Matrix.Include[j].When.ReplaceVariables(replacements, arrayReplacements) + } } } else { p.Finally[i].DisplayName = substitution.ApplyReplacements(p.Finally[i].DisplayName, replacements) diff --git a/pkg/reconciler/pipelinerun/resources/apply_test.go b/pkg/reconciler/pipelinerun/resources/apply_test.go index 507d53c17b3..01eca2d6c17 100644 --- a/pkg/reconciler/pipelinerun/resources/apply_test.go +++ b/pkg/reconciler/pipelinerun/resources/apply_test.go @@ -3405,6 +3405,103 @@ func TestContext(t *testing.T) { } } +func TestApplyMatrixIncludeWhenExpressions(t *testing.T) { + for _, tc := range []struct { + description string + pt *v1.PipelineTask + pr *v1.PipelineRun + want v1.IncludeParamsList + }{ + { + description: "string replacement", + pt: &v1.PipelineTask{ + Matrix: &v1.Matrix{ + Include: v1.IncludeParamsList{ + { + When: v1.WhenExpressions{ + { + Input: "arm64", + Operator: selection.In, + Values: []string{"$(params.platform)"}, + }, + }, + }, + }, + }, + }, + pr: &v1.PipelineRun{ + Spec: v1.PipelineRunSpec{ + Params: v1.Params{ + { + Name: "platform", + Value: *v1.NewStructuredValues("arm64"), + }, + }, + }, + }, + want: v1.IncludeParamsList{ + { + When: v1.WhenExpressions{ + { + Input: "arm64", + Operator: selection.In, + Values: []string{"arm64"}, + }, + }, + }, + }, + }, { + description: "array replacement", + pt: &v1.PipelineTask{ + Matrix: &v1.Matrix{ + Include: v1.IncludeParamsList{ + { + When: v1.WhenExpressions{ + { + Input: "arm64", + Operator: selection.In, + Values: []string{"$(params.platform[*])"}, + }, + }, + }, + }, + }, + }, + pr: &v1.PipelineRun{ + Spec: v1.PipelineRunSpec{ + Params: v1.Params{ + { + Name: "platform", + Value: v1.ParamValue{ + Type: v1.ParamTypeArray, + ArrayVal: []string{"arm64", "amd64"}, + }, + }, + }, + }, + }, + want: v1.IncludeParamsList{ + { + When: v1.WhenExpressions{ + { + Input: "arm64", + Operator: selection.In, + Values: []string{"arm64", "amd64"}, + }, + }, + }, + }, + }, + } { + t.Run(tc.description, func(t *testing.T) { + resources.ApplyMatrixIncludeWhenExpressions(tc.pt, tc.pr) + if d := cmp.Diff(tc.want, tc.pt.Matrix.Include); d != "" { + t.Errorf(diff.PrintWantGot(d)) + } + }) + } +} + func TestApplyPipelineTaskContexts(t *testing.T) { for _, tc := range []struct { description string diff --git a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go index db1fd1ffb25..de3eb719d40 100644 --- a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go +++ b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go @@ -601,6 +601,7 @@ func ResolvePipelineTask( ApplyTaskResults(PipelineRunState{&rpt}, resolvedResultRefs) if rpt.PipelineTask.IsMatrixed() { + ApplyMatrixIncludeWhenExpressions(rpt.PipelineTask, &pipelineRun) numCombinations = rpt.PipelineTask.Matrix.CountCombinations() } if rpt.IsCustomTask() {