diff --git a/Makefile b/Makefile index 1a017fdc..2c80c41d 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ rego: fmt-rego test-rego .PHONY: fmt-rego fmt-rego: - opa fmt -w rules/cloud/policies + opa fmt -w rules/ .PHONY: test-rego test-rego: diff --git a/cmd/avd_generator/main.go b/cmd/avd_generator/main.go index 953f3617..c737c1f2 100644 --- a/cmd/avd_generator/main.go +++ b/cmd/avd_generator/main.go @@ -12,7 +12,7 @@ import ( "github.com/aquasecurity/defsec/pkg/framework" - _ "github.com/aquasecurity/trivy-policies/pkg/rego" + _ "github.com/aquasecurity/trivy-policies/pkg/rego/embed" registered "github.com/aquasecurity/trivy-policies/pkg/rules" "github.com/aquasecurity/trivy-policies/pkg/types" ) diff --git a/internal/rego/compiler.go b/internal/rego/compiler.go deleted file mode 100644 index 70012cc2..00000000 --- a/internal/rego/compiler.go +++ /dev/null @@ -1,10 +0,0 @@ -package rego - -import "github.com/open-policy-agent/opa/ast" - -func NewRegoCompiler(schemas *ast.SchemaSet) *ast.Compiler { - return ast.NewCompiler(). - WithUseTypeCheckAnnotations(true). - WithCapabilities(ast.CapabilitiesForThisVersion()). - WithSchemas(schemas) -} diff --git a/pkg/rego/build.go b/pkg/rego/build.go new file mode 100644 index 00000000..82b144bc --- /dev/null +++ b/pkg/rego/build.go @@ -0,0 +1,84 @@ +package rego + +import ( + "io/fs" + "path/filepath" + "strings" + + "github.com/aquasecurity/defsec/pkg/types" + "github.com/aquasecurity/trivy-policies/pkg/rego/schemas" + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/util" +) + +func BuildSchemaSetFromPolicies(policies map[string]*ast.Module, paths []string, fsys fs.FS) (*ast.SchemaSet, bool, error) { + schemaSet := ast.NewSchemaSet() + schemaSet.Put(ast.MustParseRef("schema.input"), map[string]interface{}{}) // for backwards compat only + var customFound bool + for _, policy := range policies { + for _, annotation := range policy.Annotations { + for _, ss := range annotation.Schemas { + schemaName, err := ss.Schema.Ptr() + if err != nil { + continue + } + if schemaName != "input" { + if schema, ok := schemas.SchemaMap[types.Source(schemaName)]; ok { + customFound = true + schemaSet.Put(ast.MustParseRef(ss.Schema.String()), util.MustUnmarshalJSON([]byte(schema))) + } else { + b, err := findSchemaInFS(paths, fsys, schemaName) + if err != nil { + return schemaSet, true, err + } + if b != nil { + customFound = true + schemaSet.Put(ast.MustParseRef(ss.Schema.String()), util.MustUnmarshalJSON(b)) + } + } + } + } + } + } + + return schemaSet, customFound, nil +} + +// findSchemaInFS tries to find the schema anywhere in the specified FS +func findSchemaInFS(paths []string, srcFS fs.FS, schemaName string) ([]byte, error) { + var schema []byte + for _, path := range paths { + if err := fs.WalkDir(srcFS, sanitisePath(path), func(path string, info fs.DirEntry, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + if !IsJSONFile(info.Name()) { + return nil + } + if info.Name() == schemaName+".json" { + schema, err = fs.ReadFile(srcFS, filepath.ToSlash(path)) + if err != nil { + return err + } + return nil + } + return nil + }); err != nil { + return nil, err + } + } + return schema, nil +} + +func IsJSONFile(name string) bool { + return strings.HasSuffix(name, ".json") +} + +func sanitisePath(path string) string { + vol := filepath.VolumeName(path) + path = strings.TrimPrefix(path, vol) + return strings.TrimPrefix(strings.TrimPrefix(filepath.ToSlash(path), "./"), "/") +} diff --git a/pkg/rego/embed.go b/pkg/rego/embed.go deleted file mode 100644 index dd44d469..00000000 --- a/pkg/rego/embed.go +++ /dev/null @@ -1,110 +0,0 @@ -package rego - -import ( - "context" - "embed" - "path/filepath" - "strings" - - "github.com/aquasecurity/trivy-policies/internal/rules" - rules2 "github.com/aquasecurity/trivy-policies/rules" - "github.com/open-policy-agent/opa/ast" -) - -func init() { - - modules, err := loadEmbeddedPolicies() - if err != nil { - // we should panic as the policies were not embedded properly - panic(err) - } - loadedLibs, err := loadEmbeddedLibraries() - if err != nil { - panic(err) - } - for name, policy := range loadedLibs { - modules[name] = policy - } - - RegisterRegoRules(modules) -} - -func RegisterRegoRules(modules map[string]*ast.Module) { - ctx := context.TODO() - - schemaSet, _, _ := BuildSchemaSetFromPolicies(modules, nil, nil) - - compiler := ast.NewCompiler(). - WithSchemas(schemaSet). - WithCapabilities(nil). - WithUseTypeCheckAnnotations(true) - - compiler.Compile(modules) - if compiler.Failed() { - // we should panic as the embedded rego policies are syntactically incorrect... - panic(compiler.Errors) - } - - retriever := NewMetadataRetriever(compiler) - for _, module := range modules { - metadata, err := retriever.RetrieveMetadata(ctx, module) - if err != nil { - continue - } - if metadata.AVDID == "" { - continue - } - rules.Register( - metadata.ToRule(), - nil, - ) - } -} - -func loadEmbeddedPolicies() (map[string]*ast.Module, error) { - return RecurseEmbeddedModules(rules2.EmbeddedPolicyFileSystem, ".") -} - -func loadEmbeddedLibraries() (map[string]*ast.Module, error) { - return RecurseEmbeddedModules(rules2.EmbeddedLibraryFileSystem, ".") -} - -func RecurseEmbeddedModules(fs embed.FS, dir string) (map[string]*ast.Module, error) { - if strings.HasSuffix(dir, "policies/advanced/optional") { - return nil, nil - } - dir = strings.TrimPrefix(dir, "./") - modules := make(map[string]*ast.Module) - entries, err := fs.ReadDir(filepath.ToSlash(dir)) - if err != nil { - return nil, err - } - for _, entry := range entries { - if entry.IsDir() { - subs, err := RecurseEmbeddedModules(fs, strings.Join([]string{dir, entry.Name()}, "/")) - if err != nil { - return nil, err - } - for key, val := range subs { - modules[key] = val - } - continue - } - if !IsRegoFile(entry.Name()) || IsDotFile(entry.Name()) { - continue - } - fullPath := strings.Join([]string{dir, entry.Name()}, "/") - data, err := fs.ReadFile(filepath.ToSlash(fullPath)) - if err != nil { - return nil, err - } - mod, err := ast.ParseModuleWithOpts(fullPath, string(data), ast.ParserOptions{ - ProcessAnnotation: true, - }) - if err != nil { - return nil, err - } - modules[fullPath] = mod - } - return modules, nil -} diff --git a/pkg/rego/embed/embed.go b/pkg/rego/embed/embed.go new file mode 100644 index 00000000..5bee8ce7 --- /dev/null +++ b/pkg/rego/embed/embed.go @@ -0,0 +1,125 @@ +package embed + +import ( + "context" + "io/fs" + "path/filepath" + "strings" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/bundle" + + "github.com/aquasecurity/trivy-policies/pkg/rego" + "github.com/aquasecurity/trivy-policies/pkg/rules" + rules2 "github.com/aquasecurity/trivy-policies/rules" +) + +func init() { + + modules, err := LoadEmbeddedPolicies() + if err != nil { + // we should panic as the policies were not embedded properly + panic(err) + } + loadedLibs, err := LoadEmbeddedLibraries() + if err != nil { + panic(err) + } + for name, policy := range loadedLibs { + modules[name] = policy + } + + RegisterRegoRules(modules) +} + +func RegisterRegoRules(modules map[string]*ast.Module) { + ctx := context.TODO() + + schemaSet, _, _ := rego.BuildSchemaSetFromPolicies(modules, nil, nil) + + compiler := ast.NewCompiler(). + WithSchemas(schemaSet). + WithCapabilities(nil). + WithUseTypeCheckAnnotations(true) + + compiler.Compile(modules) + if compiler.Failed() { + // we should panic as the embedded rego policies are syntactically incorrect... + panic(compiler.Errors) + } + + retriever := rego.NewMetadataRetriever(compiler) + for _, module := range modules { + metadata, err := retriever.RetrieveMetadata(ctx, module) + if err != nil { + continue + } + if metadata.AVDID == "" { + continue + } + rules.Register( + metadata.ToRule(), + nil, + ) + } +} + +func LoadEmbeddedPolicies() (map[string]*ast.Module, error) { + return LoadPoliciesFromDirs(rules2.EmbeddedPolicyFileSystem, ".") +} + +func LoadEmbeddedLibraries() (map[string]*ast.Module, error) { + return LoadPoliciesFromDirs(rules2.EmbeddedLibraryFileSystem, ".") +} + +func IsRegoFile(name string) bool { + return strings.HasSuffix(name, bundle.RegoExt) && !strings.HasSuffix(name, "_test"+bundle.RegoExt) +} + +func IsDotFile(name string) bool { + return strings.HasPrefix(name, ".") +} + +func sanitisePath(path string) string { + vol := filepath.VolumeName(path) + path = strings.TrimPrefix(path, vol) + return strings.TrimPrefix(strings.TrimPrefix(filepath.ToSlash(path), "./"), "/") +} + +func LoadPoliciesFromDirs(target fs.FS, paths ...string) (map[string]*ast.Module, error) { + modules := make(map[string]*ast.Module) + for _, path := range paths { + if err := fs.WalkDir(target, sanitisePath(path), func(path string, info fs.DirEntry, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + if strings.HasSuffix(filepath.Dir(filepath.ToSlash(path)), "policies/advanced/optional") { + return fs.SkipDir + } + + if !IsRegoFile(info.Name()) || IsDotFile(info.Name()) { + return nil + } + data, err := fs.ReadFile(target, filepath.ToSlash(path)) + if err != nil { + return err + } + module, err := ast.ParseModuleWithOpts(path, string(data), ast.ParserOptions{ + ProcessAnnotation: true, + }) + if err != nil { + // s.debug.Log("Failed to load module: %s, err: %s", filepath.ToSlash(path), err.Error()) + return err + } + modules[path] = module + return nil + }); err != nil { + return nil, err + } + } + return modules, nil +} diff --git a/pkg/rego/embed_test.go b/pkg/rego/embed/embed_test.go similarity index 92% rename from pkg/rego/embed_test.go rename to pkg/rego/embed/embed_test.go index a4787fb1..8bb1a154 100644 --- a/pkg/rego/embed_test.go +++ b/pkg/rego/embed/embed_test.go @@ -1,9 +1,9 @@ -package rego +package embed import ( "testing" - "github.com/aquasecurity/trivy-policies/internal/rules" + "github.com/aquasecurity/trivy-policies/pkg/rules" rules2 "github.com/aquasecurity/trivy-policies/rules" "github.com/open-policy-agent/opa/ast" "github.com/stretchr/testify/assert" @@ -12,7 +12,7 @@ import ( func Test_EmbeddedLoading(t *testing.T) { - frameworkRules := rules.GetFrameworkRules() + frameworkRules := rules.GetRegistered() var found bool for _, rule := range frameworkRules { if rule.GetRule().RegoPackage != "" { @@ -102,7 +102,7 @@ deny[res]{ for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - policies, err := RecurseEmbeddedModules(rules2.EmbeddedLibraryFileSystem, ".") + policies, err := LoadPoliciesFromDirs(rules2.EmbeddedLibraryFileSystem, ".") require.NoError(t, err) newRule, err := ast.ParseModuleWithOpts("/rules/newrule.rego", tc.inputPolicy, ast.ParserOptions{ ProcessAnnotation: true, diff --git a/pkg/rego/exceptions.go b/pkg/rego/exceptions.go deleted file mode 100644 index ab202ec0..00000000 --- a/pkg/rego/exceptions.go +++ /dev/null @@ -1,33 +0,0 @@ -package rego - -import ( - "context" - "fmt" -) - -func (s *Scanner) isIgnored(ctx context.Context, namespace string, ruleName string, input interface{}) (bool, error) { - if ignored, err := s.isNamespaceIgnored(ctx, namespace, input); err != nil { - return false, err - } else if ignored { - return true, nil - } - return s.isRuleIgnored(ctx, namespace, ruleName, input) -} - -func (s *Scanner) isNamespaceIgnored(ctx context.Context, namespace string, input interface{}) (bool, error) { - exceptionQuery := fmt.Sprintf("data.namespace.exceptions.exception[_] == %q", namespace) - result, _, err := s.runQuery(ctx, exceptionQuery, input, true) - if err != nil { - return false, fmt.Errorf("query namespace exceptions: %w", err) - } - return result.Allowed(), nil -} - -func (s *Scanner) isRuleIgnored(ctx context.Context, namespace string, ruleName string, input interface{}) (bool, error) { - exceptionQuery := fmt.Sprintf("endswith(%q, data.%s.exception[_][_])", ruleName, namespace) - result, _, err := s.runQuery(ctx, exceptionQuery, input, true) - if err != nil { - return false, err - } - return result.Allowed(), nil -} diff --git a/pkg/rego/load.go b/pkg/rego/load.go deleted file mode 100644 index d7ddf1b8..00000000 --- a/pkg/rego/load.go +++ /dev/null @@ -1,269 +0,0 @@ -package rego - -import ( - "context" - "fmt" - "io" - "io/fs" - "path/filepath" - "strings" - - "github.com/aquasecurity/trivy-policies/internal/rego" - - "github.com/open-policy-agent/opa/ast" - "github.com/open-policy-agent/opa/bundle" -) - -func IsRegoFile(name string) bool { - return strings.HasSuffix(name, bundle.RegoExt) && !strings.HasSuffix(name, "_test"+bundle.RegoExt) -} - -func IsDotFile(name string) bool { - return strings.HasPrefix(name, ".") -} - -func IsJSONFile(name string) bool { - return strings.HasSuffix(name, ".json") -} - -func sanitisePath(path string) string { - vol := filepath.VolumeName(path) - path = strings.TrimPrefix(path, vol) - - return strings.TrimPrefix(strings.TrimPrefix(filepath.ToSlash(path), "./"), "/") -} - -func (s *Scanner) loadPoliciesFromDirs(target fs.FS, paths []string) (map[string]*ast.Module, error) { - modules := make(map[string]*ast.Module) - for _, path := range paths { - if err := fs.WalkDir(target, sanitisePath(path), func(path string, info fs.DirEntry, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - if !IsRegoFile(info.Name()) || IsDotFile(info.Name()) { - return nil - } - data, err := fs.ReadFile(target, filepath.ToSlash(path)) - if err != nil { - return err - } - module, err := ast.ParseModuleWithOpts(path, string(data), ast.ParserOptions{ - ProcessAnnotation: true, - }) - if err != nil { - s.debug.Log("Failed to load module: %s, err: %s", filepath.ToSlash(path), err.Error()) - return nil - } - modules[path] = module - return nil - }); err != nil { - return nil, err - } - } - return modules, nil -} - -func (s *Scanner) loadPoliciesFromReaders(readers []io.Reader) (map[string]*ast.Module, error) { - modules := make(map[string]*ast.Module) - for i, r := range readers { - moduleName := fmt.Sprintf("reader_%d", i) - data, err := io.ReadAll(r) - if err != nil { - return nil, err - } - module, err := ast.ParseModuleWithOpts(moduleName, string(data), ast.ParserOptions{ - ProcessAnnotation: true, - }) - if err != nil { - return nil, err - } - modules[moduleName] = module - } - return modules, nil -} - -func (s *Scanner) LoadEmbeddedLibraries() error { - if s.policies == nil { - s.policies = make(map[string]*ast.Module) - } - loadedLibs, err := loadEmbeddedLibraries() - if err != nil { - return fmt.Errorf("failed to load embedded rego libraries: %w", err) - } - for name, policy := range loadedLibs { - s.policies[name] = policy - } - s.debug.Log("Loaded %d embedded libraries (without embedded policies).", len(loadedLibs)) - return nil -} - -func (s *Scanner) loadEmbedded(enableEmbeddedLibraries, enableEmbeddedPolicies bool) error { - if enableEmbeddedLibraries { - loadedLibs, errLoad := loadEmbeddedLibraries() - if errLoad != nil { - return fmt.Errorf("failed to load embedded rego libraries: %w", errLoad) - } - for name, policy := range loadedLibs { - s.policies[name] = policy - } - s.debug.Log("Loaded %d embedded libraries.", len(loadedLibs)) - } - - if enableEmbeddedPolicies { - loaded, err := loadEmbeddedPolicies() - if err != nil { - return fmt.Errorf("failed to load embedded rego policies: %w", err) - } - for name, policy := range loaded { - s.policies[name] = policy - } - s.debug.Log("Loaded %d embedded policies.", len(loaded)) - } - - return nil -} - -func (s *Scanner) LoadPolicies(enableEmbeddedLibraries, enableEmbeddedPolicies bool, srcFS fs.FS, paths []string, readers []io.Reader) error { - - if s.policies == nil { - s.policies = make(map[string]*ast.Module) - } - - if s.policyFS != nil { - s.debug.Log("Overriding filesystem for policies!") - srcFS = s.policyFS - } - - if err := s.loadEmbedded(enableEmbeddedLibraries, enableEmbeddedPolicies); err != nil { - return err - } - - var err error - if len(paths) > 0 { - loaded, err := s.loadPoliciesFromDirs(srcFS, paths) - if err != nil { - return fmt.Errorf("failed to load rego policies from %s: %w", paths, err) - } - for name, policy := range loaded { - s.policies[name] = policy - } - s.debug.Log("Loaded %d policies from disk.", len(loaded)) - } - - if len(readers) > 0 { - loaded, err := s.loadPoliciesFromReaders(readers) - if err != nil { - return fmt.Errorf("failed to load rego policies from reader(s): %w", err) - } - for name, policy := range loaded { - s.policies[name] = policy - } - s.debug.Log("Loaded %d policies from reader(s).", len(loaded)) - } - - // gather namespaces - uniq := make(map[string]struct{}) - for _, module := range s.policies { - namespace := getModuleNamespace(module) - uniq[namespace] = struct{}{} - } - var namespaces []string - for namespace := range uniq { - namespaces = append(namespaces, namespace) - } - - dataFS := srcFS - if s.dataFS != nil { - s.debug.Log("Overriding filesystem for data!") - dataFS = s.dataFS - } - store, err := initStore(dataFS, s.dataDirs, namespaces) - if err != nil { - return fmt.Errorf("unable to load data: %w", err) - } - s.store = store - - return s.compilePolicies(srcFS, paths) -} - -func (s *Scanner) prunePoliciesWithError(compiler *ast.Compiler) error { - if len(compiler.Errors) > s.regoErrorLimit { - s.debug.Log("Error(s) occurred while loading policies") - return compiler.Errors - } - - for _, e := range compiler.Errors { - s.debug.Log("Error occurred while parsing: %s, %s", e.Location.File, e.Error()) - delete(s.policies, e.Location.File) - } - return nil -} - -func (s *Scanner) compilePolicies(srcFS fs.FS, paths []string) error { - - schemaSet, custom, err := BuildSchemaSetFromPolicies(s.policies, paths, srcFS) - if err != nil { - return err - } - if custom { - s.inputSchema = nil // discard auto detected input schema in favour of policy defined schema - } - - compiler := rego.NewRegoCompiler(schemaSet) - - compiler.Compile(s.policies) - if compiler.Failed() { - if err := s.prunePoliciesWithError(compiler); err != nil { - return err - } - return s.compilePolicies(srcFS, paths) - } - retriever := NewMetadataRetriever(compiler) - - if err := s.filterModules(retriever); err != nil { - return err - } - if s.inputSchema != nil { - schemaSet := ast.NewSchemaSet() - schemaSet.Put(ast.MustParseRef("schema.input"), s.inputSchema) - compiler.WithSchemas(schemaSet) - compiler.Compile(s.policies) - if compiler.Failed() { - if err := s.prunePoliciesWithError(compiler); err != nil { - return err - } - return s.compilePolicies(srcFS, paths) - } - } - s.compiler = compiler - s.retriever = retriever - return nil -} - -func (s *Scanner) filterModules(retriever *MetadataRetriever) error { - - filtered := make(map[string]*ast.Module) - for name, module := range s.policies { - meta, err := retriever.RetrieveMetadata(context.TODO(), module) - if err != nil { - return err - } - if len(meta.InputOptions.Selectors) == 0 { - s.debug.Log("WARNING: Module %s has no input selectors - it will be loaded for all inputs!", name) - filtered[name] = module - continue - } - for _, selector := range meta.InputOptions.Selectors { - if selector.Type == string(s.sourceType) { - filtered[name] = module - break - } - } - } - - s.policies = filtered - return nil -} diff --git a/pkg/rego/load_test.go b/pkg/rego/load_test.go deleted file mode 100644 index 02197b23..00000000 --- a/pkg/rego/load_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package rego - -import ( - "bytes" - "embed" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/aquasecurity/defsec/pkg/types" - "github.com/stretchr/testify/require" -) - -//go:embed all:testdata/policies -var testEmbedFS embed.FS - -func Test_RegoScanning_WithSomeInvalidPolicies(t *testing.T) { - t.Run("allow no errors", func(t *testing.T) { - var debugBuf bytes.Buffer - scanner := NewScanner(types.SourceDockerfile) - scanner.SetRegoErrorLimit(0) - scanner.SetDebugWriter(&debugBuf) - p, _ := RecurseEmbeddedModules(testEmbedFS, ".") - require.NotNil(t, p) - - scanner.policies = p - err := scanner.compilePolicies(testEmbedFS, []string{"policies"}) - require.ErrorContains(t, err, `want (one of): ["Cmd" "EndLine" "Flags" "JSON" "Original" "Path" "Stage" "StartLine" "SubCmd" "Value"]`) - assert.Contains(t, debugBuf.String(), "Error(s) occurred while loading policies") - }) - - t.Run("allow up to max 1 error", func(t *testing.T) { - var debugBuf bytes.Buffer - scanner := NewScanner(types.SourceDockerfile) - scanner.SetRegoErrorLimit(1) - scanner.SetDebugWriter(&debugBuf) - - p, _ := RecurseEmbeddedModules(testEmbedFS, ".") - scanner.policies = p - - err := scanner.compilePolicies(testEmbedFS, []string{"policies"}) - require.NoError(t, err) - - assert.Contains(t, debugBuf.String(), "Error occurred while parsing: testdata/policies/invalid.rego, testdata/policies/invalid.rego:7") - }) - -} diff --git a/pkg/rego/metadata.go b/pkg/rego/metadata.go index 80bbe263..ee6b5d1d 100644 --- a/pkg/rego/metadata.go +++ b/pkg/rego/metadata.go @@ -3,8 +3,6 @@ package rego import ( "context" "fmt" - "io/fs" - "path/filepath" "strings" "github.com/aquasecurity/defsec/pkg/framework" @@ -15,7 +13,6 @@ import ( "github.com/mitchellh/mapstructure" "github.com/open-policy-agent/opa/ast" "github.com/open-policy-agent/opa/rego" - "github.com/open-policy-agent/opa/util" ) type StaticMetadata struct { @@ -38,6 +35,128 @@ type StaticMetadata struct { Terraform *scan.EngineMetadata } +func NewStaticMetadata(pkgPath string, inputOpt InputOptions) *StaticMetadata { + return &StaticMetadata{ + ID: "N/A", + Title: "N/A", + Severity: "UNKNOWN", + Description: fmt.Sprintf("Rego module: %s", pkgPath), + Package: pkgPath, + InputOptions: inputOpt, + Frameworks: make(map[framework.Framework][]string), + } +} + +func (sm *StaticMetadata) Update(meta map[string]any) error { + + upd := func(field *string, key string) { + if raw, ok := meta[key]; ok { + *field = fmt.Sprintf("%s", raw) + } + } + + upd(&sm.ID, "id") + upd(&sm.AVDID, "avd_id") + upd(&sm.Title, "title") + upd(&sm.ShortCode, "short_code") + upd(&sm.Description, "description") + upd(&sm.Service, "service") + upd(&sm.Provider, "provider") + upd(&sm.RecommendedActions, "recommended_actions") + upd(&sm.RecommendedActions, "recommended_action") + + if raw, ok := meta["severity"]; ok { + sm.Severity = strings.ToUpper(fmt.Sprintf("%s", raw)) + } + + if raw, ok := meta["library"]; ok { + if lib, ok := raw.(bool); ok { + sm.Library = lib + } + } + + if raw, ok := meta["url"]; ok { + sm.References = append(sm.References, fmt.Sprintf("%s", raw)) + } + if raw, ok := meta["frameworks"]; ok { + frameworks, ok := raw.(map[string][]string) + if !ok { + return fmt.Errorf("failed to parse framework metadata: not an object") + } + for fw, sections := range frameworks { + sm.Frameworks[framework.Framework(fw)] = sections + } + } + if raw, ok := meta["related_resources"]; ok { + if relatedResources, ok := raw.([]map[string]any); ok { + for _, relatedResource := range relatedResources { + if raw, ok := relatedResource["ref"]; ok { + sm.References = append(sm.References, fmt.Sprintf("%s", raw)) + } + } + } else if relatedResources, ok := raw.([]string); ok { + sm.References = append(sm.References, relatedResources...) + } + } + + var err error + if sm.CloudFormation, err = NewEngineMetadata("cloud_formation", meta); err != nil { + return err + } + + if sm.Terraform, err = NewEngineMetadata("terraform", meta); err != nil { + return err + } + + return nil +} + +func (sm *StaticMetadata) FromAnnotations(annotations *ast.Annotations) error { + sm.Title = annotations.Title + sm.Description = annotations.Description + for _, resource := range annotations.RelatedResources { + if !resource.Ref.IsAbs() { + continue + } + sm.References = append(sm.References, resource.Ref.String()) + } + if custom := annotations.Custom; custom != nil { + if err := sm.Update(custom); err != nil { + return err + } + } + if len(annotations.RelatedResources) > 0 { + sm.PrimaryURL = annotations.RelatedResources[0].Ref.String() + } + return nil +} + +func NewEngineMetadata(schema string, meta map[string]interface{}) (*scan.EngineMetadata, error) { + var sMap map[string]interface{} + if raw, ok := meta[schema]; ok { + sMap, ok = raw.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("failed to parse %s metadata: not an object", schema) + } + } + + var em scan.EngineMetadata + if val, ok := sMap["good_examples"].(string); ok { + em.GoodExamples = []string{val} + } + if val, ok := sMap["bad_examples"].(string); ok { + em.BadExamples = []string{val} + } + if val, ok := sMap["links"].(string); ok { + em.Links = []string{val} + } + if val, ok := sMap["remediation_markdown"].(string); ok { + em.RemediationMarkdown = val + } + + return &em, nil +} + type InputOptions struct { Combined bool Selectors []Selector @@ -99,7 +218,7 @@ func NewMetadataRetriever(compiler *ast.Compiler) *MetadataRetriever { } } -func (m *MetadataRetriever) findPackageAnnotation(module *ast.Module) *ast.Annotations { +func (m *MetadataRetriever) findPackageAnnotations(module *ast.Module) *ast.Annotations { annotationSet := m.compiler.GetAnnotationSet() if annotationSet == nil { return nil @@ -113,24 +232,19 @@ func (m *MetadataRetriever) findPackageAnnotation(module *ast.Module) *ast.Annot return nil } -func (m *MetadataRetriever) RetrieveMetadata(ctx context.Context, module *ast.Module, inputs ...Input) (*StaticMetadata, error) { +func (m *MetadataRetriever) RetrieveMetadata(ctx context.Context, module *ast.Module, contents ...any) (*StaticMetadata, error) { - metadata := StaticMetadata{ - ID: "N/A", - Title: "N/A", - Severity: "UNKNOWN", - Description: fmt.Sprintf("Rego module: %s", module.Package.Path.String()), - Package: module.Package.Path.String(), - InputOptions: m.queryInputOptions(ctx, module), - Frameworks: make(map[framework.Framework][]string), - } + metadata := NewStaticMetadata( + module.Package.Path.String(), + m.queryInputOptions(ctx, module), + ) // read metadata from official rego annotations if possible - if annotation := m.findPackageAnnotation(module); annotation != nil { - if err := m.fromAnnotation(&metadata, annotation); err != nil { + if annotations := m.findPackageAnnotations(module); annotations != nil { + if err := metadata.FromAnnotations(annotations); err != nil { return nil, err } - return &metadata, nil + return metadata, nil } // otherwise, try to read metadata from the rego module itself - we used to do this before annotations were a thing @@ -143,8 +257,8 @@ func (m *MetadataRetriever) RetrieveMetadata(ctx context.Context, module *ast.Mo rego.Capabilities(nil), } // support dynamic metadata fields - for _, in := range inputs { - options = append(options, rego.Input(in.Contents)) + for _, in := range contents { + options = append(options, rego.Input(in)) } instance := rego.New(options...) @@ -155,7 +269,7 @@ func (m *MetadataRetriever) RetrieveMetadata(ctx context.Context, module *ast.Mo // no metadata supplied if set == nil { - return &metadata, nil + return metadata, nil } if len(set) != 1 { @@ -170,133 +284,11 @@ func (m *MetadataRetriever) RetrieveMetadata(ctx context.Context, module *ast.Mo return nil, fmt.Errorf("failed to parse metadata: not an object") } - err = m.updateMetadata(meta, &metadata) - if err != nil { + if err := metadata.Update(meta); err != nil { return nil, err } - return &metadata, nil -} - -// nolint -func (m *MetadataRetriever) updateMetadata(meta map[string]interface{}, metadata *StaticMetadata) error { - if raw, ok := meta["id"]; ok { - metadata.ID = fmt.Sprintf("%s", raw) - } - if raw, ok := meta["avd_id"]; ok { - metadata.AVDID = fmt.Sprintf("%s", raw) - } - if raw, ok := meta["title"]; ok { - metadata.Title = fmt.Sprintf("%s", raw) - } - if raw, ok := meta["short_code"]; ok { - metadata.ShortCode = fmt.Sprintf("%s", raw) - } - if raw, ok := meta["severity"]; ok { - metadata.Severity = strings.ToUpper(fmt.Sprintf("%s", raw)) - } - if raw, ok := meta["description"]; ok { - metadata.Description = fmt.Sprintf("%s", raw) - } - if raw, ok := meta["service"]; ok { - metadata.Service = fmt.Sprintf("%s", raw) - } - if raw, ok := meta["provider"]; ok { - metadata.Provider = fmt.Sprintf("%s", raw) - } - if raw, ok := meta["library"]; ok { - if lib, ok := raw.(bool); ok { - metadata.Library = lib - } - } - if raw, ok := meta["recommended_actions"]; ok { - metadata.RecommendedActions = fmt.Sprintf("%s", raw) - } - if raw, ok := meta["recommended_action"]; ok { - metadata.RecommendedActions = fmt.Sprintf("%s", raw) - } - if raw, ok := meta["url"]; ok { - metadata.References = append(metadata.References, fmt.Sprintf("%s", raw)) - } - if raw, ok := meta["frameworks"]; ok { - frameworks, ok := raw.(map[string][]string) - if !ok { - return fmt.Errorf("failed to parse framework metadata: not an object") - } - for fw, sections := range frameworks { - metadata.Frameworks[framework.Framework(fw)] = sections - } - } - if raw, ok := meta["related_resources"]; ok { - if relatedResources, ok := raw.([]interface{}); ok { - for _, relatedResource := range relatedResources { - if relatedResourceMap, ok := relatedResource.(map[string]interface{}); ok { - if raw, ok := relatedResourceMap["ref"]; ok { - metadata.References = append(metadata.References, fmt.Sprintf("%s", raw)) - } - } else if relatedResourceString, ok := relatedResource.(string); ok { - metadata.References = append(metadata.References, relatedResourceString) - } - } - } - } - - var err error - if metadata.CloudFormation, err = m.getEngineMetadata("cloud_formation", meta); err != nil { - return err - } - - if metadata.Terraform, err = m.getEngineMetadata("terraform", meta); err != nil { - return err - } - - return nil -} - -func (m *MetadataRetriever) getEngineMetadata(schema string, meta map[string]interface{}) (*scan.EngineMetadata, error) { - var sMap map[string]interface{} - if raw, ok := meta[schema]; ok { - sMap, ok = raw.(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("failed to parse %s metadata: not an object", schema) - } - } - - var em scan.EngineMetadata - if val, ok := sMap["good_examples"].(string); ok { - em.GoodExamples = []string{val} - } - if val, ok := sMap["bad_examples"].(string); ok { - em.BadExamples = []string{val} - } - if val, ok := sMap["links"].(string); ok { - em.Links = []string{val} - } - if val, ok := sMap["remediation_markdown"].(string); ok { - em.RemediationMarkdown = val - } - - return &em, nil -} - -func (m *MetadataRetriever) fromAnnotation(metadata *StaticMetadata, annotation *ast.Annotations) error { - metadata.Title = annotation.Title - metadata.Description = annotation.Description - for _, resource := range annotation.RelatedResources { - if !resource.Ref.IsAbs() { - continue - } - metadata.References = append(metadata.References, resource.Ref.String()) - } - if custom := annotation.Custom; custom != nil { - if err := m.updateMetadata(custom, metadata); err != nil { - return err - } - } - if len(annotation.RelatedResources) > 0 { - metadata.PrimaryURL = annotation.RelatedResources[0].Ref.String() - } - return nil + return metadata, nil } // nolint: cyclop @@ -310,7 +302,7 @@ func (m *MetadataRetriever) queryInputOptions(ctx context.Context, module *ast.M var metadata map[string]interface{} // read metadata from official rego annotations if possible - if annotation := m.findPackageAnnotation(module); annotation != nil && annotation.Custom != nil { + if annotation := m.findPackageAnnotations(module); annotation != nil && annotation.Custom != nil { if input, ok := annotation.Custom["input"]; ok { if mapped, ok := input.(map[string]interface{}); ok { metadata = mapped @@ -383,64 +375,6 @@ func (m *MetadataRetriever) queryInputOptions(ctx context.Context, module *ast.M } -func BuildSchemaSetFromPolicies(policies map[string]*ast.Module, paths []string, srcFS fs.FS) (*ast.SchemaSet, bool, error) { - schemaSet := ast.NewSchemaSet() - schemaSet.Put(ast.MustParseRef("schema.input"), map[string]interface{}{}) // for backwards compat only - var customFound bool - for _, policy := range policies { - for _, annotation := range policy.Annotations { - for _, ss := range annotation.Schemas { - schemaName, err := ss.Schema.Ptr() - if err != nil { - continue - } - if schemaName != "input" { - if schema, ok := SchemaMap[defsecTypes.Source(schemaName)]; ok { - customFound = true - schemaSet.Put(ast.MustParseRef(ss.Schema.String()), util.MustUnmarshalJSON([]byte(schema))) - } else { - b, err := findSchemaInFS(paths, srcFS, schemaName) - if err != nil { - return schemaSet, true, err - } - if b != nil { - customFound = true - schemaSet.Put(ast.MustParseRef(ss.Schema.String()), util.MustUnmarshalJSON(b)) - } - } - } - } - } - } - - return schemaSet, customFound, nil -} - -// findSchemaInFS tries to find the schema anywhere in the specified FS -func findSchemaInFS(paths []string, srcFS fs.FS, schemaName string) ([]byte, error) { - var schema []byte - for _, path := range paths { - if err := fs.WalkDir(srcFS, sanitisePath(path), func(path string, info fs.DirEntry, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - if !IsJSONFile(info.Name()) { - return nil - } - if info.Name() == schemaName+".json" { - schema, err = fs.ReadFile(srcFS, filepath.ToSlash(path)) - if err != nil { - return err - } - return nil - } - return nil - }); err != nil { - return nil, err - } - } - return schema, nil +func getModuleNamespace(module *ast.Module) string { + return strings.TrimPrefix(module.Package.Path.String(), "data.") } diff --git a/pkg/rego/metadata_test.go b/pkg/rego/metadata_test.go index 1f8b0e6d..935c027d 100644 --- a/pkg/rego/metadata_test.go +++ b/pkg/rego/metadata_test.go @@ -3,9 +3,119 @@ package rego import ( "testing" + "github.com/aquasecurity/defsec/pkg/framework" + "github.com/aquasecurity/defsec/pkg/scan" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func Test_UpdateStaticMetadata(t *testing.T) { + t.Run("happy", func(t *testing.T) { + sm := StaticMetadata{ + ID: "i", + AVDID: "a", + Title: "t", + ShortCode: "sc", + Description: "d", + Severity: "s", + RecommendedActions: "ra", + PrimaryURL: "pu", + References: []string{"r"}, + Package: "pkg", + Provider: "pr", + Service: "srvc", + Library: false, + Frameworks: map[framework.Framework][]string{ + framework.Default: {"dd"}, + }, + } + + require.NoError(t, sm.Update( + map[string]any{ + "id": "i_n", + "avd_id": "a_n", + "title": "t_n", + "short_code": "sc_n", + "description": "d_n", + "service": "srvc_n", + "provider": "pr_n", + "recommended_actions": "ra_n", + "severity": "s_n", + "library": true, + "url": "r_n", + "frameworks": map[string][]string{ + "all": {"aa"}, + }, + }, + )) + + expected := StaticMetadata{ + ID: "i_n", + AVDID: "a_n", + Title: "t_n", + ShortCode: "sc_n", + Description: "d_n", + Severity: "S_N", + RecommendedActions: "ra_n", + PrimaryURL: "pu", + References: []string{"r", "r_n"}, + Package: "pkg", + Provider: "pr_n", + Service: "srvc_n", + Library: true, + Frameworks: map[framework.Framework][]string{ + framework.Default: {"dd"}, + framework.ALL: {"aa"}, + }, + CloudFormation: &scan.EngineMetadata{}, + Terraform: &scan.EngineMetadata{}, + } + + assert.Equal(t, expected, sm) + }) + + t.Run("related resources are a map", func(t *testing.T) { + sm := StaticMetadata{ + References: []string{"r"}, + } + require.NoError(t, sm.Update(map[string]any{ + "related_resources": []map[string]any{ + { + "ref": "r1_n", + }, + { + "ref": "r2_n", + }, + }, + })) + + expected := StaticMetadata{ + References: []string{"r", "r1_n", "r2_n"}, + CloudFormation: &scan.EngineMetadata{}, + Terraform: &scan.EngineMetadata{}, + } + + assert.Equal(t, expected, sm) + }) + + t.Run("related resources are a string", func(t *testing.T) { + sm := StaticMetadata{ + References: []string{"r"}, + } + require.NoError(t, sm.Update(map[string]any{ + "related_resources": []string{"r1_n", "r2_n"}, + })) + + expected := StaticMetadata{ + References: []string{"r", "r1_n", "r2_n"}, + CloudFormation: &scan.EngineMetadata{}, + Terraform: &scan.EngineMetadata{}, + } + + assert.Equal(t, expected, sm) + }) +} + func Test_getEngineMetadata(t *testing.T) { inputSchema := map[string]interface{}{ "terraform": map[string]interface{}{ @@ -70,8 +180,7 @@ Resources: for _, tc := range testCases { t.Run(tc.schema, func(t *testing.T) { - var m MetadataRetriever - em, err := m.getEngineMetadata(tc.schema, inputSchema) + em, err := NewEngineMetadata(tc.schema, inputSchema) assert.NoError(t, err) assert.Equal(t, tc.want, em.GoodExamples[0]) }) diff --git a/pkg/rego/result.go b/pkg/rego/result.go deleted file mode 100644 index a0c1ef70..00000000 --- a/pkg/rego/result.go +++ /dev/null @@ -1,168 +0,0 @@ -package rego - -import ( - "fmt" - "io/fs" - "strconv" - - defsecTypes "github.com/aquasecurity/defsec/pkg/types" - - "github.com/aquasecurity/defsec/pkg/scan" - - "github.com/open-policy-agent/opa/rego" -) - -type regoResult struct { - Filepath string - Resource string - StartLine int - EndLine int - SourcePrefix string - Message string - Explicit bool - Managed bool - FSKey string - FS fs.FS - Parent *regoResult -} - -func (r regoResult) GetMetadata() defsecTypes.Metadata { - var m defsecTypes.Metadata - if !r.Managed { - m = defsecTypes.NewUnmanagedMetadata() - } else { - rng := defsecTypes.NewRangeWithFSKey(r.Filepath, r.StartLine, r.EndLine, r.SourcePrefix, r.FSKey, r.FS) - if r.Explicit { - m = defsecTypes.NewExplicitMetadata(rng, r.Resource) - } else { - m = defsecTypes.NewMetadata(rng, r.Resource) - } - } - if r.Parent != nil { - return m.WithParent(r.Parent.GetMetadata()) - } - return m -} - -func (r regoResult) GetRawValue() interface{} { - return nil -} - -func parseResult(raw interface{}) *regoResult { - var result regoResult - result.Managed = true - switch val := raw.(type) { - case []interface{}: - var msg string - for _, item := range val { - switch raw := item.(type) { - case map[string]interface{}: - result = parseCause(raw) - case string: - msg = raw - } - } - result.Message = msg - case string: - result.Message = val - case map[string]interface{}: - result = parseCause(val) - default: - result.Message = "Rego policy resulted in DENY" - } - return &result -} - -func parseCause(cause map[string]interface{}) regoResult { - var result regoResult - result.Managed = true - if msg, ok := cause["msg"]; ok { - result.Message = fmt.Sprintf("%s", msg) - } - if filepath, ok := cause["filepath"]; ok { - result.Filepath = fmt.Sprintf("%s", filepath) - } - if msg, ok := cause["fskey"]; ok { - result.FSKey = fmt.Sprintf("%s", msg) - } - if msg, ok := cause["resource"]; ok { - result.Resource = fmt.Sprintf("%s", msg) - } - if start, ok := cause["startline"]; ok { - result.StartLine = parseLineNumber(start) - } - if end, ok := cause["endline"]; ok { - result.EndLine = parseLineNumber(end) - } - if prefix, ok := cause["sourceprefix"]; ok { - result.SourcePrefix = fmt.Sprintf("%s", prefix) - } - if explicit, ok := cause["explicit"]; ok { - if set, ok := explicit.(bool); ok { - result.Explicit = set - } - } - if managed, ok := cause["managed"]; ok { - if set, ok := managed.(bool); ok { - result.Managed = set - } - } - if parent, ok := cause["parent"]; ok { - if m, ok := parent.(map[string]interface{}); ok { - parentResult := parseCause(m) - result.Parent = &parentResult - } - } - return result -} - -func parseLineNumber(raw interface{}) int { - str := fmt.Sprintf("%s", raw) - n, _ := strconv.Atoi(str) - return n -} - -func (s *Scanner) convertResults(set rego.ResultSet, input Input, namespace string, rule string, traces []string) scan.Results { - var results scan.Results - - offset := 0 - if input.Contents != nil { - if xx, ok := input.Contents.(map[string]interface{}); ok { - if md, ok := xx["__defsec_metadata"]; ok { - if md2, ok := md.(map[string]interface{}); ok { - if sl, ok := md2["offset"]; ok { - offset, _ = sl.(int) - } - } - } - } - } - for _, result := range set { - for _, expression := range result.Expressions { - values, ok := expression.Value.([]interface{}) - if !ok { - values = []interface{}{expression.Value} - } - - for _, value := range values { - regoResult := parseResult(value) - regoResult.FS = input.FS - if regoResult.Filepath == "" && input.Path != "" { - regoResult.Filepath = input.Path - } - if regoResult.Message == "" { - regoResult.Message = fmt.Sprintf("Rego policy rule: %s.%s", namespace, rule) - } - regoResult.StartLine += offset - regoResult.EndLine += offset - results.AddRego(regoResult.Message, namespace, rule, traces, regoResult) - } - } - } - return results -} - -func (s *Scanner) embellishResultsWithRuleMetadata(results scan.Results, metadata StaticMetadata) scan.Results { - results.SetRule(metadata.ToRule()) - return results -} diff --git a/pkg/rego/result_test.go b/pkg/rego/result_test.go deleted file mode 100644 index d958f796..00000000 --- a/pkg/rego/result_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package rego - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_parseResult(t *testing.T) { - var testCases = []struct { - name string - input interface{} - want regoResult - }{ - { - name: "unknown", - input: nil, - want: regoResult{ - Managed: true, - Message: "Rego policy resulted in DENY", - }, - }, - { - name: "string", - input: "message", - want: regoResult{ - Managed: true, - Message: "message", - }, - }, - { - name: "strings", - input: []interface{}{"message"}, - want: regoResult{ - Managed: true, - Message: "message", - }, - }, - { - name: "maps", - input: []interface{}{ - "message", - map[string]interface{}{ - "filepath": "a.out", - }, - }, - want: regoResult{ - Managed: true, - Message: "message", - Filepath: "a.out", - }, - }, - { - name: "map", - input: map[string]interface{}{ - "msg": "message", - "filepath": "a.out", - "fskey": "abcd", - "resource": "resource", - "startline": "123", - "endline": "456", - "sourceprefix": "git", - "explicit": true, - "managed": true, - }, - want: regoResult{ - Message: "message", - Filepath: "a.out", - Resource: "resource", - StartLine: 123, - EndLine: 456, - SourcePrefix: "git", - FSKey: "abcd", - Explicit: true, - Managed: true, - }, - }, - { - name: "parent", - input: map[string]interface{}{ - "msg": "child", - "parent": map[string]interface{}{ - "msg": "parent", - }, - }, - want: regoResult{ - Message: "child", - Managed: true, - Parent: ®oResult{ - Message: "parent", - Managed: true, - }, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - have := parseResult(tc.input) - assert.NotNil(t, have) - assert.Equal(t, tc.want, *have) - }) - } -} diff --git a/pkg/rego/runtime.go b/pkg/rego/runtime.go deleted file mode 100644 index 6e28268d..00000000 --- a/pkg/rego/runtime.go +++ /dev/null @@ -1,28 +0,0 @@ -package rego - -import ( - "os" - "strings" - - "github.com/open-policy-agent/opa/ast" - "github.com/open-policy-agent/opa/version" -) - -func addRuntimeValues() *ast.Term { - env := ast.NewObject() - for _, pair := range os.Environ() { - parts := strings.SplitN(pair, "=", 2) - if len(parts) == 1 { - env.Insert(ast.StringTerm(parts[0]), ast.NullTerm()) - } else if len(parts) > 1 { - env.Insert(ast.StringTerm(parts[0]), ast.StringTerm(parts[1])) - } - } - - obj := ast.NewObject() - obj.Insert(ast.StringTerm("env"), ast.NewTerm(env)) - obj.Insert(ast.StringTerm("version"), ast.StringTerm(version.Version)) - obj.Insert(ast.StringTerm("commit"), ast.StringTerm(version.Vcs)) - - return ast.NewTerm(obj) -} diff --git a/pkg/rego/scanner.go b/pkg/rego/scanner.go deleted file mode 100644 index cb94a276..00000000 --- a/pkg/rego/scanner.go +++ /dev/null @@ -1,420 +0,0 @@ -package rego - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "io/fs" - "strings" - - "github.com/aquasecurity/defsec/pkg/debug" - "github.com/aquasecurity/defsec/pkg/framework" - "github.com/aquasecurity/defsec/pkg/scan" - "github.com/aquasecurity/defsec/pkg/scanners/options" - "github.com/aquasecurity/defsec/pkg/types" - "github.com/aquasecurity/trivy-policies/pkg/rego/schemas" - "github.com/open-policy-agent/opa/ast" - "github.com/open-policy-agent/opa/rego" - "github.com/open-policy-agent/opa/storage" -) - -var _ options.ConfigurableScanner = (*Scanner)(nil) - -type Scanner struct { - ruleNamespaces map[string]struct{} - policies map[string]*ast.Module - store storage.Store - dataDirs []string - runtimeValues *ast.Term - compiler *ast.Compiler - regoErrorLimit int - debug debug.Logger - traceWriter io.Writer - tracePerResult bool - retriever *MetadataRetriever - policyFS fs.FS - dataFS fs.FS - frameworks []framework.Framework - spec string - inputSchema interface{} // unmarshalled into this from a json schema document - sourceType types.Source -} - -func (s *Scanner) SetUseEmbeddedLibraries(b bool) { - // handled externally -} - -func (s *Scanner) SetSpec(spec string) { - s.spec = spec -} - -func (s *Scanner) SetRegoOnly(bool) {} - -func (s *Scanner) SetFrameworks(frameworks []framework.Framework) { - s.frameworks = frameworks -} - -func (s *Scanner) SetUseEmbeddedPolicies(b bool) { - // handled externally -} - -func (s *Scanner) trace(heading string, input interface{}) { - if s.traceWriter == nil { - return - } - data, err := json.MarshalIndent(input, "", " ") - if err != nil { - return - } - _, _ = fmt.Fprintf(s.traceWriter, "REGO %[1]s:\n%s\nEND REGO %[1]s\n\n", heading, string(data)) -} - -func (s *Scanner) SetPolicyFilesystem(fs fs.FS) { - s.policyFS = fs -} - -func (s *Scanner) SetDataFilesystem(fs fs.FS) { - s.dataFS = fs -} - -func (s *Scanner) SetPolicyReaders(_ []io.Reader) { - // NOTE: Policy readers option not applicable for rego, policies are loaded on-demand by other scanners. -} - -func (s *Scanner) SetDebugWriter(writer io.Writer) { - s.debug = debug.New(writer, "rego", "scanner") -} - -func (s *Scanner) SetTraceWriter(writer io.Writer) { - s.traceWriter = writer -} - -func (s *Scanner) SetPerResultTracingEnabled(b bool) { - s.tracePerResult = b -} - -func (s *Scanner) SetPolicyDirs(_ ...string) { - // NOTE: Policy dirs option not applicable for rego, policies are loaded on-demand by other scanners. -} - -func (s *Scanner) SetDataDirs(dirs ...string) { - s.dataDirs = dirs -} - -func (s *Scanner) SetPolicyNamespaces(namespaces ...string) { - for _, namespace := range namespaces { - s.ruleNamespaces[namespace] = struct{}{} - } -} - -func (s *Scanner) SetSkipRequiredCheck(_ bool) { - // NOTE: Skip required option not applicable for rego. -} - -func (s *Scanner) SetRegoErrorLimit(limit int) { - s.regoErrorLimit = limit -} - -type DynamicMetadata struct { - Warning bool - Filepath string - Message string - StartLine int - EndLine int -} - -var SchemaMap = map[types.Source]schemas.Schema{ - types.SourceDefsec: schemas.Cloud, - types.SourceCloud: schemas.Cloud, - types.SourceKubernetes: schemas.Kubernetes, - types.SourceRbac: schemas.Kubernetes, - types.SourceDockerfile: schemas.Dockerfile, - types.SourceTOML: schemas.Anything, - types.SourceYAML: schemas.Anything, - types.SourceJSON: schemas.Anything, -} - -func NewScanner(source types.Source, options ...options.ScannerOption) *Scanner { - - schema, ok := SchemaMap[source] - if !ok { - schema = schemas.Anything - } - - s := &Scanner{ - regoErrorLimit: ast.CompileErrorLimitDefault, - sourceType: source, - ruleNamespaces: map[string]struct{}{ - "builtin": {}, - "appshield": {}, - "defsec": {}, - }, - runtimeValues: addRuntimeValues(), - } - for _, opt := range options { - opt(s) - } - if schema != schemas.None { - err := json.Unmarshal([]byte(schema), &s.inputSchema) - if err != nil { - panic(err) - } - } - return s -} - -func (s *Scanner) SetParentDebugLogger(l debug.Logger) { - s.debug = l.Extend("rego") -} - -func getModuleNamespace(module *ast.Module) string { - return strings.TrimPrefix(module.Package.Path.String(), "data.") -} - -func (s *Scanner) runQuery(ctx context.Context, query string, input interface{}, disableTracing bool) (rego.ResultSet, []string, error) { - - trace := (s.traceWriter != nil || s.tracePerResult) && !disableTracing - - regoOptions := []func(*rego.Rego){ - rego.Query(query), - rego.Compiler(s.compiler), - rego.Store(s.store), - rego.Runtime(s.runtimeValues), - rego.Trace(trace), - } - - if s.inputSchema != nil { - schemaSet := ast.NewSchemaSet() - schemaSet.Put(ast.MustParseRef("schema.input"), s.inputSchema) - regoOptions = append(regoOptions, rego.Schemas(schemaSet)) - } - - if input != nil { - regoOptions = append(regoOptions, rego.Input(input)) - } - - instance := rego.New(regoOptions...) - set, err := instance.Eval(ctx) - if err != nil { - return nil, nil, err - } - - // we also build a slice of trace lines for per-result tracing - primarily for fanal/trivy - var traces []string - - if trace { - if s.traceWriter != nil { - rego.PrintTrace(s.traceWriter, instance) - } - if s.tracePerResult { - traceBuffer := bytes.NewBuffer([]byte{}) - rego.PrintTrace(traceBuffer, instance) - traces = strings.Split(traceBuffer.String(), "\n") - } - } - return set, traces, nil -} - -type Input struct { - Path string `json:"path"` - FS fs.FS `json:"-"` - Contents interface{} `json:"contents"` -} - -func (s *Scanner) ScanInput(ctx context.Context, inputs ...Input) (scan.Results, error) { - - s.debug.Log("Scanning %d inputs...", len(inputs)) - - var results scan.Results - - for _, module := range s.policies { - - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - namespace := getModuleNamespace(module) - topLevel := strings.Split(namespace, ".")[0] - if _, ok := s.ruleNamespaces[topLevel]; !ok { - continue - } - - staticMeta, err := s.retriever.RetrieveMetadata(ctx, module, inputs...) - if err != nil { - return nil, err - } - - if isPolicyWithSubtype(s.sourceType) { - // skip if policy isn't relevant to what is being scanned - if !isPolicyApplicable(staticMeta, inputs...) { - continue - } - } - - if len(inputs) == 0 { - continue - } - - usedRules := make(map[string]struct{}) - - // all rules - for _, rule := range module.Rules { - ruleName := rule.Head.Name.String() - if _, ok := usedRules[ruleName]; ok { - continue - } - usedRules[ruleName] = struct{}{} - if isEnforcedRule(ruleName) { - ruleResults, err := s.applyRule(ctx, namespace, ruleName, inputs, staticMeta.InputOptions.Combined) - if err != nil { - return nil, err - } - results = append(results, s.embellishResultsWithRuleMetadata(ruleResults, *staticMeta)...) - } - } - - } - - return results, nil -} - -func isPolicyWithSubtype(sourceType types.Source) bool { - for _, s := range []types.Source{types.SourceCloud, types.SourceDefsec} { // TODO(simar): Add types.Kubernetes once all k8s policy have subtype - if sourceType == s { - return true - } - } - return false -} - -func checkSubtype(ii map[string]interface{}, provider string, subTypes []SubType) bool { - if len(subTypes) == 0 { - return true - } - - for _, st := range subTypes { - switch services := ii[provider].(type) { - case map[string]interface{}: // cloud - for service := range services { - if (service == st.Service) && (st.Provider == provider) { - return true - } - } - case string: // k8s - // TODO(simar): This logic probably needs to be revisited - if services == st.Group || - services == st.Version || - services == st.Kind { - return true - } - } - } - return false -} - -func isPolicyApplicable(staticMetadata *StaticMetadata, inputs ...Input) bool { - for _, input := range inputs { - if ii, ok := input.Contents.(map[string]interface{}); ok { - for provider := range ii { - // TODO(simar): Add other providers - if !strings.Contains(strings.Join([]string{"kind", "aws", "azure"}, ","), provider) { - continue - } - - if len(staticMetadata.InputOptions.Selectors) == 0 { // policy always applies if no selectors - return true - } - - // check metadata for subtype - for _, s := range staticMetadata.InputOptions.Selectors { - if checkSubtype(ii, provider, s.Subtypes) { - return true - } - } - } - } - } - return false -} - -func (s *Scanner) applyRule(ctx context.Context, namespace string, rule string, inputs []Input, combined bool) (scan.Results, error) { - - // handle combined evaluations if possible - if combined { - s.trace("INPUT", inputs) - return s.applyRuleCombined(ctx, namespace, rule, inputs) - } - - var results scan.Results - qualified := fmt.Sprintf("data.%s.%s", namespace, rule) - for _, input := range inputs { - s.trace("INPUT", input) - if ignored, err := s.isIgnored(ctx, namespace, rule, input.Contents); err != nil { - return nil, err - } else if ignored { - var result regoResult - result.FS = input.FS - result.Filepath = input.Path - result.Managed = true - results.AddIgnored(result) - continue - } - set, traces, err := s.runQuery(ctx, qualified, input.Contents, false) - if err != nil { - return nil, err - } - s.trace("RESULTSET", set) - ruleResults := s.convertResults(set, input, namespace, rule, traces) - if len(ruleResults) == 0 { // It passed because we didn't find anything wrong (NOT because it didn't exist) - var result regoResult - result.FS = input.FS - result.Filepath = input.Path - result.Managed = true - results.AddPassedRego(namespace, rule, traces, result) - continue - } - results = append(results, ruleResults...) - } - - return results, nil -} - -func (s *Scanner) applyRuleCombined(ctx context.Context, namespace string, rule string, inputs []Input) (scan.Results, error) { - if len(inputs) == 0 { - return nil, nil - } - var results scan.Results - qualified := fmt.Sprintf("data.%s.%s", namespace, rule) - if ignored, err := s.isIgnored(ctx, namespace, rule, inputs); err != nil { - return nil, err - } else if ignored { - for _, input := range inputs { - var result regoResult - result.FS = input.FS - result.Filepath = input.Path - result.Managed = true - results.AddIgnored(result) - } - return results, nil - } - set, traces, err := s.runQuery(ctx, qualified, inputs, false) - if err != nil { - return nil, err - } - return s.convertResults(set, inputs[0], namespace, rule, traces), nil -} - -// severity is now set with metadata, so deny/warn/violation now behave the same way -func isEnforcedRule(name string) bool { - switch { - case name == "deny", strings.HasPrefix(name, "deny_"), - name == "warn", strings.HasPrefix(name, "warn_"), - name == "violation", strings.HasPrefix(name, "violation_"): - return true - } - return false -} diff --git a/pkg/rego/scanner_test.go b/pkg/rego/scanner_test.go deleted file mode 100644 index 245f4088..00000000 --- a/pkg/rego/scanner_test.go +++ /dev/null @@ -1,977 +0,0 @@ -package rego - -import ( - "bytes" - "context" - "github.com/liamg/memoryfs" - "io/fs" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/aquasecurity/defsec/pkg/scanners/options" - "github.com/aquasecurity/defsec/pkg/severity" - "github.com/aquasecurity/defsec/pkg/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func CreateFS(t *testing.T, files map[string]string) fs.FS { - memfs := memoryfs.New() - for name, content := range files { - name := strings.TrimPrefix(name, "/") - err := memfs.MkdirAll(filepath.Dir(name), 0o700) - require.NoError(t, err) - err = memfs.WriteFile(name, []byte(content), 0o644) - require.NoError(t, err) - } - return memfs -} - -func Test_RegoScanning_Deny(t *testing.T) { - - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -deny { - input.evil -} -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - FS: srcFS, - }) - require.NoError(t, err) - - require.Equal(t, 1, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) - - assert.Equal(t, "/evil.lol", results.GetFailed()[0].Metadata().Range().GetFilename()) - assert.False(t, results.GetFailed()[0].IsWarning()) -} - -func Test_RegoScanning_AbsolutePolicyPath_Deny(t *testing.T) { - - tmp := t.TempDir() - require.NoError(t, os.Mkdir(filepath.Join(tmp, "policies"), 0755)) - require.NoError(t, os.WriteFile(filepath.Join(tmp, "policies", "test.rego"), []byte(`package defsec.test - -deny { - input.evil -}`), 0600)) - - srcFS := os.DirFS(tmp) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"/policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - FS: srcFS, - }) - require.NoError(t, err) - - require.Equal(t, 1, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) - - assert.Equal(t, "/evil.lol", results.GetFailed()[0].Metadata().Range().GetFilename()) - assert.False(t, results.GetFailed()[0].IsWarning()) -} - -func Test_RegoScanning_Warn(t *testing.T) { - - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -warn { - input.evil -} -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - }) - require.NoError(t, err) - - require.Equal(t, 1, len(results.GetFailed())) - require.Equal(t, 0, len(results.GetPassed())) - require.Equal(t, 0, len(results.GetIgnored())) - - assert.True(t, results.GetFailed()[0].IsWarning()) -} - -func Test_RegoScanning_Allow(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -deny { - input.evil -} -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": false, - }, - }) - require.NoError(t, err) - - assert.Equal(t, 0, len(results.GetFailed())) - require.Equal(t, 1, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) - - assert.Equal(t, "/evil.lol", results.GetPassed()[0].Metadata().Range().GetFilename()) -} - -func Test_RegoScanning_Namespace_Exception(t *testing.T) { - - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -deny { - input.evil -} -`, - "policies/exceptions.rego": ` -package namespace.exceptions - -import data.namespaces - -exception[ns] { - ns := data.namespaces[_] - startswith(ns, "defsec") -} -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - }) - require.NoError(t, err) - - assert.Equal(t, 0, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 1, len(results.GetIgnored())) - -} - -func Test_RegoScanning_Namespace_Exception_WithoutMatch(t *testing.T) { - - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -deny { - input.evil -} -`, "policies/something.rego": ` -package builtin.test - -deny_something { - input.something -} -`, - "policies/exceptions.rego": ` -package namespace.exceptions - -import data.namespaces - -exception[ns] { - ns := data.namespaces[_] - startswith(ns, "builtin") -} -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - }) - require.NoError(t, err) - - assert.Equal(t, 1, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 1, len(results.GetIgnored())) - -} - -func Test_RegoScanning_Rule_Exception(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test -deny_evil { - input.evil -} -`, - "policies/exceptions.rego": ` -package defsec.test - -exception[rules] { - rules := ["evil"] -} -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - }) - require.NoError(t, err) - - assert.Equal(t, 0, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 1, len(results.GetIgnored())) -} - -func Test_RegoScanning_Rule_Exception_WithoutMatch(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test -deny_evil { - input.evil -} -`, - "policies/exceptions.rego": ` -package defsec.test - -exception[rules] { - rules := ["good"] -} -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - }) - require.NoError(t, err) - - assert.Equal(t, 1, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) -} - -func Test_RegoScanning_WithRuntimeValues(t *testing.T) { - - _ = os.Setenv("DEFSEC_RUNTIME_VAL", "AOK") - - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -deny_evil { - output := opa.runtime() - output.env.DEFSEC_RUNTIME_VAL == "AOK" -} -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - }) - require.NoError(t, err) - - assert.Equal(t, 1, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) -} - -func Test_RegoScanning_WithDenyMessage(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -deny[msg] { - input.evil - msg := "oh no" -} -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - }) - require.NoError(t, err) - - require.Equal(t, 1, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) - - assert.Equal(t, "oh no", results.GetFailed()[0].Description()) - assert.Equal(t, "/evil.lol", results.GetFailed()[0].Metadata().Range().GetFilename()) -} - -func Test_RegoScanning_WithDenyMetadata_ImpliedPath(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -deny[res] { - input.evil - res := { - "msg": "oh no", - "startline": 123, - "endline": 456, - } -} -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - }) - require.NoError(t, err) - - require.Equal(t, 1, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) - - assert.Equal(t, "oh no", results.GetFailed()[0].Description()) - assert.Equal(t, "/evil.lol", results.GetFailed()[0].Metadata().Range().GetFilename()) - assert.Equal(t, 123, results.GetFailed()[0].Metadata().Range().GetStartLine()) - assert.Equal(t, 456, results.GetFailed()[0].Metadata().Range().GetEndLine()) - -} - -func Test_RegoScanning_WithDenyMetadata_PersistedPath(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -deny[res] { - input.evil - res := { - "msg": "oh no", - "startline": 123, - "endline": 456, - "filepath": "/blah.txt", - } -} -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - }) - require.NoError(t, err) - - require.Equal(t, 1, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) - - assert.Equal(t, "oh no", results.GetFailed()[0].Description()) - assert.Equal(t, "/blah.txt", results.GetFailed()[0].Metadata().Range().GetFilename()) - assert.Equal(t, 123, results.GetFailed()[0].Metadata().Range().GetStartLine()) - assert.Equal(t, 456, results.GetFailed()[0].Metadata().Range().GetEndLine()) - -} - -func Test_RegoScanning_WithStaticMetadata(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -__rego_metadata__ := { - "id": "AA001", - "avd_id": "AVD-XX-9999", - "title": "This is a title", - "short_code": "short-code", - "severity": "LOW", - "type": "Dockerfile Security Check", - "description": "This is a description", - "recommended_actions": "This is a recommendation", - "url": "https://google.com", -} - -deny[res] { - input.evil - res := { - "msg": "oh no", - "startline": 123, - "endline": 456, - "filepath": "/blah.txt", - } -} -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - }) - require.NoError(t, err) - - require.Equal(t, 1, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) - - failure := results.GetFailed()[0] - - assert.Equal(t, "oh no", failure.Description()) - assert.Equal(t, "/blah.txt", failure.Metadata().Range().GetFilename()) - assert.Equal(t, 123, failure.Metadata().Range().GetStartLine()) - assert.Equal(t, 456, failure.Metadata().Range().GetEndLine()) - assert.Equal(t, "AVD-XX-9999", failure.Rule().AVDID) - assert.True(t, failure.Rule().HasID("AA001")) - assert.Equal(t, "This is a title", failure.Rule().Summary) - assert.Equal(t, severity.Low, failure.Rule().Severity) - assert.Equal(t, "This is a recommendation", failure.Rule().Resolution) - assert.Equal(t, "https://google.com", failure.Rule().Links[0]) - -} - -func Test_RegoScanning_WithMatchingInputSelector(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -__rego_input__ := { - "selector": [{"type": "json"}], -} - -deny { - input.evil -} - -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - }) - require.NoError(t, err) - - assert.Equal(t, 1, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) -} - -func Test_RegoScanning_WithNonMatchingInputSelector(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -__rego_input__ := { - "selector": [{"type": "testing"}], -} - -deny { - input.evil -} -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - }) - require.NoError(t, err) - - assert.Equal(t, 0, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) -} - -func Test_RegoScanning_NoTracingByDefault(t *testing.T) { - - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -deny { - input.evil -} -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - }) - require.NoError(t, err) - - assert.Equal(t, 1, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) - - assert.Len(t, results.GetFailed()[0].Traces(), 0) -} - -func Test_RegoScanning_GlobalTracingEnabled(t *testing.T) { - - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -deny { - input.evil -} -`, - }) - - traceBuffer := bytes.NewBuffer([]byte{}) - - scanner := NewScanner(types.SourceJSON, options.ScannerWithTrace(traceBuffer)) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - }) - require.NoError(t, err) - - assert.Equal(t, 1, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) - - assert.Len(t, results.GetFailed()[0].Traces(), 0) - assert.Greater(t, len(traceBuffer.Bytes()), 0) -} - -func Test_RegoScanning_PerResultTracingEnabled(t *testing.T) { - - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -deny { - input.evil -} -`, - }) - - scanner := NewScanner(types.SourceJSON, options.ScannerWithPerResultTracing(true)) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "evil": true, - }, - }) - require.NoError(t, err) - - assert.Equal(t, 1, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) - - assert.Greater(t, len(results.GetFailed()[0].Traces()), 0) -} - -func Test_dynamicMetadata(t *testing.T) { - - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -__rego_metadata__ := { - "title" : sprintf("i am %s",[input.text]) -} - -deny { - input.text -} - -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "text": "dynamic", - }, - }) - require.NoError(t, err) - assert.Equal(t, results[0].Rule().Summary, "i am dynamic") -} - -func Test_staticMetadata(t *testing.T) { - - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test - -__rego_metadata__ := { - "title" : "i am static" -} - -deny { - input.text -} - -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "text": "test", - }, - }) - require.NoError(t, err) - assert.Equal(t, results[0].Rule().Summary, "i am static") -} - -func Test_annotationMetadata(t *testing.T) { - - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": `# METADATA -# title: i am a title -# description: i am a description -# related_resources: -# - https://google.com -# custom: -# id: EG123 -# avd_id: AVD-EG-0123 -# severity: LOW -# recommended_action: have a cup of tea -package defsec.test - -deny { - input.text -} - -`, - "policies/test2.rego": `# METADATA -# title: i am another title -package defsec.test2 - -deny { - input.blah -} - -`, - }) - - scanner := NewScanner(types.SourceJSON) - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{ - Path: "/evil.lol", - Contents: map[string]interface{}{ - "text": "test", - }, - }) - require.NoError(t, err) - require.Len(t, results.GetFailed(), 1) - failure := results.GetFailed()[0].Rule() - assert.Equal(t, "i am a title", failure.Summary) - assert.Equal(t, "i am a description", failure.Explanation) - require.Len(t, failure.Links, 1) - assert.Equal(t, "https://google.com", failure.Links[0]) - assert.Equal(t, "AVD-EG-0123", failure.AVDID) - assert.Equal(t, severity.Low, failure.Severity) - assert.Equal(t, "have a cup of tea", failure.Resolution) -} - -func Test_RegoScanning_WithInvalidInputSchema(t *testing.T) { - - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": `# METADATA -# schemas: -# - input: schema["input"] -package defsec.test - -deny { - input.evil == "lol" -} -`, - }) - - scanner := NewScanner(types.SourceDockerfile) - scanner.SetRegoErrorLimit(0) // override to not allow any errors - assert.ErrorContains( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - "undefined ref: input.evil", - ) -} - -func Test_RegoScanning_WithValidInputSchema(t *testing.T) { - - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": `# METADATA -# schemas: -# - input: schema["input"] -package defsec.test - -deny { - input.Stages[0].Commands[0].Cmd == "lol" -} -`, - }) - - scanner := NewScanner(types.SourceDockerfile) - assert.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) -} - -func Test_RegoScanning_WithFilepathToSchema(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": `# METADATA -# schemas: -# - input: schema["dockerfile"] -package defsec.test - -deny { - input.evil == "lol" -} -`, - }) - scanner := NewScanner(types.SourceJSON) - scanner.SetRegoErrorLimit(0) // override to not allow any errors - assert.ErrorContains( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - "undefined ref: input.evil", - ) -} - -func Test_RegoScanning_CustomData(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test -import data.settings.DS123.foo_bar_baz - -deny { - not foo_bar_baz -} -`, - }) - - dataFS := CreateFS(t, map[string]string{ - "data/data.json": `{ - "settings": { - "DS123":{ - "foo_bar_baz":false - } - } -}`, - "data/junk.txt": "this file should be ignored", - }) - - scanner := NewScanner(types.SourceJSON) - scanner.SetDataFilesystem(dataFS) - scanner.SetDataDirs(".") - - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{}) - require.NoError(t, err) - - assert.Equal(t, 1, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) -} - -func Test_RegoScanning_InvalidFS(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ - "policies/test.rego": ` -package defsec.test -import data.settings.DS123.foo_bar_baz - -deny { - not foo_bar_baz -} -`, - }) - - dataFS := CreateFS(t, map[string]string{ - "data/data.json": `{ - "settings": { - "DS123":{ - "foo_bar_baz":false - } - } -}`, - "data/junk.txt": "this file should be ignored", - }) - - scanner := NewScanner(types.SourceJSON) - scanner.SetDataFilesystem(dataFS) - scanner.SetDataDirs("X://") - - require.NoError( - t, - scanner.LoadPolicies(false, false, srcFS, []string{"policies"}, nil), - ) - - results, err := scanner.ScanInput(context.TODO(), Input{}) - require.NoError(t, err) - - assert.Equal(t, 1, len(results.GetFailed())) - assert.Equal(t, 0, len(results.GetPassed())) - assert.Equal(t, 0, len(results.GetIgnored())) -} diff --git a/pkg/rego/schemas/schemas.go b/pkg/rego/schemas/schemas.go new file mode 100644 index 00000000..1ece0358 --- /dev/null +++ b/pkg/rego/schemas/schemas.go @@ -0,0 +1,16 @@ +package schemas + +import ( + "github.com/aquasecurity/defsec/pkg/types" +) + +var SchemaMap = map[types.Source]Schema{ + types.SourceDefsec: Cloud, + types.SourceCloud: Cloud, + types.SourceKubernetes: Kubernetes, + types.SourceRbac: Kubernetes, + types.SourceDockerfile: Dockerfile, + types.SourceTOML: Anything, + types.SourceYAML: Anything, + types.SourceJSON: Anything, +} diff --git a/pkg/rego/store.go b/pkg/rego/store.go deleted file mode 100644 index 127b1d8d..00000000 --- a/pkg/rego/store.go +++ /dev/null @@ -1,48 +0,0 @@ -package rego - -import ( - "fmt" - "io/fs" - "os" - "path/filepath" - "strings" - - "github.com/open-policy-agent/opa/loader" - "github.com/open-policy-agent/opa/storage" -) - -// initialise a store populated with OPA data files found in dataPaths -func initStore(dataFS fs.FS, dataPaths, namespaces []string) (storage.Store, error) { - // FilteredPaths will recursively find all file paths that contain a valid document - // extension from the given list of data paths. - allDocumentPaths, _ := loader.FilteredPathsFS(dataFS, dataPaths, func(abspath string, info os.FileInfo, depth int) bool { - if info.IsDir() { - return false // filter in, include - } - ext := strings.ToLower(filepath.Ext(info.Name())) - for _, filter := range []string{ - ".yaml", - ".yml", - ".json", - } { - if filter == ext { - return false // filter in, include - } - } - return true // filter out, exclude - }) - - documents, err := loader.NewFileLoader().WithFS(dataFS).All(allDocumentPaths) - if err != nil { - return nil, fmt.Errorf("load documents: %w", err) - } - - // pass all namespaces so that rego rule can refer to namespaces as data.namespaces - documents.Documents["namespaces"] = namespaces - - store, err := documents.Store() - if err != nil { - return nil, fmt.Errorf("get documents store: %w", err) - } - return store, nil -} diff --git a/pkg/rego/testdata/policies/._sysfile.rego b/pkg/rego/testdata/policies/._sysfile.rego deleted file mode 100644 index e69de29b..00000000 diff --git a/pkg/rego/testdata/policies/invalid.rego b/pkg/rego/testdata/policies/invalid.rego deleted file mode 100644 index a2ef3607..00000000 --- a/pkg/rego/testdata/policies/invalid.rego +++ /dev/null @@ -1,8 +0,0 @@ -# METADATA -# schemas: -# - input: schema["input"] -package defsec.test_invalid - -deny { - input.Stages[0].Commands[0].FooBarNothingBurger == "lol" -} diff --git a/pkg/rego/testdata/policies/valid.rego b/pkg/rego/testdata/policies/valid.rego deleted file mode 100644 index 74a96afe..00000000 --- a/pkg/rego/testdata/policies/valid.rego +++ /dev/null @@ -1,8 +0,0 @@ -# METADATA -# schemas: -# - input: schema["input"] -package defsec.test_valid - -deny { - input.Stages[0].Commands[0].Cmd == "lol" -} diff --git a/test/rego_test.go b/test/rego_test.go index 8b0e01ce..3aa3fa1c 100644 --- a/test/rego_test.go +++ b/test/rego_test.go @@ -9,15 +9,14 @@ import ( "strings" "testing" - "github.com/aquasecurity/trivy-policies/pkg/rego/schemas" - - "github.com/stretchr/testify/assert" - - ir "github.com/aquasecurity/trivy-policies/internal/rego" - dr "github.com/aquasecurity/trivy-policies/pkg/rego" "github.com/open-policy-agent/opa/ast" "github.com/open-policy-agent/opa/rego" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + trivyRego "github.com/aquasecurity/trivy-policies/pkg/rego" + _ "github.com/aquasecurity/trivy-policies/pkg/rego/embed" + "github.com/aquasecurity/trivy-policies/pkg/rego/schemas" ) func Test_AllRegoCloudRulesMatchSchema(t *testing.T) { @@ -60,7 +59,10 @@ func Test_AllRegoCloudRulesMatchSchema(t *testing.T) { schemaSet := ast.NewSchemaSet() schemaSet.Put(ast.MustParseRef("schema.cloud"), schema) - compiler := ir.NewRegoCompiler(schemaSet) + compiler := ast.NewCompiler(). + WithUseTypeCheckAnnotations(true). + WithCapabilities(ast.CapabilitiesForThisVersion()). + WithSchemas(schemaSet) compiler.Compile(baseModules) assert.False(t, compiler.Failed(), "compilation failed: %s", compiler.Errors) @@ -109,14 +111,17 @@ func Test_AllRegoRules(t *testing.T) { schemaSet.Put(ast.MustParseRef("schema.cloud"), map[string]interface{}{}) schemaSet.Put(ast.MustParseRef("schema.kubernetes"), map[string]interface{}{}) - compiler := ir.NewRegoCompiler(schemaSet) + compiler := ast.NewCompiler(). + WithUseTypeCheckAnnotations(true). + WithCapabilities(ast.CapabilitiesForThisVersion()). + WithSchemas(schemaSet) compiler.Compile(baseModules) if compiler.Failed() { t.Fatal(compiler.Errors) } - retriever := dr.NewMetadataRetriever(compiler) + retriever := trivyRego.NewMetadataRetriever(compiler) ctx := context.Background()