diff --git a/cmd/osv-scanner/__snapshots__/main_test.snap b/cmd/osv-scanner/__snapshots__/main_test.snap index b8c76d60fd..bc8e5c9ab7 100755 --- a/cmd/osv-scanner/__snapshots__/main_test.snap +++ b/cmd/osv-scanner/__snapshots__/main_test.snap @@ -320,9 +320,9 @@ Scanning dir ./fixtures/maven-transitive Scanned /fixtures/maven-transitive/pom.xml file and found 3 packages Package npm/ansi-html/0.0.1 has been filtered out because: (no reason given) Package npm/balanced-match/1.0.2 has been filtered out because: (no reason given) +Package Maven/org.apache.logging.log4j:log4j-web/2.14.1 has been filtered out because: it makes the table output really really long Package Maven/org.apache.logging.log4j:log4j-api/2.14.1 has been filtered out because: it makes the table output really really long Package Maven/org.apache.logging.log4j:log4j-core/2.14.1 has been filtered out because: it makes the table output really really long -Package Maven/org.apache.logging.log4j:log4j-web/2.14.1 has been filtered out because: it makes the table output really really long Filtered 5 ignored package/s from the scan. overriding license for package Alpine/alpine-baselayout-data/3.4.0-r0 with MIT overriding license for package Alpine/alpine-baselayout/3.4.0-r0 with MIT @@ -2264,7 +2264,7 @@ No issues found --- [TestRun_LockfileWithExplicitParseAs/empty_works_as_an_escape_(no_fixture_because_it's_not_valid_on_Windows) - 2] -open /path/to/my:file: no such file or directory +stat /path/to/my:file: no such file or directory --- @@ -2273,7 +2273,7 @@ open /path/to/my:file: no such file or directory --- [TestRun_LockfileWithExplicitParseAs/empty_works_as_an_escape_(no_fixture_because_it's_not_valid_on_Windows)#01 - 2] -open /path/to/my:project/package-lock.json: no such file or directory +stat /path/to/my:project/package-lock.json: no such file or directory --- @@ -2340,7 +2340,7 @@ No issues found --- [TestRun_LockfileWithExplicitParseAs/parse-as_takes_priority,_even_if_it's_wrong - 2] -(extracting as package-lock.json) could not extract from /fixtures/locks-many/yarn.lock: invalid character '#' looking for beginning of value +(extracting as package-lock.json) could not extract from "/fixtures/locks-many/yarn.lock": invalid character '#' looking for beginning of value --- diff --git a/cmd/osv-scanner/fixtures/locks-requirements/my-requirements.txt b/cmd/osv-scanner/fixtures/locks-requirements/my-requirements.txt index 7e1060246f..0e463a4d02 100644 --- a/cmd/osv-scanner/fixtures/locks-requirements/my-requirements.txt +++ b/cmd/osv-scanner/fixtures/locks-requirements/my-requirements.txt @@ -1 +1 @@ -flask +flask==1.0.0 diff --git a/cmd/osv-scanner/fixtures/locks-requirements/requirements-dev.txt b/cmd/osv-scanner/fixtures/locks-requirements/requirements-dev.txt index 7e66a17d49..4fae28300e 100644 --- a/cmd/osv-scanner/fixtures/locks-requirements/requirements-dev.txt +++ b/cmd/osv-scanner/fixtures/locks-requirements/requirements-dev.txt @@ -1 +1 @@ -black +black==1.0.0 diff --git a/cmd/osv-scanner/fixtures/locks-requirements/requirements.txt b/cmd/osv-scanner/fixtures/locks-requirements/requirements.txt index d0dae5a60f..911f55bcf9 100644 --- a/cmd/osv-scanner/fixtures/locks-requirements/requirements.txt +++ b/cmd/osv-scanner/fixtures/locks-requirements/requirements.txt @@ -1,3 +1,3 @@ -flask -flask-cors +flask==1.0.0 +flask-cors==1.0.0 pandas==0.23.4 diff --git a/cmd/osv-scanner/fixtures/locks-requirements/the_requirements_for_test.txt b/cmd/osv-scanner/fixtures/locks-requirements/the_requirements_for_test.txt index e079f8a603..35663c020e 100644 --- a/cmd/osv-scanner/fixtures/locks-requirements/the_requirements_for_test.txt +++ b/cmd/osv-scanner/fixtures/locks-requirements/the_requirements_for_test.txt @@ -1 +1 @@ -pytest +pytest==1.0.0 diff --git a/cmd/osv-scanner/fixtures/sbom-insecure/osv-scanner.toml b/cmd/osv-scanner/fixtures/sbom-insecure/osv-scanner.toml index 80e5b8b2ca..4a3e9070b8 100644 --- a/cmd/osv-scanner/fixtures/sbom-insecure/osv-scanner.toml +++ b/cmd/osv-scanner/fixtures/sbom-insecure/osv-scanner.toml @@ -1,64 +1,3 @@ -[[IgnoredVulns]] -id = "GO-2022-0274" -# ignoreUntil = n/a -reason = "This is an intentionally vulnerable test sbom" - -[[IgnoredVulns]] -id = "GO-2022-0493" -# ignoreUntil = n/a -reason = "This is an intentionally vulnerable test sbom" - -[[IgnoredVulns]] -id = "GHSA-vpvm-3wq2-2wvm" -# ignoreUntil = n/a -reason = "This is an intentionally vulnerable test sbom" - -[[IgnoredVulns]] -id = "GHSA-m8cg-xc2p-r3fc" -# ignoreUntil = n/a -reason = "This is an intentionally vulnerable test sbom" - -[[IgnoredVulns]] -id = "GHSA-g2j6-57v7-gm8c" -# ignoreUntil = n/a -reason = "This is an intentionally vulnerable test sbom" - -[[IgnoredVulns]] -id = "GHSA-f3fp-gc8g-vw66" -# ignoreUntil = n/a -reason = "This is an intentionally vulnerable test sbom" - -[[IgnoredVulns]] -id = "DLA-3008-1" -# ignoreUntil = n/a -reason = "This is an intentionally vulnerable test sbom" - -[[IgnoredVulns]] -id = "DLA-3012-1" -# ignoreUntil = n/a -reason = "This is an intentionally vulnerable test sbom" - -[[IgnoredVulns]] -id = "DLA-3022-1" -# ignoreUntil = n/a -reason = "This is an intentionally vulnerable test sbom" - -[[IgnoredVulns]] -id = "DLA-3051-1" -# ignoreUntil = n/a -reason = "This is an intentionally vulnerable test sbom" - -[[IgnoredVulns]] -id = "CVE-2022-37434" -# ignoreUntil = n/a -reason = "This is an intentionally vulnerable test sbom" - -[[IgnoredVulns]] -id = "CVE-2018-25032" -# ignoreUntil = n/a -reason = "This is an intentionally vulnerable test sbom" - -[[IgnoredVulns]] -id = "GHSA-xr7r-f8xq-vfvv" -# ignoreUntil = n/a +[[PackageOverrides]] +ignore = true reason = "This is an intentionally vulnerable test sbom" diff --git a/go.mod b/go.mod index caf5f92b7b..e2b7240290 100644 --- a/go.mod +++ b/go.mod @@ -13,12 +13,12 @@ require ( github.com/charmbracelet/bubbletea v1.1.1 github.com/charmbracelet/glamour v0.8.0 github.com/charmbracelet/lipgloss v0.13.0 - github.com/dghubble/trie v0.1.0 github.com/gkampitakis/go-snaps v0.5.7 github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.12.0 github.com/google/go-cmp v0.6.0 github.com/google/go-containerregistry v0.20.2 + github.com/google/osv-scalibr v0.1.4-0.20241014113419-c36dd4d15223 github.com/ianlancetaylor/demangle v0.0.0-20240912202439-0a2b6291aafd github.com/jedib0t/go-pretty/v6 v6.6.0 github.com/muesli/reflow v0.3.0 @@ -31,7 +31,6 @@ require ( github.com/tidwall/sjson v1.2.5 github.com/urfave/cli/v2 v2.27.5 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c - golang.org/x/mod v0.21.0 golang.org/x/net v0.30.0 golang.org/x/sync v0.8.0 golang.org/x/term v0.25.0 @@ -44,7 +43,7 @@ require ( require ( dario.cat/mergo v1.0.0 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/anchore/go-struct-converter v0.0.0-20230627203149-c72ef8859ca9 // indirect @@ -58,8 +57,6 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/dlclark/regexp2 v1.11.0 // indirect - github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker-credential-helpers v0.8.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/gkampitakis/ciinfo v0.3.0 // indirect @@ -82,14 +79,13 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc3 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sahilm/fuzzy v0.1.1 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect github.com/spdx/gordf v0.0.0-20221230105357-b735bd5aac89 // indirect github.com/tidwall/match v1.1.1 // indirect @@ -100,6 +96,7 @@ require ( github.com/yuin/goldmark v1.7.4 // indirect github.com/yuin/goldmark-emoji v1.0.3 // indirect golang.org/x/crypto v0.28.0 // indirect + golang.org/x/mod v0.21.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.19.0 // indirect golang.org/x/tools v0.26.0 // indirect diff --git a/go.sum b/go.sum index 6c92190e56..c14589d9ad 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,8 @@ github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2 github.com/CycloneDX/cyclonedx-go v0.9.1 h1:yffaWOZsv77oTJa/SdVZYdgAgFioCeycBUKkqS2qzQM= github.com/CycloneDX/cyclonedx-go v0.9.1/go.mod h1:NE/EWvzELOFlG6+ljX/QeMlVt9VKcTwu8u0ccsACEsw= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/alecthomas/assert/v2 v2.7.0 h1:QtqSACNS3tF7oasA8CU6A6sXZSBDqnm7RfpLl9bZqbE= @@ -69,8 +69,6 @@ github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dghubble/trie v0.1.0 h1:kJnjBLFFElBwS60N4tkPvnLhnpcDxbBjIulgI8CpNGM= -github.com/dghubble/trie v0.1.0/go.mod h1:sOmnzfBNH7H92ow2292dDFWNsVQuh/izuD7otCYb1ak= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/cli v27.1.1+incompatible h1:goaZxOqs4QKxznZjjBWKONQci/MywhtRv2oNn0GkeZE= @@ -111,6 +109,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l/DSArMxlbwseo= github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8= +github.com/google/osv-scalibr v0.1.4-0.20241014113419-c36dd4d15223 h1:Yie21Xk5WBewZFHnt9AI27EZMbEjbwzvXwv5HM9AUDE= +github.com/google/osv-scalibr v0.1.4-0.20241014113419-c36dd4d15223/go.mod h1:MbEYB+PKqEGjwMdpcoO5DWpi0+57jYgYcw2jlRy8O9Q= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -161,8 +161,8 @@ github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= -github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= github.com/owenrumney/go-sarif/v2 v2.3.3 h1:ubWDJcF5i3L/EIOER+ZyQ03IfplbSU1BLOE26uKQIIU= github.com/owenrumney/go-sarif/v2 v2.3.3/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= diff --git a/internal/image/extractor.go b/internal/image/extractor.go index 6ddb7f9f16..d96c4eacf6 100644 --- a/internal/image/extractor.go +++ b/internal/image/extractor.go @@ -1,134 +1,96 @@ package image import ( + "context" "errors" "fmt" - "os" - "path" - "sort" - + "io/fs" + "path/filepath" + + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/packagejson" + "github.com/google/osv-scalibr/extractor/filesystem/os/apk" + "github.com/google/osv-scanner/internal/lockfilescalibr" "github.com/google/osv-scanner/pkg/lockfile" ) // artifactExtractors contains only extractors for artifacts that are important in // the final layer of a container image -var artifactExtractors map[string]lockfile.Extractor = map[string]lockfile.Extractor{ - "node_modules": lockfile.NodeModulesExtractor{}, - "apk-installed": lockfile.ApkInstalledExtractor{}, - "dpkg": lockfile.DpkgStatusExtractor{}, - "go-binary": lockfile.GoBinaryExtractor{}, -} - -type extractorPair struct { - extractor lockfile.Extractor - name string +var artifactExtractors map[string]filesystem.Extractor = map[string]filesystem.Extractor{ + "packagejson": packagejson.New(packagejson.DefaultConfig()), + "apk-installed": apk.New(apk.DefaultConfig()), + // "dpkg": lockfile.DpkgStatusExtractor{}, + // "go-binary": lockfile.GoBinaryExtractor{}, } -func findArtifactExtractor(path string) []extractorPair { +func findArtifactExtractor(path string, fileInfo fs.FileInfo) []filesystem.Extractor { // Use ShouldExtract to collect and return a slice of artifactExtractors - var extractors []extractorPair - for name, extractor := range artifactExtractors { - if extractor.ShouldExtract(path) { - extractors = append(extractors, extractorPair{extractor, name}) + var extractors []filesystem.Extractor + for _, extractor := range artifactExtractors { + if extractor.FileRequired(path, fileInfo) { + extractors = append(extractors, extractor) } } return extractors } -func extractArtifactDeps(path string, layer *Layer) (lockfile.Lockfile, error) { - foundExtractors := findArtifactExtractor(path) +// Note: Output is non deterministic +func extractArtifactDeps(path string, layer *Layer) ([]*extractor.Inventory, error) { + pathFileInfo, err := layer.Stat(path) + if err != nil { + return nil, fmt.Errorf("attempted to get FileInfo but failed: %w", err) + } + + scalibrPath, _ := filepath.Rel("/", path) + foundExtractors := findArtifactExtractor(scalibrPath, pathFileInfo) if len(foundExtractors) == 0 { - return lockfile.Lockfile{}, fmt.Errorf("%w for %s", lockfile.ErrExtractorNotFound, path) + return nil, fmt.Errorf("%w for %s", lockfilescalibr.ErrExtractorNotFound, path) } - packages := []lockfile.PackageDetails{} + inventories := []*extractor.Inventory{} var extractedAs string - for _, extPair := range foundExtractors { + for _, extractor := range foundExtractors { // File has to be reopened per extractor as each extractor moves the read cursor - f, err := OpenLayerFile(path, layer) + f, err := layer.Open(path) if err != nil { - return lockfile.Lockfile{}, fmt.Errorf("attempted to open file but failed: %w", err) + return nil, fmt.Errorf("attempted to open file but failed: %w", err) + } + + scanInput := &filesystem.ScanInput{ + FS: layer, + Path: scalibrPath, + Root: "/", + Reader: f, + Info: pathFileInfo, } - newPackages, err := extPair.extractor.Extract(f) + newPackages, err := extractor.Extract(context.Background(), scanInput) f.Close() if err != nil { - if errors.Is(err, lockfile.ErrIncompatibleFileFormat) { + if errors.Is(lockfile.ErrIncompatibleFileFormat, err) { continue } - return lockfile.Lockfile{}, fmt.Errorf("(extracting as %s) %w", extPair.name, err) + return nil, fmt.Errorf("(extracting as %s) %w", extractor.Name(), err) } - extractedAs = extPair.name - packages = newPackages - // TODO(rexpan): Determine if it's acceptable to have multiple extractors + for i := range newPackages { + newPackages[i].Extractor = extractor + } + + extractedAs = extractor.Name() + inventories = newPackages + // TODO(rexpan): Determine if this it's acceptable to have multiple extractors // extract from the same file successfully break } if extractedAs == "" { - return lockfile.Lockfile{}, fmt.Errorf("%w for %s", lockfile.ErrExtractorNotFound, path) - } - - // Sort to have deterministic output, and to match behavior of lockfile.extractDeps - sort.Slice(packages, func(i, j int) bool { - if packages[i].Name == packages[j].Name { - return packages[i].Version < packages[j].Version - } - - return packages[i].Name < packages[j].Name - }) - - return lockfile.Lockfile{ - FilePath: path, - ParsedAs: extractedAs, - Packages: packages, - }, nil -} - -// A File represents a file that exists in an image -type File struct { - *os.File - - layer *Layer - path string -} - -func (f File) Open(openPath string) (lockfile.NestedDepFile, error) { - // use path instead of filepath, because container is always in Unix paths (for now) - if path.IsAbs(openPath) { - return OpenLayerFile(openPath, f.layer) - } - - absPath := path.Join(f.path, openPath) - - return OpenLayerFile(absPath, f.layer) -} - -func (f File) Path() string { - return f.path -} - -func OpenLayerFile(path string, layer *Layer) (File, error) { - fileNode, err := layer.getFileNode(path) - if err != nil { - return File{}, err + return nil, fmt.Errorf("%w for %s", lockfilescalibr.ErrExtractorNotFound, path) } - file, err := fileNode.Open() - if err != nil { - return File{}, err - } - - return File{ - File: file, - path: path, - layer: layer, - }, nil + return inventories, nil } - -var _ lockfile.DepFile = File{} -var _ lockfile.NestedDepFile = File{} diff --git a/internal/image/fixtures/alpine-3.19-alpine-release b/internal/image/fixtures/alpine-3.18-alpine-release similarity index 100% rename from internal/image/fixtures/alpine-3.19-alpine-release rename to internal/image/fixtures/alpine-3.18-alpine-release diff --git a/internal/image/fixtures/alpine-3.18-os-release b/internal/image/fixtures/alpine-3.18-os-release new file mode 100644 index 0000000000..ffb92a8cd4 --- /dev/null +++ b/internal/image/fixtures/alpine-3.18-os-release @@ -0,0 +1,7 @@ +/ # cat /etc/os-release +NAME="Alpine Linux" +ID=alpine +VERSION_ID=3.18.1 +PRETTY_NAME="Alpine Linux v3.18" +HOME_URL="https://alpinelinux.org/" +BUG_REPORT_URL="https://gitlab.alpinelinux.org/alpine/aports/-/issues" diff --git a/internal/image/fixtures/test-alpine.Dockerfile b/internal/image/fixtures/test-alpine.Dockerfile index 5cf22e2812..d6aa79f1c8 100644 --- a/internal/image/fixtures/test-alpine.Dockerfile +++ b/internal/image/fixtures/test-alpine.Dockerfile @@ -1,4 +1,5 @@ FROM alpine:3.10@sha256:451eee8bedcb2f029756dc3e9d73bab0e7943c1ac55cff3a4861c52a0fdd3e98 -# Switch the version to 3.19 to show the advisories published for the latest alpine versions -COPY "alpine-3.19-alpine-release" "/etc/alpine-release" +# Switch the version to 3.18 to show the advisories published for the latest alpine versions +COPY "alpine-3.18-alpine-release" "/etc/alpine-release" +COPY "alpine-3.18-os-release" "/etc/os-release" diff --git a/internal/image/image.go b/internal/image/image.go index be3bd3171e..af1f7dcf01 100644 --- a/internal/image/image.go +++ b/internal/image/image.go @@ -11,9 +11,9 @@ import ( "path/filepath" "strings" - "github.com/dghubble/trie" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/osv-scanner/internal/image/thirdparty/trie" "github.com/google/osv-scanner/pkg/lockfile" ) diff --git a/internal/image/image_test.go b/internal/image/image_test.go index 90bd028524..be9901c756 100644 --- a/internal/image/image_test.go +++ b/internal/image/image_test.go @@ -3,7 +3,6 @@ package image_test import ( "errors" "os" - "sort" "testing" "github.com/google/osv-scanner/internal/image" @@ -66,14 +65,9 @@ func TestScanImage(t *testing.T) { want: testutility.NewSnapshot(), wantErr: false, }, - { - name: "scanning go binaries that's been overwritten for package tracing", - args: args{imagePath: "fixtures/test-package-tracing.tar"}, - want: testutility.NewSnapshot(), - wantErr: false, - }, } for _, tt := range tests { + tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() @@ -94,10 +88,6 @@ func TestScanImage(t *testing.T) { } } - sort.Slice(got.Lockfiles, func(i, j int) bool { - return got.Lockfiles[i].FilePath < got.Lockfiles[j].FilePath - }) - tt.want.MatchJSON(t, got) }) } diff --git a/internal/image/layer.go b/internal/image/layer.go index 9e100dc03f..186ddd1f46 100644 --- a/internal/image/layer.go +++ b/internal/image/layer.go @@ -4,8 +4,9 @@ import ( "io/fs" "os" "path/filepath" + "time" - "github.com/dghubble/trie" + "github.com/google/osv-scanner/internal/image/thirdparty/trie" ) type fileType int @@ -26,6 +27,69 @@ type FileNode struct { permission fs.FileMode } +var _ fs.DirEntry = FileNode{} + +func (f FileNode) IsDir() bool { + return f.fileType == Dir +} + +func (f FileNode) Name() string { + return filepath.Base(f.virtualPath) +} + +func (f FileNode) Type() fs.FileMode { + return f.permission +} + +func (f FileNode) Info() (fs.FileInfo, error) { + return f.Stat() +} + +type FileNodeFileInfo struct { + baseFileInfo fs.FileInfo + fileNode *FileNode +} + +var _ fs.FileInfo = FileNodeFileInfo{} + +func (f FileNodeFileInfo) Name() string { + return filepath.Base(f.fileNode.virtualPath) +} + +func (f FileNodeFileInfo) Size() int64 { + return f.baseFileInfo.Size() +} + +func (f FileNodeFileInfo) Mode() fs.FileMode { + return f.fileNode.permission +} + +func (f FileNodeFileInfo) ModTime() time.Time { + return f.baseFileInfo.ModTime() +} + +func (f FileNodeFileInfo) IsDir() bool { + return f.fileNode.fileType == Dir +} + +func (f FileNodeFileInfo) Sys() any { + return nil +} + +// Stat returns the FileInfo structure describing file. +func (f *FileNode) Stat() (fs.FileInfo, error) { + baseFileInfo, err := os.Stat(f.absoluteDiskPath()) + if err != nil { + return nil, err + } + + return FileNodeFileInfo{ + baseFileInfo: baseFileInfo, + fileNode: f, + }, nil +} + +// Open returns a file handle for the file func (f *FileNode) Open() (*os.File, error) { if f.isWhiteout { return nil, fs.ErrNotExist @@ -47,7 +111,52 @@ type Layer struct { // TODO: Use hashmap to speed up path lookups } +func (filemap Layer) Open(path string) (fs.File, error) { + node, err := filemap.getFileNode(path) + if err != nil { + return nil, err + } + + return node.Open() +} + +func (filemap Layer) Stat(path string) (fs.FileInfo, error) { + node, err := filemap.getFileNode(path) + if err != nil { + return nil, err + } + + return node.Stat() +} + +func (filemap Layer) ReadDir(path string) ([]fs.DirEntry, error) { + output := []fs.DirEntry{} + err := filemap.fileNodeTrie.WalkChildren(path, func(path string, value interface{}) error { + if value == nil { + panic("TODO: Unexpected, corrupted tar?, we should be storing all directories") + } + + output = append(output, value.(FileNode)) + + return nil + }) + + if err != nil { + return []fs.DirEntry{}, err + } + + return output, nil +} + +var _ fs.FS = Layer{} +var _ fs.StatFS = Layer{} +var _ fs.ReadDirFS = Layer{} + func (filemap Layer) getFileNode(path string) (FileNode, error) { + if !filepath.IsAbs(path) { + path = filepath.Join("/", path) + } + node, ok := filemap.fileNodeTrie.Get(path).(FileNode) if !ok { return FileNode{}, fs.ErrNotExist diff --git a/internal/image/scan.go b/internal/image/scan.go index 9bfc8ae02d..cd125b2c7f 100644 --- a/internal/image/scan.go +++ b/internal/image/scan.go @@ -1,14 +1,21 @@ package image import ( + "cmp" "errors" "fmt" "io/fs" "log" + "path/filepath" + "slices" + "strings" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scanner/internal/lockfilescalibr" "github.com/google/osv-scanner/pkg/lockfile" "github.com/google/osv-scanner/pkg/models" "github.com/google/osv-scanner/pkg/reporter" + "golang.org/x/exp/maps" ) // ScanImage scans an exported docker image .tar file @@ -22,33 +29,101 @@ func ScanImage(r reporter.Reporter, imagePath string) (ScanResults, error) { allFiles := img.LastLayer().AllFiles() - scannedLockfiles := ScanResults{ + scanResults := ScanResults{ ImagePath: imagePath, } + + inventories := []*extractor.Inventory{} + for _, file := range allFiles { if file.fileType != RegularFile { continue } - parsedLockfile, err := extractArtifactDeps(file.virtualPath, img.LastLayer()) + + // TODO: Currently osv-scalibr does not correctly annotate OS packages + // causing artifact extractors to double extract elements here. + // So let's skip all these directories for now. + // See (b/364536788) + if strings.HasPrefix(file.virtualPath, "/usr/local/") || + strings.HasPrefix(file.virtualPath, "/opt/") { + continue + } + + extractedInventories, err := extractArtifactDeps(file.virtualPath, img.LastLayer()) if err != nil { - if !errors.Is(err, lockfile.ErrExtractorNotFound) { + if !errors.Is(err, lockfilescalibr.ErrExtractorNotFound) { r.Errorf("Attempted to extract lockfile but failed: %s - %v\n", file.virtualPath, err) } continue } + inventories = append(inventories, extractedInventories...) + } + + // TODO: Remove the lockfile.Lockfile conversion + // Temporarily convert back to lockfile.Lockfiles to minimize snapshot changes + // This is done to verify the scanning behavior have not changed with this refactor + // and to minimize changes in the initial PR. + lockfiles := map[string]lockfile.Lockfile{} + for _, i := range inventories { + if len(i.Annotations) > 1 { + log.Printf("%v", i.Annotations) + } + lf, exists := lockfiles[filepath.Join("/", i.Locations[0])] + if !exists { + lf = lockfile.Lockfile{ + FilePath: filepath.Join("/", i.Locations[0]), + ParsedAs: i.Extractor.Name(), + } + } + + pkg := lockfile.PackageDetails{ + Name: i.Name, + Version: i.Version, + Ecosystem: lockfile.Ecosystem(i.Ecosystem()), + CompareAs: lockfile.Ecosystem(strings.Split(i.Ecosystem(), ":")[0]), + } + if i.SourceCode != nil { + pkg.Commit = i.SourceCode.Commit + } + + lf.Packages = append(lf.Packages, pkg) - scannedLockfiles.Lockfiles = append(scannedLockfiles.Lockfiles, parsedLockfile) + lockfiles[filepath.Join("/", i.Locations[0])] = lf } - traceOrigin(img, &scannedLockfiles) + for _, l := range lockfiles { + slices.SortFunc(l.Packages, func(a, b lockfile.PackageDetails) int { + return cmp.Or( + strings.Compare(a.Name, b.Name), + strings.Compare(a.Version, b.Version), + ) + }) + } + + scanResults.Lockfiles = maps.Values(lockfiles) + slices.SortFunc(scanResults.Lockfiles, func(a, b lockfile.Lockfile) int { + return strings.Compare(a.FilePath, b.FilePath) + }) + + traceOrigin(img, &scanResults) + + // TODO: Reenable this sort when removing lockfile.Lockfile + // Sort to have deterministic output, and to match behavior of lockfile.extractDeps + // slices.SortFunc(scanResults.Inventories, func(a, b *extractor.Inventory) int { + // // TODO: Should we consider errors here? + // aPURL, _ := a.Extractor.ToPURL(a) + // bPURL, _ := b.Extractor.ToPURL(b) + + // return strings.Compare(aPURL.ToString(), bPURL.ToString()) + // }) err = img.Cleanup() if err != nil { err = fmt.Errorf("failed to cleanup: %w", img.Cleanup()) } - return scannedLockfiles, err + return scanResults, err } // traceOrigin fills out the originLayerID for each package in ScanResults @@ -57,18 +132,26 @@ func traceOrigin(img *Image, scannedLockfiles *ScanResults) { for _, file := range scannedLockfiles.Lockfiles { // Defined locally as this is the only place this is used. type PDKey struct { - Name string - Version string - Commit string - Ecosystem lockfile.Ecosystem + Name string + Version string + // Commit string + Ecosystem string } makePDKey := func(pd lockfile.PackageDetails) PDKey { + return PDKey{ + Name: pd.Name, + Version: pd.Version, + // Commit: pd.Commit, + Ecosystem: string(pd.Ecosystem), + } + } + + makePDKey2 := func(pd *extractor.Inventory) PDKey { return PDKey{ Name: pd.Name, Version: pd.Version, - Commit: pd.Commit, - Ecosystem: pd.Ecosystem, + Ecosystem: pd.Ecosystem(), } } @@ -124,8 +207,8 @@ func traceOrigin(img *Image, scannedLockfiles *ScanResults) { } // For each package in the old version, check if it existed in the newer layer, if so, the origin must be this layer or earlier. - for _, pkg := range oldDeps.Packages { - key := makePDKey(pkg) + for _, pkg := range oldDeps { + key := makePDKey2(pkg) if val, ok := sourceLayerIdx[key]; ok && val == prevLayerIdx { sourceLayerIdx[key] = layerIdx } diff --git a/internal/image/thirdparty/trie/CHANGES.md b/internal/image/thirdparty/trie/CHANGES.md new file mode 100644 index 0000000000..125d6c77e3 --- /dev/null +++ b/internal/image/thirdparty/trie/CHANGES.md @@ -0,0 +1,15 @@ +# Changes from upstream: + +- Removed rune trie. +- Added GetChildren() function +- Removed go mod + +# trie Changelog + +Notable changes between releases. + +## Latest + +## v0.1.0 + +* Tag an official release to ease usage for some folks diff --git a/internal/image/thirdparty/trie/LICENSE b/internal/image/thirdparty/trie/LICENSE new file mode 100644 index 0000000000..f40430ab11 --- /dev/null +++ b/internal/image/thirdparty/trie/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Dalton Hubble + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/internal/image/thirdparty/trie/README.md b/internal/image/thirdparty/trie/README.md new file mode 100644 index 0000000000..a10a342952 --- /dev/null +++ b/internal/image/thirdparty/trie/README.md @@ -0,0 +1,55 @@ +# Trie +[![GoDoc](https://pkg.go.dev/badge/github.com/dghubble/trie.svg)](https://pkg.go.dev/github.com/dghubble/trie) +[![Workflow](https://github.com/dghubble/trie/actions/workflows/test.yaml/badge.svg)](https://github.com/dghubble/trie/actions/workflows/test.yaml?query=branch%3Amain) +[![Sponsors](https://img.shields.io/github/sponsors/dghubble?logo=github)](https://github.com/sponsors/dghubble) +[![Mastodon](https://img.shields.io/badge/follow-news-6364ff?logo=mastodon)](https://fosstodon.org/@dghubble) + +Package `trie` implements rune-wise and path-wise [Tries](https://en.wikipedia.org/wiki/Trie) optimized for `Get` performance and to allocate 0 bytes of heap memory (i.e. garbage) per `Get`. + +A typical use case is to perform any `Put` or `Delete` operations upfront to populate the trie, then perform `Get` operations very quickly. The Tries do not synchronize access (not thread-safe). + +When Tries are chosen over maps, it is typically for their space efficiency. However, in situations where direct key lookup is not possible (e.g. routers), tries can provide faster lookups and avoid key iteration. + +## Install + +``` +$ go get github.com/dghubble/trie +``` + +## Documentation + +Read [Godoc](https://godoc.org/github.com/dghubble/trie) + +## Performance + +RuneTrie is a typical Trie which segments strings rune-wise (i.e. by unicode code point). These benchmarks perform Puts and Gets of random string keys that are 30 bytes long and of random '/' separated paths that have 3 parts and are 30 bytes long (longer if you count the '/' seps). + +``` +BenchmarkRuneTriePutStringKey-8 3000000 437 ns/op 9 B/op 1 allocs/op +BenchmarkRuneTrieGetStringKey-8 3000000 411 ns/op 0 B/op 0 allocs/op +BenchmarkRuneTriePutPathKey-8 3000000 464 ns/op 9 B/op 1 allocs/op +BenchmarkRuneTrieGetPathKey-8 3000000 429 ns/op 0 B/op 0 allocs/op +``` + +PathTrie segments strings by forward slash separators which can boost performance +for some use cases. These benchmarks perform Puts and Gets of random string keys that are 30 bytes long and of random '/' separated paths that have 3 parts and are 30 bytes long (longer if you count the '/' seps). + +``` +BenchmarkPathTriePutStringKey-8 30000000 55.5 ns/op 8 B/op 1 allocs/op +BenchmarkPathTrieGetStringKey-8 50000000 37.9 ns/op 0 B/op 0 allocs/op +BenchmarkPathTriePutPathKey-8 20000000 88.7 ns/op 8 B/op 1 allocs/op +BenchmarkPathTrieGetPathKey-8 20000000 68.6 ns/op 0 B/op 0 allocs/op +``` + +Note that for random string Puts and Gets, the PathTrie is effectively a map as every node is a direct child of the root (except for strings that happen to have a slash). + +This benchmark measures the performance of the PathSegmenter alone. It is used to segment random paths that have 3 '/' separated parts and are 30 bytes long. + +``` +BenchmarkPathSegmenter-8 50000000 32.0 ns/op 0 B/op 0 allocs/op +``` + +## License + +[MIT License](LICENSE) + diff --git a/internal/image/thirdparty/trie/bench_test.go b/internal/image/thirdparty/trie/bench_test.go new file mode 100644 index 0000000000..18edde0b20 --- /dev/null +++ b/internal/image/thirdparty/trie/bench_test.go @@ -0,0 +1,102 @@ +package trie + +import ( + "crypto/rand" + "testing" +) + +var stringKeys [1000]string // random string keys +const bytesPerKey = 30 + +var pathKeys [1000]string // random /paths/of/parts keys +const partsPerKey = 3 // (e.g. /a/b/c has parts /a, /b, /c) +const bytesPerPart = 10 + +func init() { + // string keys + for i := 0; i < len(stringKeys); i++ { + key := make([]byte, bytesPerKey) + if _, err := rand.Read(key); err != nil { + panic("error generating random byte slice") + } + stringKeys[i] = string(key) + } + + // path keys + for i := 0; i < len(pathKeys); i++ { + var key string + for j := 0; j < partsPerKey; j++ { + key += "/" + part := make([]byte, bytesPerPart) + if _, err := rand.Read(part); err != nil { + panic("error generating random byte slice") + } + key += string(part) + } + pathKeys[i] = key + } +} + +// PathTrie +/////////////////////////////////////////////////////////////////////////////// + +// string keys + +func BenchmarkPathTriePutStringKey(b *testing.B) { + trie := NewPathTrie() + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + trie.Put(stringKeys[i%len(stringKeys)], i) + } +} + +func BenchmarkPathTrieGetStringKey(b *testing.B) { + trie := NewPathTrie() + for i := 0; i < b.N; i++ { + trie.Put(stringKeys[i%len(stringKeys)], i) + } + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + trie.Get(stringKeys[i%len(stringKeys)]) + } +} + +// path keys + +func BenchmarkPathTriePutPathKey(b *testing.B) { + trie := NewPathTrie() + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + trie.Put(pathKeys[i%len(pathKeys)], i) + } +} + +func BenchmarkPathTrieGetPathKey(b *testing.B) { + trie := NewPathTrie() + for i := 0; i < b.N; i++ { + trie.Put(pathKeys[i%len(pathKeys)], i) + } + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + trie.Get(pathKeys[i%len(pathKeys)]) + } +} + +// benchmark PathSegmenter + +func BenchmarkPathSegmenter(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + for j := 0; j < b.N; j++ { + for part, i := PathSegmenter(pathKeys[j%len(pathKeys)], 0); ; part, i = PathSegmenter(pathKeys[j%len(pathKeys)], i) { + var _ = part // NoOp 'use' the key part + if i == -1 { + break + } + } + } +} diff --git a/internal/image/thirdparty/trie/common.go b/internal/image/thirdparty/trie/common.go new file mode 100644 index 0000000000..bea317cc5e --- /dev/null +++ b/internal/image/thirdparty/trie/common.go @@ -0,0 +1,30 @@ +package trie + +import ( + "strings" +) + +// WalkFunc defines some action to take on the given key and value during +// a Trie Walk. Returning a non-nil error will terminate the Walk. +type WalkFunc func(key string, value interface{}) error + +// StringSegmenter takes a string key with a starting index and returns +// the first segment after the start and the ending index. When the end is +// reached, the returned nextIndex should be -1. +// Implementations should NOT allocate heap memory as Trie Segmenters are +// called upon Gets. See PathSegmenter. +type StringSegmenter func(key string, start int) (segment string, nextIndex int) + +// PathSegmenter segments string key paths by slash separators. For example, +// "/a/b/c" -> ("/a", 2), ("/b", 4), ("/c", -1) in successive calls. It does +// not allocate any heap memory. +func PathSegmenter(path string, start int) (segment string, next int) { + if len(path) == 0 || start < 0 || start > len(path)-1 { + return "", -1 + } + end := strings.IndexRune(path[start+1:], '/') // next '/' after 0th rune + if end == -1 { + return path[start:], -1 + } + return path[start : start+end+1], start + end + 1 +} diff --git a/internal/image/thirdparty/trie/doc.go b/internal/image/thirdparty/trie/doc.go new file mode 100644 index 0000000000..301925f217 --- /dev/null +++ b/internal/image/thirdparty/trie/doc.go @@ -0,0 +1,12 @@ +/* +Package trie implements several types of performant Tries (e.g. rune-wise, +path-wise). + +The implementations are optimized for Get performance and to allocate 0 bytes +of heap memory (i.e. garbage) per Get. + +The Tries do not synchronize access (not thread-safe). A typical use case is +to perform Puts and Deletes upfront to populate the Trie, then perform Gets +very quickly. +*/ +package trie diff --git a/internal/image/thirdparty/trie/path_trie.go b/internal/image/thirdparty/trie/path_trie.go new file mode 100644 index 0000000000..02c263ffdd --- /dev/null +++ b/internal/image/thirdparty/trie/path_trie.go @@ -0,0 +1,202 @@ +package trie + +// PathTrie is a trie of paths with string keys and interface{} values. + +// PathTrie is a trie of string keys and interface{} values. Internal nodes +// have nil values so stored nil values cannot be distinguished and are +// excluded from walks. By default, PathTrie will segment keys by forward +// slashes with PathSegmenter (e.g. "/a/b/c" -> "/a", "/b", "/c"). A custom +// StringSegmenter may be used to customize how strings are segmented into +// nodes. A classic trie might segment keys by rune (i.e. unicode points). +type PathTrie struct { + segmenter StringSegmenter // key segmenter, must not cause heap allocs + value interface{} + children map[string]*PathTrie +} + +// PathTrieConfig for building a path trie with different segmenter +type PathTrieConfig struct { + Segmenter StringSegmenter +} + +// NewPathTrie allocates and returns a new *PathTrie. +func NewPathTrie() *PathTrie { + return &PathTrie{ + segmenter: PathSegmenter, + } +} + +// NewPathTrieWithConfig allocates and returns a new *PathTrie with the given *PathTrieConfig +func NewPathTrieWithConfig(config *PathTrieConfig) *PathTrie { + segmenter := PathSegmenter + if config != nil && config.Segmenter != nil { + segmenter = config.Segmenter + } + + return &PathTrie{ + segmenter: segmenter, + } +} + +// newPathTrieFromTrie returns new trie while preserving its config +func (trie *PathTrie) newPathTrie() *PathTrie { + return &PathTrie{ + segmenter: trie.segmenter, + } +} + +// Get returns the value stored at the given key. Returns nil for internal +// nodes or for nodes with a value of nil. +func (trie *PathTrie) Get(key string) interface{} { + node := trie + for part, i := trie.segmenter(key, 0); part != ""; part, i = trie.segmenter(key, i) { + node = node.children[part] + if node == nil { + return nil + } + } + return node.value +} + +// Put inserts the value into the trie at the given key, replacing any +// existing items. It returns true if the put adds a new value, false +// if it replaces an existing value. +// Note that internal nodes have nil values so a stored nil value will not +// be distinguishable and will not be included in Walks. +func (trie *PathTrie) Put(key string, value interface{}) bool { + node := trie + for part, i := trie.segmenter(key, 0); part != ""; part, i = trie.segmenter(key, i) { + child := node.children[part] + if child == nil { + if node.children == nil { + node.children = map[string]*PathTrie{} + } + child = trie.newPathTrie() + node.children[part] = child + } + node = child + } + // does node have an existing value? + isNewVal := node.value == nil + node.value = value + return isNewVal +} + +// Delete removes the value associated with the given key. Returns true if a +// node was found for the given key. If the node or any of its ancestors +// becomes childless as a result, it is removed from the trie. +func (trie *PathTrie) Delete(key string) bool { + var path []nodeStr // record ancestors to check later + node := trie + for part, i := trie.segmenter(key, 0); part != ""; part, i = trie.segmenter(key, i) { + path = append(path, nodeStr{part: part, node: node}) + node = node.children[part] + if node == nil { + // node does not exist + return false + } + } + // delete the node value + node.value = nil + // if leaf, remove it from its parent's children map. Repeat for ancestor path. + if node.isLeaf() { + // iterate backwards over path + for i := len(path) - 1; i >= 0; i-- { + parent := path[i].node + part := path[i].part + delete(parent.children, part) + if !parent.isLeaf() { + // parent has other children, stop + break + } + parent.children = nil + if parent.value != nil { + // parent has a value, stop + break + } + } + } + return true // node (internal or not) existed and its value was nil'd +} + +// Walk iterates over each key/value stored in the trie and calls the given +// walker function with the key and value. If the walker function returns +// an error, the walk is aborted. +// The traversal is depth first with no guaranteed order. +func (trie *PathTrie) Walk(walker WalkFunc) error { + return trie.walk("", walker) +} + +// WalkPath iterates over each key/value in the path in trie from the root to +// the node at the given key, calling the given walker function for each +// key/value. If the walker function returns an error, the walk is aborted. +func (trie *PathTrie) WalkPath(key string, walker WalkFunc) error { + // Get root value if one exists. + if trie.value != nil { + if err := walker("", trie.value); err != nil { + return err + } + } + for part, i := trie.segmenter(key, 0); ; part, i = trie.segmenter(key, i) { + if trie = trie.children[part]; trie == nil { + return nil + } + if trie.value != nil { + var k string + if i == -1 { + k = key + } else { + k = key[0:i] + } + if err := walker(k, trie.value); err != nil { + return err + } + } + if i == -1 { + break + } + } + return nil +} + +func (trie *PathTrie) WalkChildren(key string, walker WalkFunc) error { + node := trie + for part, i := trie.segmenter(key, 0); part != ""; part, i = trie.segmenter(key, i) { + node = node.children[part] + if node == nil { + return nil + } + } + + for k, child := range node.children { + if err := walker(key+k, child.value); err != nil { + return err + } + } + + return nil +} + +// PathTrie node and the part string key of the child the path descends into. +type nodeStr struct { + node *PathTrie + part string +} + +func (trie *PathTrie) walk(key string, walker WalkFunc) error { + if trie.value != nil { + if err := walker(key, trie.value); err != nil { + return err + } + } + for part, child := range trie.children { + if err := child.walk(key+part, walker); err != nil { + return err + } + } + return nil +} + +func (trie *PathTrie) isLeaf() bool { + return len(trie.children) == 0 +} diff --git a/internal/image/thirdparty/trie/segmenter_test.go b/internal/image/thirdparty/trie/segmenter_test.go new file mode 100644 index 0000000000..26f414e568 --- /dev/null +++ b/internal/image/thirdparty/trie/segmenter_test.go @@ -0,0 +1,122 @@ +package trie + +import ( + "strings" + "testing" +) + +// test splitting /path/keys/ into parts (e.g. /path, /keys, /) +func TestPathSegmenter(t *testing.T) { + cases := []struct { + key string + parts []string + indices []int // indexes to use as next start, in order + }{ + {"", []string{""}, []int{-1}}, + {"/", []string{"/"}, []int{-1}}, + {"static_file", []string{"static_file"}, []int{-1}}, + {"/users/scott", []string{"/users", "/scott"}, []int{6, -1}}, + {"users/scott", []string{"users", "/scott"}, []int{5, -1}}, + {"/users/ramona/", []string{"/users", "/ramona", "/"}, []int{6, 13, -1}}, + {"users/ramona/", []string{"users", "/ramona", "/"}, []int{5, 12, -1}}, + {"//", []string{"/", "/"}, []int{1, -1}}, + {"/a/b/c", []string{"/a", "/b", "/c"}, []int{2, 4, -1}}, + } + + for _, c := range cases { + partNum := 0 + for prefix, i := PathSegmenter(c.key, 0); ; prefix, i = PathSegmenter(c.key, i) { + if prefix != c.parts[partNum] { + t.Errorf("expected part %d of key '%s' to be '%s', got '%s'", partNum, c.key, c.parts[partNum], prefix) + } + if i != c.indices[partNum] { + t.Errorf("in iteration %d, expected next index of key '%s' to be '%d', got '%d'", partNum, c.key, c.indices[partNum], i) + } + partNum++ + if i == -1 { + break + } + } + if partNum != len(c.parts) { + t.Errorf("expected '%s' to have %d parts, got %d", c.key, len(c.parts), partNum) + } + } +} + +func TestPathSegmenterEdgeCases(t *testing.T) { + cases := []struct { + path string + start int + segment string + nextIndex int + }{ + {"", 0, "", -1}, + {"", 10, "", -1}, + {"/", 0, "/", -1}, + {"/", 10, "", -1}, + {"/", -10, "", -1}, + {"/", 1, "", -1}, + {"//", 0, "/", 1}, + {"//", 1, "/", -1}, + {"//", 2, "", -1}, + {" /", 0, " ", 1}, + {" /", 1, "/", -1}, + } + + for _, c := range cases { + segment, nextIndex := PathSegmenter(c.path, c.start) + if segment != c.segment { + t.Errorf("expected segment %s starting at %d in path %s, got %s", c.segment, c.start, c.path, segment) + } + if nextIndex != c.nextIndex { + t.Errorf("expected nextIndex %d starting at %d in path %s, got %d", c.nextIndex, c.start, c.path, nextIndex) + } + } +} + +func testPathSegmenterDot(path string, start int) (segment string, next int) { + if len(path) == 0 || start < 0 || start > len(path)-1 { + return "", -1 + } + end := strings.IndexRune(path[start+1:], '.') + if end == -1 { + return path[start:], -1 + } + return path[start : start+end+1], start + end + 1 +} + +func TestCustomPathSegmenter(t *testing.T) { + depthTest := map[string]bool{ + "a": false, + "a.b": false, + "a.b.c": false, + "a.b.c.d": false, + "a.b.c.d.e": false, + ".a": false, + ".a.b": false, + ".a.b.c": false, + ".a.b.c.d": false, + ".a.b.c.d.e": false, + } + + tie := NewPathTrieWithConfig(&PathTrieConfig{Segmenter: testPathSegmenterDot}) + for k := range depthTest { + tie.Put(k, true) + } + + for k := range depthTest { + tie.WalkPath(k, func(k string, v interface{}) error { + out := tie.Get(k) + if out != nil { + depthTest[k] = true + } + return nil + }) + } + for k, ok := range depthTest { + if !ok { + t.Errorf("did not walk thru %s node", k) + } + } + +} diff --git a/internal/image/thirdparty/trie/trie.go b/internal/image/thirdparty/trie/trie.go new file mode 100644 index 0000000000..c6cef990e5 --- /dev/null +++ b/internal/image/thirdparty/trie/trie.go @@ -0,0 +1,11 @@ +package trie + +// Trier exposes the Trie structure capabilities. +type Trier interface { + Get(key string) interface{} + Put(key string, value interface{}) bool + Delete(key string) bool + Walk(walker WalkFunc) error + WalkPath(key string, walker WalkFunc) error + WalkChildren(key string, walker WalkFunc) error +} diff --git a/internal/image/thirdparty/trie/trie_test.go b/internal/image/thirdparty/trie/trie_test.go new file mode 100644 index 0000000000..a03c66a002 --- /dev/null +++ b/internal/image/thirdparty/trie/trie_test.go @@ -0,0 +1,468 @@ +package trie + +import ( + "errors" + "slices" + "testing" +) + +// PathTrie + +func TestPathTrie(t *testing.T) { + trie := NewPathTrie() + testTrie(t, trie) +} + +func TestPathTrieWithNilConfig(t *testing.T) { + trie := NewPathTrieWithConfig(nil) + testTrie(t, trie) +} + +func TestPathTrieWithEmptyConfig(t *testing.T) { + trie := NewPathTrieWithConfig(&PathTrieConfig{}) + testTrie(t, trie) +} + +func TestPathTrieWithConfig(t *testing.T) { + trie := NewPathTrieWithConfig(&PathTrieConfig{PathSegmenter}) + testTrie(t, trie) +} + +func TestPathTrieNilBehavior(t *testing.T) { + trie := NewPathTrie() + testNilBehavior(t, trie) +} + +func TestPathTrieRoot(t *testing.T) { + trie := NewPathTrie() + testTrieRoot(t, trie) + + trie = NewPathTrie() + if !trie.isLeaf() { + t.Error("root of empty tree should be leaf") + } + trie.Put("", "root") + if !trie.isLeaf() { + t.Error("root should not have children, only value") + } +} + +func TestPathTrieWalk(t *testing.T) { + trie := NewPathTrie() + testTrieWalk(t, trie) +} + +func TestPathTrieWalkError(t *testing.T) { + trie := NewPathTrie() + testTrieWalkError(t, trie) +} + +func TestPathTrieWalkChildren(t *testing.T) { + trie := NewPathTrie() + testTrieWalkChildren(t, trie) +} + +func TestPathTrieWalkPath(t *testing.T) { + trie := NewPathTrie() + testTrieWalkPath(t, trie) +} + +func TestPathTrieWalkPathError(t *testing.T) { + trie := NewPathTrie() + testTrieWalkPathError(t, trie) +} + +func testTrie(t *testing.T, trie Trier) { + const firstPutValue = "first put" + cases := []struct { + key string + value interface{} + }{ + {"fish", 0}, + {"/cat", 1}, + {"/dog", 2}, + {"/cats", 3}, + {"/caterpillar", 4}, + {"/cat/gideon", 5}, + {"/cat/giddy", 6}, + } + + // get missing keys + for _, c := range cases { + if value := trie.Get(c.key); value != nil { + t.Errorf("expected key %s to be missing, found value %v", c.key, value) + } + } + + // initial put + for _, c := range cases { + if isNew := trie.Put(c.key, firstPutValue); !isNew { + t.Errorf("expected key %s to be missing", c.key) + } + } + + // subsequent put + for _, c := range cases { + if isNew := trie.Put(c.key, c.value); isNew { + t.Errorf("expected key %s to have a value already", c.key) + } + } + + // get + for _, c := range cases { + if value := trie.Get(c.key); value != c.value { + t.Errorf("expected key %s to have value %v, got %v", c.key, c.value, value) + } + } + + // delete, expect Delete to return true indicating a node was nil'd + for _, c := range cases { + if deleted := trie.Delete(c.key); !deleted { + t.Errorf("expected key %s to be deleted", c.key) + } + } + + // delete cleaned all the way to the first character + // expect Delete to return false bc no node existed to nil + for _, c := range cases { + if deleted := trie.Delete(string(c.key[0])); deleted { + t.Errorf("expected key %s to be cleaned by delete", string(c.key[0])) + } + } + + // get deleted keys + for _, c := range cases { + if value := trie.Get(c.key); value != nil { + t.Errorf("expected key %s to be deleted, got value %v", c.key, value) + } + } +} + +func testNilBehavior(t *testing.T, trie Trier) { + cases := []struct { + key string + value interface{} + }{ + {"/cat", 1}, + {"/catamaran", 2}, + {"/caterpillar", nil}, + } + expectNilValues := []string{"/", "/c", "/ca", "/caterpillar", "/other"} + + // initial put + for _, c := range cases { + if isNew := trie.Put(c.key, c.value); !isNew { + t.Errorf("expected key %s to be missing", c.key) + } + } + + // get nil + for _, key := range expectNilValues { + if value := trie.Get(key); value != nil { + t.Errorf("expected key %s to have value nil, got %v", key, value) + } + } +} + +func testTrieRoot(t *testing.T, trie Trier) { + const firstPutValue = "first put" + const putValue = "value" + + if value := trie.Get(""); value != nil { + t.Errorf("expected key '' to be missing, found value %v", value) + } + if !trie.Put("", firstPutValue) { + t.Error("expected key '' to be missing") + } + if trie.Put("", putValue) { + t.Error("expected key '' to have a value already") + } + if value := trie.Get(""); value != putValue { + t.Errorf("expected key '' to have value %v, got %v", putValue, value) + } + if !trie.Delete("") { + t.Error("expected key '' to be deleted") + } + if value := trie.Get(""); value != nil { + t.Errorf("expected key '' to be deleted, got value %v", value) + } +} + +func testTrieWalk(t *testing.T, trie Trier) { + table := map[string]interface{}{ + "": -1, + "fish": 0, + "/cat": 1, + "/dog": 2, + "/cats": 3, + "/caterpillar": 4, + "/notes": 30, + "/notes/new": 31, + "/notes/:id": 32, + } + // key -> times walked + walked := make(map[string]int) + for key := range table { + walked[key] = 0 + } + + for key, value := range table { + if isNew := trie.Put(key, value); !isNew { + t.Errorf("expected key %s to be missing", key) + } + } + + walker := func(key string, value interface{}) error { + // value for each walked key is correct + if value != table[key] { + t.Errorf("expected key %s to have value %v, got %v", key, table[key], value) + } + walked[key]++ + return nil + } + if err := trie.Walk(walker); err != nil { + t.Errorf("expected error nil, got %v", err) + } + + // each key/value walked exactly once + for key, walkedCount := range walked { + if walkedCount != 1 { + t.Errorf("expected key %s to be walked exactly once, got %v", key, walkedCount) + } + } +} + +func testTrieWalkError(t *testing.T, trie Trier) { + table := map[string]interface{}{ + "/L1/L2A": 1, + "/L1/L2B/L3A": 2, + "/L1/L2B/L3B/L4": 42, + "/L1/L2B/L3C": 4, + "/L1/L2C": 5, + } + + walkerError := errors.New("walker error") + walked := 0 + + for key, value := range table { + trie.Put(key, value) + } + walker := func(key string, value interface{}) error { + if value == 42 { + return walkerError + } + walked++ + return nil + } + if err := trie.Walk(walker); err != walkerError { + t.Errorf("expected walker error, got %v", err) + } + if len(table) == walked { + t.Errorf("expected nodes walked < %d, got %d", len(table), walked) + } +} + +func testTrieWalkPath(t *testing.T, trie Trier) { + table := map[string]interface{}{ + "fish": 0, + "/cat": 1, + "/dog": 2, + "/cats": 3, + "/caterpillar": 4, + "/notes": 30, + "/notes/new": 31, + "/notes/new/noise": 32, + } + // key -> times walked + walked := make(map[string]int) + for key := range table { + walked[key] = 0 + } + + for key, value := range table { + if isNew := trie.Put(key, value); !isNew { + t.Errorf("expected key %s to be missing", key) + } + } + + walker := func(key string, value interface{}) error { + // value for each walked key is correct + if value != table[key] { + t.Errorf("expected key %s to have value %v, got %v", key, table[key], value) + } + walked[key]++ + return nil + } + if err := trie.WalkPath("/notes/new/noise", walker); err != nil { + t.Errorf("expected error nil, got %v", err) + } + + // expect each key/value in path walked exactly once, and not other keys + for key, walkedCount := range walked { + switch key { + case "/notes", "/notes/new", "/notes/new/noise": + if walkedCount != 1 { + t.Errorf("expected key %s to be walked exactly once, got %v", key, walkedCount) + } + default: + if walkedCount != 0 { + t.Errorf("expected key %s to not be walked, got %v", key, walkedCount) + } + } + } + + for key := range table { + walked[key] = 0 + } + if err := trie.WalkPath("/notes/new/nose", walker); err != nil { + t.Errorf("expected error nil, got %v", err) + } + // expect each key/value in path walked exactly once, and not other keys + for key, walkedCount := range walked { + switch key { + case "/notes", "/notes/new": + if walkedCount != 1 { + t.Errorf("expected key %s to be walked exactly once, got %v", key, walkedCount) + } + default: + if walkedCount != 0 { + t.Errorf("expected key %s to not be walked, got %v", key, walkedCount) + } + } + } + + var foundRoot bool + trie.Put("", "ROOT") + trie.WalkPath("/notes/new/noise", func(key string, value interface{}) error { + if key == "" && value == "ROOT" { + foundRoot = true + } + return nil + }) + if !foundRoot { + t.Error("did not find root") + } +} + +func testTrieWalkPathError(t *testing.T, trie Trier) { + table := map[string]interface{}{ + "/L1/L2A": 1, + "/L1/L2A/L3B": 99, + "/L1/L2A/L3B/L4": 2, + "/L1/L2B/L3B": 3, + "/L1/L2C": 4, + } + + walkerError := errors.New("walker error") + + walked := make(map[string]int) + for key := range table { + walked[key] = 0 + } + + for key, value := range table { + trie.Put(key, value) + } + walker := func(key string, value interface{}) error { + if value == 99 { + return walkerError + } + walked[key]++ + return nil + } + if err := trie.WalkPath("/L1/L2A/L3B", walker); err != walkerError { + t.Errorf("expected walker error, got %v", err) + } + // expect each key/value in path, up to error and not including key with + // value 99, walked exactly once, and not other keys + var walkedTotal int + for key, walkedCount := range walked { + switch key { + case "/L1/L2A": + if walkedCount != 1 { + t.Errorf("expected key %s to be walked exactly once, got %v", key, walkedCount) + } + walkedTotal++ + default: + if walkedCount != 0 { + t.Errorf("expected key %s to not be walked, got %v", key, walkedCount) + } + } + } + if walkedTotal != 1 { + t.Errorf("expected 1 nodes walked, got %d", walkedTotal) + } + + rootError := errors.New("error at root") + trie.Put("", "ROOT") + err := trie.WalkPath("/L1/L2A/L3B/L4", func(key string, value interface{}) error { + if key == "" && value == "ROOT" { + return rootError + } + return nil + }) + if err != rootError { + t.Errorf("expected %s, got %s", rootError, err) + } +} + +func testTrieWalkChildren(t *testing.T, trie Trier) { + table := map[string]interface{}{ + "/L1/L2A": 1, + "/L1/L2A/L3B": 99, + "/L1/L2A/L3B/L4": 2, + "/L1/L2B/L3B": 3, + "/L1/L2C": 4, + } + + walked := make(map[string]int) + for key := range table { + walked[key] = 0 + } + + for key, value := range table { + trie.Put(key, value) + } + + type pairs struct { + isNil bool + key string + found bool + } + + expectedPairs := []pairs{ + { + key: "/L1/L2A", + isNil: false, + }, + { + key: "/L1/L2B", + isNil: true, + }, + { + key: "/L1/L2C", + isNil: false, + }, + } + + count := 0 + _ = trie.WalkChildren("/L1", func(key string, value interface{}) error { + idx := slices.Index(expectedPairs, pairs{ + key: key, + isNil: value == nil, + }) + + if idx == -1 { + t.Fatalf("did not expect: (%s, %s)", key, value) + } + + expectedPairs[idx].found = true + count += 1 + + return nil + }) + + if count != len(expectedPairs) { + t.Errorf("only found %d entries, when expecting %d", count, len(expectedPairs)) + } +} diff --git a/internal/lockfilescalibr/errors.go b/internal/lockfilescalibr/errors.go new file mode 100644 index 0000000000..005ee0012b --- /dev/null +++ b/internal/lockfilescalibr/errors.go @@ -0,0 +1,9 @@ +package lockfilescalibr + +import "errors" + +var ErrIncompatibleFileFormat = errors.New("file format is incompatible, but this is expected") +var ErrNotImplemented = errors.New("not implemented") +var ErrWrongExtractor = errors.New("this extractor did not create this inventory") +var ErrExtractorNotFound = errors.New("could not determine extractor") +var ErrNoExtractorsFound = errors.New("no extractors found to be suitable to this file") diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/extractor.go b/internal/lockfilescalibr/language/java/pomxmlnet/extractor.go new file mode 100644 index 0000000000..df989141fa --- /dev/null +++ b/internal/lockfilescalibr/language/java/pomxmlnet/extractor.go @@ -0,0 +1,174 @@ +// Package pomxmlnet extracts Maven's pom.xml format with transitive dependency resolution. +package pomxmlnet + +import ( + "context" + "fmt" + "io/fs" + "path/filepath" + + "golang.org/x/exp/maps" + + mavenresolve "deps.dev/util/resolve/maven" + mavenutil "github.com/google/osv-scanner/internal/utility/maven" + + "deps.dev/util/maven" + "deps.dev/util/resolve" + "deps.dev/util/resolve/dep" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/extractor/filesystem/osv" + "github.com/google/osv-scalibr/plugin" + "github.com/google/osv-scalibr/purl" + "github.com/google/osv-scanner/internal/resolution/client" + "github.com/google/osv-scanner/internal/resolution/datasource" +) + +// Extractor extracts osv packages from osv-scanner json output. +type Extractor struct { + client.DependencyClient + *datasource.MavenRegistryAPIClient +} + +// Name of the extractor. +func (e Extractor) Name() string { return "osv/pomxmlnet" } + +// Version of the extractor. +func (e Extractor) Version() int { return 0 } + +// Requirements of the extractor. +func (e Extractor) Requirements() *plugin.Capabilities { + return &plugin.Capabilities{ + Network: true, + } +} + +// FileRequired never returns true, as this is for the osv-scanner json output. +func (e Extractor) FileRequired(path string, _ fs.FileInfo) bool { + return filepath.Base(path) == "pom.xml" +} + +// Extract extracts packages from yarn.lock files passed through the scan input. +func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]*extractor.Inventory, error) { + var project maven.Project + if err := datasource.NewMavenDecoder(input.Reader).Decode(&project); err != nil { + return nil, fmt.Errorf("could not extract from %s: %w", input.Path, err) + } + // Merging parents data by parsing local parent pom.xml or fetching from upstream. + if err := mavenutil.MergeParents(ctx, e.MavenRegistryAPIClient, &project, project.Parent, 1, input.Path, true); err != nil { + return nil, fmt.Errorf("failed to merge parents: %w", err) + } + // Process the dependencies: + // - dedupe dependencies and dependency management + // - import dependency management + // - fill in missing dependency version requirement + project.ProcessDependencies(func(groupID, artifactID, version maven.String) (maven.DependencyManagement, error) { + root := maven.Parent{ProjectKey: maven.ProjectKey{GroupID: groupID, ArtifactID: artifactID, Version: version}} + var result maven.Project + if err := mavenutil.MergeParents(ctx, e.MavenRegistryAPIClient, &result, root, 0, input.Path, false); err != nil { + return maven.DependencyManagement{}, err + } + + return result.DependencyManagement, nil + }) + + overrideClient := client.NewOverrideClient(e.DependencyClient) + resolver := mavenresolve.NewResolver(overrideClient) + + // Resolve the dependencies. + root := resolve.Version{ + VersionKey: resolve.VersionKey{ + PackageKey: resolve.PackageKey{ + System: resolve.Maven, + Name: project.ProjectKey.Name(), + }, + VersionType: resolve.Concrete, + Version: string(project.Version), + }} + reqs := make([]resolve.RequirementVersion, len(project.Dependencies)+len(project.DependencyManagement.Dependencies)) + for i, d := range project.Dependencies { + reqs[i] = resolve.RequirementVersion{ + VersionKey: resolve.VersionKey{ + PackageKey: resolve.PackageKey{ + System: resolve.Maven, + Name: d.Name(), + }, + VersionType: resolve.Requirement, + Version: string(d.Version), + }, + Type: resolve.MavenDepType(d, ""), + } + } + for i, d := range project.DependencyManagement.Dependencies { + reqs[len(project.Dependencies)+i] = resolve.RequirementVersion{ + VersionKey: resolve.VersionKey{ + PackageKey: resolve.PackageKey{ + System: resolve.Maven, + Name: d.Name(), + }, + VersionType: resolve.Requirement, + Version: string(d.Version), + }, + Type: resolve.MavenDepType(d, mavenutil.OriginManagement), + } + } + overrideClient.AddVersion(root, reqs) + + g, err := resolver.Resolve(ctx, root.VersionKey) + if err != nil { + return nil, fmt.Errorf("failed resolving %v: %w", root, err) + } + for i, e := range g.Edges { + e.Type = dep.Type{} + g.Edges[i] = e + } + + details := map[string]*extractor.Inventory{} + for i := 1; i < len(g.Nodes); i++ { + // Ignore the first node which is the root. + node := g.Nodes[i] + depGroups := []string{} + inventory := extractor.Inventory{ + Name: node.Version.Name, + Version: node.Version.Version, + // TODO(rexpan): Add merged paths in here as well + Locations: []string{input.Path}, + } + // We are only able to know dependency groups of direct dependencies but + // not transitive dependencies because the nodes in the resolve graph does + // not have the scope information. + for _, dep := range project.Dependencies { + if dep.Name() != inventory.Name { + continue + } + if dep.Scope != "" && dep.Scope != "compile" { + depGroups = append(depGroups, string(dep.Scope)) + } + } + inventory.Metadata = osv.DepGroupMetadata{ + DepGroupVals: depGroups, + } + details[inventory.Name] = &inventory + } + + return maps.Values(details), nil +} + +// ToPURL converts an inventory created by this extractor into a PURL. +func (e Extractor) ToPURL(i *extractor.Inventory) (*purl.PackageURL, error) { + return &purl.PackageURL{ + Type: purl.TypeMaven, + Name: i.Name, + Version: i.Version, + }, nil +} + +// ToCPEs is not applicable as this extractor does not infer CPEs from the Inventory. +func (e Extractor) ToCPEs(_ *extractor.Inventory) ([]string, error) { return []string{}, nil } + +// Ecosystem returns the OSV ecosystem ('npm') of the software extracted by this extractor. +func (e Extractor) Ecosystem(_ *extractor.Inventory) string { + return "Maven" +} + +var _ filesystem.Extractor = Extractor{} diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/extractor_test.go b/internal/lockfilescalibr/language/java/pomxmlnet/extractor_test.go new file mode 100644 index 0000000000..5b6f21d967 --- /dev/null +++ b/internal/lockfilescalibr/language/java/pomxmlnet/extractor_test.go @@ -0,0 +1,360 @@ +package pomxmlnet_test + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem/osv" + "github.com/google/osv-scalibr/testing/extracttest" + "github.com/google/osv-scanner/internal/lockfilescalibr/language/java/pomxmlnet" + "github.com/google/osv-scanner/internal/resolution/clienttest" + "github.com/google/osv-scanner/internal/resolution/datasource" + "github.com/google/osv-scanner/internal/testutility" +) + +func TestMavenResolverExtractor_FileRequired(t *testing.T) { + t.Parallel() + + tests := []struct { + path string + want bool + }{ + { + path: "", + want: false, + }, + { + path: "pom.xml", + want: true, + }, + { + path: "path/to/my/pom.xml", + want: true, + }, + { + path: "path/to/my/pom.xml/file", + want: false, + }, + { + path: "path/to/my/pom.xml.file", + want: false, + }, + { + path: "path.to.my.pom.xml", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + t.Parallel() + e := pomxmlnet.Extractor{} + got := e.FileRequired(tt.path, nil) + if got != tt.want { + t.Errorf("Extract() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestExtractor_Extract(t *testing.T) { + t.Parallel() + + tests := []extracttest.TestTableEntry{ + { + Name: "Not a pom file", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/maven/not-pom.txt", + }, + WantErr: extracttest.ContainsErrStr{Str: "could not extract from"}, + }, + { + Name: "invalid xml syntax", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/maven/invalid-syntax.xml", + }, + WantErr: extracttest.ContainsErrStr{Str: "XML syntax error"}, + }, + { + Name: "empty", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/maven/empty.xml", + }, + WantInventory: []*extractor.Inventory{}, + }, + { + Name: "one package", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/maven/one-package.xml", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "org.apache.maven:maven-artifact", + Version: "1.0.0", + Locations: []string{"testdata/maven/one-package.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + }, + }, + { + Name: "two packages", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/maven/two-packages.xml", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "io.netty:netty-all", + Version: "4.1.42.Final", + Locations: []string{"testdata/maven/two-packages.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + { + Name: "org.slf4j:slf4j-log4j12", + Version: "1.7.25", + Locations: []string{"testdata/maven/two-packages.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + }, + }, + { + Name: "with dependency management", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/maven/with-dependency-management.xml", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "io.netty:netty-all", + Version: "4.1.9", + Locations: []string{"testdata/maven/with-dependency-management.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + { + Name: "org.slf4j:slf4j-log4j12", + Version: "1.7.25", + Locations: []string{"testdata/maven/with-dependency-management.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + }, + }, + { + Name: "interpolation", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/maven/interpolation.xml", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "org.mine:mypackage", + Version: "1.0.0", + Locations: []string{"testdata/maven/interpolation.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + { + Name: "org.mine:my.package", + Version: "2.3.4", + Locations: []string{"testdata/maven/interpolation.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + { + Name: "org.mine:ranged-package", + Version: "9.4.37", + Locations: []string{"testdata/maven/interpolation.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + }, + }, + { + Name: "with scope / dep groups", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/maven/with-scope.xml", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "junit:junit", + Version: "4.12", + Locations: []string{"testdata/maven/with-scope.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{"runtime"}}, + }, + }, + }, + { + Name: "transitive dependencies", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/maven/transitive.xml", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "org.direct:alice", + Version: "1.0.0", + Locations: []string{"testdata/maven/transitive.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + { + Name: "org.direct:bob", + Version: "2.0.0", + Locations: []string{"testdata/maven/transitive.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + { + Name: "org.direct:chris", + Version: "3.0.0", + Locations: []string{"testdata/maven/transitive.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + { + Name: "org.transitive:chuck", + Version: "1.1.1", + Locations: []string{"testdata/maven/transitive.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + { + Name: "org.transitive:dave", + Version: "2.2.2", + Locations: []string{"testdata/maven/transitive.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + { + Name: "org.transitive:eve", + Version: "3.3.3", + Locations: []string{"testdata/maven/transitive.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + { + Name: "org.transitive:frank", + Version: "4.4.4", + Locations: []string{"testdata/maven/transitive.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + t.Parallel() + + resolutionClient := clienttest.NewMockResolutionClient(t, "testdata/universe/basic-universe.yaml") + extr := pomxmlnet.Extractor{ + DependencyClient: resolutionClient, + } + + scanInput := extracttest.GenerateScanInputMock(t, tt.InputConfig) + defer extracttest.CloseTestScanInput(t, scanInput) + + got, err := extr.Extract(context.Background(), &scanInput) + + if diff := cmp.Diff(tt.WantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s.Extract(%q) error diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) + return + } + + if diff := cmp.Diff(tt.WantInventory, got, cmpopts.SortSlices(extracttest.InventoryCmpLess)); diff != "" { + t.Errorf("%s.Extract(%q) diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) + } + }) + } +} + +func TestExtractor_Extract_WithMockServer(t *testing.T) { + t.Parallel() + + tt := extracttest.TestTableEntry{ + // Name: "with parent", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/maven/with-parent.xml", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "org.alice:alice", + Version: "1.0.0", + Locations: []string{"testdata/maven/with-parent.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + { + Name: "org.bob:bob", + Version: "2.0.0", + Locations: []string{"testdata/maven/with-parent.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + { + Name: "org.chuck:chuck", + Version: "3.0.0", + Locations: []string{"testdata/maven/with-parent.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + { + Name: "org.dave:dave", + Version: "4.0.0", + Locations: []string{"testdata/maven/with-parent.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + { + Name: "org.eve:eve", + Version: "5.0.0", + Locations: []string{"testdata/maven/with-parent.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + { + Name: "org.frank:frank", + Version: "6.0.0", + Locations: []string{"testdata/maven/with-parent.xml"}, + Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, + }, + }, + } + + srv := testutility.NewMockHTTPServer(t) + srv.SetResponse(t, "org/upstream/parent-pom/1.0/parent-pom-1.0.pom", []byte(` + + org.upstream + parent-pom + 1.0 + pom + + + org.eve + eve + 5.0.0 + + + + `)) + srv.SetResponse(t, "org/import/import/1.2.3/import-1.2.3.pom", []byte(` + + org.import + import + 1.2.3 + pom + + + + org.frank + frank + 6.0.0 + + + + + `)) + + resolutionClient := clienttest.NewMockResolutionClient(t, "testdata/universe/basic-universe.yaml") + extr := pomxmlnet.Extractor{ + DependencyClient: resolutionClient, + MavenRegistryAPIClient: datasource.NewMavenRegistryAPIClient(srv.URL), + } + + scanInput := extracttest.GenerateScanInputMock(t, tt.InputConfig) + defer extracttest.CloseTestScanInput(t, scanInput) + + got, err := extr.Extract(context.Background(), &scanInput) + + if diff := cmp.Diff(tt.WantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s.Extract(%q) error diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) + return + } + + if diff := cmp.Diff(tt.WantInventory, got, cmpopts.SortSlices(extracttest.InventoryCmpLess)); diff != "" { + t.Errorf("%s.Extract(%q) diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) + } +} diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/empty.xml b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/empty.xml new file mode 100644 index 0000000000..8cfeebaaa4 --- /dev/null +++ b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/empty.xml @@ -0,0 +1,7 @@ + + 4.0.0 + + com.mycompany.app + my-app + 1 + diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/interpolation.xml b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/interpolation.xml new file mode 100644 index 0000000000..6b7f761afc --- /dev/null +++ b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/interpolation.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + + io.library + my-library + 1.0-SNAPSHOT + jar + + + 1.0.0 + 2.3.4 + [9.4.35.v20201120,9.5) + + + + + org.mine + mypackage + ${mypackageVersion} + + + + org.mine + my.package + ${my.package.version} + + + + org.mine + ranged-package + ${version-range} + + + + diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/invalid-syntax.xml b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/invalid-syntax.xml new file mode 100644 index 0000000000..761a32c1ab --- /dev/null +++ b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/invalid-syntax.xml @@ -0,0 +1,13 @@ + + + <${Id}.version>${project.version} + + + + + io.netty + netty-all + 4.1.42.Final + + + diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/not-pom.txt b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/not-pom.txt new file mode 100644 index 0000000000..f9df712bcb --- /dev/null +++ b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/not-pom.txt @@ -0,0 +1 @@ +this is not a pom.xml file! diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/one-package.xml b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/one-package.xml new file mode 100644 index 0000000000..bbb1359e9d --- /dev/null +++ b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/one-package.xml @@ -0,0 +1,17 @@ + + com.mycompany.app + my-app + 1.0 + + + 3.0 + + + + + org.apache.maven + maven-artifact + 1.0.0 + + + diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/parent/pom.xml b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/parent/pom.xml new file mode 100644 index 0000000000..3751df6be3 --- /dev/null +++ b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/parent/pom.xml @@ -0,0 +1,21 @@ + + org.local + parent-pom + 1.0 + + pom + + + org.upstream + parent-pom + 1.0 + + + + + org.dave + dave + 4.0.0 + + + diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/transitive.xml b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/transitive.xml new file mode 100644 index 0000000000..52e416a0bc --- /dev/null +++ b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/transitive.xml @@ -0,0 +1,33 @@ + + com.mycompany.app + my-app + 1.0 + + + + + org.transitive + frank + 4.4.4 + + + + + + + org.direct + alice + 1.0.0 + + + org.direct + bob + 2.0.0 + + + org.direct + chris + 3.0.0 + + + diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/two-packages.xml b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/two-packages.xml new file mode 100644 index 0000000000..897f648a1e --- /dev/null +++ b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/two-packages.xml @@ -0,0 +1,22 @@ + + com.mycompany.app + my-app + 1.0 + + + 3.0 + + + + + io.netty + netty-all + 4.1.42.Final + + + org.slf4j + slf4j-log4j12 + 1.7.25 + + + diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/with-dependency-management.xml b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/with-dependency-management.xml new file mode 100644 index 0000000000..1928688e94 --- /dev/null +++ b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/with-dependency-management.xml @@ -0,0 +1,37 @@ + + com.mycompany.app + my-app + 1.0 + + + 3.0 + + + + + io.netty + netty-all + 4.1.9 + + + org.slf4j + slf4j-log4j12 + 1.7.25 + + + + + + + io.netty + netty-all + 4.1.42.Final + + + com.google.code.findbugs + jsr305 + 3.0.2 + + + + diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/with-parent.xml b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/with-parent.xml new file mode 100644 index 0000000000..602b8b877f --- /dev/null +++ b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/with-parent.xml @@ -0,0 +1,54 @@ + + com.mycompany.app + my-app + 1.0 + + + org.local + parent-pom + 1.0 + ./parent/pom.xml + + + + 2.0.0 + + + + + org.alice + alice + 1.0.0 + + + org.bob + bob + ${bob.version} + + + org.chuck + chuck + + + org.frank + frank + + + + + + + org.chuck + chuck + 3.0.0 + + + org.import + import + 1.2.3 + pom + import + + + + diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/with-scope.xml b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/with-scope.xml new file mode 100644 index 0000000000..688c6bb7bc --- /dev/null +++ b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/with-scope.xml @@ -0,0 +1,14 @@ + + com.mycompany.app + my-app + 1.0 + + + + junit + junit + 4.12 + runtime + + + diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/universe/basic-universe.yaml b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/universe/basic-universe.yaml new file mode 100644 index 0000000000..2bf2b32724 --- /dev/null +++ b/internal/lockfilescalibr/language/java/pomxmlnet/testdata/universe/basic-universe.yaml @@ -0,0 +1,60 @@ +system: maven +schema: | + com.google.code.findbugs:jsr305 + 3.0.2 + io.netty:netty-all + 4.1.9 + 4.1.42.Final + junit:junit + 4.12 + org.alice:alice + 1.0.0 + org.apache.maven:maven-artifact + 1.0.0 + org.bob:bob + 2.0.0 + org.chuck:chuck + 3.0.0 + org.dave:dave + 4.0.0 + org.direct:alice + 1.0.0 + org.transitive:chuck@1.1.1 + org.transitive:dave@2.2.2 + org.direct:bob + 2.0.0 + org.transitive:eve@3.3.3 + org.direct:chris + 3.0.0 + org.transitive:frank@3.3.3 + org.eve:eve + 5.0.0 + org.frank:frank + 6.0.0 + org.mine:my.package + 2.3.4 + org.mine:mypackage + 1.0.0 + org.mine:ranged-package + 9.4.35 + 9.4.36 + 9.4.37 + 9.5 + org.slf4j:slf4j-log4j12 + 1.7.25 + org.transitive:chuck + 1.1.1 + 2.2.2 + org.transitive:eve@2.2.2 + 3.3.3 + org.transitive:dave + 1.1.1 + 2.2.2 + 3.3.3 + org.transitive:eve + 1.1.1 + 2.2.2 + 3.3.3 + org.transitive:frank + 3.3.3 + 4.4.4 diff --git a/internal/lockfilescalibr/language/osv/osvscannerjson/extractor.go b/internal/lockfilescalibr/language/osv/osvscannerjson/extractor.go new file mode 100644 index 0000000000..74002b4b20 --- /dev/null +++ b/internal/lockfilescalibr/language/osv/osvscannerjson/extractor.go @@ -0,0 +1,87 @@ +// Package osvscannerjson extracts osv-scanner's json output. +package osvscannerjson + +import ( + "context" + "encoding/json" + "fmt" + "io/fs" + + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/plugin" + "github.com/google/osv-scalibr/purl" + "github.com/google/osv-scanner/pkg/models" +) + +// Extractor extracts osv packages from osv-scanner json output. +type Extractor struct{} + +// Name of the extractor. +func (e Extractor) Name() string { return "osv/osvscannerjson" } + +// Version of the extractor. +func (e Extractor) Version() int { return 0 } + +// Requirements of the extractor. +func (e Extractor) Requirements() *plugin.Capabilities { + return &plugin.Capabilities{} +} + +// FileRequired never returns true, as this is for the osv-scanner json output. +func (e Extractor) FileRequired(_ string, _ fs.FileInfo) bool { + return false +} + +// Extract extracts packages from yarn.lock files passed through the scan input. +func (e Extractor) Extract(_ context.Context, input *filesystem.ScanInput) ([]*extractor.Inventory, error) { + parsedResults := models.VulnerabilityResults{} + err := json.NewDecoder(input.Reader).Decode(&parsedResults) + + if err != nil { + return nil, fmt.Errorf("could not extract from %s: %w", input.Path, err) + } + + packages := []*extractor.Inventory{} + for _, res := range parsedResults.Results { + for _, pkg := range res.Packages { + inventory := extractor.Inventory{ + Name: pkg.Package.Name, + Version: pkg.Package.Version, + Metadata: Metadata{ + Ecosystem: pkg.Package.Ecosystem, + SourceInfo: res.Source, + }, + Locations: []string{input.Path}, + } + if pkg.Package.Commit != "" { + inventory.SourceCode = &extractor.SourceCodeIdentifier{ + Commit: pkg.Package.Commit, + } + } + packages = append(packages, &inventory) + + } + } + + return packages, nil +} + +// ToPURL converts an inventory created by this extractor into a PURL. +func (e Extractor) ToPURL(i *extractor.Inventory) (*purl.PackageURL, error) { + return &purl.PackageURL{ + Type: purl.TypeNPM, + Name: i.Name, + Version: i.Version, + }, nil +} + +// ToCPEs is not applicable as this extractor does not infer CPEs from the Inventory. +func (e Extractor) ToCPEs(i *extractor.Inventory) ([]string, error) { return []string{}, nil } + +// Ecosystem returns the OSV ecosystem ('npm') of the software extracted by this extractor. +func (e Extractor) Ecosystem(i *extractor.Inventory) string { + return i.Metadata.(Metadata).Ecosystem +} + +var _ filesystem.Extractor = Extractor{} diff --git a/internal/lockfilescalibr/language/osv/osvscannerjson/extractor_test.go b/internal/lockfilescalibr/language/osv/osvscannerjson/extractor_test.go new file mode 100644 index 0000000000..73a904a392 --- /dev/null +++ b/internal/lockfilescalibr/language/osv/osvscannerjson/extractor_test.go @@ -0,0 +1,137 @@ +package osvscannerjson_test + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/testing/extracttest" + "github.com/google/osv-scanner/internal/lockfilescalibr/language/osv/osvscannerjson" + "github.com/google/osv-scanner/pkg/models" +) + +func TestExtractor_Extract(t *testing.T) { + tests := []extracttest.TestTableEntry{ + { + Name: "invalid yaml", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/not-json.txt", + }, + WantErr: extracttest.ContainsErrStr{Str: "could not extract from"}, + }, + { + Name: "empty", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/empty.json", + }, + WantInventory: []*extractor.Inventory{}, + }, + { + Name: "one package", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/one-package.json", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "activesupport", + Version: "7.0.7", + Locations: []string{"testdata/one-package.json"}, + Metadata: osvscannerjson.Metadata{ + Ecosystem: "RubyGems", + SourceInfo: models.SourceInfo{ + Path: "/path/to/Gemfile.lock", + Type: "lockfile", + }, + }, + }, + }, + }, + { + Name: "one package with commit", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/one-package-commit.json", + }, + WantInventory: []*extractor.Inventory{ + { + Locations: []string{"testdata/one-package-commit.json"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "9a6bd55c9d0722cb101fe85a3b22d89e4ff4fe52", + }, + Metadata: osvscannerjson.Metadata{ + SourceInfo: models.SourceInfo{ + Path: "/path/to/Gemfile.lock", + Type: "lockfile", + }, + }, + }, + }, + }, + { + Name: "multiple packages", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/multiple-packages-with-vulns.json", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "crossbeam-utils", + Version: "0.6.6", + Locations: []string{"testdata/multiple-packages-with-vulns.json"}, + Metadata: osvscannerjson.Metadata{ + Ecosystem: "crates.io", + SourceInfo: models.SourceInfo{ + Path: "/path/to/Cargo.lock", + Type: "lockfile", + }, + }, + }, + { + Name: "memoffset", + Version: "0.5.6", + Locations: []string{"testdata/multiple-packages-with-vulns.json"}, + Metadata: osvscannerjson.Metadata{ + Ecosystem: "crates.io", + SourceInfo: models.SourceInfo{ + Path: "/path/to/Cargo.lock", + Type: "lockfile", + }, + }, + }, + { + Name: "smallvec", + Version: "1.6.0", + Locations: []string{"testdata/multiple-packages-with-vulns.json"}, + Metadata: osvscannerjson.Metadata{ + Ecosystem: "crates.io", + SourceInfo: models.SourceInfo{ + Path: "/path/to/Cargo.lock", + Type: "lockfile", + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + t.Parallel() + extr := osvscannerjson.Extractor{} + + scanInput := extracttest.GenerateScanInputMock(t, tt.InputConfig) + defer extracttest.CloseTestScanInput(t, scanInput) + + got, err := extr.Extract(context.Background(), &scanInput) + + if diff := cmp.Diff(tt.WantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s.Extract(%q) error diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) + return + } + + if diff := cmp.Diff(tt.WantInventory, got, cmpopts.SortSlices(extracttest.InventoryCmpLess)); diff != "" { + t.Errorf("%s.Extract(%q) diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) + } + }) + } +} diff --git a/internal/lockfilescalibr/language/osv/osvscannerjson/metadata.go b/internal/lockfilescalibr/language/osv/osvscannerjson/metadata.go new file mode 100644 index 0000000000..45c9e2c966 --- /dev/null +++ b/internal/lockfilescalibr/language/osv/osvscannerjson/metadata.go @@ -0,0 +1,9 @@ +package osvscannerjson + +import "github.com/google/osv-scanner/pkg/models" + +// Metadata holds the metadata for osvscanner.json +type Metadata struct { + Ecosystem string + SourceInfo models.SourceInfo +} diff --git a/pkg/lockfile/fixtures/osvscannerresults/empty.json b/internal/lockfilescalibr/language/osv/osvscannerjson/testdata/empty.json similarity index 100% rename from pkg/lockfile/fixtures/osvscannerresults/empty.json rename to internal/lockfilescalibr/language/osv/osvscannerjson/testdata/empty.json diff --git a/pkg/lockfile/fixtures/osvscannerresults/multi-packages-with-vulns.json b/internal/lockfilescalibr/language/osv/osvscannerjson/testdata/multiple-packages-with-vulns.json similarity index 100% rename from pkg/lockfile/fixtures/osvscannerresults/multi-packages-with-vulns.json rename to internal/lockfilescalibr/language/osv/osvscannerjson/testdata/multiple-packages-with-vulns.json diff --git a/pkg/lockfile/fixtures/osvscannerresults/not-json.txt b/internal/lockfilescalibr/language/osv/osvscannerjson/testdata/not-json.txt similarity index 100% rename from pkg/lockfile/fixtures/osvscannerresults/not-json.txt rename to internal/lockfilescalibr/language/osv/osvscannerjson/testdata/not-json.txt diff --git a/pkg/lockfile/fixtures/osvscannerresults/one-package-commit.json b/internal/lockfilescalibr/language/osv/osvscannerjson/testdata/one-package-commit.json similarity index 100% rename from pkg/lockfile/fixtures/osvscannerresults/one-package-commit.json rename to internal/lockfilescalibr/language/osv/osvscannerjson/testdata/one-package-commit.json diff --git a/pkg/lockfile/fixtures/osvscannerresults/one-package.json b/internal/lockfilescalibr/language/osv/osvscannerjson/testdata/one-package.json similarity index 100% rename from pkg/lockfile/fixtures/osvscannerresults/one-package.json rename to internal/lockfilescalibr/language/osv/osvscannerjson/testdata/one-package.json diff --git a/internal/lockfilescalibr/translation.go b/internal/lockfilescalibr/translation.go new file mode 100644 index 0000000000..b4a67a81c3 --- /dev/null +++ b/internal/lockfilescalibr/translation.go @@ -0,0 +1,212 @@ +package lockfilescalibr + +import ( + "context" + "fmt" + "io/fs" + "os" + "sort" + + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/extractor/filesystem/language/dart/pubspec" + "github.com/google/osv-scalibr/extractor/filesystem/language/dotnet/packageslockjson" + "github.com/google/osv-scalibr/extractor/filesystem/language/erlang/mixlock" + "github.com/google/osv-scalibr/extractor/filesystem/language/golang/gomod" + "github.com/google/osv-scalibr/extractor/filesystem/language/java/gradlelockfile" + "github.com/google/osv-scalibr/extractor/filesystem/language/java/gradleverificationmetadataxml" + "github.com/google/osv-scalibr/extractor/filesystem/language/java/pomxml" + "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/packagelockjson" + "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/pnpmlock" + "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/yarnlock" + "github.com/google/osv-scalibr/extractor/filesystem/language/php/composerlock" + "github.com/google/osv-scalibr/extractor/filesystem/language/python/pdmlock" + "github.com/google/osv-scalibr/extractor/filesystem/language/python/pipfilelock" + "github.com/google/osv-scalibr/extractor/filesystem/language/python/poetrylock" + "github.com/google/osv-scalibr/extractor/filesystem/language/python/requirements" + "github.com/google/osv-scalibr/extractor/filesystem/language/r/renvlock" + "github.com/google/osv-scalibr/extractor/filesystem/language/ruby/gemfilelock" + "github.com/google/osv-scalibr/extractor/filesystem/language/rust/cargolock" + + scalibrfs "github.com/google/osv-scalibr/fs" +) + +// type PackageDetailsParser = func(pathToLockfile string) ([]PackageDetails, error) + +// IsDevGroup returns if any string in groups indicates the development dependency group for the specified ecosystem. +// func (sys Ecosystem) IsDevGroup(groups []string) bool { +// dev := "" +// switch sys { +// case ComposerEcosystem, NpmEcosystem, PipEcosystem, PubEcosystem: +// // Also PnpmEcosystem(=NpmEcosystem) and PipenvEcosystem(=PipEcosystem). +// dev = "dev" +// case ConanEcosystem: +// dev = "build-requires" +// case MavenEcosystem: +// dev = "test" +// case AlpineEcosystem, BundlerEcosystem, CargoEcosystem, CRANEcosystem, +// DebianEcosystem, GoEcosystem, MixEcosystem, NuGetEcosystem: +// // We are not able to report development dependencies for these ecosystems. +// return false +// } + +// for _, g := range groups { +// if g == dev { +// return true +// } +// } + +// return false +// } + +var lockfileExtractors = []filesystem.Extractor{ + // conanlock.Extractor{}, + packageslockjson.Extractor{}, + mixlock.Extractor{}, + pubspec.Extractor{}, + gomod.Extractor{}, + pomxml.Extractor{}, + gradlelockfile.Extractor{}, + gradleverificationmetadataxml.Extractor{}, + packagelockjson.Extractor{}, + pnpmlock.Extractor{}, + yarnlock.Extractor{}, + composerlock.Extractor{}, + pipfilelock.Extractor{}, + pdmlock.Extractor{}, + poetrylock.Extractor{}, + requirements.Extractor{}, + renvlock.Extractor{}, + gemfilelock.Extractor{}, + cargolock.Extractor{}, +} + +var lockfileExtractorMapping = map[string]string{ + "pubspec.lock": "dart/pubspec", + "pnpm-lock.yaml": "javascript/pnpmlock", + "yarn.lock": "javascript/yarnlock", + "package-lock.json": "javascript/packagelockjson", + "pom.xml": "java/pomxml", + "buildscript-gradle.lockfile": "java/gradlelockfile", + "gradle.lockfile": "java/gradlelockfile", + "verification-metadata.xml": "java/gradleverificationmetadataxml", + "poetry.lock": "python/poetrylock", + "Pipfile.lock": "python/Pipfilelock", + "pdm.lock": "python/pdmlock", + "requirements.txt": "python/requirements", + "Cargo.lock": "rust/Cargolock", + "composer.lock": "php/composerlock", + "mix.lock": "erlang/mixlock", + "renv.lock": "r/renvlock", + "packages.lock.json": "dotnet/packageslockjson", + // "conan.lock": "cpp/conanlock", + "go.mod": "go/gomod", + "Gemfile.lock": "ruby/gemfilelock", +} + +func ExtractWithExtractor(ctx context.Context, localPath string, ext filesystem.Extractor) ([]*extractor.Inventory, error) { + info, err := os.Stat(localPath) + if err != nil { + return nil, err + } + + si, err := createScanInput(localPath, info) + if err != nil { + return nil, err + } + inv, err := ext.Extract(ctx, si) + if err != nil { + return nil, fmt.Errorf("(extracting as %s) %w", ext.Name(), err) + } + + for i := range inv { + inv[i].Extractor = ext + } + + return inv, nil +} + +func Extract(ctx context.Context, localPath string, extractAs string) ([]*extractor.Inventory, error) { + info, err := os.Stat(localPath) + if err != nil { + return nil, err + } + + if extractAs != "" { + for _, ext := range lockfileExtractors { + if lockfileExtractorMapping[extractAs] == ext.Name() { + si, err := createScanInput(localPath, info) + if err != nil { + return nil, err + } + + inv, err := ext.Extract(ctx, si) + if err != nil { + return nil, fmt.Errorf("(extracting as %s) %w", extractAs, err) + } + + for i := range inv { + inv[i].Extractor = ext + } + + return inv, nil + } + } + + return nil, fmt.Errorf("%w, requested %s", ErrExtractorNotFound, extractAs) + } + + output := []*extractor.Inventory{} + extractorFound := false + + for _, ext := range lockfileExtractors { + if ext.FileRequired(localPath, info) { + extractorFound = true + si, err := createScanInput(localPath, info) + if err != nil { + return nil, err + } + + inv, err := ext.Extract(ctx, si) + if err != nil { + return nil, fmt.Errorf("(extracting as %s) %w", ext.Name(), err) + } + + for i := range inv { + inv[i].Extractor = ext + } + output = append(output, inv...) + } + } + + if !extractorFound { + return nil, ErrNoExtractorsFound + } + + sort.Slice(output, func(i, j int) bool { + if output[i].Name == output[j].Name { + return output[i].Version < output[j].Version + } + + return output[i].Name < output[j].Name + }) + + return output, nil +} + +func createScanInput(path string, fileInfo fs.FileInfo) (*filesystem.ScanInput, error) { + reader, err := os.Open(path) + if err != nil { + return nil, err + } + + si := filesystem.ScanInput{ + FS: os.DirFS("/").(scalibrfs.FS), + Path: path, + Root: "/", + Reader: reader, + Info: fileInfo, + } + + return &si, nil +} diff --git a/internal/lockfilescalibr/translation_test.go b/internal/lockfilescalibr/translation_test.go new file mode 100644 index 0000000000..b374cd3396 --- /dev/null +++ b/internal/lockfilescalibr/translation_test.go @@ -0,0 +1,21 @@ +package lockfilescalibr + +import ( + "testing" +) + +func TestLockfileScalibrMappingExists(t *testing.T) { + for _, target := range lockfileExtractorMapping { + found := false + for _, ext := range lockfileExtractors { + if target == ext.Name() { + found = true + break + } + } + + if !found { + t.Errorf("Extractor %v not found.", target) + } + } +} diff --git a/internal/manifest/maven_test.go b/internal/manifest/maven_test.go index 882affb701..fb393b8877 100644 --- a/internal/manifest/maven_test.go +++ b/internal/manifest/maven_test.go @@ -11,7 +11,7 @@ import ( "github.com/google/osv-scanner/pkg/lockfile" ) -func TestMavenResolverExtractor_ShouldExtract(t *testing.T) { +func TestMavenResolverExtractor_FileRequired(t *testing.T) { t.Parallel() tests := []struct { diff --git a/internal/remediation/fixtures/santatracker/osv-scanner.toml b/internal/remediation/fixtures/santatracker/osv-scanner.toml index b399bb4c28..db94704b26 100644 --- a/internal/remediation/fixtures/santatracker/osv-scanner.toml +++ b/internal/remediation/fixtures/santatracker/osv-scanner.toml @@ -1,191 +1,4 @@ [[PackageOverrides]] -name = "@babel/traverse" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "@grpc/grpc-js" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "acorn" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "ajv" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "ansi-regex" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "braces" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "browserslist" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "dat.gui" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "get-func-name" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "glob-parent" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "google-closure-library" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "html-minifier" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "json-schema" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "json5" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "lodash" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "minimatch" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "minimist" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "node-fetch" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "node-forge " -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "node-forge" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "path-parse" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "pathval" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "postcss" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "protobufjs" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "qs" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "request" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "semver" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "terser" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "tough-cookie" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "ws" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "y18n" -ecosystem = "npm" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "yargs-parser" ecosystem = "npm" ignore = true reason = "This is an intentionally vulnerable test project" diff --git a/internal/remediation/fixtures/zeppelin-server/osv-scanner.toml b/internal/remediation/fixtures/zeppelin-server/osv-scanner.toml index 250f7b7530..d84c70b89e 100644 --- a/internal/remediation/fixtures/zeppelin-server/osv-scanner.toml +++ b/internal/remediation/fixtures/zeppelin-server/osv-scanner.toml @@ -1,143 +1,4 @@ [[PackageOverrides]] -name = "com.fasterxml.jackson.core:jackson-databind" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "com.google.guava:guava" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "com.jcraft:jsch" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "com.nimbusds:nimbus-jose-jwt" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "io.atomix:atomix" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "io.netty:netty-codec" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "io.netty:netty-handler" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "org.apache.commons:commons-compress" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "org.apache.commons:commons-configuration2" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "org.apache.directory.api:api-ldap-model" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "org.apache.mina:mina-core" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "org.apache.pdfbox:pdfbox" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "org.apache.shiro:shiro-core" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "org.apache.shiro:shiro-web" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "org.apache.thrift:libthrift" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "org.bouncycastle:bcprov-jdk15on" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "org.codehaus.jackson:jackson-mapper-asl" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "org.eclipse.jgit:org.eclipse.jgit" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "org.glassfish.jersey.core:jersey-common" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "com.google.code.gson:gson" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "commons-collections:commons-collections" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "org.apache.httpcomponents:httpclient" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "org.eclipse.jetty:jetty-webapp" -ecosystem = "Maven" -ignore = true -reason = "This is an intentionally vulnerable test project" - -[[PackageOverrides]] -name = "org.quartz-scheduler:quartz" ecosystem = "Maven" ignore = true reason = "This is an intentionally vulnerable test project" diff --git a/internal/utility/purl/composer_test.go b/internal/utility/purl/composer_test.go index 65139f7550..8e671a3b65 100644 --- a/internal/utility/purl/composer_test.go +++ b/internal/utility/purl/composer_test.go @@ -8,7 +8,7 @@ import ( "github.com/google/osv-scanner/pkg/models" ) -func TestComposerExtraction_shouldExtractPackages(t *testing.T) { +func TestComposerExtraction_FileRequiredPackages(t *testing.T) { t.Parallel() testCase := struct { packageInfo models.PackageInfo diff --git a/internal/utility/purl/golang_test.go b/internal/utility/purl/golang_test.go index 65d8ce6e93..77440517ae 100644 --- a/internal/utility/purl/golang_test.go +++ b/internal/utility/purl/golang_test.go @@ -8,7 +8,7 @@ import ( "github.com/google/osv-scanner/pkg/models" ) -func TestGolangExtraction_shouldExtractPackages(t *testing.T) { +func TestGolangExtraction_FileRequiredPackages(t *testing.T) { t.Parallel() testCases := []struct { name string diff --git a/internal/utility/purl/maven_test.go b/internal/utility/purl/maven_test.go index fbc2dae94b..9ef86d2fea 100644 --- a/internal/utility/purl/maven_test.go +++ b/internal/utility/purl/maven_test.go @@ -8,7 +8,7 @@ import ( "github.com/google/osv-scanner/pkg/models" ) -func TestMavenExtraction_shouldExtractPackages(t *testing.T) { +func TestMavenExtraction_FileRequiredPackages(t *testing.T) { t.Parallel() testCase := struct { packageInfo models.PackageInfo diff --git a/pkg/lockfile/another.py b/pkg/lockfile/another.py new file mode 100644 index 0000000000..8ecc1996b8 --- /dev/null +++ b/pkg/lockfile/another.py @@ -0,0 +1,106 @@ +""" + +Helper script, remember to remove later!! +""" + +import sys +import re +from dataclasses import dataclass + +filename = sys.argv[1] + +file = open(filename) +allLines = file.readlines() + +output = "" +extractorName = "" +extractorFileName = "" + +insertLine = """ +// Name of the extractor +func (e [-extractname]) Name() string { return "go/gomod" } + +// Version of the extractor +func (e [-extractname]) Version() int { return 0 } + +func (e [-extractname]) Requirements() Requirements { + return Requirements{} +} +""" + +insertLineTwo = """ +func (e [-extractname]) FileRequired(path string, fileInfo fs.FileInfo) bool { + return filepath.Base(path) == "[-extractFileName]" +} +""" + +replaceLineThree = "func (e [-extractname]) Extract(ctx context.Context, input *ScanInput) ([]*Inventory, error) {" + +insertLineFour = """ + +// ToPURL converts an inventory created by this extractor into a PURL. +func (e [-extractname]) ToPURL(i *Inventory) (*packageurl.PackageURL, error) { + return &packageurl.PackageURL{ + Type: packageurl.--, + Name: i.Name, + Version: i.Version, + + }, nil +} + +// ToCPEs is not applicable as this extractor does not infer CPEs from the Inventory. +func (e [-extractname]) ToCPEs(i *Inventory) ([]string, error) { return []string{}, nil } + +func (e [-extractname]) Ecosystem(i *Inventory) (string, error) { + switch i.Extractor.(type) { + case [-extractname]: + return string(--), nil + default: + return "", ErrWrongExtractor + } +} +""" + +removeNextLines = 0 + +for line in allLines: + matchOne = re.match("type ([a-zA-Z]+Extractor) struct{}", line) + if matchOne: + extractorName = matchOne.group(1) + output += line + output += insertLine.replace("[-extractname]", extractorName) + continue + + matchTwo = re.match("func .* ShouldExtract\\(path.* bool", line) + if matchTwo: + continue + + matchThree = re.match('.*return filepath\\.Base\\(path\\) == \\"(.*?)\\"', line) + + if matchThree: + print("YOOO") + extractorFileName = matchThree.group(1) + output += insertLineTwo.replace("[-extractname]", extractorName).replace("[-extractFileName]", extractorFileName) + removeNextLines = 1 + continue + + matchFour = re.match('func \\(.* Extract\\(f DepFile\\)', line) + if matchFour: + output += replaceLineThree.replace("[-extractname]", extractorName) + continue + + matchFive = re.match('var _ Extractor = ', line) + if matchFive: + output += insertLineFour.replace("[-extractname]", extractorName) + output += line + continue + + if removeNextLines > 0: + removeNextLines -= 1 + else: + output += line.replace("(f)", "(input.Reader)").replace("f.Path()", "input.Path").replace("PackageDetails", "*Inventory") + + +f = open(filename, "w") +f.write(output) +f.close() diff --git a/pkg/lockfile/apk-installed.go b/pkg/lockfile/apk-installed.go deleted file mode 100644 index 184f7213aa..0000000000 --- a/pkg/lockfile/apk-installed.go +++ /dev/null @@ -1,149 +0,0 @@ -package lockfile - -import ( - "bufio" - "fmt" - "io" - "sort" - "strings" -) - -const AlpineEcosystem Ecosystem = "Alpine" -const AlpineFallbackVersion = "v3.20" - -func groupApkPackageLines(scanner *bufio.Scanner) [][]string { - var groups [][]string - var group []string - - for scanner.Scan() { - line := scanner.Text() - - if line != "" { - group = append(group, line) - continue - } - if len(group) > 0 { - groups = append(groups, group) - } - group = make([]string, 0) - } - - if len(group) > 0 { - groups = append(groups, group) - } - - return groups -} - -func parseApkPackageGroup(group []string) PackageDetails { - var pkg = PackageDetails{ - Ecosystem: AlpineEcosystem, - CompareAs: AlpineEcosystem, - } - - // File SPECS: https://wiki.alpinelinux.org/wiki/Apk_spec - for _, line := range group { - switch { - case strings.HasPrefix(line, "P:"): - pkg.Name = strings.TrimPrefix(line, "P:") - case strings.HasPrefix(line, "V:"): - pkg.Version = strings.TrimPrefix(line, "V:") - case strings.HasPrefix(line, "c:"): - pkg.Commit = strings.TrimPrefix(line, "c:") - } - } - - return pkg -} - -// Deprecated: use ApkInstalledExtractor.Extract instead -func ParseApkInstalled(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, ApkInstalledExtractor{}) -} - -type ApkInstalledExtractor struct{} - -func (e ApkInstalledExtractor) ShouldExtract(path string) bool { - return path == "/lib/apk/db/installed" -} - -func (e ApkInstalledExtractor) Extract(f DepFile) ([]PackageDetails, error) { - scanner := bufio.NewScanner(f) - - packageGroups := groupApkPackageLines(scanner) - - packages := make([]PackageDetails, 0, len(packageGroups)) - - for _, group := range packageGroups { - pkg := parseApkPackageGroup(group) - - if pkg.Name == "" { - continue - } - - packages = append(packages, pkg) - } - - alpineVersion, alpineVerErr := alpineReleaseExtractor(f) - if alpineVerErr != nil { // TODO: Log error? We might not be on a alpine system - // Alpine ecosystems MUST have a version suffix. Fallback to the latest version. - alpineVersion = AlpineFallbackVersion - } - for i := range packages { - packages[i].Ecosystem = Ecosystem(string(packages[i].Ecosystem) + ":" + alpineVersion) - } - - if err := scanner.Err(); err != nil { - return packages, fmt.Errorf("error while scanning %s: %w", f.Path(), err) - } - - return packages, nil -} - -// alpineReleaseExtractor extracts the release version for an alpine distro -// will return "" if no release version can be found, or if distro is not alpine -func alpineReleaseExtractor(opener DepFile) (string, error) { - alpineReleaseFile, err := opener.Open("/etc/alpine-release") - if err != nil { - return "", err - } - defer alpineReleaseFile.Close() - - // Read to string - buf := new(strings.Builder) - _, err = io.Copy(buf, alpineReleaseFile) - if err != nil { - return "", err - } - - // We only care about the major and minor version - // because that's the Alpine version that advisories are published against - // - // E.g. 3.20.0_alpha20231219 ---> v3.20 - valueSplit := strings.Split(buf.String(), ".") - returnVersion := "v" + valueSplit[0] + "." + valueSplit[1] - - return returnVersion, nil -} - -var _ Extractor = ApkInstalledExtractor{} - -// FromApkInstalled attempts to parse the given file as an "apk-installed" lockfile -// used by the Alpine Package Keeper (apk) to record installed packages. -func FromApkInstalled(pathToInstalled string) (Lockfile, error) { - packages, err := ParseApkInstalled(pathToInstalled) - - sort.Slice(packages, func(i, j int) bool { - if packages[i].Name == packages[j].Name { - return packages[i].Version < packages[j].Version - } - - return packages[i].Name < packages[j].Name - }) - - return Lockfile{ - FilePath: pathToInstalled, - ParsedAs: "apk-installed", - Packages: packages, - }, err -} diff --git a/pkg/lockfile/apk-installed_test.go b/pkg/lockfile/apk-installed_test.go deleted file mode 100644 index 852d2e4d56..0000000000 --- a/pkg/lockfile/apk-installed_test.go +++ /dev/null @@ -1,137 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -const alpineEcosystem = lockfile.AlpineEcosystem + ":" + lockfile.AlpineFallbackVersion - -func TestParseApkInstalled_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseApkInstalled("fixtures/apk/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseApkInstalled_Empty(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseApkInstalled("fixtures/apk/empty_installed") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseApkInstalled_NotAnInstalled(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseApkInstalled("fixtures/apk/not_installed") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseApkInstalled_Malformed(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseApkInstalled("fixtures/apk/malformed_installed") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "busybox", - Version: "", - Commit: "1dbf7a793afae640ea643a055b6dd4f430ac116b", - Ecosystem: alpineEcosystem, - CompareAs: lockfile.AlpineEcosystem, - }, - }) -} - -func TestParseApkInstalled_Single(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseApkInstalled("fixtures/apk/single_installed") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "apk-tools", - Version: "2.12.10-r1", - Commit: "0188f510baadbae393472103427b9c1875117136", - Ecosystem: alpineEcosystem, - CompareAs: lockfile.AlpineEcosystem, - }, - }) -} - -func TestParseApkInstalled_Shuffled(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseApkInstalled("fixtures/apk/shuffled_installed") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "apk-tools", - Version: "2.12.10-r1", - Commit: "0188f510baadbae393472103427b9c1875117136", - Ecosystem: alpineEcosystem, - CompareAs: lockfile.AlpineEcosystem, - }, - }) -} - -func TestParseApkInstalled_Multiple(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseApkInstalled("fixtures/apk/multiple_installed") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "alpine-baselayout-data", - Version: "3.4.0-r0", - Commit: "bd965a7ebf7fd8f07d7a0cc0d7375bf3e4eb9b24", - Ecosystem: alpineEcosystem, - CompareAs: lockfile.AlpineEcosystem, - }, - { - Name: "musl", - Version: "1.2.3-r4", - Commit: "f93af038c3de7146121c2ea8124ba5ce29b4b058", - Ecosystem: alpineEcosystem, - CompareAs: lockfile.AlpineEcosystem, - }, - { - Name: "busybox", - Version: "1.35.0-r29", - Commit: "1dbf7a793afae640ea643a055b6dd4f430ac116b", - Ecosystem: alpineEcosystem, - CompareAs: lockfile.AlpineEcosystem, - }, - }) -} diff --git a/pkg/lockfile/csv.go b/pkg/lockfile/csv.go deleted file mode 100644 index 949e269f5d..0000000000 --- a/pkg/lockfile/csv.go +++ /dev/null @@ -1,129 +0,0 @@ -package lockfile - -import ( - "encoding/csv" - "errors" - "fmt" - "io" - "sort" - "strings" -) - -var errCSVRecordNotEnoughFields = errors.New("not enough fields (expected at least four)") -var errCSVRecordMissingPackageField = errors.New("field 3 is empty (must be the name of a package)") -var errCSVRecordMissingCommitField = errors.New("field 4 is empty (must be a commit)") - -func fromCSVRecord(lines []string) (PackageDetails, error) { - if len(lines) < 4 { - return PackageDetails{}, errCSVRecordNotEnoughFields - } - - ecosystem := Ecosystem(lines[0]) - compareAs := Ecosystem(lines[1]) - name := lines[2] - version := lines[3] - commit := "" - - if compareAs == "" { - compareAs = ecosystem - } - - if ecosystem == "" { - if version == "" { - return PackageDetails{}, errCSVRecordMissingCommitField - } - - commit = version - version = "" - } - - if name == "" { - return PackageDetails{}, errCSVRecordMissingPackageField - } - - return PackageDetails{ - Name: name, - Version: version, - Ecosystem: ecosystem, - CompareAs: compareAs, - Commit: commit, - }, nil -} - -func fromCSV(reader io.Reader) ([]PackageDetails, error) { - var packages []PackageDetails - - i := 0 - r := csv.NewReader(reader) - - for { - i++ - record, err := r.Read() - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return packages, fmt.Errorf("%w", err) - } - - details, err := fromCSVRecord(record) - if err != nil { - return packages, fmt.Errorf("row %d: %w", i, err) - } - - packages = append(packages, details) - } - - sort.Slice(packages, func(i, j int) bool { - if packages[i].Name == packages[j].Name { - return packages[i].Version < packages[j].Version - } - - return packages[i].Name < packages[j].Name - }) - - return packages, nil -} - -type CSVExtractor struct{} - -func (e CSVExtractor) ShouldExtract(_ string) bool { - // the csv extractor should never implicitly extract a file - return false -} - -func (e CSVExtractor) Extract(f DepFile) ([]PackageDetails, error) { - return fromCSV(f) -} - -var _ Extractor = CSVExtractor{} - -func FromCSVRows(filePath string, parseAs string, rows []string) (Lockfile, error) { - packages, err := fromCSV(strings.NewReader(strings.Join(rows, "\n"))) - - return Lockfile{ - FilePath: filePath, - ParsedAs: parseAs, - Packages: packages, - }, err -} - -func FromCSVFile(pathToCSV string, parseAs string) (Lockfile, error) { - file, err := OpenLocalDepFile(pathToCSV) - if err != nil { - return Lockfile{}, fmt.Errorf("could not read %s: %w", pathToCSV, err) - } - defer file.Close() - - packages, err := fromCSV(file) - - if err != nil { - err = fmt.Errorf("%s: %w", pathToCSV, err) - } - - return Lockfile{ - FilePath: pathToCSV, - ParsedAs: parseAs, - Packages: packages, - }, err -} diff --git a/pkg/lockfile/csv_test.go b/pkg/lockfile/csv_test.go deleted file mode 100644 index e0af55a2bf..0000000000 --- a/pkg/lockfile/csv_test.go +++ /dev/null @@ -1,545 +0,0 @@ -package lockfile_test - -import ( - "reflect" - "strings" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestFromCSVRows(t *testing.T) { - t.Parallel() - - type args struct { - filePath string - parseAs string - rows []string - } - tests := []struct { - name string - args args - want lockfile.Lockfile - }{ - { - name: "", - args: args{ - filePath: "-", - parseAs: "-", - rows: nil, - }, - want: lockfile.Lockfile{ - FilePath: "-", - ParsedAs: "-", - Packages: nil, - }, - }, - { - name: "", - args: args{ - filePath: "-", - parseAs: "csv-row", - rows: []string{ - "crates.io,,addr2line,0.15.2", - "npm,,@typescript-eslint/types,5.13.0", - "crates.io,,wasi,0.10.2+wasi-snapshot-preview1", - "Packagist,,sentry/sdk,2.0.4", - }, - }, - want: lockfile.Lockfile{ - FilePath: "-", - ParsedAs: "csv-row", - Packages: []lockfile.PackageDetails{ - { - Name: "@typescript-eslint/types", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "addr2line", - Version: "0.15.2", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - { - Name: "sentry/sdk", - Version: "2.0.4", - Ecosystem: lockfile.ComposerEcosystem, - CompareAs: lockfile.ComposerEcosystem, - }, - { - Name: "wasi", - Version: "0.10.2+wasi-snapshot-preview1", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - }, - }, - }, - { - name: "", - args: args{ - filePath: "-", - parseAs: "-", - rows: []string{ - "NuGet,,Yarp.ReverseProxy,", - "npm,,@typescript-eslint/types,5.13.0", - }, - }, - want: lockfile.Lockfile{ - FilePath: "-", - ParsedAs: "-", - Packages: []lockfile.PackageDetails{ - { - Name: "@typescript-eslint/types", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "Yarp.ReverseProxy", - Version: "", - Ecosystem: "NuGet", - CompareAs: "NuGet", - }, - }, - }, - }, - { - name: "", - args: args{ - filePath: "-", - parseAs: "-", - rows: []string{ - "NuGet,,Yarp.ReverseProxy,", - ",,vue,bb253db0b3e17124b6d1fe93fbf2db35470a1347", - }, - }, - want: lockfile.Lockfile{ - FilePath: "-", - ParsedAs: "-", - Packages: []lockfile.PackageDetails{ - { - Name: "Yarp.ReverseProxy", - Version: "", - Ecosystem: "NuGet", - CompareAs: "NuGet", - }, - { - Name: "vue", - Version: "", - Ecosystem: "", - CompareAs: "", - Commit: "bb253db0b3e17124b6d1fe93fbf2db35470a1347", - }, - }, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got, err := lockfile.FromCSVRows(tt.args.filePath, tt.args.parseAs, tt.args.rows) - if err != nil { - t.Errorf("FromCSVFile() error = %v, was not expected", err) - - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("FromCSVRows() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestFromCSVRows_Errors(t *testing.T) { - t.Parallel() - - type args struct { - filePath string - parseAs string - rows []string - } - tests := []struct { - name string - args args - wantErrMsg string - }{ - { - name: "", - args: args{ - filePath: "", - parseAs: "", - rows: []string{"one,,,"}, - }, - wantErrMsg: "row 1: field 3 is empty (must be the name of a package)", - }, - { - name: "", - args: args{ - filePath: "", - parseAs: "", - rows: []string{ - "crates.io,,addr2line,", - ",,,", - }, - }, - wantErrMsg: "row 2: field 4 is empty (must be a commit)", - }, - { - name: "", - args: args{ - filePath: "", - parseAs: "", - rows: []string{ - "crates.io,,addr2line,", - "npm,,,", - }, - }, - wantErrMsg: "row 2: field 3 is empty (must be the name of a package)", - }, - { - name: "", - args: args{ - filePath: "", - parseAs: "", - rows: []string{ - "crates.io,,addr2line,", - ",,,,", - }, - }, - wantErrMsg: "record on line 2: wrong number of fields", - }, - { - name: "", - args: args{ - filePath: "", - parseAs: "", - rows: []string{ - "crates.io,,addr2line,", - ",,,,", - }, - }, - wantErrMsg: "record on line 2: wrong number of fields", - }, - { - name: "", - args: args{ - filePath: "", - parseAs: "", - rows: []string{ - "NuGet,", - "npm,,@typescript-eslint/types,5.13.0", - }, - }, - wantErrMsg: "row 1: not enough fields (expected at least four)", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - _, err := lockfile.FromCSVRows(tt.args.filePath, tt.args.parseAs, tt.args.rows) - - if err == nil { - t.Errorf("FromCSVRows() did not error") - - return - } - - if !strings.Contains(err.Error(), tt.wantErrMsg) { - t.Errorf("FromCSVRows() error = \"%v\", wanted \"%s\"", err, tt.wantErrMsg) - } - }) - } -} - -func TestFromCSVFile(t *testing.T) { - t.Parallel() - - type args struct { - pathToCSV string - parseAs string - } - tests := []struct { - name string - args args - want lockfile.Lockfile - }{ - { - name: "", - args: args{ - pathToCSV: "fixtures/csv/empty.csv", - parseAs: "csv-file", - }, - want: lockfile.Lockfile{ - FilePath: "fixtures/csv/empty.csv", - ParsedAs: "csv-file", - Packages: nil, - }, - }, - { - name: "", - args: args{ - pathToCSV: "fixtures/csv/multiple-rows.csv", - parseAs: "csv-file", - }, - want: lockfile.Lockfile{ - FilePath: "fixtures/csv/multiple-rows.csv", - ParsedAs: "csv-file", - Packages: []lockfile.PackageDetails{ - { - Name: "@typescript-eslint/types", - Version: "4.9.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "@typescript-eslint/types", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "addr2line", - Version: "0.15.2", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - { - Name: "sentry/sdk", - Version: "2.0.4", - Ecosystem: lockfile.ComposerEcosystem, - CompareAs: lockfile.ComposerEcosystem, - }, - { - Name: "wasi", - Version: "0.10.2+wasi-snapshot-preview1", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - }, - }, - }, - { - name: "", - args: args{ - pathToCSV: "fixtures/csv/with-extra-columns.csv", - parseAs: "csv-file", - }, - want: lockfile.Lockfile{ - FilePath: "fixtures/csv/with-extra-columns.csv", - ParsedAs: "csv-file", - Packages: []lockfile.PackageDetails{ - { - Name: "@typescript-eslint/types", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "addr2line", - Version: "0.15.2", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - { - Name: "sentry/sdk", - Version: "2.0.4", - Ecosystem: lockfile.ComposerEcosystem, - CompareAs: lockfile.ComposerEcosystem, - }, - { - Name: "wasi", - Version: "0.10.2+wasi-snapshot-preview1", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - }, - }, - }, - { - name: "", - args: args{ - pathToCSV: "fixtures/csv/one-row.csv", - parseAs: "-", - }, - want: lockfile.Lockfile{ - FilePath: "fixtures/csv/one-row.csv", - ParsedAs: "-", - Packages: []lockfile.PackageDetails{ - { - Name: "@typescript-eslint/types", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: "NuGet", - }, - }, - }, - }, - { - name: "", - args: args{ - pathToCSV: "fixtures/csv/two-rows.csv", - parseAs: "-", - }, - want: lockfile.Lockfile{ - FilePath: "fixtures/csv/two-rows.csv", - ParsedAs: "-", - Packages: []lockfile.PackageDetails{ - { - Name: "@typescript-eslint/types", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "Yarp.ReverseProxy", - Version: "", - Ecosystem: "NuGet", - CompareAs: "NuGet", - }, - }, - }, - }, - { - name: "", - args: args{ - pathToCSV: "fixtures/csv/with-headers.csv", - parseAs: "-", - }, - want: lockfile.Lockfile{ - FilePath: "fixtures/csv/with-headers.csv", - ParsedAs: "-", - Packages: []lockfile.PackageDetails{ - { - Name: "@typescript-eslint/types", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "Package", - Version: "Version", - Ecosystem: "Ecosystem", - CompareAs: "CompareAs", - }, - { - Name: "sentry/sdk", - Version: "2.0.4", - Ecosystem: lockfile.ComposerEcosystem, - CompareAs: lockfile.ComposerEcosystem, - }, - }, - }, - }, - { - name: "", - args: args{ - pathToCSV: "fixtures/csv/commits.csv", - parseAs: "-", - }, - want: lockfile.Lockfile{ - FilePath: "fixtures/csv/commits.csv", - ParsedAs: "-", - Packages: []lockfile.PackageDetails{ - { - Name: "@typescript-eslint/types", - Version: "4.9.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "addr2line", - Version: "0.15.2", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - { - Name: "babel-preset-php", - Version: "", - Ecosystem: "", - CompareAs: "", - Commit: "c5a7ba5e0ad98b8db1cb8ce105403dd4b768cced", - }, - { - Name: "vue", - Version: "", - Ecosystem: "", - CompareAs: "", - Commit: "bb253db0b3e17124b6d1fe93fbf2db35470a1347", - }, - }, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got, err := lockfile.FromCSVFile(tt.args.pathToCSV, tt.args.parseAs) - if err != nil { - t.Errorf("FromCSVFile() error = %v, was not expected", err) - - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("FromCSVFile() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestFromCSVFile_Errors(t *testing.T) { - t.Parallel() - - type args struct { - pathToCSV string - parseAs string - } - tests := []struct { - name string - args args - wantErrMsg string - }{ - { - name: "", - args: args{ - pathToCSV: "fixtures/csv/does-not-exist", - parseAs: "csv-file", - }, - wantErrMsg: "could not read fixtures/csv/does-not-exist", - }, - { - name: "", - args: args{ - pathToCSV: "fixtures/csv/not-a-csv.xml", - parseAs: "csv-file", - }, - wantErrMsg: "fixtures/csv/not-a-csv.xml: row 1: not enough fields (expected at least four)", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - _, err := lockfile.FromCSVFile(tt.args.pathToCSV, tt.args.parseAs) - - if err == nil { - t.Errorf("FromCSVFile() did not error") - - return - } - - if !strings.Contains(err.Error(), tt.wantErrMsg) { - t.Errorf("FromCSVFile() error = \"%v\", wanted \"%s\"", err, tt.wantErrMsg) - } - }) - } -} diff --git a/pkg/lockfile/dpkg-status.go b/pkg/lockfile/dpkg-status.go deleted file mode 100644 index 8c2a11a4a3..0000000000 --- a/pkg/lockfile/dpkg-status.go +++ /dev/null @@ -1,188 +0,0 @@ -package lockfile - -import ( - "bufio" - "fmt" - "sort" - "strings" - - "github.com/google/osv-scanner/internal/cachedregexp" -) - -const DebianEcosystem Ecosystem = "Debian" - -func groupDpkgPackageLines(scanner *bufio.Scanner) [][]string { - var groups [][]string - var group []string - - for scanner.Scan() { - line := scanner.Text() - - if line != "" { - group = append(group, line) - continue - } - if len(group) > 0 { - groups = append(groups, group) - } - group = make([]string, 0) - } - - if len(group) > 0 { - groups = append(groups, group) - } - - return groups -} - -// Return name and version if "Source" field contains them -func parseSourceField(source string) (string, string) { - // Pattern: name (version) - re := cachedregexp.MustCompile(`^(.*)\((.*)\)`) - matches := re.FindStringSubmatch(source) - if len(matches) == 3 { - return strings.TrimSpace(matches[1]), strings.TrimSpace(matches[2]) - } - // If it not matches the pattern "name (version)", it is only "name" - return strings.TrimSpace(source), "" -} - -func parseDpkgPackageGroup(group []string) PackageDetails { - var pkg = PackageDetails{ - Ecosystem: DebianEcosystem, - CompareAs: DebianEcosystem, - } - - sourcePresent := false - sourceHasVersion := false - for _, line := range group { - switch { - // Status field SPECS: http://www.fifi.org/doc/libapt-pkg-doc/dpkg-tech.html/ch1.html#s1.2 - case strings.HasPrefix(line, "Status:"): - status := strings.TrimPrefix(line, "Status:") - tokens := strings.Fields(status) - // Status field is malformed. Expected: "Status: Want Flag Status" - if len(tokens) != 3 { - return PackageDetails{} - } - // Status field has correct number of fields but package is not installed or has only config files left - // various other field values indicate partial install/uninstall (e.g. failure of some pre/post install scripts) - // since it's not clear if failure has left package active on system, cautiously add it to queries to osv.dev - if tokens[2] == "not-installed" || tokens[2] == "config-files" { - return PackageDetails{} - } - - case strings.HasPrefix(line, "Source:"): - sourcePresent = true - source := strings.TrimPrefix(line, "Source:") - name, version := parseSourceField(source) - pkg.Name = name // can be "" - if version != "" { - sourceHasVersion = true - pkg.Version = version - } - - // If Source field has no version, use Version field - case strings.HasPrefix(line, "Version:"): - if !sourceHasVersion { - pkg.Version = strings.TrimPrefix(line, "Version:") - pkg.Version = strings.TrimSpace(pkg.Version) - } - - // Some packages have no Source field (e.g. sudo) so we use Package value - case strings.HasPrefix(line, "Package:"): - if !sourcePresent { - pkg.Name = strings.TrimPrefix(line, "Package:") - pkg.Name = strings.TrimSpace(pkg.Name) - } - } - } - - return pkg -} - -// Deprecated: use DpkgStatusExtractor.Extract instead -func ParseDpkgStatus(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, DpkgStatusExtractor{}) -} - -type DpkgStatusExtractor struct{} - -func (e DpkgStatusExtractor) ShouldExtract(path string) bool { - return path == "/var/lib/dpkg/status" -} - -func (e DpkgStatusExtractor) Extract(f DepFile) ([]PackageDetails, error) { - scanner := bufio.NewScanner(f) - packageGroups := groupDpkgPackageLines(scanner) - - packages := make([]PackageDetails, 0, len(packageGroups)) - - for _, group := range packageGroups { - pkg := parseDpkgPackageGroup(group) - - // PackageDetails does not contain any field that represent a "not installed" state - // To manage this state and avoid false positives, empty ecosystem means "not installed" so skip it - if pkg.Ecosystem == "" { - continue - } - - if pkg.Name == "" { - continue - } - - packages = append(packages, pkg) - } - - debianReleaseVersion := getReleaseVersion(packages) - if debianReleaseVersion != "" { - for i := range packages { - packages[i].Ecosystem = Ecosystem(string(packages[i].Ecosystem) + ":" + debianReleaseVersion) - } - } - - if err := scanner.Err(); err != nil { - return packages, fmt.Errorf("error while scanning %s: %w", f.Path(), err) - } - - return packages, nil -} - -func getReleaseVersion(packages []PackageDetails) string { - for _, pkg := range packages { - if pkg.Name != "base-files" { - continue - } - - // We only care about the major version - // Example base-files version: 12.4+deb12u5 - versionWithMinor, _, _ := strings.Cut(pkg.Version, "+") - majorVersion, _, _ := strings.Cut(versionWithMinor, ".") - - return majorVersion - } - - return "" -} - -var _ Extractor = DpkgStatusExtractor{} - -// FromDpkgStatus attempts to parse the given file as an "dpkg-status" lockfile -// used by the Debian Package (dpkg) to record installed packages. -func FromDpkgStatus(pathToStatus string) (Lockfile, error) { - packages, err := ParseDpkgStatus(pathToStatus) - - sort.Slice(packages, func(i, j int) bool { - if packages[i].Name == packages[j].Name { - return packages[i].Version < packages[j].Version - } - - return packages[i].Name < packages[j].Name - }) - - return Lockfile{ - FilePath: pathToStatus, - ParsedAs: "dpkg-status", - Packages: packages, - }, err -} diff --git a/pkg/lockfile/dpkg-status_test.go b/pkg/lockfile/dpkg-status_test.go deleted file mode 100644 index 4cf60cdd25..0000000000 --- a/pkg/lockfile/dpkg-status_test.go +++ /dev/null @@ -1,160 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestParseDpkgStatus_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseDpkgStatus_Empty(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/empty_status") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseDpkgStatus_NotAStatus(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/not_status") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseDpkgStatus_Malformed(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/malformed_status") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "bash", - Version: "", - Ecosystem: lockfile.DebianEcosystem, - CompareAs: lockfile.DebianEcosystem, - }, - { - Name: "util-linux", - Version: "2.36.1-8+deb11u1", - Ecosystem: lockfile.DebianEcosystem, - CompareAs: lockfile.DebianEcosystem, - }, - }) -} - -func TestParseDpkgStatus_Single(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/single_status") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "sudo", - Version: "1.8.27-1+deb10u1", - Ecosystem: lockfile.DebianEcosystem, - CompareAs: lockfile.DebianEcosystem, - }, - }) -} - -func TestParseDpkgStatus_Shuffled(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/shuffled_status") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "glibc", - Version: "2.31-13+deb11u5", - Ecosystem: lockfile.DebianEcosystem, - CompareAs: lockfile.DebianEcosystem, - }, - }) -} - -func TestParseDpkgStatus_Multiple(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/multiple_status") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "bash", - Version: "5.1-2+deb11u1", - Ecosystem: lockfile.DebianEcosystem + ":12", - CompareAs: lockfile.DebianEcosystem, - }, - { - Name: "util-linux", - Version: "2.36.1-8+deb11u1", - Ecosystem: lockfile.DebianEcosystem + ":12", - CompareAs: lockfile.DebianEcosystem, - }, - { - Name: "glibc", - Version: "2.31-13+deb11u5", - Ecosystem: lockfile.DebianEcosystem + ":12", - CompareAs: lockfile.DebianEcosystem, - }, - { - Name: "base-files", - Version: "12.4+deb12u5", - Ecosystem: lockfile.DebianEcosystem + ":12", - CompareAs: lockfile.DebianEcosystem, - }, - }) -} - -func TestParseDpkgStatus_Source_Ver_Override(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/source_ver_override_status") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "lvm2", - Version: "2.02.176-4.1ubuntu3", - Ecosystem: lockfile.DebianEcosystem, - CompareAs: lockfile.DebianEcosystem, - }, - }) -} diff --git a/pkg/lockfile/ecosystems.go b/pkg/lockfile/ecosystems.go index 3f6583456d..ad5784c072 100644 --- a/pkg/lockfile/ecosystems.go +++ b/pkg/lockfile/ecosystems.go @@ -21,3 +21,20 @@ func KnownEcosystems() []Ecosystem { // AlpineEcosystem, } } + +const ( + NpmEcosystem Ecosystem = "npm" + NuGetEcosystem Ecosystem = "NuGet" + CargoEcosystem Ecosystem = "crates.io" + BundlerEcosystem Ecosystem = "RubyGems" + ComposerEcosystem Ecosystem = "Packagist" + GoEcosystem Ecosystem = "Go" + MixEcosystem Ecosystem = "Hex" + MavenEcosystem Ecosystem = "Maven" + PipEcosystem Ecosystem = "PyPI" + PubEcosystem Ecosystem = "Pub" + ConanEcosystem Ecosystem = "ConanCenter" + CRANEcosystem Ecosystem = "CRAN" + AlpineEcosystem Ecosystem = "Alpine" + DebianEcosystem Ecosystem = "Debian" +) diff --git a/pkg/lockfile/ecosystems_test.go b/pkg/lockfile/ecosystems_test.go deleted file mode 100644 index 7c0c99ec2c..0000000000 --- a/pkg/lockfile/ecosystems_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package lockfile_test - -import ( - "os" - "strings" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func numberOfLockfileParsers(t *testing.T) int { - t.Helper() - - directories, err := os.ReadDir(".") - - if err != nil { - t.Fatalf("unable to read current directory: ") - } - - count := 0 - - for _, directory := range directories { - if strings.HasPrefix(directory.Name(), "parse-") && - !strings.HasSuffix(directory.Name(), "_test.go") { - count++ - } - } - - return count -} - -func TestKnownEcosystems(t *testing.T) { - t.Parallel() - - expectedCount := numberOfLockfileParsers(t) - - // - npm, yarn, and pnpm, - // - pip, poetry, pdm and pipenv, - // - maven, gradle, and gradle/verification-metadata - // all use the same ecosystem so "ignore" those parsers in the count - expectedCount -= 7 - - ecosystems := lockfile.KnownEcosystems() - - if knownCount := len(ecosystems); knownCount != expectedCount { - t.Errorf("Expected to know about %d ecosystems, but knew about %d", expectedCount, knownCount) - } - - uniq := make(map[lockfile.Ecosystem]int) - - for _, ecosystem := range ecosystems { - uniq[ecosystem]++ - - if uniq[ecosystem] > 1 { - t.Errorf(`Ecosystem "%s" was listed more than once`, ecosystem) - } - } -} diff --git a/pkg/lockfile/extract.go b/pkg/lockfile/extract.go deleted file mode 100644 index 99a553aaee..0000000000 --- a/pkg/lockfile/extract.go +++ /dev/null @@ -1,80 +0,0 @@ -package lockfile - -import ( - "errors" - "fmt" - "sort" - "strings" -) - -var lockfileExtractors = map[string]Extractor{} - -func registerExtractor(name string, extractor Extractor) { - if _, ok := lockfileExtractors[name]; ok { - panic("an extractor is already registered as " + name) - } - - lockfileExtractors[name] = extractor -} - -func FindExtractor(path, extractAs string) (Extractor, string) { - if extractAs != "" { - return lockfileExtractors[extractAs], extractAs - } - - for name, extractor := range lockfileExtractors { - if extractor.ShouldExtract(path) { - return extractor, name - } - } - - return nil, "" -} - -func ListExtractors() []string { - es := make([]string, 0, len(lockfileExtractors)) - - for s := range lockfileExtractors { - es = append(es, s) - } - - sort.Slice(es, func(i, j int) bool { - return strings.ToLower(es[i]) < strings.ToLower(es[j]) - }) - - return es -} - -var ErrExtractorNotFound = errors.New("could not determine extractor") - -func ExtractDeps(f DepFile, extractAs string) (Lockfile, error) { - extractor, extractedAs := FindExtractor(f.Path(), extractAs) - - if extractor == nil { - if extractAs != "" { - return Lockfile{}, fmt.Errorf("%w, requested %s", ErrExtractorNotFound, extractAs) - } - - return Lockfile{}, fmt.Errorf("%w for %s", ErrExtractorNotFound, f.Path()) - } - - packages, err := extractor.Extract(f) - - if err != nil && extractedAs != "" { - err = fmt.Errorf("(extracting as %s) %w", extractedAs, err) - } - - sort.Slice(packages, func(i, j int) bool { - if packages[i].Name == packages[j].Name { - return packages[i].Version < packages[j].Version - } - - return packages[i].Name < packages[j].Name - }) - - return Lockfile{ - FilePath: f.Path(), - ParsedAs: extractedAs, - Packages: packages, - }, err -} diff --git a/pkg/lockfile/extract_test.go b/pkg/lockfile/extract_test.go deleted file mode 100644 index 9300c244fe..0000000000 --- a/pkg/lockfile/extract_test.go +++ /dev/null @@ -1,172 +0,0 @@ -package lockfile_test - -import ( - "errors" - "io" - "strings" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -type TestDepFile struct { - io.Reader - - path string -} - -func (f TestDepFile) Open(_ string) (lockfile.NestedDepFile, error) { - return TestDepFile{}, errors.New("file opening is not supported") -} - -func (f TestDepFile) Path() string { return f.path } -func (f TestDepFile) Close() error { return nil } - -func openTestDepFile(p string) TestDepFile { - return TestDepFile{strings.NewReader(""), p} -} - -var _ lockfile.DepFile = TestDepFile{} -var _ lockfile.NestedDepFile = TestDepFile{} - -func TestFindExtractor(t *testing.T) { - t.Parallel() - - lockfiles := map[string]string{ - "buildscript-gradle.lockfile": "gradle.lockfile", - "Cargo.lock": "Cargo.lock", - "composer.lock": "composer.lock", - "Gemfile.lock": "Gemfile.lock", - "go.mod": "go.mod", - "gradle/verification-metadata.xml": "gradle/verification-metadata.xml", - "gradle.lockfile": "gradle.lockfile", - "mix.lock": "mix.lock", - "pdm.lock": "pdm.lock", - "Pipfile.lock": "Pipfile.lock", - "package-lock.json": "package-lock.json", - "packages.lock.json": "packages.lock.json", - "pnpm-lock.yaml": "pnpm-lock.yaml", - "poetry.lock": "poetry.lock", - "pom.xml": "pom.xml", - "pubspec.lock": "pubspec.lock", - "renv.lock": "renv.lock", - "requirements.txt": "requirements.txt", - "yarn.lock": "yarn.lock", - } - - for file, extractAs := range lockfiles { - extractor, extractedAs := lockfile.FindExtractor("/path/to/my/"+file, "") - - if extractor == nil { - t.Errorf("Expected a extractor to be found for %s but did not", file) - } - - if extractAs != extractedAs { - t.Errorf("Expected extractedAs to be %s but got %s instead", file, extractedAs) - } - } -} - -func TestFindExtractor_ExplicitExtractAs(t *testing.T) { - t.Parallel() - - extractor, extractedAs := lockfile.FindExtractor("/path/to/my/package-lock.json", "composer.lock") - - if extractor == nil { - t.Errorf("Expected a extractor to be found for package-lock.json (overridden as composer.lock) but did not") - } - - if extractedAs != "composer.lock" { - t.Errorf("Expected extractedAs to be composer.lock but got %s instead", extractedAs) - } -} - -func TestExtractDeps_FindsExpectedExtractor(t *testing.T) { - t.Parallel() - - lockfiles := []string{ - "buildscript-gradle.lockfile", - "Cargo.lock", - "composer.lock", - "conan.lock", - "Gemfile.lock", - "go.mod", - "gradle.lockfile", - "gradle/verification-metadata.xml", - "mix.lock", - "pdm.lock", - "Pipfile.lock", - "package-lock.json", - "packages.lock.json", - "pnpm-lock.yaml", - "poetry.lock", - "pom.xml", - "pubspec.lock", - "renv.lock", - "requirements.txt", - "yarn.lock", - } - - count := 0 - - for _, file := range lockfiles { - _, err := lockfile.ExtractDeps(openTestDepFile("/path/to/my/"+file), "") - - if errors.Is(err, lockfile.ErrExtractorNotFound) { - t.Errorf("No extractor was found for %s", file) - } - - count++ - } - - // gradle.lockfile and buildscript-gradle.lockfile use the same parser - count -= 1 - - expectNumberOfParsersCalled(t, count) -} - -func TestExtractDeps_ExtractorNotFound(t *testing.T) { - t.Parallel() - - _, err := lockfile.ExtractDeps(openTestDepFile("/path/to/my/"), "") - - if err == nil { - t.Errorf("Expected to get an error but did not") - } - - if !errors.Is(err, lockfile.ErrExtractorNotFound) { - t.Errorf("Did not get the expected ErrExtractorNotFound error - got %v instead", err) - } -} - -func TestExtractDeps_ExtractorNotFound_WithExplicitExtractAs(t *testing.T) { - t.Parallel() - - _, err := lockfile.ExtractDeps(openTestDepFile("/path/to/my/"), "unsupported") - - if err == nil { - t.Errorf("Expected to get an error but did not") - } - - if !errors.Is(err, lockfile.ErrExtractorNotFound) { - t.Errorf("Did not get the expected ErrExtractorNotFound error - got %v instead", err) - } -} - -func TestListExtractors(t *testing.T) { - t.Parallel() - - extractors := lockfile.ListExtractors() - - firstExpected := "Cargo.lock" - //nolint:ifshort - lastExpected := "yarn.lock" - - if first := extractors[0]; first != firstExpected { - t.Errorf("Expected first element to be %s, but got %s", firstExpected, first) - } - - if last := extractors[len(extractors)-1]; last != lastExpected { - t.Errorf("Expected last element to be %s, but got %s", lastExpected, last) - } -} diff --git a/pkg/lockfile/extractor.go b/pkg/lockfile/extractor.go index 738847ebff..cafcd8bd5a 100644 --- a/pkg/lockfile/extractor.go +++ b/pkg/lockfile/extractor.go @@ -69,15 +69,3 @@ func OpenLocalDepFile(path string) (NestedDepFile, error) { var _ DepFile = LocalFile{} var _ NestedDepFile = LocalFile{} - -func extractFromFile(pathToLockfile string, extractor Extractor) ([]PackageDetails, error) { - f, err := OpenLocalDepFile(pathToLockfile) - - if err != nil { - return []PackageDetails{}, err - } - - defer f.Close() - - return extractor.Extract(f) -} diff --git a/pkg/lockfile/fixtures/apk/with-os-release/empty_installed b/pkg/lockfile/fixtures/apk/with-os-release/empty_installed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pkg/lockfile/fixtures/apk/with-os-release/etc/alpine-release b/pkg/lockfile/fixtures/apk/with-os-release/etc/alpine-release new file mode 100644 index 0000000000..359a2a436b --- /dev/null +++ b/pkg/lockfile/fixtures/apk/with-os-release/etc/alpine-release @@ -0,0 +1 @@ +3.20.0_alpha20231219 diff --git a/pkg/lockfile/fixtures/apk/with-os-release/malformed_installed b/pkg/lockfile/fixtures/apk/with-os-release/malformed_installed new file mode 100644 index 0000000000..ac23d9633b --- /dev/null +++ b/pkg/lockfile/fixtures/apk/with-os-release/malformed_installed @@ -0,0 +1,131 @@ + +This is a malformed APK installed file + + + + no package: + +Z:Q1olh8TpdAi2QnTl4FK3TjdUiSwTo= +R:loader_attic.so +R:afalg.so +Z:Q1mcqLbO6iQe8TmQCoRDozFWScisQ= +F:usr/lib +r:libcrypto1.1 +F:etc +F:lib +F:usr/lib/ossl-modules +R:libcrypto.so.3 +a:0:0:755 +Z:Q1AFDkJxxzzc5SCdOjdbK1BA/vbsY= +o:openssl +I:4206592 +R:legacy.so +Z:Q1fqYq/iJ6x71cTpr8fcO4/6IgyQg= +C:Q1TjmrAEa5PKd40PNz6OIpo+bBeW0= +a:0:0:755 +R:openssl.cnf +R:openssl.cnf.dist +F:etc/ssl/private +R:padlock.so +Z:Q1zr8y7mYzOdgG1uz+DmGLBLOZ/jM= +a:0:0:755 +R:capi.so +S:1707237 +F:etc/ssl +a:0:0:777 +p:so:libcrypto.so.3=3 +R:ct_log_list.cnf.dist +F:usr/lib/engines-3 +V:3.0.7-r0 +Z:Q1WjKZkr5xeMyOxhVrCQfW04JiiME= +F:etc/ssl/certs +c:37a47708fdd97644624ff4b7238bb3299e037eaf +a:0:0:777 +t:1667317778 +a:0:0:755 +Z:Q1op76VCo7av+GQqk9nT9kEezP1I8= +L:Apache-2.0 +R:tsget +a:0:0:755 +R:CA.pl +R:ct_log_list.cnf +D:so:libc.musl-x86_64.so.1 +Z:Q1XK8nt7AyX7GIGpMOLlkJk5dy81c= +R:libcrypto.so.3 +T:Crypto library from openssl +Z:Q1fqYq/iJ6x71cTpr8fcO4/6IgyQg= +Z:Q1olh8TpdAi2QnTl4FK3TjdUiSwTo= +F:usr +Z:Q13NVgfr7dQUuGYxur0tNalH6EIjU= +R:tsget.pl +Z:Q1yzxstq05Nm+4DAS0gR/XScMthRY= +a:0:0:755 +a:0:0:755 +a:0:0:755 +Z:Q1nwKfkE6NiHpVJ8wZRoQYglMEYwQ= +U:https://www.openssl.org/ +F:etc/ssl/misc +A:x86_64 +Z:Q1Uv35WBwtuePGrdxQuLDKbHVVTT4= + +nothing here: +Z:Q1yVNLeeB7VouhCO/kz+dbfL3dY4c= +F:sbin +Z:Q1EgLFjj67ou3eMqp4m3r2ZjnQ7QU= +Z:Q1ORf+lPRKuYgdkBBcKoevR1t60Q4= +M:0:0:1777 +Z:Q1mB95Hq2NUTZ599RDiSsj9w5FrOU= +F:etc +F:var +F:etc/network/if-pre-up.d +Z:Q1HWpG3eQD8Uoi4mks2E3SSvOAUhY= +F:usr/sbin + + + + + + + + +no version +F:var/lib +A:x86_64 +F:var/cache +F:etc/network/if-up.d +F:usr/share +t:1668852790 +F:etc/network/if-down.d +a:0:0:755 +C:Q1NN3sp0yr99btRysqty3nQUrWHaY= +S:509600 +a:0:0:755 +F:etc/network/if-post-up.d +o:busybox +R:udhcpd.conf +U:https://busybox.net/ +F:var/cache/misc +a:0:0:775 +F:usr/share/udhcpc +p:cmd:busybox=1.35.0-r29 +R:dad +R:securetty +F:etc/network/if-post-down.d +I:962560 +F:etc/logrotate.d +D:so:libc.musl-x86_64.so.1 +F:bin +c:1dbf7a793afae640ea643a055b6dd4f430ac116b +F:var/lib/udhcpd +F:tmp +Z:Q1TylyCINVmnS+A/Tead4vZhE7Bks= +T:Size optimized toolbox of many common UNIX utilities +F:etc/network/if-pre-down.d +R:default.script +R:acpid +r:busybox-initscripts +R:busybox +L:GPL-2.0-only +F:usr +F:etc/network +P:busybox \ No newline at end of file diff --git a/pkg/lockfile/fixtures/apk/with-os-release/multiple_installed b/pkg/lockfile/fixtures/apk/with-os-release/multiple_installed new file mode 100644 index 0000000000..9d0fa07ce6 --- /dev/null +++ b/pkg/lockfile/fixtures/apk/with-os-release/multiple_installed @@ -0,0 +1,123 @@ +C:Q1/JgpM8J6DWI/541tUX+uHEzSjqo= +P:alpine-baselayout-data +V:3.4.0-r0 +A:x86_64 +S:11664 +I:77824 +T:Alpine base dir structure and init scripts +U:https://git.alpinelinux.org/cgit/aports/tree/main/alpine-baselayout +L:GPL-2.0-only +o:alpine-baselayout +m:redacted +t:1668104640 +c:f93af038c3de7146121c2ea8124ba5ce29b4b058 +p:so:libc.musl-x86_64.so.1=1 +F:lib +R:ld-musl-x86_64.so.1 +a:0:0:755 +Z:Q1tGxgx2FLrD+0Uk03NUBwbbEiRCU= +R:libc.musl-x86_64.so.1 +a:0:0:777 +Z:Q17yJ3JFNypA4mxhJJr0ou6CzsJVI= + +C:Q1NN3sp0yr99btRysqty3nQUrWHaY= +P:busybox +V:1.35.0-r29 +A:x86_64 +S:509600 +I:962560 +T:Size optimized toolbox of many common UNIX utilities +U:https://busybox.net/ +L:GPL-2.0-only +o:busybox +m:redacted +t:1668852790 +c:1dbf7a793afae640ea643a055b6dd4f430ac116b +D:so:libc.musl-x86_64.so.1 +p:cmd:busybox=1.35.0-r29 +r:busybox-initscripts +F:bin +R:busybox +a:0:0:755 +Z:Q1yVNLeeB7VouhCO/kz+dbfL3dY4c= +F:etc +R:securetty +Z:Q1mB95Hq2NUTZ599RDiSsj9w5FrOU= +R:udhcpd.conf +Z:Q1EgLFjj67ou3eMqp4m3r2ZjnQ7QU= +F:etc/logrotate.d +R:acpid +Z:Q1TylyCINVmnS+A/Tead4vZhE7Bks= +F:etc/network +F:etc/network/if-down.d +F:etc/network/if-post-down.d +F:etc/network/if-post-up.d +F:etc/network/if-pre-down.d +F:etc/network/if-pre-up.d +F:etc/network/if-up.d +R:dad +a:0:0:775 +Z:Q1ORf+lPRKuYgdkBBcKoevR1t60Q4= +F:sbin +F:tmp +M:0:0:1777 +F:usr +F:usr/sbin +F:usr/share +F:usr/share/udhcpc +R:default.script +a:0:0:755 +Z:Q1HWpG3eQD8Uoi4mks2E3SSvOAUhY= +F:var +F:var/cache +F:var/cache/misc +F:var/lib +F:var/lib/udhcpd diff --git a/pkg/lockfile/fixtures/apk/with-os-release/not_installed b/pkg/lockfile/fixtures/apk/with-os-release/not_installed new file mode 100644 index 0000000000..0302b39143 --- /dev/null +++ b/pkg/lockfile/fixtures/apk/with-os-release/not_installed @@ -0,0 +1,12 @@ +Not an APK file! +[[package]] +name = "hello_world" +version = "0.1.0" +dependencies = [ + "regex 1.5.0 (git+https://github.com/rust-lang/regex.git#9f9f693768c584971a4d53bc3c586c33ed3a6831)", +] + +[[package]] +name = "regex" +version = "1.5.0" +source = "git+https://github.com/rust-lang/regex.git#9f9f693768c584971a4d53bc3c586c33ed3a6831" diff --git a/pkg/lockfile/fixtures/apk/with-os-release/shuffled_installed b/pkg/lockfile/fixtures/apk/with-os-release/shuffled_installed new file mode 100644 index 0000000000..c1cbf66baa --- /dev/null +++ b/pkg/lockfile/fixtures/apk/with-os-release/shuffled_installed @@ -0,0 +1,32 @@ +F:lib/apk/exec +F:var/lib +F:sbin +L:GPL-2.0-only +D:musl>=1.2 ca-certificates-bundle so:libc.musl-x86_64.so.1 so:libcrypto.so.3 so:libssl.so.3 so:libz.so.1 +m:Natanael Copa +Z:Q1/4bmOPe/H1YhHRzlrj27oufThMw= +I:307200 +S:120973 +c:0188f510baadbae393472103427b9c1875117136 +F:etc +t:1666552494 +A:x86_64 +V:2.12.10-r1 +F:lib/apk +F:var/lib/apk +a:0:0:755 +C:Q1Ef3iwt+cMdGngEgaFr2URIJhKzQ= +T:Alpine Package Keeper - package manager for alpine +R:apk +P:apk-tools +F:etc/apk/protected_paths.d +F:etc/apk/keys +F:lib +Z:Q1opjpYqXgzmOVo7EbNe8l5Xol08g= +F:var +p:so:libapk.so.3.12.0=3.12.0 cmd:apk=2.12.10-r1 +R:libapk.so.3.12.0 +a:0:0:755 +F:etc/apk +o:apk-tools +U:https://gitlab.alpinelinux.org/alpine/apk-tools \ No newline at end of file diff --git a/pkg/lockfile/fixtures/apk/with-os-release/single_installed b/pkg/lockfile/fixtures/apk/with-os-release/single_installed new file mode 100644 index 0000000000..181245d826 --- /dev/null +++ b/pkg/lockfile/fixtures/apk/with-os-release/single_installed @@ -0,0 +1,32 @@ +C:Q1Ef3iwt+cMdGngEgaFr2URIJhKzQ= +P:apk-tools +V:2.12.10-r1 +A:x86_64 +S:120973 +I:307200 +T:Alpine Package Keeper - package manager for alpine +U:https://gitlab.alpinelinux.org/alpine/apk-tools +L:GPL-2.0-only +o:apk-tools +m:Natanael Copa +t:1666552494 +c:0188f510baadbae393472103427b9c1875117136 +D:musl>=1.2 ca-certificates-bundle so:libc.musl-x86_64.so.1 so:libcrypto.so.3 so:libssl.so.3 so:libz.so.1 +p:so:libapk.so.3.12.0=3.12.0 cmd:apk=2.12.10-r1 +F:etc +F:etc/apk +F:etc/apk/keys +F:etc/apk/protected_paths.d +F:lib +R:libapk.so.3.12.0 +a:0:0:755 +Z:Q1opjpYqXgzmOVo7EbNe8l5Xol08g= +F:lib/apk +F:lib/apk/exec +F:sbin +R:apk +a:0:0:755 +Z:Q1/4bmOPe/H1YhHRzlrj27oufThMw= +F:var +F:var/lib +F:var/lib/apk \ No newline at end of file diff --git a/pkg/lockfile/fixtures/apk/without-os-release/multiple_installed b/pkg/lockfile/fixtures/apk/without-os-release/multiple_installed new file mode 100644 index 0000000000..9d0fa07ce6 --- /dev/null +++ b/pkg/lockfile/fixtures/apk/without-os-release/multiple_installed @@ -0,0 +1,123 @@ +C:Q1/JgpM8J6DWI/541tUX+uHEzSjqo= +P:alpine-baselayout-data +V:3.4.0-r0 +A:x86_64 +S:11664 +I:77824 +T:Alpine base dir structure and init scripts +U:https://git.alpinelinux.org/cgit/aports/tree/main/alpine-baselayout +L:GPL-2.0-only +o:alpine-baselayout +m:redacted +t:1668104640 +c:f93af038c3de7146121c2ea8124ba5ce29b4b058 +p:so:libc.musl-x86_64.so.1=1 +F:lib +R:ld-musl-x86_64.so.1 +a:0:0:755 +Z:Q1tGxgx2FLrD+0Uk03NUBwbbEiRCU= +R:libc.musl-x86_64.so.1 +a:0:0:777 +Z:Q17yJ3JFNypA4mxhJJr0ou6CzsJVI= + +C:Q1NN3sp0yr99btRysqty3nQUrWHaY= +P:busybox +V:1.35.0-r29 +A:x86_64 +S:509600 +I:962560 +T:Size optimized toolbox of many common UNIX utilities +U:https://busybox.net/ +L:GPL-2.0-only +o:busybox +m:redacted +t:1668852790 +c:1dbf7a793afae640ea643a055b6dd4f430ac116b +D:so:libc.musl-x86_64.so.1 +p:cmd:busybox=1.35.0-r29 +r:busybox-initscripts +F:bin +R:busybox +a:0:0:755 +Z:Q1yVNLeeB7VouhCO/kz+dbfL3dY4c= +F:etc +R:securetty +Z:Q1mB95Hq2NUTZ599RDiSsj9w5FrOU= +R:udhcpd.conf +Z:Q1EgLFjj67ou3eMqp4m3r2ZjnQ7QU= +F:etc/logrotate.d +R:acpid +Z:Q1TylyCINVmnS+A/Tead4vZhE7Bks= +F:etc/network +F:etc/network/if-down.d +F:etc/network/if-post-down.d +F:etc/network/if-post-up.d +F:etc/network/if-pre-down.d +F:etc/network/if-pre-up.d +F:etc/network/if-up.d +R:dad +a:0:0:775 +Z:Q1ORf+lPRKuYgdkBBcKoevR1t60Q4= +F:sbin +F:tmp +M:0:0:1777 +F:usr +F:usr/sbin +F:usr/share +F:usr/share/udhcpc +R:default.script +a:0:0:755 +Z:Q1HWpG3eQD8Uoi4mks2E3SSvOAUhY= +F:var +F:var/cache +F:var/cache/misc +F:var/lib +F:var/lib/udhcpd diff --git a/pkg/lockfile/go-binary.go b/pkg/lockfile/go-binary.go deleted file mode 100644 index d06005614b..0000000000 --- a/pkg/lockfile/go-binary.go +++ /dev/null @@ -1,69 +0,0 @@ -package lockfile - -import ( - "bytes" - "debug/buildinfo" - "io" - "path/filepath" - "strings" -) - -type GoBinaryExtractor struct{} - -func (e GoBinaryExtractor) ShouldExtract(path string) bool { - if path == "" { - return false - } - - if strings.HasSuffix(path, string(filepath.Separator)) { // Don't extract directories - return false - } - - // TODO: Filter by executable bit when we have access to full FS - - // Any other path can be a go binary - return true -} - -func (e GoBinaryExtractor) Extract(f DepFile) ([]PackageDetails, error) { - var readerAt io.ReaderAt - if fileWithReaderAt, ok := f.(io.ReaderAt); ok { - readerAt = fileWithReaderAt - } else { - buf := bytes.NewBuffer([]byte{}) - _, err := io.Copy(buf, f) - if err != nil { - return []PackageDetails{}, err - } - readerAt = bytes.NewReader(buf.Bytes()) - } - - info, err := buildinfo.Read(readerAt) - if err != nil { - return []PackageDetails{}, ErrIncompatibleFileFormat - } - - pkgs := make([]PackageDetails, 0, len(info.Deps)+1) - pkgs = append(pkgs, PackageDetails{ - Name: "stdlib", - Version: strings.TrimPrefix(info.GoVersion, "go"), - Ecosystem: GoEcosystem, - CompareAs: GoEcosystem, - }) - - for _, dep := range info.Deps { - if dep.Replace != nil { // Use the replaced dep if it has been replaced - dep = dep.Replace - } - pkgs = append(pkgs, PackageDetails{ - Name: dep.Path, - Version: strings.TrimPrefix(dep.Version, "v"), - Ecosystem: GoEcosystem, - CompareAs: GoEcosystem, - }) - } - - return pkgs, nil -} - -var _ Extractor = GoBinaryExtractor{} diff --git a/pkg/lockfile/go-binary_test.go b/pkg/lockfile/go-binary_test.go deleted file mode 100644 index ed3e2e9682..0000000000 --- a/pkg/lockfile/go-binary_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package lockfile_test - -import ( - "testing" - - "github.com/google/osv-scanner/internal/testutility" - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestGoBinaryExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: testutility.ValueIfOnWindows("path\\to\\dir\\", "path/to/dir/"), - want: false, - }, - { - name: "", - path: "binary.json", - want: true, - }, - { - name: "", - path: "path/to/my/binary.json", - want: true, - }, - { - name: "", - path: "path/to/my/binary-lock.json/file", - want: true, - }, - { - name: "", - path: "path/to/my/binary", - want: true, - }, - { - name: "", - path: "path/to/my/binary.exe", - want: true, - }, - { - name: "", - path: "path/to/my/.hidden-binary", - want: true, - }, - { - name: "", - path: "path/to/my/binary.exe.1", - want: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.GoBinaryExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract(%v) got = %v, want %v", tt.path, got, tt.want) - } - }) - } -} - -func TestExtractGoBinary_NoPackages(t *testing.T) { - t.Parallel() - - file, err := lockfile.OpenLocalDepFile("fixtures/go/binaries/just-go") - if err != nil { - t.Fatalf("could not open file %v", err) - } - - packages, err := lockfile.GoBinaryExtractor{}.Extract(file) - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "stdlib", - Version: "1.21.10", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - }) -} - -func TestExtractGoBinary_OnePackage(t *testing.T) { - t.Parallel() - - file, err := lockfile.OpenLocalDepFile("fixtures/go/binaries/has-one-dep") - if err != nil { - t.Fatalf("could not open file %v", err) - } - - packages, err := lockfile.GoBinaryExtractor{}.Extract(file) - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "stdlib", - Version: "1.21.10", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - { - Name: "github.com/BurntSushi/toml", - Version: "1.4.0", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - }) -} - -func TestExtractGoBinary_NotAGoBinary(t *testing.T) { - t.Parallel() - - file, err := lockfile.OpenLocalDepFile("fixtures/go/one-package.mod") - if err != nil { - t.Fatalf("could not open file %v", err) - } - - packages, err := lockfile.GoBinaryExtractor{}.Extract(file) - if err == nil { - t.Errorf("did not get expected error when extracting") - } - - if len(packages) != 0 { - t.Errorf("packages not empty") - } -} diff --git a/pkg/lockfile/node-modules-npm-v1_test.go b/pkg/lockfile/node-modules-npm-v1_test.go deleted file mode 100644 index 7054f0c82e..0000000000 --- a/pkg/lockfile/node-modules-npm-v1_test.go +++ /dev/null @@ -1,407 +0,0 @@ -package lockfile_test - -import ( - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestNodeModulesExtractor_Extract_npm_v1_InvalidJson(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/not-json.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestNodeModulesExtractor_Extract_npm_v1_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/empty.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestNodeModulesExtractor_Extract_npm_v1_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/one-package.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v1_OnePackageDev(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/one-package-dev.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - DepGroups: []string{"dev"}, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v1_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/two-packages.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "supports-color", - Version: "5.5.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v1_ScopedPackages(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/scoped-packages.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "@babel/code-frame", - Version: "7.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v1_NestedDependencies(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/nested-dependencies.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "postcss", - Version: "6.0.23", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "postcss", - Version: "7.0.16", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "postcss-calc", - Version: "7.0.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "supports-color", - Version: "6.1.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "supports-color", - Version: "5.5.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v1_NestedDependenciesDup(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/nested-dependencies-dup.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - // todo: convert to using expectPackages w/ listing all expected packages - if len(packages) != 39 { - t.Errorf("Expected to get 39 packages, but got %d", len(packages)) - } - - expectPackage(t, packages, lockfile.PackageDetails{ - Name: "supports-color", - Version: "6.1.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }) - - expectPackage(t, packages, lockfile.PackageDetails{ - Name: "supports-color", - Version: "5.5.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }) - - expectPackage(t, packages, lockfile.PackageDetails{ - Name: "supports-color", - Version: "2.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v1_Commits(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/commits.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@segment/analytics.js-integration-facebook-pixel", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "3b1bb80b302c2e552685dc8a029797ec832ea7c9", - }, - { - Name: "ansi-styles", - Version: "1.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - { - Name: "babel-preset-php", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "c5a7ba5e0ad98b8db1cb8ce105403dd4b768cced", - }, - { - Name: "is-number-1", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-1", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "be5935f8d2595bcd97b05718ef1eeae08d812e10", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-2", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", - }, - { - Name: "is-number-2", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "82dcc8e914dabd9305ab9ae580709a7825e824f5", - }, - { - Name: "is-number-3", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-3", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "82ae8802978da40d7f1be5ad5943c9e550ab2c89", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-4", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-5", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-6", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", - DepGroups: []string{"dev"}, - }, - { - Name: "postcss-calc", - Version: "7.0.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - { - Name: "raven-js", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "c2b377e7a254264fd4a1fe328e4e3cfc9e245570", - }, - { - Name: "slick-carousel", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "280b560161b751ba226d50c7db1e0a14a78c2de0", - DepGroups: []string{"dev"}, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v1_Files(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/files.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "lodash", - Version: "1.3.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - { - Name: "other_package", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v1_Alias(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/alias.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@babel/code-frame", - Version: "7.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "string-width", - Version: "4.2.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "string-width", - Version: "5.1.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v1_OptionalPackage(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/optional-package.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - DepGroups: []string{"dev", "optional"}, - }, - { - Name: "supports-color", - Version: "5.5.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - DepGroups: []string{"optional"}, - }, - }) -} diff --git a/pkg/lockfile/node-modules-npm-v2_test.go b/pkg/lockfile/node-modules-npm-v2_test.go deleted file mode 100644 index e1ad77d7f8..0000000000 --- a/pkg/lockfile/node-modules-npm-v2_test.go +++ /dev/null @@ -1,401 +0,0 @@ -package lockfile_test - -import ( - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestNodeModulesExtractor_Extract_npm_v2_InvalidJson(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/not-json.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestNodeModulesExtractor_Extract_npm_v2_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/empty.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestNodeModulesExtractor_Extract_npm_v2_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/one-package.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v2_OnePackageDev(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/one-package-dev.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - DepGroups: []string{"dev"}, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v2_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/two-packages.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "supports-color", - Version: "5.5.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v2_ScopedPackages(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/scoped-packages.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "@babel/code-frame", - Version: "7.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v2_NestedDependencies(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/nested-dependencies.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "postcss", - Version: "6.0.23", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "postcss", - Version: "7.0.16", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "postcss-calc", - Version: "7.0.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "supports-color", - Version: "6.1.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "supports-color", - Version: "5.5.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v2_NestedDependenciesDup(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/nested-dependencies-dup.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "supports-color", - Version: "6.1.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "supports-color", - Version: "2.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v2_Commits(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/commits.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@segment/analytics.js-integration-facebook-pixel", - Version: "2.4.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "3b1bb80b302c2e552685dc8a029797ec832ea7c9", - }, - { - Name: "ansi-styles", - Version: "1.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - { - Name: "babel-preset-php", - Version: "1.1.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "c5a7ba5e0ad98b8db1cb8ce105403dd4b768cced", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-1", - Version: "3.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-1", - Version: "3.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "be5935f8d2595bcd97b05718ef1eeae08d812e10", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-2", - Version: "2.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-2", - Version: "2.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "82dcc8e914dabd9305ab9ae580709a7825e824f5", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-3", - Version: "2.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-3", - Version: "3.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "82ae8802978da40d7f1be5ad5943c9e550ab2c89", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-4", - Version: "3.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-5", - Version: "3.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", - DepGroups: []string{"dev"}, - }, - { - Name: "postcss-calc", - Version: "7.0.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - { - Name: "raven-js", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "c2b377e7a254264fd4a1fe328e4e3cfc9e245570", - }, - { - Name: "slick-carousel", - Version: "1.7.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "280b560161b751ba226d50c7db1e0a14a78c2de0", - DepGroups: []string{"dev"}, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v2_Files(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/files.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "etag", - Version: "1.8.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - DepGroups: []string{"dev"}, - }, - { - Name: "abbrev", - Version: "1.0.9", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - DepGroups: []string{"dev"}, - }, - { - Name: "abbrev", - Version: "2.3.4", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - DepGroups: []string{"dev"}, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v2_Alias(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/alias.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@babel/code-frame", - Version: "7.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "string-width", - Version: "4.2.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "string-width", - Version: "5.1.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestNodeModulesExtractor_Extract_npm_v2_OptionalPackage(t *testing.T) { - t.Parallel() - - packages, err := testParsingNodeModules(t, "fixtures/npm/optional-package.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - DepGroups: []string{"optional"}, - }, - { - Name: "supports-color", - Version: "5.5.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - DepGroups: []string{"dev", "optional"}, - }, - }) -} diff --git a/pkg/lockfile/node-modules.go b/pkg/lockfile/node-modules.go deleted file mode 100644 index 28a1b4fa86..0000000000 --- a/pkg/lockfile/node-modules.go +++ /dev/null @@ -1,19 +0,0 @@ -package lockfile - -import ( - "path/filepath" -) - -type NodeModulesExtractor struct{} - -func (e NodeModulesExtractor) ShouldExtract(path string) bool { - return filepath.Base(filepath.Dir(path)) == "node_modules" && filepath.Base(path) == ".package-lock.json" -} - -func (e NodeModulesExtractor) Extract(f DepFile) ([]PackageDetails, error) { - extractor := NpmLockExtractor{} - - return extractor.Extract(f) -} - -var _ Extractor = NodeModulesExtractor{} diff --git a/pkg/lockfile/node-modules_test.go b/pkg/lockfile/node-modules_test.go deleted file mode 100644 index d576db2f81..0000000000 --- a/pkg/lockfile/node-modules_test.go +++ /dev/null @@ -1,117 +0,0 @@ -package lockfile_test - -import ( - "os" - "path/filepath" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func createTestDirWithNodeModulesDir(t *testing.T) (string, func()) { - t.Helper() - - testDir, cleanupTestDir := createTestDir(t) - - if err := os.Mkdir(filepath.Join(testDir, "node_modules"), 0750); err != nil { - cleanupTestDir() - t.Fatalf("could not create node_modules directory: %v", err) - } - - return testDir, cleanupTestDir -} - -func testParsingNodeModules(t *testing.T, fixture string) ([]lockfile.PackageDetails, error) { - t.Helper() - - testDir, cleanupTestDir := createTestDirWithNodeModulesDir(t) - defer cleanupTestDir() - - file := copyFile(t, fixture, filepath.Join(testDir, "node_modules", ".package-lock.json")) - - f, err := lockfile.OpenLocalDepFile(file) - - if err != nil { - t.Fatalf("could not open file %v", err) - } - - defer f.Close() - - return lockfile.NodeModulesExtractor{}.Extract(f) -} - -func TestNodeModulesExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "package-lock.json", - want: false, - }, - { - name: "", - path: "path/to/my/package-lock.json", - want: false, - }, - { - name: "", - path: "path/to/my/package-lock.json/file", - want: false, - }, - { - name: "", - path: "path/to/my/package-lock.json.file", - want: false, - }, - { - name: "", - path: ".package-lock.json", - want: false, - }, - { - name: "", - path: "node_modules/.package-lock.json", - want: true, - }, - { - name: "", - path: "path/to/my/node_modules/.package-lock.json", - want: true, - }, - { - name: "", - path: "path/to/my/node_modules/.package-lock.json/file", - want: false, - }, - { - name: "", - path: "path/to/my/node_modules/.package-lock.json.file", - want: false, - }, - { - name: "", - path: "path.to.my.node_modules.package-lock.json", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.NodeModulesExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/lockfile/osv-vuln-result_test.go b/pkg/lockfile/osv-vuln-result_test.go deleted file mode 100644 index be69443b0d..0000000000 --- a/pkg/lockfile/osv-vuln-result_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestParseOSVScannerResults_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseOSVScannerResults_InvalidJSON(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/not-json.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseOSVScannerResults_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/empty.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseOSVScannerResults_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/one-package.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "activesupport", - Version: "7.0.7", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - }) -} - -func TestParseOSVScannerResults_OnePackageCommit(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/one-package-commit.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Commit: "9a6bd55c9d0722cb101fe85a3b22d89e4ff4fe52", - }, - }) -} - -func TestParseOSVScannerResults_MultiPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/multi-packages-with-vulns.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "crossbeam-utils", - Version: "0.6.6", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - { - Name: "memoffset", - Version: "0.5.6", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - { - Name: "smallvec", - Version: "1.6.0", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - }) -} diff --git a/pkg/lockfile/osv-vuln-results.go b/pkg/lockfile/osv-vuln-results.go deleted file mode 100644 index b9d461d772..0000000000 --- a/pkg/lockfile/osv-vuln-results.go +++ /dev/null @@ -1,63 +0,0 @@ -package lockfile - -import ( - "encoding/json" - "fmt" - - "github.com/google/osv-scanner/pkg/models" -) - -// Deprecated: use OSVScannerResultsExtractor.Extract instead -func ParseOSVScannerResults(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, OSVScannerResultsExtractor{}) -} - -type OSVScannerResultsExtractor struct{} - -func (e OSVScannerResultsExtractor) ShouldExtract(_ string) bool { - // The output will always be a custom json file, so don't return a default should extract - return false -} - -func (e OSVScannerResultsExtractor) Extract(f DepFile) ([]PackageDetails, error) { - parsedResults := models.VulnerabilityResults{} - err := json.NewDecoder(f).Decode(&parsedResults) - - if err != nil { - return nil, fmt.Errorf("could not extract from %s: %w", f.Path(), err) - } - - packages := []PackageDetails{} - for _, res := range parsedResults.Results { - for _, pkg := range res.Packages { - if pkg.Package.Commit != "" { // Prioritize results - packages = append(packages, PackageDetails{ - Commit: pkg.Package.Commit, - Name: pkg.Package.Name, - }) - } else { - packages = append(packages, PackageDetails{ - Name: pkg.Package.Name, - Ecosystem: Ecosystem(pkg.Package.Ecosystem), - Version: pkg.Package.Version, - CompareAs: Ecosystem(pkg.Package.Ecosystem), - }) - } - } - } - - return packages, nil -} - -var _ Extractor = OSVScannerResultsExtractor{} - -// FromOSVScannerResults attempts to extract packages stored in the OSVScannerResults format -func FromOSVScannerResults(pathToInstalled string) (Lockfile, error) { - packages, err := extractFromFile(pathToInstalled, OSVScannerResultsExtractor{}) - - return Lockfile{ - FilePath: pathToInstalled, - ParsedAs: "osv-scanner", - Packages: packages, - }, err -} diff --git a/pkg/lockfile/parse-cargo-lock.go b/pkg/lockfile/parse-cargo-lock.go deleted file mode 100644 index 185105c01e..0000000000 --- a/pkg/lockfile/parse-cargo-lock.go +++ /dev/null @@ -1,61 +0,0 @@ -package lockfile - -import ( - "fmt" - "path/filepath" - - "github.com/BurntSushi/toml" -) - -type CargoLockPackage struct { - Name string `toml:"name"` - Version string `toml:"version"` -} - -type CargoLockFile struct { - Version int `toml:"version"` - Packages []CargoLockPackage `toml:"package"` -} - -const CargoEcosystem Ecosystem = "crates.io" - -type CargoLockExtractor struct{} - -func (e CargoLockExtractor) ShouldExtract(path string) bool { - return filepath.Base(path) == "Cargo.lock" -} - -func (e CargoLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - var parsedLockfile *CargoLockFile - - _, err := toml.NewDecoder(f).Decode(&parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err) - } - - packages := make([]PackageDetails, 0, len(parsedLockfile.Packages)) - - for _, lockPackage := range parsedLockfile.Packages { - packages = append(packages, PackageDetails{ - Name: lockPackage.Name, - Version: lockPackage.Version, - Ecosystem: CargoEcosystem, - CompareAs: CargoEcosystem, - }) - } - - return packages, nil -} - -var _ Extractor = CargoLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("Cargo.lock", CargoLockExtractor{}) -} - -// Deprecated: use CargoLockExtractor.Extract instead -func ParseCargoLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, CargoLockExtractor{}) -} diff --git a/pkg/lockfile/parse-cargo-lock_test.go b/pkg/lockfile/parse-cargo-lock_test.go deleted file mode 100644 index 6952cb3109..0000000000 --- a/pkg/lockfile/parse-cargo-lock_test.go +++ /dev/null @@ -1,177 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestCargoLockExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "Cargo.lock", - want: true, - }, - { - name: "", - path: "path/to/my/Cargo.lock", - want: true, - }, - { - name: "", - path: "path/to/my/Cargo.lock/file", - want: false, - }, - { - name: "", - path: "path/to/my/Cargo.lock.file", - want: false, - }, - { - name: "", - path: "path.to.my.Cargo.lock", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.CargoLockExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParseCargoLock_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseCargoLock("fixtures/cargo/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseCargoLock_InvalidToml(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseCargoLock("fixtures/cargo/not-toml.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseCargoLock_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseCargoLock("fixtures/cargo/empty.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseCargoLock_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseCargoLock("fixtures/cargo/one-package.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "addr2line", - Version: "0.15.2", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - }) -} - -func TestParseCargoLock_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseCargoLock("fixtures/cargo/two-packages.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "addr2line", - Version: "0.15.2", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - { - Name: "syn", - Version: "1.0.73", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - }) -} - -func TestParseCargoLock_TwoPackagesWithLocal(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseCargoLock("fixtures/cargo/two-packages-with-local.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "addr2line", - Version: "0.15.2", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - { - Name: "local-rust-pkg", - Version: "0.1.0", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - }) -} - -func TestParseCargoLock_PackageWithBuildString(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseCargoLock("fixtures/cargo/package-with-build-string.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wasi", - Version: "0.10.2+wasi-snapshot-preview1", - Ecosystem: lockfile.CargoEcosystem, - CompareAs: lockfile.CargoEcosystem, - }, - }) -} diff --git a/pkg/lockfile/parse-composer-lock.go b/pkg/lockfile/parse-composer-lock.go deleted file mode 100644 index 8546d595c7..0000000000 --- a/pkg/lockfile/parse-composer-lock.go +++ /dev/null @@ -1,80 +0,0 @@ -package lockfile - -import ( - "encoding/json" - "fmt" - "path/filepath" -) - -type ComposerPackage struct { - Name string `json:"name"` - Version string `json:"version"` - Dist struct { - Reference string `json:"reference"` - } `json:"dist"` -} - -type ComposerLock struct { - Packages []ComposerPackage `json:"packages"` - PackagesDev []ComposerPackage `json:"packages-dev"` -} - -const ComposerEcosystem Ecosystem = "Packagist" - -type ComposerLockExtractor struct{} - -func (e ComposerLockExtractor) ShouldExtract(path string) bool { - return filepath.Base(path) == "composer.lock" -} - -func (e ComposerLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - var parsedLockfile *ComposerLock - - err := json.NewDecoder(f).Decode(&parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err) - } - - packages := make( - []PackageDetails, - 0, - // len cannot return negative numbers, but the types can't reflect that - uint64(len(parsedLockfile.Packages))+uint64(len(parsedLockfile.PackagesDev)), - ) - - for _, composerPackage := range parsedLockfile.Packages { - packages = append(packages, PackageDetails{ - Name: composerPackage.Name, - Version: composerPackage.Version, - Commit: composerPackage.Dist.Reference, - Ecosystem: ComposerEcosystem, - CompareAs: ComposerEcosystem, - }) - } - - for _, composerPackage := range parsedLockfile.PackagesDev { - packages = append(packages, PackageDetails{ - Name: composerPackage.Name, - Version: composerPackage.Version, - Commit: composerPackage.Dist.Reference, - Ecosystem: ComposerEcosystem, - CompareAs: ComposerEcosystem, - DepGroups: []string{"dev"}, - }) - } - - return packages, nil -} - -var _ Extractor = ComposerLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("composer.lock", ComposerLockExtractor{}) -} - -// Deprecated: use ComposerLockExtractor.Extract instead -func ParseComposerLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, ComposerLockExtractor{}) -} diff --git a/pkg/lockfile/parse-composer-lock_test.go b/pkg/lockfile/parse-composer-lock_test.go deleted file mode 100644 index e2e1f3ee0c..0000000000 --- a/pkg/lockfile/parse-composer-lock_test.go +++ /dev/null @@ -1,185 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestComposerLockExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "composer.lock", - want: true, - }, - { - name: "", - path: "path/to/my/composer.lock", - want: true, - }, - { - name: "", - path: "path/to/my/composer.lock/file", - want: false, - }, - { - name: "", - path: "path/to/my/composer.lock.file", - want: false, - }, - { - name: "", - path: "path.to.my.composer.lock", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.ComposerLockExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParseComposerLock_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseComposerLock("fixtures/composer/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseComposerLock_InvalidJson(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseComposerLock("fixtures/composer/not-json.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseComposerLock_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseComposerLock("fixtures/composer/empty.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseComposerLock_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseComposerLock("fixtures/composer/one-package.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "sentry/sdk", - Version: "2.0.4", - Commit: "4c115873c86ad5bd0ac6d962db70ca53bf8fb874", - Ecosystem: lockfile.ComposerEcosystem, - CompareAs: lockfile.ComposerEcosystem, - }, - }) -} - -func TestParseComposerLock_OnePackageDev(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseComposerLock("fixtures/composer/one-package-dev.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "sentry/sdk", - Version: "2.0.4", - Commit: "4c115873c86ad5bd0ac6d962db70ca53bf8fb874", - Ecosystem: lockfile.ComposerEcosystem, - CompareAs: lockfile.ComposerEcosystem, - DepGroups: []string{"dev"}, - }, - }) -} - -func TestParseComposerLock_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseComposerLock("fixtures/composer/two-packages.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "sentry/sdk", - Version: "2.0.4", - Commit: "4c115873c86ad5bd0ac6d962db70ca53bf8fb874", - Ecosystem: lockfile.ComposerEcosystem, - CompareAs: lockfile.ComposerEcosystem, - }, - { - Name: "theseer/tokenizer", - Version: "1.1.3", - Commit: "11336f6f84e16a720dae9d8e6ed5019efa85a0f9", - Ecosystem: lockfile.ComposerEcosystem, - CompareAs: lockfile.ComposerEcosystem, - DepGroups: []string{"dev"}, - }, - }) -} - -func TestParseComposerLock_TwoPackagesAlt(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseComposerLock("fixtures/composer/two-packages-alt.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "sentry/sdk", - Version: "2.0.4", - Commit: "4c115873c86ad5bd0ac6d962db70ca53bf8fb874", - Ecosystem: lockfile.ComposerEcosystem, - CompareAs: lockfile.ComposerEcosystem, - }, - { - Name: "theseer/tokenizer", - Version: "1.1.3", - Commit: "11336f6f84e16a720dae9d8e6ed5019efa85a0f9", - Ecosystem: lockfile.ComposerEcosystem, - CompareAs: lockfile.ComposerEcosystem, - }, - }) -} diff --git a/pkg/lockfile/parse-conan-lock-v1-revisions_test.go b/pkg/lockfile/parse-conan-lock-v1-revisions_test.go deleted file mode 100644 index e20101a3dd..0000000000 --- a/pkg/lockfile/parse-conan-lock-v1-revisions_test.go +++ /dev/null @@ -1,163 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestParseConanLock_v1_revisions_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseConanLock_v1_revisions_InvalidJson(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/not-json.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseConanLock_v1_revisions_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/empty.v1.revisions.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseConanLock_v1_revisions_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/one-package.v1.revisions.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.11", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - }) -} - -func TestParseConanLock_v1_revisions_NoName(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/no-name.v1.revisions.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.11", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - }) -} - -func TestParseConanLock_v1_revisions_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/two-packages.v1.revisions.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.11", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - { - Name: "bzip2", - Version: "1.0.8", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - }) -} - -func TestParseConanLock_v1_revisions_NestedDependencies(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/nested-dependencies.v1.revisions.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.13", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - { - Name: "bzip2", - Version: "1.0.8", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - { - Name: "freetype", - Version: "2.12.1", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - { - Name: "libpng", - Version: "1.6.39", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - { - Name: "brotli", - Version: "1.0.9", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - }) -} - -func TestParseConanLock_v1_revisions_OnePackageDev(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/one-package-dev.v1.revisions.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "ninja", - Version: "1.11.1", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - }) -} diff --git a/pkg/lockfile/parse-conan-lock-v1_test.go b/pkg/lockfile/parse-conan-lock-v1_test.go deleted file mode 100644 index 390232aa2e..0000000000 --- a/pkg/lockfile/parse-conan-lock-v1_test.go +++ /dev/null @@ -1,239 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestParseConanLock_v1_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseConanLock_v1_InvalidJson(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/not-json.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseConanLock_v1_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/empty.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseConanLock_v1_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/one-package.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.11", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - }) -} - -func TestParseConanLock_v1_NoName(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/no-name.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.11", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - }) -} - -func TestParseConanLock_v1_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/two-packages.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.11", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - { - Name: "bzip2", - Version: "1.0.8", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - }) -} - -func TestParseConanLock_v1_NestedDependencies(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/nested-dependencies.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.13", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - { - Name: "bzip2", - Version: "1.0.8", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - { - Name: "freetype", - Version: "2.12.1", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - { - Name: "libpng", - Version: "1.6.39", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - { - Name: "brotli", - Version: "1.0.9", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - }) -} - -func TestParseConanLock_v1_OnePackageDev(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/one-package-dev.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "ninja", - Version: "1.11.1", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - }) -} - -func TestParseConanLock_v1_OldFormat00(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/old-format-0.0.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.11", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - }) -} - -func TestParseConanLock_v1_OldFormat01(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/old-format-0.1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.11", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - }) -} - -func TestParseConanLock_v1_OldFormat02(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/old-format-0.2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.11", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - }) -} - -func TestParseConanLock_v1_OldFormat03(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/old-format-0.3.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.11", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - }, - }) -} diff --git a/pkg/lockfile/parse-conan-lock-v2_test.go b/pkg/lockfile/parse-conan-lock-v2_test.go deleted file mode 100644 index 58dbb6e37e..0000000000 --- a/pkg/lockfile/parse-conan-lock-v2_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestParseConanLock_v2_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseConanLock_v2_InvalidJson(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/not-json.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseConanLock_v2_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/empty.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseConanLock_v2_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/one-package.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.11", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - DepGroups: []string{"requires"}, - }, - }) -} - -func TestParseConanLock_v2_NoName(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/no-name.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.11", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - DepGroups: []string{"requires"}, - }, - }) -} - -func TestParseConanLock_v2_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/two-packages.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.11", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - DepGroups: []string{"requires"}, - }, - { - Name: "bzip2", - Version: "1.0.8", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - DepGroups: []string{"requires"}, - }, - }) -} - -func TestParseConanLock_v2_NestedDependencies(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/nested-dependencies.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zlib", - Version: "1.2.13", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - DepGroups: []string{"requires"}, - }, - { - Name: "bzip2", - Version: "1.0.8", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - DepGroups: []string{"requires"}, - }, - { - Name: "freetype", - Version: "2.12.1", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - DepGroups: []string{"requires"}, - }, - { - Name: "libpng", - Version: "1.6.39", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - DepGroups: []string{"requires"}, - }, - { - Name: "brotli", - Version: "1.0.9", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - DepGroups: []string{"requires"}, - }, - }) -} - -func TestParseConanLock_v2_OnePackageDev(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseConanLock("fixtures/conan/one-package-dev.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "ninja", - Version: "1.11.1", - Ecosystem: lockfile.ConanEcosystem, - CompareAs: lockfile.ConanEcosystem, - DepGroups: []string{"build-requires"}, - }, - }) -} diff --git a/pkg/lockfile/parse-conan-lock.go b/pkg/lockfile/parse-conan-lock.go deleted file mode 100644 index eee1eac241..0000000000 --- a/pkg/lockfile/parse-conan-lock.go +++ /dev/null @@ -1,203 +0,0 @@ -package lockfile - -import ( - "encoding/json" - "fmt" - "path/filepath" - "strings" -) - -type ConanReference struct { - Name string - Version string - Username string - Channel string - RecipeRevision string - PackageID string - PackageRevision string - TimeStamp string -} - -type ConanGraphNode struct { - Pref string `json:"pref"` - Ref string `json:"ref"` - Options string `json:"options"` - PackageID string `json:"package_id"` - Prev string `json:"prev"` - Path string `json:"path"` - Context string `json:"context"` -} - -type ConanGraphLock struct { - Nodes map[string]ConanGraphNode `json:"nodes"` -} - -type ConanLockFile struct { - Version string `json:"version"` - // conan v0.4- lockfiles use "graph_lock", "profile_host" and "profile_build" - GraphLock ConanGraphLock `json:"graph_lock,omitempty"` - ProfileHost string `json:"profile_host,omitempty"` - ProfileBuild string `json:"profile_build,omitempty"` - // conan v0.5+ lockfiles use "requires", "build_requires" and "python_requires" - Requires []string `json:"requires,omitempty"` - BuildRequires []string `json:"build_requires,omitempty"` - PythonRequires []string `json:"python_requires,omitempty"` -} - -// TODO this is tentative and subject to change depending on the OSV schema -const ConanEcosystem Ecosystem = "ConanCenter" - -func parseConanReference(ref string) ConanReference { - // very flexible format name/version[@username[/channel]][#rrev][:pkgid[#prev]][%timestamp] - var reference ConanReference - - parts := strings.SplitN(ref, "%", 2) - if len(parts) == 2 { - ref = parts[0] - reference.TimeStamp = parts[1] - } - - parts = strings.SplitN(ref, ":", 2) - if len(parts) == 2 { - ref = parts[0] - parts = strings.SplitN(parts[1], "#", 2) - reference.PackageID = parts[0] - if len(parts) == 2 { - reference.PackageRevision = parts[1] - } - } - - parts = strings.SplitN(ref, "#", 2) - if len(parts) == 2 { - ref = parts[0] - reference.RecipeRevision = parts[1] - } - - parts = strings.SplitN(ref, "@", 2) - if len(parts) == 2 { - ref = parts[0] - UsernameChannel := parts[1] - - parts = strings.SplitN(UsernameChannel, "/", 2) - reference.Username = parts[0] - if len(parts) == 2 { - reference.Channel = parts[1] - } - } - - parts = strings.SplitN(ref, "/", 2) - if len(parts) == 2 { - reference.Name = parts[0] - reference.Version = parts[1] - } else { - // consumer conanfile.txt or conanfile.py might not have a name - reference.Name = "" - reference.Version = ref - } - - return reference -} - -func parseConanV1Lock(lockfile ConanLockFile) []PackageDetails { - var reference ConanReference - packages := make([]PackageDetails, 0, len(lockfile.GraphLock.Nodes)) - - for _, node := range lockfile.GraphLock.Nodes { - if node.Path != "" { - // a local "conanfile.txt", skip - continue - } - - if node.Pref != "" { - // old format 0.3 (conan 1.27-) lockfiles use "pref" instead of "ref" - reference = parseConanReference(node.Pref) - } else if node.Ref != "" { - reference = parseConanReference(node.Ref) - } else { - continue - } - // skip entries with no name, they are most likely consumer's conanfiles - // and not dependencies to be searched in a database anyway - if reference.Name == "" { - continue - } - packages = append(packages, PackageDetails{ - Name: reference.Name, - Version: reference.Version, - Ecosystem: ConanEcosystem, - CompareAs: ConanEcosystem, - }) - } - - return packages -} - -func parseConanRequires(packages *[]PackageDetails, requires []string, group string) { - for _, ref := range requires { - reference := parseConanReference(ref) - // skip entries with no name, they are most likely consumer's conanfiles - // and not dependencies to be searched in a database anyway - if reference.Name == "" { - continue - } - - *packages = append(*packages, PackageDetails{ - Name: reference.Name, - Version: reference.Version, - Ecosystem: ConanEcosystem, - CompareAs: ConanEcosystem, - DepGroups: []string{group}, - }) - } -} - -func parseConanV2Lock(lockfile ConanLockFile) []PackageDetails { - packages := make( - []PackageDetails, - 0, - uint64(len(lockfile.Requires))+uint64(len(lockfile.BuildRequires))+uint64(len(lockfile.PythonRequires)), - ) - - parseConanRequires(&packages, lockfile.Requires, "requires") - parseConanRequires(&packages, lockfile.BuildRequires, "build-requires") - parseConanRequires(&packages, lockfile.PythonRequires, "python-requires") - - return packages -} - -func parseConanLock(lockfile ConanLockFile) []PackageDetails { - if lockfile.GraphLock.Nodes != nil { - return parseConanV1Lock(lockfile) - } - - return parseConanV2Lock(lockfile) -} - -type ConanLockExtractor struct{} - -func (e ConanLockExtractor) ShouldExtract(path string) bool { - return filepath.Base(path) == "conan.lock" -} - -func (e ConanLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - var parsedLockfile *ConanLockFile - - err := json.NewDecoder(f).Decode(&parsedLockfile) - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err) - } - - return parseConanLock(*parsedLockfile), nil -} - -var _ Extractor = ConanLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("conan.lock", ConanLockExtractor{}) -} - -// Deprecated: use ConanLockExtractor.Extract instead -func ParseConanLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, ConanLockExtractor{}) -} diff --git a/pkg/lockfile/parse-conan-lock_test.go b/pkg/lockfile/parse-conan-lock_test.go deleted file mode 100644 index b72f9d34a2..0000000000 --- a/pkg/lockfile/parse-conan-lock_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package lockfile_test - -import ( - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestConanLockExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "conan.lock", - want: true, - }, - { - name: "", - path: "path/to/my/conan.lock", - want: true, - }, - { - name: "", - path: "path/to/my/conan.lock/file", - want: false, - }, - { - name: "", - path: "path/to/my/conan.lock.file", - want: false, - }, - { - name: "", - path: "path.to.my.conan.lock", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.ConanLockExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/lockfile/parse-gemfile-lock.go b/pkg/lockfile/parse-gemfile-lock.go deleted file mode 100644 index c2c3fe64c8..0000000000 --- a/pkg/lockfile/parse-gemfile-lock.go +++ /dev/null @@ -1,194 +0,0 @@ -package lockfile - -import ( - "bufio" - "fmt" - "log" - "path/filepath" - "strings" - - "github.com/google/osv-scanner/internal/cachedregexp" -) - -const BundlerEcosystem Ecosystem = "RubyGems" - -const lockfileSectionBUNDLED = "BUNDLED WITH" -const lockfileSectionDEPENDENCIES = "DEPENDENCIES" -const lockfileSectionPLATFORMS = "PLATFORMS" -const lockfileSectionRUBY = "RUBY VERSION" -const lockfileSectionGIT = "GIT" -const lockfileSectionGEM = "GEM" -const lockfileSectionPATH = "PATH" -const lockfileSectionPLUGIN = "PLUGIN SOURCE" - -type parserState string - -const parserStateSource parserState = "source" -const parserStateDependency parserState = "dependency" -const parserStatePlatform parserState = "platform" -const parserStateRuby parserState = "ruby" -const parserStateBundledWith parserState = "bundled_with" - -func isSourceSection(line string) bool { - return strings.Contains(line, lockfileSectionGIT) || - strings.Contains(line, lockfileSectionGEM) || - strings.Contains(line, lockfileSectionPATH) || - strings.Contains(line, lockfileSectionPLUGIN) -} - -type gemfileLockfileParser struct { - state parserState - dependencies []PackageDetails - bundlerVersion string - rubyVersion string - - // holds the commit of the gem that is currently being parsed, if found - currentGemCommit string -} - -func (parser *gemfileLockfileParser) addDependency(name string, version string) { - parser.dependencies = append(parser.dependencies, PackageDetails{ - Name: name, - Version: version, - Ecosystem: BundlerEcosystem, - CompareAs: BundlerEcosystem, - Commit: parser.currentGemCommit, - }) -} - -func (parser *gemfileLockfileParser) parseSpec(line string) { - // nameVersionReg := cachedregexp.MustCompile(`^( {2}| {4}| {6})(?! )(.*?)(?: \(([^-]*)(?:-(.*))?\))?(!)?$`) - nameVersionReg := cachedregexp.MustCompile(`^( +)(.*?)(?: \(([^-]*)(?:-(.*))?\))?(!)?$`) - - results := nameVersionReg.FindStringSubmatch(line) - - if results == nil { - return - } - - spaces := results[1] - - if spaces == "" { - log.Fatal("Weird error when parsing spec in Gemfile.lock (unexpectedly had no spaces) - please report this") - } - - if len(spaces) == 4 { - parser.addDependency(results[2], results[3]) - } -} - -func (parser *gemfileLockfileParser) parseSource(line string) { - if line == " specs" { - // todo: skip for now - return - } - - // OPTIONS = /^ ([a-z]+): (.*)$/i.freeze - optionsRegexp := cachedregexp.MustCompile(`(?i)^ {2}([a-z]+): (.*)$`) - - // todo: support - options := optionsRegexp.FindStringSubmatch(line) - - if options != nil { - commit := strings.TrimPrefix(options[0], " revision: ") - - // if the prefix was removed then the gem being parsed is git based, so - // we store the commit to be included later - if commit != options[0] { - parser.currentGemCommit = commit - } - - return - } - - // todo: source check - - parser.parseSpec(line) -} - -func isNotIndented(line string) bool { - re := cachedregexp.MustCompile(`^\S`) - - return re.MatchString(line) -} - -func (parser *gemfileLockfileParser) parseLineBasedOnState(line string) { - switch parser.state { - case parserStateDependency: - case parserStatePlatform: - break - case parserStateRuby: - parser.rubyVersion = strings.TrimSpace(line) - case parserStateBundledWith: - parser.bundlerVersion = strings.TrimSpace(line) - case parserStateSource: - parser.parseSource(line) - default: - log.Fatalf("Unknown supported '%s'\n", parser.state) - } -} - -func (parser *gemfileLockfileParser) parse(line string) { - if isSourceSection(line) { - // clear the stateful package details, - // since we're now parsing a new group - parser.currentGemCommit = "" - parser.state = parserStateSource - parser.parseSource(line) - - return - } - - switch line { - case lockfileSectionDEPENDENCIES: - parser.state = parserStateDependency - case lockfileSectionPLATFORMS: - parser.state = parserStatePlatform - case lockfileSectionRUBY: - parser.state = parserStateRuby - case lockfileSectionBUNDLED: - parser.state = parserStateBundledWith - default: - if isNotIndented(line) { - parser.state = "" - } - - if parser.state != "" { - parser.parseLineBasedOnState(line) - } - } -} - -type GemfileLockExtractor struct{} - -func (e GemfileLockExtractor) ShouldExtract(path string) bool { - return filepath.Base(path) == "Gemfile.lock" -} - -func (e GemfileLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - var parser gemfileLockfileParser - - scanner := bufio.NewScanner(f) - - for scanner.Scan() { - parser.parse(scanner.Text()) - } - - if err := scanner.Err(); err != nil { - return []PackageDetails{}, fmt.Errorf("error while scanning %s: %w", f.Path(), err) - } - - return parser.dependencies, nil -} - -var _ Extractor = GemfileLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("Gemfile.lock", GemfileLockExtractor{}) -} - -// Deprecated: use GemfileLockExtractor.Extract instead -func ParseGemfileLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, GemfileLockExtractor{}) -} diff --git a/pkg/lockfile/parse-gemfile-lock_test.go b/pkg/lockfile/parse-gemfile-lock_test.go deleted file mode 100644 index 2cf3c24d2a..0000000000 --- a/pkg/lockfile/parse-gemfile-lock_test.go +++ /dev/null @@ -1,806 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestGemfileLockExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "Gemfile.lock", - want: true, - }, - { - name: "", - path: "path/to/my/Gemfile.lock", - want: true, - }, - { - name: "", - path: "path/to/my/Gemfile.lock/file", - want: false, - }, - { - name: "", - path: "path/to/my/Gemfile.lock.file", - want: false, - }, - { - name: "", - path: "path.to.my.Gemfile.lock", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.GemfileLockExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParseGemfileLock_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGemfileLock("fixtures/bundler/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseGemfileLock_NoSpecSection(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGemfileLock("fixtures/bundler/no-spec-section.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseGemfileLock_NoGemSection(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGemfileLock("fixtures/bundler/no-gem-section.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseGemfileLock_NoGems(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGemfileLock("fixtures/bundler/no-gems.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseGemfileLock_OneGem(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGemfileLock("fixtures/bundler/one-gem.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "ast", - Version: "2.4.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - }) -} - -func TestParseGemfileLock_SomeGems(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGemfileLock("fixtures/bundler/some-gems.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "coderay", - Version: "1.1.3", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "method_source", - Version: "1.0.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "pry", - Version: "0.14.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - }) -} - -func TestParseGemfileLock_MultipleGems(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGemfileLock("fixtures/bundler/multiple-gems.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "bundler-audit", - Version: "0.9.0.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "coderay", - Version: "1.1.3", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "dotenv", - Version: "2.7.6", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "method_source", - Version: "1.0.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "pry", - Version: "0.14.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "thor", - Version: "1.2.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - }) -} - -func TestParseGemfileLock_Rails(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGemfileLock("fixtures/bundler/rails.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "actioncable", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "actionmailbox", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "actionmailer", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "actionpack", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "actiontext", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "actionview", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "activejob", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "activemodel", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "activerecord", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "activestorage", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "activesupport", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "builder", - Version: "3.2.4", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "concurrent-ruby", - Version: "1.1.9", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "crass", - Version: "1.0.6", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "digest", - Version: "3.1.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "erubi", - Version: "1.10.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "globalid", - Version: "1.0.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "i18n", - Version: "1.10.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "io-wait", - Version: "0.2.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "loofah", - Version: "2.14.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "mail", - Version: "2.7.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "marcel", - Version: "1.0.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "method_source", - Version: "1.0.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "mini_mime", - Version: "1.1.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "minitest", - Version: "5.15.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "net-imap", - Version: "0.2.3", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "net-pop", - Version: "0.1.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "net-protocol", - Version: "0.1.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "net-smtp", - Version: "0.3.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "nio4r", - Version: "2.5.8", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "racc", - Version: "1.6.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "rack", - Version: "2.2.3", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "rack-test", - Version: "1.1.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "rails", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "rails-dom-testing", - Version: "2.0.3", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "rails-html-sanitizer", - Version: "1.4.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "railties", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "rake", - Version: "13.0.6", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "strscan", - Version: "3.0.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "thor", - Version: "1.2.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "timeout", - Version: "0.2.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "tzinfo", - Version: "2.0.4", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "websocket-driver", - Version: "0.7.5", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "websocket-extensions", - Version: "0.1.5", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "zeitwerk", - Version: "2.5.4", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "nokogiri", - Version: "1.13.3", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - }) -} - -func TestParseGemfileLock_Rubocop(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGemfileLock("fixtures/bundler/rubocop.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "ast", - Version: "2.4.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "parallel", - Version: "1.21.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "parser", - Version: "3.1.1.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "rainbow", - Version: "3.1.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "regexp_parser", - Version: "2.2.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "rexml", - Version: "3.2.5", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "rubocop", - Version: "1.25.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "rubocop-ast", - Version: "1.16.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "ruby-progressbar", - Version: "1.11.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "unicode-display_width", - Version: "2.1.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - }) -} - -func TestParseGemfileLock_HasLocalGem(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGemfileLock("fixtures/bundler/has-local-gem.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "backbone-on-rails", - Version: "1.2.0.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "actionpack", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "actionview", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "activesupport", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "builder", - Version: "3.2.4", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "coffee-script", - Version: "2.4.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "coffee-script-source", - Version: "1.12.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "concurrent-ruby", - Version: "1.1.9", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "crass", - Version: "1.0.6", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "eco", - Version: "1.0.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "ejs", - Version: "1.1.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "erubi", - Version: "1.10.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "execjs", - Version: "2.8.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "i18n", - Version: "1.10.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "jquery-rails", - Version: "4.4.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "loofah", - Version: "2.14.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "method_source", - Version: "1.0.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "minitest", - Version: "5.15.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "racc", - Version: "1.6.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "rack", - Version: "2.2.3", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "rack-test", - Version: "1.1.0", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "rails-dom-testing", - Version: "2.0.3", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "rails-html-sanitizer", - Version: "1.4.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "railties", - Version: "7.0.2.2", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "rake", - Version: "13.0.6", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "thor", - Version: "1.2.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "tzinfo", - Version: "2.0.4", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "zeitwerk", - Version: "2.5.4", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "nokogiri", - Version: "1.13.3", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - { - Name: "eco-source", - Version: "1.1.0.rc.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - }, - }) -} - -func TestParseGemfileLock_HasGitGem(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGemfileLock("fixtures/bundler/has-git-gem.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "hanami-controller", - Version: "2.0.0.alpha1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - Commit: "027dbe2e56397b534e859fc283990cad1b6addd6", - }, - { - Name: "hanami-utils", - Version: "2.0.0.alpha1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - Commit: "5904fc9a70683b8749aa2861257d0c8c01eae4aa", - }, - { - Name: "concurrent-ruby", - Version: "1.1.7", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - Commit: "", - }, - { - Name: "rack", - Version: "2.2.3", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - Commit: "", - }, - { - Name: "transproc", - Version: "1.1.1", - Ecosystem: lockfile.BundlerEcosystem, - CompareAs: lockfile.BundlerEcosystem, - Commit: "", - }, - }) -} diff --git a/pkg/lockfile/parse-go-lock.go b/pkg/lockfile/parse-go-lock.go deleted file mode 100644 index cd0477e979..0000000000 --- a/pkg/lockfile/parse-go-lock.go +++ /dev/null @@ -1,108 +0,0 @@ -package lockfile - -import ( - "fmt" - "io" - "path/filepath" - "strings" - - "golang.org/x/exp/maps" - "golang.org/x/mod/modfile" -) - -const GoEcosystem Ecosystem = "Go" - -func deduplicatePackages(packages map[string]PackageDetails) map[string]PackageDetails { - details := map[string]PackageDetails{} - - for _, detail := range packages { - details[detail.Name+"@"+detail.Version] = detail - } - - return details -} - -type GoLockExtractor struct{} - -func (e GoLockExtractor) ShouldExtract(path string) bool { - return filepath.Base(path) == "go.mod" -} - -func (e GoLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - var parsedLockfile *modfile.File - - b, err := io.ReadAll(f) - - if err == nil { - parsedLockfile, err = modfile.Parse(f.Path(), b, nil) - } - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err) - } - - packages := map[string]PackageDetails{} - - for _, require := range parsedLockfile.Require { - packages[require.Mod.Path+"@"+require.Mod.Version] = PackageDetails{ - Name: require.Mod.Path, - Version: strings.TrimPrefix(require.Mod.Version, "v"), - Ecosystem: GoEcosystem, - CompareAs: GoEcosystem, - } - } - - for _, replace := range parsedLockfile.Replace { - var replacements []string - - if replace.Old.Version == "" { - // If the left version is omitted, all versions of the module are replaced. - for k, pkg := range packages { - if pkg.Name == replace.Old.Path { - replacements = append(replacements, k) - } - } - } else { - // If a version is present on the left side of the arrow (=>), - // only that specific version of the module is replaced - s := replace.Old.Path + "@" + replace.Old.Version - - // A `replace` directive has no effect if the module version on the left side is not required. - if _, ok := packages[s]; ok { - replacements = []string{s} - } - } - - for _, replacement := range replacements { - packages[replacement] = PackageDetails{ - Name: replace.New.Path, - Version: strings.TrimPrefix(replace.New.Version, "v"), - Ecosystem: GoEcosystem, - CompareAs: GoEcosystem, - } - } - } - - if parsedLockfile.Go != nil && parsedLockfile.Go.Version != "" { - packages["stdlib"] = PackageDetails{ - Name: "stdlib", - Version: parsedLockfile.Go.Version, - Ecosystem: GoEcosystem, - CompareAs: GoEcosystem, - } - } - - return maps.Values(deduplicatePackages(packages)), nil -} - -var _ Extractor = GoLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("go.mod", GoLockExtractor{}) -} - -// Deprecated: use GoLockExtractor.Extract instead -func ParseGoLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, GoLockExtractor{}) -} diff --git a/pkg/lockfile/parse-go-lock_test.go b/pkg/lockfile/parse-go-lock_test.go deleted file mode 100644 index 14ccd66e95..0000000000 --- a/pkg/lockfile/parse-go-lock_test.go +++ /dev/null @@ -1,326 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestGoLockExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "go.mod", - want: true, - }, - { - name: "", - path: "path/to/my/go.mod", - want: true, - }, - { - name: "", - path: "path/to/my/go.mod/file", - want: false, - }, - { - name: "", - path: "path/to/my/go.mod.file", - want: false, - }, - { - name: "", - path: "path.to.my.go.mod", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.GoLockExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParseGoLock_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGoLock("fixtures/go/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseGoLock_Invalid(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGoLock("fixtures/go/not-go-mod.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseGoLock_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGoLock("fixtures/go/empty.mod") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseGoLock_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGoLock("fixtures/go/one-package.mod") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "github.com/BurntSushi/toml", - Version: "1.0.0", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - }) -} - -func TestParseGoLock_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGoLock("fixtures/go/two-packages.mod") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "github.com/BurntSushi/toml", - Version: "1.0.0", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - { - Name: "gopkg.in/yaml.v2", - Version: "2.4.0", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - { - Name: "stdlib", - Version: "1.17", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - }) -} - -func TestParseGoLock_IndirectPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGoLock("fixtures/go/indirect-packages.mod") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "github.com/BurntSushi/toml", - Version: "1.0.0", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - { - Name: "gopkg.in/yaml.v2", - Version: "2.4.0", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - { - Name: "github.com/mattn/go-colorable", - Version: "0.1.9", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - { - Name: "github.com/mattn/go-isatty", - Version: "0.0.14", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - { - Name: "golang.org/x/sys", - Version: "0.0.0-20210630005230-0f9fa26af87c", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - { - Name: "stdlib", - Version: "1.17", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - }) -} - -func TestParseGoLock_Replacements_One(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGoLock("fixtures/go/replace-one.mod") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "example.com/fork/net", - Version: "1.4.5", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - }) -} - -func TestParseGoLock_Replacements_Mixed(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGoLock("fixtures/go/replace-mixed.mod") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "example.com/fork/net", - Version: "1.4.5", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - { - Name: "golang.org/x/net", - Version: "0.5.6", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - }) -} - -func TestParseGoLock_Replacements_Local(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGoLock("fixtures/go/replace-local.mod") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "./fork/net", - Version: "", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - { - Name: "github.com/BurntSushi/toml", - Version: "1.0.0", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - }) -} - -func TestParseGoLock_Replacements_Different(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGoLock("fixtures/go/replace-different.mod") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "example.com/fork/foe", - Version: "1.4.5", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - { - Name: "example.com/fork/foe", - Version: "1.4.2", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - }) -} - -func TestParseGoLock_Replacements_NotRequired(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGoLock("fixtures/go/replace-not-required.mod") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "golang.org/x/net", - Version: "0.5.6", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - { - Name: "github.com/BurntSushi/toml", - Version: "1.0.0", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - }) -} - -func TestParseGoLock_Replacements_NoVersion(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGoLock("fixtures/go/replace-no-version.mod") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "example.com/fork/net", - Version: "1.4.5", - Ecosystem: lockfile.GoEcosystem, - CompareAs: lockfile.GoEcosystem, - }, - }) -} diff --git a/pkg/lockfile/parse-gradle-lock.go b/pkg/lockfile/parse-gradle-lock.go deleted file mode 100644 index 5ad5e1c97e..0000000000 --- a/pkg/lockfile/parse-gradle-lock.go +++ /dev/null @@ -1,88 +0,0 @@ -package lockfile - -import ( - "bufio" - "fmt" - "path/filepath" - "strings" -) - -const ( - gradleLockFileCommentPrefix = "#" - gradleLockFileEmptyPrefix = "empty=" -) - -func isGradleLockFileDepLine(line string) bool { - ret := strings.HasPrefix(line, gradleLockFileCommentPrefix) || - strings.HasPrefix(line, gradleLockFileEmptyPrefix) - - return !ret -} - -func parseToGradlePackageDetail(line string) (PackageDetails, error) { - parts := strings.SplitN(line, ":", 3) - if len(parts) < 3 { - return PackageDetails{}, fmt.Errorf("invalid line in gradle lockfile: %s", line) - } - - group, artifact, version := parts[0], parts[1], parts[2] - version = strings.SplitN(version, "=", 2)[0] - - return PackageDetails{ - Name: fmt.Sprintf("%s:%s", group, artifact), - Version: version, - Ecosystem: MavenEcosystem, - CompareAs: MavenEcosystem, - }, nil -} - -type GradleLockExtractor struct{} - -func (e GradleLockExtractor) ShouldExtract(path string) bool { - base := filepath.Base(path) - - for _, lockfile := range []string{"buildscript-gradle.lockfile", "gradle.lockfile"} { - if lockfile == base { - return true - } - } - - return false -} - -func (e GradleLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - pkgs := make([]PackageDetails, 0) - scanner := bufio.NewScanner(f) - - for scanner.Scan() { - lockLine := strings.TrimSpace(scanner.Text()) - if !isGradleLockFileDepLine(lockLine) { - continue - } - - pkg, err := parseToGradlePackageDetail(lockLine) - if err != nil { - continue - } - - pkgs = append(pkgs, pkg) - } - - if err := scanner.Err(); err != nil { - return []PackageDetails{}, fmt.Errorf("failed to read: %w", err) - } - - return pkgs, nil -} - -var _ Extractor = GradleLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("gradle.lockfile", GradleLockExtractor{}) -} - -// Deprecated: use GradleLockExtractor.Extract instead -func ParseGradleLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, GradleLockExtractor{}) -} diff --git a/pkg/lockfile/parse-gradle-lock_test.go b/pkg/lockfile/parse-gradle-lock_test.go deleted file mode 100644 index 34d3b3e19f..0000000000 --- a/pkg/lockfile/parse-gradle-lock_test.go +++ /dev/null @@ -1,205 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestGradleLockExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "buildscript-gradle.lockfile", - want: true, - }, - { - name: "", - path: "path/to/my/buildscript-gradle.lockfile", - want: true, - }, - { - name: "", - path: "path/to/my/buildscript-gradle.lockfile/file", - want: false, - }, - { - name: "", - path: "path/to/my/buildscript-gradle.lockfile.file", - want: false, - }, - { - name: "", - path: "path.to.my.buildscript-gradle.lockfile", - want: false, - }, - { - name: "", - path: "gradle.lockfile", - want: true, - }, - { - name: "", - path: "path/to/my/gradle.lockfile", - want: true, - }, - { - name: "", - path: "path/to/my/gradle.lockfile/file", - want: false, - }, - { - name: "", - path: "path/to/my/gradle.lockfile.file", - want: false, - }, - { - name: "", - path: "path.to.my.gradle.lockfile", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.GradleLockExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParseGradleLock_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGradleLock("fixtures/gradle/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseGradleLock_OnlyComments(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGradleLock("fixtures/gradle/only-comments") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseGradleLock_EmptyStatement(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGradleLock("fixtures/gradle/only-empty") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseGradleLock_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGradleLock("fixtures/gradle/one-pkg") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "org.springframework.security:spring-security-crypto", - Version: "5.7.3", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - }) -} - -func TestParseGradleLock_MultiplePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGradleLock("fixtures/gradle/5-pkg") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "org.springframework.boot:spring-boot-autoconfigure", - Version: "2.7.4", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.springframework.boot:spring-boot-configuration-processor", - Version: "2.7.5", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.springframework.boot:spring-boot-devtools", - Version: "2.7.6", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - - { - Name: "org.springframework.boot:spring-boot-starter-aop", - Version: "2.7.7", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.springframework.boot:spring-boot-starter-data-jpa", - Version: "2.7.8", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - }) -} - -func TestParseGradleLock_WithInvalidLines(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGradleLock("fixtures/gradle/with-bad-pkg") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "org.springframework.boot:spring-boot-autoconfigure", - Version: "2.7.4", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.springframework.boot:spring-boot-configuration-processor", - Version: "2.7.5", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - }) -} diff --git a/pkg/lockfile/parse-gradle-verification-metadata.go b/pkg/lockfile/parse-gradle-verification-metadata.go deleted file mode 100644 index 07260bcf49..0000000000 --- a/pkg/lockfile/parse-gradle-verification-metadata.go +++ /dev/null @@ -1,56 +0,0 @@ -package lockfile - -import ( - "encoding/xml" - "fmt" - "path/filepath" -) - -type GradleVerificationMetadataFile struct { - Components []struct { - Group string `xml:"group,attr"` - Name string `xml:"name,attr"` - Version string `xml:"version,attr"` - } `xml:"components>component"` -} - -type GradleVerificationMetadataExtractor struct{} - -func (e GradleVerificationMetadataExtractor) ShouldExtract(path string) bool { - return filepath.Base(filepath.Dir(path)) == "gradle" && filepath.Base(path) == "verification-metadata.xml" -} - -func (e GradleVerificationMetadataExtractor) Extract(f DepFile) ([]PackageDetails, error) { - var parsedLockfile *GradleVerificationMetadataFile - - err := xml.NewDecoder(f).Decode(&parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err) - } - - pkgs := make([]PackageDetails, 0, len(parsedLockfile.Components)) - - for _, component := range parsedLockfile.Components { - pkgs = append(pkgs, PackageDetails{ - Name: component.Group + ":" + component.Name, - Version: component.Version, - Ecosystem: MavenEcosystem, - CompareAs: MavenEcosystem, - }) - } - - return pkgs, nil -} - -var _ Extractor = GradleVerificationMetadataExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("gradle/verification-metadata.xml", GradleVerificationMetadataExtractor{}) -} - -// Deprecated: use GradleVerificationMetadataExtractor.Extract instead -func ParseGradleVerificationMetadata(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, GradleVerificationMetadataExtractor{}) -} diff --git a/pkg/lockfile/parse-gradle-verification-metadata_test.go b/pkg/lockfile/parse-gradle-verification-metadata_test.go deleted file mode 100644 index 68c5768bc9..0000000000 --- a/pkg/lockfile/parse-gradle-verification-metadata_test.go +++ /dev/null @@ -1,688 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestGradleVerificationMetadataExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "verification-metadata.xml", - want: false, - }, - { - name: "", - path: "gradle/verification-metadata.xml", - want: true, - }, - { - name: "", - path: "path/to/my/verification-metadata.xml", - want: false, - }, - { - name: "", - path: "path/to/my/verification-metadata.xml/file", - want: false, - }, - { - name: "", - path: "path/to/my/verification-metadata.xml.file", - want: false, - }, - { - name: "", - path: "path.to.my.verification-metadata.xml", - want: false, - }, - { - name: "", - path: "path/to/my/gradle/verification-metadata.xml", - want: true, - }, - { - name: "", - path: "path/to/my/gradle/verification-metadata.xml/file", - want: false, - }, - { - name: "", - path: "path/to/my/gradle/verification-metadata.xml.file", - want: false, - }, - { - name: "", - path: "path.to.my.gradle.verification-metadata.xml", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.GradleVerificationMetadataExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParseGradleVerificationMetadata_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGradleVerificationMetadata("fixtures/gradle-verification-metadata/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseGradleVerificationMetadata_InvalidXml(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGradleVerificationMetadata("fixtures/gradle-verification-metadata/not-xml.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseGradleVerificationMetadata_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGradleVerificationMetadata("fixtures/gradle-verification-metadata/empty.xml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseGradleVerificationMetadata_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGradleVerificationMetadata("fixtures/gradle-verification-metadata/one-package.xml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "org.apache.pdfbox:pdfbox", - Version: "2.0.17", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - }) -} - -func TestParseGradleVerificationMetadata_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGradleVerificationMetadata("fixtures/gradle-verification-metadata/two-packages.xml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "org.apache.pdfbox:pdfbox", - Version: "2.0.17", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.github.javaparser:javaparser-core", - Version: "3.6.11", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - }) -} - -func TestParseGradleVerificationMetadata_MultipleVersions(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGradleVerificationMetadata("fixtures/gradle-verification-metadata/multiple-versions.xml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "androidx.activity:activity", - Version: "1.2.1", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "androidx.activity:activity", - Version: "1.2.3", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "androidx.activity:activity", - Version: "1.5.1", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "androidx.activity:activity", - Version: "1.6.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "androidx.activity:activity", - Version: "1.7.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "androidx.activity:activity", - Version: "1.7.2", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "androidx.activity:activity-compose", - Version: "1.5.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "androidx.activity:activity-compose", - Version: "1.7.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "androidx.activity:activity-compose", - Version: "1.7.2", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "androidx.activity:activity-ktx", - Version: "1.5.1", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "androidx.activity:activity-ktx", - Version: "1.7.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "androidx.activity:activity-ktx", - Version: "1.7.2", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "io.ktor:ktor-serialization-jvm", - Version: "2.0.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "io.ktor:ktor-serialization-jvm", - Version: "2.0.0-beta-1", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "io.ktor:ktor-serialization-jvm", - Version: "2.0.3", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.auto.service:auto-service", - Version: "1.0-rc4", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.auto.service:auto-service", - Version: "1.0.1", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.auto.service:auto-service", - Version: "1.1.1", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - }) -} - -func TestParseGradleVerificationMetadata_Complex(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseGradleVerificationMetadata("fixtures/gradle-verification-metadata/complex.xml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "com.google:google", - Version: "1", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.errorprone:javac", - Version: "9+181-r4173-1", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.android.tools:sdklib", - Version: "31.3.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.android.tools.build:aapt2", - Version: "8.3.0-10880808", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.android.tools.build:aapt2-proto", - Version: "8.3.0-10880808", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.android.tools.build:transform-api", - Version: "2.0.0-deprecated-use-gradle-api", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.android.tools.build.jetifier:jetifier-core", - Version: "1.0.0-beta10", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.android.tools.build.jetifier:jetifier-processor", - Version: "1.0.0-beta10", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.android.tools.emulator:proto", - Version: "31.3.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.android.tools.external.com-intellij:intellij-core", - Version: "31.3.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.android.tools.external.com-intellij:kotlin-compiler", - Version: "31.3.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.android.tools.external.org-jetbrains:uast", - Version: "31.3.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.facebook:ktfmt", - Version: "0.47", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.github.ben-manes:gradle-versions-plugin", - Version: "0.51.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.github.spullara.mustache.java:compiler", - Version: "0.9.6", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.android:annotations", - Version: "4.1.1.4", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.apis:google-api-services-androidpublisher", - Version: "v3-rev20231115-2.0.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.devtools.ksp:symbol-processing", - Version: "1.9.22-1.0.17", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.devtools.ksp:symbol-processing-api", - Version: "1.9.22-1.0.17", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.devtools.ksp:symbol-processing-gradle-plugin", - Version: "1.9.22-1.0.17", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.googlejavaformat:google-java-format", - Version: "1.8", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.guava:guava", - Version: "32.0.0-jre", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.guava:guava", - Version: "32.0.1-jre", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.guava:guava", - Version: "32.1.3-jre", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.guava:listenablefuture", - Version: "1.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.guava:listenablefuture", - Version: "9999.0-empty-to-avoid-conflict-with-guava", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.j2objc:j2objc-annotations", - Version: "2.8", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.jimfs:jimfs", - Version: "1.1", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.testing.platform:core", - Version: "0.0.9-alpha02", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.testing.platform:core-proto", - Version: "0.0.9-alpha02", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.testing.platform:launcher", - Version: "0.0.9-alpha02", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.h2database:h2", - Version: "2.1.214", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.hankcs:aho-corasick-double-array-trie", - Version: "1.2.3", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.jakewharton.android.repackaged:dalvik-dx", - Version: "9.0.0_r3", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.samskivert:jmustache", - Version: "1.15", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.squareup.curtains:curtains", - Version: "1.2.4", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.vaadin.external.google:android-json", - Version: "0.0.20131108.vaadin1", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "commons-logging:commons-logging", - Version: "1.2", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "commons-logging:commons-logging", - Version: "1.3.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "commons-validator:commons-validator", - Version: "1.7", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "commons-validator:commons-validator", - Version: "1.8.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "de.mannodermaus.gradle.plugins:android-junit5", - Version: "1.10.0.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "io.netty:netty-codec-http", - Version: "4.1.93.Final", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "io.netty:netty-codec-http2", - Version: "4.1.93.Final", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "javax.inject:javax.inject", - Version: "1", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.jetbrains.intellij.deps:trove4j", - Version: "1.0.20200330", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.sonatype.ossindex:ossindex-service-client", - Version: "1.8.2", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.tensorflow:tensorflow-lite-metadata", - Version: "0.1.0-rc2", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.tukaani:xz", - Version: "1.9", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.whitesource:pecoff4j", - Version: "0.0.2.1", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.yaml:snakeyaml", - Version: "2.2", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "us.springett:cpe-parser", - Version: "2.0.3", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.json:json", - Version: "20180813", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.json:json", - Version: "20211205", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.json:json", - Version: "20220320", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "junit:junit", - Version: "4.12", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "junit:junit", - Version: "4.13", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "junit:junit", - Version: "4.13.1", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "junit:junit", - Version: "4.13.2", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.almworks.sqlite4java:sqlite4java", - Version: "0.282", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.almworks.sqlite4java:sqlite4java", - Version: "1.0.392", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.apache:apache", - Version: "13", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.apache:apache", - Version: "15", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.apache:apache", - Version: "16", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.apache:apache", - Version: "5", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.apache:apache", - Version: "9", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - }) -} diff --git a/pkg/lockfile/parse-maven-lock.go b/pkg/lockfile/parse-maven-lock.go deleted file mode 100644 index 0d64422dc4..0000000000 --- a/pkg/lockfile/parse-maven-lock.go +++ /dev/null @@ -1,169 +0,0 @@ -package lockfile - -import ( - "encoding/xml" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/google/osv-scanner/internal/cachedregexp" - "golang.org/x/exp/maps" -) - -type MavenLockDependency struct { - XMLName xml.Name `xml:"dependency"` - GroupID string `xml:"groupId"` - ArtifactID string `xml:"artifactId"` - Version string `xml:"version"` - Scope string `xml:"scope"` -} - -func (mld MavenLockDependency) parseResolvedVersion(version string) string { - versionRequirementReg := cachedregexp.MustCompile(`[[(]?(.*?)(?:,|[)\]]|$)`) - - results := versionRequirementReg.FindStringSubmatch(version) - - if results == nil || results[1] == "" { - return "0" - } - - return results[1] -} - -func (mld MavenLockDependency) resolveVersionValue(lockfile MavenLockFile) string { - interpolationReg := cachedregexp.MustCompile(`\${(.+)}`) - - results := interpolationReg.FindStringSubmatch(mld.Version) - - // no interpolation, so just return the version as-is - if results == nil { - return mld.Version - } - if val, ok := lockfile.Properties.m[results[1]]; ok { - return val - } - - fmt.Fprintf( - os.Stderr, - "Failed to resolve version of %s: property \"%s\" could not be found for \"%s\"\n", - mld.GroupID+":"+mld.ArtifactID, - results[1], - lockfile.GroupID+":"+lockfile.ArtifactID, - ) - - return "0" -} - -func (mld MavenLockDependency) ResolveVersion(lockfile MavenLockFile) string { - version := mld.resolveVersionValue(lockfile) - - return mld.parseResolvedVersion(version) -} - -type MavenLockFile struct { - XMLName xml.Name `xml:"project"` - ModelVersion string `xml:"modelVersion"` - GroupID string `xml:"groupId"` - ArtifactID string `xml:"artifactId"` - Properties MavenLockProperties `xml:"properties"` - Dependencies []MavenLockDependency `xml:"dependencies>dependency"` - ManagedDependencies []MavenLockDependency `xml:"dependencyManagement>dependencies>dependency"` -} - -const MavenEcosystem Ecosystem = "Maven" - -type MavenLockProperties struct { - m map[string]string -} - -func (p *MavenLockProperties) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { - p.m = map[string]string{} - - for { - t, err := d.Token() - if err != nil { - return err - } - - switch tt := t.(type) { - case xml.StartElement: - var s string - - if err := d.DecodeElement(&s, &tt); err != nil { - return fmt.Errorf("%w", err) - } - - p.m[tt.Name.Local] = s - - case xml.EndElement: - if tt.Name == start.Name { - return nil - } - } - } -} - -type MavenLockExtractor struct{} - -func (e MavenLockExtractor) ShouldExtract(path string) bool { - return filepath.Base(path) == "pom.xml" -} - -func (e MavenLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - var parsedLockfile *MavenLockFile - - err := xml.NewDecoder(f).Decode(&parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err) - } - - details := map[string]PackageDetails{} - - for _, lockPackage := range parsedLockfile.ManagedDependencies { - finalName := lockPackage.GroupID + ":" + lockPackage.ArtifactID - pkgDetails := PackageDetails{ - Name: finalName, - Version: lockPackage.ResolveVersion(*parsedLockfile), - Ecosystem: MavenEcosystem, - CompareAs: MavenEcosystem, - } - if scope := strings.TrimSpace(lockPackage.Scope); scope != "" && scope != "compile" { - // Only append non-default scope (compile is the default scope). - pkgDetails.DepGroups = append(pkgDetails.DepGroups, scope) - } - details[finalName] = pkgDetails - } - - // standard dependencies take precedent over managed dependencies - for _, lockPackage := range parsedLockfile.Dependencies { - finalName := lockPackage.GroupID + ":" + lockPackage.ArtifactID - - pkgDetails := PackageDetails{ - Name: finalName, - Version: lockPackage.ResolveVersion(*parsedLockfile), - Ecosystem: MavenEcosystem, - CompareAs: MavenEcosystem, - } - if scope := strings.TrimSpace(lockPackage.Scope); scope != "" && scope != "compile" { - // Only append non-default scope (compile is the default scope). - pkgDetails.DepGroups = append(pkgDetails.DepGroups, scope) - } - details[finalName] = pkgDetails - } - - return maps.Values(details), nil -} - -var _ Extractor = MavenLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("pom.xml", MavenLockExtractor{}) -} - -// Deprecated: use MavenLockExtractor.Extract instead -func ParseMavenLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, MavenLockExtractor{}) -} diff --git a/pkg/lockfile/parse-maven-lock_test.go b/pkg/lockfile/parse-maven-lock_test.go deleted file mode 100644 index b49064e712..0000000000 --- a/pkg/lockfile/parse-maven-lock_test.go +++ /dev/null @@ -1,316 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestMavenLockExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "pom.xml", - want: true, - }, - { - name: "", - path: "path/to/my/pom.xml", - want: true, - }, - { - name: "", - path: "path/to/my/pom.xml/file", - want: false, - }, - { - name: "", - path: "path/to/my/pom.xml.file", - want: false, - }, - { - name: "", - path: "path.to.my.pom.xml", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.MavenLockExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParseMavenLock_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseMavenLock("fixtures/maven/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseMavenLock_Invalid(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseMavenLock("fixtures/maven/not-pom.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseMavenLock_InvalidSyntax(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseMavenLock("fixtures/maven/invalid-syntax.xml") - - expectErrContaining(t, err, "XML syntax error") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseMavenLock_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseMavenLock("fixtures/maven/empty.xml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseMavenLock_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseMavenLock("fixtures/maven/one-package.xml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "org.apache.maven:maven-artifact", - Version: "1.0.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - }) -} - -func TestParseMavenLock_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseMavenLock("fixtures/maven/two-packages.xml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "io.netty:netty-all", - Version: "4.1.42.Final", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.slf4j:slf4j-log4j12", - Version: "1.7.25", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - }) -} - -func TestParseMavenLock_WithDependencyManagement(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseMavenLock("fixtures/maven/with-dependency-management.xml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "io.netty:netty-all", - Version: "4.1.9", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.slf4j:slf4j-log4j12", - Version: "1.7.25", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "com.google.code.findbugs:jsr305", - Version: "3.0.2", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - }) -} - -func TestParseMavenLock_Interpolation(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseMavenLock("fixtures/maven/interpolation.xml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "org.mine:mypackage", - Version: "1.0.0", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.mine:my.package", - Version: "2.3.4", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "org.mine:ranged-package", - Version: "9.4.35.v20201120", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - }) -} - -func TestMavenLockDependency_ResolveVersion(t *testing.T) { - t.Parallel() - - type fields struct { - Version string - } - type args struct { - lockfile lockfile.MavenLockFile - } - tests := []struct { - name string - fields fields - args args - want string - }{ - // 1.0: Soft requirement for 1.0. Use 1.0 if no other version appears earlier in the dependency tree. - { - name: "", - fields: fields{Version: "1.0"}, - args: args{lockfile: lockfile.MavenLockFile{}}, - want: "1.0", - }, - // [1.0]: Hard requirement for 1.0. Use 1.0 and only 1.0. - { - name: "", - fields: fields{Version: "[1.0]"}, - args: args{lockfile: lockfile.MavenLockFile{}}, - want: "1.0", - }, - // (,1.0]: Hard requirement for any version <= 1.0. - { - name: "", - fields: fields{Version: "(,1.0]"}, - args: args{lockfile: lockfile.MavenLockFile{}}, - want: "0", - }, - // [1.2,1.3]: Hard requirement for any version between 1.2 and 1.3 inclusive. - { - name: "", - fields: fields{Version: "[1.2,1.3]"}, - args: args{lockfile: lockfile.MavenLockFile{}}, - want: "1.2", - }, - // [1.0,2.0): 1.0 <= x < 2.0; Hard requirement for any version between 1.0 inclusive and 2.0 exclusive. - { - name: "", - fields: fields{Version: "[1.0,2.0)"}, - args: args{lockfile: lockfile.MavenLockFile{}}, - want: "1.0", - }, - // [1.5,): Hard requirement for any version greater than or equal to 1.5. - { - name: "", - fields: fields{Version: "[1.5,)"}, - args: args{lockfile: lockfile.MavenLockFile{}}, - want: "1.5", - }, - // (,1.0],[1.2,): Hard requirement for any version less than or equal to 1.0 than or greater than or equal to 1.2, but not 1.1. - { - name: "", - fields: fields{Version: "(,1.0],[1.2,)"}, - args: args{lockfile: lockfile.MavenLockFile{}}, - want: "0", - }, - // (,1.1),(1.1,): Hard requirement for any version except 1.1; for example because 1.1 has a critical vulnerability. - { - name: "", - fields: fields{Version: "(,1.1),(1.1,)"}, - args: args{lockfile: lockfile.MavenLockFile{}}, - want: "0", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - mld := lockfile.MavenLockDependency{ - Version: tt.fields.Version, - } - if got := mld.ResolveVersion(tt.args.lockfile); got != tt.want { - t.Errorf("ResolveVersion() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParseMavenLock_WithScope(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseMavenLock("fixtures/maven/with-scope.xml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "abc:xyz", - Version: "1.2.3", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - }, - { - Name: "junit:junit", - Version: "4.12", - Ecosystem: lockfile.MavenEcosystem, - CompareAs: lockfile.MavenEcosystem, - DepGroups: []string{"test"}, - }, - }) -} diff --git a/pkg/lockfile/parse-mix-lock.go b/pkg/lockfile/parse-mix-lock.go deleted file mode 100644 index 06d917a576..0000000000 --- a/pkg/lockfile/parse-mix-lock.go +++ /dev/null @@ -1,91 +0,0 @@ -package lockfile - -import ( - "bufio" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/google/osv-scanner/internal/cachedregexp" -) - -const MixEcosystem Ecosystem = "Hex" - -type MixLockExtractor struct{} - -func (e MixLockExtractor) ShouldExtract(path string) bool { - return filepath.Base(path) == "mix.lock" -} - -func (e MixLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - re := cachedregexp.MustCompile(`^ +"(\w+)": \{.+,$`) - - scanner := bufio.NewScanner(f) - - var packages []PackageDetails - - for scanner.Scan() { - line := scanner.Text() - - match := re.FindStringSubmatch(line) - - if match == nil { - continue - } - - // we only care about the third and fourth "rows" which are both strings, - // so we can safely split the line as if it's a set of comma-separated fields - // even though that'll actually poorly represent nested arrays & objects - fields := strings.FieldsFunc(line, func(r rune) bool { - return r == ',' - }) - - if len(fields) < 4 { - _, _ = fmt.Fprintf( - os.Stderr, - "Found less than four fields when parsing a line that looks like a dependency in a mix.lock - please report this!\n", - ) - - continue - } - - name := match[1] - version := strings.TrimSpace(fields[2]) - commit := strings.TrimSpace(fields[3]) - - version = strings.TrimSuffix(strings.TrimPrefix(version, `"`), `"`) - commit = strings.TrimSuffix(strings.TrimPrefix(commit, `"`), `"`) - - if strings.HasSuffix(fields[0], ":git") { - commit = version - version = "" - } - - packages = append(packages, PackageDetails{ - Name: name, - Version: version, - Ecosystem: MixEcosystem, - CompareAs: MixEcosystem, - Commit: commit, - }) - } - - if err := scanner.Err(); err != nil { - return []PackageDetails{}, fmt.Errorf("error while scanning %s: %w", f.Path(), err) - } - - return packages, nil -} - -var _ Extractor = MixLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("mix.lock", MixLockExtractor{}) -} - -// Deprecated: use MixLockExtractor.Extract instead -func ParseMixLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, MixLockExtractor{}) -} diff --git a/pkg/lockfile/parse-mix-lock_test.go b/pkg/lockfile/parse-mix-lock_test.go deleted file mode 100644 index 534f6e707f..0000000000 --- a/pkg/lockfile/parse-mix-lock_test.go +++ /dev/null @@ -1,321 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestMixLockExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "mix.lock", - want: true, - }, - { - name: "", - path: "path/to/my/mix.lock", - want: true, - }, - { - name: "", - path: "path/to/my/mix.lock/file", - want: false, - }, - { - name: "", - path: "path/to/my/mix.lock.file", - want: false, - }, - { - name: "", - path: "path.to.my.mix.lock", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.MixLockExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParseMixLock_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseMixLock("fixtures/mix/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseMixLock_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseMixLock("fixtures/mix/empty.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseMixLock_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseMixLock("fixtures/mix/one-package.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "plug", - Version: "1.11.1", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "f2992bac66fdae679453c9e86134a4201f6f43a687d8ff1cd1b2862d53c80259", - }, - }) -} - -func TestParseMixLock_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseMixLock("fixtures/mix/two-packages.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "plug", - Version: "1.11.1", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "f2992bac66fdae679453c9e86134a4201f6f43a687d8ff1cd1b2862d53c80259", - }, - { - Name: "plug_crypto", - Version: "1.2.2", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", - }, - }) -} - -func TestParseMixLock_Many(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseMixLock("fixtures/mix/many.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "backoff", - Version: "1.1.6", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "83b72ed2108ba1ee8f7d1c22e0b4a00cfe3593a67dbc792799e8cce9f42f796b", - }, - { - Name: "decimal", - Version: "2.0.0", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", - }, - { - Name: "dialyxir", - Version: "1.1.0", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", - }, - { - Name: "earmark", - Version: "1.4.3", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", - }, - { - Name: "earmark_parser", - Version: "1.4.10", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "6603d7a603b9c18d3d20db69921527f82ef09990885ed7525003c7fe7dc86c56", - }, - { - Name: "ecto", - Version: "3.5.5", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "48219a991bb86daba6e38a1e64f8cea540cded58950ff38fbc8163e062281a07", - }, - { - Name: "erlex", - Version: "0.2.6", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", - }, - { - Name: "ex_doc", - Version: "0.23.0", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "a069bc9b0bf8efe323ecde8c0d62afc13d308b1fa3d228b65bca5cf8703a529d", - }, - { - Name: "makeup", - Version: "1.0.5", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", - }, - { - Name: "makeup_elixir", - Version: "0.15.0", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "98312c9f0d3730fde4049985a1105da5155bfe5c11e47bdc7406d88e01e4219b", - }, - { - Name: "meck", - Version: "0.9.2", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", - }, - { - Name: "mime", - Version: "1.5.0", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "203ef35ef3389aae6d361918bf3f952fa17a09e8e43b5aa592b93eba05d0fb8d", - }, - { - Name: "nimble_parsec", - Version: "1.1.0", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", - }, - { - Name: "phoenix", - Version: "1.4.17", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "1b1bd4cff7cfc87c94deaa7d60dd8c22e04368ab95499483c50640ef3bd838d8", - }, - { - Name: "phoenix_html", - Version: "2.14.3", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "51f720d0d543e4e157ff06b65de38e13303d5778a7919bcc696599e5934271b8", - }, - { - Name: "phoenix_pubsub", - Version: "1.1.2", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "496c303bdf1b2e98a9d26e89af5bba3ab487ba3a3735f74bf1f4064d2a845a3e", - }, - { - Name: "plug", - Version: "1.11.1", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "f2992bac66fdae679453c9e86134a4201f6f43a687d8ff1cd1b2862d53c80259", - }, - { - Name: "plug_crypto", - Version: "1.2.2", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", - }, - { - Name: "poolboy", - Version: "1.5.2", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", - }, - { - Name: "pow", - Version: "1.0.15", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "9267b5c75df2d59968585c042e2a0ec6217b1959d3afd629817461f0a20e903c", - }, - { - Name: "telemetry", - Version: "0.4.2", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", - }, - }) -} - -func TestParseMixLock_GitPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseMixLock("fixtures/mix/git.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "foe", - Version: "", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "a9574ab75d6ed01e1288c453ae1d943d7a964595", - }, - { - Name: "foo", - Version: "", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "fc94cce7830fa4dc455024bc2a83720afe244531", - }, - { - Name: "bar", - Version: "", - Ecosystem: lockfile.MixEcosystem, - CompareAs: lockfile.MixEcosystem, - Commit: "bef3ee1d3618017061498b96c75043e8449ef9b5", - }, - }) -} diff --git a/pkg/lockfile/parse-npm-lock-v1_test.go b/pkg/lockfile/parse-npm-lock-v1_test.go deleted file mode 100644 index 8c07224f53..0000000000 --- a/pkg/lockfile/parse-npm-lock-v1_test.go +++ /dev/null @@ -1,449 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestParseNpmLock_v1_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseNpmLock_v1_InvalidJson(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/not-json.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseNpmLock_v1_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/empty.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseNpmLock_v1_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/one-package.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestParseNpmLock_v1_OnePackageDev(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/one-package-dev.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - DepGroups: []string{"dev"}, - }, - }) -} - -func TestParseNpmLock_v1_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/two-packages.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "supports-color", - Version: "5.5.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestParseNpmLock_v1_ScopedPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/scoped-packages.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "@babel/code-frame", - Version: "7.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestParseNpmLock_v1_NestedDependencies(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/nested-dependencies.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "postcss", - Version: "6.0.23", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "postcss", - Version: "7.0.16", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "postcss-calc", - Version: "7.0.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "supports-color", - Version: "6.1.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "supports-color", - Version: "5.5.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestParseNpmLock_v1_NestedDependenciesDup(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/nested-dependencies-dup.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - // todo: convert to using expectPackages w/ listing all expected packages - if len(packages) != 39 { - t.Errorf("Expected to get 39 packages, but got %d", len(packages)) - } - - expectPackage(t, packages, lockfile.PackageDetails{ - Name: "supports-color", - Version: "6.1.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }) - - expectPackage(t, packages, lockfile.PackageDetails{ - Name: "supports-color", - Version: "5.5.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }) - - expectPackage(t, packages, lockfile.PackageDetails{ - Name: "supports-color", - Version: "2.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }) -} - -func TestParseNpmLock_v1_Commits(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/commits.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@segment/analytics.js-integration-facebook-pixel", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "3b1bb80b302c2e552685dc8a029797ec832ea7c9", - }, - { - Name: "ansi-styles", - Version: "1.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - { - Name: "babel-preset-php", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "c5a7ba5e0ad98b8db1cb8ce105403dd4b768cced", - }, - { - Name: "is-number-1", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-1", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "be5935f8d2595bcd97b05718ef1eeae08d812e10", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-2", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", - }, - { - Name: "is-number-2", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "82dcc8e914dabd9305ab9ae580709a7825e824f5", - }, - { - Name: "is-number-3", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-3", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "82ae8802978da40d7f1be5ad5943c9e550ab2c89", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-4", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-5", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-6", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", - DepGroups: []string{"dev"}, - }, - { - Name: "postcss-calc", - Version: "7.0.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - { - Name: "raven-js", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "c2b377e7a254264fd4a1fe328e4e3cfc9e245570", - }, - { - Name: "slick-carousel", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "280b560161b751ba226d50c7db1e0a14a78c2de0", - DepGroups: []string{"dev"}, - }, - }) -} - -func TestParseNpmLock_v1_Files(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/files.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "lodash", - Version: "1.3.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - { - Name: "other_package", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - }) -} - -func TestParseNpmLock_v1_Alias(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/alias.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@babel/code-frame", - Version: "7.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "string-width", - Version: "4.2.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "string-width", - Version: "5.1.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestParseNpmLock_v1_OptionalPackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/optional-package.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - DepGroups: []string{"dev", "optional"}, - }, - { - Name: "supports-color", - Version: "5.5.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - DepGroups: []string{"optional"}, - }, - }) -} - -func TestParseNpmLock_v1_SamePackageDifferentGroups(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/same-package-different-groups.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "eslint", - Version: "1.2.3", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - DepGroups: []string{"dev"}, - }, - { - Name: "table", - Version: "1.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "ajv", - Version: "5.5.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} diff --git a/pkg/lockfile/parse-npm-lock-v2_test.go b/pkg/lockfile/parse-npm-lock-v2_test.go deleted file mode 100644 index 13a62ee5bd..0000000000 --- a/pkg/lockfile/parse-npm-lock-v2_test.go +++ /dev/null @@ -1,443 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestParseNpmLock_v2_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseNpmLock_v2_InvalidJson(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/not-json.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseNpmLock_v2_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/empty.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseNpmLock_v2_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/one-package.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestParseNpmLock_v2_OnePackageDev(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/one-package-dev.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - DepGroups: []string{"dev"}, - }, - }) -} - -func TestParseNpmLock_v2_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/two-packages.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "supports-color", - Version: "5.5.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestParseNpmLock_v2_ScopedPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/scoped-packages.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "@babel/code-frame", - Version: "7.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestParseNpmLock_v2_NestedDependencies(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/nested-dependencies.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "postcss", - Version: "6.0.23", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "postcss", - Version: "7.0.16", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "postcss-calc", - Version: "7.0.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "supports-color", - Version: "6.1.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "supports-color", - Version: "5.5.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestParseNpmLock_v2_NestedDependenciesDup(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/nested-dependencies-dup.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "supports-color", - Version: "6.1.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "supports-color", - Version: "2.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestParseNpmLock_v2_Commits(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/commits.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@segment/analytics.js-integration-facebook-pixel", - Version: "2.4.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "3b1bb80b302c2e552685dc8a029797ec832ea7c9", - }, - { - Name: "ansi-styles", - Version: "1.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - { - Name: "babel-preset-php", - Version: "1.1.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "c5a7ba5e0ad98b8db1cb8ce105403dd4b768cced", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-1", - Version: "3.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-1", - Version: "3.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "be5935f8d2595bcd97b05718ef1eeae08d812e10", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-2", - Version: "2.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-2", - Version: "2.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "82dcc8e914dabd9305ab9ae580709a7825e824f5", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-3", - Version: "2.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-3", - Version: "3.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "82ae8802978da40d7f1be5ad5943c9e550ab2c89", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-4", - Version: "3.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", - DepGroups: []string{"dev"}, - }, - { - Name: "is-number-5", - Version: "3.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", - DepGroups: []string{"dev"}, - }, - { - Name: "postcss-calc", - Version: "7.0.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - { - Name: "raven-js", - Version: "", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "c2b377e7a254264fd4a1fe328e4e3cfc9e245570", - }, - { - Name: "slick-carousel", - Version: "1.7.1", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "280b560161b751ba226d50c7db1e0a14a78c2de0", - DepGroups: []string{"dev"}, - }, - }) -} - -func TestParseNpmLock_v2_Files(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/files.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "etag", - Version: "1.8.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - DepGroups: []string{"dev"}, - }, - { - Name: "abbrev", - Version: "1.0.9", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - DepGroups: []string{"dev"}, - }, - { - Name: "abbrev", - Version: "2.3.4", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - DepGroups: []string{"dev"}, - }, - }) -} - -func TestParseNpmLock_v2_Alias(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/alias.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@babel/code-frame", - Version: "7.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "string-width", - Version: "4.2.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "string-width", - Version: "5.1.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} - -func TestParseNpmLock_v2_OptionalPackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/optional-package.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - DepGroups: []string{"optional"}, - }, - { - Name: "supports-color", - Version: "5.5.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - DepGroups: []string{"dev", "optional"}, - }, - }) -} - -func TestParseNpmLock_v2_SamePackageDifferentGroups(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNpmLock("fixtures/npm/same-package-different-groups.v2.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "eslint", - Version: "1.2.3", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - DepGroups: []string{"dev"}, - }, - { - Name: "table", - Version: "1.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - { - Name: "ajv", - Version: "5.5.2", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - }, - }) -} diff --git a/pkg/lockfile/parse-npm-lock.go b/pkg/lockfile/parse-npm-lock.go index 7347fae172..7340b36b6f 100644 --- a/pkg/lockfile/parse-npm-lock.go +++ b/pkg/lockfile/parse-npm-lock.go @@ -48,7 +48,7 @@ type NpmLockfile struct { Packages map[string]NpmLockPackage `json:"packages,omitempty"` } -const NpmEcosystem Ecosystem = "npm" +// const NpmEcosystem Ecosystem = "npm" type npmPackageDetailsMap map[string]PackageDetails @@ -123,7 +123,7 @@ func parseNpmLockDependencies(dependencies map[string]NpmLockDependency) map[str if strings.HasPrefix(detail.Version, "file:") { finalVersion = "" } else { - commit = tryExtractCommit(detail.Version) + // commit = tryExtractCommit(detail.Version) // if there is a commit, we want to deduplicate based on that rather than // the version (the versions must match anyway for the commits to match) @@ -188,20 +188,20 @@ func parseNpmLockPackages(packages map[string]NpmLockPackage) map[string]Package finalVersion := detail.Version - commit := tryExtractCommit(detail.Resolved) + // commit := tryExtractCommit(detail.Resolved) // if there is a commit, we want to deduplicate based on that rather than // the version (the versions must match anyway for the commits to match) - if commit != "" { - finalVersion = commit - } + // if commit != "" { + // finalVersion = commit + // } details.add(finalName+"@"+finalVersion, PackageDetails{ Name: finalName, Version: detail.Version, Ecosystem: NpmEcosystem, CompareAs: NpmEcosystem, - Commit: commit, + // Commit: commit, DepGroups: detail.depGroups(), }) } @@ -236,13 +236,3 @@ func (e NpmLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { } var _ Extractor = NpmLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("package-lock.json", NpmLockExtractor{}) -} - -// Deprecated: use NpmLockExtractor.Extract instead -func ParseNpmLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, NpmLockExtractor{}) -} diff --git a/pkg/lockfile/parse-npm-lock_test.go b/pkg/lockfile/parse-npm-lock_test.go deleted file mode 100644 index 83d19a014c..0000000000 --- a/pkg/lockfile/parse-npm-lock_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package lockfile_test - -import ( - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestNpmLockExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "package-lock.json", - want: true, - }, - { - name: "", - path: "path/to/my/package-lock.json", - want: true, - }, - { - name: "", - path: "path/to/my/package-lock.json/file", - want: false, - }, - { - name: "", - path: "path/to/my/package-lock.json.file", - want: false, - }, - { - name: "", - path: "path.to.my.package-lock.json", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.NpmLockExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/lockfile/parse-nuget-lock-v1_test.go b/pkg/lockfile/parse-nuget-lock-v1_test.go deleted file mode 100644 index 0397720abe..0000000000 --- a/pkg/lockfile/parse-nuget-lock-v1_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestParseNuGetLock_v1_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNuGetLock("fixtures/nuget/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseNuGetLock_v1_InvalidJson(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNuGetLock("fixtures/nuget/not-json.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseNuGetLock_v1_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNuGetLock("fixtures/nuget/empty.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseNuGetLock_v1_OneFramework_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNuGetLock("fixtures/nuget/one-framework-one-package.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "Test.Core", - Version: "6.0.5", - Ecosystem: lockfile.NuGetEcosystem, - CompareAs: lockfile.NuGetEcosystem, - }, - }) -} - -func TestParseNuGetLock_v1_OneFramework_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNuGetLock("fixtures/nuget/one-framework-two-packages.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "Test.Core", - Version: "6.0.5", - Ecosystem: lockfile.NuGetEcosystem, - CompareAs: lockfile.NuGetEcosystem, - }, - { - Name: "Test.System", - Version: "0.13.0-beta4", - Ecosystem: lockfile.NuGetEcosystem, - CompareAs: lockfile.NuGetEcosystem, - }, - }) -} - -func TestParseNuGetLock_v1_TwoFrameworks_MixedPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNuGetLock("fixtures/nuget/two-frameworks-mixed-packages.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "Test.Core", - Version: "6.0.5", - Ecosystem: lockfile.NuGetEcosystem, - CompareAs: lockfile.NuGetEcosystem, - }, - { - Name: "Test.System", - Version: "0.13.0-beta4", - Ecosystem: lockfile.NuGetEcosystem, - CompareAs: lockfile.NuGetEcosystem, - }, - { - Name: "Test.System", - Version: "2.15.0", - Ecosystem: lockfile.NuGetEcosystem, - CompareAs: lockfile.NuGetEcosystem, - }, - }) -} - -func TestParseNuGetLock_v1_TwoFrameworks_DifferentPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNuGetLock("fixtures/nuget/two-frameworks-different-packages.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "Test.Core", - Version: "6.0.5", - Ecosystem: lockfile.NuGetEcosystem, - CompareAs: lockfile.NuGetEcosystem, - }, - { - Name: "Test.System", - Version: "0.13.0-beta4", - Ecosystem: lockfile.NuGetEcosystem, - CompareAs: lockfile.NuGetEcosystem, - }, - }) -} - -func TestParseNuGetLock_v1_TwoFrameworks_DuplicatePackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNuGetLock("fixtures/nuget/two-frameworks-duplicate-packages.v1.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "Test.Core", - Version: "6.0.5", - Ecosystem: lockfile.NuGetEcosystem, - CompareAs: lockfile.NuGetEcosystem, - }, - }) -} diff --git a/pkg/lockfile/parse-nuget-lock.go b/pkg/lockfile/parse-nuget-lock.go deleted file mode 100644 index c1feb3cfc6..0000000000 --- a/pkg/lockfile/parse-nuget-lock.go +++ /dev/null @@ -1,86 +0,0 @@ -package lockfile - -import ( - "encoding/json" - "fmt" - "path/filepath" - - "golang.org/x/exp/maps" -) - -type NuGetLockPackage struct { - Resolved string `json:"resolved"` -} - -// NuGetLockfile contains the required dependency information as defined in -// https://github.com/NuGet/NuGet.Client/blob/6.5.0.136/src/NuGet.Core/NuGet.ProjectModel/ProjectLockFile/PackagesLockFileFormat.cs -type NuGetLockfile struct { - Version int `json:"version"` - Dependencies map[string]map[string]NuGetLockPackage `json:"dependencies"` -} - -const NuGetEcosystem Ecosystem = "NuGet" - -func parseNuGetLockDependencies(dependencies map[string]NuGetLockPackage) map[string]PackageDetails { - details := map[string]PackageDetails{} - - for name, dependency := range dependencies { - details[name+"@"+dependency.Resolved] = PackageDetails{ - Name: name, - Version: dependency.Resolved, - Ecosystem: NuGetEcosystem, - CompareAs: NuGetEcosystem, - } - } - - return details -} - -func parseNuGetLock(lockfile NuGetLockfile) ([]PackageDetails, error) { - details := map[string]PackageDetails{} - - // go through the dependencies for each framework, e.g. `net6.0` and parse - // its dependencies, there might be different or duplicate dependencies - // between frameworks - for _, dependencies := range lockfile.Dependencies { - for name, detail := range parseNuGetLockDependencies(dependencies) { - details[name] = detail - } - } - - return maps.Values(details), nil -} - -type NuGetLockExtractor struct{} - -func (e NuGetLockExtractor) ShouldExtract(path string) bool { - return filepath.Base(path) == "packages.lock.json" -} - -func (e NuGetLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - var parsedLockfile *NuGetLockfile - - err := json.NewDecoder(f).Decode(&parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err) - } - - if parsedLockfile.Version != 1 && parsedLockfile.Version != 2 { - return []PackageDetails{}, fmt.Errorf("could not extract: unsupported lock file version %d", parsedLockfile.Version) - } - - return parseNuGetLock(*parsedLockfile) -} - -var _ Extractor = NuGetLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("packages.lock.json", NuGetLockExtractor{}) -} - -// Deprecated: use NuGetLockExtractor.Extract instead -func ParseNuGetLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, NuGetLockExtractor{}) -} diff --git a/pkg/lockfile/parse-nuget-lock_test.go b/pkg/lockfile/parse-nuget-lock_test.go deleted file mode 100644 index 2bb1c236e0..0000000000 --- a/pkg/lockfile/parse-nuget-lock_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package lockfile_test - -import ( - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestNuGetLockExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "packages.lock.json", - want: true, - }, - { - name: "", - path: "path/to/my/packages.lock.json", - want: true, - }, - { - name: "", - path: "path/to/my/packages.lock.json/file", - want: false, - }, - { - name: "", - path: "path/to/my/packages.lock.json.file", - want: false, - }, - { - name: "", - path: "path.to.my.packages.lock.json", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.NuGetLockExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParseNuGetLock_InvalidVersion(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseNuGetLock("fixtures/nuget/empty.v0.json") - - expectErrContaining(t, err, "unsupported lock file version 0") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} diff --git a/pkg/lockfile/parse-pdm-lock.go b/pkg/lockfile/parse-pdm-lock.go deleted file mode 100644 index 67e7b89cf4..0000000000 --- a/pkg/lockfile/parse-pdm-lock.go +++ /dev/null @@ -1,80 +0,0 @@ -package lockfile - -import ( - "fmt" - "path/filepath" - - "github.com/BurntSushi/toml" -) - -type PdmLockPackage struct { - Name string `toml:"name"` - Version string `toml:"version"` - Groups []string `toml:"groups"` - Revision string `toml:"revision"` -} - -type PdmLockFile struct { - Version string `toml:"lock-version"` - Packages []PdmLockPackage `toml:"package"` -} - -const PdmEcosystem = PipEcosystem - -type PdmLockExtractor struct{} - -func (p PdmLockExtractor) ShouldExtract(path string) bool { - return filepath.Base(path) == "pdm.lock" -} - -func (p PdmLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - var parsedLockFile *PdmLockFile - - _, err := toml.NewDecoder(f).Decode(&parsedLockFile) - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err) - } - packages := make([]PackageDetails, 0, len(parsedLockFile.Packages)) - - for _, pkg := range parsedLockFile.Packages { - details := PackageDetails{ - Name: pkg.Name, - Version: pkg.Version, - Ecosystem: PdmEcosystem, - CompareAs: PdmEcosystem, - } - - var optional = true - for _, gr := range pkg.Groups { - if gr == "dev" { - details.DepGroups = append(details.DepGroups, "dev") - optional = false - } else if gr == "default" { - optional = false - } - } - if optional { - details.DepGroups = append(details.DepGroups, "optional") - } - - if pkg.Revision != "" { - details.Commit = pkg.Revision - } - - packages = append(packages, details) - } - - return packages, nil -} - -var _ Extractor = PdmLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("pdm.lock", PdmLockExtractor{}) -} - -// Deprecated: use PdmLockExtractor.Extract instead -func ParsePdmLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, PdmLockExtractor{}) -} diff --git a/pkg/lockfile/parse-pdm-lock_test.go b/pkg/lockfile/parse-pdm-lock_test.go deleted file mode 100644 index f40c0da55f..0000000000 --- a/pkg/lockfile/parse-pdm-lock_test.go +++ /dev/null @@ -1,215 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestPdmExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "empty", - path: "", - want: false, - }, - { - name: "plain", - path: "pdm.lock", - want: true, - }, - { - name: "absolute", - path: "/path/to/pdm.lock", - want: true, - }, - { - name: "relative", - path: "../../pdm.lock", - want: true, - }, - { - name: "in-path", - path: "/path/with/pdm.lock/in/middle", - want: false, - }, - { - name: "invalid-suffix", - path: "pdm.lock.file", - want: false, - }, - { - name: "invalid-prefix", - path: "project.name.pdm.lock", - want: false, - }, - } - - for _, test := range tests { - tst := test - t.Run(tst.name, func(t *testing.T) { - t.Parallel() - ext := lockfile.PdmLockExtractor{} - should := ext.ShouldExtract(tst.path) - if should != tst.want { - t.Errorf("ShouldExtract() - got %v, expected %v", should, tst.want) - } - }) - } -} - -func expectNilErr(t *testing.T, err error) { - t.Helper() - if err != nil { - t.Errorf("got unexpected error: %v", err) - } -} - -func TestParsePdmLock_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePdmLock("fixtures/pdm/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePdmLock_InvalidToml(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePdmLock("fixtures/pdm/not-toml.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePdmLock_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePdmLock("fixtures/pdm/empty.toml") - - expectNilErr(t, err) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePdmLock_SinglePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePdmLock("fixtures/pdm/single-package.toml") - - expectNilErr(t, err) - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "toml", - Version: "0.10.2", - Ecosystem: lockfile.PdmEcosystem, - CompareAs: lockfile.PdmEcosystem, - }, - }) -} - -func TestParsePdmLock_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePdmLock("fixtures/pdm/two-packages.toml") - - expectNilErr(t, err) - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "toml", - Version: "0.10.2", - Ecosystem: lockfile.PdmEcosystem, - CompareAs: lockfile.PdmEcosystem, - }, - { - Name: "six", - Version: "1.16.0", - Ecosystem: lockfile.PdmEcosystem, - CompareAs: lockfile.PdmEcosystem, - }, - }) -} - -func TestParsePdmLock_PackageWithDevDependencies(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePdmLock("fixtures/pdm/dev-dependency.toml") - - expectNilErr(t, err) - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "toml", - Version: "0.10.2", - Ecosystem: lockfile.PdmEcosystem, - CompareAs: lockfile.PdmEcosystem, - }, - { - Name: "pyroute2", - Version: "0.7.11", - Ecosystem: lockfile.PdmEcosystem, - CompareAs: lockfile.PdmEcosystem, - DepGroups: []string{"dev"}, - }, - { - Name: "win-inet-pton", - Version: "1.1.0", - Ecosystem: lockfile.PdmEcosystem, - CompareAs: lockfile.PdmEcosystem, - DepGroups: []string{"dev"}, - }, - }) -} - -func TestParsePdmLock_PackageWithOptionalDependency(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePdmLock("fixtures/pdm/optional-dependency.toml") - - expectNilErr(t, err) - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "toml", - Version: "0.10.2", - Ecosystem: lockfile.PdmEcosystem, - CompareAs: lockfile.PdmEcosystem, - }, - { - Name: "pyroute2", - Version: "0.7.11", - Ecosystem: lockfile.PdmEcosystem, - CompareAs: lockfile.PdmEcosystem, - DepGroups: []string{"optional"}, - }, - { - Name: "win-inet-pton", - Version: "1.1.0", - Ecosystem: lockfile.PdmEcosystem, - CompareAs: lockfile.PdmEcosystem, - DepGroups: []string{"optional"}, - }, - }) -} - -func TestParsePdmLock_PackageWithGitDependency(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePdmLock("fixtures/pdm/git-dependency.toml") - - expectNilErr(t, err) - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "toml", - Version: "0.10.2", - Ecosystem: lockfile.PdmEcosystem, - CompareAs: lockfile.PdmEcosystem, - Commit: "65bab7582ce14c55cdeec2244c65ea23039c9e6f", - }, - }) -} diff --git a/pkg/lockfile/parse-pipenv-lock.go b/pkg/lockfile/parse-pipenv-lock.go deleted file mode 100644 index 6610f8b88c..0000000000 --- a/pkg/lockfile/parse-pipenv-lock.go +++ /dev/null @@ -1,78 +0,0 @@ -package lockfile - -import ( - "encoding/json" - "fmt" - "path/filepath" - - "golang.org/x/exp/maps" -) - -type PipenvPackage struct { - Version string `json:"version"` -} - -type PipenvLock struct { - Packages map[string]PipenvPackage `json:"default"` - PackagesDev map[string]PipenvPackage `json:"develop"` -} - -const PipenvEcosystem = PipEcosystem - -type PipenvLockExtractor struct{} - -func (e PipenvLockExtractor) ShouldExtract(path string) bool { - return filepath.Base(path) == "Pipfile.lock" -} - -func (e PipenvLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - var parsedLockfile *PipenvLock - - err := json.NewDecoder(f).Decode(&parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err) - } - - details := make(map[string]PackageDetails) - - addPkgDetails(details, parsedLockfile.Packages, "") - addPkgDetails(details, parsedLockfile.PackagesDev, "dev") - - return maps.Values(details), nil -} - -func addPkgDetails(details map[string]PackageDetails, packages map[string]PipenvPackage, group string) { - for name, pipenvPackage := range packages { - if pipenvPackage.Version == "" { - continue - } - - version := pipenvPackage.Version[2:] - - if _, ok := details[name+"@"+version]; !ok { - pkgDetails := PackageDetails{ - Name: name, - Version: version, - Ecosystem: PipenvEcosystem, - CompareAs: PipenvEcosystem, - } - if group != "" { - pkgDetails.DepGroups = append(pkgDetails.DepGroups, group) - } - details[name+"@"+version] = pkgDetails - } - } -} - -var _ Extractor = PipenvLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("Pipfile.lock", PipenvLockExtractor{}) -} - -// Deprecated: use PipenvLockExtractor.Extract instead -func ParsePipenvLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, PipenvLockExtractor{}) -} diff --git a/pkg/lockfile/parse-pipenv-lock_test.go b/pkg/lockfile/parse-pipenv-lock_test.go deleted file mode 100644 index 0da3ac930e..0000000000 --- a/pkg/lockfile/parse-pipenv-lock_test.go +++ /dev/null @@ -1,229 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestPipenvLockExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "Pipfile.lock", - want: true, - }, - { - name: "", - path: "path/to/my/Pipfile.lock", - want: true, - }, - { - name: "", - path: "path/to/my/Pipfile.lock/file", - want: false, - }, - { - name: "", - path: "path/to/my/Pipfile.lock.file", - want: false, - }, - { - name: "", - path: "path.to.my.Pipfile.lock", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.PipenvLockExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParsePipenvLock_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePipenvLock("fixtures/pipenv/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePipenvLock_InvalidJson(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePipenvLock("fixtures/pipenv/not-json.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePipenvLock_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePipenvLock("fixtures/pipenv/empty.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePipenvLock_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePipenvLock("fixtures/pipenv/one-package.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "markupsafe", - Version: "2.1.1", - Ecosystem: lockfile.PipenvEcosystem, - CompareAs: lockfile.PipenvEcosystem, - }, - }) -} - -func TestParsePipenvLock_OnePackageDev(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePipenvLock("fixtures/pipenv/one-package-dev.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "markupsafe", - Version: "2.1.1", - Ecosystem: lockfile.PipenvEcosystem, - CompareAs: lockfile.PipenvEcosystem, - DepGroups: []string{"dev"}, - }, - }) -} - -func TestParsePipenvLock_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePipenvLock("fixtures/pipenv/two-packages.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "itsdangerous", - Version: "2.1.2", - Ecosystem: lockfile.PipenvEcosystem, - CompareAs: lockfile.PipenvEcosystem, - }, - { - Name: "markupsafe", - Version: "2.1.1", - Ecosystem: lockfile.PipenvEcosystem, - CompareAs: lockfile.PipenvEcosystem, - DepGroups: []string{"dev"}, - }, - }) -} - -func TestParsePipenvLock_TwoPackagesAlt(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePipenvLock("fixtures/pipenv/two-packages-alt.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "itsdangerous", - Version: "2.1.2", - Ecosystem: lockfile.PipenvEcosystem, - CompareAs: lockfile.PipenvEcosystem, - }, - { - Name: "markupsafe", - Version: "2.1.1", - Ecosystem: lockfile.PipenvEcosystem, - CompareAs: lockfile.PipenvEcosystem, - }, - }) -} - -func TestParsePipenvLock_MultiplePackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePipenvLock("fixtures/pipenv/multiple-packages.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "itsdangerous", - Version: "2.1.2", - Ecosystem: lockfile.PipenvEcosystem, - CompareAs: lockfile.PipenvEcosystem, - }, - { - Name: "pluggy", - Version: "1.0.1", - Ecosystem: lockfile.PipenvEcosystem, - CompareAs: lockfile.PipenvEcosystem, - }, - { - Name: "pluggy", - Version: "1.0.0", - Ecosystem: lockfile.PipenvEcosystem, - CompareAs: lockfile.PipenvEcosystem, - DepGroups: []string{"dev"}, - }, - { - Name: "markupsafe", - Version: "2.1.1", - Ecosystem: lockfile.PipenvEcosystem, - CompareAs: lockfile.PipenvEcosystem, - }, - }) -} - -func TestParsePipenvLock_PackageWithoutVersion(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePipenvLock("fixtures/pipenv/no-version.json") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} diff --git a/pkg/lockfile/parse-pnpm-lock-v9_test.go b/pkg/lockfile/parse-pnpm-lock-v9_test.go deleted file mode 100644 index b0763860b2..0000000000 --- a/pkg/lockfile/parse-pnpm-lock-v9_test.go +++ /dev/null @@ -1,281 +0,0 @@ -package lockfile_test - -import ( - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestParsePnpmLock_v9_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/no-packages.v9.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePnpmLock_v9_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/one-package.v9.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "acorn", - Version: "8.11.3", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_v9_OnePackageDev(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/one-package-dev.v9.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "acorn", - Version: "8.11.3", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_v9_ScopedPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/scoped-packages.v9.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@typescript-eslint/types", - Version: "5.62.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_v9_PeerDependencies(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/peer-dependencies.v9.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "acorn-jsx", - Version: "5.3.2", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "acorn", - Version: "8.11.3", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_v9_PeerDependenciesAdvanced(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/peer-dependencies-advanced.v9.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@eslint-community/eslint-utils", - Version: "4.4.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "@eslint/eslintrc", - Version: "2.1.4", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "@typescript-eslint/eslint-plugin", - Version: "5.62.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "@typescript-eslint/parser", - Version: "5.62.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "@typescript-eslint/type-utils", - Version: "5.62.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "@typescript-eslint/typescript-estree", - Version: "5.62.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "@typescript-eslint/utils", - Version: "5.62.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "debug", - Version: "4.3.4", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "eslint", - Version: "8.57.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "has-flag", - Version: "4.0.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "supports-color", - Version: "7.2.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "tsutils", - Version: "3.21.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "typescript", - Version: "4.9.5", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_v9_MultipleVersions(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/multiple-versions.v9.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "uuid", - Version: "8.0.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "uuid", - Version: "8.3.2", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "xmlbuilder", - Version: "11.0.1", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_v9_Commits(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/commits.v9.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "ansi-regex", - Version: "6.0.1", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - Commit: "02fa893d619d3da85411acc8fd4e2eea0e95a9d9", - }, - { - Name: "is-number", - Version: "7.0.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - Commit: "98e8ff1da1a89f93d1397a24d7413ed15421c139", - }, - }) -} - -func TestParsePnpmLock_v9_MixedGroups(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/mixed-groups.v9.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "ansi-regex", - Version: "5.0.1", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "uuid", - Version: "8.3.2", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "is-number", - Version: "7.0.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} diff --git a/pkg/lockfile/parse-pnpm-lock.go b/pkg/lockfile/parse-pnpm-lock.go deleted file mode 100644 index bb220706de..0000000000 --- a/pkg/lockfile/parse-pnpm-lock.go +++ /dev/null @@ -1,221 +0,0 @@ -package lockfile - -import ( - "errors" - "fmt" - "io" - "path/filepath" - "strconv" - "strings" - - "github.com/google/osv-scanner/internal/cachedregexp" - "gopkg.in/yaml.v3" -) - -type PnpmLockPackageResolution struct { - Tarball string `yaml:"tarball"` - Commit string `yaml:"commit"` - Repo string `yaml:"repo"` - Type string `yaml:"type"` -} - -type PnpmLockPackage struct { - Resolution PnpmLockPackageResolution `yaml:"resolution"` - Name string `yaml:"name"` - Version string `yaml:"version"` - Dev bool `yaml:"dev"` -} - -type PnpmLockfile struct { - Version float64 `yaml:"lockfileVersion"` - Packages map[string]PnpmLockPackage `yaml:"packages,omitempty"` -} - -type pnpmLockfileV6 struct { - Version string `yaml:"lockfileVersion"` - Packages map[string]PnpmLockPackage `yaml:"packages,omitempty"` -} - -func (l *PnpmLockfile) UnmarshalYAML(unmarshal func(interface{}) error) error { - var lockfileV6 pnpmLockfileV6 - - if err := unmarshal(&lockfileV6); err != nil { - return err - } - - parsedVersion, err := strconv.ParseFloat(lockfileV6.Version, 64) - - if err != nil { - return err - } - - l.Version = parsedVersion - l.Packages = lockfileV6.Packages - - return nil -} - -const PnpmEcosystem = NpmEcosystem - -func startsWithNumber(str string) bool { - matcher := cachedregexp.MustCompile(`^\d`) - - return matcher.MatchString(str) -} - -// extractPnpmPackageNameAndVersion parses a dependency path, attempting to -// extract the name and version of the package it represents -func extractPnpmPackageNameAndVersion(dependencyPath string, lockfileVersion float64) (string, string) { - // file dependencies must always have a name property to be installed, - // and their dependency path never has the version encoded, so we can - // skip trying to extract either from their dependency path - if strings.HasPrefix(dependencyPath, "file:") { - return "", "" - } - - // v9.0 specifies the dependencies as @ rather than as a path - if lockfileVersion == 9.0 { - dependencyPath = strings.Trim(dependencyPath, "'") - dependencyPath, isScoped := strings.CutPrefix(dependencyPath, "@") - - name, version, _ := strings.Cut(dependencyPath, "@") - - if isScoped { - name = "@" + name - } - - return name, version - } - - parts := strings.Split(dependencyPath, "/") - var name string - - parts = parts[1:] - - if strings.HasPrefix(parts[0], "@") { - name = strings.Join(parts[:2], "/") - parts = parts[2:] - } else { - name = parts[0] - parts = parts[1:] - } - - version := "" - - if len(parts) != 0 { - version = parts[0] - } - - if version == "" { - name, version = parseNameAtVersion(name) - } - - if version == "" || !startsWithNumber(version) { - return "", "" - } - - underscoreIndex := strings.Index(version, "_") - - if underscoreIndex != -1 { - version = strings.Split(version, "_")[0] - } - - return name, version -} - -func parseNameAtVersion(value string) (name string, version string) { - // look for pattern "name@version", where name is allowed to contain zero or more "@" - matches := cachedregexp.MustCompile(`^(.+)@([\d.]+)$`).FindStringSubmatch(value) - - if len(matches) != 3 { - return name, "" - } - - return matches[1], matches[2] -} - -func parsePnpmLock(lockfile PnpmLockfile) []PackageDetails { - packages := make([]PackageDetails, 0, len(lockfile.Packages)) - - for s, pkg := range lockfile.Packages { - name, version := extractPnpmPackageNameAndVersion(s, lockfile.Version) - - // "name" is only present if it's not in the dependency path and takes - // priority over whatever name we think we've extracted (if any) - if pkg.Name != "" { - name = pkg.Name - } - - // "version" is only present if it's not in the dependency path and takes - // priority over whatever version we think we've extracted (if any) - if pkg.Version != "" { - version = pkg.Version - } - - if name == "" || version == "" { - continue - } - - commit := pkg.Resolution.Commit - - if strings.HasPrefix(pkg.Resolution.Tarball, "https://codeload.github.com") { - re := cachedregexp.MustCompile(`https://codeload\.github\.com(?:/[\w-.]+){2}/tar\.gz/(\w+)$`) - matched := re.FindStringSubmatch(pkg.Resolution.Tarball) - - if matched != nil { - commit = matched[1] - } - } - - var depGroups []string - if pkg.Dev { - depGroups = append(depGroups, "dev") - } - - packages = append(packages, PackageDetails{ - Name: name, - Version: version, - Ecosystem: PnpmEcosystem, - CompareAs: PnpmEcosystem, - Commit: commit, - DepGroups: depGroups, - }) - } - - return packages -} - -type PnpmLockExtractor struct{} - -func (e PnpmLockExtractor) ShouldExtract(path string) bool { - return filepath.Base(path) == "pnpm-lock.yaml" -} - -func (e PnpmLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - var parsedLockfile *PnpmLockfile - - err := yaml.NewDecoder(f).Decode(&parsedLockfile) - - if err != nil && !errors.Is(err, io.EOF) { - return []PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err) - } - - // this will happen if the file is empty - if parsedLockfile == nil { - parsedLockfile = &PnpmLockfile{} - } - - return parsePnpmLock(*parsedLockfile), nil -} - -var _ Extractor = PnpmLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("pnpm-lock.yaml", PnpmLockExtractor{}) -} - -// Deprecated: use PnpmLockExtractor.Extract instead -func ParsePnpmLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, PnpmLockExtractor{}) -} diff --git a/pkg/lockfile/parse-pnpm-lock_test.go b/pkg/lockfile/parse-pnpm-lock_test.go deleted file mode 100644 index a08c11596f..0000000000 --- a/pkg/lockfile/parse-pnpm-lock_test.go +++ /dev/null @@ -1,588 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestPnpmLockExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "pnpm-lock.yaml", - want: true, - }, - { - name: "", - path: "path/to/my/pnpm-lock.yaml", - want: true, - }, - { - name: "", - path: "path/to/my/pnpm-lock.yaml/file", - want: false, - }, - { - name: "", - path: "path/to/my/pnpm-lock.yaml.file", - want: false, - }, - { - name: "", - path: "path.to.my.pnpm-lock.yaml", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.PnpmLockExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParsePnpmLock_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePnpmLock_InvalidYaml(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/not-yaml.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePnpmLock_Empty(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/empty.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePnpmLock_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/no-packages.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePnpmLock_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/one-package.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "acorn", - Version: "8.7.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_OnePackageV6Lockfile(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/one-package-v6-lockfile.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "acorn", - Version: "8.7.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_OnePackageDev(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/one-package-dev.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "acorn", - Version: "8.7.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_ScopedPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/scoped-packages.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@typescript-eslint/types", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_ScopedPackagesV6Lockfile(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/scoped-packages-v6-lockfile.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@typescript-eslint/types", - Version: "5.57.1", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_PeerDependencies(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/peer-dependencies.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "acorn-jsx", - Version: "5.3.2", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "acorn", - Version: "8.7.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_PeerDependenciesAdvanced(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/peer-dependencies-advanced.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@typescript-eslint/eslint-plugin", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "@typescript-eslint/parser", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "@typescript-eslint/type-utils", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "@typescript-eslint/types", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "@typescript-eslint/typescript-estree", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "@typescript-eslint/utils", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "eslint-utils", - Version: "3.0.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "eslint", - Version: "8.10.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "tsutils", - Version: "3.21.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_MultiplePackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/multiple-packages.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "aws-sdk", - Version: "2.1087.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "base64-js", - Version: "1.5.1", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "buffer", - Version: "4.9.2", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "events", - Version: "1.1.1", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "ieee754", - Version: "1.1.13", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "isarray", - Version: "1.0.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "jmespath", - Version: "0.16.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "punycode", - Version: "1.3.2", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "querystring", - Version: "0.2.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "sax", - Version: "1.2.1", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "url", - Version: "0.10.3", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "uuid", - Version: "3.3.2", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "xml2js", - Version: "0.4.19", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "xmlbuilder", - Version: "9.0.7", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_MultipleVersions(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/multiple-versions.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "uuid", - Version: "3.3.2", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "uuid", - Version: "8.3.2", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "xmlbuilder", - Version: "9.0.7", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_Tarball(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/tarball.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@my-org/my-package", - Version: "3.2.3", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - Commit: "", - DepGroups: []string{"dev"}, - }, - }) -} - -func TestParsePnpmLock_Exotic(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/exotic.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "foo", - Version: "1.0.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "@foo/bar", - Version: "1.0.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "foo", - Version: "1.1.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "@foo/bar", - Version: "1.1.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "foo", - Version: "1.2.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "foo", - Version: "1.3.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - { - Name: "foo", - Version: "1.4.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - }, - }) -} - -func TestParsePnpmLock_Commits(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/commits.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "my-bitbucket-package", - Version: "1.0.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - Commit: "6104ae42cd32c3d724036d3964678f197b2c9cdb", - }, - { - Name: "@my-scope/my-package", - Version: "1.0.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - Commit: "267087851ad5fac92a184749c27cd539e2fc862e", - }, - { - Name: "@my-scope/my-other-package", - Version: "1.0.0", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - Commit: "fbfc962ab51eb1d754749b68c064460221fbd689", - }, - { - Name: "faker-parser", - Version: "0.0.1", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - Commit: "d2dc42a9351d4d89ec48c525e34f612b6d77993f", - }, - { - Name: "mocks", - Version: "20.0.1", - Ecosystem: lockfile.PnpmEcosystem, - CompareAs: lockfile.PnpmEcosystem, - Commit: "590f321b4eb3f692bb211bd74e22947639a6f79d", - }, - }) -} - -func TestParsePnpmLock_Files(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePnpmLock("fixtures/pnpm/files.yaml") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "my-file-package", - Version: "0.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - { - Name: "a-local-package", - Version: "1.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - { - Name: "a-nested-local-package", - Version: "1.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - { - Name: "one-up", - Version: "1.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - { - Name: "one-up-with-peer", - Version: "1.0.0", - Ecosystem: lockfile.NpmEcosystem, - CompareAs: lockfile.NpmEcosystem, - Commit: "", - }, - }) -} diff --git a/pkg/lockfile/parse-poetry-lock.go b/pkg/lockfile/parse-poetry-lock.go deleted file mode 100644 index ef5f70f45d..0000000000 --- a/pkg/lockfile/parse-poetry-lock.go +++ /dev/null @@ -1,73 +0,0 @@ -package lockfile - -import ( - "fmt" - "path/filepath" - - "github.com/BurntSushi/toml" -) - -type PoetryLockPackageSource struct { - Type string `toml:"type"` - Commit string `toml:"resolved_reference"` -} - -type PoetryLockPackage struct { - Name string `toml:"name"` - Version string `toml:"version"` - Optional bool `toml:"optional"` - Source PoetryLockPackageSource `toml:"source"` -} - -type PoetryLockFile struct { - Version int `toml:"version"` - Packages []PoetryLockPackage `toml:"package"` -} - -const PoetryEcosystem = PipEcosystem - -type PoetryLockExtractor struct{} - -func (e PoetryLockExtractor) ShouldExtract(path string) bool { - return filepath.Base(path) == "poetry.lock" -} - -func (e PoetryLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - var parsedLockfile *PoetryLockFile - - _, err := toml.NewDecoder(f).Decode(&parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err) - } - - packages := make([]PackageDetails, 0, len(parsedLockfile.Packages)) - - for _, lockPackage := range parsedLockfile.Packages { - pkgDetails := PackageDetails{ - Name: lockPackage.Name, - Version: lockPackage.Version, - Commit: lockPackage.Source.Commit, - Ecosystem: PoetryEcosystem, - CompareAs: PoetryEcosystem, - } - if lockPackage.Optional { - pkgDetails.DepGroups = append(pkgDetails.DepGroups, "optional") - } - packages = append(packages, pkgDetails) - } - - return packages, nil -} - -var _ Extractor = PoetryLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("poetry.lock", PoetryLockExtractor{}) -} - -// Deprecated: use PoetryLockExtractor.Extract instead -func ParsePoetryLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, PoetryLockExtractor{}) -} diff --git a/pkg/lockfile/parse-poetry-lock_test.go b/pkg/lockfile/parse-poetry-lock_test.go deleted file mode 100644 index 4b719e5d19..0000000000 --- a/pkg/lockfile/parse-poetry-lock_test.go +++ /dev/null @@ -1,212 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestPoetryLockExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "poetry.lock", - want: true, - }, - { - name: "", - path: "path/to/my/poetry.lock", - want: true, - }, - { - name: "", - path: "path/to/my/poetry.lock/file", - want: false, - }, - { - name: "", - path: "path/to/my/poetry.lock.file", - want: false, - }, - { - name: "", - path: "path.to.my.poetry.lock", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.PoetryLockExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParsePoetryLock_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePoetryLock("fixtures/poetry/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePoetryLock_InvalidToml(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePoetryLock("fixtures/poetry/not-toml.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePoetryLock_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePoetryLock("fixtures/poetry/empty.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePoetryLock_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePoetryLock("fixtures/poetry/one-package.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "numpy", - Version: "1.23.3", - Ecosystem: lockfile.PoetryEcosystem, - CompareAs: lockfile.PoetryEcosystem, - }, - }) -} - -func TestParsePoetryLock_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePoetryLock("fixtures/poetry/two-packages.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "proto-plus", - Version: "1.22.0", - Ecosystem: lockfile.PoetryEcosystem, - CompareAs: lockfile.PoetryEcosystem, - }, - { - Name: "protobuf", - Version: "4.21.5", - Ecosystem: lockfile.PoetryEcosystem, - CompareAs: lockfile.PoetryEcosystem, - }, - }) -} - -func TestParsePoetryLock_PackageWithMetadata(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePoetryLock("fixtures/poetry/one-package-with-metadata.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "emoji", - Version: "2.0.0", - Ecosystem: lockfile.PoetryEcosystem, - CompareAs: lockfile.PoetryEcosystem, - }, - }) -} - -func TestParsePoetryLock_PackageWithGitSource(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePoetryLock("fixtures/poetry/source-git.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "ike", - Version: "0.2.0", - Ecosystem: lockfile.PoetryEcosystem, - CompareAs: lockfile.PoetryEcosystem, - Commit: "cd66602cd29f61a2d2e7fb995fef1e61708c034d", - }, - }) -} - -func TestParsePoetryLock_PackageWithLegacySource(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePoetryLock("fixtures/poetry/source-legacy.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "appdirs", - Version: "1.4.4", - Ecosystem: lockfile.PoetryEcosystem, - CompareAs: lockfile.PoetryEcosystem, - Commit: "", - }, - }) -} - -func TestParsePoetryLock_OptionalPackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePoetryLock("fixtures/poetry/optional-package.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "numpy", - Version: "1.23.3", - Ecosystem: lockfile.PoetryEcosystem, - CompareAs: lockfile.PoetryEcosystem, - DepGroups: []string{"optional"}, - }, - }) -} diff --git a/pkg/lockfile/parse-pubspec-lock.go b/pkg/lockfile/parse-pubspec-lock.go deleted file mode 100644 index 8934a14e66..0000000000 --- a/pkg/lockfile/parse-pubspec-lock.go +++ /dev/null @@ -1,117 +0,0 @@ -package lockfile - -import ( - "errors" - "fmt" - "io" - "path/filepath" - "strings" - - "gopkg.in/yaml.v3" -) - -type PubspecLockDescription struct { - Name string `yaml:"name"` - URL string `yaml:"url"` - Path string `yaml:"path"` - Ref string `yaml:"resolved-ref"` -} - -var _ yaml.Unmarshaler = &PubspecLockDescription{} - -func (pld *PubspecLockDescription) UnmarshalYAML(value *yaml.Node) error { - var m struct { - Name string `yaml:"name"` - URL string `yaml:"url"` - Path string `yaml:"path"` - Ref string `yaml:"resolved-ref"` - } - - err := value.Decode(&m) - - if err == nil { - pld.Name = m.Name - pld.Path = m.Path - pld.URL = m.URL - pld.Ref = m.Ref - - return nil - } - - var str *string - - err = value.Decode(&str) - - if err != nil { - return err - } - - pld.Path = *str - - return nil -} - -type PubspecLockPackage struct { - Source string `yaml:"source"` - Description PubspecLockDescription `yaml:"description"` - Version string `yaml:"version"` - Dependency string `yaml:"dependency"` -} - -type PubspecLockfile struct { - Packages map[string]PubspecLockPackage `yaml:"packages,omitempty"` - Sdks map[string]string `yaml:"sdks"` -} - -const PubEcosystem Ecosystem = "Pub" - -type PubspecLockExtractor struct{} - -func (e PubspecLockExtractor) ShouldExtract(path string) bool { - return filepath.Base(path) == "pubspec.lock" -} - -func (e PubspecLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - var parsedLockfile *PubspecLockfile - - err := yaml.NewDecoder(f).Decode(&parsedLockfile) - - if err != nil && !errors.Is(err, io.EOF) { - return []PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err) - } - if parsedLockfile == nil { - return []PackageDetails{}, nil - } - - packages := make([]PackageDetails, 0, len(parsedLockfile.Packages)) - - for name, pkg := range parsedLockfile.Packages { - pkgDetails := PackageDetails{ - Name: name, - Version: pkg.Version, - Commit: pkg.Description.Ref, - Ecosystem: PubEcosystem, - } - for _, str := range strings.Split(pkg.Dependency, " ") { - if str == "dev" { - pkgDetails.DepGroups = append(pkgDetails.DepGroups, "dev") - break - } - } - packages = append(packages, pkgDetails) - } - - return packages, nil -} - -var _ Extractor = PubspecLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("pubspec.lock", PubspecLockExtractor{}) -} - -// Deprecated: use PubspecLockExtractor.Extract instead -func ParsePubspecLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, PubspecLockExtractor{}) -} diff --git a/pkg/lockfile/parse-pubspec-lock_test.go b/pkg/lockfile/parse-pubspec-lock_test.go deleted file mode 100644 index 4f20df34fb..0000000000 --- a/pkg/lockfile/parse-pubspec-lock_test.go +++ /dev/null @@ -1,276 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestPubspecLockExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "pubspec.lock", - want: true, - }, - { - name: "", - path: "path/to/my/pubspec.lock", - want: true, - }, - { - name: "", - path: "path/to/my/pubspec.lock/file", - want: false, - }, - { - name: "", - path: "path/to/my/pubspec.lock.file", - want: false, - }, - { - name: "", - path: "path.to.my.pubspec.lock", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.PubspecLockExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParsePubspecLock_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePubspecLock("fixtures/pub/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePubspecLock_InvalidYaml(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePubspecLock("fixtures/pub/not-yaml.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePubspecLock_Empty(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePubspecLock("fixtures/pub/empty.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePubspecLock_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePubspecLock("fixtures/pub/no-packages.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParsePubspecLock_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePubspecLock("fixtures/pub/one-package.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "back_button_interceptor", - Version: "6.0.1", - Ecosystem: lockfile.PubEcosystem, - }, - }) -} - -func TestParsePubspecLock_OnePackageDev(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePubspecLock("fixtures/pub/one-package-dev.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "build_runner", - Version: "2.2.1", - Ecosystem: lockfile.PubEcosystem, - DepGroups: []string{"dev"}, - }, - }) -} - -func TestParsePubspecLock_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePubspecLock("fixtures/pub/two-packages.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "shelf", - Version: "1.3.2", - Ecosystem: lockfile.PubEcosystem, - }, - { - Name: "shelf_web_socket", - Version: "1.0.2", - Ecosystem: lockfile.PubEcosystem, - }, - }) -} - -func TestParsePubspecLock_MixedPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePubspecLock("fixtures/pub/mixed-packages.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "back_button_interceptor", - Version: "6.0.1", - Ecosystem: lockfile.PubEcosystem, - }, - { - Name: "build_runner", - Version: "2.2.1", - Ecosystem: lockfile.PubEcosystem, - DepGroups: []string{"dev"}, - }, - { - Name: "shelf", - Version: "1.3.2", - Ecosystem: lockfile.PubEcosystem, - }, - { - Name: "shelf_web_socket", - Version: "1.0.2", - Ecosystem: lockfile.PubEcosystem, - }, - }) -} - -func TestParsePubspecLock_PackageWithGitSource(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePubspecLock("fixtures/pub/source-git.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "flutter_rust_bridge", - Version: "1.32.0", - Ecosystem: lockfile.PubEcosystem, - Commit: "e5adce55eea0b74d3680e66a2c5252edf17b07e1", - }, - { - Name: "screen_retriever", - Version: "0.1.2", - Ecosystem: lockfile.PubEcosystem, - Commit: "406b9b038b2c1d779f1e7bf609c8c248be247372", - }, - { - Name: "tray_manager", - Version: "0.1.8", - Ecosystem: lockfile.PubEcosystem, - Commit: "3aa37c86e47ea748e7b5507cbe59f2c54ebdb23a", - }, - { - Name: "window_manager", - Version: "0.2.7", - Ecosystem: lockfile.PubEcosystem, - Commit: "88487257cbafc501599ab4f82ec343b46acec020", - }, - { - Name: "toggle_switch", - Version: "1.4.0", - Ecosystem: lockfile.PubEcosystem, - Commit: "", - }, - }) -} - -func TestParsePubspecLock_PackageWithSdkSource(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePubspecLock("fixtures/pub/source-sdk.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "flutter_web_plugins", - Version: "0.0.0", - Ecosystem: lockfile.PubEcosystem, - Commit: "", - }, - }) -} - -func TestParsePubspecLock_PackageWithPathSource(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParsePubspecLock("fixtures/pub/source-path.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "maa_core", - Version: "0.0.1", - Ecosystem: lockfile.PubEcosystem, - Commit: "", - }, - }) -} diff --git a/pkg/lockfile/parse-renv-lock.go b/pkg/lockfile/parse-renv-lock.go deleted file mode 100644 index beb8609b15..0000000000 --- a/pkg/lockfile/parse-renv-lock.go +++ /dev/null @@ -1,65 +0,0 @@ -package lockfile - -import ( - "encoding/json" - "fmt" - "path/filepath" -) - -type RenvPackage struct { - Package string `json:"Package"` - Version string `json:"Version"` - Repository string `json:"Repository"` -} - -type RenvLockfile struct { - Packages map[string]RenvPackage `json:"Packages"` -} - -const CRANEcosystem Ecosystem = "CRAN" - -type RenvLockExtractor struct{} - -func (e RenvLockExtractor) ShouldExtract(path string) bool { - return filepath.Base(path) == "renv.lock" -} - -func (e RenvLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - var parsedLockfile *RenvLockfile - - err := json.NewDecoder(f).Decode(&parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err) - } - - packages := make([]PackageDetails, 0, len(parsedLockfile.Packages)) - - for _, pkg := range parsedLockfile.Packages { - // currently we only support CRAN - if pkg.Repository != string(CRANEcosystem) { - continue - } - - packages = append(packages, PackageDetails{ - Name: pkg.Package, - Version: pkg.Version, - Ecosystem: CRANEcosystem, - CompareAs: CRANEcosystem, - }) - } - - return packages, nil -} - -var _ Extractor = RenvLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("renv.lock", RenvLockExtractor{}) -} - -// Deprecated: use RenvLockExtractor.Extract instead -func ParseRenvLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, RenvLockExtractor{}) -} diff --git a/pkg/lockfile/parse-renv-lock_test.go b/pkg/lockfile/parse-renv-lock_test.go deleted file mode 100644 index a4f8eecb53..0000000000 --- a/pkg/lockfile/parse-renv-lock_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestParseRenvLock_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRenvLock("fixtures/renv/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseRenvLock_InvalidJson(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRenvLock("fixtures/renv/not-json.txt") - - expectErrContaining(t, err, "could not extract from") - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseRenvLock_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRenvLock("fixtures/renv/empty.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseRenvLock_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRenvLock("fixtures/renv/one-package.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "morning", - Version: "0.1.0", - Ecosystem: lockfile.CRANEcosystem, - CompareAs: lockfile.CRANEcosystem, - }, - }) -} - -func TestParseRenvLock_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRenvLock("fixtures/renv/two-packages.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "markdown", - Version: "1.0", - Ecosystem: lockfile.CRANEcosystem, - CompareAs: lockfile.CRANEcosystem, - }, - { - Name: "mime", - Version: "0.7", - Ecosystem: lockfile.CRANEcosystem, - CompareAs: lockfile.CRANEcosystem, - }, - }) -} - -func TestParseRenvLock_WithMixedSources(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRenvLock("fixtures/renv/with-mixed-sources.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "markdown", - Version: "1.0", - Ecosystem: lockfile.CRANEcosystem, - CompareAs: lockfile.CRANEcosystem, - }, - }) -} - -func TestParseRenvLock_WithBioconductor(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRenvLock("fixtures/renv/with-bioconductor.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - // currently Bioconductor is not supported - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "BH", - Version: "1.75.0-0", - Ecosystem: lockfile.CRANEcosystem, - CompareAs: lockfile.CRANEcosystem, - }, - }) -} - -func TestParseRenvLock_WithoutRepository(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRenvLock("fixtures/renv/without-repository.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} diff --git a/pkg/lockfile/parse-requirements-txt.go b/pkg/lockfile/parse-requirements-txt.go deleted file mode 100644 index f4a0f88a2f..0000000000 --- a/pkg/lockfile/parse-requirements-txt.go +++ /dev/null @@ -1,211 +0,0 @@ -package lockfile - -import ( - "bufio" - "fmt" - "path/filepath" - "strings" - - "github.com/google/osv-scanner/internal/cachedregexp" - "golang.org/x/exp/maps" -) - -const PipEcosystem Ecosystem = "PyPI" - -// todo: expand this to support more things, e.g. -// -// https://pip.pypa.io/en/stable/reference/requirements-file-format/#example -func parseLine(line string) PackageDetails { - var constraint string - name := line - - version := "0.0.0" - - if strings.Contains(line, "==") { - constraint = "==" - } - - if strings.Contains(line, ">=") { - constraint = ">=" - } - - if strings.Contains(line, "~=") { - constraint = "~=" - } - - if strings.Contains(line, "!=") { - constraint = "!=" - } - - if constraint != "" { - unprocessedName, unprocessedVersion, _ := strings.Cut(line, constraint) - name = strings.TrimSpace(unprocessedName) - - if constraint != "!=" { - version, _, _ = strings.Cut(strings.TrimSpace(unprocessedVersion), " ") - } - } - - return PackageDetails{ - Name: normalizedRequirementName(name), - Version: version, - Ecosystem: PipEcosystem, - CompareAs: PipEcosystem, - } -} - -// normalizedName ensures that the package name is normalized per PEP-0503 -// and then removing "added support" syntax if present. -// -// This is done to ensure we don't miss any advisories, as while the OSV -// specification says that the normalized name should be used for advisories, -// that's not the case currently in our databases, _and_ Pip itself supports -// non-normalized names in the requirements.txt, so we need to normalize -// on both sides to ensure we don't have false negatives. -// -// It's possible that this will cause some false positives, but that is better -// than false negatives, and can be dealt with when/if it actually happens. -func normalizedRequirementName(name string) string { - // per https://www.python.org/dev/peps/pep-0503/#normalized-names - name = cachedregexp.MustCompile(`[-_.]+`).ReplaceAllString(name, "-") - name = strings.ToLower(name) - name, _, _ = strings.Cut(name, "[") - - return name -} - -func removeComments(line string) string { - var re = cachedregexp.MustCompile(`(^|\s+)#.*$`) - - return strings.TrimSpace(re.ReplaceAllString(line, "")) -} - -func isNotRequirementLine(line string) bool { - return line == "" || - // flags are not supported - strings.HasPrefix(line, "-") || - // file urls - strings.HasPrefix(line, "https://") || - strings.HasPrefix(line, "http://") || - // file paths are not supported (relative or absolute) - strings.HasPrefix(line, ".") || - strings.HasPrefix(line, "/") -} - -func isLineContinuation(line string) bool { - // checks that the line ends with an odd number of back slashes, - // meaning the last one isn't escaped - var re = cachedregexp.MustCompile(`([^\\]|^)(\\{2})*\\$`) - - return re.MatchString(line) -} - -type RequirementsTxtExtractor struct{} - -func (e RequirementsTxtExtractor) ShouldExtract(path string) bool { - base := filepath.Base(path) - - return strings.Contains(base, "requirements") && strings.HasSuffix(base, ".txt") -} - -func (e RequirementsTxtExtractor) Extract(f DepFile) ([]PackageDetails, error) { - return parseRequirementsTxt(f, map[string]struct{}{}) -} - -func parseRequirementsTxt(f DepFile, requiredAlready map[string]struct{}) ([]PackageDetails, error) { - packages := map[string]PackageDetails{} - - group := strings.TrimSuffix(filepath.Base(f.Path()), filepath.Ext(f.Path())) - hasGroup := func(groups []string) bool { - for _, g := range groups { - if g == group { - return true - } - } - - return false - } - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - - for isLineContinuation(line) { - line = strings.TrimSuffix(line, "\\") - - if scanner.Scan() { - line += scanner.Text() - } - } - - line = removeComments(line) - if ar := strings.TrimPrefix(line, "-r "); ar != line { - err := func() error { - af, err := f.Open(ar) - - if err != nil { - return fmt.Errorf("failed to include %s: %w", line, err) - } - - defer af.Close() - - if _, ok := requiredAlready[af.Path()]; ok { - return nil - } - - requiredAlready[af.Path()] = struct{}{} - - details, err := parseRequirementsTxt(af, requiredAlready) - - if err != nil { - return fmt.Errorf("failed to include %s: %w", line, err) - } - - for _, detail := range details { - packages[detail.Name+"@"+detail.Version] = detail - } - - return nil - }() - - if err != nil { - return []PackageDetails{}, err - } - - continue - } - - if isNotRequirementLine(line) { - continue - } - - detail := parseLine(line) - key := detail.Name + "@" + detail.Version - if _, ok := packages[key]; !ok { - packages[key] = detail - } - d := packages[key] - if !hasGroup(d.DepGroups) { - d.DepGroups = append(d.DepGroups, group) - packages[key] = d - } - } - - if err := scanner.Err(); err != nil { - return []PackageDetails{}, fmt.Errorf("error while scanning %s: %w", f.Path(), err) - } - - return maps.Values(packages), nil -} - -var _ Extractor = RequirementsTxtExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("requirements.txt", RequirementsTxtExtractor{}) -} - -// Deprecated: use RequirementsTxtExtractor.Extract instead -func ParseRequirementsTxt(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, RequirementsTxtExtractor{}) -} diff --git a/pkg/lockfile/parse-requirements-txt_test.go b/pkg/lockfile/parse-requirements-txt_test.go deleted file mode 100644 index 341e976957..0000000000 --- a/pkg/lockfile/parse-requirements-txt_test.go +++ /dev/null @@ -1,793 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestRequirementsTxtExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "requirements.txt", - want: true, - }, - { - name: "", - path: "path/to/my/requirements.txt", - want: true, - }, - { - name: "", - path: "path/to/my/requirements.txt/file", - want: false, - }, - { - name: "", - path: "path/to/my/requirements.txt.file", - want: false, - }, - { - name: "", - path: "path.to.my.requirements.txt", - want: true, - }, - { - name: "", - path: "requirements-dev.txt", - want: true, - }, - { - name: "", - path: "requirements.dev.txt", - want: true, - }, - { - name: "", - path: "dev-requirements.txt", - want: true, - }, - { - name: "", - path: "requirements-ci.txt", - want: true, - }, - { - name: "", - path: "requirements.ci.txt", - want: true, - }, - { - name: "", - path: "ci-requirements.txt", - want: true, - }, - { - name: "", - path: "dev.txt", - want: false, - }, - { - name: "", - path: "ci.txt", - want: false, - }, - { - name: "", - path: "requirements", - want: false, - }, - { - name: "", - path: "requirements.json", - want: false, - }, - { - name: "", - path: "requirements/txt", - want: false, - }, - { - name: "", - path: "requirements/txt.txt", - want: false, - }, - { - name: "", - path: "path/requirements/txt.txt", - want: false, - }, - { - name: "", - path: "path/requirements/requirements.txt", - want: true, - }, - { - name: "", - path: "path/requirements/requirements-dev.txt", - want: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.RequirementsTxtExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParseRequirementsTxt_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseRequirementsTxt_Empty(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/empty.txt") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseRequirementsTxt_CommentsOnly(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/only-comments.txt") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseRequirementsTxt_OneRequirementUnconstrained(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/one-package-unconstrained.txt") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "flask", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"one-package-unconstrained"}, - }, - }) -} - -func TestParseRequirementsTxt_OneRequirementConstrained(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/one-package-constrained.txt") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "django", - Version: "2.2.24", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"one-package-constrained"}, - }, - }) -} - -func TestParseRequirementsTxt_MultipleRequirementsConstrained(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/multiple-packages-constrained.txt") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "astroid", - Version: "2.5.1", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-constrained"}, - }, - { - Name: "beautifulsoup4", - Version: "4.9.3", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-constrained"}, - }, - { - Name: "boto3", - Version: "1.17.19", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-constrained"}, - }, - { - Name: "botocore", - Version: "1.20.19", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-constrained"}, - }, - { - Name: "certifi", - Version: "2020.12.5", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-constrained"}, - }, - { - Name: "chardet", - Version: "4.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-constrained"}, - }, - { - Name: "circus", - Version: "0.17.1", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-constrained"}, - }, - { - Name: "click", - Version: "7.1.2", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-constrained"}, - }, - { - Name: "django-debug-toolbar", - Version: "3.2.1", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-constrained"}, - }, - { - Name: "django-filter", - Version: "2.4.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-constrained"}, - }, - { - Name: "django-nose", - Version: "1.4.7", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-constrained"}, - }, - { - Name: "django-storages", - Version: "1.11.1", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-constrained"}, - }, - { - Name: "django", - Version: "2.2.24", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-constrained"}, - }, - }) -} - -func TestParseRequirementsTxt_MultipleRequirementsMixed(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/multiple-packages-mixed.txt") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "flask", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed"}, - }, - { - Name: "flask-cors", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed"}, - }, - { - Name: "pandas", - Version: "0.23.4", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed"}, - }, - { - Name: "numpy", - Version: "1.16.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed"}, - }, - { - Name: "scikit-learn", - Version: "0.20.1", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed"}, - }, - { - Name: "sklearn", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed"}, - }, - { - Name: "requests", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed"}, - }, - { - Name: "gevent", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed"}, - }, - }) -} - -func TestParseRequirementsTxt_FileFormatExample(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/file-format-example.txt") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "pytest", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"file-format-example"}, - }, - { - Name: "pytest-cov", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"file-format-example"}, - }, - { - Name: "beautifulsoup4", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"file-format-example"}, - }, - { - Name: "docopt", - Version: "0.6.1", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"file-format-example"}, - }, - { - Name: "keyring", - Version: "4.1.1", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"file-format-example"}, - }, - { - Name: "coverage", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"file-format-example"}, - }, - { - Name: "mopidy-dirble", - Version: "1.1", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"file-format-example"}, - }, - { - Name: "rejected", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"file-format-example"}, - }, - { - Name: "green", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"file-format-example"}, - }, - { - Name: "django", - Version: "2.2.24", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"other-file"}, - }, - }) -} - -func TestParseRequirementsTxt_WithAddedSupport(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/with-added-support.txt") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "twisted", - Version: "20.3.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"with-added-support"}, - }, - }) -} - -func TestParseRequirementsTxt_NonNormalizedNames(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/non-normalized-names.txt") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "zope-interface", - Version: "5.4.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"non-normalized-names"}, - }, - { - Name: "pillow", - Version: "1.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"non-normalized-names"}, - }, - { - Name: "twisted", - Version: "20.3.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"non-normalized-names"}, - }, - }) -} - -func TestParseRequirementsTxt_WithMultipleROptions(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/with-multiple-r-options.txt") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "flask", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed"}, - }, - { - Name: "flask-cors", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed"}, - }, - { - Name: "pandas", - Version: "0.23.4", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed", "with-multiple-r-options"}, - }, - { - Name: "numpy", - Version: "1.16.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed"}, - }, - { - Name: "scikit-learn", - Version: "0.20.1", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed"}, - }, - { - Name: "sklearn", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed"}, - }, - { - Name: "requests", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed"}, - }, - { - Name: "gevent", - Version: "0.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"multiple-packages-mixed"}, - }, - { - Name: "requests", - Version: "1.2.3", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"with-multiple-r-options"}, - }, - { - Name: "django", - Version: "2.2.24", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"one-package-constrained"}, - }, - }) -} - -func TestParseRequirementsTxt_WithBadROption(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/with-bad-r-option.txt") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseRequirementsTxt_DuplicateROptions(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/duplicate-r-dev.txt") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "django", - Version: "0.1.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"duplicate-r-base"}, - }, - { - Name: "pandas", - Version: "0.23.4", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"duplicate-r-dev"}, - }, - { - Name: "requests", - Version: "1.2.3", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"duplicate-r-test", "duplicate-r-dev"}, - }, - { - Name: "unittest", - Version: "1.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"duplicate-r-test"}, - }, - }) -} - -func TestParseRequirementsTxt_CyclicRSelf(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/cyclic-r-self.txt") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "pandas", - Version: "0.23.4", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"cyclic-r-self"}, - }, - { - Name: "requests", - Version: "1.2.3", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"cyclic-r-self"}, - }, - }) -} - -func TestParseRequirementsTxt_CyclicRComplex(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/cyclic-r-complex-1.txt") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "cyclic-r-complex", - Version: "1", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"cyclic-r-complex-1"}, - }, - { - Name: "cyclic-r-complex", - Version: "2", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"cyclic-r-complex-2"}, - }, - { - Name: "cyclic-r-complex", - Version: "3", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"cyclic-r-complex-3"}, - }, - }) -} - -func TestParseRequirementsTxt_WithPerRequirementOptions(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/with-per-requirement-options.txt") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "boto3", - Version: "1.26.121", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"with-per-requirement-options"}, - }, - { - Name: "foo", - Version: "1.0.0", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"with-per-requirement-options"}, - }, - { - Name: "fooproject", - Version: "1.2", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"with-per-requirement-options"}, - }, - { - Name: "barproject", - Version: "1.2", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"with-per-requirement-options"}, - }, - }) -} - -func TestParseRequirementsTxt_LineContinuation(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/line-continuation.txt") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "foo", - Version: "1.2.3", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"line-continuation"}, - }, - { - Name: "bar", - Version: "4.5\\\\", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"line-continuation"}, - }, - { - Name: "baz", - Version: "7.8.9", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"line-continuation"}, - }, - { - Name: "qux", - Version: "10.11.12", - Ecosystem: lockfile.PipEcosystem, - CompareAs: lockfile.PipEcosystem, - DepGroups: []string{"line-continuation"}, - }, - }) -} diff --git a/pkg/lockfile/parse-yarn-lock-v1_test.go b/pkg/lockfile/parse-yarn-lock-v1_test.go deleted file mode 100644 index 459672fd28..0000000000 --- a/pkg/lockfile/parse-yarn-lock-v1_test.go +++ /dev/null @@ -1,478 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestParseYarnLock_v1_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseYarnLock_v1_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/empty.v1.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseYarnLock_v1_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/one-package.v1.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "balanced-match", - Version: "1.0.2", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} - -func TestParseYarnLock_v1_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/two-packages.v1.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "concat-stream", - Version: "1.6.2", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "concat-map", - Version: "0.0.1", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} - -func TestParseYarnLock_v1_WithQuotes(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/with-quotes.v1.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "concat-stream", - Version: "1.6.2", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "concat-map", - Version: "0.0.1", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} - -func TestParseYarnLock_v1_MultipleVersions(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/multiple-versions.v1.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "define-properties", - Version: "1.1.3", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "define-property", - Version: "0.2.5", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "define-property", - Version: "1.0.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "define-property", - Version: "2.0.2", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} - -func TestParseYarnLock_v1_MultipleConstraints(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/multiple-constraints.v1.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@babel/code-frame", - Version: "7.12.13", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "domelementtype", - Version: "1.3.1", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} - -func TestParseYarnLock_v1_ScopedPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/scoped-packages.v1.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@babel/code-frame", - Version: "7.12.11", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "@babel/compat-data", - Version: "7.14.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} - -func TestParseYarnLock_v1_WithPrerelease(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/with-prerelease.v1.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "css-tree", - Version: "1.0.0-alpha.37", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "gensync", - Version: "1.0.0-beta.2", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "node-fetch", - Version: "3.0.0-beta.9", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "resolve", - Version: "1.20.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "resolve", - Version: "2.0.0-next.3", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} - -func TestParseYarnLock_v1_WithBuildString(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/with-build-string.v1.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "domino", - Version: "2.1.6+git", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "tslib", - Version: "2.6.2", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} - -func TestParseYarnLock_v1_Commits(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/commits.v1.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "mine1", - Version: "1.0.0-alpha.37", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "0a2d2506c1fe299691fc5db53a2097db3bd615bc", - }, - { - Name: "mine2", - Version: "0.0.1", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "0a2d2506c1fe299691fc5db53a2097db3bd615bc", - }, - { - Name: "mine3", - Version: "1.2.3", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "094e581aaf927d010e4b61d706ba584551dac502", - }, - { - Name: "mine4", - Version: "0.0.2", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "aa3bdfcb1d845c79f14abb66f60d35b8a3ee5998", - }, - { - Name: "mine4", - Version: "0.0.4", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "aa3bdfcb1d845c79f14abb66f60d35b8a3ee5998", - }, - { - Name: "my-package", - Version: "1.8.3", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "b3bd3f1b3dad036e671251f5258beaae398f983a", - }, - { - Name: "@bower_components/angular-animate", - Version: "1.4.14", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "e7f778fc054a086ba3326d898a00fa1bc78650a8", - }, - { - Name: "@bower_components/alertify", - Version: "0.0.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "e7b6c46d76604d297c389d830817b611c9a8f17c", - }, - { - Name: "minimist", - Version: "0.0.8", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "3754568bfd43a841d2d72d7fb54598635aea8fa4", - }, - { - Name: "bats-assert", - Version: "2.0.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "4bdd58d3fbcdce3209033d44d884e87add1d8405", - }, - { - Name: "bats-support", - Version: "0.3.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "d140a65044b2d6810381935ae7f0c94c7023c8c3", - }, - { - Name: "bats", - Version: "1.5.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "172580d2ce19ee33780b5f1df817bbddced43789", - }, - { - Name: "vue", - Version: "2.6.12", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "bb253db0b3e17124b6d1fe93fbf2db35470a1347", - }, - { - Name: "kit", - Version: "1.0.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "5b6830c0252eb73c6024d40a8ff5106d3023a2a6", - }, - { - Name: "casadistance", - Version: "1.0.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "f0308391f0c50104182bfb2332a53e4e523a4603", - }, - { - Name: "babel-preset-php", - Version: "1.1.1", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "c5a7ba5e0ad98b8db1cb8ce105403dd4b768cced", - }, - { - Name: "is-number", - Version: "2.0.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", - }, - { - Name: "is-number", - Version: "5.0.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", - }, - }) -} - -func TestParseYarnLock_v1_Files(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/files.v1.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "etag", - Version: "1.8.1", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "", - }, - { - Name: "filedep", - Version: "1.2.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "", - }, - { - Name: "lodash", - Version: "1.3.1", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "", - }, - { - Name: "other_package", - Version: "0.0.2", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "", - }, - { - Name: "sprintf-js", - Version: "0.0.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "", - }, - { - Name: "etag", - Version: "1.8.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "", - }, - }) -} - -func TestParseYarnLock_v1_WithAliases(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/with-aliases.v1.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@babel/helper-validator-identifier", - Version: "7.22.20", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "ansi-regex", - Version: "6.0.1", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "ansi-regex", - Version: "5.0.1", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} diff --git a/pkg/lockfile/parse-yarn-lock-v2_test.go b/pkg/lockfile/parse-yarn-lock-v2_test.go deleted file mode 100644 index 64cde52719..0000000000 --- a/pkg/lockfile/parse-yarn-lock-v2_test.go +++ /dev/null @@ -1,342 +0,0 @@ -package lockfile_test - -import ( - "io/fs" - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestParseYarnLock_v2_FileDoesNotExist(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/does-not-exist") - - expectErrIs(t, err, fs.ErrNotExist) - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseYarnLock_v2_NoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/empty.v2.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{}) -} - -func TestParseYarnLock_v2_OnePackage(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/one-package.v2.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "balanced-match", - Version: "1.0.2", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} - -func TestParseYarnLock_v2_TwoPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/two-packages.v2.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "compare-func", - Version: "2.0.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "concat-map", - Version: "0.0.1", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} - -func TestParseYarnLock_v2_WithQuotes(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/with-quotes.v2.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "compare-func", - Version: "2.0.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "concat-map", - Version: "0.0.1", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} - -func TestParseYarnLock_v2_MultipleVersions(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/multiple-versions.v2.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "debug", - Version: "4.3.3", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "debug", - Version: "2.6.9", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "debug", - Version: "3.2.7", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} - -func TestParseYarnLock_v2_ScopedPackages(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/scoped-packages.v2.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@babel/cli", - Version: "7.16.8", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "@babel/code-frame", - Version: "7.16.7", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "@babel/compat-data", - Version: "7.16.8", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} - -func TestParseYarnLock_v2_WithPrerelease(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/with-prerelease.v2.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@nicolo-ribaudo/chokidar-2", - Version: "2.1.8-no-fsevents.3", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "gensync", - Version: "1.0.0-beta.2", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "eslint-plugin-jest", - Version: "0.0.0-use.local", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} - -func TestParseYarnLock_v2_WithBuildString(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/with-build-string.v2.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "domino", - Version: "2.1.6+git", - Commit: "f2435fe1f9f7c91ade0bd472c4723e5eacd7d19a", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "tslib", - Version: "2.6.2", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "zone.js", - Version: "0.0.0-use.local", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} - -func TestParseYarnLock_v2_Commits(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/commits.v2.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@my-scope/my-first-package", - Version: "0.0.6", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "0b824c650d3a03444dbcf2b27a5f3566f6e41358", - }, - { - Name: "my-second-package", - Version: "0.2.2", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "59e2127b9f9d4fda5f928c4204213b3502cd5bb0", - }, - { - Name: "@typegoose/typegoose", - Version: "7.2.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "3ed06e5097ab929f69755676fee419318aaec73a", - }, - { - Name: "vuejs", - Version: "2.5.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "0948d999f2fddf9f90991956493f976273c5da1f", - }, - { - Name: "my-third-package", - Version: "0.16.1-dev", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "5675a0aed98e067ff6ecccc5ac674fe8995960e0", - }, - { - Name: "my-node-sdk", - Version: "1.1.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "053dea9e0b8af442d8f867c8e690d2fb0ceb1bf5", - }, - { - Name: "is-really-great", - Version: "1.0.0", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "191eeef50c584714e1fb8927d17ee72b3b8c97c4", - }, - }) -} - -func TestParseYarnLock_v2_Files(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/files.v2.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "my-package", - Version: "0.0.2", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - Commit: "", - }, - }) -} - -func TestParseYarnLock_v2_WithAliases(t *testing.T) { - t.Parallel() - - packages, err := lockfile.ParseYarnLock("fixtures/yarn/with-aliases.v2.lock") - - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - - expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "@babel/helper-validator-identifier", - Version: "7.22.20", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "ansi-regex", - Version: "6.0.1", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "ansi-regex", - Version: "5.0.1", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - { - Name: "mine", - Version: "0.0.0-use.local", - Ecosystem: lockfile.YarnEcosystem, - CompareAs: lockfile.YarnEcosystem, - }, - }) -} diff --git a/pkg/lockfile/parse-yarn-lock.go b/pkg/lockfile/parse-yarn-lock.go deleted file mode 100644 index 1534e3ace3..0000000000 --- a/pkg/lockfile/parse-yarn-lock.go +++ /dev/null @@ -1,218 +0,0 @@ -package lockfile - -import ( - "bufio" - "fmt" - "net/url" - "os" - "path/filepath" - "strings" - - "github.com/google/osv-scanner/internal/cachedregexp" -) - -const YarnEcosystem = NpmEcosystem - -func shouldSkipYarnLine(line string) bool { - return line == "" || strings.HasPrefix(line, "#") -} - -func groupYarnPackageLines(scanner *bufio.Scanner) [][]string { - var groups [][]string - var group []string - - for scanner.Scan() { - line := scanner.Text() - - if shouldSkipYarnLine(line) { - continue - } - - // represents the start of a new dependency - if !strings.HasPrefix(line, " ") { - if len(group) > 0 { - groups = append(groups, group) - } - group = make([]string, 0) - } - - group = append(group, line) - } - - if len(group) > 0 { - groups = append(groups, group) - } - - return groups -} - -func extractYarnPackageName(str string) string { - str = strings.TrimPrefix(str, "\"") - str, _, _ = strings.Cut(str, ",") - - isScoped := strings.HasPrefix(str, "@") - - if isScoped { - str = strings.TrimPrefix(str, "@") - } - - name, right, _ := strings.Cut(str, "@") - - if strings.HasPrefix(right, "npm:") && strings.Contains(right, "@") { - return extractYarnPackageName(strings.TrimPrefix(right, "npm:")) - } - - if isScoped { - name = "@" + name - } - - return name -} - -func determineYarnPackageVersion(group []string) string { - re := cachedregexp.MustCompile(`^ {2}"?version"?:? "?([\w-.+]+)"?$`) - - for _, s := range group { - matched := re.FindStringSubmatch(s) - - if matched != nil { - return matched[1] - } - } - - // todo: decide what to do here - maybe panic...? - return "" -} - -func determineYarnPackageResolution(group []string) string { - re := cachedregexp.MustCompile(`^ {2}"?(?:resolution:|resolved)"? "([^ '"]+)"$`) - - for _, s := range group { - matched := re.FindStringSubmatch(s) - - if matched != nil { - return matched[1] - } - } - - // todo: decide what to do here - maybe panic...? - return "" -} - -func tryExtractCommit(resolution string) string { - // language=GoRegExp - matchers := []string{ - // ssh://... - // git://... - // git+ssh://... - // git+https://... - `(?:^|.+@)(?:git(?:\+(?:ssh|https))?|ssh)://.+#(\w+)$`, - // https://....git/... - `(?:^|.+@)https://.+\.git#(\w+)$`, - `https://codeload\.github\.com(?:/[\w-.]+){2}/tar\.gz/(\w+)$`, - `.+#commit[:=](\w+)$`, - // github:... - // gitlab:... - // bitbucket:... - `^(?:github|gitlab|bitbucket):.+#(\w+)$`, - } - - for _, matcher := range matchers { - re := cachedregexp.MustCompile(matcher) - matched := re.FindStringSubmatch(resolution) - - if matched != nil { - return matched[1] - } - } - - u, err := url.Parse(resolution) - - if err == nil { - gitRepoHosts := []string{ - "bitbucket.org", - "github.com", - "gitlab.com", - } - - for _, host := range gitRepoHosts { - if u.Host != host { - continue - } - - if u.RawQuery != "" { - queries := u.Query() - - if queries.Has("ref") { - return queries.Get("ref") - } - } - - return u.Fragment - } - } - - return "" -} - -func parseYarnPackageGroup(group []string) PackageDetails { - name := extractYarnPackageName(group[0]) - version := determineYarnPackageVersion(group) - resolution := determineYarnPackageResolution(group) - - if version == "" { - _, _ = fmt.Fprintf( - os.Stderr, - "Failed to determine version of %s while parsing a yarn.lock - please report this!\n", - name, - ) - } - - return PackageDetails{ - Name: name, - Version: version, - Ecosystem: YarnEcosystem, - CompareAs: YarnEcosystem, - Commit: tryExtractCommit(resolution), - } -} - -type YarnLockExtractor struct{} - -func (e YarnLockExtractor) ShouldExtract(path string) bool { - return filepath.Base(path) == "yarn.lock" -} - -func (e YarnLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { - scanner := bufio.NewScanner(f) - - packageGroups := groupYarnPackageLines(scanner) - - if err := scanner.Err(); err != nil { - return []PackageDetails{}, fmt.Errorf("error while scanning %s: %w", f.Path(), err) - } - - packages := make([]PackageDetails, 0, len(packageGroups)) - - for _, group := range packageGroups { - if group[0] == "__metadata:" { - continue - } - - packages = append(packages, parseYarnPackageGroup(group)) - } - - return packages, nil -} - -var _ Extractor = YarnLockExtractor{} - -//nolint:gochecknoinits -func init() { - registerExtractor("yarn.lock", YarnLockExtractor{}) -} - -// Deprecated: use YarnLockExtractor.Extract instead -func ParseYarnLock(pathToLockfile string) ([]PackageDetails, error) { - return extractFromFile(pathToLockfile, YarnLockExtractor{}) -} diff --git a/pkg/lockfile/parse-yarn-lock_test.go b/pkg/lockfile/parse-yarn-lock_test.go deleted file mode 100644 index 854436d47e..0000000000 --- a/pkg/lockfile/parse-yarn-lock_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package lockfile_test - -import ( - "testing" - - "github.com/google/osv-scanner/pkg/lockfile" -) - -func TestYarnLockExtractor_ShouldExtract(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - path string - want bool - }{ - { - name: "", - path: "", - want: false, - }, - { - name: "", - path: "yarn.lock", - want: true, - }, - { - name: "", - path: "path/to/my/yarn.lock", - want: true, - }, - { - name: "", - path: "path/to/my/yarn.lock/file", - want: false, - }, - { - name: "", - path: "path/to/my/yarn.lock.file", - want: false, - }, - { - name: "", - path: "path.to.my.yarn.lock", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - e := lockfile.YarnLockExtractor{} - got := e.ShouldExtract(tt.path) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/lockfile/parse.go b/pkg/lockfile/parse.go deleted file mode 100644 index af55f1e627..0000000000 --- a/pkg/lockfile/parse.go +++ /dev/null @@ -1,146 +0,0 @@ -package lockfile - -import ( - "errors" - "fmt" - "path/filepath" - "sort" - "strings" - - "golang.org/x/exp/maps" -) - -func FindParser(pathToLockfile string, parseAs string) (PackageDetailsParser, string) { - if parseAs == "" { - parseAs = filepath.Base(pathToLockfile) - } - - return parsers[parseAs], parseAs -} - -// this is an optimisation and read-only -var parsers = map[string]PackageDetailsParser{ - "buildscript-gradle.lockfile": ParseGradleLock, - "Cargo.lock": ParseCargoLock, - "composer.lock": ParseComposerLock, - "conan.lock": ParseConanLock, - "Gemfile.lock": ParseGemfileLock, - "go.mod": ParseGoLock, - "verification-metadata.xml": ParseGradleVerificationMetadata, - "gradle.lockfile": ParseGradleLock, - "mix.lock": ParseMixLock, - "Pipfile.lock": ParsePipenvLock, - "package-lock.json": ParseNpmLock, - "packages.lock.json": ParseNuGetLock, - "pdm.lock": ParsePdmLock, - "pnpm-lock.yaml": ParsePnpmLock, - "poetry.lock": ParsePoetryLock, - "pom.xml": ParseMavenLock, - "pubspec.lock": ParsePubspecLock, - "renv.lock": ParseRenvLock, - "requirements.txt": ParseRequirementsTxt, - "yarn.lock": ParseYarnLock, -} - -func ListParsers() []string { - ps := make([]string, 0, len(parsers)) - - for s := range parsers { - ps = append(ps, s) - } - - sort.Slice(ps, func(i, j int) bool { - return strings.ToLower(ps[i]) < strings.ToLower(ps[j]) - }) - - return ps -} - -var ErrParserNotFound = errors.New("could not determine parser") - -type Packages []PackageDetails - -func (ps Packages) Ecosystems() []Ecosystem { - ecosystems := make(map[Ecosystem]struct{}) - - for _, pkg := range ps { - ecosystems[pkg.Ecosystem] = struct{}{} - } - - slicedEcosystems := maps.Keys(ecosystems) - - sort.Slice(slicedEcosystems, func(i, j int) bool { - return slicedEcosystems[i] < slicedEcosystems[j] - }) - - return slicedEcosystems -} - -type Lockfile struct { - FilePath string `json:"filePath"` - ParsedAs string `json:"parsedAs"` - Packages Packages `json:"packages"` -} - -func (l Lockfile) String() string { - lines := make([]string, 0, len(l.Packages)) - - for _, details := range l.Packages { - ecosystem := details.Ecosystem - - if ecosystem == "" { - ecosystem = "" - } - - ln := fmt.Sprintf(" %s: %s", ecosystem, details.Name) - - if details.Version != "" { - ln += "@" + details.Version - } - - if details.Commit != "" { - ln += " (" + details.Commit + ")" - } - - lines = append(lines, ln) - } - - return strings.Join(lines, "\n") -} - -// Parse attempts to extract a collection of package details from a lockfile, -// using one of the native parsers. -// -// The parser is selected based on the name of the file, which can be overridden -// with the "parseAs" parameter. -func Parse(pathToLockfile string, parseAs string) (Lockfile, error) { - parser, parsedAs := FindParser(pathToLockfile, parseAs) - - if parser == nil { - if parseAs != "" { - return Lockfile{}, fmt.Errorf("%w, requested %s", ErrParserNotFound, parseAs) - } - - return Lockfile{}, fmt.Errorf("%w for %s", ErrParserNotFound, pathToLockfile) - } - - packages, err := parser(pathToLockfile) - - if err != nil && parseAs != "" { - err = fmt.Errorf("(extracting as %s) %w", parsedAs, err) - } - - sort.Slice(packages, func(i, j int) bool { - if packages[i].Name == packages[j].Name { - return packages[i].Version < packages[j].Version - } - - return packages[i].Name < packages[j].Name - }) - - return Lockfile{ - FilePath: pathToLockfile, - ParsedAs: parsedAs, - Packages: packages, - }, err -} diff --git a/pkg/lockfile/parse_test.go b/pkg/lockfile/parse_test.go index a13d02310c..e7a600acaa 100644 --- a/pkg/lockfile/parse_test.go +++ b/pkg/lockfile/parse_test.go @@ -1,316 +1,316 @@ package lockfile_test -import ( - "errors" - "os" - "reflect" - "strings" - "testing" - - "github.com/google/osv-scanner/internal/output" - "github.com/google/osv-scanner/pkg/lockfile" -) - -func expectNumberOfParsersCalled(t *testing.T, numberOfParsersCalled int) { - t.Helper() - - directories, err := os.ReadDir(".") - - if err != nil { - t.Fatalf("unable to read current directory: ") - } - - count := 0 - - for _, directory := range directories { - if strings.HasPrefix(directory.Name(), "parse-") && - !strings.HasSuffix(directory.Name(), "_test.go") { - count++ - } - } - - if numberOfParsersCalled != count { - t.Errorf( - "Expected %d %s to have been called, but had %d", - count, - output.Form(count, "parser", "parsers"), - numberOfParsersCalled, - ) - } -} - -func TestFindParser(t *testing.T) { - t.Parallel() - - lockfiles := []string{ - "buildscript-gradle.lockfile", - "Cargo.lock", - "composer.lock", - "Gemfile.lock", - "go.mod", - "gradle.lockfile", - "mix.lock", - "pdm.lock", - "Pipfile.lock", - "package-lock.json", - "packages.lock.json", - "pnpm-lock.yaml", - "poetry.lock", - "pom.xml", - "pubspec.lock", - "renv.lock", - "requirements.txt", - "yarn.lock", - } - - for _, file := range lockfiles { - parser, parsedAs := lockfile.FindParser("/path/to/my/"+file, "") - - if parser == nil { - t.Errorf("Expected a parser to be found for %s but did not", file) - } - - if file != parsedAs { - t.Errorf("Expected parsedAs to be %s but got %s instead", file, parsedAs) - } - } -} - -func TestFindParser_ExplicitParseAs(t *testing.T) { - t.Parallel() - - parser, parsedAs := lockfile.FindParser("/path/to/my/package-lock.json", "composer.lock") - - if parser == nil { - t.Errorf("Expected a parser to be found for package-lock.json (overridden as composer.lock) but did not") - } - - if parsedAs != "composer.lock" { - t.Errorf("Expected parsedAs to be composer.lock but got %s instead", parsedAs) - } -} - -func TestParse_FindsExpectedParsers(t *testing.T) { - t.Parallel() - - lockfiles := []string{ - "buildscript-gradle.lockfile", - "Cargo.lock", - "composer.lock", - "conan.lock", - "Gemfile.lock", - "go.mod", - "gradle/verification-metadata.xml", - "gradle.lockfile", - "mix.lock", - "Pipfile.lock", - "pdm.lock", - "package-lock.json", - "packages.lock.json", - "pnpm-lock.yaml", - "poetry.lock", - "pom.xml", - "pubspec.lock", - "renv.lock", - "requirements.txt", - "yarn.lock", - } - - count := 0 - - for _, file := range lockfiles { - _, err := lockfile.Parse("/path/to/my/"+file, "") - - if errors.Is(err, lockfile.ErrParserNotFound) { - t.Errorf("No parser was found for %s", file) - } - - count++ - } - - // gradle.lockfile and buildscript-gradle.lockfile use the same parser - count -= 1 - - expectNumberOfParsersCalled(t, count) -} - -func TestParse_ParserNotFound(t *testing.T) { - t.Parallel() - - _, err := lockfile.Parse("/path/to/my/", "") - - expectErrIs(t, err, lockfile.ErrParserNotFound) -} - -func TestParse_ParserNotFound_WithExplicitParseAs(t *testing.T) { - t.Parallel() - - _, err := lockfile.Parse("/path/to/my/", "unsupported") - - if err == nil { - t.Errorf("Expected to get an error but did not") - } - - if !errors.Is(err, lockfile.ErrParserNotFound) { - t.Errorf("Did not get the expected ErrParserNotFound error - got %v instead", err) - } -} - -func TestListParsers(t *testing.T) { - t.Parallel() - - parsers := lockfile.ListParsers() - - firstExpected := "buildscript-gradle.lockfile" - //nolint:ifshort - lastExpected := "yarn.lock" - - if first := parsers[0]; first != firstExpected { - t.Errorf("Expected first element to be %s, but got %s", firstExpected, first) - } - - if last := parsers[len(parsers)-1]; last != lastExpected { - t.Errorf("Expected last element to be %s, but got %s", lastExpected, last) - } -} - -func TestLockfile_String(t *testing.T) { - t.Parallel() - - expected := strings.Join([]string{ - " crates.io: addr2line@0.15.2", - " npm: @typescript-eslint/types@5.13.0", - " crates.io: wasi@0.10.2+wasi-snapshot-preview1", - " Packagist: sentry/sdk@2.0.4", - " crates.io: no-version", - " : no-ecosystem@1.2.3", - " : no-ecosystem@1.2.3 (with-commit)", - }, "\n") - - lockf := lockfile.Lockfile{ - Packages: []lockfile.PackageDetails{ - { - Name: "addr2line", - Version: "0.15.2", - Ecosystem: lockfile.CargoEcosystem, - }, - { - Name: "@typescript-eslint/types", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - }, - { - Name: "wasi", - Version: "0.10.2+wasi-snapshot-preview1", - Ecosystem: lockfile.CargoEcosystem, - }, - { - Name: "sentry/sdk", - Version: "2.0.4", - Ecosystem: lockfile.ComposerEcosystem, - }, - { - Name: "no-version", - Version: "", - Ecosystem: lockfile.CargoEcosystem, - }, - { - Name: "no-ecosystem", - Version: "1.2.3", - Ecosystem: "", - }, - { - Name: "no-ecosystem", - Version: "1.2.3", - Ecosystem: "", - Commit: "with-commit", - }, - }, - } - - if actual := lockf.String(); expected != actual { - t.Errorf("\nExpected:\n%s\nActual:\n%s", expected, actual) - } -} - -func TestPackages_Ecosystems(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - ps lockfile.Packages - want []lockfile.Ecosystem - }{ - {name: "", ps: lockfile.Packages{}, want: []lockfile.Ecosystem{}}, - { - name: "", - ps: lockfile.Packages{ - { - Name: "addr2line", - Version: "0.15.2", - Ecosystem: lockfile.CargoEcosystem, - }, - }, - want: []lockfile.Ecosystem{ - lockfile.CargoEcosystem, - }, - }, - { - name: "", - ps: lockfile.Packages{ - { - Name: "addr2line", - Version: "0.15.2", - Ecosystem: lockfile.CargoEcosystem, - }, - { - Name: "wasi", - Version: "0.10.2+wasi-snapshot-preview1", - Ecosystem: lockfile.CargoEcosystem, - }, - }, - want: []lockfile.Ecosystem{ - lockfile.CargoEcosystem, - }, - }, - { - name: "", - ps: lockfile.Packages{ - { - Name: "addr2line", - Version: "0.15.2", - Ecosystem: lockfile.CargoEcosystem, - }, - { - Name: "@typescript-eslint/types", - Version: "5.13.0", - Ecosystem: lockfile.PnpmEcosystem, - }, - { - Name: "wasi", - Version: "0.10.2+wasi-snapshot-preview1", - Ecosystem: lockfile.CargoEcosystem, - }, - { - Name: "sentry/sdk", - Version: "2.0.4", - Ecosystem: lockfile.ComposerEcosystem, - }, - }, - want: []lockfile.Ecosystem{ - lockfile.ComposerEcosystem, - lockfile.CargoEcosystem, - lockfile.PnpmEcosystem, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - if got := tt.ps.Ecosystems(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("Ecosystems() = %v, want %v", got, tt.want) - } - }) - } -} +// import ( +// "errors" +// "os" +// "reflect" +// "strings" +// "testing" + +// "github.com/google/osv-scanner/internal/output" +// "github.com/google/osv-scanner/pkg/lockfile" +// ) + +// func expectNumberOfParsersCalled(t *testing.T, numberOfParsersCalled int) { +// t.Helper() + +// directories, err := os.ReadDir(".") + +// if err != nil { +// t.Fatalf("unable to read current directory: ") +// } + +// count := 0 + +// for _, directory := range directories { +// if strings.HasPrefix(directory.Name(), "parse-") && +// !strings.HasSuffix(directory.Name(), "_test.go") { +// count++ +// } +// } + +// if numberOfParsersCalled != count { +// t.Errorf( +// "Expected %d %s to have been called, but had %d", +// count, +// output.Form(count, "parser", "parsers"), +// numberOfParsersCalled, +// ) +// } +// } + +// func TestFindParser(t *testing.T) { +// t.Parallel() + +// lockfiles := []string{ +// "buildscript-gradle.lockfile", +// "Cargo.lock", +// "composer.lock", +// "Gemfile.lock", +// "go.mod", +// "gradle.lockfile", +// "mix.lock", +// "pdm.lock", +// "Pipfile.lock", +// "package-lock.json", +// "packages.lock.json", +// "pnpm-lock.yaml", +// "poetry.lock", +// "pom.xml", +// "pubspec.lock", +// "renv.lock", +// "requirements.txt", +// "yarn.lock", +// } + +// for _, file := range lockfiles { +// parser, parsedAs := lockfile.FindParser("/path/to/my/"+file, "") + +// if parser == nil { +// t.Errorf("Expected a parser to be found for %s but did not", file) +// } + +// if file != parsedAs { +// t.Errorf("Expected parsedAs to be %s but got %s instead", file, parsedAs) +// } +// } +// } + +// func TestFindParser_ExplicitParseAs(t *testing.T) { +// t.Parallel() + +// parser, parsedAs := lockfile.FindParser("/path/to/my/package-lock.json", "composer.lock") + +// if parser == nil { +// t.Errorf("Expected a parser to be found for package-lock.json (overridden as composer.lock) but did not") +// } + +// if parsedAs != "composer.lock" { +// t.Errorf("Expected parsedAs to be composer.lock but got %s instead", parsedAs) +// } +// } + +// func TestParse_FindsExpectedParsers(t *testing.T) { +// t.Parallel() + +// lockfiles := []string{ +// "buildscript-gradle.lockfile", +// "Cargo.lock", +// "composer.lock", +// "conan.lock", +// "Gemfile.lock", +// "go.mod", +// "gradle/verification-metadata.xml", +// "gradle.lockfile", +// "mix.lock", +// "Pipfile.lock", +// "pdm.lock", +// "package-lock.json", +// "packages.lock.json", +// "pnpm-lock.yaml", +// "poetry.lock", +// "pom.xml", +// "pubspec.lock", +// "renv.lock", +// "requirements.txt", +// "yarn.lock", +// } + +// count := 0 + +// for _, file := range lockfiles { +// _, err := lockfile.Parse("/path/to/my/"+file, "") + +// if errors.Is(err, lockfile.ErrParserNotFound) { +// t.Errorf("No parser was found for %s", file) +// } + +// count++ +// } + +// // gradle.lockfile and buildscript-gradle.lockfile use the same parser +// count -= 1 + +// expectNumberOfParsersCalled(t, count) +// } + +// func TestParse_ParserNotFound(t *testing.T) { +// t.Parallel() + +// _, err := lockfile.Parse("/path/to/my/", "") + +// expectErrIs(t, err, lockfile.ErrParserNotFound) +// } + +// func TestParse_ParserNotFound_WithExplicitParseAs(t *testing.T) { +// t.Parallel() + +// _, err := lockfile.Parse("/path/to/my/", "unsupported") + +// if err == nil { +// t.Errorf("Expected to get an error but did not") +// } + +// if !errors.Is(err, lockfile.ErrParserNotFound) { +// t.Errorf("Did not get the expected ErrParserNotFound error - got %v instead", err) +// } +// } + +// func TestListParsers(t *testing.T) { +// t.Parallel() + +// parsers := lockfile.ListParsers() + +// firstExpected := "buildscript-gradle.lockfile" +// //nolint:ifshort +// lastExpected := "yarn.lock" + +// if first := parsers[0]; first != firstExpected { +// t.Errorf("Expected first element to be %s, but got %s", firstExpected, first) +// } + +// if last := parsers[len(parsers)-1]; last != lastExpected { +// t.Errorf("Expected last element to be %s, but got %s", lastExpected, last) +// } +// } + +// func TestLockfile_String(t *testing.T) { +// t.Parallel() + +// expected := strings.Join([]string{ +// " crates.io: addr2line@0.15.2", +// " npm: @typescript-eslint/types@5.13.0", +// " crates.io: wasi@0.10.2+wasi-snapshot-preview1", +// " Packagist: sentry/sdk@2.0.4", +// " crates.io: no-version", +// " : no-ecosystem@1.2.3", +// " : no-ecosystem@1.2.3 (with-commit)", +// }, "\n") + +// lockf := lockfile.Lockfile{ +// Packages: []lockfile.PackageDetails{ +// { +// Name: "addr2line", +// Version: "0.15.2", +// Ecosystem: lockfile.CargoEcosystem, +// }, +// { +// Name: "@typescript-eslint/types", +// Version: "5.13.0", +// Ecosystem: lockfile.PnpmEcosystem, +// }, +// { +// Name: "wasi", +// Version: "0.10.2+wasi-snapshot-preview1", +// Ecosystem: lockfile.CargoEcosystem, +// }, +// { +// Name: "sentry/sdk", +// Version: "2.0.4", +// Ecosystem: lockfile.ComposerEcosystem, +// }, +// { +// Name: "no-version", +// Version: "", +// Ecosystem: lockfile.CargoEcosystem, +// }, +// { +// Name: "no-ecosystem", +// Version: "1.2.3", +// Ecosystem: "", +// }, +// { +// Name: "no-ecosystem", +// Version: "1.2.3", +// Ecosystem: "", +// Commit: "with-commit", +// }, +// }, +// } + +// if actual := lockf.String(); expected != actual { +// t.Errorf("\nExpected:\n%s\nActual:\n%s", expected, actual) +// } +// } + +// func TestPackages_Ecosystems(t *testing.T) { +// t.Parallel() + +// tests := []struct { +// name string +// ps lockfile.Packages +// want []lockfile.Ecosystem +// }{ +// {name: "", ps: lockfile.Packages{}, want: []lockfile.Ecosystem{}}, +// { +// name: "", +// ps: lockfile.Packages{ +// { +// Name: "addr2line", +// Version: "0.15.2", +// Ecosystem: lockfile.CargoEcosystem, +// }, +// }, +// want: []lockfile.Ecosystem{ +// lockfile.CargoEcosystem, +// }, +// }, +// { +// name: "", +// ps: lockfile.Packages{ +// { +// Name: "addr2line", +// Version: "0.15.2", +// Ecosystem: lockfile.CargoEcosystem, +// }, +// { +// Name: "wasi", +// Version: "0.10.2+wasi-snapshot-preview1", +// Ecosystem: lockfile.CargoEcosystem, +// }, +// }, +// want: []lockfile.Ecosystem{ +// lockfile.CargoEcosystem, +// }, +// }, +// { +// name: "", +// ps: lockfile.Packages{ +// { +// Name: "addr2line", +// Version: "0.15.2", +// Ecosystem: lockfile.CargoEcosystem, +// }, +// { +// Name: "@typescript-eslint/types", +// Version: "5.13.0", +// Ecosystem: lockfile.PnpmEcosystem, +// }, +// { +// Name: "wasi", +// Version: "0.10.2+wasi-snapshot-preview1", +// Ecosystem: lockfile.CargoEcosystem, +// }, +// { +// Name: "sentry/sdk", +// Version: "2.0.4", +// Ecosystem: lockfile.ComposerEcosystem, +// }, +// }, +// want: []lockfile.Ecosystem{ +// lockfile.ComposerEcosystem, +// lockfile.CargoEcosystem, +// lockfile.PnpmEcosystem, +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// t.Parallel() + +// if got := tt.ps.Ecosystems(); !reflect.DeepEqual(got, tt.want) { +// t.Errorf("Ecosystems() = %v, want %v", got, tt.want) +// } +// }) +// } +// } diff --git a/pkg/lockfile/tester.py b/pkg/lockfile/tester.py new file mode 100644 index 0000000000..2e4301f1d2 --- /dev/null +++ b/pkg/lockfile/tester.py @@ -0,0 +1,114 @@ +""" + +Helper script, remember to remove later!! +""" + +import sys +import re +from dataclasses import dataclass + +expectCommit = False +expectDepGroups = True +startKey = "ParseMavenLock" + + +@dataclass +class InventoryItem: + """Class for keeping track of an item in inventory.""" + name: str + version: str + commit: str = "" + depGroups: list[str] = None + + +def genInventory(path, valueList: list[InventoryItem]) -> str: + buildStr = "" + for i in valueList: + current = f""" + {{ + Name: "{i.name}", + Version: "{i.version}", + Locations: []string{{"{path}"}}, +""" + if expectCommit: + current += f'SourceCode: &lockfile.SourceCodeIdentifier{{\nCommit: "{i.commit}",\n}},\n' + + if expectDepGroups: + current += f"Metadata: lockfile.DepGroupMetadata{{\nDepGroupVals: []string{{{', '.join(i.depGroups or [])}}},\n}},\n" + + current += '},' + buildStr += current + return buildStr + + +def outputValue(path, funcName, inventory: list[InventoryItem]): + name = re.sub(r'(?" + } + + ln := fmt.Sprintf(" %s: %s", ecosystem, details.Name) + + if details.Version != "" { + ln += "@" + details.Version + } + + if details.Commit != "" { + ln += " (" + details.Commit + ")" + } + + lines = append(lines, ln) + } + + return strings.Join(lines, "\n") +} diff --git a/pkg/osvscanner/osvscanner.go b/pkg/osvscanner/osvscanner.go index eb050408f6..937ec8c3f9 100644 --- a/pkg/osvscanner/osvscanner.go +++ b/pkg/osvscanner/osvscanner.go @@ -2,6 +2,7 @@ package osvscanner import ( "bufio" + "context" "crypto/md5" //nolint:gosec "errors" "fmt" @@ -11,15 +12,21 @@ import ( "path" "path/filepath" "slices" - "sort" "strings" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem/os/apk" + "github.com/google/osv-scalibr/extractor/filesystem/os/dpkg" + scalibrosv "github.com/google/osv-scalibr/extractor/filesystem/osv" + "github.com/google/osv-scanner/internal/config" "github.com/google/osv-scanner/internal/customgitignore" "github.com/google/osv-scanner/internal/depsdev" "github.com/google/osv-scanner/internal/image" "github.com/google/osv-scanner/internal/local" - "github.com/google/osv-scanner/internal/manifest" + "github.com/google/osv-scanner/internal/lockfilescalibr" + "github.com/google/osv-scanner/internal/lockfilescalibr/language/java/pomxmlnet" + "github.com/google/osv-scanner/internal/lockfilescalibr/language/osv/osvscannerjson" "github.com/google/osv-scanner/internal/output" "github.com/google/osv-scanner/internal/resolution/client" "github.com/google/osv-scanner/internal/resolution/datasource" @@ -164,17 +171,19 @@ func scanDir(r reporter.Reporter, dir string, skipGit bool, recursive bool, useG } if !info.IsDir() { - if extractor, _ := lockfile.FindExtractor(path, ""); extractor != nil { - pkgs, err := scanLockfile(r, path, "", compareOffline) - if err != nil { + pkgs, err := scanLockfile(r, path, "", compareOffline) + if err != nil { + // If no extractors found then just continue + if !errors.Is(err, lockfilescalibr.ErrNoExtractorsFound) { r.Errorf("Attempted to scan lockfile but failed: %s\n", path) } - scannedPackages = append(scannedPackages, pkgs...) } + scannedPackages = append(scannedPackages, pkgs...) + // No need to check for error // If scan fails, it means it isn't a valid SBOM file, // so just move onto the next file - pkgs, _ := scanSBOMFile(r, path, true) + pkgs, _ = scanSBOMFile(r, path, true) scannedPackages = append(scannedPackages, pkgs...) } @@ -349,27 +358,33 @@ func scanImage(r reporter.Reporter, path string) ([]scannedPackage, error) { // within to `query` func scanLockfile(r reporter.Reporter, path string, parseAs string, compareOffline bool) ([]scannedPackage, error) { var err error - var parsedLockfile lockfile.Lockfile - - f, err := lockfile.OpenLocalDepFile(path) - - if err == nil { - // special case for the APK and DPKG parsers because they have a very generic name while - // living at a specific location, so they are not included in the map of parsers - // used by lockfile.Parse to avoid false-positives when scanning projects - switch parseAs { - case "apk-installed": - parsedLockfile, err = lockfile.FromApkInstalled(path) - case "dpkg-status": - parsedLockfile, err = lockfile.FromDpkgStatus(path) - case "osv-scanner": - parsedLockfile, err = lockfile.FromOSVScannerResults(path) - default: - if !compareOffline && (parseAs == "pom.xml" || filepath.Base(path) == "pom.xml") { - parsedLockfile, err = extractMavenDeps(f) - } else { - parsedLockfile, err = lockfile.ExtractDeps(f, parseAs) + + var inventories []*extractor.Inventory + + // special case for the APK and DPKG parsers because they have a very generic name while + // living at a specific location, so they are not included in the map of parsers + // used by lockfile.Parse to avoid false-positives when scanning projects + switch parseAs { + case "apk-installed": + // inventories, err := apkinstalled.Extractor{}.Extract(context.Background(), &si) + inventories, err = lockfilescalibr.ExtractWithExtractor(context.Background(), path, apk.New(apk.DefaultConfig())) + case "dpkg-status": + inventories, err = lockfilescalibr.ExtractWithExtractor(context.Background(), path, dpkg.New(dpkg.DefaultConfig())) + case "osv-scanner": + inventories, err = lockfilescalibr.ExtractWithExtractor(context.Background(), path, osvscannerjson.Extractor{}) + default: + if !compareOffline && (parseAs == "pom.xml" || filepath.Base(path) == "pom.xml") { + depClient, depErr := client.NewDepsDevClient(depsdev.DepsdevAPI) + if depErr != nil { + return nil, depErr + } + ext := pomxmlnet.Extractor{ + DependencyClient: depClient, + MavenRegistryAPIClient: datasource.NewMavenRegistryAPIClient(datasource.MavenCentral), } + inventories, err = lockfilescalibr.ExtractWithExtractor(context.Background(), path, ext) + } else { + inventories, err = lockfilescalibr.Extract(context.Background(), path, parseAs) } } @@ -383,62 +398,85 @@ func scanLockfile(r reporter.Reporter, path string, parseAs string, compareOffli parsedAsComment = fmt.Sprintf("as a %s ", parseAs) } + pkgCount := len(inventories) + r.Infof( "Scanned %s file %sand found %d %s\n", path, parsedAsComment, - len(parsedLockfile.Packages), - output.Form(len(parsedLockfile.Packages), "package", "packages"), + pkgCount, + output.Form(pkgCount, "package", "packages"), ) - packages := make([]scannedPackage, len(parsedLockfile.Packages)) - for i, pkgDetail := range parsedLockfile.Packages { - packages[i] = scannedPackage{ - Name: pkgDetail.Name, - Version: pkgDetail.Version, - Commit: pkgDetail.Commit, - Ecosystem: pkgDetail.Ecosystem, - DepGroups: pkgDetail.DepGroups, + packages := make([]scannedPackage, 0, pkgCount) + + for _, inv := range inventories { + scannedPackage := scannedPackage{ + Name: inv.Name, + Version: inv.Version, Source: models.SourceInfo{ Path: path, Type: "lockfile", }, } - } - - return packages, nil -} + if inv.SourceCode != nil { + scannedPackage.Commit = inv.SourceCode.Commit + } + eco := inv.Ecosystem() + // TODO(rexpan): Refactor these minor patches to individual items + // TODO: Ecosystem should be pared with Enum : Suffix + if eco == "Alpine" { + eco = "Alpine:v3.20" + } -func extractMavenDeps(f lockfile.DepFile) (lockfile.Lockfile, error) { - depClient, err := client.NewDepsDevClient(depsdev.DepsdevAPI) - if err != nil { - return lockfile.Lockfile{}, err - } - extractor := manifest.MavenResolverExtractor{ - DependencyClient: depClient, - MavenRegistryAPIClient: datasource.NewMavenRegistryAPIClient(datasource.MavenCentral), - } - packages, err := extractor.Extract(f) - if err != nil { - err = fmt.Errorf("failed extracting %s: %w", f.Path(), err) - } + scannedPackage.Ecosystem = lockfile.Ecosystem(eco) - // Sort packages for testing convenience. - sort.Slice(packages, func(i, j int) bool { - if packages[i].Name == packages[j].Name { - return packages[i].Version < packages[j].Version + if dg, ok := inv.Metadata.(scalibrosv.DepGroups); ok { + scannedPackage.DepGroups = dg.DepGroups() } - return packages[i].Name < packages[j].Name - }) + packages = append(packages, scannedPackage) + } - return lockfile.Lockfile{ - FilePath: f.Path(), - ParsedAs: "pom.xml", - Packages: packages, - }, err + return packages, nil } +// func extractMavenDeps(context ,f lockfile.DepFile) ([]*extractor.Inventory, error) { +// depClient, err := client.NewDepsDevClient(depsdev.DepsdevAPI) +// if err != nil { +// return nil, err +// } +// ext := manifest.MavenResolverExtractor{ +// DependencyClient: depClient, +// MavenRegistryAPIClient: datasource.NewMavenRegistryAPIClient(datasource.MavenCentral), +// } +// packages, err := ext.Extract(f) +// if err != nil { +// err = fmt.Errorf("failed extracting %s: %w", f.Path(), err) +// return nil, err +// } + +// // Sort packages for testing convenience. +// sort.Slice(packages, func(i, j int) bool { +// if packages[i].Name == packages[j].Name { +// return packages[i].Version < packages[j].Version +// } + +// return packages[i].Name < packages[j].Name +// }) + +// inv := make([]*extractor.Inventory, len(packages)) + +// for i, pkg := range packages { +// inv[i] = &extractor.Inventory{ +// Name: pkg.Name, +// Version: pkg.Version, +// Locations: f.Path(), +// } +// } +// return inv, err +// } + // scanSBOMFile will load, identify, and parse the SBOM path passed in, and add the dependencies specified // within to `query` func scanSBOMFile(r reporter.Reporter, path string, fromFSScan bool) ([]scannedPackage, error) { @@ -1029,7 +1067,12 @@ func filterIgnoredPackages(r reporter.Reporter, packages []scannedPackage, confi } if ignore, ignoreLine := configToUse.ShouldIgnorePackage(pkg); ignore { - pkgString := fmt.Sprintf("%s/%s/%s", p.Ecosystem, p.Name, p.Version) + var pkgString string + if p.PURL != "" { + pkgString = p.PURL + } else { + pkgString = fmt.Sprintf("%s/%s/%s", p.Ecosystem, p.Name, p.Version) + } reason := ignoreLine.Reason if reason == "" {