Skip to content

Commit

Permalink
Run contextual analysis and secret detection in Docker scans (#10)
Browse files Browse the repository at this point in the history
  • Loading branch information
guyshe-jfrog authored May 30, 2024
1 parent e4bd282 commit d867456
Show file tree
Hide file tree
Showing 21 changed files with 276 additions and 138 deletions.
25 changes: 11 additions & 14 deletions commands/audit/audit.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,22 @@ package audit
import (
"errors"
"fmt"
"os"

"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
"github.com/jfrog/jfrog-cli-security/scangraph"
"github.com/jfrog/jfrog-cli-security/utils"

"github.com/jfrog/jfrog-cli-security/jas"

"github.com/jfrog/jfrog-cli-security/jas/applicability"
"github.com/jfrog/jfrog-cli-security/jas/runner"
"github.com/jfrog/jfrog-cli-security/jas/secrets"
clientutils "github.com/jfrog/jfrog-client-go/utils"
"github.com/jfrog/jfrog-client-go/utils/log"
"github.com/jfrog/jfrog-client-go/xray"
"github.com/jfrog/jfrog-client-go/xray/services"
xscservices "github.com/jfrog/jfrog-client-go/xsc/services"
"golang.org/x/sync/errgroup"
"os"

xrayutils "github.com/jfrog/jfrog-cli-security/utils"
)
Expand Down Expand Up @@ -182,15 +188,15 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error)
return
}
results.XrayVersion = auditParams.xrayVersion
results.ExtendedScanResults.EntitledForJas, err = isEntitledForJas(xrayManager, auditParams.xrayVersion)
results.ExtendedScanResults.EntitledForJas, err = jas.IsEntitledForJas(xrayManager, auditParams.xrayVersion)
if err != nil {
return
}

errGroup := new(errgroup.Group)
if results.ExtendedScanResults.EntitledForJas {
// Download (if needed) the analyzer manager in a background routine.
errGroup.Go(utils.DownloadAnalyzerManagerIfNeeded)
errGroup.Go(xrayutils.DownloadAnalyzerManagerIfNeeded)
}

results.MultiScanId = auditParams.XrayGraphScanParams().MultiScanId
Expand All @@ -205,16 +211,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error)

// Run scanners only if the user is entitled for Advanced Security
if results.ExtendedScanResults.EntitledForJas {
results.JasError = runJasScannersAndSetResults(results, auditParams.DirectDependencies(), serverDetails, auditParams.workingDirs, auditParams.Progress(), auditParams.thirdPartyApplicabilityScan, auditParams.XrayGraphScanParams().MultiScanId)
}
return
}

func isEntitledForJas(xrayManager *xray.XrayServicesManager, xrayVersion string) (entitled bool, err error) {
if e := clientutils.ValidateMinimumVersion(clientutils.Xray, xrayVersion, xrayutils.EntitlementsMinVersion); e != nil {
log.Debug(e)
return
results.JasError = runner.RunJasScannersAndSetResults(results.ExtendedScanResults, results.GetScaScannedTechnologies(), results.GetScaScansXrayResults(), auditParams.DirectDependencies(), serverDetails, auditParams.workingDirs, auditParams.Progress(), auditParams.thirdPartyApplicabilityScan, auditParams.XrayGraphScanParams().MultiScanId, applicability.ApplicabilityScannerType, secrets.SecretsScannerType)
}
entitled, err = xrayManager.IsEntitled(xrayutils.ApplicabilityFeatureId)
return
}
63 changes: 0 additions & 63 deletions commands/audit/jasrunner.go

This file was deleted.

10 changes: 6 additions & 4 deletions commands/scan/dockerscan.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,17 @@ package scan
import (
"bytes"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"

"github.com/jfrog/jfrog-cli-core/v2/common/spec"
xrayutils "github.com/jfrog/jfrog-cli-security/utils"
clientutils "github.com/jfrog/jfrog-client-go/utils"
"github.com/jfrog/jfrog-client-go/utils/errorutils"
"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
"github.com/jfrog/jfrog-client-go/utils/log"
"os"
"os/exec"
"path/filepath"
"strings"
)

const (
Expand Down Expand Up @@ -81,6 +82,7 @@ func (dsc *DockerScanCommand) Run() (err error) {
Pattern(imageTarPath).
Target(dsc.targetRepoPath).
BuildSpec()).SetThreads(1)
dsc.ScanCommand.SetRunJasScans(true)
err = dsc.setCredentialEnvsForIndexerApp()
if err != nil {
return errorutils.CheckError(err)
Expand Down
83 changes: 71 additions & 12 deletions commands/scan/scan.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,19 @@ import (
"regexp"
"strings"

"golang.org/x/exp/maps"
"golang.org/x/exp/slices"

"github.com/jfrog/jfrog-cli-security/jas"
"github.com/jfrog/jfrog-cli-security/jas/applicability"
"github.com/jfrog/jfrog-cli-security/jas/runner"
"github.com/jfrog/jfrog-cli-security/jas/secrets"
"github.com/jfrog/jfrog-cli-security/scangraph"
xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
"golang.org/x/sync/errgroup"

"github.com/jfrog/gofrog/parallel"
"github.com/jfrog/jfrog-cli-core/v2/common/format"
outputFormat "github.com/jfrog/jfrog-cli-core/v2/common/format"
"github.com/jfrog/jfrog-cli-core/v2/common/spec"
"github.com/jfrog/jfrog-cli-core/v2/utils/config"
"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
Expand All @@ -35,8 +42,9 @@ type FileContext func(string) parallel.TaskFunc
type indexFileHandlerFunc func(file string)

type ScanInfo struct {
Target string
Result *services.ScanResponse
Target string
Result *services.ScanResponse
ExtendedScanResults *utils.ExtendedScanResults
}

const (
Expand All @@ -52,7 +60,7 @@ type ScanCommand struct {
// The location of the downloaded Xray indexer binary on the local file system.
indexerPath string
indexerTempDir string
outputFormat outputFormat.OutputFormat
outputFormat format.OutputFormat
projectKey string
minSeverityFilter string
watches []string
Expand All @@ -63,6 +71,7 @@ type ScanCommand struct {
bypassArchiveLimits bool
fixableOnly bool
progress ioUtils.ProgressMgr
commandSupportsJAS bool
}

func (scanCmd *ScanCommand) SetMinSeverityFilter(minSeverityFilter string) *ScanCommand {
Expand All @@ -75,6 +84,11 @@ func (scanCmd *ScanCommand) SetFixableOnly(fixable bool) *ScanCommand {
return scanCmd
}

func (scanCmd *ScanCommand) SetRunJasScans(run bool) *ScanCommand {
scanCmd.commandSupportsJAS = run
return scanCmd
}

func (scanCmd *ScanCommand) SetProgress(progress ioUtils.ProgressMgr) {
scanCmd.progress = progress
}
Expand All @@ -84,7 +98,7 @@ func (scanCmd *ScanCommand) SetThreads(threads int) *ScanCommand {
return scanCmd
}

func (scanCmd *ScanCommand) SetOutputFormat(format outputFormat.OutputFormat) *ScanCommand {
func (scanCmd *ScanCommand) SetOutputFormat(format format.OutputFormat) *ScanCommand {
scanCmd.outputFormat = format
return scanCmd
}
Expand Down Expand Up @@ -182,6 +196,16 @@ func (scanCmd *ScanCommand) Run() (err error) {
return err
}

scanResults := xrutils.NewAuditResults()
scanResults.XrayVersion = xrayVersion

scanResults.ExtendedScanResults.EntitledForJas, err = jas.IsEntitledForJas(xrayManager, xrayVersion)
errGroup := new(errgroup.Group)
if scanResults.ExtendedScanResults.EntitledForJas {
// Download (if needed) the analyzer manager in a background routine.
errGroup.Go(xrutils.DownloadAnalyzerManagerIfNeeded)
}

// Validate Xray minimum version for graph scan command
err = clientutils.ValidateMinimumVersion(clientutils.Xray, xrayVersion, scangraph.GraphScanMinXrayVersion)
if err != nil {
Expand Down Expand Up @@ -226,16 +250,20 @@ func (scanCmd *ScanCommand) Run() (err error) {
fileCollectingErrorsQueue := clientutils.NewErrorsQueue(1)
// Start walking on the filesystem to "produce" files that match the given pattern
// while the consumer uses the indexer to index those files.
scanCmd.prepareScanTasks(fileProducerConsumer, indexedFileProducerConsumer, resultsArr, fileProducerErrors, indexedFileProducerErrors, fileCollectingErrorsQueue, xrayVersion)
scanCmd.prepareScanTasks(fileProducerConsumer, indexedFileProducerConsumer, scanResults.ExtendedScanResults.EntitledForJas, resultsArr, fileProducerErrors, indexedFileProducerErrors, fileCollectingErrorsQueue, xrayVersion)
scanCmd.performScanTasks(fileProducerConsumer, indexedFileProducerConsumer)

// Handle results
flatResults := []xrutils.ScaScanResult{}

for _, arr := range resultsArr {
for _, res := range arr {
flatResults = append(flatResults, xrutils.ScaScanResult{Target: res.Target, XrayResults: []services.ScanResponse{*res.Result}})
scanResults.ExtendedScanResults.ApplicabilityScanResults = append(scanResults.ExtendedScanResults.ApplicabilityScanResults, res.ExtendedScanResults.ApplicabilityScanResults...)
scanResults.ExtendedScanResults.SecretsScanResults = append(scanResults.ExtendedScanResults.SecretsScanResults, res.ExtendedScanResults.SecretsScanResults...)
}
}

if scanCmd.progress != nil {
if err = scanCmd.progress.Quit(); err != nil {
return err
Expand All @@ -251,10 +279,13 @@ func (scanCmd *ScanCommand) Run() (err error) {
scanErrors = appendErrorSlice(scanErrors, fileProducerErrors)
scanErrors = appendErrorSlice(scanErrors, indexedFileProducerErrors)

scanResults := xrutils.NewAuditResults()
scanResults.XrayVersion = xrayVersion
scanResults.ScaResults = flatResults

// Wait for the Download of the AnalyzerManager to complete.
if err = errGroup.Wait(); err != nil {
err = errors.New("failed while trying to get Analyzer Manager: " + err.Error())
}

if err = xrutils.NewResultsWriter(scanResults).
SetOutputFormat(scanCmd.outputFormat).
SetIncludeVulnerabilities(scanCmd.includeVulnerabilities).
Expand Down Expand Up @@ -296,14 +327,14 @@ func (scanCmd *ScanCommand) CommandName() string {
return "xr_scan"
}

func (scanCmd *ScanCommand) prepareScanTasks(fileProducer, indexedFileProducer parallel.Runner, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors [][]formats.SimpleJsonError, fileCollectingErrorsQueue *clientutils.ErrorsQueue, xrayVersion string) {
func (scanCmd *ScanCommand) prepareScanTasks(fileProducer, indexedFileProducer parallel.Runner, entitledForJas bool, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors [][]formats.SimpleJsonError, fileCollectingErrorsQueue *clientutils.ErrorsQueue, xrayVersion string) {
go func() {
defer fileProducer.Done()
// Iterate over file-spec groups and produce indexing tasks.
// When encountering an error, log and move to next group.
specFiles := scanCmd.spec.Files
for i := range specFiles {
artifactHandlerFunc := scanCmd.createIndexerHandlerFunc(&specFiles[i], indexedFileProducer, resultsArr, fileErrors, indexedFileErrors, xrayVersion)
artifactHandlerFunc := scanCmd.createIndexerHandlerFunc(&specFiles[i], entitledForJas, indexedFileProducer, resultsArr, fileErrors, indexedFileErrors, xrayVersion)
taskHandler := getAddTaskToProducerFunc(fileProducer, artifactHandlerFunc)

err := collectFilesForIndexing(specFiles[i], taskHandler)
Expand All @@ -315,7 +346,7 @@ func (scanCmd *ScanCommand) prepareScanTasks(fileProducer, indexedFileProducer p
}()
}

func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, indexedFileProducer parallel.Runner, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors [][]formats.SimpleJsonError, xrayVersion string) FileContext {
func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, entitledForJas bool, indexedFileProducer parallel.Runner, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors [][]formats.SimpleJsonError, xrayVersion string) FileContext {
return func(filePath string) parallel.TaskFunc {
return func(threadId int) (err error) {
logMsgPrefix := clientutils.GetLogMsgPrefix(threadId, false)
Expand Down Expand Up @@ -360,12 +391,26 @@ func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, indexedFil
return err
}
scanResults, err := scangraph.RunScanGraphAndGetResults(scanGraphParams, xrayManager)

if err != nil {
log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error()))
indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()})
return
}
resultsArr[threadId] = append(resultsArr[threadId], &ScanInfo{Target: filePath, Result: scanResults})

extendedScanResults := utils.ExtendedScanResults{}
if entitledForJas && scanCmd.commandSupportsJAS {
// Run Jas scans
workingDirs := []string{filePath}
err = runner.RunJasScannersAndSetResults(&extendedScanResults, []coreutils.Technology{coreutils.Technology(scanResults.ScannedPackageType)}, []services.ScanResponse{*scanResults}, depsListFromVulnerabilities(*scanResults), scanCmd.serverDetails, workingDirs, nil, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType)

if err != nil {
log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error()))
indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()})
return
}
}
resultsArr[threadId] = append(resultsArr[threadId], &ScanInfo{Target: filePath, Result: scanResults, ExtendedScanResults: &extendedScanResults})
return
}

Expand Down Expand Up @@ -469,6 +514,20 @@ func appendErrorSlice(scanErrors []formats.SimpleJsonError, errorsToAdd [][]form
return scanErrors
}

func depsListFromVulnerabilities(scanResult ...services.ScanResponse) (depsList []string) {
for _, result := range scanResult {
for _, vulnerability := range result.Vulnerabilities {
dependencies := maps.Keys(vulnerability.Components)
for _, dependency := range dependencies {
if !slices.Contains(depsList, dependency) {
depsList = append(depsList, dependency)
}
}
}
}
return
}

func ConditionalUploadDefaultScanFunc(serverDetails *config.ServerDetails, fileSpec *spec.SpecFiles, threads int, scanOutputFormat format.OutputFormat) error {
return NewScanCommand().SetServerDetails(serverDetails).SetSpec(fileSpec).SetThreads(threads).SetOutputFormat(scanOutputFormat).SetFail(true).SetPrintExtendedTable(false).Run()
}
1 change: 1 addition & 0 deletions formats/conversion.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ func ConvertToVulnerabilityScanTableRow(rows []VulnerabilityOrViolationRow) (tab
tableRows = append(tableRows, vulnerabilityScanTableRow{
severity: rows[i].Severity,
severityNumValue: rows[i].SeverityNumValue,
applicable: rows[i].Applicable,
impactedPackageName: rows[i].ImpactedDependencyName,
impactedPackageVersion: rows[i].ImpactedDependencyVersion,
ImpactedPackageType: rows[i].ImpactedDependencyType,
Expand Down
3 changes: 2 additions & 1 deletion formats/table.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ type vulnerabilityTableRow struct {
}

type vulnerabilityScanTableRow struct {
severity string `col-name:"Severity"`
severity string `col-name:"Severity"`
applicable string `col-name:"Contextual\nAnalysis" omitempty:"true"`
// For sorting
severityNumValue int
directPackages []directPackagesTableRow `embed-table:"true"`
Expand Down
Loading

0 comments on commit d867456

Please sign in to comment.