use new grpc abstraction

Dockerfile

@@ -13,7 +13,6 @@ RUN tar -xjf /vectorscan.tar.bz2 -C / && rm /vectorscan.tar.bz2
 
 WORKDIR /home/deepfence/src/SecretScanner
 COPY . .
-RUN make clean
 RUN make
 
 FROM alpine:3.18
@@ -30,7 +29,7 @@ RUN apk add --no-cache --upgrade tar libstdc++ libgcc docker skopeo bash podman
 RUN <<EOF
 set -eux
 
-apk update && apk add --no-cache --upgrade curl 
+apk update && apk add --no-cache --upgrade curl
 
 NERDCTL_VERSION=1.4.0
 curl -fsSLO https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-${TARGETARCH}.tar.gz

Makefile

@@ -7,8 +7,8 @@ clean:
 	-rm ./SecretScanner
 
 SecretScanner: $(PWD)/**/*.go $(PWD)/agent-plugins-grpc/**/*.go
-	go mod tidy -v
-	go mod vendor
+	-go mod tidy -v
+	-go mod vendor
 	go build -ldflags="-extldflags=-static" -buildvcs=false -v .
 
 .PHONY: clean bootstrap

go.mod

@@ -4,10 +4,8 @@ go 1.21.0
 
 replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc
 
-replace github.com/deepfence/YaraHunter => ../YaraHunter
-
 require (
-	github.com/deepfence/YaraHunter v0.0.0-00010101000000-000000000000
+	github.com/deepfence/YaraHunter v0.0.0-20240708090804-4196e3bbd2c1
 	github.com/deepfence/agent-plugins-grpc v0.0.0-00010101000000-000000000000
 	github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9
 	github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240626143546-e4ec9311fdf9

go.sum

@@ -34,6 +34,8 @@ github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3H
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deepfence/YaraHunter v0.0.0-20240708090804-4196e3bbd2c1 h1:VuKkE8OKmM+hII/fViwYuLL1DTq5ncczMKWuOKg+EZc=
+github.com/deepfence/YaraHunter v0.0.0-20240708090804-4196e3bbd2c1/go.mod h1:SSZ34MXU0qR30p6/rJ/PpY7OgwljWF3go6K/ZfYqHZ0=
 github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9 h1:hI/Fv6XabkERGza4E8g7XLhlkuzWjoXQwTmtq0WIC+Y=
 github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9/go.mod h1:+rchMc4YNjCoHo0YAwKsT+DRBNr1hdDG0WrvAOOCc5k=
 github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240626143546-e4ec9311fdf9 h1:/1olNPTiYUFxuYsP79DKLEEieNWIdPDFxs+B58jysRA=

jobs/scan.go

@@ -2,77 +2,15 @@ package jobs
 
 import (
 	"encoding/json"
-	"fmt"
 	"strings"
 	"sync"
-	"time"
-
-	"github.com/deepfence/SecretScanner/core"
-	"github.com/deepfence/SecretScanner/output"
-	"github.com/deepfence/SecretScanner/scan"
-	"github.com/deepfence/golang_deepfence_sdk/utils/tasks"
 
 	pb "github.com/deepfence/agent-plugins-grpc/srcgo"
-	cfg "github.com/deepfence/match-scanner/pkg/config"
 	log "github.com/sirupsen/logrus"
 )
 
 var ScanMap sync.Map
 
-func DispatchScan(r *pb.FindRequest) {
-	go func() {
-		startScanJob()
-		defer stopScanJob()
-
-		var err error
-		res, scanCtx := tasks.StartStatusReporter(
-			r.ScanId,
-			func(ss tasks.ScanStatus) error {
-				return writeSecretScanStatus(ss.ScanStatus, ss.ScanId, ss.ScanMessage)
-			},
-			tasks.StatusValues{
-				IN_PROGRESS: "IN_PROGRESS",
-				CANCELLED:   "CANCELLED",
-				FAILED:      "ERROR",
-				SUCCESS:     "COMPLETE",
-			},
-			time.Minute*20,
-		)
-
-		ScanMap.Store(r.ScanId, scanCtx)
-
-		defer func() {
-			ScanMap.Delete(r.ScanId)
-			res <- err
-			close(res)
-		}()
-
-		var (
-			scanType scan.ScanType
-			nodeID   string
-		)
-
-		if r.GetPath() != "" {
-			scanType = scan.DirScan
-			nodeID = r.GetPath()
-		} else if r.GetImage() != nil && r.GetImage().Name != "" {
-			scanType = scan.ImageScan
-			nodeID = r.GetImage().Name
-		} else if r.GetContainer() != nil && r.GetContainer().Id != "" {
-			scanType = scan.ContainerScan
-			nodeID = r.GetContainer().Id
-		} else {
-			err = fmt.Errorf("Invalid request")
-			return
-		}
-
-		filters := cfg.Config2Filter(core.GetSession().ExtractorConfig)
-		err = scan.Scan(scanCtx, scanType, filters, "", nodeID, r.GetScanId(), func(sf output.SecretFound, s string) {
-			writeSingleScanData(output.SecretToSecretInfo(sf), r.ScanId)
-		})
-	}()
-}
-
 type SecretScanDoc struct {
 	pb.SecretInfo
 	ScanID string `json:"scan_id,omitempty"`
@@ -100,7 +38,7 @@ func writeMultiScanData(secrets []*pb.SecretInfo, scan_id string) {
 	}
 }
 
-func writeSingleScanData(secret *pb.SecretInfo, scan_id string) {
+func WriteSingleScanData(secret *pb.SecretInfo, scan_id string) {
 	if SecretScanDir == HostMountDir {
 		secret.GetMatch().FullFilename = strings.Replace(secret.GetMatch().GetFullFilename(), SecretScanDir, "", 1)
 	}

main.go

@@ -39,12 +39,17 @@ import (
 	"github.com/deepfence/SecretScanner/core"
 	"github.com/deepfence/SecretScanner/output"
 	"github.com/deepfence/SecretScanner/scan"
+	"github.com/deepfence/SecretScanner/server"
 	"github.com/deepfence/SecretScanner/signature"
 	"github.com/deepfence/golang_deepfence_sdk/utils/tasks"
 	"github.com/deepfence/match-scanner/pkg/config"
 	log "github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
 
 	"github.com/deepfence/YaraHunter/pkg/runner"
+	yaraserver "github.com/deepfence/YaraHunter/pkg/server"
+
+	pb "github.com/deepfence/agent-plugins-grpc/srcgo"
 )
 
 const (
@@ -206,5 +211,15 @@ func main() {
 		go runner.ScheduleYaraHunterUpdater(ctx, runnerOpts)
 	}
 
-	runner.StartYaraHunter(ctx, runnerOpts, core.GetSession().ExtractorConfig)
+	runner.StartYaraHunter(ctx, runnerOpts, core.GetSession().ExtractorConfig,
+
+		func(base *yaraserver.GRPCScannerServer) server.SecretGRPCServer {
+			return server.SecretGRPCServer{
+				GRPCScannerServer:                base,
+				UnimplementedSecretScannerServer: pb.UnimplementedSecretScannerServer{},
+			}
+		},
+		func(s grpc.ServiceRegistrar, impl any) {
+			pb.RegisterSecretScannerServer(s, impl.(pb.SecretScannerServer))
+		})
 }

output/output.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/deepfence/YaraHunter/pkg/output"
 	pb "github.com/deepfence/agent-plugins-grpc/srcgo"
 	"github.com/fatih/color"
 	tw "github.com/olekukonko/tablewriter"
@@ -180,30 +181,32 @@ func removeFirstLastChar(input string) string {
 	return input[1 : len(input)-1]
 }
 
-func SecretsToSecretInfos(out []SecretFound) []*pb.SecretInfo {
+func SecretsToSecretInfos(out []output.IOCFound) []*pb.SecretInfo {
 	res := make([]*pb.SecretInfo, 0)
 	for _, v := range out {
 		res = append(res, SecretToSecretInfo(v))
 	}
 	return res
 }
 
-func SecretToSecretInfo(out SecretFound) *pb.SecretInfo {
+func SecretToSecretInfo(out output.IOCFound) *pb.SecretInfo {
+	matchedContent := ""
+	if len(out.Meta) != 0 {
+		matchedContent = out.Meta[0]
+	}
+	signature := ""
+	if len(out.StringsToMatch) != 0 {
+		signature = out.StringsToMatch[0]
+	}
 	return &pb.SecretInfo{
 		ImageLayerId: out.LayerID,
 		Rule: &pb.MatchRule{
-			Id:               int32(out.RuleID),
 			Name:             out.RuleName,
-			Part:             out.PartToMatch,
-			StringToMatch:    out.Match,
-			SignatureToMatch: out.Regex,
+			SignatureToMatch: signature,
 		},
 		Match: &pb.Match{
-			StartingIndex:         int64(out.PrintBufferStartIndex),
-			RelativeStartingIndex: int64(out.MatchFromByte),
-			RelativeEndingIndex:   int64(out.MatchToByte),
-			FullFilename:          out.CompleteFilename,
-			MatchedContent:        jsonMarshal(out.MatchedContents),
+			FullFilename:   out.CompleteFilename,
+			MatchedContent: matchedContent,
 		},
 		Severity: &pb.Severity{
 			Level: out.Severity,

rules/yara.rules

@@ -0,0 +1,11 @@
+rule github_personal_access_token {
+    meta:
+        description = "Rule to match GitHub Personal Access Tokens (classic), Fine-grained & Github Actions Token"
+        author = "deepfence.io"
+
+    strings:
+        $github_pat = /^gh[ps]_[a-zA-Z0-9]{36}|github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}/
+
+    condition:
+        $github_pat
+}

server/grpc.go

@@ -2,95 +2,59 @@ package server
 
 import (
 	"context"
-	"fmt"
-	"net"
-	"sync"
 
 	"github.com/deepfence/SecretScanner/jobs"
+	"github.com/deepfence/SecretScanner/output"
+	out "github.com/deepfence/YaraHunter/pkg/output"
+	"github.com/deepfence/YaraHunter/pkg/server"
 	pb "github.com/deepfence/agent-plugins-grpc/srcgo"
-
+	"github.com/sirupsen/logrus"
 	//nolint:typecheck
-	"github.com/deepfence/golang_deepfence_sdk/utils/tasks"
-	log "github.com/sirupsen/logrus"
-	"google.golang.org/grpc"
 )
 
-type gRPCServer struct {
-	socket_path string
-	plugin_name string
+type SecretGRPCServer struct {
+	*server.GRPCScannerServer
 	pb.UnimplementedSecretScannerServer
-	pb.UnimplementedAgentPluginServer
-	pb.UnimplementedScannersServer
-}
-
-func (s *gRPCServer) ReportJobsStatus(context.Context, *pb.Empty) (*pb.JobReports, error) {
-	return &pb.JobReports{
-		RunningJobs: jobs.GetRunningJobCount(),
-	}, nil
-}
-
-func (s *gRPCServer) StopScan(c context.Context, req *pb.StopScanRequest) (*pb.StopScanResult, error) {
-	log.Errorf("Received StopScanRequest: %v", *req)
-	scanID := req.ScanId
-	result := &pb.StopScanResult{
-		Success:     true,
-		Description: "",
-	}
-
-	obj, found := jobs.ScanMap.Load(scanID)
-	if !found {
-		log.Errorf("SecretScanner::Failed to Stop scan, may have already completed successfully or errored out, scan_id: %s", scanID)
-		result.Success = false
-		result.Description = "SecretScanner::Failed to Stop scan"
-		return result, nil
-	} else {
-		log.Errorf("SecretScanner::Stop request submitted")
-		result.Description = "SecretScanner::Stop request submitted"
-	}
-
-	scanCtx := obj.(*tasks.ScanContext)
-	scanCtx.StopTriggered.Store(true)
-	scanCtx.Cancel()
-	return result, nil
-}
-
-func (s *gRPCServer) GetName(context.Context, *pb.Empty) (*pb.Name, error) {
-	return &pb.Name{Str: s.plugin_name}, nil
 }
 
-func (s *gRPCServer) GetUID(context.Context, *pb.Empty) (*pb.Uid, error) {
-	return &pb.Uid{Str: fmt.Sprintf("%s-%s", s.plugin_name, s.socket_path)}, nil
-}
-
-func (s *gRPCServer) FindSecretInfo(c context.Context, r *pb.FindRequest) (*pb.FindResult, error) {
-	jobs.DispatchScan(r)
-	return &pb.FindResult{}, nil
-}
-
-func RunServer(ctx context.Context, socket_path string, plugin_name string) error {
-
-	lis, err := net.Listen("unix", fmt.Sprintf("%s", socket_path))
+func (s *SecretGRPCServer) FindSecretInfo(c context.Context, r *pb.FindRequest) (*pb.FindResult, error) {
+	yaraScanner, err := s.YaraRules.NewScanner()
 	if err != nil {
-		return err
+		return &pb.FindResult{}, err
 	}
-	s := grpc.NewServer()
 
-	jobs.ScanMap = sync.Map{}
-
-	impl := &gRPCServer{socket_path: socket_path, plugin_name: plugin_name}
-	pb.RegisterAgentPluginServer(s, impl)
-	pb.RegisterSecretScannerServer(s, impl)
-	pb.RegisterScannersServer(s, impl)
-	log.Infof("main: server listening at %v", lis.Addr())
 	go func() {
-		if err := s.Serve(lis); err != nil {
-			log.Errorf("server: %v", err)
+		logrus.Infof("request to scan %+v", r)
+
+		namespace := ""
+		container := ""
+		image := ""
+		path := ""
+		switch {
+		case r.GetContainer() != nil:
+			namespace = r.GetContainer().GetNamespace()
+			container = r.GetContainer().GetId()
+		case r.GetImage() != nil:
+			image = r.GetImage().GetName()
+		default:
+			path = r.GetPath()
 		}
-	}()
-
-	<-ctx.Done()
-	s.GracefulStop()
 
-	log.Infof("main: exiting gracefully")
-	return nil
+		server.DoScan(
+			r.ScanId,
+			s.HostMountPath,
+			s.ExtractorConfig,
+			s.InactiveThreshold,
+			&s.ScanMap,
+			namespace,
+			path,
+			image,
+			container,
+			yaraScanner,
+			func(res out.IOCFound, scanID string) {
+				jobs.WriteSingleScanData(output.SecretToSecretInfo(res), scanID)
+			},
+		)
+	}()
+	return &pb.FindResult{}, nil
 }