working copy

cmd/friller/Makefile

@@ -8,15 +8,16 @@ PLATFORMS=darwin linux windows
 ARCHITECTURES=amd64
 
 LDFLAGS=-ldflags "-X main.Version=${VERSION} -X main.Build=${BUILD} -X main.SigFile=${SigFile} -w -s" 
+BLDFLAGS=-buildmode=pie
 
 all: clean build_all
 
 build:
-	go build ${LDFLAGS} -o ${BINARY}
+	go build ${BLDFLAGS} ${LDFLAGS} -o ${BINARY}
 
 build_all:
 	$(foreach GOOS, $(PLATFORMS),\
-	$(foreach GOARCH, $(ARCHITECTURES), $(shell export GOOS=$(GOOS); export GOARCH=$(GOARCH); go build $(LDFLAGS) -v -o $(BINARY)-$(GOOS))))
+	$(foreach GOARCH, $(ARCHITECTURES), $(shell export GOOS=$(GOOS); export GOARCH=$(GOARCH); go build $(BLDFLAGS) $(LDFLAGS) -v -o $(BINARY)-$(GOOS))))
 	mv friller-darwin friller && tar cvfz friller_macos_$(VERSION).tar.gz friller
 	rm friller
 	mv friller-linux friller && tar cvfz friller_linux_$(VERSION).tar.gz friller

cmd/friller/main.go

@@ -25,9 +25,9 @@ import (
 	"strings"
 
 	"github.com/gomodule/redigo/redis"
-	flag "github.com/spf13/pflag"
 
 	fdr "github.com/dla-marbach/filedriller"
+	flag "github.com/spf13/pflag"
 )
 
 // Version holds the version of filedriller
@@ -40,8 +40,8 @@ var Build string
 var SigFile string
 
 // we make this two gloablly available for usage inside imported customized loggers
-var	logFile = flag.StringP("log", "l", "logs.txt", "Log file")
-var	errlogFile = flag.StringP("errlog", "w", "errorlogs.txt", "Error log file")
+var logFile = flag.StringP("log", "l", "logs.txt", "Log file")
+var errlogFile = flag.StringP("errlog", "w", "errorlogs.txt", "Error log file")
 
 func main() {
 
@@ -59,6 +59,7 @@ func main() {
 
 	flag.Parse()
 
+	// Create two loggers for two log files, standard and error
 	fdr.CreateLogger(*logFile)
 	fdr.CreateErrorLogger(*errlogFile)
 
@@ -180,18 +181,18 @@ func main() {
 	log.Println("info: Output written to " + *oFile)
 	fdr.InfoLogger.Println("Output written to " + *oFile)
 	log.Println("info: Log file written to " + *logFile)
-       
-        // Delete empty error log file 
-        if fi, err := os.Stat(*errlogFile); err == nil && fi.Size() == 0 {
-            err := os.Remove(*errlogFile)
-            if err != nil {
-                log.Println("error: Could not delete empty error log file.")
-                log.Println(err)
-            }
-      	
-        } else {
-                log.Println("info: Error log file written to " + *errlogFile)
-       }
+
+	// Delete empty error log file
+	if fi, err := os.Stat(*errlogFile); err == nil && fi.Size() == 0 {
+		err := os.Remove(*errlogFile)
+		if err != nil {
+			log.Println("error: Could not delete empty error log file.")
+			log.Println(err)
+		}
+
+	} else {
+		log.Println("info: Error log file written to " + *errlogFile)
+	}
 
 	log.Println("info: friller ended")
 	fdr.InfoLogger.Println("friller stopped")

go.mod

@@ -3,6 +3,7 @@ module github.com/dla-marbach/filedriller
 go 1.15
 
 require (
+	github.com/cheggaaa/pb/v3 v3.0.8
 	github.com/gomodule/redigo v1.8.3
 	github.com/google/uuid v1.1.4
 	github.com/pkg/errors v0.9.1

go.sum

@@ -1,9 +1,21 @@
+github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
+github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA=
+github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/gomodule/redigo v1.8.3 h1:HR0kYDX2RJZvAup8CsiJwxB4dTCSC0AaUq6S4SiLwUc=
 github.com/gomodule/redigo v1.8.3/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0=
 github.com/google/uuid v1.1.4 h1:0ecGp3skIrHWPNGPJDaBIghfA6Sp7Ruo2Io8eLKzWm0=
 github.com/google/uuid v1.1.4/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow=
+github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -21,6 +33,9 @@ github.com/richardlehane/siegfried v1.9.1/go.mod h1:6Xdr8SWONtMJiCkDvM/mH5z/W1Rd
 github.com/richardlehane/webarchive v1.0.0/go.mod h1:R+J0ocBfFb0A3vzwXZwrYoPe69PJLEejLzFL5ceDroM=
 github.com/richardlehane/xmldetect v1.0.2 h1:/3ooFuJwtgpMMe14/7m8a/JIvECMx6SpsPcDRiNyR8o=
 github.com/richardlehane/xmldetect v1.0.2/go.mod h1:Zp1lhTLRJa2p2QKA4jOruVQYc0NFQDO0YUz3k/k6JcE=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/ross-spencer/spargo v0.0.0-20200323024642-38971d4365a7 h1:G50l+RXrUyL5DE+Mj1+OOJgOR+hq8Ghf/ozx3FFcffQ=
 github.com/ross-spencer/spargo v0.0.0-20200323024642-38971d4365a7/go.mod h1:5mytCwysAzmwG9GJTFD7GR8+ZrhStjTOe3krU9Rlm8c=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -33,9 +48,12 @@ golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5
 golang.org/x/image v0.0.0-20200922025426-e59bae62ef32 h1:E+SEVulmY8U4+i6vSB88YSc2OKAFfvbHPU/uDTdQu7M=
 golang.org/x/image v0.0.0-20200922025426-e59bae62ef32/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=

process.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/cheggaaa/pb/v3"
 	"github.com/gomodule/redigo/redis"
 	"github.com/richardlehane/siegfried"
 )
@@ -31,7 +32,6 @@ func CreateFileList(rootDir string) []string {
 
 // IdentifyFiles creates metadata with siegfried and hashsum
 func IdentifyFiles(fileList []string, hashDigest string, nsrlEnabled bool, conn redis.Conn, entroEnabled bool) []string {
-
 	var resultList []string
 	s, err := siegfried.Load("pronom.sig")
 	if err != nil {
@@ -45,6 +45,8 @@ func IdentifyFiles(fileList []string, hashDigest string, nsrlEnabled bool, conn
 
 	var entroFile float64
 
+	bar := pb.StartNew(len(fileList))
+
 	for _, filePath := range fileList {
 		successful, oneFileResult := siegfriedIdent(s, filePath)
 		if !successful {
@@ -53,6 +55,9 @@ func IdentifyFiles(fileList []string, hashDigest string, nsrlEnabled bool, conn
 
 		onefilehash := hex.EncodeToString(Hashit(filePath, hashDigest))
 		oneFile := oneFileResult + ",\"" + onefilehash + "\",\"" + CreateUUID() + "\","
+
+		// we need a sha1 for redis. if sha1 is not used in this run we
+		// need to calculate sha1 for redis if nsrl is enabled
 		if nsrlEnabled {
 			var nsrlHash string
 			if calcNSRL {
@@ -79,7 +84,10 @@ func IdentifyFiles(fileList []string, hashDigest string, nsrlEnabled bool, conn
 		}
 
 		resultList = append(resultList, oneFile)
+		bar.Increment()
 	}
 
+	bar.Finish()
+
 	return resultList
 }

redis.go

@@ -4,6 +4,12 @@ import (
 	"github.com/gomodule/redigo/redis"
 )
 
+// RedisConf holds the config of a redis server
+type RedisConf struct {
+	Server *string
+	Port   *string
+}
+
 // RedisConnect creates a connection to a Redis server
 func RedisConnect(r RedisConf) redis.Conn {
 	conn, err := redis.Dial("tcp", *r.Server+":"+*r.Port)

siegfried.go

@@ -1,7 +1,7 @@
 package filedriller
 
 import (
-	"log"
+	//"log"
 	"os"
 	"strconv"
 
@@ -11,29 +11,31 @@ import (
 func siegfriedIdent(s *siegfried.Siegfried, inFile string) (bool, string) {
 	var oneFile string
 	var resultBool bool
+
 	f, err := os.Open(inFile)
 	if err != nil {
-		log.Println(err)
 		return resultBool, err.Error()
-	} else {
-		defer f.Close()
-		fi, _ := f.Stat()
-		if fi.Size() == 0 {
-			return resultBool, inFile + ",,,,,,,,"
-		}
-		ids, err := s.Identify(f, "", "")
-		if err != nil {
-			ret := inFile + " : " + err.Error()
-			log.Println(ret)
-			return resultBool, ret
-		}
-		for _, id := range ids {
-			values := id.Values()
-			for _, value := range values {
-				oneFile += "\"" + value + "\"" + ","
-			}
-			oneFile = "\"" + inFile + "\",\"" + strconv.Itoa(int(fi.Size())) + "\"," + oneFile[:len(oneFile)-1] // remove last comma
+	}
+
+	defer f.Close()
+
+	fi, _ := f.Stat()
+	if fi.Size() == 0 {
+		return resultBool, "empty file : " + inFile + ",,,,,,,,"
+	}
+
+	ids, err := s.Identify(f, "", "")
+	if err != nil {
+		ret := inFile + " : " + err.Error()
+		return resultBool, ret
+	}
+
+	for _, id := range ids {
+		values := id.Values()
+		for _, value := range values {
+			oneFile += "\"" + value + "\"" + ","
 		}
+		oneFile = "\"" + inFile + "\",\"" + strconv.Itoa(int(fi.Size())) + "\"," + oneFile[:len(oneFile)-1] // remove last comma
 	}
 
 	return true, oneFile