junk

go.mod

@@ -8,11 +8,14 @@ replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleak
 
 replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2
 
+replace github.com/bill-rich/gitparser => /home/hrich/go/src/github.com/bill-rich/gitparse
+
 require (
 	cloud.google.com/go/secretmanager v1.5.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
 	github.com/aws/aws-sdk-go v1.44.61
 	github.com/bill-rich/disk-buffer-reader v0.1.2
+	github.com/bill-rich/gitparser v0.0.0-00010101000000-000000000000
 	github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
 	github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
 	github.com/bradleyfalzon/ghinstallation/v2 v2.1.0
@@ -20,7 +23,6 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v0.6.7
 	github.com/fatih/color v1.13.0
 	github.com/felixge/fgprof v0.9.2
-	github.com/gitleaks/go-gitdiff v0.7.4
 	github.com/go-errors/errors v1.4.2
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/golang-jwt/jwt v3.2.2+incompatible
@@ -42,7 +44,7 @@ require (
 	github.com/stretchr/testify v1.8.0
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
 	github.com/xanzy/go-gitlab v0.69.0
-	github.com/zricethezav/gitleaks/v8 v8.5.2
+	github.com/zricethezav/gitleaks/v8 v8.10.3
 	golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e
 	golang.org/x/net v0.0.0-20220617184016-355a448f1bc9
 	golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb
@@ -74,6 +76,7 @@ require (
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/dsnet/compress v0.0.1 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
+	github.com/gitleaks/go-gitdiff v0.7.6 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-git/go-billy/v5 v5.3.1 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
@@ -84,7 +87,7 @@ require (
 	github.com/google/go-cmp v0.5.8 // indirect
 	github.com/google/go-github/v45 v45.2.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
+	github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
 	github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
@@ -108,7 +111,7 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/mod v0.5.0 // indirect
-	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+	golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
 	golang.org/x/tools v0.1.7 // indirect

go.sum

@@ -260,8 +260,9 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y=
 github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
+github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1 h1:8pyqKJvrJqUYaKS851Ule26pwWvey6IDMiczaBLDKLQ=
+github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1/go.mod h1:gSuNB+gJaOiQKLEZ+q+PK9Mq3SOzhRcw2GsGS/FhYDk=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
@@ -627,8 +628,9 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

main.go

@@ -192,15 +192,15 @@ func run(state overseer.State) {
 			defer os.RemoveAll(repoPath)
 		}
 
-		g := func(c sources.Config) {
+		config := sources.Config{
-			c.RepoPath = repoPath
+			RepoPath: repoPath,
-			c.HeadRef = *gitScanBranch
+			HeadRef:  *gitScanBranch,
-			c.BaseRef = *gitScanSinceCommit
+			BaseRef:  *gitScanSinceCommit,
-			c.MaxDepth = *gitScanMaxDepth
+			MaxDepth: *gitScanMaxDepth,
-			c.Filter = filter
+			Filter:   filter,
 		}
 
-		if err = e.ScanGit(ctx, sources.NewConfig(g)); err != nil {
+		if err = e.ScanGit(ctx, config); err != nil {
 			logrus.WithError(err).Fatal("Failed to scan Git.")
 		}
 	case githubScan.FullCommand():

pkg/gitparse/main.go

@@ -0,0 +1,240 @@
+//package main
+package gitparse
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"log"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// DateFormat is the standard date format for git.
+const DateFormat = "Mon Jan 02 15:04:05 2006 -0700"
+
+// Commit contains commit header info and diffs.
+type Commit struct {
+	Hash    string
+	Author  string
+	Date    time.Time
+	Message strings.Builder
+	Diffs   []Diff
+}
+
+// Diff contains the info about a file diff in a commit.
+type Diff struct {
+	PathA     string
+	PathB     string
+	LineStart int
+	Content   bytes.Buffer
+	IsBinary  bool
+}
+
+/*
+func main() {
+	source := "/home/hrich/go/src/github.com/hashicorp/terraform"
+	start := time.Now()
+	fileChan := BillParser(source)
+	for range fileChan {
+	}
+	newTime := time.Now().Sub(start)
+
+	start = time.Now()
+	errChan := make(chan error)
+	fileChan2, _ := glgo.GitLog(source, glgo.LogOpts{}, errChan)
+	for range fileChan2 {
+	}
+	oldTime := time.Now().Sub(start)
+
+	fmt.Printf("Gitleaks parser: %d seconds\nNew parser: %d seconds", oldTime.Seconds(), newTime.Seconds())
+
+}
+*/
+
+func BillParser(source string) chan Commit {
+	cmd := exec.Command("git", "-C", source, "log", "-p", "-U0", "--full-history", "--all", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z")
+
+	stdOut, err := cmd.StdoutPipe()
+	if err != nil {
+		errs(err)
+	}
+	// TODO: Handle errors
+	// stdErr, err := cmd.StderrPipe()
+	// if err != nil {
+	// 	errs(err)
+	// }
+
+	err = cmd.Start()
+	if err != nil {
+		errs(err)
+	}
+
+	outReader := bufio.NewReader(stdOut)
+	var currentCommit *Commit
+	var currentDiff *Diff
+
+	commitChan := make(chan Commit)
+
+	// TODO: Something about nil handling maybe? I don't remember
+	go func() {
+		for {
+			line, _, err := outReader.ReadLine()
+			if err != nil {
+				fmt.Println(err)
+				break
+			}
+			logrus.Debugf("Line: %s", line)
+			switch {
+			case isCommitLine(line):
+				logrus.Debugf("commitline: %s", line)
+				if currentCommit != nil {
+					commitChan <- *currentCommit
+				}
+				currentCommit = &Commit{
+					Message: strings.Builder{},
+				}
+				currentCommit.Hash = string(line[7:])
+			case isAuthorLine(line):
+				currentCommit.Author = string(line[8:])
+			case isDateLine(line):
+				date, err := time.Parse(DateFormat, strings.TrimSpace(string(line[6:])))
+				if err != nil {
+					errs(err)
+				}
+				currentCommit.Date = date
+			case isDiffLine(line):
+				if currentDiff != nil {
+					currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+				}
+				currentDiff = &Diff{}
+			case isModeLine(line):
+			case isIndexLine(line):
+			case isPlusFileLine(line):
+				currentDiff.PathB = string(line[4:])
+			case isMinusFileLine(line):
+				currentDiff.PathA = string(line[4:])
+			case isPlusDiffLine(line):
+				currentDiff.Content.Write(line[1:])
+				currentDiff.Content.Write([]byte("\n"))
+			case isMinusDiffLine(line):
+			case isMessageLine(line):
+				currentCommit.Message.Write(line[4:])
+			case isBinaryLine(line):
+				currentDiff.IsBinary = true
+				currentDiff.PathB = pathFromBinaryLine(line, source)
+			}
+
+		}
+		if currentDiff != nil && currentDiff.Content.Len() > 0 {
+			currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+		}
+		if currentCommit != nil {
+			commitChan <- *currentCommit
+		}
+		cmd.Wait()
+		close(commitChan)
+	}()
+	return commitChan
+}
+
+func isDateLine(line []byte) bool {
+	if len(line) > 7 && bytes.Equal(line[:5], []byte("Date:")) {
+		return true
+	}
+	return false
+}
+
+func isAuthorLine(line []byte) bool {
+	if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
+		return true
+	}
+	return false
+}
+
+func isCommitLine(line []byte) bool {
+	if len(line) > 7 && bytes.Equal(line[:6], []byte("commit")) {
+		return true
+	}
+	return false
+}
+
+func isDiffLine(line []byte) bool {
+	if len(line) > 5 && bytes.Equal(line[:4], []byte("diff")) {
+		return true
+	}
+	return false
+}
+
+func isIndexLine(line []byte) bool {
+	if len(line) > 6 && bytes.Equal(line[:5], []byte("index")) {
+		return true
+	}
+	return false
+}
+
+func isModeLine(line []byte) bool {
+	if len(line) > 8 && bytes.Equal(line[:8], []byte("new mode")) {
+		return true
+	}
+	return false
+}
+
+func isMinusFileLine(line []byte) bool {
+	if len(line) > 3 && bytes.Equal(line[:3], []byte("---")) {
+		return true
+	}
+	return false
+}
+
+func isPlusFileLine(line []byte) bool {
+	if len(line) > 3 && bytes.Equal(line[:3], []byte("+++")) {
+		return true
+	}
+	return false
+}
+
+func isPlusDiffLine(line []byte) bool {
+	if len(line) > 1 && bytes.Equal(line[:1], []byte("+")) {
+		return true
+	}
+	return false
+}
+
+func isMinusDiffLine(line []byte) bool {
+	if len(line) > 1 && bytes.Equal(line[:1], []byte("-")) {
+		return true
+	}
+	return false
+}
+
+func isMessageLine(line []byte) bool {
+	if len(line) > 4 && bytes.Equal(line[:4], []byte("    ")) {
+		return true
+	}
+	return false
+}
+
+func isBinaryLine(line []byte) bool {
+	if len(line) > 7 && bytes.Equal(line[:6], []byte("Binary")) {
+		return true
+	}
+	return false
+}
+
+func pathFromBinaryLine(line []byte, source string) string {
+	sbytes := bytes.Split(line, []byte(" and "))
+	if len(sbytes) != 2 {
+		logrus.Errorf("Expected binary line to be in 'Binary files a/filaA and b/fileB differ' format. Got: %s", line)
+		return ""
+	}
+	bRaw := sbytes[1]
+	return string(bRaw[2 : len(bRaw)-7]) // drop the "b/" and " differ"
+}
+
+func errs(err error) {
+	log.Println(err)
+}

pkg/gitparse/main_test.go

@@ -0,0 +1,30 @@
+package gitparse
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	glgo "github.com/zricethezav/gitleaks/v8/detect/git"
+)
+
+func TestMain(t *testing.T) {
+	source := "/home/hrich/go/src/github.com/hashicorp/terraform"
+	start := time.Now()
+	commitChan := BillParser(source)
+	for _ = range commitChan {
+	}
+	fmt.Printf("BillParser took %f seconds.", time.Now().Sub(start).Seconds())
+	start = time.Now()
+	logOpts := glgo.LogOpts{
+		DisableSafeDir: true,
+	}
+	errChan := make(chan error)
+	fileChan, err := glgo.GitLog(source, logOpts, errChan)
+	if err != nil {
+		//errs(err)
+	}
+	for _ = range fileChan {
+	}
+	fmt.Printf("GitLeaksParser took %f seconds.", time.Now().Sub(start).Seconds())
+}

pkg/sources/git/git.go

@@ -15,14 +15,13 @@ import (
 	"time"
 
 	diskbufferreader "github.com/bill-rich/disk-buffer-reader"
-	"github.com/gitleaks/go-gitdiff/gitdiff"
+	gitparse "github.com/bill-rich/gitparser"
 	"github.com/go-errors/errors"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/google/go-github/v42/github"
 	"github.com/rs/zerolog"
 	log "github.com/sirupsen/logrus"
-	glgo "github.com/zricethezav/gitleaks/v8/detect/git"
 	"golang.org/x/oauth2"
 	"golang.org/x/sync/semaphore"
 	"google.golang.org/protobuf/proto"
@@ -283,100 +282,91 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
 	}
 
 	// Errors returned on errChan aren't blocking, so just ignore them.
-	errChan := make(chan error)
 	var gitLogArgs []string
 	if scanOptions.HeadHash != "" {
 		gitLogArgs = append(gitLogArgs, scanOptions.HeadHash)
 	}
-	logOpts := glgo.LogOpts{
+	commitChan := gitparse.BillParser(path)
-		Args:           gitLogArgs,
-		DisableSafeDir: true,
-	}
-	fileChan, err := glgo.GitLog(path, logOpts, errChan)
-	if err != nil {
-		return errors.WrapPrefix(err, "could not open repo path", 0)
-	}
 	// parser can return nil chan and nil error
-	if fileChan == nil {
-		return errors.New("nothing to scan")
-	}
-
 	// get the URL metadata for reporting (may be empty)
 	urlMetadata := getSafeRemoteURL(repo, "origin")
 
 	var depth int64
 	var reachedBase = false
-	for file := range fileChan {
+	// for file := range fileChan {
-		if file == nil || file.PatchHeader == nil {
+	log.Debugf("Scanning repo")
-			log.Debugf("file missing patch header, skipping")
+	for commit := range commitChan {
-			continue
+		log.Debugf("Commit: %s", commit.Hash)
-		}
+		for _, diff := range commit.Diffs {
-		log.WithField("commit", file.PatchHeader.SHA).WithField("file", file.NewName).Trace("Scanning file from git")
+			log.Debugf("Chunk: %s", diff.Content.Bytes())
-		if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {
+			log.WithField("commit", commit.Hash).WithField("file", diff.PathB).Trace("Scanning file from git")
-			log.Debugf("reached max depth")
+			if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {
-			break
+				log.Debugf("reached max depth")
-		}
+				break
-		depth++
-		if reachedBase && file.PatchHeader.SHA != scanOptions.BaseHash {
-			break
-		}
-		if len(scanOptions.BaseHash) > 0 {
-			if file.PatchHeader.SHA == scanOptions.BaseHash {
-				log.Debugf("Reached base commit. Finishing scanning files.")
-				reachedBase = true
 			}
-		}
+			depth++
-		if !scanOptions.Filter.Pass(file.NewName) {
+			if reachedBase && commit.Hash != scanOptions.BaseHash {
-			continue
+				break
-		}
-
-		fileName := file.NewName
-		if fileName == "" {
-			continue
-		}
-		var email, hash, when string
-		if file.PatchHeader != nil {
-			if file.PatchHeader.Author != nil {
-				email = file.PatchHeader.Author.Email
 			}
-			hash = file.PatchHeader.SHA
+			if len(scanOptions.BaseHash) > 0 {
-			when = file.PatchHeader.AuthorDate.String()
+				if commit.Hash == scanOptions.BaseHash {
-		}
+					log.Debugf("Reached base commit. Finishing scanning files.")
-
+					reachedBase = true
-		// Handle binary files by reading the entire file rather than using the diff.
-		if file.IsBinary {
-			commitHash := plumbing.NewHash(hash)
-			metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, 0)
-			chunkSkel := &sources.Chunk{
-				SourceName:     s.sourceName,
-				SourceID:       s.sourceID,
-				SourceType:     s.sourceType,
-				SourceMetadata: metadata,
-				Verify:         s.verify,
-			}
-			if err = handleBinary(repo, chunksChan, chunkSkel, commitHash, fileName); err != nil {
-				log.WithError(err).Error("Error handling binary file")
-			}
-			continue
-		}
-
-		for _, frag := range file.TextFragments {
-			var sb strings.Builder
-			newLineNumber := frag.NewPosition
-			for _, line := range frag.Lines {
-				if line.Op == gitdiff.OpAdd {
-					sb.WriteString(line.Line)
 				}
 			}
-			metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, newLineNumber)
+			/*
+				if !scanOptions.Filter.Pass(diff.PathB) {
+					continue
+				}
+			*/
+
+			fileName := diff.PathB
+			if fileName == "" {
+				continue
+			}
+			var email, hash, when string
+			email = commit.Author
+			hash = commit.Hash
+			when = commit.Date.GoString()
+
+			// Handle binary files by reading the entire file rather than using the diff.
+			if diff.IsBinary {
+				commitHash := plumbing.NewHash(hash)
+				metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, 0)
+				chunkSkel := &sources.Chunk{
+					SourceName:     s.sourceName,
+					SourceID:       s.sourceID,
+					SourceType:     s.sourceType,
+					SourceMetadata: metadata,
+					Verify:         s.verify,
+				}
+				if err := handleBinary(repo, chunksChan, chunkSkel, commitHash, fileName); err != nil {
+					log.WithError(err).Error("Error handling binary file")
+				}
+				continue
+			}
+
+			/*
+				for _, frag := range file.TextFragments {
+					var sb strings.Builder
+					newLineNumber := frag.NewPosition
+					for _, line := range frag.Lines {
+						if line.Op == gitdiff.OpAdd {
+							sb.WriteString(line.Line)
+						}
+					}
+			*/
+
+			metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, int64(diff.LineStart))
 			chunksChan <- &sources.Chunk{
 				SourceName:     s.sourceName,
 				SourceID:       s.sourceID,
 				SourceType:     s.sourceType,
 				SourceMetadata: metadata,
-				Data:           []byte(sb.String()),
+				Data:           diff.Content.Bytes(),
 				Verify:         s.verify,
 			}
+			//}
 		}
 	}
 	return nil
@@ -401,9 +391,11 @@ func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunk
 			return err
 		}
 		for fh := range status {
-			if !scanOptions.Filter.Pass(fh) {
+			/*
-				continue
+				if !scanOptions.Filter.Pass(fh) {
-			}
+					continue
+				}
+			*/
 			metadata := s.sourceMetadataFunc(
 				fh, "unstaged", "unstaged", time.Now().String(), urlMetadata, 0,
 			)
@@ -643,6 +635,7 @@ func handleBinary(repo *git.Repository, chunksChan chan *sources.Chunk, chunkSke
 		return err
 	}
 
+	log.Error(path)
 	file, err := commit.File(path)
 	if err != nil {
 		return err