remove _ from npm_ and update ahocorasick to our fork

go.mod

@@ -8,7 +8,6 @@ require (
 	cloud.google.com/go/secretmanager v1.11.1
 	cloud.google.com/go/storage v1.31.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
-	github.com/BobuSumisu/aho-corasick v1.0.3
 	github.com/TheZeroSlave/zapsentry v1.17.0
 	github.com/aws/aws-sdk-go v1.44.83
 	github.com/bill-rich/disk-buffer-reader v0.1.7
@@ -151,6 +150,7 @@ require (
 	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/skeema/knownhosts v1.1.1 // indirect
 	github.com/therootcompany/xz v1.0.1 // indirect
+	github.com/trufflesecurity/aho-corasick v0.1.0
 	github.com/ulikunitz/xz v0.5.10 // indirect
 	github.com/vbatts/tar-split v0.11.3 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect

go.sum

@@ -51,8 +51,6 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g=
-github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -447,6 +445,8 @@ github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HP
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
 github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
 github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
+github.com/trufflesecurity/aho-corasick v0.1.0 h1:bwtdjWFkuzDRLWjJi4fCxkBSv2axhYXJryCqsSgMHSg=
+github.com/trufflesecurity/aho-corasick v0.1.0/go.mod h1:JKKZLgcVlyWn0LsPezrqEIQPi76i2URv44hC0gJY4Vo=
 github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
 github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=

pkg/context/context.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"runtime/debug"
-	"sync"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -29,9 +28,8 @@ type CancelFunc = context.CancelFunc
 type logCtx struct {
 	// Embed context.Context to get all methods for free.
 	context.Context
-	log     logr.Logger
-	err     *error
-	errLock *sync.Mutex
+	log logr.Logger
+	err *error
 }
 
 // Logger returns a structured logger.
@@ -147,11 +145,8 @@ func captureCancelCallstack(ctx logCtx, f context.CancelFunc) (Context, context.
 	if ctx.err == nil {
 		var err error
 		ctx.err = &err
-		ctx.errLock = &sync.Mutex{}
 	}
 	return ctx, func() {
-		ctx.errLock.Lock()
-		defer ctx.errLock.Unlock()
 		// We must check Err() before calling f() since f() sets the error.
 		// If there's already an error, do nothing special.
 		if ctx.Err() != nil {

pkg/context/context_test.go

@@ -212,9 +212,3 @@ func TestErrCallstackTimeoutCancel(t *testing.T) {
 	cancel()
 	assert.Equal(t, err, ctx.Err())
 }
-
-func TestRace(t *testing.T) {
-	_, cancel := WithCancel(Background())
-	go cancel()
-	cancel()
-}

pkg/detectors/npmtokenv2/npmtokenv2.go

@@ -29,7 +29,7 @@ var (
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"npm_"}
+	return []string{"npm"}
 }
 
 // FromData will find and optionally verify NpmTokenV2 secrets in a given set of bytes.

pkg/engine/engine.go

@@ -10,7 +10,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	ahocorasick "github.com/BobuSumisu/aho-corasick"
+	ahocorasick "github.com/trufflesecurity/aho-corasick"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 
@@ -43,7 +43,7 @@ type Engine struct {
 
 	// prefilter is a ahocorasick struct used for doing efficient string
 	// matching given a set of words (keywords from the rules in the config)
-	prefilter ahocorasick.Trie
+	prefilter ahocorasick.AhoCorasick
 }
 
 type EngineOption func(*Engine)
@@ -145,16 +145,18 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	// on keywords
 	keywords := []string{}
 	for _, d := range e.detectors[false] {
-		for _, kw := range d.Keywords() {
-			keywords = append(keywords, strings.ToLower(kw))
-		}
+		keywords = append(keywords, d.Keywords()...)
 	}
 	for _, d := range e.detectors[true] {
-		for _, kw := range d.Keywords() {
-			keywords = append(keywords, strings.ToLower(kw))
-		}
+		keywords = append(keywords, d.Keywords()...)
 	}
-	e.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
+	builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{
+		AsciiCaseInsensitive: true,
+		MatchOnlyWholeWords:  false,
+		MatchKind:            ahocorasick.StandardMatch,
+		DFA:                  false,
+	})
+	e.prefilter = builder.Build(keywords)
 
 	ctx.Logger().Info("loaded decoders", "count", len(e.decoders))
 	ctx.Logger().Info("loaded detectors",
@@ -295,8 +297,8 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 				}
 
 				// build a map of all keywords that were matched in the chunk
-				for _, m := range e.prefilter.MatchString(strings.ToLower(string(decoded.Data))) {
-					matchedKeywords[strings.ToLower(m.MatchString())] = struct{}{}
+				for _, m := range e.prefilter.FindAll(string(decoded.Data)) {
+					matchedKeywords[strings.ToLower(string(decoded.Data[m.Start():m.End()]))] = struct{}{}
 				}
 
 				for verify, detectorsSet := range e.detectors {
@@ -305,7 +307,6 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 						for _, kw := range detector.Keywords() {
 							if _, ok := matchedKeywords[strings.ToLower(kw)]; ok {
 								chunkContainsKeyword = true
-								break
 							}
 						}
 
@@ -349,15 +350,8 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 								continue
 							}
 							result.DecoderType = decoderType
-							if result.ExtraData == nil && chunk.HandleMetadata != nil {
-								result.ExtraData = chunk.HandleMetadata
-							} else {
-								for k, v := range chunk.HandleMetadata {
-									// TODO: Check key collisions.
-									result.ExtraData[k] = v
-								}
-							}
 							chunkResults = append(chunkResults, detectors.CopyMetadata(resultChunk, result))
+
 						}
 						if len(results) > 0 {
 							elapsed := time.Since(start)

pkg/gitparse/gitparse.go

@@ -51,47 +51,6 @@ type Parser struct {
 	dateFormat    string
 }
 
-type ParseState int
-
-const (
-	Initial ParseState = iota
-	CommitLine
-	AuthorLine
-	DateLine
-	MessageStartLine
-	MessageLine
-	MessageEndLine
-	DiffLine
-	ModeLine
-	IndexLine
-	FromFileLine
-	ToFileLine
-	BinaryFileLine
-	HunkLineNumberLine
-	HunkContentLine
-	ParseFailure
-)
-
-func (state ParseState) String() string {
-	return [...]string{
-		"Initial",
-		"CommitLine",
-		"AuthorLine",
-		"DateLine",
-		"MessageStartLine",
-		"MessageLine",
-		"MessageEndLine",
-		"DiffLine",
-		"ModeLine",
-		"IndexLine",
-		"FromFileLine",
-		"ToFileLine",
-		"BinaryFileLine",
-		"HunkLineNumberLine",
-		"HunkContentLine",
-	}[state]
-}
-
 // WithMaxDiffSize sets maxDiffSize option. Diffs larger than maxDiffSize will
 // be truncated.
 func WithMaxDiffSize(maxDiffSize int) Option {
@@ -178,7 +137,7 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
 		cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
 	}
 
-	return c.executeCommand(ctx, cmd, false)
+	return c.executeCommand(ctx, cmd)
 }
 
 // Staged parses the output of the `git diff` command for the `source` path.
@@ -193,11 +152,11 @@ func (c *Parser) Staged(ctx context.Context, source string) (chan Commit, error)
 		cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
 	}
 
-	return c.executeCommand(ctx, cmd, true)
+	return c.executeCommand(ctx, cmd)
 }
 
 // executeCommand runs an exec.Cmd, reads stdout and stderr, and waits for the Cmd to complete.
-func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged bool) (chan Commit, error) {
+func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit, error) {
 	commitChan := make(chan Commit, 64)
 
 	stdOut, err := cmd.StdoutPipe()
@@ -222,7 +181,7 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged boo
 	}()
 
 	go func() {
-		c.FromReader(ctx, stdOut, commitChan, isStaged)
+		c.FromReader(ctx, stdOut, commitChan)
 		if err := cmd.Wait(); err != nil {
 			ctx.Logger().V(2).Info("Error waiting for git command to complete.", "error", err)
 		}
@@ -231,15 +190,14 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged boo
 	return commitChan, nil
 }
 
-func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit, isStaged bool) {
+func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
 	outReader := bufio.NewReader(stdOut)
 	var (
-		currentCommit *Commit
-		currentDiff   *Diff
-
-		totalLogSize int
+		currentCommit        *Commit
+		currentDiff          *Diff
+		recentlyPassedAuthor bool
+		totalLogSize         int
 	)
-	var latestState = Initial
 
 	defer common.RecoverWithExit(ctx)
 	defer close(commitChan)
@@ -247,16 +205,12 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 		if common.IsDone(ctx) {
 			break
 		}
-
 		line, err := outReader.ReadBytes([]byte("\n")[0])
 		if err != nil && len(line) == 0 {
 			break
 		}
-
 		switch {
-		case isCommitLine(isStaged, latestState, line):
-			latestState = CommitLine
-
+		case isCommitLine(line):
 			// If there is a currentDiff, add it to currentCommit.
 			if currentDiff != nil && currentDiff.Content.Len() > 0 {
 				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
@@ -276,31 +230,16 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			if len(line) >= 47 {
 				currentCommit.Hash = string(line[7:47])
 			}
-		case isAuthorLine(isStaged, latestState, line):
-			latestState = AuthorLine
-
+		case isAuthorLine(line):
 			currentCommit.Author = strings.TrimRight(string(line[8:]), "\n")
-		case isDateLine(isStaged, latestState, line):
-			latestState = DateLine
-
+			recentlyPassedAuthor = true
+		case isDateLine(line):
 			date, err := time.Parse(c.dateFormat, strings.TrimSpace(string(line[6:])))
 			if err != nil {
 				ctx.Logger().V(2).Info("Could not parse date from git stream.", "error", err)
 			}
 			currentCommit.Date = date
-		case isMessageStartLine(isStaged, latestState, line):
-			latestState = MessageStartLine
-			// NoOp
-		case isMessageLine(isStaged, latestState, line):
-			latestState = MessageLine
-
-			currentCommit.Message.Write(line[4:])
-		case isMessageEndLine(isStaged, latestState, line):
-			latestState = MessageEndLine
-			// NoOp
-		case isDiffLine(isStaged, latestState, line):
-			latestState = DiffLine
-
+		case isDiffLine(line):
 			// This should never be nil, but check in case the stdin stream is messed up.
 			if currentCommit == nil {
 				currentCommit = &Commit{}
@@ -328,29 +267,27 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 				}
 			}
 			currentDiff = &Diff{}
-		case isModeLine(isStaged, latestState, line):
-			latestState = ModeLine
+		case isModeLine(line):
 			// NoOp
-		case isIndexLine(isStaged, latestState, line):
-			latestState = IndexLine
+		case isIndexLine(line):
+			recentlyPassedAuthor = false
 			// NoOp
-		case isBinaryLine(isStaged, latestState, line):
-			latestState = BinaryFileLine
-
+		case isPlusFileLine(line):
+			currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
+		case isMinusFileLine(line):
+			// NoOp
+		case isPlusDiffLine(line):
+			currentDiff.Content.Write(line[1:])
+		case isMinusDiffLine(line):
+			// NoOp. We only care about additions.
+		case (isMessageLine(line) && recentlyPassedAuthor):
+			currentCommit.Message.Write(line[4:])
+		case isContextDiffLine(line):
+			currentDiff.Content.Write([]byte("\n"))
+		case isBinaryLine(line):
 			currentDiff.IsBinary = true
 			currentDiff.PathB = pathFromBinaryLine(line)
-		case isFromFileLine(isStaged, latestState, line):
-			latestState = FromFileLine
-			// NoOp
-		case isToFileLine(isStaged, latestState, line):
-			latestState = ToFileLine
-
-			// TODO: Is this fix still required?
-			currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
-		case isHunkLineNumberLine(isStaged, latestState, line):
-			latestState = HunkLineNumberLine
-
-			// TODO: Is it still necessary to check whether the currentDiff is nil?
+		case isLineNumberDiffLine(line):
 			if currentDiff != nil && currentDiff.Content.Len() > 0 {
 				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 			}
@@ -368,54 +305,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 					currentDiff.LineStart = lineStart
 				}
 			}
-		case isHunkContextLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-			// TODO: Why do we care about this? It creates empty lines in the diff. If there are no plusLines, it's just newlines.
-			currentDiff.Content.Write([]byte("\n"))
-		case isHunkPlusLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-
-			currentDiff.Content.Write(line[1:])
-		case isHunkMinusLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-			// NoOp. We only care about additions.
-		case isHunkNewlineWarningLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-			// NoOp
-		case isHunkEmptyLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-			// NoOp
-		case isCommitSeparatorLine(isStaged, latestState, line):
-			// NoOp
-		default:
-			// Skip ahead until we find the next diff or commit.
-			if latestState == ParseFailure {
-				continue
-			}
-
-			// Here be dragons...
-			// Build an informative error message.
-			var err error
-			if currentCommit != nil && currentCommit.Hash != "" {
-				err = fmt.Errorf(`failed to parse line "%s" after state "%s" (commit=%s)`, line, latestState, currentCommit.Hash)
-			} else {
-				err = fmt.Errorf(`failed to parse line "%s" after state "%s"`, line, latestState)
-			}
-			ctx.Logger().V(2).Error(err, "Recovering at the latest commit or diff...\n")
-
-			latestState = ParseFailure
 		}
-
 		if currentDiff.Content.Len() > c.maxDiffSize {
 			ctx.Logger().V(2).Info(fmt.Sprintf(
 				"Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize,
@@ -428,147 +318,130 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 	ctx.Logger().V(2).Info("finished parsing git log.", "total_log_size", totalLogSize)
 }
 
-// commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05
-func isCommitLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || !(latestState == Initial ||
-		latestState == MessageStartLine ||
-		latestState == MessageEndLine ||
-		latestState == ModeLine ||
-		latestState == IndexLine ||
-		latestState == BinaryFileLine ||
-		latestState == ToFileLine ||
-		latestState == HunkContentLine ||
-		latestState == ParseFailure) {
-		return false
+func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
+	if currentDiff != nil && currentDiff.Content.Len() > 0 {
+		currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 	}
-
-	if len(line) > 7 && bytes.Equal(line[:7], []byte("commit ")) {
-		return true
+	if currentCommit != nil {
+		commitChan <- *currentCommit
+		if totalLogSize != nil {
+			*totalLogSize += currentCommit.Size
+		}
 	}
-	return false
-}
-
-// Author: Bill Rich <bill.rich@trufflesec.com>
-func isAuthorLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || !(latestState == CommitLine) {
-		return false
-	}
-	if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
-		return true
-	}
-	return false
 }
 
 // Date:   Tue Aug 10 15:20:40 2021 +0100
-func isDateLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || latestState != AuthorLine {
-		return false
-	}
+func isDateLine(line []byte) bool {
 	if len(line) > 7 && bytes.Equal(line[:5], []byte("Date:")) {
 		return true
 	}
 	return false
 }
 
-// Line directly after Date with only a newline.
-func isMessageStartLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || latestState != DateLine {
-		return false
-	}
-	// TODO: Improve the implementation of this and isMessageEndLine
-	if len(strings.TrimRight(string(line[:]), "\r\n")) == 0 {
+// Author: Bill Rich <bill.rich@trufflesec.com>
+func isAuthorLine(line []byte) bool {
+	if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
 		return true
 	}
 	return false
 }
 
-// Line that starts with 4 spaces
-func isMessageLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || !(latestState == MessageStartLine || latestState == MessageLine) {
-		return false
-	}
-	if len(line) > 4 && bytes.Equal(line[:4], []byte("    ")) {
-		return true
-	}
-	return false
-}
-
-// Line directly after MessageLine with only a newline.
-func isMessageEndLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || latestState != MessageLine {
-		return false
-	}
-	if len(strings.TrimRight(string(line[:]), "\r\n")) == 0 {
+// commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05
+func isCommitLine(line []byte) bool {
+	if len(line) > 7 && bytes.Equal(line[:6], []byte("commit")) {
 		return true
 	}
 	return false
 }
 
 // diff --git a/internal/addrs/move_endpoint_module.go b/internal/addrs/move_endpoint_module.go
-func isDiffLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == MessageStartLine || // Empty commit messages can go from MessageStart->Diff
-		latestState == MessageEndLine ||
-		latestState == BinaryFileLine ||
-		latestState == IndexLine ||
-		latestState == HunkContentLine ||
-		latestState == ParseFailure) {
-		if latestState == Initial && !isStaged {
-			return false
-		}
-	}
-	if len(line) > 11 && bytes.Equal(line[:11], []byte("diff --git ")) {
-		return true
-	}
-	return false
-}
-
-// old mode 100644
-// new mode 100755
-// new file mode 100644
-// similarity index 100%
-// rename from old.txt
-// rename to new.txt
-// deleted file mode 100644
-func isModeLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == DiffLine || latestState == ModeLine) {
-		return false
-	}
-	// This could probably be better written.
-	if (len(line) > 17 && bytes.Equal(line[:17], []byte("deleted file mode"))) ||
-		(len(line) > 16 && bytes.Equal(line[:16], []byte("similarity index"))) ||
-		(len(line) > 13 && bytes.Equal(line[:13], []byte("new file mode"))) ||
-		(len(line) > 11 && bytes.Equal(line[:11], []byte("rename from"))) ||
-		(len(line) > 9 && bytes.Equal(line[:9], []byte("rename to"))) ||
-		(len(line) > 8 && bytes.Equal(line[:8], []byte("old mode"))) ||
-		(len(line) > 8 && bytes.Equal(line[:8], []byte("new mode"))) {
+func isDiffLine(line []byte) bool {
+	if len(line) > 5 && bytes.Equal(line[:4], []byte("diff")) {
 		return true
 	}
 	return false
 }
 
 // index 1ed6fbee1..aea1e643a 100644
-// index 00000000..e69de29b
-func isIndexLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == DiffLine || latestState == ModeLine) {
-		return false
+func isIndexLine(line []byte) bool {
+	if len(line) > 6 && bytes.Equal(line[:5], []byte("index")) {
+		return true
 	}
-	if len(line) > 6 && bytes.Equal(line[:6], []byte("index ")) {
+	return false
+}
+
+// new file mode 100644
+func isModeLine(line []byte) bool {
+	if len(line) > 13 && bytes.Equal(line[:13], []byte("new file mode")) {
+		return true
+	}
+	return false
+}
+
+// --- a/internal/addrs/move_endpoint_module.go
+func isMinusFileLine(line []byte) bool {
+	if len(line) >= 6 && bytes.Equal(line[:3], []byte("---")) {
+		return true
+	}
+	return false
+}
+
+// +++ b/internal/addrs/move_endpoint_module.go
+func isPlusFileLine(line []byte) bool {
+	if len(line) >= 6 && bytes.Equal(line[:4], []byte("+++ ")) {
+		return true
+	}
+	return false
+}
+
+// +fmt.Println("ok")
+func isPlusDiffLine(line []byte) bool {
+	if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) {
+		return true
+	}
+	return false
+}
+
+// -fmt.Println("ok")
+func isMinusDiffLine(line []byte) bool {
+	if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) {
+		return true
+	}
+	return false
+}
+
+// fmt.Println("ok")
+func isContextDiffLine(line []byte) bool {
+	if len(line) >= 1 && bytes.Equal(line[:1], []byte(" ")) {
+		return true
+	}
+	return false
+}
+
+// Line that starts with 4 spaces
+func isMessageLine(line []byte) bool {
+	if len(line) > 4 && bytes.Equal(line[:4], []byte("    ")) {
 		return true
 	}
 	return false
 }
 
 // Binary files /dev/null and b/plugin.sig differ
-func isBinaryLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if latestState != IndexLine {
-		return false
-	}
+func isBinaryLine(line []byte) bool {
 	if len(line) > 7 && bytes.Equal(line[:6], []byte("Binary")) {
 		return true
 	}
 	return false
 }
 
+// @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)
+func isLineNumberDiffLine(line []byte) bool {
+	if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) {
+		return true
+	}
+	return false
+}
+
 // Get the b/ file path. Ignoring the edge case of files having `and /b` in the name for simplicity.
 func pathFromBinaryLine(line []byte) string {
 	logger := context.Background().Logger()
@@ -580,117 +453,3 @@ func pathFromBinaryLine(line []byte) string {
 	bRaw := sbytes[1]
 	return string(bRaw[:len(bRaw)-7]) // drop the "b/" and " differ"
 }
-
-// --- a/internal/addrs/move_endpoint_module.go
-func isFromFileLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if latestState != IndexLine {
-		return false
-	}
-	if len(line) >= 6 && bytes.Equal(line[:4], []byte("--- ")) {
-		return true
-	}
-	return false
-}
-
-// +++ b/internal/addrs/move_endpoint_module.go
-func isToFileLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if latestState != FromFileLine {
-		return false
-	}
-	if len(line) >= 6 && bytes.Equal(line[:4], []byte("+++ ")) {
-		return true
-	}
-	return false
-}
-
-// @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)
-func isHunkLineNumberLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == ToFileLine || latestState == HunkContentLine) {
-		return false
-	}
-	if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) {
-		return true
-	}
-	return false
-}
-
-// fmt.Println("ok")
-// (There's a space before `fmt` that gets removed by the formatter.)
-func isHunkContextLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
-		return false
-	}
-	if len(line) >= 1 && bytes.Equal(line[:1], []byte(" ")) {
-		return true
-	}
-	return false
-}
-
-// +fmt.Println("ok")
-func isHunkPlusLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
-		return false
-	}
-	if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) {
-		return true
-	}
-	return false
-}
-
-// -fmt.Println("ok")
-func isHunkMinusLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
-		return false
-	}
-	if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) {
-		return true
-	}
-	return false
-}
-
-// \ No newline at end of file
-func isHunkNewlineWarningLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if latestState != HunkContentLine {
-		return false
-	}
-	if len(line) >= 27 && bytes.Equal(line[:27], []byte("\\ No newline at end of file")) {
-		return true
-	}
-	return false
-}
-
-// Newline after hunk, or an empty line, e.g.
-// +}
-//
-// commit 00920984e3435057f09cee5468850f7546dfa637 (tag: v3.42.0)
-func isHunkEmptyLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
-		return false
-	}
-	// TODO: Can this also be `\n\r`?
-	if len(line) == 1 && bytes.Equal(line[:1], []byte("\n")) {
-		return true
-	}
-	return false
-}
-
-func isCommitSeparatorLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if (latestState == ModeLine || latestState == IndexLine || latestState == BinaryFileLine || latestState == ToFileLine) &&
-		len(line) == 1 && bytes.Equal(line[:1], []byte("\n")) {
-		return true
-	}
-	return false
-}
-
-func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
-	// Ignore empty or binary diffs (this condition may be redundant).
-	if currentDiff != nil && currentDiff.Content.Len() > 0 {
-		currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
-	}
-	if currentCommit != nil {
-		commitChan <- *currentCommit
-		if totalLogSize != nil {
-			*totalLogSize += currentCommit.Size
-		}
-	}
-}

pkg/handlers/archive.go

@@ -6,14 +6,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"strings"
 	"time"
 
 	"github.com/mholt/archiver/v4"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
 type ctxKey int
@@ -54,14 +52,14 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
 }
 
 // FromFile extracts the files from an archive.
-func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan ChunkOpt {
-	archiveChan := make(chan ChunkOpt, 512)
+func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan ([]byte) {
+	archiveChan := make(chan ([]byte), 512)
 	go func() {
 		ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
 		logger := logContext.AddLogger(ctx).Logger()
 		defer cancel()
 		defer close(archiveChan)
-		err := d.openArchive(ctx, nil, data, archiveChan)
+		err := d.openArchive(ctx, 0, data, archiveChan)
 		if err != nil {
 			if errors.Is(err, archiver.ErrNoMatch) {
 				return
@@ -73,21 +71,18 @@ func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan Chu
 }
 
 // openArchive takes a reader and extracts the contents up to the maximum depth.
-func (d *Archive) openArchive(ctx context.Context, depth []string, reader io.Reader, archiveChan chan ChunkOpt) error {
-	if len(depth) >= maxDepth {
+func (d *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan ([]byte)) error {
+	if depth >= maxDepth {
 		return fmt.Errorf("max archive depth reached")
 	}
 	format, reader, err := archiver.Identify("", reader)
 	if err != nil {
-		if errors.Is(err, archiver.ErrNoMatch) && len(depth) > 0 {
+		if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
 			chunkSize := 10 * 1024
 			for {
-				data := make([]byte, chunkSize)
-				n, _ := reader.Read(data)
-				archiveChan <- func(chunk *sources.Chunk) {
-					chunk.Data = data
-					chunk.HandleMetadata = map[string]string{"archive path": strings.Join(depth, " > ")}
-				}
+				chunk := make([]byte, chunkSize)
+				n, _ := reader.Read(chunk)
+				archiveChan <- chunk
 				if n < chunkSize {
 					break
 				}
@@ -107,9 +102,9 @@ func (d *Archive) openArchive(ctx context.Context, depth []string, reader io.Rea
 			return err
 		}
 		newReader := bytes.NewReader(fileBytes)
-		return d.openArchive(ctx, append(depth, "decompressed"), newReader, archiveChan)
+		return d.openArchive(ctx, depth+1, newReader, archiveChan)
 	case archiver.Extractor:
-		err := archive.Extract(context.WithValue(ctx, depthKey, depth), reader, nil, d.extractorHandler(archiveChan))
+		err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, d.extractorHandler(archiveChan))
 		if err != nil {
 			return err
 		}
@@ -134,12 +129,12 @@ func (d *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader,
 }
 
 // extractorHandler is applied to each file in an archiver.Extractor file.
-func (d *Archive) extractorHandler(archiveChan chan ChunkOpt) func(context.Context, archiver.File) error {
+func (d *Archive) extractorHandler(archiveChan chan ([]byte)) func(context.Context, archiver.File) error {
 	return func(ctx context.Context, f archiver.File) error {
 		logger := logContext.AddLogger(ctx).Logger()
 		logger.V(5).Info("Handling extracted file.", "filename", f.Name())
-		var depth []string
-		if ctxDepth, ok := ctx.Value(depthKey).([]string); ok {
+		depth := 0
+		if ctxDepth, ok := ctx.Value(depthKey).(int); ok {
 			depth = ctxDepth
 		}
 
@@ -153,7 +148,7 @@ func (d *Archive) extractorHandler(archiveChan chan ChunkOpt) func(context.Conte
 		}
 		fileContent := bytes.NewReader(fileBytes)
 
-		err = d.openArchive(ctx, append(depth, f.Name()), fileContent, archiveChan)
+		err = d.openArchive(ctx, depth, fileContent, archiveChan)
 		if err != nil {
 			return err
 		}

pkg/handlers/archive_test.go

@@ -85,11 +85,9 @@ func TestArchiveHandler(t *testing.T) {
 		count := 0
 		re := regexp.MustCompile(testCase.matchString)
 		matched := false
-		var chunk sources.Chunk
-		for opt := range archiveChan {
+		for chunk := range archiveChan {
 			count++
-			opt(&chunk)
-			if re.Match(chunk.Data) {
+			if re.Match(chunk) {
 				matched = true
 			}
 		}

pkg/handlers/handlers.go

@@ -13,10 +13,8 @@ func DefaultHandlers() []Handler {
 	}
 }
 
-type ChunkOpt func(*sources.Chunk)
-
 type Handler interface {
-	FromFile(context.Context, io.Reader) chan ChunkOpt
+	FromFile(context.Context, io.Reader) chan ([]byte)
 	IsFiletype(context.Context, io.Reader) (io.Reader, bool)
 	New()
 }
@@ -40,13 +38,13 @@ func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, c
 	handlerChan := handler.FromFile(ctx, file)
 	for {
 		select {
-		case opt, open := <-handlerChan:
+		case data, open := <-handlerChan:
 			if !open {
 				// We finished reading everything from handlerChan.
 				return true
 			}
 			chunk := *chunkSkel
-			opt(&chunk)
+			chunk.Data = data
 			// Send data on chunksChan.
 			select {
 			case chunksChan <- &chunk:

pkg/output/plain.go

@@ -96,9 +96,9 @@ func structToMap(obj interface{}) (m map[string]map[string]interface{}, err erro
 }
 
 type outputFormat struct {
-	DetectorType string
-	DecoderType  string
-	Verified     bool
-	Raw          string
+	DetectorType,
+	DecoderType string
+	Verified bool
+	Raw      string
 	*source_metadatapb.MetaData
 }

pkg/sources/sources.go

@@ -21,8 +21,7 @@ type Chunk struct {
 	SourceType sourcespb.SourceType
 	// SourceMetadata holds the context of where the Chunk was found.
 	SourceMetadata *source_metadatapb.MetaData
-	// HandleMetadata holds the metadata from a handler if one was used.
-	HandleMetadata map[string]string
+
 	// Data is the data to decode and scan.
 	Data []byte
 	// Verify specifies whether any secrets in the Chunk should be verified.