saving progress

go.mod

@@ -8,7 +8,6 @@ require (
 	cloud.google.com/go/secretmanager v1.11.1
 	cloud.google.com/go/storage v1.31.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
-	github.com/BobuSumisu/aho-corasick v1.0.3
 	github.com/TheZeroSlave/zapsentry v1.17.0
 	github.com/aws/aws-sdk-go v1.44.83
 	github.com/bill-rich/disk-buffer-reader v0.1.7
@@ -45,6 +44,7 @@ require (
 	github.com/mholt/archiver/v4 v4.0.0-alpha.8
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/paulbellamy/ratecounter v0.2.0
+	github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.16.0
 	github.com/rabbitmq/amqp091-go v1.8.1
@@ -94,7 +94,7 @@ require (
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/connesc/cipherio v0.2.1 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
-	github.com/couchbase/gocbcore/v10 v10.2.4 // indirect
+	github.com/couchbase/gocbcore/v10 v10.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/docker/cli v23.0.5+incompatible // indirect

go.sum

@@ -51,8 +51,6 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g=
-github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -121,9 +119,8 @@ github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSk
 github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
 github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
 github.com/couchbase/gocb/v2 v2.6.3/go.mod h1:yF5F6BHTZ/ZowhEuZbySbXrlI4rHd1TIhm5azOaMbJU=
+github.com/couchbase/gocbcore/v10 v10.2.3 h1:PEkRSNSkKjUBXx82Ucr094+anoiCG5GleOOQZOHo6D4=
 github.com/couchbase/gocbcore/v10 v10.2.3/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
-github.com/couchbase/gocbcore/v10 v10.2.4 h1:TDTQ1mSBUw9eajuV71ZDJcLRbmfGcsvZV9SMeUDAVGc=
-github.com/couchbase/gocbcore/v10 v10.2.4/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
 github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
 github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4=
 github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
@@ -385,6 +382,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
 github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
+github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
+github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
 github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=

pkg/context/context.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"runtime/debug"
-	"sync"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -29,9 +28,8 @@ type CancelFunc = context.CancelFunc
 type logCtx struct {
 	// Embed context.Context to get all methods for free.
 	context.Context
-	log     logr.Logger
-	err     *error
-	errLock *sync.Mutex
+	log logr.Logger
+	err *error
 }
 
 // Logger returns a structured logger.
@@ -147,11 +145,8 @@ func captureCancelCallstack(ctx logCtx, f context.CancelFunc) (Context, context.
 	if ctx.err == nil {
 		var err error
 		ctx.err = &err
-		ctx.errLock = &sync.Mutex{}
 	}
 	return ctx, func() {
-		ctx.errLock.Lock()
-		defer ctx.errLock.Unlock()
 		// We must check Err() before calling f() since f() sets the error.
 		// If there's already an error, do nothing special.
 		if ctx.Err() != nil {

pkg/context/context_test.go

@@ -212,9 +212,3 @@ func TestErrCallstackTimeoutCancel(t *testing.T) {
 	cancel()
 	assert.Equal(t, err, ctx.Err())
 }
-
-func TestRace(t *testing.T) {
-	_, cancel := WithCancel(Background())
-	go cancel()
-	cancel()
-}

pkg/detectors/npmtoken/npmtoken.go

@@ -36,6 +36,7 @@ func (s Scanner) Keywords() []string {
 // FromData will find and optionally verify NpmToken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
+
 	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 	for _, match := range matches {
 		if len(match) != 2 {

pkg/detectors/postgres/postgres.go

@@ -0,0 +1,105 @@
+package postgres
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	client *http.Client
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	defaultClient = common.SaneHttpClient()
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"postgres"}) + `\b([0-9a-zA-Z]{23}_[0-9a-zA-Z]{8})\b`)
+)
+
+// find various connection strings
+
+// postgresql://user:secret@localhost:5432/mydb
+// "host=localhost dbname=publishing user=www password=foo"
+
+// detectors.PrefixRegex([]string{"nytimes"})
+
+//var (
+//	connectionStringPats = map[string]*regexp.Regexp{
+//		"KeyValue":               regexp.MustCompile(`xoxb\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
+//		"":              regexp.MustCompile(`xoxp\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
+//		"Slack Workspace Access Token":  regexp.MustCompile(`xoxa\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
+//		"Slack Workspace Refresh Token": regexp.MustCompile(`xoxr\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
+//	}
+//	verifyURL = "https://slack.com/api/auth.test"
+//)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"postgres://", ""}
+}
+
+// FromData will find and optionally verify Postgres secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		if len(match) != 2 {
+			continue
+		}
+		resMatch := strings.TrimSpace(match[1])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_Postgres,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://eth-mainnet.g.postgres.com/v2/"+resMatch+"/getNFTs/?owner=vitalik.eth", nil)
+			if err != nil {
+				continue
+			}
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
+					s1.Verified = true
+				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
+					}
+
+					if res.StatusCode != 401 {
+						s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
+					}
+				}
+			} else {
+				s1.VerificationError = err
+			}
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Postgres
+}

pkg/detectors/postgres/postgres_test.go

@@ -0,0 +1,160 @@
+//go:build detectors
+// +build detectors
+
+package postgres
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPostgres_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("POSTGRES")
+	inactiveSecret := testSecrets.MustGetField("POSTGRES_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postgres secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     true,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postgres secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postgres secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantStatusHttpClient(404)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postgres secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Postgres.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationErr)
+				}
+				got[i].VerificationError = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/engine/engine.go

@@ -10,7 +10,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	ahocorasick "github.com/BobuSumisu/aho-corasick"
+	ahocorasick "github.com/petar-dambovaliev/aho-corasick"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 
@@ -43,7 +43,7 @@ type Engine struct {
 
 	// prefilter is a ahocorasick struct used for doing efficient string
 	// matching given a set of words (keywords from the rules in the config)
-	prefilter ahocorasick.Trie
+	prefilter ahocorasick.AhoCorasick
 }
 
 type EngineOption func(*Engine)
@@ -145,16 +145,18 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	// on keywords
 	keywords := []string{}
 	for _, d := range e.detectors[false] {
-		for _, kw := range d.Keywords() {
-			keywords = append(keywords, strings.ToLower(kw))
-		}
+		keywords = append(keywords, d.Keywords()...)
 	}
 	for _, d := range e.detectors[true] {
-		for _, kw := range d.Keywords() {
-			keywords = append(keywords, strings.ToLower(kw))
-		}
+		keywords = append(keywords, d.Keywords()...)
 	}
-	e.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
+	builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{
+		AsciiCaseInsensitive: true,
+		MatchOnlyWholeWords:  false,
+		MatchKind:            ahocorasick.LeftMostLongestMatch,
+		DFA:                  true,
+	})
+	e.prefilter = builder.Build(keywords)
 
 	ctx.Logger().Info("loaded decoders", "count", len(e.decoders))
 	ctx.Logger().Info("loaded detectors",
@@ -295,8 +297,8 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 				}
 
 				// build a map of all keywords that were matched in the chunk
-				for _, m := range e.prefilter.MatchString(strings.ToLower(string(decoded.Data))) {
-					matchedKeywords[strings.ToLower(m.MatchString())] = struct{}{}
+				for _, m := range e.prefilter.FindAll(string(decoded.Data)) {
+					matchedKeywords[strings.ToLower(string(decoded.Data[m.Start():m.End()]))] = struct{}{}
 				}
 
 				for verify, detectorsSet := range e.detectors {
@@ -349,15 +351,8 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 								continue
 							}
 							result.DecoderType = decoderType
-							if result.ExtraData == nil && chunk.HandleMetadata != nil {
-								result.ExtraData = chunk.HandleMetadata
-							} else {
-								for k, v := range chunk.HandleMetadata {
-									// TODO: Check key collisions.
-									result.ExtraData[k] = v
-								}
-							}
 							chunkResults = append(chunkResults, detectors.CopyMetadata(resultChunk, result))
+
 						}
 						if len(results) > 0 {
 							elapsed := time.Since(start)

pkg/gitparse/gitparse.go

@@ -51,47 +51,6 @@ type Parser struct {
 	dateFormat    string
 }
 
-type ParseState int
-
-const (
-	Initial ParseState = iota
-	CommitLine
-	AuthorLine
-	DateLine
-	MessageStartLine
-	MessageLine
-	MessageEndLine
-	DiffLine
-	ModeLine
-	IndexLine
-	FromFileLine
-	ToFileLine
-	BinaryFileLine
-	HunkLineNumberLine
-	HunkContentLine
-	ParseFailure
-)
-
-func (state ParseState) String() string {
-	return [...]string{
-		"Initial",
-		"CommitLine",
-		"AuthorLine",
-		"DateLine",
-		"MessageStartLine",
-		"MessageLine",
-		"MessageEndLine",
-		"DiffLine",
-		"ModeLine",
-		"IndexLine",
-		"FromFileLine",
-		"ToFileLine",
-		"BinaryFileLine",
-		"HunkLineNumberLine",
-		"HunkContentLine",
-	}[state]
-}
-
 // WithMaxDiffSize sets maxDiffSize option. Diffs larger than maxDiffSize will
 // be truncated.
 func WithMaxDiffSize(maxDiffSize int) Option {
@@ -178,7 +137,7 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
 		cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
 	}
 
-	return c.executeCommand(ctx, cmd, false)
+	return c.executeCommand(ctx, cmd)
 }
 
 // Staged parses the output of the `git diff` command for the `source` path.
@@ -193,11 +152,11 @@ func (c *Parser) Staged(ctx context.Context, source string) (chan Commit, error)
 		cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
 	}
 
-	return c.executeCommand(ctx, cmd, true)
+	return c.executeCommand(ctx, cmd)
 }
 
 // executeCommand runs an exec.Cmd, reads stdout and stderr, and waits for the Cmd to complete.
-func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged bool) (chan Commit, error) {
+func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit, error) {
 	commitChan := make(chan Commit, 64)
 
 	stdOut, err := cmd.StdoutPipe()
@@ -222,7 +181,7 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged boo
 	}()
 
 	go func() {
-		c.FromReader(ctx, stdOut, commitChan, isStaged)
+		c.FromReader(ctx, stdOut, commitChan)
 		if err := cmd.Wait(); err != nil {
 			ctx.Logger().V(2).Info("Error waiting for git command to complete.", "error", err)
 		}
@@ -231,15 +190,14 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged boo
 	return commitChan, nil
 }
 
-func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit, isStaged bool) {
+func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
 	outReader := bufio.NewReader(stdOut)
 	var (
-		currentCommit *Commit
-		currentDiff   *Diff
-
-		totalLogSize int
+		currentCommit        *Commit
+		currentDiff          *Diff
+		recentlyPassedAuthor bool
+		totalLogSize         int
 	)
-	var latestState = Initial
 
 	defer common.RecoverWithExit(ctx)
 	defer close(commitChan)
@@ -247,16 +205,12 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 		if common.IsDone(ctx) {
 			break
 		}
-
 		line, err := outReader.ReadBytes([]byte("\n")[0])
 		if err != nil && len(line) == 0 {
 			break
 		}
-
 		switch {
-		case isCommitLine(isStaged, latestState, line):
-			latestState = CommitLine
-
+		case isCommitLine(line):
 			// If there is a currentDiff, add it to currentCommit.
 			if currentDiff != nil && currentDiff.Content.Len() > 0 {
 				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
@@ -276,31 +230,16 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			if len(line) >= 47 {
 				currentCommit.Hash = string(line[7:47])
 			}
-		case isAuthorLine(isStaged, latestState, line):
-			latestState = AuthorLine
-
+		case isAuthorLine(line):
 			currentCommit.Author = strings.TrimRight(string(line[8:]), "\n")
-		case isDateLine(isStaged, latestState, line):
-			latestState = DateLine
-
+			recentlyPassedAuthor = true
+		case isDateLine(line):
 			date, err := time.Parse(c.dateFormat, strings.TrimSpace(string(line[6:])))
 			if err != nil {
 				ctx.Logger().V(2).Info("Could not parse date from git stream.", "error", err)
 			}
 			currentCommit.Date = date
-		case isMessageStartLine(isStaged, latestState, line):
-			latestState = MessageStartLine
-			// NoOp
-		case isMessageLine(isStaged, latestState, line):
-			latestState = MessageLine
-
-			currentCommit.Message.Write(line[4:])
-		case isMessageEndLine(isStaged, latestState, line):
-			latestState = MessageEndLine
-			// NoOp
-		case isDiffLine(isStaged, latestState, line):
-			latestState = DiffLine
-
+		case isDiffLine(line):
 			// This should never be nil, but check in case the stdin stream is messed up.
 			if currentCommit == nil {
 				currentCommit = &Commit{}
@@ -328,29 +267,27 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 				}
 			}
 			currentDiff = &Diff{}
-		case isModeLine(isStaged, latestState, line):
-			latestState = ModeLine
+		case isModeLine(line):
 			// NoOp
-		case isIndexLine(isStaged, latestState, line):
-			latestState = IndexLine
+		case isIndexLine(line):
+			recentlyPassedAuthor = false
 			// NoOp
-		case isBinaryLine(isStaged, latestState, line):
-			latestState = BinaryFileLine
-
+		case isPlusFileLine(line):
+			currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
+		case isMinusFileLine(line):
+			// NoOp
+		case isPlusDiffLine(line):
+			currentDiff.Content.Write(line[1:])
+		case isMinusDiffLine(line):
+			// NoOp. We only care about additions.
+		case (isMessageLine(line) && recentlyPassedAuthor):
+			currentCommit.Message.Write(line[4:])
+		case isContextDiffLine(line):
+			currentDiff.Content.Write([]byte("\n"))
+		case isBinaryLine(line):
 			currentDiff.IsBinary = true
 			currentDiff.PathB = pathFromBinaryLine(line)
-		case isFromFileLine(isStaged, latestState, line):
-			latestState = FromFileLine
-			// NoOp
-		case isToFileLine(isStaged, latestState, line):
-			latestState = ToFileLine
-
-			// TODO: Is this fix still required?
-			currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
-		case isHunkLineNumberLine(isStaged, latestState, line):
-			latestState = HunkLineNumberLine
-
-			// TODO: Is it still necessary to check whether the currentDiff is nil?
+		case isLineNumberDiffLine(line):
 			if currentDiff != nil && currentDiff.Content.Len() > 0 {
 				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 			}
@@ -368,54 +305,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 					currentDiff.LineStart = lineStart
 				}
 			}
-		case isHunkContextLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-			// TODO: Why do we care about this? It creates empty lines in the diff. If there are no plusLines, it's just newlines.
-			currentDiff.Content.Write([]byte("\n"))
-		case isHunkPlusLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-
-			currentDiff.Content.Write(line[1:])
-		case isHunkMinusLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-			// NoOp. We only care about additions.
-		case isHunkNewlineWarningLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-			// NoOp
-		case isHunkEmptyLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-			// NoOp
-		case isCommitSeparatorLine(isStaged, latestState, line):
-			// NoOp
-		default:
-			// Skip ahead until we find the next diff or commit.
-			if latestState == ParseFailure {
-				continue
-			}
-
-			// Here be dragons...
-			// Build an informative error message.
-			var err error
-			if currentCommit != nil && currentCommit.Hash != "" {
-				err = fmt.Errorf(`failed to parse line "%s" after state "%s" (commit=%s)`, line, latestState, currentCommit.Hash)
-			} else {
-				err = fmt.Errorf(`failed to parse line "%s" after state "%s"`, line, latestState)
-			}
-			ctx.Logger().V(2).Error(err, "Recovering at the latest commit or diff...\n")
-
-			latestState = ParseFailure
 		}
-
 		if currentDiff.Content.Len() > c.maxDiffSize {
 			ctx.Logger().V(2).Info(fmt.Sprintf(
 				"Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize,
@@ -428,147 +318,130 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 	ctx.Logger().V(2).Info("finished parsing git log.", "total_log_size", totalLogSize)
 }
 
-// commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05
-func isCommitLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || !(latestState == Initial ||
-		latestState == MessageStartLine ||
-		latestState == MessageEndLine ||
-		latestState == ModeLine ||
-		latestState == IndexLine ||
-		latestState == BinaryFileLine ||
-		latestState == ToFileLine ||
-		latestState == HunkContentLine ||
-		latestState == ParseFailure) {
-		return false
+func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
+	if currentDiff != nil && currentDiff.Content.Len() > 0 {
+		currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 	}
-
-	if len(line) > 7 && bytes.Equal(line[:7], []byte("commit ")) {
-		return true
+	if currentCommit != nil {
+		commitChan <- *currentCommit
+		if totalLogSize != nil {
+			*totalLogSize += currentCommit.Size
+		}
 	}
-	return false
-}
-
-// Author: Bill Rich <bill.rich@trufflesec.com>
-func isAuthorLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || !(latestState == CommitLine) {
-		return false
-	}
-	if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
-		return true
-	}
-	return false
 }
 
 // Date:   Tue Aug 10 15:20:40 2021 +0100
-func isDateLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || latestState != AuthorLine {
-		return false
-	}
+func isDateLine(line []byte) bool {
 	if len(line) > 7 && bytes.Equal(line[:5], []byte("Date:")) {
 		return true
 	}
 	return false
 }
 
-// Line directly after Date with only a newline.
-func isMessageStartLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || latestState != DateLine {
-		return false
-	}
-	// TODO: Improve the implementation of this and isMessageEndLine
-	if len(strings.TrimRight(string(line[:]), "\r\n")) == 0 {
+// Author: Bill Rich <bill.rich@trufflesec.com>
+func isAuthorLine(line []byte) bool {
+	if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
 		return true
 	}
 	return false
 }
 
-// Line that starts with 4 spaces
-func isMessageLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || !(latestState == MessageStartLine || latestState == MessageLine) {
-		return false
-	}
-	if len(line) > 4 && bytes.Equal(line[:4], []byte("    ")) {
-		return true
-	}
-	return false
-}
-
-// Line directly after MessageLine with only a newline.
-func isMessageEndLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || latestState != MessageLine {
-		return false
-	}
-	if len(strings.TrimRight(string(line[:]), "\r\n")) == 0 {
+// commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05
+func isCommitLine(line []byte) bool {
+	if len(line) > 7 && bytes.Equal(line[:6], []byte("commit")) {
 		return true
 	}
 	return false
 }
 
 // diff --git a/internal/addrs/move_endpoint_module.go b/internal/addrs/move_endpoint_module.go
-func isDiffLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == MessageStartLine || // Empty commit messages can go from MessageStart->Diff
-		latestState == MessageEndLine ||
-		latestState == BinaryFileLine ||
-		latestState == IndexLine ||
-		latestState == HunkContentLine ||
-		latestState == ParseFailure) {
-		if latestState == Initial && !isStaged {
-			return false
-		}
-	}
-	if len(line) > 11 && bytes.Equal(line[:11], []byte("diff --git ")) {
-		return true
-	}
-	return false
-}
-
-// old mode 100644
-// new mode 100755
-// new file mode 100644
-// similarity index 100%
-// rename from old.txt
-// rename to new.txt
-// deleted file mode 100644
-func isModeLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == DiffLine || latestState == ModeLine) {
-		return false
-	}
-	// This could probably be better written.
-	if (len(line) > 17 && bytes.Equal(line[:17], []byte("deleted file mode"))) ||
-		(len(line) > 16 && bytes.Equal(line[:16], []byte("similarity index"))) ||
-		(len(line) > 13 && bytes.Equal(line[:13], []byte("new file mode"))) ||
-		(len(line) > 11 && bytes.Equal(line[:11], []byte("rename from"))) ||
-		(len(line) > 9 && bytes.Equal(line[:9], []byte("rename to"))) ||
-		(len(line) > 8 && bytes.Equal(line[:8], []byte("old mode"))) ||
-		(len(line) > 8 && bytes.Equal(line[:8], []byte("new mode"))) {
+func isDiffLine(line []byte) bool {
+	if len(line) > 5 && bytes.Equal(line[:4], []byte("diff")) {
 		return true
 	}
 	return false
 }
 
 // index 1ed6fbee1..aea1e643a 100644
-// index 00000000..e69de29b
-func isIndexLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == DiffLine || latestState == ModeLine) {
-		return false
+func isIndexLine(line []byte) bool {
+	if len(line) > 6 && bytes.Equal(line[:5], []byte("index")) {
+		return true
 	}
-	if len(line) > 6 && bytes.Equal(line[:6], []byte("index ")) {
+	return false
+}
+
+// new file mode 100644
+func isModeLine(line []byte) bool {
+	if len(line) > 13 && bytes.Equal(line[:13], []byte("new file mode")) {
+		return true
+	}
+	return false
+}
+
+// --- a/internal/addrs/move_endpoint_module.go
+func isMinusFileLine(line []byte) bool {
+	if len(line) >= 6 && bytes.Equal(line[:3], []byte("---")) {
+		return true
+	}
+	return false
+}
+
+// +++ b/internal/addrs/move_endpoint_module.go
+func isPlusFileLine(line []byte) bool {
+	if len(line) >= 6 && bytes.Equal(line[:4], []byte("+++ ")) {
+		return true
+	}
+	return false
+}
+
+// +fmt.Println("ok")
+func isPlusDiffLine(line []byte) bool {
+	if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) {
+		return true
+	}
+	return false
+}
+
+// -fmt.Println("ok")
+func isMinusDiffLine(line []byte) bool {
+	if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) {
+		return true
+	}
+	return false
+}
+
+// fmt.Println("ok")
+func isContextDiffLine(line []byte) bool {
+	if len(line) >= 1 && bytes.Equal(line[:1], []byte(" ")) {
+		return true
+	}
+	return false
+}
+
+// Line that starts with 4 spaces
+func isMessageLine(line []byte) bool {
+	if len(line) > 4 && bytes.Equal(line[:4], []byte("    ")) {
 		return true
 	}
 	return false
 }
 
 // Binary files /dev/null and b/plugin.sig differ
-func isBinaryLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if latestState != IndexLine {
-		return false
-	}
+func isBinaryLine(line []byte) bool {
 	if len(line) > 7 && bytes.Equal(line[:6], []byte("Binary")) {
 		return true
 	}
 	return false
 }
 
+// @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)
+func isLineNumberDiffLine(line []byte) bool {
+	if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) {
+		return true
+	}
+	return false
+}
+
 // Get the b/ file path. Ignoring the edge case of files having `and /b` in the name for simplicity.
 func pathFromBinaryLine(line []byte) string {
 	logger := context.Background().Logger()
@@ -580,117 +453,3 @@ func pathFromBinaryLine(line []byte) string {
 	bRaw := sbytes[1]
 	return string(bRaw[:len(bRaw)-7]) // drop the "b/" and " differ"
 }
-
-// --- a/internal/addrs/move_endpoint_module.go
-func isFromFileLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if latestState != IndexLine {
-		return false
-	}
-	if len(line) >= 6 && bytes.Equal(line[:4], []byte("--- ")) {
-		return true
-	}
-	return false
-}
-
-// +++ b/internal/addrs/move_endpoint_module.go
-func isToFileLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if latestState != FromFileLine {
-		return false
-	}
-	if len(line) >= 6 && bytes.Equal(line[:4], []byte("+++ ")) {
-		return true
-	}
-	return false
-}
-
-// @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)
-func isHunkLineNumberLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == ToFileLine || latestState == HunkContentLine) {
-		return false
-	}
-	if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) {
-		return true
-	}
-	return false
-}
-
-// fmt.Println("ok")
-// (There's a space before `fmt` that gets removed by the formatter.)
-func isHunkContextLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
-		return false
-	}
-	if len(line) >= 1 && bytes.Equal(line[:1], []byte(" ")) {
-		return true
-	}
-	return false
-}
-
-// +fmt.Println("ok")
-func isHunkPlusLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
-		return false
-	}
-	if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) {
-		return true
-	}
-	return false
-}
-
-// -fmt.Println("ok")
-func isHunkMinusLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
-		return false
-	}
-	if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) {
-		return true
-	}
-	return false
-}
-
-// \ No newline at end of file
-func isHunkNewlineWarningLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if latestState != HunkContentLine {
-		return false
-	}
-	if len(line) >= 27 && bytes.Equal(line[:27], []byte("\\ No newline at end of file")) {
-		return true
-	}
-	return false
-}
-
-// Newline after hunk, or an empty line, e.g.
-// +}
-//
-// commit 00920984e3435057f09cee5468850f7546dfa637 (tag: v3.42.0)
-func isHunkEmptyLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
-		return false
-	}
-	// TODO: Can this also be `\n\r`?
-	if len(line) == 1 && bytes.Equal(line[:1], []byte("\n")) {
-		return true
-	}
-	return false
-}
-
-func isCommitSeparatorLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if (latestState == ModeLine || latestState == IndexLine || latestState == BinaryFileLine || latestState == ToFileLine) &&
-		len(line) == 1 && bytes.Equal(line[:1], []byte("\n")) {
-		return true
-	}
-	return false
-}
-
-func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
-	// Ignore empty or binary diffs (this condition may be redundant).
-	if currentDiff != nil && currentDiff.Content.Len() > 0 {
-		currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
-	}
-	if currentCommit != nil {
-		commitChan <- *currentCommit
-		if totalLogSize != nil {
-			*totalLogSize += currentCommit.Size
-		}
-	}
-}

pkg/handlers/archive.go

@@ -6,14 +6,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"strings"
 	"time"
 
 	"github.com/mholt/archiver/v4"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
 type ctxKey int
@@ -54,14 +52,14 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
 }
 
 // FromFile extracts the files from an archive.
-func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan ChunkOpt {
-	archiveChan := make(chan ChunkOpt, 512)
+func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan ([]byte) {
+	archiveChan := make(chan ([]byte), 512)
 	go func() {
 		ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
 		logger := logContext.AddLogger(ctx).Logger()
 		defer cancel()
 		defer close(archiveChan)
-		err := d.openArchive(ctx, nil, data, archiveChan)
+		err := d.openArchive(ctx, 0, data, archiveChan)
 		if err != nil {
 			if errors.Is(err, archiver.ErrNoMatch) {
 				return
@@ -73,21 +71,18 @@ func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan Chu
 }
 
 // openArchive takes a reader and extracts the contents up to the maximum depth.
-func (d *Archive) openArchive(ctx context.Context, depth []string, reader io.Reader, archiveChan chan ChunkOpt) error {
-	if len(depth) >= maxDepth {
+func (d *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan ([]byte)) error {
+	if depth >= maxDepth {
 		return fmt.Errorf("max archive depth reached")
 	}
 	format, reader, err := archiver.Identify("", reader)
 	if err != nil {
-		if errors.Is(err, archiver.ErrNoMatch) && len(depth) > 0 {
+		if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
 			chunkSize := 10 * 1024
 			for {
-				data := make([]byte, chunkSize)
-				n, _ := reader.Read(data)
-				archiveChan <- func(chunk *sources.Chunk) {
-					chunk.Data = data
-					chunk.HandleMetadata = map[string]string{"archive path": strings.Join(depth, " > ")}
-				}
+				chunk := make([]byte, chunkSize)
+				n, _ := reader.Read(chunk)
+				archiveChan <- chunk
 				if n < chunkSize {
 					break
 				}
@@ -107,9 +102,9 @@ func (d *Archive) openArchive(ctx context.Context, depth []string, reader io.Rea
 			return err
 		}
 		newReader := bytes.NewReader(fileBytes)
-		return d.openArchive(ctx, append(depth, "decompressed"), newReader, archiveChan)
+		return d.openArchive(ctx, depth+1, newReader, archiveChan)
 	case archiver.Extractor:
-		err := archive.Extract(context.WithValue(ctx, depthKey, depth), reader, nil, d.extractorHandler(archiveChan))
+		err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, d.extractorHandler(archiveChan))
 		if err != nil {
 			return err
 		}
@@ -134,12 +129,12 @@ func (d *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader,
 }
 
 // extractorHandler is applied to each file in an archiver.Extractor file.
-func (d *Archive) extractorHandler(archiveChan chan ChunkOpt) func(context.Context, archiver.File) error {
+func (d *Archive) extractorHandler(archiveChan chan ([]byte)) func(context.Context, archiver.File) error {
 	return func(ctx context.Context, f archiver.File) error {
 		logger := logContext.AddLogger(ctx).Logger()
 		logger.V(5).Info("Handling extracted file.", "filename", f.Name())
-		var depth []string
-		if ctxDepth, ok := ctx.Value(depthKey).([]string); ok {
+		depth := 0
+		if ctxDepth, ok := ctx.Value(depthKey).(int); ok {
 			depth = ctxDepth
 		}
 
@@ -153,7 +148,7 @@ func (d *Archive) extractorHandler(archiveChan chan ChunkOpt) func(context.Conte
 		}
 		fileContent := bytes.NewReader(fileBytes)
 
-		err = d.openArchive(ctx, append(depth, f.Name()), fileContent, archiveChan)
+		err = d.openArchive(ctx, depth, fileContent, archiveChan)
 		if err != nil {
 			return err
 		}

pkg/handlers/archive_test.go

@@ -85,11 +85,9 @@ func TestArchiveHandler(t *testing.T) {
 		count := 0
 		re := regexp.MustCompile(testCase.matchString)
 		matched := false
-		var chunk sources.Chunk
-		for opt := range archiveChan {
+		for chunk := range archiveChan {
 			count++
-			opt(&chunk)
-			if re.Match(chunk.Data) {
+			if re.Match(chunk) {
 				matched = true
 			}
 		}

pkg/handlers/handlers.go

@@ -13,10 +13,8 @@ func DefaultHandlers() []Handler {
 	}
 }
 
-type ChunkOpt func(*sources.Chunk)
-
 type Handler interface {
-	FromFile(context.Context, io.Reader) chan ChunkOpt
+	FromFile(context.Context, io.Reader) chan ([]byte)
 	IsFiletype(context.Context, io.Reader) (io.Reader, bool)
 	New()
 }
@@ -40,13 +38,13 @@ func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, c
 	handlerChan := handler.FromFile(ctx, file)
 	for {
 		select {
-		case opt, open := <-handlerChan:
+		case data, open := <-handlerChan:
 			if !open {
 				// We finished reading everything from handlerChan.
 				return true
 			}
 			chunk := *chunkSkel
-			opt(&chunk)
+			chunk.Data = data
 			// Send data on chunksChan.
 			select {
 			case chunksChan <- &chunk:

pkg/output/plain.go

@@ -96,9 +96,9 @@ func structToMap(obj interface{}) (m map[string]map[string]interface{}, err erro
 }
 
 type outputFormat struct {
-	DetectorType string
-	DecoderType  string
-	Verified     bool
-	Raw          string
+	DetectorType,
+	DecoderType string
+	Verified bool
+	Raw      string
 	*source_metadatapb.MetaData
 }

pkg/sources/chunker.go

@@ -12,8 +12,6 @@ const (
 	ChunkSize = 10 * 1024
 	// PeekSize is the size of the peek into the previous chunk.
 	PeekSize = 3 * 1024
-	// TotalChunkSize is the total size of a chunk with peek data.
-	TotalChunkSize = ChunkSize + PeekSize
 )
 
 // Chunker takes a chunk and splits it into chunks of ChunkSize.
@@ -21,31 +19,27 @@ func Chunker(originalChunk *Chunk) chan *Chunk {
 	chunkChan := make(chan *Chunk)
 	go func() {
 		defer close(chunkChan)
-		if len(originalChunk.Data) <= TotalChunkSize {
+		if len(originalChunk.Data) <= ChunkSize+PeekSize {
 			chunkChan <- originalChunk
 			return
 		}
 		r := bytes.NewReader(originalChunk.Data)
 		reader := bufio.NewReaderSize(bufio.NewReader(r), ChunkSize)
 		for {
-			chunkBytes := make([]byte, TotalChunkSize)
+			chunkBytes := make([]byte, ChunkSize)
 			chunk := *originalChunk
-			chunkBytes = chunkBytes[:ChunkSize]
 			n, err := reader.Read(chunkBytes)
 			if err != nil && !errors.Is(err, io.EOF) {
 				break
 			}
-			if n == 0 {
-				if errors.Is(err, io.EOF) {
-					break
-				}
-				continue
-			}
 			peekData, _ := reader.Peek(PeekSize)
-			copy(chunkBytes[n:], peekData)
-			chunk.Data = chunkBytes[:n+len(peekData)]
-
-			chunkChan <- &chunk
+			chunk.Data = append(chunkBytes[:n], peekData...)
+			if n > 0 {
+				chunkChan <- &chunk
+			}
+			if errors.Is(err, io.EOF) {
+				break
+			}
 		}
 	}()
 	return chunkChan

pkg/sources/chunker_test.go

@@ -72,15 +72,3 @@ func TestChunker(t *testing.T) {
 	}
 
 }
-
-func BenchmarkChunker(b *testing.B) {
-	data := bytes.Repeat([]byte("a"), ChunkSize*100)
-	chunk := &Chunk{
-		Data: data,
-	}
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		for range Chunker(chunk) {
-		}
-	}
-}

pkg/sources/git/git.go

@@ -51,14 +51,9 @@ type Git struct {
 	jobID              int64
 	sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData
 	verify             bool
-	metrics            metrics
 	concurrency        *semaphore.Weighted
 }
 
-type metrics struct {
-	commitsScanned int
-}
-
 func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
 	sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData,
 ) *Git {
@@ -344,10 +339,6 @@ func CloneRepoUsingSSH(ctx context.Context, gitUrl string, args ...string) (stri
 	return CloneRepo(ctx, userInfo, gitUrl, args...)
 }
 
-func (s *Git) CommitsScanned() int {
-	return s.metrics.commitsScanned
-}
-
 func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
 	if err := GitCmdCheck(); err != nil {
 		return err
@@ -380,7 +371,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 			break
 		}
 		depth++
-		s.metrics.commitsScanned++
 		logger.V(5).Info("scanning commit", "commit", commit.Hash)
 		for _, diff := range commit.Diffs {
 			if !scanOptions.Filter.Pass(diff.PathB) {

pkg/sources/sources.go

@@ -21,8 +21,7 @@ type Chunk struct {
 	SourceType sourcespb.SourceType
 	// SourceMetadata holds the context of where the Chunk was found.
 	SourceMetadata *source_metadatapb.MetaData
-	// HandleMetadata holds the metadata from a handler if one was used.
-	HandleMetadata map[string]string
+
 	// Data is the data to decode and scan.
 	Data []byte
 	// Verify specifies whether any secrets in the Chunk should be verified.

proto/detectors.proto

@@ -932,6 +932,7 @@ enum DetectorType {
   Dockerhub = 921;
   TrufflehogEnterprise = 922;
   EnvoyApiKey = 923;
+  Postgres = 925;
 }
 
 message Result {