Update test

Currently it's a unstructured map[string]string that gets copied into
a Result.ExtraData after detection, but we might want to revisit this
decision when we get more handlers.

go.mod

@@ -8,6 +8,7 @@ require (
 	cloud.google.com/go/secretmanager v1.11.1
 	cloud.google.com/go/storage v1.31.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
+	github.com/BobuSumisu/aho-corasick v1.0.3
 	github.com/TheZeroSlave/zapsentry v1.17.0
 	github.com/aws/aws-sdk-go v1.44.83
 	github.com/bill-rich/disk-buffer-reader v0.1.7
@@ -44,7 +45,6 @@ require (
 	github.com/mholt/archiver/v4 v4.0.0-alpha.8
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/paulbellamy/ratecounter v0.2.0
-	github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.16.0
 	github.com/rabbitmq/amqp091-go v1.8.1
@@ -94,7 +94,7 @@ require (
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/connesc/cipherio v0.2.1 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
-	github.com/couchbase/gocbcore/v10 v10.2.3 // indirect
+	github.com/couchbase/gocbcore/v10 v10.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/docker/cli v23.0.5+incompatible // indirect

go.sum

@@ -51,6 +51,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
+github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g=
+github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -119,8 +121,9 @@ github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSk
 github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
 github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
 github.com/couchbase/gocb/v2 v2.6.3/go.mod h1:yF5F6BHTZ/ZowhEuZbySbXrlI4rHd1TIhm5azOaMbJU=
-github.com/couchbase/gocbcore/v10 v10.2.3 h1:PEkRSNSkKjUBXx82Ucr094+anoiCG5GleOOQZOHo6D4=
 github.com/couchbase/gocbcore/v10 v10.2.3/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
+github.com/couchbase/gocbcore/v10 v10.2.4 h1:TDTQ1mSBUw9eajuV71ZDJcLRbmfGcsvZV9SMeUDAVGc=
+github.com/couchbase/gocbcore/v10 v10.2.4/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
 github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
 github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4=
 github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
@@ -382,8 +385,6 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
 github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
-github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
-github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
 github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=

pkg/context/context.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"runtime/debug"
+	"sync"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -28,8 +29,9 @@ type CancelFunc = context.CancelFunc
 type logCtx struct {
 	// Embed context.Context to get all methods for free.
 	context.Context
-	log logr.Logger
-	err *error
+	log     logr.Logger
+	err     *error
+	errLock *sync.Mutex
 }
 
 // Logger returns a structured logger.
@@ -145,8 +147,11 @@ func captureCancelCallstack(ctx logCtx, f context.CancelFunc) (Context, context.
 	if ctx.err == nil {
 		var err error
 		ctx.err = &err
+		ctx.errLock = &sync.Mutex{}
 	}
 	return ctx, func() {
+		ctx.errLock.Lock()
+		defer ctx.errLock.Unlock()
 		// We must check Err() before calling f() since f() sets the error.
 		// If there's already an error, do nothing special.
 		if ctx.Err() != nil {

pkg/context/context_test.go

@@ -212,3 +212,9 @@ func TestErrCallstackTimeoutCancel(t *testing.T) {
 	cancel()
 	assert.Equal(t, err, ctx.Err())
 }
+
+func TestRace(t *testing.T) {
+	_, cancel := WithCancel(Background())
+	go cancel()
+	cancel()
+}

pkg/detectors/npmtoken/npmtoken.go

@@ -36,7 +36,6 @@ func (s Scanner) Keywords() []string {
 // FromData will find and optionally verify NpmToken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
-
 	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 	for _, match := range matches {
 		if len(match) != 2 {

pkg/detectors/okta/okta.go

@@ -19,8 +19,8 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	domainPat = regexp.MustCompile(`[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com`)
-	tokenPat  = regexp.MustCompile(`00[a-zA-Z0-9_-]{40}`)
+	domainPat = regexp.MustCompile(`\b[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com\b`)
+	tokenPat  = regexp.MustCompile(`\b00[a-zA-Z0-9_-]{40}\b`)
 	// TODO: Oauth client secrets
 )
 

pkg/engine/circleci.go

@@ -1,14 +1,16 @@
 package engine
 
 import (
+	"fmt"
 	"runtime"
 
+	"github.com/go-errors/errors"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
 )
 
@@ -27,16 +29,23 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
 		return err
 	}
 
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Circle CI", (&circleci.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			circleSource := &circleci.Source{}
-			if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			return circleSource, nil
-		})
+	circleSource := circleci.Source{}
+	ctx = context.WithValues(ctx,
+		"source_type", circleSource.Type().String(),
+		"source_name", "Circle CI",
+	)
+	err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU())
 	if err != nil {
-		return err
+		return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
 	}
-	return e.sourceManager.ScheduleRun(ctx, handle)
+
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := circleSource.Chunks(ctx, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("error scanning CircleCI: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/engine/docker.go

@@ -1,27 +1,38 @@
 package engine
 
 import (
+	"fmt"
 	"runtime"
 
+	"github.com/go-errors/errors"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
 )
 
 // ScanDocker scans a given docker connection.
 func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error {
-	handler, err := e.sourceManager.Enroll(ctx, "trufflehog - docker", (&docker.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			dockerSource := &docker.Source{}
-			if err := dockerSource.Init(ctx, "trufflehog - docker", jobID, sourceID, true, conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			return dockerSource, nil
-		})
+	dockerSource := docker.Source{}
+
+	ctx = context.WithValues(ctx,
+		"source_type", dockerSource.Type().String(),
+		"source_name", "docker",
+	)
+	err := dockerSource.Init(ctx, "trufflehog - docker", 0, int64(sourcespb.SourceType_SOURCE_TYPE_DOCKER), true, conn, runtime.NumCPU())
 	if err != nil {
-		return err
+		return errors.WrapPrefix(err, "could not init docker source", 0)
 	}
-	return e.sourceManager.ScheduleRun(ctx, handler)
+
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := dockerSource.Chunks(ctx, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("error scanning docker image: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/engine/engine.go

@@ -10,7 +10,8 @@ import (
 	"sync/atomic"
 	"time"
 
-	ahocorasick "github.com/petar-dambovaliev/aho-corasick"
+	ahocorasick "github.com/BobuSumisu/aho-corasick"
+	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -26,13 +27,14 @@ import (
 
 type Engine struct {
 	concurrency     int
-	sourceManager   *sources.SourceManager
+	chunks          chan *sources.Chunk
 	results         chan detectors.ResultWithMetadata
 	decoders        []decoders.Decoder
 	detectors       map[bool][]detectors.Detector
 	chunksScanned   uint64
 	bytesScanned    uint64
 	detectorAvgTime sync.Map
+	sourcesWg       *errgroup.Group
 	workersWg       sync.WaitGroup
 	// filterUnverified is used to reduce the number of unverified results.
 	// If there are multiple unverified results for the same chunk for the same detector,
@@ -41,7 +43,7 @@ type Engine struct {
 
 	// prefilter is a ahocorasick struct used for doing efficient string
 	// matching given a set of words (keywords from the rules in the config)
-	prefilter ahocorasick.AhoCorasick
+	prefilter ahocorasick.Trie
 }
 
 type EngineOption func(*Engine)
@@ -108,8 +110,10 @@ func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors
 
 func Start(ctx context.Context, options ...EngineOption) *Engine {
 	e := &Engine{
+		chunks:          make(chan *sources.Chunk),
 		results:         make(chan detectors.ResultWithMetadata),
 		detectorAvgTime: sync.Map{},
+		sourcesWg:       &errgroup.Group{},
 	}
 
 	for _, option := range options {
@@ -124,6 +128,9 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	}
 	ctx.Logger().V(2).Info("engine started", "workers", e.concurrency)
 
+	// Limit number of concurrent goroutines dedicated to chunking a source.
+	e.sourcesWg.SetLimit(e.concurrency)
+
 	if len(e.decoders) == 0 {
 		e.decoders = decoders.DefaultDecoders()
 	}
@@ -134,25 +141,20 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 		e.detectors[false] = []detectors.Detector{}
 	}
 
-	// Setup source manager now that we know the concurrency.
-	e.sourceManager = sources.NewManager(sources.WithConcurrency(e.concurrency))
-
 	// build ahocorasick prefilter for efficient string matching
 	// on keywords
 	keywords := []string{}
 	for _, d := range e.detectors[false] {
-		keywords = append(keywords, d.Keywords()...)
+		for _, kw := range d.Keywords() {
+			keywords = append(keywords, strings.ToLower(kw))
+		}
 	}
 	for _, d := range e.detectors[true] {
-		keywords = append(keywords, d.Keywords()...)
+		for _, kw := range d.Keywords() {
+			keywords = append(keywords, strings.ToLower(kw))
+		}
 	}
-	builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{
-		AsciiCaseInsensitive: true,
-		MatchOnlyWholeWords:  false,
-		MatchKind:            ahocorasick.LeftMostLongestMatch,
-		DFA:                  true,
-	})
-	e.prefilter = builder.Build(keywords)
+	e.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
 
 	ctx.Logger().Info("loaded decoders", "count", len(e.decoders))
 	ctx.Logger().Info("loaded detectors",
@@ -180,7 +182,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	for i := 0; i < e.concurrency; i++ {
 		e.workersWg.Add(1)
 		go func() {
-			defer common.RecoverWithExit(ctx)
+			defer common.Recover(ctx)
 			defer e.workersWg.Done()
 			e.detectorWorker(ctx)
 		}()
@@ -194,12 +196,13 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 // more sources may be scanned by the engine.
 func (e *Engine) Finish(ctx context.Context, logFunc func(error, string, ...any)) {
 	defer common.RecoverWithExit(ctx)
-	// Wait for the sources to finish putting chunks onto the chunks channel.
-	sourceErr := e.sourceManager.Wait()
+	// wait for the sources to finish putting chunks onto the chunks channel
+	sourceErr := e.sourcesWg.Wait()
 	if sourceErr != nil {
 		logFunc(sourceErr, "error occurred while collecting chunks")
 	}
 
+	close(e.chunks)
 	// wait for the workers to finish processing all of the chunks and putting
 	// results onto the results channel
 	e.workersWg.Wait()
@@ -212,8 +215,8 @@ func (e *Engine) Finish(ctx context.Context, logFunc func(error, string, ...any)
 	close(e.results)
 }
 
-func (e *Engine) ChunksChan() <-chan *sources.Chunk {
-	return e.sourceManager.Chunks()
+func (e *Engine) ChunksChan() chan *sources.Chunk {
+	return e.chunks
 }
 
 func (e *Engine) ResultsChan() chan detectors.ResultWithMetadata {
@@ -266,7 +269,7 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
 }
 
 func (e *Engine) detectorWorker(ctx context.Context) {
-	for originalChunk := range e.ChunksChan() {
+	for originalChunk := range e.chunks {
 		for chunk := range sources.Chunker(originalChunk) {
 			var chunkResults []detectors.ResultWithMetadata
 			matchedKeywords := make(map[string]struct{})
@@ -292,8 +295,8 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 				}
 
 				// build a map of all keywords that were matched in the chunk
-				for _, m := range e.prefilter.FindAll(string(decoded.Data)) {
-					matchedKeywords[strings.ToLower(string(decoded.Data[m.Start():m.End()]))] = struct{}{}
+				for _, m := range e.prefilter.MatchString(strings.ToLower(string(decoded.Data))) {
+					matchedKeywords[strings.ToLower(m.MatchString())] = struct{}{}
 				}
 
 				for verify, detectorsSet := range e.detectors {
@@ -346,8 +349,15 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 								continue
 							}
 							result.DecoderType = decoderType
+							if result.ExtraData == nil && chunk.HandleMetadata != nil {
+								result.ExtraData = chunk.HandleMetadata
+							} else {
+								for k, v := range chunk.HandleMetadata {
+									// TODO: Check key collisions.
+									result.ExtraData[k] = v
+								}
+							}
 							chunkResults = append(chunkResults, detectors.CopyMetadata(resultChunk, result))
-
 						}
 						if len(results) > 0 {
 							elapsed := time.Since(start)

pkg/engine/filesystem.go

@@ -1,11 +1,14 @@
 package engine
 
 import (
+	"fmt"
 	"runtime"
 
+	"github.com/go-errors/errors"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -24,17 +27,39 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
 		return err
 	}
 
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", (&filesystem.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			fileSystemSource := &filesystem.Source{}
-			fileSystemSource.WithFilter(c.Filter)
-			if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			return fileSystemSource, nil
-		})
+	fileSystemSource := filesystem.Source{}
+	fileSystemSource.WithFilter(c.Filter)
+
+	// TODO: This is how it would be used; everything after these comments would be removed.
+	// handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", fileSystemSource.Type(),
+	// 	func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
+	// 		fileSystemSource := filesystem.Source{}
+	// 		fileSystemSource.WithFilter(c.Filter)
+	// 		if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+	// 			return nil, err
+	// 		}
+	// 		return &fileSystemSource, nil
+	// 	})
+	// if err != nil {
+	// 	return err
+	// }
+	// return e.sourceManager.ScheduleRun(ctx, handle)
+
+	ctx = context.WithValues(ctx,
+		"source_type", fileSystemSource.Type().String(),
+		"source_name", "filesystem",
+	)
+	err = fileSystemSource.Init(ctx, "trufflehog - filesystem", 0, int64(sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM), true, &conn, runtime.NumCPU())
 	if err != nil {
-		return err
+		return errors.WrapPrefix(err, "could not init filesystem source", 0)
 	}
-	return e.sourceManager.ScheduleRun(ctx, handle)
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := fileSystemSource.Chunks(ctx, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("error scanning filesystem: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/engine/gcs.go

@@ -6,6 +6,7 @@ import (
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -44,18 +45,23 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
 		return fmt.Errorf("failed to marshal GCS connection: %w", err)
 	}
 
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - GCS", (&gcs.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			source := &gcs.Source{}
-			if err := source.Init(ctx, "trufflehog - GCS", jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
-				return nil, err
-			}
-			return source, nil
-		})
-	if err != nil {
-		return err
+	source := gcs.Source{}
+	ctx = context.WithValues(ctx,
+		"source_type", source.Type().String(),
+		"source_name", "gcs",
+	)
+	if err = source.Init(ctx, "trufflehog - GCS", 0, 0, true, &conn, int(c.Concurrency)); err != nil {
+		return fmt.Errorf("failed to initialize GCS source: %w", err)
 	}
-	return e.sourceManager.ScheduleRun(ctx, handle)
+
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		if err := source.Chunks(ctx, e.ChunksChan()); err != nil {
+			return fmt.Errorf("could not scan GCS: %w", err)
+		}
+		return nil
+	})
+	return nil
 }
 
 func isAuthValid(ctx context.Context, c sources.GCSConfig, connection *sourcespb.GCS) bool {

pkg/engine/git.go

@@ -1,13 +1,14 @@
 package engine
 
 import (
+	"fmt"
 	"runtime"
 
 	gogit "github.com/go-git/go-git/v5"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
@@ -21,6 +22,16 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 		git.ScanOptionLogOptions(logOptions),
 	}
 
+	options := &gogit.PlainOpenOptions{
+		DetectDotGit:          true,
+		EnableDotGitCommonDir: true,
+	}
+
+	repo, err := gogit.PlainOpenWithOptions(c.RepoPath, options)
+	if err != nil {
+		return fmt.Errorf("could not open repo: %s: %w", c.RepoPath, err)
+	}
+
 	if c.MaxDepth != 0 {
 		opts = append(opts, git.ScanOptionMaxDepth(int64(c.MaxDepth)))
 	}
@@ -35,34 +46,33 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 	}
 	scanOptions := git.NewScanOptions(opts...)
 
-	connection := &sourcespb.Git{
-		// Using Directories here allows us to not pass any
-		// authentication. Also by this point, the c.RepoPath should
-		// still have been prepared and downloaded to a temporary
-		// directory if it was a URL.
-		Directories: []string{c.RepoPath},
-	}
-	var conn anypb.Any
-	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
-	if err != nil {
-		ctx.Logger().Error(err, "failed to marshal git connection")
-		return err
-	}
-
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - git", (&git.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			gitSource := &git.Source{}
-			if err := gitSource.Init(ctx, "trufflehog - git", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
+	gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "trufflehog - git", true, runtime.NumCPU(),
+		func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
+			return &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_Git{
+					Git: &source_metadatapb.Git{
+						Commit:     commit,
+						File:       file,
+						Email:      email,
+						Repository: repository,
+						Timestamp:  timestamp,
+						Line:       line,
+					},
+				},
 			}
-			gitSource.WithScanOptions(scanOptions)
-			// Don't try to clean up the provided directory. That's
-			// handled by the caller of ScanGit.
-			gitSource.WithPreserveTempDirs(true)
-			return gitSource, nil
 		})
-	if err != nil {
-		return err
-	}
-	return e.sourceManager.ScheduleRun(ctx, handle)
+
+	ctx = context.WithValues(ctx,
+		"source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),
+		"source_name", "git",
+	)
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("could not scan repo: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/engine/github.go

@@ -1,10 +1,13 @@
 package engine
 
 import (
+	"fmt"
+
 	gogit "github.com/go-git/go-git/v5"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -14,6 +17,8 @@ import (
 
 // ScanGitHub scans Github with the provided options.
 func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
+	source := github.Source{}
+
 	connection := sourcespb.GitHub{
 		Endpoint:      c.Endpoint,
 		Organizations: c.Orgs,
@@ -37,24 +42,31 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
 		return err
 	}
 
+	ctx = context.WithValues(ctx,
+		"source_type", source.Type().String(),
+		"source_name", "github",
+	)
+	err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency)
+	if err != nil {
+		ctx.Logger().Error(err, "failed to initialize github source")
+		return err
+	}
+
 	logOptions := &gogit.LogOptions{}
 	opts := []git.ScanOption{
 		git.ScanOptionFilter(c.Filter),
 		git.ScanOptionLogOptions(logOptions),
 	}
 	scanOptions := git.NewScanOptions(opts...)
+	source.WithScanOptions(scanOptions)
 
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - GitHub", (&github.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			source := &github.Source{}
-			source.WithScanOptions(scanOptions)
-			if err := source.Init(ctx, "trufflehog - GitHub", jobID, sourceID, false, &conn, c.Concurrency); err != nil {
-				return nil, err
-			}
-			return source, nil
-		})
-	if err != nil {
-		return err
-	}
-	return e.sourceManager.Run(ctx, handle)
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := source.Chunks(ctx, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("could not scan github: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/engine/gitlab.go

@@ -4,10 +4,12 @@ import (
 	"fmt"
 	"runtime"
 
+	"github.com/go-errors/errors"
 	gogit "github.com/go-git/go-git/v5"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -50,17 +52,24 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
 		return err
 	}
 
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - GitLab", (&gitlab.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			gitlabSource := &gitlab.Source{}
-			gitlabSource.WithScanOptions(scanOptions)
-			if err := gitlabSource.Init(ctx, "trufflehog - GitLab", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			return gitlabSource, nil
-		})
+	gitlabSource := gitlab.Source{}
+	ctx = context.WithValues(ctx,
+		"source_type", gitlabSource.Type().String(),
+		"source_name", "gitlab",
+	)
+	err = gitlabSource.Init(ctx, "trufflehog - gitlab", 0, int64(sourcespb.SourceType_SOURCE_TYPE_GITLAB), true, &conn, runtime.NumCPU())
 	if err != nil {
-		return err
+		return errors.WrapPrefix(err, "could not init GitLab source", 0)
 	}
-	return e.sourceManager.ScheduleRun(ctx, handle)
+	gitlabSource.WithScanOptions(scanOptions)
+
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := gitlabSource.Chunks(ctx, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("error scanning GitLab: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/engine/s3.go

@@ -4,9 +4,11 @@ import (
 	"fmt"
 	"runtime"
 
+	"github.com/go-errors/errors"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -53,16 +55,23 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
 		return err
 	}
 
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - S3", (&s3.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			s3Source := &s3.Source{}
-			if err := s3Source.Init(ctx, "trufflehog - S3", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			return s3Source, nil
-		})
+	s3Source := s3.Source{}
+	ctx = context.WithValues(ctx,
+		"source_type", s3Source.Type().String(),
+		"source_name", "s3",
+	)
+	err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU())
 	if err != nil {
-		return err
+		return errors.WrapPrefix(err, "failed to init S3 source", 0)
 	}
-	return e.sourceManager.ScheduleRun(ctx, handle)
+
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := s3Source.Chunks(ctx, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("error scanning S3: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/engine/syslog.go

@@ -1,12 +1,14 @@
 package engine
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/go-errors/errors"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -40,17 +42,25 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
 	if err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
-
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - syslog", (&syslog.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			source := &syslog.Source{}
-			if err := source.Init(ctx, "trufflehog - syslog", jobID, sourceID, false, &conn, c.Concurrency); err != nil {
-				return nil, err
-			}
-			return source, nil
-		})
+	source := syslog.Source{}
+	ctx = context.WithValues(ctx,
+		"source_type", source.Type().String(),
+		"source_name", "syslog",
+	)
+	err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, c.Concurrency)
+	source.InjectConnection(connection)
 	if err != nil {
+		ctx.Logger().Error(err, "failed to initialize syslog source")
 		return err
 	}
-	return e.sourceManager.ScheduleRun(ctx, handle)
+
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := source.Chunks(ctx, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("could not scan syslog: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/gitparse/gitparse.go

@@ -51,6 +51,47 @@ type Parser struct {
 	dateFormat    string
 }
 
+type ParseState int
+
+const (
+	Initial ParseState = iota
+	CommitLine
+	AuthorLine
+	DateLine
+	MessageStartLine
+	MessageLine
+	MessageEndLine
+	DiffLine
+	ModeLine
+	IndexLine
+	FromFileLine
+	ToFileLine
+	BinaryFileLine
+	HunkLineNumberLine
+	HunkContentLine
+	ParseFailure
+)
+
+func (state ParseState) String() string {
+	return [...]string{
+		"Initial",
+		"CommitLine",
+		"AuthorLine",
+		"DateLine",
+		"MessageStartLine",
+		"MessageLine",
+		"MessageEndLine",
+		"DiffLine",
+		"ModeLine",
+		"IndexLine",
+		"FromFileLine",
+		"ToFileLine",
+		"BinaryFileLine",
+		"HunkLineNumberLine",
+		"HunkContentLine",
+	}[state]
+}
+
 // WithMaxDiffSize sets maxDiffSize option. Diffs larger than maxDiffSize will
 // be truncated.
 func WithMaxDiffSize(maxDiffSize int) Option {
@@ -137,7 +178,7 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
 		cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
 	}
 
-	return c.executeCommand(ctx, cmd)
+	return c.executeCommand(ctx, cmd, false)
 }
 
 // Staged parses the output of the `git diff` command for the `source` path.
@@ -152,11 +193,11 @@ func (c *Parser) Staged(ctx context.Context, source string) (chan Commit, error)
 		cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
 	}
 
-	return c.executeCommand(ctx, cmd)
+	return c.executeCommand(ctx, cmd, true)
 }
 
 // executeCommand runs an exec.Cmd, reads stdout and stderr, and waits for the Cmd to complete.
-func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit, error) {
+func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged bool) (chan Commit, error) {
 	commitChan := make(chan Commit, 64)
 
 	stdOut, err := cmd.StdoutPipe()
@@ -181,7 +222,7 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit
 	}()
 
 	go func() {
-		c.FromReader(ctx, stdOut, commitChan)
+		c.FromReader(ctx, stdOut, commitChan, isStaged)
 		if err := cmd.Wait(); err != nil {
 			ctx.Logger().V(2).Info("Error waiting for git command to complete.", "error", err)
 		}
@@ -190,14 +231,15 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit
 	return commitChan, nil
 }
 
-func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
+func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit, isStaged bool) {
 	outReader := bufio.NewReader(stdOut)
 	var (
-		currentCommit        *Commit
-		currentDiff          *Diff
-		recentlyPassedAuthor bool
-		totalLogSize         int
+		currentCommit *Commit
+		currentDiff   *Diff
+
+		totalLogSize int
 	)
+	var latestState = Initial
 
 	defer common.RecoverWithExit(ctx)
 	defer close(commitChan)
@@ -205,12 +247,16 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 		if common.IsDone(ctx) {
 			break
 		}
+
 		line, err := outReader.ReadBytes([]byte("\n")[0])
 		if err != nil && len(line) == 0 {
 			break
 		}
+
 		switch {
-		case isCommitLine(line):
+		case isCommitLine(isStaged, latestState, line):
+			latestState = CommitLine
+
 			// If there is a currentDiff, add it to currentCommit.
 			if currentDiff != nil && currentDiff.Content.Len() > 0 {
 				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
@@ -230,16 +276,31 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			if len(line) >= 47 {
 				currentCommit.Hash = string(line[7:47])
 			}
-		case isAuthorLine(line):
+		case isAuthorLine(isStaged, latestState, line):
+			latestState = AuthorLine
+
 			currentCommit.Author = strings.TrimRight(string(line[8:]), "\n")
-			recentlyPassedAuthor = true
-		case isDateLine(line):
+		case isDateLine(isStaged, latestState, line):
+			latestState = DateLine
+
 			date, err := time.Parse(c.dateFormat, strings.TrimSpace(string(line[6:])))
 			if err != nil {
 				ctx.Logger().V(2).Info("Could not parse date from git stream.", "error", err)
 			}
 			currentCommit.Date = date
-		case isDiffLine(line):
+		case isMessageStartLine(isStaged, latestState, line):
+			latestState = MessageStartLine
+			// NoOp
+		case isMessageLine(isStaged, latestState, line):
+			latestState = MessageLine
+
+			currentCommit.Message.Write(line[4:])
+		case isMessageEndLine(isStaged, latestState, line):
+			latestState = MessageEndLine
+			// NoOp
+		case isDiffLine(isStaged, latestState, line):
+			latestState = DiffLine
+
 			// This should never be nil, but check in case the stdin stream is messed up.
 			if currentCommit == nil {
 				currentCommit = &Commit{}
@@ -267,27 +328,29 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 				}
 			}
 			currentDiff = &Diff{}
-		case isModeLine(line):
+		case isModeLine(isStaged, latestState, line):
+			latestState = ModeLine
 			// NoOp
-		case isIndexLine(line):
-			recentlyPassedAuthor = false
+		case isIndexLine(isStaged, latestState, line):
+			latestState = IndexLine
 			// NoOp
-		case isPlusFileLine(line):
-			currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
-		case isMinusFileLine(line):
-			// NoOp
-		case isPlusDiffLine(line):
-			currentDiff.Content.Write(line[1:])
-		case isMinusDiffLine(line):
-			// NoOp. We only care about additions.
-		case (isMessageLine(line) && recentlyPassedAuthor):
-			currentCommit.Message.Write(line[4:])
-		case isContextDiffLine(line):
-			currentDiff.Content.Write([]byte("\n"))
-		case isBinaryLine(line):
+		case isBinaryLine(isStaged, latestState, line):
+			latestState = BinaryFileLine
+
 			currentDiff.IsBinary = true
 			currentDiff.PathB = pathFromBinaryLine(line)
-		case isLineNumberDiffLine(line):
+		case isFromFileLine(isStaged, latestState, line):
+			latestState = FromFileLine
+			// NoOp
+		case isToFileLine(isStaged, latestState, line):
+			latestState = ToFileLine
+
+			// TODO: Is this fix still required?
+			currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
+		case isHunkLineNumberLine(isStaged, latestState, line):
+			latestState = HunkLineNumberLine
+
+			// TODO: Is it still necessary to check whether the currentDiff is nil?
 			if currentDiff != nil && currentDiff.Content.Len() > 0 {
 				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 			}
@@ -305,7 +368,54 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 					currentDiff.LineStart = lineStart
 				}
 			}
+		case isHunkContextLine(isStaged, latestState, line):
+			if latestState != HunkContentLine {
+				latestState = HunkContentLine
+			}
+			// TODO: Why do we care about this? It creates empty lines in the diff. If there are no plusLines, it's just newlines.
+			currentDiff.Content.Write([]byte("\n"))
+		case isHunkPlusLine(isStaged, latestState, line):
+			if latestState != HunkContentLine {
+				latestState = HunkContentLine
+			}
+
+			currentDiff.Content.Write(line[1:])
+		case isHunkMinusLine(isStaged, latestState, line):
+			if latestState != HunkContentLine {
+				latestState = HunkContentLine
+			}
+			// NoOp. We only care about additions.
+		case isHunkNewlineWarningLine(isStaged, latestState, line):
+			if latestState != HunkContentLine {
+				latestState = HunkContentLine
+			}
+			// NoOp
+		case isHunkEmptyLine(isStaged, latestState, line):
+			if latestState != HunkContentLine {
+				latestState = HunkContentLine
+			}
+			// NoOp
+		case isCommitSeparatorLine(isStaged, latestState, line):
+			// NoOp
+		default:
+			// Skip ahead until we find the next diff or commit.
+			if latestState == ParseFailure {
+				continue
+			}
+
+			// Here be dragons...
+			// Build an informative error message.
+			var err error
+			if currentCommit != nil && currentCommit.Hash != "" {
+				err = fmt.Errorf(`failed to parse line "%s" after state "%s" (commit=%s)`, line, latestState, currentCommit.Hash)
+			} else {
+				err = fmt.Errorf(`failed to parse line "%s" after state "%s"`, line, latestState)
+			}
+			ctx.Logger().V(2).Error(err, "Recovering at the latest commit or diff...\n")
+
+			latestState = ParseFailure
 		}
+
 		if currentDiff.Content.Len() > c.maxDiffSize {
 			ctx.Logger().V(2).Info(fmt.Sprintf(
 				"Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize,
@@ -318,125 +428,142 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 	ctx.Logger().V(2).Info("finished parsing git log.", "total_log_size", totalLogSize)
 }
 
-func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
-	if currentDiff != nil && currentDiff.Content.Len() > 0 {
-		currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+// commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05
+func isCommitLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if isStaged || !(latestState == Initial ||
+		latestState == MessageStartLine ||
+		latestState == MessageEndLine ||
+		latestState == ModeLine ||
+		latestState == IndexLine ||
+		latestState == BinaryFileLine ||
+		latestState == ToFileLine ||
+		latestState == HunkContentLine ||
+		latestState == ParseFailure) {
+		return false
 	}
-	if currentCommit != nil {
-		commitChan <- *currentCommit
-		if totalLogSize != nil {
-			*totalLogSize += currentCommit.Size
-		}
-	}
-}
 
-// Date:   Tue Aug 10 15:20:40 2021 +0100
-func isDateLine(line []byte) bool {
-	if len(line) > 7 && bytes.Equal(line[:5], []byte("Date:")) {
+	if len(line) > 7 && bytes.Equal(line[:7], []byte("commit ")) {
 		return true
 	}
 	return false
 }
 
 // Author: Bill Rich <bill.rich@trufflesec.com>
-func isAuthorLine(line []byte) bool {
+func isAuthorLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if isStaged || !(latestState == CommitLine) {
+		return false
+	}
 	if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
 		return true
 	}
 	return false
 }
 
-// commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05
-func isCommitLine(line []byte) bool {
-	if len(line) > 7 && bytes.Equal(line[:6], []byte("commit")) {
+// Date:   Tue Aug 10 15:20:40 2021 +0100
+func isDateLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if isStaged || latestState != AuthorLine {
+		return false
+	}
+	if len(line) > 7 && bytes.Equal(line[:5], []byte("Date:")) {
 		return true
 	}
 	return false
 }
 
-// diff --git a/internal/addrs/move_endpoint_module.go b/internal/addrs/move_endpoint_module.go
-func isDiffLine(line []byte) bool {
-	if len(line) > 5 && bytes.Equal(line[:4], []byte("diff")) {
-		return true
+// Line directly after Date with only a newline.
+func isMessageStartLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if isStaged || latestState != DateLine {
+		return false
 	}
-	return false
-}
-
-// index 1ed6fbee1..aea1e643a 100644
-func isIndexLine(line []byte) bool {
-	if len(line) > 6 && bytes.Equal(line[:5], []byte("index")) {
-		return true
-	}
-	return false
-}
-
-// new file mode 100644
-func isModeLine(line []byte) bool {
-	if len(line) > 13 && bytes.Equal(line[:13], []byte("new file mode")) {
-		return true
-	}
-	return false
-}
-
-// --- a/internal/addrs/move_endpoint_module.go
-func isMinusFileLine(line []byte) bool {
-	if len(line) >= 6 && bytes.Equal(line[:3], []byte("---")) {
-		return true
-	}
-	return false
-}
-
-// +++ b/internal/addrs/move_endpoint_module.go
-func isPlusFileLine(line []byte) bool {
-	if len(line) >= 6 && bytes.Equal(line[:4], []byte("+++ ")) {
-		return true
-	}
-	return false
-}
-
-// +fmt.Println("ok")
-func isPlusDiffLine(line []byte) bool {
-	if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) {
-		return true
-	}
-	return false
-}
-
-// -fmt.Println("ok")
-func isMinusDiffLine(line []byte) bool {
-	if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) {
-		return true
-	}
-	return false
-}
-
-// fmt.Println("ok")
-func isContextDiffLine(line []byte) bool {
-	if len(line) >= 1 && bytes.Equal(line[:1], []byte(" ")) {
+	// TODO: Improve the implementation of this and isMessageEndLine
+	if len(strings.TrimRight(string(line[:]), "\r\n")) == 0 {
 		return true
 	}
 	return false
 }
 
 // Line that starts with 4 spaces
-func isMessageLine(line []byte) bool {
+func isMessageLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if isStaged || !(latestState == MessageStartLine || latestState == MessageLine) {
+		return false
+	}
 	if len(line) > 4 && bytes.Equal(line[:4], []byte("    ")) {
 		return true
 	}
 	return false
 }
 
-// Binary files /dev/null and b/plugin.sig differ
-func isBinaryLine(line []byte) bool {
-	if len(line) > 7 && bytes.Equal(line[:6], []byte("Binary")) {
+// Line directly after MessageLine with only a newline.
+func isMessageEndLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if isStaged || latestState != MessageLine {
+		return false
+	}
+	if len(strings.TrimRight(string(line[:]), "\r\n")) == 0 {
 		return true
 	}
 	return false
 }
 
-// @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)
-func isLineNumberDiffLine(line []byte) bool {
-	if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) {
+// diff --git a/internal/addrs/move_endpoint_module.go b/internal/addrs/move_endpoint_module.go
+func isDiffLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if !(latestState == MessageStartLine || // Empty commit messages can go from MessageStart->Diff
+		latestState == MessageEndLine ||
+		latestState == BinaryFileLine ||
+		latestState == IndexLine ||
+		latestState == HunkContentLine ||
+		latestState == ParseFailure) {
+		if latestState == Initial && !isStaged {
+			return false
+		}
+	}
+	if len(line) > 11 && bytes.Equal(line[:11], []byte("diff --git ")) {
+		return true
+	}
+	return false
+}
+
+// old mode 100644
+// new mode 100755
+// new file mode 100644
+// similarity index 100%
+// rename from old.txt
+// rename to new.txt
+// deleted file mode 100644
+func isModeLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if !(latestState == DiffLine || latestState == ModeLine) {
+		return false
+	}
+	// This could probably be better written.
+	if (len(line) > 17 && bytes.Equal(line[:17], []byte("deleted file mode"))) ||
+		(len(line) > 16 && bytes.Equal(line[:16], []byte("similarity index"))) ||
+		(len(line) > 13 && bytes.Equal(line[:13], []byte("new file mode"))) ||
+		(len(line) > 11 && bytes.Equal(line[:11], []byte("rename from"))) ||
+		(len(line) > 9 && bytes.Equal(line[:9], []byte("rename to"))) ||
+		(len(line) > 8 && bytes.Equal(line[:8], []byte("old mode"))) ||
+		(len(line) > 8 && bytes.Equal(line[:8], []byte("new mode"))) {
+		return true
+	}
+	return false
+}
+
+// index 1ed6fbee1..aea1e643a 100644
+// index 00000000..e69de29b
+func isIndexLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if !(latestState == DiffLine || latestState == ModeLine) {
+		return false
+	}
+	if len(line) > 6 && bytes.Equal(line[:6], []byte("index ")) {
+		return true
+	}
+	return false
+}
+
+// Binary files /dev/null and b/plugin.sig differ
+func isBinaryLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if latestState != IndexLine {
+		return false
+	}
+	if len(line) > 7 && bytes.Equal(line[:6], []byte("Binary")) {
 		return true
 	}
 	return false
@@ -453,3 +580,117 @@ func pathFromBinaryLine(line []byte) string {
 	bRaw := sbytes[1]
 	return string(bRaw[:len(bRaw)-7]) // drop the "b/" and " differ"
 }
+
+// --- a/internal/addrs/move_endpoint_module.go
+func isFromFileLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if latestState != IndexLine {
+		return false
+	}
+	if len(line) >= 6 && bytes.Equal(line[:4], []byte("--- ")) {
+		return true
+	}
+	return false
+}
+
+// +++ b/internal/addrs/move_endpoint_module.go
+func isToFileLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if latestState != FromFileLine {
+		return false
+	}
+	if len(line) >= 6 && bytes.Equal(line[:4], []byte("+++ ")) {
+		return true
+	}
+	return false
+}
+
+// @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)
+func isHunkLineNumberLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if !(latestState == ToFileLine || latestState == HunkContentLine) {
+		return false
+	}
+	if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) {
+		return true
+	}
+	return false
+}
+
+// fmt.Println("ok")
+// (There's a space before `fmt` that gets removed by the formatter.)
+func isHunkContextLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
+		return false
+	}
+	if len(line) >= 1 && bytes.Equal(line[:1], []byte(" ")) {
+		return true
+	}
+	return false
+}
+
+// +fmt.Println("ok")
+func isHunkPlusLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
+		return false
+	}
+	if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) {
+		return true
+	}
+	return false
+}
+
+// -fmt.Println("ok")
+func isHunkMinusLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
+		return false
+	}
+	if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) {
+		return true
+	}
+	return false
+}
+
+// \ No newline at end of file
+func isHunkNewlineWarningLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if latestState != HunkContentLine {
+		return false
+	}
+	if len(line) >= 27 && bytes.Equal(line[:27], []byte("\\ No newline at end of file")) {
+		return true
+	}
+	return false
+}
+
+// Newline after hunk, or an empty line, e.g.
+// +}
+//
+// commit 00920984e3435057f09cee5468850f7546dfa637 (tag: v3.42.0)
+func isHunkEmptyLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
+		return false
+	}
+	// TODO: Can this also be `\n\r`?
+	if len(line) == 1 && bytes.Equal(line[:1], []byte("\n")) {
+		return true
+	}
+	return false
+}
+
+func isCommitSeparatorLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if (latestState == ModeLine || latestState == IndexLine || latestState == BinaryFileLine || latestState == ToFileLine) &&
+		len(line) == 1 && bytes.Equal(line[:1], []byte("\n")) {
+		return true
+	}
+	return false
+}
+
+func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
+	// Ignore empty or binary diffs (this condition may be redundant).
+	if currentDiff != nil && currentDiff.Content.Len() > 0 {
+		currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+	}
+	if currentCommit != nil {
+		commitChan <- *currentCommit
+		if totalLogSize != nil {
+			*totalLogSize += currentCommit.Size
+		}
+	}
+}

pkg/handlers/archive.go

@@ -6,12 +6,14 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"strings"
 	"time"
 
 	"github.com/mholt/archiver/v4"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
 type ctxKey int
@@ -52,14 +54,14 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
 }
 
 // FromFile extracts the files from an archive.
-func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan ([]byte) {
-	archiveChan := make(chan ([]byte), 512)
+func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan ChunkOpt {
+	archiveChan := make(chan ChunkOpt, 512)
 	go func() {
 		ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
 		logger := logContext.AddLogger(ctx).Logger()
 		defer cancel()
 		defer close(archiveChan)
-		err := d.openArchive(ctx, 0, data, archiveChan)
+		err := d.openArchive(ctx, nil, data, archiveChan)
 		if err != nil {
 			if errors.Is(err, archiver.ErrNoMatch) {
 				return
@@ -71,18 +73,21 @@ func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan ([]
 }
 
 // openArchive takes a reader and extracts the contents up to the maximum depth.
-func (d *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan ([]byte)) error {
-	if depth >= maxDepth {
+func (d *Archive) openArchive(ctx context.Context, depth []string, reader io.Reader, archiveChan chan ChunkOpt) error {
+	if len(depth) >= maxDepth {
 		return fmt.Errorf("max archive depth reached")
 	}
 	format, reader, err := archiver.Identify("", reader)
 	if err != nil {
-		if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
+		if errors.Is(err, archiver.ErrNoMatch) && len(depth) > 0 {
 			chunkSize := 10 * 1024
 			for {
-				chunk := make([]byte, chunkSize)
-				n, _ := reader.Read(chunk)
-				archiveChan <- chunk
+				data := make([]byte, chunkSize)
+				n, _ := reader.Read(data)
+				archiveChan <- func(chunk *sources.Chunk) {
+					chunk.Data = data
+					chunk.HandleMetadata = map[string]string{"archive path": strings.Join(depth, " > ")}
+				}
 				if n < chunkSize {
 					break
 				}
@@ -102,9 +107,9 @@ func (d *Archive) openArchive(ctx context.Context, depth int, reader io.Reader,
 			return err
 		}
 		newReader := bytes.NewReader(fileBytes)
-		return d.openArchive(ctx, depth+1, newReader, archiveChan)
+		return d.openArchive(ctx, append(depth, "decompressed"), newReader, archiveChan)
 	case archiver.Extractor:
-		err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, d.extractorHandler(archiveChan))
+		err := archive.Extract(context.WithValue(ctx, depthKey, depth), reader, nil, d.extractorHandler(archiveChan))
 		if err != nil {
 			return err
 		}
@@ -129,12 +134,12 @@ func (d *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader,
 }
 
 // extractorHandler is applied to each file in an archiver.Extractor file.
-func (d *Archive) extractorHandler(archiveChan chan ([]byte)) func(context.Context, archiver.File) error {
+func (d *Archive) extractorHandler(archiveChan chan ChunkOpt) func(context.Context, archiver.File) error {
 	return func(ctx context.Context, f archiver.File) error {
 		logger := logContext.AddLogger(ctx).Logger()
 		logger.V(5).Info("Handling extracted file.", "filename", f.Name())
-		depth := 0
-		if ctxDepth, ok := ctx.Value(depthKey).(int); ok {
+		var depth []string
+		if ctxDepth, ok := ctx.Value(depthKey).([]string); ok {
 			depth = ctxDepth
 		}
 
@@ -148,7 +153,7 @@ func (d *Archive) extractorHandler(archiveChan chan ([]byte)) func(context.Conte
 		}
 		fileContent := bytes.NewReader(fileBytes)
 
-		err = d.openArchive(ctx, depth, fileContent, archiveChan)
+		err = d.openArchive(ctx, append(depth, f.Name()), fileContent, archiveChan)
 		if err != nil {
 			return err
 		}

pkg/handlers/archive_test.go

@@ -85,9 +85,11 @@ func TestArchiveHandler(t *testing.T) {
 		count := 0
 		re := regexp.MustCompile(testCase.matchString)
 		matched := false
-		for chunk := range archiveChan {
+		var chunk sources.Chunk
+		for opt := range archiveChan {
 			count++
-			if re.Match(chunk) {
+			opt(&chunk)
+			if re.Match(chunk.Data) {
 				matched = true
 			}
 		}

pkg/handlers/handlers.go

@@ -13,8 +13,10 @@ func DefaultHandlers() []Handler {
 	}
 }
 
+type ChunkOpt func(*sources.Chunk)
+
 type Handler interface {
-	FromFile(context.Context, io.Reader) chan ([]byte)
+	FromFile(context.Context, io.Reader) chan ChunkOpt
 	IsFiletype(context.Context, io.Reader) (io.Reader, bool)
 	New()
 }
@@ -38,13 +40,13 @@ func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, c
 	handlerChan := handler.FromFile(ctx, file)
 	for {
 		select {
-		case data, open := <-handlerChan:
+		case opt, open := <-handlerChan:
 			if !open {
 				// We finished reading everything from handlerChan.
 				return true
 			}
 			chunk := *chunkSkel
-			chunk.Data = data
+			opt(&chunk)
 			// Send data on chunksChan.
 			select {
 			case chunksChan <- &chunk:

pkg/output/plain.go

@@ -96,9 +96,9 @@ func structToMap(obj interface{}) (m map[string]map[string]interface{}, err erro
 }
 
 type outputFormat struct {
-	DetectorType,
-	DecoderType string
-	Verified bool
-	Raw      string
+	DetectorType string
+	DecoderType  string
+	Verified     bool
+	Raw          string
 	*source_metadatapb.MetaData
 }

pkg/sources/chunker.go

@@ -12,6 +12,8 @@ const (
 	ChunkSize = 10 * 1024
 	// PeekSize is the size of the peek into the previous chunk.
 	PeekSize = 3 * 1024
+	// TotalChunkSize is the total size of a chunk with peek data.
+	TotalChunkSize = ChunkSize + PeekSize
 )
 
 // Chunker takes a chunk and splits it into chunks of ChunkSize.
@@ -19,27 +21,31 @@ func Chunker(originalChunk *Chunk) chan *Chunk {
 	chunkChan := make(chan *Chunk)
 	go func() {
 		defer close(chunkChan)
-		if len(originalChunk.Data) <= ChunkSize+PeekSize {
+		if len(originalChunk.Data) <= TotalChunkSize {
 			chunkChan <- originalChunk
 			return
 		}
 		r := bytes.NewReader(originalChunk.Data)
 		reader := bufio.NewReaderSize(bufio.NewReader(r), ChunkSize)
 		for {
-			chunkBytes := make([]byte, ChunkSize)
+			chunkBytes := make([]byte, TotalChunkSize)
 			chunk := *originalChunk
+			chunkBytes = chunkBytes[:ChunkSize]
 			n, err := reader.Read(chunkBytes)
 			if err != nil && !errors.Is(err, io.EOF) {
 				break
 			}
+			if n == 0 {
+				if errors.Is(err, io.EOF) {
+					break
+				}
+				continue
+			}
 			peekData, _ := reader.Peek(PeekSize)
-			chunk.Data = append(chunkBytes[:n], peekData...)
-			if n > 0 {
-				chunkChan <- &chunk
-			}
-			if errors.Is(err, io.EOF) {
-				break
-			}
+			copy(chunkBytes[n:], peekData)
+			chunk.Data = chunkBytes[:n+len(peekData)]
+
+			chunkChan <- &chunk
 		}
 	}()
 	return chunkChan

pkg/sources/chunker_test.go

@@ -72,3 +72,15 @@ func TestChunker(t *testing.T) {
 	}
 
 }
+
+func BenchmarkChunker(b *testing.B) {
+	data := bytes.Repeat([]byte("a"), ChunkSize*100)
+	chunk := &Chunk{
+		Data: data,
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for range Chunker(chunk) {
+		}
+	}
+}

pkg/sources/git/git.go

@@ -41,11 +41,7 @@ type Source struct {
 	verify   bool
 	git      *Git
 	sources.Progress
-	conn        *sourcespb.Git
-	scanOptions *ScanOptions
-	// Kludge to preserve engine.ScanGit functionality which doesn't expect
-	// the scanning to clean up the directory.
-	preserveTempDirs bool
+	conn *sourcespb.Git
 }
 
 type Git struct {
@@ -55,9 +51,14 @@ type Git struct {
 	jobID              int64
 	sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData
 	verify             bool
+	metrics            metrics
 	concurrency        *semaphore.Weighted
 }
 
+type metrics struct {
+	commitsScanned int
+}
+
 func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
 	sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData,
 ) *Git {
@@ -90,19 +91,6 @@ func (s *Source) JobID() int64 {
 	return s.jobId
 }
 
-// WithScanOptions sets the scan options.
-func (s *Source) WithScanOptions(scanOptions *ScanOptions) {
-	s.scanOptions = scanOptions
-}
-
-// WithPreserveTempDirs sets whether to preserve temp directories when scanning
-// the provided list of s.conn.Directories. NOTE: This is *only* for
-// s.conn.Directories, not all temp directories created. This is also a kludge
-// and should be refactored away.
-func (s *Source) WithPreserveTempDirs(preserve bool) {
-	s.preserveTempDirs = preserve
-}
-
 // Init returns an initialized GitHub source.
 func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
 	s.name = name
@@ -146,29 +134,8 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 
 // Chunks emits chunks of bytes over a channel.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	if err := s.scanRepos(ctx, chunksChan); err != nil {
-		return err
-	}
-	if err := s.scanDirs(ctx, chunksChan); err != nil {
-		return err
-	}
-
-	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
-	ctx.Logger().V(1).Info("Git source finished scanning", "repo_count", totalRepos)
-	s.SetProgressComplete(
-		totalRepos, totalRepos,
-		fmt.Sprintf("Completed scanning source %s", s.name), "",
-	)
-	return nil
-}
-
-// scanRepos scans the configured repositories in s.conn.Repositories.
-func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	if len(s.conn.Repositories) == 0 {
-		return nil
-	}
-	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
 	// TODO: refactor to remove duplicate code
+	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
 	switch cred := s.conn.GetCredential().(type) {
 	case *sourcespb.Git_BasicAuth:
 		user := cred.BasicAuth.Username
@@ -185,7 +152,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 				if err != nil {
 					return err
 				}
-				return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
+				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			}(repoURI)
 			if err != nil {
 				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
@@ -204,7 +171,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 				if err != nil {
 					return err
 				}
-				return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
+				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			}(repoURI)
 			if err != nil {
 				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
@@ -223,7 +190,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 				if err != nil {
 					return err
 				}
-				return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
+				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			}(repoURI)
 			if err != nil {
 				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
@@ -233,42 +200,41 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 	default:
 		return errors.New("invalid connection type for git source")
 	}
-	return nil
-}
 
-// scanDirs scans the configured directories in s.conn.Directories.
-func (s *Source) scanDirs(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
 	for i, gitDir := range s.conn.Directories {
 		s.SetProgressComplete(len(s.conn.Repositories)+i, totalRepos, fmt.Sprintf("Repo: %s", gitDir), "")
 
 		if len(gitDir) == 0 {
 			continue
 		}
-		if strings.HasSuffix(gitDir, "git") {
-			// TODO: Figure out why we skip directories ending in "git".
-			continue
-		}
-		// try paths instead of url
-		repo, err := RepoFromPath(gitDir)
-		if err != nil {
-			ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
-			continue
-		}
-
-		err = func(repoPath string) error {
-			if !s.preserveTempDirs && strings.HasPrefix(repoPath, filepath.Join(os.TempDir(), "trufflehog")) {
-				defer os.RemoveAll(repoPath)
+		if !strings.HasSuffix(gitDir, "git") {
+			// try paths instead of url
+			repo, err := RepoFromPath(gitDir)
+			if err != nil {
+				ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
+				continue
 			}
 
-			return s.git.ScanRepo(ctx, repo, repoPath, s.scanOptions, chunksChan)
-		}(gitDir)
-		if err != nil {
-			ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
-			continue
+			err = func(repoPath string) error {
+				if strings.HasPrefix(repoPath, filepath.Join(os.TempDir(), "trufflehog")) {
+					defer os.RemoveAll(repoPath)
+				}
+
+				return s.git.ScanRepo(ctx, repo, repoPath, NewScanOptions(), chunksChan)
+			}(gitDir)
+			if err != nil {
+				ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
+				continue
+			}
 		}
 
 	}
+
+	ctx.Logger().V(1).Info("Git source finished scanning", "repo-count", totalRepos)
+	s.SetProgressComplete(
+		totalRepos, totalRepos,
+		fmt.Sprintf("Completed scanning source %s", s.name), "",
+	)
 	return nil
 }
 
@@ -378,6 +344,10 @@ func CloneRepoUsingSSH(ctx context.Context, gitUrl string, args ...string) (stri
 	return CloneRepo(ctx, userInfo, gitUrl, args...)
 }
 
+func (s *Git) CommitsScanned() int {
+	return s.metrics.commitsScanned
+}
+
 func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
 	if err := GitCmdCheck(); err != nil {
 		return err
@@ -410,6 +380,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 			break
 		}
 		depth++
+		s.metrics.commitsScanned++
 		logger.V(5).Info("scanning commit", "commit", commit.Hash)
 		for _, diff := range commit.Diffs {
 			if !scanOptions.Filter.Pass(diff.PathB) {

pkg/sources/source_manager.go

@@ -18,7 +18,7 @@ type handle int64
 
 // SourceInitFunc is a function that takes a source and job ID and returns an
 // initialized Source.
-type SourceInitFunc func(ctx context.Context, sourceID int64, jobID int64) (Source, error)
+type SourceInitFunc func(sourceID int64, jobID int64) (Source, error)
 
 type SourceManager struct {
 	api apiClient
@@ -50,12 +50,12 @@ func WithConcurrency(concurrency int) func(*SourceManager) {
 }
 
 // NewManager creates a new manager with the provided options.
-func NewManager(opts ...func(*SourceManager)) *SourceManager {
+func NewManager(outputChunks chan *Chunk, opts ...func(*SourceManager)) *SourceManager {
 	man := SourceManager{
 		// Default to the headless API. Can be overwritten by the WithAPI option.
 		api:          &headlessAPI{},
 		handles:      make(map[handle]SourceInitFunc),
-		outputChunks: make(chan *Chunk),
+		outputChunks: outputChunks,
 	}
 	for _, opt := range opts {
 		opt(&man)
@@ -77,7 +77,7 @@ func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.
 		return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
 	}
 	s.handles[handleID] = f
-	return handleID, nil
+	return 0, nil
 }
 
 // Run blocks until a resource is available to run the source, then synchronously runs it.
@@ -119,21 +119,6 @@ func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) error {
 	return nil
 }
 
-// Chunks returns the read only channel of all the chunks produced by all of
-// the sources managed by this manager.
-func (s *SourceManager) Chunks() <-chan *Chunk {
-	return s.outputChunks
-}
-
-// Wait blocks until all running sources are completed and returns the first
-// error encountered if any. It also closes the channel returned by Chunks().
-// The manager should not be reused after calling this method.
-func (s *SourceManager) Wait() error {
-	// TODO: Aggregate all errors from all sources.
-	defer close(s.outputChunks)
-	return s.pool.Wait()
-}
-
 // run is a helper method to sychronously run the source. It does not check for
 // acquired resources.
 func (s *SourceManager) run(ctx context.Context, handle handle) error {
@@ -145,7 +130,7 @@ func (s *SourceManager) run(ctx context.Context, handle handle) error {
 	if !ok {
 		return fmt.Errorf("unrecognized handle")
 	}
-	source, err := initFunc(ctx, jobID, int64(handle))
+	source, err := initFunc(jobID, int64(handle))
 	if err != nil {
 		return err
 	}

pkg/sources/sources.go

@@ -21,7 +21,8 @@ type Chunk struct {
 	SourceType sourcespb.SourceType
 	// SourceMetadata holds the context of where the Chunk was found.
 	SourceMetadata *source_metadatapb.MetaData
-
+	// HandleMetadata holds the metadata from a handler if one was used.
+	HandleMetadata map[string]string
 	// Data is the data to decode and scan.
 	Data []byte
 	// Verify specifies whether any secrets in the Chunk should be verified.