remove _ from npm_ and update ahocorasick to our fork

* Add commitsScanned metrics

* Just keep commit count

go.mod

@@ -44,7 +44,6 @@ require (
 	github.com/mholt/archiver/v4 v4.0.0-alpha.8
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/paulbellamy/ratecounter v0.2.0
-	github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.16.0
 	github.com/rabbitmq/amqp091-go v1.8.1
@@ -94,7 +93,7 @@ require (
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/connesc/cipherio v0.2.1 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
-	github.com/couchbase/gocbcore/v10 v10.2.3 // indirect
+	github.com/couchbase/gocbcore/v10 v10.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/docker/cli v23.0.5+incompatible // indirect
@@ -151,6 +150,7 @@ require (
 	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/skeema/knownhosts v1.1.1 // indirect
 	github.com/therootcompany/xz v1.0.1 // indirect
+	github.com/trufflesecurity/aho-corasick v0.1.0
 	github.com/ulikunitz/xz v0.5.10 // indirect
 	github.com/vbatts/tar-split v0.11.3 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect

go.sum

@@ -119,8 +119,9 @@ github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSk
 github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
 github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
 github.com/couchbase/gocb/v2 v2.6.3/go.mod h1:yF5F6BHTZ/ZowhEuZbySbXrlI4rHd1TIhm5azOaMbJU=
-github.com/couchbase/gocbcore/v10 v10.2.3 h1:PEkRSNSkKjUBXx82Ucr094+anoiCG5GleOOQZOHo6D4=
 github.com/couchbase/gocbcore/v10 v10.2.3/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
+github.com/couchbase/gocbcore/v10 v10.2.4 h1:TDTQ1mSBUw9eajuV71ZDJcLRbmfGcsvZV9SMeUDAVGc=
+github.com/couchbase/gocbcore/v10 v10.2.4/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
 github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
 github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4=
 github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
@@ -382,8 +383,6 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
 github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
-github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
-github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
 github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
@@ -446,6 +445,8 @@ github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HP
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
 github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
 github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
+github.com/trufflesecurity/aho-corasick v0.1.0 h1:bwtdjWFkuzDRLWjJi4fCxkBSv2axhYXJryCqsSgMHSg=
+github.com/trufflesecurity/aho-corasick v0.1.0/go.mod h1:JKKZLgcVlyWn0LsPezrqEIQPi76i2URv44hC0gJY4Vo=
 github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
 github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=

pkg/detectors/npmtoken/npmtoken.go

@@ -36,7 +36,6 @@ func (s Scanner) Keywords() []string {
 // FromData will find and optionally verify NpmToken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
-
 	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 	for _, match := range matches {
 		if len(match) != 2 {

pkg/detectors/npmtokenv2/npmtokenv2.go

@@ -29,7 +29,7 @@ var (
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"npm_"}
+	return []string{"npm"}
 }
 
 // FromData will find and optionally verify NpmTokenV2 secrets in a given set of bytes.

pkg/detectors/okta/okta.go

@@ -19,8 +19,8 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	domainPat = regexp.MustCompile(`[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com`)
-	tokenPat  = regexp.MustCompile(`00[a-zA-Z0-9_-]{40}`)
+	domainPat = regexp.MustCompile(`\b[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com\b`)
+	tokenPat  = regexp.MustCompile(`\b00[a-zA-Z0-9_-]{40}\b`)
 	// TODO: Oauth client secrets
 )
 

pkg/engine/circleci.go

@@ -1,14 +1,16 @@
 package engine
 
 import (
+	"fmt"
 	"runtime"
 
+	"github.com/go-errors/errors"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
 )
 
@@ -27,16 +29,23 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
 		return err
 	}
 
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Circle CI", (&circleci.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			circleSource := &circleci.Source{}
-			if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			return circleSource, nil
-		})
+	circleSource := circleci.Source{}
+	ctx = context.WithValues(ctx,
+		"source_type", circleSource.Type().String(),
+		"source_name", "Circle CI",
+	)
+	err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU())
 	if err != nil {
-		return err
+		return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
 	}
-	return e.sourceManager.ScheduleRun(ctx, handle)
+
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := circleSource.Chunks(ctx, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("error scanning CircleCI: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/engine/docker.go

@@ -1,27 +1,38 @@
 package engine
 
 import (
+	"fmt"
 	"runtime"
 
+	"github.com/go-errors/errors"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
 )
 
 // ScanDocker scans a given docker connection.
 func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error {
-	handler, err := e.sourceManager.Enroll(ctx, "trufflehog - docker", (&docker.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			dockerSource := &docker.Source{}
-			if err := dockerSource.Init(ctx, "trufflehog - docker", jobID, sourceID, true, conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			return dockerSource, nil
-		})
+	dockerSource := docker.Source{}
+
+	ctx = context.WithValues(ctx,
+		"source_type", dockerSource.Type().String(),
+		"source_name", "docker",
+	)
+	err := dockerSource.Init(ctx, "trufflehog - docker", 0, int64(sourcespb.SourceType_SOURCE_TYPE_DOCKER), true, conn, runtime.NumCPU())
 	if err != nil {
-		return err
+		return errors.WrapPrefix(err, "could not init docker source", 0)
 	}
-	return e.sourceManager.ScheduleRun(ctx, handler)
+
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := dockerSource.Chunks(ctx, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("error scanning docker image: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/engine/engine.go

@@ -10,7 +10,8 @@ import (
 	"sync/atomic"
 	"time"
 
-	ahocorasick "github.com/petar-dambovaliev/aho-corasick"
+	ahocorasick "github.com/trufflesecurity/aho-corasick"
+	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -26,13 +27,14 @@ import (
 
 type Engine struct {
 	concurrency     int
-	sourceManager   *sources.SourceManager
+	chunks          chan *sources.Chunk
 	results         chan detectors.ResultWithMetadata
 	decoders        []decoders.Decoder
 	detectors       map[bool][]detectors.Detector
 	chunksScanned   uint64
 	bytesScanned    uint64
 	detectorAvgTime sync.Map
+	sourcesWg       *errgroup.Group
 	workersWg       sync.WaitGroup
 	// filterUnverified is used to reduce the number of unverified results.
 	// If there are multiple unverified results for the same chunk for the same detector,
@@ -108,8 +110,10 @@ func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors
 
 func Start(ctx context.Context, options ...EngineOption) *Engine {
 	e := &Engine{
+		chunks:          make(chan *sources.Chunk),
 		results:         make(chan detectors.ResultWithMetadata),
 		detectorAvgTime: sync.Map{},
+		sourcesWg:       &errgroup.Group{},
 	}
 
 	for _, option := range options {
@@ -124,6 +128,9 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	}
 	ctx.Logger().V(2).Info("engine started", "workers", e.concurrency)
 
+	// Limit number of concurrent goroutines dedicated to chunking a source.
+	e.sourcesWg.SetLimit(e.concurrency)
+
 	if len(e.decoders) == 0 {
 		e.decoders = decoders.DefaultDecoders()
 	}
@@ -134,9 +141,6 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 		e.detectors[false] = []detectors.Detector{}
 	}
 
-	// Setup source manager now that we know the concurrency.
-	e.sourceManager = sources.NewManager(sources.WithConcurrency(e.concurrency))
-
 	// build ahocorasick prefilter for efficient string matching
 	// on keywords
 	keywords := []string{}
@@ -149,8 +153,8 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{
 		AsciiCaseInsensitive: true,
 		MatchOnlyWholeWords:  false,
-		MatchKind:            ahocorasick.LeftMostLongestMatch,
-		DFA:                  true,
+		MatchKind:            ahocorasick.StandardMatch,
+		DFA:                  false,
 	})
 	e.prefilter = builder.Build(keywords)
 
@@ -180,7 +184,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	for i := 0; i < e.concurrency; i++ {
 		e.workersWg.Add(1)
 		go func() {
-			defer common.RecoverWithExit(ctx)
+			defer common.Recover(ctx)
 			defer e.workersWg.Done()
 			e.detectorWorker(ctx)
 		}()
@@ -194,12 +198,13 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 // more sources may be scanned by the engine.
 func (e *Engine) Finish(ctx context.Context, logFunc func(error, string, ...any)) {
 	defer common.RecoverWithExit(ctx)
-	// Wait for the sources to finish putting chunks onto the chunks channel.
-	sourceErr := e.sourceManager.Wait()
+	// wait for the sources to finish putting chunks onto the chunks channel
+	sourceErr := e.sourcesWg.Wait()
 	if sourceErr != nil {
 		logFunc(sourceErr, "error occurred while collecting chunks")
 	}
 
+	close(e.chunks)
 	// wait for the workers to finish processing all of the chunks and putting
 	// results onto the results channel
 	e.workersWg.Wait()
@@ -212,8 +217,8 @@ func (e *Engine) Finish(ctx context.Context, logFunc func(error, string, ...any)
 	close(e.results)
 }
 
-func (e *Engine) ChunksChan() <-chan *sources.Chunk {
-	return e.sourceManager.Chunks()
+func (e *Engine) ChunksChan() chan *sources.Chunk {
+	return e.chunks
 }
 
 func (e *Engine) ResultsChan() chan detectors.ResultWithMetadata {
@@ -266,7 +271,7 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
 }
 
 func (e *Engine) detectorWorker(ctx context.Context) {
-	for originalChunk := range e.ChunksChan() {
+	for originalChunk := range e.chunks {
 		for chunk := range sources.Chunker(originalChunk) {
 			var chunkResults []detectors.ResultWithMetadata
 			matchedKeywords := make(map[string]struct{})
@@ -302,7 +307,6 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 						for _, kw := range detector.Keywords() {
 							if _, ok := matchedKeywords[strings.ToLower(kw)]; ok {
 								chunkContainsKeyword = true
-								break
 							}
 						}
 

pkg/engine/filesystem.go

@@ -1,11 +1,14 @@
 package engine
 
 import (
+	"fmt"
 	"runtime"
 
+	"github.com/go-errors/errors"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -24,17 +27,39 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
 		return err
 	}
 
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", (&filesystem.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			fileSystemSource := &filesystem.Source{}
-			fileSystemSource.WithFilter(c.Filter)
-			if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			return fileSystemSource, nil
-		})
+	fileSystemSource := filesystem.Source{}
+	fileSystemSource.WithFilter(c.Filter)
+
+	// TODO: This is how it would be used; everything after these comments would be removed.
+	// handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", fileSystemSource.Type(),
+	// 	func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
+	// 		fileSystemSource := filesystem.Source{}
+	// 		fileSystemSource.WithFilter(c.Filter)
+	// 		if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+	// 			return nil, err
+	// 		}
+	// 		return &fileSystemSource, nil
+	// 	})
+	// if err != nil {
+	// 	return err
+	// }
+	// return e.sourceManager.ScheduleRun(ctx, handle)
+
+	ctx = context.WithValues(ctx,
+		"source_type", fileSystemSource.Type().String(),
+		"source_name", "filesystem",
+	)
+	err = fileSystemSource.Init(ctx, "trufflehog - filesystem", 0, int64(sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM), true, &conn, runtime.NumCPU())
 	if err != nil {
-		return err
+		return errors.WrapPrefix(err, "could not init filesystem source", 0)
 	}
-	return e.sourceManager.ScheduleRun(ctx, handle)
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := fileSystemSource.Chunks(ctx, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("error scanning filesystem: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/engine/gcs.go

@@ -6,6 +6,7 @@ import (
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -44,18 +45,23 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
 		return fmt.Errorf("failed to marshal GCS connection: %w", err)
 	}
 
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - GCS", (&gcs.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			source := &gcs.Source{}
-			if err := source.Init(ctx, "trufflehog - GCS", jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
-				return nil, err
-			}
-			return source, nil
-		})
-	if err != nil {
-		return err
+	source := gcs.Source{}
+	ctx = context.WithValues(ctx,
+		"source_type", source.Type().String(),
+		"source_name", "gcs",
+	)
+	if err = source.Init(ctx, "trufflehog - GCS", 0, 0, true, &conn, int(c.Concurrency)); err != nil {
+		return fmt.Errorf("failed to initialize GCS source: %w", err)
 	}
-	return e.sourceManager.ScheduleRun(ctx, handle)
+
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		if err := source.Chunks(ctx, e.ChunksChan()); err != nil {
+			return fmt.Errorf("could not scan GCS: %w", err)
+		}
+		return nil
+	})
+	return nil
 }
 
 func isAuthValid(ctx context.Context, c sources.GCSConfig, connection *sourcespb.GCS) bool {

pkg/engine/git.go

@@ -1,13 +1,14 @@
 package engine
 
 import (
+	"fmt"
 	"runtime"
 
 	gogit "github.com/go-git/go-git/v5"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
@@ -21,6 +22,16 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 		git.ScanOptionLogOptions(logOptions),
 	}
 
+	options := &gogit.PlainOpenOptions{
+		DetectDotGit:          true,
+		EnableDotGitCommonDir: true,
+	}
+
+	repo, err := gogit.PlainOpenWithOptions(c.RepoPath, options)
+	if err != nil {
+		return fmt.Errorf("could not open repo: %s: %w", c.RepoPath, err)
+	}
+
 	if c.MaxDepth != 0 {
 		opts = append(opts, git.ScanOptionMaxDepth(int64(c.MaxDepth)))
 	}
@@ -35,34 +46,33 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 	}
 	scanOptions := git.NewScanOptions(opts...)
 
-	connection := &sourcespb.Git{
-		// Using Directories here allows us to not pass any
-		// authentication. Also by this point, the c.RepoPath should
-		// still have been prepared and downloaded to a temporary
-		// directory if it was a URL.
-		Directories: []string{c.RepoPath},
-	}
-	var conn anypb.Any
-	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
-	if err != nil {
-		ctx.Logger().Error(err, "failed to marshal git connection")
-		return err
-	}
-
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - git", (&git.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			gitSource := &git.Source{}
-			if err := gitSource.Init(ctx, "trufflehog - git", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
+	gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "trufflehog - git", true, runtime.NumCPU(),
+		func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
+			return &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_Git{
+					Git: &source_metadatapb.Git{
+						Commit:     commit,
+						File:       file,
+						Email:      email,
+						Repository: repository,
+						Timestamp:  timestamp,
+						Line:       line,
+					},
+				},
 			}
-			gitSource.WithScanOptions(scanOptions)
-			// Don't try to clean up the provided directory. That's
-			// handled by the caller of ScanGit.
-			gitSource.WithPreserveTempDirs(true)
-			return gitSource, nil
 		})
-	if err != nil {
-		return err
-	}
-	return e.sourceManager.ScheduleRun(ctx, handle)
+
+	ctx = context.WithValues(ctx,
+		"source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),
+		"source_name", "git",
+	)
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("could not scan repo: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/engine/github.go

@@ -1,10 +1,13 @@
 package engine
 
 import (
+	"fmt"
+
 	gogit "github.com/go-git/go-git/v5"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -14,6 +17,8 @@ import (
 
 // ScanGitHub scans Github with the provided options.
 func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
+	source := github.Source{}
+
 	connection := sourcespb.GitHub{
 		Endpoint:      c.Endpoint,
 		Organizations: c.Orgs,
@@ -37,24 +42,31 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
 		return err
 	}
 
+	ctx = context.WithValues(ctx,
+		"source_type", source.Type().String(),
+		"source_name", "github",
+	)
+	err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency)
+	if err != nil {
+		ctx.Logger().Error(err, "failed to initialize github source")
+		return err
+	}
+
 	logOptions := &gogit.LogOptions{}
 	opts := []git.ScanOption{
 		git.ScanOptionFilter(c.Filter),
 		git.ScanOptionLogOptions(logOptions),
 	}
 	scanOptions := git.NewScanOptions(opts...)
+	source.WithScanOptions(scanOptions)
 
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - GitHub", (&github.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			source := &github.Source{}
-			source.WithScanOptions(scanOptions)
-			if err := source.Init(ctx, "trufflehog - GitHub", jobID, sourceID, false, &conn, c.Concurrency); err != nil {
-				return nil, err
-			}
-			return source, nil
-		})
-	if err != nil {
-		return err
-	}
-	return e.sourceManager.Run(ctx, handle)
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := source.Chunks(ctx, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("could not scan github: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/engine/gitlab.go

@@ -4,10 +4,12 @@ import (
 	"fmt"
 	"runtime"
 
+	"github.com/go-errors/errors"
 	gogit "github.com/go-git/go-git/v5"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -50,17 +52,24 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
 		return err
 	}
 
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - GitLab", (&gitlab.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			gitlabSource := &gitlab.Source{}
-			gitlabSource.WithScanOptions(scanOptions)
-			if err := gitlabSource.Init(ctx, "trufflehog - GitLab", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			return gitlabSource, nil
-		})
+	gitlabSource := gitlab.Source{}
+	ctx = context.WithValues(ctx,
+		"source_type", gitlabSource.Type().String(),
+		"source_name", "gitlab",
+	)
+	err = gitlabSource.Init(ctx, "trufflehog - gitlab", 0, int64(sourcespb.SourceType_SOURCE_TYPE_GITLAB), true, &conn, runtime.NumCPU())
 	if err != nil {
-		return err
+		return errors.WrapPrefix(err, "could not init GitLab source", 0)
 	}
-	return e.sourceManager.ScheduleRun(ctx, handle)
+	gitlabSource.WithScanOptions(scanOptions)
+
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := gitlabSource.Chunks(ctx, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("error scanning GitLab: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/engine/s3.go

@@ -4,9 +4,11 @@ import (
 	"fmt"
 	"runtime"
 
+	"github.com/go-errors/errors"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -53,16 +55,23 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
 		return err
 	}
 
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - S3", (&s3.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			s3Source := &s3.Source{}
-			if err := s3Source.Init(ctx, "trufflehog - S3", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			return s3Source, nil
-		})
+	s3Source := s3.Source{}
+	ctx = context.WithValues(ctx,
+		"source_type", s3Source.Type().String(),
+		"source_name", "s3",
+	)
+	err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU())
 	if err != nil {
-		return err
+		return errors.WrapPrefix(err, "failed to init S3 source", 0)
 	}
-	return e.sourceManager.ScheduleRun(ctx, handle)
+
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := s3Source.Chunks(ctx, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("error scanning S3: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/engine/syslog.go

@@ -1,12 +1,14 @@
 package engine
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/go-errors/errors"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -40,17 +42,25 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
 	if err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
-
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - syslog", (&syslog.Source{}).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			source := &syslog.Source{}
-			if err := source.Init(ctx, "trufflehog - syslog", jobID, sourceID, false, &conn, c.Concurrency); err != nil {
-				return nil, err
-			}
-			return source, nil
-		})
+	source := syslog.Source{}
+	ctx = context.WithValues(ctx,
+		"source_type", source.Type().String(),
+		"source_name", "syslog",
+	)
+	err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, c.Concurrency)
+	source.InjectConnection(connection)
 	if err != nil {
+		ctx.Logger().Error(err, "failed to initialize syslog source")
 		return err
 	}
-	return e.sourceManager.ScheduleRun(ctx, handle)
+
+	e.sourcesWg.Go(func() error {
+		defer common.RecoverWithExit(ctx)
+		err := source.Chunks(ctx, e.ChunksChan())
+		if err != nil {
+			return fmt.Errorf("could not scan syslog: %w", err)
+		}
+		return nil
+	})
+	return nil
 }

pkg/sources/chunker.go

@@ -12,6 +12,8 @@ const (
 	ChunkSize = 10 * 1024
 	// PeekSize is the size of the peek into the previous chunk.
 	PeekSize = 3 * 1024
+	// TotalChunkSize is the total size of a chunk with peek data.
+	TotalChunkSize = ChunkSize + PeekSize
 )
 
 // Chunker takes a chunk and splits it into chunks of ChunkSize.
@@ -19,27 +21,31 @@ func Chunker(originalChunk *Chunk) chan *Chunk {
 	chunkChan := make(chan *Chunk)
 	go func() {
 		defer close(chunkChan)
-		if len(originalChunk.Data) <= ChunkSize+PeekSize {
+		if len(originalChunk.Data) <= TotalChunkSize {
 			chunkChan <- originalChunk
 			return
 		}
 		r := bytes.NewReader(originalChunk.Data)
 		reader := bufio.NewReaderSize(bufio.NewReader(r), ChunkSize)
 		for {
-			chunkBytes := make([]byte, ChunkSize)
+			chunkBytes := make([]byte, TotalChunkSize)
 			chunk := *originalChunk
+			chunkBytes = chunkBytes[:ChunkSize]
 			n, err := reader.Read(chunkBytes)
 			if err != nil && !errors.Is(err, io.EOF) {
 				break
 			}
+			if n == 0 {
+				if errors.Is(err, io.EOF) {
+					break
+				}
+				continue
+			}
 			peekData, _ := reader.Peek(PeekSize)
-			chunk.Data = append(chunkBytes[:n], peekData...)
-			if n > 0 {
-				chunkChan <- &chunk
-			}
-			if errors.Is(err, io.EOF) {
-				break
-			}
+			copy(chunkBytes[n:], peekData)
+			chunk.Data = chunkBytes[:n+len(peekData)]
+
+			chunkChan <- &chunk
 		}
 	}()
 	return chunkChan

pkg/sources/chunker_test.go

@@ -72,3 +72,15 @@ func TestChunker(t *testing.T) {
 	}
 
 }
+
+func BenchmarkChunker(b *testing.B) {
+	data := bytes.Repeat([]byte("a"), ChunkSize*100)
+	chunk := &Chunk{
+		Data: data,
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for range Chunker(chunk) {
+		}
+	}
+}

pkg/sources/git/git.go

@@ -41,11 +41,7 @@ type Source struct {
 	verify   bool
 	git      *Git
 	sources.Progress
-	conn        *sourcespb.Git
-	scanOptions *ScanOptions
-	// Kludge to preserve engine.ScanGit functionality which doesn't expect
-	// the scanning to clean up the directory.
-	preserveTempDirs bool
+	conn *sourcespb.Git
 }
 
 type Git struct {
@@ -55,9 +51,14 @@ type Git struct {
 	jobID              int64
 	sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData
 	verify             bool
+	metrics            metrics
 	concurrency        *semaphore.Weighted
 }
 
+type metrics struct {
+	commitsScanned int
+}
+
 func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
 	sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData,
 ) *Git {
@@ -90,19 +91,6 @@ func (s *Source) JobID() int64 {
 	return s.jobId
 }
 
-// WithScanOptions sets the scan options.
-func (s *Source) WithScanOptions(scanOptions *ScanOptions) {
-	s.scanOptions = scanOptions
-}
-
-// WithPreserveTempDirs sets whether to preserve temp directories when scanning
-// the provided list of s.conn.Directories. NOTE: This is *only* for
-// s.conn.Directories, not all temp directories created. This is also a kludge
-// and should be refactored away.
-func (s *Source) WithPreserveTempDirs(preserve bool) {
-	s.preserveTempDirs = preserve
-}
-
 // Init returns an initialized GitHub source.
 func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
 	s.name = name
@@ -146,29 +134,8 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 
 // Chunks emits chunks of bytes over a channel.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	if err := s.scanRepos(ctx, chunksChan); err != nil {
-		return err
-	}
-	if err := s.scanDirs(ctx, chunksChan); err != nil {
-		return err
-	}
-
-	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
-	ctx.Logger().V(1).Info("Git source finished scanning", "repo_count", totalRepos)
-	s.SetProgressComplete(
-		totalRepos, totalRepos,
-		fmt.Sprintf("Completed scanning source %s", s.name), "",
-	)
-	return nil
-}
-
-// scanRepos scans the configured repositories in s.conn.Repositories.
-func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	if len(s.conn.Repositories) == 0 {
-		return nil
-	}
-	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
 	// TODO: refactor to remove duplicate code
+	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
 	switch cred := s.conn.GetCredential().(type) {
 	case *sourcespb.Git_BasicAuth:
 		user := cred.BasicAuth.Username
@@ -185,7 +152,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 				if err != nil {
 					return err
 				}
-				return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
+				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			}(repoURI)
 			if err != nil {
 				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
@@ -204,7 +171,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 				if err != nil {
 					return err
 				}
-				return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
+				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			}(repoURI)
 			if err != nil {
 				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
@@ -223,7 +190,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 				if err != nil {
 					return err
 				}
-				return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
+				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			}(repoURI)
 			if err != nil {
 				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
@@ -233,42 +200,41 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 	default:
 		return errors.New("invalid connection type for git source")
 	}
-	return nil
-}
 
-// scanDirs scans the configured directories in s.conn.Directories.
-func (s *Source) scanDirs(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
 	for i, gitDir := range s.conn.Directories {
 		s.SetProgressComplete(len(s.conn.Repositories)+i, totalRepos, fmt.Sprintf("Repo: %s", gitDir), "")
 
 		if len(gitDir) == 0 {
 			continue
 		}
-		if strings.HasSuffix(gitDir, "git") {
-			// TODO: Figure out why we skip directories ending in "git".
-			continue
-		}
-		// try paths instead of url
-		repo, err := RepoFromPath(gitDir)
-		if err != nil {
-			ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
-			continue
-		}
-
-		err = func(repoPath string) error {
-			if !s.preserveTempDirs && strings.HasPrefix(repoPath, filepath.Join(os.TempDir(), "trufflehog")) {
-				defer os.RemoveAll(repoPath)
+		if !strings.HasSuffix(gitDir, "git") {
+			// try paths instead of url
+			repo, err := RepoFromPath(gitDir)
+			if err != nil {
+				ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
+				continue
 			}
 
-			return s.git.ScanRepo(ctx, repo, repoPath, s.scanOptions, chunksChan)
-		}(gitDir)
-		if err != nil {
-			ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
-			continue
+			err = func(repoPath string) error {
+				if strings.HasPrefix(repoPath, filepath.Join(os.TempDir(), "trufflehog")) {
+					defer os.RemoveAll(repoPath)
+				}
+
+				return s.git.ScanRepo(ctx, repo, repoPath, NewScanOptions(), chunksChan)
+			}(gitDir)
+			if err != nil {
+				ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
+				continue
+			}
 		}
 
 	}
+
+	ctx.Logger().V(1).Info("Git source finished scanning", "repo-count", totalRepos)
+	s.SetProgressComplete(
+		totalRepos, totalRepos,
+		fmt.Sprintf("Completed scanning source %s", s.name), "",
+	)
 	return nil
 }
 
@@ -378,6 +344,10 @@ func CloneRepoUsingSSH(ctx context.Context, gitUrl string, args ...string) (stri
 	return CloneRepo(ctx, userInfo, gitUrl, args...)
 }
 
+func (s *Git) CommitsScanned() int {
+	return s.metrics.commitsScanned
+}
+
 func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
 	if err := GitCmdCheck(); err != nil {
 		return err
@@ -410,6 +380,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 			break
 		}
 		depth++
+		s.metrics.commitsScanned++
 		logger.V(5).Info("scanning commit", "commit", commit.Hash)
 		for _, diff := range commit.Diffs {
 			if !scanOptions.Filter.Pass(diff.PathB) {

pkg/sources/source_manager.go

@@ -18,7 +18,7 @@ type handle int64
 
 // SourceInitFunc is a function that takes a source and job ID and returns an
 // initialized Source.
-type SourceInitFunc func(ctx context.Context, sourceID int64, jobID int64) (Source, error)
+type SourceInitFunc func(sourceID int64, jobID int64) (Source, error)
 
 type SourceManager struct {
 	api apiClient
@@ -50,12 +50,12 @@ func WithConcurrency(concurrency int) func(*SourceManager) {
 }
 
 // NewManager creates a new manager with the provided options.
-func NewManager(opts ...func(*SourceManager)) *SourceManager {
+func NewManager(outputChunks chan *Chunk, opts ...func(*SourceManager)) *SourceManager {
 	man := SourceManager{
 		// Default to the headless API. Can be overwritten by the WithAPI option.
 		api:          &headlessAPI{},
 		handles:      make(map[handle]SourceInitFunc),
-		outputChunks: make(chan *Chunk),
+		outputChunks: outputChunks,
 	}
 	for _, opt := range opts {
 		opt(&man)
@@ -77,7 +77,7 @@ func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.
 		return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
 	}
 	s.handles[handleID] = f
-	return handleID, nil
+	return 0, nil
 }
 
 // Run blocks until a resource is available to run the source, then synchronously runs it.
@@ -119,21 +119,6 @@ func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) error {
 	return nil
 }
 
-// Chunks returns the read only channel of all the chunks produced by all of
-// the sources managed by this manager.
-func (s *SourceManager) Chunks() <-chan *Chunk {
-	return s.outputChunks
-}
-
-// Wait blocks until all running sources are completed and returns the first
-// error encountered if any. It also closes the channel returned by Chunks().
-// The manager should not be reused after calling this method.
-func (s *SourceManager) Wait() error {
-	// TODO: Aggregate all errors from all sources.
-	defer close(s.outputChunks)
-	return s.pool.Wait()
-}
-
 // run is a helper method to sychronously run the source. It does not check for
 // acquired resources.
 func (s *SourceManager) run(ctx context.Context, handle handle) error {
@@ -145,7 +130,7 @@ func (s *SourceManager) run(ctx context.Context, handle handle) error {
 	if !ok {
 		return fmt.Errorf("unrecognized handle")
 	}
-	source, err := initFunc(ctx, jobID, int64(handle))
+	source, err := initFunc(jobID, int64(handle))
 	if err != nil {
 		return err
 	}