remove mutex usage around failFast.

* updating sendgrid detector to use tri-state verification

main.go

@@ -53,6 +53,7 @@ var (
 	printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()
 	noUpdate             = cli.Flag("no-update", "Don't check for updates.").Bool()
 	fail                 = cli.Flag("fail", "Exit with code 183 if results are found.").Bool()
+	failFast             = cli.Flag("fail-fast", "Exit with code 1 as soon as a single result is found.").Bool()
 	verifiers            = cli.Flag("verifier", "Set custom verification endpoints.").StringMap()
 	archiveMaxSize       = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()
 	archiveMaxDepth      = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
@@ -105,11 +106,11 @@ var (
 	filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
 	filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
 
-	s3Scan    = cli.Command("s3", "Find credentials in S3 buckets.")
-	s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
-	s3ScanRoleArns     = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
-	s3ScanSecret       = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
-	s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
+	s3Scan              = cli.Command("s3", "Find credentials in S3 buckets.")
+	s3ScanKey           = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
+	s3ScanRoleArns      = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
+	s3ScanSecret        = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
+	s3ScanSessionToken  = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
 	s3ScanCloudEnv      = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
 	s3ScanBuckets       = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
 	s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()
@@ -366,6 +367,7 @@ func run(state overseer.State) {
 		engine.WithOnlyVerified(*onlyVerified),
 		engine.WithPrintAvgDetectorTime(*printAvgDetectorTime),
 		engine.WithPrinter(printer),
+		engine.WithFailFast(*failFast),
 	)
 	if err != nil {
 		logFatal(err, "error initializing engine")
@@ -469,7 +471,7 @@ func run(state overseer.State) {
 			Secret:        *s3ScanSecret,
 			SessionToken:  *s3ScanSessionToken,
 			Buckets:       *s3ScanBuckets,
-			Roles:	       *s3ScanRoleArns,
+			Roles:         *s3ScanRoleArns,
 			CloudCred:     *s3ScanCloudEnv,
 			MaxObjectSize: int64(*s3ScanMaxObjectSize),
 		}
@@ -533,6 +535,9 @@ func run(state overseer.State) {
 	if err = e.Finish(ctx); err != nil {
 		logFatal(err, "engine failed to finish execution")
 	}
+	if e.IsTerminated() {
+		os.Exit(1)
+	}
 
 	metrics := e.GetMetrics()
 	// Print results.

pkg/detectors/sendgrid/sendgrid.go

@@ -13,13 +13,15 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
 
-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	defaultClient = common.SaneHttpClient()
 
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendgrid"}) + `(SG\.[\w\-_]{20,24}\.[\w\-_]{39,50})\b`)
 )
@@ -42,7 +44,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 		res := strings.TrimSpace(match[1])
 
-		s := detectors.Result{
+		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_SendGrid,
 			Raw:          []byte(res),
 		}
@@ -52,6 +54,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			// 403 will be issued if the scope is wrong but the key is correct
 			baseURL := "https://api.sendgrid.com/v3/templates"
 
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
+
 			// test `read_user` scope
 			req, err := http.NewRequestWithContext(ctx, "GET", baseURL, nil)
 			if err != nil {
@@ -67,16 +74,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				// 403 means good key but not the right scope
 				// 401 is bad key
 				if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
-					s.Verified = true
+					s1.Verified = true
+				} else if res.StatusCode != http.StatusUnauthorized {
+					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 				}
 			}
 		}
 
-		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
 			continue
 		}
 
-		results = append(results, s)
+		results = append(results, s1)
 	}
 
 	return

pkg/detectors/sendgrid/sendgrid_test.go

@@ -9,7 +9,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -18,7 +19,7 @@ import (
 func TestSendgrid_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
@@ -30,11 +31,12 @@ func TestSendgrid_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
@@ -50,7 +52,8 @@ func TestSendgrid_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -66,7 +69,8 @@ func TestSendgrid_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "not found",
@@ -76,22 +80,54 @@ func TestSendgrid_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:    nil,
-			wantErr: false,
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SendGrid,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SendGrid,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Sendgrid.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			for i := range got {
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
 				t.Errorf("Sendgrid.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/engine/engine.go

@@ -51,6 +51,53 @@ type Printer interface {
 	Print(ctx context.Context, r *detectors.ResultWithMetadata) error
 }
 
+// terminationSignal provides a mechanism to safely signal a termination
+// event across different goroutines. It uses a combination of a condition variable
+// and a mutex to enable safe signaling and waiting, ensuring that the signaling
+// is consistent and that there are no race conditions when checking or updating
+// the status of the termination.
+type terminationSignal struct {
+	// isTerminated indicates the signal state (true if the engine is terminated).
+	isTerminated bool
+	// isFailFast indicates if the engine is in fail-fast mode.
+	isFailFast bool
+	// mutex is used to protect access to isFailFast and to synchronize
+	// the condition variable.
+	mutex sync.Mutex
+	// failFastCondition is a condition variable that's signaled when
+	// a fail-fast event occurs. It's used in Finish to wait for or
+	// react to this signal.
+	failFastCondition *sync.Cond
+}
+
+// newTerminationSignal initializes a new terminationSignal.
+// The failFast argument determines the initial state.
+func newTerminationSignal(failFast bool) *terminationSignal {
+	ts := &terminationSignal{isFailFast: failFast}
+	// Initialize the condition variable with the mutex.
+	ts.failFastCondition = sync.NewCond(&ts.mutex)
+	return ts
+}
+
+// signalFinish signals that the engine has finished its work.
+// It uses the condition variable to notify any waiting goroutines
+// (e.g., the Finish method).
+func (t *terminationSignal) signalFinish() {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+	t.isTerminated = true
+
+	t.failFastCondition.Signal() // Signal the condition to unblock Finish.
+}
+
+func (t *terminationSignal) wait() {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+	for !t.isTerminated {
+		t.failFastCondition.Wait()
+	}
+}
+
 type Engine struct {
 	// CLI flags.
 	concurrency uint8
@@ -74,6 +121,11 @@ type Engine struct {
 	workersWg            sync.WaitGroup
 	wgDetectorWorkers    sync.WaitGroup
 	WgNotifier           sync.WaitGroup
+	// termSignal is used to signal the engine to terminate.
+	// It is used to implement the fail-fast feature,
+	// where the engine will terminate scanning as soon as
+	// the first potential secret is detected.
+	termSignal *terminationSignal
 
 	// Runtime information.
 	metrics runtimeMetrics
@@ -169,6 +221,19 @@ func WithPrinter(printer Printer) EngineOption {
 	}
 }
 
+// WithFailFast returns an EngineOption that configures the engine's termination behavior.
+// If 'failFast' is enabled, the engine will terminate scanning upon detecting the first potential secret.
+// This mode is valuable in contexts like CI/CD pipelines or preliminary scans where
+// immediate feedback is more important than a comprehensive review. In the non-failFast mode,
+// the engine continues its operation without waiting for a result.
+// The boolean parameter 'failFast' dictates the termination behavior.
+func WithFailFast(failFast bool) EngineOption {
+	return func(e *Engine) {
+		// Configure the termination signal with the desired mode (failFast or not).
+		e.termSignal = newTerminationSignal(failFast)
+	}
+}
+
 func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors.Detector) []detectors.Detector {
 	var output []detectors.Detector
 	for _, detector := range input {
@@ -276,6 +341,7 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
 		dedupeCache:          cache,
 		printer:              new(output.PlainPrinter), // default printer
 		metrics:              runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}},
+		termSignal:           &terminationSignal{},
 	}
 
 	for _, option := range options {
@@ -367,22 +433,15 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
 		}()
 	}
 
-	// We want 1/4th of the notifier workers as the number of scanner workers.
-	const notifierWorkerRatio = 4
-	maxNotifierWorkers := 1
-	if numWorkers := e.concurrency / notifierWorkerRatio; numWorkers > 0 {
-		maxNotifierWorkers = int(numWorkers)
-	}
-	ctx.Logger().V(2).Info("starting notifier workers", "count", maxNotifierWorkers)
-	for worker := 0; worker < maxNotifierWorkers; worker++ {
-		e.WgNotifier.Add(1)
-		go func() {
-			ctx := context.WithValue(ctx, "notifier_worker_id", common.RandomID(5))
-			defer common.Recover(ctx)
-			defer e.WgNotifier.Done()
-			e.notifyResults(ctx)
-		}()
-	}
+	// We want only one notifier worker.
+	ctx.Logger().V(2).Info("starting notifier worker")
+	e.WgNotifier.Add(1)
+	go func() {
+		ctx := context.WithValue(ctx, "notifier_worker_id", common.RandomID(5))
+		defer common.Recover(ctx)
+		defer e.WgNotifier.Done()
+		e.notifyResults(ctx)
+	}()
 
 	return e, nil
 }
@@ -392,6 +451,13 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
 // more sources may be scanned by the engine.
 func (e *Engine) Finish(ctx context.Context) error {
 	defer common.RecoverWithExit(ctx)
+
+	// Handle the fail-fast termination signal.
+	if e.termSignal.isFailFast {
+		e.termSignal.wait()
+		return fmt.Errorf("terminated early due to fail-fast flag after printing the first result")
+	}
+
 	// Wait for the sources to finish putting chunks onto the chunks channel.
 	err := e.sourceManager.Wait()
 
@@ -407,6 +473,15 @@ func (e *Engine) Finish(ctx context.Context) error {
 	return err
 }
 
+// IsTerminated checks if the engine received a termination signal.
+// This can be used, for example, to determine if the engine
+// terminated early due to the fail-fast feature being activated.
+func (e *Engine) IsTerminated() bool {
+	e.termSignal.mutex.Lock()
+	defer e.termSignal.mutex.Unlock()
+	return e.termSignal.isTerminated
+}
+
 func (e *Engine) ChunksChan() <-chan *sources.Chunk {
 	return e.sourceManager.Chunks()
 }
@@ -559,27 +634,51 @@ func (e *Engine) processResult(data detectableChunk, res detectors.Result) {
 }
 
 func (e *Engine) notifyResults(ctx context.Context) {
+	if e.termSignal.isFailFast {
+		e.processResultsWithFailFast(ctx)
+	} else {
+		e.processRegularResults(ctx)
+	}
+}
+
+func (e *Engine) processResultsWithFailFast(ctx context.Context) {
 	for r := range e.ResultsChan() {
-		if e.onlyVerified && !r.Verified {
-			continue
-		}
-		atomic.AddUint32(&e.numFoundResults, 1)
+		e.processSingleResult(ctx, &r)
 
-		key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
-		if _, ok := e.dedupeCache.Get(key); ok {
-			continue
-		}
-		e.dedupeCache.Add(key, struct{}{})
+		// If fail-fast is enabled, signal finish after processing the first result.
+		e.termSignal.signalFinish()
+		return
+	}
+}
 
-		if r.Verified {
-			atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)
-		} else {
-			atomic.AddUint64(&e.metrics.UnverifiedSecretsFound, 1)
-		}
+func (e *Engine) processRegularResults(ctx context.Context) {
+	for r := range e.ResultsChan() {
+		e.processSingleResult(ctx, &r)
+	}
+}
 
-		if err := e.printer.Print(ctx, &r); err != nil {
-			ctx.Logger().Error(err, "error printing result")
-		}
+func (e *Engine) processSingleResult(ctx context.Context, r *detectors.ResultWithMetadata) {
+	if e.onlyVerified && !r.Verified {
+		return
+	}
+
+	atomic.AddUint32(&e.numFoundResults, 1)
+
+	key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
+	if _, ok := e.dedupeCache.Get(key); ok {
+		return
+	}
+
+	e.dedupeCache.Add(key, struct{}{})
+
+	if r.Verified {
+		atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)
+	} else {
+		atomic.AddUint64(&e.metrics.UnverifiedSecretsFound, 1)
+	}
+
+	if err := e.printer.Print(ctx, r); err != nil {
+		ctx.Logger().Error(err, "error printing result")
 	}
 }
 

pkg/output/github_actions.go

@@ -4,7 +4,6 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
-	"sync"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,7 +13,7 @@ import (
 var dedupeCache = make(map[string]struct{})
 
 // GitHubActionsPrinter is a printer that prints results in GitHub Actions format.
-type GitHubActionsPrinter struct{ mu sync.Mutex }
+type GitHubActionsPrinter struct{}
 
 func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
 	out := gitHubActionsOutputFormat{
@@ -57,8 +56,6 @@ func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithM
 	}
 	dedupeCache[key] = struct{}{}
 
-	p.mu.Lock()
-	defer p.mu.Unlock()
 	message := fmt.Sprintf("Found %s %s result 🐷🔑\n", verifiedStatus, out.DetectorType)
 	if r.Result.DecoderType != detectorspb.DecoderType_PLAIN {
 		message = fmt.Sprintf("Found %s %s result with %s encoding 🐷🔑\n", verifiedStatus, out.DetectorType, out.DecoderType)

pkg/output/json.go

@@ -3,7 +3,6 @@ package output
 import (
 	"encoding/json"
 	"fmt"
-	"sync"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,7 +12,7 @@ import (
 )
 
 // JSONPrinter is a printer that prints results in JSON format.
-type JSONPrinter struct{ mu sync.Mutex }
+type JSONPrinter struct{}
 
 func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
 	v := &struct {
@@ -62,8 +61,6 @@ func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
 		return fmt.Errorf("could not marshal result: %w", err)
 	}
 
-	p.mu.Lock()
 	fmt.Println(string(out))
-	p.mu.Unlock()
 	return nil
 }

pkg/output/legacy_json.go

@@ -7,7 +7,6 @@ import (
 	"net/url"
 	"os"
 	"strings"
-	"sync"
 
 	gogit "github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
@@ -21,7 +20,7 @@ import (
 )
 
 // LegacyJSONPrinter is a printer that prints results in legacy JSON format for backwards compatibility.
-type LegacyJSONPrinter struct{ mu sync.Mutex }
+type LegacyJSONPrinter struct{}
 
 func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMetadata) error {
 	var repo string
@@ -54,9 +53,7 @@ func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMe
 		return fmt.Errorf("could not marshal result: %w", err)
 	}
 
-	p.mu.Lock()
 	fmt.Println(string(out))
-	p.mu.Unlock()
 	return nil
 }
 

pkg/output/plain.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"sort"
 	"strings"
-	"sync"
 
 	"github.com/fatih/color"
 	"golang.org/x/text/cases"
@@ -23,7 +22,7 @@ var (
 )
 
 // PlainPrinter is a printer that prints results in plain text format.
-type PlainPrinter struct{ mu sync.Mutex }
+type PlainPrinter struct{}
 
 func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
 	out := outputFormat{
@@ -40,8 +39,6 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
 	}
 
 	printer := greenPrinter
-	p.mu.Lock()
-	defer p.mu.Unlock()
 
 	if out.Verified {
 		yellowPrinter.Print("Found verified result 🐷🔑\n")

pkg/sources/circleci/circleci.go

@@ -75,7 +75,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	projects, err := s.projects(ctx)
 	if err != nil {
 		return fmt.Errorf("error getting projects: %w", err)

pkg/sources/docker/docker.go

@@ -85,7 +85,7 @@ type layerInfo struct {
 }
 
 // Chunks emits data over a channel that is decoded and scanned for secrets.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	ctx = context.WithValues(ctx, "source_type", s.Type(), "source_name", s.name)
 
 	workers := new(errgroup.Group)

pkg/sources/filesystem/filesystem.go

@@ -77,7 +77,7 @@ func (s *Source) WithFilter(filter *common.Filter) {
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	for i, path := range s.paths {
 		logger := ctx.Logger().WithValues("path", path)
 		if common.IsDone(ctx) {

pkg/sources/gcs/gcs.go

@@ -248,7 +248,7 @@ func (s *Source) enumerate(ctx context.Context) error {
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	persistableCache := s.setupCache(ctx)
 
 	objectCh, err := s.gcsManager.ListObjects(ctx)

pkg/sources/git/git.go

@@ -153,7 +153,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	if err := s.scanRepos(ctx, chunksChan); err != nil {
 		return err
 	}

pkg/sources/github/github.go

@@ -413,7 +413,7 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	apiEndpoint := s.conn.Endpoint
 	if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) {
 		apiEndpoint = "https://api.github.com"

pkg/sources/gitlab/gitlab.go

@@ -139,7 +139,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	// Start client.
 	apiClient, err := s.newClient()
 	if err != nil {

pkg/sources/s3/s3.go

@@ -232,7 +232,7 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
 		s.scanBuckets(c, defaultRegionClient, roleArn, buckets, chunksChan)
 	}

pkg/sources/source_manager_test.go

@@ -6,10 +6,11 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"google.golang.org/protobuf/types/known/anypb"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"google.golang.org/protobuf/types/known/anypb"
 )
 
 // DummySource implements Source and is used for testing a SourceManager.
@@ -31,7 +32,7 @@ func (d *DummySource) GetProgress() *Progress { return nil }
 
 // Interface to easily test different chunking methods.
 type chunker interface {
-	Chunks(context.Context, chan *Chunk) error
+	Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error
 	ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
 	Enumerate(ctx context.Context, reporter UnitReporter) error
 }
@@ -42,7 +43,7 @@ type counterChunker struct {
 	count        int
 }
 
-func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
+func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
 	for i := 0; i < c.count; i++ {
 		select {
 		case ch <- &Chunk{Data: []byte{c.chunkCounter}}:
@@ -75,9 +76,9 @@ func (c *counterChunker) ChunkUnit(ctx context.Context, unit SourceUnit, reporte
 // Chunk method that always returns an error.
 type errorChunker struct{ error }
 
-func (c errorChunker) Chunks(context.Context, chan *Chunk) error                  { return c }
-func (c errorChunker) Enumerate(context.Context, UnitReporter) error              { return c }
-func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
+func (c errorChunker) Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error { return c }
+func (c errorChunker) Enumerate(context.Context, UnitReporter) error                { return c }
+func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error   { return c }
 
 // enrollDummy is a helper function to enroll a DummySource with a SourceManager.
 func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {
@@ -176,7 +177,7 @@ type unitChunk struct {
 
 type unitChunker struct{ steps []unitChunk }
 
-func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
+func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
 	for _, step := range c.steps {
 		if step.err != "" {
 			continue
@@ -294,7 +295,9 @@ type callbackChunker struct {
 	cb func(context.Context, chan *Chunk) error
 }
 
-func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk) error           { return c.cb(ctx, ch) }
+func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
+	return c.cb(ctx, ch)
+}
 func (c callbackChunker) Enumerate(context.Context, UnitReporter) error              { return nil }
 func (c callbackChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return nil }
 

pkg/sources/sources.go

@@ -30,6 +30,16 @@ type Chunk struct {
 	Verify bool
 }
 
+// ChunkingTarget specifies criteria for a targeted chunking process.
+// Instead of collecting data indiscriminately, this struct allows the caller
+// to specify particular subsets of data they're interested in. This becomes
+// especially useful when one needs to verify or recheck specific data points
+// without processing the entire dataset.
+type ChunkingTarget struct {
+	// QueryCriteria represents specific parameters or conditions to target the chunking process.
+	QueryCriteria source_metadatapb.MetaData
+}
+
 // Source defines the interface required to implement a source chunker.
 type Source interface {
 	// Type returns the source type, used for matching against configuration and jobs.
@@ -40,8 +50,12 @@ type Source interface {
 	JobID() int64
 	// Init initializes the source.
 	Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error
-	// Chunks emits data over a channel that is decoded and scanned for secrets.
-	Chunks(ctx context.Context, chunksChan chan *Chunk) error
+	// Chunks emits data over a channel which is then decoded and scanned for secrets.
+	// By default, data is obtained indiscriminately. However, by providing one or more
+	// ChunkingTarget parameters, the caller can direct the function to retrieve
+	// specific chunks of data. This targeted approach allows for efficient and
+	// intentional data processing, beneficial when verifying or rechecking specific data points.
+	Chunks(ctx context.Context, chunksChan chan *Chunk, targets ...ChunkingTarget) error
 	// GetProgress is the completion progress (percentage) for Scanned Source.
 	GetProgress() *Progress
 }

pkg/sources/syslog/syslog.go

@@ -182,7 +182,7 @@ func (s *Source) verifyConnectionConfig() error {
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	switch {
 	case s.conn.TlsCert != nilString || s.conn.TlsKey != nilString:
 		cert, err := tls.X509KeyPair([]byte(s.conn.TlsCert), []byte(s.conn.TlsKey))