Add method to check for unverified.

Co-authored-by: Zachary Rice <zachary.rice@trufflesec.com>

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
 	github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
 	github.com/bradleyfalzon/ghinstallation/v2 v2.4.0
+	github.com/couchbase/gocb/v2 v2.6.3
 	github.com/crewjam/rfc5424 v0.1.0
 	github.com/denisenkom/go-mssqldb v0.12.3
 	github.com/envoyproxy/protoc-gen-validate v1.0.1
@@ -50,7 +51,7 @@ require (
 	github.com/sergi/go-diff v1.3.1
 	github.com/stretchr/testify v1.8.4
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
-	github.com/xanzy/go-gitlab v0.85.0
+	github.com/xanzy/go-gitlab v0.86.0
 	go.mongodb.org/mongo-driver v1.12.0
 	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.10.0
@@ -93,6 +94,7 @@ require (
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/connesc/cipherio v0.2.1 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
+  github.com/couchbase/gocbcore/v10 v10.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/docker/cli v23.0.5+incompatible // indirect

go.sum

@@ -115,6 +115,13 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/connesc/cipherio v0.2.1 h1:FGtpTPMbKNNWByNrr9aEBtaJtXjqOzkIXNYJp6OEycw=
 github.com/connesc/cipherio v0.2.1/go.mod h1:ukY0MWJDFnJEbXMQtOcn2VmTpRfzcTz4OoVrWGGJZcA=
+github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
+github.com/couchbase/gocb/v2 v2.6.3/go.mod h1:yF5F6BHTZ/ZowhEuZbySbXrlI4rHd1TIhm5azOaMbJU=
+github.com/couchbase/gocbcore/v10 v10.2.3 h1:PEkRSNSkKjUBXx82Ucr094+anoiCG5GleOOQZOHo6D4=
+github.com/couchbase/gocbcore/v10 v10.2.3/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
+github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
+github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4=
+github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
 github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
 github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@@ -421,6 +428,7 @@ github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt
 github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -431,6 +439,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
@@ -445,8 +454,8 @@ github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o
 github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
 github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
 github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
-github.com/xanzy/go-gitlab v0.85.0 h1:E/wjnsd/mM5kV6O9y5+i6zxjx+wfAwa97sgcT1ETNwk=
-github.com/xanzy/go-gitlab v0.85.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
+github.com/xanzy/go-gitlab v0.86.0 h1:jR8V9cK9jXRQDb46KOB20NCF3ksY09luaG0IfXE6p7w=
+github.com/xanzy/go-gitlab v0.86.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=

main.go

@@ -476,7 +476,7 @@ func run(state overseer.State) {
 		}
 	}
 	// asynchronously wait for scanning to finish and cleanup
-	go e.Finish(ctx)
+	go e.Finish(ctx, logFatal)
 
 	if !*jsonLegacy && !*jsonOut {
 		fmt.Fprintf(os.Stderr, "🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")

pkg/common/patterns.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 )
@@ -15,7 +16,11 @@ const RegexPattern = "0-9a-z"
 const AlphaNumPattern = "0-9a-zA-Z"
 const HexPattern = "0-9a-f"
 
-//Custom Regex functions
+type RegexState struct {
+	compiledRegex *regexp.Regexp
+}
+
+// Custom Regex functions
 func BuildRegex(pattern string, specialChar string, length int) string {
 	return fmt.Sprintf(`\b([%s%s]{%s})\b`, pattern, specialChar, strconv.Itoa(length))
 }
@@ -37,3 +42,30 @@ func RangeValidation(rangeInput string) bool {
 func ToUpperCase(input string) string {
 	return strings.ToUpper(input)
 }
+
+func (r RegexState) Matches(data []byte) []string {
+	matches := r.compiledRegex.FindAllStringSubmatch(string(data), -1)
+
+	res := make([]string, 0, len(matches))
+
+	// trim off spaces and different quote types ('").
+	for i := range matches {
+		res = append(res, strings.Trim(matches[i][1], `"' )`))
+	}
+
+	return res
+}
+
+// UsernameRegexCheck constructs an username usernameRegex pattern from a given pattern of excluded characters.
+func UsernameRegexCheck(pattern string) RegexState {
+	raw := fmt.Sprintf(`(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v]{4,40})\b`, pattern)
+
+	return RegexState{regexp.MustCompile(raw)}
+}
+
+// PasswordRegexCheck constructs an username usernameRegex pattern from a given pattern of excluded characters.
+func PasswordRegexCheck(pattern string) RegexState {
+	raw := fmt.Sprintf(`(?im)(?:pass|password)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v]{4,40})`, pattern)
+
+	return RegexState{regexp.MustCompile(raw)}
+}

pkg/common/patterns_test.go

@@ -0,0 +1,59 @@
+package common
+
+import (
+	"github.com/stretchr/testify/assert"
+	"regexp"
+	"testing"
+)
+
+const (
+	usernamePattern = `?()/\+=\s\n`
+	passwordPattern = `^<>;.*&|£\n\s`
+	usernameRegex   = `(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:?()/\+=\s\n]{4,40})\b`
+	passwordRegex   = `(?im)(?:pass|password)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:^<>;.*&|£\n\s]{4,40})`
+)
+
+func TestUsernameRegexCheck(t *testing.T) {
+	usernameRegexPat := UsernameRegexCheck(usernamePattern)
+
+	expectedRegexPattern := regexp.MustCompile(usernameRegex)
+
+	if usernameRegexPat.compiledRegex.String() != expectedRegexPattern.String() {
+		t.Errorf("\n got %v \n want %v", usernameRegexPat.compiledRegex, expectedRegexPattern)
+	}
+
+	testString := `username = "johnsmith123"
+                   username='johnsmith123'
+				   username:="johnsmith123"
+                   username = johnsmith123
+                   username=johnsmith123`
+
+	expectedStr := []string{"johnsmith123", "johnsmith123", "johnsmith123", "johnsmith123", "johnsmith123"}
+
+	usernameRegexMatches := usernameRegexPat.Matches([]byte(testString))
+
+	assert.Exactly(t, usernameRegexMatches, expectedStr)
+
+}
+
+func TestPasswordRegexCheck(t *testing.T) {
+	passwordRegexPat := PasswordRegexCheck(passwordPattern)
+
+	expectedRegexPattern := regexp.MustCompile(passwordRegex)
+	assert.Equal(t, passwordRegexPat.compiledRegex, expectedRegexPattern)
+
+	testString := `password = "johnsmith123$!"
+                   password='johnsmith123$!'
+				   password:="johnsmith123$!"
+                   password = johnsmith123$!
+                   password=johnsmith123$!
+				   PasswordAuthenticator(username, "johnsmith123$!")`
+
+	expectedStr := []string{"johnsmith123$!", "johnsmith123$!", "johnsmith123$!", "johnsmith123$!", "johnsmith123$!",
+		"johnsmith123$!"}
+
+	passwordRegexMatches := passwordRegexPat.Matches([]byte(testString))
+
+	assert.Exactly(t, passwordRegexMatches, expectedStr)
+
+}

pkg/detectors/couchbase/couchbase.go

@@ -0,0 +1,152 @@
+package couchbase
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"regexp"
+	"strings"
+	"time"
+	"unicode"
+
+	"github.com/couchbase/gocb/v2"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	connectionStringPat = regexp.MustCompile(`\bcb\.[a-z0-9]+\.cloud\.couchbase\.com\b`)
+	usernamePat         = `?()/\+=\s\n`
+	passwordPat         = `^<>;.*&|£\n\s`
+	//passwordPat         = regexp.MustCompile(`(?i)(?:pass|pwd)(?:.|[\n\r]){0,15}(\b[^<>;.*&|£\n\s]{8,100}$)`)
+	// passwordPat = regexp.MustCompile(`(?im)(?:pass|pwd)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:?()/\+=\s\n]{4,40})\b`)
+)
+
+func meetsCouchbasePasswordRequirements(password string) (string, bool) {
+	var hasLower, hasUpper, hasNumber, hasSpecialChar bool
+	for _, char := range password {
+		switch {
+		case unicode.IsLower(char):
+			hasLower = true
+		case unicode.IsUpper(char):
+			hasUpper = true
+		case unicode.IsNumber(char):
+			hasNumber = true
+		case unicode.IsPunct(char) || unicode.IsSymbol(char):
+			hasSpecialChar = true
+		}
+
+		if hasLower && hasUpper && hasNumber && hasSpecialChar {
+			return password, true
+		}
+	}
+
+	return "", false
+}
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"couchbase://", "couchbases://"}
+}
+
+// FromData will find and optionally verify Couchbase secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	connectionStringMatches := connectionStringPat.FindAllStringSubmatch(dataStr, -1)
+
+	// prepend 'couchbases://' to the connection string as the connection
+	// string format is couchbases://cb.stuff.cloud.couchbase.com but the
+	// cb.stuff.cloud.couchbase.com may be separated from the couchbases:// in codebases.
+	for i, connectionStringMatch := range connectionStringMatches {
+		connectionStringMatches[i][0] = "couchbases://" + connectionStringMatch[0]
+	}
+
+	usernameRegexState := common.UsernameRegexCheck(usernamePat)
+	usernameMatches := usernameRegexState.Matches(data)
+
+	passwordRegexState := common.PasswordRegexCheck(passwordPat)
+	passwordMatches := passwordRegexState.Matches(data)
+
+	for _, connectionStringMatch := range connectionStringMatches {
+		resConnectionStringMatch := strings.TrimSpace(connectionStringMatch[0])
+
+		for _, resUsernameMatch := range usernameMatches {
+
+			for _, resPasswordMatch := range passwordMatches {
+				_, metPasswordRequirements := meetsCouchbasePasswordRequirements(resPasswordMatch)
+
+				if !metPasswordRequirements {
+					continue
+				}
+
+				s1 := detectors.Result{
+					DetectorType: detectorspb.DetectorType_Couchbase,
+					Raw:          []byte(fmt.Sprintf("%s:%s@%s", resUsernameMatch, resPasswordMatch, resConnectionStringMatch)),
+				}
+
+				if verify {
+
+					options := gocb.ClusterOptions{
+						Authenticator: gocb.PasswordAuthenticator{
+							Username: resUsernameMatch,
+							Password: resPasswordMatch,
+						},
+					}
+
+					// Sets a pre-configured profile called "wan-development" to help avoid latency issues
+					// when accessing Capella from a different Wide Area Network
+					// or Availability Zone (e.g. your laptop).
+					if err := options.ApplyProfile(gocb.ClusterConfigProfileWanDevelopment); err != nil {
+						log.Fatal("apply profile err", err)
+					}
+
+					// Initialize the Connection
+					cluster, err := gocb.Connect(resConnectionStringMatch, options)
+					if err != nil {
+						continue
+					}
+
+					// We'll ping the KV nodes in our cluster.
+					pings, err := cluster.Ping(&gocb.PingOptions{
+						Timeout: time.Second * 5,
+					})
+
+					if err != nil {
+						continue
+					}
+
+					for _, ping := range pings.Services {
+						for _, pingEndpoint := range ping {
+							if pingEndpoint.State == gocb.PingStateOk {
+								s1.Verified = true
+								break
+							} else {
+								// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+								if detectors.IsKnownFalsePositive(resPasswordMatch, detectors.DefaultFalsePositives, true) {
+									continue
+								}
+							}
+						}
+					}
+				}
+
+				results = append(results, s1)
+			}
+		}
+	}
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Couchbase
+}

pkg/detectors/couchbase/couchbase_test.go

@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package couchbase
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCouchbase_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	endpoint := testSecrets.MustGetField("COUCHBASE_ENDPOINT")
+	username := testSecrets.MustGetField("COUCHBASE_USERNAME")
+	password := testSecrets.MustGetField("COUCHBASE_PASSWORD")
+	inactiveSecret := testSecrets.MustGetField("COUCHBASE_INACTIVE_PASSWORD")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, password)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Couchbase,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Couchbase,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Couchbase.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Couchbase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/detectors.go

@@ -39,6 +39,8 @@ type EndpointCustomizer interface {
 	DefaultEndpoint() string
 }
 
+// Result is a secret that has been found by a Detector and optionally verified.
+// It contains the raw secret identification data, redacted data, and extra data.
 type Result struct {
 	// DetectorType is the type of Detector.
 	DetectorType detectorspb.DetectorType
@@ -46,7 +48,15 @@ type Result struct {
 	DetectorName string
 	// DecoderType is the type of Decoder.
 	DecoderType detectorspb.DecoderType
-	Verified    bool
+	// Verified is true if the secret has been confirmed to be a valid secret.
+	Verified bool
+	// HasVerificationIndeterminate is true if the secret could NOT be verified.
+	// This is used to differentiate between secrets that are NOT verified and secrets that COULD NOT be verified.
+	// Ex: A 429 response from a service, or a timeout would set this to true.
+	HasVerificationIndeterminate bool
+	// VerificationIndeterminateError holds the error if the secret could NOT be verified.
+	// This is used specifically for cases that resulted in indeterminate resolution, such as a 429 or timeout.
+	VerificationIndeterminateError error
 	// Raw contains the raw secret identifier data. Prefer IDs over secrets since it is used for deduping after hashing.
 	Raw []byte
 	// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
@@ -59,6 +69,17 @@ type Result struct {
 	StructuredData *detectorspb.StructuredData
 }
 
+// IsVerified checks if a result is guaranteed to be verified.
+func (r Result) IsVerified() bool {
+	return r.Verified && !r.HasVerificationIndeterminate
+}
+
+// IsUnverified checks if a result is guaranteed to be unverified.
+// This is used to differentiate between secrets that are NOT verified and secrets that COULD NOT be verified.
+func (r Result) IsUnverified() bool {
+	return !r.Verified && !r.HasVerificationIndeterminate
+}
+
 type ResultWithMetadata struct {
 	// SourceMetadata contains source-specific contextual information.
 	SourceMetadata *source_metadatapb.MetaData

pkg/detectors/detectors_test.go

@@ -3,7 +3,9 @@
 
 package detectors
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestPrefixRegex(t *testing.T) {
 	tests := []struct {
@@ -37,3 +39,101 @@ func BenchmarkPrefixRegex(b *testing.B) {
 		PrefixRegex(kws)
 	}
 }
+
+func TestIsVerified(t *testing.T) {
+	tests := []struct {
+		name   string
+		result Result
+		want   bool
+	}{
+		{
+			name: "Verified true, HasVerificationIndeterminate false",
+			result: Result{
+				Verified:                     true,
+				HasVerificationIndeterminate: false,
+			},
+			want: true,
+		},
+		{
+			name: "Verified false, HasVerificationIndeterminate false",
+			result: Result{
+				Verified:                     false,
+				HasVerificationIndeterminate: false,
+			},
+			want: false,
+		},
+		{
+			name: "Verified true, HasVerificationIndeterminate true",
+			result: Result{
+				Verified:                     true,
+				HasVerificationIndeterminate: true,
+			},
+			want: false,
+		},
+		{
+			name: "Verified false, HasVerificationIndeterminate true",
+			result: Result{
+				Verified:                     false,
+				HasVerificationIndeterminate: true,
+			},
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.result.IsVerified(); got != tt.want {
+				t.Errorf("IsVerified() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIsUnverified(t *testing.T) {
+	tests := []struct {
+		name   string
+		result Result
+		want   bool
+	}{
+		{
+			name: "Verified true, HasVerificationIndeterminate false",
+			result: Result{
+				Verified:                     true,
+				HasVerificationIndeterminate: false,
+			},
+			want: false,
+		},
+		{
+			name: "Verified false, HasVerificationIndeterminate false",
+			result: Result{
+				Verified:                     false,
+				HasVerificationIndeterminate: false,
+			},
+			want: true,
+		},
+		{
+			name: "Verified true, HasVerificationIndeterminate true",
+			result: Result{
+				Verified:                     true,
+				HasVerificationIndeterminate: true,
+			},
+			want: false,
+		},
+		{
+			name: "Verified false, HasVerificationIndeterminate true",
+			result: Result{
+				Verified:                     false,
+				HasVerificationIndeterminate: true,
+			},
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.result.IsUnverified(); got != tt.want {
+				t.Errorf("IsUnverified() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/engine/circleci.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"fmt"
 	"runtime"
 
 	"github.com/go-errors/errors"
@@ -38,14 +39,13 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
 		return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
 	}
 
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := circleSource.Chunks(ctx, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "error scanning Circle CI")
+			return fmt.Errorf("error scanning CircleCI: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/engine/docker.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"fmt"
 	"runtime"
 
 	"github.com/go-errors/errors"
@@ -25,14 +26,13 @@ func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error {
 		return errors.WrapPrefix(err, "could not init docker source", 0)
 	}
 
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := dockerSource.Chunks(ctx, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "error scanning docker image")
+			return fmt.Errorf("error scanning docker image: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/engine/engine.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	ahocorasick "github.com/petar-dambovaliev/aho-corasick"
+	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -32,7 +33,7 @@ type Engine struct {
 	chunksScanned   uint64
 	bytesScanned    uint64
 	detectorAvgTime sync.Map
-	sourcesWg       sync.WaitGroup
+	sourcesWg       *errgroup.Group
 	workersWg       sync.WaitGroup
 	// filterUnverified is used to reduce the number of unverified results.
 	// If there are multiple unverified results for the same chunk for the same detector,
@@ -123,6 +124,11 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	}
 	ctx.Logger().V(2).Info("engine started", "workers", e.concurrency)
 
+	sourcesWg, egCtx := errgroup.WithContext(ctx)
+	sourcesWg.SetLimit(e.concurrency)
+	e.sourcesWg = sourcesWg
+	ctx.SetParent(egCtx)
+
 	if len(e.decoders) == 0 {
 		e.decoders = decoders.DefaultDecoders()
 	}
@@ -188,10 +194,14 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 // Finish waits for running sources to complete and workers to finish scanning
 // chunks before closing their respective channels. Once Finish is called, no
 // more sources may be scanned by the engine.
-func (e *Engine) Finish(ctx context.Context) {
+func (e *Engine) Finish(ctx context.Context, logFunc func(error, string, ...any)) {
 	defer common.RecoverWithExit(ctx)
 	// wait for the sources to finish putting chunks onto the chunks channel
-	e.sourcesWg.Wait()
+	sourceErr := e.sourcesWg.Wait()
+	if sourceErr != nil {
+		logFunc(sourceErr, "error occurred while collecting chunks")
+	}
+
 	close(e.chunks)
 	// wait for the workers to finish processing all of the chunks and putting
 	// results onto the results channel

pkg/engine/filesystem.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"fmt"
 	"runtime"
 
 	"github.com/go-errors/errors"
@@ -37,14 +38,13 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
 		return errors.WrapPrefix(err, "could not init filesystem source", 0)
 	}
 	fileSystemSource.WithFilter(c.Filter)
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := fileSystemSource.Chunks(ctx, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "error scanning filesystem")
+			return fmt.Errorf("error scanning filesystem: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/engine/gcs.go

@@ -54,14 +54,13 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
 		return fmt.Errorf("failed to initialize GCS source: %w", err)
 	}
 
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		if err := source.Chunks(ctx, e.ChunksChan()); err != nil {
-			ctx.Logger().Error(err, "could not scan GCS")
+			return fmt.Errorf("could not scan GCS: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }
 

pkg/engine/gcs_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
@@ -55,12 +56,30 @@ func TestScanGCS(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			e := &Engine{}
-			err := e.ScanGCS(context.Background(), test.gcsConfig)
+			ctx, cancel := context.WithCancel(context.TODO())
+			defer cancel()
+
+			e := Start(ctx,
+				WithConcurrency(1),
+				WithDecoders(decoders.DefaultDecoders()...),
+				WithDetectors(false, DefaultDetectors()...),
+			)
+			go func() {
+				resultCount := 0
+				for range e.ResultsChan() {
+					resultCount++
+				}
+			}()
+
+			err := e.ScanGCS(ctx, test.gcsConfig)
 			if err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {
 				t.Errorf("ScanGCS() got: %v, want: %v", err, nil)
 				return
 			}
+			logFatalFunc := func(_ error, _ string, _ ...any) {
+				t.Fatalf("error logging function should not have been called")
+			}
+			e.Finish(ctx, logFatalFunc)
 
 			if err == nil && test.wantErr {
 				t.Errorf("ScanGCS() got: %v, want: %v", err, "error")

pkg/engine/git.go

@@ -66,14 +66,13 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 		"source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),
 		"source_name", "git",
 	)
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "could not scan repo")
+			return fmt.Errorf("could not scan repo: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/engine/git_test.go

@@ -53,39 +53,45 @@ func TestGitEngine(t *testing.T) {
 			base:   "2f251b8c1e72135a375b659951097ec7749d4af9",
 		},
 	} {
-		e := Start(ctx,
-			WithConcurrency(1),
-			WithDecoders(decoders.DefaultDecoders()...),
-			WithDetectors(false, DefaultDetectors()...),
-		)
-		cfg := sources.GitConfig{
-			RepoPath: path,
-			HeadRef:  tTest.branch,
-			BaseRef:  tTest.base,
-			MaxDepth: tTest.maxDepth,
-			Filter:   tTest.filter,
-		}
-		if err := e.ScanGit(ctx, cfg); err != nil {
-			return
-		}
-		go e.Finish(ctx)
-		resultCount := 0
-		for result := range e.ResultsChan() {
-			switch meta := result.SourceMetadata.GetData().(type) {
-			case *source_metadatapb.MetaData_Git:
-				if tTest.expected[meta.Git.Commit].B != string(result.Raw) {
-					t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit].B)
-				}
-				if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
-					t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
-				}
+		t.Run(tName, func(t *testing.T) {
+			e := Start(ctx,
+				WithConcurrency(1),
+				WithDecoders(decoders.DefaultDecoders()...),
+				WithDetectors(false, DefaultDetectors()...),
+			)
+			cfg := sources.GitConfig{
+				RepoPath: path,
+				HeadRef:  tTest.branch,
+				BaseRef:  tTest.base,
+				MaxDepth: tTest.maxDepth,
+				Filter:   tTest.filter,
+			}
+			if err := e.ScanGit(ctx, cfg); err != nil {
+				return
 			}
-			resultCount++
 
-		}
-		if resultCount != len(tTest.expected) {
-			t.Errorf("%s: unexpected number of results. Got: %d, Expected: %d", tName, resultCount, len(tTest.expected))
-		}
+			logFatalFunc := func(_ error, _ string, _ ...any) {
+				t.Fatalf("error logging function should not have been called")
+			}
+			go e.Finish(ctx, logFatalFunc)
+			resultCount := 0
+			for result := range e.ResultsChan() {
+				switch meta := result.SourceMetadata.GetData().(type) {
+				case *source_metadatapb.MetaData_Git:
+					if tTest.expected[meta.Git.Commit].B != string(result.Raw) {
+						t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit].B)
+					}
+					if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
+						t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
+					}
+				}
+				resultCount++
+
+			}
+			if resultCount != len(tTest.expected) {
+				t.Errorf("%s: unexpected number of results. Got: %d, Expected: %d", tName, resultCount, len(tTest.expected))
+			}
+		})
 	}
 }
 
@@ -124,5 +130,8 @@ func BenchmarkGitEngine(b *testing.B) {
 			return
 		}
 	}
-	e.Finish(ctx)
+	logFatalFunc := func(_ error, _ string, _ ...any) {
+		b.Fatalf("error logging function should not have been called")
+	}
+	e.Finish(ctx, logFatalFunc)
 }

pkg/engine/github.go

@@ -1,6 +1,8 @@
 package engine
 
 import (
+	"fmt"
+
 	gogit "github.com/go-git/go-git/v5"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
@@ -58,14 +60,13 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
 	scanOptions := git.NewScanOptions(opts...)
 	source.WithScanOptions(scanOptions)
 
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := source.Chunks(ctx, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "could not scan github")
+			return fmt.Errorf("could not scan github: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/engine/gitlab.go

@@ -63,14 +63,13 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
 	}
 	gitlabSource.WithScanOptions(scanOptions)
 
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := gitlabSource.Chunks(ctx, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "error scanning GitLab")
+			return fmt.Errorf("error scanning GitLab: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/engine/s3.go

@@ -65,14 +65,13 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
 		return errors.WrapPrefix(err, "failed to init S3 source", 0)
 	}
 
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := s3Source.Chunks(ctx, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "error scanning S3")
+			return fmt.Errorf("error scanning S3: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/engine/syslog.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/go-errors/errors"
@@ -53,14 +54,13 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
 		return err
 	}
 
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := source.Chunks(ctx, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "could not scan syslog")
+			return fmt.Errorf("could not scan syslog: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/pb/detectorspb/detectors.pb.go

@@ -991,6 +991,7 @@ const (
 	DetectorType_Aiven                         DetectorType = 917
 	DetectorType_Prefect                       DetectorType = 918
 	DetectorType_Docusign                      DetectorType = 919
+	DetectorType_Couchbase                     DetectorType = 920
 )
 
 // Enum value maps for DetectorType.
@@ -1912,6 +1913,7 @@ var (
 		917: "Aiven",
 		918: "Prefect",
 		919: "Docusign",
+		920: "Couchbase",
 	}
 	DetectorType_value = map[string]int32{
 		"Alibaba":                       0,
@@ -2830,6 +2832,7 @@ var (
 		"Aiven":                         917,
 		"Prefect":                       918,
 		"Docusign":                      919,
+		"Couchbase":                     920,
 	}
 )
 
@@ -3194,7 +3197,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x0a, 0x0b, 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,
 	0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c,
 	0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10,
-	0x02, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xf9, 0x72, 0x0a,
+	0x02, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x89, 0x73, 0x0a,
 	0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,
 	0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d,
 	0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a,
@@ -4114,7 +4117,8 @@ var file_detectors_proto_rawDesc = []byte{
 	0x0b, 0x4e, 0x75, 0x47, 0x65, 0x74, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x10, 0x94, 0x07, 0x12,
 	0x0a, 0x0a, 0x05, 0x41, 0x69, 0x76, 0x65, 0x6e, 0x10, 0x95, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x50,
 	0x72, 0x65, 0x66, 0x65, 0x63, 0x74, 0x10, 0x96, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x44, 0x6f, 0x63,
-	0x75, 0x73, 0x69, 0x67, 0x6e, 0x10, 0x97, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68,
+	0x75, 0x73, 0x69, 0x67, 0x6e, 0x10, 0x97, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x43, 0x6f, 0x75, 0x63,
+	0x68, 0x62, 0x61, 0x73, 0x65, 0x10, 0x98, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68,
 	0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,
 	0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,
 	0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65,

pkg/sources/git/git.go

@@ -739,7 +739,7 @@ func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) (
 	var path string
 	switch {
 	case uri.User != nil:
-		ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri)
+		ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri.Redacted())
 		password, ok := uri.User.Password()
 		if !ok {
 			return "", true, fmt.Errorf("password must be included in Git repo URL when username is provided")
@@ -777,7 +777,7 @@ func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) {
 		remote = true
 		switch {
 		case uri.User != nil:
-			ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri)
+			ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri.Redacted())
 			password, ok := uri.User.Password()
 			if !ok {
 				return "", remote, fmt.Errorf("password must be included in Git repo URL when username is provided")

pkg/sources/s3/s3.go

@@ -197,7 +197,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			})
 
 		if err != nil {
-			s.log.Error(err, "could not list objects in s3 bucket", "bucket", bucket)
+			return fmt.Errorf(
+				"could not list objects in s3 bucket: bucket %s: %w",
+				bucket,
+				err)
 		}
 	}
 	s.SetProgressComplete(len(bucketsToScan), len(bucketsToScan), fmt.Sprintf("Completed scanning source %s. %d objects scanned.", s.name, objectCount), "")

proto/detectors.proto

@@ -928,6 +928,7 @@ enum DetectorType {
   Aiven = 917;
   Prefect = 918;
   Docusign = 919;
+  Couchbase = 920;
 }
 
 message Result {