updating secret to s1 to match convention

* add client, add known false positive check

* fix idiosyncracies

* cleanup

* cleanup comment

* add unexpected error test case

---------

Co-authored-by: Dustin Decker <dustin@trufflesec.com>

main.go

@@ -105,11 +105,11 @@ var (
 	filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
 	filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
 
-	s3Scan    = cli.Command("s3", "Find credentials in S3 buckets.")
-	s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
-	s3ScanRoleArns     = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
-	s3ScanSecret       = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
-	s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
+	s3Scan              = cli.Command("s3", "Find credentials in S3 buckets.")
+	s3ScanKey           = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
+	s3ScanRoleArns      = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
+	s3ScanSecret        = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
+	s3ScanSessionToken  = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
 	s3ScanCloudEnv      = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
 	s3ScanBuckets       = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
 	s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()
@@ -354,6 +354,10 @@ func run(state overseer.State) {
 		printer = new(output.PlainPrinter)
 	}
 
+	if !*jsonLegacy && !*jsonOut {
+		fmt.Fprintf(os.Stderr, "🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
+	}
+
 	e, err := engine.Start(ctx,
 		engine.WithConcurrency(uint8(*concurrency)),
 		engine.WithDecoders(decoders.DefaultDecoders()...),
@@ -469,7 +473,7 @@ func run(state overseer.State) {
 			Secret:        *s3ScanSecret,
 			SessionToken:  *s3ScanSessionToken,
 			Buckets:       *s3ScanBuckets,
-			Roles:	       *s3ScanRoleArns,
+			Roles:         *s3ScanRoleArns,
 			CloudCred:     *s3ScanCloudEnv,
 			MaxObjectSize: int64(*s3ScanMaxObjectSize),
 		}
@@ -525,10 +529,6 @@ func run(state overseer.State) {
 		}
 	}
 
-	if !*jsonLegacy && !*jsonOut {
-		fmt.Fprintf(os.Stderr, "🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
-	}
-
 	// Wait for all workers to finish.
 	if err = e.Finish(ctx); err != nil {
 		logFatal(err, "engine failed to finish execution")

pkg/detectors/aws/aws.go

@@ -208,11 +208,10 @@ func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch str
 
 	res, err := client.Do(req)
 	if err == nil {
-
+		defer res.Body.Close()
 		if res.StatusCode >= 200 && res.StatusCode < 300 {
 			identityInfo := identityRes{}
 			err := json.NewDecoder(res.Body).Decode(&identityInfo)
-			res.Body.Close()
 			if err == nil {
 				extraData := map[string]string{
 					"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
@@ -244,7 +243,6 @@ func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch str
 			}
 			var body awsErrorResponseBody
 			err = json.NewDecoder(res.Body).Decode(&body)
-			res.Body.Close()
 			if err == nil {
 				// All instances of the code I've seen in the wild are PascalCased but this check is
 				// case-insensitive out of an abundance of caution

pkg/detectors/gitlab/gitlab.go

@@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			resMatch = keyString[len(keyString)-1]
 		}
 
-		secret := detectors.Result{
+		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Gitlab,
 			Raw:          []byte(match[1]),
 		}
@@ -84,26 +84,26 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					// 401 is bad key
 					switch res.StatusCode {
 					case http.StatusOK:
-						secret.Verified = true
+						s1.Verified = true
 					case http.StatusForbidden:
 						// Good key but not the right scope
-						secret.Verified = true
+						s1.Verified = true
 					case http.StatusUnauthorized:
 						// Nothing to do; zero values are the ones we want
 					default:
-						secret.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+						s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 					}
 				} else {
-					secret.VerificationError = err
+					s1.VerificationError = err
 				}
 			}
 		}
 
-		if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
+		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
 			continue
 		}
 
-		results = append(results, secret)
+		results = append(results, s1)
 	}
 
 	return results, nil

pkg/detectors/gitlab/gitlab_test.go

@@ -6,11 +6,12 @@ package gitlab
 import (
 	"context"
 	"fmt"
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"

pkg/detectors/gitlab/gitlabv2_test.go

@@ -1,115 +0,0 @@
-package gitlab
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestGitlabv2_FromChunk(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
-	if err != nil {
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("GITLABV2")
-	secretInactive := testSecrets.MustGetField("GITLABV2_INACTIVE")
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
-	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
-	}{
-		{
-			name: "found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Gitlab,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found, unverified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Gitlab,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatal("no raw secret present")
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
-
-func BenchmarkFromData2(benchmark *testing.B) {
-	ctx := context.Background()
-	s := Scanner{}
-	for name, data := range detectors.MustGetBenchmarkData() {
-		benchmark.Run(name, func(b *testing.B) {
-			b.ResetTimer()
-			for n := 0; n < b.N; n++ {
-				_, err := s.FromData(ctx, false, data)
-				if err != nil {
-					b.Fatal(err)
-				}
-			}
-		})
-	}
-}

pkg/detectors/gitlabv2/gitlab.go

@@ -11,7 +11,10 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{ detectors.EndpointSetter }
+type Scanner struct {
+	client *http.Client
+	detectors.EndpointSetter
+}
 
 // Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
@@ -22,7 +25,8 @@ func (Scanner) Version() int            { return 2 }
 func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" }
 
 var (
-	keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
+	defaultClient = common.SaneHttpClient()
+	keyPat        = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -42,7 +46,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			continue
 		}
 
-		secret := detectors.Result{
+		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Gitlab,
 			Raw:          []byte(match[1]),
 		}
@@ -52,7 +56,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			// they all grant access to different parts of the API. I couldn't find an endpoint that every
 			// one of these scopes has access to, so we just check an example endpoint for each scope. If any
 			// of them contain data, we know we have a valid key, but if they all fail, we don't
-			client := common.SaneHttpClient()
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
 			for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
 				// test `read_user` scope
 				req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
@@ -67,19 +74,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					// 200 means good key and has `read_user` scope
 					// 403 means good key but not the right scope
 					// 401 is bad key
-					if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
-						secret.Verified = true
+					switch res.StatusCode {
+					case http.StatusOK:
+						s1.Verified = true
+					case http.StatusForbidden:
+						// Good key but not the right scope
+						s1.Verified = true
+					case http.StatusUnauthorized:
+						// Nothing to do; zero values are the ones we want
+					default:
+						s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 					}
+				} else {
+					s1.VerificationError = err
 				}
 			}
 
 		}
 
-		if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
+		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
 			continue
 		}
 
-		results = append(results, secret)
+		results = append(results, s1)
 	}
 
 	return

pkg/detectors/gitlabv2/gitlab_test.go

@@ -9,7 +9,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -18,7 +19,7 @@ import (
 func TestGitlab_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
@@ -31,18 +32,19 @@ func TestGitlab_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a aws secret %s within", secret)),
+				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secret)),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -58,10 +60,65 @@ func TestGitlab_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a aws secret %s within", secretInactive)),
+				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
 				verify: true,
 			},
-			want:    nil,
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Gitlab,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, would be verified but for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Gitlab,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found and valid but unexpected api response",
+			s:    Scanner{client: common.ConstantResponseHttpClient(500, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Gitlab,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, good key but wrong scope",
+			s:    Scanner{client: common.ConstantResponseHttpClient(403, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Gitlab,
+					Verified:     true,
+				},
+			},
 			wantErr: false,
 		},
 		{
@@ -78,8 +135,7 @@ func TestGitlab_FromChunk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -88,9 +144,12 @@ func TestGitlab_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatal("no raw secret present")
 				}
-				got[i].Raw = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
+				}
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
+			opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, opts); diff != "" {
 				t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/detectors/sentrytoken/sentrytoken.go

@@ -122,6 +122,9 @@ func verifyToken(ctx context.Context, client *http.Client, token string) (bool,
 	if err = json.Unmarshal(bytes, &resp); err != nil {
 		return false, err
 	}
+	if len(resp) == 0 {
+		return false, fmt.Errorf("unexpected response body: %s", string(bytes))
+	}
 
 	return isVerified, err
 }

pkg/detectors/sentrytoken/sentrytoken_test.go

@@ -123,6 +123,40 @@ func TestSentryToken_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "found, account deactivated",
+			s:    Scanner{client: common.ConstantResponseHttpClient(200, reponseAccountDeactivated)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SentryToken,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, account deactivated",
+			s:    Scanner{client: common.ConstantResponseHttpClient(200, responseEnmpty)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SentryToken,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
 		{
 			name: "not found",
 			s:    Scanner{},
@@ -159,7 +193,8 @@ func TestSentryToken_FromChunk(t *testing.T) {
 	}
 }
 
-const responseBody403 = `
+const (
+	responseBody403 = `
 [
   {
     "organization": {
@@ -174,6 +209,9 @@ const responseBody403 = `
   }
 ]
 `
+	reponseAccountDeactivated = `{"detail": "Authentication credentials were not provided"}`
+	responseEnmpty            = `[]`
+)
 
 func BenchmarkFromData(benchmark *testing.B) {
 	ctx := context.Background()

pkg/detectors/slack/slack.go

@@ -13,13 +13,16 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
 
-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}
 
 // Check that the Slack scanner implements the SecretScanner interface at compile time.
 var _ detectors.Detector = Scanner{}
 
 var (
-	tokenPats = map[string]*regexp.Regexp{
+	defaultClient = common.SaneHttpClient()
+	tokenPats     = map[string]*regexp.Regexp{
 		"Slack Bot Token":               regexp.MustCompile(`xoxb\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
 		"Slack User Token":              regexp.MustCompile(`xoxp\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
 		"Slack Workspace Access Token":  regexp.MustCompile(`xoxa\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
@@ -53,34 +56,54 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		tokens := tokenPat.FindAllString(dataStr, -1)
 
 		for _, token := range tokens {
-			s := detectors.Result{
+			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Slack,
 				Raw:          []byte(token),
 			}
 			if verify {
-				client := common.SaneHttpClient()
+				client := s.client
+				if s.client == nil {
+					client = defaultClient
+				}
+
 				req, err := http.NewRequestWithContext(ctx, "POST", verifyURL, nil)
 				if err != nil {
 					continue
 				}
+
 				req.Header.Add("Content-Type", "application/json; charset=utf-8")
 				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					var authResponse authRes
 					if err := json.NewDecoder(res.Body).Decode(&authResponse); err != nil {
-						continue
+						s1.VerificationError = fmt.Errorf("failed to decode auth response: %w", err)
 					}
-					s.Verified = authResponse.Ok
+
+					if authResponse.Ok {
+						s1.Verified = true
+						// Slack API returns 200 even if the token is invalid. We need to check the error field.
+					} else if authResponse.Error == "invalid_auth" {
+						// The secret is determinately not verified (nothing to do)
+					} else {
+						s1.VerificationError = fmt.Errorf("unexpected error auth response %+v", authResponse.Error)
+					}
+				} else {
+					s1.VerificationError = err
 				}
 			}
 
-			results = append(results, s)
+			if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
+				continue
+			}
+
+			results = append(results, s1)
 		}
 	}
 
-	return
+	return results, nil
 }
 
 func (s Scanner) Type() detectorspb.DetectorType {

pkg/detectors/slack/slack_test.go

@@ -9,13 +9,15 @@ import (
 	"testing"
 	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestScanner_FromChunk(t *testing.T) {
+func TestSlack_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
 	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
@@ -24,17 +26,28 @@ func TestScanner_FromChunk(t *testing.T) {
 	}
 	secret := testSecrets.MustGetField("SLACK")
 	secretInactive := testSecrets.MustGetField("SLACK_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+
 	tests := []struct {
-		name        string
-		data        []byte
-		verify      bool
-		wantResults []detectors.Result
-		wantErr     bool
+		name                string
+		s                   Scanner
+		args                args
+		wantResults         []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
-			name:   "found, verified",
-			data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
-			verify: true,
+			name: "found, verified",
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
+				verify: true,
+			},
 			wantResults: []detectors.Result{
 				{
 					DetectorType: detectorspb.DetectorType_Slack,
@@ -44,9 +57,12 @@ func TestScanner_FromChunk(t *testing.T) {
 			wantErr: false,
 		},
 		{
-			name:   "found but unverified",
-			data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
-			verify: true,
+			name: "found but unverified",
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
+				verify: true,
+			},
 			wantResults: []detectors.Result{
 				{
 					DetectorType: detectorspb.DetectorType_Slack,
@@ -55,23 +71,62 @@ func TestScanner_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
+				verify: true,
+			},
+			wantResults: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Slack,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "unexpected auth response",
+			s:    Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "unexpected_error"}`)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
+				verify: true,
+			},
+			wantResults: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Slack,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(context.Background(), tt.verify, tt.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("Scanner.FromData) error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("Slack.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
+
 			for i := range got {
 				if len(got[i].Raw) == 0 {
 					t.Fatal("no raw secret present")
 				}
 				got[i].Raw = nil
+
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
 			}
-			if diff := pretty.Compare(got, tt.wantResults); diff != "" {
-				t.Errorf("Scanner.FromData) %s diff: (-got +want)\n%s", tt.name, diff)
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.wantResults, ignoreOpts); diff != "" {
+				t.Errorf("Slack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}

pkg/engine/engine.go

@@ -565,11 +565,18 @@ func (e *Engine) notifyResults(ctx context.Context) {
 		}
 		atomic.AddUint32(&e.numFoundResults, 1)
 
+		// Dedupe results by comparing the detector type, raw result, and source metadata.
+		// We want to avoid duplicate results with different decoder types, but we also
+		// want to include duplicate results with the same decoder type.
+		// Duplicate results with the same decoder type SHOULD have their own entry in the
+		// results list, this would happen if the same secret is found multiple times.
 		key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
-		if _, ok := e.dedupeCache.Get(key); ok {
-			continue
+		if val, ok := e.dedupeCache.Get(key); ok {
+			if res, ok := val.(detectorspb.DecoderType); ok && res != r.DecoderType {
+				continue
+			}
 		}
-		e.dedupeCache.Add(key, struct{}{})
+		e.dedupeCache.Add(key, r.DecoderType)
 
 		if r.Verified {
 			atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)

pkg/engine/engine_test.go

@@ -2,8 +2,13 @@ package engine
 
 import (
 	"fmt"
+	"path/filepath"
 	"testing"
+	"time"
 
+	"github.com/stretchr/testify/assert"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -171,3 +176,32 @@ func BenchmarkSupportsLineNumbersLoop(b *testing.B) {
 		_ = SupportsLineNumbers(sourceType)
 	}
 }
+
+// TestEngine_DuplicatSecrets is a test that detects ALL duplicate secrets with the same decoder.
+func TestEngine_DuplicatSecrets(t *testing.T) {
+	ctx := context.Background()
+
+	absPath, err := filepath.Abs("./testdata")
+	assert.Nil(t, err)
+
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	e, err := Start(ctx,
+		WithConcurrency(1),
+		WithDecoders(decoders.DefaultDecoders()...),
+		WithDetectors(true, DefaultDetectors()...),
+		WithPrinter(new(discardPrinter)),
+	)
+	assert.Nil(t, err)
+
+	cfg := sources.FilesystemConfig{Paths: []string{absPath}}
+	if err := e.ScanFileSystem(ctx, cfg); err != nil {
+		return
+	}
+
+	// Wait for all the chunks to be processed.
+	assert.Nil(t, e.Finish(ctx))
+	want := uint64(5)
+	assert.Equal(t, want, e.GetMetrics().UnverifiedSecretsFound)
+}

pkg/engine/testdata/secrets.txt

@@ -0,0 +1,6 @@
+  1 aws AKIAWARWQKZNHMZBLY4I
+  2 secret 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
+ sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
+ sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
+ sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
+ sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90

pkg/giturl/giturl_test.go

@@ -160,6 +160,24 @@ func TestGenerateLink(t *testing.T) {
 			},
 			want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=20",
 		},
+		{
+			name: "Unknown provider on-prem instance",
+			args: args{
+				repo:   "https://onprem.customdomain.com/org/repo.git",
+				commit: "xyz123",
+				file:   "main.go",
+				line:   int64(30),
+			},
+			want: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L30",
+		},
+		{
+			name: "Unknown provider on-prem instance - no file",
+			args: args{
+				repo:   "https://onprem.customdomain.com/org/repo.git",
+				commit: "xyz123",
+			},
+			want: "https://onprem.customdomain.com/org/repo/commit/xyz123",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {