Merge branch 'feature-bytification' into detectors-9

main.go

@@ -105,11 +105,11 @@ var (
 	filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
 	filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
 
-	s3Scan              = cli.Command("s3", "Find credentials in S3 buckets.")
-	s3ScanKey           = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
-	s3ScanRoleArns      = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
-	s3ScanSecret        = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
-	s3ScanSessionToken  = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
+	s3Scan    = cli.Command("s3", "Find credentials in S3 buckets.")
+	s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
+	s3ScanRoleArns     = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
+	s3ScanSecret       = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
+	s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
 	s3ScanCloudEnv      = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
 	s3ScanBuckets       = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
 	s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()
@@ -354,10 +354,6 @@ func run(state overseer.State) {
 		printer = new(output.PlainPrinter)
 	}
 
-	if !*jsonLegacy && !*jsonOut {
-		fmt.Fprintf(os.Stderr, "🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
-	}
-
 	e, err := engine.Start(ctx,
 		engine.WithConcurrency(uint8(*concurrency)),
 		engine.WithDecoders(decoders.DefaultDecoders()...),
@@ -473,7 +469,7 @@ func run(state overseer.State) {
 			Secret:        *s3ScanSecret,
 			SessionToken:  *s3ScanSessionToken,
 			Buckets:       *s3ScanBuckets,
-			Roles:         *s3ScanRoleArns,
+			Roles:	       *s3ScanRoleArns,
 			CloudCred:     *s3ScanCloudEnv,
 			MaxObjectSize: int64(*s3ScanMaxObjectSize),
 		}
@@ -529,6 +525,10 @@ func run(state overseer.State) {
 		}
 	}
 
+	if !*jsonLegacy && !*jsonOut {
+		fmt.Fprintf(os.Stderr, "🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
+	}
+
 	// Wait for all workers to finish.
 	if err = e.Finish(ctx); err != nil {
 		logFatal(err, "engine failed to finish execution")

pkg/detectors/aws/aws.go

@@ -118,21 +118,118 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			if verify {
-				verified, extraData, verificationErr := s.verifyMatch(ctx, resIDMatch, resSecretMatch, true)
-				s1.Verified = verified
-				s1.ExtraData = extraData
-				s1.VerificationError = verificationErr
+				// REQUEST VALUES.
+				method := "GET"
+				service := "sts"
+				host := "sts.amazonaws.com"
+				region := "us-east-1"
+				endpoint := "https://sts.amazonaws.com"
+				now := time.Now().UTC()
+				datestamp := now.Format("20060102")
+				amzDate := now.Format("20060102T150405Z0700")
+
+				req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
+				if err != nil {
+					continue
+				}
+				req.Header.Set("Accept", "application/json")
+
+				// TASK 1: CREATE A CANONICAL REQUEST.
+				// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+				canonicalURI := "/"
+				canonicalHeaders := "host:" + host + "\n"
+				signedHeaders := "host"
+				algorithm := "AWS4-HMAC-SHA256"
+				credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
+
+				params := req.URL.Query()
+				params.Add("Action", "GetCallerIdentity")
+				params.Add("Version", "2011-06-15")
+				params.Add("X-Amz-Algorithm", algorithm)
+				params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
+				params.Add("X-Amz-Date", amzDate)
+				params.Add("X-Amz-Expires", "30")
+				params.Add("X-Amz-SignedHeaders", signedHeaders)
+
+				canonicalQuerystring := params.Encode()
+				payloadHash := GetHash("") // empty payload
+				canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
+
+				// TASK 2: CREATE THE STRING TO SIGN.
+				stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
+
+				// TASK 3: CALCULATE THE SIGNATURE.
+				// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
+				hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
+				hash = GetHMAC(hash, []byte(region))
+				hash = GetHMAC(hash, []byte(service))
+				hash = GetHMAC(hash, []byte("aws4_request"))
+
+				signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
+				signature := hex.EncodeToString(signature2)
+
+				// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
+				params.Add("X-Amz-Signature", signature)
+				req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
+				req.URL.RawQuery = params.Encode()
+
+				client := s.verificationClient
+				if client == nil {
+					client = defaultVerificationClient
+				}
+				res, err := client.Do(req)
+				if err == nil {
+
+					if res.StatusCode >= 200 && res.StatusCode < 300 {
+						identityInfo := identityRes{}
+						err := json.NewDecoder(res.Body).Decode(&identityInfo)
+						if err == nil {
+							s1.Verified = true
+							s1.ExtraData = map[string]string{
+								"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
+								"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
+								"arn":     identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
+							}
+						} else {
+							s1.VerificationError = err
+						}
+						res.Body.Close()
+					} else {
+						// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
+						if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
+							continue
+						}
+
+						if res.StatusCode == 403 {
+							var body awsErrorResponseBody
+							err = json.NewDecoder(res.Body).Decode(&body)
+							res.Body.Close()
+							if err == nil {
+								// All instances of the code I've seen in the wild are PascalCased but this check is
+								// case-insensitive out of an abundance of caution
+								if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
+									// determinate failure - nothing to do
+								} else {
+									// We see a surprising number of false-negative signature mismatch errors here
+									// (The official SDK somehow elicits even more than just making the request
+									// ourselves)
+									s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
+								}
+							} else {
+								s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
+							}
+						} else {
+							s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
+						}
+					}
+				} else {
+					s1.VerificationError = err
+				}
 			}
 
-			if !s1.Verified {
-				// Unverified results that contain common test words are probably not secrets
-				if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
-					continue
-				}
-				// Unverified results that look like hashes are probably not secrets
-				if falsePositiveSecretCheck.MatchString(resSecretMatch) {
-					continue
-				}
+			// If the result is unverified and matches something like a git hash, don't include it in the results.
+			if !s1.Verified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
+				continue
 			}
 
 			results = append(results, s1)
@@ -145,123 +242,6 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return awsCustomCleanResults(results), nil
 }
 
-func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, retryOn403 bool) (bool, map[string]string, error) {
-	// REQUEST VALUES.
-	method := "GET"
-	service := "sts"
-	host := "sts.amazonaws.com"
-	region := "us-east-1"
-	endpoint := "https://sts.amazonaws.com"
-	now := time.Now().UTC()
-	datestamp := now.Format("20060102")
-	amzDate := now.Format("20060102T150405Z0700")
-
-	req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
-	if err != nil {
-		return false, nil, err
-	}
-	req.Header.Set("Accept", "application/json")
-
-	// TASK 1: CREATE A CANONICAL REQUEST.
-	// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
-	canonicalURI := "/"
-	canonicalHeaders := "host:" + host + "\n"
-	signedHeaders := "host"
-	algorithm := "AWS4-HMAC-SHA256"
-	credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
-
-	params := req.URL.Query()
-	params.Add("Action", "GetCallerIdentity")
-	params.Add("Version", "2011-06-15")
-	params.Add("X-Amz-Algorithm", algorithm)
-	params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
-	params.Add("X-Amz-Date", amzDate)
-	params.Add("X-Amz-Expires", "30")
-	params.Add("X-Amz-SignedHeaders", signedHeaders)
-
-	canonicalQuerystring := params.Encode()
-	payloadHash := GetHash("") // empty payload
-	canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
-
-	// TASK 2: CREATE THE STRING TO SIGN.
-	stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
-
-	// TASK 3: CALCULATE THE SIGNATURE.
-	// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
-	hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
-	hash = GetHMAC(hash, []byte(region))
-	hash = GetHMAC(hash, []byte(service))
-	hash = GetHMAC(hash, []byte("aws4_request"))
-
-	signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
-	signature := hex.EncodeToString(signature2)
-
-	// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
-	params.Add("X-Amz-Signature", signature)
-	req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
-	req.URL.RawQuery = params.Encode()
-
-	client := s.verificationClient
-	if client == nil {
-		client = defaultVerificationClient
-	}
-
-	res, err := client.Do(req)
-	if err == nil {
-		defer res.Body.Close()
-		if res.StatusCode >= 200 && res.StatusCode < 300 {
-			identityInfo := identityRes{}
-			err := json.NewDecoder(res.Body).Decode(&identityInfo)
-			if err == nil {
-				extraData := map[string]string{
-					"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
-					"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
-					"arn":     identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
-				}
-				return true, extraData, nil
-			} else {
-				return false, nil, err
-			}
-		} else if res.StatusCode == 403 {
-			// Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that
-			// share a key ID but are signed with different secrets the second one will be rejected with a 403 that
-			// carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is
-			// valid. Since this is exactly our access pattern, we need to work around it.
-			//
-			// Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The
-			// response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of
-			// the response to the original second request, or read to its end, or the resubmission will also yield a
-			// SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only
-			// SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch
-			// after decoding its response body, which requires reading the entire response body, which disables the
-			// workaround.
-			//
-			// We are clearly deep in the guts of AWS implementation details here, so this all might change with no
-			// notice. If you're here because something in this detector broke, you have my condolences.
-			if retryOn403 {
-				return s.verifyMatch(ctx, resIDMatch, resSecretMatch, false)
-			}
-			var body awsErrorResponseBody
-			err = json.NewDecoder(res.Body).Decode(&body)
-			if err == nil {
-				// All instances of the code I've seen in the wild are PascalCased but this check is
-				// case-insensitive out of an abundance of caution
-				if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
-					return false, nil, nil
-				} else {
-					return false, nil, fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
-				}
-			} else {
-				return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err)
-			}
-		} else {
-			return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
-		}
-	} else {
-		return false, nil, err
-	}
-}
-
 func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
 	if len(results) == 0 {
 		return results

pkg/detectors/aws/aws_test.go

@@ -248,28 +248,6 @@ func TestAWS_FromChunk(t *testing.T) {
 			wantErr:               false,
 			wantVerificationError: true,
 		},
-		{
-			name: "verified secret checked directly after unverified secret with same key id",
-			s:    scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_AWS,
-					Verified:     true,
-					Redacted:     "AKIASP2TPHJSQH3FJRUX",
-					ExtraData: map[string]string{
-						"account": "171436882533",
-						"arn":     "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
-						"user_id": "AIDASP2TPHJSUFRSTTZX4",
-					},
-				},
-			},
-			wantErr: false,
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/detectors/dailyco/dailyco.go

@@ -1,11 +1,11 @@
 package dailyco
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"daily"}) + `\b([0-9a-f]{64})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"daily"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("daily")}
 }
 
-// FromData will find and optionally verify DailyCO secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_DailyCO,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
-			if err == nil {
+			if err == nil && res != nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/dandelion/dandelion.go

@@ -1,11 +1,11 @@
 package dandelion
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,51 +14,44 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"dandelion"}) + `\b([a-z0-9]{32})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"dandelion"}) + `\b([0-9a-z]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"dandelion"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("dandelion")}
 }
 
-// FromData will find and optionally verify Dandelion secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Dandelion,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.dandelion.eu/datatxt/li/v1/?text=Smart&token=%s", resMatch), nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.dandelion.eu/datatxt/li/v1/?text=Smart&token=%s", string(resMatch)), nil)
 			if err != nil {
 				continue
 			}
+			req.Header.Add("Content-Type", "application/json")
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/dareboost/dareboost.go

@@ -1,11 +1,11 @@
 package dareboost
 
 import (
+	"bytes"
 	"context"
 	"io"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,41 +14,35 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"dareboost"}) + `\b([0-9a-zA-Z]{60})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"dareboost"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("dareboost")}
 }
 
-// FromData will find and optionally verify Dareboost secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Dareboost,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			payload := strings.NewReader(`{    "token": "` + resMatch + `",    "location": "Paris"}`)
+			payload := bytes.NewBuffer([]byte(`{    "token": "` + string(resMatch) + `",    "location": "Paris"}`))
 
 			req, err := http.NewRequestWithContext(ctx, "POST", "https://api.dareboost.com/0.8/config", payload)
 			if err != nil {
@@ -62,8 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 
-				bodyString := string(bodyBytes)
-				validResponse := strings.Contains(bodyString, `"status":200`)
+				validResponse := bytes.Contains(bodyBytes, []byte(`"status":200`))
 
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
@@ -73,7 +66,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						s1.Verified = false
 					}
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/databox/databox.go

@@ -1,12 +1,12 @@
 package databox
 
 import (
+	"bytes"
 	"context"
 	b64 "encoding/base64"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,49 +15,42 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"databox"}) + common.BuildRegex(common.RegexPattern, "", 21))
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"databox"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("databox")}
 }
 
-// FromData will find and optionally verify Databox secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Databox,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 		if verify {
-			data := fmt.Sprintf("%s:", resMatch)
+			data := fmt.Sprintf("%s:", string(resMatch))
 			sEnc := b64.StdEncoding.EncodeToString([]byte(data))
-			payload := strings.NewReader(`{
+			payload := bytes.NewBuffer([]byte(`{
 				"data":[
 				  {
 					"$sales": 420,
 					"$visitors": 123000
 				  }
 				]
-			  }`)
+			  }`))
 
 			req, err := http.NewRequestWithContext(ctx, "POST", "https://push.databox.com", payload)
 			if err != nil {
@@ -72,7 +65,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/databrickstoken/databrickstoken.go

@@ -1,11 +1,11 @@
 package databrickstoken
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -27,48 +27,46 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"databricks", "dapi"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("databricks"), []byte("dapi")}
 }
 
 // FromData will find and optionally verify Databrickstoken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	domainMatches := domain.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
+	domainMatches := domain.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		for _, domainmatch := range domainMatches {
 			if len(domainmatch) != 2 {
 				continue
 			}
-			resDomainMatch := strings.TrimSpace(domainmatch[1])
+
+			resDomainMatch := bytes.TrimSpace(domainmatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_DatabricksToken,
-				Raw:          []byte(resMatch),
-				RawV2:        []byte(resMatch + resDomainMatch),
+				Raw:          resMatch,
+				RawV2:        append(resMatch, resDomainMatch...),
 			}
 
 			if verify {
-				req, err := http.NewRequestWithContext(ctx, "GET", resDomainMatch + "/api/2.0/clusters/list", nil)
+				req, err := http.NewRequestWithContext(ctx, "GET", string(resDomainMatch)+"/api/2.0/clusters/list", nil)
 				if err != nil {
 					continue
 				}
-				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}

pkg/detectors/datadogtoken/datadogtoken.go

@@ -1,10 +1,10 @@
 package datadogtoken
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,108 +13,61 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
 	apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"datadog"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("datadog")}
 }
 
-// FromData will find and optionally verify DatadogToken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
+	matches := keyPat.FindAllSubmatch(data, -1)
+	apiMatches := apiPat.FindAllSubmatch(data, -1)
 
-	appMatches := appPat.FindAllStringSubmatch(dataStr, -1)
-	apiMatches := apiPat.FindAllStringSubmatch(dataStr, -1)
-
-	for _, apiMatch := range apiMatches {
-		if len(apiMatch) != 2 {
+	for _, match := range matches {
+		if len(match) != 2 {
 			continue
 		}
-		resApiMatch := strings.TrimSpace(apiMatch[1])
-		appIncluded := false
-		for _, appMatch := range appMatches {
-			if len(appMatch) != 2 {
+		resMatch := bytes.TrimSpace(match[1])
+		for _, apimatch := range apiMatches {
+			if len(apimatch) != 2 {
 				continue
 			}
-			resAppMatch := strings.TrimSpace(appMatch[1])
+			resApiMatch := bytes.TrimSpace(apimatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_DatadogToken,
-				Raw:          []byte(resAppMatch),
-				RawV2:        []byte(resAppMatch + resApiMatch),
-				ExtraData: map[string]string{
-					"Type": "Application+APIKey",
-				},
+				Raw:          resMatch,
+				RawV2:        append(resMatch, resApiMatch...),
 			}
 
 			if verify {
-
 				req, err := http.NewRequestWithContext(ctx, "GET", "https://api.datadoghq.com/api/v2/users", nil)
 				if err != nil {
 					continue
 				}
 				req.Header.Add("Content-Type", "application/json")
-				req.Header.Add("DD-API-KEY", resApiMatch)
-				req.Header.Add("DD-APPLICATION-KEY", resAppMatch)
+				req.Header.Add("DD-API-KEY", string(resApiMatch))
+				req.Header.Add("DD-APPLICATION-KEY", string(resMatch))
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-						if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) {
+						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}
 					}
 				}
 			}
-			appIncluded = true
-			results = append(results, s1)
-		}
 
-		if !appIncluded {
-			s1 := detectors.Result{
-				DetectorType: detectorspb.DetectorType_DatadogToken,
-				Raw:          []byte(resApiMatch),
-				RawV2:        []byte(resApiMatch),
-				ExtraData: map[string]string{
-					"Type": "APIKeyOnly",
-				},
-			}
-
-			if verify {
-
-				req, err := http.NewRequestWithContext(ctx, "GET", "https://api.datadoghq.com/api/v1/validate", nil)
-				if err != nil {
-					continue
-				}
-				req.Header.Add("Content-Type", "application/json")
-				req.Header.Add("DD-API-KEY", resApiMatch)
-				res, err := client.Do(req)
-				if err == nil {
-					defer res.Body.Close()
-					if res.StatusCode >= 200 && res.StatusCode < 300 {
-						s1.Verified = true
-					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-						if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) {
-							continue
-						}
-					}
-				}
-			}
 			results = append(results, s1)
 		}
 	}

pkg/detectors/datafire/datafire.go

@@ -1,11 +1,11 @@
 package datafire
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datafire"}) + `\b([a-z0-9\S]{175,190})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"datafire"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("datafire")}
 }
 
-// FromData will find and optionally verify DataFire secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_DataFire,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,14 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/datagov/datagov.go

@@ -1,11 +1,11 @@
 package datagov
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,41 +14,33 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"data.gov"}) + `\b([a-zA-Z0-9]{40})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"data.gov"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("data.gov")}
 }
 
-// FromData will find and optionally verify DataGov secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_DataGov,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.ers.usda.gov/data/arms/state?api_key=%s", resMatch), nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.ers.usda.gov/data/arms/state?api_key=%s", string(resMatch)), nil)
 			if err != nil {
 				continue
 			}
@@ -58,7 +50,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
@@ -68,7 +59,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 		results = append(results, s1)
 	}
-
 	return results, nil
 }
 

pkg/detectors/debounce/debounce.go

@@ -1,10 +1,10 @@
 package debounce
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,41 +13,37 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"debounce"}) + `\b([a-zA-Z0-9]{13})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"debounce"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("debounce")}
 }
 
-// FromData will find and optionally verify Debounce secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Debounce,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.debounce.io/v1/?api="+resMatch+"&email=some@gmail.com", nil)
+			api := append([]byte("https://api.debounce.io/v1/?api="), resMatch...)
+			url := append(api, []byte("&email=some@gmail.com")...)
+
+			req, err := http.NewRequestWithContext(ctx, "GET", string(url), nil)
 			if err != nil {
 				continue
 			}
@@ -58,7 +54,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/deepai/deepai.go

@@ -7,7 +7,6 @@ import (
 	"mime/multipart"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -26,27 +25,22 @@ var (
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"deepai"}) + `\b([a-z0-9-]{36})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"deepai"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("deepai")}
 }
 
-// FromData will find and optionally verify DeepAI secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_DeepAI,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -56,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			_, err = io.Copy(fw, strings.NewReader("test"))
+			_, err = io.Copy(fw, bytes.NewReader([]byte("test")))
 			if err != nil {
 				continue
 			}
@@ -66,15 +60,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", writer.FormDataContentType())
-			req.Header.Add("api-key", resMatch)
+			req.Header.Add("api-key", string(resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+					if detectors.IsKnownFalsePositive([]byte(resMatch), detectors.DefaultFalsePositives, true) {
 						continue
 					}
 				}

pkg/detectors/deepgram/deepgram.go

@@ -1,11 +1,11 @@
 package deepgram
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"deepgram"}) + `\b([0-9a-z]{40})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"deepgram"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("deepgram")}
 }
 
-// FromData will find and optionally verify Deepgram secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Deepgram,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -53,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Token %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/delighted/delighted.go

@@ -1,12 +1,12 @@
 package delighted
 
 import (
+	"bytes"
 	"context"
 	b64 "encoding/base64"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,69 +15,56 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"delighted"}) + `\b([a-z0-9A-Z]{32})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"delighted"}) + `\b([a-zA-Z0-9]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"delighted"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("delighted")}
 }
 
-// FromData will find and optionally verify Delighted secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Delighted,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			data := fmt.Sprintf("%s:", resMatch)
-			sEnc := b64.StdEncoding.EncodeToString([]byte(data))
-			payload := strings.NewReader(`{
+			sEnc := b64.StdEncoding.EncodeToString(resMatch)
+			payload := bytes.NewReader([]byte(`{
 				"email": "jony@appleseed.com",
 				"properties": { "Purchase Experience": "Mobile App", "State": "CA" }
-				}`)
+				}`))
 			req, err := http.NewRequestWithContext(ctx, "POST", "https://api.delighted.com/v1/people.json", payload)
-			if err != nil {
-				continue
-			}
-			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
-			res, err := client.Do(req)
 			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-						continue
+				req.Header.Add("Content-Type", "application/json")
+				req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
+				res, err := client.Do(req)
+				if err == nil {
+					defer res.Body.Close()
+					if res.StatusCode >= 200 && res.StatusCode < 300 {
+						s1.Verified = true
+					} else {
+						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+							continue
+						}
 					}
 				}
 			}
 		}
-
 		results = append(results, s1)
 	}
-
 	return results, nil
 }
 

pkg/detectors/demio/demio.go

@@ -1,11 +1,11 @@
 package demio
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,70 +14,59 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
 	keyPat    = regexp.MustCompile(detectors.PrefixRegex([]string{"demio"}) + `\b([a-z0-9A-Z]{32})\b`)
 	secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"demio"}) + `\b([a-z0-9A-Z]{10,20})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"demio"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("demio")}
 }
 
-// FromData will find and optionally verify Demio secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
+	keyMatches := keyPat.FindAllSubmatch(data, -1)
+	secretMatches := secretPat.FindAllSubmatch(data, -1)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	idMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
-	for _, match := range matches {
+	for _, match := range keyMatches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
-		for _, idmatch := range idMatches {
-			if len(match) != 2 {
+		resMatch := bytes.TrimSpace(match[1])
+
+		for _, idmatch := range secretMatches {
+			if len(idmatch) != 2 {
 				continue
 			}
-			resIdMatch := strings.TrimSpace(idmatch[1])
+			resIdMatch := bytes.TrimSpace(idmatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Demio,
-				Raw:          []byte(resMatch),
+				Raw:          resMatch,
 			}
 
 			if verify {
-				url := fmt.Sprintf("https://my.demio.com/api/v1/ping/query?api_key=%s&api_secret=%s", resMatch, resIdMatch)
+				url := fmt.Sprintf("https://my.demio.com/api/v1/ping/query?api_key=%s&api_secret=%s", string(resMatch), string(resIdMatch))
+
 				req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 				if err != nil {
 					continue
 				}
 				res, err := client.Do(req)
-				if err == nil {
-					defer res.Body.Close()
-					if res.StatusCode >= 200 && res.StatusCode < 300 {
-						s1.Verified = true
-					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
-						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-							continue
-						}
+				if err != nil || res.StatusCode < 200 || res.StatusCode >= 300 {
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
 					}
+				} else {
+					s1.Verified = true
 				}
 			}
-
 			results = append(results, s1)
 		}
-
 	}
-
 	return results, nil
 }
 

pkg/detectors/deputy/deputy.go

@@ -1,11 +1,11 @@
 package deputy
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,64 +14,58 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"deputy"}) + `\b([0-9a-z]{32})\b`)
 	urlPat = regexp.MustCompile(`\b([0-9a-z]{1,}\.as\.deputy\.com)\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"deputy"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("deputy")}
 }
 
-// FromData will find and optionally verify Deputy secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	urlMatches := urlPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
+	urlMatches := urlPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		for _, urlMatch := range urlMatches {
 			if len(urlMatch) != 2 {
 				continue
 			}
-			resURL := strings.TrimSpace(urlMatch[1])
+			resURL := bytes.TrimSpace(urlMatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Deputy,
-				Raw:          []byte(resMatch),
+				Raw:          resMatch,
 			}
 
 			if verify {
-				req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/api/v1/me", resURL), nil)
+				req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/api/v1/me", string(resURL)), nil)
 				if err != nil {
 					continue
 				}
 				req.Header.Add("Content-Type", "application/json")
-				req.Header.Add("Authorization", fmt.Sprintf("OAuth %s", resMatch))
+				req.Header.Add("Authorization", fmt.Sprintf("OAuth %s", string(resMatch)))
 				res, err := client.Do(req)
-				if err == nil {
-					defer res.Body.Close()
-					if res.StatusCode >= 200 && res.StatusCode < 300 {
-						s1.Verified = true
-					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-							continue
-						}
+				if err != nil {
+					continue
+				}
+				defer res.Body.Close()
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
+					s1.Verified = true
+				} else {
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
 					}
 				}
 			}
@@ -79,6 +73,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			results = append(results, s1)
 		}
 	}
+
 	return results, nil
 }
 

pkg/detectors/detectify/detectify.go

@@ -1,10 +1,10 @@
 package detectify
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,37 +13,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"detectify"}) + `\b([0-9a-z]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"detectify"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("detectify")}
 }
 
-// FromData will find and optionally verify Detectify secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Detectify,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -51,14 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-Detectify-Key", resMatch)
+			req.Header.Add("X-Detectify-Key", string(resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/detectlanguage/detectlanguage.go

@@ -1,11 +1,11 @@
 package detectlanguage
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"detectlanguage"}) + `\b([a-z0-9]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"detectlanguage"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("detectlanguage")}
 }
 
-// FromData will find and optionally verify DetectLanguage secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_DetectLanguage,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/detectors.go

@@ -20,7 +20,7 @@ type Detector interface {
 	FromData(ctx context.Context, verify bool, data []byte) ([]Result, error)
 	// Keywords are used for efficiently pre-filtering chunks using substring operations.
 	// Use unique identifiers that are part of the secret if you can, or the provider name.
-	Keywords() []string
+	Keywords() [][]byte
 	// Type returns the DetectorType number from detectors.proto for the given detector.
 	Type() detectorspb.DetectorType
 }

pkg/detectors/dfuse/dfuse.go

@@ -1,10 +1,10 @@
 package dfuse
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -25,29 +25,27 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"dfuse"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("dfuse")}
 }
 
 // FromData will find and optionally verify Dfuse secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Dfuse,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			payload := strings.NewReader(`{"api_key":"` + resMatch + `"}`)
+			payload := bytes.NewBuffer([]byte(`{"api_key":"` + string(resMatch) + `"}`))
 			req, err := http.NewRequestWithContext(ctx, "POST", "https://auth.dfuse.io/v1/auth/issue", payload)
 			if err != nil {
 				continue
@@ -59,7 +57,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/diffbot/diffbot.go

@@ -1,12 +1,12 @@
 package diffbot
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
 	"net/http"
 	"regexp"
-	"strings"
 	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -16,43 +16,36 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"diffbot"}) + `\b([a-z0-9]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"diffbot"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("diffbot")}
 }
 
-// FromData will find and optionally verify Diffbot secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Diffbot,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
 			timeout := 10 * time.Second
 			client.Timeout = timeout
-			req, err := http.NewRequest("GET", fmt.Sprintf("https://api.diffbot.com/v4/account?token=%s", resMatch), nil)
+			req, err := http.NewRequest("GET", fmt.Sprintf("https://api.diffbot.com/v4/account?token=%s", string(resMatch)), nil)
 			if err != nil {
 				continue
 			}
@@ -60,8 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err == nil {
 				bodyBytes, err := io.ReadAll(res.Body)
 				if err == nil {
-					bodyString := string(bodyBytes)
-					validResponse := strings.Contains(bodyString, `"token":`) && strings.Contains(bodyString, `"planCredits":`)
+					validResponse := bytes.Contains(bodyBytes, []byte(`"token":`)) && bytes.Contains(bodyBytes, []byte(`"planCredits":`))
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						if validResponse {
@@ -70,7 +62,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 							s1.Verified = false
 						}
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}

pkg/detectors/diggernaut/diggernaut.go

@@ -1,11 +1,11 @@
 package diggernaut
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"diggernaut"}) + `\b([0-9a-z]{40})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"diggernaut"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("diggernaut")}
 }
 
-// FromData will find and optionally verify Diggernaut secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Diggernaut,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -53,14 +46,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Accept", "application/json")
-			req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Token %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/gitlab/gitlab.go

@@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			resMatch = keyString[len(keyString)-1]
 		}
 
-		s1 := detectors.Result{
+		secret := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Gitlab,
 			Raw:          []byte(match[1]),
 		}
@@ -84,26 +84,26 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					// 401 is bad key
 					switch res.StatusCode {
 					case http.StatusOK:
-						s1.Verified = true
+						secret.Verified = true
 					case http.StatusForbidden:
 						// Good key but not the right scope
-						s1.Verified = true
+						secret.Verified = true
 					case http.StatusUnauthorized:
 						// Nothing to do; zero values are the ones we want
 					default:
-						s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+						secret.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 					}
 				} else {
-					s1.VerificationError = err
+					secret.VerificationError = err
 				}
 			}
 		}
 
-		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
+		if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
 			continue
 		}
 
-		results = append(results, s1)
+		results = append(results, secret)
 	}
 
 	return results, nil

pkg/detectors/gitlab/gitlab_test.go

@@ -6,11 +6,10 @@ package gitlab
 import (
 	"context"
 	"fmt"
-	"testing"
-	"time"
-
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"testing"
+	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"

pkg/detectors/gitlab/gitlabv2_test.go

@@ -1,7 +1,4 @@
-//go:build detectors
-// +build detectors
-
-package tailscale
+package gitlab
 
 import (
 	"context"
@@ -10,23 +7,20 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestTailscaleapi_FromChunk(t *testing.T) {
+func TestGitlabv2_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
 	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
-	if err != nil && false {
+	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
-	secret := testSecrets.MustGetField("TAILSCALEAPI")
-	inactiveSecret := testSecrets.MustGetField("TAILSCALEAPI_INACTIVE")
-
+	secret := testSecrets.MustGetField("GITLABV2")
+	secretInactive := testSecrets.MustGetField("GITLABV2_INACTIVE")
 	type args struct {
 		ctx    context.Context
 		data   []byte
@@ -40,16 +34,16 @@ func TestTailscaleapi_FromChunk(t *testing.T) {
 		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name: "found",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within", secret)),
+				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secret)),
 				verify: true,
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_Tailscale,
+					DetectorType: detectorspb.DetectorType_Gitlab,
 					Verified:     true,
 				},
 			},
@@ -60,12 +54,12 @@ func TestTailscaleapi_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
 				verify: true,
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_Tailscale,
+					DetectorType: detectorspb.DetectorType_Gitlab,
 					Verified:     false,
 				},
 			},
@@ -88,27 +82,28 @@ func TestTailscaleapi_FromChunk(t *testing.T) {
 			s := Scanner{}
 			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("Tailscaleapi.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			for i := range got {
 				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
+					t.Fatal("no raw secret present")
 				}
 				got[i].Raw = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Tailscaleapi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+				t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}
 }
 
-func BenchmarkFromData(benchmark *testing.B) {
+func BenchmarkFromData2(benchmark *testing.B) {
 	ctx := context.Background()
 	s := Scanner{}
 	for name, data := range detectors.MustGetBenchmarkData() {
 		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
 			for n := 0; n < b.N; n++ {
 				_, err := s.FromData(ctx, false, data)
 				if err != nil {

pkg/detectors/gitlabv2/gitlab.go

@@ -11,10 +11,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct {
-	client *http.Client
-	detectors.EndpointSetter
-}
+type Scanner struct{ detectors.EndpointSetter }
 
 // Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
@@ -25,8 +22,7 @@ func (Scanner) Version() int            { return 2 }
 func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" }
 
 var (
-	defaultClient = common.SaneHttpClient()
-	keyPat        = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
+	keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -46,7 +42,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			continue
 		}
 
-		s1 := detectors.Result{
+		secret := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Gitlab,
 			Raw:          []byte(match[1]),
 		}
@@ -56,10 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			// they all grant access to different parts of the API. I couldn't find an endpoint that every
 			// one of these scopes has access to, so we just check an example endpoint for each scope. If any
 			// of them contain data, we know we have a valid key, but if they all fail, we don't
-			client := s.client
-			if client == nil {
-				client = defaultClient
-			}
+			client := common.SaneHttpClient()
 			for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
 				// test `read_user` scope
 				req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
@@ -74,29 +67,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					// 200 means good key and has `read_user` scope
 					// 403 means good key but not the right scope
 					// 401 is bad key
-					switch res.StatusCode {
-					case http.StatusOK:
-						s1.Verified = true
-					case http.StatusForbidden:
-						// Good key but not the right scope
-						s1.Verified = true
-					case http.StatusUnauthorized:
-						// Nothing to do; zero values are the ones we want
-					default:
-						s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+					if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
+						secret.Verified = true
 					}
-				} else {
-					s1.VerificationError = err
 				}
 			}
 
 		}
 
-		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
+		if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
 			continue
 		}
 
-		results = append(results, s1)
+		results = append(results, secret)
 	}
 
 	return

pkg/detectors/gitlabv2/gitlab_test.go

@@ -9,8 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -19,7 +18,7 @@ import (
 func TestGitlab_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
@@ -32,19 +31,18 @@ func TestGitlab_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secret)),
+				data:   []byte(fmt.Sprintf("You can find a aws secret %s within", secret)),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -60,65 +58,10 @@ func TestGitlab_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
+				data:   []byte(fmt.Sprintf("You can find a aws secret %s within", secretInactive)),
 				verify: true,
 			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Gitlab,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found, would be verified but for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Gitlab,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found and valid but unexpected api response",
-			s:    Scanner{client: common.ConstantResponseHttpClient(500, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Gitlab,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, good key but wrong scope",
-			s:    Scanner{client: common.ConstantResponseHttpClient(403, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Gitlab,
-					Verified:     true,
-				},
-			},
+			want:    nil,
 			wantErr: false,
 		},
 		{
@@ -135,7 +78,8 @@ func TestGitlab_FromChunk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -144,12 +88,9 @@ func TestGitlab_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatal("no raw secret present")
 				}
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
-				}
+				got[i].Raw = nil
 			}
-			opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
-			if diff := cmp.Diff(got, tt.want, opts); diff != "" {
+			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/detectors/jiratoken/jiratoken.go

@@ -13,18 +13,16 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 var _ detectors.Versioner = (*Scanner)(nil)
 
-func (Scanner) Version() int { return 1 }
+func (Scanner) Version() int            { return 1 }
 
 var (
-	defaultClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	tokenPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{24})\b`)
@@ -75,11 +73,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 
 				if verify {
-					client := s.client
-					if client == nil {
-						client = defaultClient
-					}
-
 					data := fmt.Sprintf("%s:%s", resEmail, resToken)
 					sEnc := b64.StdEncoding.EncodeToString([]byte(data))
 					req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
@@ -95,15 +88,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						// If the request is successful and the login reason is not failed authentication, then the token is valid.
 						// This is because Jira returns a 200 status code even if the token is invalid.
 						// Jira returns a default dashboard page.
-						if res.StatusCode >= 200 && res.StatusCode < 300 {
-							if res.Header.Get(loginReasonHeaderKey) != failedAuth {
-								s1.Verified = true
-							}
-						} else {
-							s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+						if (res.StatusCode >= 200 && res.StatusCode < 300) && res.Header.Get(loginReasonHeaderKey) != failedAuth {
+							s1.Verified = true
 						}
-					} else {
-						s1.VerificationError = err
 					}
 				}
 

pkg/detectors/jiratoken/jiratoken_test.go

@@ -9,8 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -24,8 +23,8 @@ func TestJiraToken_FromChunk(t *testing.T) {
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
-	secret := testSecrets.MustGetField("JIRA_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("JIRA_INACTIVE")
+	token := testSecrets.MustGetField("JIRA_TOKEN")
+	inactiveToken := testSecrets.MustGetField("JIRA_INACTIVE")
 	email := testSecrets.MustGetField("JIRA_EMAIL")
 	domain := testSecrets.MustGetField("JIRA_DOMAIN")
 
@@ -35,19 +34,18 @@ func TestJiraToken_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", token, email, domain)),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -56,15 +54,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "found, unverified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveSecret, email, domain)), // the secret would satisfy the regex but not pass validation
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveToken, email, domain)), // the secret would satisfy the regex but not pass validation
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -73,8 +70,7 @@ func TestJiraToken_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "not found",
@@ -84,48 +80,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_JiraToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_JiraToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
+			want:    nil,
+			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -134,12 +96,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
-				}
+				got[i].Raw = nil
+				got[i].RawV2 = nil
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/detectors/jiratoken_v2/jiratoken_v2.go

@@ -13,18 +13,16 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 var _ detectors.Versioner = (*Scanner)(nil)
 
-func (Scanner) Version() int { return 2 }
+func (Scanner) Version() int            { return 2 }
 
 var (
-	defaultClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
 
 	// https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/
 	// Tokens created after Jan 18 2023 use a variable length
@@ -76,11 +74,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 
 				if verify {
-					client := s.client
-					if client == nil {
-						client = defaultClient
-					}
-
 					data := fmt.Sprintf("%s:%s", resEmail, resToken)
 					sEnc := b64.StdEncoding.EncodeToString([]byte(data))
 					req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
@@ -96,15 +89,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						// If the request is successful and the login reason is not failed authentication, then the token is valid.
 						// This is because Jira returns a 200 status code even if the token is invalid.
 						// Jira returns a default dashboard page.
-						if res.StatusCode >= 200 && res.StatusCode < 300 {
-							if res.Header.Get(loginReasonHeaderKey) != failedAuth {
-								s1.Verified = true
-							}
+						if (res.StatusCode >= 200 && res.StatusCode < 300) && res.Header.Get(loginReasonHeaderKey) != failedAuth {
+							s1.Verified = true
 						} else {
-							s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+							s1.VerificationError = err
 						}
-					} else {
-						s1.VerificationError = err
 					}
 				}
 

pkg/detectors/jiratoken_v2/jiratoken_v2_test.go

@@ -9,8 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -24,8 +23,8 @@ func TestJiraToken_FromChunk(t *testing.T) {
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
-	secret := testSecrets.MustGetField("JIRA_V2_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("JIRA_V2_INACTIVE")
+	token := testSecrets.MustGetField("JIRA_V2_TOKEN")
+	inactiveToken := testSecrets.MustGetField("JIRA_V2_INACTIVE")
 	email := testSecrets.MustGetField("JIRA_V2_EMAIL")
 	domain := testSecrets.MustGetField("JIRA_V2_DOMAIN")
 
@@ -35,19 +34,18 @@ func TestJiraToken_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", token, email, domain)),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -56,15 +54,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "found, unverified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveSecret, email, domain)), // the secret would satisfy the regex but not pass validation
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveToken, email, domain)), // the secret would satisfy the regex but not pass validation
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -73,8 +70,7 @@ func TestJiraToken_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "not found",
@@ -84,48 +80,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_JiraToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_JiraToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
+			want:    nil,
+			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -134,12 +96,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
-				}
+				got[i].Raw = nil
+				got[i].RawV2 = nil
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/detectors/pagerdutyapikey/pagerdutyapikey.go

@@ -21,13 +21,13 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pd"}) + `\b([a-zA-Z0-9_+-]{20})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty"}) + `\b([a-z]{1}\+[a-zA-Z]{9}\-[a-z]{2}\-[a-z0-9]{5})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"pagerduty", "pd"}
+	return []string{"pagerduty"}
 }
 
 // FromData will find and optionally verify PagerDutyApiKey secrets in a given set of bytes.

pkg/detectors/sendbird/sendbird.go

@@ -2,7 +2,6 @@ package sendbird
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"regexp"
@@ -13,25 +12,19 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	defaultClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat   = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9a-f]{40})\b`)
 	appIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})\b`)
 )
 
-type userResp struct {
-	Code int `json:"code"`
-}
-
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
@@ -68,40 +61,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				req.Header.Add("Api-Token", resMatch)
-
-				client := s.client
-				if client == nil {
-					client = defaultClient
-				}
-
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
-					} else if res.StatusCode == 400 { // Sendbird returns 400 for all errors
-						var userResp userResp
-						err := json.NewDecoder(res.Body).Decode(&userResp)
-						if err != nil {
-							s1.VerificationError = fmt.Errorf("error decoding json response body: %w", err)
-						} else if userResp.Code != 400401 {
-							// https://sendbird.com/docs/chat/platform-api/v3/error-codes
-							// Sendbird always includes its own error codes with 400 responses
-							// 400401 (InvalidApiToken) is the only one that indicates a bad token
-							s1.VerificationError = fmt.Errorf("unexpected response code: %d", userResp.Code)
-						}
 					} else {
-						s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+							continue
+						}
 					}
-				} else {
-					s1.VerificationError = err
 				}
 			}
 
-			// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-			if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-				continue
-			}
 			results = append(results, s1)
 		}
 	}

pkg/detectors/sendbird/sendbird_test.go

@@ -9,8 +9,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/kylelemons/godebug/pretty"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -19,7 +19,7 @@ import (
 func TestSendbird_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
@@ -33,12 +33,11 @@ func TestSendbird_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
@@ -54,8 +53,7 @@ func TestSendbird_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -71,8 +69,7 @@ func TestSendbird_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "not found",
@@ -82,82 +79,14 @@ func TestSendbird_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Sendbird,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Sendbird,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected error code",
-			s:    Scanner{client: common.ConstantResponseHttpClient(400, `{"code":400402}`)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Sendbird,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but failed to decode json",
-			s:    Scanner{client: common.ConstantResponseHttpClient(400, `{`)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Sendbird,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
+			want:    nil,
+			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Sendbird.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -166,12 +95,9 @@ func TestSendbird_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
-				}
+				got[i].Raw = nil
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/detectors/sendgrid/sendgrid.go

@@ -13,15 +13,13 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	defaultClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
 
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendgrid"}) + `(SG\.[\w\-_]{20,24}\.[\w\-_]{39,50})\b`)
 )
@@ -44,7 +42,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 		res := strings.TrimSpace(match[1])
 
-		s1 := detectors.Result{
+		s := detectors.Result{
 			DetectorType: detectorspb.DetectorType_SendGrid,
 			Raw:          []byte(res),
 		}
@@ -54,11 +52,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			// 403 will be issued if the scope is wrong but the key is correct
 			baseURL := "https://api.sendgrid.com/v3/templates"
 
-			client := s.client
-			if client == nil {
-				client = defaultClient
-			}
-
 			// test `read_user` scope
 			req, err := http.NewRequestWithContext(ctx, "GET", baseURL, nil)
 			if err != nil {
@@ -74,18 +67,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				// 403 means good key but not the right scope
 				// 401 is bad key
 				if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
-					s1.Verified = true
-				} else if res.StatusCode != http.StatusUnauthorized {
-					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+					s.Verified = true
 				}
 			}
 		}
 
-		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
+		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
 			continue
 		}
 
-		results = append(results, s1)
+		results = append(results, s)
 	}
 
 	return

pkg/detectors/sendgrid/sendgrid_test.go

@@ -9,8 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -19,7 +18,7 @@ import (
 func TestSendgrid_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
@@ -31,12 +30,11 @@ func TestSendgrid_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
@@ -52,8 +50,7 @@ func TestSendgrid_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -69,8 +66,7 @@ func TestSendgrid_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "not found",
@@ -80,54 +76,22 @@ func TestSendgrid_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_SendGrid,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_SendGrid,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
+			want:    nil,
+			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Sendgrid.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+			for i := range got {
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("Sendgrid.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/detectors/sentrytoken/sentrytoken.go

@@ -122,9 +122,6 @@ func verifyToken(ctx context.Context, client *http.Client, token string) (bool,
 	if err = json.Unmarshal(bytes, &resp); err != nil {
 		return false, err
 	}
-	if len(resp) == 0 {
-		return false, fmt.Errorf("unexpected response body: %s", string(bytes))
-	}
 
 	return isVerified, err
 }

pkg/detectors/sentrytoken/sentrytoken_test.go

@@ -123,40 +123,6 @@ func TestSentryToken_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
-		{
-			name: "found, account deactivated",
-			s:    Scanner{client: common.ConstantResponseHttpClient(200, reponseAccountDeactivated)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_SentryToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, account deactivated",
-			s:    Scanner{client: common.ConstantResponseHttpClient(200, responseEnmpty)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_SentryToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
 		{
 			name: "not found",
 			s:    Scanner{},
@@ -193,8 +159,7 @@ func TestSentryToken_FromChunk(t *testing.T) {
 	}
 }
 
-const (
-	responseBody403 = `
+const responseBody403 = `
 [
   {
     "organization": {
@@ -209,9 +174,6 @@ const (
   }
 ]
 `
-	reponseAccountDeactivated = `{"detail": "Authentication credentials were not provided"}`
-	responseEnmpty            = `[]`
-)
 
 func BenchmarkFromData(benchmark *testing.B) {
 	ctx := context.Background()

pkg/detectors/slack/slack.go

@@ -13,16 +13,13 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Check that the Slack scanner implements the SecretScanner interface at compile time.
 var _ detectors.Detector = Scanner{}
 
 var (
-	defaultClient = common.SaneHttpClient()
-	tokenPats     = map[string]*regexp.Regexp{
+	tokenPats = map[string]*regexp.Regexp{
 		"Slack Bot Token":               regexp.MustCompile(`xoxb\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
 		"Slack User Token":              regexp.MustCompile(`xoxp\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
 		"Slack Workspace Access Token":  regexp.MustCompile(`xoxa\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
@@ -56,54 +53,34 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		tokens := tokenPat.FindAllString(dataStr, -1)
 
 		for _, token := range tokens {
-			s1 := detectors.Result{
+			s := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Slack,
 				Raw:          []byte(token),
 			}
 			if verify {
-				client := s.client
-				if s.client == nil {
-					client = defaultClient
-				}
-
+				client := common.SaneHttpClient()
 				req, err := http.NewRequestWithContext(ctx, "POST", verifyURL, nil)
 				if err != nil {
 					continue
 				}
-
 				req.Header.Add("Content-Type", "application/json; charset=utf-8")
 				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
-
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					var authResponse authRes
 					if err := json.NewDecoder(res.Body).Decode(&authResponse); err != nil {
-						s1.VerificationError = fmt.Errorf("failed to decode auth response: %w", err)
+						continue
 					}
-
-					if authResponse.Ok {
-						s1.Verified = true
-						// Slack API returns 200 even if the token is invalid. We need to check the error field.
-					} else if authResponse.Error == "invalid_auth" {
-						// The secret is determinately not verified (nothing to do)
-					} else {
-						s1.VerificationError = fmt.Errorf("unexpected error auth response %+v", authResponse.Error)
-					}
-				} else {
-					s1.VerificationError = err
+					s.Verified = authResponse.Ok
 				}
 			}
 
-			if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
-				continue
-			}
-
-			results = append(results, s1)
+			results = append(results, s)
 		}
 	}
 
-	return results, nil
+	return
 }
 
 func (s Scanner) Type() detectorspb.DetectorType {

pkg/detectors/slack/slack_test.go

@@ -9,15 +9,13 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
-
+	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestSlack_FromChunk(t *testing.T) {
+func TestScanner_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
 	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
@@ -26,28 +24,17 @@ func TestSlack_FromChunk(t *testing.T) {
 	}
 	secret := testSecrets.MustGetField("SLACK")
 	secretInactive := testSecrets.MustGetField("SLACK_INACTIVE")
-
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
-
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		wantResults         []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name        string
+		data        []byte
+		verify      bool
+		wantResults []detectors.Result
+		wantErr     bool
 	}{
 		{
-			name: "found, verified",
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
-				verify: true,
-			},
+			name:   "found, verified",
+			data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
+			verify: true,
 			wantResults: []detectors.Result{
 				{
 					DetectorType: detectorspb.DetectorType_Slack,
@@ -57,12 +44,9 @@ func TestSlack_FromChunk(t *testing.T) {
 			wantErr: false,
 		},
 		{
-			name: "found but unverified",
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
-				verify: true,
-			},
+			name:   "found but unverified",
+			data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
+			verify: true,
 			wantResults: []detectors.Result{
 				{
 					DetectorType: detectorspb.DetectorType_Slack,
@@ -71,62 +55,23 @@ func TestSlack_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
-				verify: true,
-			},
-			wantResults: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Slack,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "unexpected auth response",
-			s:    Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "unexpected_error"}`)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
-				verify: true,
-			},
-			wantResults: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Slack,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(context.Background(), tt.verify, tt.data)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("Slack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("Scanner.FromData) error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-
 			for i := range got {
 				if len(got[i].Raw) == 0 {
 					t.Fatal("no raw secret present")
 				}
 				got[i].Raw = nil
-
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
-				}
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
-			if diff := cmp.Diff(got, tt.wantResults, ignoreOpts); diff != "" {
-				t.Errorf("Slack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			if diff := pretty.Compare(got, tt.wantResults); diff != "" {
+				t.Errorf("Scanner.FromData) %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}

pkg/detectors/tailscale/tailscale.go

@@ -1,85 +0,0 @@
-package tailscale
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"net/url"
-	"regexp"
-	"strings"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-type Scanner struct{}
-
-// Ensure the Scanner satisfies the interface at compile time.
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
-	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(`\btskey-[a-z]+-[0-9A-Za-z_]+-[0-9A-Za-z_]+\b`)
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"tskey-", "tskey-api-", "tskey-oauth-"}
-}
-
-// FromData will find and optionally verify Tailscaleapi secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-
-	for _, match := range matches {
-		resMatch := strings.TrimSpace(match[0])
-
-		s1 := detectors.Result{
-			DetectorType: detectorspb.DetectorType_Tailscale,
-			Raw:          []byte(resMatch),
-		}
-
-		if verify {
-			const u = "https://api.tailscale.com/api/v2/secret-scanning/verify"
-			vals := url.Values{"key": []string{resMatch}}
-			req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, strings.NewReader(vals.Encode()))
-			if err != nil {
-				continue
-			}
-			req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				switch res.StatusCode {
-				case http.StatusNoContent:
-					s1.Verified = true
-				case http.StatusUnauthorized:
-					// The secret is determinately not verified (nothing to do)
-				default:
-					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
-				}
-			} else {
-				s1.VerificationError = err
-			}
-		}
-
-		// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-		if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-			continue
-		}
-
-		results = append(results, s1)
-	}
-
-	return results, nil
-}
-
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_Tailscale
-}

pkg/engine/defaults.go

@@ -6,7 +6,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraph"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -1543,7 +1542,6 @@ func DefaultDetectors() []detectors.Detector {
 		trufflehogenterprise.Scanner{},
 		salesforce.Scanner{},
 		sourcegraph.Scanner{},
-		tailscale.Scanner{},
 	}
 
 }

pkg/engine/engine.go

@@ -565,18 +565,11 @@ func (e *Engine) notifyResults(ctx context.Context) {
 		}
 		atomic.AddUint32(&e.numFoundResults, 1)
 
-		// Dedupe results by comparing the detector type, raw result, and source metadata.
-		// We want to avoid duplicate results with different decoder types, but we also
-		// want to include duplicate results with the same decoder type.
-		// Duplicate results with the same decoder type SHOULD have their own entry in the
-		// results list, this would happen if the same secret is found multiple times.
 		key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
-		if val, ok := e.dedupeCache.Get(key); ok {
-			if res, ok := val.(detectorspb.DecoderType); ok && res != r.DecoderType {
-				continue
-			}
+		if _, ok := e.dedupeCache.Get(key); ok {
+			continue
 		}
-		e.dedupeCache.Add(key, r.DecoderType)
+		e.dedupeCache.Add(key, struct{}{})
 
 		if r.Verified {
 			atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)

pkg/engine/engine_test.go

@@ -2,13 +2,8 @@ package engine
 
 import (
 	"fmt"
-	"path/filepath"
 	"testing"
-	"time"
 
-	"github.com/stretchr/testify/assert"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -176,32 +171,3 @@ func BenchmarkSupportsLineNumbersLoop(b *testing.B) {
 		_ = SupportsLineNumbers(sourceType)
 	}
 }
-
-// TestEngine_DuplicatSecrets is a test that detects ALL duplicate secrets with the same decoder.
-func TestEngine_DuplicatSecrets(t *testing.T) {
-	ctx := context.Background()
-
-	absPath, err := filepath.Abs("./testdata")
-	assert.Nil(t, err)
-
-	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
-	defer cancel()
-
-	e, err := Start(ctx,
-		WithConcurrency(1),
-		WithDecoders(decoders.DefaultDecoders()...),
-		WithDetectors(true, DefaultDetectors()...),
-		WithPrinter(new(discardPrinter)),
-	)
-	assert.Nil(t, err)
-
-	cfg := sources.FilesystemConfig{Paths: []string{absPath}}
-	if err := e.ScanFileSystem(ctx, cfg); err != nil {
-		return
-	}
-
-	// Wait for all the chunks to be processed.
-	assert.Nil(t, e.Finish(ctx))
-	want := uint64(5)
-	assert.Equal(t, want, e.GetMetrics().UnverifiedSecretsFound)
-}

pkg/engine/testdata/secrets.txt

@@ -1,6 +0,0 @@
-  1 aws AKIAWARWQKZNHMZBLY4I
-  2 secret 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
- sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
- sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
- sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
- sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90

pkg/giturl/giturl.go

@@ -119,18 +119,7 @@ func GenerateLink(repo, commit, file string, line int64) string {
 	case providerBitbucket:
 		return repo[:len(repo)-4] + "/commits/" + commit
 
-	case providerAzure:
-		baseLink := repo + "/commit/" + commit + "/" + file
-		if line > 0 {
-			baseLink += "?line=" + strconv.FormatInt(line, 10)
-		}
-		return baseLink
-
 	case providerGithub, providerGitlab:
-		// If the provider name isn't one of the cloud defaults, it is probably an on-prem github or gitlab.
-		// So do the same thing.
-		fallthrough
-	default:
 		var baseLink string
 		if file == "" {
 			baseLink = repo[:len(repo)-4] + "/commit/" + commit
@@ -141,5 +130,15 @@ func GenerateLink(repo, commit, file string, line int64) string {
 			}
 		}
 		return baseLink
+
+	case providerAzure:
+		baseLink := repo + "/commit/" + commit + "/" + file
+		if line > 0 {
+			baseLink += "?line=" + strconv.FormatInt(line, 10)
+		}
+		return baseLink
+
+	default:
+		return ""
 	}
 }

pkg/giturl/giturl_test.go

@@ -160,24 +160,6 @@ func TestGenerateLink(t *testing.T) {
 			},
 			want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=20",
 		},
-		{
-			name: "Unknown provider on-prem instance",
-			args: args{
-				repo:   "https://onprem.customdomain.com/org/repo.git",
-				commit: "xyz123",
-				file:   "main.go",
-				line:   int64(30),
-			},
-			want: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L30",
-		},
-		{
-			name: "Unknown provider on-prem instance - no file",
-			args: args{
-				repo:   "https://onprem.customdomain.com/org/repo.git",
-				commit: "xyz123",
-			},
-			want: "https://onprem.customdomain.com/org/repo/commit/xyz123",
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/handlers/archive.go

@@ -17,15 +17,12 @@ import (
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
 type ctxKey int
 
 const (
 	depthKey ctxKey = iota
-
-	errMaxArchiveDepthReached = "max archive depth reached"
 )
 
 var (
@@ -84,66 +81,51 @@ func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []b
 	return archiveChan
 }
 
-type decompressorInfo struct {
-	depth       int
-	reader      io.Reader
-	archiveChan chan []byte
-	archiver    archiver.Decompressor
-}
-
 // openArchive takes a reader and extracts the contents up to the maximum depth.
 func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan []byte) error {
 	if depth >= maxDepth {
-		return fmt.Errorf(errMaxArchiveDepthReached)
+		return fmt.Errorf("max archive depth reached")
 	}
-
-	format, arReader, err := archiver.Identify("", reader)
-	if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
-		return a.handleNonArchiveContent(ctx, arReader, archiveChan)
+	format, reader, err := archiver.Identify("", reader)
+	if err != nil {
+		if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
+			chunkSize := 10 * 1024
+			for {
+				chunk := make([]byte, chunkSize)
+				n, _ := reader.Read(chunk)
+				archiveChan <- chunk
+				if n < chunkSize {
+					break
+				}
+			}
+			return nil
+		}
+		return err
 	}
-
 	switch archive := format.(type) {
 	case archiver.Decompressor:
-		info := decompressorInfo{depth: depth, reader: arReader, archiveChan: archiveChan, archiver: archive}
-		return a.handleDecompressor(ctx, info)
-	case archiver.Extractor:
-		return archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
-	default:
-		return fmt.Errorf("unknown archive type: %s", format.Name())
-	}
-}
-
-func (a *Archive) handleNonArchiveContent(ctx context.Context, reader io.Reader, archiveChan chan []byte) error {
-	aCtx := logContext.AddLogger(ctx)
-	chunkReader := sources.NewChunkReader()
-	chunkResChan := chunkReader(aCtx, reader)
-	for data := range chunkResChan {
-		if err := data.Error(); err != nil {
-			aCtx.Logger().Error(err, "error reading chunk")
-			continue
-		}
-		if err := common.CancellableWrite(ctx, archiveChan, data.Bytes()); err != nil {
+		compReader, err := archive.OpenReader(reader)
+		if err != nil {
 			return err
 		}
+		fileBytes, err := a.ReadToMax(ctx, compReader)
+		if err != nil {
+			return err
+		}
+		newReader := bytes.NewReader(fileBytes)
+		return a.openArchive(ctx, depth+1, newReader, archiveChan)
+	case archiver.Extractor:
+		err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
+		if err != nil {
+			return err
+		}
+		return nil
 	}
-	return nil
-}
-
-func (a *Archive) handleDecompressor(ctx context.Context, info decompressorInfo) error {
-	compReader, err := info.archiver.OpenReader(info.reader)
-	if err != nil {
-		return err
-	}
-	fileBytes, err := a.ReadToMax(ctx, compReader)
-	if err != nil {
-		return err
-	}
-	newReader := bytes.NewReader(fileBytes)
-	return a.openArchive(ctx, info.depth+1, newReader, info.archiveChan)
+	return fmt.Errorf("Unknown archive type: %s", format.Name())
 }
 
 // IsFiletype returns true if the provided reader is an archive.
-func (a *Archive) IsFiletype(_ context.Context, reader io.Reader) (io.Reader, bool) {
+func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) {
 	format, readerB, err := archiver.Identify("", reader)
 	if err != nil {
 		return readerB, false
@@ -153,9 +135,8 @@ func (a *Archive) IsFiletype(_ context.Context, reader io.Reader) (io.Reader, bo
 		return readerB, true
 	case archiver.Decompressor:
 		return readerB, true
-	default:
-		return readerB, false
 	}
+	return readerB, false
 }
 
 // extractorHandler is applied to each file in an archiver.Extractor file.
@@ -199,7 +180,7 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
 			if e, ok := r.(error); ok {
 				err = e
 			} else {
-				err = fmt.Errorf("panic occurred: %v", r)
+				err = fmt.Errorf("Panic occurred: %v", r)
 			}
 			logger.Error(err, "Panic occurred when reading archive")
 		}
@@ -302,7 +283,7 @@ func (a *Archive) HandleSpecialized(ctx logContext.Context, reader io.Reader) (i
 // The caller is responsible for closing the returned reader.
 func (a *Archive) extractDebContent(ctx logContext.Context, file io.Reader) (io.ReadCloser, error) {
 	if a.currentDepth >= maxDepth {
-		return nil, fmt.Errorf(errMaxArchiveDepthReached)
+		return nil, fmt.Errorf("max archive depth reached")
 	}
 
 	tmpEnv, err := a.createTempEnv(ctx, file)

pkg/pb/detectorspb/detectors.pb.go

@@ -1000,7 +1000,6 @@ const (
 	DetectorType_HuggingFace                   DetectorType = 926
 	DetectorType_Snowflake                     DetectorType = 927
 	DetectorType_Sourcegraph                   DetectorType = 928
-	DetectorType_Tailscale                     DetectorType = 929
 )
 
 // Enum value maps for DetectorType.
@@ -1931,7 +1930,6 @@ var (
 		926: "HuggingFace",
 		927: "Snowflake",
 		928: "Sourcegraph",
-		929: "Tailscale",
 	}
 	DetectorType_value = map[string]int32{
 		"Alibaba":                       0,
@@ -2859,7 +2857,6 @@ var (
 		"HuggingFace":                   926,
 		"Snowflake":                     927,
 		"Sourcegraph":                   928,
-		"Tailscale":                     929,
 	}
 )
 
@@ -3238,7 +3235,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
 	0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
 	0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
-	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xae, 0x74, 0x0a, 0x0c, 0x44,
+	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x9e, 0x74, 0x0a, 0x0c, 0x44,
 	0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
 	0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
 	0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4168,8 +4165,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x6f, 0x72, 0x63, 0x65, 0x10, 0x9d, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69,
 	0x6e, 0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x6e, 0x6f,
 	0x77, 0x66, 0x6c, 0x61, 0x6b, 0x65, 0x10, 0x9f, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x53, 0x6f, 0x75,
-	0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x10, 0xa0, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x54,
-	0x61, 0x69, 0x6c, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x10, 0xa1, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67,
+	0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x10, 0xa0, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67,
 	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
 	0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
 	0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64,

pkg/sources/circleci/circleci.go

@@ -75,7 +75,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	projects, err := s.projects(ctx)
 	if err != nil {
 		return fmt.Errorf("error getting projects: %w", err)

pkg/sources/docker/docker.go

@@ -2,6 +2,7 @@ package docker
 
 import (
 	"archive/tar"
+	"bytes"
 	"compress/gzip"
 	"errors"
 	"fmt"
@@ -13,11 +14,9 @@ import (
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
-	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -25,16 +24,17 @@ import (
 )
 
 type Source struct {
-	name        string
-	sourceId    int64
-	jobId       int64
-	verify      bool
-	concurrency int
-	conn        sourcespb.Docker
+	name     string
+	sourceId int64
+	jobId    int64
+	verify   bool
+	conn     sourcespb.Docker
 	sources.Progress
 	sources.CommonSourceUnitUnmarshaller
 }
 
+var FilesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
+
 // Ensure the Source satisfies the interfaces at compile time.
 var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
@@ -59,7 +59,6 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 	s.sourceId = sourceId
 	s.jobId = jobId
 	s.verify = verify
-	s.concurrency = concurrency
 
 	// Reset metrics for this source at initialization time.
 	dockerImagesScanned.WithLabelValues(s.name).Set(0)
@@ -72,209 +71,166 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 	return nil
 }
 
-type imageInfo struct {
-	image v1.Image
-	base  string
-	tag   string
-}
-
-type layerInfo struct {
-	digest v1.Hash
-	base   string
-	tag    string
-}
-
 // Chunks emits data over a channel that is decoded and scanned for secrets.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
-	ctx = context.WithValues(ctx, "source_type", s.Type(), "source_name", s.name)
-
-	workers := new(errgroup.Group)
-	workers.SetLimit(s.concurrency)
-
-	scanErrs := sources.NewScanErrors()
-	for _, image := range s.conn.GetImages() {
-		image := image
-		workers.Go(func() error {
-			if common.IsDone(ctx) {
-				return nil
-			}
-
-			imgInfo, err := s.processImage(ctx, image)
-			if err != nil {
-				scanErrs.Add(err)
-				return nil
-			}
-
-			ctx = context.WithValues(ctx, "image", imgInfo.base, "tag", imgInfo.tag)
-			ctx.Logger().V(2).Info("scanning image")
-
-			layers, err := imgInfo.image.Layers()
-			if err != nil {
-				scanErrs.Add(err)
-				return nil
-			}
-
-			for _, layer := range layers {
-				if err := s.processLayer(ctx, layer, imgInfo, chunksChan); err != nil {
-					scanErrs.Add(err)
-					return nil
-				}
-				dockerLayersScanned.WithLabelValues(s.name).Inc()
-			}
-
-			dockerImagesScanned.WithLabelValues(s.name).Inc()
-
-			return nil
-		})
-	}
-	_ = workers.Wait()
-	if scanErrs.Count() > 0 {
-		ctx.Logger().V(2).Info("scan errors", "errors", scanErrs.String())
-	}
-
-	return nil
-}
-
-// processImage processes an individual image and prepares it for further processing.
-func (s *Source) processImage(ctx context.Context, image string) (imageInfo, error) {
-	var (
-		imgInfo   imageInfo
-		hasDigest bool
-		imageName name.Reference
-	)
-
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	remoteOpts, err := s.remoteOpts()
 	if err != nil {
-		return imgInfo, err
+		return err
 	}
 
-	const filePrefix = "file://"
-	if strings.HasPrefix(image, filePrefix) {
-		image = strings.TrimPrefix(image, filePrefix)
-		imgInfo.base = image
-		imgInfo.image, err = tarball.ImageFromPath(image, nil)
-		if err != nil {
-			return imgInfo, err
-		}
-	} else {
-		imgInfo.base, imgInfo.tag, hasDigest = baseAndTagFromImage(image)
+	logger := ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
 
-		if hasDigest {
-			imageName, err = name.NewDigest(image)
+	for _, image := range s.conn.GetImages() {
+		var img v1.Image
+		var err error
+		var base, tag string
+		var hasDigest bool
+		var imageName name.Reference
+
+		if strings.HasPrefix(image, "file://") {
+			image = strings.TrimPrefix(image, "file://")
+			base = image
+			img, err = tarball.ImageFromPath(image, nil)
+			if err != nil {
+				return err
+			}
 		} else {
-			imageName, err = name.NewTag(image)
-		}
-		if err != nil {
-			return imgInfo, err
+			base, tag, hasDigest = baseAndTagFromImage(image)
+
+			if hasDigest {
+				imageName, err = name.NewDigest(image)
+				if err != nil {
+					return err
+				}
+			} else {
+				imageName, err = name.NewTag(image)
+				if err != nil {
+					return err
+				}
+			}
+
+			img, err = remote.Image(imageName, remoteOpts...)
+			if err != nil {
+				return err
+			}
 		}
 
-		imgInfo.image, err = remote.Image(imageName, remoteOpts...)
-		if err != nil {
-			return imgInfo, err
-		}
-	}
+		logger = logger.WithValues("image", base, "tag", tag)
+		logger.V(2).Info("scanning image")
 
-	ctx.Logger().WithValues("image", imgInfo.base, "tag", imgInfo.tag).V(2).Info("scanning image")
-
-	return imgInfo, nil
-}
-
-// processLayer processes an individual layer of an image.
-func (s *Source) processLayer(ctx context.Context, layer v1.Layer, imgInfo imageInfo, chunksChan chan *sources.Chunk) error {
-	layerInfo := layerInfo{
-		base: imgInfo.base,
-		tag:  imgInfo.tag,
-	}
-
-	var err error
-	layerInfo.digest, err = layer.Digest()
-	if err != nil {
-		return err
-	}
-
-	ctx.Logger().WithValues("layer", layerInfo.digest.String()).V(2).Info("scanning layer")
-
-	rc, err := layer.Compressed()
-	if err != nil {
-		return err
-	}
-	defer rc.Close()
-
-	gzipReader, err := gzip.NewReader(rc)
-	if err != nil {
-		return err
-	}
-	defer gzipReader.Close()
-
-	tarReader := tar.NewReader(gzipReader)
-	for {
-		header, err := tarReader.Next()
-		if errors.Is(err, io.EOF) {
-			break
-		}
+		layers, err := img.Layers()
 		if err != nil {
 			return err
 		}
 
-		info := chunkProcessingInfo{size: header.Size, name: header.Name, reader: tarReader, layer: layerInfo}
-		if err := s.processChunk(ctx, info, chunksChan); err != nil {
-			return err
-		}
-	}
+		for _, layer := range layers {
 
-	return nil
-}
+			digest, err := layer.Digest()
+			if err != nil {
+				return err
+			}
 
-type chunkProcessingInfo struct {
-	size   int64
-	name   string
-	reader io.Reader
-	layer  layerInfo
-}
+			logger = logger.WithValues("layer", digest.String())
+			logger.V(2).Info("scanning layer")
 
-// processChunk processes an individual chunk of a layer.
-func (s *Source) processChunk(ctx context.Context, info chunkProcessingInfo, chunksChan chan *sources.Chunk) error {
-	const filesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
-	if info.size > filesizeLimitBytes {
-		ctx.Logger().V(4).Info("skipping large file", "file", info.name, "size", info.size)
-		return nil
-	}
+			rc, err := layer.Compressed()
+			if err != nil {
+				return err
+			}
 
-	chunkReader := sources.NewChunkReader()
-	chunkResChan := chunkReader(ctx, info.reader)
+			defer rc.Close()
 
-	for data := range chunkResChan {
-		if err := data.Error(); err != nil {
-			ctx.Logger().Error(err, "error reading chunk.")
-			continue
-		}
+			gzipReader, err := gzip.NewReader(rc)
+			if err != nil {
+				return err
+			}
 
-		chunk := &sources.Chunk{
-			SourceType: s.Type(),
-			SourceName: s.name,
-			SourceID:   s.SourceID(),
-			SourceMetadata: &source_metadatapb.MetaData{
-				Data: &source_metadatapb.MetaData_Docker{
-					Docker: &source_metadatapb.Docker{
-						File:  "/" + info.name,
-						Image: info.layer.base,
-						Tag:   info.layer.tag,
-						Layer: info.layer.digest.String(),
+			defer gzipReader.Close()
+
+			tarReader := tar.NewReader(gzipReader)
+
+			for {
+				header, err := tarReader.Next()
+				if errors.Is(err, io.EOF) {
+					break // End of archive
+				}
+				if err != nil {
+					return err
+				}
+
+				// Skip files larger than FilesizeLimitBytes
+				if header.Size > FilesizeLimitBytes {
+					continue
+				}
+
+				file := bytes.NewBuffer(make([]byte, 0, header.Size))
+
+				_, err = io.Copy(file, tarReader)
+				if err != nil {
+					return err
+				}
+
+				chunk := &sources.Chunk{
+					SourceType: s.Type(),
+					SourceName: s.name,
+					SourceID:   s.SourceID(),
+					JobID:      s.JobID(),
+					Data:       file.Bytes(),
+					SourceMetadata: &source_metadatapb.MetaData{
+						Data: &source_metadatapb.MetaData_Docker{
+							Docker: &source_metadatapb.Docker{
+								File:  "/" + header.Name, // Prepend /, because the file is relative to the root of the image
+								Image: base,
+								Tag:   tag,
+								Layer: digest.String(),
+							},
+						},
 					},
-				},
-			},
-			Verify: s.verify,
-		}
-		chunk.Data = data.Bytes()
+					Verify: s.verify,
+				}
 
-		if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
-			return err
+				select {
+				case chunksChan <- chunk:
+				case <-ctx.Done():
+					return ctx.Err()
+				}
+			}
+
+			dockerLayersScanned.WithLabelValues(s.name).Inc()
 		}
+
+		dockerImagesScanned.WithLabelValues(s.name).Inc()
 	}
 
 	return nil
 }
 
+func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
+	regRepoDelimiter := "/"
+	tagDelim := ":"
+	digestDelim := "@"
+
+	// Split on digest first, if present.
+	parts := strings.SplitN(image, digestDelim, 2)
+	if len(parts) > 1 {
+		base = parts[0]
+		tag = parts[1]
+		hasDigest = true
+		return
+	}
+	parts = strings.Split(image, tagDelim)
+	// Verify that we aren't confusing a tag for a hostname w/ port for the purposes of weak validation.
+	if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelimiter) {
+		base = strings.Join(parts[:len(parts)-1], tagDelim)
+		tag = parts[len(parts)-1]
+	} else {
+		base = image
+		tag = "latest"
+	}
+
+	return
+}
+
 func (s *Source) remoteOpts() ([]remote.Option, error) {
 	switch s.conn.GetCredential().(type) {
 	case *sourcespb.Docker_Unauthenticated:
@@ -300,40 +256,3 @@ func (s *Source) remoteOpts() ([]remote.Option, error) {
 		return nil, fmt.Errorf("unknown credential type: %T", s.conn.Credential)
 	}
 }
-
-func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
-	if base, tag, hasDigest = extractDigest(image); hasDigest {
-		return base, tag, true
-	}
-
-	base, tag = extractTagOrUseDefault(image)
-	return base, tag, false
-}
-
-// extractDigest tries to split the image string on the digest delimiter.
-// If successful, it means the image has a digest.
-func extractDigest(image string) (base, tag string, hasDigest bool) {
-	const digestDelim = "@"
-
-	if parts := strings.SplitN(image, digestDelim, 2); len(parts) > 1 {
-		return parts[0], parts[1], true
-	}
-	return "", "", false
-}
-
-// extractTagOrUseDefault extracts the tag from the image string.
-// If no tag is found, it defaults to "latest".
-func extractTagOrUseDefault(image string) (base, tag string) {
-	const (
-		tagDelim     = ":"
-		regRepoDelim = "/"
-	)
-
-	parts := strings.Split(image, tagDelim)
-
-	// Check if the last part is not a hostname with a port (for weak validation)
-	if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelim) {
-		return strings.Join(parts[:len(parts)-1], tagDelim), parts[len(parts)-1]
-	}
-	return image, "latest"
-}

pkg/sources/filesystem/filesystem.go

@@ -77,7 +77,7 @@ func (s *Source) WithFilter(filter *common.Filter) {
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	for i, path := range s.paths {
 		logger := ctx.Logger().WithValues("path", path)
 		if common.IsDone(ctx) {

pkg/sources/gcs/gcs.go

@@ -248,7 +248,7 @@ func (s *Source) enumerate(ctx context.Context) error {
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	persistableCache := s.setupCache(ctx)
 
 	objectCh, err := s.gcsManager.ListObjects(ctx)

pkg/sources/git/git.go

@@ -153,7 +153,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	if err := s.scanRepos(ctx, chunksChan); err != nil {
 		return err
 	}

pkg/sources/github/github.go

@@ -413,7 +413,7 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	apiEndpoint := s.conn.Endpoint
 	if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) {
 		apiEndpoint = "https://api.github.com"

pkg/sources/gitlab/gitlab.go

@@ -139,7 +139,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	// Start client.
 	apiClient, err := s.newClient()
 	if err != nil {

pkg/sources/s3/s3.go

@@ -31,7 +31,6 @@ import (
 )
 
 const (
-	defaultAWSRegion     = "us-east-1"
 	defaultMaxObjectSize = 250 * 1024 * 1024 // 250 MiB
 	maxObjectSizeLimit   = 250 * 1024 * 1024 // 250 MiB
 )
@@ -54,7 +53,6 @@ type Source struct {
 // Ensure the Source satisfies the interfaces at compile time
 var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
-var _ sources.Validator = (*Source)(nil)
 
 // Type returns the type of source
 func (s *Source) Type() sourcespb.SourceType {
@@ -95,23 +93,6 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 	return nil
 }
 
-func (s *Source) Validate(ctx context.Context) []error {
-	var errors []error
-	visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
-		roleErrs := s.validateBucketAccess(c, defaultRegionClient, roleArn, buckets)
-		if len(roleErrs) > 0 {
-			errors = append(errors, roleErrs...)
-		}
-	}
-
-	err := s.visitRoles(ctx, visitor)
-	if err != nil {
-		errors = append(errors, err)
-	}
-
-	return errors
-}
-
 // setMaxObjectSize sets the maximum size of objects that will be scanned. If
 // not set, set to a negative number, or set larger than the
 // maxObjectSizeLimit, the defaultMaxObjectSizeLimit will be used.
@@ -172,7 +153,7 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
 
 	res, err := client.ListBuckets(&s3.ListBucketsInput{})
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("could not list s3 buckets: %w", err)
 	}
 
 	var bucketsToScan []string
@@ -182,30 +163,34 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
 	return bucketsToScan, nil
 }
 
-func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bucketsToScan []string, chunksChan chan *sources.Chunk) {
+func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bucketsToScan []string, chunksChan chan *sources.Chunk) error {
+	const defaultAWSRegion = "us-east-1"
 	objectCount := uint64(0)
 
-	logger := s.log
-	if role != "" {
-		logger = logger.WithValues("roleArn", role)
-	}
-
 	for i, bucket := range bucketsToScan {
-		logger := logger.WithValues("bucket", bucket)
-
 		if common.IsDone(ctx) {
-			return
+			return nil
 		}
 
 		s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "")
-		logger.Info("Scanning bucket")
-
-		regionalClient, err := s.getRegionalClientForBucket(ctx, client, role, bucket)
+		s.log.Info("Scanning bucket", "bucket", bucket)
+		region, err := s3manager.GetBucketRegionWithClient(ctx, client, bucket)
 		if err != nil {
-			logger.Error(err, "could not get regional client for bucket")
+			s.log.Error(err, "could not get s3 region for bucket", "bucket: ", bucket)
 			continue
 		}
 
+		var regionalClient *s3.S3
+		if region != defaultAWSRegion {
+			regionalClient, err = s.newClient(region, role)
+			if err != nil {
+				s.log.Error(err, "could not make regional s3 client")
+				continue
+			}
+		} else {
+			regionalClient = client
+		}
+
 		errorCount := sync.Map{}
 
 		err = regionalClient.ListObjectsV2PagesWithContext(
@@ -216,46 +201,40 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
 			})
 
 		if err != nil {
-			if role == "" {
-				logger.Error(err, "could not list objects in bucket")
-			} else {
-				// Our documentation blesses specifying a role to assume without specifying buckets to scan, which will
-				// often cause this to happen a lot (because in that case the scanner tries to scan every bucket in the
-				// account, but the role probably doesn't have access to all of them). This makes it expected behavior
-				// and therefore not an error.
-				logger.V(3).Info("could not list objects in bucket",
-					"err", err)
-			}
+			s.log.Error(err, "could not list objects in s3 bucket", "bucket: ", bucket)
+			continue
 		}
 	}
 	s.SetProgressComplete(len(bucketsToScan), len(bucketsToScan), fmt.Sprintf("Completed scanning source %s. %d objects scanned.", s.name, objectCount), "")
+	return nil
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
-	visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
-		s.scanBuckets(c, defaultRegionClient, roleArn, buckets, chunksChan)
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+	const defaultAWSRegion = "us-east-1"
+
+	roles := s.conn.Roles
+	if len(roles) == 0 {
+		roles = []string{""}
 	}
 
-	return s.visitRoles(ctx, visitor)
-}
+	for _, role := range roles {
+		client, err := s.newClient(defaultAWSRegion, role)
+		if err != nil {
+			return errors.WrapPrefix(err, "could not create s3 client", 0)
+		}
 
-func (s *Source) getRegionalClientForBucket(ctx context.Context, defaultRegionClient *s3.S3, role, bucket string) (*s3.S3, error) {
-	region, err := s3manager.GetBucketRegionWithClient(ctx, defaultRegionClient, bucket)
-	if err != nil {
-		return nil, errors.WrapPrefix(err, "could not get s3 region for bucket", 0)
+		bucketsToScan, err := s.getBucketsToScan(client)
+		if err != nil {
+			return err
+		}
+
+		if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil {
+			return err
+		}
 	}
 
-	if region == defaultAWSRegion {
-		return defaultRegionClient, nil
-	}
-
-	regionalClient, err := s.newClient(region, role)
-	if err != nil {
-		return nil, errors.WrapPrefix(err, "could not create regional s3 client", 0)
-	}
-
-	return regionalClient, nil
+	return nil
 }
 
 // pageChunker emits chunks onto the given channel from a page
@@ -417,65 +396,6 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 	_ = s.jobPool.Wait()
 }
 
-func (s *Source) validateBucketAccess(ctx context.Context, client *s3.S3, roleArn string, buckets []string) []error {
-	shouldHaveAccessToAllBuckets := roleArn == ""
-	wasAbleToListAnyBucket := false
-	var errors []error
-
-	for _, bucket := range buckets {
-		if common.IsDone(ctx) {
-			return append(errors, ctx.Err())
-		}
-
-		regionalClient, err := s.getRegionalClientForBucket(ctx, client, roleArn, bucket)
-		if err != nil {
-			errors = append(errors, fmt.Errorf("could not get regional client for bucket %q: %w", bucket, err))
-			continue
-		}
-
-		_, err = regionalClient.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &bucket})
-
-		if err == nil {
-			wasAbleToListAnyBucket = true
-		} else if shouldHaveAccessToAllBuckets {
-			errors = append(errors, fmt.Errorf("could not list objects in bucket %q: %w", bucket, err))
-		}
-	}
-
-	if !wasAbleToListAnyBucket {
-		if roleArn == "" {
-			errors = append(errors, fmt.Errorf("could not list objects in any bucket"))
-		} else {
-			errors = append(errors, fmt.Errorf("role %q could not list objects in any bucket", roleArn))
-		}
-	}
-
-	return errors
-}
-
-func (s *Source) visitRoles(ctx context.Context, f func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string)) error {
-	roles := s.conn.Roles
-	if len(roles) == 0 {
-		roles = []string{""}
-	}
-
-	for _, role := range roles {
-		client, err := s.newClient(defaultAWSRegion, role)
-		if err != nil {
-			return errors.WrapPrefix(err, "could not create s3 client", 0)
-		}
-
-		bucketsToScan, err := s.getBucketsToScan(client)
-		if err != nil {
-			return fmt.Errorf("role %q could not list any s3 buckets for scanning: %w", role, err)
-		}
-
-		f(ctx, client, role, bucketsToScan)
-	}
-
-	return nil
-}
-
 // S3 links currently have the general format of:
 // https://[bucket].s3[.region unless us-east-1].amazonaws.com/[key]
 func makeS3Link(bucket, region, key string) string {

pkg/sources/s3/s3_integration_test.go

@@ -4,14 +4,10 @@
 package s3
 
 import (
-	"fmt"
-	"sync"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
 	"google.golang.org/protobuf/types/known/anypb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -49,127 +45,3 @@ func TestSource_ChunksCount(t *testing.T) {
 	}
 	assert.Greater(t, got, wantChunkCount)
 }
-
-func TestSource_Validate(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
-	defer cancel()
-
-	secret, err := common.GetTestSecret(ctx)
-	if err != nil {
-		t.Fatal(fmt.Errorf("failed to access secret: %v", err))
-	}
-
-	s3key := secret.MustGetField("AWS_S3_KEY")
-	s3secret := secret.MustGetField("AWS_S3_SECRET")
-
-	tests := []struct {
-		name         string
-		roles        []string
-		buckets      []string
-		wantErrCount int
-	}{
-		{
-			name: "buckets without roles, can access all buckets",
-			buckets: []string{
-				"truffletestbucket-s3-tests",
-			},
-			wantErrCount: 0,
-		},
-		{
-			name: "buckets without roles, one error per inaccessible bucket",
-			buckets: []string{
-				"truffletestbucket-s3-tests",
-				"truffletestbucket-s3-role-assumption",
-				"truffletestbucket-no-access",
-			},
-			wantErrCount: 2,
-		},
-		{
-			name: "roles without buckets, all can access at least one account bucket",
-			roles: []string{
-				"arn:aws:iam::619888638459:role/s3-test-assume-role",
-			},
-			wantErrCount: 0,
-		},
-		{
-			name: "roles without buckets, one error per role that cannot access any account buckets",
-			roles: []string{
-				"arn:aws:iam::619888638459:role/s3-test-assume-role",
-				"arn:aws:iam::619888638459:role/test-no-access",
-			},
-			wantErrCount: 1,
-		},
-		{
-			name: "role and buckets, can access at least one bucket",
-			roles: []string{
-				"arn:aws:iam::619888638459:role/s3-test-assume-role",
-			},
-			buckets: []string{
-				"truffletestbucket-s3-role-assumption",
-				"truffletestbucket-no-access",
-			},
-			wantErrCount: 0,
-		},
-		{
-			name: "roles and buckets, one error per role that cannot access at least one bucket",
-			roles: []string{
-				"arn:aws:iam::619888638459:role/s3-test-assume-role",
-				"arn:aws:iam::619888638459:role/test-no-access",
-			},
-			buckets: []string{
-				"truffletestbucket-s3-role-assumption",
-				"truffletestbucket-no-access",
-			},
-			wantErrCount: 1,
-		},
-		{
-			name: "role and buckets, a bucket doesn't even exist",
-			roles: []string{
-				"arn:aws:iam::619888638459:role/s3-test-assume-role",
-			},
-			buckets: []string{
-				"truffletestbucket-s3-role-assumption",
-				"not-a-real-bucket-asljdhmglasjgvklhsdaljfh", // need a bucket name that nobody is likely to ever create
-			},
-			wantErrCount: 1,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
-			var cancelOnce sync.Once
-			defer cancelOnce.Do(cancel)
-
-			// These are used by the tests that assume roles
-			t.Setenv("AWS_ACCESS_KEY_ID", s3key)
-			t.Setenv("AWS_SECRET_ACCESS_KEY", s3secret)
-
-			s := &Source{}
-
-			conn, err := anypb.New(&sourcespb.S3{
-				// These are used by the tests that don't assume roles
-				Credential: &sourcespb.S3_AccessKey{
-					AccessKey: &credentialspb.KeySecret{
-						Key:    s3key,
-						Secret: s3secret,
-					},
-				},
-				Buckets: tt.buckets,
-				Roles:   tt.roles,
-			})
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = s.Init(ctx, tt.name, 0, 0, false, conn, 0)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			errs := s.Validate(ctx)
-
-			assert.Equal(t, tt.wantErrCount, len(errs))
-		})
-	}
-}

pkg/sources/source_manager_test.go

@@ -6,11 +6,10 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"google.golang.org/protobuf/types/known/anypb"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"google.golang.org/protobuf/types/known/anypb"
 )
 
 // DummySource implements Source and is used for testing a SourceManager.
@@ -32,7 +31,7 @@ func (d *DummySource) GetProgress() *Progress { return nil }
 
 // Interface to easily test different chunking methods.
 type chunker interface {
-	Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error
+	Chunks(context.Context, chan *Chunk) error
 	ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
 	Enumerate(ctx context.Context, reporter UnitReporter) error
 }
@@ -43,7 +42,7 @@ type counterChunker struct {
 	count        int
 }
 
-func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
+func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
 	for i := 0; i < c.count; i++ {
 		select {
 		case ch <- &Chunk{Data: []byte{c.chunkCounter}}:
@@ -76,9 +75,9 @@ func (c *counterChunker) ChunkUnit(ctx context.Context, unit SourceUnit, reporte
 // Chunk method that always returns an error.
 type errorChunker struct{ error }
 
-func (c errorChunker) Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error { return c }
-func (c errorChunker) Enumerate(context.Context, UnitReporter) error                { return c }
-func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error   { return c }
+func (c errorChunker) Chunks(context.Context, chan *Chunk) error                  { return c }
+func (c errorChunker) Enumerate(context.Context, UnitReporter) error              { return c }
+func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
 
 // enrollDummy is a helper function to enroll a DummySource with a SourceManager.
 func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {
@@ -177,7 +176,7 @@ type unitChunk struct {
 
 type unitChunker struct{ steps []unitChunk }
 
-func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
+func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
 	for _, step := range c.steps {
 		if step.err != "" {
 			continue
@@ -295,9 +294,7 @@ type callbackChunker struct {
 	cb func(context.Context, chan *Chunk) error
 }
 
-func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
-	return c.cb(ctx, ch)
-}
+func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk) error           { return c.cb(ctx, ch) }
 func (c callbackChunker) Enumerate(context.Context, UnitReporter) error              { return nil }
 func (c callbackChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return nil }
 

pkg/sources/sources.go

@@ -30,16 +30,6 @@ type Chunk struct {
 	Verify bool
 }
 
-// ChunkingTarget specifies criteria for a targeted chunking process.
-// Instead of collecting data indiscriminately, this struct allows the caller
-// to specify particular subsets of data they're interested in. This becomes
-// especially useful when one needs to verify or recheck specific data points
-// without processing the entire dataset.
-type ChunkingTarget struct {
-	// QueryCriteria represents specific parameters or conditions to target the chunking process.
-	QueryCriteria source_metadatapb.MetaData
-}
-
 // Source defines the interface required to implement a source chunker.
 type Source interface {
 	// Type returns the source type, used for matching against configuration and jobs.
@@ -50,12 +40,8 @@ type Source interface {
 	JobID() int64
 	// Init initializes the source.
 	Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error
-	// Chunks emits data over a channel which is then decoded and scanned for secrets.
-	// By default, data is obtained indiscriminately. However, by providing one or more
-	// ChunkingTarget parameters, the caller can direct the function to retrieve
-	// specific chunks of data. This targeted approach allows for efficient and
-	// intentional data processing, beneficial when verifying or rechecking specific data points.
-	Chunks(ctx context.Context, chunksChan chan *Chunk, targets ...ChunkingTarget) error
+	// Chunks emits data over a channel that is decoded and scanned for secrets.
+	Chunks(ctx context.Context, chunksChan chan *Chunk) error
 	// GetProgress is the completion progress (percentage) for Scanned Source.
 	GetProgress() *Progress
 }

pkg/sources/syslog/syslog.go

@@ -182,7 +182,7 @@ func (s *Source) verifyConnectionConfig() error {
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	switch {
 	case s.conn.TlsCert != nilString || s.conn.TlsKey != nilString:
 		cert, err := tls.X509KeyPair([]byte(s.conn.TlsCert), []byte(s.conn.TlsKey))

proto/detectors.proto

@@ -937,7 +937,6 @@ enum DetectorType {
   HuggingFace = 926;
   Snowflake = 927;
   Sourcegraph = 928;
-  Tailscale = 929;
 }
 
 message Result {