Merge branch 'feature-bytification' into detectors-9

go.mod

@@ -11,7 +11,6 @@ require (
 	cloud.google.com/go/secretmanager v1.11.1
 	cloud.google.com/go/storage v1.31.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
-	github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1
 	github.com/BobuSumisu/aho-corasick v1.0.3
 	github.com/TheZeroSlave/zapsentry v1.17.0
 	github.com/aws/aws-sdk-go v1.44.83
@@ -204,7 +203,7 @@ require (
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/montanaflynn/stats v0.6.6 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
 	github.com/mtibben/percent v0.2.1 // indirect
 	github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
 	github.com/muesli/cancelreader v0.2.2 // indirect

go.sum

@@ -304,7 +304,6 @@ github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA=
 github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0=
 github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
-github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
@@ -383,7 +382,6 @@ github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8I
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
 github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -520,9 +518,8 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8Ie
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
-github.com/montanaflynn/stats v0.6.6 h1:Duep6KMIDpY4Yo11iFsvyqJDyfzLF9+sndUKT+v64GQ=
-github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs=
 github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns=
 github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 h1:kMlmsLSbjkikxQJ1IPwaM+7LJ9ltFu/fi8CRzvSnQmA=
@@ -568,7 +565,6 @@ github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw
 github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
 github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
 github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
-github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
 github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8=

main.go

@@ -105,11 +105,11 @@ var (
 	filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
 	filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
 
-	s3Scan              = cli.Command("s3", "Find credentials in S3 buckets.")
-	s3ScanKey           = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
-	s3ScanRoleArns      = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
-	s3ScanSecret        = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
-	s3ScanSessionToken  = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
+	s3Scan    = cli.Command("s3", "Find credentials in S3 buckets.")
+	s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
+	s3ScanRoleArns     = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
+	s3ScanSecret       = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
+	s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
 	s3ScanCloudEnv      = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
 	s3ScanBuckets       = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
 	s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()
@@ -354,10 +354,6 @@ func run(state overseer.State) {
 		printer = new(output.PlainPrinter)
 	}
 
-	if !*jsonLegacy && !*jsonOut {
-		fmt.Fprintf(os.Stderr, "🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
-	}
-
 	e, err := engine.Start(ctx,
 		engine.WithConcurrency(uint8(*concurrency)),
 		engine.WithDecoders(decoders.DefaultDecoders()...),
@@ -473,7 +469,7 @@ func run(state overseer.State) {
 			Secret:        *s3ScanSecret,
 			SessionToken:  *s3ScanSessionToken,
 			Buckets:       *s3ScanBuckets,
-			Roles:         *s3ScanRoleArns,
+			Roles:	       *s3ScanRoleArns,
 			CloudCred:     *s3ScanCloudEnv,
 			MaxObjectSize: int64(*s3ScanMaxObjectSize),
 		}
@@ -529,6 +525,10 @@ func run(state overseer.State) {
 		}
 	}
 
+	if !*jsonLegacy && !*jsonOut {
+		fmt.Fprintf(os.Stderr, "🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
+	}
+
 	// Wait for all workers to finish.
 	if err = e.Finish(ctx); err != nil {
 		logFatal(err, "engine failed to finish execution")

pkg/detectors/aws/aws.go

@@ -118,21 +118,118 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			if verify {
-				verified, extraData, verificationErr := s.verifyMatch(ctx, resIDMatch, resSecretMatch, true)
-				s1.Verified = verified
-				s1.ExtraData = extraData
-				s1.VerificationError = verificationErr
+				// REQUEST VALUES.
+				method := "GET"
+				service := "sts"
+				host := "sts.amazonaws.com"
+				region := "us-east-1"
+				endpoint := "https://sts.amazonaws.com"
+				now := time.Now().UTC()
+				datestamp := now.Format("20060102")
+				amzDate := now.Format("20060102T150405Z0700")
+
+				req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
+				if err != nil {
+					continue
+				}
+				req.Header.Set("Accept", "application/json")
+
+				// TASK 1: CREATE A CANONICAL REQUEST.
+				// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+				canonicalURI := "/"
+				canonicalHeaders := "host:" + host + "\n"
+				signedHeaders := "host"
+				algorithm := "AWS4-HMAC-SHA256"
+				credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
+
+				params := req.URL.Query()
+				params.Add("Action", "GetCallerIdentity")
+				params.Add("Version", "2011-06-15")
+				params.Add("X-Amz-Algorithm", algorithm)
+				params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
+				params.Add("X-Amz-Date", amzDate)
+				params.Add("X-Amz-Expires", "30")
+				params.Add("X-Amz-SignedHeaders", signedHeaders)
+
+				canonicalQuerystring := params.Encode()
+				payloadHash := GetHash("") // empty payload
+				canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
+
+				// TASK 2: CREATE THE STRING TO SIGN.
+				stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
+
+				// TASK 3: CALCULATE THE SIGNATURE.
+				// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
+				hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
+				hash = GetHMAC(hash, []byte(region))
+				hash = GetHMAC(hash, []byte(service))
+				hash = GetHMAC(hash, []byte("aws4_request"))
+
+				signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
+				signature := hex.EncodeToString(signature2)
+
+				// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
+				params.Add("X-Amz-Signature", signature)
+				req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
+				req.URL.RawQuery = params.Encode()
+
+				client := s.verificationClient
+				if client == nil {
+					client = defaultVerificationClient
+				}
+				res, err := client.Do(req)
+				if err == nil {
+
+					if res.StatusCode >= 200 && res.StatusCode < 300 {
+						identityInfo := identityRes{}
+						err := json.NewDecoder(res.Body).Decode(&identityInfo)
+						if err == nil {
+							s1.Verified = true
+							s1.ExtraData = map[string]string{
+								"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
+								"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
+								"arn":     identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
+							}
+						} else {
+							s1.VerificationError = err
+						}
+						res.Body.Close()
+					} else {
+						// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
+						if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
+							continue
+						}
+
+						if res.StatusCode == 403 {
+							var body awsErrorResponseBody
+							err = json.NewDecoder(res.Body).Decode(&body)
+							res.Body.Close()
+							if err == nil {
+								// All instances of the code I've seen in the wild are PascalCased but this check is
+								// case-insensitive out of an abundance of caution
+								if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
+									// determinate failure - nothing to do
+								} else {
+									// We see a surprising number of false-negative signature mismatch errors here
+									// (The official SDK somehow elicits even more than just making the request
+									// ourselves)
+									s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
+								}
+							} else {
+								s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
+							}
+						} else {
+							s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
+						}
+					}
+				} else {
+					s1.VerificationError = err
+				}
 			}
 
-			if !s1.Verified {
-				// Unverified results that contain common test words are probably not secrets
-				if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
-					continue
-				}
-				// Unverified results that look like hashes are probably not secrets
-				if falsePositiveSecretCheck.MatchString(resSecretMatch) {
-					continue
-				}
+			// If the result is unverified and matches something like a git hash, don't include it in the results.
+			if !s1.Verified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
+				continue
 			}
 
 			results = append(results, s1)
@@ -145,123 +242,6 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return awsCustomCleanResults(results), nil
 }
 
-func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, retryOn403 bool) (bool, map[string]string, error) {
-	// REQUEST VALUES.
-	method := "GET"
-	service := "sts"
-	host := "sts.amazonaws.com"
-	region := "us-east-1"
-	endpoint := "https://sts.amazonaws.com"
-	now := time.Now().UTC()
-	datestamp := now.Format("20060102")
-	amzDate := now.Format("20060102T150405Z0700")
-
-	req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
-	if err != nil {
-		return false, nil, err
-	}
-	req.Header.Set("Accept", "application/json")
-
-	// TASK 1: CREATE A CANONICAL REQUEST.
-	// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
-	canonicalURI := "/"
-	canonicalHeaders := "host:" + host + "\n"
-	signedHeaders := "host"
-	algorithm := "AWS4-HMAC-SHA256"
-	credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
-
-	params := req.URL.Query()
-	params.Add("Action", "GetCallerIdentity")
-	params.Add("Version", "2011-06-15")
-	params.Add("X-Amz-Algorithm", algorithm)
-	params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
-	params.Add("X-Amz-Date", amzDate)
-	params.Add("X-Amz-Expires", "30")
-	params.Add("X-Amz-SignedHeaders", signedHeaders)
-
-	canonicalQuerystring := params.Encode()
-	payloadHash := GetHash("") // empty payload
-	canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
-
-	// TASK 2: CREATE THE STRING TO SIGN.
-	stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
-
-	// TASK 3: CALCULATE THE SIGNATURE.
-	// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
-	hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
-	hash = GetHMAC(hash, []byte(region))
-	hash = GetHMAC(hash, []byte(service))
-	hash = GetHMAC(hash, []byte("aws4_request"))
-
-	signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
-	signature := hex.EncodeToString(signature2)
-
-	// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
-	params.Add("X-Amz-Signature", signature)
-	req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
-	req.URL.RawQuery = params.Encode()
-
-	client := s.verificationClient
-	if client == nil {
-		client = defaultVerificationClient
-	}
-
-	res, err := client.Do(req)
-	if err == nil {
-		defer res.Body.Close()
-		if res.StatusCode >= 200 && res.StatusCode < 300 {
-			identityInfo := identityRes{}
-			err := json.NewDecoder(res.Body).Decode(&identityInfo)
-			if err == nil {
-				extraData := map[string]string{
-					"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
-					"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
-					"arn":     identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
-				}
-				return true, extraData, nil
-			} else {
-				return false, nil, err
-			}
-		} else if res.StatusCode == 403 {
-			// Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that
-			// share a key ID but are signed with different secrets the second one will be rejected with a 403 that
-			// carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is
-			// valid. Since this is exactly our access pattern, we need to work around it.
-			//
-			// Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The
-			// response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of
-			// the response to the original second request, or read to its end, or the resubmission will also yield a
-			// SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only
-			// SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch
-			// after decoding its response body, which requires reading the entire response body, which disables the
-			// workaround.
-			//
-			// We are clearly deep in the guts of AWS implementation details here, so this all might change with no
-			// notice. If you're here because something in this detector broke, you have my condolences.
-			if retryOn403 {
-				return s.verifyMatch(ctx, resIDMatch, resSecretMatch, false)
-			}
-			var body awsErrorResponseBody
-			err = json.NewDecoder(res.Body).Decode(&body)
-			if err == nil {
-				// All instances of the code I've seen in the wild are PascalCased but this check is
-				// case-insensitive out of an abundance of caution
-				if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
-					return false, nil, nil
-				} else {
-					return false, nil, fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
-				}
-			} else {
-				return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err)
-			}
-		} else {
-			return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
-		}
-	} else {
-		return false, nil, err
-	}
-}
-
 func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
 	if len(results) == 0 {
 		return results

pkg/detectors/aws/aws_test.go

@@ -248,28 +248,6 @@ func TestAWS_FromChunk(t *testing.T) {
 			wantErr:               false,
 			wantVerificationError: true,
 		},
-		{
-			name: "verified secret checked directly after unverified secret with same key id",
-			s:    scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_AWS,
-					Verified:     true,
-					Redacted:     "AKIASP2TPHJSQH3FJRUX",
-					ExtraData: map[string]string{
-						"account": "171436882533",
-						"arn":     "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
-						"user_id": "AIDASP2TPHJSUFRSTTZX4",
-					},
-				},
-			},
-			wantErr: false,
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/detectors/dailyco/dailyco.go

@@ -1,11 +1,11 @@
 package dailyco
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"daily"}) + `\b([0-9a-f]{64})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"daily"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("daily")}
 }
 
-// FromData will find and optionally verify DailyCO secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_DailyCO,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
-			if err == nil {
+			if err == nil && res != nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/dandelion/dandelion.go

@@ -1,11 +1,11 @@
 package dandelion
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,51 +14,44 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"dandelion"}) + `\b([a-z0-9]{32})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"dandelion"}) + `\b([0-9a-z]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"dandelion"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("dandelion")}
 }
 
-// FromData will find and optionally verify Dandelion secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Dandelion,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.dandelion.eu/datatxt/li/v1/?text=Smart&token=%s", resMatch), nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.dandelion.eu/datatxt/li/v1/?text=Smart&token=%s", string(resMatch)), nil)
 			if err != nil {
 				continue
 			}
+			req.Header.Add("Content-Type", "application/json")
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/dareboost/dareboost.go

@@ -1,11 +1,11 @@
 package dareboost
 
 import (
+	"bytes"
 	"context"
 	"io"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,41 +14,35 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"dareboost"}) + `\b([0-9a-zA-Z]{60})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"dareboost"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("dareboost")}
 }
 
-// FromData will find and optionally verify Dareboost secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Dareboost,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			payload := strings.NewReader(`{    "token": "` + resMatch + `",    "location": "Paris"}`)
+			payload := bytes.NewBuffer([]byte(`{    "token": "` + string(resMatch) + `",    "location": "Paris"}`))
 
 			req, err := http.NewRequestWithContext(ctx, "POST", "https://api.dareboost.com/0.8/config", payload)
 			if err != nil {
@@ -62,8 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 
-				bodyString := string(bodyBytes)
-				validResponse := strings.Contains(bodyString, `"status":200`)
+				validResponse := bytes.Contains(bodyBytes, []byte(`"status":200`))
 
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
@@ -73,7 +66,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						s1.Verified = false
 					}
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/databox/databox.go

@@ -1,12 +1,12 @@
 package databox
 
 import (
+	"bytes"
 	"context"
 	b64 "encoding/base64"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,49 +15,42 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"databox"}) + common.BuildRegex(common.RegexPattern, "", 21))
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"databox"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("databox")}
 }
 
-// FromData will find and optionally verify Databox secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Databox,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 		if verify {
-			data := fmt.Sprintf("%s:", resMatch)
+			data := fmt.Sprintf("%s:", string(resMatch))
 			sEnc := b64.StdEncoding.EncodeToString([]byte(data))
-			payload := strings.NewReader(`{
+			payload := bytes.NewBuffer([]byte(`{
 				"data":[
 				  {
 					"$sales": 420,
 					"$visitors": 123000
 				  }
 				]
-			  }`)
+			  }`))
 
 			req, err := http.NewRequestWithContext(ctx, "POST", "https://push.databox.com", payload)
 			if err != nil {
@@ -72,7 +65,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/databrickstoken/databrickstoken.go

@@ -1,11 +1,11 @@
 package databrickstoken
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -27,48 +27,46 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"databricks", "dapi"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("databricks"), []byte("dapi")}
 }
 
 // FromData will find and optionally verify Databrickstoken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	domainMatches := domain.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
+	domainMatches := domain.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		for _, domainmatch := range domainMatches {
 			if len(domainmatch) != 2 {
 				continue
 			}
-			resDomainMatch := strings.TrimSpace(domainmatch[1])
+
+			resDomainMatch := bytes.TrimSpace(domainmatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_DatabricksToken,
-				Raw:          []byte(resMatch),
-				RawV2:        []byte(resMatch + resDomainMatch),
+				Raw:          resMatch,
+				RawV2:        append(resMatch, resDomainMatch...),
 			}
 
 			if verify {
-				req, err := http.NewRequestWithContext(ctx, "GET", resDomainMatch + "/api/2.0/clusters/list", nil)
+				req, err := http.NewRequestWithContext(ctx, "GET", string(resDomainMatch)+"/api/2.0/clusters/list", nil)
 				if err != nil {
 					continue
 				}
-				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}

pkg/detectors/datadogtoken/datadogtoken.go

@@ -1,10 +1,10 @@
 package datadogtoken
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,108 +13,61 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
 	apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"datadog"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("datadog")}
 }
 
-// FromData will find and optionally verify DatadogToken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
+	matches := keyPat.FindAllSubmatch(data, -1)
+	apiMatches := apiPat.FindAllSubmatch(data, -1)
 
-	appMatches := appPat.FindAllStringSubmatch(dataStr, -1)
-	apiMatches := apiPat.FindAllStringSubmatch(dataStr, -1)
-
-	for _, apiMatch := range apiMatches {
-		if len(apiMatch) != 2 {
+	for _, match := range matches {
+		if len(match) != 2 {
 			continue
 		}
-		resApiMatch := strings.TrimSpace(apiMatch[1])
-		appIncluded := false
-		for _, appMatch := range appMatches {
-			if len(appMatch) != 2 {
+		resMatch := bytes.TrimSpace(match[1])
+		for _, apimatch := range apiMatches {
+			if len(apimatch) != 2 {
 				continue
 			}
-			resAppMatch := strings.TrimSpace(appMatch[1])
+			resApiMatch := bytes.TrimSpace(apimatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_DatadogToken,
-				Raw:          []byte(resAppMatch),
-				RawV2:        []byte(resAppMatch + resApiMatch),
-				ExtraData: map[string]string{
-					"Type": "Application+APIKey",
-				},
+				Raw:          resMatch,
+				RawV2:        append(resMatch, resApiMatch...),
 			}
 
 			if verify {
-
 				req, err := http.NewRequestWithContext(ctx, "GET", "https://api.datadoghq.com/api/v2/users", nil)
 				if err != nil {
 					continue
 				}
 				req.Header.Add("Content-Type", "application/json")
-				req.Header.Add("DD-API-KEY", resApiMatch)
-				req.Header.Add("DD-APPLICATION-KEY", resAppMatch)
+				req.Header.Add("DD-API-KEY", string(resApiMatch))
+				req.Header.Add("DD-APPLICATION-KEY", string(resMatch))
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-						if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) {
+						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}
 					}
 				}
 			}
-			appIncluded = true
-			results = append(results, s1)
-		}
 
-		if !appIncluded {
-			s1 := detectors.Result{
-				DetectorType: detectorspb.DetectorType_DatadogToken,
-				Raw:          []byte(resApiMatch),
-				RawV2:        []byte(resApiMatch),
-				ExtraData: map[string]string{
-					"Type": "APIKeyOnly",
-				},
-			}
-
-			if verify {
-
-				req, err := http.NewRequestWithContext(ctx, "GET", "https://api.datadoghq.com/api/v1/validate", nil)
-				if err != nil {
-					continue
-				}
-				req.Header.Add("Content-Type", "application/json")
-				req.Header.Add("DD-API-KEY", resApiMatch)
-				res, err := client.Do(req)
-				if err == nil {
-					defer res.Body.Close()
-					if res.StatusCode >= 200 && res.StatusCode < 300 {
-						s1.Verified = true
-					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-						if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) {
-							continue
-						}
-					}
-				}
-			}
 			results = append(results, s1)
 		}
 	}

pkg/detectors/datafire/datafire.go

@@ -1,11 +1,11 @@
 package datafire
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datafire"}) + `\b([a-z0-9\S]{175,190})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"datafire"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("datafire")}
 }
 
-// FromData will find and optionally verify DataFire secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_DataFire,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,14 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/datagov/datagov.go

@@ -1,11 +1,11 @@
 package datagov
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,41 +14,33 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"data.gov"}) + `\b([a-zA-Z0-9]{40})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"data.gov"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("data.gov")}
 }
 
-// FromData will find and optionally verify DataGov secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_DataGov,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.ers.usda.gov/data/arms/state?api_key=%s", resMatch), nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.ers.usda.gov/data/arms/state?api_key=%s", string(resMatch)), nil)
 			if err != nil {
 				continue
 			}
@@ -58,7 +50,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
@@ -68,7 +59,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 		results = append(results, s1)
 	}
-
 	return results, nil
 }
 

pkg/detectors/debounce/debounce.go

@@ -1,10 +1,10 @@
 package debounce
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,41 +13,37 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"debounce"}) + `\b([a-zA-Z0-9]{13})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"debounce"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("debounce")}
 }
 
-// FromData will find and optionally verify Debounce secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Debounce,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.debounce.io/v1/?api="+resMatch+"&email=some@gmail.com", nil)
+			api := append([]byte("https://api.debounce.io/v1/?api="), resMatch...)
+			url := append(api, []byte("&email=some@gmail.com")...)
+
+			req, err := http.NewRequestWithContext(ctx, "GET", string(url), nil)
 			if err != nil {
 				continue
 			}
@@ -58,7 +54,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/deepai/deepai.go

@@ -7,7 +7,6 @@ import (
 	"mime/multipart"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -26,27 +25,22 @@ var (
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"deepai"}) + `\b([a-z0-9-]{36})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"deepai"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("deepai")}
 }
 
-// FromData will find and optionally verify DeepAI secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_DeepAI,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -56,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			_, err = io.Copy(fw, strings.NewReader("test"))
+			_, err = io.Copy(fw, bytes.NewReader([]byte("test")))
 			if err != nil {
 				continue
 			}
@@ -66,15 +60,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", writer.FormDataContentType())
-			req.Header.Add("api-key", resMatch)
+			req.Header.Add("api-key", string(resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+					if detectors.IsKnownFalsePositive([]byte(resMatch), detectors.DefaultFalsePositives, true) {
 						continue
 					}
 				}

pkg/detectors/deepgram/deepgram.go

@@ -1,11 +1,11 @@
 package deepgram
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"deepgram"}) + `\b([0-9a-z]{40})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"deepgram"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("deepgram")}
 }
 
-// FromData will find and optionally verify Deepgram secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Deepgram,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -53,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Token %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/delighted/delighted.go

@@ -1,12 +1,12 @@
 package delighted
 
 import (
+	"bytes"
 	"context"
 	b64 "encoding/base64"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,69 +15,56 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"delighted"}) + `\b([a-z0-9A-Z]{32})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"delighted"}) + `\b([a-zA-Z0-9]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"delighted"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("delighted")}
 }
 
-// FromData will find and optionally verify Delighted secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Delighted,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			data := fmt.Sprintf("%s:", resMatch)
-			sEnc := b64.StdEncoding.EncodeToString([]byte(data))
-			payload := strings.NewReader(`{
+			sEnc := b64.StdEncoding.EncodeToString(resMatch)
+			payload := bytes.NewReader([]byte(`{
 				"email": "jony@appleseed.com",
 				"properties": { "Purchase Experience": "Mobile App", "State": "CA" }
-				}`)
+				}`))
 			req, err := http.NewRequestWithContext(ctx, "POST", "https://api.delighted.com/v1/people.json", payload)
-			if err != nil {
-				continue
-			}
-			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
-			res, err := client.Do(req)
 			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-						continue
+				req.Header.Add("Content-Type", "application/json")
+				req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
+				res, err := client.Do(req)
+				if err == nil {
+					defer res.Body.Close()
+					if res.StatusCode >= 200 && res.StatusCode < 300 {
+						s1.Verified = true
+					} else {
+						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+							continue
+						}
 					}
 				}
 			}
 		}
-
 		results = append(results, s1)
 	}
-
 	return results, nil
 }
 

pkg/detectors/demio/demio.go

@@ -1,11 +1,11 @@
 package demio
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,70 +14,59 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
 	keyPat    = regexp.MustCompile(detectors.PrefixRegex([]string{"demio"}) + `\b([a-z0-9A-Z]{32})\b`)
 	secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"demio"}) + `\b([a-z0-9A-Z]{10,20})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"demio"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("demio")}
 }
 
-// FromData will find and optionally verify Demio secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
+	keyMatches := keyPat.FindAllSubmatch(data, -1)
+	secretMatches := secretPat.FindAllSubmatch(data, -1)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	idMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
-	for _, match := range matches {
+	for _, match := range keyMatches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
-		for _, idmatch := range idMatches {
-			if len(match) != 2 {
+		resMatch := bytes.TrimSpace(match[1])
+
+		for _, idmatch := range secretMatches {
+			if len(idmatch) != 2 {
 				continue
 			}
-			resIdMatch := strings.TrimSpace(idmatch[1])
+			resIdMatch := bytes.TrimSpace(idmatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Demio,
-				Raw:          []byte(resMatch),
+				Raw:          resMatch,
 			}
 
 			if verify {
-				url := fmt.Sprintf("https://my.demio.com/api/v1/ping/query?api_key=%s&api_secret=%s", resMatch, resIdMatch)
+				url := fmt.Sprintf("https://my.demio.com/api/v1/ping/query?api_key=%s&api_secret=%s", string(resMatch), string(resIdMatch))
+
 				req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 				if err != nil {
 					continue
 				}
 				res, err := client.Do(req)
-				if err == nil {
-					defer res.Body.Close()
-					if res.StatusCode >= 200 && res.StatusCode < 300 {
-						s1.Verified = true
-					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
-						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-							continue
-						}
+				if err != nil || res.StatusCode < 200 || res.StatusCode >= 300 {
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
 					}
+				} else {
+					s1.Verified = true
 				}
 			}
-
 			results = append(results, s1)
 		}
-
 	}
-
 	return results, nil
 }
 

pkg/detectors/deputy/deputy.go

@@ -1,11 +1,11 @@
 package deputy
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,64 +14,58 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"deputy"}) + `\b([0-9a-z]{32})\b`)
 	urlPat = regexp.MustCompile(`\b([0-9a-z]{1,}\.as\.deputy\.com)\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"deputy"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("deputy")}
 }
 
-// FromData will find and optionally verify Deputy secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	urlMatches := urlPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
+	urlMatches := urlPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		for _, urlMatch := range urlMatches {
 			if len(urlMatch) != 2 {
 				continue
 			}
-			resURL := strings.TrimSpace(urlMatch[1])
+			resURL := bytes.TrimSpace(urlMatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Deputy,
-				Raw:          []byte(resMatch),
+				Raw:          resMatch,
 			}
 
 			if verify {
-				req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/api/v1/me", resURL), nil)
+				req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/api/v1/me", string(resURL)), nil)
 				if err != nil {
 					continue
 				}
 				req.Header.Add("Content-Type", "application/json")
-				req.Header.Add("Authorization", fmt.Sprintf("OAuth %s", resMatch))
+				req.Header.Add("Authorization", fmt.Sprintf("OAuth %s", string(resMatch)))
 				res, err := client.Do(req)
-				if err == nil {
-					defer res.Body.Close()
-					if res.StatusCode >= 200 && res.StatusCode < 300 {
-						s1.Verified = true
-					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-							continue
-						}
+				if err != nil {
+					continue
+				}
+				defer res.Body.Close()
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
+					s1.Verified = true
+				} else {
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
 					}
 				}
 			}
@@ -79,6 +73,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			results = append(results, s1)
 		}
 	}
+
 	return results, nil
 }
 

pkg/detectors/detectify/detectify.go

@@ -1,10 +1,10 @@
 package detectify
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,37 +13,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"detectify"}) + `\b([0-9a-z]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"detectify"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("detectify")}
 }
 
-// FromData will find and optionally verify Detectify secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Detectify,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -51,14 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-Detectify-Key", resMatch)
+			req.Header.Add("X-Detectify-Key", string(resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/detectlanguage/detectlanguage.go

@@ -1,11 +1,11 @@
 package detectlanguage
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"detectlanguage"}) + `\b([a-z0-9]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"detectlanguage"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("detectlanguage")}
 }
 
-// FromData will find and optionally verify DetectLanguage secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_DetectLanguage,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/detectors.go

@@ -20,7 +20,7 @@ type Detector interface {
 	FromData(ctx context.Context, verify bool, data []byte) ([]Result, error)
 	// Keywords are used for efficiently pre-filtering chunks using substring operations.
 	// Use unique identifiers that are part of the secret if you can, or the provider name.
-	Keywords() []string
+	Keywords() [][]byte
 	// Type returns the DetectorType number from detectors.proto for the given detector.
 	Type() detectorspb.DetectorType
 }

pkg/detectors/dfuse/dfuse.go

@@ -1,10 +1,10 @@
 package dfuse
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -25,29 +25,27 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"dfuse"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("dfuse")}
 }
 
 // FromData will find and optionally verify Dfuse secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Dfuse,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			payload := strings.NewReader(`{"api_key":"` + resMatch + `"}`)
+			payload := bytes.NewBuffer([]byte(`{"api_key":"` + string(resMatch) + `"}`))
 			req, err := http.NewRequestWithContext(ctx, "POST", "https://auth.dfuse.io/v1/auth/issue", payload)
 			if err != nil {
 				continue
@@ -59,7 +57,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/diffbot/diffbot.go

@@ -1,12 +1,12 @@
 package diffbot
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
 	"net/http"
 	"regexp"
-	"strings"
 	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -16,43 +16,36 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"diffbot"}) + `\b([a-z0-9]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"diffbot"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("diffbot")}
 }
 
-// FromData will find and optionally verify Diffbot secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Diffbot,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
 			timeout := 10 * time.Second
 			client.Timeout = timeout
-			req, err := http.NewRequest("GET", fmt.Sprintf("https://api.diffbot.com/v4/account?token=%s", resMatch), nil)
+			req, err := http.NewRequest("GET", fmt.Sprintf("https://api.diffbot.com/v4/account?token=%s", string(resMatch)), nil)
 			if err != nil {
 				continue
 			}
@@ -60,8 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err == nil {
 				bodyBytes, err := io.ReadAll(res.Body)
 				if err == nil {
-					bodyString := string(bodyBytes)
-					validResponse := strings.Contains(bodyString, `"token":`) && strings.Contains(bodyString, `"planCredits":`)
+					validResponse := bytes.Contains(bodyBytes, []byte(`"token":`)) && bytes.Contains(bodyBytes, []byte(`"planCredits":`))
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						if validResponse {
@@ -70,7 +62,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 							s1.Verified = false
 						}
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}

pkg/detectors/diggernaut/diggernaut.go

@@ -1,11 +1,11 @@
 package diggernaut
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"diggernaut"}) + `\b([0-9a-z]{40})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"diggernaut"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("diggernaut")}
 }
 
-// FromData will find and optionally verify Diggernaut secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Diggernaut,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -53,14 +46,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Accept", "application/json")
-			req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Token %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/jiratoken/jiratoken.go

@@ -13,18 +13,16 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 var _ detectors.Versioner = (*Scanner)(nil)
 
-func (Scanner) Version() int { return 1 }
+func (Scanner) Version() int            { return 1 }
 
 var (
-	defaultClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	tokenPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{24})\b`)
@@ -75,11 +73,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 
 				if verify {
-					client := s.client
-					if client == nil {
-						client = defaultClient
-					}
-
 					data := fmt.Sprintf("%s:%s", resEmail, resToken)
 					sEnc := b64.StdEncoding.EncodeToString([]byte(data))
 					req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
@@ -95,15 +88,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						// If the request is successful and the login reason is not failed authentication, then the token is valid.
 						// This is because Jira returns a 200 status code even if the token is invalid.
 						// Jira returns a default dashboard page.
-						if res.StatusCode >= 200 && res.StatusCode < 300 {
-							if res.Header.Get(loginReasonHeaderKey) != failedAuth {
-								s1.Verified = true
-							}
-						} else {
-							s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+						if (res.StatusCode >= 200 && res.StatusCode < 300) && res.Header.Get(loginReasonHeaderKey) != failedAuth {
+							s1.Verified = true
 						}
-					} else {
-						s1.VerificationError = err
 					}
 				}
 

pkg/detectors/jiratoken/jiratoken_test.go

@@ -9,8 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -24,8 +23,8 @@ func TestJiraToken_FromChunk(t *testing.T) {
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
-	secret := testSecrets.MustGetField("JIRA_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("JIRA_INACTIVE")
+	token := testSecrets.MustGetField("JIRA_TOKEN")
+	inactiveToken := testSecrets.MustGetField("JIRA_INACTIVE")
 	email := testSecrets.MustGetField("JIRA_EMAIL")
 	domain := testSecrets.MustGetField("JIRA_DOMAIN")
 
@@ -35,19 +34,18 @@ func TestJiraToken_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", token, email, domain)),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -56,15 +54,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "found, unverified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveSecret, email, domain)), // the secret would satisfy the regex but not pass validation
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveToken, email, domain)), // the secret would satisfy the regex but not pass validation
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -73,8 +70,7 @@ func TestJiraToken_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "not found",
@@ -84,48 +80,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_JiraToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_JiraToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
+			want:    nil,
+			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -134,12 +96,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
-				}
+				got[i].Raw = nil
+				got[i].RawV2 = nil
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/detectors/jiratoken_v2/jiratoken_v2.go

@@ -13,18 +13,16 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 var _ detectors.Versioner = (*Scanner)(nil)
 
-func (Scanner) Version() int { return 2 }
+func (Scanner) Version() int            { return 2 }
 
 var (
-	defaultClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
 
 	// https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/
 	// Tokens created after Jan 18 2023 use a variable length
@@ -76,11 +74,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 
 				if verify {
-					client := s.client
-					if client == nil {
-						client = defaultClient
-					}
-
 					data := fmt.Sprintf("%s:%s", resEmail, resToken)
 					sEnc := b64.StdEncoding.EncodeToString([]byte(data))
 					req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
@@ -96,15 +89,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						// If the request is successful and the login reason is not failed authentication, then the token is valid.
 						// This is because Jira returns a 200 status code even if the token is invalid.
 						// Jira returns a default dashboard page.
-						if res.StatusCode >= 200 && res.StatusCode < 300 {
-							if res.Header.Get(loginReasonHeaderKey) != failedAuth {
-								s1.Verified = true
-							}
+						if (res.StatusCode >= 200 && res.StatusCode < 300) && res.Header.Get(loginReasonHeaderKey) != failedAuth {
+							s1.Verified = true
 						} else {
-							s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+							s1.VerificationError = err
 						}
-					} else {
-						s1.VerificationError = err
 					}
 				}
 

pkg/detectors/jiratoken_v2/jiratoken_v2_test.go

@@ -9,8 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -24,8 +23,8 @@ func TestJiraToken_FromChunk(t *testing.T) {
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
-	secret := testSecrets.MustGetField("JIRA_V2_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("JIRA_V2_INACTIVE")
+	token := testSecrets.MustGetField("JIRA_V2_TOKEN")
+	inactiveToken := testSecrets.MustGetField("JIRA_V2_INACTIVE")
 	email := testSecrets.MustGetField("JIRA_V2_EMAIL")
 	domain := testSecrets.MustGetField("JIRA_V2_DOMAIN")
 
@@ -35,19 +34,18 @@ func TestJiraToken_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", token, email, domain)),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -56,15 +54,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "found, unverified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveSecret, email, domain)), // the secret would satisfy the regex but not pass validation
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveToken, email, domain)), // the secret would satisfy the regex but not pass validation
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -73,8 +70,7 @@ func TestJiraToken_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "not found",
@@ -84,48 +80,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_JiraToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_JiraToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
+			want:    nil,
+			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -134,12 +96,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
-				}
+				got[i].Raw = nil
+				got[i].RawV2 = nil
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/detectors/pagerdutyapikey/pagerdutyapikey.go

@@ -21,13 +21,13 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pd"}) + `\b([a-zA-Z0-9_+-]{20})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty"}) + `\b([a-z]{1}\+[a-zA-Z]{9}\-[a-z]{2}\-[a-z0-9]{5})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"pagerduty", "pd"}
+	return []string{"pagerduty"}
 }
 
 // FromData will find and optionally verify PagerDutyApiKey secrets in a given set of bytes.

pkg/detectors/privatekey/cracker.go

@@ -9,7 +9,7 @@ import (
 	"golang.org/x/crypto/ssh"
 )
 
-//go:embed "list.txt"
+//go:embed "rockyou-15.txt"
 var rawCrackList []byte
 var passphrases [][]byte
 
@@ -21,17 +21,17 @@ var (
 	ErrUncrackable = errors.New("unable to crack encryption")
 )
 
-func crack(in []byte) (interface{}, string, error) {
+func crack(in []byte) (interface{}, error) {
 	for _, passphrase := range passphrases {
 		parsed, err := ssh.ParseRawPrivateKeyWithPassphrase(in, passphrase)
 		if err != nil {
 			if errors.Is(err, x509.IncorrectPasswordError) {
 				continue
 			} else {
-				return nil, "", err
+				return nil, err
 			}
 		}
-		return parsed, string(passphrase), nil
+		return parsed, nil
 	}
-	return nil, "", ErrUncrackable
+	return nil, ErrUncrackable
 }

pkg/detectors/privatekey/cracker_test.go

@@ -56,29 +56,23 @@ Hx7UPVlTK8dyvk1Z+Yw0nrfNClI=
 
 func Test_crack(t *testing.T) {
 	tests := []struct {
-		name             string
-		in               []byte
-		wantedPassphrase string
-		wantErr          bool
+		name    string
+		in      []byte
+		wantErr bool
 	}{
 		{
-			name:             "crackable",
-			wantErr:          false,
-			in:               testEncryptedKey,
-			wantedPassphrase: string(testEncryptedKeyCorrectPassword),
+			name:    "crackable",
+			wantErr: false,
+			in:      testEncryptedKey,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, passphrase, err := crack(tt.in)
+			_, err := crack(tt.in)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("crack() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-
-			if passphrase != string(tt.wantedPassphrase) {
-				t.Errorf("crack() passphrase = %v, want %v", passphrase, string(testEncryptedKeyCorrectPassword))
-			}
 		})
 	}
 }

pkg/detectors/privatekey/fingerprint.go

@@ -5,12 +5,10 @@ import (
 	"crypto/ed25519"
 	"crypto/rsa"
 	"crypto/sha1"
-	"crypto/sha256"
 	"crypto/x509"
-	"encoding/base64"
 	"encoding/hex"
 	"errors"
-	"fmt"
+	"strings"
 
 	"golang.org/x/crypto/ssh"
 )
@@ -20,8 +18,18 @@ var (
 	ErrEncryptedKey = errors.New("key is encrypted")
 )
 
-func FingerprintPEMKey(parsedKey any) (string, error) {
-	var pubKey any
+func FingerprintPEMKey(in []byte) (string, error) {
+	parsedKey, err := ssh.ParseRawPrivateKey(in)
+	if err != nil && strings.Contains(err.Error(), "private key is passphrase protected") {
+		parsedKey, err = crack(in)
+		if err != nil {
+			return "", err
+		}
+	} else if err != nil {
+		return "", err
+	}
+
+	var pubKey interface{}
 	switch privateKey := parsedKey.(type) {
 	case *rsa.PrivateKey:
 		pubKey = &privateKey.PublicKey
@@ -36,10 +44,6 @@ func FingerprintPEMKey(parsedKey any) (string, error) {
 		return "", ErrNotSupported
 	}
 
-	return fingerprintPublicKey(pubKey)
-}
-
-func fingerprintPublicKey(pubKey any) (string, error) {
 	publickeyBytes, err := x509.MarshalPKIXPublicKey(pubKey)
 	if err != nil {
 		return "", err
@@ -49,8 +53,3 @@ func fingerprintPublicKey(pubKey any) (string, error) {
 	publicKeyFingerprintInHex := hex.EncodeToString(publicKeyFingerprint[:])
 	return publicKeyFingerprintInHex, nil
 }
-
-func fingerprintSSHPublicKey(pubKey ssh.PublicKey) string {
-	publicKeyFingerprint := sha256.Sum256(pubKey.Marshal())
-	return fmt.Sprintf("SHA256:%s", base64.RawStdEncoding.EncodeToString(publicKeyFingerprint[:]))
-}

pkg/detectors/privatekey/privatekey.go

@@ -6,14 +6,11 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 	"time"
 
-	"github.com/AzureAD/microsoft-authentication-library-for-go/apps/errors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-	"golang.org/x/crypto/ssh"
 )
 
 type Scanner struct {
@@ -56,67 +53,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete
 			Redacted:     token[0:64],
 		}
 
-		secret.ExtraData = make(map[string]string)
-
-		var passphrase string
-		parsedKey, err := ssh.ParseRawPrivateKey([]byte(token))
-		if err != nil && strings.Contains(err.Error(), "private key is passphrase protected") {
-			secret.ExtraData["encrypted"] = "true"
-			parsedKey, passphrase, err = crack([]byte(token))
-			if err != nil {
-				secret.VerificationError = err
-				continue
-			}
-			if passphrase != "" {
-				secret.ExtraData["cracked_encryption_passphrase"] = "true"
-			}
-		} else if err != nil {
-			// couldn't parse key, probably invalid
-			continue
-		}
-
-		fingerprint, err := FingerprintPEMKey(parsedKey)
+		fingerprint, err := FingerprintPEMKey([]byte(token))
 		if err != nil {
 			continue
 		}
 
 		if verify {
-			verificationErrors := []string{}
 			data, err := lookupFingerprint(fingerprint, s.IncludeExpired)
 			if err == nil {
+				secret.StructuredData = data
 				if data != nil {
 					secret.Verified = true
-					secret.ExtraData["certificate_urls"] = strings.Join(data.CertificateURLs, ", ")
 				}
-			} else {
-				verificationErrors = append(verificationErrors, err.Error())
 			}
-
-			user, err := verifyGitHubUser(parsedKey)
-			if err != nil && !errors.Is(err, errPermissionDenied) {
-				verificationErrors = append(verificationErrors, err.Error())
-			}
-			if user != nil {
-				secret.Verified = true
-				secret.ExtraData["github_user"] = *user
-			}
-
-			user, err = verifyGitLabUser(parsedKey)
-			if err != nil && !errors.Is(err, errPermissionDenied) {
-				verificationErrors = append(verificationErrors, err.Error())
-			}
-			if user != nil {
-				secret.Verified = true
-				secret.ExtraData["gitlab_user"] = *user
-			}
-
-			if !secret.Verified && len(verificationErrors) > 0 {
-				secret.VerificationError = fmt.Errorf("verification failures: %s", strings.Join(verificationErrors, ", "))
-			}
-		}
-
-		if len(secret.ExtraData) == 0 {
-			secret.ExtraData = nil
 		}
 
 		results = append(results, secret)
@@ -125,30 +74,23 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete
 	return results, nil
 }
 
-type result struct {
-	CertificateURLs []string
-	GitHubUsername  string
-}
-
-func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (*result, error) {
+func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (data *detectorspb.StructuredData, err error) {
 	req, err := http.NewRequest("GET", fmt.Sprintf("https://keychecker.trufflesecurity.com/fingerprint/%s", publicKeyFingerprintInHex), nil)
 	if err != nil {
-		return nil, err
+		return
 	}
 	res, err := client.Do(req)
 	if err != nil {
-		return nil, err
+		return
 	}
 	defer res.Body.Close()
 
 	results := DriftwoodResult{}
 	err = json.NewDecoder(res.Body).Decode(&results)
 	if err != nil {
-		return nil, err
+		return
 	}
 
-	var data *result
-
 	seen := map[string]struct{}{}
 	for _, r := range results.CertificateResults {
 		if _, ok := seen[r.CertificateFingerprint]; ok {
@@ -158,13 +100,36 @@ func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (*
 			continue
 		}
 		if data == nil {
-			data = &result{}
+			data = &detectorspb.StructuredData{}
 		}
-		data.CertificateURLs = append(data.CertificateURLs, fmt.Sprintf("https://crt.sh/?q=%s", r.CertificateFingerprint))
+		if data.TlsPrivateKey == nil {
+			data.TlsPrivateKey = make([]*detectorspb.TlsPrivateKey, 0)
+		}
+		data.TlsPrivateKey = append(data.TlsPrivateKey, &detectorspb.TlsPrivateKey{
+			CertificateFingerprint: r.CertificateFingerprint,
+			ExpirationTimestamp:    r.ExpirationTimestamp.Unix(),
+			VerificationUrl:        fmt.Sprintf("https://crt.sh/?q=%s", r.CertificateFingerprint),
+		})
 		seen[r.CertificateFingerprint] = struct{}{}
 	}
 
-	return data, nil
+	for _, r := range results.GitHubSSHResults {
+		if _, ok := seen[r.Username]; ok {
+			continue
+		}
+		if data == nil {
+			data = &detectorspb.StructuredData{}
+		}
+		if data.GithubSshKey == nil {
+			data.GithubSshKey = make([]*detectorspb.GitHubSSHKey, 0)
+		}
+		data.GithubSshKey = append(data.GithubSshKey, &detectorspb.GitHubSSHKey{
+			User: r.Username,
+		})
+		seen[r.Username] = struct{}{}
+	}
+
+	return
 }
 
 type DriftwoodResult struct {

pkg/detectors/privatekey/privatekey_test.go

@@ -20,15 +20,13 @@ import (
 func TestPrivatekey_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
 	secretTLS := testSecrets.MustGetField("PRIVATEKEY_TLS")
 	secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")
-	secretGitHubEncrypted := testSecrets.MustGetField("PRIVATEKEY_GITHUB_ENCRYPTED")
 	secretInactive := testSecrets.MustGetField("PRIVATEKEY_UNVERIFIED")
-
 	type args struct {
 		ctx    context.Context
 		data   []byte
@@ -77,8 +75,19 @@ func TestPrivatekey_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_PrivateKey,
 					Verified:     true,
 					Redacted:     "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",
-					ExtraData: map[string]string{
-						"certificate_urls": "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9, https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",
+					StructuredData: &detectorspb.StructuredData{
+						TlsPrivateKey: []*detectorspb.TlsPrivateKey{
+							{
+								CertificateFingerprint: "1e20c40deb44a8539dd3ac3e8c53b72750cb19f9",
+								VerificationUrl:        "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9",
+								ExpirationTimestamp:    1497337731,
+							},
+							{
+								CertificateFingerprint: "0e9de31fb2ee16465a4d5d93b227d54f870326d1",
+								VerificationUrl:        "https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",
+								ExpirationTimestamp:    1528192832,
+							},
+						},
 					},
 				},
 			},
@@ -97,8 +106,12 @@ func TestPrivatekey_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_PrivateKey,
 					Verified:     true,
 					Redacted:     "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5v",
-					ExtraData: map[string]string{
-						"github_user": "sirdetectsalot",
+					StructuredData: &detectorspb.StructuredData{
+						GithubSshKey: []*detectorspb.GitHubSSHKey{
+							{
+								User: "thisisforgithub0",
+							},
+						},
 					},
 				},
 			},
@@ -108,8 +121,47 @@ func TestPrivatekey_FromChunk(t *testing.T) {
 			name: "found encrypted GitHub SSH private key, verified",
 			s:    Scanner{},
 			args: args{
-				ctx:    context.Background(),
-				data:   []byte(secretGitHubEncrypted),
+				ctx: context.Background(),
+				data: []byte(`-----BEGIN OPENSSH PRIVATE KEY-----
+						b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABAjNIZuun
+						xgLkM8KuzfmQuRAAAAEAAAAAEAAAGXAAAAB3NzaC1yc2EAAAADAQABAAABgQDe3Al0EMPz
+						utVNk5DixaYrGMK56RqUoqGBinke6SWVWmqom1lBcJWzor6HlnMRPPr7YCEsJKL4IpuVwu
+						inRa5kdtNTyM7yyQTSR2xXCS0fUItNuq8pUktsH8VUggpMeew8hJv7rFA7tnIg3UXCl6iF
+						OLZKbDA5aa24idpcD8b1I9/RzTOB1fu0of5xd9vgODzGw5JvHQSJ0FaA42aNBMGwrDhDB3
+						sgnRNdWf6NNIh8KpXXMKJADf3klsyn6He8L2bPMp8a4wwys2YB35p5zQ0JURovsdewlOxH
+						NT7eP19eVf4dCreibxUmRUaob5DEoHEk8WrxjKWIYUuLeD6AfcW6oXyRU2Yy8Vrt6SqFl5
+						WAi47VMFTkDZYS/eCvG53q9UBHpCj7Qvb0vSkCZXBvBIhlw193F3PX4WvO1IXsMwvQ1D1X
+						lmomsItbqM0cJyKw6LU18QWiBHvE7BqcphaoL5E08W2ATTSRIMCp6rt4rptM7KyGK8rc6W
+						UYrCnWt6KlCA8AAAWQXk+lVx6bH5itIKKYmQr6cR/5xtZ2GHAxnYtvlW3xnGhU0MHv+lJ2
+						uoWlT2RXE5pdMUQj7rNWAMqkwifSKZs9wBfYeo1TaFDmC3nW7yHSN3XTuO78mPIW5JyvmE
+						Rj5qjsUn7fNmzECoAxnVERhwnF3KqUBEPzIAc6/7v/na9NTiiGaJPco9lvCoPWbVLN08WG
+						SuyU+0x5zc3ebzuPcYqu5/c5nmiGxhALrIhjIS0OV1mtAAFhvdMjMIHOijOzSKVCC7rRk5
+						kG9EMLNvOn/DUVSRHamw5gs2V3V+Zq2g5nYWfgq8aDSTB8XlIzOj1cz3HwfN6pfSNQ/3Qe
+						wOQfWfTWdO+JSL8aoBN5Wg8tDbgmvmbFrINsJfFfSm0wZgcHhC7Ul4U3v4c8PoNdK9HXwi
+						TKKzJ9nxLYb+vDh50cnkseu2gt0KwVpjIorxEqeK755mKPao3JmOMr6uFTQsb+g+ZNgPwl
+						nRHA4Igx+zADFj3twldnKIiRpBQ5J4acur3uQ+saanBTXgul1TiFiUGT2cnz+IiCsdPovg
+						TAMt868W5LmzpfH4Cy54JtaRC4/UuMnkTGbWgutVDnWj2stOAzsQ1YmhH5igUmc94mUL+W
+						8vQDCKpeI8n+quDS9zxTvy4L4H5Iz7OZlh0h6N13BDvCYXKcNF/ugkfxZbu8mZsZQQzXNR
+						wOrEtKoHc4AnXYNzsuHEoEyLyJxGfFRDSTLbyN9wFOS/c0k9Gjte+kQRZjBVGORE5sN6X3
+						akUnTF76RhbEc+LamrwM1h5340bwosRbR8I+UrsQdFfJBEj1ZSyMRJlMkFUNi6blt7bhyx
+						ea+Pm2A614nlYUBjw2KKzzn8N/0H2NpJjIptvDsbrx3BS/rKwOeJwavRrGnIlEzuAag4vx
+						Zb2TPVta45uz7fQP5IBl83b0BJKI5Zv/fniUeLI78W/UsZqb64YQbfRyBzFtI1T/SsCi0B
+						e0EyKMzbxtSceT1Mb8eJiVIq04Xpwez9fIUt5rSedZD8KPq8P6s0cGsR7Qmw6eXZ/dBR/a
+						s5vPhfIUmQawmnwAVuWNRdQQ79jUBSn5M+ZRVVTgEG+vFyvxr/bZqOo1JCoq5BmQhLWGRJ
+						Dk9TolbeFIVFrkuXkcu99a079ux7XSkON64oPzHrcsEzjPA1GPqs9CGBSO16wq/nI3zg+E
+						kcOCaurc9yHJJPwduem0+8WLX3WoGNfQRKurtQze2ppy8KarEtDhDd96sKkhYaqOg3GOX8
+						Yx827L4vuWSJSIqKuO2kH6kOCMUNO16piv0z/8u3CJxOGh9+4FZIop81fiFTKLhV3/gwLm
+						fzFY++KIZrLfZcUjzd80NNEja69F452Eb9HrI5BurN/PznDEi9bzM598Y7beyl4/kd4R2e
+						S7SW9/LOrGw5UgxtiU+kV8nPz1PdgxO4sRlnntSBEwkQBzMkLOpq2h2BuJ2TlMP/TWuwLQ
+						sDkv1Yk1pD0roGmtMzbujnURGxqRJ8gUmuIot4hpfyRSssvnRQQZ3lQCQCwHiE+HJxXWf5
+						c58zOMjW7o21tI8e13uUnbRoQVJM9XYqk1usPXIkYPYL9uOw3AW/Zn+cnDrsXvTK9ZxgGD
+						/90b1BNwVqMlUK+QggHNwl5qD8eoXK5cDvav66te+E+V7FYFQ06w3tytRVz8SjoaiChN02
+						muIjvl6G7Hoj1hObM2t/ZheN1EShS11z868hhS6Mx7GvIdtkXuvdiBYMiBLOshJQxB8Mzx
+						iug9W+Di3upLf0UMC1TqADGphsIHRU7RbmHQ8Rwp7dogswmDfpRSapPt9p0D+6Ad5VBzi3
+						f3BPXj76UBLMEJCrZR1P28vnAA7AyNHaLvMPlWDMG5v3V/UV+ugyFcoBAOyjiQgYST8F3e
+						Hx7UPVlTK8dyvk1Z+Yw0nrfNClI=
+						-----END OPENSSH PRIVATE KEY-----
+		`),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -117,10 +169,12 @@ func TestPrivatekey_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_PrivateKey,
 					Verified:     true,
 					Redacted:     "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFl",
-					ExtraData: map[string]string{
-						"github_user":                   "sirdetectsalot",
-						"encrypted":                     "true",
-						"cracked_encryption_passphrase": "true",
+					StructuredData: &detectorspb.StructuredData{
+						GithubSshKey: []*detectorspb.GitHubSSHKey{
+							{
+								User: "thisisforgithub0",
+							},
+						},
 					},
 				},
 			},
@@ -208,7 +262,7 @@ func Test_lookupFingerprint(t *testing.T) {
 				t.Errorf("lookupFingerprint() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.CertificateURLs) > 0, tt.wantFingerprints) {
+			if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.TlsPrivateKey) > 0, tt.wantFingerprints) {
 				t.Errorf("lookupFingerprint() = %v, want %v", gotFingerprints, tt.wantFingerprints)
 			}
 		})

pkg/detectors/privatekey/ssh.go

@@ -1,125 +0,0 @@
-package privatekey
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"net"
-	"strings"
-	"time"
-
-	"golang.org/x/crypto/ssh"
-)
-
-// https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints
-var githubFingerprints = map[string]string{
-	"SHA256:uNiVztksCsDhcc0u9e8BujQXVUpKZIDTMczCvj3tD2s": "RSA",
-	"SHA256:br9IjFspm1vxR3iA35FWE+4VTyz1hYVLIE2t1/CeyWQ": "DSA - deprecated",
-	"SHA256:p2QAMXNIC1TJYWeIOttrVc98/R1BUFWu3/LiyKgUfQM": "ECDSA",
-	"SHA256:+DiY3wvvV6TuJJhbpZisF/zLDA0zPMSvHdkr4UvCOqU": "ED25519",
-}
-
-// https://docs.gitlab.com/ee/user/gitlab_com/index.html#ssh-host-keys-fingerprints
-var gitlabFingerprints = map[string]string{
-	"SHA256:HbW3g8zUjNSksFbqTiUWPWg2Bq1x8xdGUrliXFzSnUw": "ECDSA",
-	"SHA256:eUXGGm1YGsMAS7vkcx6JOJdOGHPem5gQp4taiCfCLB8": "ED25519",
-	"SHA256:ROQFvPThGrW4RuWLoL9tq9I9zJ42fK4XywyRtbOz/EQ": "RSA",
-}
-
-func firstResponseFromSSH(parsedKey any, username, hostport string) (string, error) {
-	signer, err := ssh.NewSignerFromKey(parsedKey)
-	if err != nil {
-		return "", err
-	}
-
-	// Verify the server fingerprint to ensure that there is no MITM replay attack
-	config := &ssh.ClientConfig{
-		Timeout: 3 * time.Second,
-		User:    username,
-		Auth: []ssh.AuthMethod{
-			ssh.PublicKeys(signer),
-		},
-		HostKeyCallback: func(hostname string, _ net.Addr, key ssh.PublicKey) error {
-			switch hostname {
-			case "github.com:22":
-				fingerprint := fingerprintSSHPublicKey(key)
-				if _, ok := githubFingerprints[fingerprint]; !ok {
-					return fmt.Errorf("unknown host fingerprint for github.com, got %s", fingerprint)
-				}
-			case "gitlab.com:22":
-				fingerprint := fingerprintSSHPublicKey(key)
-				if _, ok := gitlabFingerprints[fingerprint]; !ok {
-					return fmt.Errorf("unknown host fingerprint for gitlab.com, got %s", fingerprint)
-				}
-			default:
-				return errors.New("unknown host in fingerprint db")
-			}
-			return nil
-		},
-	}
-
-	client, err := ssh.Dial("tcp", hostport, config)
-	if err != nil {
-		if strings.Contains(err.Error(), "unable to authenticate") {
-			return "", errPermissionDenied
-		}
-		return "", err
-	}
-	defer client.Close()
-
-	session, err := client.NewSession()
-	if err != nil {
-		return "", err
-	}
-	defer session.Close()
-
-	var output bytes.Buffer
-	session.Stderr = &output
-
-	err = session.Shell()
-	if err != nil {
-		return "", err
-	}
-	_ = session.Wait()
-
-	return output.String(), err
-}
-
-var errPermissionDenied = errors.New("permission denied")
-
-func verifyGitHubUser(parsedKey any) (*string, error) {
-	output, err := firstResponseFromSSH(parsedKey, "git", "github.com:22")
-	if err != nil {
-		return nil, err
-	}
-
-	if strings.Contains(output, "Permission denied") {
-		return nil, errPermissionDenied
-	}
-
-	if strings.Contains(output, "successfully authenticated") {
-		username := strings.TrimSuffix(strings.Split(output, " ")[1], "!")
-		return &username, nil
-	}
-
-	return nil, nil
-}
-
-func verifyGitLabUser(parsedKey any) (*string, error) {
-	output, err := firstResponseFromSSH(parsedKey, "git", "gitlab.com:22")
-	if err != nil {
-		return nil, err
-	}
-
-	if strings.Contains(output, "Permission denied") {
-		return nil, errPermissionDenied
-	}
-
-	if strings.Contains(output, "Welcome to GitLab") {
-		split := strings.Split(output, " ")
-		username := strings.TrimPrefix(strings.TrimSuffix(split[len(split)-1], "!"), "@")
-		return &username, nil
-	}
-
-	return nil, nil
-}

pkg/detectors/privatekey/ssh_test.go

@@ -1,34 +0,0 @@
-package privatekey
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"golang.org/x/crypto/ssh"
-)
-
-func TestFirstResponseFromSSH(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
-	if err != nil {
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")
-
-	parsedKey, err := ssh.ParseRawPrivateKey([]byte(normalize(secretGitHub)))
-	if err != nil {
-		t.Fatalf("could not parse test secret: %s", err)
-	}
-
-	output, err := firstResponseFromSSH(parsedKey, "git", "github.com:22")
-	if err != nil {
-		t.Fail()
-	}
-
-	if len(output) == 0 {
-		t.Fail()
-	}
-}

pkg/detectors/sendbird/sendbird.go

@@ -2,7 +2,6 @@ package sendbird
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"regexp"
@@ -13,25 +12,19 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	defaultClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat   = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9a-f]{40})\b`)
 	appIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})\b`)
 )
 
-type userResp struct {
-	Code int `json:"code"`
-}
-
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
@@ -68,40 +61,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				req.Header.Add("Api-Token", resMatch)
-
-				client := s.client
-				if client == nil {
-					client = defaultClient
-				}
-
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
-					} else if res.StatusCode == 400 { // Sendbird returns 400 for all errors
-						var userResp userResp
-						err := json.NewDecoder(res.Body).Decode(&userResp)
-						if err != nil {
-							s1.VerificationError = fmt.Errorf("error decoding json response body: %w", err)
-						} else if userResp.Code != 400401 {
-							// https://sendbird.com/docs/chat/platform-api/v3/error-codes
-							// Sendbird always includes its own error codes with 400 responses
-							// 400401 (InvalidApiToken) is the only one that indicates a bad token
-							s1.VerificationError = fmt.Errorf("unexpected response code: %d", userResp.Code)
-						}
 					} else {
-						s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+							continue
+						}
 					}
-				} else {
-					s1.VerificationError = err
 				}
 			}
 
-			// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-			if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-				continue
-			}
 			results = append(results, s1)
 		}
 	}

pkg/detectors/sendbird/sendbird_test.go

@@ -9,8 +9,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/kylelemons/godebug/pretty"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -19,7 +19,7 @@ import (
 func TestSendbird_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
@@ -33,12 +33,11 @@ func TestSendbird_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
@@ -54,8 +53,7 @@ func TestSendbird_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -71,8 +69,7 @@ func TestSendbird_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "not found",
@@ -82,82 +79,14 @@ func TestSendbird_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Sendbird,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Sendbird,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected error code",
-			s:    Scanner{client: common.ConstantResponseHttpClient(400, `{"code":400402}`)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Sendbird,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but failed to decode json",
-			s:    Scanner{client: common.ConstantResponseHttpClient(400, `{`)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Sendbird,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
+			want:    nil,
+			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Sendbird.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -166,12 +95,9 @@ func TestSendbird_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
-				}
+				got[i].Raw = nil
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/detectors/sendgrid/sendgrid.go

@@ -13,15 +13,13 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	defaultClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
 
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendgrid"}) + `(SG\.[\w\-_]{20,24}\.[\w\-_]{39,50})\b`)
 )
@@ -44,7 +42,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 		res := strings.TrimSpace(match[1])
 
-		s1 := detectors.Result{
+		s := detectors.Result{
 			DetectorType: detectorspb.DetectorType_SendGrid,
 			Raw:          []byte(res),
 		}
@@ -54,11 +52,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			// 403 will be issued if the scope is wrong but the key is correct
 			baseURL := "https://api.sendgrid.com/v3/templates"
 
-			client := s.client
-			if client == nil {
-				client = defaultClient
-			}
-
 			// test `read_user` scope
 			req, err := http.NewRequestWithContext(ctx, "GET", baseURL, nil)
 			if err != nil {
@@ -74,18 +67,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				// 403 means good key but not the right scope
 				// 401 is bad key
 				if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
-					s1.Verified = true
-				} else if res.StatusCode != http.StatusUnauthorized {
-					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+					s.Verified = true
 				}
 			}
 		}
 
-		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
+		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
 			continue
 		}
 
-		results = append(results, s1)
+		results = append(results, s)
 	}
 
 	return

pkg/detectors/sendgrid/sendgrid_test.go

@@ -9,8 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -19,7 +18,7 @@ import (
 func TestSendgrid_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
@@ -31,12 +30,11 @@ func TestSendgrid_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
@@ -52,8 +50,7 @@ func TestSendgrid_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -69,8 +66,7 @@ func TestSendgrid_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "not found",
@@ -80,54 +76,22 @@ func TestSendgrid_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_SendGrid,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_SendGrid,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
+			want:    nil,
+			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Sendgrid.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+			for i := range got {
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("Sendgrid.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/detectors/sentrytoken/sentrytoken.go

@@ -122,9 +122,6 @@ func verifyToken(ctx context.Context, client *http.Client, token string) (bool,
 	if err = json.Unmarshal(bytes, &resp); err != nil {
 		return false, err
 	}
-	if len(resp) == 0 {
-		return false, fmt.Errorf("unexpected response body: %s", string(bytes))
-	}
 
 	return isVerified, err
 }

pkg/detectors/sentrytoken/sentrytoken_test.go

@@ -123,40 +123,6 @@ func TestSentryToken_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
-		{
-			name: "found, account deactivated",
-			s:    Scanner{client: common.ConstantResponseHttpClient(200, reponseAccountDeactivated)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_SentryToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, account deactivated",
-			s:    Scanner{client: common.ConstantResponseHttpClient(200, responseEnmpty)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_SentryToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
 		{
 			name: "not found",
 			s:    Scanner{},
@@ -193,8 +159,7 @@ func TestSentryToken_FromChunk(t *testing.T) {
 	}
 }
 
-const (
-	responseBody403 = `
+const responseBody403 = `
 [
   {
     "organization": {
@@ -209,9 +174,6 @@ const (
   }
 ]
 `
-	reponseAccountDeactivated = `{"detail": "Authentication credentials were not provided"}`
-	responseEnmpty            = `[]`
-)
 
 func BenchmarkFromData(benchmark *testing.B) {
 	ctx := context.Background()

pkg/detectors/slack/slack.go

@@ -13,16 +13,13 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Check that the Slack scanner implements the SecretScanner interface at compile time.
 var _ detectors.Detector = Scanner{}
 
 var (
-	defaultClient = common.SaneHttpClient()
-	tokenPats     = map[string]*regexp.Regexp{
+	tokenPats = map[string]*regexp.Regexp{
 		"Slack Bot Token":               regexp.MustCompile(`xoxb\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
 		"Slack User Token":              regexp.MustCompile(`xoxp\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
 		"Slack Workspace Access Token":  regexp.MustCompile(`xoxa\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
@@ -56,54 +53,34 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		tokens := tokenPat.FindAllString(dataStr, -1)
 
 		for _, token := range tokens {
-			s1 := detectors.Result{
+			s := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Slack,
 				Raw:          []byte(token),
 			}
 			if verify {
-				client := s.client
-				if s.client == nil {
-					client = defaultClient
-				}
-
+				client := common.SaneHttpClient()
 				req, err := http.NewRequestWithContext(ctx, "POST", verifyURL, nil)
 				if err != nil {
 					continue
 				}
-
 				req.Header.Add("Content-Type", "application/json; charset=utf-8")
 				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
-
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					var authResponse authRes
 					if err := json.NewDecoder(res.Body).Decode(&authResponse); err != nil {
-						s1.VerificationError = fmt.Errorf("failed to decode auth response: %w", err)
+						continue
 					}
-
-					if authResponse.Ok {
-						s1.Verified = true
-						// Slack API returns 200 even if the token is invalid. We need to check the error field.
-					} else if authResponse.Error == "invalid_auth" {
-						// The secret is determinately not verified (nothing to do)
-					} else {
-						s1.VerificationError = fmt.Errorf("unexpected error auth response %+v", authResponse.Error)
-					}
-				} else {
-					s1.VerificationError = err
+					s.Verified = authResponse.Ok
 				}
 			}
 
-			if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
-				continue
-			}
-
-			results = append(results, s1)
+			results = append(results, s)
 		}
 	}
 
-	return results, nil
+	return
 }
 
 func (s Scanner) Type() detectorspb.DetectorType {

pkg/detectors/slack/slack_test.go

@@ -9,15 +9,13 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
-
+	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestSlack_FromChunk(t *testing.T) {
+func TestScanner_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
 	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
@@ -26,28 +24,17 @@ func TestSlack_FromChunk(t *testing.T) {
 	}
 	secret := testSecrets.MustGetField("SLACK")
 	secretInactive := testSecrets.MustGetField("SLACK_INACTIVE")
-
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
-
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		wantResults         []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name        string
+		data        []byte
+		verify      bool
+		wantResults []detectors.Result
+		wantErr     bool
 	}{
 		{
-			name: "found, verified",
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
-				verify: true,
-			},
+			name:   "found, verified",
+			data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
+			verify: true,
 			wantResults: []detectors.Result{
 				{
 					DetectorType: detectorspb.DetectorType_Slack,
@@ -57,12 +44,9 @@ func TestSlack_FromChunk(t *testing.T) {
 			wantErr: false,
 		},
 		{
-			name: "found but unverified",
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
-				verify: true,
-			},
+			name:   "found but unverified",
+			data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
+			verify: true,
 			wantResults: []detectors.Result{
 				{
 					DetectorType: detectorspb.DetectorType_Slack,
@@ -71,62 +55,23 @@ func TestSlack_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
-				verify: true,
-			},
-			wantResults: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Slack,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "unexpected auth response",
-			s:    Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "unexpected_error"}`)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
-				verify: true,
-			},
-			wantResults: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Slack,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(context.Background(), tt.verify, tt.data)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("Slack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("Scanner.FromData) error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-
 			for i := range got {
 				if len(got[i].Raw) == 0 {
 					t.Fatal("no raw secret present")
 				}
 				got[i].Raw = nil
-
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
-				}
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
-			if diff := cmp.Diff(got, tt.wantResults, ignoreOpts); diff != "" {
-				t.Errorf("Slack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			if diff := pretty.Compare(got, tt.wantResults); diff != "" {
+				t.Errorf("Scanner.FromData) %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}

pkg/detectors/tailscale/tailscale.go

@@ -1,85 +0,0 @@
-package tailscale
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"net/url"
-	"regexp"
-	"strings"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-type Scanner struct{}
-
-// Ensure the Scanner satisfies the interface at compile time.
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
-	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(`\btskey-[a-z]+-[0-9A-Za-z_]+-[0-9A-Za-z_]+\b`)
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"tskey-", "tskey-api-", "tskey-oauth-"}
-}
-
-// FromData will find and optionally verify Tailscaleapi secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-
-	for _, match := range matches {
-		resMatch := strings.TrimSpace(match[0])
-
-		s1 := detectors.Result{
-			DetectorType: detectorspb.DetectorType_Tailscale,
-			Raw:          []byte(resMatch),
-		}
-
-		if verify {
-			const u = "https://api.tailscale.com/api/v2/secret-scanning/verify"
-			vals := url.Values{"key": []string{resMatch}}
-			req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, strings.NewReader(vals.Encode()))
-			if err != nil {
-				continue
-			}
-			req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				switch res.StatusCode {
-				case http.StatusNoContent:
-					s1.Verified = true
-				case http.StatusUnauthorized:
-					// The secret is determinately not verified (nothing to do)
-				default:
-					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
-				}
-			} else {
-				s1.VerificationError = err
-			}
-		}
-
-		// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-		if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-			continue
-		}
-
-		results = append(results, s1)
-	}
-
-	return results, nil
-}
-
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_Tailscale
-}

pkg/detectors/tailscale/tailscale_test.go

@@ -1,120 +0,0 @@
-//go:build detectors
-// +build detectors
-
-package tailscale
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/kylelemons/godebug/pretty"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestTailscaleapi_FromChunk(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
-	if err != nil && false {
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("TAILSCALEAPI")
-	inactiveSecret := testSecrets.MustGetField("TAILSCALEAPI_INACTIVE")
-
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
-	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
-	}{
-		{
-			name: "found, verified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Tailscale,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found, unverified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Tailscale,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Tailscaleapi.FromData() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Tailscaleapi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
-	ctx := context.Background()
-	s := Scanner{}
-	for name, data := range detectors.MustGetBenchmarkData() {
-		benchmark.Run(name, func(b *testing.B) {
-			for n := 0; n < b.N; n++ {
-				_, err := s.FromData(ctx, false, data)
-				if err != nil {
-					b.Fatal(err)
-				}
-			}
-		})
-	}
-}

pkg/engine/defaults.go

@@ -6,7 +6,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraph"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -1543,7 +1542,6 @@ func DefaultDetectors() []detectors.Detector {
 		trufflehogenterprise.Scanner{},
 		salesforce.Scanner{},
 		sourcegraph.Scanner{},
-		tailscale.Scanner{},
 	}
 
 }

pkg/engine/engine.go

@@ -565,18 +565,11 @@ func (e *Engine) notifyResults(ctx context.Context) {
 		}
 		atomic.AddUint32(&e.numFoundResults, 1)
 
-		// Dedupe results by comparing the detector type, raw result, and source metadata.
-		// We want to avoid duplicate results with different decoder types, but we also
-		// want to include duplicate results with the same decoder type.
-		// Duplicate results with the same decoder type SHOULD have their own entry in the
-		// results list, this would happen if the same secret is found multiple times.
 		key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
-		if val, ok := e.dedupeCache.Get(key); ok {
-			if res, ok := val.(detectorspb.DecoderType); ok && res != r.DecoderType {
-				continue
-			}
+		if _, ok := e.dedupeCache.Get(key); ok {
+			continue
 		}
-		e.dedupeCache.Add(key, r.DecoderType)
+		e.dedupeCache.Add(key, struct{}{})
 
 		if r.Verified {
 			atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)

pkg/engine/engine_test.go

@@ -2,13 +2,8 @@ package engine
 
 import (
 	"fmt"
-	"path/filepath"
 	"testing"
-	"time"
 
-	"github.com/stretchr/testify/assert"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -176,32 +171,3 @@ func BenchmarkSupportsLineNumbersLoop(b *testing.B) {
 		_ = SupportsLineNumbers(sourceType)
 	}
 }
-
-// TestEngine_DuplicatSecrets is a test that detects ALL duplicate secrets with the same decoder.
-func TestEngine_DuplicatSecrets(t *testing.T) {
-	ctx := context.Background()
-
-	absPath, err := filepath.Abs("./testdata")
-	assert.Nil(t, err)
-
-	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
-	defer cancel()
-
-	e, err := Start(ctx,
-		WithConcurrency(1),
-		WithDecoders(decoders.DefaultDecoders()...),
-		WithDetectors(true, DefaultDetectors()...),
-		WithPrinter(new(discardPrinter)),
-	)
-	assert.Nil(t, err)
-
-	cfg := sources.FilesystemConfig{Paths: []string{absPath}}
-	if err := e.ScanFileSystem(ctx, cfg); err != nil {
-		return
-	}
-
-	// Wait for all the chunks to be processed.
-	assert.Nil(t, e.Finish(ctx))
-	want := uint64(5)
-	assert.Equal(t, want, e.GetMetrics().UnverifiedSecretsFound)
-}

pkg/engine/testdata/secrets.txt

@@ -1,6 +0,0 @@
-  1 aws AKIAWARWQKZNHMZBLY4I
-  2 secret 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
- sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
- sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
- sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
- sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90

pkg/giturl/giturl.go

@@ -119,18 +119,7 @@ func GenerateLink(repo, commit, file string, line int64) string {
 	case providerBitbucket:
 		return repo[:len(repo)-4] + "/commits/" + commit
 
-	case providerAzure:
-		baseLink := repo + "/commit/" + commit + "/" + file
-		if line > 0 {
-			baseLink += "?line=" + strconv.FormatInt(line, 10)
-		}
-		return baseLink
-
 	case providerGithub, providerGitlab:
-		// If the provider name isn't one of the cloud defaults, it is probably an on-prem github or gitlab.
-		// So do the same thing.
-		fallthrough
-	default:
 		var baseLink string
 		if file == "" {
 			baseLink = repo[:len(repo)-4] + "/commit/" + commit
@@ -141,5 +130,15 @@ func GenerateLink(repo, commit, file string, line int64) string {
 			}
 		}
 		return baseLink
+
+	case providerAzure:
+		baseLink := repo + "/commit/" + commit + "/" + file
+		if line > 0 {
+			baseLink += "?line=" + strconv.FormatInt(line, 10)
+		}
+		return baseLink
+
+	default:
+		return ""
 	}
 }

pkg/giturl/giturl_test.go

@@ -160,24 +160,6 @@ func TestGenerateLink(t *testing.T) {
 			},
 			want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=20",
 		},
-		{
-			name: "Unknown provider on-prem instance",
-			args: args{
-				repo:   "https://onprem.customdomain.com/org/repo.git",
-				commit: "xyz123",
-				file:   "main.go",
-				line:   int64(30),
-			},
-			want: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L30",
-		},
-		{
-			name: "Unknown provider on-prem instance - no file",
-			args: args{
-				repo:   "https://onprem.customdomain.com/org/repo.git",
-				commit: "xyz123",
-			},
-			want: "https://onprem.customdomain.com/org/repo/commit/xyz123",
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/handlers/archive.go

@@ -17,15 +17,12 @@ import (
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
 type ctxKey int
 
 const (
 	depthKey ctxKey = iota
-
-	errMaxArchiveDepthReached = "max archive depth reached"
 )
 
 var (
@@ -84,66 +81,51 @@ func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []b
 	return archiveChan
 }
 
-type decompressorInfo struct {
-	depth       int
-	reader      io.Reader
-	archiveChan chan []byte
-	archiver    archiver.Decompressor
-}
-
 // openArchive takes a reader and extracts the contents up to the maximum depth.
 func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan []byte) error {
 	if depth >= maxDepth {
-		return fmt.Errorf(errMaxArchiveDepthReached)
+		return fmt.Errorf("max archive depth reached")
 	}
-
-	format, arReader, err := archiver.Identify("", reader)
-	if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
-		return a.handleNonArchiveContent(ctx, arReader, archiveChan)
+	format, reader, err := archiver.Identify("", reader)
+	if err != nil {
+		if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
+			chunkSize := 10 * 1024
+			for {
+				chunk := make([]byte, chunkSize)
+				n, _ := reader.Read(chunk)
+				archiveChan <- chunk
+				if n < chunkSize {
+					break
+				}
+			}
+			return nil
+		}
+		return err
 	}
-
 	switch archive := format.(type) {
 	case archiver.Decompressor:
-		info := decompressorInfo{depth: depth, reader: arReader, archiveChan: archiveChan, archiver: archive}
-		return a.handleDecompressor(ctx, info)
-	case archiver.Extractor:
-		return archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
-	default:
-		return fmt.Errorf("unknown archive type: %s", format.Name())
-	}
-}
-
-func (a *Archive) handleNonArchiveContent(ctx context.Context, reader io.Reader, archiveChan chan []byte) error {
-	aCtx := logContext.AddLogger(ctx)
-	chunkReader := sources.NewChunkReader()
-	chunkResChan := chunkReader(aCtx, reader)
-	for data := range chunkResChan {
-		if err := data.Error(); err != nil {
-			aCtx.Logger().Error(err, "error reading chunk")
-			continue
-		}
-		if err := common.CancellableWrite(ctx, archiveChan, data.Bytes()); err != nil {
+		compReader, err := archive.OpenReader(reader)
+		if err != nil {
 			return err
 		}
+		fileBytes, err := a.ReadToMax(ctx, compReader)
+		if err != nil {
+			return err
+		}
+		newReader := bytes.NewReader(fileBytes)
+		return a.openArchive(ctx, depth+1, newReader, archiveChan)
+	case archiver.Extractor:
+		err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
+		if err != nil {
+			return err
+		}
+		return nil
 	}
-	return nil
-}
-
-func (a *Archive) handleDecompressor(ctx context.Context, info decompressorInfo) error {
-	compReader, err := info.archiver.OpenReader(info.reader)
-	if err != nil {
-		return err
-	}
-	fileBytes, err := a.ReadToMax(ctx, compReader)
-	if err != nil {
-		return err
-	}
-	newReader := bytes.NewReader(fileBytes)
-	return a.openArchive(ctx, info.depth+1, newReader, info.archiveChan)
+	return fmt.Errorf("Unknown archive type: %s", format.Name())
 }
 
 // IsFiletype returns true if the provided reader is an archive.
-func (a *Archive) IsFiletype(_ context.Context, reader io.Reader) (io.Reader, bool) {
+func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) {
 	format, readerB, err := archiver.Identify("", reader)
 	if err != nil {
 		return readerB, false
@@ -153,9 +135,8 @@ func (a *Archive) IsFiletype(_ context.Context, reader io.Reader) (io.Reader, bo
 		return readerB, true
 	case archiver.Decompressor:
 		return readerB, true
-	default:
-		return readerB, false
 	}
+	return readerB, false
 }
 
 // extractorHandler is applied to each file in an archiver.Extractor file.
@@ -199,7 +180,7 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
 			if e, ok := r.(error); ok {
 				err = e
 			} else {
-				err = fmt.Errorf("panic occurred: %v", r)
+				err = fmt.Errorf("Panic occurred: %v", r)
 			}
 			logger.Error(err, "Panic occurred when reading archive")
 		}
@@ -302,7 +283,7 @@ func (a *Archive) HandleSpecialized(ctx logContext.Context, reader io.Reader) (i
 // The caller is responsible for closing the returned reader.
 func (a *Archive) extractDebContent(ctx logContext.Context, file io.Reader) (io.ReadCloser, error) {
 	if a.currentDepth >= maxDepth {
-		return nil, fmt.Errorf(errMaxArchiveDepthReached)
+		return nil, fmt.Errorf("max archive depth reached")
 	}
 
 	tmpEnv, err := a.createTempEnv(ctx, file)

pkg/pb/detectorspb/detectors.pb.go

@@ -1000,7 +1000,6 @@ const (
 	DetectorType_HuggingFace                   DetectorType = 926
 	DetectorType_Snowflake                     DetectorType = 927
 	DetectorType_Sourcegraph                   DetectorType = 928
-	DetectorType_Tailscale                     DetectorType = 929
 )
 
 // Enum value maps for DetectorType.
@@ -1931,7 +1930,6 @@ var (
 		926: "HuggingFace",
 		927: "Snowflake",
 		928: "Sourcegraph",
-		929: "Tailscale",
 	}
 	DetectorType_value = map[string]int32{
 		"Alibaba":                       0,
@@ -2859,7 +2857,6 @@ var (
 		"HuggingFace":                   926,
 		"Snowflake":                     927,
 		"Sourcegraph":                   928,
-		"Tailscale":                     929,
 	}
 )
 
@@ -3238,7 +3235,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
 	0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
 	0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
-	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xae, 0x74, 0x0a, 0x0c, 0x44,
+	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x9e, 0x74, 0x0a, 0x0c, 0x44,
 	0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
 	0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
 	0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4168,8 +4165,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x6f, 0x72, 0x63, 0x65, 0x10, 0x9d, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69,
 	0x6e, 0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x6e, 0x6f,
 	0x77, 0x66, 0x6c, 0x61, 0x6b, 0x65, 0x10, 0x9f, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x53, 0x6f, 0x75,
-	0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x10, 0xa0, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x54,
-	0x61, 0x69, 0x6c, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x10, 0xa1, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67,
+	0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x10, 0xa0, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67,
 	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
 	0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
 	0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64,

pkg/sources/circleci/circleci.go

@@ -75,7 +75,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	projects, err := s.projects(ctx)
 	if err != nil {
 		return fmt.Errorf("error getting projects: %w", err)

pkg/sources/docker/docker.go

@@ -2,6 +2,7 @@ package docker
 
 import (
 	"archive/tar"
+	"bytes"
 	"compress/gzip"
 	"errors"
 	"fmt"
@@ -13,11 +14,9 @@ import (
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
-	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -25,16 +24,17 @@ import (
 )
 
 type Source struct {
-	name        string
-	sourceId    int64
-	jobId       int64
-	verify      bool
-	concurrency int
-	conn        sourcespb.Docker
+	name     string
+	sourceId int64
+	jobId    int64
+	verify   bool
+	conn     sourcespb.Docker
 	sources.Progress
 	sources.CommonSourceUnitUnmarshaller
 }
 
+var FilesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
+
 // Ensure the Source satisfies the interfaces at compile time.
 var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
@@ -59,7 +59,6 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 	s.sourceId = sourceId
 	s.jobId = jobId
 	s.verify = verify
-	s.concurrency = concurrency
 
 	// Reset metrics for this source at initialization time.
 	dockerImagesScanned.WithLabelValues(s.name).Set(0)
@@ -72,209 +71,166 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 	return nil
 }
 
-type imageInfo struct {
-	image v1.Image
-	base  string
-	tag   string
-}
-
-type layerInfo struct {
-	digest v1.Hash
-	base   string
-	tag    string
-}
-
 // Chunks emits data over a channel that is decoded and scanned for secrets.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
-	ctx = context.WithValues(ctx, "source_type", s.Type(), "source_name", s.name)
-
-	workers := new(errgroup.Group)
-	workers.SetLimit(s.concurrency)
-
-	scanErrs := sources.NewScanErrors()
-	for _, image := range s.conn.GetImages() {
-		image := image
-		workers.Go(func() error {
-			if common.IsDone(ctx) {
-				return nil
-			}
-
-			imgInfo, err := s.processImage(ctx, image)
-			if err != nil {
-				scanErrs.Add(err)
-				return nil
-			}
-
-			ctx = context.WithValues(ctx, "image", imgInfo.base, "tag", imgInfo.tag)
-			ctx.Logger().V(2).Info("scanning image")
-
-			layers, err := imgInfo.image.Layers()
-			if err != nil {
-				scanErrs.Add(err)
-				return nil
-			}
-
-			for _, layer := range layers {
-				if err := s.processLayer(ctx, layer, imgInfo, chunksChan); err != nil {
-					scanErrs.Add(err)
-					return nil
-				}
-				dockerLayersScanned.WithLabelValues(s.name).Inc()
-			}
-
-			dockerImagesScanned.WithLabelValues(s.name).Inc()
-
-			return nil
-		})
-	}
-	_ = workers.Wait()
-	if scanErrs.Count() > 0 {
-		ctx.Logger().V(2).Info("scan errors", "errors", scanErrs.String())
-	}
-
-	return nil
-}
-
-// processImage processes an individual image and prepares it for further processing.
-func (s *Source) processImage(ctx context.Context, image string) (imageInfo, error) {
-	var (
-		imgInfo   imageInfo
-		hasDigest bool
-		imageName name.Reference
-	)
-
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	remoteOpts, err := s.remoteOpts()
 	if err != nil {
-		return imgInfo, err
+		return err
 	}
 
-	const filePrefix = "file://"
-	if strings.HasPrefix(image, filePrefix) {
-		image = strings.TrimPrefix(image, filePrefix)
-		imgInfo.base = image
-		imgInfo.image, err = tarball.ImageFromPath(image, nil)
-		if err != nil {
-			return imgInfo, err
-		}
-	} else {
-		imgInfo.base, imgInfo.tag, hasDigest = baseAndTagFromImage(image)
+	logger := ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
 
-		if hasDigest {
-			imageName, err = name.NewDigest(image)
+	for _, image := range s.conn.GetImages() {
+		var img v1.Image
+		var err error
+		var base, tag string
+		var hasDigest bool
+		var imageName name.Reference
+
+		if strings.HasPrefix(image, "file://") {
+			image = strings.TrimPrefix(image, "file://")
+			base = image
+			img, err = tarball.ImageFromPath(image, nil)
+			if err != nil {
+				return err
+			}
 		} else {
-			imageName, err = name.NewTag(image)
-		}
-		if err != nil {
-			return imgInfo, err
+			base, tag, hasDigest = baseAndTagFromImage(image)
+
+			if hasDigest {
+				imageName, err = name.NewDigest(image)
+				if err != nil {
+					return err
+				}
+			} else {
+				imageName, err = name.NewTag(image)
+				if err != nil {
+					return err
+				}
+			}
+
+			img, err = remote.Image(imageName, remoteOpts...)
+			if err != nil {
+				return err
+			}
 		}
 
-		imgInfo.image, err = remote.Image(imageName, remoteOpts...)
-		if err != nil {
-			return imgInfo, err
-		}
-	}
+		logger = logger.WithValues("image", base, "tag", tag)
+		logger.V(2).Info("scanning image")
 
-	ctx.Logger().WithValues("image", imgInfo.base, "tag", imgInfo.tag).V(2).Info("scanning image")
-
-	return imgInfo, nil
-}
-
-// processLayer processes an individual layer of an image.
-func (s *Source) processLayer(ctx context.Context, layer v1.Layer, imgInfo imageInfo, chunksChan chan *sources.Chunk) error {
-	layerInfo := layerInfo{
-		base: imgInfo.base,
-		tag:  imgInfo.tag,
-	}
-
-	var err error
-	layerInfo.digest, err = layer.Digest()
-	if err != nil {
-		return err
-	}
-
-	ctx.Logger().WithValues("layer", layerInfo.digest.String()).V(2).Info("scanning layer")
-
-	rc, err := layer.Compressed()
-	if err != nil {
-		return err
-	}
-	defer rc.Close()
-
-	gzipReader, err := gzip.NewReader(rc)
-	if err != nil {
-		return err
-	}
-	defer gzipReader.Close()
-
-	tarReader := tar.NewReader(gzipReader)
-	for {
-		header, err := tarReader.Next()
-		if errors.Is(err, io.EOF) {
-			break
-		}
+		layers, err := img.Layers()
 		if err != nil {
 			return err
 		}
 
-		info := chunkProcessingInfo{size: header.Size, name: header.Name, reader: tarReader, layer: layerInfo}
-		if err := s.processChunk(ctx, info, chunksChan); err != nil {
-			return err
-		}
-	}
+		for _, layer := range layers {
 
-	return nil
-}
+			digest, err := layer.Digest()
+			if err != nil {
+				return err
+			}
 
-type chunkProcessingInfo struct {
-	size   int64
-	name   string
-	reader io.Reader
-	layer  layerInfo
-}
+			logger = logger.WithValues("layer", digest.String())
+			logger.V(2).Info("scanning layer")
 
-// processChunk processes an individual chunk of a layer.
-func (s *Source) processChunk(ctx context.Context, info chunkProcessingInfo, chunksChan chan *sources.Chunk) error {
-	const filesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
-	if info.size > filesizeLimitBytes {
-		ctx.Logger().V(4).Info("skipping large file", "file", info.name, "size", info.size)
-		return nil
-	}
+			rc, err := layer.Compressed()
+			if err != nil {
+				return err
+			}
 
-	chunkReader := sources.NewChunkReader()
-	chunkResChan := chunkReader(ctx, info.reader)
+			defer rc.Close()
 
-	for data := range chunkResChan {
-		if err := data.Error(); err != nil {
-			ctx.Logger().Error(err, "error reading chunk.")
-			continue
-		}
+			gzipReader, err := gzip.NewReader(rc)
+			if err != nil {
+				return err
+			}
 
-		chunk := &sources.Chunk{
-			SourceType: s.Type(),
-			SourceName: s.name,
-			SourceID:   s.SourceID(),
-			SourceMetadata: &source_metadatapb.MetaData{
-				Data: &source_metadatapb.MetaData_Docker{
-					Docker: &source_metadatapb.Docker{
-						File:  "/" + info.name,
-						Image: info.layer.base,
-						Tag:   info.layer.tag,
-						Layer: info.layer.digest.String(),
+			defer gzipReader.Close()
+
+			tarReader := tar.NewReader(gzipReader)
+
+			for {
+				header, err := tarReader.Next()
+				if errors.Is(err, io.EOF) {
+					break // End of archive
+				}
+				if err != nil {
+					return err
+				}
+
+				// Skip files larger than FilesizeLimitBytes
+				if header.Size > FilesizeLimitBytes {
+					continue
+				}
+
+				file := bytes.NewBuffer(make([]byte, 0, header.Size))
+
+				_, err = io.Copy(file, tarReader)
+				if err != nil {
+					return err
+				}
+
+				chunk := &sources.Chunk{
+					SourceType: s.Type(),
+					SourceName: s.name,
+					SourceID:   s.SourceID(),
+					JobID:      s.JobID(),
+					Data:       file.Bytes(),
+					SourceMetadata: &source_metadatapb.MetaData{
+						Data: &source_metadatapb.MetaData_Docker{
+							Docker: &source_metadatapb.Docker{
+								File:  "/" + header.Name, // Prepend /, because the file is relative to the root of the image
+								Image: base,
+								Tag:   tag,
+								Layer: digest.String(),
+							},
+						},
 					},
-				},
-			},
-			Verify: s.verify,
-		}
-		chunk.Data = data.Bytes()
+					Verify: s.verify,
+				}
 
-		if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
-			return err
+				select {
+				case chunksChan <- chunk:
+				case <-ctx.Done():
+					return ctx.Err()
+				}
+			}
+
+			dockerLayersScanned.WithLabelValues(s.name).Inc()
 		}
+
+		dockerImagesScanned.WithLabelValues(s.name).Inc()
 	}
 
 	return nil
 }
 
+func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
+	regRepoDelimiter := "/"
+	tagDelim := ":"
+	digestDelim := "@"
+
+	// Split on digest first, if present.
+	parts := strings.SplitN(image, digestDelim, 2)
+	if len(parts) > 1 {
+		base = parts[0]
+		tag = parts[1]
+		hasDigest = true
+		return
+	}
+	parts = strings.Split(image, tagDelim)
+	// Verify that we aren't confusing a tag for a hostname w/ port for the purposes of weak validation.
+	if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelimiter) {
+		base = strings.Join(parts[:len(parts)-1], tagDelim)
+		tag = parts[len(parts)-1]
+	} else {
+		base = image
+		tag = "latest"
+	}
+
+	return
+}
+
 func (s *Source) remoteOpts() ([]remote.Option, error) {
 	switch s.conn.GetCredential().(type) {
 	case *sourcespb.Docker_Unauthenticated:
@@ -300,40 +256,3 @@ func (s *Source) remoteOpts() ([]remote.Option, error) {
 		return nil, fmt.Errorf("unknown credential type: %T", s.conn.Credential)
 	}
 }
-
-func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
-	if base, tag, hasDigest = extractDigest(image); hasDigest {
-		return base, tag, true
-	}
-
-	base, tag = extractTagOrUseDefault(image)
-	return base, tag, false
-}
-
-// extractDigest tries to split the image string on the digest delimiter.
-// If successful, it means the image has a digest.
-func extractDigest(image string) (base, tag string, hasDigest bool) {
-	const digestDelim = "@"
-
-	if parts := strings.SplitN(image, digestDelim, 2); len(parts) > 1 {
-		return parts[0], parts[1], true
-	}
-	return "", "", false
-}
-
-// extractTagOrUseDefault extracts the tag from the image string.
-// If no tag is found, it defaults to "latest".
-func extractTagOrUseDefault(image string) (base, tag string) {
-	const (
-		tagDelim     = ":"
-		regRepoDelim = "/"
-	)
-
-	parts := strings.Split(image, tagDelim)
-
-	// Check if the last part is not a hostname with a port (for weak validation)
-	if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelim) {
-		return strings.Join(parts[:len(parts)-1], tagDelim), parts[len(parts)-1]
-	}
-	return image, "latest"
-}

pkg/sources/filesystem/filesystem.go

@@ -77,7 +77,7 @@ func (s *Source) WithFilter(filter *common.Filter) {
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	for i, path := range s.paths {
 		logger := ctx.Logger().WithValues("path", path)
 		if common.IsDone(ctx) {

pkg/sources/gcs/gcs.go

@@ -248,7 +248,7 @@ func (s *Source) enumerate(ctx context.Context) error {
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	persistableCache := s.setupCache(ctx)
 
 	objectCh, err := s.gcsManager.ListObjects(ctx)

pkg/sources/git/git.go

@@ -153,7 +153,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	if err := s.scanRepos(ctx, chunksChan); err != nil {
 		return err
 	}

pkg/sources/github/github.go

@@ -413,7 +413,7 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	apiEndpoint := s.conn.Endpoint
 	if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) {
 		apiEndpoint = "https://api.github.com"

pkg/sources/gitlab/gitlab.go

@@ -139,7 +139,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	// Start client.
 	apiClient, err := s.newClient()
 	if err != nil {

pkg/sources/s3/s3.go

@@ -31,7 +31,6 @@ import (
 )
 
 const (
-	defaultAWSRegion     = "us-east-1"
 	defaultMaxObjectSize = 250 * 1024 * 1024 // 250 MiB
 	maxObjectSizeLimit   = 250 * 1024 * 1024 // 250 MiB
 )
@@ -54,7 +53,6 @@ type Source struct {
 // Ensure the Source satisfies the interfaces at compile time
 var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
-var _ sources.Validator = (*Source)(nil)
 
 // Type returns the type of source
 func (s *Source) Type() sourcespb.SourceType {
@@ -95,23 +93,6 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 	return nil
 }
 
-func (s *Source) Validate(ctx context.Context) []error {
-	var errors []error
-	visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
-		roleErrs := s.validateBucketAccess(c, defaultRegionClient, roleArn, buckets)
-		if len(roleErrs) > 0 {
-			errors = append(errors, roleErrs...)
-		}
-	}
-
-	err := s.visitRoles(ctx, visitor)
-	if err != nil {
-		errors = append(errors, err)
-	}
-
-	return errors
-}
-
 // setMaxObjectSize sets the maximum size of objects that will be scanned. If
 // not set, set to a negative number, or set larger than the
 // maxObjectSizeLimit, the defaultMaxObjectSizeLimit will be used.
@@ -172,7 +153,7 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
 
 	res, err := client.ListBuckets(&s3.ListBucketsInput{})
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("could not list s3 buckets: %w", err)
 	}
 
 	var bucketsToScan []string
@@ -182,30 +163,34 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
 	return bucketsToScan, nil
 }
 
-func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bucketsToScan []string, chunksChan chan *sources.Chunk) {
+func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bucketsToScan []string, chunksChan chan *sources.Chunk) error {
+	const defaultAWSRegion = "us-east-1"
 	objectCount := uint64(0)
 
-	logger := s.log
-	if role != "" {
-		logger = logger.WithValues("roleArn", role)
-	}
-
 	for i, bucket := range bucketsToScan {
-		logger := logger.WithValues("bucket", bucket)
-
 		if common.IsDone(ctx) {
-			return
+			return nil
 		}
 
 		s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "")
-		logger.Info("Scanning bucket")
-
-		regionalClient, err := s.getRegionalClientForBucket(ctx, client, role, bucket)
+		s.log.Info("Scanning bucket", "bucket", bucket)
+		region, err := s3manager.GetBucketRegionWithClient(ctx, client, bucket)
 		if err != nil {
-			logger.Error(err, "could not get regional client for bucket")
+			s.log.Error(err, "could not get s3 region for bucket", "bucket: ", bucket)
 			continue
 		}
 
+		var regionalClient *s3.S3
+		if region != defaultAWSRegion {
+			regionalClient, err = s.newClient(region, role)
+			if err != nil {
+				s.log.Error(err, "could not make regional s3 client")
+				continue
+			}
+		} else {
+			regionalClient = client
+		}
+
 		errorCount := sync.Map{}
 
 		err = regionalClient.ListObjectsV2PagesWithContext(
@@ -216,46 +201,40 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
 			})
 
 		if err != nil {
-			if role == "" {
-				logger.Error(err, "could not list objects in bucket")
-			} else {
-				// Our documentation blesses specifying a role to assume without specifying buckets to scan, which will
-				// often cause this to happen a lot (because in that case the scanner tries to scan every bucket in the
-				// account, but the role probably doesn't have access to all of them). This makes it expected behavior
-				// and therefore not an error.
-				logger.V(3).Info("could not list objects in bucket",
-					"err", err)
-			}
+			s.log.Error(err, "could not list objects in s3 bucket", "bucket: ", bucket)
+			continue
 		}
 	}
 	s.SetProgressComplete(len(bucketsToScan), len(bucketsToScan), fmt.Sprintf("Completed scanning source %s. %d objects scanned.", s.name, objectCount), "")
+	return nil
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
-	visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
-		s.scanBuckets(c, defaultRegionClient, roleArn, buckets, chunksChan)
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+	const defaultAWSRegion = "us-east-1"
+
+	roles := s.conn.Roles
+	if len(roles) == 0 {
+		roles = []string{""}
 	}
 
-	return s.visitRoles(ctx, visitor)
-}
+	for _, role := range roles {
+		client, err := s.newClient(defaultAWSRegion, role)
+		if err != nil {
+			return errors.WrapPrefix(err, "could not create s3 client", 0)
+		}
 
-func (s *Source) getRegionalClientForBucket(ctx context.Context, defaultRegionClient *s3.S3, role, bucket string) (*s3.S3, error) {
-	region, err := s3manager.GetBucketRegionWithClient(ctx, defaultRegionClient, bucket)
-	if err != nil {
-		return nil, errors.WrapPrefix(err, "could not get s3 region for bucket", 0)
+		bucketsToScan, err := s.getBucketsToScan(client)
+		if err != nil {
+			return err
+		}
+
+		if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil {
+			return err
+		}
 	}
 
-	if region == defaultAWSRegion {
-		return defaultRegionClient, nil
-	}
-
-	regionalClient, err := s.newClient(region, role)
-	if err != nil {
-		return nil, errors.WrapPrefix(err, "could not create regional s3 client", 0)
-	}
-
-	return regionalClient, nil
+	return nil
 }
 
 // pageChunker emits chunks onto the given channel from a page
@@ -417,65 +396,6 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 	_ = s.jobPool.Wait()
 }
 
-func (s *Source) validateBucketAccess(ctx context.Context, client *s3.S3, roleArn string, buckets []string) []error {
-	shouldHaveAccessToAllBuckets := roleArn == ""
-	wasAbleToListAnyBucket := false
-	var errors []error
-
-	for _, bucket := range buckets {
-		if common.IsDone(ctx) {
-			return append(errors, ctx.Err())
-		}
-
-		regionalClient, err := s.getRegionalClientForBucket(ctx, client, roleArn, bucket)
-		if err != nil {
-			errors = append(errors, fmt.Errorf("could not get regional client for bucket %q: %w", bucket, err))
-			continue
-		}
-
-		_, err = regionalClient.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &bucket})
-
-		if err == nil {
-			wasAbleToListAnyBucket = true
-		} else if shouldHaveAccessToAllBuckets {
-			errors = append(errors, fmt.Errorf("could not list objects in bucket %q: %w", bucket, err))
-		}
-	}
-
-	if !wasAbleToListAnyBucket {
-		if roleArn == "" {
-			errors = append(errors, fmt.Errorf("could not list objects in any bucket"))
-		} else {
-			errors = append(errors, fmt.Errorf("role %q could not list objects in any bucket", roleArn))
-		}
-	}
-
-	return errors
-}
-
-func (s *Source) visitRoles(ctx context.Context, f func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string)) error {
-	roles := s.conn.Roles
-	if len(roles) == 0 {
-		roles = []string{""}
-	}
-
-	for _, role := range roles {
-		client, err := s.newClient(defaultAWSRegion, role)
-		if err != nil {
-			return errors.WrapPrefix(err, "could not create s3 client", 0)
-		}
-
-		bucketsToScan, err := s.getBucketsToScan(client)
-		if err != nil {
-			return fmt.Errorf("role %q could not list any s3 buckets for scanning: %w", role, err)
-		}
-
-		f(ctx, client, role, bucketsToScan)
-	}
-
-	return nil
-}
-
 // S3 links currently have the general format of:
 // https://[bucket].s3[.region unless us-east-1].amazonaws.com/[key]
 func makeS3Link(bucket, region, key string) string {

pkg/sources/s3/s3_integration_test.go

@@ -4,14 +4,10 @@
 package s3
 
 import (
-	"fmt"
-	"sync"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
 	"google.golang.org/protobuf/types/known/anypb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -49,127 +45,3 @@ func TestSource_ChunksCount(t *testing.T) {
 	}
 	assert.Greater(t, got, wantChunkCount)
 }
-
-func TestSource_Validate(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
-	defer cancel()
-
-	secret, err := common.GetTestSecret(ctx)
-	if err != nil {
-		t.Fatal(fmt.Errorf("failed to access secret: %v", err))
-	}
-
-	s3key := secret.MustGetField("AWS_S3_KEY")
-	s3secret := secret.MustGetField("AWS_S3_SECRET")
-
-	tests := []struct {
-		name         string
-		roles        []string
-		buckets      []string
-		wantErrCount int
-	}{
-		{
-			name: "buckets without roles, can access all buckets",
-			buckets: []string{
-				"truffletestbucket-s3-tests",
-			},
-			wantErrCount: 0,
-		},
-		{
-			name: "buckets without roles, one error per inaccessible bucket",
-			buckets: []string{
-				"truffletestbucket-s3-tests",
-				"truffletestbucket-s3-role-assumption",
-				"truffletestbucket-no-access",
-			},
-			wantErrCount: 2,
-		},
-		{
-			name: "roles without buckets, all can access at least one account bucket",
-			roles: []string{
-				"arn:aws:iam::619888638459:role/s3-test-assume-role",
-			},
-			wantErrCount: 0,
-		},
-		{
-			name: "roles without buckets, one error per role that cannot access any account buckets",
-			roles: []string{
-				"arn:aws:iam::619888638459:role/s3-test-assume-role",
-				"arn:aws:iam::619888638459:role/test-no-access",
-			},
-			wantErrCount: 1,
-		},
-		{
-			name: "role and buckets, can access at least one bucket",
-			roles: []string{
-				"arn:aws:iam::619888638459:role/s3-test-assume-role",
-			},
-			buckets: []string{
-				"truffletestbucket-s3-role-assumption",
-				"truffletestbucket-no-access",
-			},
-			wantErrCount: 0,
-		},
-		{
-			name: "roles and buckets, one error per role that cannot access at least one bucket",
-			roles: []string{
-				"arn:aws:iam::619888638459:role/s3-test-assume-role",
-				"arn:aws:iam::619888638459:role/test-no-access",
-			},
-			buckets: []string{
-				"truffletestbucket-s3-role-assumption",
-				"truffletestbucket-no-access",
-			},
-			wantErrCount: 1,
-		},
-		{
-			name: "role and buckets, a bucket doesn't even exist",
-			roles: []string{
-				"arn:aws:iam::619888638459:role/s3-test-assume-role",
-			},
-			buckets: []string{
-				"truffletestbucket-s3-role-assumption",
-				"not-a-real-bucket-asljdhmglasjgvklhsdaljfh", // need a bucket name that nobody is likely to ever create
-			},
-			wantErrCount: 1,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
-			var cancelOnce sync.Once
-			defer cancelOnce.Do(cancel)
-
-			// These are used by the tests that assume roles
-			t.Setenv("AWS_ACCESS_KEY_ID", s3key)
-			t.Setenv("AWS_SECRET_ACCESS_KEY", s3secret)
-
-			s := &Source{}
-
-			conn, err := anypb.New(&sourcespb.S3{
-				// These are used by the tests that don't assume roles
-				Credential: &sourcespb.S3_AccessKey{
-					AccessKey: &credentialspb.KeySecret{
-						Key:    s3key,
-						Secret: s3secret,
-					},
-				},
-				Buckets: tt.buckets,
-				Roles:   tt.roles,
-			})
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = s.Init(ctx, tt.name, 0, 0, false, conn, 0)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			errs := s.Validate(ctx)
-
-			assert.Equal(t, tt.wantErrCount, len(errs))
-		})
-	}
-}

pkg/sources/source_manager_test.go

@@ -6,11 +6,10 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"google.golang.org/protobuf/types/known/anypb"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"google.golang.org/protobuf/types/known/anypb"
 )
 
 // DummySource implements Source and is used for testing a SourceManager.
@@ -32,7 +31,7 @@ func (d *DummySource) GetProgress() *Progress { return nil }
 
 // Interface to easily test different chunking methods.
 type chunker interface {
-	Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error
+	Chunks(context.Context, chan *Chunk) error
 	ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
 	Enumerate(ctx context.Context, reporter UnitReporter) error
 }
@@ -43,7 +42,7 @@ type counterChunker struct {
 	count        int
 }
 
-func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
+func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
 	for i := 0; i < c.count; i++ {
 		select {
 		case ch <- &Chunk{Data: []byte{c.chunkCounter}}:
@@ -76,9 +75,9 @@ func (c *counterChunker) ChunkUnit(ctx context.Context, unit SourceUnit, reporte
 // Chunk method that always returns an error.
 type errorChunker struct{ error }
 
-func (c errorChunker) Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error { return c }
-func (c errorChunker) Enumerate(context.Context, UnitReporter) error                { return c }
-func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error   { return c }
+func (c errorChunker) Chunks(context.Context, chan *Chunk) error                  { return c }
+func (c errorChunker) Enumerate(context.Context, UnitReporter) error              { return c }
+func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
 
 // enrollDummy is a helper function to enroll a DummySource with a SourceManager.
 func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {
@@ -177,7 +176,7 @@ type unitChunk struct {
 
 type unitChunker struct{ steps []unitChunk }
 
-func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
+func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
 	for _, step := range c.steps {
 		if step.err != "" {
 			continue
@@ -295,9 +294,7 @@ type callbackChunker struct {
 	cb func(context.Context, chan *Chunk) error
 }
 
-func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
-	return c.cb(ctx, ch)
-}
+func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk) error           { return c.cb(ctx, ch) }
 func (c callbackChunker) Enumerate(context.Context, UnitReporter) error              { return nil }
 func (c callbackChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return nil }
 

pkg/sources/sources.go

@@ -30,16 +30,6 @@ type Chunk struct {
 	Verify bool
 }
 
-// ChunkingTarget specifies criteria for a targeted chunking process.
-// Instead of collecting data indiscriminately, this struct allows the caller
-// to specify particular subsets of data they're interested in. This becomes
-// especially useful when one needs to verify or recheck specific data points
-// without processing the entire dataset.
-type ChunkingTarget struct {
-	// QueryCriteria represents specific parameters or conditions to target the chunking process.
-	QueryCriteria source_metadatapb.MetaData
-}
-
 // Source defines the interface required to implement a source chunker.
 type Source interface {
 	// Type returns the source type, used for matching against configuration and jobs.
@@ -50,12 +40,8 @@ type Source interface {
 	JobID() int64
 	// Init initializes the source.
 	Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error
-	// Chunks emits data over a channel which is then decoded and scanned for secrets.
-	// By default, data is obtained indiscriminately. However, by providing one or more
-	// ChunkingTarget parameters, the caller can direct the function to retrieve
-	// specific chunks of data. This targeted approach allows for efficient and
-	// intentional data processing, beneficial when verifying or rechecking specific data points.
-	Chunks(ctx context.Context, chunksChan chan *Chunk, targets ...ChunkingTarget) error
+	// Chunks emits data over a channel that is decoded and scanned for secrets.
+	Chunks(ctx context.Context, chunksChan chan *Chunk) error
 	// GetProgress is the completion progress (percentage) for Scanned Source.
 	GetProgress() *Progress
 }

pkg/sources/syslog/syslog.go

@@ -182,7 +182,7 @@ func (s *Source) verifyConnectionConfig() error {
 }
 
 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	switch {
 	case s.conn.TlsCert != nilString || s.conn.TlsKey != nilString:
 		cert, err := tls.X509KeyPair([]byte(s.conn.TlsCert), []byte(s.conn.TlsKey))

proto/detectors.proto

@@ -937,7 +937,6 @@ enum DetectorType {
   HuggingFace = 926;
   Snowflake = 927;
   Sourcegraph = 928;
-  Tailscale = 929;
 }
 
 message Result {