Merge branch 'feature-bytification' into detectors-34

* fix sentry detector to check response.

* use err.

* address comments.

pkg/detectors/sentrytoken/sentrytoken.go

@@ -2,7 +2,10 @@ package sentrytoken
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"regexp"
 	"strings"
@@ -12,16 +15,20 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	defaultClient = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sentry"}) + `\b([a-f0-9]{64})\b`)
+
+	errUnauthorized = fmt.Errorf("token unauthorized")
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -47,32 +54,78 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://sentry.io/api/0/projects/", nil)
-			if err != nil {
-				continue
+			client := s.client
+			if client == nil {
+				client = defaultClient
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
-			// req.Header.Add("Cookie", "sc=g2ceDPcsBwPLGXugHUv0coGBi09TQYKqmtp0u4JYdIvOTFd28yVvn1yiuXgmJJa8; session=.eJwNyE0OwiAQQGGsPwtP4kqoGQa4gUdgZ2CGiaZJCUqjGxOPbnfve7_h29QpHpVSvbw61To9StvEwzre9TkVbkPcr7jN5dPbNl7X1jUt_a4vLpEHgUycQSgFZ0YBRkGxnthxwIyBAgdngcF4YxBssiMjekGv2245_wHyySMG:1m0J5S:FjpRlBVuFx577UQp6GcSGWTq0uk")
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-						continue
-					}
-				}
+			isVerified, verificationErr := verifyToken(ctx, client, resMatch)
+
+			switch {
+			case errors.Is(verificationErr, errUnauthorized):
+				s1.Verified = false
+			case isVerified:
+				s1.Verified = true
+			default:
+				s1.VerificationError = verificationErr
 			}
 		}
 
+		if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+			continue
+		}
 		results = append(results, s1)
 	}
 
 	return results, nil
 }
 
+type Response []Project
+
+type Project struct {
+	Organization Organization `json:"organization"`
+}
+
+type Organization struct {
+	ID   string `json:"id"`
+	Name string `json:"name"`
+}
+
+func verifyToken(ctx context.Context, client *http.Client, token string) (bool, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://sentry.io/api/0/projects/", nil)
+	if err != nil {
+		return false, err
+	}
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+
+	res, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer res.Body.Close()
+
+	var isVerified bool
+	switch res.StatusCode {
+	case http.StatusOK, http.StatusForbidden:
+		isVerified = true
+	case http.StatusUnauthorized:
+		return false, errUnauthorized
+	default:
+		return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+	}
+
+	bytes, readErr := io.ReadAll(res.Body)
+	if readErr != nil {
+		return false, readErr
+	}
+
+	var resp Response
+	if err = json.Unmarshal(bytes, &resp); err != nil {
+		return false, err
+	}
+
+	return isVerified, err
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_SentryToken
 }

pkg/detectors/sentrytoken/sentrytoken_test.go

@@ -9,7 +9,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -32,11 +34,12 @@ func TestSentryToken_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
@@ -70,6 +73,56 @@ func TestSentryToken_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "found, would be verified but for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SentryToken,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found and valid but unexpected api response",
+			s:    Scanner{client: common.ConstantResponseHttpClient(500, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SentryToken,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, good key but wrong scope",
+			s:    Scanner{client: common.ConstantResponseHttpClient(403, responseBody403)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SentryToken,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
 		{
 			name: "not found",
 			s:    Scanner{},
@@ -82,27 +135,46 @@ func TestSentryToken_FromChunk(t *testing.T) {
 			wantErr: false,
 		},
 	}
+
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("SentryToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			for i := range got {
 				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
+					t.Fatal("no raw secret present")
+				}
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
 				}
-				got[i].Raw = nil
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("SentryToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, opts); diff != "" {
+				t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}
 }
 
+const responseBody403 = `
+[
+  {
+    "organization": {
+      "id": "911964",
+      "slug": "wigslap",
+      "status": {
+        "id": "active",
+        "name": "active"
+      },
+      "name": "wigslap"
+    }
+  }
+]
+`
+
 func BenchmarkFromData(benchmark *testing.B) {
 	ctx := context.Background()
 	s := Scanner{}

pkg/detectors/sourcegraph/sourcegraph.go

@@ -22,13 +22,13 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	defaultClient = common.SaneHttpClient()
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sourcegraph", "sg_token"}) + `\b(sgp_[a-z0-9]{40})\b`)
+	keyPat = regexp.MustCompile(`\b(sgp_[a-f0-9]{40})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"sourcegraph", "sg_token"}
+	return []string{"sgp_"}
 }
 
 // FromData will find and optionally verify Sourcegraph secrets in a given set of bytes.

pkg/detectors/tallyfy/tallyfy.go

@@ -1,11 +1,11 @@
 package tallyfy
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"tallyfy"}) + `\b([0-9A-Za-z]{36}\.[0-9A-Za-z]{264}\.[0-9A-Za-z\-\_]{683})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"tallyfy"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("tallyfy")}
 }
 
-// FromData will find and optionally verify Tallyfy secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Tallyfy,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -53,24 +45,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Accept", "application/json")
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
 				}
 			}
 		}
-
 		results = append(results, s1)
 	}
-
 	return results, nil
 }
 

pkg/detectors/tatumio/tatumio.go

@@ -1,10 +1,10 @@
 package tatumio
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,37 +13,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"tatum"}) + `\b([0-9a-z-]{36})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"tatum"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("tatum")}
 }
 
-// FromData will find and optionally verify TatumIO secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_TatumIO,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Accept", "application/vnd.tatumio+json; version=3")
-			req.Header.Add("x-api-key", resMatch)
+			req.Header.Add("x-api-key", string(resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/taxjar/taxjar.go

@@ -1,11 +1,11 @@
 package taxjar
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"taxjar"}) + `\b([a-z0-9]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"taxjar"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("taxjar")}
 }
 
-// FromData will find and optionally verify Taxjar secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Taxjar,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,17 +45,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-						continue
-					}
+			if err != nil {
+				continue
+			}
+			defer res.Body.Close()
+			if res.StatusCode >= 200 && res.StatusCode < 300 {
+				s1.Verified = true
+			} else {
+				if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+					continue
 				}
 			}
 		}

pkg/detectors/teamgate/teamgate.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,53 +12,48 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	client   = common.SaneHttpClient()
 	tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"teamgate"}) + `\b([a-z0-9]{40})\b`)
 	keyPat   = regexp.MustCompile(detectors.PrefixRegex([]string{"teamgate"}) + `\b([a-zA-Z0-9]{80})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"teamgate"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("teamgate")}
 }
 
-// FromData will find and optionally verify Teamgate secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := tokenPat.FindAllStringSubmatch(dataStr, -1)
-	keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := tokenPat.FindAllSubmatch(data, -1)
+	keyMatches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := match[1]
 
 		for _, keyMatch := range keyMatches {
 			if len(keyMatch) != 2 {
 				continue
 			}
 
-			resKeyMatch := strings.TrimSpace(keyMatch[1])
+			resKeyMatch := keyMatch[1]
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Teamgate,
-				Raw:          []byte(resMatch),
+				Raw:          resMatch,
 			}
+
 			if verify {
 				req, err := http.NewRequestWithContext(ctx, "GET", "https://api.teamgate.com/v4/users", nil)
 				if err != nil {
 					continue
 				}
-				req.Header.Add("X-Auth-Token", resMatch)
-				req.Header.Add("X-App-Key", resKeyMatch)
+
+				req.Header.Add("X-Auth-Token", string(resMatch))
+				req.Header.Add("X-App-Key", string(resKeyMatch))
 
 				res, err := client.Do(req)
 
@@ -68,7 +62,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}

pkg/detectors/teamworkcrm/teamworkcrm.go

@@ -1,11 +1,11 @@
 package teamworkcrm
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"teamwork", "teamworkcrm"}) + `\b(tkn\.v1_[0-9A-Za-z]{71}=[ \r\n]{1})`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"teamwork", "teamworkcrm"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("teamwork"), []byte("teamworkcrm")}
 }
 
-// FromData will find and optionally verify TeamworkCRM secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_TeamworkCRM,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -59,7 +51,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/teamworkdesk/teamworkdesk.go

@@ -1,11 +1,11 @@
 package teamworkdesk
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"teamwork", "teamworkdesk"}) + `\b(tkn\.v1_[0-9A-Za-z]{71}=[ \r\n]{1})`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"teamwork", "teamworkdesk"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("teamwork"), []byte("teamworkdesk")}
 }
 
-// FromData will find and optionally verify TeamworkDesk secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_TeamworkDesk,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,21 +44,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
 				}
 			}
 		}
-
 		results = append(results, s1)
 	}
 

pkg/detectors/teamworkspaces/teamworkspaces.go

@@ -1,11 +1,11 @@
 package teamworkspaces
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,31 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"teamwork", "teamworkspaces"}) + `\b(tkn\.v1_[0-9A-Za-z]{71}=[ \r\n]{1})`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"teamwork", "teamworkspaces"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("teamwork"), []byte("teamworkspaces")}
 }
 
-// FromData will find and optionally verify TeamworkSpaces secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_TeamworkSpaces,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,14 +46,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/technicalanalysisapi/technicalanalysisapi.go

@@ -1,11 +1,11 @@
 package technicalanalysisapi
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -27,41 +27,38 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"technicalanalysisapi"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("technicalanalysisapi")}
 }
 
 // FromData will find and optionally verify TechnicalAnalysisApi secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_TechnicalAnalysisApi,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
 			timeout := 10 * time.Second
 			client.Timeout = timeout
-			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://technical-analysis-api.com/api/v1/analysis/BTC?apiKey=%s", resMatch), nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://technical-analysis-api.com/api/v1/analysis/BTC?apiKey=%s", string(resMatch)), nil)
 			if err != nil {
 				continue
 			}
 			res, err := client.Do(req)
-			if err == nil {
+			if err == nil && res != nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/tefter/tefter.go

@@ -1,12 +1,12 @@
 package tefter
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"io"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,37 +15,31 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"tefter"}) + `\b([0-9a-zA-Z]{20})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"tefter"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("tefter")}
 }
 
-// FromData will find and optionally verify Tefter secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Tefter,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -54,21 +48,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("X-User-Token", resMatch)
+			req.Header.Add("X-User-Token", string(resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				bodyBytes, err := io.ReadAll(res.Body)
 				if err != nil {
 					continue
 				}
-				bodyString := string(bodyBytes)
-				validResponse := json.Valid([]byte(bodyString))
 
 				defer res.Body.Close()
+
+				validResponse := json.Valid(bodyBytes)
+
 				if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/telegrambottoken/telegrambottoken.go

@@ -1,11 +1,10 @@
 package telegrambottoken
 
 import (
+	"bytes"
 	"context"
-	//	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -27,30 +26,29 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"telegram"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("telegram")}
 }
 
 // FromData will find and optionally verify TelegramBotToken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		key := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_TelegramBotToken,
-			Raw:          []byte(key),
+			Raw:          resMatch,
 		}
 
 		if verify {
 			// https://core.telegram.org/bots/api#getme
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.telegram.org/bot"+key+"/getMe", nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.telegram.org/bot"+string(resMatch)+"/getMe", nil)
 			if err != nil {
 				continue
 			}
@@ -60,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					if detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) {
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
 				}

pkg/detectors/teletype/teletype.go

@@ -1,11 +1,11 @@
 package teletype
 
 import (
+	"bytes"
 	"context"
 	"io"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"teletype"}) + `\b([0-9a-zA-Z-]{64})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"teletype"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("teletype")}
 }
 
-// FromData will find and optionally verify Teletype secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Teletype,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -53,18 +45,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("X-Auth-Token", resMatch)
+			req.Header.Add("X-Auth-Token", string(resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				bodyBytes, err := io.ReadAll(res.Body)
+				defer res.Body.Close()
 				if err != nil {
 					continue
 				}
 
-				bodyString := string(bodyBytes)
-				validResponse := strings.Contains(bodyString, `"code":401`)
+				validResponse := bytes.Contains(bodyBytes, []byte(`"code":401`))
 
-				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					if validResponse {
 						s1.Verified = false
@@ -72,7 +63,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						s1.Verified = true
 					}
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/telnyx/telnyx.go

@@ -1,11 +1,11 @@
 package telnyx
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"telnyx"}) + `\b(KEY[0-9A-Za-z_-]{55})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"telnyx"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("telnyx")}
 }
 
-// FromData will find and optionally verify Telnyx secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Telnyx,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -53,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Accept", "application/vnd.telnyx+json; version=3")
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/terraformcloudpersonaltoken/terraformcloudpersonaltoken.go

@@ -1,11 +1,11 @@
 package terraformcloudpersonaltoken
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,36 +14,29 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	keyPat = regexp.MustCompile(`\b([A-Za-z0-9]{14}.atlasv1.[A-Za-z0-9]{67})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{".atlasv1."}) + `b([A-Za-z0-9]{14}.atlasv1.[A-Za-z0-9]{67})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{".atlasv1."}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte(".atlasv1.")}
 }
 
-// FromData will find and optionally verify TerraformCloudPersonalToken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_TerraformCloudPersonalToken,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -51,7 +44,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -64,7 +57,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 			}
 		}
-
 		results = append(results, s1)
 	}
 

pkg/detectors/testingbot/testingbot.go

@@ -1,10 +1,10 @@
 package testingbot
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -26,31 +26,28 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"testingbot"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("testingbot")}
 }
 
-// FromData will find and optionally verify TestingBot secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	idmatches := idPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
+	idmatches := idPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 		for _, idmatch := range idmatches {
 			if len(idmatch) != 2 {
 				continue
 			}
-			resIdMatch := strings.TrimSpace(idmatch[1])
+			resIdMatch := bytes.TrimSpace(idmatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_TestingBot,
-				Raw:          []byte(resMatch),
+				Raw:          resMatch,
 			}
 
 			if verify {
@@ -58,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if err != nil {
 					continue
 				}
-				req.SetBasicAuth(resIdMatch, resMatch)
+				req.SetBasicAuth(string(resIdMatch), string(resMatch))
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()

pkg/detectors/text2data/text2data.go

@@ -1,6 +1,7 @@
 package text2data
 
 import (
+	"bytes"
 	"context"
 	"io"
 	"net/http"
@@ -15,44 +16,36 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"text2data"}) + `\b([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"text2data"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("text2data")}
 }
 
-// FromData will find and optionally verify Text2Data secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Text2Data,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-
 			data := url.Values{}
 			data.Add("DocumentText", "Excellent location, opposite a very large mall with wide variety of shops, restaurants and more.")
-			data.Add("PrivateKey", resMatch)
+			data.Add("PrivateKey", string(resMatch))
 
 			req, err := http.NewRequestWithContext(ctx, "POST", "http://api.text2data.com/v3/Analyze", strings.NewReader(data.Encode()))
 			if err != nil {
@@ -66,19 +59,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				body, errBody := io.ReadAll(res.Body)
 
 				if errBody == nil {
-					bodyString := string(body)
-					validResponse := strings.Contains(bodyString, `"DocSentimentResultString":"positive"`)
+					validResponse := bytes.Contains(body, []byte(`"DocSentimentResultString":"positive"`))
 
 					if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse {
 						s1.Verified = true
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}
 					}
 				}
-
 			}
 		}
 

pkg/detectors/textmagic/textmagic.go

@@ -1,12 +1,12 @@
 package textmagic
 
 import (
+	"bytes"
 	"context"
 	b64 "encoding/base64"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,51 +15,46 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"textmagic"}) + `\b([0-9A-Za-z]{30})\b`)
 	userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"textmagic"}) + `\b([0-9A-Za-z]{1,25})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"textmagic"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("textmagic")}
 }
 
-// FromData will find and optionally verify Textmagic secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
+	keyMatches := keyPat.FindAllSubmatch(data, -1)
+	userMatches := userPat.FindAllSubmatch(data, -1)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	userMatches := userPat.FindAllStringSubmatch(dataStr, -1)
-
-	for _, match := range matches {
+	for _, match := range keyMatches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		for _, userMatch := range userMatches {
-			if len(match) != 2 {
+			if len(userMatch) != 2 {
 				continue
 			}
-			resUser := strings.TrimSpace(userMatch[1])
+			resUser := bytes.TrimSpace(userMatch[1])
+
+			rawV2 := append(resMatch, resUser...)
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Textmagic,
-				Raw:          []byte(resMatch),
-				RawV2:        []byte(resMatch + resUser),
+				Raw:          resMatch,
+				RawV2:        rawV2,
 			}
 
 			if verify {
-				data := fmt.Sprintf("%s:%s", resUser, resMatch)
-				sEnc := b64.StdEncoding.EncodeToString([]byte(data))
+				data := append(resUser, resMatch...)
+				sEnc := b64.StdEncoding.EncodeToString(data)
 				req, err := http.NewRequestWithContext(ctx, "GET", "https://rest.textmagic.com/api/v2/user", nil)
 				if err != nil {
 					continue
@@ -72,7 +67,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}

pkg/detectors/theoddsapi/theoddsapi.go

@@ -1,10 +1,11 @@
 package theoddsapi
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,41 +14,34 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"theoddsapi", "the-odds-api"}) + `\b([0-9a-f]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"theoddsapi", "the-odds-api"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("theoddsapi"), []byte("the-odds-api")}
 }
 
-// FromData will find and optionally verify TheOddsApi secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_TheOddsApi,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			req, err := http.NewRequest("GET", "https://api.the-odds-api.com/v4/sports/?apiKey="+resMatch, nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.the-odds-api.com/v4/sports/?apiKey=%s", resMatch), nil)
 			if err != nil {
 				continue
 			}
@@ -57,7 +51,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/thinkific/thinkific.go

@@ -1,12 +1,12 @@
 package thinkific
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -28,40 +28,39 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"thinkific"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("thinkific")}
 }
 
 // FromData will find and optionally verify Thinkific secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	domainMatches := domainPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
+	domainMatches := domainPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 		for _, domainMatch := range domainMatches {
 
 			if len(domainMatch) != 2 {
 				continue
 			}
-			resDomainMatch := strings.TrimSpace(domainMatch[1])
+			resDomainMatch := bytes.TrimSpace(domainMatch[1])
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Thinkific,
-				Raw:          []byte(resMatch),
+				Raw:          resMatch,
 			}
 
 			if verify {
-				domainRes := fmt.Sprintf("%s-s-school", resDomainMatch)
+				domainRes := fmt.Sprintf("%s-s-school", string(resDomainMatch))
 				req, err := http.NewRequestWithContext(ctx, "GET", "https://api.thinkific.com/api/public/v1/collections", nil)
 				if err != nil {
 					continue
 				}
-				req.Header.Add("X-Auth-API-Key", resMatch)
+				req.Header.Add("X-Auth-API-Key", string(resMatch))
 				req.Header.Add("X-Auth-Subdomain", domainRes)
 				req.Header.Add("Content-Type", "application/json")
 				res, err := client.Do(req)
@@ -71,22 +70,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					if err != nil {
 						continue
 					}
-					body := string(bodyBytes)
 
-					if strings.Contains(body, "API Access is not available") {
+					if bytes.Contains(bodyBytes, []byte("API Access is not available")) {
 						s1.Verified = true
 					} else {
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}
 					}
-
 				}
 			}
 
 			results = append(results, s1)
 		}
-
 	}
 
 	return results, nil

pkg/detectors/thousandeyes/thousandeyes.go

@@ -1,10 +1,10 @@
 package thousandeyes
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,46 +13,39 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"thousandeyes"}) + `\b([a-zA-Z0-9]{32})\b`)
 	email  = regexp.MustCompile(detectors.PrefixRegex([]string{"thousandeyes"}) + `\b([a-zA-Z0-9]{3,20}@[a-zA-Z0-9]{2,12}.[a-zA-Z0-9]{2,5})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"thousandeyes"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("thousandeyes")}
 }
 
-// FromData will find and optionally verify ThousandEyes secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	emailMatches := email.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
+	emailMatches := email.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		tokenPatMatch := strings.TrimSpace(match[1])
+		tokenPatMatch := bytes.TrimSpace(match[1])
 
 		for _, emailMatch := range emailMatches {
 			if len(emailMatch) != 2 {
 				continue
 			}
 
-			userPatMatch := strings.TrimSpace(emailMatch[1])
+			userPatMatch := bytes.TrimSpace(emailMatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_ThousandEyes,
-				Raw:          []byte(tokenPatMatch),
+				Raw:          tokenPatMatch,
 			}
 
 			if verify {
@@ -60,14 +53,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if err != nil {
 					continue
 				}
-				req.SetBasicAuth(userPatMatch, tokenPatMatch)
+				req.SetBasicAuth(string(userPatMatch), string(tokenPatMatch))
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 						if detectors.IsKnownFalsePositive(tokenPatMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}

pkg/detectors/ticketmaster/ticketmaster.go

@@ -1,11 +1,11 @@
 package ticketmaster
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -26,29 +26,27 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"ticketmaster"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("ticketmaster")}
 }
 
 // FromData will find and optionally verify TicketMaster secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_TicketMaster,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://app.ticketmaster.com/discovery/v2/events.json?apikey=%s", resMatch), nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://app.ticketmaster.com/discovery/v2/events.json?apikey=%s", string(resMatch)), nil)
 			if err != nil {
 				continue
 			}
@@ -58,7 +56,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/giturl/giturl.go

@@ -132,9 +132,9 @@ func GenerateLink(repo, commit, file string, line int64) string {
 		return baseLink
 
 	case providerAzure:
-		baseLink := repo + "?path=" + file + "&version=GB" + commit
+		baseLink := repo + "/commit/" + commit + "/" + file
 		if line > 0 {
-			baseLink += "&line=" + strconv.FormatInt(line, 10)
+			baseLink += "?line=" + strconv.FormatInt(line, 10)
 		}
 		return baseLink
 

pkg/giturl/giturl_test.go

@@ -148,7 +148,7 @@ func TestGenerateLink(t *testing.T) {
 				commit: "abcdef",
 				file:   "main.go",
 			},
-			want: "https://dev.azure.com/org/project/_git/repo?path=main.go&version=GBabcdef",
+			want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go",
 		},
 		{
 			name: "Azure link gen with line",
@@ -158,7 +158,7 @@ func TestGenerateLink(t *testing.T) {
 				file:   "main.go",
 				line:   int64(20),
 			},
-			want: "https://dev.azure.com/org/project/_git/repo?path=main.go&version=GBabcdef&line=20",
+			want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=20",
 		},
 	}
 	for _, tt := range tests {