Merge branch 'feature-bytification' into detectors-36

main.go

@@ -136,12 +136,6 @@ var (
 	circleCiScan      = cli.Command("circleci", "Scan CircleCI")
 	circleCiScanToken = circleCiScan.Flag("token", "CircleCI token. Can also be provided with environment variable").Envar("CIRCLECI_TOKEN").Required().String()
 
-	trelloScan      = cli.Command("trello", "Scan Trello")
-	trelloApiKey    = trelloScan.Flag("apiKey", "Trello API Key. Can also be provided with environment variable").Envar("TRELLO_API_KEY").Required().String()
-	trelloToken     = trelloScan.Flag("token", "Trello token. Can also be provided with environment variable").Envar("TRELLO_TOKEN").Required().String()
-	trelloBoardIDs  = trelloScan.Flag("boardIDs", "Trello board IDs to scan. Can also be provided with environment variable").Envar("TRELLO_BOARD_IDS").Required().Strings()
-
-
 	dockerScan       = cli.Command("docker", "Scan Docker Image")
 	dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, otherwise a image registry is assumed.").Required().Strings()
 )
@@ -498,10 +492,6 @@ func run(state overseer.State) {
 		if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
 			logFatal(err, "Failed to scan CircleCI.")
 		}
-	case trelloScan.FullCommand():
-		if err := e.ScanTrello(ctx, *trelloBoardIDs, *trelloApiKey, *trelloToken); err != nil {
-			logFatal(err, "Failed to scan Trello.")
-		}
 	case gcsScan.FullCommand():
 		cfg := sources.GCSConfig{
 			ProjectID:      *gcsProjectID,

pkg/detectors/jiratoken/jiratoken.go

@@ -13,18 +13,16 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 var _ detectors.Versioner = (*Scanner)(nil)
 
-func (Scanner) Version() int { return 1 }
+func (Scanner) Version() int            { return 1 }
 
 var (
-	defaultClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	tokenPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{24})\b`)
@@ -75,11 +73,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 
 				if verify {
-					client := s.client
-					if client == nil {
-						client = defaultClient
-					}
-
 					data := fmt.Sprintf("%s:%s", resEmail, resToken)
 					sEnc := b64.StdEncoding.EncodeToString([]byte(data))
 					req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
@@ -95,15 +88,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						// If the request is successful and the login reason is not failed authentication, then the token is valid.
 						// This is because Jira returns a 200 status code even if the token is invalid.
 						// Jira returns a default dashboard page.
-						if res.StatusCode >= 200 && res.StatusCode < 300 {
-							if res.Header.Get(loginReasonHeaderKey) != failedAuth {
-								s1.Verified = true
-							}
-						} else {
-							s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+						if (res.StatusCode >= 200 && res.StatusCode < 300) && res.Header.Get(loginReasonHeaderKey) != failedAuth {
+							s1.Verified = true
 						}
-					} else {
-						s1.VerificationError = err
 					}
 				}
 

pkg/detectors/jiratoken/jiratoken_test.go

@@ -9,8 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -24,8 +23,8 @@ func TestJiraToken_FromChunk(t *testing.T) {
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
-	secret := testSecrets.MustGetField("JIRA_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("JIRA_INACTIVE")
+	token := testSecrets.MustGetField("JIRA_TOKEN")
+	inactiveToken := testSecrets.MustGetField("JIRA_INACTIVE")
 	email := testSecrets.MustGetField("JIRA_EMAIL")
 	domain := testSecrets.MustGetField("JIRA_DOMAIN")
 
@@ -35,19 +34,18 @@ func TestJiraToken_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", token, email, domain)),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -56,15 +54,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "found, unverified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveSecret, email, domain)), // the secret would satisfy the regex but not pass validation
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveToken, email, domain)), // the secret would satisfy the regex but not pass validation
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -73,8 +70,7 @@ func TestJiraToken_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "not found",
@@ -84,48 +80,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_JiraToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_JiraToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
+			want:    nil,
+			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -134,12 +96,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
-				}
+				got[i].Raw = nil
+				got[i].RawV2 = nil
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/detectors/jiratoken_v2/jiratoken_v2.go

@@ -13,18 +13,16 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 var _ detectors.Versioner = (*Scanner)(nil)
 
-func (Scanner) Version() int { return 2 }
+func (Scanner) Version() int            { return 2 }
 
 var (
-	defaultClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
 
 	// https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/
 	// Tokens created after Jan 18 2023 use a variable length
@@ -76,11 +74,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 
 				if verify {
-					client := s.client
-					if client == nil {
-						client = defaultClient
-					}
-
 					data := fmt.Sprintf("%s:%s", resEmail, resToken)
 					sEnc := b64.StdEncoding.EncodeToString([]byte(data))
 					req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
@@ -96,15 +89,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						// If the request is successful and the login reason is not failed authentication, then the token is valid.
 						// This is because Jira returns a 200 status code even if the token is invalid.
 						// Jira returns a default dashboard page.
-						if res.StatusCode >= 200 && res.StatusCode < 300 {
-							if res.Header.Get(loginReasonHeaderKey) != failedAuth {
-								s1.Verified = true
-							}
+						if (res.StatusCode >= 200 && res.StatusCode < 300) && res.Header.Get(loginReasonHeaderKey) != failedAuth {
+							s1.Verified = true
 						} else {
-							s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+							s1.VerificationError = err
 						}
-					} else {
-						s1.VerificationError = err
 					}
 				}
 

pkg/detectors/jiratoken_v2/jiratoken_v2_test.go

@@ -9,8 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -24,8 +23,8 @@ func TestJiraToken_FromChunk(t *testing.T) {
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
-	secret := testSecrets.MustGetField("JIRA_V2_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("JIRA_V2_INACTIVE")
+	token := testSecrets.MustGetField("JIRA_V2_TOKEN")
+	inactiveToken := testSecrets.MustGetField("JIRA_V2_INACTIVE")
 	email := testSecrets.MustGetField("JIRA_V2_EMAIL")
 	domain := testSecrets.MustGetField("JIRA_V2_DOMAIN")
 
@@ -35,19 +34,18 @@ func TestJiraToken_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", token, email, domain)),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -56,15 +54,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "found, unverified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveSecret, email, domain)), // the secret would satisfy the regex but not pass validation
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveToken, email, domain)), // the secret would satisfy the regex but not pass validation
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -73,8 +70,7 @@ func TestJiraToken_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "not found",
@@ -84,48 +80,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_JiraToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_JiraToken,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
+			want:    nil,
+			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -134,12 +96,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
-				}
+				got[i].Raw = nil
+				got[i].RawV2 = nil
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/detectors/pagerdutyapikey/pagerdutyapikey.go

@@ -21,13 +21,13 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pd"}) + `\b([a-zA-Z0-9_+-]{20})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty"}) + `\b([a-z]{1}\+[a-zA-Z]{9}\-[a-z]{2}\-[a-z0-9]{5})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"pagerduty", "pd"}
+	return []string{"pagerduty"}
 }
 
 // FromData will find and optionally verify PagerDutyApiKey secrets in a given set of bytes.

pkg/detectors/sendbird/sendbird.go

@@ -2,7 +2,6 @@ package sendbird
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"regexp"
@@ -13,25 +12,19 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	defaultClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat   = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9a-f]{40})\b`)
 	appIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})\b`)
 )
 
-type userResp struct {
-	Code int `json:"code"`
-}
-
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
@@ -68,40 +61,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				req.Header.Add("Api-Token", resMatch)
-
-				client := s.client
-				if client == nil {
-					client = defaultClient
-				}
-
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
-					} else if res.StatusCode == 400 { // Sendbird returns 400 for all errors
-						var userResp userResp
-						err := json.NewDecoder(res.Body).Decode(&userResp)
-						if err != nil {
-							s1.VerificationError = fmt.Errorf("error decoding json response body: %w", err)
-						} else if userResp.Code != 400401 {
-							// https://sendbird.com/docs/chat/platform-api/v3/error-codes
-							// Sendbird always includes its own error codes with 400 responses
-							// 400401 (InvalidApiToken) is the only one that indicates a bad token
-							s1.VerificationError = fmt.Errorf("unexpected response code: %d", userResp.Code)
-						}
 					} else {
-						s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+							continue
+						}
 					}
-				} else {
-					s1.VerificationError = err
 				}
 			}
 
-			// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-			if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-				continue
-			}
 			results = append(results, s1)
 		}
 	}

pkg/detectors/sendbird/sendbird_test.go

@@ -9,8 +9,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/kylelemons/godebug/pretty"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -19,7 +19,7 @@ import (
 func TestSendbird_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
@@ -33,12 +33,11 @@ func TestSendbird_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
@@ -54,8 +53,7 @@ func TestSendbird_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -71,8 +69,7 @@ func TestSendbird_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "not found",
@@ -82,82 +79,14 @@ func TestSendbird_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Sendbird,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Sendbird,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected error code",
-			s:    Scanner{client: common.ConstantResponseHttpClient(400, `{"code":400402}`)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Sendbird,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but failed to decode json",
-			s:    Scanner{client: common.ConstantResponseHttpClient(400, `{`)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Sendbird,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
+			want:    nil,
+			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Sendbird.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -166,12 +95,9 @@ func TestSendbird_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
-				}
+				got[i].Raw = nil
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/detectors/twilio/twilio.go

@@ -28,30 +28,28 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"sid"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("sid")}
 }
 
 // FromData will find and optionally verify Twilio secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	identifierMatches := identifierPat.FindAllString(dataStr, -1)
+	identifierMatches := identifierPat.FindAll(data, -1)
 
 	if len(identifierMatches) == 0 {
 		return
 	}
 
-	keyMatches := keyPat.FindAllString(dataStr, -1)
-	sidMatches := sidPat.FindAllString(dataStr, -1)
+	keyMatches := keyPat.FindAll(data, -1)
+	sidMatches := sidPat.FindAll(data, -1)
 
 	for _, sid := range sidMatches {
 		for _, key := range keyMatches {
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Twilio,
-				Raw:          []byte(sid),
-				RawV2:        []byte(sid + key),
-				Redacted:     sid,
+				Raw:          sid,
+				RawV2:        append(append([]byte(nil), sid...), key...),
+				Redacted:     string(sid),
 			}
 
 			if verify {
@@ -67,10 +65,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 				req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 				req.Header.Add("Accept", "*/*")
-				req.SetBasicAuth(sid, key)
+				req.SetBasicAuth(string(sid), string(key))
 				res, err := client.Do(req)
 				if err == nil {
-					res.Body.Close() // The request body is unused.
+					res.Body.Close()
 
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
@@ -84,7 +82,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 			}
 
-			if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
+			if !s1.Verified && detectors.IsKnownFalsePositive(s1.Raw, detectors.DefaultFalsePositives, true) {
 				continue
 			}
 

pkg/detectors/twist/twist.go

@@ -1,11 +1,11 @@
 package twist
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -26,30 +26,28 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"twist"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("twist")}
 }
 
 // FromData will find and optionally verify Twist secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 		setAuth := resMatch
 
-		if strings.Contains(match[0], "oauth") {
-			setAuth = fmt.Sprintf("oauth2:%s", strings.TrimSpace(match[1]))
+		if bytes.Contains(match[0], []byte("oauth")) {
+			setAuth = append([]byte("oauth2:"), bytes.TrimSpace(match[1])...)
 		}
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Twist,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -57,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", setAuth))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(setAuth)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/twitch/twitch.go

@@ -1,12 +1,10 @@
 package twitch
 
 import (
+	"bytes"
 	"context"
 	"net/http"
-	"net/url"
 	"regexp"
-	"strconv"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,68 +13,57 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"twitch"}) + `\b([0-9a-z]{30})\b`)
 	idPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"twitch"}) + `\b([0-9a-z]{30})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"twitch"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("twitch")}
 }
 
-// FromData will find and optionally verify Twitch secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
+	idMatches := idPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		for _, idMatch := range idMatches {
 			if len(idMatch) != 2 {
 				continue
 			}
 
-			resIdMatch := strings.TrimSpace(idMatch[1])
+			resIdMatch := bytes.TrimSpace(idMatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Twitch,
-				Raw:          []byte(resMatch),
+				Raw:          resMatch,
 			}
 
 			if verify {
-				data := url.Values{}
-				data.Set("client_id", resIdMatch)
-				data.Set("client_secret", resMatch)
-				data.Set("grant_type", "client_credentials")
-				encodedData := data.Encode()
-				req, err := http.NewRequestWithContext(ctx, "POST", "https://id.twitch.tv/oauth2/token", strings.NewReader(encodedData))
+				data := "client_id=" + string(resIdMatch) + "&client_secret=" + string(resMatch) + "&grant_type=client_credentials"
+				req, err := http.NewRequestWithContext(ctx, "POST", "https://id.twitch.tv/oauth2/token", bytes.NewBufferString(data))
 				if err != nil {
 					continue
 				}
 				req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
-				req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))
+				req.Header.Add("Content-Length", string(len(data)))
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
-						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						if detectors.IsKnownFalsePositive([]byte(resMatch), detectors.DefaultFalsePositives, true) {
 							continue
 						}
 					}

pkg/detectors/twitter/twitter.go

@@ -1,11 +1,11 @@
 package twitter
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"twitter"}) + `\b([A-Z]{22}%[a-zA-Z-0-9]{23}%[a-zA-Z-0-9]{6}%[a-zA-Z-0-9]{3}%[a-zA-Z-0-9]{9}%[a-zA-Z-0-9]{52})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"twitter"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("twitter")}
 }
 
-// FromData will find and optionally verify Twitter secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
-
+		resMatch := bytes.TrimSpace(match[1])
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Twitter,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -61,7 +54,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 			}
 		}
-
+		if !s1.Verified {
+			if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+				continue
+			}
+		}
 		results = append(results, s1)
 	}
 

pkg/detectors/tyntec/tyntec.go

@@ -1,10 +1,10 @@
 package tyntec
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -25,25 +25,23 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"tyntec"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("tyntec")}
 }
 
 // FromData will find and optionally verify Tyntec secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Tyntec,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -51,7 +49,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("apiKey", resMatch)
+			req.Header.Add("apiKey", string(resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/typeform/typeform.go

@@ -1,11 +1,11 @@
 package typeform
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -26,25 +26,24 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"typeform"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("typeform")}
 }
 
 // FromData will find and optionally verify Typeform secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Typeform,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/typetalk/typetalk.go

@@ -1,11 +1,11 @@
 package typetalk
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -27,36 +27,34 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"typetalk"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("typetalk")}
 }
 
 // FromData will find and optionally verify Typetalk secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
+	idMatches := idPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		for _, idMatch := range idMatches {
 			if len(idMatch) != 2 {
 				continue
 			}
+			resIdMatch := bytes.TrimSpace(idMatch[1])
 
-			resIdMatch := strings.TrimSpace(idMatch[1])
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Typetalk,
-				Raw:          []byte(resMatch),
+				Raw:          resMatch,
 			}
 
 			if verify {
-				req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("https://typetalk.com/oauth2/access_token?client_id=%s&client_secret=%s&grant_type=client_credentials", resIdMatch, resMatch), nil)
+				req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("https://typetalk.com/oauth2/access_token?client_id=%s&client_secret=%s&grant_type=client_credentials", string(resIdMatch), string(resMatch)), nil)
 				if err != nil {
 					continue
 				}
@@ -66,7 +64,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}

pkg/detectors/ubidots/ubidots.go

@@ -1,10 +1,10 @@
 package ubidots
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -25,25 +25,23 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"BBFF-"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("BBFF-")}
 }
 
 // FromData will find and optionally verify Ubidots secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Ubidots,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,14 +50,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("X-Auth-Token", resMatch)
+			req.Header.Add("X-Auth-Token", string(resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/uclassify/uclassify.go

@@ -1,11 +1,11 @@
 package uclassify
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,54 +14,46 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"uclassify"}) + `\b([a-z0-9A-Z]{12})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"uclassify"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("uclassify")}
 }
 
-// FromData will find and optionally verify Uclassify secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Uclassify,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			payload := strings.NewReader(`{"texts":["I am so happy today"]}`)
-			req, err := http.NewRequestWithContext(ctx, "POST", "https://api.uclassify.com/v1/uClassify/Sentiment/classify", payload)
+			req, err := http.NewRequestWithContext(ctx, "POST", "https://api.uclassify.com/v1/uClassify/Sentiment/classify", bytes.NewReader([]byte(`{"texts":["I am so happy today"]}`)))
 			if err != nil {
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Token %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
@@ -71,7 +63,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 		results = append(results, s1)
 	}
-
 	return results, nil
 }
 

pkg/detectors/unifyid/unifyid.go

@@ -1,11 +1,11 @@
 package unifyid
 
 import (
+	"bytes"
 	"context"
 	"io"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,59 +14,55 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"unify"}) + `\b([0-9A-Za-z_=-]{44})`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"unify"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("unify")}
 }
 
-// FromData will find and optionally verify Unifyid secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_UnifyID,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			payload := strings.NewReader(`{"token": "sample"}`)
+			payload := bytes.NewReader([]byte(`{"token": "sample"}`))
 			req, err := http.NewRequestWithContext(ctx, "POST", "https://api.unify.id/v1/humandetect/verify", payload)
+
 			if err != nil {
 				continue
 			}
+
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("X-API-Key", resMatch)
+			req.Header.Add("X-API-Key", string(resMatch))
 			res, err := client.Do(req)
+
 			if err == nil {
 				defer res.Body.Close()
 				bodyBytes, err := io.ReadAll(res.Body)
+
 				if err != nil {
 					continue
 				}
-				body := string(bodyBytes)
-				if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && strings.Contains(body, "invalid token")) {
+
+				if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && bytes.Contains(bodyBytes, []byte("invalid token"))) {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/unplugg/unplugg.go

@@ -1,10 +1,10 @@
 package unplugg
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,54 +13,46 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"unplu"}) + `\b([a-z0-9]{64})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"unplu"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("unplu")}
 }
 
-// FromData will find and optionally verify Unplugg secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Unplugg,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			payload := strings.NewReader(`{"data":[{"timestamp":1466640000,"value":0.27589941523037853},{"timestamp":1466640900,"value":0.4059699097648263}],"forecast_to":1458136800,"callback":"https://yourdomain.com/samplecallback"}`)
+			payload := bytes.NewReader([]byte(`{"data":[{"timestamp":1466640000,"value":0.27589941523037853},{"timestamp":1466640900,"value":0.4059699097648263}],"forecast_to":1458136800,"callback":"https://yourdomain.com/samplecallback"}`))
 			req, err := http.NewRequestWithContext(ctx, "POST", "https://api.unplu.gg/forecast", payload)
 			if err != nil {
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("x-access-token", resMatch)
+			req.Header.Add("x-access-token", string(resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/unsplash/unsplash.go

@@ -1,10 +1,11 @@
 package unsplash
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,45 +20,40 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"unsplash"}) + `\b([0-9A-Za-z_]{43})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"unsplash"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("unsplash")}
 }
 
-// FromData will find and optionally verify Unsplash secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Unsplash,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.unsplash.com/photos/?client_id="+resMatch, nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.unsplash.com/photos/?client_id=%s", resMatch), nil)
 			if err != nil {
 				continue
 			}
+
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/upcdatabase/upcdatabase.go

@@ -1,12 +1,12 @@
 package upcdatabase
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -26,26 +26,23 @@ var (
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"upcdatabase"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("upcdatabase")}
 }
 
 // FromData will find and optionally verify UPCDatabase secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_UPCDatabase,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -57,8 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err == nil {
 				bodyBytes, err := io.ReadAll(res.Body)
 				if err == nil {
-					bodyString := string(bodyBytes)
-					validResponse := strings.Contains(bodyString, `added_time`)
+					validResponse := bytes.Contains(bodyBytes, []byte(`added_time`))
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						if validResponse {
@@ -67,16 +63,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 							s1.Verified = false
 						}
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}
 					}
 				}
 			}
-		}
 
-		results = append(results, s1)
+			results = append(results, s1)
+		}
 	}
 
 	return results, nil

pkg/detectors/uplead/uplead.go

@@ -1,10 +1,10 @@
 package uplead
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,37 +13,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"uplead"}) + `\b([a-z0-9-]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"uplead"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("uplead")}
 }
 
-// FromData will find and optionally verify Uplead secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Uplead,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -51,14 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", resMatch)
+			req.Header.Add("Authorization", string(resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/uploadcare/uploadcare.go

@@ -1,11 +1,11 @@
 package uploadcare
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,44 +14,38 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat       = regexp.MustCompile(detectors.PrefixRegex([]string{"uploadcare"}) + `\b([a-z0-9]{20})\b`)
 	publicKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"uploadcare"}) + `\b([a-z0-9]{20})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"uploadcare"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("uploadcare")}
 }
 
-// FromData will find and optionally verify UploadCare secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	publicMatches := publicKeyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
+	publicMatches := publicKeyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		for _, publicMatch := range publicMatches {
 			if len(publicMatch) != 2 {
 				continue
 			}
-			publicKeyMatch := strings.TrimSpace(publicMatch[1])
+			publicKeyMatch := bytes.TrimSpace(publicMatch[1])
+
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_UploadCare,
-				Raw:          []byte(resMatch),
+				Raw:          resMatch,
 			}
 
 			if verify {
@@ -60,17 +54,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				req.Header.Add("Accept", "application/vnd.uploadcare-v0.5+json")
-				req.Header.Add("Authorization", fmt.Sprintf("Uploadcare.Simple %s:%s", publicKeyMatch, resMatch))
+				req.Header.Add("Authorization", fmt.Sprintf("Uploadcare.Simple %s:%s", string(publicKeyMatch), string(resMatch)))
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
-					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-							continue
-						}
+					} else if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
 					}
 				}
 			}

pkg/detectors/uptimerobot/uptimerobot.go

@@ -1,11 +1,12 @@
 package uptimerobot
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"io"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -26,29 +27,28 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"uptimerobot"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("uptimerobot")}
 }
 
 // FromData will find and optionally verify UptimeRobot secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_UptimeRobot,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "POST", "https://api.uptimerobot.com/v2/getMonitors?api_key="+resMatch, nil)
+			req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("https://api.uptimerobot.com/v2/getMonitors?api_key=%s", string(resMatch)), nil)
 			if err != nil {
 				continue
 			}
@@ -58,17 +58,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if err != nil {
 					continue
 				}
-				bodyString := string(bodyBytes)
-				validResponse := strings.Contains(bodyString, `"ok"`)
+				validResponse := bytes.Contains(bodyBytes, []byte(`"ok"`))
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					if validResponse {
-						s1.Verified = true
-					} else {
-						s1.Verified = false
-					}
+					s1.Verified = validResponse
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/upwave/upwave.go

@@ -1,11 +1,11 @@
 package upwave
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"upwave"}) + `\b([0-9a-z]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"upwave"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("upwave")}
 }
 
-// FromData will find and optionally verify Upwave secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Upwave,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,15 +45,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
+
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Token %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/uri/uri.go

@@ -1,6 +1,7 @@
 package uri
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"net/url"
@@ -26,39 +27,32 @@ var (
 	client = common.SaneHttpClient()
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"http"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("http")}
 }
 
-// FromData will find and optionally verify URI secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
+		// do not convert the match[0] into a string.
 
 		if !s.allowKnownTestSites {
-			if strings.Contains(match[0], "httpbin.org") {
+			if bytes.Contains(match[0], []byte("httpbin.org")) {
 				continue
 			}
-			if strings.Contains(match[0], "httpwatch.com") {
+
+			if bytes.Contains(match[0], []byte("httpwatch.com")) {
 				continue
 			}
 		}
 
-		urlMatch := match[0]
 		password := match[1]
-
-		// Skip findings where the password only has "*" characters, this is a redacted password
-		// Also include the url encoded "*" characters: "%2A"
-		if strings.Trim(password, "*") == "" || strings.Trim(password, "%2A") == "" {
+		if bytes.Trim(password, "*") == nil || bytes.Trim(password, "%2A") == nil {
 			continue
 		}
 
-		parsedURL, err := url.Parse(urlMatch)
+		parsedURL, err := url.Parse(string(match[0]))
 		if err != nil {
 			continue
 		}
@@ -66,15 +60,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			continue
 		}
 
-		rawURL, _ := url.Parse(urlMatch)
-		rawURLStr := rawURL.String()
-		// Removing the path causes possible deduplication issues if some paths have basic auth and some do not.
+		rawURL, _ := url.Parse(string(match[0]))
+
 		rawURL.Path = ""
 
 		s := detectors.Result{
 			DetectorType: detectorspb.DetectorType_URI,
 			Raw:          []byte(rawURL.String()),
-			RawV2:        []byte(rawURLStr),
+			RawV2:        match[0],
 			Redacted:     detectors.RedactURL(*rawURL),
 		}
 
@@ -82,14 +75,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s.Verified = verifyURL(ctx, parsedURL)
 		}
 
-		if !s.Verified {
-			// Skip unverified findings where the password starts with a `$` - it's almost certainly a variable.
-			if strings.HasPrefix(password, "$") {
-				continue
-			}
+		if bytes.HasPrefix(password, []byte("$")) {
+			continue
 		}
 
-		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
+		if !s.Verified && detectors.IsKnownFalsePositive(s.Raw, detectors.DefaultFalsePositives, false) {
 			continue
 		}
 

pkg/detectors/urlscan/urlscan.go

@@ -1,10 +1,10 @@
 package urlscan
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -18,32 +18,24 @@ var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"urlscan"}) + `\b([a-z0-9-]{36})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"urlscan"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("urlscan")}
 }
 
-// FromData will find and optionally verify Urlscan secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-
+	matches := keyPat.FindAllSubmatch(data, -1)
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Urlscan,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -51,18 +43,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("API-Key", resMatch)
+			req.Header.Add("API-Key", string(resMatch))
 			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-						continue
-					}
-				}
+
+			if err == nil && res.StatusCode >= 200 && res.StatusCode < 300 {
+				s1.Verified = true
+			} else if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+				continue
 			}
 		}
 

pkg/detectors/user/user.go

@@ -1,11 +1,11 @@
 package user
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"user"}) + `\b([a-zA-Z0-9-._+=]{64})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"user"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("user")}
 }
 
-// FromData will find and optionally verify User secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_User,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,14 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Token %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/engine/trello.go

@@ -1,50 +0,0 @@
-package engine
-
-import (
-	"runtime"
-
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/anypb"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/trello"
-)
-
-// ScanTrello scans Trello boards for secrets.
-func (e *Engine) ScanTrello(ctx context.Context, apiKey string) error {
-	trelloSource := trello.Source{}
-	boardIDs, err := trelloSource.GetBoardIDs(ctx, apiKey)
-	if err != nil {
-		ctx.Logger().Error(err, "failed to get Trello board IDs")
-		return err
-	}
-
-	connection := &sourcespb.Trello{
-		Auth: &sourcespb.Trello_TrelloAuth{
-			ApiKey: apiKey,
-		},
-		Boards: boardIDs,
-	}
-
-	var conn anypb.Any
-	err = anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
-	if err != nil {
-		ctx.Logger().Error(err, "failed to marshal Trello connection")
-		return err
-	}
-
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Trello", new(trello.Source).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			if err := trelloSource.Init(ctx, "trufflehog - Trello", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			return &trelloSource, nil
-		})
-	if err != nil {
-		return err
-	}
-	_, err = e.sourceManager.ScheduleRun(ctx, handle)
-	return err
-}

pkg/handlers/archive.go

@@ -17,15 +17,12 @@ import (
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
 type ctxKey int
 
 const (
 	depthKey ctxKey = iota
-
-	errMaxArchiveDepthReached = "max archive depth reached"
 )
 
 var (
@@ -84,66 +81,51 @@ func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []b
 	return archiveChan
 }
 
-type decompressorInfo struct {
-	depth       int
-	reader      io.Reader
-	archiveChan chan []byte
-	archiver    archiver.Decompressor
-}
-
 // openArchive takes a reader and extracts the contents up to the maximum depth.
 func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan []byte) error {
 	if depth >= maxDepth {
-		return fmt.Errorf(errMaxArchiveDepthReached)
+		return fmt.Errorf("max archive depth reached")
 	}
-
-	format, arReader, err := archiver.Identify("", reader)
-	if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
-		return a.handleNonArchiveContent(ctx, arReader, archiveChan)
+	format, reader, err := archiver.Identify("", reader)
+	if err != nil {
+		if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
+			chunkSize := 10 * 1024
+			for {
+				chunk := make([]byte, chunkSize)
+				n, _ := reader.Read(chunk)
+				archiveChan <- chunk
+				if n < chunkSize {
+					break
+				}
+			}
+			return nil
+		}
+		return err
 	}
-
 	switch archive := format.(type) {
 	case archiver.Decompressor:
-		info := decompressorInfo{depth: depth, reader: arReader, archiveChan: archiveChan, archiver: archive}
-		return a.handleDecompressor(ctx, info)
-	case archiver.Extractor:
-		return archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
-	default:
-		return fmt.Errorf("unknown archive type: %s", format.Name())
-	}
-}
-
-func (a *Archive) handleNonArchiveContent(ctx context.Context, reader io.Reader, archiveChan chan []byte) error {
-	aCtx := logContext.AddLogger(ctx)
-	chunkReader := sources.NewChunkReader()
-	chunkResChan := chunkReader(aCtx, reader)
-	for data := range chunkResChan {
-		if err := data.Error(); err != nil {
-			aCtx.Logger().Error(err, "error reading chunk")
-			continue
-		}
-		if err := common.CancellableWrite(ctx, archiveChan, data.Bytes()); err != nil {
+		compReader, err := archive.OpenReader(reader)
+		if err != nil {
 			return err
 		}
+		fileBytes, err := a.ReadToMax(ctx, compReader)
+		if err != nil {
+			return err
+		}
+		newReader := bytes.NewReader(fileBytes)
+		return a.openArchive(ctx, depth+1, newReader, archiveChan)
+	case archiver.Extractor:
+		err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
+		if err != nil {
+			return err
+		}
+		return nil
 	}
-	return nil
-}
-
-func (a *Archive) handleDecompressor(ctx context.Context, info decompressorInfo) error {
-	compReader, err := info.archiver.OpenReader(info.reader)
-	if err != nil {
-		return err
-	}
-	fileBytes, err := a.ReadToMax(ctx, compReader)
-	if err != nil {
-		return err
-	}
-	newReader := bytes.NewReader(fileBytes)
-	return a.openArchive(ctx, info.depth+1, newReader, info.archiveChan)
+	return fmt.Errorf("Unknown archive type: %s", format.Name())
 }
 
 // IsFiletype returns true if the provided reader is an archive.
-func (a *Archive) IsFiletype(_ context.Context, reader io.Reader) (io.Reader, bool) {
+func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) {
 	format, readerB, err := archiver.Identify("", reader)
 	if err != nil {
 		return readerB, false
@@ -153,9 +135,8 @@ func (a *Archive) IsFiletype(_ context.Context, reader io.Reader) (io.Reader, bo
 		return readerB, true
 	case archiver.Decompressor:
 		return readerB, true
-	default:
-		return readerB, false
 	}
+	return readerB, false
 }
 
 // extractorHandler is applied to each file in an archiver.Extractor file.
@@ -199,7 +180,7 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
 			if e, ok := r.(error); ok {
 				err = e
 			} else {
-				err = fmt.Errorf("panic occurred: %v", r)
+				err = fmt.Errorf("Panic occurred: %v", r)
 			}
 			logger.Error(err, "Panic occurred when reading archive")
 		}
@@ -302,7 +283,7 @@ func (a *Archive) HandleSpecialized(ctx logContext.Context, reader io.Reader) (i
 // The caller is responsible for closing the returned reader.
 func (a *Archive) extractDebContent(ctx logContext.Context, file io.Reader) (io.ReadCloser, error) {
 	if a.currentDepth >= maxDepth {
-		return nil, fmt.Errorf(errMaxArchiveDepthReached)
+		return nil, fmt.Errorf("max archive depth reached")
 	}
 
 	tmpEnv, err := a.createTempEnv(ctx, file)

pkg/pb/sourcespb/sources.pb.go

@@ -60,7 +60,6 @@ const (
 	SourceType_SOURCE_TYPE_SHAREPOINT                 SourceType = 29
 	SourceType_SOURCE_TYPE_GCS_UNAUTHED               SourceType = 30
 	SourceType_SOURCE_TYPE_AZURE_REPOS                SourceType = 31
-	SourceType_SOURCE_TYPE_TRELLO                     SourceType = 32
 )
 
 // Enum value maps for SourceType.
@@ -98,7 +97,6 @@ var (
 		29: "SOURCE_TYPE_SHAREPOINT",
 		30: "SOURCE_TYPE_GCS_UNAUTHED",
 		31: "SOURCE_TYPE_AZURE_REPOS",
-		32: "SOURCE_TYPE_AZURE_TRELLO",
 	}
 	SourceType_value = map[string]int32{
 		"SOURCE_TYPE_AZURE_STORAGE":              0,
@@ -133,7 +131,6 @@ var (
 		"SOURCE_TYPE_SHAREPOINT":                 29,
 		"SOURCE_TYPE_GCS_UNAUTHED":               30,
 		"SOURCE_TYPE_AZURE_REPOS":                31,
-		"SOURCE_TYPE_AZURE_TRELLO":                32,
 	}
 )
 
@@ -526,80 +523,6 @@ func (*Bitbucket_Oauth) isBitbucket_Credential() {}
 
 func (*Bitbucket_BasicAuth) isBitbucket_Credential() {}
 
-type Trello struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Auth *Trello_TrelloAuth `protobuf:"bytes,1,opt,name=auth,proto3,oneof"`
-	Boards []string `protobuf:"bytes,2,rep,name=boards,proto3" json:"boards,omitempty"`
-}
-
-type Trello_TrelloAuth struct {
-	ApiKey string `protobuf:"bytes,1,opt,name=api_key,json=apiKey,proto3,oneof"`
-	Token  string `protobuf:"bytes,2,opt,name=token,proto3,oneof"`
-}
-
-func (x *Trello) Reset() {
-	*x = Trello{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_sources_proto_msgTypes[4]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *Trello) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*Trello) ProtoMessage() {}
-
-func (x *Trello) ProtoReflect() protoreflect.Message {
-	mi := &file_sources_proto_msgTypes[4]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-// Deprecated: Use Trello.ProtoReflect.Descriptor instead.
-func (*Trello) Descriptor() ([]byte, []int) {
-	return file_sources_proto_rawDescGZIP(), []int{4}  // Assuming Trello is the 5th message in source.proto file
-}
-
-func (x *Trello) GetAuth() *Trello_TrelloAuth {
-	if x != nil {
-		return x.Auth
-	}
-	return nil
-}
-
-func (t *Trello_TrelloAuth) GetApiKey() string {
-	if t != nil && t.ApiKey != nil {
-		return t.ApiKey
-	}
-	return ""
-}
-
-func (t *Trello_TrelloAuth) GetToken() string {
-	if t != nil && t.Token != nil {
-		return t.Token
-	}
-	return ""
-}
-
-func (x *Trello) GetBoards() []string {
-	if x != nil {
-		return x.Boards
-	}
-	return nil
-}
-
-
 type CircleCI struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -3770,7 +3693,6 @@ var file_sources_proto_goTypes = []interface{}{
 	(*SlackRealtime)(nil),                       // 28: sources.SlackRealtime
 	(*Sharepoint)(nil),                          // 29: sources.Sharepoint
 	(*AzureRepos)(nil),                          // 30: sources.AzureRepos
-	(*Trello)(nil),                              // 45: sources.Trello
 	(*durationpb.Duration)(nil),                 // 31: google.protobuf.Duration
 	(*anypb.Any)(nil),                           // 32: google.protobuf.Any
 	(*credentialspb.BasicAuth)(nil),             // 33: credentials.BasicAuth
@@ -4304,16 +4226,13 @@ func file_sources_proto_init() {
 		(*AzureRepos_Token)(nil),
 		(*AzureRepos_Oauth)(nil),
 	}
-	file_sources_proto_msgTypes[29].OneofWrappers = []interface{}{
-		(*Trello_TrelloAuth)(nil),
-	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_sources_proto_rawDesc,
 			NumEnums:      2,
-			NumMessages:   30,
+			NumMessages:   29,
 			NumExtensions: 0,
 			NumServices:   0,
 		},

pkg/sources/docker/docker.go

@@ -2,6 +2,7 @@ package docker
 
 import (
 	"archive/tar"
+	"bytes"
 	"compress/gzip"
 	"errors"
 	"fmt"
@@ -13,11 +14,9 @@ import (
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
-	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -25,16 +24,17 @@ import (
 )
 
 type Source struct {
-	name        string
-	sourceId    int64
-	jobId       int64
-	verify      bool
-	concurrency int
-	conn        sourcespb.Docker
+	name     string
+	sourceId int64
+	jobId    int64
+	verify   bool
+	conn     sourcespb.Docker
 	sources.Progress
 	sources.CommonSourceUnitUnmarshaller
 }
 
+var FilesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
+
 // Ensure the Source satisfies the interfaces at compile time.
 var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
@@ -59,7 +59,6 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 	s.sourceId = sourceId
 	s.jobId = jobId
 	s.verify = verify
-	s.concurrency = concurrency
 
 	// Reset metrics for this source at initialization time.
 	dockerImagesScanned.WithLabelValues(s.name).Set(0)
@@ -72,209 +71,166 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 	return nil
 }
 
-type imageInfo struct {
-	image v1.Image
-	base  string
-	tag   string
-}
-
-type layerInfo struct {
-	digest v1.Hash
-	base   string
-	tag    string
-}
-
 // Chunks emits data over a channel that is decoded and scanned for secrets.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	ctx = context.WithValues(ctx, "source_type", s.Type(), "source_name", s.name)
-
-	workers := new(errgroup.Group)
-	workers.SetLimit(s.concurrency)
-
-	scanErrs := sources.NewScanErrors()
-	for _, image := range s.conn.GetImages() {
-		image := image
-		workers.Go(func() error {
-			if common.IsDone(ctx) {
-				return nil
-			}
-
-			imgInfo, err := s.processImage(ctx, image)
-			if err != nil {
-				scanErrs.Add(err)
-				return nil
-			}
-
-			ctx = context.WithValues(ctx, "image", imgInfo.base, "tag", imgInfo.tag)
-			ctx.Logger().V(2).Info("scanning image")
-
-			layers, err := imgInfo.image.Layers()
-			if err != nil {
-				scanErrs.Add(err)
-				return nil
-			}
-
-			for _, layer := range layers {
-				if err := s.processLayer(ctx, layer, imgInfo, chunksChan); err != nil {
-					scanErrs.Add(err)
-					return nil
-				}
-				dockerLayersScanned.WithLabelValues(s.name).Inc()
-			}
-
-			dockerImagesScanned.WithLabelValues(s.name).Inc()
-
-			return nil
-		})
-	}
-	_ = workers.Wait()
-	if scanErrs.Count() > 0 {
-		ctx.Logger().V(2).Info("scan errors", "errors", scanErrs.String())
-	}
-
-	return nil
-}
-
-// processImage processes an individual image and prepares it for further processing.
-func (s *Source) processImage(ctx context.Context, image string) (imageInfo, error) {
-	var (
-		imgInfo   imageInfo
-		hasDigest bool
-		imageName name.Reference
-	)
-
 	remoteOpts, err := s.remoteOpts()
 	if err != nil {
-		return imgInfo, err
+		return err
 	}
 
-	const filePrefix = "file://"
-	if strings.HasPrefix(image, filePrefix) {
-		image = strings.TrimPrefix(image, filePrefix)
-		imgInfo.base = image
-		imgInfo.image, err = tarball.ImageFromPath(image, nil)
-		if err != nil {
-			return imgInfo, err
-		}
-	} else {
-		imgInfo.base, imgInfo.tag, hasDigest = baseAndTagFromImage(image)
+	logger := ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
 
-		if hasDigest {
-			imageName, err = name.NewDigest(image)
+	for _, image := range s.conn.GetImages() {
+		var img v1.Image
+		var err error
+		var base, tag string
+		var hasDigest bool
+		var imageName name.Reference
+
+		if strings.HasPrefix(image, "file://") {
+			image = strings.TrimPrefix(image, "file://")
+			base = image
+			img, err = tarball.ImageFromPath(image, nil)
+			if err != nil {
+				return err
+			}
 		} else {
-			imageName, err = name.NewTag(image)
-		}
-		if err != nil {
-			return imgInfo, err
+			base, tag, hasDigest = baseAndTagFromImage(image)
+
+			if hasDigest {
+				imageName, err = name.NewDigest(image)
+				if err != nil {
+					return err
+				}
+			} else {
+				imageName, err = name.NewTag(image)
+				if err != nil {
+					return err
+				}
+			}
+
+			img, err = remote.Image(imageName, remoteOpts...)
+			if err != nil {
+				return err
+			}
 		}
 
-		imgInfo.image, err = remote.Image(imageName, remoteOpts...)
-		if err != nil {
-			return imgInfo, err
-		}
-	}
+		logger = logger.WithValues("image", base, "tag", tag)
+		logger.V(2).Info("scanning image")
 
-	ctx.Logger().WithValues("image", imgInfo.base, "tag", imgInfo.tag).V(2).Info("scanning image")
-
-	return imgInfo, nil
-}
-
-// processLayer processes an individual layer of an image.
-func (s *Source) processLayer(ctx context.Context, layer v1.Layer, imgInfo imageInfo, chunksChan chan *sources.Chunk) error {
-	layerInfo := layerInfo{
-		base: imgInfo.base,
-		tag:  imgInfo.tag,
-	}
-
-	var err error
-	layerInfo.digest, err = layer.Digest()
-	if err != nil {
-		return err
-	}
-
-	ctx.Logger().WithValues("layer", layerInfo.digest.String()).V(2).Info("scanning layer")
-
-	rc, err := layer.Compressed()
-	if err != nil {
-		return err
-	}
-	defer rc.Close()
-
-	gzipReader, err := gzip.NewReader(rc)
-	if err != nil {
-		return err
-	}
-	defer gzipReader.Close()
-
-	tarReader := tar.NewReader(gzipReader)
-	for {
-		header, err := tarReader.Next()
-		if errors.Is(err, io.EOF) {
-			break
-		}
+		layers, err := img.Layers()
 		if err != nil {
 			return err
 		}
 
-		info := chunkProcessingInfo{size: header.Size, name: header.Name, reader: tarReader, layer: layerInfo}
-		if err := s.processChunk(ctx, info, chunksChan); err != nil {
-			return err
-		}
-	}
+		for _, layer := range layers {
 
-	return nil
-}
+			digest, err := layer.Digest()
+			if err != nil {
+				return err
+			}
 
-type chunkProcessingInfo struct {
-	size   int64
-	name   string
-	reader io.Reader
-	layer  layerInfo
-}
+			logger = logger.WithValues("layer", digest.String())
+			logger.V(2).Info("scanning layer")
 
-// processChunk processes an individual chunk of a layer.
-func (s *Source) processChunk(ctx context.Context, info chunkProcessingInfo, chunksChan chan *sources.Chunk) error {
-	const filesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
-	if info.size > filesizeLimitBytes {
-		ctx.Logger().V(4).Info("skipping large file", "file", info.name, "size", info.size)
-		return nil
-	}
+			rc, err := layer.Compressed()
+			if err != nil {
+				return err
+			}
 
-	chunkReader := sources.NewChunkReader()
-	chunkResChan := chunkReader(ctx, info.reader)
+			defer rc.Close()
 
-	for data := range chunkResChan {
-		if err := data.Error(); err != nil {
-			ctx.Logger().Error(err, "error reading chunk.")
-			continue
-		}
+			gzipReader, err := gzip.NewReader(rc)
+			if err != nil {
+				return err
+			}
 
-		chunk := &sources.Chunk{
-			SourceType: s.Type(),
-			SourceName: s.name,
-			SourceID:   s.SourceID(),
-			SourceMetadata: &source_metadatapb.MetaData{
-				Data: &source_metadatapb.MetaData_Docker{
-					Docker: &source_metadatapb.Docker{
-						File:  "/" + info.name,
-						Image: info.layer.base,
-						Tag:   info.layer.tag,
-						Layer: info.layer.digest.String(),
+			defer gzipReader.Close()
+
+			tarReader := tar.NewReader(gzipReader)
+
+			for {
+				header, err := tarReader.Next()
+				if errors.Is(err, io.EOF) {
+					break // End of archive
+				}
+				if err != nil {
+					return err
+				}
+
+				// Skip files larger than FilesizeLimitBytes
+				if header.Size > FilesizeLimitBytes {
+					continue
+				}
+
+				file := bytes.NewBuffer(make([]byte, 0, header.Size))
+
+				_, err = io.Copy(file, tarReader)
+				if err != nil {
+					return err
+				}
+
+				chunk := &sources.Chunk{
+					SourceType: s.Type(),
+					SourceName: s.name,
+					SourceID:   s.SourceID(),
+					JobID:      s.JobID(),
+					Data:       file.Bytes(),
+					SourceMetadata: &source_metadatapb.MetaData{
+						Data: &source_metadatapb.MetaData_Docker{
+							Docker: &source_metadatapb.Docker{
+								File:  "/" + header.Name, // Prepend /, because the file is relative to the root of the image
+								Image: base,
+								Tag:   tag,
+								Layer: digest.String(),
+							},
+						},
 					},
-				},
-			},
-			Verify: s.verify,
-		}
-		chunk.Data = data.Bytes()
+					Verify: s.verify,
+				}
 
-		if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
-			return err
+				select {
+				case chunksChan <- chunk:
+				case <-ctx.Done():
+					return ctx.Err()
+				}
+			}
+
+			dockerLayersScanned.WithLabelValues(s.name).Inc()
 		}
+
+		dockerImagesScanned.WithLabelValues(s.name).Inc()
 	}
 
 	return nil
 }
 
+func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
+	regRepoDelimiter := "/"
+	tagDelim := ":"
+	digestDelim := "@"
+
+	// Split on digest first, if present.
+	parts := strings.SplitN(image, digestDelim, 2)
+	if len(parts) > 1 {
+		base = parts[0]
+		tag = parts[1]
+		hasDigest = true
+		return
+	}
+	parts = strings.Split(image, tagDelim)
+	// Verify that we aren't confusing a tag for a hostname w/ port for the purposes of weak validation.
+	if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelimiter) {
+		base = strings.Join(parts[:len(parts)-1], tagDelim)
+		tag = parts[len(parts)-1]
+	} else {
+		base = image
+		tag = "latest"
+	}
+
+	return
+}
+
 func (s *Source) remoteOpts() ([]remote.Option, error) {
 	switch s.conn.GetCredential().(type) {
 	case *sourcespb.Docker_Unauthenticated:
@@ -300,40 +256,3 @@ func (s *Source) remoteOpts() ([]remote.Option, error) {
 		return nil, fmt.Errorf("unknown credential type: %T", s.conn.Credential)
 	}
 }
-
-func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
-	if base, tag, hasDigest = extractDigest(image); hasDigest {
-		return base, tag, true
-	}
-
-	base, tag = extractTagOrUseDefault(image)
-	return base, tag, false
-}
-
-// extractDigest tries to split the image string on the digest delimiter.
-// If successful, it means the image has a digest.
-func extractDigest(image string) (base, tag string, hasDigest bool) {
-	const digestDelim = "@"
-
-	if parts := strings.SplitN(image, digestDelim, 2); len(parts) > 1 {
-		return parts[0], parts[1], true
-	}
-	return "", "", false
-}
-
-// extractTagOrUseDefault extracts the tag from the image string.
-// If no tag is found, it defaults to "latest".
-func extractTagOrUseDefault(image string) (base, tag string) {
-	const (
-		tagDelim     = ":"
-		regRepoDelim = "/"
-	)
-
-	parts := strings.Split(image, tagDelim)
-
-	// Check if the last part is not a hostname with a port (for weak validation)
-	if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelim) {
-		return strings.Join(parts[:len(parts)-1], tagDelim), parts[len(parts)-1]
-	}
-	return image, "latest"
-}

pkg/sources/s3/s3.go

@@ -31,7 +31,6 @@ import (
 )
 
 const (
-	defaultAWSRegion     = "us-east-1"
 	defaultMaxObjectSize = 250 * 1024 * 1024 // 250 MiB
 	maxObjectSizeLimit   = 250 * 1024 * 1024 // 250 MiB
 )
@@ -54,7 +53,6 @@ type Source struct {
 // Ensure the Source satisfies the interfaces at compile time
 var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
-var _ sources.Validator = (*Source)(nil)
 
 // Type returns the type of source
 func (s *Source) Type() sourcespb.SourceType {
@@ -95,23 +93,6 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 	return nil
 }
 
-func (s *Source) Validate(ctx context.Context) []error {
-	var errors []error
-	visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
-		roleErrs := s.validateBucketAccess(c, defaultRegionClient, roleArn, buckets)
-		if len(roleErrs) > 0 {
-			errors = append(errors, roleErrs...)
-		}
-	}
-
-	err := s.visitRoles(ctx, visitor)
-	if err != nil {
-		errors = append(errors, err)
-	}
-
-	return errors
-}
-
 // setMaxObjectSize sets the maximum size of objects that will be scanned. If
 // not set, set to a negative number, or set larger than the
 // maxObjectSizeLimit, the defaultMaxObjectSizeLimit will be used.
@@ -172,7 +153,7 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
 
 	res, err := client.ListBuckets(&s3.ListBucketsInput{})
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("could not list s3 buckets: %w", err)
 	}
 
 	var bucketsToScan []string
@@ -182,30 +163,34 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
 	return bucketsToScan, nil
 }
 
-func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bucketsToScan []string, chunksChan chan *sources.Chunk) {
+func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bucketsToScan []string, chunksChan chan *sources.Chunk) error {
+	const defaultAWSRegion = "us-east-1"
 	objectCount := uint64(0)
 
-	logger := s.log
-	if role != "" {
-		logger = logger.WithValues("roleArn", role)
-	}
-
 	for i, bucket := range bucketsToScan {
-		logger := logger.WithValues("bucket", bucket)
-
 		if common.IsDone(ctx) {
-			return
+			return nil
 		}
 
 		s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "")
-		logger.Info("Scanning bucket")
-
-		regionalClient, err := s.getRegionalClientForBucket(ctx, client, role, bucket)
+		s.log.Info("Scanning bucket", "bucket", bucket)
+		region, err := s3manager.GetBucketRegionWithClient(ctx, client, bucket)
 		if err != nil {
-			logger.Error(err, "could not get regional client for bucket")
+			s.log.Error(err, "could not get s3 region for bucket", "bucket: ", bucket)
 			continue
 		}
 
+		var regionalClient *s3.S3
+		if region != defaultAWSRegion {
+			regionalClient, err = s.newClient(region, role)
+			if err != nil {
+				s.log.Error(err, "could not make regional s3 client")
+				continue
+			}
+		} else {
+			regionalClient = client
+		}
+
 		errorCount := sync.Map{}
 
 		err = regionalClient.ListObjectsV2PagesWithContext(
@@ -216,46 +201,40 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
 			})
 
 		if err != nil {
-			if role == "" {
-				logger.Error(err, "could not list objects in bucket")
-			} else {
-				// Our documentation blesses specifying a role to assume without specifying buckets to scan, which will
-				// often cause this to happen a lot (because in that case the scanner tries to scan every bucket in the
-				// account, but the role probably doesn't have access to all of them). This makes it expected behavior
-				// and therefore not an error.
-				logger.V(3).Info("could not list objects in bucket",
-					"err", err)
-			}
+			s.log.Error(err, "could not list objects in s3 bucket", "bucket: ", bucket)
+			continue
 		}
 	}
 	s.SetProgressComplete(len(bucketsToScan), len(bucketsToScan), fmt.Sprintf("Completed scanning source %s. %d objects scanned.", s.name, objectCount), "")
+	return nil
 }
 
 // Chunks emits chunks of bytes over a channel.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
-		s.scanBuckets(c, defaultRegionClient, roleArn, buckets, chunksChan)
+	const defaultAWSRegion = "us-east-1"
+
+	roles := s.conn.Roles
+	if len(roles) == 0 {
+		roles = []string{""}
 	}
 
-	return s.visitRoles(ctx, visitor)
-}
+	for _, role := range roles {
+		client, err := s.newClient(defaultAWSRegion, role)
+		if err != nil {
+			return errors.WrapPrefix(err, "could not create s3 client", 0)
+		}
 
-func (s *Source) getRegionalClientForBucket(ctx context.Context, defaultRegionClient *s3.S3, role, bucket string) (*s3.S3, error) {
-	region, err := s3manager.GetBucketRegionWithClient(ctx, defaultRegionClient, bucket)
-	if err != nil {
-		return nil, errors.WrapPrefix(err, "could not get s3 region for bucket", 0)
+		bucketsToScan, err := s.getBucketsToScan(client)
+		if err != nil {
+			return err
+		}
+
+		if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil {
+			return err
+		}
 	}
 
-	if region == defaultAWSRegion {
-		return defaultRegionClient, nil
-	}
-
-	regionalClient, err := s.newClient(region, role)
-	if err != nil {
-		return nil, errors.WrapPrefix(err, "could not create regional s3 client", 0)
-	}
-
-	return regionalClient, nil
+	return nil
 }
 
 // pageChunker emits chunks onto the given channel from a page
@@ -417,65 +396,6 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 	_ = s.jobPool.Wait()
 }
 
-func (s *Source) validateBucketAccess(ctx context.Context, client *s3.S3, roleArn string, buckets []string) []error {
-	shouldHaveAccessToAllBuckets := roleArn == ""
-	wasAbleToListAnyBucket := false
-	var errors []error
-
-	for _, bucket := range buckets {
-		if common.IsDone(ctx) {
-			return append(errors, ctx.Err())
-		}
-
-		regionalClient, err := s.getRegionalClientForBucket(ctx, client, roleArn, bucket)
-		if err != nil {
-			errors = append(errors, fmt.Errorf("could not get regional client for bucket %q: %w", bucket, err))
-			continue
-		}
-
-		_, err = regionalClient.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &bucket})
-
-		if err == nil {
-			wasAbleToListAnyBucket = true
-		} else if shouldHaveAccessToAllBuckets {
-			errors = append(errors, fmt.Errorf("could not list objects in bucket %q: %w", bucket, err))
-		}
-	}
-
-	if !wasAbleToListAnyBucket {
-		if roleArn == "" {
-			errors = append(errors, fmt.Errorf("could not list objects in any bucket"))
-		} else {
-			errors = append(errors, fmt.Errorf("role %q could not list objects in any bucket", roleArn))
-		}
-	}
-
-	return errors
-}
-
-func (s *Source) visitRoles(ctx context.Context, f func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string)) error {
-	roles := s.conn.Roles
-	if len(roles) == 0 {
-		roles = []string{""}
-	}
-
-	for _, role := range roles {
-		client, err := s.newClient(defaultAWSRegion, role)
-		if err != nil {
-			return errors.WrapPrefix(err, "could not create s3 client", 0)
-		}
-
-		bucketsToScan, err := s.getBucketsToScan(client)
-		if err != nil {
-			return fmt.Errorf("role %q could not list any s3 buckets for scanning: %w", role, err)
-		}
-
-		f(ctx, client, role, bucketsToScan)
-	}
-
-	return nil
-}
-
 // S3 links currently have the general format of:
 // https://[bucket].s3[.region unless us-east-1].amazonaws.com/[key]
 func makeS3Link(bucket, region, key string) string {

pkg/sources/s3/s3_integration_test.go

@@ -4,14 +4,10 @@
 package s3
 
 import (
-	"fmt"
-	"sync"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
 	"google.golang.org/protobuf/types/known/anypb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -49,127 +45,3 @@ func TestSource_ChunksCount(t *testing.T) {
 	}
 	assert.Greater(t, got, wantChunkCount)
 }
-
-func TestSource_Validate(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
-	defer cancel()
-
-	secret, err := common.GetTestSecret(ctx)
-	if err != nil {
-		t.Fatal(fmt.Errorf("failed to access secret: %v", err))
-	}
-
-	s3key := secret.MustGetField("AWS_S3_KEY")
-	s3secret := secret.MustGetField("AWS_S3_SECRET")
-
-	tests := []struct {
-		name         string
-		roles        []string
-		buckets      []string
-		wantErrCount int
-	}{
-		{
-			name: "buckets without roles, can access all buckets",
-			buckets: []string{
-				"truffletestbucket-s3-tests",
-			},
-			wantErrCount: 0,
-		},
-		{
-			name: "buckets without roles, one error per inaccessible bucket",
-			buckets: []string{
-				"truffletestbucket-s3-tests",
-				"truffletestbucket-s3-role-assumption",
-				"truffletestbucket-no-access",
-			},
-			wantErrCount: 2,
-		},
-		{
-			name: "roles without buckets, all can access at least one account bucket",
-			roles: []string{
-				"arn:aws:iam::619888638459:role/s3-test-assume-role",
-			},
-			wantErrCount: 0,
-		},
-		{
-			name: "roles without buckets, one error per role that cannot access any account buckets",
-			roles: []string{
-				"arn:aws:iam::619888638459:role/s3-test-assume-role",
-				"arn:aws:iam::619888638459:role/test-no-access",
-			},
-			wantErrCount: 1,
-		},
-		{
-			name: "role and buckets, can access at least one bucket",
-			roles: []string{
-				"arn:aws:iam::619888638459:role/s3-test-assume-role",
-			},
-			buckets: []string{
-				"truffletestbucket-s3-role-assumption",
-				"truffletestbucket-no-access",
-			},
-			wantErrCount: 0,
-		},
-		{
-			name: "roles and buckets, one error per role that cannot access at least one bucket",
-			roles: []string{
-				"arn:aws:iam::619888638459:role/s3-test-assume-role",
-				"arn:aws:iam::619888638459:role/test-no-access",
-			},
-			buckets: []string{
-				"truffletestbucket-s3-role-assumption",
-				"truffletestbucket-no-access",
-			},
-			wantErrCount: 1,
-		},
-		{
-			name: "role and buckets, a bucket doesn't even exist",
-			roles: []string{
-				"arn:aws:iam::619888638459:role/s3-test-assume-role",
-			},
-			buckets: []string{
-				"truffletestbucket-s3-role-assumption",
-				"not-a-real-bucket-asljdhmglasjgvklhsdaljfh", // need a bucket name that nobody is likely to ever create
-			},
-			wantErrCount: 1,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
-			var cancelOnce sync.Once
-			defer cancelOnce.Do(cancel)
-
-			// These are used by the tests that assume roles
-			t.Setenv("AWS_ACCESS_KEY_ID", s3key)
-			t.Setenv("AWS_SECRET_ACCESS_KEY", s3secret)
-
-			s := &Source{}
-
-			conn, err := anypb.New(&sourcespb.S3{
-				// These are used by the tests that don't assume roles
-				Credential: &sourcespb.S3_AccessKey{
-					AccessKey: &credentialspb.KeySecret{
-						Key:    s3key,
-						Secret: s3secret,
-					},
-				},
-				Buckets: tt.buckets,
-				Roles:   tt.roles,
-			})
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = s.Init(ctx, tt.name, 0, 0, false, conn, 0)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			errs := s.Validate(ctx)
-
-			assert.Equal(t, tt.wantErrCount, len(errs))
-		})
-	}
-}

pkg/sources/trello/trello.go

@@ -1,239 +0,0 @@
-package trello
-
-import (
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"sync/atomic"
-
-	"github.com/go-errors/errors"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/anypb"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
-)
-
-const baseURL = "https://api.trello.com/1/"
-
-type Source struct {
-	name     string
-	apiKey   string
-	token    string
-	boardIDs []string
-	sourceId int64
-	jobId    int64
-	verify   bool
-	jobPool  *errgroup.Group
-	sources.Progress
-	client *http.Client
-	sources.CommonSourceUnitUnmarshaller
-}
-
-// Type returns the type of the source.
-func (s *Source) Type() sourcespb.SourceType {
-	return sourcespb.SourceType_SOURCE_TYPE_TRELLO
-}
-
-// Init returns an initialized Source.
-func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
-	s.name = name
-	s.sourceId = sourceId
-	s.jobId = jobId
-	s.verify = verify
-	s.jobPool = &errgroup.Group{}
-	s.jobPool.SetLimit(concurrency)
-	s.client = common.RetryableHttpClientTimeout(3)
-
-	var conn sourcespb.Trello
-	if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
-		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
-	}
-
-	s.apiKey = conn.GetAuth().GetApiKey()
-	s.token = conn.GetAuth().GetToken()
-	s.boardIDs = conn.GetBoards()
-
-	return nil
-}
-
-// Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	scanErrs := sources.NewScanErrors()
-
-	for _, boardID := range s.boardIDs {
-		board, err := s.getBoard(ctx, boardID)
-		if err != nil {
-			scanErrs.Add(err)
-			return nil
-		}
-
-		if err = s.chunkBoard(ctx, board, chunksChan); err != nil {
-			scanErrs.Add(err)
-			return nil
-		}
-	}
-
-	_ = s.jobPool.Wait()
-	if scanErrs.Count() > 0 {
-		ctx.Logger().V(2).Info("encountered errors while scanning", "count", scanErrs.Count(), "errors", scanErrs)
-	}
-
-	return nil
-}
-
-type board struct {
-	Name  string `json:"name"`
-	Desc  string `json:"desc"`
-	ID    string `json:"id"`
-	URL   string `json:"url"`
-}
-
-type card struct {
-	Name  string `json:"name"`
-	Desc  string `json:"desc"`
-	ID    string `json:"id"`
-	URL   string `json:"url"`
-}
-
-type action struct {
-	Type string `json:"type"`
-	Data struct {
-		Text string `json:"text"`
-	} `json:"data"`
-}
-
-// Add these two methods to Source
-func (s *Source) getCards(_ context.Context, boardID string) ([]card, error) {
-	reqURL := fmt.Sprintf("%sboards/%s/cards?key=%s&token=%s", baseURL, boardID, s.apiKey, s.token)
-	req, err := http.NewRequest("GET", reqURL, nil)
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set("Accept", "application/json")
-	res, err := s.client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer res.Body.Close()
-
-	var cards []card
-	if err := json.NewDecoder(res.Body).Decode(&cards); err != nil {
-		return nil, err
-	}
-
-	return cards, nil
-}
-
-func (s *Source) getComments(_ context.Context, cardID string) ([]action, error) {
-	reqURL := fmt.Sprintf("%scards/%s/actions?key=%s&token=%s", baseURL, cardID, s.apiKey, s.token)
-	req, err := http.NewRequest("GET", reqURL, nil)
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set("Accept", "application/json")
-	res, err := s.client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer res.Body.Close()
-
-	var actions []action
-	if err := json.NewDecoder(res.Body).Decode(&actions); err != nil {
-		return nil, err
-	}
-
-	return actions, nil
-}
-
-func (s *Source) getBoard(_ context.Context, boardID string) (*board, error) {
-	reqURL := fmt.Sprintf("%sboards/%s?key=%s&token=%s", baseURL, boardID, s.apiKey, s.token)
-	req, err := http.NewRequest("GET", reqURL, nil)
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set("Accept", "application/json")
-	res, err := s.client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer res.Body.Close()
-
-	var b board
-	if err := json.NewDecoder(res.Body).Decode(&b); err != nil {
-		return nil, err
-	}
-
-	return &b, nil
-}
-
-func (s *Source) chunkBoard(ctx context.Context, board *board, chunksChan chan *sources.Chunk) error {
-	err := s.chunkItem(ctx, chunksChan, board.ID, board.Desc, board.URL, source_metadatapb.ItemType_ITEM_TYPE_BOARD, board.Name)
-	if err != nil {
-		return err
-	}
-
-	cards, err := s.getCards(ctx, board.ID)
-	if err != nil {
-		return err
-	}
-
-	for _, card := range cards {
-		err := s.chunkItem(ctx, chunksChan, card.ID, card.Desc, card.URL, source_metadatapb.ItemType_ITEM_TYPE_CARD, card.Name)
-		if err != nil {
-			return err
-		}
-
-		comments, err := s.getComments(ctx, card.ID)
-		if err != nil {
-			return err
-		}
-
-		for _, comment := range comments {
-			if comment.Type != "commentCard" {
-				continue
-			}
-
-			err := s.chunkItem(ctx, chunksChan, card.ID, comment.Data.Text, card.URL, source_metadatapb.ItemType_ITEM_TYPE_COMMENT, "")
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (s *Source) chunkItem(ctx context.Context, chunksChan chan *sources.Chunk, id, text, url string, itemType source_metadatapb.ItemType, name string) error {
-	data := []byte(text)
-
-	chunk := &sources.Chunk{
-		SourceType: s.Type(),
-		SourceName: s.name,
-		SourceID:   s.sourceId,
-		JobID:      s.jobId,
-		Data:       data,
-		SourceMetadata: &source_metadatapb.MetaData{
-			Data: &source_metadatapb.MetaData_Trello{
-				Trello: &source_metadatapb.Trello{
-					Id:   id,
-					Url:  url,
-					Type: itemType,
-					Name: name
-				},
-			},
-		},
-		Verify: s.verify,
-	}
-
-	atomic.AddInt64(&s.scannedBytes, int64(len(data)))
-
-	if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
-		return err
-	}
-
-	return nil
-}

pkg/sources/trello/trello_test.go

@@ -1,94 +0,0 @@
-package trello
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/kylelemons/godebug/pretty"
-	"google.golang.org/protobuf/types/known/anypb"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
-)
-
-func TestSource_Scan(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
-	defer cancel()
-
-	secret, err := common.GetTestSecret(ctx)
-	if err != nil {
-		t.Fatal(fmt.Errorf("failed to access secret: %v", err))
-	}
-	apiKey := secret.MustGetField("TRELLO_API_KEY")
-	token := secret.MustGetField("TRELLO_TOKEN")
-
-	type init struct {
-		name       string
-		verify     bool
-		connection *sourcespb.Trello
-	}
-	tests := []struct {
-		name               string
-		init               init
-		wantSourceMetadata *source_metadatapb.MetaData
-		wantErr            bool
-	}{
-		{
-			name: "get a chunk",
-			init: init{
-				name: "example board",
-				connection: &sourcespb.Trello{
-					Auth: &sourcespb.Trello_TrelloAuth{
-						ApiKey: apiKey,
-						Token:  token,
-					},
-					Boards: []string{"boardID1"},
-				},
-				verify: true,
-			},
-			wantSourceMetadata: &source_metadatapb.MetaData{
-				Data: &source_metadatapb.MetaData_Trello{
-					Trello: &source_metadatapb.Trello{
-						BoardId:   "boardID1",
-						BoardName: "Example Board",
-						CardId:    "cardID1",
-					},
-				},
-			},
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Source{}
-
-			conn, err := anypb.New(tt.init.connection)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if err := s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5); (err != nil) != tt.wantErr {
-				t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-
-			chunksCh := make(chan *sources.Chunk, 1)
-			go func() {
-				if err := s.Chunks(ctx, chunksCh); (err != nil) != tt.wantErr {
-					t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
-					return
-				}
-			}()
-
-			gotChunk := <-chunksCh
-			if diff := pretty.Compare(gotChunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {
-				t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}

pkg/tui/pages/source_select/source_select.go

@@ -68,7 +68,6 @@ func New(c common.Common) *SourceSelect {
 		OssItem("Filesystem", "Scan your filesystem by selecting what directories to scan."),
 		OssItem("AWS S3", "Scan Amazon S3 buckets."),
 		OssItem("CircleCI", "Scan CircleCI, a CI/CD platform."),
-		OssItem("Trello", "Scan Trello, a ticket system."),
 		OssItem("Syslog", "Scan syslog, event data logs."),
 		OssItem("Docker", "Scan a Docker instance, a containerized application."),
 		OssItem("GCS (Google Cloud Storage)", "Scan a Google Cloud Storage instance."),

pkg/tui/sources/sources.go

@@ -5,7 +5,6 @@ import (
 
 	tea "github.com/charmbracelet/bubbletea"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/circleci"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/trello"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/docker"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/filesystem"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/gcs"
@@ -53,8 +52,6 @@ func GetSourceFields(sourceName string) CmdModel {
 		return syslog.GetFields()
 	case "circleci":
 		return circleci.GetFields()
-	case "trello":
-		return trello.GetFields()
 	case "docker":
 		return docker.GetFields()
 	}

pkg/tui/sources/trello/trello.go

@@ -1,64 +0,0 @@
-package trello
-
-import (
-	"strings"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs"
-)
-
-type trelloCmdModel struct {
-	textinputs.Model
-}
-
-func GetFields() trelloCmdModel {
-	apiKey := textinputs.InputConfig{
-		Label:       "API Key",
-		Key:         "api_key",
-		Required:    true,
-		Placeholder: "Your Trello API key",
-	}
-
-	token := textinputs.InputConfig{
-		Label:       "Token",
-		Key:         "token",
-		Required:    true,
-		Placeholder: "Your Trello token",
-	}
-
-	boardIds := textinputs.InputConfig{
-		Label:       "Board IDs",
-		Key:         "board_ids",
-		Required:    true,
-		Placeholder: "Comma-separated list of Trello board ID(s)",
-	}
-
-	return trelloCmdModel{textinputs.New([]textinputs.InputConfig{apiKey, token, boardIds})}
-}
-
-func (m trelloCmdModel) Cmd() string {
-	var command []string
-	command = append(command, "trufflehog", "trello")
-
-	inputs := m.GetInputs()
-
-	if inputs["api_key"] != "" {
-		command = append(command, "--api-key="+inputs["api_key"])
-	}
-	if inputs["token"] != "" {
-		command = append(command, "--token="+inputs["token"])
-	}
-	if inputs["board_ids"] != "" {
-		command = append(command, "--board-ids="+inputs["board_ids"])
-	}
-
-	return strings.Join(command, " ")
-}
-
-func (m trelloCmdModel) Summary() string {
-	inputs := m.GetInputs()
-	labels := m.GetLabels()
-	keys := []string{"api_key", "token", "board_ids"}
-
-	return common.SummarizeSource(keys, inputs, labels)
-}

proto/source_metadata.proto

@@ -186,17 +186,6 @@ message Test {
   string file = 1;
 }
 
-message Trello {
-  string board_id = 1;
-  string card_id = 2;
-  string list_id = 3;
-  string action_id = 4;
-  string user_id = 5;
-  string username = 6;
-  string timestamp = 7;
-  string link = 8;
-}
-
 message Jenkins {
   string project_name = 1;
   int64 build_number = 2;
@@ -305,6 +294,5 @@ message MetaData {
     SharePoint sharepoint = 25;
     GoogleDrive googleDrive = 26;
     AzureRepos azureRepos = 27;
-    Trello trello = 28;
   }
 }

proto/sources.proto

@@ -44,7 +44,6 @@ enum SourceType {
   SOURCE_TYPE_SHAREPOINT = 29;
   SOURCE_TYPE_GCS_UNAUTHED = 30;
   SOURCE_TYPE_AZURE_REPOS = 31;
-  SOURCE_TYPE_TRELLO = 32;
 }
 
 message LocalSource {
@@ -266,17 +265,6 @@ message Jenkins {
   }
 }
 
-message Trello {
-  message TrelloAuth {
-    string api_key = 1;
-    string token = 2;
-  }
-
-  TrelloAuth auth = 1;
-  repeated string boards = 2;
-}
-
-
 message Teams {
   string endpoint = 1 [(validate.rules).string.uri_ref = true];
   oneof credential {