Add RunOnce and ScheduleRunOnce to SourceManager

This allows the manager to run an already initialized source.

README.md

@@ -32,7 +32,7 @@ Have questions? Feedback? Jump in slack or discord and hang out with us
 
 Join our [Slack Community](https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ)
 
-Join the [Secret Scanning Discord](https://discord.gg/sydS6AHTUP)
+Join the [Secret Scanning Discord](https://discord.gg/8Hzbrnkr7E)
 
 # :tv: Demo
 

pkg/context/context.go

@@ -26,9 +26,10 @@ type Context interface {
 	Logger() logr.Logger
 }
 
-// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
-// the same types.
+// CancelFunc and CancelCauseFunc are type aliases to allow use as if they are
+// the same types as the standard library variants.
 type CancelFunc = context.CancelFunc
+type CancelCauseFunc = context.CancelCauseFunc
 
 // logCtx implements Context.
 type logCtx struct {
@@ -68,6 +69,16 @@ func WithCancel(parent Context) (Context, context.CancelFunc) {
 	return lCtx, cancel
 }
 
+// WithCancelCause returns context.WithCancelCause with the log object propagated.
+func WithCancelCause(parent Context) (Context, context.CancelCauseFunc) {
+	ctx, cancel := context.WithCancelCause(parent)
+	lCtx := logCtx{
+		log:     parent.Logger(),
+		Context: ctx,
+	}
+	return lCtx, cancel
+}
+
 // WithDeadline returns context.WithDeadline with the log object propagated and
 // the deadline added to the structured log values.
 func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
@@ -79,6 +90,17 @@ func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
 	return lCtx, cancel
 }
 
+// WithDeadlineCause returns context.WithDeadlineCause with the log object
+// propagated and the deadline added to the structured log values.
+func WithDeadlineCause(parent Context, d time.Time, cause error) (Context, context.CancelFunc) {
+	ctx, cancel := context.WithDeadlineCause(parent, d, cause)
+	lCtx := logCtx{
+		log:     parent.Logger().WithValues("deadline", d),
+		Context: ctx,
+	}
+	return lCtx, cancel
+}
+
 // WithTimeout returns context.WithTimeout with the log object propagated and
 // the timeout added to the structured log values.
 func WithTimeout(parent Context, timeout time.Duration) (Context, context.CancelFunc) {
@@ -90,6 +112,22 @@ func WithTimeout(parent Context, timeout time.Duration) (Context, context.Cancel
 	return lCtx, cancel
 }
 
+// WithTimeoutCause returns context.WithTimeoutCause with the log object
+// propagated and the timeout added to the structured log values.
+func WithTimeoutCause(parent Context, timeout time.Duration, cause error) (Context, context.CancelFunc) {
+	ctx, cancel := context.WithTimeoutCause(parent, timeout, cause)
+	lCtx := logCtx{
+		log:     parent.Logger().WithValues("timeout", timeout),
+		Context: ctx,
+	}
+	return lCtx, cancel
+}
+
+// Cause returns the context.Cause of the context.
+func Cause(ctx context.Context) error {
+	return context.Cause(ctx)
+}
+
 // WithValue returns context.WithValue with the log object propagated and
 // the value added to the structured log values (if the key is a string).
 func WithValue(parent Context, key, val any) Context {

pkg/context/context_test.go

@@ -3,6 +3,7 @@ package context
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"strings"
 	"testing"
 	"time"
@@ -177,3 +178,10 @@ func TestRace(t *testing.T) {
 	cancel()
 	_ = ctx.Err()
 }
+
+func TestCause(t *testing.T) {
+	ctx, cancel := WithCancelCause(Background())
+	err := fmt.Errorf("oh no")
+	cancel(err)
+	assert.Equal(t, err, Cause(ctx))
+}

pkg/detectors/appcues/appcues.go

@@ -1,11 +1,11 @@
 package appcues
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,59 +20,64 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client = common.SaneHttpClient()
 
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"appcues"}) + `\b([a-z0-9-]{36})\b`)
 	userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appcues"}) + `\b([a-z0-9-]{39})\b`)
 	idPat   = regexp.MustCompile(detectors.PrefixRegex([]string{"appcues"}) + `\b([0-9]{5})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("appcues")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"appcues"}
 }
 
+// FromData will find and optionally verify Appcues secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
 
-	keyMatches := keyPat.FindAllSubmatch(data, -1)
-	userMatches := userPat.FindAllSubmatch(data, -1)
-	idMatches := idPat.FindAllSubmatch(data, -1)
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	userMatches := userPat.FindAllStringSubmatch(dataStr, -1)
+	idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
 
-	for _, match := range keyMatches {
+	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		for _, userMatch := range userMatches {
 			if len(userMatch) != 2 {
 				continue
 			}
 
-			resUserMatch := bytes.TrimSpace(userMatch[1])
+			resUserMatch := strings.TrimSpace(userMatch[1])
 
 			for _, idMatch := range idMatches {
 				if len(idMatch) != 2 {
 					continue
 				}
 
-				resIdMatch := bytes.TrimSpace(idMatch[1])
+				resIdMatch := strings.TrimSpace(idMatch[1])
 
 				s1 := detectors.Result{
 					DetectorType: detectorspb.DetectorType_Appcues,
-					Raw:          resMatch,
-					RawV2:        append(resMatch, resUserMatch...),
+					Raw:          []byte(resMatch),
+					RawV2:        []byte(resMatch + resUserMatch),
 				}
-
 				if verify {
-					req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.appcues.com/v2/accounts/%s/flows", string(resIdMatch)), nil)
+					req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.appcues.com/v2/accounts/%s/flows", resIdMatch), nil)
 					if err != nil {
 						continue
 					}
-					req.SetBasicAuth(string(resUserMatch), string(resMatch))
+					req.SetBasicAuth(resUserMatch, resMatch)
 					res, err := client.Do(req)
 					if err == nil {
 						defer res.Body.Close()
 						if res.StatusCode >= 200 && res.StatusCode < 300 {
 							s1.Verified = true
 						} else {
+							// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 							if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 								continue
 							}

pkg/detectors/appfollow/appfollow.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,35 +15,42 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appfollow"}) + `\b([0-9A-Za-z]{20})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("appfollow")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"appfollow"}
 }
 
+// FromData will find and optionally verify Appfollow secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-
-		resMatch := match[1]
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Appfollow,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			data := append(resMatch, ':')
-			sEnc := b64.StdEncoding.EncodeToString(data)
+			data := fmt.Sprintf("%s:", resMatch)
+			sEnc := b64.StdEncoding.EncodeToString([]byte(data))
 			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.appfollow.io/test", nil)
 			if err != nil {
 				continue
@@ -54,6 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/appointedd/appointedd.go

@@ -1,11 +1,11 @@
 package appointedd
 
 import (
-	"bytes"
 	"context"
 	"io"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,53 +14,66 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appointedd"}) + `\b([a-zA-Z0-9=+]{88})`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("appointedd")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"appointedd"}
 }
 
+// FromData will find and optionally verify appointedd secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Appointedd,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 		if verify {
 			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.appointedd.com/v1/availability/slots", nil)
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-API-KEY", string(resMatch))
+			req.Header.Add("X-API-KEY", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
+				bodyBytes, err := io.ReadAll(res.Body)
+				if err != nil {
+					continue
+				}
+				body := string(bodyBytes)
 
-				if bodyBytes, err := io.ReadAll(res.Body); err == nil {
-					if bytes.Contains(bodyBytes, []byte("total")) {
-						s1.Verified = true
-					} else {
-						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-							continue
-						}
+				if strings.Contains(body, "total") {
+					s1.Verified = true
+				} else {
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
 					}
 				}
 			}
 		}
+
 		results = append(results, s1)
 	}
+
 	return results, nil
 }
 

pkg/detectors/appsynergy/appsynergy.go

@@ -1,11 +1,11 @@
 package appsynergy
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,34 +14,42 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appsynergy"}) + `\b([a-z0-9]{64})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("appsynergy")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"appsynergy"}
 }
 
+// FromData will find and optionally verify AppSynergy secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_AppSynergy,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			payload := bytes.NewBuffer([]byte(`{"html":"<html><body><h1>Hello World</h1></body></html>","filename":"HelloWorld.pdf"}`))
-			req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("https://www.appsynergy.com/api?action=HTML2PDF&apiKey=%s", string(resMatch)), payload)
+			payload := strings.NewReader(`{"html":"<html><body><h1>Hello World</h1></body></html>","filename":"HelloWorld.pdf"}`)
+			req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("https://www.appsynergy.com/api?action=HTML2PDF&apiKey=%s", resMatch), payload)
 			if err != nil {
 				continue
 			}
@@ -51,6 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/apptivo/apptivo.go

@@ -1,13 +1,12 @@
 package apptivo
 
 import (
-	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"regexp"
-
-	"io/ioutil"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -16,52 +15,60 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apptivo"}) + `\b([a-z0-9-]{36})\b`)
 	idPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"apptivo"}) + `\b([a-zA-Z0-9-]{32})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("apptivo")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"apptivo"}
 }
 
+// FromData will find and optionally verify Apptivo secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
-	idMatches := idPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 		for _, idMatch := range idMatches {
 			if len(idMatch) != 2 {
 				continue
 			}
-			resIdMatch := bytes.TrimSpace(idMatch[1])
+			resIdMatch := strings.TrimSpace(idMatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Apptivo,
-				Raw:          resMatch,
-				RawV2:        append(resMatch, resIdMatch...),
+				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}
 
 			if verify {
-				req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.apptivo.com/app/dao/v6/leads?a=getConfigData&apiKey=%s&accessKey=%s", string(resMatch), string(resIdMatch)), nil)
+				req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.apptivo.com/app/dao/v6/leads?a=getConfigData&apiKey=%s&accessKey=%s", resMatch, resIdMatch), nil)
 				if err != nil {
 					continue
 				}
 				res, err := client.Do(req)
 				if err == nil {
-					bodyBytes, err := ioutil.ReadAll(res.Body)
+					bodyBytes, err := io.ReadAll(res.Body)
 					if err != nil {
 						continue
 					}
-					validResponse := bytes.Contains(bodyBytes, []byte(`displayName`))
+					bodyString := string(bodyBytes)
+					validResponse := strings.Contains(bodyString, `displayName`)
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						if validResponse {
@@ -70,6 +77,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 							s1.Verified = false
 						}
 					} else {
+						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}

pkg/detectors/artifactory/artifactory.go

@@ -1,10 +1,10 @@
 package artifactory
 
 import (
-	"bytes"
 	"context"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,54 +19,58 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client = common.SaneHttpClient()
 
-	// Keep the regexp patterns unchanged
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{73})`)
 	URLPat = regexp.MustCompile(`\b([A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])\.jfrog\.io)`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("artifactory")}
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"artifactory"}
 }
 
 // FromData will find and optionally verify Artifactory secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	URLmatches := URLPat.FindAllSubmatch(data, -1)
-	matches := keyPat.FindAllSubmatch(data, -1)
-
-	var resURLMatch []byte
+	dataStr := string(data)
+	URLmatches := URLPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
+	resURLMatch := ""
 	for _, URLmatch := range URLmatches {
 		if len(URLmatch) != 2 {
 			continue
 		}
-		resURLMatch = bytes.TrimSpace(URLmatch[1])
+		resURLMatch = strings.TrimSpace(URLmatch[1])
+
 	}
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
-			Raw:          resMatch,
-			RawV2:        append(resMatch, resURLMatch...),
+			Raw:          []byte(resMatch),
+			RawV2:        []byte(resMatch + resURLMatch),
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://"+string(resURLMatch)+"/artifactory/api/storageinfo", nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resURLMatch+"/artifactory/api/storageinfo", nil)
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-JFrog-Art-Api", string(resMatch))
+
+			req.Header.Add("X-JFrog-Art-Api", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/artsy/artsy.go

@@ -1,10 +1,10 @@
 package artsy
 
 import (
-	"bytes"
 	"context"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,44 +13,51 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"artsy"}) + `\b([0-9a-zA-Z]{32})\b`)
 	idPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"artsy"}) + `\b([0-9a-zA-Z]{20})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("artsy")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"artsy"}
 }
 
+// FromData will find and optionally verify Artsy secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
-	idmatches := idPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	idmatches := idPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
 
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		for _, idmatch := range idmatches {
 			if len(idmatch) != 2 {
 				continue
 			}
-			resIdMatch := bytes.TrimSpace(idmatch[1])
+			resIdMatch := strings.TrimSpace(idmatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Artsy,
-				Raw:          resMatch,
-				RawV2:        append(resMatch, resIdMatch...),
+				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}
 
 			if verify {
-				req, err := http.NewRequestWithContext(ctx, "POST", "https://api.artsy.net/api/tokens/xapp_token?client_id="+string(resIdMatch)+"&client_secret="+string(resMatch), nil)
+				req, err := http.NewRequestWithContext(ctx, "POST", "https://api.artsy.net/api/tokens/xapp_token?client_id="+resIdMatch+"&client_secret="+resMatch, nil)
 				if err != nil {
 					continue
 				}
@@ -60,7 +67,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
-						if detectors.IsKnownFalsePositive(bytes.Join([][]byte{resMatch, resIdMatch}, nil), detectors.DefaultFalsePositives, true) {
+						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}
 					}
@@ -69,6 +77,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 			results = append(results, s1)
 		}
+
 	}
 
 	return results, nil

pkg/detectors/asanaoauth/asanaoauth.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,25 +20,31 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client = common.SaneHttpClient()
 
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"asana"}) + `\b([a-z\/:0-9]{51})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("asana")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"asana"}
 }
 
+// FromData will find and optionally verify AsanaOauth secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := match[1]
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_AsanaOauth,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -45,13 +52,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
 			res, err := client.Do(req)
-			if err == nil && res.Body != nil {
+			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken.go

@@ -1,11 +1,11 @@
 package asanapersonalaccesstoken
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,6 +14,7 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
@@ -22,22 +23,27 @@ var (
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"asana"}) + `\b([0-9]{1,}\/[0-9]{16,}:[A-Za-z0-9]{32,})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("asana")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"asana"}
 }
 
+// FromData will find and optionally verify AsanaPersonalAccessToken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_AsanaPersonalAccessToken,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -45,13 +51,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/assemblyai/assemblyai.go

@@ -1,10 +1,10 @@
 package assemblyai
 
 import (
-	"bytes"
 	"context"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,28 +19,31 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client = common.SaneHttpClient()
 
-	// Keep the regexp patterns unchanged
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"assemblyai"}) + `\b([0-9a-z]{32})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("assemblyai")}
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"assemblyai"}
 }
 
 // FromData will find and optionally verify Assemblyai secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_AssemblyAI,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -49,13 +52,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("Authorization", string(resMatch))
+			req.Header.Add("Authorization", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/atera/atera.go

@@ -1,10 +1,10 @@
 package atera
 
 import (
-	"bytes"
 	"context"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -18,27 +18,32 @@ var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atera"}) + `\b([0-9a-z]{32})\b`)
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atera"}) + `\b([[0-9a-z]{32})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("atera")}
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"atera"}
 }
 
 // FromData will find and optionally verify Atera secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Atera,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -47,14 +52,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Accept", "application/json")
-			req.Header.Add("X-API-KEY", string(resMatch))
+			req.Header.Add("X-API-KEY", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// Consider adding the []byte conversion here if the input also changed to a []byte.
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/audd/audd.go

@@ -1,12 +1,12 @@
 package audd
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"io"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -26,26 +26,30 @@ var (
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("audd")}
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"audd"}
 }
 
+// FromData will find and optionally verify Audd secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Audd,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.audd.io/setCallbackUrl/?api_token=%s&url=https://yourwebsite.com/callbacks_handler/", string(resMatch)), nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.audd.io/setCallbackUrl/?api_token=%s&url=https://yourwebsite.com/callbacks_handler/", resMatch), nil)
 			if err != nil {
 				continue
 			}
@@ -53,8 +57,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err == nil {
 				bodyBytes, err := io.ReadAll(res.Body)
 				if err == nil {
+					bodyString := string(bodyBytes)
+					validResponse := strings.Contains(bodyString, `"status":"success"`)
 					defer res.Body.Close()
-					validResponse := bytes.Contains(bodyBytes, []byte(`"status":"success"`))
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						if validResponse {
 							s1.Verified = true
@@ -62,6 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 							s1.Verified = false
 						}
 					} else {
+						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}

pkg/detectors/auth0managementapitoken/auth0managementapitoken.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,28 +14,36 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
+	// long jwt token but note this is default 8640000 seconds = 24 hours but could be set to maximum 2592000 seconds = 720 hours = 30 days
+	// at https://manage.auth0.com/dashboard/us/dev-63memjo3/apis/management/explorer
 	managementApiTokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"auth0"}) + `\b(ey[a-zA-Z0-9._-]+)\b`)
-	domainPat             = regexp.MustCompile(`([a-zA-Z0-9\-]{2,16}\.[a-zA-Z0-9_-]{2,3}\.auth0\.com)`)
+	domainPat             = regexp.MustCompile(`([a-zA-Z0-9\-]{2,16}\.[a-zA-Z0-9_-]{2,3}\.auth0\.com)`) // could be part of url
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("auth0")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"auth0"}
 }
 
+// FromData will find and optionally verify Auth0ManagementApiToken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	managementApiTokenMatches := managementApiTokenPat.FindAllSubmatch(data, -1)
-	domainMatches := domainPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	managementApiTokenMatches := managementApiTokenPat.FindAllStringSubmatch(dataStr, -1)
+	domainMatches := domainPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, managementApiTokenMatch := range managementApiTokenMatches {
 		if len(managementApiTokenMatch) != 2 {
 			continue
 		}
-		managementApiTokenRes := managementApiTokenMatch[1]
+		managementApiTokenRes := strings.TrimSpace(managementApiTokenMatch[1])
 		if len(managementApiTokenRes) < 2000 || len(managementApiTokenRes) > 5000 {
 			continue
 		}
@@ -43,22 +52,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if len(domainMatch) != 2 {
 				continue
 			}
-
-			domainRes := domainMatch[1]
+			domainRes := strings.TrimSpace(domainMatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken,
-				Redacted:     string(domainRes),
-				Raw:          managementApiTokenRes,
-				RawV2:        append(managementApiTokenRes, domainRes...),
+				Redacted:     domainRes,
+				Raw:          []byte(managementApiTokenRes),
+				RawV2:        []byte(managementApiTokenRes + domainRes),
 			}
 
 			if verify {
-				req, err := http.NewRequestWithContext(ctx, "GET", "https://"+string(domainRes)+"/api/v2/users", nil)
+				/*
+				   curl -H "Authorization: Bearer $token" https://domain/api/v2/users
+				*/
+
+				req, err := http.NewRequestWithContext(ctx, "GET", "https://"+domainRes+"/api/v2/users", nil)
 				if err != nil {
 					continue
 				}
-				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(managementApiTokenRes)))
+				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", managementApiTokenRes))
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()

pkg/detectors/auth0oauth/auth0oauth.go

@@ -1,7 +1,6 @@
 package auth0oauth
 
 import (
-	"bytes"
 	"context"
 	"io"
 	"net/http"
@@ -16,6 +15,7 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
@@ -23,54 +23,64 @@ var (
 
 	clientIdPat     = regexp.MustCompile(detectors.PrefixRegex([]string{"auth0"}) + `\b([a-zA-Z0-9_-]{32,60})\b`)
 	clientSecretPat = regexp.MustCompile(`\b([a-zA-Z0-9_-]{64,})\b`)
-	domainPat       = regexp.MustCompile(`\b([a-zA-Z0-9][a-zA-Z0-9._-]*auth0\.com)\b`)
+	domainPat       = regexp.MustCompile(`\b([a-zA-Z0-9][a-zA-Z0-9._-]*auth0\.com)\b`) // could be part of url
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("auth0")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"auth0"}
 }
 
+// FromData will find and optionally verify Auth0oauth secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
 
-	clientIdMatches := clientIdPat.FindAllSubmatch(data, -1)
-	clientSecretMatches := clientSecretPat.FindAllSubmatch(data, -1)
-	domainMatches := domainPat.FindAllSubmatch(data, -1)
+	clientIdMatches := clientIdPat.FindAllStringSubmatch(dataStr, -1)
+	clientSecretMatches := clientSecretPat.FindAllStringSubmatch(dataStr, -1)
+	domainMatches := domainPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, clientIdMatch := range clientIdMatches {
 		if len(clientIdMatch) != 2 {
 			continue
 		}
-		clientIdRes := bytes.TrimSpace(clientIdMatch[1])
+		clientIdRes := strings.TrimSpace(clientIdMatch[1])
 
 		for _, clientSecretMatch := range clientSecretMatches {
 			if len(clientSecretMatch) != 2 {
 				continue
 			}
-			clientSecretRes := bytes.TrimSpace(clientSecretMatch[1])
+			clientSecretRes := strings.TrimSpace(clientSecretMatch[1])
 
 			for _, domainMatch := range domainMatches {
 				if len(domainMatch) != 2 {
 					continue
 				}
-				domainRes := bytes.TrimSpace(domainMatch[1])
+				domainRes := strings.TrimSpace(domainMatch[1])
 
 				s1 := detectors.Result{
 					DetectorType: detectorspb.DetectorType_Auth0oauth,
-					Redacted:     string(clientIdRes),
-					Raw:          clientSecretRes,
-					RawV2:        append(clientIdRes, clientSecretRes...),
+					Redacted:     clientIdRes,
+					Raw:          []byte(clientSecretRes),
+					RawV2:        []byte(clientIdRes + clientSecretRes),
 				}
 
 				if verify {
+					/*
+					   curl --request POST \
+					     --url 'https://YOUR_DOMAIN/oauth/token' \
+					     --header 'content-type: application/x-www-form-urlencoded' \
+					     --data 'grant_type=authorization_code&client_id=W44JmL3qD6LxHeEJyKe9lMuhcwvPOaOq&client_secret=YOUR_CLIENT_SECRET&code=AUTHORIZATION_CODE&redirect_uri=undefined'
+					*/
 
 					data := url.Values{}
 					data.Set("grant_type", "authorization_code")
-					data.Set("client_id", string(clientIdRes))
-					data.Set("client_secret", string(clientSecretRes))
+					data.Set("client_id", clientIdRes)
+					data.Set("client_secret", clientSecretRes)
 					data.Set("code", "AUTHORIZATION_CODE")
 					data.Set("redirect_uri", "undefined")
 
-					req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://"+string(domainRes)+"/oauth/token", strings.NewReader(data.Encode()))
+					req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://"+domainRes+"/oauth/token", strings.NewReader(data.Encode())) // URL-encoded payload
 					if err != nil {
 						continue
 					}
@@ -84,7 +94,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						}
 						body := string(bodyBytes)
 
-						if !bytes.Contains(bodyBytes, []byte("access_denied")) {
+						// if client_id and client_secret is valid -> 403 {"error":"invalid_grant","error_description":"Invalid authorization code"}
+						// if invalid -> 401 {"error":"access_denied","error_description":"Unauthorized"}
+						// ingenious!
+
+						if !strings.Contains(body, "access_denied") {
 							s1.Verified = true
 						} else {
 							if detectors.IsKnownFalsePositive(clientIdRes, detectors.DefaultFalsePositives, true) {

pkg/detectors/autodesk/autodesk.go

@@ -1,11 +1,11 @@
 package autodesk
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,41 +14,49 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client    = common.SaneHttpClient()
+	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat    = regexp.MustCompile(detectors.PrefixRegex([]string{"autodesk"}) + `\b([0-9A-Za-z]{32})\b`)
 	secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"autodesk"}) + `\b([0-9A-Za-z]{16})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("autodesk")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"autodesk"}
 }
 
+// FromData will find and optionally verify Autodesk secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
-	secretMatches := secretPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 		for _, secretMatch := range secretMatches {
 			if len(secretMatch) != 2 {
 				continue
 			}
-			resSecret := bytes.TrimSpace(secretMatch[1])
+			resSecret := strings.TrimSpace(secretMatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Autodesk,
-				Raw:          resMatch,
-				RawV2:        append(resMatch, resSecret...),
+				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resSecret),
 			}
 
 			if verify {
-				payload := bytes.NewReader([]byte(fmt.Sprintf(`grant_type=client_credentials&client_id=%s&client_secret=%s`, string(resMatch), string(resSecret))))
+				payload := strings.NewReader(fmt.Sprintf(`grant_type=client_credentials&client_id=%s&client_secret=%s`, resMatch, resSecret))
 				req, err := http.NewRequestWithContext(ctx, "POST", "https://developer.api.autodesk.com/authentication/v1/authenticate", payload)
 				if err != nil {
 					continue
@@ -60,6 +68,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
+						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}

pkg/detectors/autoklose/autoklose.go

@@ -1,13 +1,13 @@
 package autoklose
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -16,34 +16,41 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"autoklose"}) + `\b([a-zA-Z0-9-]{32})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("autoklose")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"autoklose"}
 }
 
+// FromData will find and optionally verify Autoklose secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Autoklose,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.autoklose.com/api/campaigns/?api_token=%s", string(resMatch)), nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.autoklose.com/api/campaigns/?api_token=%s", resMatch), nil)
 			if err != nil {
 				continue
 			}
@@ -61,6 +68,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						s1.Verified = false
 					}
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/autopilot/autopilot.go

@@ -1,10 +1,10 @@
 package autopilot
 
 import (
-	"bytes"
 	"context"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,30 +13,37 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"autopilot"}) + `\b([0-9a-f]{32})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("autopilot")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"autopilot"}
 }
 
+// FromData will find and optionally verify AutoPilot secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_AutoPilot,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -44,13 +51,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("autopilotapikey", string(resMatch))
+			req.Header.Add("autopilotapikey", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken.go

@@ -1,11 +1,11 @@
 package avazapersonalaccesstoken
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,43 +14,53 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	// The number prefix increments for every Personal Access Token created.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"avaza"}) + `\b([0-9]+-[0-9a-f]{40})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("avaza")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"avaza"}
 }
 
+// FromData will find and optionally verify AvazaPersonalAccessToken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_AvazaPersonalAccessToken,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.avaza.com/api/Account", nil)
+			req, err := http.NewRequest("GET", "https://api.avaza.com/api/Account", nil)
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/aviationstack/aviationstack.go

@@ -1,11 +1,12 @@
 package aviationstack
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
+	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,31 +21,37 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client = common.SaneHttpClient()
 
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aviationstack"}) + `\b([a-z0-9]{32})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("aviationstack")}
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"aviationstack"}
 }
 
 // FromData will find and optionally verify AviationStack secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_AviationStack,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.aviationstack.com/v1/flights?access_key=%s", string(resMatch)), nil)
+			timeout := 10 * time.Second
+			client.Timeout = timeout
+			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.aviationstack.com/v1/flights?access_key=%s", resMatch), nil)
 			if err != nil {
 				continue
 			}
@@ -54,6 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/aws/aws.go

@@ -1,7 +1,6 @@
 package aws
 
 import (
-	"bytes"
 	"context"
 	"crypto/hmac"
 	"crypto/sha256"
@@ -20,8 +19,7 @@ import (
 
 type scanner struct {
 	verificationClient *http.Client
-
-	skipIDs map[string]struct{}
+	skipIDs            map[string]struct{}
 }
 
 func New(opts ...func(*scanner)) *scanner {
@@ -63,13 +61,14 @@ var (
 	falsePositiveSecretCheck = regexp.MustCompile(`[a-f0-9]{40}`)
 )
 
-// Returning the keywords as byte arrays
-func (s scanner) Keywords() [][]byte {
-	return [][]byte{
-		[]byte("AKIA"),
-		[]byte("ABIA"),
-		[]byte("ACCA"),
-		[]byte("ASIA"),
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s scanner) Keywords() []string {
+	return []string{
+		"AKIA",
+		"ABIA",
+		"ACCA",
+		"ASIA",
 	}
 }
 
@@ -88,17 +87,19 @@ func GetHMAC(key []byte, data []byte) []byte {
 
 // FromData will find and optionally verify AWS secrets in a given set of bytes.
 func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	idMatches := idPat.FindAllSubmatch(data, -1)
-	secretMatches := secretPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
+	secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, idMatch := range idMatches {
 		if len(idMatch) != 2 {
 			continue
 		}
-		resIDMatch := bytes.TrimSpace(idMatch[1])
+		resIDMatch := strings.TrimSpace(idMatch[1])
 
 		if s.skipIDs != nil {
-			if _, ok := s.skipIDs[string(resIDMatch)]; ok {
+			if _, ok := s.skipIDs[resIDMatch]; ok {
 				continue
 			}
 		}
@@ -107,13 +108,13 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if len(secretMatch) != 2 {
 				continue
 			}
-			resSecretMatch := bytes.TrimSpace(secretMatch[1])
+			resSecretMatch := strings.TrimSpace(secretMatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_AWS,
-				Raw:          resIDMatch,
-				Redacted:     string(resIDMatch),
-				RawV2:        append(append([]byte(nil), resIDMatch...), resSecretMatch...),
+				Raw:          []byte(resIDMatch),
+				Redacted:     resIDMatch,
+				RawV2:        []byte(resIDMatch + resSecretMatch),
 			}
 
 			if verify {
@@ -145,17 +146,17 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				params.Add("Action", "GetCallerIdentity")
 				params.Add("Version", "2011-06-15")
 				params.Add("X-Amz-Algorithm", algorithm)
-				params.Add("X-Amz-Credential", string(resIDMatch)+"/"+credentialScope)
+				params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
 				params.Add("X-Amz-Date", amzDate)
 				params.Add("X-Amz-Expires", "30")
 				params.Add("X-Amz-SignedHeaders", signedHeaders)
 
 				canonicalQuerystring := params.Encode()
 				payloadHash := GetHash("") // empty payload
-				canonicalRequest := string(method) + "\n" + string(canonicalURI) + "\n" + string(canonicalQuerystring) + "\n" + string(canonicalHeaders) + "\n" + string(signedHeaders) + "\n" + string(payloadHash)
+				canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
 
 				// TASK 2: CREATE THE STRING TO SIGN.
-				stringToSign := string(algorithm) + "\n" + string(amzDate) + "\n" + string(credentialScope) + "\n" + string(GetHash(canonicalRequest))
+				stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
 
 				// TASK 3: CALCULATE THE SIGNATURE.
 				// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
@@ -227,7 +228,7 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			// If the result is unverified and matches something like a git hash, don't include it in the results.
-			if !s1.Verified && falsePositiveSecretCheck.Match(resSecretMatch) {
+			if !s1.Verified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
 				continue
 			}
 
@@ -241,157 +242,6 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return awsCustomCleanResults(results), nil
 }
 
-// func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-// 	idMatches := idPat.FindAllSubmatch(data, -1)
-// 	secretMatches := secretPat.FindAllSubmatch(data, -1)
-//
-// 	for _, idMatch := range idMatches {
-// 		if len(idMatch) != 2 {
-// 			continue
-// 		}
-// 		resIDMatch := bytes.TrimSpace(idMatch[1])
-//
-// 		if s.skipIDs != nil {
-// 			if _, ok := s.skipIDs[string(resIDMatch)]; ok {
-// 				continue
-// 			}
-// 		}
-//
-// 		for _, secretMatch := range secretMatches {
-// 			if len(secretMatch) != 2 {
-// 				continue
-// 			}
-// 			resSecretMatch := bytes.TrimSpace(secretMatch[1])
-//
-// 			s1 := detectors.Result{
-// 				DetectorType: detectorspb.DetectorType_AWS,
-// 				Raw:          resIDMatch,
-// 				Redacted:     string(resIDMatch),
-// 				RawV2:        append(append([]byte(nil), resIDMatch...), resSecretMatch...),
-// 			}
-//
-// 			if verify {
-// 				// Each string operation is converted into byte operations
-//
-// 				method := []byte("GET")
-// 				service := []byte("sts")
-// 				host := []byte("sts.amazonaws.com")
-// 				region := []byte("us-east-1")
-// 				endpoint := []byte("https://sts.amazonaws.com")
-// 				datestamp := []byte(time.Now().UTC().Format("20060102"))
-// 				amzDate := []byte(time.Now().UTC().Format("20060102T150405Z0700"))
-//
-// 				req, err := http.NewRequestWithContext(ctx, string(method), string(endpoint), nil)
-// 				if err != nil {
-// 					continue
-// 				}
-// 				req.Header.Set("Accept", "application/json")
-//
-// 				// Byte operations for the subsequent string operations
-//
-// 				canonicalURI := []byte("/")
-// 				canonicalHeaders := []byte("host:" + string(host) + "\n")
-// 				signedHeaders := []byte("host")
-// 				algorithm := []byte("AWS4-HMAC-SHA256")
-// 				credentialScope := []byte(fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service))
-//
-// 				params := req.URL.Query()
-// 				params.Add("Action", "GetCallerIdentity")
-// 				params.Add("Version", "2011-06-15")
-// 				params.Add("X-Amz-Algorithm", string(algorithm))
-// 				params.Add("X-Amz-Credential", string(resIDMatch)+"/"+string(credentialScope))
-// 				params.Add("X-Amz-Date", string(amzDate))
-// 				params.Add("X-Amz-Expires", "30")
-// 				params.Add("X-Amz-SignedHeaders", string(signedHeaders))
-//
-// 				canonicalQuerystring := params.Encode()
-// 				payloadHash := GetHash([]byte("")) // empty payload
-// 				canonicalRequest := string(method) + "\n" + string(canonicalURI) + "\n" + string(canonicalQuerystring) + "\n" + string(canonicalHeaders) + "\n" + string(signedHeaders) + "\n" + string(payloadHash)
-//
-// 				// TASK 2: CREATE THE STRING TO SIGN.
-// 				stringToSign := string(algorithm) + "\n" + string(amzDate) + "\n" + string(credentialScope) + "\n" + string(GetHash([]byte(canonicalRequest)))
-//
-// 				hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", string(resSecretMatch))), datestamp)
-// 				hash = GetHMAC(hash, region)
-// 				hash = GetHMAC(hash, service)
-// 				hash = GetHMAC(hash, []byte("aws4_request"))
-//
-// 				signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
-// 				signature := hex.EncodeToString(signature2)
-//
-// 				params.Add("X-Amz-Signature", signature)
-// 				req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
-// 				req.URL.RawQuery = params.Encode()
-//
-// 				client := s.verificationClient
-// 				if client == nil {
-// 					client = defaultVerificationClient
-// 				}
-// 				res, err := client.Do(req)
-// 				if err == nil {
-//
-// 					if res.StatusCode >= 200 && res.StatusCode < 300 {
-// 						identityInfo := identityRes{}
-// 						err := json.NewDecoder(res.Body).Decode(&identityInfo)
-// 						if err == nil {
-// 							s1.Verified = true
-// 							s1.ExtraData = map[string]string{
-// 								"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
-// 								"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
-// 								"arn":     identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
-// 							}
-// 						} else {
-// 							s1.VerificationError = err
-// 						}
-// 						res.Body.Close()
-// 					} else {
-// 						// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
-// 						if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
-// 							continue
-// 						}
-//
-// 						if res.StatusCode == 403 {
-// 							var body awsErrorResponseBody
-// 							err = json.NewDecoder(res.Body).Decode(&body)
-// 							res.Body.Close()
-// 							if err == nil {
-// 								// All instances of the code I've seen in the wild are PascalCased but this check is
-// 								// case-insensitive out of an abundance of caution
-// 								if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
-// 									// determinate failure - nothing to do
-// 								} else {
-// 									// We see a surprising number of false-negative signature mismatch errors here
-// 									// (The official SDK somehow elicits even more than just making the request
-// 									// ourselves)
-// 									s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
-// 								}
-// 							} else {
-// 								s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
-// 							}
-// 						} else {
-// 							s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
-// 						}
-// 					}
-// 				} else {
-// 					s1.VerificationError = err
-// 				}
-// 			}
-//
-// 			// If the result is unverified and matches something like a git hash, don't include it in the results.
-// 			if !s1.Verified && falsePositiveSecretCheck.Match(resSecretMatch) {
-// 				continue
-// 			}
-//
-// 			results = append(results, s1)
-// 			// If we've found a verified match with this ID, we don't need to look for any more. So move on to the next ID.
-// 			if s1.Verified {
-// 				break
-// 			}
-// 		}
-// 	}
-// 	return awsCustomCleanResults(results), nil
-// }
-
 func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
 	if len(results) == 0 {
 		return results

pkg/detectors/salesforce/salesforce.go

@@ -43,8 +43,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1)
 	tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1)
 
-	fmt.Printf("instanceMatches: %v\n", instanceMatches)
-
 	for _, instance := range instanceMatches {
 		if len(instance) != 1 {
 			continue

pkg/detectors/slackwebhook/slackwebhook.go

@@ -2,6 +2,7 @@ package slackwebhook
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"net/http"
 	"regexp"
@@ -12,14 +13,15 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
-
+	defaultClient = common.SaneHttpClient()
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(`(https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[A-Za-z0-9]{23,25})`)
 )
@@ -48,6 +50,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
+
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
+
 			payload := strings.NewReader(`{"text": ""}`)
 			req, err := http.NewRequestWithContext(ctx, "POST", resMatch, payload)
 			if err != nil {
@@ -62,9 +70,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				body := string(bodyBytes)
-				if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
+
+				defer res.Body.Close()
+
+				if res.StatusCode >= 200 && res.StatusCode < 300 || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
 					s1.Verified = true
+				} else if res.StatusCode == 401 || res.StatusCode == 403 {
+					// The secret is determinately not verified (nothing to do)
+				} else {
+					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 				}
+			} else {
+				s1.VerificationError = err
 			}
 		}
 

pkg/detectors/slackwebhook/slackwebhook_test.go

@@ -6,10 +6,11 @@ package slackwebhook
 import (
 	"context"
 	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"testing"
 	"time"
 
-	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -32,11 +33,12 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
@@ -52,7 +54,8 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -68,7 +71,8 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "not found",
@@ -78,14 +82,48 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:    nil,
-			wantErr: false,
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SlackWebhook,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SlackWebhook,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("SlackWebhook.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -94,9 +132,12 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				got[i].Raw = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
 				t.Errorf("SlackWebhook.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/engine/defaults.go

@@ -2,6 +2,10 @@ package engine
 
 import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
@@ -287,6 +291,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getresponse"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getsandbox"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_old"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/githubapp"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab"
@@ -556,7 +561,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
@@ -1532,6 +1536,10 @@ func DefaultDetectors() []detectors.Detector {
 		couchbase.Scanner{},
 		envoyapikey.Scanner{},
 		github_oauth2.Scanner{},
+		snowflake.Scanner{},
+		huggingface.Scanner{},
+		trufflehogenterprise.Scanner{},
+		salesforce.Scanner{},
 	}
 
 }

pkg/handlers/archive.go

@@ -29,6 +29,8 @@ var (
 	maxDepth   = 5
 	maxSize    = 250 * 1024 * 1024 // 20MB
 	maxTimeout = time.Duration(30) * time.Second
+
+	defaultBufferSize = 512
 )
 
 // Ensure the Archive satisfies the interfaces at compile time.
@@ -62,7 +64,7 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
 
 // FromFile extracts the files from an archive.
 func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
-	archiveChan := make(chan []byte, 512)
+	archiveChan := make(chan []byte, defaultBufferSize)
 	go func() {
 		ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
 		logger := logContext.AddLogger(ctx).Logger()
@@ -183,29 +185,28 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
 			logger.Error(err, "Panic occurred when reading archive")
 		}
 	}()
-	fileContent := bytes.Buffer{}
-	logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
-	for i := 0; i <= maxSize/512; i++ {
-		if common.IsDone(ctx) {
-			return nil, ctx.Err()
-		}
-		fileChunk := make([]byte, 512)
-		bRead, err := reader.Read(fileChunk)
-		if err != nil && !errors.Is(err, io.EOF) {
-			return []byte{}, err
-		}
-		a.size += bRead
-		if len(fileChunk) > 0 {
-			fileContent.Write(fileChunk[0:bRead])
-		}
-		if bRead < 512 {
-			return fileContent.Bytes(), nil
-		}
-		if a.size >= maxSize && bRead == 512 {
-			logger.V(2).Info("Max archive size reached.")
-			return fileContent.Bytes(), nil
-		}
+
+	if common.IsDone(ctx) {
+		return nil, ctx.Err()
 	}
+
+	var fileContent bytes.Buffer
+	// Create a limited reader to ensure we don't read more than the max size.
+	lr := io.LimitReader(reader, int64(maxSize))
+
+	// Using io.CopyBuffer for performance advantages. Though buf is mandatory
+	// for the method, due to the internal implementation of io.CopyBuffer, when
+	// *bytes.Buffer implements io.WriterTo or io.ReaderFrom, the provided buf
+	// is simply ignored. Thus, we can pass nil for the buf parameter.
+	_, err = io.CopyBuffer(&fileContent, lr, nil)
+	if err != nil && !errors.Is(err, io.EOF) {
+		return nil, err
+	}
+
+	if fileContent.Len() == maxSize {
+		logger.V(2).Info("Max archive size reached.")
+	}
+
 	return fileContent.Bytes(), nil
 }
 

pkg/handlers/archive_test.go

@@ -1,6 +1,7 @@
 package handlers
 
 import (
+	"bytes"
 	"context"
 	"io"
 	"net/http"
@@ -126,6 +127,58 @@ func TestHandleFile(t *testing.T) {
 	assert.Equal(t, 1, len(ch))
 }
 
+func TestReadToMax(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    []byte
+		expected []byte
+	}{
+		{
+			name:     "read full content within maxSize",
+			input:    []byte("abcdefg"),
+			expected: []byte("abcdefg"),
+		},
+		{
+			name:     "read content larger than maxSize",
+			input:    make([]byte, maxSize+10), // this creates a byte slice 10 bytes larger than maxSize
+			expected: make([]byte, maxSize),
+		},
+		{
+			name:     "empty input",
+			input:    []byte(""),
+			expected: []byte(""),
+		},
+	}
+
+	a := &Archive{}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			reader := bytes.NewReader(tt.input)
+			output, err := a.ReadToMax(context.Background(), reader)
+			assert.Nil(t, err)
+
+			assert.Equal(t, tt.expected, output)
+		})
+	}
+}
+
+func BenchmarkReadToMax(b *testing.B) {
+	data := bytes.Repeat([]byte("a"), 1024*1000) // 1MB of data.
+	reader := bytes.NewReader(data)
+	a := &Archive{}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		_, _ = a.ReadToMax(context.Background(), reader)
+		b.StopTimer()
+
+		_, _ = reader.Seek(0, 0) // Reset the reader position.
+		a.size = 0               // Reset archive size.
+	}
+}
+
 func TestExtractDebContent(t *testing.T) {
 	// Open the sample .deb file from the testdata folder.
 	file, err := os.Open("testdata/test.deb")

pkg/output/legacy_json.go

@@ -133,7 +133,7 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
 		}
 		headCommit, err := repo.CommitObject(*headHash)
 		if err != nil {
-			logger.Error(err, "unable to get commit", "commit", headCommit.String())
+			logger.Error(err, "unable to get commit", "head_hash", headHash.String())
 			return nil
 		}
 		branches[branchName] = headCommit

pkg/sources/chunker.go

@@ -20,7 +20,7 @@ const (
 
 // Chunker takes a chunk and splits it into chunks of ChunkSize.
 func Chunker(originalChunk *Chunk) chan *Chunk {
-	chunkChan := make(chan *Chunk)
+	chunkChan := make(chan *Chunk, 1)
 	go func() {
 		defer close(chunkChan)
 		if len(originalChunk.Data) <= TotalChunkSize {

pkg/sources/circleci/circleci.go

@@ -238,6 +238,7 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
 			SourceType: s.Type(),
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
+			JobID:      s.JobID(),
 			Data:       removeCircleSha1Line(data.Bytes()),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Circleci{

pkg/sources/docker/docker.go

@@ -174,6 +174,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 					SourceType: s.Type(),
 					SourceName: s.name,
 					SourceID:   s.SourceID(),
+					JobID:      s.JobID(),
 					Data:       file.Bytes(),
 					SourceMetadata: &source_metadatapb.MetaData{
 						Data: &source_metadatapb.MetaData_Docker{

pkg/sources/filesystem/filesystem.go

@@ -161,6 +161,7 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
 		SourceType: s.Type(),
 		SourceName: s.name,
 		SourceID:   s.SourceID(),
+		JobID:      s.JobID(),
 		SourceMetadata: &source_metadatapb.MetaData{
 			Data: &source_metadatapb.MetaData_Filesystem{
 				Filesystem: &source_metadatapb.Filesystem{
@@ -191,6 +192,7 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
 			SourceType: s.Type(),
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
+			JobID:      s.JobID(),
 			Data:       data.Bytes(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Filesystem{

pkg/sources/gcs/gcs.go

@@ -324,6 +324,7 @@ func (s *Source) processObject(ctx context.Context, o object) error {
 	chunkSkel := &sources.Chunk{
 		SourceName: s.name,
 		SourceType: s.Type(),
+		JobID:      s.JobID(),
 		SourceID:   s.sourceId,
 		Verify:     s.verify,
 		SourceMetadata: &source_metadatapb.MetaData{

pkg/sources/git/git.go

@@ -446,6 +446,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 				chunkSkel := &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
+					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Verify:         s.verify,
@@ -464,6 +465,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 			chunksChan <- &sources.Chunk{
 				SourceName:     s.sourceName,
 				SourceID:       s.sourceID,
+				JobID:          s.jobID,
 				SourceType:     s.sourceType,
 				SourceMetadata: metadata,
 				Data:           diff.Content.Bytes(),
@@ -490,6 +492,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 				chunksChan <- &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
+					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Data:           append([]byte{}, newChunkBuffer.Bytes()...),
@@ -504,6 +507,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 				chunksChan <- &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
+					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Data:           line,
@@ -523,6 +527,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 		chunksChan <- &sources.Chunk{
 			SourceName:     s.sourceName,
 			SourceID:       s.sourceID,
+			JobID:          s.jobID,
 			SourceType:     s.sourceType,
 			SourceMetadata: metadata,
 			Data:           append([]byte{}, newChunkBuffer.Bytes()...),
@@ -588,6 +593,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 				chunkSkel := &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
+					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Verify:         s.verify,
@@ -602,6 +608,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 			chunksChan <- &sources.Chunk{
 				SourceName:     s.sourceName,
 				SourceID:       s.sourceID,
+				JobID:          s.jobID,
 				SourceType:     s.sourceType,
 				SourceMetadata: metadata,
 				Data:           diff.Content.Bytes(),

pkg/sources/github/github.go

@@ -149,14 +149,16 @@ func (s *Source) newFilteredRepoCache(c cache.Cache, include, exclude []string)
 	for _, ig := range include {
 		g, err := glob.Compile(ig)
 		if err != nil {
-			s.log.V(1).Info("invalid include glob", "glob", g, "err", err)
+			s.log.V(1).Info("invalid include glob", "include_value", ig, "err", err)
+			continue
 		}
 		includeGlobs = append(includeGlobs, g)
 	}
 	for _, eg := range exclude {
 		g, err := glob.Compile(eg)
 		if err != nil {
-			s.log.V(1).Info("invalid exclude glob", "glob", g, "err", err)
+			s.log.V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
+			continue
 		}
 		excludeGlobs = append(excludeGlobs, g)
 	}
@@ -1223,6 +1225,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
 		chunk := &sources.Chunk{
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
+			JobID:      s.JobID(),
 			SourceType: s.Type(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Github{
@@ -1256,6 +1259,7 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
 			SourceType: s.Type(),
+			JobID:      s.JobID(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Github{
 					Github: &source_metadatapb.Github{
@@ -1287,6 +1291,7 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
 			SourceType: s.Type(),
+			JobID:      s.JobID(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Github{
 					Github: &source_metadatapb.Github{

pkg/sources/job_progress.go

@@ -71,11 +71,11 @@ func (r *JobProgressRef) Done() <-chan struct{} {
 // CancelRun requests that the job this is referencing is cancelled and stops
 // running. This method will have no effect if the job does not allow
 // cancellation.
-func (r *JobProgressRef) CancelRun() {
+func (r *JobProgressRef) CancelRun(cause error) {
 	if r.jobProgress == nil || r.jobProgress.jobCancel == nil {
 		return
 	}
-	r.jobProgress.jobCancel()
+	r.jobProgress.jobCancel(cause)
 }
 
 // Fatal is a wrapper around error to differentiate non-fatal errors from fatal
@@ -108,7 +108,7 @@ type JobProgress struct {
 	ctx    context.Context
 	cancel context.CancelFunc
 	// Requests to cancel the job.
-	jobCancel context.CancelFunc
+	jobCancel context.CancelCauseFunc
 	// Metrics.
 	metrics     JobProgressMetrics
 	metricsLock sync.Mutex
@@ -150,7 +150,7 @@ func WithHooks(hooks ...JobProgressHook) func(*JobProgress) {
 }
 
 // WithCancel allows cancelling the job by the JobProgressRef.
-func WithCancel(cancel context.CancelFunc) func(*JobProgress) {
+func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) {
 	return func(jp *JobProgress) { jp.jobCancel = cancel }
 }
 

pkg/sources/s3/s3.go

@@ -104,22 +104,34 @@ func (s *Source) setMaxObjectSize(maxObjectSize int64) {
 	}
 }
 
-func (s *Source) newClient(region string) (*s3.S3, error) {
+func (s *Source) newClient(region, roleArn string) (*s3.S3, error) {
 	cfg := aws.NewConfig()
 	cfg.CredentialsChainVerboseErrors = aws.Bool(true)
 	cfg.Region = aws.String(region)
 
-	switch cred := s.conn.GetCredential().(type) {
-	case *sourcespb.S3_SessionToken:
-		cfg.Credentials = credentials.NewStaticCredentials(cred.SessionToken.Key, cred.SessionToken.Secret, cred.SessionToken.SessionToken)
-	case *sourcespb.S3_AccessKey:
-		cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")
-	case *sourcespb.S3_Unauthenticated:
-		cfg.Credentials = credentials.AnonymousCredentials
-	case *sourcespb.S3_CloudEnvironment:
-		// Nothing needs to be done!
-	default:
-		return nil, errors.Errorf("invalid configuration given for %s source", s.name)
+	if roleArn == "" {
+		switch cred := s.conn.GetCredential().(type) {
+		case *sourcespb.S3_SessionToken:
+			cfg.Credentials = credentials.NewStaticCredentials(cred.SessionToken.Key, cred.SessionToken.Secret, cred.SessionToken.SessionToken)
+		case *sourcespb.S3_AccessKey:
+			cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")
+		case *sourcespb.S3_Unauthenticated:
+			cfg.Credentials = credentials.AnonymousCredentials
+		case *sourcespb.S3_CloudEnvironment:
+			// Nothing needs to be done!
+		default:
+			return nil, errors.Errorf("invalid configuration given for %s source", s.name)
+		}
+	} else {
+		sess, err := session.NewSession(cfg)
+		if err != nil {
+			return nil, err
+		}
+
+		stsClient := sts.New(sess)
+		cfg.Credentials = stscreds.NewCredentialsWithClient(stsClient, roleArn, func(p *stscreds.AssumeRoleProvider) {
+			p.RoleSessionName = "trufflehog"
+		})
 	}
 
 	sess, err := session.NewSessionWithOptions(session.Options{
@@ -133,40 +145,6 @@ func (s *Source) newClient(region string) (*s3.S3, error) {
 	return s3.New(sess), nil
 }
 
-// Separate role assumption functionality into a different newClient function
-func (s *Source) newRoleClient(region string, roleArn string) (*s3.S3, error) {
-
-	cfg := aws.NewConfig()
-	cfg.CredentialsChainVerboseErrors = aws.Bool(true)
-	cfg.Region = aws.String(region)
-
-	var baseCredentials *credentials.Credentials
-
-	sess, err := session.NewSession(cfg)
-	if err != nil {
-		return nil, err
-	}
-
-	stsClient := sts.New(sess)
-	baseCredentials = stscreds.NewCredentialsWithClient(stsClient, roleArn, func(p *stscreds.AssumeRoleProvider) {
-		p.RoleSessionName = "trufflehog"
-	})
-
-	cfg.Credentials = baseCredentials
-
-	sess, err = session.NewSessionWithOptions(session.Options{
-		SharedConfigState: session.SharedConfigEnable,
-		Config:            *cfg,
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	// Create and return the S3 client with the final configuration
-	return s3.New(sess), nil
-}
-
 // IAM identity needs s3:ListBuckets permission
 func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
 	if len(s.conn.Buckets) > 0 {
@@ -204,13 +182,10 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
 
 		var regionalClient *s3.S3
 		if region != defaultAWSRegion {
-			if role != "" {
-				regionalClient, err = s.newRoleClient(region, role)
-			} else {
-				regionalClient, err = s.newClient(region)
-			}
+			regionalClient, err = s.newClient(region, role)
 			if err != nil {
 				s.log.Error(err, "could not make regional s3 client")
+				continue
 			}
 		} else {
 			regionalClient = client
@@ -238,24 +213,13 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	const defaultAWSRegion = "us-east-1"
 
-	if len(s.conn.Roles) > 0 {
-		for _, role := range s.conn.Roles {
-			client, err := s.newRoleClient(defaultAWSRegion, role)
-			if err != nil {
-				return errors.WrapPrefix(err, "could not create s3 client", 0)
-			}
+	roles := s.conn.Roles
+	if len(roles) == 0 {
+		roles = []string{""}
+	}
 
-			bucketsToScan, err := s.getBucketsToScan(client)
-			if err != nil {
-				return err
-			}
-
-			if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil {
-				return err
-			}
-		}
-	} else {
-		client, err := s.newClient(defaultAWSRegion)
+	for _, role := range roles {
+		client, err := s.newClient(defaultAWSRegion, role)
 		if err != nil {
 			return errors.WrapPrefix(err, "could not create s3 client", 0)
 		}
@@ -265,7 +229,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			return err
 		}
 
-		if err := s.scanBuckets(ctx, client, "", bucketsToScan, chunksChan); err != nil {
+		if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil {
 			return err
 		}
 	}
@@ -376,6 +340,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 				SourceType: s.Type(),
 				SourceName: s.name,
 				SourceID:   s.SourceID(),
+				JobID:      s.JobID(),
 				SourceMetadata: &source_metadatapb.MetaData{
 					Data: &source_metadatapb.MetaData_S3{
 						S3: &source_metadatapb.S3{

pkg/sources/source_manager.go

@@ -26,6 +26,7 @@ type SourceInitFunc func(ctx context.Context, jobID, sourceID int64) (Source, er
 // initialization.
 type sourceInfo struct {
 	initFunc SourceInitFunc
+	sourceID int64
 	name     string
 }
 
@@ -36,7 +37,10 @@ type SourceManager struct {
 	handles     map[handle]sourceInfo
 	handlesLock sync.Mutex
 	// Pool limiting the amount of concurrent sources running.
-	pool            errgroup.Group
+	pool                errgroup.Group
+	poolLimit           int
+	currentRunningCount int32
+	// Max number of units to scan concurrently per source.
 	concurrentUnits int
 	// Run the sources using source unit enumeration / chunking if available.
 	useSourceUnits bool
@@ -68,7 +72,10 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {
 
 // WithConcurrentSources limits the concurrent number of sources a manager can run.
 func WithConcurrentSources(concurrency int) func(*SourceManager) {
-	return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
+	return func(mgr *SourceManager) {
+		mgr.pool.SetLimit(concurrency)
+		mgr.poolLimit = concurrency
+	}
 }
 
 // WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
@@ -120,6 +127,7 @@ func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.
 	}
 	s.handles[handleID] = sourceInfo{
 		initFunc: f,
+		sourceID: int64(handleID),
 		name:     name,
 	}
 	return handleID, nil
@@ -128,7 +136,18 @@ func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.
 // Run blocks until a resource is available to run the source, then
 // synchronously runs it. The first fatal error, if any, will be returned.
 func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) {
-	progress, err := s.asyncRun(ctx, handle)
+	progress, err := s.ScheduleRun(ctx, handle)
+	if err != nil {
+		return progress, err
+	}
+	<-progress.Done()
+	return progress, progress.Snapshot().FatalError()
+}
+
+// RunOnce blocks until a resource is available to run the source, then
+// synchronously runs it. The first fatal error, if any, will be returned.
+func (s *SourceManager) RunOnce(ctx context.Context, name string, source Source) (JobProgressRef, error) {
+	progress, err := s.ScheduleRunOnce(ctx, name, source)
 	if err != nil {
 		return progress, err
 	}
@@ -143,15 +162,22 @@ func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProg
 	return s.asyncRun(ctx, handle)
 }
 
+// ScheduleRunOnce blocks until a resource is available to run the source, then
+// asynchronously runs it. Error information is stored and accessible via the
+// JobProgressRef as it becomes available.
+func (s *SourceManager) ScheduleRunOnce(ctx context.Context, name string, source Source) (JobProgressRef, error) {
+	return s.asyncRunOnce(ctx, name, source)
+}
+
 // asyncRun is a helper method to asynchronously run the Source. It calls out
 // to the API to get a job ID for this run, creates a JobProgress object, then
 // waits for an available goroutine to asynchronously run it.
 func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) {
 	// Do preflight checks before waiting on the pool.
-	if err := s.preflightChecks(ctx, handle); err != nil {
+	if err := s.preflightChecks(ctx); err != nil {
 		return JobProgressRef{}, err
 	}
-	// Get the name. Should never fail due to preflight checks.
+	// Get the name.
 	sourceInfo, ok := s.getSourceInfo(handle)
 	if !ok {
 		return JobProgressRef{SourceID: int64(handle)}, fmt.Errorf("unrecognized handle")
@@ -164,16 +190,47 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
 		return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err
 	}
 	// Create a JobProgress object for tracking progress.
-	ctx, cancel := context.WithCancel(ctx)
+	ctx, cancel := context.WithCancelCause(ctx)
 	progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
+	s.pool.Go(func() error {
+		atomic.AddInt32(&s.currentRunningCount, 1)
+		defer atomic.AddInt32(&s.currentRunningCount, -1)
+		ctx := context.WithValues(ctx,
+			"job_id", jobID,
+			"source_manager_worker_id", common.RandomID(5),
+		)
+		defer common.Recover(ctx)
+		defer cancel(nil)
+		return s.run(ctx, sourceInfo, jobID, progress)
+	})
+	return progress.Ref(), nil
+}
+
+// asyncRunOnce is a helper method to asynchronously run an already initialized
+// Source once. Unlike asyncRun, this method doesn't call out to the API for a
+// job ID since it's already available in the Source.
+func (s *SourceManager) asyncRunOnce(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
+	if err := s.preflightChecks(ctx); err != nil {
+		return JobProgressRef{}, err
+	}
+	ctx = context.WithValue(ctx, "source_id", source.SourceID())
+	jobID := source.JobID()
+	// Create a JobProgress object for tracking progress.
+	ctx, cancel := context.WithCancelCause(ctx)
+	progress := NewJobProgress(jobID, source.SourceID(), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
 	s.pool.Go(func() error {
 		ctx := context.WithValues(ctx,
 			"job_id", jobID,
 			"source_manager_worker_id", common.RandomID(5),
 		)
 		defer common.Recover(ctx)
-		defer cancel()
-		return s.run(ctx, handle, jobID, progress)
+		defer cancel(nil)
+		sourceInfo := sourceInfo{
+			initFunc: func(context.Context, int64, int64) (Source, error) { return source, nil },
+			sourceID: source.SourceID(),
+			name:     sourceName,
+		}
+		return s.run(ctx, sourceInfo, jobID, progress)
 	})
 	return progress.Ref(), nil
 }
@@ -207,37 +264,41 @@ func (s *SourceManager) ScanChunk(chunk *Chunk) {
 	s.outputChunks <- chunk
 }
 
-// preflightChecks is a helper method to check the Manager or the context isn't
-// done and that the handle is valid.
-func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
+// AvailableCapacity returns the number of concurrent jobs the manager can
+// accommodate at this time. If there is no limit, -1 is returned.
+func (s *SourceManager) AvailableCapacity() int {
+	if s.poolLimit == 0 {
+		return -1
+	}
+	runCount := atomic.LoadInt32(&s.currentRunningCount)
+	return s.poolLimit - int(runCount)
+}
+
+// preflightChecks is a helper method to check the Manager or the context isn't done.
+func (s *SourceManager) preflightChecks(ctx context.Context) error {
 	// Check if the manager has been Waited.
 	if s.done {
 		return fmt.Errorf("manager is done")
 	}
-	// Check the handle is valid.
-	if _, ok := s.getSourceInfo(handle); !ok {
-		return fmt.Errorf("unrecognized handle")
-	}
 	return ctx.Err()
 }
 
 // run is a helper method to sychronously run the source. It does not check for
 // acquired resources. An error is returned if there was a fatal error during
 // the run. This information is also recorded in the JobProgress.
-func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, report *JobProgress) error {
+func (s *SourceManager) run(ctx context.Context, sourceInfo sourceInfo, jobID int64, report *JobProgress) error {
 	defer report.Finish()
 	report.Start(time.Now())
 	defer func() { report.End(time.Now()) }()
 
+	defer func() {
+		if err := context.Cause(ctx); err != nil {
+			report.ReportError(Fatal{err})
+		}
+	}()
+
 	// Initialize the source.
-	sourceInfo, ok := s.getSourceInfo(handle)
-	if !ok {
-		// Shouldn't happen due to preflight checks.
-		err := fmt.Errorf("unrecognized handle")
-		report.ReportError(Fatal{err})
-		return Fatal{err}
-	}
-	source, err := sourceInfo.initFunc(ctx, jobID, int64(handle))
+	source, err := sourceInfo.initFunc(ctx, jobID, sourceInfo.sourceID)
 	if err != nil {
 		report.ReportError(Fatal{err})
 		return Fatal{err}
@@ -249,14 +310,14 @@ func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, rep
 	)
 	// Check for the preferred method of tracking source units.
 	if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {
-		return s.runWithUnits(ctx, handle, enumChunker, report)
+		return s.runWithUnits(ctx, enumChunker, report)
 	}
-	return s.runWithoutUnits(ctx, handle, source, report)
+	return s.runWithoutUnits(ctx, source, report)
 }
 
 // runWithoutUnits is a helper method to run a Source. It has coarse-grained
 // job reporting.
-func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobProgress) error {
+func (s *SourceManager) runWithoutUnits(ctx context.Context, source Source, report *JobProgress) error {
 	// Introspect on the chunks we get from the Chunks method.
 	ch := make(chan *Chunk, 1)
 	var wg sync.WaitGroup
@@ -286,7 +347,7 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, sour
 // runWithUnits is a helper method to run a Source that is also a
 // SourceUnitEnumChunker. This allows better introspection of what is getting
 // scanned and any errors encountered.
-func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobProgress) error {
+func (s *SourceManager) runWithUnits(ctx context.Context, source SourceUnitEnumChunker, report *JobProgress) error {
 	unitReporter := &mgrUnitReporter{
 		unitCh: make(chan SourceUnit, 1),
 		report: report,

pkg/sources/source_manager_test.go

@@ -14,8 +14,9 @@ import (
 
 // DummySource implements Source and is used for testing a SourceManager.
 type DummySource struct {
-	sourceID int64
-	jobID    int64
+	sourceID   int64
+	jobID      int64
+	initCalled bool
 	chunker
 }
 
@@ -23,6 +24,7 @@ func (d *DummySource) Type() sourcespb.SourceType { return 1337 }
 func (d *DummySource) SourceID() int64            { return d.sourceID }
 func (d *DummySource) JobID() int64               { return d.jobID }
 func (d *DummySource) Init(_ context.Context, _ string, jobID, sourceID int64, _ bool, _ *anypb.Any, _ int) error {
+	d.initCalled = true
 	d.sourceID = sourceID
 	d.jobID = jobID
 	return nil
@@ -312,8 +314,49 @@ func TestSourceManagerCancelRun(t *testing.T) {
 	ref, err := mgr.ScheduleRun(context.Background(), handle)
 	assert.NoError(t, err)
 
-	ref.CancelRun()
+	cancelErr := fmt.Errorf("abort! abort!")
+	ref.CancelRun(cancelErr)
 	<-ref.Done()
 	assert.Error(t, ref.Snapshot().FatalError())
 	assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr))
+	assert.True(t, errors.Is(ref.Snapshot().FatalErrors(), cancelErr))
+}
+
+func TestSourceManagerAvailableCapacity(t *testing.T) {
+	mgr := NewManager(WithConcurrentSources(1337))
+	start, end := make(chan struct{}), make(chan struct{})
+	handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
+		start <- struct{}{} // Send start signal.
+		<-end               // Wait for end signal.
+		return nil
+	}})
+	assert.NoError(t, err)
+
+	assert.Equal(t, 1337, mgr.AvailableCapacity())
+	ref, err := mgr.ScheduleRun(context.Background(), handle)
+	assert.NoError(t, err)
+
+	<-start // Wait for start signal.
+	assert.Equal(t, 1336, mgr.AvailableCapacity())
+	end <- struct{}{} // Send end signal.
+	<-ref.Done()      // Wait for the job to finish.
+	assert.Equal(t, 1337, mgr.AvailableCapacity())
+}
+
+func TestSourceManagerRunOnce(t *testing.T) {
+	mgr := NewManager()
+	var chunksCalled bool
+	source := &DummySource{sourceID: 123, jobID: 456, chunker: callbackChunker{
+		func(context.Context, chan *Chunk) error {
+			chunksCalled = true
+			return nil
+		},
+	}}
+	ref, err := mgr.RunOnce(context.Background(), "test", source)
+	assert.NoError(t, err)
+	assert.Equal(t, source.jobID, ref.JobID)
+	assert.Equal(t, source.sourceID, ref.SourceID)
+	assert.Equal(t, "test", ref.SourceName)
+	assert.True(t, chunksCalled)
+	assert.False(t, source.initCalled)
 }

pkg/sources/syslog/syslog.go

@@ -272,6 +272,7 @@ func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksCha
 			SourceName:     s.syslog.sourceName,
 			SourceID:       s.syslog.sourceID,
 			SourceType:     s.syslog.sourceType,
+			JobID:          s.JobID(),
 			SourceMetadata: metadata,
 			Data:           input,
 			Verify:         s.verify,
@@ -313,6 +314,7 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.Packe
 		chunksChan <- &sources.Chunk{
 			SourceName:     s.syslog.sourceName,
 			SourceID:       s.syslog.sourceID,
+			JobID:          s.JobID(),
 			SourceType:     s.syslog.sourceType,
 			SourceMetadata: metadata,
 			Data:           input,