PR 3 of 40

README.md

@@ -32,7 +32,7 @@ Have questions? Feedback? Jump in slack or discord and hang out with us
 
 Join our [Slack Community](https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ)
 
-Join the [Secret Scanning Discord](https://discord.gg/8Hzbrnkr7E)
+Join the [Secret Scanning Discord](https://discord.gg/sydS6AHTUP)
 
 # :tv: Demo
 

pkg/context/context.go

@@ -26,10 +26,9 @@ type Context interface {
 	Logger() logr.Logger
 }
 
-// CancelFunc and CancelCauseFunc are type aliases to allow use as if they are
-// the same types as the standard library variants.
+// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
+// the same types.
 type CancelFunc = context.CancelFunc
-type CancelCauseFunc = context.CancelCauseFunc
 
 // logCtx implements Context.
 type logCtx struct {
@@ -69,16 +68,6 @@ func WithCancel(parent Context) (Context, context.CancelFunc) {
 	return lCtx, cancel
 }
 
-// WithCancelCause returns context.WithCancelCause with the log object propagated.
-func WithCancelCause(parent Context) (Context, context.CancelCauseFunc) {
-	ctx, cancel := context.WithCancelCause(parent)
-	lCtx := logCtx{
-		log:     parent.Logger(),
-		Context: ctx,
-	}
-	return lCtx, cancel
-}
-
 // WithDeadline returns context.WithDeadline with the log object propagated and
 // the deadline added to the structured log values.
 func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
@@ -90,17 +79,6 @@ func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
 	return lCtx, cancel
 }
 
-// WithDeadlineCause returns context.WithDeadlineCause with the log object
-// propagated and the deadline added to the structured log values.
-func WithDeadlineCause(parent Context, d time.Time, cause error) (Context, context.CancelFunc) {
-	ctx, cancel := context.WithDeadlineCause(parent, d, cause)
-	lCtx := logCtx{
-		log:     parent.Logger().WithValues("deadline", d),
-		Context: ctx,
-	}
-	return lCtx, cancel
-}
-
 // WithTimeout returns context.WithTimeout with the log object propagated and
 // the timeout added to the structured log values.
 func WithTimeout(parent Context, timeout time.Duration) (Context, context.CancelFunc) {
@@ -112,22 +90,6 @@ func WithTimeout(parent Context, timeout time.Duration) (Context, context.Cancel
 	return lCtx, cancel
 }
 
-// WithTimeoutCause returns context.WithTimeoutCause with the log object
-// propagated and the timeout added to the structured log values.
-func WithTimeoutCause(parent Context, timeout time.Duration, cause error) (Context, context.CancelFunc) {
-	ctx, cancel := context.WithTimeoutCause(parent, timeout, cause)
-	lCtx := logCtx{
-		log:     parent.Logger().WithValues("timeout", timeout),
-		Context: ctx,
-	}
-	return lCtx, cancel
-}
-
-// Cause returns the context.Cause of the context.
-func Cause(ctx context.Context) error {
-	return context.Cause(ctx)
-}
-
 // WithValue returns context.WithValue with the log object propagated and
 // the value added to the structured log values (if the key is a string).
 func WithValue(parent Context, key, val any) Context {

pkg/context/context_test.go

@@ -3,7 +3,6 @@ package context
 import (
 	"bytes"
 	"context"
-	"fmt"
 	"strings"
 	"testing"
 	"time"
@@ -178,10 +177,3 @@ func TestRace(t *testing.T) {
 	cancel()
 	_ = ctx.Err()
 }
-
-func TestCause(t *testing.T) {
-	ctx, cancel := WithCancelCause(Background())
-	err := fmt.Errorf("oh no")
-	cancel(err)
-	assert.Equal(t, err, Cause(ctx))
-}

pkg/detectors/axonaut/axonaut.go

@@ -1,10 +1,10 @@
 package axonaut
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,37 +13,29 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"axonaut"}) + `\b([a-z0-9]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"axonaut"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("axonaut")}
 }
 
-// FromData will find and optionally verify Axonaut secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Axonaut,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -51,14 +43,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("userApiKey", resMatch)
+			req.Header.Add("userApiKey", string(resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/aylien/aylien.go

@@ -1,10 +1,10 @@
 package aylien
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -26,49 +26,48 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"aylien"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("aylien")}
 }
 
 // FromData will find and optionally verify Aylien secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
+	idMatches := idPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		for _, idMatch := range idMatches {
 			if len(idMatch) != 2 {
 				continue
 			}
-
-			resIdMatch := strings.TrimSpace(idMatch[1])
+			resIdMatch := bytes.TrimSpace(idMatch[1])
+			newSecret := append(resMatch, resIdMatch...)
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Aylien,
-				Raw:          []byte(resMatch),
-				RawV2:        []byte(resMatch + resIdMatch),
+				Raw:          resMatch,
+				RawV2:        newSecret,
 			}
+
 			if verify {
 				req, err := http.NewRequestWithContext(ctx, "GET", "https://api.aylien.com/news/stories", nil)
 				if err != nil {
 					continue
 				}
-				req.Header.Add("X-AYLIEN-NewsAPI-Application-ID", resIdMatch)
-				req.Header.Add("X-AYLIEN-NewsAPI-Application-Key", resMatch)
+				req.Header.Add("X-AYLIEN-NewsAPI-Application-ID", string(resMatch))
+				req.Header.Add("X-AYLIEN-NewsAPI-Application-Key", string(resIdMatch))
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						if detectors.IsKnownFalsePositive(newSecret, detectors.DefaultFalsePositives, true) {
 							continue
 						}
 					}

pkg/detectors/ayrshare/ayrshare.go

@@ -1,11 +1,11 @@
 package ayrshare
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,28 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ayrshare"}) + `\b([A-Z]{7}-[A-Z0-9]{7}-[A-Z0-9]{7}-[A-Z0-9]{7})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"ayrshare"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("ayrshare")}
 }
 
-// FromData will find and optionally verify Ayrshare secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
-
+		resMatch := bytes.TrimSpace(match[1])
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Ayrshare,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,24 +43,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
 				}
 			}
 		}
-
 		results = append(results, s1)
 	}
-
 	return results, nil
 }
 

pkg/detectors/azure/azure.go

@@ -7,7 +7,6 @@ import (
 	"strings"
 
 	"github.com/Azure/go-autorest/autorest/azure/auth"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
@@ -31,35 +30,33 @@ var (
 	tenantIDPat = mustFmtPat("tenant_id", idPatFmt)
 
 	// TODO: support old patterns
-	secretPatFmt    = `(?i)(%s).{0,20}([a-z0-9_\.\-~]{34})`
+	secretPatFmt    = `(?i)(%s).{0,20}([a-z0-9_.\-~]{34})`
 	clientSecretPat = mustFmtPat("client_secret", secretPatFmt)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"azure"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("azure")}
 }
 
 // FromData will find and optionally verify Azure secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
+	clientSecretMatches := clientSecretPat.FindAllSubmatch(data, -1)
 
-	clientSecretMatches := clientSecretPat.FindAllStringSubmatch(dataStr, -1)
 	for _, clientSecret := range clientSecretMatches {
-		tenantIDMatches := tenantIDPat.FindAllStringSubmatch(dataStr, -1)
+		tenantIDMatches := tenantIDPat.FindAllSubmatch(data, -1)
 		for _, tenantID := range tenantIDMatches {
-			clientIDMatches := clientIDPat.FindAllStringSubmatch(dataStr, -1)
+			clientIDMatches := clientIDPat.FindAllSubmatch(data, -1)
 			for _, clientID := range clientIDMatches {
 				s := detectors.Result{
 					DetectorType: detectorspb.DetectorType_Azure,
-					Raw:          []byte(clientSecret[2]),
-					RawV2:        []byte(clientID[2] + clientSecret[2] + tenantID[2]),
-					Redacted:     clientID[2],
+					Raw:          clientSecret[2],
+					RawV2:        append(clientID[2], append(clientSecret[2], tenantID[2]...)...),
+					Redacted:     string(clientID[2]),
 				}
 
 				if verify {
-					cred := auth.NewClientCredentialsConfig(clientID[2], clientSecret[2], tenantID[2])
+					cred := auth.NewClientCredentialsConfig(string(clientID[2]), string(clientSecret[2]), string(tenantID[2]))
 					token, err := cred.ServicePrincipalToken()
 					if err != nil {
 						continue
@@ -71,10 +68,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 
 				if !s.Verified {
-					if detectors.IsKnownFalsePositive(s.Redacted, detectors.DefaultFalsePositives, true) {
+					if detectors.IsKnownFalsePositive([]byte(s.Redacted), detectors.DefaultFalsePositives, true) {
 						continue
 					}
-					if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+					if detectors.IsKnownFalsePositive(s.Raw, detectors.DefaultFalsePositives, true) {
 						continue
 					}
 				}

pkg/detectors/bannerbear/bannerbear.go

@@ -1,11 +1,11 @@
 package bannerbear
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,31 +20,29 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	// Keep the regexp patterns unchanged
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bannerbear"}) + `\b([0-9a-zA-Z]{22}tt)\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"bannerbear"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("bannerbear")}
 }
 
 // FromData will find and optionally verify Bannerbear secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Bannerbear,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -59,7 +57,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/baremetrics/baremetrics.go

@@ -1,11 +1,11 @@
 package baremetrics
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"baremetrics"}) + `\b([a-zA-Z0-9_]{25})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"baremetrics"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("baremetrics")}
 }
 
-// FromData will find and optionally verify Baremetrics secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Baremetrics,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/baseapiio/baseapiio.go

@@ -1,12 +1,11 @@
 package baseapiio
 
 import (
+	"bytes"
 	"context"
 	"fmt"
-	"io"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,37 +14,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"baseapi", "base-api"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"baseapi", "base-api"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("baseapi"), []byte("base-api")}
 }
 
-// FromData will find and optionally verify BaseApiIO secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_BaseApiIO,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -53,24 +45,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
-				bodyBytes, err := io.ReadAll(res.Body)
-				if err != nil {
-					continue
-				}
-				body := string(bodyBytes)
-
-				if strings.Contains(body, "items") {
+				if bytes.Contains(resMatch, []byte("items")) {
 					s1.Verified = true
 				} else {
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
 				}
-
 			}
 		}
 

pkg/detectors/beamer/beamer.go

@@ -1,10 +1,10 @@
 package beamer
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,31 +19,27 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"beamer"}) + `\b([a-zA-Z0-9_+/]{45}=)`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"beamer"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("beamer")}
 }
 
 // FromData will find and optionally verify Beamer secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Beamer,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -51,14 +47,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Beamer-Api-Key", resMatch)
+			req.Header.Add("Beamer-Api-Key", string(resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/beebole/beebole.go

@@ -1,12 +1,12 @@
 package beebole
 
 import (
+	"bytes"
 	"context"
 	b64 "encoding/base64"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,56 +15,50 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"beebole"}) + `\b([0-9a-z]{40})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"beebole"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("beebole")}
 }
 
-// FromData will find and optionally verify Beebole secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Beebole,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			data := fmt.Sprintf("%s:X", resMatch)
+			data := fmt.Sprintf("%s:X", string(resMatch))
 			sEnc := b64.StdEncoding.EncodeToString([]byte(data))
-			payload := strings.NewReader(`{"service": "custom_field.list"}`)
+			payload := bytes.NewReader([]byte(`{"service": "custom_field.list"}`))
 			req, err := http.NewRequestWithContext(ctx, "POST", "https://beebole-apps.com/api/v2", payload)
 			if err != nil {
 				continue
 			}
+
 			req.Header.Add("Content-Type", "application/json")
 			req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
 			res, err := client.Do(req)
+
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/besnappy/besnappy.go

@@ -1,12 +1,12 @@
 package besnappy
 
 import (
+	b "bytes"
 	"context"
 	b64 "encoding/base64"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,37 +15,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"besnappy"}) + `\b([a-f0-9]{64})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"besnappy"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("besnappy")}
 }
 
-// FromData will find and optionally verify Besnappy secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := b.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Besnappy,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 		if verify {
 			data := fmt.Sprintf("%s:x", resMatch)
@@ -61,7 +54,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/besttime/besttime.go

@@ -1,11 +1,11 @@
 package besttime
 
 import (
+	"bytes"
 	"context"
 	"io"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,61 +14,54 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"besttime"}) + `\b([0-9A-Za-z_]{36})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"besttime"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("besttime")}
 }
 
-// FromData will find and optionally verify Besttime secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Besttime,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://besttime.app/api/v1/keys/"+resMatch, nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://besttime.app/api/v1/keys/"+string(resMatch), nil)
 			if err != nil {
 				continue
 			}
+
 			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				bodyBytes, err := io.ReadAll(res.Body)
-				if err != nil {
+			if err != nil {
+				continue
+			}
+			defer res.Body.Close()
+			bodyBytes, err := io.ReadAll(res.Body)
+			if err != nil {
+				continue
+			}
+
+			if bytes.Contains(bodyBytes, []byte(`"status": "OK"`)) {
+				s1.Verified = true
+			} else {
+				if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 					continue
 				}
-				body := string(bodyBytes)
-
-				if strings.Contains(body, `"status": "OK"`) {
-					s1.Verified = true
-				} else {
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-						continue
-					}
-				}
-
 			}
 		}
 

pkg/detectors/billomat/billomat.go

@@ -1,11 +1,11 @@
 package billomat
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,55 +20,49 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"billomat"}) + `\b([0-9a-z]{32})\b`)
 	idPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"billomat"}) + `\b([0-9a-z]{1,})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"billomat"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("billomat")}
 }
 
-// FromData will find and optionally verify Billomat secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
+	idMatches := idPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
+
 		for _, idMatch := range idMatches {
 			if len(idMatch) != 2 {
 				continue
 			}
-			resId := strings.TrimSpace(idMatch[1])
+			resId := bytes.TrimSpace(idMatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Billomat,
-				Raw:          []byte(resMatch),
-				RawV2:        []byte(resMatch + resId),
+				Raw:          resMatch,
+				RawV2:        append(resMatch, resId...),
 			}
 
 			if verify {
-				req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s.billomat.net/api/v2/clients/myself", resId), nil)
+				req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s.billomat.net/api/v2/clients/myself", string(resId)), nil)
 				if err != nil {
 					continue
 				}
 				req.Header.Add("Content-Type", "application/json")
-				req.Header.Add("X-BillomatApiKey", resMatch)
+				req.Header.Add("X-BillomatApiKey", string(resMatch))
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}

pkg/detectors/bitbar/bitbar.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,42 +14,34 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitbar"}) + `\b([0-9a-z]{32})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"bitbar"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("bitbar")}
 }
 
-// FromData will find and optionally verify Bitbar secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := match[1]
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Bitbar,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
-			data := fmt.Sprintf("%s:", resMatch)
-			baseToken := b64.StdEncoding.EncodeToString([]byte(data))
+			data := []byte(fmt.Sprintf("%s:", resMatch))
+			baseToken := b64.StdEncoding.EncodeToString(data)
 			req, err := http.NewRequestWithContext(ctx, "GET", "https://cloud.bitbar.com/api/me", nil)
 			if err != nil {
 				continue
@@ -63,7 +54,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/bitcoinaverage/bitcoinaverage.go

@@ -1,10 +1,10 @@
 package bitcoinaverage
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,45 +19,38 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitcoinaverage"}) + `\b([a-zA-Z0-9]{43})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"bitcoinaverage"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("bitcoinaverage")}
 }
 
-// FromData will find and optionally verify BitcoinAverage secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_BitcoinAverage,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 		if verify {
 			req, err := http.NewRequestWithContext(ctx, "GET", "https://apiv2.bitcoinaverage.com/websocket/get_ticket", nil)
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-ba-key", resMatch)
+			req.Header.Add("x-ba-key", string(resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/bitfinex/bitfinex.go

@@ -1,99 +1,80 @@
 package bitfinex
 
 import (
+	"bytes"
 	"context"
-	"flag"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/bitfinexcom/bitfinex-api-go/v2/rest"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
-
-	// related resource https://medium.com/@Bitfinex/api-development-update-april-65fe52f84124
+	client       = common.SaneHttpClient()
 	apiKeyPat    = regexp.MustCompile(detectors.PrefixRegex([]string{"bitfinex"}) + `\b([A-Za-z0-9_-]{43})\b`)
 	apiSecretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitfinex"}) + `\b([A-Za-z0-9_-]{43})\b`)
 )
 
-var (
-	api = flag.String("api", "https://api-pub.bitfinex.com/v2/", "v2 REST API URL")
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"bitfinex"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("bitfinex")}
 }
 
-// FromData will find and optionally verify Bitfinex secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	apiKeyMatches := apiKeyPat.FindAllStringSubmatch(dataStr, -1)
-	apiSecretMatches := apiSecretPat.FindAllStringSubmatch(dataStr, -1)
+	apiKeyMatches := apiKeyPat.FindAllSubmatch(data, -1)
+	apiSecretMatches := apiSecretPat.FindAllSubmatch(data, -1)
 
 	for _, apiKeyMatch := range apiKeyMatches {
 		if len(apiKeyMatch) != 2 {
 			continue
 		}
-		apiKeyRes := strings.TrimSpace(apiKeyMatch[1])
+		apiKeyRes := bytes.TrimSpace(apiKeyMatch[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Bitfinex,
-			Raw:          []byte(apiKeyRes),
+			Raw:          apiKeyRes,
 		}
 
 		for _, apiSecretMatch := range apiSecretMatches {
 			if len(apiSecretMatch) != 2 {
 				continue
 			}
-			apiSecretRes := strings.TrimSpace(apiSecretMatch[1])
+			apiSecretRes := bytes.TrimSpace(apiSecretMatch[1])
 
-			if apiKeyRes == apiSecretRes {
+			if bytes.Equal(apiKeyRes, apiSecretRes) {
 				continue
 			}
 
 			if verify {
-				// thankfully official golang examples exist but you just need to dig their many repos https://github.com/bitfinexcom/bitfinex-api-go/blob/master/examples/v2/rest-orders/main.go
-				key := apiKeyRes
-				secret := apiSecretRes
-				http.DefaultClient = client // filed https://github.com/bitfinexcom/bitfinex-api-go/issues/238 to improve this
-				c := rest.NewClientWithURL(*api).Credentials(key, secret)
+				key := string(apiKeyRes)
+				secret := string(apiSecretRes)
+				http.DefaultClient = client
+				c := rest.NewClient().Credentials(key, secret)
 
-				isValid := true // assume valid
+				isValid := true
 				_, err = c.Orders.AllHistory()
 				if err != nil {
-					if strings.HasPrefix(err.Error(), "POST https://") { // eg POST https://api-pub.bitfinex.com/v2/auth/r/orders/hist: 500 apikey: digest invalid (10100)
+					if bytes.HasPrefix([]byte(err.Error()), []byte("POST https://")) {
 						isValid = false
-					} else {
-						if detectors.IsKnownFalsePositive(apiKeyRes, detectors.DefaultFalsePositives, true) {
-							continue
-						}
+					} else if detectors.IsKnownFalsePositive(apiKeyRes, detectors.DefaultFalsePositives, true) {
+						continue
 					}
 				}
 
 				s1.Verified = isValid
-				// If there is a valid one, we need to stop iterating now and return the valid result
 				if isValid {
 					break
 				}
 			}
 		}
 
-		// By appending resutls in the outer loop we can reduce false positives if there are multiple
-		// combinations of secrets and IDs found.
 		if len(apiSecretMatches) > 0 {
 			results = append(results, s1)
 		}

pkg/detectors/bitlyaccesstoken/bitlyaccesstoken.go

@@ -1,11 +1,11 @@
 package bitlyaccesstoken
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitly"}) + `\b([a-zA-Z-0-9]{40})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"bitly"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("bitly")}
 }
 
-// FromData will find and optionally verify BitLyAccessToken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_BitLyAccessToken,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -52,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/bitmex/bitmex.go

@@ -1,6 +1,7 @@
 package bitmex
 
 import (
+	"bytes"
 	"context"
 	"crypto/hmac"
 	"crypto/sha256"
@@ -9,7 +10,6 @@ import (
 	"net/url"
 	"regexp"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -19,46 +19,40 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat    = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + `([ \r\n]{1}[0-9a-zA-Z\-\_]{24}[ \r\n]{1})`)
-	secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + `([ \r\n]{1}[0-9a-zA-Z\-\_]{48}[ \r\n]{1})`)
+	keyPat    = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + string([]byte{32, 13, 10}) + `[0-9a-zA-Z\-\_]{24}` + string([]byte{32, 13, 10}))
+	secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + string([]byte{32, 13, 10}) + `[0-9a-zA-Z\-\_]{48}` + string([]byte{32, 13, 10}))
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"bitmex"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("bitmex")}
 }
 
-// FromData will find and optionally verify Bitmex secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
+	secretMatches := secretPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		for _, secretMatch := range secretMatches {
 			if len(secretMatch) != 2 {
 				continue
 			}
-			resSecretMatch := strings.TrimSpace(secretMatch[1])
+			resSecretMatch := bytes.TrimSpace(secretMatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Bitmex,
-				Raw:          []byte(resSecretMatch),
-				RawV2:        []byte(resMatch + resSecretMatch),
+				Raw:          resSecretMatch,
+				RawV2:        append(resMatch, resSecretMatch...),
 			}
 
 			if verify {
@@ -68,14 +62,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				path := "/api/v1/user"
 				payload := url.Values{}
 
-				signature := getBitmexSignature(timestamp, resSecretMatch, action, path, payload.Encode())
+				signature := getBitmexSignature(timestamp, string(resSecretMatch), action, path, payload.Encode())
+
+				req, err := http.NewRequestWithContext(ctx, action, "https://www.bitmex.com"+path, bytes.NewReader([]byte(payload.Encode())))
 
-				req, err := http.NewRequestWithContext(ctx, action, "https://www.bitmex.com"+path, strings.NewReader(payload.Encode()))
 				if err != nil {
 					continue
 				}
 				req.Header.Add("api-expires", timestamp)
-				req.Header.Add("api-key", resMatch)
+				req.Header.Add("api-key", string(resMatch))
 				req.Header.Add("api-signature", signature)
 				res, err := client.Do(req)
 				if err == nil {
@@ -83,7 +78,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}

pkg/detectors/blablabus/blablabus.go

@@ -1,11 +1,11 @@
 package blablabus
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blablabus"}) + `\b([0-9A-Za-z]{22})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"blablabus"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("blablabus")}
 }
 
-// FromData will find and optionally verify Blablabus secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Blablabus,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -53,14 +46,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Token %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/blazemeter/blazemeter.go

@@ -1,11 +1,11 @@
 package blazemeter
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,31 +20,25 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blazemeter", "runscope"}) + `\b([0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"blazemeter", "runscope"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("blazemeter"), []byte("runscope")}
 }
 
-// FromData will find and optionally verify Blazemeter secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Blazemeter,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -53,14 +47,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/blitapp/blitapp.go

@@ -1,10 +1,10 @@
 package blitapp
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,37 +13,30 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blitapp"}) + `\b([a-zA-Z0-9_-]{39})\b`)
 )
 
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"blitapp"}
+func (s Scanner) Keywords() [][]byte {
+	return [][]byte{[]byte("blitapp")}
 }
 
-// FromData will find and optionally verify BlitApp secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllSubmatch(data, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resMatch := bytes.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_BlitApp,
-			Raw:          []byte(resMatch),
+			Raw:          resMatch,
 		}
 
 		if verify {
@@ -51,14 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("API-Key", resMatch)
+			req.Header.Add("API-Key", string(resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/salesforce/salesforce.go

@@ -43,6 +43,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1)
 	tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1)
 
+	fmt.Printf("instanceMatches: %v\n", instanceMatches)
+
 	for _, instance := range instanceMatches {
 		if len(instance) != 1 {
 			continue

pkg/detectors/slackwebhook/slackwebhook.go

@@ -2,7 +2,6 @@ package slackwebhook
 
 import (
 	"context"
-	"fmt"
 	"io"
 	"net/http"
 	"regexp"
@@ -13,15 +12,14 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	defaultClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
+
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(`(https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[A-Za-z0-9]{23,25})`)
 )
@@ -50,12 +48,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-
-			client := s.client
-			if client == nil {
-				client = defaultClient
-			}
-
 			payload := strings.NewReader(`{"text": ""}`)
 			req, err := http.NewRequestWithContext(ctx, "POST", resMatch, payload)
 			if err != nil {
@@ -70,18 +62,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				body := string(bodyBytes)
-
-				defer res.Body.Close()
-
-				if res.StatusCode >= 200 && res.StatusCode < 300 || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
+				if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
 					s1.Verified = true
-				} else if res.StatusCode == 401 || res.StatusCode == 403 {
-					// The secret is determinately not verified (nothing to do)
-				} else {
-					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 				}
-			} else {
-				s1.VerificationError = err
 			}
 		}
 

pkg/detectors/slackwebhook/slackwebhook_test.go

@@ -6,11 +6,10 @@ package slackwebhook
 import (
 	"context"
 	"fmt"
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 	"testing"
 	"time"
 
+	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -33,12 +32,11 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
@@ -54,8 +52,7 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -71,8 +68,7 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "not found",
@@ -82,48 +78,14 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_SlackWebhook,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_SlackWebhook,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
+			want:    nil,
+			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("SlackWebhook.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -132,12 +94,9 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
-				}
+				got[i].Raw = nil
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("SlackWebhook.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/engine/defaults.go

@@ -2,10 +2,6 @@ package engine
 
 import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
@@ -291,7 +287,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getresponse"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getsandbox"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_old"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/githubapp"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab"
@@ -561,6 +556,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
@@ -1536,10 +1532,6 @@ func DefaultDetectors() []detectors.Detector {
 		couchbase.Scanner{},
 		envoyapikey.Scanner{},
 		github_oauth2.Scanner{},
-		snowflake.Scanner{},
-		huggingface.Scanner{},
-		trufflehogenterprise.Scanner{},
-		salesforce.Scanner{},
 	}
 
 }

pkg/handlers/archive.go

@@ -29,8 +29,6 @@ var (
 	maxDepth   = 5
 	maxSize    = 250 * 1024 * 1024 // 20MB
 	maxTimeout = time.Duration(30) * time.Second
-
-	defaultBufferSize = 512
 )
 
 // Ensure the Archive satisfies the interfaces at compile time.
@@ -64,7 +62,7 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
 
 // FromFile extracts the files from an archive.
 func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
-	archiveChan := make(chan []byte, defaultBufferSize)
+	archiveChan := make(chan []byte, 512)
 	go func() {
 		ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
 		logger := logContext.AddLogger(ctx).Logger()
@@ -185,28 +183,29 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
 			logger.Error(err, "Panic occurred when reading archive")
 		}
 	}()
-
-	if common.IsDone(ctx) {
-		return nil, ctx.Err()
+	fileContent := bytes.Buffer{}
+	logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
+	for i := 0; i <= maxSize/512; i++ {
+		if common.IsDone(ctx) {
+			return nil, ctx.Err()
+		}
+		fileChunk := make([]byte, 512)
+		bRead, err := reader.Read(fileChunk)
+		if err != nil && !errors.Is(err, io.EOF) {
+			return []byte{}, err
+		}
+		a.size += bRead
+		if len(fileChunk) > 0 {
+			fileContent.Write(fileChunk[0:bRead])
+		}
+		if bRead < 512 {
+			return fileContent.Bytes(), nil
+		}
+		if a.size >= maxSize && bRead == 512 {
+			logger.V(2).Info("Max archive size reached.")
+			return fileContent.Bytes(), nil
+		}
 	}
-
-	var fileContent bytes.Buffer
-	// Create a limited reader to ensure we don't read more than the max size.
-	lr := io.LimitReader(reader, int64(maxSize))
-
-	// Using io.CopyBuffer for performance advantages. Though buf is mandatory
-	// for the method, due to the internal implementation of io.CopyBuffer, when
-	// *bytes.Buffer implements io.WriterTo or io.ReaderFrom, the provided buf
-	// is simply ignored. Thus, we can pass nil for the buf parameter.
-	_, err = io.CopyBuffer(&fileContent, lr, nil)
-	if err != nil && !errors.Is(err, io.EOF) {
-		return nil, err
-	}
-
-	if fileContent.Len() == maxSize {
-		logger.V(2).Info("Max archive size reached.")
-	}
-
 	return fileContent.Bytes(), nil
 }
 

pkg/handlers/archive_test.go

@@ -1,7 +1,6 @@
 package handlers
 
 import (
-	"bytes"
 	"context"
 	"io"
 	"net/http"
@@ -127,58 +126,6 @@ func TestHandleFile(t *testing.T) {
 	assert.Equal(t, 1, len(ch))
 }
 
-func TestReadToMax(t *testing.T) {
-	tests := []struct {
-		name     string
-		input    []byte
-		expected []byte
-	}{
-		{
-			name:     "read full content within maxSize",
-			input:    []byte("abcdefg"),
-			expected: []byte("abcdefg"),
-		},
-		{
-			name:     "read content larger than maxSize",
-			input:    make([]byte, maxSize+10), // this creates a byte slice 10 bytes larger than maxSize
-			expected: make([]byte, maxSize),
-		},
-		{
-			name:     "empty input",
-			input:    []byte(""),
-			expected: []byte(""),
-		},
-	}
-
-	a := &Archive{}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			reader := bytes.NewReader(tt.input)
-			output, err := a.ReadToMax(context.Background(), reader)
-			assert.Nil(t, err)
-
-			assert.Equal(t, tt.expected, output)
-		})
-	}
-}
-
-func BenchmarkReadToMax(b *testing.B) {
-	data := bytes.Repeat([]byte("a"), 1024*1000) // 1MB of data.
-	reader := bytes.NewReader(data)
-	a := &Archive{}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		b.StartTimer()
-		_, _ = a.ReadToMax(context.Background(), reader)
-		b.StopTimer()
-
-		_, _ = reader.Seek(0, 0) // Reset the reader position.
-		a.size = 0               // Reset archive size.
-	}
-}
-
 func TestExtractDebContent(t *testing.T) {
 	// Open the sample .deb file from the testdata folder.
 	file, err := os.Open("testdata/test.deb")

pkg/output/legacy_json.go

@@ -133,7 +133,7 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
 		}
 		headCommit, err := repo.CommitObject(*headHash)
 		if err != nil {
-			logger.Error(err, "unable to get commit", "head_hash", headHash.String())
+			logger.Error(err, "unable to get commit", "commit", headCommit.String())
 			return nil
 		}
 		branches[branchName] = headCommit

pkg/sources/chunker.go

@@ -20,7 +20,7 @@ const (
 
 // Chunker takes a chunk and splits it into chunks of ChunkSize.
 func Chunker(originalChunk *Chunk) chan *Chunk {
-	chunkChan := make(chan *Chunk, 1)
+	chunkChan := make(chan *Chunk)
 	go func() {
 		defer close(chunkChan)
 		if len(originalChunk.Data) <= TotalChunkSize {

pkg/sources/circleci/circleci.go

@@ -238,7 +238,6 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
 			SourceType: s.Type(),
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
-			JobID:      s.JobID(),
 			Data:       removeCircleSha1Line(data.Bytes()),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Circleci{

pkg/sources/docker/docker.go

@@ -174,7 +174,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 					SourceType: s.Type(),
 					SourceName: s.name,
 					SourceID:   s.SourceID(),
-					JobID:      s.JobID(),
 					Data:       file.Bytes(),
 					SourceMetadata: &source_metadatapb.MetaData{
 						Data: &source_metadatapb.MetaData_Docker{

pkg/sources/filesystem/filesystem.go

@@ -161,7 +161,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
 		SourceType: s.Type(),
 		SourceName: s.name,
 		SourceID:   s.SourceID(),
-		JobID:      s.JobID(),
 		SourceMetadata: &source_metadatapb.MetaData{
 			Data: &source_metadatapb.MetaData_Filesystem{
 				Filesystem: &source_metadatapb.Filesystem{
@@ -192,7 +191,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
 			SourceType: s.Type(),
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
-			JobID:      s.JobID(),
 			Data:       data.Bytes(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Filesystem{

pkg/sources/gcs/gcs.go

@@ -324,7 +324,6 @@ func (s *Source) processObject(ctx context.Context, o object) error {
 	chunkSkel := &sources.Chunk{
 		SourceName: s.name,
 		SourceType: s.Type(),
-		JobID:      s.JobID(),
 		SourceID:   s.sourceId,
 		Verify:     s.verify,
 		SourceMetadata: &source_metadatapb.MetaData{

pkg/sources/git/git.go

@@ -446,7 +446,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 				chunkSkel := &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
-					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Verify:         s.verify,
@@ -465,7 +464,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 			chunksChan <- &sources.Chunk{
 				SourceName:     s.sourceName,
 				SourceID:       s.sourceID,
-				JobID:          s.jobID,
 				SourceType:     s.sourceType,
 				SourceMetadata: metadata,
 				Data:           diff.Content.Bytes(),
@@ -492,7 +490,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 				chunksChan <- &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
-					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Data:           append([]byte{}, newChunkBuffer.Bytes()...),
@@ -507,7 +504,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 				chunksChan <- &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
-					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Data:           line,
@@ -527,7 +523,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 		chunksChan <- &sources.Chunk{
 			SourceName:     s.sourceName,
 			SourceID:       s.sourceID,
-			JobID:          s.jobID,
 			SourceType:     s.sourceType,
 			SourceMetadata: metadata,
 			Data:           append([]byte{}, newChunkBuffer.Bytes()...),
@@ -593,7 +588,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 				chunkSkel := &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
-					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Verify:         s.verify,
@@ -608,7 +602,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 			chunksChan <- &sources.Chunk{
 				SourceName:     s.sourceName,
 				SourceID:       s.sourceID,
-				JobID:          s.jobID,
 				SourceType:     s.sourceType,
 				SourceMetadata: metadata,
 				Data:           diff.Content.Bytes(),

pkg/sources/github/github.go

@@ -149,16 +149,14 @@ func (s *Source) newFilteredRepoCache(c cache.Cache, include, exclude []string)
 	for _, ig := range include {
 		g, err := glob.Compile(ig)
 		if err != nil {
-			s.log.V(1).Info("invalid include glob", "include_value", ig, "err", err)
-			continue
+			s.log.V(1).Info("invalid include glob", "glob", g, "err", err)
 		}
 		includeGlobs = append(includeGlobs, g)
 	}
 	for _, eg := range exclude {
 		g, err := glob.Compile(eg)
 		if err != nil {
-			s.log.V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
-			continue
+			s.log.V(1).Info("invalid exclude glob", "glob", g, "err", err)
 		}
 		excludeGlobs = append(excludeGlobs, g)
 	}
@@ -1225,7 +1223,6 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
 		chunk := &sources.Chunk{
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
-			JobID:      s.JobID(),
 			SourceType: s.Type(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Github{
@@ -1259,7 +1256,6 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
 			SourceType: s.Type(),
-			JobID:      s.JobID(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Github{
 					Github: &source_metadatapb.Github{
@@ -1291,7 +1287,6 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
 			SourceType: s.Type(),
-			JobID:      s.JobID(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Github{
 					Github: &source_metadatapb.Github{

pkg/sources/job_progress.go

@@ -71,11 +71,11 @@ func (r *JobProgressRef) Done() <-chan struct{} {
 // CancelRun requests that the job this is referencing is cancelled and stops
 // running. This method will have no effect if the job does not allow
 // cancellation.
-func (r *JobProgressRef) CancelRun(cause error) {
+func (r *JobProgressRef) CancelRun() {
 	if r.jobProgress == nil || r.jobProgress.jobCancel == nil {
 		return
 	}
-	r.jobProgress.jobCancel(cause)
+	r.jobProgress.jobCancel()
 }
 
 // Fatal is a wrapper around error to differentiate non-fatal errors from fatal
@@ -108,7 +108,7 @@ type JobProgress struct {
 	ctx    context.Context
 	cancel context.CancelFunc
 	// Requests to cancel the job.
-	jobCancel context.CancelCauseFunc
+	jobCancel context.CancelFunc
 	// Metrics.
 	metrics     JobProgressMetrics
 	metricsLock sync.Mutex
@@ -150,7 +150,7 @@ func WithHooks(hooks ...JobProgressHook) func(*JobProgress) {
 }
 
 // WithCancel allows cancelling the job by the JobProgressRef.
-func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) {
+func WithCancel(cancel context.CancelFunc) func(*JobProgress) {
 	return func(jp *JobProgress) { jp.jobCancel = cancel }
 }
 

pkg/sources/s3/s3.go

@@ -104,34 +104,22 @@ func (s *Source) setMaxObjectSize(maxObjectSize int64) {
 	}
 }
 
-func (s *Source) newClient(region, roleArn string) (*s3.S3, error) {
+func (s *Source) newClient(region string) (*s3.S3, error) {
 	cfg := aws.NewConfig()
 	cfg.CredentialsChainVerboseErrors = aws.Bool(true)
 	cfg.Region = aws.String(region)
 
-	if roleArn == "" {
-		switch cred := s.conn.GetCredential().(type) {
-		case *sourcespb.S3_SessionToken:
-			cfg.Credentials = credentials.NewStaticCredentials(cred.SessionToken.Key, cred.SessionToken.Secret, cred.SessionToken.SessionToken)
-		case *sourcespb.S3_AccessKey:
-			cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")
-		case *sourcespb.S3_Unauthenticated:
-			cfg.Credentials = credentials.AnonymousCredentials
-		case *sourcespb.S3_CloudEnvironment:
-			// Nothing needs to be done!
-		default:
-			return nil, errors.Errorf("invalid configuration given for %s source", s.name)
-		}
-	} else {
-		sess, err := session.NewSession(cfg)
-		if err != nil {
-			return nil, err
-		}
-
-		stsClient := sts.New(sess)
-		cfg.Credentials = stscreds.NewCredentialsWithClient(stsClient, roleArn, func(p *stscreds.AssumeRoleProvider) {
-			p.RoleSessionName = "trufflehog"
-		})
+	switch cred := s.conn.GetCredential().(type) {
+	case *sourcespb.S3_SessionToken:
+		cfg.Credentials = credentials.NewStaticCredentials(cred.SessionToken.Key, cred.SessionToken.Secret, cred.SessionToken.SessionToken)
+	case *sourcespb.S3_AccessKey:
+		cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")
+	case *sourcespb.S3_Unauthenticated:
+		cfg.Credentials = credentials.AnonymousCredentials
+	case *sourcespb.S3_CloudEnvironment:
+		// Nothing needs to be done!
+	default:
+		return nil, errors.Errorf("invalid configuration given for %s source", s.name)
 	}
 
 	sess, err := session.NewSessionWithOptions(session.Options{
@@ -145,6 +133,40 @@ func (s *Source) newClient(region, roleArn string) (*s3.S3, error) {
 	return s3.New(sess), nil
 }
 
+// Separate role assumption functionality into a different newClient function
+func (s *Source) newRoleClient(region string, roleArn string) (*s3.S3, error) {
+
+	cfg := aws.NewConfig()
+	cfg.CredentialsChainVerboseErrors = aws.Bool(true)
+	cfg.Region = aws.String(region)
+
+	var baseCredentials *credentials.Credentials
+
+	sess, err := session.NewSession(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	stsClient := sts.New(sess)
+	baseCredentials = stscreds.NewCredentialsWithClient(stsClient, roleArn, func(p *stscreds.AssumeRoleProvider) {
+		p.RoleSessionName = "trufflehog"
+	})
+
+	cfg.Credentials = baseCredentials
+
+	sess, err = session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+		Config:            *cfg,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Create and return the S3 client with the final configuration
+	return s3.New(sess), nil
+}
+
 // IAM identity needs s3:ListBuckets permission
 func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
 	if len(s.conn.Buckets) > 0 {
@@ -182,10 +204,13 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
 
 		var regionalClient *s3.S3
 		if region != defaultAWSRegion {
-			regionalClient, err = s.newClient(region, role)
+			if role != "" {
+				regionalClient, err = s.newRoleClient(region, role)
+			} else {
+				regionalClient, err = s.newClient(region)
+			}
 			if err != nil {
 				s.log.Error(err, "could not make regional s3 client")
-				continue
 			}
 		} else {
 			regionalClient = client
@@ -213,13 +238,24 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	const defaultAWSRegion = "us-east-1"
 
-	roles := s.conn.Roles
-	if len(roles) == 0 {
-		roles = []string{""}
-	}
+	if len(s.conn.Roles) > 0 {
+		for _, role := range s.conn.Roles {
+			client, err := s.newRoleClient(defaultAWSRegion, role)
+			if err != nil {
+				return errors.WrapPrefix(err, "could not create s3 client", 0)
+			}
 
-	for _, role := range roles {
-		client, err := s.newClient(defaultAWSRegion, role)
+			bucketsToScan, err := s.getBucketsToScan(client)
+			if err != nil {
+				return err
+			}
+
+			if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil {
+				return err
+			}
+		}
+	} else {
+		client, err := s.newClient(defaultAWSRegion)
 		if err != nil {
 			return errors.WrapPrefix(err, "could not create s3 client", 0)
 		}
@@ -229,7 +265,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			return err
 		}
 
-		if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil {
+		if err := s.scanBuckets(ctx, client, "", bucketsToScan, chunksChan); err != nil {
 			return err
 		}
 	}
@@ -340,7 +376,6 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 				SourceType: s.Type(),
 				SourceName: s.name,
 				SourceID:   s.SourceID(),
-				JobID:      s.JobID(),
 				SourceMetadata: &source_metadatapb.MetaData{
 					Data: &source_metadatapb.MetaData_S3{
 						S3: &source_metadatapb.S3{

pkg/sources/source_manager.go

@@ -26,7 +26,6 @@ type SourceInitFunc func(ctx context.Context, jobID, sourceID int64) (Source, er
 // initialization.
 type sourceInfo struct {
 	initFunc SourceInitFunc
-	sourceID int64
 	name     string
 }
 
@@ -37,10 +36,7 @@ type SourceManager struct {
 	handles     map[handle]sourceInfo
 	handlesLock sync.Mutex
 	// Pool limiting the amount of concurrent sources running.
-	pool                errgroup.Group
-	poolLimit           int
-	currentRunningCount int32
-	// Max number of units to scan concurrently per source.
+	pool            errgroup.Group
 	concurrentUnits int
 	// Run the sources using source unit enumeration / chunking if available.
 	useSourceUnits bool
@@ -72,10 +68,7 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {
 
 // WithConcurrentSources limits the concurrent number of sources a manager can run.
 func WithConcurrentSources(concurrency int) func(*SourceManager) {
-	return func(mgr *SourceManager) {
-		mgr.pool.SetLimit(concurrency)
-		mgr.poolLimit = concurrency
-	}
+	return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
 }
 
 // WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
@@ -127,7 +120,6 @@ func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.
 	}
 	s.handles[handleID] = sourceInfo{
 		initFunc: f,
-		sourceID: int64(handleID),
 		name:     name,
 	}
 	return handleID, nil
@@ -136,18 +128,7 @@ func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.
 // Run blocks until a resource is available to run the source, then
 // synchronously runs it. The first fatal error, if any, will be returned.
 func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) {
-	progress, err := s.ScheduleRun(ctx, handle)
-	if err != nil {
-		return progress, err
-	}
-	<-progress.Done()
-	return progress, progress.Snapshot().FatalError()
-}
-
-// RunOnce blocks until a resource is available to run the source, then
-// synchronously runs it. The first fatal error, if any, will be returned.
-func (s *SourceManager) RunOnce(ctx context.Context, name string, source Source) (JobProgressRef, error) {
-	progress, err := s.ScheduleRunOnce(ctx, name, source)
+	progress, err := s.asyncRun(ctx, handle)
 	if err != nil {
 		return progress, err
 	}
@@ -162,22 +143,15 @@ func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProg
 	return s.asyncRun(ctx, handle)
 }
 
-// ScheduleRunOnce blocks until a resource is available to run the source, then
-// asynchronously runs it. Error information is stored and accessible via the
-// JobProgressRef as it becomes available.
-func (s *SourceManager) ScheduleRunOnce(ctx context.Context, name string, source Source) (JobProgressRef, error) {
-	return s.asyncRunOnce(ctx, name, source)
-}
-
 // asyncRun is a helper method to asynchronously run the Source. It calls out
 // to the API to get a job ID for this run, creates a JobProgress object, then
 // waits for an available goroutine to asynchronously run it.
 func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) {
 	// Do preflight checks before waiting on the pool.
-	if err := s.preflightChecks(ctx); err != nil {
+	if err := s.preflightChecks(ctx, handle); err != nil {
 		return JobProgressRef{}, err
 	}
-	// Get the name.
+	// Get the name. Should never fail due to preflight checks.
 	sourceInfo, ok := s.getSourceInfo(handle)
 	if !ok {
 		return JobProgressRef{SourceID: int64(handle)}, fmt.Errorf("unrecognized handle")
@@ -190,47 +164,16 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
 		return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err
 	}
 	// Create a JobProgress object for tracking progress.
-	ctx, cancel := context.WithCancelCause(ctx)
+	ctx, cancel := context.WithCancel(ctx)
 	progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
 	s.pool.Go(func() error {
-		atomic.AddInt32(&s.currentRunningCount, 1)
-		defer atomic.AddInt32(&s.currentRunningCount, -1)
 		ctx := context.WithValues(ctx,
 			"job_id", jobID,
 			"source_manager_worker_id", common.RandomID(5),
 		)
 		defer common.Recover(ctx)
-		defer cancel(nil)
-		return s.run(ctx, sourceInfo, jobID, progress)
-	})
-	return progress.Ref(), nil
-}
-
-// asyncRunOnce is a helper method to asynchronously run an already initialized
-// Source once. Unlike asyncRun, this method doesn't call out to the API for a
-// job ID since it's already available in the Source.
-func (s *SourceManager) asyncRunOnce(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
-	if err := s.preflightChecks(ctx); err != nil {
-		return JobProgressRef{}, err
-	}
-	ctx = context.WithValue(ctx, "source_id", source.SourceID())
-	jobID := source.JobID()
-	// Create a JobProgress object for tracking progress.
-	ctx, cancel := context.WithCancelCause(ctx)
-	progress := NewJobProgress(jobID, source.SourceID(), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
-	s.pool.Go(func() error {
-		ctx := context.WithValues(ctx,
-			"job_id", jobID,
-			"source_manager_worker_id", common.RandomID(5),
-		)
-		defer common.Recover(ctx)
-		defer cancel(nil)
-		sourceInfo := sourceInfo{
-			initFunc: func(context.Context, int64, int64) (Source, error) { return source, nil },
-			sourceID: source.SourceID(),
-			name:     sourceName,
-		}
-		return s.run(ctx, sourceInfo, jobID, progress)
+		defer cancel()
+		return s.run(ctx, handle, jobID, progress)
 	})
 	return progress.Ref(), nil
 }
@@ -264,41 +207,37 @@ func (s *SourceManager) ScanChunk(chunk *Chunk) {
 	s.outputChunks <- chunk
 }
 
-// AvailableCapacity returns the number of concurrent jobs the manager can
-// accommodate at this time. If there is no limit, -1 is returned.
-func (s *SourceManager) AvailableCapacity() int {
-	if s.poolLimit == 0 {
-		return -1
-	}
-	runCount := atomic.LoadInt32(&s.currentRunningCount)
-	return s.poolLimit - int(runCount)
-}
-
-// preflightChecks is a helper method to check the Manager or the context isn't done.
-func (s *SourceManager) preflightChecks(ctx context.Context) error {
+// preflightChecks is a helper method to check the Manager or the context isn't
+// done and that the handle is valid.
+func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
 	// Check if the manager has been Waited.
 	if s.done {
 		return fmt.Errorf("manager is done")
 	}
+	// Check the handle is valid.
+	if _, ok := s.getSourceInfo(handle); !ok {
+		return fmt.Errorf("unrecognized handle")
+	}
 	return ctx.Err()
 }
 
 // run is a helper method to sychronously run the source. It does not check for
 // acquired resources. An error is returned if there was a fatal error during
 // the run. This information is also recorded in the JobProgress.
-func (s *SourceManager) run(ctx context.Context, sourceInfo sourceInfo, jobID int64, report *JobProgress) error {
+func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, report *JobProgress) error {
 	defer report.Finish()
 	report.Start(time.Now())
 	defer func() { report.End(time.Now()) }()
 
-	defer func() {
-		if err := context.Cause(ctx); err != nil {
-			report.ReportError(Fatal{err})
-		}
-	}()
-
 	// Initialize the source.
-	source, err := sourceInfo.initFunc(ctx, jobID, sourceInfo.sourceID)
+	sourceInfo, ok := s.getSourceInfo(handle)
+	if !ok {
+		// Shouldn't happen due to preflight checks.
+		err := fmt.Errorf("unrecognized handle")
+		report.ReportError(Fatal{err})
+		return Fatal{err}
+	}
+	source, err := sourceInfo.initFunc(ctx, jobID, int64(handle))
 	if err != nil {
 		report.ReportError(Fatal{err})
 		return Fatal{err}
@@ -310,14 +249,14 @@ func (s *SourceManager) run(ctx context.Context, sourceInfo sourceInfo, jobID in
 	)
 	// Check for the preferred method of tracking source units.
 	if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {
-		return s.runWithUnits(ctx, enumChunker, report)
+		return s.runWithUnits(ctx, handle, enumChunker, report)
 	}
-	return s.runWithoutUnits(ctx, source, report)
+	return s.runWithoutUnits(ctx, handle, source, report)
 }
 
 // runWithoutUnits is a helper method to run a Source. It has coarse-grained
 // job reporting.
-func (s *SourceManager) runWithoutUnits(ctx context.Context, source Source, report *JobProgress) error {
+func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobProgress) error {
 	// Introspect on the chunks we get from the Chunks method.
 	ch := make(chan *Chunk, 1)
 	var wg sync.WaitGroup
@@ -347,7 +286,7 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, source Source, repo
 // runWithUnits is a helper method to run a Source that is also a
 // SourceUnitEnumChunker. This allows better introspection of what is getting
 // scanned and any errors encountered.
-func (s *SourceManager) runWithUnits(ctx context.Context, source SourceUnitEnumChunker, report *JobProgress) error {
+func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobProgress) error {
 	unitReporter := &mgrUnitReporter{
 		unitCh: make(chan SourceUnit, 1),
 		report: report,

pkg/sources/source_manager_test.go

@@ -14,9 +14,8 @@ import (
 
 // DummySource implements Source and is used for testing a SourceManager.
 type DummySource struct {
-	sourceID   int64
-	jobID      int64
-	initCalled bool
+	sourceID int64
+	jobID    int64
 	chunker
 }
 
@@ -24,7 +23,6 @@ func (d *DummySource) Type() sourcespb.SourceType { return 1337 }
 func (d *DummySource) SourceID() int64            { return d.sourceID }
 func (d *DummySource) JobID() int64               { return d.jobID }
 func (d *DummySource) Init(_ context.Context, _ string, jobID, sourceID int64, _ bool, _ *anypb.Any, _ int) error {
-	d.initCalled = true
 	d.sourceID = sourceID
 	d.jobID = jobID
 	return nil
@@ -314,49 +312,8 @@ func TestSourceManagerCancelRun(t *testing.T) {
 	ref, err := mgr.ScheduleRun(context.Background(), handle)
 	assert.NoError(t, err)
 
-	cancelErr := fmt.Errorf("abort! abort!")
-	ref.CancelRun(cancelErr)
+	ref.CancelRun()
 	<-ref.Done()
 	assert.Error(t, ref.Snapshot().FatalError())
 	assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr))
-	assert.True(t, errors.Is(ref.Snapshot().FatalErrors(), cancelErr))
-}
-
-func TestSourceManagerAvailableCapacity(t *testing.T) {
-	mgr := NewManager(WithConcurrentSources(1337))
-	start, end := make(chan struct{}), make(chan struct{})
-	handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
-		start <- struct{}{} // Send start signal.
-		<-end               // Wait for end signal.
-		return nil
-	}})
-	assert.NoError(t, err)
-
-	assert.Equal(t, 1337, mgr.AvailableCapacity())
-	ref, err := mgr.ScheduleRun(context.Background(), handle)
-	assert.NoError(t, err)
-
-	<-start // Wait for start signal.
-	assert.Equal(t, 1336, mgr.AvailableCapacity())
-	end <- struct{}{} // Send end signal.
-	<-ref.Done()      // Wait for the job to finish.
-	assert.Equal(t, 1337, mgr.AvailableCapacity())
-}
-
-func TestSourceManagerRunOnce(t *testing.T) {
-	mgr := NewManager()
-	var chunksCalled bool
-	source := &DummySource{sourceID: 123, jobID: 456, chunker: callbackChunker{
-		func(context.Context, chan *Chunk) error {
-			chunksCalled = true
-			return nil
-		},
-	}}
-	ref, err := mgr.RunOnce(context.Background(), "test", source)
-	assert.NoError(t, err)
-	assert.Equal(t, source.jobID, ref.JobID)
-	assert.Equal(t, source.sourceID, ref.SourceID)
-	assert.Equal(t, "test", ref.SourceName)
-	assert.True(t, chunksCalled)
-	assert.False(t, source.initCalled)
 }

pkg/sources/syslog/syslog.go

@@ -272,7 +272,6 @@ func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksCha
 			SourceName:     s.syslog.sourceName,
 			SourceID:       s.syslog.sourceID,
 			SourceType:     s.syslog.sourceType,
-			JobID:          s.JobID(),
 			SourceMetadata: metadata,
 			Data:           input,
 			Verify:         s.verify,
@@ -314,7 +313,6 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.Packe
 		chunksChan <- &sources.Chunk{
 			SourceName:     s.syslog.sourceName,
 			SourceID:       s.syslog.sourceID,
-			JobID:          s.JobID(),
 			SourceType:     s.syslog.sourceType,
 			SourceMetadata: metadata,
 			Data:           input,