Add RunOnce and ScheduleRunOnce to SourceManager

This allows the manager to run an already initialized source.

README.md

@@ -32,7 +32,7 @@ Have questions? Feedback? Jump in slack or discord and hang out with us
 
 Join our [Slack Community](https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ)
 
-Join the [Secret Scanning Discord](https://discord.gg/sydS6AHTUP)
+Join the [Secret Scanning Discord](https://discord.gg/8Hzbrnkr7E)
 
 # :tv: Demo
 

pkg/context/context.go

@@ -26,9 +26,10 @@ type Context interface {
 	Logger() logr.Logger
 }
 
-// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
-// the same types.
+// CancelFunc and CancelCauseFunc are type aliases to allow use as if they are
+// the same types as the standard library variants.
 type CancelFunc = context.CancelFunc
+type CancelCauseFunc = context.CancelCauseFunc
 
 // logCtx implements Context.
 type logCtx struct {
@@ -68,6 +69,16 @@ func WithCancel(parent Context) (Context, context.CancelFunc) {
 	return lCtx, cancel
 }
 
+// WithCancelCause returns context.WithCancelCause with the log object propagated.
+func WithCancelCause(parent Context) (Context, context.CancelCauseFunc) {
+	ctx, cancel := context.WithCancelCause(parent)
+	lCtx := logCtx{
+		log:     parent.Logger(),
+		Context: ctx,
+	}
+	return lCtx, cancel
+}
+
 // WithDeadline returns context.WithDeadline with the log object propagated and
 // the deadline added to the structured log values.
 func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
@@ -79,6 +90,17 @@ func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
 	return lCtx, cancel
 }
 
+// WithDeadlineCause returns context.WithDeadlineCause with the log object
+// propagated and the deadline added to the structured log values.
+func WithDeadlineCause(parent Context, d time.Time, cause error) (Context, context.CancelFunc) {
+	ctx, cancel := context.WithDeadlineCause(parent, d, cause)
+	lCtx := logCtx{
+		log:     parent.Logger().WithValues("deadline", d),
+		Context: ctx,
+	}
+	return lCtx, cancel
+}
+
 // WithTimeout returns context.WithTimeout with the log object propagated and
 // the timeout added to the structured log values.
 func WithTimeout(parent Context, timeout time.Duration) (Context, context.CancelFunc) {
@@ -90,6 +112,22 @@ func WithTimeout(parent Context, timeout time.Duration) (Context, context.Cancel
 	return lCtx, cancel
 }
 
+// WithTimeoutCause returns context.WithTimeoutCause with the log object
+// propagated and the timeout added to the structured log values.
+func WithTimeoutCause(parent Context, timeout time.Duration, cause error) (Context, context.CancelFunc) {
+	ctx, cancel := context.WithTimeoutCause(parent, timeout, cause)
+	lCtx := logCtx{
+		log:     parent.Logger().WithValues("timeout", timeout),
+		Context: ctx,
+	}
+	return lCtx, cancel
+}
+
+// Cause returns the context.Cause of the context.
+func Cause(ctx context.Context) error {
+	return context.Cause(ctx)
+}
+
 // WithValue returns context.WithValue with the log object propagated and
 // the value added to the structured log values (if the key is a string).
 func WithValue(parent Context, key, val any) Context {

pkg/context/context_test.go

@@ -3,6 +3,7 @@ package context
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"strings"
 	"testing"
 	"time"
@@ -177,3 +178,10 @@ func TestRace(t *testing.T) {
 	cancel()
 	_ = ctx.Err()
 }
+
+func TestCause(t *testing.T) {
+	ctx, cancel := WithCancelCause(Background())
+	err := fmt.Errorf("oh no")
+	cancel(err)
+	assert.Equal(t, err, Cause(ctx))
+}

pkg/detectors/oopspam/oopspam.go

@@ -1,10 +1,10 @@
 package oopspam
 
 import (
-	"bytes"
 	"context"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,49 +13,54 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"oopspam"}) + `\b([a-zA-Z0-9]{40})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("oopspam")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"oopspam"}
 }
 
+// FromData will find and optionally verify OOPSpam secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_OOPSpam,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			payload := bytes.NewBuffer([]byte(`{"checkForLength":true,"content":"Dear Agent, We are a manufacturing company which specializes in supplying Aluminum Rod with Zinc Alloy Rod to customers worldwide, based in Japan, Asia. We have been unable to follow up payments effectively for transactions with debtor customers in your country due to our distant locations, thus our reason for requesting for your services representation.","senderIP":"185.234.219.246","allowedCountries":["it","us"],"allowedLanguages":["en"]}`))
+			payload := strings.NewReader(`{"checkForLength":true,"content":"Dear Agent, We are a manufacturing company which specializes in supplying Aluminum Rod with Zinc Alloy Rod to customers worldwide, based in Japan, Asia. We have been unable to follow up payments effectively for transactions with debtor customers in your country due to our distant locations, thus our reason for requesting for your services representation.","senderIP":"185.234.219.246","allowedCountries":["it","us"],"allowedLanguages":["en"]}`)
 			req, err := http.NewRequestWithContext(ctx, "POST", "https://api.oopspam.com/v1/spamdetection", payload)
 			if err != nil {
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("X-Api-Key", string(resMatch))
-
+			req.Header.Add("X-Api-Key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
-
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/openai/openai.go

@@ -1,7 +1,6 @@
 package openai
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -20,8 +19,10 @@ type Scanner struct{}
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
-var keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"T3BlbkFJ"}) + `(\bsk-[[:alnum:]]{20}T3BlbkFJ[[:alnum:]]{20}\b)`)
+// The magic string T3BlbkFJ is the base64-encoded string: OpenAI
+var keyPat = regexp.MustCompile(`\b(sk-[[:alnum:]]{20}T3BlbkFJ[[:alnum:]]{20})\b`)
 
+// TODO: Add secret context?? Information about access, ownership etc
 type orgResponse struct {
 	Data []organization `json:"data"`
 }
@@ -36,44 +37,48 @@ type organization struct {
 	Role        string `json:"role"`
 }
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("T3BlbkFJ")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"T3BlbkFJ"}
 }
 
+// FromData will find and optionally verify OpenAI secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
+		// First match is entire regex, second is the first group.
 		if len(match) != 2 {
 			continue
 		}
 
-		token := bytes.TrimSpace(match[1])
-
-		redacted := append(append(token[:3], []byte("...")...), token[47:]...)
+		token := match[1]
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_OpenAI,
-			Redacted:     string(redacted),
-			Raw:          token,
+			Redacted:     token[:3] + "..." + token[47:],
+			Raw:          []byte(token),
 		}
 
 		if verify {
 			client := common.SaneHttpClient()
+			// Undocumented API
+			// https://api.openai.com/v1/organizations
 			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.openai.com/v1/organizations", nil)
 			if err != nil {
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json; charset=utf-8")
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(token)))
-
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
 			res, err := client.Do(req)
-
 			if err == nil {
-				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					var orgs orgResponse
 					err = json.NewDecoder(res.Body).Decode(&orgs)
+					res.Body.Close()
 					if err == nil {
 						s1.Verified = true
 						org := orgs.Data[0]
@@ -92,7 +97,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 		}
 
-		if !s1.Verified && detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {
+		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
 			continue
 		}
 

pkg/detectors/opencagedata/opencagedata.go

@@ -1,11 +1,11 @@
 package opencagedata
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -21,34 +21,34 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"opencagedata"}) + `\b([0-9a-z]{32})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"opencagedata"}) + `\b([a-z0-9]{32})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("opencagedata")}
+func (s Scanner) Keywords() []string {
+	return []string{"opencagedata"}
 }
 
 // FromData will find and optionally verify OpenCageData secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
 
-	matches := keyPat.FindAllSubmatch(data, -1)
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_OpenCageData,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.opencagedata.com/geocode/v1/json?q=12.8797,121.7740&key=%s", string(resMatch)), nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.opencagedata.com/geocode/v1/json?q=12.8797,121.7740&key=%s", resMatch), nil)
 			if err != nil {
 				continue
 			}
@@ -58,12 +58,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
 				}
 			}
 		}
+
 		results = append(results, s1)
 	}
 

pkg/detectors/opengraphr/opengraphr.go

@@ -1,10 +1,10 @@
 package opengraphr
 
 import (
-	"bytes"
 	"context"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,34 +13,41 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"opengraphr"}) + `\b([0-9Aa-zA-Z]{80})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("opengraphr")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"opengraphr"}
 }
 
+// FromData will find and optionally verify Opengraphr secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Opengraphr,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opengraphr.com/v1/og?api_token="+string(resMatch)+"&url=https://www.marlonpamisa.com/", nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opengraphr.com/v1/og?api_token="+resMatch+"&url=https://www.marlonpamisa.com/", nil)
 			if err != nil {
 				continue
 			}
@@ -50,6 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/openuv/openuv.go

@@ -1,10 +1,10 @@
 package openuv
 
 import (
-	"bytes"
 	"context"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,29 +13,37 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"openuv"}) + `\b([0-9a-z]{32})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("openuv")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"openuv"}
 }
 
+// FromData will find and optionally verify Openuv secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Openuv,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -43,13 +51,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-access-token", string(resMatch))
+			req.Header.Add("x-access-token", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/openweather/openweather.go

@@ -1,10 +1,10 @@
 package openweather
 
 import (
-	"bytes"
 	"context"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,33 +13,41 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"openweather"}) + `\b([a-z0-9]{32})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("openweather")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"openweather"}
 }
 
+// FromData will find and optionally verify OpenWeather secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_OpenWeather,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.openweathermap.org/data/2.5/weather?id=524901&=fr&appid="+string(resMatch), nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.openweathermap.org/data/2.5/weather?id=524901&lang=fr&appid="+resMatch, nil)
 			if err != nil {
 				continue
 			}
@@ -50,6 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/opsgenie/opsgenie.go

@@ -1,11 +1,11 @@
 package opsgenie
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,10 +14,13 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"opsgenie"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
 )
 
@@ -25,36 +28,42 @@ func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_Opsgenie
 }
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("opsgenie")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"opsgenie"}
 }
 
+// FromData will find and optionally verify Opsgenie secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Opsgenie,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/users", nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/alerts", nil)
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("GenieKey %s", string(resMatch)))
+			req.Header.Add("Authorization", fmt.Sprintf("GenieKey %s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/optimizely/optimizely.go

@@ -1,11 +1,11 @@
 package optimizely
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -24,23 +24,27 @@ var (
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"optimizely"}) + `\b([0-9A-Za-z-:]{54})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("optimizely")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"optimizely"}
 }
 
+// FromData will find and optionally verify Optimizely secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
 
-	matches := keyPat.FindAllSubmatch(data, -1)
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Optimizely,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -48,19 +52,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
 				}
 			}
-
 		}
 
 		results = append(results, s1)

pkg/detectors/owlbot/owlbot.go

@@ -1,11 +1,11 @@
 package owlbot
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,29 +14,37 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"owlbot"}) + `\b([a-z0-9]{40})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("owlbot")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"owlbot"}
 }
 
+// FromData will find and optionally verify Owlbot secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Owlbot,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -44,13 +52,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Token %s", string(resMatch)))
+			req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/packagecloud/packagecloud.go

@@ -1,10 +1,10 @@
 package packagecloud
 
 import (
-	"bytes"
 	"context"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,29 +13,37 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"packagecloud"}) + `\b([0-9a-f]{48})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("packagecloud")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"packagecloud"}
 }
 
+// FromData will find and optionally verify PackageCloud secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_PackageCloud,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -43,14 +51,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.SetBasicAuth(string(resMatch), "")
+			req.SetBasicAuth(resMatch, "")
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
-					if detectors.IsKnownFalsePositive(match[1], detectors.DefaultFalsePositives, true) {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
 				}

pkg/detectors/pagerdutyapikey/pagerdutyapikey.go

@@ -1,11 +1,11 @@
 package pagerdutyapikey
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,30 +14,37 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty"}) + `\b([a-z]{1}\+[a-zA-Z]{9}\-[a-z]{2}\-[a-z0-9]{5})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("pagerduty")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"pagerduty"}
 }
 
+// FromData will find and optionally verify PagerDutyApiKey secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -58,7 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if !s1.Verified {
-			if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+			if detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
 				continue
 			}
 		}

pkg/detectors/pandadoc/pandadoc.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -25,23 +26,25 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("pandadoc")}
+func (s Scanner) Keywords() []string {
+	return []string{"pandadoc"}
 }
 
 // FromData will find and optionally verify Pandadoc secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := match[1]
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Pandadoc,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -49,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("API-Key %s", string(resMatch)))
+			req.Header.Add("Authorization", fmt.Sprintf("API-Key %s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/pandascore/pandascore.go

@@ -1,11 +1,11 @@
 package pandascore
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,39 +14,46 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pandascore"}) + `([ \r\n]{0,1}[0-9A-Za-z\-\_]{51}[ \r\n]{1})`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("pandascore")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"pandascore"}
 }
 
+// FromData will find and optionally verify PandaScore secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_PandaScore,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.pandascore.co/videogames", nil)
+			req, err := http.NewRequest("GET", "https://api.pandascore.co/videogames", nil)
 			if err != nil {
 				continue
 			}
 			req.Header.Add("Accept", "application/json")
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -54,9 +61,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					s1.Verified = true
 				} else {
 					s1.Verified = false
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-						continue
-					}
 				}
 			}
 		}

pkg/detectors/paperform/paperform.go

@@ -1,11 +1,11 @@
 package paperform
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,26 +20,31 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client = common.SaneHttpClient()
 
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"paperform"}) + `\b([a-z0-9A-Z-._]{850,1000})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("paperform")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"paperform"}
 }
 
+// FromData will find and optionally verify Paperform secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
 
-	matches := keyPat.FindAllSubmatch(data, -1)
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Paperform,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -47,13 +52,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/paralleldots/paralleldots.go

@@ -7,6 +7,7 @@ import (
 	"mime/multipart"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,29 +16,37 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"paralleldots"}) + `\b([0-9A-Za-z]{43})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("paralleldots")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"paralleldots"}
 }
 
+// FromData will find and optionally verify Paralleldots secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_ParallelDots,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -47,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			_, err = io.Copy(fw, bytes.NewReader(resMatch))
+			_, err = io.Copy(fw, strings.NewReader(resMatch))
 			if err != nil {
 				continue
 			}
@@ -55,7 +64,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			_, err = io.Copy(fw, bytes.NewReader([]byte("sample text")))
+			_, err = io.Copy(fw, strings.NewReader("sample text"))
 			if err != nil {
 				continue
 			}
@@ -72,9 +81,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if err != nil {
 					continue
 				}
-				if (res.StatusCode >= 200 && res.StatusCode < 300) && bytes.Contains(bodyBytes, []byte("intent")) {
+				body := string(bodyBytes)
+				if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "intent") {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
@@ -84,6 +95,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 		results = append(results, s1)
 	}
+
 	return results, nil
 }
 

pkg/detectors/parsehub/parsehub.go

@@ -1,11 +1,11 @@
 package parsehub
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,34 +14,41 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"parsehub"}) + `\b([0-9a-zA-Z]{12})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("parsehub")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"parsehub"}
 }
 
+// FromData will find and optionally verify Parsehub secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
 
-	matches := keyPat.FindAllSubmatch(data, -1)
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Parsehub,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://www.parsehub.com/api/v2/projects?api_key=%s&limit=1", string(resMatch)), nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://www.parsehub.com/api/v2/projects?api_key=%s&limit=1", resMatch), nil)
 			if err != nil {
 				continue
 			}
@@ -51,6 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/parsers/parsers.go

@@ -1,10 +1,10 @@
 package parsers
 
 import (
-	"bytes"
 	"context"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,54 +13,64 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
 
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"parsers"}) + `\b([0-9a-z]{64})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("parsers")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"parsers"}
 }
 
+// FromData will find and optionally verify Parsers secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Parsers,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			payload := bytes.NewBuffer([]byte(`{"sql": "CREATE TABLE t();SELECT 1; SELECT 1 FROM ; SELECT 2"}`))
+			payload := strings.NewReader(`{"sql": "CREATE TABLE t();SELECT 1; SELECT 1 FROM ; SELECT 2"}`)
 			req, err := http.NewRequestWithContext(ctx, "POST", "https://api.parsers.dev/api/v1/parse/postgresql/", payload)
 			if err != nil {
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("x-api-key", string(resMatch))
+			req.Header.Add("x-api-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
 				}
 			}
 		}
+
 		results = append(results, s1)
 	}
+
 	return results, nil
 }
 

pkg/detectors/parseur/parseur.go

@@ -1,11 +1,11 @@
 package parseur
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,29 +14,37 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"parseur"}) + `\b([a-f0-9]{40})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("parseur")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"parseur"}
 }
 
+// FromData will find and optionally verify Parseur secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Parseur,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -44,13 +52,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Token %s", string(resMatch)))
+			req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/partnerstack/partnerstack.go

@@ -1,11 +1,11 @@
 package partnerstack
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,30 +14,37 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"partnerstack"}) + `\b([0-9A-Za-z]{64})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("partnerstack")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"partnerstack"}
 }
 
+// FromData will find and optionally verify Partnerstack secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Partnerstack,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -46,21 +53,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Accept", "application/json")
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
 				}
 			}
 		}
+
 		results = append(results, s1)
 	}
+
 	return results, nil
 }
 

pkg/detectors/passbase/passbase.go

@@ -1,10 +1,10 @@
 package passbase
 
 import (
-	"bytes"
 	"context"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,29 +13,37 @@ import (
 
 type Scanner struct{}
 
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"passbase"}) + `\b([a-zA-Z0-9]{128})\b`)
 )
 
-func (s Scanner) Keywords() [][]byte {
-	return [][]byte{[]byte("passbase")}
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"passbase"}
 }
 
+// FromData will find and optionally verify Passbase secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	matches := keyPat.FindAllSubmatch(data, -1)
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := bytes.TrimSpace(match[1])
+		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Passbase,
-			Raw:          resMatch,
+			Raw:          []byte(resMatch),
 		}
 
 		if verify {
@@ -43,13 +51,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-API-KEY", string(resMatch))
+			req.Header.Add("X-API-KEY", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}

pkg/detectors/salesforce/salesforce.go

@@ -43,8 +43,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1)
 	tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1)
 
-	fmt.Printf("instanceMatches: %v\n", instanceMatches)
-
 	for _, instance := range instanceMatches {
 		if len(instance) != 1 {
 			continue

pkg/detectors/slackwebhook/slackwebhook.go

@@ -2,6 +2,7 @@ package slackwebhook
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"net/http"
 	"regexp"
@@ -12,14 +13,15 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
-
+	defaultClient = common.SaneHttpClient()
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(`(https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[A-Za-z0-9]{23,25})`)
 )
@@ -48,6 +50,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
+
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
+
 			payload := strings.NewReader(`{"text": ""}`)
 			req, err := http.NewRequestWithContext(ctx, "POST", resMatch, payload)
 			if err != nil {
@@ -62,9 +70,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				body := string(bodyBytes)
-				if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
+
+				defer res.Body.Close()
+
+				if res.StatusCode >= 200 && res.StatusCode < 300 || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
 					s1.Verified = true
+				} else if res.StatusCode == 401 || res.StatusCode == 403 {
+					// The secret is determinately not verified (nothing to do)
+				} else {
+					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 				}
+			} else {
+				s1.VerificationError = err
 			}
 		}
 

pkg/detectors/slackwebhook/slackwebhook_test.go

@@ -6,10 +6,11 @@ package slackwebhook
 import (
 	"context"
 	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"testing"
 	"time"
 
-	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -32,11 +33,12 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
@@ -52,7 +54,8 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -68,7 +71,8 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "not found",
@@ -78,14 +82,48 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:    nil,
-			wantErr: false,
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SlackWebhook,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SlackWebhook,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("SlackWebhook.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -94,9 +132,12 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				got[i].Raw = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
 				t.Errorf("SlackWebhook.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/engine/defaults.go

@@ -2,6 +2,10 @@ package engine
 
 import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
@@ -287,6 +291,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getresponse"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getsandbox"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_old"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/githubapp"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab"
@@ -556,7 +561,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
@@ -1532,6 +1536,10 @@ func DefaultDetectors() []detectors.Detector {
 		couchbase.Scanner{},
 		envoyapikey.Scanner{},
 		github_oauth2.Scanner{},
+		snowflake.Scanner{},
+		huggingface.Scanner{},
+		trufflehogenterprise.Scanner{},
+		salesforce.Scanner{},
 	}
 
 }

pkg/handlers/archive.go

@@ -29,6 +29,8 @@ var (
 	maxDepth   = 5
 	maxSize    = 250 * 1024 * 1024 // 20MB
 	maxTimeout = time.Duration(30) * time.Second
+
+	defaultBufferSize = 512
 )
 
 // Ensure the Archive satisfies the interfaces at compile time.
@@ -62,7 +64,7 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
 
 // FromFile extracts the files from an archive.
 func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
-	archiveChan := make(chan []byte, 512)
+	archiveChan := make(chan []byte, defaultBufferSize)
 	go func() {
 		ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
 		logger := logContext.AddLogger(ctx).Logger()
@@ -183,29 +185,28 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
 			logger.Error(err, "Panic occurred when reading archive")
 		}
 	}()
-	fileContent := bytes.Buffer{}
-	logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
-	for i := 0; i <= maxSize/512; i++ {
-		if common.IsDone(ctx) {
-			return nil, ctx.Err()
-		}
-		fileChunk := make([]byte, 512)
-		bRead, err := reader.Read(fileChunk)
-		if err != nil && !errors.Is(err, io.EOF) {
-			return []byte{}, err
-		}
-		a.size += bRead
-		if len(fileChunk) > 0 {
-			fileContent.Write(fileChunk[0:bRead])
-		}
-		if bRead < 512 {
-			return fileContent.Bytes(), nil
-		}
-		if a.size >= maxSize && bRead == 512 {
-			logger.V(2).Info("Max archive size reached.")
-			return fileContent.Bytes(), nil
-		}
+
+	if common.IsDone(ctx) {
+		return nil, ctx.Err()
 	}
+
+	var fileContent bytes.Buffer
+	// Create a limited reader to ensure we don't read more than the max size.
+	lr := io.LimitReader(reader, int64(maxSize))
+
+	// Using io.CopyBuffer for performance advantages. Though buf is mandatory
+	// for the method, due to the internal implementation of io.CopyBuffer, when
+	// *bytes.Buffer implements io.WriterTo or io.ReaderFrom, the provided buf
+	// is simply ignored. Thus, we can pass nil for the buf parameter.
+	_, err = io.CopyBuffer(&fileContent, lr, nil)
+	if err != nil && !errors.Is(err, io.EOF) {
+		return nil, err
+	}
+
+	if fileContent.Len() == maxSize {
+		logger.V(2).Info("Max archive size reached.")
+	}
+
 	return fileContent.Bytes(), nil
 }
 

pkg/handlers/archive_test.go

@@ -1,6 +1,7 @@
 package handlers
 
 import (
+	"bytes"
 	"context"
 	"io"
 	"net/http"
@@ -126,6 +127,58 @@ func TestHandleFile(t *testing.T) {
 	assert.Equal(t, 1, len(ch))
 }
 
+func TestReadToMax(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    []byte
+		expected []byte
+	}{
+		{
+			name:     "read full content within maxSize",
+			input:    []byte("abcdefg"),
+			expected: []byte("abcdefg"),
+		},
+		{
+			name:     "read content larger than maxSize",
+			input:    make([]byte, maxSize+10), // this creates a byte slice 10 bytes larger than maxSize
+			expected: make([]byte, maxSize),
+		},
+		{
+			name:     "empty input",
+			input:    []byte(""),
+			expected: []byte(""),
+		},
+	}
+
+	a := &Archive{}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			reader := bytes.NewReader(tt.input)
+			output, err := a.ReadToMax(context.Background(), reader)
+			assert.Nil(t, err)
+
+			assert.Equal(t, tt.expected, output)
+		})
+	}
+}
+
+func BenchmarkReadToMax(b *testing.B) {
+	data := bytes.Repeat([]byte("a"), 1024*1000) // 1MB of data.
+	reader := bytes.NewReader(data)
+	a := &Archive{}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		_, _ = a.ReadToMax(context.Background(), reader)
+		b.StopTimer()
+
+		_, _ = reader.Seek(0, 0) // Reset the reader position.
+		a.size = 0               // Reset archive size.
+	}
+}
+
 func TestExtractDebContent(t *testing.T) {
 	// Open the sample .deb file from the testdata folder.
 	file, err := os.Open("testdata/test.deb")

pkg/output/legacy_json.go

@@ -133,7 +133,7 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
 		}
 		headCommit, err := repo.CommitObject(*headHash)
 		if err != nil {
-			logger.Error(err, "unable to get commit", "commit", headCommit.String())
+			logger.Error(err, "unable to get commit", "head_hash", headHash.String())
 			return nil
 		}
 		branches[branchName] = headCommit

pkg/sources/chunker.go

@@ -20,7 +20,7 @@ const (
 
 // Chunker takes a chunk and splits it into chunks of ChunkSize.
 func Chunker(originalChunk *Chunk) chan *Chunk {
-	chunkChan := make(chan *Chunk)
+	chunkChan := make(chan *Chunk, 1)
 	go func() {
 		defer close(chunkChan)
 		if len(originalChunk.Data) <= TotalChunkSize {

pkg/sources/circleci/circleci.go

@@ -238,6 +238,7 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
 			SourceType: s.Type(),
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
+			JobID:      s.JobID(),
 			Data:       removeCircleSha1Line(data.Bytes()),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Circleci{

pkg/sources/docker/docker.go

@@ -174,6 +174,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 					SourceType: s.Type(),
 					SourceName: s.name,
 					SourceID:   s.SourceID(),
+					JobID:      s.JobID(),
 					Data:       file.Bytes(),
 					SourceMetadata: &source_metadatapb.MetaData{
 						Data: &source_metadatapb.MetaData_Docker{

pkg/sources/filesystem/filesystem.go

@@ -161,6 +161,7 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
 		SourceType: s.Type(),
 		SourceName: s.name,
 		SourceID:   s.SourceID(),
+		JobID:      s.JobID(),
 		SourceMetadata: &source_metadatapb.MetaData{
 			Data: &source_metadatapb.MetaData_Filesystem{
 				Filesystem: &source_metadatapb.Filesystem{
@@ -191,6 +192,7 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
 			SourceType: s.Type(),
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
+			JobID:      s.JobID(),
 			Data:       data.Bytes(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Filesystem{

pkg/sources/gcs/gcs.go

@@ -324,6 +324,7 @@ func (s *Source) processObject(ctx context.Context, o object) error {
 	chunkSkel := &sources.Chunk{
 		SourceName: s.name,
 		SourceType: s.Type(),
+		JobID:      s.JobID(),
 		SourceID:   s.sourceId,
 		Verify:     s.verify,
 		SourceMetadata: &source_metadatapb.MetaData{

pkg/sources/git/git.go

@@ -446,6 +446,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 				chunkSkel := &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
+					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Verify:         s.verify,
@@ -464,6 +465,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 			chunksChan <- &sources.Chunk{
 				SourceName:     s.sourceName,
 				SourceID:       s.sourceID,
+				JobID:          s.jobID,
 				SourceType:     s.sourceType,
 				SourceMetadata: metadata,
 				Data:           diff.Content.Bytes(),
@@ -490,6 +492,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 				chunksChan <- &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
+					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Data:           append([]byte{}, newChunkBuffer.Bytes()...),
@@ -504,6 +507,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 				chunksChan <- &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
+					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Data:           line,
@@ -523,6 +527,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 		chunksChan <- &sources.Chunk{
 			SourceName:     s.sourceName,
 			SourceID:       s.sourceID,
+			JobID:          s.jobID,
 			SourceType:     s.sourceType,
 			SourceMetadata: metadata,
 			Data:           append([]byte{}, newChunkBuffer.Bytes()...),
@@ -588,6 +593,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 				chunkSkel := &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
+					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Verify:         s.verify,
@@ -602,6 +608,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 			chunksChan <- &sources.Chunk{
 				SourceName:     s.sourceName,
 				SourceID:       s.sourceID,
+				JobID:          s.jobID,
 				SourceType:     s.sourceType,
 				SourceMetadata: metadata,
 				Data:           diff.Content.Bytes(),

pkg/sources/github/github.go

@@ -149,14 +149,16 @@ func (s *Source) newFilteredRepoCache(c cache.Cache, include, exclude []string)
 	for _, ig := range include {
 		g, err := glob.Compile(ig)
 		if err != nil {
-			s.log.V(1).Info("invalid include glob", "glob", g, "err", err)
+			s.log.V(1).Info("invalid include glob", "include_value", ig, "err", err)
+			continue
 		}
 		includeGlobs = append(includeGlobs, g)
 	}
 	for _, eg := range exclude {
 		g, err := glob.Compile(eg)
 		if err != nil {
-			s.log.V(1).Info("invalid exclude glob", "glob", g, "err", err)
+			s.log.V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
+			continue
 		}
 		excludeGlobs = append(excludeGlobs, g)
 	}
@@ -1223,6 +1225,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
 		chunk := &sources.Chunk{
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
+			JobID:      s.JobID(),
 			SourceType: s.Type(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Github{
@@ -1256,6 +1259,7 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
 			SourceType: s.Type(),
+			JobID:      s.JobID(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Github{
 					Github: &source_metadatapb.Github{
@@ -1287,6 +1291,7 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
 			SourceType: s.Type(),
+			JobID:      s.JobID(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Github{
 					Github: &source_metadatapb.Github{

pkg/sources/job_progress.go

@@ -71,11 +71,11 @@ func (r *JobProgressRef) Done() <-chan struct{} {
 // CancelRun requests that the job this is referencing is cancelled and stops
 // running. This method will have no effect if the job does not allow
 // cancellation.
-func (r *JobProgressRef) CancelRun() {
+func (r *JobProgressRef) CancelRun(cause error) {
 	if r.jobProgress == nil || r.jobProgress.jobCancel == nil {
 		return
 	}
-	r.jobProgress.jobCancel()
+	r.jobProgress.jobCancel(cause)
 }
 
 // Fatal is a wrapper around error to differentiate non-fatal errors from fatal
@@ -108,7 +108,7 @@ type JobProgress struct {
 	ctx    context.Context
 	cancel context.CancelFunc
 	// Requests to cancel the job.
-	jobCancel context.CancelFunc
+	jobCancel context.CancelCauseFunc
 	// Metrics.
 	metrics     JobProgressMetrics
 	metricsLock sync.Mutex
@@ -150,7 +150,7 @@ func WithHooks(hooks ...JobProgressHook) func(*JobProgress) {
 }
 
 // WithCancel allows cancelling the job by the JobProgressRef.
-func WithCancel(cancel context.CancelFunc) func(*JobProgress) {
+func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) {
 	return func(jp *JobProgress) { jp.jobCancel = cancel }
 }
 

pkg/sources/s3/s3.go

@@ -104,22 +104,34 @@ func (s *Source) setMaxObjectSize(maxObjectSize int64) {
 	}
 }
 
-func (s *Source) newClient(region string) (*s3.S3, error) {
+func (s *Source) newClient(region, roleArn string) (*s3.S3, error) {
 	cfg := aws.NewConfig()
 	cfg.CredentialsChainVerboseErrors = aws.Bool(true)
 	cfg.Region = aws.String(region)
 
-	switch cred := s.conn.GetCredential().(type) {
-	case *sourcespb.S3_SessionToken:
-		cfg.Credentials = credentials.NewStaticCredentials(cred.SessionToken.Key, cred.SessionToken.Secret, cred.SessionToken.SessionToken)
-	case *sourcespb.S3_AccessKey:
-		cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")
-	case *sourcespb.S3_Unauthenticated:
-		cfg.Credentials = credentials.AnonymousCredentials
-	case *sourcespb.S3_CloudEnvironment:
-		// Nothing needs to be done!
-	default:
-		return nil, errors.Errorf("invalid configuration given for %s source", s.name)
+	if roleArn == "" {
+		switch cred := s.conn.GetCredential().(type) {
+		case *sourcespb.S3_SessionToken:
+			cfg.Credentials = credentials.NewStaticCredentials(cred.SessionToken.Key, cred.SessionToken.Secret, cred.SessionToken.SessionToken)
+		case *sourcespb.S3_AccessKey:
+			cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")
+		case *sourcespb.S3_Unauthenticated:
+			cfg.Credentials = credentials.AnonymousCredentials
+		case *sourcespb.S3_CloudEnvironment:
+			// Nothing needs to be done!
+		default:
+			return nil, errors.Errorf("invalid configuration given for %s source", s.name)
+		}
+	} else {
+		sess, err := session.NewSession(cfg)
+		if err != nil {
+			return nil, err
+		}
+
+		stsClient := sts.New(sess)
+		cfg.Credentials = stscreds.NewCredentialsWithClient(stsClient, roleArn, func(p *stscreds.AssumeRoleProvider) {
+			p.RoleSessionName = "trufflehog"
+		})
 	}
 
 	sess, err := session.NewSessionWithOptions(session.Options{
@@ -133,40 +145,6 @@ func (s *Source) newClient(region string) (*s3.S3, error) {
 	return s3.New(sess), nil
 }
 
-// Separate role assumption functionality into a different newClient function
-func (s *Source) newRoleClient(region string, roleArn string) (*s3.S3, error) {
-
-	cfg := aws.NewConfig()
-	cfg.CredentialsChainVerboseErrors = aws.Bool(true)
-	cfg.Region = aws.String(region)
-
-	var baseCredentials *credentials.Credentials
-
-	sess, err := session.NewSession(cfg)
-	if err != nil {
-		return nil, err
-	}
-
-	stsClient := sts.New(sess)
-	baseCredentials = stscreds.NewCredentialsWithClient(stsClient, roleArn, func(p *stscreds.AssumeRoleProvider) {
-		p.RoleSessionName = "trufflehog"
-	})
-
-	cfg.Credentials = baseCredentials
-
-	sess, err = session.NewSessionWithOptions(session.Options{
-		SharedConfigState: session.SharedConfigEnable,
-		Config:            *cfg,
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	// Create and return the S3 client with the final configuration
-	return s3.New(sess), nil
-}
-
 // IAM identity needs s3:ListBuckets permission
 func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
 	if len(s.conn.Buckets) > 0 {
@@ -204,13 +182,10 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
 
 		var regionalClient *s3.S3
 		if region != defaultAWSRegion {
-			if role != "" {
-				regionalClient, err = s.newRoleClient(region, role)
-			} else {
-				regionalClient, err = s.newClient(region)
-			}
+			regionalClient, err = s.newClient(region, role)
 			if err != nil {
 				s.log.Error(err, "could not make regional s3 client")
+				continue
 			}
 		} else {
 			regionalClient = client
@@ -238,24 +213,13 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	const defaultAWSRegion = "us-east-1"
 
-	if len(s.conn.Roles) > 0 {
-		for _, role := range s.conn.Roles {
-			client, err := s.newRoleClient(defaultAWSRegion, role)
-			if err != nil {
-				return errors.WrapPrefix(err, "could not create s3 client", 0)
-			}
+	roles := s.conn.Roles
+	if len(roles) == 0 {
+		roles = []string{""}
+	}
 
-			bucketsToScan, err := s.getBucketsToScan(client)
-			if err != nil {
-				return err
-			}
-
-			if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil {
-				return err
-			}
-		}
-	} else {
-		client, err := s.newClient(defaultAWSRegion)
+	for _, role := range roles {
+		client, err := s.newClient(defaultAWSRegion, role)
 		if err != nil {
 			return errors.WrapPrefix(err, "could not create s3 client", 0)
 		}
@@ -265,7 +229,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			return err
 		}
 
-		if err := s.scanBuckets(ctx, client, "", bucketsToScan, chunksChan); err != nil {
+		if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil {
 			return err
 		}
 	}
@@ -376,6 +340,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 				SourceType: s.Type(),
 				SourceName: s.name,
 				SourceID:   s.SourceID(),
+				JobID:      s.JobID(),
 				SourceMetadata: &source_metadatapb.MetaData{
 					Data: &source_metadatapb.MetaData_S3{
 						S3: &source_metadatapb.S3{

pkg/sources/source_manager.go

@@ -26,6 +26,7 @@ type SourceInitFunc func(ctx context.Context, jobID, sourceID int64) (Source, er
 // initialization.
 type sourceInfo struct {
 	initFunc SourceInitFunc
+	sourceID int64
 	name     string
 }
 
@@ -36,7 +37,10 @@ type SourceManager struct {
 	handles     map[handle]sourceInfo
 	handlesLock sync.Mutex
 	// Pool limiting the amount of concurrent sources running.
-	pool            errgroup.Group
+	pool                errgroup.Group
+	poolLimit           int
+	currentRunningCount int32
+	// Max number of units to scan concurrently per source.
 	concurrentUnits int
 	// Run the sources using source unit enumeration / chunking if available.
 	useSourceUnits bool
@@ -68,7 +72,10 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {
 
 // WithConcurrentSources limits the concurrent number of sources a manager can run.
 func WithConcurrentSources(concurrency int) func(*SourceManager) {
-	return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
+	return func(mgr *SourceManager) {
+		mgr.pool.SetLimit(concurrency)
+		mgr.poolLimit = concurrency
+	}
 }
 
 // WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
@@ -120,6 +127,7 @@ func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.
 	}
 	s.handles[handleID] = sourceInfo{
 		initFunc: f,
+		sourceID: int64(handleID),
 		name:     name,
 	}
 	return handleID, nil
@@ -128,7 +136,18 @@ func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.
 // Run blocks until a resource is available to run the source, then
 // synchronously runs it. The first fatal error, if any, will be returned.
 func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) {
-	progress, err := s.asyncRun(ctx, handle)
+	progress, err := s.ScheduleRun(ctx, handle)
+	if err != nil {
+		return progress, err
+	}
+	<-progress.Done()
+	return progress, progress.Snapshot().FatalError()
+}
+
+// RunOnce blocks until a resource is available to run the source, then
+// synchronously runs it. The first fatal error, if any, will be returned.
+func (s *SourceManager) RunOnce(ctx context.Context, name string, source Source) (JobProgressRef, error) {
+	progress, err := s.ScheduleRunOnce(ctx, name, source)
 	if err != nil {
 		return progress, err
 	}
@@ -143,15 +162,22 @@ func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProg
 	return s.asyncRun(ctx, handle)
 }
 
+// ScheduleRunOnce blocks until a resource is available to run the source, then
+// asynchronously runs it. Error information is stored and accessible via the
+// JobProgressRef as it becomes available.
+func (s *SourceManager) ScheduleRunOnce(ctx context.Context, name string, source Source) (JobProgressRef, error) {
+	return s.asyncRunOnce(ctx, name, source)
+}
+
 // asyncRun is a helper method to asynchronously run the Source. It calls out
 // to the API to get a job ID for this run, creates a JobProgress object, then
 // waits for an available goroutine to asynchronously run it.
 func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) {
 	// Do preflight checks before waiting on the pool.
-	if err := s.preflightChecks(ctx, handle); err != nil {
+	if err := s.preflightChecks(ctx); err != nil {
 		return JobProgressRef{}, err
 	}
-	// Get the name. Should never fail due to preflight checks.
+	// Get the name.
 	sourceInfo, ok := s.getSourceInfo(handle)
 	if !ok {
 		return JobProgressRef{SourceID: int64(handle)}, fmt.Errorf("unrecognized handle")
@@ -164,16 +190,47 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
 		return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err
 	}
 	// Create a JobProgress object for tracking progress.
-	ctx, cancel := context.WithCancel(ctx)
+	ctx, cancel := context.WithCancelCause(ctx)
 	progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
+	s.pool.Go(func() error {
+		atomic.AddInt32(&s.currentRunningCount, 1)
+		defer atomic.AddInt32(&s.currentRunningCount, -1)
+		ctx := context.WithValues(ctx,
+			"job_id", jobID,
+			"source_manager_worker_id", common.RandomID(5),
+		)
+		defer common.Recover(ctx)
+		defer cancel(nil)
+		return s.run(ctx, sourceInfo, jobID, progress)
+	})
+	return progress.Ref(), nil
+}
+
+// asyncRunOnce is a helper method to asynchronously run an already initialized
+// Source once. Unlike asyncRun, this method doesn't call out to the API for a
+// job ID since it's already available in the Source.
+func (s *SourceManager) asyncRunOnce(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
+	if err := s.preflightChecks(ctx); err != nil {
+		return JobProgressRef{}, err
+	}
+	ctx = context.WithValue(ctx, "source_id", source.SourceID())
+	jobID := source.JobID()
+	// Create a JobProgress object for tracking progress.
+	ctx, cancel := context.WithCancelCause(ctx)
+	progress := NewJobProgress(jobID, source.SourceID(), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
 	s.pool.Go(func() error {
 		ctx := context.WithValues(ctx,
 			"job_id", jobID,
 			"source_manager_worker_id", common.RandomID(5),
 		)
 		defer common.Recover(ctx)
-		defer cancel()
-		return s.run(ctx, handle, jobID, progress)
+		defer cancel(nil)
+		sourceInfo := sourceInfo{
+			initFunc: func(context.Context, int64, int64) (Source, error) { return source, nil },
+			sourceID: source.SourceID(),
+			name:     sourceName,
+		}
+		return s.run(ctx, sourceInfo, jobID, progress)
 	})
 	return progress.Ref(), nil
 }
@@ -207,37 +264,41 @@ func (s *SourceManager) ScanChunk(chunk *Chunk) {
 	s.outputChunks <- chunk
 }
 
-// preflightChecks is a helper method to check the Manager or the context isn't
-// done and that the handle is valid.
-func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
+// AvailableCapacity returns the number of concurrent jobs the manager can
+// accommodate at this time. If there is no limit, -1 is returned.
+func (s *SourceManager) AvailableCapacity() int {
+	if s.poolLimit == 0 {
+		return -1
+	}
+	runCount := atomic.LoadInt32(&s.currentRunningCount)
+	return s.poolLimit - int(runCount)
+}
+
+// preflightChecks is a helper method to check the Manager or the context isn't done.
+func (s *SourceManager) preflightChecks(ctx context.Context) error {
 	// Check if the manager has been Waited.
 	if s.done {
 		return fmt.Errorf("manager is done")
 	}
-	// Check the handle is valid.
-	if _, ok := s.getSourceInfo(handle); !ok {
-		return fmt.Errorf("unrecognized handle")
-	}
 	return ctx.Err()
 }
 
 // run is a helper method to sychronously run the source. It does not check for
 // acquired resources. An error is returned if there was a fatal error during
 // the run. This information is also recorded in the JobProgress.
-func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, report *JobProgress) error {
+func (s *SourceManager) run(ctx context.Context, sourceInfo sourceInfo, jobID int64, report *JobProgress) error {
 	defer report.Finish()
 	report.Start(time.Now())
 	defer func() { report.End(time.Now()) }()
 
+	defer func() {
+		if err := context.Cause(ctx); err != nil {
+			report.ReportError(Fatal{err})
+		}
+	}()
+
 	// Initialize the source.
-	sourceInfo, ok := s.getSourceInfo(handle)
-	if !ok {
-		// Shouldn't happen due to preflight checks.
-		err := fmt.Errorf("unrecognized handle")
-		report.ReportError(Fatal{err})
-		return Fatal{err}
-	}
-	source, err := sourceInfo.initFunc(ctx, jobID, int64(handle))
+	source, err := sourceInfo.initFunc(ctx, jobID, sourceInfo.sourceID)
 	if err != nil {
 		report.ReportError(Fatal{err})
 		return Fatal{err}
@@ -249,14 +310,14 @@ func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, rep
 	)
 	// Check for the preferred method of tracking source units.
 	if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {
-		return s.runWithUnits(ctx, handle, enumChunker, report)
+		return s.runWithUnits(ctx, enumChunker, report)
 	}
-	return s.runWithoutUnits(ctx, handle, source, report)
+	return s.runWithoutUnits(ctx, source, report)
 }
 
 // runWithoutUnits is a helper method to run a Source. It has coarse-grained
 // job reporting.
-func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobProgress) error {
+func (s *SourceManager) runWithoutUnits(ctx context.Context, source Source, report *JobProgress) error {
 	// Introspect on the chunks we get from the Chunks method.
 	ch := make(chan *Chunk, 1)
 	var wg sync.WaitGroup
@@ -286,7 +347,7 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, sour
 // runWithUnits is a helper method to run a Source that is also a
 // SourceUnitEnumChunker. This allows better introspection of what is getting
 // scanned and any errors encountered.
-func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobProgress) error {
+func (s *SourceManager) runWithUnits(ctx context.Context, source SourceUnitEnumChunker, report *JobProgress) error {
 	unitReporter := &mgrUnitReporter{
 		unitCh: make(chan SourceUnit, 1),
 		report: report,

pkg/sources/source_manager_test.go

@@ -14,8 +14,9 @@ import (
 
 // DummySource implements Source and is used for testing a SourceManager.
 type DummySource struct {
-	sourceID int64
-	jobID    int64
+	sourceID   int64
+	jobID      int64
+	initCalled bool
 	chunker
 }
 
@@ -23,6 +24,7 @@ func (d *DummySource) Type() sourcespb.SourceType { return 1337 }
 func (d *DummySource) SourceID() int64            { return d.sourceID }
 func (d *DummySource) JobID() int64               { return d.jobID }
 func (d *DummySource) Init(_ context.Context, _ string, jobID, sourceID int64, _ bool, _ *anypb.Any, _ int) error {
+	d.initCalled = true
 	d.sourceID = sourceID
 	d.jobID = jobID
 	return nil
@@ -312,8 +314,49 @@ func TestSourceManagerCancelRun(t *testing.T) {
 	ref, err := mgr.ScheduleRun(context.Background(), handle)
 	assert.NoError(t, err)
 
-	ref.CancelRun()
+	cancelErr := fmt.Errorf("abort! abort!")
+	ref.CancelRun(cancelErr)
 	<-ref.Done()
 	assert.Error(t, ref.Snapshot().FatalError())
 	assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr))
+	assert.True(t, errors.Is(ref.Snapshot().FatalErrors(), cancelErr))
+}
+
+func TestSourceManagerAvailableCapacity(t *testing.T) {
+	mgr := NewManager(WithConcurrentSources(1337))
+	start, end := make(chan struct{}), make(chan struct{})
+	handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
+		start <- struct{}{} // Send start signal.
+		<-end               // Wait for end signal.
+		return nil
+	}})
+	assert.NoError(t, err)
+
+	assert.Equal(t, 1337, mgr.AvailableCapacity())
+	ref, err := mgr.ScheduleRun(context.Background(), handle)
+	assert.NoError(t, err)
+
+	<-start // Wait for start signal.
+	assert.Equal(t, 1336, mgr.AvailableCapacity())
+	end <- struct{}{} // Send end signal.
+	<-ref.Done()      // Wait for the job to finish.
+	assert.Equal(t, 1337, mgr.AvailableCapacity())
+}
+
+func TestSourceManagerRunOnce(t *testing.T) {
+	mgr := NewManager()
+	var chunksCalled bool
+	source := &DummySource{sourceID: 123, jobID: 456, chunker: callbackChunker{
+		func(context.Context, chan *Chunk) error {
+			chunksCalled = true
+			return nil
+		},
+	}}
+	ref, err := mgr.RunOnce(context.Background(), "test", source)
+	assert.NoError(t, err)
+	assert.Equal(t, source.jobID, ref.JobID)
+	assert.Equal(t, source.sourceID, ref.SourceID)
+	assert.Equal(t, "test", ref.SourceName)
+	assert.True(t, chunksCalled)
+	assert.False(t, source.initCalled)
 }

pkg/sources/syslog/syslog.go

@@ -272,6 +272,7 @@ func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksCha
 			SourceName:     s.syslog.sourceName,
 			SourceID:       s.syslog.sourceID,
 			SourceType:     s.syslog.sourceType,
+			JobID:          s.JobID(),
 			SourceMetadata: metadata,
 			Data:           input,
 			Verify:         s.verify,
@@ -313,6 +314,7 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.Packe
 		chunksChan <- &sources.Chunk{
 			SourceName:     s.syslog.sourceName,
 			SourceID:       s.syslog.sourceID,
+			JobID:          s.JobID(),
 			SourceType:     s.syslog.sourceType,
 			SourceMetadata: metadata,
 			Data:           input,