[chore] Fix glob package name

* Detector-Competition-Feat: Add Privacy.com API key detector

* Detector-Competition-Feat: Add Privacy.com API key detector

* cleanup: fix prefix

go.mod

@@ -81,6 +81,7 @@ require (
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/h2non/gock.v1 v1.1.2
+	pgregory.net/rapid v1.1.0
 	sigs.k8s.io/yaml v1.3.0
 )
 

go.sum

@@ -1013,6 +1013,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
+pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

pkg/common/glob/glob.go

@@ -0,0 +1,120 @@
+package glob
+
+import (
+	"fmt"
+
+	"github.com/gobwas/glob"
+)
+
+// Filter is a generic filter for excluding and including globs (limited
+// regular expressions). Exclusion takes precedence if both include and exclude
+// lists are provided.
+type Filter struct {
+	exclude []glob.Glob
+	include []glob.Glob
+}
+
+type globFilterOpt func(*Filter) error
+
+// WithExcludeGlobs adds exclude globs to the filter.
+func WithExcludeGlobs(excludes ...string) globFilterOpt {
+	return func(f *Filter) error {
+		for _, exclude := range excludes {
+			g, err := glob.Compile(exclude)
+			if err != nil {
+				return fmt.Errorf("invalid exclude glob %q: %w", exclude, err)
+			}
+			f.exclude = append(f.exclude, g)
+		}
+		return nil
+	}
+}
+
+// WithIncludeGlobs adds include globs to the filter.
+func WithIncludeGlobs(includes ...string) globFilterOpt {
+	return func(f *Filter) error {
+		for _, include := range includes {
+			g, err := glob.Compile(include)
+			if err != nil {
+				return fmt.Errorf("invalid include glob %q: %w", include, err)
+			}
+			f.include = append(f.include, g)
+		}
+		return nil
+	}
+}
+
+// NewGlobFilter creates a new Filter with the provided options.
+func NewGlobFilter(opts ...globFilterOpt) (*Filter, error) {
+	filter := &Filter{}
+	for _, opt := range opts {
+		if err := opt(filter); err != nil {
+			return nil, err
+		}
+	}
+	return filter, nil
+}
+
+// ShouldInclude returns whether the object is in the include list or not in
+// the exclude list (exclude taking precedence).
+func (f *Filter) ShouldInclude(object string) bool {
+	if f == nil {
+		return true
+	}
+	exclude, include := len(f.exclude), len(f.include)
+	if exclude == 0 && include == 0 {
+		return true
+	} else if exclude > 0 && include == 0 {
+		return f.shouldIncludeFromExclude(object)
+	} else if exclude == 0 && include > 0 {
+		return f.shouldIncludeFromInclude(object)
+	} else {
+		if ok, err := f.shouldIncludeFromBoth(object); err == nil {
+			return ok
+		}
+		// Ambiguous case.
+		return false
+	}
+}
+
+// shouldIncludeFromExclude checks for explicitly excluded paths. This should
+// only be called when the include list is empty.
+func (f *Filter) shouldIncludeFromExclude(object string) bool {
+	for _, glob := range f.exclude {
+		if glob.Match(object) {
+			return false
+		}
+	}
+	return true
+}
+
+// shouldIncludeFromInclude checks for explicitly included paths. This should
+// only be called when the exclude list is empty.
+func (f *Filter) shouldIncludeFromInclude(object string) bool {
+	for _, glob := range f.include {
+		if glob.Match(object) {
+			return true
+		}
+	}
+	return false
+}
+
+// shouldIncludeFromBoth checks for either excluded or included paths. Exclusion
+// takes precedence. If neither list contains the object, true is returned.
+func (f *Filter) shouldIncludeFromBoth(object string) (bool, error) {
+	// Exclude takes precedence. If we find the object in the exclude list,
+	// we should not match.
+	for _, glob := range f.exclude {
+		if glob.Match(object) {
+			return false, nil
+		}
+	}
+	// If we find the object in the include list, we should match.
+	for _, glob := range f.include {
+		if glob.Match(object) {
+			return true, nil
+		}
+	}
+	// If we find it in neither, return an error to let the caller decide.
+	return false, fmt.Errorf("ambiguous match")
+}

pkg/common/glob/glob_test.go

@@ -0,0 +1,138 @@
+package glob
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"pgregory.net/rapid"
+)
+
+type globTest struct {
+	input         string
+	shouldInclude bool
+}
+
+func testGlobs(t *testing.T, filter *Filter, tests ...globTest) {
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			// Invert because mentally it's easier to say whether an
+			// input should be included.
+			assert.Equal(t, tt.shouldInclude, filter.ShouldInclude(tt.input))
+		})
+	}
+}
+
+func TestGlobFilterExclude(t *testing.T) {
+	filter, err := NewGlobFilter(WithExcludeGlobs("foo", "bar*"))
+	assert.NoError(t, err)
+
+	testGlobs(t, filter,
+		globTest{"foo", false},
+		globTest{"bar", false},
+		globTest{"bara", false},
+		globTest{"barb", false},
+		globTest{"barbosa", false},
+		globTest{"foobar", true},
+		globTest{"food", true},
+		globTest{"anything else", true},
+	)
+}
+
+func TestGlobFilterInclude(t *testing.T) {
+	filter, err := NewGlobFilter(WithIncludeGlobs("foo", "bar*"))
+	assert.NoError(t, err)
+
+	testGlobs(t, filter,
+		globTest{"foo", true},
+		globTest{"bar", true},
+		globTest{"bara", true},
+		globTest{"barb", true},
+		globTest{"barbosa", true},
+		globTest{"foobar", false},
+		globTest{"food", false},
+		globTest{"anything else", false},
+	)
+}
+
+func TestGlobFilterEmpty(t *testing.T) {
+	filter, err := NewGlobFilter()
+	assert.NoError(t, err)
+
+	testGlobs(t, filter,
+		globTest{"foo", true},
+		globTest{"bar", true},
+		globTest{"bara", true},
+		globTest{"barb", true},
+		globTest{"barbosa", true},
+		globTest{"foobar", true},
+		globTest{"food", true},
+		globTest{"anything else", true},
+	)
+}
+
+func TestGlobFilterExcludeInclude(t *testing.T) {
+	filter, err := NewGlobFilter(WithExcludeGlobs("/foo/bar/**"), WithIncludeGlobs("/foo/**"))
+	assert.NoError(t, err)
+
+	testGlobs(t, filter,
+		globTest{"/foo/a", true},
+		globTest{"/foo/b", true},
+		globTest{"/foo/c/d/e", true},
+		globTest{"/foo/bar/a", false},
+		globTest{"/foo/bar/b", false},
+		globTest{"/foo/bar/c/d/e", false},
+		globTest{"/any/other/path", false},
+	)
+}
+
+func TestGlobFilterExcludePrecedence(t *testing.T) {
+	filter, err := NewGlobFilter(WithExcludeGlobs("foo"), WithIncludeGlobs("foo*"))
+	assert.NoError(t, err)
+
+	testGlobs(t, filter,
+		globTest{"foo", false},
+		globTest{"foobar", true},
+	)
+}
+
+func TestGlobErrorContainsGlob(t *testing.T) {
+	invalidGlob := "[this is invalid because it doesn't close the capture group"
+	_, err := NewGlobFilter(WithExcludeGlobs(invalidGlob))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), invalidGlob)
+}
+
+// The filters in this test should be mutually exclusive because one includes
+// and the other excludes the same glob.
+func TestGlobInverse(t *testing.T) {
+	for _, glob := range []string{
+		"a",
+		"a*",
+		"a**",
+		"*a",
+		"**a",
+		"*",
+	} {
+		include, err := NewGlobFilter(WithIncludeGlobs(glob))
+		assert.NoError(t, err)
+		exclude, err := NewGlobFilter(WithExcludeGlobs(glob))
+		assert.NoError(t, err)
+		rapid.Check(t, func(t *rapid.T) {
+			input := rapid.String().Draw(t, "input")
+			a, b := include.ShouldInclude(input), exclude.ShouldInclude(input)
+			if a == b {
+				t.Fatalf("Filter(Include(%q)) == Filter(Exclude(%q)) == %v for input %q", glob, glob, a, input)
+			}
+		})
+	}
+}
+
+func TestGlobDefaultFilters(t *testing.T) {
+	for _, filter := range []*Filter{nil, {}} {
+		rapid.Check(t, func(t *rapid.T) {
+			if !filter.ShouldInclude(rapid.String().Draw(t, "input")) {
+				t.Fatalf("filter %#v did not include input", filter)
+			}
+		})
+	}
+}

pkg/detectors/pagerdutyapikey/pagerdutyapikey.go

@@ -24,7 +24,7 @@ const verifyURL = "https://api.pagerduty.com/users"
 var (
 	defaultClient = common.SaneHttpClient()
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b([a-zA-Z0-9_+-]{20})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b(u\+[a-zA-Z0-9_+-]{18})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.

pkg/detectors/plaidkey/plaidkey.go

@@ -2,6 +2,7 @@ package plaidkey
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,32 +54,36 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				DetectorType: detectorspb.DetectorType_PlaidKey,
 				Raw:          []byte(resMatch),
 			}
-
+			environments := []string{"development", "production"}
 			if verify {
-				payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
-				req, err := http.NewRequestWithContext(ctx, "POST", "https://development.plaid.com/link/token/create", payload)
-				if err != nil {
-					continue
-				}
-				req.Header.Add("Content-Type", "application/json")
-				res, err := client.Do(req)
-				if err == nil {
-					defer res.Body.Close()
-					if res.StatusCode >= 200 && res.StatusCode < 300 {
-						s1.Verified = true
-					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-							continue
+				for _, env := range environments {
+					payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
+					req, err := http.NewRequestWithContext(ctx, "POST", "https://"+env+".plaid.com/link/token/create", payload)
+					if err != nil {
+						continue
+					}
+					req.Header.Add("Content-Type", "application/json")
+					res, err := client.Do(req)
+					if err == nil {
+						defer res.Body.Close()
+						if res.StatusCode >= 200 && res.StatusCode < 300 {
+							s1.Verified = true
+							s1.ExtraData = map[string]string{"environment": fmt.Sprintf("https://%s.plaid.com", env)}
+						} else {
+							// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+							if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+								continue
+							}
 						}
 					}
 				}
+				results = append(results, s1)
+				// if the environment is dev, we don't need to check production
+				if s1.Verified {
+					break
+				}
 			}
-
-			results = append(results, s1)
-
 		}
-
 	}
 
 	return results, nil

pkg/detectors/plaidkey/plaidkey_test.go

@@ -26,6 +26,8 @@ func TestPlaidKey_FromChunk(t *testing.T) {
 	secret := testSecrets.MustGetField("PLAIDKEY_SECRET")
 	inactiveSecret := testSecrets.MustGetField("PLAIDKEY_SECRET_INACTIVE")
 	id := testSecrets.MustGetField("PLAIDKEY_CLIENTID")
+	// env := testSecrets.MustGetField("PLAIDKEY_ENVIRONMENT") // development or production
+	env := "development"
 
 	type args struct {
 		ctx    context.Context
@@ -51,6 +53,9 @@ func TestPlaidKey_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_PlaidKey,
 					Verified:     true,
+					ExtraData: map[string]string{
+						"environment": fmt.Sprintf("https://%s.plaid.com", env),
+					},
 				},
 			},
 			wantErr: false,

pkg/detectors/posthog/posthog.go

@@ -20,13 +20,13 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(`\b(phc_[a-zA-Z0-9_]{43})\b`)
+	keyPat = regexp.MustCompile(`\b(phx_[a-zA-Z0-9_]{43})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"phc_"}
+	return []string{"phx_"}
 }
 
 // FromData will find and optionally verify AppPosthog secrets in a given set of bytes.
@@ -47,20 +47,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			payload := strings.NewReader(` {
-				"api_key": "` + resMatch + `",
-				"distinct_id": "1234"
-				}`)
-			req, err := http.NewRequestWithContext(ctx, "POST", "https://app.posthog.com/decide/", payload)
-			if err != nil {
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://app.posthog.com/api/event/?personal_api_key="+resMatch, nil)
+			reqEU, errEU := http.NewRequestWithContext(ctx, "GET", "https://eu.posthog.com/api/event/?personal_api_key="+resMatch, nil)
+
+			if err != nil || errEU != nil {
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
+			reqEU.Header.Add("Content-Type", "application/json")
+
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
+				} else if res.StatusCode == 401 {
+					//Try EU Endpoint only if other one fails.
+					res, err := client.Do(reqEU)
+					if err == nil {
+						defer res.Body.Close()
+						if res.StatusCode >= 200 && res.StatusCode < 300 {
+							s1.Verified = true
+						} else {
+							if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+								continue
+							}
+						}
+					}
 				} else {
 					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {

pkg/detectors/privacy/privacy.go

@@ -0,0 +1,89 @@
+package privacy
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	client *http.Client
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	defaultClient = common.SaneHttpClient()
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"privacy"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"privacy"}
+}
+
+// FromData will find and optionally verify Privacy secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		if len(match) != 2 {
+			continue
+		}
+		resMatch := strings.TrimSpace(match[1])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_Privacy,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.privacy.com/v1/card?page=1&page_size=50", nil)
+			if err != nil {
+				continue
+			}
+			req.Header.Set("Authorization", "api-key "+resMatch)
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
+					s1.Verified = true
+				} else if res.StatusCode == 401 {
+					// The secret is determinately not verified (nothing to do)
+				} else {
+					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+				}
+			} else {
+				s1.VerificationError = err
+			}
+		}
+
+		// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+		if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+			continue
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Privacy
+}

pkg/detectors/privacy/privacy_test.go

@@ -0,0 +1,161 @@
+//go:build detectors
+// +build detectors
+
+package privacy
+
+import (
+	"context"
+	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"testing"
+	"time"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPrivacy_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PRIVACY")
+	inactiveSecret := testSecrets.MustGetField("PRIVACY_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Privacy,
+					Verified:     true,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a privacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Privacy,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Privacy,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Privacy,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Privacy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
+			}
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("Privacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/twist/twist.go

@@ -21,7 +21,7 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b([0-9a-f:]{40,47})\b`)
+	accessToken = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b(?:oauth2:)?([0-9a-f]{40})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -34,7 +34,7 @@ func (s Scanner) Keywords() []string {
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := accessToken.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {

pkg/detectors/voiceflow/voiceflow.go

@@ -0,0 +1,99 @@
+package voiceflow
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	client *http.Client
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	defaultClient = common.SaneHttpClient()
+	// Reference: https://developer.voiceflow.com/reference/project#dialog-manager-api-keys
+	//
+	//TODO: This includes Workspace and Legacy Workspace API keys; I haven't validated whether these actually work.
+	// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
+	keyPat = regexp.MustCompile(`\b(VF\.(?:(?:DM|WS)\.)?[a-fA-F0-9]{24}\.[a-zA-Z0-9]{16})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"vf", "dm"}
+}
+
+// FromData will find and optionally verify Voiceflow secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		if len(match) != 2 {
+			continue
+		}
+		resMatch := strings.TrimSpace(match[1])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_Voiceflow,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
+			// Fetch the state for a random user.
+			payload := []byte(`{"question": "why is the sky blue?"}`)
+			req, err := http.NewRequestWithContext(ctx, "POST", "https://general-runtime.voiceflow.com/knowledge-base/query", bytes.NewBuffer(payload))
+			if err != nil {
+				continue
+			}
+			req.Header.Set("Accept", "application/json")
+			req.Header.Set("Authorization", resMatch)
+			req.Header.Set("Content-Type", "application/json")
+
+			res, err := client.Do(req)
+			if err == nil {
+				if res.StatusCode == http.StatusOK {
+					s1.Verified = true
+				} else if res.StatusCode == http.StatusUnauthorized {
+					// The secret is determinately not verified (nothing to do)
+				} else {
+					var buf bytes.Buffer
+					var bodyString string
+					_, err = io.Copy(&buf, res.Body)
+					if err == nil {
+						bodyString = buf.String()
+					}
+					s1.VerificationError = fmt.Errorf("unexpected HTTP response [status=%d, body=%s]", res.StatusCode, bodyString)
+				}
+				_ = res.Body.Close()
+			} else {
+				s1.VerificationError = err
+			}
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Voiceflow
+}

pkg/detectors/voiceflow/voiceflow_test.go

@@ -0,0 +1,302 @@
+//go:build detectors
+// +build detectors
+
+package voiceflow
+
+import (
+	"context"
+	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"testing"
+	"time"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestVoiceflow_Pattern(t *testing.T) {
+	tests := []struct {
+		name        string
+		data        string
+		shouldMatch bool
+		match       string
+	}{
+		// True positives
+		// https://github.com/funDAOmental/endlessquest/blob/5c008f7c6a7e58c45a88b72fef4b965c258d665c/Voiceflow/agent-api/index.js#L6
+		{
+			name: `valid_result1`,
+			data: `// z0MG IT'S NOT A SECRET (but we'll delete it)
+const API_KEY = "VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
+			shouldMatch: true,
+			match:       `VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF`,
+		},
+		// https://github.com/sherifButt/ll-site/blob/b98b268214324da42a84e996e4c03c242e122680/src/components/Chatbot.jsx#L14
+		{
+			name: `valid_result2`,
+			data: `  const runtime = useRuntime({
+    verify: { authorization: 'VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f' },
+    session: { userID: 'user_123' },
+  });`,
+			shouldMatch: true,
+			match:       `VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f`,
+		},
+		// https://github.com/the-vv/Voiceflow-chatbot/blob/324db17693dd46387ea7a020e92c4e79b94306c6/src/app/chat/chat.component.ts#L27
+		{
+			name: `valid_result3`,
+			data: `    this.http.delete('https://general-runtime.voiceflow.com/state/user/TEST_USER', {
+      headers: {
+        Authorization: "VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq"
+      }
+    }).subscribe(res => {
+      this.loading = false;
+      this.doPrompt('', { action: { type: 'launch' } });
+    })`,
+			shouldMatch: true,
+			match:       `VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq`,
+		},
+		// https://github.com/legionX7/Graduation-Project-API/blob/451431771d3fba1d8c634b8855274b414d7aed6d/mainAPI.py#L547
+		{
+			name: `valid_result4`,
+			data: `
+API_KEY = 'VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl'
+VERSION_ID = '646bc'`,
+			shouldMatch: true,
+			match:       `VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl`,
+		},
+		// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
+		{
+			name: `valid_result5`,
+			data: ` it('extracts ID from a Dialog Manager API key', () => {
+      // eslint-disable-next-line no-secrets/no-secrets
+      const key = 'VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai';
+      const result = utils.extractAPIKeyID(key);
+
+      expect(result).to.equal('628d5d92faf688001bda7907');
+    });`,
+			shouldMatch: true,
+			match:       `VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai`,
+		},
+		{
+			name: `valid_result6_legacy`,
+			data: `    it('extracts ID from a Workspace API key', () => {
+      // eslint-disable-next-line no-secrets/no-secrets
+      const key = 'VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa';
+      const result = utils.extractAPIKeyID(key);
+
+      expect(result).to.equal('62bcb0cca5184300066f5ac7');
+    });
+`,
+			shouldMatch: true,
+			match:       `VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa`,
+		},
+		{
+			name: `valid_result7_legacy`,
+			data: `    it('extracts ID from a Legacy Workspace API key', () => {
+      // eslint-disable-next-line no-secrets/no-secrets
+      const key = 'VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az';
+      const result = utils.extractAPIKeyID(key);
+
+      expect(result).to.equal('62bcb0cca5184300066f5ac7');
+    });`,
+			shouldMatch: true,
+			match:       `VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az`,
+		},
+
+		// False positives
+		// https://github.com/ImperialCollegeLondon/voiceflow-integration-whatsapp/blob/0f3d6a5638b9acb4989d5bf8e77081cc78e9b976/README.md?plain=1#L155
+		{
+			name:        `invalid_result1`,
+			data:        "Now, paste it in your .env file for the **VF_PROJECT_API** variable<br>\n```VF_PROJECT_API='VF.DM.62xxxxxxxxxxxxxxxxxxxxxxx'```",
+			shouldMatch: false,
+		},
+		// https://github.com/voiceflow/api-examples/blob/c3d8ba9ee8eced7ec8d241973b1eb0284aaec212/rust/src/main.rs#L5
+		{
+			name:        `invalid_result2`,
+			data:        `const API_KEY: &str = "YOUR_API_KEY_HERE"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
+			shouldMatch: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			s := Scanner{}
+
+			results, err := s.FromData(context.Background(), true, []byte(test.data))
+			if err != nil {
+				t.Errorf("CoinbaseWaaS.FromData() error = %v", err)
+				return
+			}
+
+			if test.shouldMatch {
+				if len(results) == 0 {
+					t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
+					return
+				}
+				expected := test.data
+				if test.match != "" {
+					expected = test.match
+				}
+				result := results[0]
+				if result.VerificationError != nil {
+					fmt.Printf("VerificationError: %v\n", result.VerificationError)
+				}
+				resultData := string(result.Raw)
+				if resultData != expected {
+					t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t  actual: '%s'", test.name, expected, resultData)
+					return
+				}
+			} else {
+				if len(results) > 0 {
+					t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
+					return
+				}
+			}
+		})
+	}
+}
+
+func TestVoiceflow_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("VOICEFLOW")
+	inactiveSecret := testSecrets.MustGetField("VOICEFLOW_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Voiceflow,
+					Verified:     true,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a voiceflow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Voiceflow,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Voiceflow,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Voiceflow,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Voiceflow.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
+			}
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("Voiceflow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/engine/defaults.go

@@ -11,6 +11,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraphcody"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/voiceflow"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
@@ -1554,6 +1555,7 @@ func DefaultDetectors() []detectors.Detector {
 		&ramp.Scanner{},
 		&anthropic.Scanner{},
 		&sourcegraphcody.Scanner{},
+		voiceflow.Scanner{},
 	}
 
 }

pkg/pb/detectorspb/detectors.pb.go

@@ -1008,6 +1008,8 @@ const (
 	DetectorType_Ramp                          DetectorType = 934
 	DetectorType_Klaviyo                       DetectorType = 935
 	DetectorType_SourcegraphCody               DetectorType = 936
+	DetectorType_Voiceflow                     DetectorType = 937
+	DetectorType_Privacy                       DetectorType = 938
 )
 
 // Enum value maps for DetectorType.
@@ -1946,6 +1948,8 @@ var (
 		934: "Ramp",
 		935: "Klaviyo",
 		936: "SourcegraphCody",
+		937: "Voiceflow",
+		938: "Privacy",
 	}
 	DetectorType_value = map[string]int32{
 		"Alibaba":                       0,
@@ -2881,6 +2885,8 @@ var (
 		"Ramp":                          934,
 		"Klaviyo":                       935,
 		"SourcegraphCody":               936,
+		"Voiceflow":                     937,
+		"Privacy":                       938,
 	}
 )
 
@@ -3259,7 +3265,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
 	0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
 	0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
-	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xa6, 0x75, 0x0a, 0x0c, 0x44,
+	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xc4, 0x75, 0x0a, 0x0c, 0x44,
 	0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
 	0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
 	0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4198,11 +4204,13 @@ var file_detectors_proto_rawDesc = []byte{
 	0x63, 0x10, 0xa5, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x52, 0x61, 0x6d, 0x70, 0x10, 0xa6, 0x07, 0x12,
 	0x0c, 0x0a, 0x07, 0x4b, 0x6c, 0x61, 0x76, 0x69, 0x79, 0x6f, 0x10, 0xa7, 0x07, 0x12, 0x14, 0x0a,
 	0x0f, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x43, 0x6f, 0x64, 0x79,
-	0x10, 0xa8, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
-	0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
-	0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
-	0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
-	0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x10, 0xa8, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x56, 0x6f, 0x69, 0x63, 0x65, 0x66, 0x6c, 0x6f, 0x77,
+	0x10, 0xa9, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x50, 0x72, 0x69, 0x76, 0x61, 0x63, 0x79, 0x10, 0xaa,
+	0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f,
+	0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f,
+	0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b,
+	0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62,
+	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

pkg/sources/github/github.go

@@ -222,7 +222,10 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
 	}
 	s.conn = &conn
 
-	s.filteredRepoCache = s.newFilteredRepoCache(memory.New(), s.conn.GetRepositories(), s.conn.GetIgnoreRepos())
+	s.filteredRepoCache = s.newFilteredRepoCache(memory.New(),
+		append(s.conn.GetRepositories(), s.conn.GetIncludeRepos()...),
+		s.conn.GetIgnoreRepos(),
+	)
 	s.memberCache = make(map[string]struct{})
 
 	s.repoSizes = newRepoSize()
@@ -508,12 +511,12 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint strin
 
 	for _, org := range s.orgsCache.Keys() {
 		if err := s.getReposByOrg(ctx, org); err != nil {
-			s.log.Error(err, "error fetching repos for org or user")
+			s.log.Error(err, "error fetching repos for org")
 		}
 
 		// We probably don't need to do this, since getting repos by org makes more sense?
 		if err := s.getReposByUser(ctx, org); err != nil {
-			s.log.Error(err, "error fetching repos for org or user")
+			s.log.Error(err, "error fetching repos for user")
 		}
 
 		if s.conn.ScanUsers {
@@ -825,38 +828,51 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
 // Unauthenticated access to most github endpoints has a rate limit of 60 requests per hour.
 // This will likely only be exhausted if many users/orgs are scanned without auth
 func (s *Source) handleRateLimit(errIn error, res *github.Response) bool {
-	limit, ok := errIn.(*github.RateLimitError)
-	if !ok {
+	var (
+		knownWait  = true
+		remaining  = 0
+		retryAfter time.Duration
+	)
+
+	// GitHub has both primary (RateLimit) and secondary (AbuseRateLimit) errors.
+	var rateLimit *github.RateLimitError
+	var abuseLimit *github.AbuseRateLimitError
+	if errors.As(errIn, &rateLimit) {
+		// Do nothing
+	} else if errors.As(errIn, &abuseLimit) {
+		retryAfter = abuseLimit.GetRetryAfter()
+	} else {
 		return false
 	}
 
 	githubNumRateLimitEncountered.WithLabelValues(s.name).Inc()
-
-	if res != nil {
-		knownWait := true
-		remaining, err := strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
+	// Parse retry information from response headers, unless a Retry-After value was already provided.
+	// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#exceeding-the-rate-limit
+	if retryAfter <= 0 && res != nil {
+		var err error
+		remaining, err = strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
 		if err != nil {
 			knownWait = false
 		}
+
 		resetTime, err := strconv.Atoi(res.Header.Get("x-ratelimit-reset"))
 		if err != nil || resetTime == 0 {
 			knownWait = false
-		}
-
-		if knownWait && remaining == 0 {
-			waitTime := int64(resetTime) - time.Now().Unix()
-			if waitTime > 0 {
-				duration := time.Duration(waitTime+1) * time.Second
-				s.log.V(2).Info("rate limited", "resumeTime", time.Now().Add(duration).String())
-				time.Sleep(duration)
-				githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(duration.Seconds())
-				return true
-			}
+		} else if resetTime > 0 {
+			retryAfter = time.Duration(int64(resetTime)-time.Now().Unix()) * time.Second
 		}
 	}
 
-	s.log.V(2).Info("handling rate limit (5 minutes retry)", "retry-after", limit.Message)
-	time.Sleep(time.Minute * 5)
+	resumeTime := time.Now().Add(retryAfter).String()
+	if knownWait && remaining == 0 && retryAfter > 0 {
+		s.log.V(2).Info("rate limited", "retry_after", retryAfter.String(), "resume_time", resumeTime)
+	} else {
+		// TODO: Use exponential backoff instead of static retry time.
+		retryAfter = time.Minute * 5
+		s.log.V(2).Error(errIn, "unexpected rate limit error", "retry_after", retryAfter.String(), "resume_time", resumeTime)
+	}
+	time.Sleep(retryAfter)
+	githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(retryAfter.Seconds())
 	return true
 }
 
@@ -1076,7 +1092,7 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
 }
 
 func (s *Source) processGistComments(ctx context.Context, repoPath string, trimmedURL []string, repoURL *url.URL, chunksChan chan *sources.Chunk) error {
-	s.log.Info("scanning github gist comments", "repository", repoPath)
+	ctx.Logger().V(2).Info("scanning github gist comments", "repository", repoPath)
 	// GitHub Gist URL.
 	gistID, err := extractGistID(trimmedURL)
 	if err != nil {
@@ -1131,9 +1147,10 @@ var (
 )
 
 type repoInfo struct {
-	owner    string
-	repo     string
-	repoPath string
+	owner      string
+	repo       string
+	repoPath   string
+	visibility source_metadatapb.Visibility
 }
 
 func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimmedURL []string, repoURL *url.URL, chunksChan chan *sources.Chunk) error {
@@ -1144,22 +1161,33 @@ func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimm
 	owner := trimmedURL[1]
 	repo := trimmedURL[2]
 
-	repoInfo := repoInfo{owner: owner, repo: repo, repoPath: repoPath}
+	if !(s.includeIssueComments || s.includePRComments) {
+		return nil
+	}
+
+	repoInfo := repoInfo{
+		owner:      owner,
+		repo:       repo,
+		repoPath:   repoPath,
+		visibility: s.visibilityOf(ctx, repoPath),
+	}
 
 	if s.includeIssueComments {
-		if err := s.processIssueComments(ctx, repoInfo, chunksChan); err != nil {
+		ctx.Logger().V(2).Info("scanning github issues", "repository", repoInfo.repoPath)
+		if err := s.processIssues(ctx, repoInfo, chunksChan); err != nil {
 			return err
 		}
-		if err := s.processIssues(ctx, repoInfo, chunksChan); err != nil {
+		if err := s.processIssueComments(ctx, repoInfo, chunksChan); err != nil {
 			return err
 		}
 	}
 
 	if s.includePRComments {
-		if err := s.processPRComments(ctx, repoInfo, chunksChan); err != nil {
+		ctx.Logger().V(2).Info("scanning github pull requests", "repository", repoInfo.repoPath)
+		if err := s.processPRs(ctx, repoInfo, chunksChan); err != nil {
 			return err
 		}
-		if err := s.processPRs(ctx, repoInfo, chunksChan); err != nil {
+		if err := s.processPRComments(ctx, repoInfo, chunksChan); err != nil {
 			return err
 		}
 	}
@@ -1169,8 +1197,6 @@ func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimm
 }
 
 func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
-	s.log.Info("scanning github issue descriptions", "repository", info.repoPath)
-
 	bodyTextsOpts := &github.IssueListByRepoOptions{
 		Sort:      sortType,
 		Direction: directionType,
@@ -1191,7 +1217,7 @@ func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan ch
 			return err
 		}
 
-		if err = s.chunkIssues(ctx, info.repo, info.repoPath, issues, chunksChan); err != nil {
+		if err = s.chunkIssues(ctx, info, issues, chunksChan); err != nil {
 			return err
 		}
 
@@ -1205,8 +1231,6 @@ func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan ch
 }
 
 func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
-	s.log.Info("scanning github issue comments", "repository", info.repoPath)
-
 	issueOpts := &github.IssueListCommentsOptions{
 		Sort:      &sortType,
 		Direction: &directionType,
@@ -1226,7 +1250,7 @@ func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunks
 			return err
 		}
 
-		if err = s.chunkIssueComments(ctx, info.repo, info.repoPath, issueComments, chunksChan); err != nil {
+		if err = s.chunkIssueComments(ctx, info, issueComments, chunksChan); err != nil {
 			return err
 		}
 
@@ -1240,8 +1264,6 @@ func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunks
 }
 
 func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
-	s.log.Info("scanning github pull request descriptions", "repository", info.repoPath)
-
 	prOpts := &github.PullRequestListOptions{
 		Sort:      sortType,
 		Direction: directionType,
@@ -1262,7 +1284,7 @@ func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan
 			return err
 		}
 
-		if err = s.chunkPullRequests(ctx, info.repo, prs, chunksChan); err != nil {
+		if err = s.chunkPullRequests(ctx, info, prs, chunksChan); err != nil {
 			return err
 		}
 
@@ -1276,8 +1298,6 @@ func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan
 }
 
 func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
-	s.log.Info("scanning github pull request comments", "repository", info.repoPath)
-
 	prOpts := &github.PullRequestListCommentsOptions{
 		Sort:      sortType,
 		Direction: directionType,
@@ -1297,7 +1317,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
 			return err
 		}
 
-		if err = s.chunkPullRequestComments(ctx, info.repo, prComments, chunksChan); err != nil {
+		if err = s.chunkPullRequestComments(ctx, info, prComments, chunksChan); err != nil {
 			return err
 		}
 
@@ -1310,7 +1330,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
 	return nil
 }
 
-func (s *Source) chunkIssues(ctx context.Context, repo, repoPath string, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
+func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
 	for _, issue := range issues {
 
 		// Skip pull requests since covered by processPRs.
@@ -1330,9 +1350,9 @@ func (s *Source) chunkIssues(ctx context.Context, repo, repoPath string, issues
 						Link:       sanitizer.UTF8(issue.GetHTMLURL()),
 						Username:   sanitizer.UTF8(issue.GetUser().GetLogin()),
 						Email:      sanitizer.UTF8(issue.GetUser().GetEmail()),
-						Repository: sanitizer.UTF8(repo),
+						Repository: sanitizer.UTF8(repoInfo.repo),
 						Timestamp:  sanitizer.UTF8(issue.GetCreatedAt().String()),
-						Visibility: s.visibilityOf(ctx, repoPath),
+						Visibility: repoInfo.visibility,
 					},
 				},
 			},
@@ -1349,7 +1369,7 @@ func (s *Source) chunkIssues(ctx context.Context, repo, repoPath string, issues
 	return nil
 }
 
-func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
+func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
 	for _, comment := range comments {
 		// Create chunk and send it to the channel.
 		chunk := &sources.Chunk{
@@ -1363,9 +1383,9 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
 						Link:       sanitizer.UTF8(comment.GetHTMLURL()),
 						Username:   sanitizer.UTF8(comment.GetUser().GetLogin()),
 						Email:      sanitizer.UTF8(comment.GetUser().GetEmail()),
-						Repository: sanitizer.UTF8(repo),
+						Repository: sanitizer.UTF8(repoInfo.repo),
 						Timestamp:  sanitizer.UTF8(comment.GetCreatedAt().String()),
-						Visibility: s.visibilityOf(ctx, repoPath),
+						Visibility: repoInfo.visibility,
 					},
 				},
 			},
@@ -1382,7 +1402,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
 	return nil
 }
 
-func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
+func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
 	for _, comment := range comments {
 		// Create chunk and send it to the channel.
 		chunk := &sources.Chunk{
@@ -1396,8 +1416,9 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
 						Link:       sanitizer.UTF8(comment.GetHTMLURL()),
 						Username:   sanitizer.UTF8(comment.GetUser().GetLogin()),
 						Email:      sanitizer.UTF8(comment.GetUser().GetEmail()),
-						Repository: sanitizer.UTF8(repo),
+						Repository: sanitizer.UTF8(repoInfo.repo),
 						Timestamp:  sanitizer.UTF8(comment.GetCreatedAt().String()),
+						Visibility: repoInfo.visibility,
 					},
 				},
 			},
@@ -1414,7 +1435,7 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
 	return nil
 }
 
-func (s *Source) chunkPullRequests(ctx context.Context, repo string, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
+func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
 	for _, pr := range prs {
 		// Create chunk and send it to the channel.
 		chunk := &sources.Chunk{
@@ -1428,8 +1449,9 @@ func (s *Source) chunkPullRequests(ctx context.Context, repo string, prs []*gith
 						Link:       sanitizer.UTF8(pr.GetHTMLURL()),
 						Username:   sanitizer.UTF8(pr.GetUser().GetLogin()),
 						Email:      sanitizer.UTF8(pr.GetUser().GetEmail()),
-						Repository: sanitizer.UTF8(repo),
+						Repository: sanitizer.UTF8(repoInfo.repo),
 						Timestamp:  sanitizer.UTF8(pr.GetCreatedAt().String()),
+						Visibility: repoInfo.visibility,
 					},
 				},
 			},
@@ -1462,8 +1484,7 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
 						Email:      sanitizer.UTF8(comment.GetUser().GetEmail()),
 						Repository: sanitizer.UTF8(gistUrl),
 						Timestamp:  sanitizer.UTF8(comment.GetCreatedAt().String()),
-						// Fetching this information requires making an additional API call.
-						// We may want to include this in the future.
+						// TODO: Fetching this requires making an additional API call. We may want to include this in the future.
 						// Visibility: s.visibilityOf(ctx, repoPath),
 					},
 				},

pkg/sources/job_progress.go

@@ -89,14 +89,14 @@ func (f Fatal) Unwrap() error { return f.error }
 // ChunkError is a custom error type for errors encountered during chunking of
 // a specific unit.
 type ChunkError struct {
-	unit SourceUnit
-	err  error
+	Unit SourceUnit
+	Err  error
 }
 
 func (f ChunkError) Error() string {
-	return fmt.Sprintf("error chunking unit %q: %s", f.unit.SourceUnitID(), f.err.Error())
+	return fmt.Sprintf("error chunking unit %q: %s", f.Unit.SourceUnitID(), f.Err.Error())
 }
-func (f ChunkError) Unwrap() error { return f.err }
+func (f ChunkError) Unwrap() error { return f.Err }
 
 // JobProgress aggregates information about a run of a Source.
 type JobProgress struct {
@@ -351,3 +351,17 @@ func (m JobProgressMetrics) ElapsedTime() time.Duration {
 	}
 	return m.EndTime.Sub(m.StartTime)
 }
+
+// ErrorsFor returns all the errors for the given SourceUnit. If there are no
+// errors, including the case that the unit has not been encountered, nil will
+// be returned.
+func (m JobProgressMetrics) ErrorsFor(unit SourceUnit) []error {
+	var errs []error
+	for _, err := range m.Errors {
+		var chunkErr ChunkError
+		if errors.As(err, &chunkErr) && chunkErr.Unit == unit {
+			errs = append(errs, err)
+		}
+	}
+	return errs
+}

pkg/sources/job_progress_test.go

@@ -126,3 +126,26 @@ func TestJobProgressElapsedTime(t *testing.T) {
 	metrics.EndTime = metrics.StartTime.Add(1 * time.Hour)
 	assert.Equal(t, metrics.ElapsedTime(), 1*time.Hour)
 }
+
+func TestJobProgressErrorsFor(t *testing.T) {
+	metrics := JobProgressMetrics{
+		Errors: []error{
+			Fatal{ChunkError{
+				Unit: CommonSourceUnit{ID: "foo"},
+				Err:  fmt.Errorf("foo error"),
+			}},
+			ChunkError{
+				Unit: CommonSourceUnit{ID: "foo"},
+				Err:  fmt.Errorf("foo again error"),
+			},
+			ChunkError{
+				Unit: CommonSourceUnit{ID: "bar"},
+				Err:  fmt.Errorf("bar error"),
+			},
+			fmt.Errorf("hi there"),
+		},
+	}
+	assert.Equal(t, 2, len(metrics.ErrorsFor(CommonSourceUnit{ID: "foo"})))
+	assert.Equal(t, 1, len(metrics.ErrorsFor(CommonSourceUnit{ID: "bar"})))
+	assert.Equal(t, 0, len(metrics.ErrorsFor(CommonSourceUnit{ID: "baz"})))
+}

proto/detectors.proto

@@ -945,6 +945,8 @@ enum DetectorType {
   Ramp = 934;
   Klaviyo = 935;
   SourcegraphCody = 936;
+  Voiceflow = 937;
+  Privacy = 938;
 }
 
 message Result {