Refactor channel to use an LRU cache instead

An LRU cache has a more favorable failure mode than the channel. With
the channel, if the consumer stopped consuming metrics, scanning would
block. With the LRU cache, metrics will be dropped when space runs out
and a log message emitted.

go.mod

@@ -47,6 +47,7 @@ require (
 	github.com/h2non/filetype v1.1.3
 	github.com/hashicorp/go-retryablehttp v0.7.4
 	github.com/hashicorp/golang-lru v0.5.1
+	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/jlaffaye/ftp v0.2.0
 	github.com/joho/godotenv v1.5.1
 	github.com/jpillora/overseer v1.1.6
@@ -65,7 +66,6 @@ require (
 	github.com/prometheus/client_golang v1.16.0
 	github.com/rabbitmq/amqp091-go v1.8.1
 	github.com/sergi/go-diff v1.3.1
-	github.com/shuheiktgw/go-travis v0.3.1
 	github.com/snowflakedb/gosnowflake v1.6.23
 	github.com/stretchr/testify v1.8.4
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
@@ -82,6 +82,7 @@ require (
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/h2non/gock.v1 v1.1.2
+	pgregory.net/rapid v1.1.0
 	sigs.k8s.io/yaml v1.3.0
 )
 

go.sum

@@ -369,7 +369,6 @@ github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5Px
 github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
 github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=
 github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=
-github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
@@ -419,6 +418,8 @@ github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
@@ -606,8 +607,6 @@ github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
 github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
-github.com/shuheiktgw/go-travis v0.3.1 h1:SAT16mi77ccqogOslnXxBXzXbpeyChaIYUwi2aJpVZY=
-github.com/shuheiktgw/go-travis v0.3.1/go.mod h1:avnFFDqJDdRHwlF9tgqvYi3asQCm/HGL8aLxYiKa4Yg=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
@@ -1016,6 +1015,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
+pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

main.go

@@ -139,9 +139,6 @@ var (
 
 	dockerScan       = cli.Command("docker", "Scan Docker Image")
 	dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, otherwise a image registry is assumed.").Required().Strings()
-
-	travisCiScan      = cli.Command("travisci", "Scan TravisCI")
-	travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()
 )
 
 func init() {
@@ -155,6 +152,9 @@ func init() {
 
 	cli.Version("trufflehog " + version.BuildVersion)
 
+	//Support -h for help
+	cli.HelpFlag.Short('h')
+
 	if len(os.Args) <= 1 && isatty.IsTerminal(os.Stdout.Fd()) {
 		args := tui.Run()
 		if len(args) == 0 {
@@ -501,10 +501,6 @@ func run(state overseer.State) {
 		if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
 			logFatal(err, "Failed to scan CircleCI.")
 		}
-	case travisCiScan.FullCommand():
-		if err := e.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
-			logFatal(err, "Failed to scan TravisCI.")
-		}
 	case gcsScan.FullCommand():
 		cfg := sources.GCSConfig{
 			ProjectID:      *gcsProjectID,

pkg/common/glob/glob.go

@@ -0,0 +1,120 @@
+package common
+
+import (
+	"fmt"
+
+	"github.com/gobwas/glob"
+)
+
+// Filter is a generic filter for excluding and including globs (limited
+// regular expressions). Exclusion takes precedence if both include and exclude
+// lists are provided.
+type Filter struct {
+	exclude []glob.Glob
+	include []glob.Glob
+}
+
+type globFilterOpt func(*Filter) error
+
+// WithExcludeGlobs adds exclude globs to the filter.
+func WithExcludeGlobs(excludes ...string) globFilterOpt {
+	return func(f *Filter) error {
+		for _, exclude := range excludes {
+			g, err := glob.Compile(exclude)
+			if err != nil {
+				return fmt.Errorf("invalid exclude glob %q: %w", exclude, err)
+			}
+			f.exclude = append(f.exclude, g)
+		}
+		return nil
+	}
+}
+
+// WithIncludeGlobs adds include globs to the filter.
+func WithIncludeGlobs(includes ...string) globFilterOpt {
+	return func(f *Filter) error {
+		for _, include := range includes {
+			g, err := glob.Compile(include)
+			if err != nil {
+				return fmt.Errorf("invalid include glob %q: %w", include, err)
+			}
+			f.include = append(f.include, g)
+		}
+		return nil
+	}
+}
+
+// NewGlobFilter creates a new Filter with the provided options.
+func NewGlobFilter(opts ...globFilterOpt) (*Filter, error) {
+	filter := &Filter{}
+	for _, opt := range opts {
+		if err := opt(filter); err != nil {
+			return nil, err
+		}
+	}
+	return filter, nil
+}
+
+// ShouldInclude returns whether the object is in the include list or not in
+// the exclude list (exclude taking precedence).
+func (f *Filter) ShouldInclude(object string) bool {
+	if f == nil {
+		return true
+	}
+	exclude, include := len(f.exclude), len(f.include)
+	if exclude == 0 && include == 0 {
+		return true
+	} else if exclude > 0 && include == 0 {
+		return f.shouldIncludeFromExclude(object)
+	} else if exclude == 0 && include > 0 {
+		return f.shouldIncludeFromInclude(object)
+	} else {
+		if ok, err := f.shouldIncludeFromBoth(object); err == nil {
+			return ok
+		}
+		// Ambiguous case.
+		return false
+	}
+}
+
+// shouldIncludeFromExclude checks for explicitly excluded paths. This should
+// only be called when the include list is empty.
+func (f *Filter) shouldIncludeFromExclude(object string) bool {
+	for _, glob := range f.exclude {
+		if glob.Match(object) {
+			return false
+		}
+	}
+	return true
+}
+
+// shouldIncludeFromInclude checks for explicitly included paths. This should
+// only be called when the exclude list is empty.
+func (f *Filter) shouldIncludeFromInclude(object string) bool {
+	for _, glob := range f.include {
+		if glob.Match(object) {
+			return true
+		}
+	}
+	return false
+}
+
+// shouldIncludeFromBoth checks for either excluded or included paths. Exclusion
+// takes precedence. If neither list contains the object, true is returned.
+func (f *Filter) shouldIncludeFromBoth(object string) (bool, error) {
+	// Exclude takes precedence. If we find the object in the exclude list,
+	// we should not match.
+	for _, glob := range f.exclude {
+		if glob.Match(object) {
+			return false, nil
+		}
+	}
+	// If we find the object in the include list, we should match.
+	for _, glob := range f.include {
+		if glob.Match(object) {
+			return true, nil
+		}
+	}
+	// If we find it in neither, return an error to let the caller decide.
+	return false, fmt.Errorf("ambiguous match")
+}

pkg/common/glob/glob_test.go

@@ -0,0 +1,138 @@
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"pgregory.net/rapid"
+)
+
+type globTest struct {
+	input         string
+	shouldInclude bool
+}
+
+func testGlobs(t *testing.T, filter *Filter, tests ...globTest) {
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			// Invert because mentally it's easier to say whether an
+			// input should be included.
+			assert.Equal(t, tt.shouldInclude, filter.ShouldInclude(tt.input))
+		})
+	}
+}
+
+func TestGlobFilterExclude(t *testing.T) {
+	filter, err := NewGlobFilter(WithExcludeGlobs("foo", "bar*"))
+	assert.NoError(t, err)
+
+	testGlobs(t, filter,
+		globTest{"foo", false},
+		globTest{"bar", false},
+		globTest{"bara", false},
+		globTest{"barb", false},
+		globTest{"barbosa", false},
+		globTest{"foobar", true},
+		globTest{"food", true},
+		globTest{"anything else", true},
+	)
+}
+
+func TestGlobFilterInclude(t *testing.T) {
+	filter, err := NewGlobFilter(WithIncludeGlobs("foo", "bar*"))
+	assert.NoError(t, err)
+
+	testGlobs(t, filter,
+		globTest{"foo", true},
+		globTest{"bar", true},
+		globTest{"bara", true},
+		globTest{"barb", true},
+		globTest{"barbosa", true},
+		globTest{"foobar", false},
+		globTest{"food", false},
+		globTest{"anything else", false},
+	)
+}
+
+func TestGlobFilterEmpty(t *testing.T) {
+	filter, err := NewGlobFilter()
+	assert.NoError(t, err)
+
+	testGlobs(t, filter,
+		globTest{"foo", true},
+		globTest{"bar", true},
+		globTest{"bara", true},
+		globTest{"barb", true},
+		globTest{"barbosa", true},
+		globTest{"foobar", true},
+		globTest{"food", true},
+		globTest{"anything else", true},
+	)
+}
+
+func TestGlobFilterExcludeInclude(t *testing.T) {
+	filter, err := NewGlobFilter(WithExcludeGlobs("/foo/bar/**"), WithIncludeGlobs("/foo/**"))
+	assert.NoError(t, err)
+
+	testGlobs(t, filter,
+		globTest{"/foo/a", true},
+		globTest{"/foo/b", true},
+		globTest{"/foo/c/d/e", true},
+		globTest{"/foo/bar/a", false},
+		globTest{"/foo/bar/b", false},
+		globTest{"/foo/bar/c/d/e", false},
+		globTest{"/any/other/path", false},
+	)
+}
+
+func TestGlobFilterExcludePrecedence(t *testing.T) {
+	filter, err := NewGlobFilter(WithExcludeGlobs("foo"), WithIncludeGlobs("foo*"))
+	assert.NoError(t, err)
+
+	testGlobs(t, filter,
+		globTest{"foo", false},
+		globTest{"foobar", true},
+	)
+}
+
+func TestGlobErrorContainsGlob(t *testing.T) {
+	invalidGlob := "[this is invalid because it doesn't close the capture group"
+	_, err := NewGlobFilter(WithExcludeGlobs(invalidGlob))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), invalidGlob)
+}
+
+// The filters in this test should be mutually exclusive because one includes
+// and the other excludes the same glob.
+func TestGlobInverse(t *testing.T) {
+	for _, glob := range []string{
+		"a",
+		"a*",
+		"a**",
+		"*a",
+		"**a",
+		"*",
+	} {
+		include, err := NewGlobFilter(WithIncludeGlobs(glob))
+		assert.NoError(t, err)
+		exclude, err := NewGlobFilter(WithExcludeGlobs(glob))
+		assert.NoError(t, err)
+		rapid.Check(t, func(t *rapid.T) {
+			input := rapid.String().Draw(t, "input")
+			a, b := include.ShouldInclude(input), exclude.ShouldInclude(input)
+			if a == b {
+				t.Fatalf("Filter(Include(%q)) == Filter(Exclude(%q)) == %v for input %q", glob, glob, a, input)
+			}
+		})
+	}
+}
+
+func TestGlobDefaultFilters(t *testing.T) {
+	for _, filter := range []*Filter{nil, {}} {
+		rapid.Check(t, func(t *rapid.T) {
+			if !filter.ShouldInclude(rapid.String().Draw(t, "input")) {
+				t.Fatalf("filter %#v did not include input", filter)
+			}
+		})
+	}
+}

pkg/detectors/falsepositives_test.go

@@ -74,6 +74,13 @@ func TestStringShannonEntropy(t *testing.T) {
 			},
 			want: 0.22228483068568816,
 		},
+		{
+			name: "empty",
+			args: args{
+				input: "",
+			},
+			want: 0.0,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/detectors/pagerdutyapikey/pagerdutyapikey.go

@@ -24,7 +24,7 @@ const verifyURL = "https://api.pagerduty.com/users"
 var (
 	defaultClient = common.SaneHttpClient()
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b([a-zA-Z0-9_+-]{20})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b(u\+[a-zA-Z0-9_+-]{18})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.

pkg/detectors/plaidkey/plaidkey.go

@@ -2,6 +2,7 @@ package plaidkey
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,32 +54,36 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				DetectorType: detectorspb.DetectorType_PlaidKey,
 				Raw:          []byte(resMatch),
 			}
-
+			environments := []string{"development", "production"}
 			if verify {
-				payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
-				req, err := http.NewRequestWithContext(ctx, "POST", "https://development.plaid.com/link/token/create", payload)
-				if err != nil {
-					continue
-				}
-				req.Header.Add("Content-Type", "application/json")
-				res, err := client.Do(req)
-				if err == nil {
-					defer res.Body.Close()
-					if res.StatusCode >= 200 && res.StatusCode < 300 {
-						s1.Verified = true
-					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-							continue
+				for _, env := range environments {
+					payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
+					req, err := http.NewRequestWithContext(ctx, "POST", "https://"+env+".plaid.com/link/token/create", payload)
+					if err != nil {
+						continue
+					}
+					req.Header.Add("Content-Type", "application/json")
+					res, err := client.Do(req)
+					if err == nil {
+						defer res.Body.Close()
+						if res.StatusCode >= 200 && res.StatusCode < 300 {
+							s1.Verified = true
+							s1.ExtraData = map[string]string{"environment": fmt.Sprintf("https://%s.plaid.com", env)}
+						} else {
+							// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+							if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+								continue
+							}
 						}
 					}
 				}
+				results = append(results, s1)
+				// if the environment is dev, we don't need to check production
+				if s1.Verified {
+					break
+				}
 			}
-
-			results = append(results, s1)
-
 		}
-
 	}
 
 	return results, nil

pkg/detectors/plaidkey/plaidkey_test.go

@@ -26,6 +26,8 @@ func TestPlaidKey_FromChunk(t *testing.T) {
 	secret := testSecrets.MustGetField("PLAIDKEY_SECRET")
 	inactiveSecret := testSecrets.MustGetField("PLAIDKEY_SECRET_INACTIVE")
 	id := testSecrets.MustGetField("PLAIDKEY_CLIENTID")
+	// env := testSecrets.MustGetField("PLAIDKEY_ENVIRONMENT") // development or production
+	env := "development"
 
 	type args struct {
 		ctx    context.Context
@@ -51,6 +53,9 @@ func TestPlaidKey_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_PlaidKey,
 					Verified:     true,
+					ExtraData: map[string]string{
+						"environment": fmt.Sprintf("https://%s.plaid.com", env),
+					},
 				},
 			},
 			wantErr: false,

pkg/detectors/posthog/posthog.go

@@ -20,13 +20,13 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(`\b(phc_[a-zA-Z0-9_]{43})\b`)
+	keyPat = regexp.MustCompile(`\b(phx_[a-zA-Z0-9_]{43})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"phc_"}
+	return []string{"phx_"}
 }
 
 // FromData will find and optionally verify AppPosthog secrets in a given set of bytes.
@@ -47,20 +47,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			payload := strings.NewReader(` {
-				"api_key": "` + resMatch + `",
-				"distinct_id": "1234"
-				}`)
-			req, err := http.NewRequestWithContext(ctx, "POST", "https://app.posthog.com/decide/", payload)
-			if err != nil {
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://app.posthog.com/api/event/?personal_api_key="+resMatch, nil)
+			reqEU, errEU := http.NewRequestWithContext(ctx, "GET", "https://eu.posthog.com/api/event/?personal_api_key="+resMatch, nil)
+
+			if err != nil || errEU != nil {
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
+			reqEU.Header.Add("Content-Type", "application/json")
+
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
+				} else if res.StatusCode == 401 {
+					//Try EU Endpoint only if other one fails.
+					res, err := client.Do(reqEU)
+					if err == nil {
+						defer res.Body.Close()
+						if res.StatusCode >= 200 && res.StatusCode < 300 {
+							s1.Verified = true
+						} else {
+							if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+								continue
+							}
+						}
+					}
 				} else {
 					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {

pkg/detectors/twist/twist.go

@@ -21,7 +21,7 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b([0-9a-f:]{40,47})\b`)
+	accessToken = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b(?:oauth2:)?([0-9a-f]{40})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -34,7 +34,7 @@ func (s Scanner) Keywords() []string {
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := accessToken.FindAllStringSubmatch(dataStr, -1)
 
 	for _, match := range matches {
 		if len(match) != 2 {

pkg/detectors/voiceflow/voiceflow.go

@@ -0,0 +1,99 @@
+package voiceflow
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	client *http.Client
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	defaultClient = common.SaneHttpClient()
+	// Reference: https://developer.voiceflow.com/reference/project#dialog-manager-api-keys
+	//
+	//TODO: This includes Workspace and Legacy Workspace API keys; I haven't validated whether these actually work.
+	// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
+	keyPat = regexp.MustCompile(`\b(VF\.(?:(?:DM|WS)\.)?[a-fA-F0-9]{24}\.[a-zA-Z0-9]{16})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"vf", "dm"}
+}
+
+// FromData will find and optionally verify Voiceflow secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		if len(match) != 2 {
+			continue
+		}
+		resMatch := strings.TrimSpace(match[1])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_Voiceflow,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
+			// Fetch the state for a random user.
+			payload := []byte(`{"question": "why is the sky blue?"}`)
+			req, err := http.NewRequestWithContext(ctx, "POST", "https://general-runtime.voiceflow.com/knowledge-base/query", bytes.NewBuffer(payload))
+			if err != nil {
+				continue
+			}
+			req.Header.Set("Accept", "application/json")
+			req.Header.Set("Authorization", resMatch)
+			req.Header.Set("Content-Type", "application/json")
+
+			res, err := client.Do(req)
+			if err == nil {
+				if res.StatusCode == http.StatusOK {
+					s1.Verified = true
+				} else if res.StatusCode == http.StatusUnauthorized {
+					// The secret is determinately not verified (nothing to do)
+				} else {
+					var buf bytes.Buffer
+					var bodyString string
+					_, err = io.Copy(&buf, res.Body)
+					if err == nil {
+						bodyString = buf.String()
+					}
+					s1.VerificationError = fmt.Errorf("unexpected HTTP response [status=%d, body=%s]", res.StatusCode, bodyString)
+				}
+				_ = res.Body.Close()
+			} else {
+				s1.VerificationError = err
+			}
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Voiceflow
+}

pkg/detectors/voiceflow/voiceflow_test.go

@@ -0,0 +1,302 @@
+//go:build detectors
+// +build detectors
+
+package voiceflow
+
+import (
+	"context"
+	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"testing"
+	"time"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestVoiceflow_Pattern(t *testing.T) {
+	tests := []struct {
+		name        string
+		data        string
+		shouldMatch bool
+		match       string
+	}{
+		// True positives
+		// https://github.com/funDAOmental/endlessquest/blob/5c008f7c6a7e58c45a88b72fef4b965c258d665c/Voiceflow/agent-api/index.js#L6
+		{
+			name: `valid_result1`,
+			data: `// z0MG IT'S NOT A SECRET (but we'll delete it)
+const API_KEY = "VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
+			shouldMatch: true,
+			match:       `VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF`,
+		},
+		// https://github.com/sherifButt/ll-site/blob/b98b268214324da42a84e996e4c03c242e122680/src/components/Chatbot.jsx#L14
+		{
+			name: `valid_result2`,
+			data: `  const runtime = useRuntime({
+    verify: { authorization: 'VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f' },
+    session: { userID: 'user_123' },
+  });`,
+			shouldMatch: true,
+			match:       `VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f`,
+		},
+		// https://github.com/the-vv/Voiceflow-chatbot/blob/324db17693dd46387ea7a020e92c4e79b94306c6/src/app/chat/chat.component.ts#L27
+		{
+			name: `valid_result3`,
+			data: `    this.http.delete('https://general-runtime.voiceflow.com/state/user/TEST_USER', {
+      headers: {
+        Authorization: "VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq"
+      }
+    }).subscribe(res => {
+      this.loading = false;
+      this.doPrompt('', { action: { type: 'launch' } });
+    })`,
+			shouldMatch: true,
+			match:       `VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq`,
+		},
+		// https://github.com/legionX7/Graduation-Project-API/blob/451431771d3fba1d8c634b8855274b414d7aed6d/mainAPI.py#L547
+		{
+			name: `valid_result4`,
+			data: `
+API_KEY = 'VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl'
+VERSION_ID = '646bc'`,
+			shouldMatch: true,
+			match:       `VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl`,
+		},
+		// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
+		{
+			name: `valid_result5`,
+			data: ` it('extracts ID from a Dialog Manager API key', () => {
+      // eslint-disable-next-line no-secrets/no-secrets
+      const key = 'VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai';
+      const result = utils.extractAPIKeyID(key);
+
+      expect(result).to.equal('628d5d92faf688001bda7907');
+    });`,
+			shouldMatch: true,
+			match:       `VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai`,
+		},
+		{
+			name: `valid_result6_legacy`,
+			data: `    it('extracts ID from a Workspace API key', () => {
+      // eslint-disable-next-line no-secrets/no-secrets
+      const key = 'VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa';
+      const result = utils.extractAPIKeyID(key);
+
+      expect(result).to.equal('62bcb0cca5184300066f5ac7');
+    });
+`,
+			shouldMatch: true,
+			match:       `VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa`,
+		},
+		{
+			name: `valid_result7_legacy`,
+			data: `    it('extracts ID from a Legacy Workspace API key', () => {
+      // eslint-disable-next-line no-secrets/no-secrets
+      const key = 'VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az';
+      const result = utils.extractAPIKeyID(key);
+
+      expect(result).to.equal('62bcb0cca5184300066f5ac7');
+    });`,
+			shouldMatch: true,
+			match:       `VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az`,
+		},
+
+		// False positives
+		// https://github.com/ImperialCollegeLondon/voiceflow-integration-whatsapp/blob/0f3d6a5638b9acb4989d5bf8e77081cc78e9b976/README.md?plain=1#L155
+		{
+			name:        `invalid_result1`,
+			data:        "Now, paste it in your .env file for the **VF_PROJECT_API** variable<br>\n```VF_PROJECT_API='VF.DM.62xxxxxxxxxxxxxxxxxxxxxxx'```",
+			shouldMatch: false,
+		},
+		// https://github.com/voiceflow/api-examples/blob/c3d8ba9ee8eced7ec8d241973b1eb0284aaec212/rust/src/main.rs#L5
+		{
+			name:        `invalid_result2`,
+			data:        `const API_KEY: &str = "YOUR_API_KEY_HERE"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
+			shouldMatch: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			s := Scanner{}
+
+			results, err := s.FromData(context.Background(), true, []byte(test.data))
+			if err != nil {
+				t.Errorf("CoinbaseWaaS.FromData() error = %v", err)
+				return
+			}
+
+			if test.shouldMatch {
+				if len(results) == 0 {
+					t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
+					return
+				}
+				expected := test.data
+				if test.match != "" {
+					expected = test.match
+				}
+				result := results[0]
+				if result.VerificationError != nil {
+					fmt.Printf("VerificationError: %v\n", result.VerificationError)
+				}
+				resultData := string(result.Raw)
+				if resultData != expected {
+					t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t  actual: '%s'", test.name, expected, resultData)
+					return
+				}
+			} else {
+				if len(results) > 0 {
+					t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
+					return
+				}
+			}
+		})
+	}
+}
+
+func TestVoiceflow_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("VOICEFLOW")
+	inactiveSecret := testSecrets.MustGetField("VOICEFLOW_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Voiceflow,
+					Verified:     true,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a voiceflow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Voiceflow,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Voiceflow,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Voiceflow,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Voiceflow.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
+			}
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("Voiceflow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/engine/defaults.go

@@ -11,6 +11,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraphcody"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/voiceflow"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
@@ -1554,6 +1555,7 @@ func DefaultDetectors() []detectors.Detector {
 		&ramp.Scanner{},
 		&anthropic.Scanner{},
 		&sourcegraphcody.Scanner{},
+		voiceflow.Scanner{},
 	}
 
 }

pkg/engine/engine.go

@@ -306,7 +306,6 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
 	e.sourceManager = sources.NewManager(
 		sources.WithConcurrentSources(int(e.concurrency)),
 		sources.WithConcurrentUnits(int(e.concurrency)),
-		sources.WithSourceUnits(),
 	)
 
 	if len(e.decoders) == 0 {
@@ -636,7 +635,7 @@ func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64,
 	if !found {
 		return 0, false
 	}
-	lineNumber := int64(bytes.Count(before, []byte("\n")))
+	lineNumber := int64(bytes.Count(before, []byte("\n")) + 1)
 	// If the line contains the ignore tag, we should ignore the result.
 	endLine := bytes.Index(after, []byte("\n"))
 	if endLine == -1 {

pkg/engine/engine_test.go

@@ -32,7 +32,7 @@ func TestFragmentLineOffset(t *testing.T) {
 			result: &detectors.Result{
 				Raw: []byte("secret here"),
 			},
-			expectedLine: 2,
+			expectedLine: 3,
 			ignore:       true,
 		},
 		{
@@ -43,7 +43,7 @@ func TestFragmentLineOffset(t *testing.T) {
 			result: &detectors.Result{
 				Raw: []byte("secret here"),
 			},
-			expectedLine: 2,
+			expectedLine: 3,
 			ignore:       false,
 		},
 		{
@@ -54,7 +54,7 @@ func TestFragmentLineOffset(t *testing.T) {
 			result: &detectors.Result{
 				Raw: []byte("secret here"),
 			},
-			expectedLine: 4,
+			expectedLine: 5,
 			ignore:       false,
 		},
 		{
@@ -65,7 +65,7 @@ func TestFragmentLineOffset(t *testing.T) {
 			result: &detectors.Result{
 				Raw: []byte("secret\nhere"),
 			},
-			expectedLine: 4,
+			expectedLine: 5,
 			ignore:       false,
 		},
 		{
@@ -76,7 +76,7 @@ func TestFragmentLineOffset(t *testing.T) {
 			result: &detectors.Result{
 				Raw: []byte("secret\nhere"),
 			},
-			expectedLine: 3,
+			expectedLine: 4,
 			ignore:       true,
 		},
 		{
@@ -87,7 +87,7 @@ func TestFragmentLineOffset(t *testing.T) {
 			result: &detectors.Result{
 				Raw: []byte("secret here"),
 			},
-			expectedLine: 3,
+			expectedLine: 4,
 			ignore:       true,
 		},
 	}

pkg/engine/travisci.go

@@ -1,38 +0,0 @@
-package engine
-
-import (
-	"runtime"
-
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/anypb"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/travisci"
-)
-
-// ScanTravisCI scans TravisCI logs.
-func (e *Engine) ScanTravisCI(ctx context.Context, token string) error {
-	connection := &sourcespb.TravisCI{
-		Credential: &sourcespb.TravisCI_Token{
-			Token: token,
-		},
-	}
-
-	var conn anypb.Any
-	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
-	if err != nil {
-		ctx.Logger().Error(err, "failed to marshal Travis CI connection")
-		return err
-	}
-
-	sourceName := "trufflehog - Travis CI"
-	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, travisci.SourceType)
-
-	travisSource := &travisci.Source{}
-	if err := travisSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-		return err
-	}
-	_, err = e.sourceManager.Run(ctx, sourceName, travisSource)
-	return err
-}

pkg/giturl/giturl.go

@@ -158,6 +158,11 @@ func UpdateLinkLineNumber(ctx context.Context, link string, newLine int64) strin
 		return link
 	}
 
+	if newLine <= 0 {
+		// Don't change the link if the line number is 0.
+		return link
+	}
+
 	switch determineProvider(link) {
 	case providerBitbucket:
 		// For Bitbucket, it doesn't support line links (based on the GenerateLink function).

pkg/giturl/giturl_test.go

@@ -251,6 +251,14 @@ func TestUpdateLinkLineNumber(t *testing.T) {
 			},
 			want: "https://onprem.customdomain.com/org/repo/commit/xyz123#L50",
 		},
+		{
+			name: "Don't include line when it's 0",
+			args: args{
+				link:    "https://github.com/coinbase/cbpay-js/issues/181",
+				newLine: int64(0),
+			},
+			want: "https://github.com/coinbase/cbpay-js/issues/181",
+		},
 		{
 			name: "Invalid link",
 			args: args{

pkg/pb/detectorspb/detectors.pb.go

@@ -1008,6 +1008,7 @@ const (
 	DetectorType_Ramp                          DetectorType = 934
 	DetectorType_Klaviyo                       DetectorType = 935
 	DetectorType_SourcegraphCody               DetectorType = 936
+	DetectorType_Voiceflow                     DetectorType = 937
 )
 
 // Enum value maps for DetectorType.
@@ -1946,6 +1947,7 @@ var (
 		934: "Ramp",
 		935: "Klaviyo",
 		936: "SourcegraphCody",
+		937: "Voiceflow",
 	}
 	DetectorType_value = map[string]int32{
 		"Alibaba":                       0,
@@ -2881,6 +2883,7 @@ var (
 		"Ramp":                          934,
 		"Klaviyo":                       935,
 		"SourcegraphCody":               936,
+		"Voiceflow":                     937,
 	}
 )
 
@@ -3259,7 +3262,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
 	0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
 	0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
-	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xa6, 0x75, 0x0a, 0x0c, 0x44,
+	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xb6, 0x75, 0x0a, 0x0c, 0x44,
 	0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
 	0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
 	0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4198,7 +4201,8 @@ var file_detectors_proto_rawDesc = []byte{
 	0x63, 0x10, 0xa5, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x52, 0x61, 0x6d, 0x70, 0x10, 0xa6, 0x07, 0x12,
 	0x0c, 0x0a, 0x07, 0x4b, 0x6c, 0x61, 0x76, 0x69, 0x79, 0x6f, 0x10, 0xa7, 0x07, 0x12, 0x14, 0x0a,
 	0x0f, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x43, 0x6f, 0x64, 0x79,
-	0x10, 0xa8, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
+	0x10, 0xa8, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x56, 0x6f, 0x69, 0x63, 0x65, 0x66, 0x6c, 0x6f, 0x77,
+	0x10, 0xa9, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
 	0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
 	0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
 	0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,

pkg/pb/source_metadatapb/source_metadata.pb.validate.go

@@ -486,117 +486,6 @@ var _ interface {
 	ErrorName() string
 } = CircleCIValidationError{}
 
-// Validate checks the field values on TravisCI with the rules defined in the
-// proto definition for this message. If any rules are violated, the first
-// error encountered is returned, or nil if there are no violations.
-func (m *TravisCI) Validate() error {
-	return m.validate(false)
-}
-
-// ValidateAll checks the field values on TravisCI with the rules defined in
-// the proto definition for this message. If any rules are violated, the
-// result is a list of violation errors wrapped in TravisCIMultiError, or nil
-// if none found.
-func (m *TravisCI) ValidateAll() error {
-	return m.validate(true)
-}
-
-func (m *TravisCI) validate(all bool) error {
-	if m == nil {
-		return nil
-	}
-
-	var errors []error
-
-	// no validation rules for Username
-
-	// no validation rules for Repository
-
-	// no validation rules for BuildNumber
-
-	// no validation rules for JobNumber
-
-	// no validation rules for Link
-
-	// no validation rules for Public
-
-	if len(errors) > 0 {
-		return TravisCIMultiError(errors)
-	}
-
-	return nil
-}
-
-// TravisCIMultiError is an error wrapping multiple validation errors returned
-// by TravisCI.ValidateAll() if the designated constraints aren't met.
-type TravisCIMultiError []error
-
-// Error returns a concatenation of all the error messages it wraps.
-func (m TravisCIMultiError) Error() string {
-	var msgs []string
-	for _, err := range m {
-		msgs = append(msgs, err.Error())
-	}
-	return strings.Join(msgs, "; ")
-}
-
-// AllErrors returns a list of validation violation errors.
-func (m TravisCIMultiError) AllErrors() []error { return m }
-
-// TravisCIValidationError is the validation error returned by
-// TravisCI.Validate if the designated constraints aren't met.
-type TravisCIValidationError struct {
-	field  string
-	reason string
-	cause  error
-	key    bool
-}
-
-// Field function returns field value.
-func (e TravisCIValidationError) Field() string { return e.field }
-
-// Reason function returns reason value.
-func (e TravisCIValidationError) Reason() string { return e.reason }
-
-// Cause function returns cause value.
-func (e TravisCIValidationError) Cause() error { return e.cause }
-
-// Key function returns key value.
-func (e TravisCIValidationError) Key() bool { return e.key }
-
-// ErrorName returns error name.
-func (e TravisCIValidationError) ErrorName() string { return "TravisCIValidationError" }
-
-// Error satisfies the builtin error interface
-func (e TravisCIValidationError) Error() string {
-	cause := ""
-	if e.cause != nil {
-		cause = fmt.Sprintf(" | caused by: %v", e.cause)
-	}
-
-	key := ""
-	if e.key {
-		key = "key for "
-	}
-
-	return fmt.Sprintf(
-		"invalid %sTravisCI.%s: %s%s",
-		key,
-		e.field,
-		e.reason,
-		cause)
-}
-
-var _ error = TravisCIValidationError{}
-
-var _ interface {
-	Field() string
-	Reason() string
-	Key() bool
-	Cause() error
-	ErrorName() string
-} = TravisCIValidationError{}
-
 // Validate checks the field values on Confluence with the rules defined in the
 // proto definition for this message. If any rules are violated, the first
 // error encountered is returned, or nil if there are no violations.
@@ -4037,37 +3926,6 @@ func (m *MetaData) validate(all bool) error {
 			}
 		}
 
-	case *MetaData_TravisCI:
-
-		if all {
-			switch v := interface{}(m.GetTravisCI()).(type) {
-			case interface{ ValidateAll() error }:
-				if err := v.ValidateAll(); err != nil {
-					errors = append(errors, MetaDataValidationError{
-						field:  "TravisCI",
-						reason: "embedded message failed validation",
-						cause:  err,
-					})
-				}
-			case interface{ Validate() error }:
-				if err := v.Validate(); err != nil {
-					errors = append(errors, MetaDataValidationError{
-						field:  "TravisCI",
-						reason: "embedded message failed validation",
-						cause:  err,
-					})
-				}
-			}
-		} else if v, ok := interface{}(m.GetTravisCI()).(interface{ Validate() error }); ok {
-			if err := v.Validate(); err != nil {
-				return MetaDataValidationError{
-					field:  "TravisCI",
-					reason: "embedded message failed validation",
-					cause:  err,
-				}
-			}
-		}
-
 	}
 
 	if len(errors) > 0 {

pkg/pb/sourcespb/sources.pb.validate.go

@@ -667,124 +667,6 @@ var _ interface {
 	ErrorName() string
 } = CircleCIValidationError{}
 
-// Validate checks the field values on TravisCI with the rules defined in the
-// proto definition for this message. If any rules are violated, the first
-// error encountered is returned, or nil if there are no violations.
-func (m *TravisCI) Validate() error {
-	return m.validate(false)
-}
-
-// ValidateAll checks the field values on TravisCI with the rules defined in
-// the proto definition for this message. If any rules are violated, the
-// result is a list of violation errors wrapped in TravisCIMultiError, or nil
-// if none found.
-func (m *TravisCI) ValidateAll() error {
-	return m.validate(true)
-}
-
-func (m *TravisCI) validate(all bool) error {
-	if m == nil {
-		return nil
-	}
-
-	var errors []error
-
-	if _, err := url.Parse(m.GetEndpoint()); err != nil {
-		err = TravisCIValidationError{
-			field:  "Endpoint",
-			reason: "value must be a valid URI",
-			cause:  err,
-		}
-		if !all {
-			return err
-		}
-		errors = append(errors, err)
-	}
-
-	switch m.Credential.(type) {
-
-	case *TravisCI_Token:
-		// no validation rules for Token
-
-	}
-
-	if len(errors) > 0 {
-		return TravisCIMultiError(errors)
-	}
-
-	return nil
-}
-
-// TravisCIMultiError is an error wrapping multiple validation errors returned
-// by TravisCI.ValidateAll() if the designated constraints aren't met.
-type TravisCIMultiError []error
-
-// Error returns a concatenation of all the error messages it wraps.
-func (m TravisCIMultiError) Error() string {
-	var msgs []string
-	for _, err := range m {
-		msgs = append(msgs, err.Error())
-	}
-	return strings.Join(msgs, "; ")
-}
-
-// AllErrors returns a list of validation violation errors.
-func (m TravisCIMultiError) AllErrors() []error { return m }
-
-// TravisCIValidationError is the validation error returned by
-// TravisCI.Validate if the designated constraints aren't met.
-type TravisCIValidationError struct {
-	field  string
-	reason string
-	cause  error
-	key    bool
-}
-
-// Field function returns field value.
-func (e TravisCIValidationError) Field() string { return e.field }
-
-// Reason function returns reason value.
-func (e TravisCIValidationError) Reason() string { return e.reason }
-
-// Cause function returns cause value.
-func (e TravisCIValidationError) Cause() error { return e.cause }
-
-// Key function returns key value.
-func (e TravisCIValidationError) Key() bool { return e.key }
-
-// ErrorName returns error name.
-func (e TravisCIValidationError) ErrorName() string { return "TravisCIValidationError" }
-
-// Error satisfies the builtin error interface
-func (e TravisCIValidationError) Error() string {
-	cause := ""
-	if e.cause != nil {
-		cause = fmt.Sprintf(" | caused by: %v", e.cause)
-	}
-
-	key := ""
-	if e.key {
-		key = "key for "
-	}
-
-	return fmt.Sprintf(
-		"invalid %sTravisCI.%s: %s%s",
-		key,
-		e.field,
-		e.reason,
-		cause)
-}
-
-var _ error = TravisCIValidationError{}
-
-var _ interface {
-	Field() string
-	Reason() string
-	Key() bool
-	Cause() error
-	ErrorName() string
-} = TravisCIValidationError{}
-
 // Validate checks the field values on Confluence with the rules defined in the
 // proto definition for this message. If any rules are violated, the first
 // error encountered is returned, or nil if there are no violations.

pkg/sources/errors_test.go

@@ -1,8 +1,8 @@
 package sources
 
 import (
-	"errors"
 	"fmt"
+	"reflect"
 	"sync"
 	"testing"
 )
@@ -40,7 +40,7 @@ func TestNewScanErrors(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			got := NewScanErrors()
 
-			if !errors.Is(got.Errors(), tc.want.Errors()) {
+			if !reflect.DeepEqual(got, tc.want) {
 				t.Errorf("got %+v, want %+v", got, tc.want)
 			}
 		})

pkg/sources/github/github.go

@@ -233,7 +233,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
 			aCtx.Logger().Error(err, "invalid repository", "repo", repo)
 			continue
 		}
-		s.filteredRepoCache.Set(r, r)
+		s.filteredRepoCache.Set(repo, r)
 	}
 
 	s.includeIssueComments = s.conn.IncludeIssueComments
@@ -825,38 +825,51 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
 // Unauthenticated access to most github endpoints has a rate limit of 60 requests per hour.
 // This will likely only be exhausted if many users/orgs are scanned without auth
 func (s *Source) handleRateLimit(errIn error, res *github.Response) bool {
-	limit, ok := errIn.(*github.RateLimitError)
-	if !ok {
+	var (
+		knownWait  = true
+		remaining  = 0
+		retryAfter time.Duration
+	)
+
+	// GitHub has both primary (RateLimit) and secondary (AbuseRateLimit) errors.
+	var rateLimit *github.RateLimitError
+	var abuseLimit *github.AbuseRateLimitError
+	if errors.As(errIn, &rateLimit) {
+		// Do nothing
+	} else if errors.As(errIn, &abuseLimit) {
+		retryAfter = abuseLimit.GetRetryAfter()
+	} else {
 		return false
 	}
 
 	githubNumRateLimitEncountered.WithLabelValues(s.name).Inc()
-
-	if res != nil {
-		knownWait := true
-		remaining, err := strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
+	// Parse retry information from response headers, unless a Retry-After value was already provided.
+	// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#exceeding-the-rate-limit
+	if retryAfter <= 0 && res != nil {
+		var err error
+		remaining, err = strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
 		if err != nil {
 			knownWait = false
 		}
+
 		resetTime, err := strconv.Atoi(res.Header.Get("x-ratelimit-reset"))
 		if err != nil || resetTime == 0 {
 			knownWait = false
-		}
-
-		if knownWait && remaining == 0 {
-			waitTime := int64(resetTime) - time.Now().Unix()
-			if waitTime > 0 {
-				duration := time.Duration(waitTime+1) * time.Second
-				s.log.V(2).Info("rate limited", "resumeTime", time.Now().Add(duration).String())
-				time.Sleep(duration)
-				githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(duration.Seconds())
-				return true
-			}
+		} else if resetTime > 0 {
+			retryAfter = time.Duration(int64(resetTime)-time.Now().Unix()) * time.Second
 		}
 	}
 
-	s.log.V(2).Info("handling rate limit (5 minutes retry)", "retry-after", limit.Message)
-	time.Sleep(time.Minute * 5)
+	resumeTime := time.Now().Add(retryAfter).String()
+	if knownWait && remaining == 0 && retryAfter > 0 {
+		s.log.V(2).Info("rate limited", "retry_after", retryAfter.String(), "resume_time", resumeTime)
+	} else {
+		// TODO: Use exponential backoff instead of static retry time.
+		retryAfter = time.Minute * 5
+		s.log.V(2).Error(errIn, "unexpected rate limit error", "retry_after", retryAfter.String(), "resume_time", resumeTime)
+	}
+	time.Sleep(retryAfter)
+	githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(retryAfter.Seconds())
 	return true
 }
 
@@ -1131,9 +1144,10 @@ var (
 )
 
 type repoInfo struct {
-	owner    string
-	repo     string
-	repoPath string
+	owner      string
+	repo       string
+	repoPath   string
+	visibility source_metadatapb.Visibility
 }
 
 func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimmedURL []string, repoURL *url.URL, chunksChan chan *sources.Chunk) error {
@@ -1144,7 +1158,16 @@ func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimm
 	owner := trimmedURL[1]
 	repo := trimmedURL[2]
 
-	repoInfo := repoInfo{owner: owner, repo: repo, repoPath: repoPath}
+	if !(s.includeIssueComments || s.includePRComments) {
+		return nil
+	}
+
+	repoInfo := repoInfo{
+		owner:      owner,
+		repo:       repo,
+		repoPath:   repoPath,
+		visibility: s.visibilityOf(ctx, repoPath),
+	}
 
 	if s.includeIssueComments {
 		if err := s.processIssueComments(ctx, repoInfo, chunksChan); err != nil {
@@ -1191,7 +1214,7 @@ func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan ch
 			return err
 		}
 
-		if err = s.chunkIssues(ctx, info.repo, info.repoPath, issues, chunksChan); err != nil {
+		if err = s.chunkIssues(ctx, info, issues, chunksChan); err != nil {
 			return err
 		}
 
@@ -1226,7 +1249,7 @@ func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunks
 			return err
 		}
 
-		if err = s.chunkIssueComments(ctx, info.repo, info.repoPath, issueComments, chunksChan); err != nil {
+		if err = s.chunkIssueComments(ctx, info, issueComments, chunksChan); err != nil {
 			return err
 		}
 
@@ -1262,7 +1285,7 @@ func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan
 			return err
 		}
 
-		if err = s.chunkPullRequests(ctx, info.repo, prs, chunksChan); err != nil {
+		if err = s.chunkPullRequests(ctx, info, prs, chunksChan); err != nil {
 			return err
 		}
 
@@ -1297,7 +1320,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
 			return err
 		}
 
-		if err = s.chunkPullRequestComments(ctx, info.repo, prComments, chunksChan); err != nil {
+		if err = s.chunkPullRequestComments(ctx, info, prComments, chunksChan); err != nil {
 			return err
 		}
 
@@ -1310,7 +1333,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
 	return nil
 }
 
-func (s *Source) chunkIssues(ctx context.Context, repo, repoPath string, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
+func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
 	for _, issue := range issues {
 
 		// Skip pull requests since covered by processPRs.
@@ -1330,9 +1353,9 @@ func (s *Source) chunkIssues(ctx context.Context, repo, repoPath string, issues
 						Link:       sanitizer.UTF8(issue.GetHTMLURL()),
 						Username:   sanitizer.UTF8(issue.GetUser().GetLogin()),
 						Email:      sanitizer.UTF8(issue.GetUser().GetEmail()),
-						Repository: sanitizer.UTF8(repo),
+						Repository: sanitizer.UTF8(repoInfo.repo),
 						Timestamp:  sanitizer.UTF8(issue.GetCreatedAt().String()),
-						Visibility: s.visibilityOf(ctx, repoPath),
+						Visibility: repoInfo.visibility,
 					},
 				},
 			},
@@ -1349,7 +1372,7 @@ func (s *Source) chunkIssues(ctx context.Context, repo, repoPath string, issues
 	return nil
 }
 
-func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
+func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
 	for _, comment := range comments {
 		// Create chunk and send it to the channel.
 		chunk := &sources.Chunk{
@@ -1363,9 +1386,9 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
 						Link:       sanitizer.UTF8(comment.GetHTMLURL()),
 						Username:   sanitizer.UTF8(comment.GetUser().GetLogin()),
 						Email:      sanitizer.UTF8(comment.GetUser().GetEmail()),
-						Repository: sanitizer.UTF8(repo),
+						Repository: sanitizer.UTF8(repoInfo.repo),
 						Timestamp:  sanitizer.UTF8(comment.GetCreatedAt().String()),
-						Visibility: s.visibilityOf(ctx, repoPath),
+						Visibility: repoInfo.visibility,
 					},
 				},
 			},
@@ -1382,7 +1405,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
 	return nil
 }
 
-func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
+func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
 	for _, comment := range comments {
 		// Create chunk and send it to the channel.
 		chunk := &sources.Chunk{
@@ -1396,8 +1419,9 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
 						Link:       sanitizer.UTF8(comment.GetHTMLURL()),
 						Username:   sanitizer.UTF8(comment.GetUser().GetLogin()),
 						Email:      sanitizer.UTF8(comment.GetUser().GetEmail()),
-						Repository: sanitizer.UTF8(repo),
+						Repository: sanitizer.UTF8(repoInfo.repo),
 						Timestamp:  sanitizer.UTF8(comment.GetCreatedAt().String()),
+						Visibility: repoInfo.visibility,
 					},
 				},
 			},
@@ -1414,7 +1438,7 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
 	return nil
 }
 
-func (s *Source) chunkPullRequests(ctx context.Context, repo string, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
+func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
 	for _, pr := range prs {
 		// Create chunk and send it to the channel.
 		chunk := &sources.Chunk{
@@ -1428,8 +1452,9 @@ func (s *Source) chunkPullRequests(ctx context.Context, repo string, prs []*gith
 						Link:       sanitizer.UTF8(pr.GetHTMLURL()),
 						Username:   sanitizer.UTF8(pr.GetUser().GetLogin()),
 						Email:      sanitizer.UTF8(pr.GetUser().GetEmail()),
-						Repository: sanitizer.UTF8(repo),
+						Repository: sanitizer.UTF8(repoInfo.repo),
 						Timestamp:  sanitizer.UTF8(pr.GetCreatedAt().String()),
+						Visibility: repoInfo.visibility,
 					},
 				},
 			},
@@ -1462,8 +1487,7 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
 						Email:      sanitizer.UTF8(comment.GetUser().GetEmail()),
 						Repository: sanitizer.UTF8(gistUrl),
 						Timestamp:  sanitizer.UTF8(comment.GetCreatedAt().String()),
-						// Fetching this information requires making an additional API call.
-						// We may want to include this in the future.
+						// TODO: Fetching this requires making an additional API call. We may want to include this in the future.
 						// Visibility: s.visibilityOf(ctx, repoPath),
 					},
 				},

pkg/sources/job_progress.go

@@ -89,14 +89,14 @@ func (f Fatal) Unwrap() error { return f.error }
 // ChunkError is a custom error type for errors encountered during chunking of
 // a specific unit.
 type ChunkError struct {
-	unit SourceUnit
-	err  error
+	Unit SourceUnit
+	Err  error
 }
 
 func (f ChunkError) Error() string {
-	return fmt.Sprintf("error chunking unit %q: %s", f.unit.SourceUnitID(), f.err.Error())
+	return fmt.Sprintf("error chunking unit %q: %s", f.Unit.SourceUnitID(), f.Err.Error())
 }
-func (f ChunkError) Unwrap() error { return f.err }
+func (f ChunkError) Unwrap() error { return f.Err }
 
 // JobProgress aggregates information about a run of a Source.
 type JobProgress struct {
@@ -351,3 +351,17 @@ func (m JobProgressMetrics) ElapsedTime() time.Duration {
 	}
 	return m.EndTime.Sub(m.StartTime)
 }
+
+// ErrorsFor returns all the errors for the given SourceUnit. If there are no
+// errors, including the case that the unit has not been encountered, nil will
+// be returned.
+func (m JobProgressMetrics) ErrorsFor(unit SourceUnit) []error {
+	var errs []error
+	for _, err := range m.Errors {
+		var chunkErr ChunkError
+		if errors.As(err, &chunkErr) && chunkErr.Unit == unit {
+			errs = append(errs, err)
+		}
+	}
+	return errs
+}

pkg/sources/job_progress_hook.go

@@ -0,0 +1,218 @@
+package sources
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	lru "github.com/hashicorp/golang-lru/v2"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+)
+
+// UnitHook implements JobProgressHook for tracking the progress of each
+// individual unit.
+type UnitHook struct {
+	metrics *lru.Cache[string, *UnitMetrics]
+	mu      sync.Mutex
+	NoopHook
+}
+
+type UnitHookOpt func(*UnitHook)
+
+func WithUnitHookCache(cache *lru.Cache[string, *UnitMetrics]) UnitHookOpt {
+	return func(hook *UnitHook) { hook.metrics = cache }
+}
+
+func NewUnitHook(ctx context.Context, opts ...UnitHookOpt) *UnitHook {
+	// lru.NewWithEvict can only fail if the size is < 0.
+	cache, _ := lru.NewWithEvict(1024, func(key string, value *UnitMetrics) {
+		if value.handled {
+			return
+		}
+		ctx.Logger().Error(fmt.Errorf("eviction"), "dropping unit metric",
+			"id", key,
+			"metric", value,
+		)
+	})
+	hook := UnitHook{metrics: cache}
+	for _, opt := range opts {
+		opt(&hook)
+	}
+	return &hook
+}
+
+// id is a helper method to generate an ID for the given job and unit.
+func (u *UnitHook) id(ref JobProgressRef, unit SourceUnit) string {
+	unitID := ""
+	if unit != nil {
+		unitID = unit.SourceUnitID()
+	}
+	return fmt.Sprintf("%d/%d/%s", ref.SourceID, ref.JobID, unitID)
+}
+
+func (u *UnitHook) StartUnitChunking(ref JobProgressRef, unit SourceUnit, start time.Time) {
+	id := u.id(ref, unit)
+	u.mu.Lock()
+	defer u.mu.Unlock()
+
+	u.metrics.Add(id, &UnitMetrics{
+		Unit:      unit,
+		Parent:    ref,
+		StartTime: start,
+	})
+}
+
+func (u *UnitHook) EndUnitChunking(ref JobProgressRef, unit SourceUnit, end time.Time) {
+	id := u.id(ref, unit)
+	u.mu.Lock()
+	defer u.mu.Unlock()
+
+	metrics, ok := u.metrics.Get(id)
+	if !ok {
+		return
+	}
+	metrics.EndTime = end
+}
+
+func (u *UnitHook) ReportChunk(ref JobProgressRef, unit SourceUnit, chunk *Chunk) {
+	id := u.id(ref, unit)
+	u.mu.Lock()
+	defer u.mu.Unlock()
+
+	metrics, ok := u.metrics.Get(id)
+	if !ok && unit != nil {
+		// The unit has been evicted.
+		return
+	} else if !ok && unit == nil {
+		// This is a chunk from a non-unit source.
+		metrics = &UnitMetrics{
+			Unit:      nil,
+			Parent:    ref,
+			StartTime: ref.Snapshot().StartTime,
+		}
+		u.metrics.Add(id, metrics)
+	}
+	metrics.TotalChunks++
+	metrics.TotalBytes += uint64(len(chunk.Data))
+}
+
+func (u *UnitHook) ReportError(ref JobProgressRef, err error) {
+	u.mu.Lock()
+	defer u.mu.Unlock()
+
+	// Always add the error to the nil unit if it exists.
+	if metrics, ok := u.metrics.Get(u.id(ref, nil)); ok {
+		metrics.Errors = append(metrics.Errors, err)
+	}
+
+	// Check if it's a ChunkError for a specific unit.
+	var chunkErr ChunkError
+	if !errors.As(err, &chunkErr) {
+		return
+	}
+	id := u.id(ref, chunkErr.Unit)
+
+	metrics, ok := u.metrics.Get(id)
+	if !ok {
+		return
+	}
+	metrics.Errors = append(metrics.Errors, err)
+}
+
+func (u *UnitHook) Finish(ref JobProgressRef) {
+	u.mu.Lock()
+	defer u.mu.Unlock()
+	// Clear out any metrics on this job. This covers the case for the
+	// source running without unit support.
+	prefix := u.id(ref, nil)
+	for _, id := range u.metrics.Keys() {
+		if !strings.HasPrefix(id, prefix) {
+			continue
+		}
+		metric, ok := u.metrics.Get(id)
+		if !ok {
+			continue
+		}
+		// If the unit is nil, the source does not support units.
+		// Use the overall job metrics instead.
+		if metric.Unit == nil {
+			snap := ref.Snapshot()
+			metric.StartTime = snap.StartTime
+			metric.EndTime = snap.EndTime
+			metric.Errors = snap.Errors
+		}
+	}
+}
+
+// UnitMetrics gets all the currently active or newly finished metrics for this
+// job. If a unit returned from this method has finished, it will be removed
+// from the cache and no longer returned in successive calls to UnitMetrics().
+func (u *UnitHook) UnitMetrics() []UnitMetrics {
+	u.mu.Lock()
+	defer u.mu.Unlock()
+	output := make([]UnitMetrics, 0, u.metrics.Len())
+	for _, id := range u.metrics.Keys() {
+		metric, ok := u.metrics.Get(id)
+		if !ok {
+			continue
+		}
+		output = append(output, *metric)
+		if metric.IsFinished() {
+			metric.handled = true
+			u.metrics.Remove(id)
+		}
+	}
+	return output
+}
+
+type UnitMetrics struct {
+	Unit   SourceUnit
+	Parent JobProgressRef
+	// Start and end time for chunking this unit.
+	StartTime time.Time
+	EndTime   time.Time
+	// Total number of chunks produced from this unit.
+	TotalChunks uint64
+	// Total number of bytes produced from this unit.
+	TotalBytes uint64
+	// All errors encountered by this unit.
+	Errors []error
+	// Flag to mark that these metrics were intentionally evicted from
+	// the cache.
+	handled bool
+}
+
+func (u UnitMetrics) IsFinished() bool {
+	return !u.EndTime.IsZero()
+}
+
+// ElapsedTime is a convenience method that provides the elapsed time the job
+// has been running. If it hasn't started yet, 0 is returned. If it has
+// finished, the total time is returned.
+func (u UnitMetrics) ElapsedTime() time.Duration {
+	if u.StartTime.IsZero() {
+		return 0
+	}
+	if u.EndTime.IsZero() {
+		return time.Since(u.StartTime)
+	}
+	return u.EndTime.Sub(u.StartTime)
+}
+
+// NoopHook implements JobProgressHook by doing nothing. This is useful for
+// embedding in other structs to overwrite only the methods of the interface
+// that you care about.
+type NoopHook struct{}
+
+func (NoopHook) Start(JobProgressRef, time.Time)                         {}
+func (NoopHook) End(JobProgressRef, time.Time)                           {}
+func (NoopHook) StartEnumerating(JobProgressRef, time.Time)              {}
+func (NoopHook) EndEnumerating(JobProgressRef, time.Time)                {}
+func (NoopHook) StartUnitChunking(JobProgressRef, SourceUnit, time.Time) {}
+func (NoopHook) EndUnitChunking(JobProgressRef, SourceUnit, time.Time)   {}
+func (NoopHook) ReportError(JobProgressRef, error)                       {}
+func (NoopHook) ReportUnit(JobProgressRef, SourceUnit)                   {}
+func (NoopHook) ReportChunk(JobProgressRef, SourceUnit, *Chunk)          {}
+func (NoopHook) Finish(JobProgressRef)                                   {}

pkg/sources/job_progress_test.go

@@ -126,3 +126,26 @@ func TestJobProgressElapsedTime(t *testing.T) {
 	metrics.EndTime = metrics.StartTime.Add(1 * time.Hour)
 	assert.Equal(t, metrics.ElapsedTime(), 1*time.Hour)
 }
+
+func TestJobProgressErrorsFor(t *testing.T) {
+	metrics := JobProgressMetrics{
+		Errors: []error{
+			Fatal{ChunkError{
+				Unit: CommonSourceUnit{ID: "foo"},
+				Err:  fmt.Errorf("foo error"),
+			}},
+			ChunkError{
+				Unit: CommonSourceUnit{ID: "foo"},
+				Err:  fmt.Errorf("foo again error"),
+			},
+			ChunkError{
+				Unit: CommonSourceUnit{ID: "bar"},
+				Err:  fmt.Errorf("bar error"),
+			},
+			fmt.Errorf("hi there"),
+		},
+	}
+	assert.Equal(t, 2, len(metrics.ErrorsFor(CommonSourceUnit{ID: "foo"})))
+	assert.Equal(t, 1, len(metrics.ErrorsFor(CommonSourceUnit{ID: "bar"})))
+	assert.Equal(t, 0, len(metrics.ErrorsFor(CommonSourceUnit{ID: "baz"})))
+}

pkg/sources/source_manager_test.go

@@ -3,8 +3,10 @@ package sources
 import (
 	"errors"
 	"fmt"
+	"sort"
 	"testing"
 
+	lru "github.com/hashicorp/golang-lru/v2"
 	"github.com/stretchr/testify/assert"
 	"google.golang.org/protobuf/types/known/anypb"
 
@@ -200,6 +202,8 @@ func (c *unitChunker) ChunkUnit(ctx context.Context, unit SourceUnit, rep ChunkR
 			if err := rep.ChunkErr(ctx, fmt.Errorf(step.err)); err != nil {
 				return err
 			}
+		}
+		if step.output == "" {
 			continue
 		}
 		if err := rep.ChunkOk(ctx, Chunk{Data: []byte(step.output)}); err != nil {
@@ -310,3 +314,114 @@ func TestSourceManagerAvailableCapacity(t *testing.T) {
 	<-ref.Done()      // Wait for the job to finish.
 	assert.Equal(t, 1337, mgr.AvailableCapacity())
 }
+
+func TestSourceManagerUnitHook(t *testing.T) {
+	hook := NewUnitHook(context.TODO())
+
+	input := []unitChunk{
+		{unit: "one", output: "bar"},
+		{unit: "two", err: "oh no"},
+		{unit: "three", err: "not again"},
+	}
+	mgr := NewManager(
+		WithBufferedOutput(8),
+		WithSourceUnits(), WithConcurrentUnits(1),
+		WithReportHook(hook),
+	)
+	source, err := buildDummy(&unitChunker{input})
+	assert.NoError(t, err)
+	ref, err := mgr.Run(context.Background(), "dummy", source)
+	assert.NoError(t, err)
+	<-ref.Done()
+
+	metrics := hook.UnitMetrics()
+	assert.Equal(t, 3, len(metrics))
+	sort.Slice(metrics, func(i, j int) bool {
+		return metrics[i].EndTime.Before(metrics[j].EndTime)
+	})
+	m0, m1, m2 := metrics[0], metrics[1], metrics[2]
+
+	assert.Equal(t, "one", m0.Unit.SourceUnitID())
+	assert.Equal(t, uint64(1), m0.TotalChunks)
+	assert.Equal(t, uint64(3), m0.TotalBytes)
+	assert.NotZero(t, m0.StartTime)
+	assert.NotZero(t, m0.EndTime)
+	assert.NotZero(t, m0.ElapsedTime())
+	assert.Equal(t, 0, len(m0.Errors))
+
+	assert.Equal(t, "two", m1.Unit.SourceUnitID())
+	assert.Equal(t, uint64(0), m1.TotalChunks)
+	assert.Equal(t, uint64(0), m1.TotalBytes)
+	assert.NotZero(t, m1.StartTime)
+	assert.NotZero(t, m1.EndTime)
+	assert.NotZero(t, m1.ElapsedTime())
+	assert.Equal(t, 1, len(m1.Errors))
+
+	assert.Equal(t, "three", m2.Unit.SourceUnitID())
+	assert.Equal(t, uint64(0), m2.TotalChunks)
+	assert.Equal(t, uint64(0), m2.TotalBytes)
+	assert.NotZero(t, m2.StartTime)
+	assert.NotZero(t, m2.EndTime)
+	assert.NotZero(t, m2.ElapsedTime())
+	assert.Equal(t, 1, len(m2.Errors))
+}
+
+// TestSourceManagerUnitHookNoBlock tests that the UnitHook drops metrics if
+// they aren't handled fast enough.
+func TestSourceManagerUnitHookNoBlock(t *testing.T) {
+	var evictedKeys []string
+	cache, _ := lru.NewWithEvict(1, func(key string, _ *UnitMetrics) {
+		evictedKeys = append(evictedKeys, key)
+	})
+	hook := NewUnitHook(context.TODO(), WithUnitHookCache(cache))
+
+	input := []unitChunk{
+		{unit: "one", output: "bar"},
+		{unit: "two", err: "oh no"},
+		{unit: "three", err: "not again"},
+	}
+	mgr := NewManager(
+		WithBufferedOutput(8),
+		WithSourceUnits(), WithConcurrentUnits(1),
+		WithReportHook(hook),
+	)
+	source, err := buildDummy(&unitChunker{input})
+	assert.NoError(t, err)
+	ref, err := mgr.Run(context.Background(), "dummy", source)
+	assert.NoError(t, err)
+	<-ref.Done()
+
+	assert.Equal(t, 2, len(evictedKeys))
+	metrics := hook.UnitMetrics()
+	assert.Equal(t, 1, len(metrics))
+	assert.Equal(t, "three", metrics[0].Unit.SourceUnitID())
+}
+
+// TestSourceManagerUnitHookNoUnits tests whether the UnitHook works for
+// sources that don't support units.
+func TestSourceManagerUnitHookNoUnits(t *testing.T) {
+	hook := NewUnitHook(context.TODO())
+
+	mgr := NewManager(
+		WithBufferedOutput(8),
+		WithReportHook(hook),
+	)
+	source, err := buildDummy(&counterChunker{count: 5})
+	assert.NoError(t, err)
+
+	ref, err := mgr.Run(context.Background(), "dummy", source)
+	assert.NoError(t, err)
+	<-ref.Done()
+
+	metrics := hook.UnitMetrics()
+	assert.Equal(t, 1, len(metrics))
+
+	m := metrics[0]
+	assert.Equal(t, nil, m.Unit)
+	assert.Equal(t, uint64(5), m.TotalChunks)
+	assert.Equal(t, uint64(5), m.TotalBytes)
+	assert.NotZero(t, m.StartTime)
+	assert.NotZero(t, m.EndTime)
+	assert.NotZero(t, m.ElapsedTime())
+	assert.Equal(t, 0, len(m.Errors))
+}

pkg/sources/travisci/travisci.go

@@ -1,223 +0,0 @@
-package travisci
-
-import (
-	"fmt"
-	"strconv"
-
-	"github.com/go-errors/errors"
-	"github.com/shuheiktgw/go-travis"
-	"golang.org/x/sync/errgroup"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/anypb"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
-)
-
-const (
-	SourceType = sourcespb.SourceType_SOURCE_TYPE_TRAVISCI
-
-	baseURL = "https://api.travis-ci.com/"
-
-	pageSize = 100
-)
-
-type Source struct {
-	name     string
-	sourceId sources.SourceID
-	jobId    sources.JobID
-	verify   bool
-	jobPool  *errgroup.Group
-	sources.Progress
-	client *travis.Client
-	sources.CommonSourceUnitUnmarshaller
-	returnAfterFirstChunk bool
-}
-
-// Ensure the Source satisfies the interfaces at compile time.
-var _ sources.Source = (*Source)(nil)
-var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
-
-// Type returns the type of source.
-// It is used for matching source types in configuration and job input.
-func (s *Source) Type() sourcespb.SourceType {
-	return SourceType
-}
-
-func (s *Source) SourceID() sources.SourceID {
-	return s.sourceId
-}
-
-func (s *Source) JobID() sources.JobID {
-	return s.jobId
-}
-
-// Init returns an initialized TravisCI source.
-func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
-	s.name = name
-	s.sourceId = sourceId
-	s.jobId = jobId
-	s.verify = verify
-	s.jobPool = &errgroup.Group{}
-	s.jobPool.SetLimit(concurrency)
-
-	var conn sourcespb.TravisCI
-	if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
-		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
-	}
-
-	switch conn.Credential.(type) {
-	case *sourcespb.TravisCI_Token:
-		if conn.GetToken() == "" {
-			return errors.New("token is empty")
-		}
-		s.client = travis.NewClient(baseURL, conn.GetToken())
-		s.client.HTTPClient = common.RetryableHttpClientTimeout(3)
-
-		user, _, err := s.client.User.Current(ctx, nil)
-		if err != nil {
-			return errors.WrapPrefix(err, "error getting testing travis client", 0)
-		}
-		ctx.Logger().V(2).Info("authenticated to Travis CI with user", "username", user.Login)
-	default:
-		return errors.New("credential type not implemented for Travis CI")
-	}
-
-	return nil
-}
-
-func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {
-	for repoPage := 0; ; repoPage++ {
-		repositories, _, err := s.client.Repositories.List(ctx, &travis.RepositoriesOption{
-			Limit:  pageSize,
-			Offset: repoPage * pageSize,
-		})
-		if err != nil {
-			if repoPage == 0 {
-				return fmt.Errorf("error listing repositories: %w", err)
-			}
-			err = reporter.UnitErr(ctx, err)
-			if err != nil {
-				return fmt.Errorf("error reporting error: %w", err)
-			}
-		}
-
-		if len(repositories) == 0 {
-			break
-		}
-
-		for _, repo := range repositories {
-			err = reporter.UnitOk(ctx, sources.CommonSourceUnit{
-				ID: strconv.Itoa(int(*repo.Id)),
-			})
-			if err != nil {
-				return fmt.Errorf("error reporting unit: %w", err)
-			}
-			ctx.Logger().V(2).Info("enumerated repository", "id", repo.Id, "name", repo.Name)
-		}
-	}
-
-	return nil
-}
-
-// ChunkUnit implements SourceUnitChunker interface.
-func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
-	repo, _, err := s.client.Repositories.Find(ctx, unit.SourceUnitID(), nil)
-	if err != nil {
-		return fmt.Errorf("error finding repository: %w", err)
-	}
-	logger := ctx.Logger().WithValues("repo", *repo.Name)
-	logger.V(2).Info("scanning repository")
-
-	// Counts continuous errors from ListByRepoSlug. Used to quit early in
-	// case the API always returns an error.
-	var buildPageErrs int
-	for buildPage := 0; ; buildPage++ {
-		builds, _, err := s.client.Builds.ListByRepoSlug(ctx, *repo.Slug, &travis.BuildsByRepoOption{
-			Limit:  pageSize,
-			Offset: buildPage * pageSize,
-		})
-		if err != nil {
-			if err := reporter.ChunkErr(ctx, err); err != nil {
-				return err
-			}
-			buildPageErrs++
-			if buildPageErrs >= 5 {
-				return fmt.Errorf("encountered too many errors listing builds, aborting")
-			}
-			continue
-		}
-		// Reset the page error counter.
-		buildPageErrs = 0
-
-		if len(builds) == 0 {
-			break
-		}
-
-		for _, build := range builds {
-			jobs, _, err := s.client.Jobs.ListByBuild(ctx, *build.Id)
-			if err != nil {
-				if err := reporter.ChunkErr(ctx, err); err != nil {
-					return err
-				}
-				continue
-			}
-
-			if len(jobs) == 0 {
-				break
-			}
-
-			for _, job := range jobs {
-				log, _, err := s.client.Logs.FindByJobId(ctx, *job.Id)
-				if err != nil {
-					if err := reporter.ChunkErr(ctx, err); err != nil {
-						return err
-					}
-					continue
-				}
-
-				logger.V(3).Info("scanning job", "id", *job.Id, "number", *job.Number)
-
-				chunk := sources.Chunk{
-					SourceType: s.Type(),
-					SourceName: s.name,
-					SourceID:   s.SourceID(),
-					JobID:      s.JobID(),
-					Data:       []byte(*log.Content),
-					SourceMetadata: &source_metadatapb.MetaData{
-						Data: &source_metadatapb.MetaData_TravisCI{
-							TravisCI: &source_metadatapb.TravisCI{
-								Username:    *job.Owner.Login,
-								Repository:  *repo.Name,
-								BuildNumber: *build.Number,
-								JobNumber:   *job.Number,
-								Link:        fmt.Sprintf("https://app.travis-ci.com/github/%s/%s/jobs/%d", *job.Owner.Login, *repo.Name, *job.Id),
-								Public:      !*repo.Private,
-							},
-						},
-					},
-					Verify: s.verify,
-				}
-
-				if err := reporter.ChunkOk(ctx, chunk); err != nil {
-					return err
-				}
-
-				if s.returnAfterFirstChunk {
-					return nil
-				}
-			}
-		}
-	}
-
-	return nil
-}
-
-// Chunks emits chunks of bytes over a channel.
-// It's a no-op because we've implemented the SourceUnitChunker interface instead.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
-	return nil
-}

pkg/sources/travisci/travisci_test.go

@@ -1,99 +0,0 @@
-package travisci
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/kylelemons/godebug/pretty"
-	"google.golang.org/protobuf/types/known/anypb"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest"
-)
-
-func TestSource_Scan(t *testing.T) {
-	ctx := context.Background()
-
-	secret, err := common.GetTestSecret(ctx)
-	if err != nil {
-		t.Fatal(fmt.Errorf("failed to access secret: %v", err))
-	}
-	token := secret.MustGetField("TRAVISCI_TOKEN")
-
-	type init struct {
-		name       string
-		verify     bool
-		connection *sourcespb.TravisCI
-	}
-	tests := []struct {
-		name               string
-		init               init
-		wantSourceMetadata *source_metadatapb.MetaData
-		wantErr            bool
-	}{
-		{
-			name: "get a chunk",
-			init: init{
-				name: "example repo",
-				connection: &sourcespb.TravisCI{
-					Credential: &sourcespb.TravisCI_Token{
-						Token: token,
-					},
-				},
-				verify: true,
-			},
-			wantSourceMetadata: &source_metadatapb.MetaData{
-				Data: &source_metadatapb.MetaData_TravisCI{
-					TravisCI: &source_metadatapb.TravisCI{
-						Username:    "truffle-sandbox",
-						Repository:  "test-repo",
-						BuildNumber: "1",
-						JobNumber:   "1.1",
-						Link:        "https://app.travis-ci.com/github/truffle-sandbox/test-repo/jobs/611053994",
-						Public:      false,
-					},
-				},
-			},
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Source{}
-
-			conn, err := anypb.New(tt.init.connection)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5)
-			if (err != nil) != tt.wantErr {
-				t.Fatalf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
-			}
-
-			reporter := sourcestest.TestReporter{}
-			s.returnAfterFirstChunk = true
-
-			err = s.Enumerate(ctx, &reporter)
-			if (err != nil) != tt.wantErr {
-				t.Fatalf("Source.Enumerate() error = %v, wantErr %v", err, tt.wantErr)
-			}
-
-			for _, unit := range reporter.Units {
-				err = s.ChunkUnit(ctx, unit, &reporter)
-				if (err != nil) != tt.wantErr {
-					t.Fatalf("Source.ChunkUnit() error = %v, wantErr %v", err, tt.wantErr)
-				}
-			}
-
-			for _, chunk := range reporter.Chunks {
-				if diff := pretty.Compare(chunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {
-					t.Fatalf("comparing chunks: %s diff: (-got +want)\n%s", tt.name, diff)
-				}
-			}
-		})
-	}
-}

proto/detectors.proto

@@ -945,6 +945,7 @@ enum DetectorType {
   Ramp = 934;
   Klaviyo = 935;
   SourcegraphCody = 936;
+  Voiceflow = 937;
 }
 
 message Result {

proto/source_metadata.proto

@@ -44,16 +44,6 @@ message CircleCI {
   string email = 7;
 }
 
-message TravisCI {
-  string username = 2;
-  string repository = 3;
-  string build_number = 4;
-  string job_number  = 5;
-  string link = 6;
-  bool public = 7;
-}
-
-
 message Confluence {
   string page = 1;
   string space = 2;
@@ -304,6 +294,5 @@ message MetaData {
     SharePoint sharepoint = 25;
     GoogleDrive googleDrive = 26;
     AzureRepos azureRepos = 27;
-    TravisCI travisCI = 28;
   }
 }

proto/sources.proto

@@ -44,7 +44,6 @@ enum SourceType {
   SOURCE_TYPE_SHAREPOINT = 29;
   SOURCE_TYPE_GCS_UNAUTHED = 30;
   SOURCE_TYPE_AZURE_REPOS = 31;
-  SOURCE_TYPE_TRAVISCI = 32;
 }
 
 message LocalSource {
@@ -84,13 +83,6 @@ message CircleCI {
   }
 }
 
-message TravisCI {
-  string endpoint = 1 [(validate.rules).string.uri_ref = true];
-  oneof credential {
-    string token = 2;
-  }
-}
-
 message Confluence {
   string endpoint = 1 [(validate.rules).string.uri_ref = true];
   oneof credential {