Compare commits

..

6 Commits

Author SHA1 Message Date
Dustin Decker 065984ee58 review updates 2023-10-18 09:05:07 -07:00
Miccah Castorina fe90effec1 Continue chunking on error 2023-10-16 16:42:33 -07:00
Miccah Castorina e5a34a9809 Remove jobPage loop
ListByBuild does not support pagination, so this was infinitely
repeating. https://developer.travis-ci.com/resource/jobs#find
2023-10-16 16:21:56 -07:00
Dustin Decker ec3c59d025 update test to use sourcestest 2023-10-16 14:00:22 -07:00
Dustin Decker c99e0b93f7 Merge branch 'main' of github.com:trufflesecurity/trufflehog into travis-scanning 2023-10-16 13:42:35 -07:00
Dustin Decker b960231155 Add TravisCI source 2023-10-08 19:44:13 -07:00
35 changed files with 2086 additions and 2315 deletions
+1 -2
View File
@@ -47,7 +47,6 @@ require (
github.com/h2non/filetype v1.1.3
github.com/hashicorp/go-retryablehttp v0.7.4
github.com/hashicorp/golang-lru v0.5.1
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/jlaffaye/ftp v0.2.0
github.com/joho/godotenv v1.5.1
github.com/jpillora/overseer v1.1.6
@@ -66,6 +65,7 @@ require (
github.com/prometheus/client_golang v1.16.0
github.com/rabbitmq/amqp091-go v1.8.1
github.com/sergi/go-diff v1.3.1
github.com/shuheiktgw/go-travis v0.3.1
github.com/snowflakedb/gosnowflake v1.6.23
github.com/stretchr/testify v1.8.4
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
@@ -82,7 +82,6 @@ require (
google.golang.org/protobuf v1.31.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/h2non/gock.v1 v1.1.2
pgregory.net/rapid v1.1.0
sigs.k8s.io/yaml v1.3.0
)
+3 -4
View File
@@ -369,6 +369,7 @@ github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5Px
github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=
github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
@@ -418,8 +419,6 @@ github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
@@ -607,6 +606,8 @@ github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
github.com/shuheiktgw/go-travis v0.3.1 h1:SAT16mi77ccqogOslnXxBXzXbpeyChaIYUwi2aJpVZY=
github.com/shuheiktgw/go-travis v0.3.1/go.mod h1:avnFFDqJDdRHwlF9tgqvYi3asQCm/HGL8aLxYiKa4Yg=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
@@ -1015,8 +1016,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+7 -3
View File
@@ -139,6 +139,9 @@ var (
dockerScan = cli.Command("docker", "Scan Docker Image")
dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, otherwise a image registry is assumed.").Required().Strings()
travisCiScan = cli.Command("travisci", "Scan TravisCI")
travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()
)
func init() {
@@ -152,9 +155,6 @@ func init() {
cli.Version("trufflehog " + version.BuildVersion)
//Support -h for help
cli.HelpFlag.Short('h')
if len(os.Args) <= 1 && isatty.IsTerminal(os.Stdout.Fd()) {
args := tui.Run()
if len(args) == 0 {
@@ -501,6 +501,10 @@ func run(state overseer.State) {
if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
logFatal(err, "Failed to scan CircleCI.")
}
case travisCiScan.FullCommand():
if err := e.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
logFatal(err, "Failed to scan TravisCI.")
}
case gcsScan.FullCommand():
cfg := sources.GCSConfig{
ProjectID: *gcsProjectID,
-120
View File
@@ -1,120 +0,0 @@
package common
import (
"fmt"
"github.com/gobwas/glob"
)
// Filter is a generic filter for excluding and including globs (limited
// regular expressions). Exclusion takes precedence if both include and exclude
// lists are provided.
type Filter struct {
exclude []glob.Glob
include []glob.Glob
}
type globFilterOpt func(*Filter) error
// WithExcludeGlobs adds exclude globs to the filter.
func WithExcludeGlobs(excludes ...string) globFilterOpt {
return func(f *Filter) error {
for _, exclude := range excludes {
g, err := glob.Compile(exclude)
if err != nil {
return fmt.Errorf("invalid exclude glob %q: %w", exclude, err)
}
f.exclude = append(f.exclude, g)
}
return nil
}
}
// WithIncludeGlobs adds include globs to the filter.
func WithIncludeGlobs(includes ...string) globFilterOpt {
return func(f *Filter) error {
for _, include := range includes {
g, err := glob.Compile(include)
if err != nil {
return fmt.Errorf("invalid include glob %q: %w", include, err)
}
f.include = append(f.include, g)
}
return nil
}
}
// NewGlobFilter creates a new Filter with the provided options.
func NewGlobFilter(opts ...globFilterOpt) (*Filter, error) {
filter := &Filter{}
for _, opt := range opts {
if err := opt(filter); err != nil {
return nil, err
}
}
return filter, nil
}
// ShouldInclude returns whether the object is in the include list or not in
// the exclude list (exclude taking precedence).
func (f *Filter) ShouldInclude(object string) bool {
if f == nil {
return true
}
exclude, include := len(f.exclude), len(f.include)
if exclude == 0 && include == 0 {
return true
} else if exclude > 0 && include == 0 {
return f.shouldIncludeFromExclude(object)
} else if exclude == 0 && include > 0 {
return f.shouldIncludeFromInclude(object)
} else {
if ok, err := f.shouldIncludeFromBoth(object); err == nil {
return ok
}
// Ambiguous case.
return false
}
}
// shouldIncludeFromExclude checks for explicitly excluded paths. This should
// only be called when the include list is empty.
func (f *Filter) shouldIncludeFromExclude(object string) bool {
for _, glob := range f.exclude {
if glob.Match(object) {
return false
}
}
return true
}
// shouldIncludeFromInclude checks for explicitly included paths. This should
// only be called when the exclude list is empty.
func (f *Filter) shouldIncludeFromInclude(object string) bool {
for _, glob := range f.include {
if glob.Match(object) {
return true
}
}
return false
}
// shouldIncludeFromBoth checks for either excluded or included paths. Exclusion
// takes precedence. If neither list contains the object, true is returned.
func (f *Filter) shouldIncludeFromBoth(object string) (bool, error) {
// Exclude takes precedence. If we find the object in the exclude list,
// we should not match.
for _, glob := range f.exclude {
if glob.Match(object) {
return false, nil
}
}
// If we find the object in the include list, we should match.
for _, glob := range f.include {
if glob.Match(object) {
return true, nil
}
}
// If we find it in neither, return an error to let the caller decide.
return false, fmt.Errorf("ambiguous match")
}
-138
View File
@@ -1,138 +0,0 @@
package common
import (
"testing"
"github.com/stretchr/testify/assert"
"pgregory.net/rapid"
)
type globTest struct {
input string
shouldInclude bool
}
func testGlobs(t *testing.T, filter *Filter, tests ...globTest) {
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
// Invert because mentally it's easier to say whether an
// input should be included.
assert.Equal(t, tt.shouldInclude, filter.ShouldInclude(tt.input))
})
}
}
func TestGlobFilterExclude(t *testing.T) {
filter, err := NewGlobFilter(WithExcludeGlobs("foo", "bar*"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", false},
globTest{"bar", false},
globTest{"bara", false},
globTest{"barb", false},
globTest{"barbosa", false},
globTest{"foobar", true},
globTest{"food", true},
globTest{"anything else", true},
)
}
func TestGlobFilterInclude(t *testing.T) {
filter, err := NewGlobFilter(WithIncludeGlobs("foo", "bar*"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", true},
globTest{"bar", true},
globTest{"bara", true},
globTest{"barb", true},
globTest{"barbosa", true},
globTest{"foobar", false},
globTest{"food", false},
globTest{"anything else", false},
)
}
func TestGlobFilterEmpty(t *testing.T) {
filter, err := NewGlobFilter()
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", true},
globTest{"bar", true},
globTest{"bara", true},
globTest{"barb", true},
globTest{"barbosa", true},
globTest{"foobar", true},
globTest{"food", true},
globTest{"anything else", true},
)
}
func TestGlobFilterExcludeInclude(t *testing.T) {
filter, err := NewGlobFilter(WithExcludeGlobs("/foo/bar/**"), WithIncludeGlobs("/foo/**"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"/foo/a", true},
globTest{"/foo/b", true},
globTest{"/foo/c/d/e", true},
globTest{"/foo/bar/a", false},
globTest{"/foo/bar/b", false},
globTest{"/foo/bar/c/d/e", false},
globTest{"/any/other/path", false},
)
}
func TestGlobFilterExcludePrecedence(t *testing.T) {
filter, err := NewGlobFilter(WithExcludeGlobs("foo"), WithIncludeGlobs("foo*"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", false},
globTest{"foobar", true},
)
}
func TestGlobErrorContainsGlob(t *testing.T) {
invalidGlob := "[this is invalid because it doesn't close the capture group"
_, err := NewGlobFilter(WithExcludeGlobs(invalidGlob))
assert.Error(t, err)
assert.Contains(t, err.Error(), invalidGlob)
}
// The filters in this test should be mutually exclusive because one includes
// and the other excludes the same glob.
func TestGlobInverse(t *testing.T) {
for _, glob := range []string{
"a",
"a*",
"a**",
"*a",
"**a",
"*",
} {
include, err := NewGlobFilter(WithIncludeGlobs(glob))
assert.NoError(t, err)
exclude, err := NewGlobFilter(WithExcludeGlobs(glob))
assert.NoError(t, err)
rapid.Check(t, func(t *rapid.T) {
input := rapid.String().Draw(t, "input")
a, b := include.ShouldInclude(input), exclude.ShouldInclude(input)
if a == b {
t.Fatalf("Filter(Include(%q)) == Filter(Exclude(%q)) == %v for input %q", glob, glob, a, input)
}
})
}
}
func TestGlobDefaultFilters(t *testing.T) {
for _, filter := range []*Filter{nil, {}} {
rapid.Check(t, func(t *rapid.T) {
if !filter.ShouldInclude(rapid.String().Draw(t, "input")) {
t.Fatalf("filter %#v did not include input", filter)
}
})
}
}
-7
View File
@@ -74,13 +74,6 @@ func TestStringShannonEntropy(t *testing.T) {
},
want: 0.22228483068568816,
},
{
name: "empty",
args: args{
input: "",
},
want: 0.0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -24,7 +24,7 @@ const verifyURL = "https://api.pagerduty.com/users"
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b(u\+[a-zA-Z0-9_+-]{18})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b([a-zA-Z0-9_+-]{20})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
+20 -25
View File
@@ -2,7 +2,6 @@ package plaidkey
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
@@ -54,36 +53,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
DetectorType: detectorspb.DetectorType_PlaidKey,
Raw: []byte(resMatch),
}
environments := []string{"development", "production"}
if verify {
for _, env := range environments {
payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://"+env+".plaid.com/link/token/create", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
s1.ExtraData = map[string]string{"environment": fmt.Sprintf("https://%s.plaid.com", env)}
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://development.plaid.com/link/token/create", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
results = append(results, s1)
// if the environment is dev, we don't need to check production
if s1.Verified {
break
}
}
results = append(results, s1)
}
}
return results, nil
-5
View File
@@ -26,8 +26,6 @@ func TestPlaidKey_FromChunk(t *testing.T) {
secret := testSecrets.MustGetField("PLAIDKEY_SECRET")
inactiveSecret := testSecrets.MustGetField("PLAIDKEY_SECRET_INACTIVE")
id := testSecrets.MustGetField("PLAIDKEY_CLIENTID")
// env := testSecrets.MustGetField("PLAIDKEY_ENVIRONMENT") // development or production
env := "development"
type args struct {
ctx context.Context
@@ -53,9 +51,6 @@ func TestPlaidKey_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_PlaidKey,
Verified: true,
ExtraData: map[string]string{
"environment": fmt.Sprintf("https://%s.plaid.com", env),
},
},
},
wantErr: false,
+8 -21
View File
@@ -20,13 +20,13 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b(phx_[a-zA-Z0-9_]{43})\b`)
keyPat = regexp.MustCompile(`\b(phc_[a-zA-Z0-9_]{43})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"phx_"}
return []string{"phc_"}
}
// FromData will find and optionally verify AppPosthog secrets in a given set of bytes.
@@ -47,33 +47,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.posthog.com/api/event/?personal_api_key="+resMatch, nil)
reqEU, errEU := http.NewRequestWithContext(ctx, "GET", "https://eu.posthog.com/api/event/?personal_api_key="+resMatch, nil)
if err != nil || errEU != nil {
payload := strings.NewReader(` {
"api_key": "` + resMatch + `",
"distinct_id": "1234"
}`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://app.posthog.com/decide/", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
reqEU.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
//Try EU Endpoint only if other one fails.
res, err := client.Do(reqEU)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+2 -2
View File
@@ -21,7 +21,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
accessToken = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b(?:oauth2:)?([0-9a-f]{40})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b([0-9a-f:]{40,47})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -34,7 +34,7 @@ func (s Scanner) Keywords() []string {
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := accessToken.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
-99
View File
@@ -1,99 +0,0 @@
package voiceflow
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Reference: https://developer.voiceflow.com/reference/project#dialog-manager-api-keys
//
//TODO: This includes Workspace and Legacy Workspace API keys; I haven't validated whether these actually work.
// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
keyPat = regexp.MustCompile(`\b(VF\.(?:(?:DM|WS)\.)?[a-fA-F0-9]{24}\.[a-zA-Z0-9]{16})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"vf", "dm"}
}
// FromData will find and optionally verify Voiceflow secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Voiceflow,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
// Fetch the state for a random user.
payload := []byte(`{"question": "why is the sky blue?"}`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://general-runtime.voiceflow.com/knowledge-base/query", bytes.NewBuffer(payload))
if err != nil {
continue
}
req.Header.Set("Accept", "application/json")
req.Header.Set("Authorization", resMatch)
req.Header.Set("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
if res.StatusCode == http.StatusOK {
s1.Verified = true
} else if res.StatusCode == http.StatusUnauthorized {
// The secret is determinately not verified (nothing to do)
} else {
var buf bytes.Buffer
var bodyString string
_, err = io.Copy(&buf, res.Body)
if err == nil {
bodyString = buf.String()
}
s1.VerificationError = fmt.Errorf("unexpected HTTP response [status=%d, body=%s]", res.StatusCode, bodyString)
}
_ = res.Body.Close()
} else {
s1.VerificationError = err
}
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Voiceflow
}
-302
View File
@@ -1,302 +0,0 @@
//go:build detectors
// +build detectors
package voiceflow
import (
"context"
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"testing"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestVoiceflow_Pattern(t *testing.T) {
tests := []struct {
name string
data string
shouldMatch bool
match string
}{
// True positives
// https://github.com/funDAOmental/endlessquest/blob/5c008f7c6a7e58c45a88b72fef4b965c258d665c/Voiceflow/agent-api/index.js#L6
{
name: `valid_result1`,
data: `// z0MG IT'S NOT A SECRET (but we'll delete it)
const API_KEY = "VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
shouldMatch: true,
match: `VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF`,
},
// https://github.com/sherifButt/ll-site/blob/b98b268214324da42a84e996e4c03c242e122680/src/components/Chatbot.jsx#L14
{
name: `valid_result2`,
data: ` const runtime = useRuntime({
verify: { authorization: 'VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f' },
session: { userID: 'user_123' },
});`,
shouldMatch: true,
match: `VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f`,
},
// https://github.com/the-vv/Voiceflow-chatbot/blob/324db17693dd46387ea7a020e92c4e79b94306c6/src/app/chat/chat.component.ts#L27
{
name: `valid_result3`,
data: ` this.http.delete('https://general-runtime.voiceflow.com/state/user/TEST_USER', {
headers: {
Authorization: "VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq"
}
}).subscribe(res => {
this.loading = false;
this.doPrompt('', { action: { type: 'launch' } });
})`,
shouldMatch: true,
match: `VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq`,
},
// https://github.com/legionX7/Graduation-Project-API/blob/451431771d3fba1d8c634b8855274b414d7aed6d/mainAPI.py#L547
{
name: `valid_result4`,
data: `
API_KEY = 'VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl'
VERSION_ID = '646bc'`,
shouldMatch: true,
match: `VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl`,
},
// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
{
name: `valid_result5`,
data: ` it('extracts ID from a Dialog Manager API key', () => {
// eslint-disable-next-line no-secrets/no-secrets
const key = 'VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai';
const result = utils.extractAPIKeyID(key);
expect(result).to.equal('628d5d92faf688001bda7907');
});`,
shouldMatch: true,
match: `VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai`,
},
{
name: `valid_result6_legacy`,
data: ` it('extracts ID from a Workspace API key', () => {
// eslint-disable-next-line no-secrets/no-secrets
const key = 'VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa';
const result = utils.extractAPIKeyID(key);
expect(result).to.equal('62bcb0cca5184300066f5ac7');
});
`,
shouldMatch: true,
match: `VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa`,
},
{
name: `valid_result7_legacy`,
data: ` it('extracts ID from a Legacy Workspace API key', () => {
// eslint-disable-next-line no-secrets/no-secrets
const key = 'VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az';
const result = utils.extractAPIKeyID(key);
expect(result).to.equal('62bcb0cca5184300066f5ac7');
});`,
shouldMatch: true,
match: `VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az`,
},
// False positives
// https://github.com/ImperialCollegeLondon/voiceflow-integration-whatsapp/blob/0f3d6a5638b9acb4989d5bf8e77081cc78e9b976/README.md?plain=1#L155
{
name: `invalid_result1`,
data: "Now, paste it in your .env file for the **VF_PROJECT_API** variable<br>\n```VF_PROJECT_API='VF.DM.62xxxxxxxxxxxxxxxxxxxxxxx'```",
shouldMatch: false,
},
// https://github.com/voiceflow/api-examples/blob/c3d8ba9ee8eced7ec8d241973b1eb0284aaec212/rust/src/main.rs#L5
{
name: `invalid_result2`,
data: `const API_KEY: &str = "YOUR_API_KEY_HERE"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
shouldMatch: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
s := Scanner{}
results, err := s.FromData(context.Background(), true, []byte(test.data))
if err != nil {
t.Errorf("CoinbaseWaaS.FromData() error = %v", err)
return
}
if test.shouldMatch {
if len(results) == 0 {
t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
return
}
expected := test.data
if test.match != "" {
expected = test.match
}
result := results[0]
if result.VerificationError != nil {
fmt.Printf("VerificationError: %v\n", result.VerificationError)
}
resultData := string(result.Raw)
if resultData != expected {
t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, resultData)
return
}
} else {
if len(results) > 0 {
t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
return
}
}
})
}
}
func TestVoiceflow_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("VOICEFLOW")
inactiveSecret := testSecrets.MustGetField("VOICEFLOW_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Voiceflow,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Voiceflow,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Voiceflow,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Voiceflow,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Voiceflow.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Voiceflow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
-2
View File
@@ -11,7 +11,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraphcody"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/voiceflow"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
@@ -1555,7 +1554,6 @@ func DefaultDetectors() []detectors.Detector {
&ramp.Scanner{},
&anthropic.Scanner{},
&sourcegraphcody.Scanner{},
voiceflow.Scanner{},
}
}
+2 -1
View File
@@ -306,6 +306,7 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
e.sourceManager = sources.NewManager(
sources.WithConcurrentSources(int(e.concurrency)),
sources.WithConcurrentUnits(int(e.concurrency)),
sources.WithSourceUnits(),
)
if len(e.decoders) == 0 {
@@ -635,7 +636,7 @@ func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64,
if !found {
return 0, false
}
lineNumber := int64(bytes.Count(before, []byte("\n")) + 1)
lineNumber := int64(bytes.Count(before, []byte("\n")))
// If the line contains the ignore tag, we should ignore the result.
endLine := bytes.Index(after, []byte("\n"))
if endLine == -1 {
+6 -6
View File
@@ -32,7 +32,7 @@ func TestFragmentLineOffset(t *testing.T) {
result: &detectors.Result{
Raw: []byte("secret here"),
},
expectedLine: 3,
expectedLine: 2,
ignore: true,
},
{
@@ -43,7 +43,7 @@ func TestFragmentLineOffset(t *testing.T) {
result: &detectors.Result{
Raw: []byte("secret here"),
},
expectedLine: 3,
expectedLine: 2,
ignore: false,
},
{
@@ -54,7 +54,7 @@ func TestFragmentLineOffset(t *testing.T) {
result: &detectors.Result{
Raw: []byte("secret here"),
},
expectedLine: 5,
expectedLine: 4,
ignore: false,
},
{
@@ -65,7 +65,7 @@ func TestFragmentLineOffset(t *testing.T) {
result: &detectors.Result{
Raw: []byte("secret\nhere"),
},
expectedLine: 5,
expectedLine: 4,
ignore: false,
},
{
@@ -76,7 +76,7 @@ func TestFragmentLineOffset(t *testing.T) {
result: &detectors.Result{
Raw: []byte("secret\nhere"),
},
expectedLine: 4,
expectedLine: 3,
ignore: true,
},
{
@@ -87,7 +87,7 @@ func TestFragmentLineOffset(t *testing.T) {
result: &detectors.Result{
Raw: []byte("secret here"),
},
expectedLine: 4,
expectedLine: 3,
ignore: true,
},
}
+38
View File
@@ -0,0 +1,38 @@
package engine
import (
"runtime"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/travisci"
)
// ScanTravisCI scans TravisCI logs.
func (e *Engine) ScanTravisCI(ctx context.Context, token string) error {
connection := &sourcespb.TravisCI{
Credential: &sourcespb.TravisCI_Token{
Token: token,
},
}
var conn anypb.Any
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal Travis CI connection")
return err
}
sourceName := "trufflehog - Travis CI"
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, travisci.SourceType)
travisSource := &travisci.Source{}
if err := travisSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
return err
}
_, err = e.sourceManager.Run(ctx, sourceName, travisSource)
return err
}
-5
View File
@@ -158,11 +158,6 @@ func UpdateLinkLineNumber(ctx context.Context, link string, newLine int64) strin
return link
}
if newLine <= 0 {
// Don't change the link if the line number is 0.
return link
}
switch determineProvider(link) {
case providerBitbucket:
// For Bitbucket, it doesn't support line links (based on the GenerateLink function).
-8
View File
@@ -251,14 +251,6 @@ func TestUpdateLinkLineNumber(t *testing.T) {
},
want: "https://onprem.customdomain.com/org/repo/commit/xyz123#L50",
},
{
name: "Don't include line when it's 0",
args: args{
link: "https://github.com/coinbase/cbpay-js/issues/181",
newLine: int64(0),
},
want: "https://github.com/coinbase/cbpay-js/issues/181",
},
{
name: "Invalid link",
args: args{
+2 -6
View File
@@ -1008,7 +1008,6 @@ const (
DetectorType_Ramp DetectorType = 934
DetectorType_Klaviyo DetectorType = 935
DetectorType_SourcegraphCody DetectorType = 936
DetectorType_Voiceflow DetectorType = 937
)
// Enum value maps for DetectorType.
@@ -1947,7 +1946,6 @@ var (
934: "Ramp",
935: "Klaviyo",
936: "SourcegraphCody",
937: "Voiceflow",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@@ -2883,7 +2881,6 @@ var (
"Ramp": 934,
"Klaviyo": 935,
"SourcegraphCody": 936,
"Voiceflow": 937,
}
)
@@ -3262,7 +3259,7 @@ var file_detectors_proto_rawDesc = []byte{
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xb6, 0x75, 0x0a, 0x0c, 0x44,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xa6, 0x75, 0x0a, 0x0c, 0x44,
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4201,8 +4198,7 @@ var file_detectors_proto_rawDesc = []byte{
0x63, 0x10, 0xa5, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x52, 0x61, 0x6d, 0x70, 0x10, 0xa6, 0x07, 0x12,
0x0c, 0x0a, 0x07, 0x4b, 0x6c, 0x61, 0x76, 0x69, 0x79, 0x6f, 0x10, 0xa7, 0x07, 0x12, 0x14, 0x0a,
0x0f, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x43, 0x6f, 0x64, 0x79,
0x10, 0xa8, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x56, 0x6f, 0x69, 0x63, 0x65, 0x66, 0x6c, 0x6f, 0x77,
0x10, 0xa9, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
0x10, 0xa8, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
File diff suppressed because it is too large Load Diff
@@ -486,6 +486,117 @@ var _ interface {
ErrorName() string
} = CircleCIValidationError{}
// Validate checks the field values on TravisCI with the rules defined in the
// proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
func (m *TravisCI) Validate() error {
return m.validate(false)
}
// ValidateAll checks the field values on TravisCI with the rules defined in
// the proto definition for this message. If any rules are violated, the
// result is a list of violation errors wrapped in TravisCIMultiError, or nil
// if none found.
func (m *TravisCI) ValidateAll() error {
return m.validate(true)
}
func (m *TravisCI) validate(all bool) error {
if m == nil {
return nil
}
var errors []error
// no validation rules for Username
// no validation rules for Repository
// no validation rules for BuildNumber
// no validation rules for JobNumber
// no validation rules for Link
// no validation rules for Public
if len(errors) > 0 {
return TravisCIMultiError(errors)
}
return nil
}
// TravisCIMultiError is an error wrapping multiple validation errors returned
// by TravisCI.ValidateAll() if the designated constraints aren't met.
type TravisCIMultiError []error
// Error returns a concatenation of all the error messages it wraps.
func (m TravisCIMultiError) Error() string {
var msgs []string
for _, err := range m {
msgs = append(msgs, err.Error())
}
return strings.Join(msgs, "; ")
}
// AllErrors returns a list of validation violation errors.
func (m TravisCIMultiError) AllErrors() []error { return m }
// TravisCIValidationError is the validation error returned by
// TravisCI.Validate if the designated constraints aren't met.
type TravisCIValidationError struct {
field string
reason string
cause error
key bool
}
// Field function returns field value.
func (e TravisCIValidationError) Field() string { return e.field }
// Reason function returns reason value.
func (e TravisCIValidationError) Reason() string { return e.reason }
// Cause function returns cause value.
func (e TravisCIValidationError) Cause() error { return e.cause }
// Key function returns key value.
func (e TravisCIValidationError) Key() bool { return e.key }
// ErrorName returns error name.
func (e TravisCIValidationError) ErrorName() string { return "TravisCIValidationError" }
// Error satisfies the builtin error interface
func (e TravisCIValidationError) Error() string {
cause := ""
if e.cause != nil {
cause = fmt.Sprintf(" | caused by: %v", e.cause)
}
key := ""
if e.key {
key = "key for "
}
return fmt.Sprintf(
"invalid %sTravisCI.%s: %s%s",
key,
e.field,
e.reason,
cause)
}
var _ error = TravisCIValidationError{}
var _ interface {
Field() string
Reason() string
Key() bool
Cause() error
ErrorName() string
} = TravisCIValidationError{}
// Validate checks the field values on Confluence with the rules defined in the
// proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
@@ -3926,6 +4037,37 @@ func (m *MetaData) validate(all bool) error {
}
}
case *MetaData_TravisCI:
if all {
switch v := interface{}(m.GetTravisCI()).(type) {
case interface{ ValidateAll() error }:
if err := v.ValidateAll(); err != nil {
errors = append(errors, MetaDataValidationError{
field: "TravisCI",
reason: "embedded message failed validation",
cause: err,
})
}
case interface{ Validate() error }:
if err := v.Validate(); err != nil {
errors = append(errors, MetaDataValidationError{
field: "TravisCI",
reason: "embedded message failed validation",
cause: err,
})
}
}
} else if v, ok := interface{}(m.GetTravisCI()).(interface{ Validate() error }); ok {
if err := v.Validate(); err != nil {
return MetaDataValidationError{
field: "TravisCI",
reason: "embedded message failed validation",
cause: err,
}
}
}
}
if len(errors) > 0 {
File diff suppressed because it is too large Load Diff
+118
View File
@@ -667,6 +667,124 @@ var _ interface {
ErrorName() string
} = CircleCIValidationError{}
// Validate checks the field values on TravisCI with the rules defined in the
// proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
func (m *TravisCI) Validate() error {
return m.validate(false)
}
// ValidateAll checks the field values on TravisCI with the rules defined in
// the proto definition for this message. If any rules are violated, the
// result is a list of violation errors wrapped in TravisCIMultiError, or nil
// if none found.
func (m *TravisCI) ValidateAll() error {
return m.validate(true)
}
func (m *TravisCI) validate(all bool) error {
if m == nil {
return nil
}
var errors []error
if _, err := url.Parse(m.GetEndpoint()); err != nil {
err = TravisCIValidationError{
field: "Endpoint",
reason: "value must be a valid URI",
cause: err,
}
if !all {
return err
}
errors = append(errors, err)
}
switch m.Credential.(type) {
case *TravisCI_Token:
// no validation rules for Token
}
if len(errors) > 0 {
return TravisCIMultiError(errors)
}
return nil
}
// TravisCIMultiError is an error wrapping multiple validation errors returned
// by TravisCI.ValidateAll() if the designated constraints aren't met.
type TravisCIMultiError []error
// Error returns a concatenation of all the error messages it wraps.
func (m TravisCIMultiError) Error() string {
var msgs []string
for _, err := range m {
msgs = append(msgs, err.Error())
}
return strings.Join(msgs, "; ")
}
// AllErrors returns a list of validation violation errors.
func (m TravisCIMultiError) AllErrors() []error { return m }
// TravisCIValidationError is the validation error returned by
// TravisCI.Validate if the designated constraints aren't met.
type TravisCIValidationError struct {
field string
reason string
cause error
key bool
}
// Field function returns field value.
func (e TravisCIValidationError) Field() string { return e.field }
// Reason function returns reason value.
func (e TravisCIValidationError) Reason() string { return e.reason }
// Cause function returns cause value.
func (e TravisCIValidationError) Cause() error { return e.cause }
// Key function returns key value.
func (e TravisCIValidationError) Key() bool { return e.key }
// ErrorName returns error name.
func (e TravisCIValidationError) ErrorName() string { return "TravisCIValidationError" }
// Error satisfies the builtin error interface
func (e TravisCIValidationError) Error() string {
cause := ""
if e.cause != nil {
cause = fmt.Sprintf(" | caused by: %v", e.cause)
}
key := ""
if e.key {
key = "key for "
}
return fmt.Sprintf(
"invalid %sTravisCI.%s: %s%s",
key,
e.field,
e.reason,
cause)
}
var _ error = TravisCIValidationError{}
var _ interface {
Field() string
Reason() string
Key() bool
Cause() error
ErrorName() string
} = TravisCIValidationError{}
// Validate checks the field values on Confluence with the rules defined in the
// proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
+2 -2
View File
@@ -1,8 +1,8 @@
package sources
import (
"errors"
"fmt"
"reflect"
"sync"
"testing"
)
@@ -40,7 +40,7 @@ func TestNewScanErrors(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
got := NewScanErrors()
if !reflect.DeepEqual(got, tc.want) {
if !errors.Is(got.Errors(), tc.want.Errors()) {
t.Errorf("got %+v, want %+v", got, tc.want)
}
})
+40 -64
View File
@@ -233,7 +233,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
aCtx.Logger().Error(err, "invalid repository", "repo", repo)
continue
}
s.filteredRepoCache.Set(repo, r)
s.filteredRepoCache.Set(r, r)
}
s.includeIssueComments = s.conn.IncludeIssueComments
@@ -825,51 +825,38 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
// Unauthenticated access to most github endpoints has a rate limit of 60 requests per hour.
// This will likely only be exhausted if many users/orgs are scanned without auth
func (s *Source) handleRateLimit(errIn error, res *github.Response) bool {
var (
knownWait = true
remaining = 0
retryAfter time.Duration
)
// GitHub has both primary (RateLimit) and secondary (AbuseRateLimit) errors.
var rateLimit *github.RateLimitError
var abuseLimit *github.AbuseRateLimitError
if errors.As(errIn, &rateLimit) {
// Do nothing
} else if errors.As(errIn, &abuseLimit) {
retryAfter = abuseLimit.GetRetryAfter()
} else {
limit, ok := errIn.(*github.RateLimitError)
if !ok {
return false
}
githubNumRateLimitEncountered.WithLabelValues(s.name).Inc()
// Parse retry information from response headers, unless a Retry-After value was already provided.
// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#exceeding-the-rate-limit
if retryAfter <= 0 && res != nil {
var err error
remaining, err = strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
if res != nil {
knownWait := true
remaining, err := strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
if err != nil {
knownWait = false
}
resetTime, err := strconv.Atoi(res.Header.Get("x-ratelimit-reset"))
if err != nil || resetTime == 0 {
knownWait = false
} else if resetTime > 0 {
retryAfter = time.Duration(int64(resetTime)-time.Now().Unix()) * time.Second
}
if knownWait && remaining == 0 {
waitTime := int64(resetTime) - time.Now().Unix()
if waitTime > 0 {
duration := time.Duration(waitTime+1) * time.Second
s.log.V(2).Info("rate limited", "resumeTime", time.Now().Add(duration).String())
time.Sleep(duration)
githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(duration.Seconds())
return true
}
}
}
resumeTime := time.Now().Add(retryAfter).String()
if knownWait && remaining == 0 && retryAfter > 0 {
s.log.V(2).Info("rate limited", "retry_after", retryAfter.String(), "resume_time", resumeTime)
} else {
// TODO: Use exponential backoff instead of static retry time.
retryAfter = time.Minute * 5
s.log.V(2).Error(errIn, "unexpected rate limit error", "retry_after", retryAfter.String(), "resume_time", resumeTime)
}
time.Sleep(retryAfter)
githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(retryAfter.Seconds())
s.log.V(2).Info("handling rate limit (5 minutes retry)", "retry-after", limit.Message)
time.Sleep(time.Minute * 5)
return true
}
@@ -1144,10 +1131,9 @@ var (
)
type repoInfo struct {
owner string
repo string
repoPath string
visibility source_metadatapb.Visibility
owner string
repo string
repoPath string
}
func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimmedURL []string, repoURL *url.URL, chunksChan chan *sources.Chunk) error {
@@ -1158,16 +1144,7 @@ func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimm
owner := trimmedURL[1]
repo := trimmedURL[2]
if !(s.includeIssueComments || s.includePRComments) {
return nil
}
repoInfo := repoInfo{
owner: owner,
repo: repo,
repoPath: repoPath,
visibility: s.visibilityOf(ctx, repoPath),
}
repoInfo := repoInfo{owner: owner, repo: repo, repoPath: repoPath}
if s.includeIssueComments {
if err := s.processIssueComments(ctx, repoInfo, chunksChan); err != nil {
@@ -1214,7 +1191,7 @@ func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan ch
return err
}
if err = s.chunkIssues(ctx, info, issues, chunksChan); err != nil {
if err = s.chunkIssues(ctx, info.repo, info.repoPath, issues, chunksChan); err != nil {
return err
}
@@ -1249,7 +1226,7 @@ func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunks
return err
}
if err = s.chunkIssueComments(ctx, info, issueComments, chunksChan); err != nil {
if err = s.chunkIssueComments(ctx, info.repo, info.repoPath, issueComments, chunksChan); err != nil {
return err
}
@@ -1285,7 +1262,7 @@ func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan
return err
}
if err = s.chunkPullRequests(ctx, info, prs, chunksChan); err != nil {
if err = s.chunkPullRequests(ctx, info.repo, prs, chunksChan); err != nil {
return err
}
@@ -1320,7 +1297,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
return err
}
if err = s.chunkPullRequestComments(ctx, info, prComments, chunksChan); err != nil {
if err = s.chunkPullRequestComments(ctx, info.repo, prComments, chunksChan); err != nil {
return err
}
@@ -1333,7 +1310,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
return nil
}
func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
func (s *Source) chunkIssues(ctx context.Context, repo, repoPath string, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
for _, issue := range issues {
// Skip pull requests since covered by processPRs.
@@ -1353,9 +1330,9 @@ func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*g
Link: sanitizer.UTF8(issue.GetHTMLURL()),
Username: sanitizer.UTF8(issue.GetUser().GetLogin()),
Email: sanitizer.UTF8(issue.GetUser().GetEmail()),
Repository: sanitizer.UTF8(repoInfo.repo),
Repository: sanitizer.UTF8(repo),
Timestamp: sanitizer.UTF8(issue.GetCreatedAt().String()),
Visibility: repoInfo.visibility,
Visibility: s.visibilityOf(ctx, repoPath),
},
},
},
@@ -1372,7 +1349,7 @@ func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*g
return nil
}
func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
for _, comment := range comments {
// Create chunk and send it to the channel.
chunk := &sources.Chunk{
@@ -1386,9 +1363,9 @@ func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comm
Link: sanitizer.UTF8(comment.GetHTMLURL()),
Username: sanitizer.UTF8(comment.GetUser().GetLogin()),
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
Repository: sanitizer.UTF8(repoInfo.repo),
Repository: sanitizer.UTF8(repo),
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
Visibility: repoInfo.visibility,
Visibility: s.visibilityOf(ctx, repoPath),
},
},
},
@@ -1405,7 +1382,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comm
return nil
}
func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
for _, comment := range comments {
// Create chunk and send it to the channel.
chunk := &sources.Chunk{
@@ -1419,9 +1396,8 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo
Link: sanitizer.UTF8(comment.GetHTMLURL()),
Username: sanitizer.UTF8(comment.GetUser().GetLogin()),
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
Repository: sanitizer.UTF8(repoInfo.repo),
Repository: sanitizer.UTF8(repo),
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
Visibility: repoInfo.visibility,
},
},
},
@@ -1438,7 +1414,7 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo
return nil
}
func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
func (s *Source) chunkPullRequests(ctx context.Context, repo string, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
for _, pr := range prs {
// Create chunk and send it to the channel.
chunk := &sources.Chunk{
@@ -1452,9 +1428,8 @@ func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs [
Link: sanitizer.UTF8(pr.GetHTMLURL()),
Username: sanitizer.UTF8(pr.GetUser().GetLogin()),
Email: sanitizer.UTF8(pr.GetUser().GetEmail()),
Repository: sanitizer.UTF8(repoInfo.repo),
Repository: sanitizer.UTF8(repo),
Timestamp: sanitizer.UTF8(pr.GetCreatedAt().String()),
Visibility: repoInfo.visibility,
},
},
},
@@ -1487,7 +1462,8 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
Repository: sanitizer.UTF8(gistUrl),
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
// TODO: Fetching this requires making an additional API call. We may want to include this in the future.
// Fetching this information requires making an additional API call.
// We may want to include this in the future.
// Visibility: s.visibilityOf(ctx, repoPath),
},
},
+4 -18
View File
@@ -89,14 +89,14 @@ func (f Fatal) Unwrap() error { return f.error }
// ChunkError is a custom error type for errors encountered during chunking of
// a specific unit.
type ChunkError struct {
Unit SourceUnit
Err error
unit SourceUnit
err error
}
func (f ChunkError) Error() string {
return fmt.Sprintf("error chunking unit %q: %s", f.Unit.SourceUnitID(), f.Err.Error())
return fmt.Sprintf("error chunking unit %q: %s", f.unit.SourceUnitID(), f.err.Error())
}
func (f ChunkError) Unwrap() error { return f.Err }
func (f ChunkError) Unwrap() error { return f.err }
// JobProgress aggregates information about a run of a Source.
type JobProgress struct {
@@ -351,17 +351,3 @@ func (m JobProgressMetrics) ElapsedTime() time.Duration {
}
return m.EndTime.Sub(m.StartTime)
}
// ErrorsFor returns all the errors for the given SourceUnit. If there are no
// errors, including the case that the unit has not been encountered, nil will
// be returned.
func (m JobProgressMetrics) ErrorsFor(unit SourceUnit) []error {
var errs []error
for _, err := range m.Errors {
var chunkErr ChunkError
if errors.As(err, &chunkErr) && chunkErr.Unit == unit {
errs = append(errs, err)
}
}
return errs
}
-218
View File
@@ -1,218 +0,0 @@
package sources
import (
"errors"
"fmt"
"strings"
"sync"
"time"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
// UnitHook implements JobProgressHook for tracking the progress of each
// individual unit.
type UnitHook struct {
metrics *lru.Cache[string, *UnitMetrics]
mu sync.Mutex
NoopHook
}
type UnitHookOpt func(*UnitHook)
func WithUnitHookCache(cache *lru.Cache[string, *UnitMetrics]) UnitHookOpt {
return func(hook *UnitHook) { hook.metrics = cache }
}
func NewUnitHook(ctx context.Context, opts ...UnitHookOpt) *UnitHook {
// lru.NewWithEvict can only fail if the size is < 0.
cache, _ := lru.NewWithEvict(1024, func(key string, value *UnitMetrics) {
if value.handled {
return
}
ctx.Logger().Error(fmt.Errorf("eviction"), "dropping unit metric",
"id", key,
"metric", value,
)
})
hook := UnitHook{metrics: cache}
for _, opt := range opts {
opt(&hook)
}
return &hook
}
// id is a helper method to generate an ID for the given job and unit.
func (u *UnitHook) id(ref JobProgressRef, unit SourceUnit) string {
unitID := ""
if unit != nil {
unitID = unit.SourceUnitID()
}
return fmt.Sprintf("%d/%d/%s", ref.SourceID, ref.JobID, unitID)
}
func (u *UnitHook) StartUnitChunking(ref JobProgressRef, unit SourceUnit, start time.Time) {
id := u.id(ref, unit)
u.mu.Lock()
defer u.mu.Unlock()
u.metrics.Add(id, &UnitMetrics{
Unit: unit,
Parent: ref,
StartTime: start,
})
}
func (u *UnitHook) EndUnitChunking(ref JobProgressRef, unit SourceUnit, end time.Time) {
id := u.id(ref, unit)
u.mu.Lock()
defer u.mu.Unlock()
metrics, ok := u.metrics.Get(id)
if !ok {
return
}
metrics.EndTime = end
}
func (u *UnitHook) ReportChunk(ref JobProgressRef, unit SourceUnit, chunk *Chunk) {
id := u.id(ref, unit)
u.mu.Lock()
defer u.mu.Unlock()
metrics, ok := u.metrics.Get(id)
if !ok && unit != nil {
// The unit has been evicted.
return
} else if !ok && unit == nil {
// This is a chunk from a non-unit source.
metrics = &UnitMetrics{
Unit: nil,
Parent: ref,
StartTime: ref.Snapshot().StartTime,
}
u.metrics.Add(id, metrics)
}
metrics.TotalChunks++
metrics.TotalBytes += uint64(len(chunk.Data))
}
func (u *UnitHook) ReportError(ref JobProgressRef, err error) {
u.mu.Lock()
defer u.mu.Unlock()
// Always add the error to the nil unit if it exists.
if metrics, ok := u.metrics.Get(u.id(ref, nil)); ok {
metrics.Errors = append(metrics.Errors, err)
}
// Check if it's a ChunkError for a specific unit.
var chunkErr ChunkError
if !errors.As(err, &chunkErr) {
return
}
id := u.id(ref, chunkErr.Unit)
metrics, ok := u.metrics.Get(id)
if !ok {
return
}
metrics.Errors = append(metrics.Errors, err)
}
func (u *UnitHook) Finish(ref JobProgressRef) {
u.mu.Lock()
defer u.mu.Unlock()
// Clear out any metrics on this job. This covers the case for the
// source running without unit support.
prefix := u.id(ref, nil)
for _, id := range u.metrics.Keys() {
if !strings.HasPrefix(id, prefix) {
continue
}
metric, ok := u.metrics.Get(id)
if !ok {
continue
}
// If the unit is nil, the source does not support units.
// Use the overall job metrics instead.
if metric.Unit == nil {
snap := ref.Snapshot()
metric.StartTime = snap.StartTime
metric.EndTime = snap.EndTime
metric.Errors = snap.Errors
}
}
}
// UnitMetrics gets all the currently active or newly finished metrics for this
// job. If a unit returned from this method has finished, it will be removed
// from the cache and no longer returned in successive calls to UnitMetrics().
func (u *UnitHook) UnitMetrics() []UnitMetrics {
u.mu.Lock()
defer u.mu.Unlock()
output := make([]UnitMetrics, 0, u.metrics.Len())
for _, id := range u.metrics.Keys() {
metric, ok := u.metrics.Get(id)
if !ok {
continue
}
output = append(output, *metric)
if metric.IsFinished() {
metric.handled = true
u.metrics.Remove(id)
}
}
return output
}
type UnitMetrics struct {
Unit SourceUnit
Parent JobProgressRef
// Start and end time for chunking this unit.
StartTime time.Time
EndTime time.Time
// Total number of chunks produced from this unit.
TotalChunks uint64
// Total number of bytes produced from this unit.
TotalBytes uint64
// All errors encountered by this unit.
Errors []error
// Flag to mark that these metrics were intentionally evicted from
// the cache.
handled bool
}
func (u UnitMetrics) IsFinished() bool {
return !u.EndTime.IsZero()
}
// ElapsedTime is a convenience method that provides the elapsed time the job
// has been running. If it hasn't started yet, 0 is returned. If it has
// finished, the total time is returned.
func (u UnitMetrics) ElapsedTime() time.Duration {
if u.StartTime.IsZero() {
return 0
}
if u.EndTime.IsZero() {
return time.Since(u.StartTime)
}
return u.EndTime.Sub(u.StartTime)
}
// NoopHook implements JobProgressHook by doing nothing. This is useful for
// embedding in other structs to overwrite only the methods of the interface
// that you care about.
type NoopHook struct{}
func (NoopHook) Start(JobProgressRef, time.Time) {}
func (NoopHook) End(JobProgressRef, time.Time) {}
func (NoopHook) StartEnumerating(JobProgressRef, time.Time) {}
func (NoopHook) EndEnumerating(JobProgressRef, time.Time) {}
func (NoopHook) StartUnitChunking(JobProgressRef, SourceUnit, time.Time) {}
func (NoopHook) EndUnitChunking(JobProgressRef, SourceUnit, time.Time) {}
func (NoopHook) ReportError(JobProgressRef, error) {}
func (NoopHook) ReportUnit(JobProgressRef, SourceUnit) {}
func (NoopHook) ReportChunk(JobProgressRef, SourceUnit, *Chunk) {}
func (NoopHook) Finish(JobProgressRef) {}
-23
View File
@@ -126,26 +126,3 @@ func TestJobProgressElapsedTime(t *testing.T) {
metrics.EndTime = metrics.StartTime.Add(1 * time.Hour)
assert.Equal(t, metrics.ElapsedTime(), 1*time.Hour)
}
func TestJobProgressErrorsFor(t *testing.T) {
metrics := JobProgressMetrics{
Errors: []error{
Fatal{ChunkError{
Unit: CommonSourceUnit{ID: "foo"},
Err: fmt.Errorf("foo error"),
}},
ChunkError{
Unit: CommonSourceUnit{ID: "foo"},
Err: fmt.Errorf("foo again error"),
},
ChunkError{
Unit: CommonSourceUnit{ID: "bar"},
Err: fmt.Errorf("bar error"),
},
fmt.Errorf("hi there"),
},
}
assert.Equal(t, 2, len(metrics.ErrorsFor(CommonSourceUnit{ID: "foo"})))
assert.Equal(t, 1, len(metrics.ErrorsFor(CommonSourceUnit{ID: "bar"})))
assert.Equal(t, 0, len(metrics.ErrorsFor(CommonSourceUnit{ID: "baz"})))
}
-115
View File
@@ -3,10 +3,8 @@ package sources
import (
"errors"
"fmt"
"sort"
"testing"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/stretchr/testify/assert"
"google.golang.org/protobuf/types/known/anypb"
@@ -202,8 +200,6 @@ func (c *unitChunker) ChunkUnit(ctx context.Context, unit SourceUnit, rep ChunkR
if err := rep.ChunkErr(ctx, fmt.Errorf(step.err)); err != nil {
return err
}
}
if step.output == "" {
continue
}
if err := rep.ChunkOk(ctx, Chunk{Data: []byte(step.output)}); err != nil {
@@ -314,114 +310,3 @@ func TestSourceManagerAvailableCapacity(t *testing.T) {
<-ref.Done() // Wait for the job to finish.
assert.Equal(t, 1337, mgr.AvailableCapacity())
}
func TestSourceManagerUnitHook(t *testing.T) {
hook := NewUnitHook(context.TODO())
input := []unitChunk{
{unit: "one", output: "bar"},
{unit: "two", err: "oh no"},
{unit: "three", err: "not again"},
}
mgr := NewManager(
WithBufferedOutput(8),
WithSourceUnits(), WithConcurrentUnits(1),
WithReportHook(hook),
)
source, err := buildDummy(&unitChunker{input})
assert.NoError(t, err)
ref, err := mgr.Run(context.Background(), "dummy", source)
assert.NoError(t, err)
<-ref.Done()
metrics := hook.UnitMetrics()
assert.Equal(t, 3, len(metrics))
sort.Slice(metrics, func(i, j int) bool {
return metrics[i].EndTime.Before(metrics[j].EndTime)
})
m0, m1, m2 := metrics[0], metrics[1], metrics[2]
assert.Equal(t, "one", m0.Unit.SourceUnitID())
assert.Equal(t, uint64(1), m0.TotalChunks)
assert.Equal(t, uint64(3), m0.TotalBytes)
assert.NotZero(t, m0.StartTime)
assert.NotZero(t, m0.EndTime)
assert.NotZero(t, m0.ElapsedTime())
assert.Equal(t, 0, len(m0.Errors))
assert.Equal(t, "two", m1.Unit.SourceUnitID())
assert.Equal(t, uint64(0), m1.TotalChunks)
assert.Equal(t, uint64(0), m1.TotalBytes)
assert.NotZero(t, m1.StartTime)
assert.NotZero(t, m1.EndTime)
assert.NotZero(t, m1.ElapsedTime())
assert.Equal(t, 1, len(m1.Errors))
assert.Equal(t, "three", m2.Unit.SourceUnitID())
assert.Equal(t, uint64(0), m2.TotalChunks)
assert.Equal(t, uint64(0), m2.TotalBytes)
assert.NotZero(t, m2.StartTime)
assert.NotZero(t, m2.EndTime)
assert.NotZero(t, m2.ElapsedTime())
assert.Equal(t, 1, len(m2.Errors))
}
// TestSourceManagerUnitHookNoBlock tests that the UnitHook drops metrics if
// they aren't handled fast enough.
func TestSourceManagerUnitHookNoBlock(t *testing.T) {
var evictedKeys []string
cache, _ := lru.NewWithEvict(1, func(key string, _ *UnitMetrics) {
evictedKeys = append(evictedKeys, key)
})
hook := NewUnitHook(context.TODO(), WithUnitHookCache(cache))
input := []unitChunk{
{unit: "one", output: "bar"},
{unit: "two", err: "oh no"},
{unit: "three", err: "not again"},
}
mgr := NewManager(
WithBufferedOutput(8),
WithSourceUnits(), WithConcurrentUnits(1),
WithReportHook(hook),
)
source, err := buildDummy(&unitChunker{input})
assert.NoError(t, err)
ref, err := mgr.Run(context.Background(), "dummy", source)
assert.NoError(t, err)
<-ref.Done()
assert.Equal(t, 2, len(evictedKeys))
metrics := hook.UnitMetrics()
assert.Equal(t, 1, len(metrics))
assert.Equal(t, "three", metrics[0].Unit.SourceUnitID())
}
// TestSourceManagerUnitHookNoUnits tests whether the UnitHook works for
// sources that don't support units.
func TestSourceManagerUnitHookNoUnits(t *testing.T) {
hook := NewUnitHook(context.TODO())
mgr := NewManager(
WithBufferedOutput(8),
WithReportHook(hook),
)
source, err := buildDummy(&counterChunker{count: 5})
assert.NoError(t, err)
ref, err := mgr.Run(context.Background(), "dummy", source)
assert.NoError(t, err)
<-ref.Done()
metrics := hook.UnitMetrics()
assert.Equal(t, 1, len(metrics))
m := metrics[0]
assert.Equal(t, nil, m.Unit)
assert.Equal(t, uint64(5), m.TotalChunks)
assert.Equal(t, uint64(5), m.TotalBytes)
assert.NotZero(t, m.StartTime)
assert.NotZero(t, m.EndTime)
assert.NotZero(t, m.ElapsedTime())
assert.Equal(t, 0, len(m.Errors))
}
+223
View File
@@ -0,0 +1,223 @@
package travisci
import (
"fmt"
"strconv"
"github.com/go-errors/errors"
"github.com/shuheiktgw/go-travis"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
const (
SourceType = sourcespb.SourceType_SOURCE_TYPE_TRAVISCI
baseURL = "https://api.travis-ci.com/"
pageSize = 100
)
type Source struct {
name string
sourceId sources.SourceID
jobId sources.JobID
verify bool
jobPool *errgroup.Group
sources.Progress
client *travis.Client
sources.CommonSourceUnitUnmarshaller
returnAfterFirstChunk bool
}
// Ensure the Source satisfies the interfaces at compile time.
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
func (s *Source) Type() sourcespb.SourceType {
return SourceType
}
func (s *Source) SourceID() sources.SourceID {
return s.sourceId
}
func (s *Source) JobID() sources.JobID {
return s.jobId
}
// Init returns an initialized TravisCI source.
func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
s.name = name
s.sourceId = sourceId
s.jobId = jobId
s.verify = verify
s.jobPool = &errgroup.Group{}
s.jobPool.SetLimit(concurrency)
var conn sourcespb.TravisCI
if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
}
switch conn.Credential.(type) {
case *sourcespb.TravisCI_Token:
if conn.GetToken() == "" {
return errors.New("token is empty")
}
s.client = travis.NewClient(baseURL, conn.GetToken())
s.client.HTTPClient = common.RetryableHttpClientTimeout(3)
user, _, err := s.client.User.Current(ctx, nil)
if err != nil {
return errors.WrapPrefix(err, "error getting testing travis client", 0)
}
ctx.Logger().V(2).Info("authenticated to Travis CI with user", "username", user.Login)
default:
return errors.New("credential type not implemented for Travis CI")
}
return nil
}
func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {
for repoPage := 0; ; repoPage++ {
repositories, _, err := s.client.Repositories.List(ctx, &travis.RepositoriesOption{
Limit: pageSize,
Offset: repoPage * pageSize,
})
if err != nil {
if repoPage == 0 {
return fmt.Errorf("error listing repositories: %w", err)
}
err = reporter.UnitErr(ctx, err)
if err != nil {
return fmt.Errorf("error reporting error: %w", err)
}
}
if len(repositories) == 0 {
break
}
for _, repo := range repositories {
err = reporter.UnitOk(ctx, sources.CommonSourceUnit{
ID: strconv.Itoa(int(*repo.Id)),
})
if err != nil {
return fmt.Errorf("error reporting unit: %w", err)
}
ctx.Logger().V(2).Info("enumerated repository", "id", repo.Id, "name", repo.Name)
}
}
return nil
}
// ChunkUnit implements SourceUnitChunker interface.
func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
repo, _, err := s.client.Repositories.Find(ctx, unit.SourceUnitID(), nil)
if err != nil {
return fmt.Errorf("error finding repository: %w", err)
}
logger := ctx.Logger().WithValues("repo", *repo.Name)
logger.V(2).Info("scanning repository")
// Counts continuous errors from ListByRepoSlug. Used to quit early in
// case the API always returns an error.
var buildPageErrs int
for buildPage := 0; ; buildPage++ {
builds, _, err := s.client.Builds.ListByRepoSlug(ctx, *repo.Slug, &travis.BuildsByRepoOption{
Limit: pageSize,
Offset: buildPage * pageSize,
})
if err != nil {
if err := reporter.ChunkErr(ctx, err); err != nil {
return err
}
buildPageErrs++
if buildPageErrs >= 5 {
return fmt.Errorf("encountered too many errors listing builds, aborting")
}
continue
}
// Reset the page error counter.
buildPageErrs = 0
if len(builds) == 0 {
break
}
for _, build := range builds {
jobs, _, err := s.client.Jobs.ListByBuild(ctx, *build.Id)
if err != nil {
if err := reporter.ChunkErr(ctx, err); err != nil {
return err
}
continue
}
if len(jobs) == 0 {
break
}
for _, job := range jobs {
log, _, err := s.client.Logs.FindByJobId(ctx, *job.Id)
if err != nil {
if err := reporter.ChunkErr(ctx, err); err != nil {
return err
}
continue
}
logger.V(3).Info("scanning job", "id", *job.Id, "number", *job.Number)
chunk := sources.Chunk{
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
Data: []byte(*log.Content),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_TravisCI{
TravisCI: &source_metadatapb.TravisCI{
Username: *job.Owner.Login,
Repository: *repo.Name,
BuildNumber: *build.Number,
JobNumber: *job.Number,
Link: fmt.Sprintf("https://app.travis-ci.com/github/%s/%s/jobs/%d", *job.Owner.Login, *repo.Name, *job.Id),
Public: !*repo.Private,
},
},
},
Verify: s.verify,
}
if err := reporter.ChunkOk(ctx, chunk); err != nil {
return err
}
if s.returnAfterFirstChunk {
return nil
}
}
}
}
return nil
}
// Chunks emits chunks of bytes over a channel.
// It's a no-op because we've implemented the SourceUnitChunker interface instead.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
return nil
}
+99
View File
@@ -0,0 +1,99 @@
package travisci
import (
"fmt"
"testing"
"github.com/kylelemons/godebug/pretty"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest"
)
func TestSource_Scan(t *testing.T) {
ctx := context.Background()
secret, err := common.GetTestSecret(ctx)
if err != nil {
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
}
token := secret.MustGetField("TRAVISCI_TOKEN")
type init struct {
name string
verify bool
connection *sourcespb.TravisCI
}
tests := []struct {
name string
init init
wantSourceMetadata *source_metadatapb.MetaData
wantErr bool
}{
{
name: "get a chunk",
init: init{
name: "example repo",
connection: &sourcespb.TravisCI{
Credential: &sourcespb.TravisCI_Token{
Token: token,
},
},
verify: true,
},
wantSourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_TravisCI{
TravisCI: &source_metadatapb.TravisCI{
Username: "truffle-sandbox",
Repository: "test-repo",
BuildNumber: "1",
JobNumber: "1.1",
Link: "https://app.travis-ci.com/github/truffle-sandbox/test-repo/jobs/611053994",
Public: false,
},
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Source{}
conn, err := anypb.New(tt.init.connection)
if err != nil {
t.Fatal(err)
}
err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5)
if (err != nil) != tt.wantErr {
t.Fatalf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
}
reporter := sourcestest.TestReporter{}
s.returnAfterFirstChunk = true
err = s.Enumerate(ctx, &reporter)
if (err != nil) != tt.wantErr {
t.Fatalf("Source.Enumerate() error = %v, wantErr %v", err, tt.wantErr)
}
for _, unit := range reporter.Units {
err = s.ChunkUnit(ctx, unit, &reporter)
if (err != nil) != tt.wantErr {
t.Fatalf("Source.ChunkUnit() error = %v, wantErr %v", err, tt.wantErr)
}
}
for _, chunk := range reporter.Chunks {
if diff := pretty.Compare(chunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {
t.Fatalf("comparing chunks: %s diff: (-got +want)\n%s", tt.name, diff)
}
}
})
}
}
-1
View File
@@ -945,7 +945,6 @@ enum DetectorType {
Ramp = 934;
Klaviyo = 935;
SourcegraphCody = 936;
Voiceflow = 937;
}
message Result {
+11
View File
@@ -44,6 +44,16 @@ message CircleCI {
string email = 7;
}
message TravisCI {
string username = 2;
string repository = 3;
string build_number = 4;
string job_number = 5;
string link = 6;
bool public = 7;
}
message Confluence {
string page = 1;
string space = 2;
@@ -294,5 +304,6 @@ message MetaData {
SharePoint sharepoint = 25;
GoogleDrive googleDrive = 26;
AzureRepos azureRepos = 27;
TravisCI travisCI = 28;
}
}
+8
View File
@@ -44,6 +44,7 @@ enum SourceType {
SOURCE_TYPE_SHAREPOINT = 29;
SOURCE_TYPE_GCS_UNAUTHED = 30;
SOURCE_TYPE_AZURE_REPOS = 31;
SOURCE_TYPE_TRAVISCI = 32;
}
message LocalSource {
@@ -83,6 +84,13 @@ message CircleCI {
}
}
message TravisCI {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
string token = 2;
}
}
message Confluence {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {