Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 065984ee58 | |||
| fe90effec1 | |||
| e5a34a9809 | |||
| ec3c59d025 | |||
| c99e0b93f7 | |||
| b960231155 |
@@ -47,7 +47,6 @@ require (
|
||||
github.com/h2non/filetype v1.1.3
|
||||
github.com/hashicorp/go-retryablehttp v0.7.4
|
||||
github.com/hashicorp/golang-lru v0.5.1
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/jlaffaye/ftp v0.2.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/jpillora/overseer v1.1.6
|
||||
@@ -66,6 +65,7 @@ require (
|
||||
github.com/prometheus/client_golang v1.16.0
|
||||
github.com/rabbitmq/amqp091-go v1.8.1
|
||||
github.com/sergi/go-diff v1.3.1
|
||||
github.com/shuheiktgw/go-travis v0.3.1
|
||||
github.com/snowflakedb/gosnowflake v1.6.23
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
@@ -82,7 +82,6 @@ require (
|
||||
google.golang.org/protobuf v1.31.0
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6
|
||||
gopkg.in/h2non/gock.v1 v1.1.2
|
||||
pgregory.net/rapid v1.1.0
|
||||
sigs.k8s.io/yaml v1.3.0
|
||||
)
|
||||
|
||||
|
||||
@@ -369,6 +369,7 @@ github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5Px
|
||||
github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
|
||||
github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=
|
||||
github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=
|
||||
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
@@ -418,8 +419,6 @@ github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
|
||||
@@ -607,6 +606,8 @@ github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
|
||||
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
|
||||
github.com/shuheiktgw/go-travis v0.3.1 h1:SAT16mi77ccqogOslnXxBXzXbpeyChaIYUwi2aJpVZY=
|
||||
github.com/shuheiktgw/go-travis v0.3.1/go.mod h1:avnFFDqJDdRHwlF9tgqvYi3asQCm/HGL8aLxYiKa4Yg=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
@@ -1015,8 +1016,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
|
||||
pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
|
||||
@@ -139,6 +139,9 @@ var (
|
||||
|
||||
dockerScan = cli.Command("docker", "Scan Docker Image")
|
||||
dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, otherwise a image registry is assumed.").Required().Strings()
|
||||
|
||||
travisCiScan = cli.Command("travisci", "Scan TravisCI")
|
||||
travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -152,9 +155,6 @@ func init() {
|
||||
|
||||
cli.Version("trufflehog " + version.BuildVersion)
|
||||
|
||||
//Support -h for help
|
||||
cli.HelpFlag.Short('h')
|
||||
|
||||
if len(os.Args) <= 1 && isatty.IsTerminal(os.Stdout.Fd()) {
|
||||
args := tui.Run()
|
||||
if len(args) == 0 {
|
||||
@@ -501,6 +501,10 @@ func run(state overseer.State) {
|
||||
if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
|
||||
logFatal(err, "Failed to scan CircleCI.")
|
||||
}
|
||||
case travisCiScan.FullCommand():
|
||||
if err := e.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
|
||||
logFatal(err, "Failed to scan TravisCI.")
|
||||
}
|
||||
case gcsScan.FullCommand():
|
||||
cfg := sources.GCSConfig{
|
||||
ProjectID: *gcsProjectID,
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/gobwas/glob"
|
||||
)
|
||||
|
||||
// Filter is a generic filter for excluding and including globs (limited
|
||||
// regular expressions). Exclusion takes precedence if both include and exclude
|
||||
// lists are provided.
|
||||
type Filter struct {
|
||||
exclude []glob.Glob
|
||||
include []glob.Glob
|
||||
}
|
||||
|
||||
type globFilterOpt func(*Filter) error
|
||||
|
||||
// WithExcludeGlobs adds exclude globs to the filter.
|
||||
func WithExcludeGlobs(excludes ...string) globFilterOpt {
|
||||
return func(f *Filter) error {
|
||||
for _, exclude := range excludes {
|
||||
g, err := glob.Compile(exclude)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid exclude glob %q: %w", exclude, err)
|
||||
}
|
||||
f.exclude = append(f.exclude, g)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithIncludeGlobs adds include globs to the filter.
|
||||
func WithIncludeGlobs(includes ...string) globFilterOpt {
|
||||
return func(f *Filter) error {
|
||||
for _, include := range includes {
|
||||
g, err := glob.Compile(include)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid include glob %q: %w", include, err)
|
||||
}
|
||||
f.include = append(f.include, g)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewGlobFilter creates a new Filter with the provided options.
|
||||
func NewGlobFilter(opts ...globFilterOpt) (*Filter, error) {
|
||||
filter := &Filter{}
|
||||
for _, opt := range opts {
|
||||
if err := opt(filter); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return filter, nil
|
||||
}
|
||||
|
||||
// ShouldInclude returns whether the object is in the include list or not in
|
||||
// the exclude list (exclude taking precedence).
|
||||
func (f *Filter) ShouldInclude(object string) bool {
|
||||
if f == nil {
|
||||
return true
|
||||
}
|
||||
exclude, include := len(f.exclude), len(f.include)
|
||||
if exclude == 0 && include == 0 {
|
||||
return true
|
||||
} else if exclude > 0 && include == 0 {
|
||||
return f.shouldIncludeFromExclude(object)
|
||||
} else if exclude == 0 && include > 0 {
|
||||
return f.shouldIncludeFromInclude(object)
|
||||
} else {
|
||||
if ok, err := f.shouldIncludeFromBoth(object); err == nil {
|
||||
return ok
|
||||
}
|
||||
// Ambiguous case.
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// shouldIncludeFromExclude checks for explicitly excluded paths. This should
|
||||
// only be called when the include list is empty.
|
||||
func (f *Filter) shouldIncludeFromExclude(object string) bool {
|
||||
for _, glob := range f.exclude {
|
||||
if glob.Match(object) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// shouldIncludeFromInclude checks for explicitly included paths. This should
|
||||
// only be called when the exclude list is empty.
|
||||
func (f *Filter) shouldIncludeFromInclude(object string) bool {
|
||||
for _, glob := range f.include {
|
||||
if glob.Match(object) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// shouldIncludeFromBoth checks for either excluded or included paths. Exclusion
|
||||
// takes precedence. If neither list contains the object, true is returned.
|
||||
func (f *Filter) shouldIncludeFromBoth(object string) (bool, error) {
|
||||
// Exclude takes precedence. If we find the object in the exclude list,
|
||||
// we should not match.
|
||||
for _, glob := range f.exclude {
|
||||
if glob.Match(object) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
// If we find the object in the include list, we should match.
|
||||
for _, glob := range f.include {
|
||||
if glob.Match(object) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
// If we find it in neither, return an error to let the caller decide.
|
||||
return false, fmt.Errorf("ambiguous match")
|
||||
}
|
||||
@@ -1,138 +0,0 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"pgregory.net/rapid"
|
||||
)
|
||||
|
||||
type globTest struct {
|
||||
input string
|
||||
shouldInclude bool
|
||||
}
|
||||
|
||||
func testGlobs(t *testing.T, filter *Filter, tests ...globTest) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
// Invert because mentally it's easier to say whether an
|
||||
// input should be included.
|
||||
assert.Equal(t, tt.shouldInclude, filter.ShouldInclude(tt.input))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobFilterExclude(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithExcludeGlobs("foo", "bar*"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", false},
|
||||
globTest{"bar", false},
|
||||
globTest{"bara", false},
|
||||
globTest{"barb", false},
|
||||
globTest{"barbosa", false},
|
||||
globTest{"foobar", true},
|
||||
globTest{"food", true},
|
||||
globTest{"anything else", true},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterInclude(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithIncludeGlobs("foo", "bar*"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", true},
|
||||
globTest{"bar", true},
|
||||
globTest{"bara", true},
|
||||
globTest{"barb", true},
|
||||
globTest{"barbosa", true},
|
||||
globTest{"foobar", false},
|
||||
globTest{"food", false},
|
||||
globTest{"anything else", false},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterEmpty(t *testing.T) {
|
||||
filter, err := NewGlobFilter()
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", true},
|
||||
globTest{"bar", true},
|
||||
globTest{"bara", true},
|
||||
globTest{"barb", true},
|
||||
globTest{"barbosa", true},
|
||||
globTest{"foobar", true},
|
||||
globTest{"food", true},
|
||||
globTest{"anything else", true},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterExcludeInclude(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithExcludeGlobs("/foo/bar/**"), WithIncludeGlobs("/foo/**"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"/foo/a", true},
|
||||
globTest{"/foo/b", true},
|
||||
globTest{"/foo/c/d/e", true},
|
||||
globTest{"/foo/bar/a", false},
|
||||
globTest{"/foo/bar/b", false},
|
||||
globTest{"/foo/bar/c/d/e", false},
|
||||
globTest{"/any/other/path", false},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterExcludePrecedence(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithExcludeGlobs("foo"), WithIncludeGlobs("foo*"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", false},
|
||||
globTest{"foobar", true},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobErrorContainsGlob(t *testing.T) {
|
||||
invalidGlob := "[this is invalid because it doesn't close the capture group"
|
||||
_, err := NewGlobFilter(WithExcludeGlobs(invalidGlob))
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), invalidGlob)
|
||||
}
|
||||
|
||||
// The filters in this test should be mutually exclusive because one includes
|
||||
// and the other excludes the same glob.
|
||||
func TestGlobInverse(t *testing.T) {
|
||||
for _, glob := range []string{
|
||||
"a",
|
||||
"a*",
|
||||
"a**",
|
||||
"*a",
|
||||
"**a",
|
||||
"*",
|
||||
} {
|
||||
include, err := NewGlobFilter(WithIncludeGlobs(glob))
|
||||
assert.NoError(t, err)
|
||||
exclude, err := NewGlobFilter(WithExcludeGlobs(glob))
|
||||
assert.NoError(t, err)
|
||||
rapid.Check(t, func(t *rapid.T) {
|
||||
input := rapid.String().Draw(t, "input")
|
||||
a, b := include.ShouldInclude(input), exclude.ShouldInclude(input)
|
||||
if a == b {
|
||||
t.Fatalf("Filter(Include(%q)) == Filter(Exclude(%q)) == %v for input %q", glob, glob, a, input)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobDefaultFilters(t *testing.T) {
|
||||
for _, filter := range []*Filter{nil, {}} {
|
||||
rapid.Check(t, func(t *rapid.T) {
|
||||
if !filter.ShouldInclude(rapid.String().Draw(t, "input")) {
|
||||
t.Fatalf("filter %#v did not include input", filter)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -74,13 +74,6 @@ func TestStringShannonEntropy(t *testing.T) {
|
||||
},
|
||||
want: 0.22228483068568816,
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
args: args{
|
||||
input: "",
|
||||
},
|
||||
want: 0.0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -24,7 +24,7 @@ const verifyURL = "https://api.pagerduty.com/users"
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b(u\+[a-zA-Z0-9_+-]{18})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b([a-zA-Z0-9_+-]{20})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
|
||||
@@ -2,7 +2,6 @@ package plaidkey
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -54,36 +53,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
DetectorType: detectorspb.DetectorType_PlaidKey,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
environments := []string{"development", "production"}
|
||||
|
||||
if verify {
|
||||
for _, env := range environments {
|
||||
payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://"+env+".plaid.com/link/token/create", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
s1.ExtraData = map[string]string{"environment": fmt.Sprintf("https://%s.plaid.com", env)}
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://development.plaid.com/link/token/create", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
results = append(results, s1)
|
||||
// if the environment is dev, we don't need to check production
|
||||
if s1.Verified {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return results, nil
|
||||
|
||||
@@ -26,8 +26,6 @@ func TestPlaidKey_FromChunk(t *testing.T) {
|
||||
secret := testSecrets.MustGetField("PLAIDKEY_SECRET")
|
||||
inactiveSecret := testSecrets.MustGetField("PLAIDKEY_SECRET_INACTIVE")
|
||||
id := testSecrets.MustGetField("PLAIDKEY_CLIENTID")
|
||||
// env := testSecrets.MustGetField("PLAIDKEY_ENVIRONMENT") // development or production
|
||||
env := "development"
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
@@ -53,9 +51,6 @@ func TestPlaidKey_FromChunk(t *testing.T) {
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PlaidKey,
|
||||
Verified: true,
|
||||
ExtraData: map[string]string{
|
||||
"environment": fmt.Sprintf("https://%s.plaid.com", env),
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
|
||||
@@ -20,13 +20,13 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(phx_[a-zA-Z0-9_]{43})\b`)
|
||||
keyPat = regexp.MustCompile(`\b(phc_[a-zA-Z0-9_]{43})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"phx_"}
|
||||
return []string{"phc_"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify AppPosthog secrets in a given set of bytes.
|
||||
@@ -47,33 +47,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.posthog.com/api/event/?personal_api_key="+resMatch, nil)
|
||||
reqEU, errEU := http.NewRequestWithContext(ctx, "GET", "https://eu.posthog.com/api/event/?personal_api_key="+resMatch, nil)
|
||||
|
||||
if err != nil || errEU != nil {
|
||||
payload := strings.NewReader(` {
|
||||
"api_key": "` + resMatch + `",
|
||||
"distinct_id": "1234"
|
||||
}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://app.posthog.com/decide/", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
reqEU.Header.Add("Content-Type", "application/json")
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
//Try EU Endpoint only if other one fails.
|
||||
res, err := client.Do(reqEU)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
|
||||
@@ -21,7 +21,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
|
||||
accessToken = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b(?:oauth2:)?([0-9a-f]{40})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b([0-9a-f:]{40,47})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -34,7 +34,7 @@ func (s Scanner) Keywords() []string {
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := accessToken.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
package voiceflow
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Reference: https://developer.voiceflow.com/reference/project#dialog-manager-api-keys
|
||||
//
|
||||
//TODO: This includes Workspace and Legacy Workspace API keys; I haven't validated whether these actually work.
|
||||
// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
|
||||
keyPat = regexp.MustCompile(`\b(VF\.(?:(?:DM|WS)\.)?[a-fA-F0-9]{24}\.[a-zA-Z0-9]{16})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"vf", "dm"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Voiceflow secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Voiceflow,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
// Fetch the state for a random user.
|
||||
payload := []byte(`{"question": "why is the sky blue?"}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://general-runtime.voiceflow.com/knowledge-base/query", bytes.NewBuffer(payload))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", resMatch)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
if res.StatusCode == http.StatusOK {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == http.StatusUnauthorized {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
var buf bytes.Buffer
|
||||
var bodyString string
|
||||
_, err = io.Copy(&buf, res.Body)
|
||||
if err == nil {
|
||||
bodyString = buf.String()
|
||||
}
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response [status=%d, body=%s]", res.StatusCode, bodyString)
|
||||
}
|
||||
_ = res.Body.Close()
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Voiceflow
|
||||
}
|
||||
@@ -1,302 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package voiceflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestVoiceflow_Pattern(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data string
|
||||
shouldMatch bool
|
||||
match string
|
||||
}{
|
||||
// True positives
|
||||
// https://github.com/funDAOmental/endlessquest/blob/5c008f7c6a7e58c45a88b72fef4b965c258d665c/Voiceflow/agent-api/index.js#L6
|
||||
{
|
||||
name: `valid_result1`,
|
||||
data: `// z0MG IT'S NOT A SECRET (but we'll delete it)
|
||||
const API_KEY = "VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
|
||||
shouldMatch: true,
|
||||
match: `VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF`,
|
||||
},
|
||||
// https://github.com/sherifButt/ll-site/blob/b98b268214324da42a84e996e4c03c242e122680/src/components/Chatbot.jsx#L14
|
||||
{
|
||||
name: `valid_result2`,
|
||||
data: ` const runtime = useRuntime({
|
||||
verify: { authorization: 'VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f' },
|
||||
session: { userID: 'user_123' },
|
||||
});`,
|
||||
shouldMatch: true,
|
||||
match: `VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f`,
|
||||
},
|
||||
// https://github.com/the-vv/Voiceflow-chatbot/blob/324db17693dd46387ea7a020e92c4e79b94306c6/src/app/chat/chat.component.ts#L27
|
||||
{
|
||||
name: `valid_result3`,
|
||||
data: ` this.http.delete('https://general-runtime.voiceflow.com/state/user/TEST_USER', {
|
||||
headers: {
|
||||
Authorization: "VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq"
|
||||
}
|
||||
}).subscribe(res => {
|
||||
this.loading = false;
|
||||
this.doPrompt('', { action: { type: 'launch' } });
|
||||
})`,
|
||||
shouldMatch: true,
|
||||
match: `VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq`,
|
||||
},
|
||||
// https://github.com/legionX7/Graduation-Project-API/blob/451431771d3fba1d8c634b8855274b414d7aed6d/mainAPI.py#L547
|
||||
{
|
||||
name: `valid_result4`,
|
||||
data: `
|
||||
API_KEY = 'VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl'
|
||||
VERSION_ID = '646bc'`,
|
||||
shouldMatch: true,
|
||||
match: `VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl`,
|
||||
},
|
||||
// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
|
||||
{
|
||||
name: `valid_result5`,
|
||||
data: ` it('extracts ID from a Dialog Manager API key', () => {
|
||||
// eslint-disable-next-line no-secrets/no-secrets
|
||||
const key = 'VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai';
|
||||
const result = utils.extractAPIKeyID(key);
|
||||
|
||||
expect(result).to.equal('628d5d92faf688001bda7907');
|
||||
});`,
|
||||
shouldMatch: true,
|
||||
match: `VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai`,
|
||||
},
|
||||
{
|
||||
name: `valid_result6_legacy`,
|
||||
data: ` it('extracts ID from a Workspace API key', () => {
|
||||
// eslint-disable-next-line no-secrets/no-secrets
|
||||
const key = 'VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa';
|
||||
const result = utils.extractAPIKeyID(key);
|
||||
|
||||
expect(result).to.equal('62bcb0cca5184300066f5ac7');
|
||||
});
|
||||
`,
|
||||
shouldMatch: true,
|
||||
match: `VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa`,
|
||||
},
|
||||
{
|
||||
name: `valid_result7_legacy`,
|
||||
data: ` it('extracts ID from a Legacy Workspace API key', () => {
|
||||
// eslint-disable-next-line no-secrets/no-secrets
|
||||
const key = 'VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az';
|
||||
const result = utils.extractAPIKeyID(key);
|
||||
|
||||
expect(result).to.equal('62bcb0cca5184300066f5ac7');
|
||||
});`,
|
||||
shouldMatch: true,
|
||||
match: `VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az`,
|
||||
},
|
||||
|
||||
// False positives
|
||||
// https://github.com/ImperialCollegeLondon/voiceflow-integration-whatsapp/blob/0f3d6a5638b9acb4989d5bf8e77081cc78e9b976/README.md?plain=1#L155
|
||||
{
|
||||
name: `invalid_result1`,
|
||||
data: "Now, paste it in your .env file for the **VF_PROJECT_API** variable<br>\n```VF_PROJECT_API='VF.DM.62xxxxxxxxxxxxxxxxxxxxxxx'```",
|
||||
shouldMatch: false,
|
||||
},
|
||||
// https://github.com/voiceflow/api-examples/blob/c3d8ba9ee8eced7ec8d241973b1eb0284aaec212/rust/src/main.rs#L5
|
||||
{
|
||||
name: `invalid_result2`,
|
||||
data: `const API_KEY: &str = "YOUR_API_KEY_HERE"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
|
||||
results, err := s.FromData(context.Background(), true, []byte(test.data))
|
||||
if err != nil {
|
||||
t.Errorf("CoinbaseWaaS.FromData() error = %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if test.shouldMatch {
|
||||
if len(results) == 0 {
|
||||
t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
|
||||
return
|
||||
}
|
||||
expected := test.data
|
||||
if test.match != "" {
|
||||
expected = test.match
|
||||
}
|
||||
result := results[0]
|
||||
if result.VerificationError != nil {
|
||||
fmt.Printf("VerificationError: %v\n", result.VerificationError)
|
||||
}
|
||||
resultData := string(result.Raw)
|
||||
if resultData != expected {
|
||||
t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, resultData)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
if len(results) > 0 {
|
||||
t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVoiceflow_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("VOICEFLOW")
|
||||
inactiveSecret := testSecrets.MustGetField("VOICEFLOW_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Voiceflow,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Voiceflow,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Voiceflow,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Voiceflow,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Voiceflow.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Voiceflow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraphcody"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/voiceflow"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
|
||||
@@ -1555,7 +1554,6 @@ func DefaultDetectors() []detectors.Detector {
|
||||
&ramp.Scanner{},
|
||||
&anthropic.Scanner{},
|
||||
&sourcegraphcody.Scanner{},
|
||||
voiceflow.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -306,6 +306,7 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
|
||||
e.sourceManager = sources.NewManager(
|
||||
sources.WithConcurrentSources(int(e.concurrency)),
|
||||
sources.WithConcurrentUnits(int(e.concurrency)),
|
||||
sources.WithSourceUnits(),
|
||||
)
|
||||
|
||||
if len(e.decoders) == 0 {
|
||||
@@ -635,7 +636,7 @@ func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64,
|
||||
if !found {
|
||||
return 0, false
|
||||
}
|
||||
lineNumber := int64(bytes.Count(before, []byte("\n")) + 1)
|
||||
lineNumber := int64(bytes.Count(before, []byte("\n")))
|
||||
// If the line contains the ignore tag, we should ignore the result.
|
||||
endLine := bytes.Index(after, []byte("\n"))
|
||||
if endLine == -1 {
|
||||
|
||||
@@ -32,7 +32,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret here"),
|
||||
},
|
||||
expectedLine: 3,
|
||||
expectedLine: 2,
|
||||
ignore: true,
|
||||
},
|
||||
{
|
||||
@@ -43,7 +43,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret here"),
|
||||
},
|
||||
expectedLine: 3,
|
||||
expectedLine: 2,
|
||||
ignore: false,
|
||||
},
|
||||
{
|
||||
@@ -54,7 +54,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret here"),
|
||||
},
|
||||
expectedLine: 5,
|
||||
expectedLine: 4,
|
||||
ignore: false,
|
||||
},
|
||||
{
|
||||
@@ -65,7 +65,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret\nhere"),
|
||||
},
|
||||
expectedLine: 5,
|
||||
expectedLine: 4,
|
||||
ignore: false,
|
||||
},
|
||||
{
|
||||
@@ -76,7 +76,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret\nhere"),
|
||||
},
|
||||
expectedLine: 4,
|
||||
expectedLine: 3,
|
||||
ignore: true,
|
||||
},
|
||||
{
|
||||
@@ -87,7 +87,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret here"),
|
||||
},
|
||||
expectedLine: 4,
|
||||
expectedLine: 3,
|
||||
ignore: true,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/travisci"
|
||||
)
|
||||
|
||||
// ScanTravisCI scans TravisCI logs.
|
||||
func (e *Engine) ScanTravisCI(ctx context.Context, token string) error {
|
||||
connection := &sourcespb.TravisCI{
|
||||
Credential: &sourcespb.TravisCI_Token{
|
||||
Token: token,
|
||||
},
|
||||
}
|
||||
|
||||
var conn anypb.Any
|
||||
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "failed to marshal Travis CI connection")
|
||||
return err
|
||||
}
|
||||
|
||||
sourceName := "trufflehog - Travis CI"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, travisci.SourceType)
|
||||
|
||||
travisSource := &travisci.Source{}
|
||||
if err := travisSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, travisSource)
|
||||
return err
|
||||
}
|
||||
@@ -158,11 +158,6 @@ func UpdateLinkLineNumber(ctx context.Context, link string, newLine int64) strin
|
||||
return link
|
||||
}
|
||||
|
||||
if newLine <= 0 {
|
||||
// Don't change the link if the line number is 0.
|
||||
return link
|
||||
}
|
||||
|
||||
switch determineProvider(link) {
|
||||
case providerBitbucket:
|
||||
// For Bitbucket, it doesn't support line links (based on the GenerateLink function).
|
||||
|
||||
@@ -251,14 +251,6 @@ func TestUpdateLinkLineNumber(t *testing.T) {
|
||||
},
|
||||
want: "https://onprem.customdomain.com/org/repo/commit/xyz123#L50",
|
||||
},
|
||||
{
|
||||
name: "Don't include line when it's 0",
|
||||
args: args{
|
||||
link: "https://github.com/coinbase/cbpay-js/issues/181",
|
||||
newLine: int64(0),
|
||||
},
|
||||
want: "https://github.com/coinbase/cbpay-js/issues/181",
|
||||
},
|
||||
{
|
||||
name: "Invalid link",
|
||||
args: args{
|
||||
|
||||
@@ -1008,7 +1008,6 @@ const (
|
||||
DetectorType_Ramp DetectorType = 934
|
||||
DetectorType_Klaviyo DetectorType = 935
|
||||
DetectorType_SourcegraphCody DetectorType = 936
|
||||
DetectorType_Voiceflow DetectorType = 937
|
||||
)
|
||||
|
||||
// Enum value maps for DetectorType.
|
||||
@@ -1947,7 +1946,6 @@ var (
|
||||
934: "Ramp",
|
||||
935: "Klaviyo",
|
||||
936: "SourcegraphCody",
|
||||
937: "Voiceflow",
|
||||
}
|
||||
DetectorType_value = map[string]int32{
|
||||
"Alibaba": 0,
|
||||
@@ -2883,7 +2881,6 @@ var (
|
||||
"Ramp": 934,
|
||||
"Klaviyo": 935,
|
||||
"SourcegraphCody": 936,
|
||||
"Voiceflow": 937,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -3262,7 +3259,7 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
|
||||
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
|
||||
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xb6, 0x75, 0x0a, 0x0c, 0x44,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xa6, 0x75, 0x0a, 0x0c, 0x44,
|
||||
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
|
||||
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
|
||||
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
|
||||
@@ -4201,8 +4198,7 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x63, 0x10, 0xa5, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x52, 0x61, 0x6d, 0x70, 0x10, 0xa6, 0x07, 0x12,
|
||||
0x0c, 0x0a, 0x07, 0x4b, 0x6c, 0x61, 0x76, 0x69, 0x79, 0x6f, 0x10, 0xa7, 0x07, 0x12, 0x14, 0x0a,
|
||||
0x0f, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x43, 0x6f, 0x64, 0x79,
|
||||
0x10, 0xa8, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x56, 0x6f, 0x69, 0x63, 0x65, 0x66, 0x6c, 0x6f, 0x77,
|
||||
0x10, 0xa9, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
|
||||
0x10, 0xa8, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
|
||||
0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
|
||||
0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
|
||||
0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -486,6 +486,117 @@ var _ interface {
|
||||
ErrorName() string
|
||||
} = CircleCIValidationError{}
|
||||
|
||||
// Validate checks the field values on TravisCI with the rules defined in the
|
||||
// proto definition for this message. If any rules are violated, the first
|
||||
// error encountered is returned, or nil if there are no violations.
|
||||
func (m *TravisCI) Validate() error {
|
||||
return m.validate(false)
|
||||
}
|
||||
|
||||
// ValidateAll checks the field values on TravisCI with the rules defined in
|
||||
// the proto definition for this message. If any rules are violated, the
|
||||
// result is a list of violation errors wrapped in TravisCIMultiError, or nil
|
||||
// if none found.
|
||||
func (m *TravisCI) ValidateAll() error {
|
||||
return m.validate(true)
|
||||
}
|
||||
|
||||
func (m *TravisCI) validate(all bool) error {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var errors []error
|
||||
|
||||
// no validation rules for Username
|
||||
|
||||
// no validation rules for Repository
|
||||
|
||||
// no validation rules for BuildNumber
|
||||
|
||||
// no validation rules for JobNumber
|
||||
|
||||
// no validation rules for Link
|
||||
|
||||
// no validation rules for Public
|
||||
|
||||
if len(errors) > 0 {
|
||||
return TravisCIMultiError(errors)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TravisCIMultiError is an error wrapping multiple validation errors returned
|
||||
// by TravisCI.ValidateAll() if the designated constraints aren't met.
|
||||
type TravisCIMultiError []error
|
||||
|
||||
// Error returns a concatenation of all the error messages it wraps.
|
||||
func (m TravisCIMultiError) Error() string {
|
||||
var msgs []string
|
||||
for _, err := range m {
|
||||
msgs = append(msgs, err.Error())
|
||||
}
|
||||
return strings.Join(msgs, "; ")
|
||||
}
|
||||
|
||||
// AllErrors returns a list of validation violation errors.
|
||||
func (m TravisCIMultiError) AllErrors() []error { return m }
|
||||
|
||||
// TravisCIValidationError is the validation error returned by
|
||||
// TravisCI.Validate if the designated constraints aren't met.
|
||||
type TravisCIValidationError struct {
|
||||
field string
|
||||
reason string
|
||||
cause error
|
||||
key bool
|
||||
}
|
||||
|
||||
// Field function returns field value.
|
||||
func (e TravisCIValidationError) Field() string { return e.field }
|
||||
|
||||
// Reason function returns reason value.
|
||||
func (e TravisCIValidationError) Reason() string { return e.reason }
|
||||
|
||||
// Cause function returns cause value.
|
||||
func (e TravisCIValidationError) Cause() error { return e.cause }
|
||||
|
||||
// Key function returns key value.
|
||||
func (e TravisCIValidationError) Key() bool { return e.key }
|
||||
|
||||
// ErrorName returns error name.
|
||||
func (e TravisCIValidationError) ErrorName() string { return "TravisCIValidationError" }
|
||||
|
||||
// Error satisfies the builtin error interface
|
||||
func (e TravisCIValidationError) Error() string {
|
||||
cause := ""
|
||||
if e.cause != nil {
|
||||
cause = fmt.Sprintf(" | caused by: %v", e.cause)
|
||||
}
|
||||
|
||||
key := ""
|
||||
if e.key {
|
||||
key = "key for "
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"invalid %sTravisCI.%s: %s%s",
|
||||
key,
|
||||
e.field,
|
||||
e.reason,
|
||||
cause)
|
||||
}
|
||||
|
||||
var _ error = TravisCIValidationError{}
|
||||
|
||||
var _ interface {
|
||||
Field() string
|
||||
Reason() string
|
||||
Key() bool
|
||||
Cause() error
|
||||
ErrorName() string
|
||||
} = TravisCIValidationError{}
|
||||
|
||||
// Validate checks the field values on Confluence with the rules defined in the
|
||||
// proto definition for this message. If any rules are violated, the first
|
||||
// error encountered is returned, or nil if there are no violations.
|
||||
@@ -3926,6 +4037,37 @@ func (m *MetaData) validate(all bool) error {
|
||||
}
|
||||
}
|
||||
|
||||
case *MetaData_TravisCI:
|
||||
|
||||
if all {
|
||||
switch v := interface{}(m.GetTravisCI()).(type) {
|
||||
case interface{ ValidateAll() error }:
|
||||
if err := v.ValidateAll(); err != nil {
|
||||
errors = append(errors, MetaDataValidationError{
|
||||
field: "TravisCI",
|
||||
reason: "embedded message failed validation",
|
||||
cause: err,
|
||||
})
|
||||
}
|
||||
case interface{ Validate() error }:
|
||||
if err := v.Validate(); err != nil {
|
||||
errors = append(errors, MetaDataValidationError{
|
||||
field: "TravisCI",
|
||||
reason: "embedded message failed validation",
|
||||
cause: err,
|
||||
})
|
||||
}
|
||||
}
|
||||
} else if v, ok := interface{}(m.GetTravisCI()).(interface{ Validate() error }); ok {
|
||||
if err := v.Validate(); err != nil {
|
||||
return MetaDataValidationError{
|
||||
field: "TravisCI",
|
||||
reason: "embedded message failed validation",
|
||||
cause: err,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if len(errors) > 0 {
|
||||
|
||||
+741
-640
File diff suppressed because it is too large
Load Diff
@@ -667,6 +667,124 @@ var _ interface {
|
||||
ErrorName() string
|
||||
} = CircleCIValidationError{}
|
||||
|
||||
// Validate checks the field values on TravisCI with the rules defined in the
|
||||
// proto definition for this message. If any rules are violated, the first
|
||||
// error encountered is returned, or nil if there are no violations.
|
||||
func (m *TravisCI) Validate() error {
|
||||
return m.validate(false)
|
||||
}
|
||||
|
||||
// ValidateAll checks the field values on TravisCI with the rules defined in
|
||||
// the proto definition for this message. If any rules are violated, the
|
||||
// result is a list of violation errors wrapped in TravisCIMultiError, or nil
|
||||
// if none found.
|
||||
func (m *TravisCI) ValidateAll() error {
|
||||
return m.validate(true)
|
||||
}
|
||||
|
||||
func (m *TravisCI) validate(all bool) error {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var errors []error
|
||||
|
||||
if _, err := url.Parse(m.GetEndpoint()); err != nil {
|
||||
err = TravisCIValidationError{
|
||||
field: "Endpoint",
|
||||
reason: "value must be a valid URI",
|
||||
cause: err,
|
||||
}
|
||||
if !all {
|
||||
return err
|
||||
}
|
||||
errors = append(errors, err)
|
||||
}
|
||||
|
||||
switch m.Credential.(type) {
|
||||
|
||||
case *TravisCI_Token:
|
||||
// no validation rules for Token
|
||||
|
||||
}
|
||||
|
||||
if len(errors) > 0 {
|
||||
return TravisCIMultiError(errors)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TravisCIMultiError is an error wrapping multiple validation errors returned
|
||||
// by TravisCI.ValidateAll() if the designated constraints aren't met.
|
||||
type TravisCIMultiError []error
|
||||
|
||||
// Error returns a concatenation of all the error messages it wraps.
|
||||
func (m TravisCIMultiError) Error() string {
|
||||
var msgs []string
|
||||
for _, err := range m {
|
||||
msgs = append(msgs, err.Error())
|
||||
}
|
||||
return strings.Join(msgs, "; ")
|
||||
}
|
||||
|
||||
// AllErrors returns a list of validation violation errors.
|
||||
func (m TravisCIMultiError) AllErrors() []error { return m }
|
||||
|
||||
// TravisCIValidationError is the validation error returned by
|
||||
// TravisCI.Validate if the designated constraints aren't met.
|
||||
type TravisCIValidationError struct {
|
||||
field string
|
||||
reason string
|
||||
cause error
|
||||
key bool
|
||||
}
|
||||
|
||||
// Field function returns field value.
|
||||
func (e TravisCIValidationError) Field() string { return e.field }
|
||||
|
||||
// Reason function returns reason value.
|
||||
func (e TravisCIValidationError) Reason() string { return e.reason }
|
||||
|
||||
// Cause function returns cause value.
|
||||
func (e TravisCIValidationError) Cause() error { return e.cause }
|
||||
|
||||
// Key function returns key value.
|
||||
func (e TravisCIValidationError) Key() bool { return e.key }
|
||||
|
||||
// ErrorName returns error name.
|
||||
func (e TravisCIValidationError) ErrorName() string { return "TravisCIValidationError" }
|
||||
|
||||
// Error satisfies the builtin error interface
|
||||
func (e TravisCIValidationError) Error() string {
|
||||
cause := ""
|
||||
if e.cause != nil {
|
||||
cause = fmt.Sprintf(" | caused by: %v", e.cause)
|
||||
}
|
||||
|
||||
key := ""
|
||||
if e.key {
|
||||
key = "key for "
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"invalid %sTravisCI.%s: %s%s",
|
||||
key,
|
||||
e.field,
|
||||
e.reason,
|
||||
cause)
|
||||
}
|
||||
|
||||
var _ error = TravisCIValidationError{}
|
||||
|
||||
var _ interface {
|
||||
Field() string
|
||||
Reason() string
|
||||
Key() bool
|
||||
Cause() error
|
||||
ErrorName() string
|
||||
} = TravisCIValidationError{}
|
||||
|
||||
// Validate checks the field values on Confluence with the rules defined in the
|
||||
// proto definition for this message. If any rules are violated, the first
|
||||
// error encountered is returned, or nil if there are no violations.
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
@@ -40,7 +40,7 @@ func TestNewScanErrors(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := NewScanErrors()
|
||||
|
||||
if !reflect.DeepEqual(got, tc.want) {
|
||||
if !errors.Is(got.Errors(), tc.want.Errors()) {
|
||||
t.Errorf("got %+v, want %+v", got, tc.want)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -233,7 +233,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
|
||||
aCtx.Logger().Error(err, "invalid repository", "repo", repo)
|
||||
continue
|
||||
}
|
||||
s.filteredRepoCache.Set(repo, r)
|
||||
s.filteredRepoCache.Set(r, r)
|
||||
}
|
||||
|
||||
s.includeIssueComments = s.conn.IncludeIssueComments
|
||||
@@ -825,51 +825,38 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
|
||||
// Unauthenticated access to most github endpoints has a rate limit of 60 requests per hour.
|
||||
// This will likely only be exhausted if many users/orgs are scanned without auth
|
||||
func (s *Source) handleRateLimit(errIn error, res *github.Response) bool {
|
||||
var (
|
||||
knownWait = true
|
||||
remaining = 0
|
||||
retryAfter time.Duration
|
||||
)
|
||||
|
||||
// GitHub has both primary (RateLimit) and secondary (AbuseRateLimit) errors.
|
||||
var rateLimit *github.RateLimitError
|
||||
var abuseLimit *github.AbuseRateLimitError
|
||||
if errors.As(errIn, &rateLimit) {
|
||||
// Do nothing
|
||||
} else if errors.As(errIn, &abuseLimit) {
|
||||
retryAfter = abuseLimit.GetRetryAfter()
|
||||
} else {
|
||||
limit, ok := errIn.(*github.RateLimitError)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
githubNumRateLimitEncountered.WithLabelValues(s.name).Inc()
|
||||
// Parse retry information from response headers, unless a Retry-After value was already provided.
|
||||
// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#exceeding-the-rate-limit
|
||||
if retryAfter <= 0 && res != nil {
|
||||
var err error
|
||||
remaining, err = strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
|
||||
|
||||
if res != nil {
|
||||
knownWait := true
|
||||
remaining, err := strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
|
||||
if err != nil {
|
||||
knownWait = false
|
||||
}
|
||||
|
||||
resetTime, err := strconv.Atoi(res.Header.Get("x-ratelimit-reset"))
|
||||
if err != nil || resetTime == 0 {
|
||||
knownWait = false
|
||||
} else if resetTime > 0 {
|
||||
retryAfter = time.Duration(int64(resetTime)-time.Now().Unix()) * time.Second
|
||||
}
|
||||
|
||||
if knownWait && remaining == 0 {
|
||||
waitTime := int64(resetTime) - time.Now().Unix()
|
||||
if waitTime > 0 {
|
||||
duration := time.Duration(waitTime+1) * time.Second
|
||||
s.log.V(2).Info("rate limited", "resumeTime", time.Now().Add(duration).String())
|
||||
time.Sleep(duration)
|
||||
githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(duration.Seconds())
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resumeTime := time.Now().Add(retryAfter).String()
|
||||
if knownWait && remaining == 0 && retryAfter > 0 {
|
||||
s.log.V(2).Info("rate limited", "retry_after", retryAfter.String(), "resume_time", resumeTime)
|
||||
} else {
|
||||
// TODO: Use exponential backoff instead of static retry time.
|
||||
retryAfter = time.Minute * 5
|
||||
s.log.V(2).Error(errIn, "unexpected rate limit error", "retry_after", retryAfter.String(), "resume_time", resumeTime)
|
||||
}
|
||||
time.Sleep(retryAfter)
|
||||
githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(retryAfter.Seconds())
|
||||
s.log.V(2).Info("handling rate limit (5 minutes retry)", "retry-after", limit.Message)
|
||||
time.Sleep(time.Minute * 5)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -1144,10 +1131,9 @@ var (
|
||||
)
|
||||
|
||||
type repoInfo struct {
|
||||
owner string
|
||||
repo string
|
||||
repoPath string
|
||||
visibility source_metadatapb.Visibility
|
||||
owner string
|
||||
repo string
|
||||
repoPath string
|
||||
}
|
||||
|
||||
func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimmedURL []string, repoURL *url.URL, chunksChan chan *sources.Chunk) error {
|
||||
@@ -1158,16 +1144,7 @@ func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimm
|
||||
owner := trimmedURL[1]
|
||||
repo := trimmedURL[2]
|
||||
|
||||
if !(s.includeIssueComments || s.includePRComments) {
|
||||
return nil
|
||||
}
|
||||
|
||||
repoInfo := repoInfo{
|
||||
owner: owner,
|
||||
repo: repo,
|
||||
repoPath: repoPath,
|
||||
visibility: s.visibilityOf(ctx, repoPath),
|
||||
}
|
||||
repoInfo := repoInfo{owner: owner, repo: repo, repoPath: repoPath}
|
||||
|
||||
if s.includeIssueComments {
|
||||
if err := s.processIssueComments(ctx, repoInfo, chunksChan); err != nil {
|
||||
@@ -1214,7 +1191,7 @@ func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan ch
|
||||
return err
|
||||
}
|
||||
|
||||
if err = s.chunkIssues(ctx, info, issues, chunksChan); err != nil {
|
||||
if err = s.chunkIssues(ctx, info.repo, info.repoPath, issues, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1249,7 +1226,7 @@ func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunks
|
||||
return err
|
||||
}
|
||||
|
||||
if err = s.chunkIssueComments(ctx, info, issueComments, chunksChan); err != nil {
|
||||
if err = s.chunkIssueComments(ctx, info.repo, info.repoPath, issueComments, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1285,7 +1262,7 @@ func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan
|
||||
return err
|
||||
}
|
||||
|
||||
if err = s.chunkPullRequests(ctx, info, prs, chunksChan); err != nil {
|
||||
if err = s.chunkPullRequests(ctx, info.repo, prs, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1320,7 +1297,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
|
||||
return err
|
||||
}
|
||||
|
||||
if err = s.chunkPullRequestComments(ctx, info, prComments, chunksChan); err != nil {
|
||||
if err = s.chunkPullRequestComments(ctx, info.repo, prComments, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1333,7 +1310,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) chunkIssues(ctx context.Context, repo, repoPath string, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
|
||||
for _, issue := range issues {
|
||||
|
||||
// Skip pull requests since covered by processPRs.
|
||||
@@ -1353,9 +1330,9 @@ func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*g
|
||||
Link: sanitizer.UTF8(issue.GetHTMLURL()),
|
||||
Username: sanitizer.UTF8(issue.GetUser().GetLogin()),
|
||||
Email: sanitizer.UTF8(issue.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(repoInfo.repo),
|
||||
Repository: sanitizer.UTF8(repo),
|
||||
Timestamp: sanitizer.UTF8(issue.GetCreatedAt().String()),
|
||||
Visibility: repoInfo.visibility,
|
||||
Visibility: s.visibilityOf(ctx, repoPath),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1372,7 +1349,7 @@ func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*g
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
|
||||
for _, comment := range comments {
|
||||
// Create chunk and send it to the channel.
|
||||
chunk := &sources.Chunk{
|
||||
@@ -1386,9 +1363,9 @@ func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comm
|
||||
Link: sanitizer.UTF8(comment.GetHTMLURL()),
|
||||
Username: sanitizer.UTF8(comment.GetUser().GetLogin()),
|
||||
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(repoInfo.repo),
|
||||
Repository: sanitizer.UTF8(repo),
|
||||
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
|
||||
Visibility: repoInfo.visibility,
|
||||
Visibility: s.visibilityOf(ctx, repoPath),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1405,7 +1382,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comm
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
|
||||
for _, comment := range comments {
|
||||
// Create chunk and send it to the channel.
|
||||
chunk := &sources.Chunk{
|
||||
@@ -1419,9 +1396,8 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo
|
||||
Link: sanitizer.UTF8(comment.GetHTMLURL()),
|
||||
Username: sanitizer.UTF8(comment.GetUser().GetLogin()),
|
||||
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(repoInfo.repo),
|
||||
Repository: sanitizer.UTF8(repo),
|
||||
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
|
||||
Visibility: repoInfo.visibility,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1438,7 +1414,7 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) chunkPullRequests(ctx context.Context, repo string, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
|
||||
for _, pr := range prs {
|
||||
// Create chunk and send it to the channel.
|
||||
chunk := &sources.Chunk{
|
||||
@@ -1452,9 +1428,8 @@ func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs [
|
||||
Link: sanitizer.UTF8(pr.GetHTMLURL()),
|
||||
Username: sanitizer.UTF8(pr.GetUser().GetLogin()),
|
||||
Email: sanitizer.UTF8(pr.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(repoInfo.repo),
|
||||
Repository: sanitizer.UTF8(repo),
|
||||
Timestamp: sanitizer.UTF8(pr.GetCreatedAt().String()),
|
||||
Visibility: repoInfo.visibility,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1487,7 +1462,8 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
|
||||
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(gistUrl),
|
||||
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
|
||||
// TODO: Fetching this requires making an additional API call. We may want to include this in the future.
|
||||
// Fetching this information requires making an additional API call.
|
||||
// We may want to include this in the future.
|
||||
// Visibility: s.visibilityOf(ctx, repoPath),
|
||||
},
|
||||
},
|
||||
|
||||
@@ -89,14 +89,14 @@ func (f Fatal) Unwrap() error { return f.error }
|
||||
// ChunkError is a custom error type for errors encountered during chunking of
|
||||
// a specific unit.
|
||||
type ChunkError struct {
|
||||
Unit SourceUnit
|
||||
Err error
|
||||
unit SourceUnit
|
||||
err error
|
||||
}
|
||||
|
||||
func (f ChunkError) Error() string {
|
||||
return fmt.Sprintf("error chunking unit %q: %s", f.Unit.SourceUnitID(), f.Err.Error())
|
||||
return fmt.Sprintf("error chunking unit %q: %s", f.unit.SourceUnitID(), f.err.Error())
|
||||
}
|
||||
func (f ChunkError) Unwrap() error { return f.Err }
|
||||
func (f ChunkError) Unwrap() error { return f.err }
|
||||
|
||||
// JobProgress aggregates information about a run of a Source.
|
||||
type JobProgress struct {
|
||||
@@ -351,17 +351,3 @@ func (m JobProgressMetrics) ElapsedTime() time.Duration {
|
||||
}
|
||||
return m.EndTime.Sub(m.StartTime)
|
||||
}
|
||||
|
||||
// ErrorsFor returns all the errors for the given SourceUnit. If there are no
|
||||
// errors, including the case that the unit has not been encountered, nil will
|
||||
// be returned.
|
||||
func (m JobProgressMetrics) ErrorsFor(unit SourceUnit) []error {
|
||||
var errs []error
|
||||
for _, err := range m.Errors {
|
||||
var chunkErr ChunkError
|
||||
if errors.As(err, &chunkErr) && chunkErr.Unit == unit {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
@@ -1,218 +0,0 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
// UnitHook implements JobProgressHook for tracking the progress of each
|
||||
// individual unit.
|
||||
type UnitHook struct {
|
||||
metrics *lru.Cache[string, *UnitMetrics]
|
||||
mu sync.Mutex
|
||||
NoopHook
|
||||
}
|
||||
|
||||
type UnitHookOpt func(*UnitHook)
|
||||
|
||||
func WithUnitHookCache(cache *lru.Cache[string, *UnitMetrics]) UnitHookOpt {
|
||||
return func(hook *UnitHook) { hook.metrics = cache }
|
||||
}
|
||||
|
||||
func NewUnitHook(ctx context.Context, opts ...UnitHookOpt) *UnitHook {
|
||||
// lru.NewWithEvict can only fail if the size is < 0.
|
||||
cache, _ := lru.NewWithEvict(1024, func(key string, value *UnitMetrics) {
|
||||
if value.handled {
|
||||
return
|
||||
}
|
||||
ctx.Logger().Error(fmt.Errorf("eviction"), "dropping unit metric",
|
||||
"id", key,
|
||||
"metric", value,
|
||||
)
|
||||
})
|
||||
hook := UnitHook{metrics: cache}
|
||||
for _, opt := range opts {
|
||||
opt(&hook)
|
||||
}
|
||||
return &hook
|
||||
}
|
||||
|
||||
// id is a helper method to generate an ID for the given job and unit.
|
||||
func (u *UnitHook) id(ref JobProgressRef, unit SourceUnit) string {
|
||||
unitID := ""
|
||||
if unit != nil {
|
||||
unitID = unit.SourceUnitID()
|
||||
}
|
||||
return fmt.Sprintf("%d/%d/%s", ref.SourceID, ref.JobID, unitID)
|
||||
}
|
||||
|
||||
func (u *UnitHook) StartUnitChunking(ref JobProgressRef, unit SourceUnit, start time.Time) {
|
||||
id := u.id(ref, unit)
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
|
||||
u.metrics.Add(id, &UnitMetrics{
|
||||
Unit: unit,
|
||||
Parent: ref,
|
||||
StartTime: start,
|
||||
})
|
||||
}
|
||||
|
||||
func (u *UnitHook) EndUnitChunking(ref JobProgressRef, unit SourceUnit, end time.Time) {
|
||||
id := u.id(ref, unit)
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
|
||||
metrics, ok := u.metrics.Get(id)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
metrics.EndTime = end
|
||||
}
|
||||
|
||||
func (u *UnitHook) ReportChunk(ref JobProgressRef, unit SourceUnit, chunk *Chunk) {
|
||||
id := u.id(ref, unit)
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
|
||||
metrics, ok := u.metrics.Get(id)
|
||||
if !ok && unit != nil {
|
||||
// The unit has been evicted.
|
||||
return
|
||||
} else if !ok && unit == nil {
|
||||
// This is a chunk from a non-unit source.
|
||||
metrics = &UnitMetrics{
|
||||
Unit: nil,
|
||||
Parent: ref,
|
||||
StartTime: ref.Snapshot().StartTime,
|
||||
}
|
||||
u.metrics.Add(id, metrics)
|
||||
}
|
||||
metrics.TotalChunks++
|
||||
metrics.TotalBytes += uint64(len(chunk.Data))
|
||||
}
|
||||
|
||||
func (u *UnitHook) ReportError(ref JobProgressRef, err error) {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
|
||||
// Always add the error to the nil unit if it exists.
|
||||
if metrics, ok := u.metrics.Get(u.id(ref, nil)); ok {
|
||||
metrics.Errors = append(metrics.Errors, err)
|
||||
}
|
||||
|
||||
// Check if it's a ChunkError for a specific unit.
|
||||
var chunkErr ChunkError
|
||||
if !errors.As(err, &chunkErr) {
|
||||
return
|
||||
}
|
||||
id := u.id(ref, chunkErr.Unit)
|
||||
|
||||
metrics, ok := u.metrics.Get(id)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
metrics.Errors = append(metrics.Errors, err)
|
||||
}
|
||||
|
||||
func (u *UnitHook) Finish(ref JobProgressRef) {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
// Clear out any metrics on this job. This covers the case for the
|
||||
// source running without unit support.
|
||||
prefix := u.id(ref, nil)
|
||||
for _, id := range u.metrics.Keys() {
|
||||
if !strings.HasPrefix(id, prefix) {
|
||||
continue
|
||||
}
|
||||
metric, ok := u.metrics.Get(id)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
// If the unit is nil, the source does not support units.
|
||||
// Use the overall job metrics instead.
|
||||
if metric.Unit == nil {
|
||||
snap := ref.Snapshot()
|
||||
metric.StartTime = snap.StartTime
|
||||
metric.EndTime = snap.EndTime
|
||||
metric.Errors = snap.Errors
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// UnitMetrics gets all the currently active or newly finished metrics for this
|
||||
// job. If a unit returned from this method has finished, it will be removed
|
||||
// from the cache and no longer returned in successive calls to UnitMetrics().
|
||||
func (u *UnitHook) UnitMetrics() []UnitMetrics {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
output := make([]UnitMetrics, 0, u.metrics.Len())
|
||||
for _, id := range u.metrics.Keys() {
|
||||
metric, ok := u.metrics.Get(id)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
output = append(output, *metric)
|
||||
if metric.IsFinished() {
|
||||
metric.handled = true
|
||||
u.metrics.Remove(id)
|
||||
}
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
type UnitMetrics struct {
|
||||
Unit SourceUnit
|
||||
Parent JobProgressRef
|
||||
// Start and end time for chunking this unit.
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
// Total number of chunks produced from this unit.
|
||||
TotalChunks uint64
|
||||
// Total number of bytes produced from this unit.
|
||||
TotalBytes uint64
|
||||
// All errors encountered by this unit.
|
||||
Errors []error
|
||||
// Flag to mark that these metrics were intentionally evicted from
|
||||
// the cache.
|
||||
handled bool
|
||||
}
|
||||
|
||||
func (u UnitMetrics) IsFinished() bool {
|
||||
return !u.EndTime.IsZero()
|
||||
}
|
||||
|
||||
// ElapsedTime is a convenience method that provides the elapsed time the job
|
||||
// has been running. If it hasn't started yet, 0 is returned. If it has
|
||||
// finished, the total time is returned.
|
||||
func (u UnitMetrics) ElapsedTime() time.Duration {
|
||||
if u.StartTime.IsZero() {
|
||||
return 0
|
||||
}
|
||||
if u.EndTime.IsZero() {
|
||||
return time.Since(u.StartTime)
|
||||
}
|
||||
return u.EndTime.Sub(u.StartTime)
|
||||
}
|
||||
|
||||
// NoopHook implements JobProgressHook by doing nothing. This is useful for
|
||||
// embedding in other structs to overwrite only the methods of the interface
|
||||
// that you care about.
|
||||
type NoopHook struct{}
|
||||
|
||||
func (NoopHook) Start(JobProgressRef, time.Time) {}
|
||||
func (NoopHook) End(JobProgressRef, time.Time) {}
|
||||
func (NoopHook) StartEnumerating(JobProgressRef, time.Time) {}
|
||||
func (NoopHook) EndEnumerating(JobProgressRef, time.Time) {}
|
||||
func (NoopHook) StartUnitChunking(JobProgressRef, SourceUnit, time.Time) {}
|
||||
func (NoopHook) EndUnitChunking(JobProgressRef, SourceUnit, time.Time) {}
|
||||
func (NoopHook) ReportError(JobProgressRef, error) {}
|
||||
func (NoopHook) ReportUnit(JobProgressRef, SourceUnit) {}
|
||||
func (NoopHook) ReportChunk(JobProgressRef, SourceUnit, *Chunk) {}
|
||||
func (NoopHook) Finish(JobProgressRef) {}
|
||||
@@ -126,26 +126,3 @@ func TestJobProgressElapsedTime(t *testing.T) {
|
||||
metrics.EndTime = metrics.StartTime.Add(1 * time.Hour)
|
||||
assert.Equal(t, metrics.ElapsedTime(), 1*time.Hour)
|
||||
}
|
||||
|
||||
func TestJobProgressErrorsFor(t *testing.T) {
|
||||
metrics := JobProgressMetrics{
|
||||
Errors: []error{
|
||||
Fatal{ChunkError{
|
||||
Unit: CommonSourceUnit{ID: "foo"},
|
||||
Err: fmt.Errorf("foo error"),
|
||||
}},
|
||||
ChunkError{
|
||||
Unit: CommonSourceUnit{ID: "foo"},
|
||||
Err: fmt.Errorf("foo again error"),
|
||||
},
|
||||
ChunkError{
|
||||
Unit: CommonSourceUnit{ID: "bar"},
|
||||
Err: fmt.Errorf("bar error"),
|
||||
},
|
||||
fmt.Errorf("hi there"),
|
||||
},
|
||||
}
|
||||
assert.Equal(t, 2, len(metrics.ErrorsFor(CommonSourceUnit{ID: "foo"})))
|
||||
assert.Equal(t, 1, len(metrics.ErrorsFor(CommonSourceUnit{ID: "bar"})))
|
||||
assert.Equal(t, 0, len(metrics.ErrorsFor(CommonSourceUnit{ID: "baz"})))
|
||||
}
|
||||
|
||||
@@ -3,10 +3,8 @@ package sources
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
@@ -202,8 +200,6 @@ func (c *unitChunker) ChunkUnit(ctx context.Context, unit SourceUnit, rep ChunkR
|
||||
if err := rep.ChunkErr(ctx, fmt.Errorf(step.err)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if step.output == "" {
|
||||
continue
|
||||
}
|
||||
if err := rep.ChunkOk(ctx, Chunk{Data: []byte(step.output)}); err != nil {
|
||||
@@ -314,114 +310,3 @@ func TestSourceManagerAvailableCapacity(t *testing.T) {
|
||||
<-ref.Done() // Wait for the job to finish.
|
||||
assert.Equal(t, 1337, mgr.AvailableCapacity())
|
||||
}
|
||||
|
||||
func TestSourceManagerUnitHook(t *testing.T) {
|
||||
hook := NewUnitHook(context.TODO())
|
||||
|
||||
input := []unitChunk{
|
||||
{unit: "one", output: "bar"},
|
||||
{unit: "two", err: "oh no"},
|
||||
{unit: "three", err: "not again"},
|
||||
}
|
||||
mgr := NewManager(
|
||||
WithBufferedOutput(8),
|
||||
WithSourceUnits(), WithConcurrentUnits(1),
|
||||
WithReportHook(hook),
|
||||
)
|
||||
source, err := buildDummy(&unitChunker{input})
|
||||
assert.NoError(t, err)
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
assert.NoError(t, err)
|
||||
<-ref.Done()
|
||||
|
||||
metrics := hook.UnitMetrics()
|
||||
assert.Equal(t, 3, len(metrics))
|
||||
sort.Slice(metrics, func(i, j int) bool {
|
||||
return metrics[i].EndTime.Before(metrics[j].EndTime)
|
||||
})
|
||||
m0, m1, m2 := metrics[0], metrics[1], metrics[2]
|
||||
|
||||
assert.Equal(t, "one", m0.Unit.SourceUnitID())
|
||||
assert.Equal(t, uint64(1), m0.TotalChunks)
|
||||
assert.Equal(t, uint64(3), m0.TotalBytes)
|
||||
assert.NotZero(t, m0.StartTime)
|
||||
assert.NotZero(t, m0.EndTime)
|
||||
assert.NotZero(t, m0.ElapsedTime())
|
||||
assert.Equal(t, 0, len(m0.Errors))
|
||||
|
||||
assert.Equal(t, "two", m1.Unit.SourceUnitID())
|
||||
assert.Equal(t, uint64(0), m1.TotalChunks)
|
||||
assert.Equal(t, uint64(0), m1.TotalBytes)
|
||||
assert.NotZero(t, m1.StartTime)
|
||||
assert.NotZero(t, m1.EndTime)
|
||||
assert.NotZero(t, m1.ElapsedTime())
|
||||
assert.Equal(t, 1, len(m1.Errors))
|
||||
|
||||
assert.Equal(t, "three", m2.Unit.SourceUnitID())
|
||||
assert.Equal(t, uint64(0), m2.TotalChunks)
|
||||
assert.Equal(t, uint64(0), m2.TotalBytes)
|
||||
assert.NotZero(t, m2.StartTime)
|
||||
assert.NotZero(t, m2.EndTime)
|
||||
assert.NotZero(t, m2.ElapsedTime())
|
||||
assert.Equal(t, 1, len(m2.Errors))
|
||||
}
|
||||
|
||||
// TestSourceManagerUnitHookNoBlock tests that the UnitHook drops metrics if
|
||||
// they aren't handled fast enough.
|
||||
func TestSourceManagerUnitHookNoBlock(t *testing.T) {
|
||||
var evictedKeys []string
|
||||
cache, _ := lru.NewWithEvict(1, func(key string, _ *UnitMetrics) {
|
||||
evictedKeys = append(evictedKeys, key)
|
||||
})
|
||||
hook := NewUnitHook(context.TODO(), WithUnitHookCache(cache))
|
||||
|
||||
input := []unitChunk{
|
||||
{unit: "one", output: "bar"},
|
||||
{unit: "two", err: "oh no"},
|
||||
{unit: "three", err: "not again"},
|
||||
}
|
||||
mgr := NewManager(
|
||||
WithBufferedOutput(8),
|
||||
WithSourceUnits(), WithConcurrentUnits(1),
|
||||
WithReportHook(hook),
|
||||
)
|
||||
source, err := buildDummy(&unitChunker{input})
|
||||
assert.NoError(t, err)
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
assert.NoError(t, err)
|
||||
<-ref.Done()
|
||||
|
||||
assert.Equal(t, 2, len(evictedKeys))
|
||||
metrics := hook.UnitMetrics()
|
||||
assert.Equal(t, 1, len(metrics))
|
||||
assert.Equal(t, "three", metrics[0].Unit.SourceUnitID())
|
||||
}
|
||||
|
||||
// TestSourceManagerUnitHookNoUnits tests whether the UnitHook works for
|
||||
// sources that don't support units.
|
||||
func TestSourceManagerUnitHookNoUnits(t *testing.T) {
|
||||
hook := NewUnitHook(context.TODO())
|
||||
|
||||
mgr := NewManager(
|
||||
WithBufferedOutput(8),
|
||||
WithReportHook(hook),
|
||||
)
|
||||
source, err := buildDummy(&counterChunker{count: 5})
|
||||
assert.NoError(t, err)
|
||||
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
assert.NoError(t, err)
|
||||
<-ref.Done()
|
||||
|
||||
metrics := hook.UnitMetrics()
|
||||
assert.Equal(t, 1, len(metrics))
|
||||
|
||||
m := metrics[0]
|
||||
assert.Equal(t, nil, m.Unit)
|
||||
assert.Equal(t, uint64(5), m.TotalChunks)
|
||||
assert.Equal(t, uint64(5), m.TotalBytes)
|
||||
assert.NotZero(t, m.StartTime)
|
||||
assert.NotZero(t, m.EndTime)
|
||||
assert.NotZero(t, m.ElapsedTime())
|
||||
assert.Equal(t, 0, len(m.Errors))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
package travisci
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/shuheiktgw/go-travis"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const (
|
||||
SourceType = sourcespb.SourceType_SOURCE_TYPE_TRAVISCI
|
||||
|
||||
baseURL = "https://api.travis-ci.com/"
|
||||
|
||||
pageSize = 100
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
verify bool
|
||||
jobPool *errgroup.Group
|
||||
sources.Progress
|
||||
client *travis.Client
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
returnAfterFirstChunk bool
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return SourceType
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
// Init returns an initialized TravisCI source.
|
||||
func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
s.verify = verify
|
||||
s.jobPool = &errgroup.Group{}
|
||||
s.jobPool.SetLimit(concurrency)
|
||||
|
||||
var conn sourcespb.TravisCI
|
||||
if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
|
||||
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
|
||||
}
|
||||
|
||||
switch conn.Credential.(type) {
|
||||
case *sourcespb.TravisCI_Token:
|
||||
if conn.GetToken() == "" {
|
||||
return errors.New("token is empty")
|
||||
}
|
||||
s.client = travis.NewClient(baseURL, conn.GetToken())
|
||||
s.client.HTTPClient = common.RetryableHttpClientTimeout(3)
|
||||
|
||||
user, _, err := s.client.User.Current(ctx, nil)
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "error getting testing travis client", 0)
|
||||
}
|
||||
ctx.Logger().V(2).Info("authenticated to Travis CI with user", "username", user.Login)
|
||||
default:
|
||||
return errors.New("credential type not implemented for Travis CI")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {
|
||||
for repoPage := 0; ; repoPage++ {
|
||||
repositories, _, err := s.client.Repositories.List(ctx, &travis.RepositoriesOption{
|
||||
Limit: pageSize,
|
||||
Offset: repoPage * pageSize,
|
||||
})
|
||||
if err != nil {
|
||||
if repoPage == 0 {
|
||||
return fmt.Errorf("error listing repositories: %w", err)
|
||||
}
|
||||
err = reporter.UnitErr(ctx, err)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reporting error: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(repositories) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, repo := range repositories {
|
||||
err = reporter.UnitOk(ctx, sources.CommonSourceUnit{
|
||||
ID: strconv.Itoa(int(*repo.Id)),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reporting unit: %w", err)
|
||||
}
|
||||
ctx.Logger().V(2).Info("enumerated repository", "id", repo.Id, "name", repo.Name)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ChunkUnit implements SourceUnitChunker interface.
|
||||
func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
|
||||
repo, _, err := s.client.Repositories.Find(ctx, unit.SourceUnitID(), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error finding repository: %w", err)
|
||||
}
|
||||
logger := ctx.Logger().WithValues("repo", *repo.Name)
|
||||
logger.V(2).Info("scanning repository")
|
||||
|
||||
// Counts continuous errors from ListByRepoSlug. Used to quit early in
|
||||
// case the API always returns an error.
|
||||
var buildPageErrs int
|
||||
for buildPage := 0; ; buildPage++ {
|
||||
builds, _, err := s.client.Builds.ListByRepoSlug(ctx, *repo.Slug, &travis.BuildsByRepoOption{
|
||||
Limit: pageSize,
|
||||
Offset: buildPage * pageSize,
|
||||
})
|
||||
if err != nil {
|
||||
if err := reporter.ChunkErr(ctx, err); err != nil {
|
||||
return err
|
||||
}
|
||||
buildPageErrs++
|
||||
if buildPageErrs >= 5 {
|
||||
return fmt.Errorf("encountered too many errors listing builds, aborting")
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Reset the page error counter.
|
||||
buildPageErrs = 0
|
||||
|
||||
if len(builds) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, build := range builds {
|
||||
jobs, _, err := s.client.Jobs.ListByBuild(ctx, *build.Id)
|
||||
if err != nil {
|
||||
if err := reporter.ChunkErr(ctx, err); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if len(jobs) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, job := range jobs {
|
||||
log, _, err := s.client.Logs.FindByJobId(ctx, *job.Id)
|
||||
if err != nil {
|
||||
if err := reporter.ChunkErr(ctx, err); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
logger.V(3).Info("scanning job", "id", *job.Id, "number", *job.Number)
|
||||
|
||||
chunk := sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
Data: []byte(*log.Content),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_TravisCI{
|
||||
TravisCI: &source_metadatapb.TravisCI{
|
||||
Username: *job.Owner.Login,
|
||||
Repository: *repo.Name,
|
||||
BuildNumber: *build.Number,
|
||||
JobNumber: *job.Number,
|
||||
Link: fmt.Sprintf("https://app.travis-ci.com/github/%s/%s/jobs/%d", *job.Owner.Login, *repo.Name, *job.Id),
|
||||
Public: !*repo.Private,
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
|
||||
if err := reporter.ChunkOk(ctx, chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.returnAfterFirstChunk {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
// It's a no-op because we've implemented the SourceUnitChunker interface instead.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package travisci
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest"
|
||||
)
|
||||
|
||||
func TestSource_Scan(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
secret, err := common.GetTestSecret(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||
}
|
||||
token := secret.MustGetField("TRAVISCI_TOKEN")
|
||||
|
||||
type init struct {
|
||||
name string
|
||||
verify bool
|
||||
connection *sourcespb.TravisCI
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
init init
|
||||
wantSourceMetadata *source_metadatapb.MetaData
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "get a chunk",
|
||||
init: init{
|
||||
name: "example repo",
|
||||
connection: &sourcespb.TravisCI{
|
||||
Credential: &sourcespb.TravisCI_Token{
|
||||
Token: token,
|
||||
},
|
||||
},
|
||||
verify: true,
|
||||
},
|
||||
wantSourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_TravisCI{
|
||||
TravisCI: &source_metadatapb.TravisCI{
|
||||
Username: "truffle-sandbox",
|
||||
Repository: "test-repo",
|
||||
BuildNumber: "1",
|
||||
JobNumber: "1.1",
|
||||
Link: "https://app.travis-ci.com/github/truffle-sandbox/test-repo/jobs/611053994",
|
||||
Public: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Source{}
|
||||
|
||||
conn, err := anypb.New(tt.init.connection)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
|
||||
reporter := sourcestest.TestReporter{}
|
||||
s.returnAfterFirstChunk = true
|
||||
|
||||
err = s.Enumerate(ctx, &reporter)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("Source.Enumerate() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
|
||||
for _, unit := range reporter.Units {
|
||||
err = s.ChunkUnit(ctx, unit, &reporter)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("Source.ChunkUnit() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
for _, chunk := range reporter.Chunks {
|
||||
if diff := pretty.Compare(chunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {
|
||||
t.Fatalf("comparing chunks: %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -945,7 +945,6 @@ enum DetectorType {
|
||||
Ramp = 934;
|
||||
Klaviyo = 935;
|
||||
SourcegraphCody = 936;
|
||||
Voiceflow = 937;
|
||||
}
|
||||
|
||||
message Result {
|
||||
|
||||
@@ -44,6 +44,16 @@ message CircleCI {
|
||||
string email = 7;
|
||||
}
|
||||
|
||||
message TravisCI {
|
||||
string username = 2;
|
||||
string repository = 3;
|
||||
string build_number = 4;
|
||||
string job_number = 5;
|
||||
string link = 6;
|
||||
bool public = 7;
|
||||
}
|
||||
|
||||
|
||||
message Confluence {
|
||||
string page = 1;
|
||||
string space = 2;
|
||||
@@ -294,5 +304,6 @@ message MetaData {
|
||||
SharePoint sharepoint = 25;
|
||||
GoogleDrive googleDrive = 26;
|
||||
AzureRepos azureRepos = 27;
|
||||
TravisCI travisCI = 28;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ enum SourceType {
|
||||
SOURCE_TYPE_SHAREPOINT = 29;
|
||||
SOURCE_TYPE_GCS_UNAUTHED = 30;
|
||||
SOURCE_TYPE_AZURE_REPOS = 31;
|
||||
SOURCE_TYPE_TRAVISCI = 32;
|
||||
}
|
||||
|
||||
message LocalSource {
|
||||
@@ -83,6 +84,13 @@ message CircleCI {
|
||||
}
|
||||
}
|
||||
|
||||
message TravisCI {
|
||||
string endpoint = 1 [(validate.rules).string.uri_ref = true];
|
||||
oneof credential {
|
||||
string token = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message Confluence {
|
||||
string endpoint = 1 [(validate.rules).string.uri_ref = true];
|
||||
oneof credential {
|
||||
|
||||
Reference in New Issue
Block a user