Compare commits

..

23 Commits

Author SHA1 Message Date
Ahrav Dutta 584ef1429b exported some methods and constructor since it will need to be be used by the enterprise pipeline as well 2023-10-22 19:18:41 -07:00
Ahrav Dutta f010f06edc address comments 2023-10-20 16:12:56 -07:00
Ahrav Dutta 60df4d6684 fix 2023-10-12 13:27:35 -07:00
ahrav 0a4a5f3ab4 Merge branch 'main' into filter-detectors-before-detecting 2023-10-12 13:26:40 -07:00
Ahrav Dutta 88c8f69532 update comments 2023-10-12 13:22:01 -07:00
Ahrav Dutta d144e9dc9b move structs to ahocorasickcore 2023-10-12 13:18:18 -07:00
Ahrav Dutta e43d95f1c5 update comments 2023-10-12 13:17:03 -07:00
Ahrav Dutta 0064b00c5f refactor ahocorasickcore into own file. 2023-10-12 13:10:38 -07:00
Ahrav Dutta ff3751fda2 remove old code. 2023-08-29 14:35:38 -07:00
Ahrav Dutta 5089383e71 delete the pool. 2023-08-29 14:32:06 -07:00
Ahrav Dutta 1311e7df40 use regular map. 2023-08-28 09:10:15 -07:00
Ahrav Dutta b90bb8cbe2 update comments. 2023-08-28 07:16:18 -07:00
Ahrav Dutta 621bb75f40 remove Engine prefix. 2023-08-28 07:01:09 -07:00
Ahrav Dutta 916a4f9ecf add comment. 2023-08-28 06:56:50 -07:00
Ahrav Dutta 11a0d1d31b Update benchmark. 2023-08-27 19:57:48 -07:00
Ahrav Dutta 04e4bc1dcf cleanup. 2023-08-27 16:55:39 -07:00
Ahrav Dutta 7dd3aa639d Break apart Start. 2023-08-27 16:42:22 -07:00
Ahrav Dutta 22564bc483 allow versioned detectors. 2023-08-27 16:23:24 -07:00
Ahrav Dutta 32bacdda7e use correct verify. 2023-08-27 15:47:28 -07:00
Ahrav Dutta 4c44a31d10 lint. 2023-08-27 11:36:46 -07:00
Ahrav Dutta 913a94f4b4 use sync.Map for concurrent access. 2023-08-27 09:54:28 -07:00
Ahrav Dutta 9f4623958d Optimize the engine to prevent iterating overing all detectors. 2023-08-26 21:29:12 -07:00
Ahrav Dutta f071a2ff14 pre filter detectors that include the keywords in the chunk. 2023-08-26 13:52:08 -07:00
45 changed files with 138 additions and 2754 deletions
+2 -2
View File
@@ -164,7 +164,7 @@ trufflehog docker --image trufflesecurity/secrets --only-verified
+ All I see is `🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷` and the program exits, what gives?
+ That means no secrets were detected
+ Why is the scan taking a long time when I scan a GitHub org
+ Why is the scan is taking a long time when I scan a GitHub org
+ Unauthenticated GitHub scans have rate limits. To improve your rate limits, include the `--token` flag with a personal access token
+ It says a private key was verified, what does that mean?
+ Check out our Driftwood blog post to learn how to do this, in short we've confirmed the key can be used live for SSH or SSL [Blog post](https://trufflesecurity.com/blog/driftwood-know-if-private-keys-are-sensitive/)
@@ -219,7 +219,7 @@ Flags:
--concurrency=10 Number of concurrent workers.
--no-verification Don't verify the results.
--only-verified Only output verified results.
--filter-unverified Only output first unverified result per chunk per detector if there is more than one result.
--filter-unverified Only output first unverified result per chunk per detector if there are more than one results.
--config=CONFIG Path to configuration file.
--print-avg-detector-time Print the average time spent on each detector.
--no-update Don't check for updates.
-2
View File
@@ -47,7 +47,6 @@ require (
github.com/h2non/filetype v1.1.3
github.com/hashicorp/go-retryablehttp v0.7.4
github.com/hashicorp/golang-lru v0.5.1
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/jlaffaye/ftp v0.2.0
github.com/joho/godotenv v1.5.1
github.com/jpillora/overseer v1.1.6
@@ -82,7 +81,6 @@ require (
google.golang.org/protobuf v1.31.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/h2non/gock.v1 v1.1.2
pgregory.net/rapid v1.1.0
sigs.k8s.io/yaml v1.3.0
)
-4
View File
@@ -418,8 +418,6 @@ github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
@@ -1015,8 +1013,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-3
View File
@@ -152,9 +152,6 @@ func init() {
cli.Version("trufflehog " + version.BuildVersion)
//Support -h for help
cli.HelpFlag.Short('h')
if len(os.Args) <= 1 && isatty.IsTerminal(os.Stdout.Fd()) {
args := tui.Run()
if len(args) == 0 {
-120
View File
@@ -1,120 +0,0 @@
package glob
import (
"fmt"
"github.com/gobwas/glob"
)
// Filter is a generic filter for excluding and including globs (limited
// regular expressions). Exclusion takes precedence if both include and exclude
// lists are provided.
type Filter struct {
exclude []glob.Glob
include []glob.Glob
}
type globFilterOpt func(*Filter) error
// WithExcludeGlobs adds exclude globs to the filter.
func WithExcludeGlobs(excludes ...string) globFilterOpt {
return func(f *Filter) error {
for _, exclude := range excludes {
g, err := glob.Compile(exclude)
if err != nil {
return fmt.Errorf("invalid exclude glob %q: %w", exclude, err)
}
f.exclude = append(f.exclude, g)
}
return nil
}
}
// WithIncludeGlobs adds include globs to the filter.
func WithIncludeGlobs(includes ...string) globFilterOpt {
return func(f *Filter) error {
for _, include := range includes {
g, err := glob.Compile(include)
if err != nil {
return fmt.Errorf("invalid include glob %q: %w", include, err)
}
f.include = append(f.include, g)
}
return nil
}
}
// NewGlobFilter creates a new Filter with the provided options.
func NewGlobFilter(opts ...globFilterOpt) (*Filter, error) {
filter := &Filter{}
for _, opt := range opts {
if err := opt(filter); err != nil {
return nil, err
}
}
return filter, nil
}
// ShouldInclude returns whether the object is in the include list or not in
// the exclude list (exclude taking precedence).
func (f *Filter) ShouldInclude(object string) bool {
if f == nil {
return true
}
exclude, include := len(f.exclude), len(f.include)
if exclude == 0 && include == 0 {
return true
} else if exclude > 0 && include == 0 {
return f.shouldIncludeFromExclude(object)
} else if exclude == 0 && include > 0 {
return f.shouldIncludeFromInclude(object)
} else {
if ok, err := f.shouldIncludeFromBoth(object); err == nil {
return ok
}
// Ambiguous case.
return false
}
}
// shouldIncludeFromExclude checks for explicitly excluded paths. This should
// only be called when the include list is empty.
func (f *Filter) shouldIncludeFromExclude(object string) bool {
for _, glob := range f.exclude {
if glob.Match(object) {
return false
}
}
return true
}
// shouldIncludeFromInclude checks for explicitly included paths. This should
// only be called when the exclude list is empty.
func (f *Filter) shouldIncludeFromInclude(object string) bool {
for _, glob := range f.include {
if glob.Match(object) {
return true
}
}
return false
}
// shouldIncludeFromBoth checks for either excluded or included paths. Exclusion
// takes precedence. If neither list contains the object, true is returned.
func (f *Filter) shouldIncludeFromBoth(object string) (bool, error) {
// Exclude takes precedence. If we find the object in the exclude list,
// we should not match.
for _, glob := range f.exclude {
if glob.Match(object) {
return false, nil
}
}
// If we find the object in the include list, we should match.
for _, glob := range f.include {
if glob.Match(object) {
return true, nil
}
}
// If we find it in neither, return an error to let the caller decide.
return false, fmt.Errorf("ambiguous match")
}
-138
View File
@@ -1,138 +0,0 @@
package glob
import (
"testing"
"github.com/stretchr/testify/assert"
"pgregory.net/rapid"
)
type globTest struct {
input string
shouldInclude bool
}
func testGlobs(t *testing.T, filter *Filter, tests ...globTest) {
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
// Invert because mentally it's easier to say whether an
// input should be included.
assert.Equal(t, tt.shouldInclude, filter.ShouldInclude(tt.input))
})
}
}
func TestGlobFilterExclude(t *testing.T) {
filter, err := NewGlobFilter(WithExcludeGlobs("foo", "bar*"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", false},
globTest{"bar", false},
globTest{"bara", false},
globTest{"barb", false},
globTest{"barbosa", false},
globTest{"foobar", true},
globTest{"food", true},
globTest{"anything else", true},
)
}
func TestGlobFilterInclude(t *testing.T) {
filter, err := NewGlobFilter(WithIncludeGlobs("foo", "bar*"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", true},
globTest{"bar", true},
globTest{"bara", true},
globTest{"barb", true},
globTest{"barbosa", true},
globTest{"foobar", false},
globTest{"food", false},
globTest{"anything else", false},
)
}
func TestGlobFilterEmpty(t *testing.T) {
filter, err := NewGlobFilter()
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", true},
globTest{"bar", true},
globTest{"bara", true},
globTest{"barb", true},
globTest{"barbosa", true},
globTest{"foobar", true},
globTest{"food", true},
globTest{"anything else", true},
)
}
func TestGlobFilterExcludeInclude(t *testing.T) {
filter, err := NewGlobFilter(WithExcludeGlobs("/foo/bar/**"), WithIncludeGlobs("/foo/**"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"/foo/a", true},
globTest{"/foo/b", true},
globTest{"/foo/c/d/e", true},
globTest{"/foo/bar/a", false},
globTest{"/foo/bar/b", false},
globTest{"/foo/bar/c/d/e", false},
globTest{"/any/other/path", false},
)
}
func TestGlobFilterExcludePrecedence(t *testing.T) {
filter, err := NewGlobFilter(WithExcludeGlobs("foo"), WithIncludeGlobs("foo*"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", false},
globTest{"foobar", true},
)
}
func TestGlobErrorContainsGlob(t *testing.T) {
invalidGlob := "[this is invalid because it doesn't close the capture group"
_, err := NewGlobFilter(WithExcludeGlobs(invalidGlob))
assert.Error(t, err)
assert.Contains(t, err.Error(), invalidGlob)
}
// The filters in this test should be mutually exclusive because one includes
// and the other excludes the same glob.
func TestGlobInverse(t *testing.T) {
for _, glob := range []string{
"a",
"a*",
"a**",
"*a",
"**a",
"*",
} {
include, err := NewGlobFilter(WithIncludeGlobs(glob))
assert.NoError(t, err)
exclude, err := NewGlobFilter(WithExcludeGlobs(glob))
assert.NoError(t, err)
rapid.Check(t, func(t *rapid.T) {
input := rapid.String().Draw(t, "input")
a, b := include.ShouldInclude(input), exclude.ShouldInclude(input)
if a == b {
t.Fatalf("Filter(Include(%q)) == Filter(Exclude(%q)) == %v for input %q", glob, glob, a, input)
}
})
}
}
func TestGlobDefaultFilters(t *testing.T) {
for _, filter := range []*Filter{nil, {}} {
rapid.Check(t, func(t *rapid.T) {
if !filter.ShouldInclude(rapid.String().Draw(t, "input")) {
t.Fatalf("filter %#v did not include input", filter)
}
})
}
}
+2 -27
View File
@@ -2,7 +2,6 @@ package airtableapikey
import (
"context"
"encoding/json"
"fmt"
"net/http"
"regexp"
@@ -21,9 +20,8 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
appPat = regexp.MustCompile(`(app[a-zA-Z0-9_-]{14})`) // could be part of url
keyPat = regexp.MustCompile(`\b(key[a-zA-Z0-9_-]{14})\b`)
personalPat = regexp.MustCompile(`(\bpat[[:alnum:]]{14}\.[[:alnum:]]{64}\b)`)
appPat = regexp.MustCompile(`(app[a-zA-Z0-9_-]{14})`) // could be part of url
keyPat = regexp.MustCompile(`\b(key[a-zA-Z0-9_-]{14})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -32,31 +30,17 @@ func (s Scanner) Keywords() []string {
return []string{"airtable"}
}
type response struct {
Error struct {
Type string `json:"type"`
Message string `json:"message"`
} `json:"error"`
}
// FromData will find and optionally verify AirtableApiKey secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
appMatches := appPat.FindAllStringSubmatch(dataStr, -1)
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
personalKeyMatches := personalPat.FindAllStringSubmatch(dataStr, -1)
if len(keyMatches) == 0 {
keyMatches = personalKeyMatches
}
for _, keyMatch := range keyMatches {
if len(keyMatch) != 2 {
continue
}
keyRes := strings.TrimSpace(keyMatch[1])
for _, appMatch := range appMatches {
@@ -83,15 +67,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 403 {
var resp response
if err = json.NewDecoder(res.Body).Decode(&resp); err == nil {
// check if the error is due to invalid permissions or model not found
if resp.Error.Type == "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND" {
// The key is verified as it works, but the user must enumerate the tables or permissions for the key.
s1.Verified = true
}
}
} else {
if detectors.IsKnownFalsePositive(keyRes, detectors.DefaultFalsePositives, true) {
continue
@@ -9,10 +9,9 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -100,9 +99,8 @@ func TestAirtableApiKey_FromChunk(t *testing.T) {
}
got[i].Raw = nil
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Airtable.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("AirtableApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
+1 -15
View File
@@ -2,7 +2,6 @@ package codeclimate
import (
"context"
"encoding/json"
"fmt"
"net/http"
"regexp"
@@ -31,12 +30,6 @@ func (s Scanner) Keywords() []string {
return []string{"codeclimate"}
}
type response struct {
Data struct {
Id string `json:"id"`
} `json:"data"`
}
// FromData will find and optionally verify Codeclimate secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -65,14 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
var r response
if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
s1.VerificationError = err
continue
}
if r.Data.Id != "" {
s1.Verified = true
}
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-7
View File
@@ -74,13 +74,6 @@ func TestStringShannonEntropy(t *testing.T) {
},
want: 0.22228483068568816,
},
{
name: "empty",
args: args{
input: "",
},
want: 0.0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
-105
View File
@@ -1,105 +0,0 @@
package instamojo
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
//KeyPat is client_id
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"instamojo"}) + `\b([0-9a-zA-Z]{40})\b`)
//Secretpat is Client_secret
secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"instamojo"}) + `\b([0-9a-zA-Z]{128})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"instamojo"}
}
// FromData will find and optionally verify Instamojo secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
clientIdmatches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range secretMatches {
if len(match) != 2 {
continue
}
resSecret := strings.TrimSpace(match[1])
for _, clientIdMatch := range clientIdmatches {
if len(clientIdMatch) != 2 {
continue
}
resClientId := strings.TrimSpace(clientIdMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Instamojo,
Raw: []byte(resClientId),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
payload := strings.NewReader("grant_type=client_credentials&client_id=" + resClientId + "&client_secret=" + resSecret)
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.instamojo.com/oauth2/token/", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
continue
}
body := string(bodyBytes)
if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "access_token") {
s1.Verified = true
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Instamojo
}
-124
View File
@@ -1,124 +0,0 @@
//go:build detectors
// +build detectors
package instamojo
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestInstamojo_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
id := testSecrets.MustGetField("INSTAMOJO_CLIENT_ID")
secret := testSecrets.MustGetField("INSTAMOJO_SECRET")
inactiveSecret := testSecrets.MustGetField("INSTAMOJO_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a instamojo secret %s within id %s", secret, id)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Instamojo,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a instamojo secret %s within but not valid, within id %s", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Instamojo,
VerificationError: fmt.Errorf("unexpected HTTP response status 401"),
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Instamojo.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Instamojo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
-84
View File
@@ -1,84 +0,0 @@
package ip2location
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ip2location"}) + `\b([0-9A-Z]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"ip2location"}
}
// FromData will find and optionally verify Ip2location secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Ip2location,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.ip2location.io/?key=" + resMatch, nil)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Ip2location
}
@@ -1,128 +0,0 @@
//go:build detectors
// +build detectors
package ip2location
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestIp2location_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("IP2LOCATION")
inactiveSecret := testSecrets.MustGetField("IP2LOCATION_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a ip2location secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Ip2location,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a ip2location secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Ip2location,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Ip2location.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Ip2location.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
-89
View File
@@ -1,89 +0,0 @@
package ipinfo
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ipinfo"}) + `\b([a-f0-9]{14})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"ipinfo"}
}
// FromData will find and optionally verify Ipinfo secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_IPInfo,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://ipinfo.io/json?token="+resMatch, nil)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
fmt.Println(res.Status, resMatch)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 403 {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_IPInfo
}
-161
View File
@@ -1,161 +0,0 @@
//go:build detectors
// +build detectors
package ipinfo
import (
"context"
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"testing"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestIpinfo_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("IPINFO")
inactiveSecret := testSecrets.MustGetField("IPINFO_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_IPInfo,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_IPInfo,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_IPInfo,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_IPInfo,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Ipinfo.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Ipinfo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
@@ -24,7 +24,7 @@ const verifyURL = "https://api.pagerduty.com/users"
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b(u\+[a-zA-Z0-9_+-]{18})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b([a-zA-Z0-9_+-]{20})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
+20 -25
View File
@@ -2,7 +2,6 @@ package plaidkey
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
@@ -54,36 +53,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
DetectorType: detectorspb.DetectorType_PlaidKey,
Raw: []byte(resMatch),
}
environments := []string{"development", "production"}
if verify {
for _, env := range environments {
payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://"+env+".plaid.com/link/token/create", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
s1.ExtraData = map[string]string{"environment": fmt.Sprintf("https://%s.plaid.com", env)}
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://development.plaid.com/link/token/create", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
results = append(results, s1)
// if the environment is dev, we don't need to check production
if s1.Verified {
break
}
}
results = append(results, s1)
}
}
return results, nil
-5
View File
@@ -26,8 +26,6 @@ func TestPlaidKey_FromChunk(t *testing.T) {
secret := testSecrets.MustGetField("PLAIDKEY_SECRET")
inactiveSecret := testSecrets.MustGetField("PLAIDKEY_SECRET_INACTIVE")
id := testSecrets.MustGetField("PLAIDKEY_CLIENTID")
// env := testSecrets.MustGetField("PLAIDKEY_ENVIRONMENT") // development or production
env := "development"
type args struct {
ctx context.Context
@@ -53,9 +51,6 @@ func TestPlaidKey_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_PlaidKey,
Verified: true,
ExtraData: map[string]string{
"environment": fmt.Sprintf("https://%s.plaid.com", env),
},
},
},
wantErr: false,
+8 -21
View File
@@ -20,13 +20,13 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b(phx_[a-zA-Z0-9_]{43})\b`)
keyPat = regexp.MustCompile(`\b(phc_[a-zA-Z0-9_]{43})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"phx_"}
return []string{"phc_"}
}
// FromData will find and optionally verify AppPosthog secrets in a given set of bytes.
@@ -47,33 +47,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.posthog.com/api/event/?personal_api_key="+resMatch, nil)
reqEU, errEU := http.NewRequestWithContext(ctx, "GET", "https://eu.posthog.com/api/event/?personal_api_key="+resMatch, nil)
if err != nil || errEU != nil {
payload := strings.NewReader(` {
"api_key": "` + resMatch + `",
"distinct_id": "1234"
}`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://app.posthog.com/decide/", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
reqEU.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
//Try EU Endpoint only if other one fails.
res, err := client.Do(reqEU)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-89
View File
@@ -1,89 +0,0 @@
package privacy
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"privacy"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"privacy"}
}
// FromData will find and optionally verify Privacy secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Privacy,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.privacy.com/v1/card?page=1&page_size=50", nil)
if err != nil {
continue
}
req.Header.Set("Authorization", "api-key "+resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Privacy
}
-161
View File
@@ -1,161 +0,0 @@
//go:build detectors
// +build detectors
package privacy
import (
"context"
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"testing"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestPrivacy_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("PRIVACY")
inactiveSecret := testSecrets.MustGetField("PRIVACY_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Privacy,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a privacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Privacy,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Privacy,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Privacy,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Privacy.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Privacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
@@ -1,91 +0,0 @@
package sourcegraphcody
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b(slk_[a-f0-9]{64})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"slk_"}
}
// FromData will find and optionally verify Sourcegraphcody secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_SourcegraphCody,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://cody-gateway.sourcegraph.com/v1/limits", nil)
req.Header.Add("Authorization", "Bearer "+resMatch)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 403 || res.StatusCode == 401 {
// 403 when the gateway is not configured to accept the token,
// 401 when the token is invalid.
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_SourcegraphCody
}
@@ -1,162 +0,0 @@
//go:build detectors
// +build detectors
package sourcegraphcody
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestSourcegraphcody_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("SOURCEGRAPHCODY")
inactiveSecret := testSecrets.MustGetField("SOURCEGRAPHCODY_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SourcegraphCody,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SourcegraphCody,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SourcegraphCody,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SourcegraphCody,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Sourcegraphcody.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Sourcegraphcody.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+1 -1
View File
@@ -47,7 +47,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.supernotes.app/v1/user", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.supernotes.app/v1/user/", nil)
if err != nil {
continue
}
+2 -1
View File
@@ -23,6 +23,7 @@ var (
identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive.
sidPat = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`)
keyPat = regexp.MustCompile(`\b[0-9a-f]{32}\b`)
client = common.SaneHttpClient()
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -58,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
client := s.client
client = s.client
if client == nil {
client = defaultClient
}
+2 -2
View File
@@ -21,7 +21,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
accessToken = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b(?:oauth2:)?([0-9a-f]{40})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b([0-9a-f:]{40,47})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -34,7 +34,7 @@ func (s Scanner) Keywords() []string {
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := accessToken.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
-99
View File
@@ -1,99 +0,0 @@
package voiceflow
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Reference: https://developer.voiceflow.com/reference/project#dialog-manager-api-keys
//
//TODO: This includes Workspace and Legacy Workspace API keys; I haven't validated whether these actually work.
// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
keyPat = regexp.MustCompile(`\b(VF\.(?:(?:DM|WS)\.)?[a-fA-F0-9]{24}\.[a-zA-Z0-9]{16})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"vf", "dm"}
}
// FromData will find and optionally verify Voiceflow secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Voiceflow,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
// Fetch the state for a random user.
payload := []byte(`{"question": "why is the sky blue?"}`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://general-runtime.voiceflow.com/knowledge-base/query", bytes.NewBuffer(payload))
if err != nil {
continue
}
req.Header.Set("Accept", "application/json")
req.Header.Set("Authorization", resMatch)
req.Header.Set("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
if res.StatusCode == http.StatusOK {
s1.Verified = true
} else if res.StatusCode == http.StatusUnauthorized {
// The secret is determinately not verified (nothing to do)
} else {
var buf bytes.Buffer
var bodyString string
_, err = io.Copy(&buf, res.Body)
if err == nil {
bodyString = buf.String()
}
s1.VerificationError = fmt.Errorf("unexpected HTTP response [status=%d, body=%s]", res.StatusCode, bodyString)
}
_ = res.Body.Close()
} else {
s1.VerificationError = err
}
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Voiceflow
}
-302
View File
@@ -1,302 +0,0 @@
//go:build detectors
// +build detectors
package voiceflow
import (
"context"
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"testing"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestVoiceflow_Pattern(t *testing.T) {
tests := []struct {
name string
data string
shouldMatch bool
match string
}{
// True positives
// https://github.com/funDAOmental/endlessquest/blob/5c008f7c6a7e58c45a88b72fef4b965c258d665c/Voiceflow/agent-api/index.js#L6
{
name: `valid_result1`,
data: `// z0MG IT'S NOT A SECRET (but we'll delete it)
const API_KEY = "VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
shouldMatch: true,
match: `VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF`,
},
// https://github.com/sherifButt/ll-site/blob/b98b268214324da42a84e996e4c03c242e122680/src/components/Chatbot.jsx#L14
{
name: `valid_result2`,
data: ` const runtime = useRuntime({
verify: { authorization: 'VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f' },
session: { userID: 'user_123' },
});`,
shouldMatch: true,
match: `VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f`,
},
// https://github.com/the-vv/Voiceflow-chatbot/blob/324db17693dd46387ea7a020e92c4e79b94306c6/src/app/chat/chat.component.ts#L27
{
name: `valid_result3`,
data: ` this.http.delete('https://general-runtime.voiceflow.com/state/user/TEST_USER', {
headers: {
Authorization: "VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq"
}
}).subscribe(res => {
this.loading = false;
this.doPrompt('', { action: { type: 'launch' } });
})`,
shouldMatch: true,
match: `VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq`,
},
// https://github.com/legionX7/Graduation-Project-API/blob/451431771d3fba1d8c634b8855274b414d7aed6d/mainAPI.py#L547
{
name: `valid_result4`,
data: `
API_KEY = 'VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl'
VERSION_ID = '646bc'`,
shouldMatch: true,
match: `VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl`,
},
// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
{
name: `valid_result5`,
data: ` it('extracts ID from a Dialog Manager API key', () => {
// eslint-disable-next-line no-secrets/no-secrets
const key = 'VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai';
const result = utils.extractAPIKeyID(key);
expect(result).to.equal('628d5d92faf688001bda7907');
});`,
shouldMatch: true,
match: `VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai`,
},
{
name: `valid_result6_legacy`,
data: ` it('extracts ID from a Workspace API key', () => {
// eslint-disable-next-line no-secrets/no-secrets
const key = 'VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa';
const result = utils.extractAPIKeyID(key);
expect(result).to.equal('62bcb0cca5184300066f5ac7');
});
`,
shouldMatch: true,
match: `VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa`,
},
{
name: `valid_result7_legacy`,
data: ` it('extracts ID from a Legacy Workspace API key', () => {
// eslint-disable-next-line no-secrets/no-secrets
const key = 'VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az';
const result = utils.extractAPIKeyID(key);
expect(result).to.equal('62bcb0cca5184300066f5ac7');
});`,
shouldMatch: true,
match: `VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az`,
},
// False positives
// https://github.com/ImperialCollegeLondon/voiceflow-integration-whatsapp/blob/0f3d6a5638b9acb4989d5bf8e77081cc78e9b976/README.md?plain=1#L155
{
name: `invalid_result1`,
data: "Now, paste it in your .env file for the **VF_PROJECT_API** variable<br>\n```VF_PROJECT_API='VF.DM.62xxxxxxxxxxxxxxxxxxxxxxx'```",
shouldMatch: false,
},
// https://github.com/voiceflow/api-examples/blob/c3d8ba9ee8eced7ec8d241973b1eb0284aaec212/rust/src/main.rs#L5
{
name: `invalid_result2`,
data: `const API_KEY: &str = "YOUR_API_KEY_HERE"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
shouldMatch: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
s := Scanner{}
results, err := s.FromData(context.Background(), true, []byte(test.data))
if err != nil {
t.Errorf("CoinbaseWaaS.FromData() error = %v", err)
return
}
if test.shouldMatch {
if len(results) == 0 {
t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
return
}
expected := test.data
if test.match != "" {
expected = test.match
}
result := results[0]
if result.VerificationError != nil {
fmt.Printf("VerificationError: %v\n", result.VerificationError)
}
resultData := string(result.Raw)
if resultData != expected {
t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, resultData)
return
}
} else {
if len(results) > 0 {
t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
return
}
}
})
}
}
func TestVoiceflow_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("VOICEFLOW")
inactiveSecret := testSecrets.MustGetField("VOICEFLOW_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Voiceflow,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Voiceflow,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Voiceflow,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Voiceflow,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Voiceflow.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Voiceflow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+8 -7
View File
@@ -18,10 +18,10 @@ type detectorKey struct {
version int
}
// DetectorInfo is used to store a detector and whether it should be verified.
type DetectorInfo struct {
// detectorInfo is used to store a detector and whether it should be verified.
type detectorInfo struct {
detectors.Detector
ShouldVerify bool
shouldVerify bool
}
// AhoCorasickCore encapsulates the operations and data structures used for keyword matching via the
@@ -32,7 +32,7 @@ type AhoCorasickCore struct {
// matching given a set of words. (keywords from the rules in the config)
prefilter ahocorasick.Trie
// Maps for efficient lookups during detection.
detectorTypeToDetectorInfo map[detectorKey]DetectorInfo
detectorTypeToDetectorInfo map[detectorKey]detectorInfo
detectors map[bool][]detectors.Detector
keywordsToDetectors map[string][]detectorKey
}
@@ -45,7 +45,7 @@ func NewAhoCorasickCore(detectors map[bool][]detectors.Detector) *AhoCorasickCor
return &AhoCorasickCore{
keywordsToDetectors: make(map[string][]detectorKey),
detectors: detectors,
detectorTypeToDetectorInfo: make(map[detectorKey]DetectorInfo, len(detectors[true])+len(detectors[false])),
detectorTypeToDetectorInfo: make(map[detectorKey]detectorInfo, len(detectors[true])+len(detectors[false])),
}
}
@@ -57,7 +57,7 @@ func (ac *AhoCorasickCore) Setup(ctx context.Context) {
for verify, detectorsSet := range ac.detectors {
for _, d := range detectorsSet {
key := createDetectorKey(d)
ac.detectorTypeToDetectorInfo[key] = DetectorInfo{Detector: d, ShouldVerify: verify}
ac.detectorTypeToDetectorInfo[key] = detectorInfo{Detector: d, shouldVerify: verify}
keywords = ac.extractAndMapKeywords(d, key, keywords)
}
}
@@ -65,6 +65,7 @@ func (ac *AhoCorasickCore) Setup(ctx context.Context) {
// Use the Ahocorasick algorithm to create a trie structure for efficient keyword matching.
// This ensures that we can rapidly match against a vast set of keywords without individually comparing each one.
ac.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
ctx.Logger().V(4).Info("AhoCorasickCore lookups built")
ctx.Logger().V(4).Info("AhoCorasickCore Setup complete")
}
@@ -99,7 +100,7 @@ func (ac *AhoCorasickCore) MatchString(input string) []*ahocorasick.Match {
// PopulateDetectorsByMatch populates the given detectorMap based on the Aho-Corasick match results.
// This method is designed to reuse the same map for performance optimization,
// reducing the need for repeated allocations within each detector worker in the engine.
func (ac *AhoCorasickCore) PopulateDetectorsByMatch(match *ahocorasick.Match, detectors map[detectorspb.DetectorType]DetectorInfo) bool {
func (ac *AhoCorasickCore) PopulateDetectorsByMatch(match *ahocorasick.Match, detectors map[detectorspb.DetectorType]detectorInfo) bool {
matchedKeys, ok := ac.keywordsToDetectors[match.MatchString()]
if !ok {
return false
-6
View File
@@ -4,15 +4,12 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/anthropic"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ip2location"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ramp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraph"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraphcody"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/voiceflow"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
@@ -1555,9 +1552,6 @@ func DefaultDetectors() []detectors.Detector {
web3storage.Scanner{},
&ramp.Scanner{},
&anthropic.Scanner{},
&sourcegraphcody.Scanner{},
voiceflow.Scanner{},
ip2location.Scanner{},
}
}
+3 -3
View File
@@ -457,7 +457,7 @@ func (e *Engine) detectorWorker(ctx context.Context) {
// Reuse the same map to avoid allocations.
const avgDetectorsPerChunk = 2
chunkSpecificDetectors := make(map[detectorspb.DetectorType]DetectorInfo, avgDetectorsPerChunk)
chunkSpecificDetectors := make(map[detectorspb.DetectorType]detectorInfo, avgDetectorsPerChunk)
for originalChunk := range e.ChunksChan() {
for chunk := range sources.Chunker(originalChunk) {
atomic.AddUint64(&e.metrics.BytesScanned, uint64(len(chunk.Data)))
@@ -475,7 +475,7 @@ func (e *Engine) detectorWorker(ctx context.Context) {
}
for k, detector := range chunkSpecificDetectors {
decoded.Chunk.Verify = detector.ShouldVerify
decoded.Chunk.Verify = detector.shouldVerify
wgDetect.Add(1)
e.detectableChunksChan <- detectableChunk{
chunk: *decoded.Chunk,
@@ -623,7 +623,7 @@ func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64,
if !found {
return 0, false
}
lineNumber := int64(bytes.Count(before, []byte("\n")) + 1)
lineNumber := int64(bytes.Count(before, []byte("\n")))
// If the line contains the ignore tag, we should ignore the result.
endLine := bytes.Index(after, []byte("\n"))
if endLine == -1 {
+6 -6
View File
@@ -32,7 +32,7 @@ func TestFragmentLineOffset(t *testing.T) {
result: &detectors.Result{
Raw: []byte("secret here"),
},
expectedLine: 3,
expectedLine: 2,
ignore: true,
},
{
@@ -43,7 +43,7 @@ func TestFragmentLineOffset(t *testing.T) {
result: &detectors.Result{
Raw: []byte("secret here"),
},
expectedLine: 3,
expectedLine: 2,
ignore: false,
},
{
@@ -54,7 +54,7 @@ func TestFragmentLineOffset(t *testing.T) {
result: &detectors.Result{
Raw: []byte("secret here"),
},
expectedLine: 5,
expectedLine: 4,
ignore: false,
},
{
@@ -65,7 +65,7 @@ func TestFragmentLineOffset(t *testing.T) {
result: &detectors.Result{
Raw: []byte("secret\nhere"),
},
expectedLine: 5,
expectedLine: 4,
ignore: false,
},
{
@@ -76,7 +76,7 @@ func TestFragmentLineOffset(t *testing.T) {
result: &detectors.Result{
Raw: []byte("secret\nhere"),
},
expectedLine: 4,
expectedLine: 3,
ignore: true,
},
{
@@ -87,7 +87,7 @@ func TestFragmentLineOffset(t *testing.T) {
result: &detectors.Result{
Raw: []byte("secret here"),
},
expectedLine: 4,
expectedLine: 3,
ignore: true,
},
}
-5
View File
@@ -158,11 +158,6 @@ func UpdateLinkLineNumber(ctx context.Context, link string, newLine int64) strin
return link
}
if newLine <= 0 {
// Don't change the link if the line number is 0.
return link
}
switch determineProvider(link) {
case providerBitbucket:
// For Bitbucket, it doesn't support line links (based on the GenerateLink function).
-8
View File
@@ -251,14 +251,6 @@ func TestUpdateLinkLineNumber(t *testing.T) {
},
want: "https://onprem.customdomain.com/org/repo/commit/xyz123#L50",
},
{
name: "Don't include line when it's 0",
args: args{
link: "https://github.com/coinbase/cbpay-js/issues/181",
newLine: int64(0),
},
want: "https://github.com/coinbase/cbpay-js/issues/181",
},
{
name: "Invalid link",
args: args{
+7 -31
View File
@@ -1007,12 +1007,6 @@ const (
DetectorType_Anthropic DetectorType = 933
DetectorType_Ramp DetectorType = 934
DetectorType_Klaviyo DetectorType = 935
DetectorType_SourcegraphCody DetectorType = 936
DetectorType_Voiceflow DetectorType = 937
DetectorType_Privacy DetectorType = 938
DetectorType_IPInfo DetectorType = 939
DetectorType_Ip2location DetectorType = 940
DetectorType_Instamojo DetectorType = 941
)
// Enum value maps for DetectorType.
@@ -1950,12 +1944,6 @@ var (
933: "Anthropic",
934: "Ramp",
935: "Klaviyo",
936: "SourcegraphCody",
937: "Voiceflow",
938: "Privacy",
939: "IPInfo",
940: "Ip2location",
941: "Instamojo",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@@ -2890,12 +2878,6 @@ var (
"Anthropic": 933,
"Ramp": 934,
"Klaviyo": 935,
"SourcegraphCody": 936,
"Voiceflow": 937,
"Privacy": 938,
"IPInfo": 939,
"Ip2location": 940,
"Instamojo": 941,
}
)
@@ -3274,7 +3256,7 @@ var file_detectors_proto_rawDesc = []byte{
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xf3, 0x75, 0x0a, 0x0c, 0x44,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x90, 0x75, 0x0a, 0x0c, 0x44,
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4211,18 +4193,12 @@ var file_detectors_proto_rawDesc = []byte{
0x12, 0x12, 0x0a, 0x0d, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x74, 0x53, 0x63, 0x61, 0x6c, 0x65, 0x44,
0x62, 0x10, 0xa4, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x41, 0x6e, 0x74, 0x68, 0x72, 0x6f, 0x70, 0x69,
0x63, 0x10, 0xa5, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x52, 0x61, 0x6d, 0x70, 0x10, 0xa6, 0x07, 0x12,
0x0c, 0x0a, 0x07, 0x4b, 0x6c, 0x61, 0x76, 0x69, 0x79, 0x6f, 0x10, 0xa7, 0x07, 0x12, 0x14, 0x0a,
0x0f, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x43, 0x6f, 0x64, 0x79,
0x10, 0xa8, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x56, 0x6f, 0x69, 0x63, 0x65, 0x66, 0x6c, 0x6f, 0x77,
0x10, 0xa9, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x50, 0x72, 0x69, 0x76, 0x61, 0x63, 0x79, 0x10, 0xaa,
0x07, 0x12, 0x0b, 0x0a, 0x06, 0x49, 0x50, 0x49, 0x6e, 0x66, 0x6f, 0x10, 0xab, 0x07, 0x12, 0x10,
0x0a, 0x0b, 0x49, 0x70, 0x32, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0xac, 0x07,
0x12, 0x0e, 0x0a, 0x09, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6d, 0x6f, 0x6a, 0x6f, 0x10, 0xad, 0x07,
0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74,
0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74,
0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67,
0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62,
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x0c, 0x0a, 0x07, 0x4b, 0x6c, 0x61, 0x76, 0x69, 0x79, 0x6f, 0x10, 0xa7, 0x07, 0x42, 0x3d, 0x5a,
0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66,
0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66,
0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62,
0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,
}
var (
+2 -1
View File
@@ -39,7 +39,8 @@ type Source struct {
// Ensure the Source satisfies the interfaces at compile time
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
var _ sources.SourceUnitEnumChunker = (*Source)(nil)
var _ sources.SourceUnitEnumerator = (*Source)(nil)
var _ sources.SourceUnitChunker = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
+10 -195
View File
@@ -15,7 +15,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest"
)
func TestSource_Scan(t *testing.T) {
@@ -85,15 +84,23 @@ func TestSource_Scan(t *testing.T) {
}
func TestScanFile(t *testing.T) {
tmpfile, err := os.CreateTemp("", "example.txt")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
chunkSize := sources.ChunkSize
secretPart1 := "SECRET"
secretPart2 := "SPLIT"
// Split the secret into two parts and pad the rest of the chunk with A's.
data := strings.Repeat("A", chunkSize-len(secretPart1)) + secretPart1 + secretPart2 + strings.Repeat("A", chunkSize-len(secretPart2))
tmpfile, cleanup, err := createTempFile("", data)
_, err = tmpfile.Write([]byte(data))
assert.Nil(t, err)
err = tmpfile.Close()
assert.Nil(t, err)
defer cleanup()
source := &Source{}
chunksChan := make(chan *sources.Chunk, 2)
@@ -113,195 +120,3 @@ func TestScanFile(t *testing.T) {
assert.Contains(t, foundSecret, secretPart1+secretPart2)
}
func TestEnumerate(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Setup the connection to test enumeration.
units := []string{
"/one", "/two", "/three",
"/path/to/dir/", "/path/to/another/dir/",
}
conn, err := anypb.New(&sourcespb.Filesystem{
Paths: units[0:3],
Directories: units[3:],
})
assert.NoError(t, err)
// Initialize the source.
s := Source{}
err = s.Init(ctx, "test enumerate", 0, 0, true, conn, 1)
assert.NoError(t, err)
reporter := sourcestest.TestReporter{}
err = s.Enumerate(ctx, &reporter)
assert.NoError(t, err)
assert.Equal(t, len(units), len(reporter.Units))
assert.Equal(t, 0, len(reporter.UnitErrs))
for _, unit := range reporter.Units {
assert.Contains(t, units, unit.SourceUnitID())
}
for _, unit := range units {
assert.Contains(t, reporter.Units, sources.CommonSourceUnit{ID: unit})
}
}
func TestChunkUnit(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Setup test file to chunk.
fileContents := "TestChunkUnit"
tmpfile, cleanup, err := createTempFile("", fileContents)
assert.NoError(t, err)
defer cleanup()
tmpdir, cleanup, err := createTempDir("", "foo", "bar", "baz")
assert.NoError(t, err)
defer cleanup()
conn, err := anypb.New(&sourcespb.Filesystem{})
assert.NoError(t, err)
// Initialize the source.
s := Source{}
err = s.Init(ctx, "test chunk unit", 0, 0, true, conn, 1)
assert.NoError(t, err)
// Happy path single file.
reporter := sourcestest.TestReporter{}
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: tmpfile.Name(),
}, &reporter)
assert.NoError(t, err)
// Happy path directory.
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: tmpdir,
}, &reporter)
assert.NoError(t, err)
// Error path.
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: "/file/not/found",
}, &reporter)
assert.NoError(t, err)
assert.Equal(t, 4, len(reporter.Chunks))
assert.Equal(t, 1, len(reporter.ChunkErrs))
dataFound := make(map[string]struct{}, 4)
for _, chunk := range reporter.Chunks {
dataFound[string(chunk.Data)] = struct{}{}
}
assert.Contains(t, dataFound, fileContents)
assert.Contains(t, dataFound, "foo")
assert.Contains(t, dataFound, "bar")
assert.Contains(t, dataFound, "baz")
}
func TestEnumerateReporterErr(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Setup the connection to test enumeration.
units := []string{
"/one", "/two", "/three",
"/path/to/dir/", "/path/to/another/dir/",
}
conn, err := anypb.New(&sourcespb.Filesystem{
Paths: units[0:3],
Directories: units[3:],
})
assert.NoError(t, err)
// Initialize the source.
s := Source{}
err = s.Init(ctx, "test enumerate", 0, 0, true, conn, 1)
assert.NoError(t, err)
// Enumerate should always return an error if the reporter returns an
// error.
reporter := sourcestest.ErrReporter{}
err = s.Enumerate(ctx, &reporter)
assert.Error(t, err)
}
func TestChunkUnitReporterErr(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Setup test file to chunk.
tmpfile, err := os.CreateTemp("", "example.txt")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
fileContents := []byte("TestChunkUnit")
_, err = tmpfile.Write(fileContents)
assert.NoError(t, err)
assert.NoError(t, tmpfile.Close())
conn, err := anypb.New(&sourcespb.Filesystem{})
assert.NoError(t, err)
// Initialize the source.
s := Source{}
err = s.Init(ctx, "test chunk unit", 0, 0, true, conn, 1)
assert.NoError(t, err)
// Happy path. ChunkUnit should always return an error if the reporter
// returns an error.
reporter := sourcestest.ErrReporter{}
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: tmpfile.Name(),
}, &reporter)
assert.Error(t, err)
// Error path. ChunkUnit should always return an error if the reporter
// returns an error.
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: "/file/not/found",
}, &reporter)
assert.Error(t, err)
}
// createTempFile is a helper function to create a temporary file in the given
// directory with the provided contents. If dir is "", the operating system's
// temp directory is used.
func createTempFile(dir string, contents string) (*os.File, func(), error) {
tmpfile, err := os.CreateTemp(dir, "trufflehogtest")
if err != nil {
return nil, nil, err
}
if _, err := tmpfile.Write([]byte(contents)); err != nil {
_ = os.Remove(tmpfile.Name())
return nil, nil, err
}
if err := tmpfile.Close(); err != nil {
_ = os.Remove(tmpfile.Name())
return nil, nil, err
}
return tmpfile, func() { _ = os.Remove(tmpfile.Name()) }, nil
}
// createTempDir is a helper function to create a temporary directory in the
// given directory with files containing the provided contents. If dir is "",
// the operating system's temp directory is used.
func createTempDir(dir string, contents ...string) (string, func(), error) {
tmpdir, err := os.MkdirTemp(dir, "trufflehogtest")
if err != nil {
return "", nil, err
}
for _, content := range contents {
if _, _, err := createTempFile(tmpdir, content); err != nil {
_ = os.RemoveAll(tmpdir)
return "", nil, err
}
}
return tmpdir, func() { _ = os.RemoveAll(tmpdir) }, nil
}
+56 -77
View File
@@ -222,10 +222,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
}
s.conn = &conn
s.filteredRepoCache = s.newFilteredRepoCache(memory.New(),
append(s.conn.GetRepositories(), s.conn.GetIncludeRepos()...),
s.conn.GetIgnoreRepos(),
)
s.filteredRepoCache = s.newFilteredRepoCache(memory.New(), s.conn.GetRepositories(), s.conn.GetIgnoreRepos())
s.memberCache = make(map[string]struct{})
s.repoSizes = newRepoSize()
@@ -236,7 +233,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
aCtx.Logger().Error(err, "invalid repository", "repo", repo)
continue
}
s.filteredRepoCache.Set(repo, r)
s.filteredRepoCache.Set(r, r)
}
s.includeIssueComments = s.conn.IncludeIssueComments
@@ -511,12 +508,12 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint strin
for _, org := range s.orgsCache.Keys() {
if err := s.getReposByOrg(ctx, org); err != nil {
s.log.Error(err, "error fetching repos for org")
s.log.Error(err, "error fetching repos for org or user")
}
// We probably don't need to do this, since getting repos by org makes more sense?
if err := s.getReposByUser(ctx, org); err != nil {
s.log.Error(err, "error fetching repos for user")
s.log.Error(err, "error fetching repos for org or user")
}
if s.conn.ScanUsers {
@@ -828,51 +825,38 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
// Unauthenticated access to most github endpoints has a rate limit of 60 requests per hour.
// This will likely only be exhausted if many users/orgs are scanned without auth
func (s *Source) handleRateLimit(errIn error, res *github.Response) bool {
var (
knownWait = true
remaining = 0
retryAfter time.Duration
)
// GitHub has both primary (RateLimit) and secondary (AbuseRateLimit) errors.
var rateLimit *github.RateLimitError
var abuseLimit *github.AbuseRateLimitError
if errors.As(errIn, &rateLimit) {
// Do nothing
} else if errors.As(errIn, &abuseLimit) {
retryAfter = abuseLimit.GetRetryAfter()
} else {
limit, ok := errIn.(*github.RateLimitError)
if !ok {
return false
}
githubNumRateLimitEncountered.WithLabelValues(s.name).Inc()
// Parse retry information from response headers, unless a Retry-After value was already provided.
// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#exceeding-the-rate-limit
if retryAfter <= 0 && res != nil {
var err error
remaining, err = strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
if res != nil {
knownWait := true
remaining, err := strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
if err != nil {
knownWait = false
}
resetTime, err := strconv.Atoi(res.Header.Get("x-ratelimit-reset"))
if err != nil || resetTime == 0 {
knownWait = false
} else if resetTime > 0 {
retryAfter = time.Duration(int64(resetTime)-time.Now().Unix()) * time.Second
}
if knownWait && remaining == 0 {
waitTime := int64(resetTime) - time.Now().Unix()
if waitTime > 0 {
duration := time.Duration(waitTime+1) * time.Second
s.log.V(2).Info("rate limited", "resumeTime", time.Now().Add(duration).String())
time.Sleep(duration)
githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(duration.Seconds())
return true
}
}
}
resumeTime := time.Now().Add(retryAfter).String()
if knownWait && remaining == 0 && retryAfter > 0 {
s.log.V(2).Info("rate limited", "retry_after", retryAfter.String(), "resume_time", resumeTime)
} else {
// TODO: Use exponential backoff instead of static retry time.
retryAfter = time.Minute * 5
s.log.V(2).Error(errIn, "unexpected rate limit error", "retry_after", retryAfter.String(), "resume_time", resumeTime)
}
time.Sleep(retryAfter)
githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(retryAfter.Seconds())
s.log.V(2).Info("handling rate limit (5 minutes retry)", "retry-after", limit.Message)
time.Sleep(time.Minute * 5)
return true
}
@@ -1092,7 +1076,7 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
}
func (s *Source) processGistComments(ctx context.Context, repoPath string, trimmedURL []string, repoURL *url.URL, chunksChan chan *sources.Chunk) error {
ctx.Logger().V(2).Info("scanning github gist comments", "repository", repoPath)
s.log.Info("scanning github gist comments", "repository", repoPath)
// GitHub Gist URL.
gistID, err := extractGistID(trimmedURL)
if err != nil {
@@ -1147,10 +1131,9 @@ var (
)
type repoInfo struct {
owner string
repo string
repoPath string
visibility source_metadatapb.Visibility
owner string
repo string
repoPath string
}
func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimmedURL []string, repoURL *url.URL, chunksChan chan *sources.Chunk) error {
@@ -1161,33 +1144,22 @@ func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimm
owner := trimmedURL[1]
repo := trimmedURL[2]
if !(s.includeIssueComments || s.includePRComments) {
return nil
}
repoInfo := repoInfo{
owner: owner,
repo: repo,
repoPath: repoPath,
visibility: s.visibilityOf(ctx, repoPath),
}
repoInfo := repoInfo{owner: owner, repo: repo, repoPath: repoPath}
if s.includeIssueComments {
ctx.Logger().V(2).Info("scanning github issues", "repository", repoInfo.repoPath)
if err := s.processIssues(ctx, repoInfo, chunksChan); err != nil {
if err := s.processIssueComments(ctx, repoInfo, chunksChan); err != nil {
return err
}
if err := s.processIssueComments(ctx, repoInfo, chunksChan); err != nil {
if err := s.processIssues(ctx, repoInfo, chunksChan); err != nil {
return err
}
}
if s.includePRComments {
ctx.Logger().V(2).Info("scanning github pull requests", "repository", repoInfo.repoPath)
if err := s.processPRs(ctx, repoInfo, chunksChan); err != nil {
if err := s.processPRComments(ctx, repoInfo, chunksChan); err != nil {
return err
}
if err := s.processPRComments(ctx, repoInfo, chunksChan); err != nil {
if err := s.processPRs(ctx, repoInfo, chunksChan); err != nil {
return err
}
}
@@ -1197,6 +1169,8 @@ func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimm
}
func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
s.log.Info("scanning github issue descriptions", "repository", info.repoPath)
bodyTextsOpts := &github.IssueListByRepoOptions{
Sort: sortType,
Direction: directionType,
@@ -1217,7 +1191,7 @@ func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan ch
return err
}
if err = s.chunkIssues(ctx, info, issues, chunksChan); err != nil {
if err = s.chunkIssues(ctx, info.repo, info.repoPath, issues, chunksChan); err != nil {
return err
}
@@ -1231,6 +1205,8 @@ func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan ch
}
func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
s.log.Info("scanning github issue comments", "repository", info.repoPath)
issueOpts := &github.IssueListCommentsOptions{
Sort: &sortType,
Direction: &directionType,
@@ -1250,7 +1226,7 @@ func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunks
return err
}
if err = s.chunkIssueComments(ctx, info, issueComments, chunksChan); err != nil {
if err = s.chunkIssueComments(ctx, info.repo, info.repoPath, issueComments, chunksChan); err != nil {
return err
}
@@ -1264,6 +1240,8 @@ func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunks
}
func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
s.log.Info("scanning github pull request descriptions", "repository", info.repoPath)
prOpts := &github.PullRequestListOptions{
Sort: sortType,
Direction: directionType,
@@ -1284,7 +1262,7 @@ func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan
return err
}
if err = s.chunkPullRequests(ctx, info, prs, chunksChan); err != nil {
if err = s.chunkPullRequests(ctx, info.repo, prs, chunksChan); err != nil {
return err
}
@@ -1298,6 +1276,8 @@ func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan
}
func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
s.log.Info("scanning github pull request comments", "repository", info.repoPath)
prOpts := &github.PullRequestListCommentsOptions{
Sort: sortType,
Direction: directionType,
@@ -1317,7 +1297,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
return err
}
if err = s.chunkPullRequestComments(ctx, info, prComments, chunksChan); err != nil {
if err = s.chunkPullRequestComments(ctx, info.repo, prComments, chunksChan); err != nil {
return err
}
@@ -1330,7 +1310,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
return nil
}
func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
func (s *Source) chunkIssues(ctx context.Context, repo, repoPath string, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
for _, issue := range issues {
// Skip pull requests since covered by processPRs.
@@ -1350,9 +1330,9 @@ func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*g
Link: sanitizer.UTF8(issue.GetHTMLURL()),
Username: sanitizer.UTF8(issue.GetUser().GetLogin()),
Email: sanitizer.UTF8(issue.GetUser().GetEmail()),
Repository: sanitizer.UTF8(repoInfo.repo),
Repository: sanitizer.UTF8(repo),
Timestamp: sanitizer.UTF8(issue.GetCreatedAt().String()),
Visibility: repoInfo.visibility,
Visibility: s.visibilityOf(ctx, repoPath),
},
},
},
@@ -1369,7 +1349,7 @@ func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*g
return nil
}
func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
for _, comment := range comments {
// Create chunk and send it to the channel.
chunk := &sources.Chunk{
@@ -1383,9 +1363,9 @@ func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comm
Link: sanitizer.UTF8(comment.GetHTMLURL()),
Username: sanitizer.UTF8(comment.GetUser().GetLogin()),
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
Repository: sanitizer.UTF8(repoInfo.repo),
Repository: sanitizer.UTF8(repo),
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
Visibility: repoInfo.visibility,
Visibility: s.visibilityOf(ctx, repoPath),
},
},
},
@@ -1402,7 +1382,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comm
return nil
}
func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
for _, comment := range comments {
// Create chunk and send it to the channel.
chunk := &sources.Chunk{
@@ -1416,9 +1396,8 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo
Link: sanitizer.UTF8(comment.GetHTMLURL()),
Username: sanitizer.UTF8(comment.GetUser().GetLogin()),
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
Repository: sanitizer.UTF8(repoInfo.repo),
Repository: sanitizer.UTF8(repo),
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
Visibility: repoInfo.visibility,
},
},
},
@@ -1435,7 +1414,7 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo
return nil
}
func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
func (s *Source) chunkPullRequests(ctx context.Context, repo string, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
for _, pr := range prs {
// Create chunk and send it to the channel.
chunk := &sources.Chunk{
@@ -1449,9 +1428,8 @@ func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs [
Link: sanitizer.UTF8(pr.GetHTMLURL()),
Username: sanitizer.UTF8(pr.GetUser().GetLogin()),
Email: sanitizer.UTF8(pr.GetUser().GetEmail()),
Repository: sanitizer.UTF8(repoInfo.repo),
Repository: sanitizer.UTF8(repo),
Timestamp: sanitizer.UTF8(pr.GetCreatedAt().String()),
Visibility: repoInfo.visibility,
},
},
},
@@ -1484,7 +1462,8 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
Repository: sanitizer.UTF8(gistUrl),
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
// TODO: Fetching this requires making an additional API call. We may want to include this in the future.
// Fetching this information requires making an additional API call.
// We may want to include this in the future.
// Visibility: s.visibilityOf(ctx, repoPath),
},
},
+4 -18
View File
@@ -89,14 +89,14 @@ func (f Fatal) Unwrap() error { return f.error }
// ChunkError is a custom error type for errors encountered during chunking of
// a specific unit.
type ChunkError struct {
Unit SourceUnit
Err error
unit SourceUnit
err error
}
func (f ChunkError) Error() string {
return fmt.Sprintf("error chunking unit %q: %s", f.Unit.SourceUnitID(), f.Err.Error())
return fmt.Sprintf("error chunking unit %q: %s", f.unit.SourceUnitID(), f.err.Error())
}
func (f ChunkError) Unwrap() error { return f.Err }
func (f ChunkError) Unwrap() error { return f.err }
// JobProgress aggregates information about a run of a Source.
type JobProgress struct {
@@ -351,17 +351,3 @@ func (m JobProgressMetrics) ElapsedTime() time.Duration {
}
return m.EndTime.Sub(m.StartTime)
}
// ErrorsFor returns all the errors for the given SourceUnit. If there are no
// errors, including the case that the unit has not been encountered, nil will
// be returned.
func (m JobProgressMetrics) ErrorsFor(unit SourceUnit) []error {
var errs []error
for _, err := range m.Errors {
var chunkErr ChunkError
if errors.As(err, &chunkErr) && chunkErr.Unit == unit {
errs = append(errs, err)
}
}
return errs
}
-218
View File
@@ -1,218 +0,0 @@
package sources
import (
"errors"
"fmt"
"strings"
"sync"
"time"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
// UnitHook implements JobProgressHook for tracking the progress of each
// individual unit.
type UnitHook struct {
metrics *lru.Cache[string, *UnitMetrics]
mu sync.Mutex
NoopHook
}
type UnitHookOpt func(*UnitHook)
func WithUnitHookCache(cache *lru.Cache[string, *UnitMetrics]) UnitHookOpt {
return func(hook *UnitHook) { hook.metrics = cache }
}
func NewUnitHook(ctx context.Context, opts ...UnitHookOpt) *UnitHook {
// lru.NewWithEvict can only fail if the size is < 0.
cache, _ := lru.NewWithEvict(1024, func(key string, value *UnitMetrics) {
if value.handled {
return
}
ctx.Logger().Error(fmt.Errorf("eviction"), "dropping unit metric",
"id", key,
"metric", value,
)
})
hook := UnitHook{metrics: cache}
for _, opt := range opts {
opt(&hook)
}
return &hook
}
// id is a helper method to generate an ID for the given job and unit.
func (u *UnitHook) id(ref JobProgressRef, unit SourceUnit) string {
unitID := ""
if unit != nil {
unitID = unit.SourceUnitID()
}
return fmt.Sprintf("%d/%d/%s", ref.SourceID, ref.JobID, unitID)
}
func (u *UnitHook) StartUnitChunking(ref JobProgressRef, unit SourceUnit, start time.Time) {
id := u.id(ref, unit)
u.mu.Lock()
defer u.mu.Unlock()
u.metrics.Add(id, &UnitMetrics{
Unit: unit,
Parent: ref,
StartTime: start,
})
}
func (u *UnitHook) EndUnitChunking(ref JobProgressRef, unit SourceUnit, end time.Time) {
id := u.id(ref, unit)
u.mu.Lock()
defer u.mu.Unlock()
metrics, ok := u.metrics.Get(id)
if !ok {
return
}
metrics.EndTime = end
}
func (u *UnitHook) ReportChunk(ref JobProgressRef, unit SourceUnit, chunk *Chunk) {
id := u.id(ref, unit)
u.mu.Lock()
defer u.mu.Unlock()
metrics, ok := u.metrics.Get(id)
if !ok && unit != nil {
// The unit has been evicted.
return
} else if !ok && unit == nil {
// This is a chunk from a non-unit source.
metrics = &UnitMetrics{
Unit: nil,
Parent: ref,
StartTime: ref.Snapshot().StartTime,
}
u.metrics.Add(id, metrics)
}
metrics.TotalChunks++
metrics.TotalBytes += uint64(len(chunk.Data))
}
func (u *UnitHook) ReportError(ref JobProgressRef, err error) {
u.mu.Lock()
defer u.mu.Unlock()
// Always add the error to the nil unit if it exists.
if metrics, ok := u.metrics.Get(u.id(ref, nil)); ok {
metrics.Errors = append(metrics.Errors, err)
}
// Check if it's a ChunkError for a specific unit.
var chunkErr ChunkError
if !errors.As(err, &chunkErr) {
return
}
id := u.id(ref, chunkErr.Unit)
metrics, ok := u.metrics.Get(id)
if !ok {
return
}
metrics.Errors = append(metrics.Errors, err)
}
func (u *UnitHook) Finish(ref JobProgressRef) {
u.mu.Lock()
defer u.mu.Unlock()
// Clear out any metrics on this job. This covers the case for the
// source running without unit support.
prefix := u.id(ref, nil)
for _, id := range u.metrics.Keys() {
if !strings.HasPrefix(id, prefix) {
continue
}
metric, ok := u.metrics.Get(id)
if !ok {
continue
}
// If the unit is nil, the source does not support units.
// Use the overall job metrics instead.
if metric.Unit == nil {
snap := ref.Snapshot()
metric.StartTime = snap.StartTime
metric.EndTime = snap.EndTime
metric.Errors = snap.Errors
}
}
}
// UnitMetrics gets all the currently active or newly finished metrics for this
// job. If a unit returned from this method has finished, it will be removed
// from the cache and no longer returned in successive calls to UnitMetrics().
func (u *UnitHook) UnitMetrics() []UnitMetrics {
u.mu.Lock()
defer u.mu.Unlock()
output := make([]UnitMetrics, 0, u.metrics.Len())
for _, id := range u.metrics.Keys() {
metric, ok := u.metrics.Get(id)
if !ok {
continue
}
output = append(output, *metric)
if metric.IsFinished() {
metric.handled = true
u.metrics.Remove(id)
}
}
return output
}
type UnitMetrics struct {
Unit SourceUnit
Parent JobProgressRef
// Start and end time for chunking this unit.
StartTime time.Time
EndTime time.Time
// Total number of chunks produced from this unit.
TotalChunks uint64
// Total number of bytes produced from this unit.
TotalBytes uint64
// All errors encountered by this unit.
Errors []error
// Flag to mark that these metrics were intentionally evicted from
// the cache.
handled bool
}
func (u UnitMetrics) IsFinished() bool {
return !u.EndTime.IsZero()
}
// ElapsedTime is a convenience method that provides the elapsed time the job
// has been running. If it hasn't started yet, 0 is returned. If it has
// finished, the total time is returned.
func (u UnitMetrics) ElapsedTime() time.Duration {
if u.StartTime.IsZero() {
return 0
}
if u.EndTime.IsZero() {
return time.Since(u.StartTime)
}
return u.EndTime.Sub(u.StartTime)
}
// NoopHook implements JobProgressHook by doing nothing. This is useful for
// embedding in other structs to overwrite only the methods of the interface
// that you care about.
type NoopHook struct{}
func (NoopHook) Start(JobProgressRef, time.Time) {}
func (NoopHook) End(JobProgressRef, time.Time) {}
func (NoopHook) StartEnumerating(JobProgressRef, time.Time) {}
func (NoopHook) EndEnumerating(JobProgressRef, time.Time) {}
func (NoopHook) StartUnitChunking(JobProgressRef, SourceUnit, time.Time) {}
func (NoopHook) EndUnitChunking(JobProgressRef, SourceUnit, time.Time) {}
func (NoopHook) ReportError(JobProgressRef, error) {}
func (NoopHook) ReportUnit(JobProgressRef, SourceUnit) {}
func (NoopHook) ReportChunk(JobProgressRef, SourceUnit, *Chunk) {}
func (NoopHook) Finish(JobProgressRef) {}
-23
View File
@@ -126,26 +126,3 @@ func TestJobProgressElapsedTime(t *testing.T) {
metrics.EndTime = metrics.StartTime.Add(1 * time.Hour)
assert.Equal(t, metrics.ElapsedTime(), 1*time.Hour)
}
func TestJobProgressErrorsFor(t *testing.T) {
metrics := JobProgressMetrics{
Errors: []error{
Fatal{ChunkError{
Unit: CommonSourceUnit{ID: "foo"},
Err: fmt.Errorf("foo error"),
}},
ChunkError{
Unit: CommonSourceUnit{ID: "foo"},
Err: fmt.Errorf("foo again error"),
},
ChunkError{
Unit: CommonSourceUnit{ID: "bar"},
Err: fmt.Errorf("bar error"),
},
fmt.Errorf("hi there"),
},
}
assert.Equal(t, 2, len(metrics.ErrorsFor(CommonSourceUnit{ID: "foo"})))
assert.Equal(t, 1, len(metrics.ErrorsFor(CommonSourceUnit{ID: "bar"})))
assert.Equal(t, 0, len(metrics.ErrorsFor(CommonSourceUnit{ID: "baz"})))
}
-115
View File
@@ -3,10 +3,8 @@ package sources
import (
"errors"
"fmt"
"sort"
"testing"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/stretchr/testify/assert"
"google.golang.org/protobuf/types/known/anypb"
@@ -202,8 +200,6 @@ func (c *unitChunker) ChunkUnit(ctx context.Context, unit SourceUnit, rep ChunkR
if err := rep.ChunkErr(ctx, fmt.Errorf(step.err)); err != nil {
return err
}
}
if step.output == "" {
continue
}
if err := rep.ChunkOk(ctx, Chunk{Data: []byte(step.output)}); err != nil {
@@ -314,114 +310,3 @@ func TestSourceManagerAvailableCapacity(t *testing.T) {
<-ref.Done() // Wait for the job to finish.
assert.Equal(t, 1337, mgr.AvailableCapacity())
}
func TestSourceManagerUnitHook(t *testing.T) {
hook := NewUnitHook(context.TODO())
input := []unitChunk{
{unit: "one", output: "bar"},
{unit: "two", err: "oh no"},
{unit: "three", err: "not again"},
}
mgr := NewManager(
WithBufferedOutput(8),
WithSourceUnits(), WithConcurrentUnits(1),
WithReportHook(hook),
)
source, err := buildDummy(&unitChunker{input})
assert.NoError(t, err)
ref, err := mgr.Run(context.Background(), "dummy", source)
assert.NoError(t, err)
<-ref.Done()
metrics := hook.UnitMetrics()
assert.Equal(t, 3, len(metrics))
sort.Slice(metrics, func(i, j int) bool {
return metrics[i].EndTime.Before(metrics[j].EndTime)
})
m0, m1, m2 := metrics[0], metrics[1], metrics[2]
assert.Equal(t, "one", m0.Unit.SourceUnitID())
assert.Equal(t, uint64(1), m0.TotalChunks)
assert.Equal(t, uint64(3), m0.TotalBytes)
assert.NotZero(t, m0.StartTime)
assert.NotZero(t, m0.EndTime)
assert.NotZero(t, m0.ElapsedTime())
assert.Equal(t, 0, len(m0.Errors))
assert.Equal(t, "two", m1.Unit.SourceUnitID())
assert.Equal(t, uint64(0), m1.TotalChunks)
assert.Equal(t, uint64(0), m1.TotalBytes)
assert.NotZero(t, m1.StartTime)
assert.NotZero(t, m1.EndTime)
assert.NotZero(t, m1.ElapsedTime())
assert.Equal(t, 1, len(m1.Errors))
assert.Equal(t, "three", m2.Unit.SourceUnitID())
assert.Equal(t, uint64(0), m2.TotalChunks)
assert.Equal(t, uint64(0), m2.TotalBytes)
assert.NotZero(t, m2.StartTime)
assert.NotZero(t, m2.EndTime)
assert.NotZero(t, m2.ElapsedTime())
assert.Equal(t, 1, len(m2.Errors))
}
// TestSourceManagerUnitHookNoBlock tests that the UnitHook drops metrics if
// they aren't handled fast enough.
func TestSourceManagerUnitHookNoBlock(t *testing.T) {
var evictedKeys []string
cache, _ := lru.NewWithEvict(1, func(key string, _ *UnitMetrics) {
evictedKeys = append(evictedKeys, key)
})
hook := NewUnitHook(context.TODO(), WithUnitHookCache(cache))
input := []unitChunk{
{unit: "one", output: "bar"},
{unit: "two", err: "oh no"},
{unit: "three", err: "not again"},
}
mgr := NewManager(
WithBufferedOutput(8),
WithSourceUnits(), WithConcurrentUnits(1),
WithReportHook(hook),
)
source, err := buildDummy(&unitChunker{input})
assert.NoError(t, err)
ref, err := mgr.Run(context.Background(), "dummy", source)
assert.NoError(t, err)
<-ref.Done()
assert.Equal(t, 2, len(evictedKeys))
metrics := hook.UnitMetrics()
assert.Equal(t, 1, len(metrics))
assert.Equal(t, "three", metrics[0].Unit.SourceUnitID())
}
// TestSourceManagerUnitHookNoUnits tests whether the UnitHook works for
// sources that don't support units.
func TestSourceManagerUnitHookNoUnits(t *testing.T) {
hook := NewUnitHook(context.TODO())
mgr := NewManager(
WithBufferedOutput(8),
WithReportHook(hook),
)
source, err := buildDummy(&counterChunker{count: 5})
assert.NoError(t, err)
ref, err := mgr.Run(context.Background(), "dummy", source)
assert.NoError(t, err)
<-ref.Done()
metrics := hook.UnitMetrics()
assert.Equal(t, 1, len(metrics))
m := metrics[0]
assert.Equal(t, nil, m.Unit)
assert.Equal(t, uint64(5), m.TotalChunks)
assert.Equal(t, uint64(5), m.TotalBytes)
assert.NotZero(t, m.StartTime)
assert.NotZero(t, m.EndTime)
assert.NotZero(t, m.ElapsedTime())
assert.Equal(t, 0, len(m.Errors))
}
-61
View File
@@ -1,61 +0,0 @@
package sourcestest
import (
"fmt"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
type reporter interface {
sources.UnitReporter
sources.ChunkReporter
}
var (
_ reporter = (*TestReporter)(nil)
_ reporter = (*ErrReporter)(nil)
)
// TestReporter is a helper struct that implements both UnitReporter and
// ChunkReporter by simply recording the values passed in the methods.
type TestReporter struct {
Units []sources.SourceUnit
UnitErrs []error
Chunks []sources.Chunk
ChunkErrs []error
}
func (t *TestReporter) UnitOk(_ context.Context, unit sources.SourceUnit) error {
t.Units = append(t.Units, unit)
return nil
}
func (t *TestReporter) UnitErr(_ context.Context, err error) error {
t.UnitErrs = append(t.UnitErrs, err)
return nil
}
func (t *TestReporter) ChunkOk(_ context.Context, chunk sources.Chunk) error {
t.Chunks = append(t.Chunks, chunk)
return nil
}
func (t *TestReporter) ChunkErr(_ context.Context, err error) error {
t.ChunkErrs = append(t.ChunkErrs, err)
return nil
}
// ErrReporter implements UnitReporter and ChunkReporter but always returns an
// error.
type ErrReporter struct{}
func (ErrReporter) UnitOk(context.Context, sources.SourceUnit) error {
return fmt.Errorf("ErrReporter: UnitOk error")
}
func (ErrReporter) UnitErr(context.Context, error) error {
return fmt.Errorf("ErrReporter: UnitErr error")
}
func (ErrReporter) ChunkOk(context.Context, sources.Chunk) error {
return fmt.Errorf("ErrReporter: ChunkOk error")
}
func (ErrReporter) ChunkErr(context.Context, error) error {
return fmt.Errorf("ErrReporter: ChunkErr error")
}
-6
View File
@@ -944,12 +944,6 @@ enum DetectorType {
Anthropic = 933;
Ramp = 934;
Klaviyo = 935;
SourcegraphCody = 936;
Voiceflow = 937;
Privacy = 938;
IPInfo = 939;
Ip2location = 940;
Instamojo = 941;
}
message Result {