diff --git a/hack/generate/generate.go b/hack/generate/generate.go index a95251da..7fd8a899 100644 --- a/hack/generate/generate.go +++ b/hack/generate/generate.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "io/ioutil" "log" "os" @@ -95,11 +96,14 @@ func mustWriteTemplates(jobs []templateJob) { if err != nil { log.Fatal(err) } - tmpl.Execute(f, templateData{ + err = tmpl.Execute(f, templateData{ NameTitle: nameTitle, NameLower: nameLower, NameUpper: nameUpper, }) + if err != nil { + log.Fatal(fmt.Errorf("failed to execute template: %w", err)) + } } } diff --git a/pkg/common/filter_test.go b/pkg/common/filter_test.go index eac6fd62..c7a7e406 100644 --- a/pkg/common/filter_test.go +++ b/pkg/common/filter_test.go @@ -163,6 +163,9 @@ func testFilterWriteFile(filename string, content []byte) error { if err != nil { return err } - f.Write(content) + _, err = f.Write(content) + if err != nil { + return err + } return f.Close() } diff --git a/pkg/detectors/bitfinex/bitfinex.go b/pkg/detectors/bitfinex/bitfinex.go index 59b285bd..5c1be198 100644 --- a/pkg/detectors/bitfinex/bitfinex.go +++ b/pkg/detectors/bitfinex/bitfinex.go @@ -3,6 +3,7 @@ package bitfinex import ( "context" "flag" + "net/http" "regexp" "strings" @@ -27,8 +28,7 @@ var ( ) var ( - orderid = flag.String("id", "", "lookup trades for an order ID") - api = flag.String("api", "https://api-pub.bitfinex.com/v2/", "v2 REST API URL") + api = flag.String("api", "https://api-pub.bitfinex.com/v2/", "v2 REST API URL") ) // Keywords are used for efficiently pre-filtering chunks. @@ -69,6 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result // thankfully official golang examples exist but you just need to dig their many repos https://github.com/bitfinexcom/bitfinex-api-go/blob/master/examples/v2/rest-orders/main.go key := apiKeyRes secret := apiSecretRes + http.DefaultClient = client // filed https://github.com/bitfinexcom/bitfinex-api-go/issues/238 to improve this c := rest.NewClientWithURL(*api).Credentials(key, secret) isValid := true // assume valid diff --git a/pkg/detectors/bitfinex/bitfinex_test.go b/pkg/detectors/bitfinex/bitfinex_test.go index 62f237f1..c2a71b43 100644 --- a/pkg/detectors/bitfinex/bitfinex_test.go +++ b/pkg/detectors/bitfinex/bitfinex_test.go @@ -50,10 +50,6 @@ func TestBitfinex_FromChunk(t *testing.T) { DetectorType: detectorspb.DetectorType_Bitfinex, Verified: true, }, - { - DetectorType: detectorspb.DetectorType_Bitfinex, - Verified: false, - }, }, wantErr: false, }, @@ -70,10 +66,6 @@ func TestBitfinex_FromChunk(t *testing.T) { DetectorType: detectorspb.DetectorType_Bitfinex, Verified: false, }, - { - DetectorType: detectorspb.DetectorType_Bitfinex, - Verified: false, - }, }, wantErr: false, }, diff --git a/pkg/detectors/buddyns/buddyns/buddyns.go b/pkg/detectors/buddyns/buddyns/buddyns.go deleted file mode 100644 index 20572ab8..00000000 --- a/pkg/detectors/buddyns/buddyns/buddyns.go +++ /dev/null @@ -1,75 +0,0 @@ -package buddyns - -import ( - "context" - "fmt" - "net/http" - "regexp" - "strings" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -type Scanner struct{} - -// Ensure the Scanner satisfies the interface at compile time -var _ detectors.Detector = (*Scanner)(nil) - -var ( - client = common.SaneHttpClient() - - //Make sure that your group is surrounded in boundry characters such as below to reduce false positives - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"buddyns"}) + `\b([0-9a-z]{40})\b`) -) - -// Keywords are used for efficiently pre-filtering chunks. -// Use identifiers in the secret preferably, or the provider name. -func (s Scanner) Keywords() []string { - return []string{"buddyns"} -} - -// FromData will find and optionally verify Buddyns secrets in a given set of bytes. -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - dataStr := string(data) - - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - - for _, match := range matches { - if len(match) != 2 { - continue - } - resMatch := strings.TrimSpace(match[1]) - - s1 := detectors.Result{ - DetectorType: detectorspb.DetectorType_BuddyNS, - Raw: []byte(resMatch), - } - - if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://www.buddyns.com/api/v2/zone/", nil) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch)) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } - } - - results = append(results, s1) - } - - return detectors.CleanResults(results), nil -} diff --git a/pkg/detectors/buddyns/buddyns/buddyns_test.go b/pkg/detectors/buddyns/buddyns/buddyns_test.go deleted file mode 100644 index 5b33e30a..00000000 --- a/pkg/detectors/buddyns/buddyns/buddyns_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package buddyns - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/kylelemons/godebug/pretty" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -func TestBuddyns_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BUDDYNS") - inactiveSecret := testSecrets.MustGetField("BUDDYNS_INACTIVE") - - type args struct { - ctx context.Context - data []byte - verify bool - } - tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - }{ - { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a buddyns secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BuddyNS, - Verified: true, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a buddyns secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BuddyNS, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Buddyns.FromData() error = %v, wantErr %v", err, tt.wantErr) - return - } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil - } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Buddyns.FromData() %s diff: (-got +want)\n%s", tt.name, diff) - } - }) - } -} - -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - for n := 0; n < b.N; n++ { - s.FromData(ctx, false, data) - } - }) - } -} diff --git a/pkg/detectors/elasticemail/elasticemail.go b/pkg/detectors/elasticemail/elasticemail.go index d8261bea..e895a288 100644 --- a/pkg/detectors/elasticemail/elasticemail.go +++ b/pkg/detectors/elasticemail/elasticemail.go @@ -91,7 +91,3 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } return detectors.CleanResults(results), nil } -func prettyPrint(i interface{}) string { - s, _ := json.MarshalIndent(i, "", "\t") - return string(s) -} diff --git a/pkg/detectors/generic/generic.go b/pkg/detectors/generic/generic.go index fd7f779a..44cc264c 100644 --- a/pkg/detectors/generic/generic.go +++ b/pkg/detectors/generic/generic.go @@ -118,22 +118,22 @@ func hasReMatch(matchers []*regexp.Regexp, token string) bool { return false } -func hasDictWord(wordList []string, token string) bool { - lower := strings.ToLower(token) - for _, word := range wordList { - if strings.Contains(lower, word) { - return true - } - } - return false -} +// func hasDictWord(wordList []string, token string) bool { +// lower := strings.ToLower(token) +// for _, word := range wordList { +// if strings.Contains(lower, word) { +// return true +// } +// } +// return false +// } -func bytesToCleanWordList(data []byte) []string { - words := []string{} - for _, word := range strings.Split(string(data), "\n") { - if strings.TrimSpace(word) != "" { - words = append(words, strings.TrimSpace(strings.ToLower(word))) - } - } - return words -} +// func bytesToCleanWordList(data []byte) []string { +// words := []string{} +// for _, word := range strings.Split(string(data), "\n") { +// if strings.TrimSpace(word) != "" { +// words = append(words, strings.TrimSpace(strings.ToLower(word))) +// } +// } +// return words +// } diff --git a/pkg/detectors/privatekey/normalize.go b/pkg/detectors/privatekey/normalize.go index cc8df87d..8b232ab3 100644 --- a/pkg/detectors/privatekey/normalize.go +++ b/pkg/detectors/privatekey/normalize.go @@ -4,8 +4,6 @@ import ( "strings" ) -const b64chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" - func normalize(in string) string { in = strings.ReplaceAll(in, `"`, "") in = strings.ReplaceAll(in, `'`, "") diff --git a/pkg/detectors/privatekey/privatekey.go b/pkg/detectors/privatekey/privatekey.go index 1c5622a2..beebdf61 100644 --- a/pkg/detectors/privatekey/privatekey.go +++ b/pkg/detectors/privatekey/privatekey.go @@ -23,6 +23,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( + // TODO: add base64 encoded key support client = common.RetryableHttpClient() keyPat = regexp.MustCompile(`(?i)-----\s*?BEGIN[ A-Z0-9_-]*?PRIVATE KEY\s*?-----[\s\S]*?----\s*?END[ A-Z0-9_-]*? PRIVATE KEY\s*?-----`) ) diff --git a/pkg/output/legacy_json.go b/pkg/output/legacy_json.go index bf3eac95..08c80a38 100644 --- a/pkg/output/legacy_json.go +++ b/pkg/output/legacy_json.go @@ -73,7 +73,7 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) { return branches, err } - branchIter.ForEach(func(branchRef *plumbing.Reference) error { + err = branchIter.ForEach(func(branchRef *plumbing.Reference) error { branchName := branchRef.Name().String() headHash, err := repo.ResolveRevision(plumbing.Revision(branchName)) if err != nil { @@ -88,7 +88,7 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) { branches[branchName] = headCommit return nil }) - return branches, nil + return branches, err } // FindBranch returns the first branch a commit is a part of. Not the most accurate, but it should work similar to pre v3.0.