Add method to check for unverified.

pkg/common/utils.go

@@ -1,11 +1,5 @@
 package common
 
-import (
-	"bufio"
-	"io"
-	"strings"
-)
-
 func AddStringSliceItem(item string, slice *[]string) {
 	for _, i := range *slice {
 		if i == item {
@@ -23,16 +17,3 @@ func RemoveStringSliceItem(item string, slice *[]string) {
 		}
 	}
 }
-
-func ResponseContainsSubstring(reader io.ReadCloser, target string) (bool, error) {
-	scanner := bufio.NewScanner(reader)
-	for scanner.Scan() {
-		if strings.Contains(scanner.Text(), target) {
-			return true, nil
-		}
-	}
-	if err := scanner.Err(); err != nil {
-		return false, err
-	}
-	return false, nil
-}

pkg/common/utils_test.go

@@ -1,9 +1,7 @@
 package common
 
 import (
-	"io"
 	"reflect"
-	"strings"
 	"testing"
 )
 
@@ -76,60 +74,3 @@ func TestRemoveItem(t *testing.T) {
 		}
 	}
 }
-
-// Test ParseResponseForKeywords with a reader that contains the keyword and a reader that doesn't.
-func TestParseResponseForKeywords(t *testing.T) {
-	testCases := []struct {
-		name     string
-		input    string
-		keyword  string
-		expected bool
-	}{
-		{
-			name:     "Should find keyword",
-			input:    "ey: abc",
-			keyword:  "ey",
-			expected: true,
-		},
-		{
-			name:     "Should not find keyword",
-			input:    "fake response",
-			keyword:  "ey",
-			expected: false,
-		},
-		{
-			name:     "Empty string",
-			input:    "",
-			keyword:  "ey",
-			expected: false,
-		},
-		{
-			name:     "Keyword at end",
-			input:    "abc ey",
-			keyword:  "ey",
-			expected: true,
-		},
-		{
-			name:     "Keyword at start",
-			input:    "ey abc",
-			keyword:  "ey",
-			expected: true,
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			testReader := strings.NewReader(tc.input)
-			testReadCloser := io.NopCloser(testReader)
-			found, err := ResponseContainsSubstring(testReadCloser, tc.keyword)
-
-			if err != nil {
-				t.Errorf("Error: %v", err)
-			}
-
-			if found != tc.expected {
-				t.Errorf("Expected %v, got %v", tc.expected, found)
-			}
-		})
-	}
-}

pkg/detectors/detectors.go

@@ -39,6 +39,8 @@ type EndpointCustomizer interface {
 	DefaultEndpoint() string
 }
 
+// Result is a secret that has been found by a Detector and optionally verified.
+// It contains the raw secret identification data, redacted data, and extra data.
 type Result struct {
 	// DetectorType is the type of Detector.
 	DetectorType detectorspb.DetectorType
@@ -46,7 +48,15 @@ type Result struct {
 	DetectorName string
 	// DecoderType is the type of Decoder.
 	DecoderType detectorspb.DecoderType
-	Verified    bool
+	// Verified is true if the secret has been confirmed to be a valid secret.
+	Verified bool
+	// HasVerificationIndeterminate is true if the secret could NOT be verified.
+	// This is used to differentiate between secrets that are NOT verified and secrets that COULD NOT be verified.
+	// Ex: A 429 response from a service, or a timeout would set this to true.
+	HasVerificationIndeterminate bool
+	// VerificationIndeterminateError holds the error if the secret could NOT be verified.
+	// This is used specifically for cases that resulted in indeterminate resolution, such as a 429 or timeout.
+	VerificationIndeterminateError error
 	// Raw contains the raw secret identifier data. Prefer IDs over secrets since it is used for deduping after hashing.
 	Raw []byte
 	// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
@@ -59,6 +69,17 @@ type Result struct {
 	StructuredData *detectorspb.StructuredData
 }
 
+// IsVerified checks if a result is guaranteed to be verified.
+func (r Result) IsVerified() bool {
+	return r.Verified && !r.HasVerificationIndeterminate
+}
+
+// IsUnverified checks if a result is guaranteed to be unverified.
+// This is used to differentiate between secrets that are NOT verified and secrets that COULD NOT be verified.
+func (r Result) IsUnverified() bool {
+	return !r.Verified && !r.HasVerificationIndeterminate
+}
+
 type ResultWithMetadata struct {
 	// SourceMetadata contains source-specific contextual information.
 	SourceMetadata *source_metadatapb.MetaData

pkg/detectors/detectors_test.go

@@ -3,7 +3,9 @@
 
 package detectors
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestPrefixRegex(t *testing.T) {
 	tests := []struct {
@@ -37,3 +39,101 @@ func BenchmarkPrefixRegex(b *testing.B) {
 		PrefixRegex(kws)
 	}
 }
+
+func TestIsVerified(t *testing.T) {
+	tests := []struct {
+		name   string
+		result Result
+		want   bool
+	}{
+		{
+			name: "Verified true, HasVerificationIndeterminate false",
+			result: Result{
+				Verified:                     true,
+				HasVerificationIndeterminate: false,
+			},
+			want: true,
+		},
+		{
+			name: "Verified false, HasVerificationIndeterminate false",
+			result: Result{
+				Verified:                     false,
+				HasVerificationIndeterminate: false,
+			},
+			want: false,
+		},
+		{
+			name: "Verified true, HasVerificationIndeterminate true",
+			result: Result{
+				Verified:                     true,
+				HasVerificationIndeterminate: true,
+			},
+			want: false,
+		},
+		{
+			name: "Verified false, HasVerificationIndeterminate true",
+			result: Result{
+				Verified:                     false,
+				HasVerificationIndeterminate: true,
+			},
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.result.IsVerified(); got != tt.want {
+				t.Errorf("IsVerified() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIsUnverified(t *testing.T) {
+	tests := []struct {
+		name   string
+		result Result
+		want   bool
+	}{
+		{
+			name: "Verified true, HasVerificationIndeterminate false",
+			result: Result{
+				Verified:                     true,
+				HasVerificationIndeterminate: false,
+			},
+			want: false,
+		},
+		{
+			name: "Verified false, HasVerificationIndeterminate false",
+			result: Result{
+				Verified:                     false,
+				HasVerificationIndeterminate: false,
+			},
+			want: true,
+		},
+		{
+			name: "Verified true, HasVerificationIndeterminate true",
+			result: Result{
+				Verified:                     true,
+				HasVerificationIndeterminate: true,
+			},
+			want: false,
+		},
+		{
+			name: "Verified false, HasVerificationIndeterminate true",
+			result: Result{
+				Verified:                     false,
+				HasVerificationIndeterminate: true,
+			},
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.result.IsUnverified(); got != tt.want {
+				t.Errorf("IsUnverified() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/detectors/docusign/docusign.go

@@ -3,8 +3,10 @@ package docusign
 import (
 	"context"
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"github.com/go-errors/errors"
+	"io"
 	"net/http"
 	"regexp"
 	"strings"
@@ -81,15 +83,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					return nil, errors.WrapPrefix(err, "Error making request", 0)
 				}
 
-				verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "ey")
-				res.Body.Close()
+				// Read the response body
+				body, err := io.ReadAll(res.Body)
 
 				if err != nil {
-					return nil, err
+					return nil, errors.WrapPrefix(err, "Error reading response body", 0)
+				}
+
+				// Close the response body
+				res.Body.Close()
+
+				// Parse the response body into a Response struct
+				var parsedResponse Response
+				err = json.Unmarshal(body, &parsedResponse)
+				if err != nil {
+					return nil, errors.WrapPrefix(err, "Error parsing response", 0)
 				}
 
 				if err == nil {
-					if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
+					if res.StatusCode >= 200 && res.StatusCode < 300 && strings.HasPrefix(parsedResponse.AccessToken, "ey") {
 						s1.Verified = true
 					} else {
 						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.

pkg/detectors/okta/okta.go

@@ -63,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 				resp, err := common.SaneHttpClient().Do(req)
 				if err != nil {
-					continue
+					return results, err
 				}
 				defer resp.Body.Close()
 				if resp.StatusCode >= 200 && resp.StatusCode < 300 {

pkg/detectors/opsgenie/opsgenie.go

@@ -1,78 +0,0 @@
-package opsgenie
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"regexp"
-	"strings"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-type Scanner struct{}
-
-// Ensure the Scanner satisfies the interface at compile time.
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
-	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"opsgenie"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
-)
-
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_Opsgenie
-}
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"opsgenie"}
-}
-
-// FromData will find and optionally verify Opsgenie secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-
-	for _, match := range matches {
-		if len(match) != 2 {
-			continue
-		}
-		resMatch := strings.TrimSpace(match[1])
-
-		s1 := detectors.Result{
-			DetectorType: detectorspb.DetectorType_Opsgenie,
-			Raw:          []byte(resMatch),
-		}
-
-		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/users", nil)
-			if err != nil {
-				continue
-			}
-			req.Header.Add("Authorization", fmt.Sprintf("GenieKey %s", resMatch))
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-						continue
-					}
-				}
-			}
-		}
-
-		results = append(results, s1)
-	}
-
-	return detectors.CleanResults(results), nil
-}

pkg/detectors/opsgenie/opsgenie_test.go

@@ -1,116 +0,0 @@
-package opsgenie
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestOpsgenie_FromChunk(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
-	if err != nil {
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("OPSGENIE")
-	inactiveSecret := testSecrets.MustGetField("OPSGENIE_INACTIVE")
-
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
-	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
-	}{
-		{
-			name: "found, verified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a opsgenie secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Opsgenie,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found, unverified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a opsgenie secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Opsgenie,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Opsgenie.FromData() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Opsgenie.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
-	ctx := context.Background()
-	s := Scanner{}
-	for name, data := range detectors.MustGetBenchmarkData() {
-		benchmark.Run(name, func(b *testing.B) {
-			for n := 0; n < b.N; n++ {
-				_, err := s.FromData(ctx, false, data)
-				if err != nil {
-					b.Fatal(err)
-				}
-			}
-		})
-	}
-}

pkg/engine/defaults.go

@@ -459,7 +459,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/opengraphr"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/openuv"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/openweather"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/opsgenie"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/optimizely"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/owlbot"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/packagecloud"
@@ -1526,7 +1525,6 @@ func DefaultDetectors() []detectors.Detector {
 		aiven.Scanner{},
 		prefect.Scanner{},
 		buildkitev2.Scanner{},
-		opsgenie.Scanner{},
 	}
 
 }