Set total count during enumeration.

pkg/detectors/aws/aws.go

@@ -15,7 +15,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-	"google.golang.org/protobuf/types/known/structpb"
 )
 
 type scanner struct {
@@ -180,12 +179,10 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						err := json.NewDecoder(res.Body).Decode(&identityInfo)
 						if err == nil {
 							s1.Verified = true
-							s1.ExtraData = &structpb.Struct{
-								Fields: map[string]*structpb.Value{
-									"account": structpb.NewStringValue(identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account),
-									"user_id": structpb.NewStringValue(identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID),
-									"arn":     structpb.NewStringValue(identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn),
-								},
+							s1.ExtraData = map[string]string{
+								"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
+								"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
+								"arn":     identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
 							}
 						}
 						res.Body.Close()

pkg/detectors/aws/aws_test.go

@@ -15,8 +15,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/structpb"
 )
 
 func TestAWS_FromChunk(t *testing.T) {
@@ -58,13 +56,11 @@ func TestAWS_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     true,
-					Redacted:     "AKIASP2TPHJSQH3FJRUX",
-					ExtraData: &structpb.Struct{
-						Fields: map[string]*structpb.Value{
-							"account": structpb.NewStringValue("171436882533"),
-							"arn":     structpb.NewStringValue("arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793"),
-							"user_id": structpb.NewStringValue("AIDASP2TPHJSUFRSTTZX4"),
-						},
+					Redacted:     "AKIAWARWQKZNHMZBLY4I",
+					ExtraData: map[string]string{
+						"account": "413504919130",
+						"arn":     "arn:aws:iam::413504919130:root",
+						"user_id": "413504919130",
 					},
 				},
 			},
@@ -82,7 +78,7 @@ func TestAWS_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     false,
-					Redacted:     "AKIASP2TPHJSQH3FJRUX",
+					Redacted:     "AKIAWARWQKZNHMZBLY4I",
 					ExtraData:    nil,
 				},
 			},
@@ -112,12 +108,10 @@ func TestAWS_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     true,
 					Redacted:     id,
-					ExtraData: &structpb.Struct{
-						Fields: map[string]*structpb.Value{
-							"account": structpb.NewStringValue("171436882533"),
-							"arn":     structpb.NewStringValue("arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793"),
-							"user_id": structpb.NewStringValue("AIDASP2TPHJSUFRSTTZX4"),
-						},
+					ExtraData: map[string]string{
+						"account": "413504919130",
+						"arn":     "arn:aws:iam::413504919130:root",
+						"user_id": "413504919130",
 					},
 				},
 				{
@@ -152,12 +146,10 @@ func TestAWS_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     true,
 					Redacted:     id,
-					ExtraData: &structpb.Struct{
-						Fields: map[string]*structpb.Value{
-							"account": structpb.NewStringValue("171436882533"),
-							"arn":     structpb.NewStringValue("arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793"),
-							"user_id": structpb.NewStringValue("AIDASP2TPHJSUFRSTTZX4"),
-						},
+					ExtraData: map[string]string{
+						"account": "413504919130",
+						"arn":     "arn:aws:iam::413504919130:root",
+						"user_id": "413504919130",
 					},
 				},
 				{
@@ -180,7 +172,7 @@ func TestAWS_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     false,
-					Redacted:     "AKIASP2TPHJSQH3FJRUX",
+					Redacted:     "AKIAWARWQKZNHMZBLY4I",
 				},
 			},
 			wantErr: false,
@@ -189,7 +181,7 @@ func TestAWS_FromChunk(t *testing.T) {
 			name: "skipped",
 			s: scanner{
 				skipIDs: map[string]struct{}{
-					"AKIASP2TPHJSQH3FJRUX": {},
+					"AKIAWARWQKZNHMZBLY4I": {},
 				},
 			},
 			args: args{
@@ -209,16 +201,11 @@ func TestAWS_FromChunk(t *testing.T) {
 				return
 			}
 			for i := range got {
-				if tt.want[i].ExtraData != nil {
-					if !proto.Equal(got[i].ExtraData, tt.want[i].ExtraData) {
-						t.Errorf("AWS.FromData() %s extra data not equal: got %+v, want %+v", tt.name, got[i].ExtraData, tt.want[i].ExtraData)
-					}
-				}
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "ExtraData")
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw")
 			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
 				t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}

pkg/detectors/detectors.go

@@ -12,7 +12,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
-	"google.golang.org/protobuf/types/known/structpb"
 )
 
 // Detector defines an interface for scanning for and verifying secrets.
@@ -46,7 +45,7 @@ type Result struct {
 	// Redacted contains the redacted version of the raw secret identification data for display purposes.
 	// A secret ID should be used if available.
 	Redacted       string
-	ExtraData      *structpb.Struct
+	ExtraData      map[string]string
 	StructuredData *detectorspb.StructuredData
 }
 

pkg/detectors/fastlypersonaltoken/fastlypersonaltoken.go

@@ -3,6 +3,7 @@ package fastlypersonaltoken
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"regexp"
@@ -11,7 +12,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-	"google.golang.org/protobuf/types/known/structpb"
 )
 
 type Scanner struct{}
@@ -76,14 +76,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						continue
 					}
 					s1.Verified = true
-					s1.ExtraData = &structpb.Struct{
-						Fields: map[string]*structpb.Value{
-							"username":                structpb.NewStringValue(userRes.Login),
-							"name":                    structpb.NewStringValue(userRes.Name),
-							"role":                    structpb.NewStringValue(userRes.Role),
-							"locked":                  structpb.NewBoolValue(userRes.Locked),
-							"two_factor_auth_enabled": structpb.NewBoolValue(userRes.TwoFactorAuthEnabled),
-						},
+					s1.ExtraData = map[string]string{
+						"username":                userRes.Login,
+						"name":                    userRes.Name,
+						"role":                    userRes.Role,
+						"locked":                  fmt.Sprintf("%t", userRes.Locked),
+						"two_factor_auth_enabled": fmt.Sprintf("%t", userRes.TwoFactorAuthEnabled),
 					}
 				} else {
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {

pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_test.go

@@ -95,7 +95,6 @@ func TestFastlyPersonalToken_FromChunk(t *testing.T) {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
-				got[i].ExtraData = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("FastlyPersonalToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)

pkg/detectors/github/github.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-	"google.golang.org/protobuf/types/known/structpb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
@@ -86,15 +85,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					res.Body.Close()
 					if err == nil {
 						s1.Verified = true
-						s1.ExtraData = &structpb.Struct{
-							Fields: map[string]*structpb.Value{
-								"username":     structpb.NewStringValue(userResponse.Login),
-								"url":          structpb.NewStringValue(userResponse.UserURL),
-								"account_type": structpb.NewStringValue(userResponse.Type),
-								"site_admin":   structpb.NewBoolValue(userResponse.SiteAdmin),
-								"name":         structpb.NewStringValue(userResponse.Name),
-								"company":      structpb.NewStringValue(userResponse.Company),
-							},
+						s1.ExtraData = map[string]string{
+							"username":     userResponse.Login,
+							"url":          userResponse.UserURL,
+							"account_type": userResponse.Type,
+							"site_admin":   fmt.Sprintf("%t", userResponse.SiteAdmin),
+							"name":         userResponse.Name,
+							"company":      userResponse.Company,
 						}
 					}
 				}

pkg/detectors/openai/openai.go

@@ -6,10 +6,10 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
+	"strconv"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-	"google.golang.org/protobuf/types/known/structpb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
@@ -83,17 +83,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					if err == nil {
 						s1.Verified = true
 						org := orgs.Data[0]
-						s1.ExtraData = &structpb.Struct{
-							Fields: map[string]*structpb.Value{
-								"id":          structpb.NewStringValue(org.Id),
-								"title":       structpb.NewStringValue(org.Title),
-								"user":        structpb.NewStringValue(org.User),
-								"description": structpb.NewStringValue(org.Description),
-								"role":        structpb.NewStringValue(org.Role),
-								"is_personal": structpb.NewBoolValue(org.Personal),
-								"is_default":  structpb.NewBoolValue(org.Default),
-								"total_orgs":  structpb.NewNumberValue(float64(len(orgs.Data))),
-							},
+						s1.ExtraData = map[string]string{
+							"id":          org.Id,
+							"title":       org.Title,
+							"user":        org.User,
+							"description": org.Description,
+							"role":        org.Role,
+							"is_personal": strconv.FormatBool(org.Personal),
+							"is_default":  strconv.FormatBool(org.Default),
+							"total_orgs":  fmt.Sprintf("%d", len(orgs.Data)),
 						}
 					}
 				}

pkg/detectors/openai/openai_test.go

@@ -69,6 +69,16 @@ func TestOpenAI_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_OpenAI,
 					Verified:     true,
 					Redacted:     "sk-...xxxx",
+					ExtraData: map[string]string{
+						"description": "",
+						"id":          "",
+						"is_default":  "",
+						"is_personal": "",
+						"role":        "",
+						"title":       "",
+						"total_orgs":  "",
+						"user":        "",
+					},
 				},
 			},
 			wantErr: false,

pkg/detectors/shopify/shopify.go

@@ -10,7 +10,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-	"google.golang.org/protobuf/types/known/structpb"
 )
 
 type Scanner struct{}
@@ -63,15 +62,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						shopifyTokenAccessScopes := shopifyTokenAccessScopes{}
 						err := json.NewDecoder(res.Body).Decode(&shopifyTokenAccessScopes)
 						if err == nil {
-							var handleArray []*structpb.Value
+							var handleArray []string
 							for _, handle := range shopifyTokenAccessScopes.AccessScopes {
-								handleArray = append(handleArray, structpb.NewStringValue(handle.Handle))
+								handleArray = append(handleArray, handle.Handle)
+
 							}
 							s1.Verified = true
-							s1.ExtraData = &structpb.Struct{
-								Fields: map[string]*structpb.Value{
-									"access_scopes": structpb.NewListValue(&structpb.ListValue{Values: handleArray}),
-								},
+							s1.ExtraData = map[string]string{
+								"access_scopes": strings.Join(handleArray, ","),
 							}
 						}
 						res.Body.Close()

pkg/detectors/shopify/shopify_test.go

@@ -6,11 +6,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/structpb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -52,13 +49,8 @@ func TestShopify_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_Shopify,
 					Redacted:     domain,
 					Verified:     true,
-					ExtraData: &structpb.Struct{
-						Fields: map[string]*structpb.Value{
-							"access_scopes": structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{
-								structpb.NewStringValue("read_analytics"),
-								structpb.NewStringValue("unauthenticated_read_selling_plans"),
-							}}),
-						},
+					ExtraData: map[string]string{
+						"access_scopes": "read_analytics,unauthenticated_read_selling_plans",
 					},
 				},
 			},
@@ -103,19 +95,13 @@ func TestShopify_FromChunk(t *testing.T) {
 				return
 			}
 			for i := range got {
-				if tt.want[i].ExtraData != nil {
-					if !proto.Equal(got[i].ExtraData, tt.want[i].ExtraData) {
-						t.Errorf("AWS.FromData() %s extra data not equal: got %+v, want %+v", tt.name, got[i].ExtraData, tt.want[i].ExtraData)
-					}
-				}
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "ExtraData")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
-				t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Shopify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}

pkg/pb/detectorspb/detectors.pb.validate.go

@@ -64,34 +64,7 @@ func (m *Result) validate(all bool) error {
 
 	// no validation rules for Hash
 
-	if all {
-		switch v := interface{}(m.GetExtraData()).(type) {
-		case interface{ ValidateAll() error }:
-			if err := v.ValidateAll(); err != nil {
-				errors = append(errors, ResultValidationError{
-					field:  "ExtraData",
-					reason: "embedded message failed validation",
-					cause:  err,
-				})
-			}
-		case interface{ Validate() error }:
-			if err := v.Validate(); err != nil {
-				errors = append(errors, ResultValidationError{
-					field:  "ExtraData",
-					reason: "embedded message failed validation",
-					cause:  err,
-				})
-			}
-		}
-	} else if v, ok := interface{}(m.GetExtraData()).(interface{ Validate() error }); ok {
-		if err := v.Validate(); err != nil {
-			return ResultValidationError{
-				field:  "ExtraData",
-				reason: "embedded message failed validation",
-				cause:  err,
-			}
-		}
-	}
+	// no validation rules for ExtraData
 
 	if all {
 		switch v := interface{}(m.GetStructuredData()).(type) {

pkg/sources/gcs/gcs.go

@@ -173,7 +173,7 @@ type objectsProgress struct {
 }
 
 func newObjectsProgress(cnt int) *objectsProgress {
-	return &objectsProgress{TotalCnt: cnt, Processing: map[string]struct{}{}}
+	return &objectsProgress{TotalCnt: cnt}
 }
 
 // newProgressInfo constructs a progressInfo from the Source's Progress information.
@@ -208,6 +208,9 @@ func (p *progressInfo) setProcessStatus(obj object, fn progressStateFn) {
 }
 
 func setProcessingBucketObject(bucket, obj string, progress *progressInfo) {
+	if progress.bucketObjects[bucket].Processing == nil {
+		progress.bucketObjects[bucket].Processing = make(map[string]struct{})
+	}
 	progress.processing(bucket, obj)
 }
 

pkg/sources/gcs/gcs_test.go

@@ -350,7 +350,7 @@ func TestSourceInit_Enumerate(t *testing.T) {
 	assert.Equal(t, uint32(1), source.stats.numBuckets)
 	assert.Equal(t, uint64(5), source.stats.bucketObjects[testBucket])
 	// Ensure progress is initialized correctly.
-	p := map[string]*objectsProgress{testBucket: {TotalCnt: 5, Processing: map[string]struct{}{}}}
+	p := map[string]*objectsProgress{testBucket: {IsBucketProcessed: false, ProcessedCnt: 0, TotalCnt: 5}}
 	b, err := json.Marshal(p)
 	assert.Nil(t, err)
 	assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)

proto/detectors.proto

@@ -4,8 +4,6 @@ package detectors;
 
 option go_package = "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb";
 
-import "google/protobuf/struct.proto";
-
 enum DecoderType {
   UNKNOWN = 0;
   PLAIN = 1;
@@ -928,11 +926,12 @@ message Result {
   string redacted = 3;
   bool verified = 4;
   string hash = 5;
-  google.protobuf.Struct extra_data = 6;
+  map<string, string> extra_data = 6;
   StructuredData structured_data = 7;
   string hash_v2 = 8;
   DecoderType decoder_type = 9;
 }
+
 message StructuredData {
   repeated TlsPrivateKey tls_private_key = 1;
   repeated GitHubSSHKey github_ssh_key = 2;