Use structpb for Extra Data to support other types

pkg/detectors/aws/aws.go

@@ -15,6 +15,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"google.golang.org/protobuf/types/known/structpb"
 )
 
 type scanner struct {
@@ -179,10 +180,12 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						err := json.NewDecoder(res.Body).Decode(&identityInfo)
 						if err == nil {
 							s1.Verified = true
-							s1.ExtraData = map[string]string{
+							s1.ExtraData = &structpb.Struct{
-								"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
+								Fields: map[string]*structpb.Value{
-								"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
+									"account": structpb.NewStringValue(identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account),
-								"arn":     identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
+									"user_id": structpb.NewStringValue(identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID),
+									"arn":     structpb.NewStringValue(identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn),
+								},
 							}
 						}
 						res.Body.Close()

pkg/detectors/aws/aws_test.go

@@ -15,6 +15,8 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
 )
 
 func TestAWS_FromChunk(t *testing.T) {
@@ -56,11 +58,13 @@ func TestAWS_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     true,
-					Redacted:     "AKIAWARWQKZNHMZBLY4I",
+					Redacted:     "AKIASP2TPHJSQH3FJRUX",
-					ExtraData: map[string]string{
+					ExtraData: &structpb.Struct{
-						"account": "413504919130",
+						Fields: map[string]*structpb.Value{
-						"arn":     "arn:aws:iam::413504919130:root",
+							"account": structpb.NewStringValue("171436882533"),
-						"user_id": "413504919130",
+							"arn":     structpb.NewStringValue("arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793"),
+							"user_id": structpb.NewStringValue("AIDASP2TPHJSUFRSTTZX4"),
+						},
 					},
 				},
 			},
@@ -78,7 +82,7 @@ func TestAWS_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     false,
-					Redacted:     "AKIAWARWQKZNHMZBLY4I",
+					Redacted:     "AKIASP2TPHJSQH3FJRUX",
 					ExtraData:    nil,
 				},
 			},
@@ -108,10 +112,12 @@ func TestAWS_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     true,
 					Redacted:     id,
-					ExtraData: map[string]string{
+					ExtraData: &structpb.Struct{
-						"account": "413504919130",
+						Fields: map[string]*structpb.Value{
-						"arn":     "arn:aws:iam::413504919130:root",
+							"account": structpb.NewStringValue("171436882533"),
-						"user_id": "413504919130",
+							"arn":     structpb.NewStringValue("arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793"),
+							"user_id": structpb.NewStringValue("AIDASP2TPHJSUFRSTTZX4"),
+						},
 					},
 				},
 				{
@@ -146,10 +152,12 @@ func TestAWS_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     true,
 					Redacted:     id,
-					ExtraData: map[string]string{
+					ExtraData: &structpb.Struct{
-						"account": "413504919130",
+						Fields: map[string]*structpb.Value{
-						"arn":     "arn:aws:iam::413504919130:root",
+							"account": structpb.NewStringValue("171436882533"),
-						"user_id": "413504919130",
+							"arn":     structpb.NewStringValue("arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793"),
+							"user_id": structpb.NewStringValue("AIDASP2TPHJSUFRSTTZX4"),
+						},
 					},
 				},
 				{
@@ -172,7 +180,7 @@ func TestAWS_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     false,
-					Redacted:     "AKIAWARWQKZNHMZBLY4I",
+					Redacted:     "AKIASP2TPHJSQH3FJRUX",
 				},
 			},
 			wantErr: false,
@@ -181,7 +189,7 @@ func TestAWS_FromChunk(t *testing.T) {
 			name: "skipped",
 			s: scanner{
 				skipIDs: map[string]struct{}{
-					"AKIAWARWQKZNHMZBLY4I": {},
+					"AKIASP2TPHJSQH3FJRUX": {},
 				},
 			},
 			args: args{
@@ -201,11 +209,16 @@ func TestAWS_FromChunk(t *testing.T) {
 				return
 			}
 			for i := range got {
+				if tt.want[i].ExtraData != nil {
+					if !proto.Equal(got[i].ExtraData, tt.want[i].ExtraData) {
+						t.Errorf("AWS.FromData() %s extra data not equal: got %+v, want %+v", tt.name, got[i].ExtraData, tt.want[i].ExtraData)
+					}
+				}
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw")
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "ExtraData")
 			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
 				t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}

pkg/detectors/detectors.go

@@ -12,6 +12,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"google.golang.org/protobuf/types/known/structpb"
 )
 
 // Detector defines an interface for scanning for and verifying secrets.
@@ -45,7 +46,7 @@ type Result struct {
 	// Redacted contains the redacted version of the raw secret identification data for display purposes.
 	// A secret ID should be used if available.
 	Redacted       string
-	ExtraData      map[string]string
+	ExtraData      *structpb.Struct
 	StructuredData *detectorspb.StructuredData
 }
 

pkg/detectors/fastlypersonaltoken/fastlypersonaltoken.go

@@ -3,7 +3,6 @@ package fastlypersonaltoken
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"io"
 	"net/http"
 	"regexp"
@@ -12,6 +11,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"google.golang.org/protobuf/types/known/structpb"
 )
 
 type Scanner struct{}
@@ -76,12 +76,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						continue
 					}
 					s1.Verified = true
-					s1.ExtraData = map[string]string{
+					s1.ExtraData = &structpb.Struct{
-						"username":                userRes.Login,
+						Fields: map[string]*structpb.Value{
-						"name":                    userRes.Name,
+							"username":                structpb.NewStringValue(userRes.Login),
-						"role":                    userRes.Role,
+							"name":                    structpb.NewStringValue(userRes.Name),
-						"locked":                  fmt.Sprintf("%t", userRes.Locked),
+							"role":                    structpb.NewStringValue(userRes.Role),
-						"two_factor_auth_enabled": fmt.Sprintf("%t", userRes.TwoFactorAuthEnabled),
+							"locked":                  structpb.NewBoolValue(userRes.Locked),
+							"two_factor_auth_enabled": structpb.NewBoolValue(userRes.TwoFactorAuthEnabled),
+						},
 					}
 				} else {
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {

pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_test.go

@@ -95,6 +95,7 @@ func TestFastlyPersonalToken_FromChunk(t *testing.T) {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
+				got[i].ExtraData = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("FastlyPersonalToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)

pkg/detectors/github/github.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"google.golang.org/protobuf/types/known/structpb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
@@ -85,13 +86,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					res.Body.Close()
 					if err == nil {
 						s1.Verified = true
-						s1.ExtraData = map[string]string{
+						s1.ExtraData = &structpb.Struct{
-							"username":     userResponse.Login,
+							Fields: map[string]*structpb.Value{
-							"url":          userResponse.UserURL,
+								"username":     structpb.NewStringValue(userResponse.Login),
-							"account_type": userResponse.Type,
+								"url":          structpb.NewStringValue(userResponse.UserURL),
-							"site_admin":   fmt.Sprintf("%t", userResponse.SiteAdmin),
+								"account_type": structpb.NewStringValue(userResponse.Type),
-							"name":         userResponse.Name,
+								"site_admin":   structpb.NewBoolValue(userResponse.SiteAdmin),
-							"company":      userResponse.Company,
+								"name":         structpb.NewStringValue(userResponse.Name),
+								"company":      structpb.NewStringValue(userResponse.Company),
+							},
 						}
 					}
 				}

pkg/detectors/openai/openai.go

@@ -6,10 +6,10 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
-	"strconv"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"google.golang.org/protobuf/types/known/structpb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
@@ -83,15 +83,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					if err == nil {
 						s1.Verified = true
 						org := orgs.Data[0]
-						s1.ExtraData = map[string]string{
+						s1.ExtraData = &structpb.Struct{
-							"id":          org.Id,
+							Fields: map[string]*structpb.Value{
-							"title":       org.Title,
+								"id":          structpb.NewStringValue(org.Id),
-							"user":        org.User,
+								"title":       structpb.NewStringValue(org.Title),
-							"description": org.Description,
+								"user":        structpb.NewStringValue(org.User),
-							"role":        org.Role,
+								"description": structpb.NewStringValue(org.Description),
-							"is_personal": strconv.FormatBool(org.Personal),
+								"role":        structpb.NewStringValue(org.Role),
-							"is_default":  strconv.FormatBool(org.Default),
+								"is_personal": structpb.NewBoolValue(org.Personal),
-							"total_orgs":  fmt.Sprintf("%d", len(orgs.Data)),
+								"is_default":  structpb.NewBoolValue(org.Default),
+								"total_orgs":  structpb.NewNumberValue(float64(len(orgs.Data))),
+							},
 						}
 					}
 				}

pkg/detectors/openai/openai_test.go

@@ -69,16 +69,6 @@ func TestOpenAI_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_OpenAI,
 					Verified:     true,
 					Redacted:     "sk-...xxxx",
-					ExtraData: map[string]string{
-						"description": "",
-						"id":          "",
-						"is_default":  "",
-						"is_personal": "",
-						"role":        "",
-						"title":       "",
-						"total_orgs":  "",
-						"user":        "",
-					},
 				},
 			},
 			wantErr: false,

pkg/detectors/shopify/shopify.go

@@ -10,6 +10,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"google.golang.org/protobuf/types/known/structpb"
 )
 
 type Scanner struct{}
@@ -62,14 +63,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						shopifyTokenAccessScopes := shopifyTokenAccessScopes{}
 						err := json.NewDecoder(res.Body).Decode(&shopifyTokenAccessScopes)
 						if err == nil {
-							var handleArray []string
+							var handleArray []*structpb.Value
 							for _, handle := range shopifyTokenAccessScopes.AccessScopes {
-								handleArray = append(handleArray, handle.Handle)
+								handleArray = append(handleArray, structpb.NewStringValue(handle.Handle))
-
 							}
 							s1.Verified = true
-							s1.ExtraData = map[string]string{
+							s1.ExtraData = &structpb.Struct{
-								"access_scopes": strings.Join(handleArray, ","),
+								Fields: map[string]*structpb.Value{
+									"access_scopes": structpb.NewListValue(&structpb.ListValue{Values: handleArray}),
+								},
 							}
 						}
 						res.Body.Close()

pkg/detectors/shopify/shopify_test.go

@@ -6,8 +6,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -49,8 +52,13 @@ func TestShopify_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_Shopify,
 					Redacted:     domain,
 					Verified:     true,
-					ExtraData: map[string]string{
+					ExtraData: &structpb.Struct{
-						"access_scopes": "read_analytics,unauthenticated_read_selling_plans",
+						Fields: map[string]*structpb.Value{
+							"access_scopes": structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{
+								structpb.NewStringValue("read_analytics"),
+								structpb.NewStringValue("unauthenticated_read_selling_plans"),
+							}}),
+						},
 					},
 				},
 			},
@@ -95,13 +103,19 @@ func TestShopify_FromChunk(t *testing.T) {
 				return
 			}
 			for i := range got {
+				if tt.want[i].ExtraData != nil {
+					if !proto.Equal(got[i].ExtraData, tt.want[i].ExtraData) {
+						t.Errorf("AWS.FromData() %s extra data not equal: got %+v, want %+v", tt.name, got[i].ExtraData, tt.want[i].ExtraData)
+					}
+				}
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "ExtraData")
-				t.Errorf("Shopify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}

pkg/pb/detectorspb/detectors.pb.validate.go

@@ -64,7 +64,34 @@ func (m *Result) validate(all bool) error {
 
 	// no validation rules for Hash
 
-	// no validation rules for ExtraData
+	if all {
+		switch v := interface{}(m.GetExtraData()).(type) {
+		case interface{ ValidateAll() error }:
+			if err := v.ValidateAll(); err != nil {
+				errors = append(errors, ResultValidationError{
+					field:  "ExtraData",
+					reason: "embedded message failed validation",
+					cause:  err,
+				})
+			}
+		case interface{ Validate() error }:
+			if err := v.Validate(); err != nil {
+				errors = append(errors, ResultValidationError{
+					field:  "ExtraData",
+					reason: "embedded message failed validation",
+					cause:  err,
+				})
+			}
+		}
+	} else if v, ok := interface{}(m.GetExtraData()).(interface{ Validate() error }); ok {
+		if err := v.Validate(); err != nil {
+			return ResultValidationError{
+				field:  "ExtraData",
+				reason: "embedded message failed validation",
+				cause:  err,
+			}
+		}
+	}
 
 	if all {
 		switch v := interface{}(m.GetStructuredData()).(type) {

proto/detectors.proto

@@ -4,6 +4,8 @@ package detectors;
 
 option go_package = "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb";
 
+import "google/protobuf/struct.proto";
+
 enum DecoderType {
   UNKNOWN = 0;
   PLAIN = 1;
@@ -926,12 +928,11 @@ message Result {
   string redacted = 3;
   bool verified = 4;
   string hash = 5;
-  map<string, string> extra_data = 6;
+  google.protobuf.Struct extra_data = 6;
   StructuredData structured_data = 7;
   string hash_v2 = 8;
   DecoderType decoder_type = 9;
 }
-
 message StructuredData {
   repeated TlsPrivateKey tls_private_key = 1;
   repeated GitHubSSHKey github_ssh_key = 2;