From c2b4f0bc39aa8dd92372ed066040c8289b2703a2 Mon Sep 17 00:00:00 2001 From: Bill Rich Date: Tue, 22 Mar 2022 08:27:15 -0800 Subject: [PATCH] Include line numbers in git metadata (#97) * Include line numbers in git metadata * Update tests for fragments and line numbers --- hack/snifftest/main.go | 2 +- main.go | 3 +- pkg/engine/engine.go | 59 ++++++++++++++++++- pkg/engine/git.go | 5 +- pkg/engine/git_test.go | 28 +++++---- pkg/output/plain.go | 2 +- .../source_metadatapb/source_metadata.pb.go | 40 ++++++------- pkg/sources/git/git.go | 43 +++++++------- pkg/sources/git/git_test.go | 28 +++++---- pkg/sources/github/github.go | 3 +- pkg/sources/gitlab/gitlab.go | 3 +- proto/source_metadata.proto | 10 ++-- 12 files changed, 149 insertions(+), 77 deletions(-) diff --git a/hack/snifftest/main.go b/hack/snifftest/main.go index 34e8e1fb..2de774af 100644 --- a/hack/snifftest/main.go +++ b/hack/snifftest/main.go @@ -170,7 +170,7 @@ func main() { log.Infof("cloned %s", r) s := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "snifftest", false, runtime.NumCPU(), - func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData { + func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData { return &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Git{ Git: &source_metadatapb.Git{ diff --git a/main.go b/main.go index 481a5a6a..f7d563df 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "gopkg.in/alecthomas/kingpin.v2" "log" "net/http" _ "net/http/pprof" @@ -17,8 +18,6 @@ import ( "github.com/gorilla/mux" "github.com/sirupsen/logrus" - "gopkg.in/alecthomas/kingpin.v2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/decoders" "github.com/trufflesecurity/trufflehog/v3/pkg/engine" diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 7ad06f0f..fa5d2242 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -1,7 +1,10 @@ package engine import ( + "bytes" "context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "runtime" "strings" "sync" @@ -174,7 +177,13 @@ func (e *Engine) detectorWorker(ctx context.Context) { continue } for _, result := range results { - e.results <- detectors.CopyMetadata(chunk, result) + targetChunk := chunk + if isGitSource(chunk.SourceType) { + copyChunk := *chunk + targetChunk = ©Chunk + SetLineNumber(©Chunk, &result) + } + e.results <- detectors.CopyMetadata(targetChunk, result) } if len(results) > 0 { elasped := time.Since(start) @@ -196,3 +205,51 @@ func (e *Engine) detectorWorker(ctx context.Context) { atomic.AddUint64(&e.chunksScanned, 1) } } + +// gitSources is a list of sources that utilize the Git source. It is stored this way because slice consts are not +// supported. +func gitSources() []sourcespb.SourceType { + return []sourcespb.SourceType{ + sourcespb.SourceType_SOURCE_TYPE_GIT, + sourcespb.SourceType_SOURCE_TYPE_GITHUB, + sourcespb.SourceType_SOURCE_TYPE_GITLAB, + sourcespb.SourceType_SOURCE_TYPE_BITBUCKET, + sourcespb.SourceType_SOURCE_TYPE_GERRIT, + sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG, + sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT, + } +} + +func isGitSource(sourceType sourcespb.SourceType) bool { + for _, i := range gitSources() { + if i == sourceType { + return true + } + } + return false +} + +func SetLineNumber(chunk *sources.Chunk, result *detectors.Result) { + var startingLine *int64 + switch metadata := chunk.SourceMetadata.GetData().(type) { + case *source_metadatapb.MetaData_Git: + startingLine = &metadata.Git.Line + case *source_metadatapb.MetaData_Github: + startingLine = &metadata.Github.Line + case *source_metadatapb.MetaData_Gitlab: + startingLine = &metadata.Gitlab.Line + case *source_metadatapb.MetaData_Bitbucket: + startingLine = &metadata.Bitbucket.Line + case *source_metadatapb.MetaData_Gerrit: + startingLine = &metadata.Gerrit.Line + } + lines := bytes.Split(chunk.Data, []byte("\n")) + + for i, line := range lines { + if bytes.Contains(line, result.Raw) { + *startingLine = *startingLine + int64(i) + return + } + } + return +} diff --git a/pkg/engine/git.go b/pkg/engine/git.go index 5377b746..bc2f869a 100644 --- a/pkg/engine/git.go +++ b/pkg/engine/git.go @@ -86,8 +86,8 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string, } scanOptions := git.NewScanOptions(opts...) - gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "local", true, runtime.NumCPU(), - func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData { + gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "trufflehog - git", true, runtime.NumCPU(), + func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData { return &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Git{ Git: &source_metadatapb.Git{ @@ -96,6 +96,7 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string, Email: email, Repository: repository, Timestamp: timestamp, + Line: line, }, }, } diff --git a/pkg/engine/git_test.go b/pkg/engine/git_test.go index a0feb5be..834101b1 100644 --- a/pkg/engine/git_test.go +++ b/pkg/engine/git_test.go @@ -11,6 +11,11 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" ) +type expResult struct { + B string + LineNumber int64 +} + func TestGitEngine(t *testing.T) { repoUrl := "https://github.com/dustin-decker/secretsandstuff.git" path, _, err := git.PrepareRepo(repoUrl) @@ -22,7 +27,7 @@ func TestGitEngine(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() type testProfile struct { - expected map[string]string + expected map[string]expResult branch string base string maxDepth int @@ -30,17 +35,17 @@ func TestGitEngine(t *testing.T) { } for tName, tTest := range map[string]testProfile{ "all_secrets": { - expected: map[string]string{ - "70001020fab32b1fcf2f1f0e5c66424eae649826": "AKIAXYZDQCEN4B6JSJQI", - "90c75f884c65dc3638ca1610bd9844e668f213c2": "AKIAILE3JG6KMS3HZGCA", - "8afb0ecd4998b1179e428db5ebbcdc8221214432": "369963c1434c377428ca8531fbc46c0c43d037a0", - "27fbead3bf883cdb7de9d7825ed401f28f9398f1": "ffc7e0f9400fb6300167009e42d2f842cd7956e2", + expected: map[string]expResult{ + "70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2}, + "90c75f884c65dc3638ca1610bd9844e668f213c2": {"AKIAILE3JG6KMS3HZGCA", 4}, + "8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3}, + "27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7}, }, filter: common.FilterEmpty(), }, "base_commit": { - expected: map[string]string{ - "70001020fab32b1fcf2f1f0e5c66424eae649826": "AKIAXYZDQCEN4B6JSJQI", + expected: map[string]expResult{ + "70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2}, }, filter: common.FilterEmpty(), base: "2f251b8c1e72135a375b659951097ec7749d4af9", @@ -56,8 +61,11 @@ func TestGitEngine(t *testing.T) { for result := range e.ResultsChan() { switch meta := result.SourceMetadata.GetData().(type) { case *source_metadatapb.MetaData_Git: - if tTest.expected[meta.Git.Commit] != string(result.Raw) { - t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit]) + if tTest.expected[meta.Git.Commit].B != string(result.Raw) { + t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit].B) + } + if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line { + t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber) } } resultCount++ diff --git a/pkg/output/plain.go b/pkg/output/plain.go index 814d131e..ea7887dc 100644 --- a/pkg/output/plain.go +++ b/pkg/output/plain.go @@ -42,7 +42,7 @@ func PrintPlainOutput(r *detectors.ResultWithMetadata) { printer.Printf("Raw result: %s\n", whitePrinter.Sprint(out.Raw)) for _, data := range meta { for k, v := range data { - printer.Printf("%s: %s\n", strings.Title(k), v) + printer.Printf("%s: %v\n", strings.Title(k), v) } } fmt.Println("") diff --git a/pkg/pb/source_metadatapb/source_metadata.pb.go b/pkg/pb/source_metadatapb/source_metadata.pb.go index 95557dd7..7c3d3d9f 100644 --- a/pkg/pb/source_metadatapb/source_metadata.pb.go +++ b/pkg/pb/source_metadatapb/source_metadata.pb.go @@ -113,7 +113,7 @@ type Bitbucket struct { Email string `protobuf:"bytes,7,opt,name=email,proto3" json:"email,omitempty"` Link string `protobuf:"bytes,8,opt,name=link,proto3" json:"link,omitempty"` Timestamp string `protobuf:"bytes,9,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - Line string `protobuf:"bytes,10,opt,name=line,proto3" json:"line,omitempty"` + Line int64 `protobuf:"varint,10,opt,name=line,proto3" json:"line,omitempty"` } func (x *Bitbucket) Reset() { @@ -211,11 +211,11 @@ func (x *Bitbucket) GetTimestamp() string { return "" } -func (x *Bitbucket) GetLine() string { +func (x *Bitbucket) GetLine() int64 { if x != nil { return x.Line } - return "" + return 0 } type Buildkite struct { @@ -742,7 +742,7 @@ type Git struct { Email string `protobuf:"bytes,3,opt,name=email,proto3" json:"email,omitempty"` Repository string `protobuf:"bytes,4,opt,name=repository,proto3" json:"repository,omitempty"` Timestamp string `protobuf:"bytes,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - Line string `protobuf:"bytes,6,opt,name=line,proto3" json:"line,omitempty"` + Line int64 `protobuf:"varint,6,opt,name=line,proto3" json:"line,omitempty"` } func (x *Git) Reset() { @@ -812,11 +812,11 @@ func (x *Git) GetTimestamp() string { return "" } -func (x *Git) GetLine() string { +func (x *Git) GetLine() int64 { if x != nil { return x.Line } - return "" + return 0 } type Github struct { @@ -831,7 +831,7 @@ type Github struct { Email string `protobuf:"bytes,5,opt,name=email,proto3" json:"email,omitempty"` File string `protobuf:"bytes,6,opt,name=file,proto3" json:"file,omitempty"` Timestamp string `protobuf:"bytes,7,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - Line string `protobuf:"bytes,8,opt,name=line,proto3" json:"line,omitempty"` + Line int64 `protobuf:"varint,8,opt,name=line,proto3" json:"line,omitempty"` } func (x *Github) Reset() { @@ -915,11 +915,11 @@ func (x *Github) GetTimestamp() string { return "" } -func (x *Github) GetLine() string { +func (x *Github) GetLine() int64 { if x != nil { return x.Line } - return "" + return 0 } type Gitlab struct { @@ -933,7 +933,7 @@ type Gitlab struct { Email string `protobuf:"bytes,4,opt,name=email,proto3" json:"email,omitempty"` Repository string `protobuf:"bytes,5,opt,name=repository,proto3" json:"repository,omitempty"` Timestamp string `protobuf:"bytes,6,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - Line string `protobuf:"bytes,7,opt,name=line,proto3" json:"line,omitempty"` + Line int64 `protobuf:"varint,7,opt,name=line,proto3" json:"line,omitempty"` } func (x *Gitlab) Reset() { @@ -1010,11 +1010,11 @@ func (x *Gitlab) GetTimestamp() string { return "" } -func (x *Gitlab) GetLine() string { +func (x *Gitlab) GetLine() int64 { if x != nil { return x.Line } - return "" + return 0 } type GCS struct { @@ -1517,7 +1517,7 @@ type Gerrit struct { Email string `protobuf:"bytes,3,opt,name=email,proto3" json:"email,omitempty"` Project string `protobuf:"bytes,4,opt,name=project,proto3" json:"project,omitempty"` // projects are what Gerrit calls repositories Timestamp string `protobuf:"bytes,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - Line string `protobuf:"bytes,6,opt,name=line,proto3" json:"line,omitempty"` + Line int64 `protobuf:"varint,6,opt,name=line,proto3" json:"line,omitempty"` } func (x *Gerrit) Reset() { @@ -1587,11 +1587,11 @@ func (x *Gerrit) GetTimestamp() string { return "" } -func (x *Gerrit) GetLine() string { +func (x *Gerrit) GetLine() int64 { if x != nil { return x.Line } - return "" + return 0 } type Test struct { @@ -2073,7 +2073,7 @@ var file_source_metadata_proto_rawDesc = []byte{ 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, - 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, + 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xa4, 0x01, 0x0a, 0x09, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f, 0x72, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x02, @@ -2140,7 +2140,7 @@ var file_source_metadata_proto_rawDesc = []byte{ 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xcc, 0x01, 0x0a, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xcc, 0x01, 0x0a, 0x06, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, @@ -2153,7 +2153,7 @@ var file_source_metadata_proto_rawDesc = []byte{ 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xb0, 0x01, 0x0a, 0x06, + 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xb0, 0x01, 0x0a, 0x06, 0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, @@ -2164,7 +2164,7 @@ var file_source_metadata_proto_rawDesc = []byte{ 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, - 0x6e, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x5b, + 0x6e, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x5b, 0x0a, 0x03, 0x47, 0x43, 0x53, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, @@ -2224,7 +2224,7 @@ var file_source_metadata_proto_rawDesc = []byte{ 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, - 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x1a, 0x0a, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x1a, 0x0a, 0x04, 0x54, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x81, 0x01, 0x0a, 0x07, 0x4a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go index ecc1e9ce..1911c7f4 100644 --- a/pkg/sources/git/git.go +++ b/pkg/sources/git/git.go @@ -47,14 +47,14 @@ type Git struct { sourceName string sourceID int64 jobID int64 - sourceMetadataFunc func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData + sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData verify bool // sem is used to limit concurrency sem *semaphore.Weighted } func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int, - sourceMetadataFunc func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData, + sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData, ) *Git { return &Git{ sourceType: sourceType, @@ -102,7 +102,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, s.conn = &conn s.git = NewGit(s.Type(), s.jobId, s.sourceId, s.name, s.verify, runtime.NumCPU(), - func(file, email, commit, repository, timestamp string) *source_metadatapb.MetaData { + func(file, email, commit, repository, timestamp string, line int64) *source_metadatapb.MetaData { return &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Git{ Git: &source_metadatapb.Git{ @@ -111,6 +111,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, Email: sanitizer.UTF8(email), Repository: sanitizer.UTF8(repository), Timestamp: sanitizer.UTF8(timestamp), + Line: line, }, }, } @@ -269,14 +270,7 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp if !scanOptions.Filter.Pass(file.NewName) { continue } - newLines := bytes.Buffer{} - for _, frag := range file.TextFragments { - for _, line := range frag.Lines { - if line.Op == gitdiff.OpAdd { - newLines.WriteString(line.String()) - } - } - } + fileName := file.NewName if fileName == "" { continue @@ -299,14 +293,23 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp return errors.WrapPrefix(err, "couldn't get repo name", 0) } - metadata := s.sourceMetadataFunc(fileName, email, hash, when, safeRepo) - chunksChan <- &sources.Chunk{ - SourceName: s.sourceName, - SourceID: s.sourceID, - SourceType: s.sourceType, - SourceMetadata: metadata, - Data: newLines.Bytes(), - Verify: s.verify, + for _, frag := range file.TextFragments { + newLines := bytes.Buffer{} + newLineNumber := frag.NewPosition + for _, line := range frag.Lines { + if line.Op == gitdiff.OpAdd { + newLines.WriteString(strings.ReplaceAll(line.String(), "\n", " ") + "\n") + } + } + metadata := s.sourceMetadataFunc(fileName, email, hash, when, safeRepo, newLineNumber) + chunksChan <- &sources.Chunk{ + SourceName: s.sourceName, + SourceID: s.sourceID, + SourceType: s.sourceType, + SourceMetadata: metadata, + Data: newLines.Bytes(), + Verify: s.verify, + } } } return nil @@ -341,7 +344,7 @@ func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunk continue } metadata := s.sourceMetadataFunc( - fh, "unstaged", "unstaged", time.Now().String(), safeRepo, + fh, "unstaged", "unstaged", time.Now().String(), safeRepo, 0, ) fileBuf := bytes.NewBuffer(nil) diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.go index 06de2023..61b9da20 100644 --- a/pkg/sources/git/git_test.go +++ b/pkg/sources/git/git_test.go @@ -149,6 +149,7 @@ func TestSource_Chunks_Integration(t *testing.T) { type byteCompare struct { B []byte Found bool + Multi bool } tests := []struct { name string @@ -168,17 +169,17 @@ func TestSource_Chunks_Integration(t *testing.T) { }, }, expectedChunkData: map[string]*byteCompare{ - "70001020fab32b1fcf2f1f0e5c66424eae649826-aws": {B: []byte("+[default]\n+aws_access_key_id = AKIAXYZDQCEN4B6JSJQI\n+aws_secret_access_key = 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\n+output = json\n+region = us-east-2\n")}, - "a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte("+f\n")}, - "07d96d011005fe8296bdd237c13a06a72e96783d-bump": {B: []byte("+ s \n")}, - "2f251b8c1e72135a375b659951097ec7749d4af9-bump": {B: []byte("+ \n")}, - "e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump": {B: []byte("+oops \n")}, - "735b52b0eb40610002bb1088e902bd61824eb305-bump": {B: []byte("+oops\n")}, - "ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("+\n")}, - "27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("+yup, just did that\n+\n+github_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n+\n+oh, goodness. there's another one!")}, - "8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("+oops might drop a slack token here\n+\n+github_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\"\n+\n+yup, just did that")}, - "8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("+blah blaj\n+\n+this is the secret: AKIA2E0A8F3B244C9986\n+\n+okay thank you bye")}, - "90c75f884c65dc3638ca1610bd9844e668f213c2-aws": {B: []byte("+this is the secret: [Default]\n+Access key Id: AKIAILE3JG6KMS3HZGCA\n+Secret Access Key: 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\n+okay thank you bye\n")}, + "70001020fab32b1fcf2f1f0e5c66424eae649826-aws": {B: []byte("+[default] \n+aws_access_key_id = AKIAXYZDQCEN4B6JSJQI \n+aws_secret_access_key = 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 \n+output = json \n+region = us-east-2 \n")}, + "a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte("+f \n")}, + "07d96d011005fe8296bdd237c13a06a72e96783d-bump": {B: []byte("+ s \n")}, + "2f251b8c1e72135a375b659951097ec7749d4af9-bump": {B: []byte("+ \n")}, + "e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump": {B: []byte("+oops \n")}, + "735b52b0eb40610002bb1088e902bd61824eb305-bump": {B: []byte("+oops \n")}, + "ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("+ \n")}, + "27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("+yup, just did that \n+ \n+github_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\" \n+ \n+oh, goodness. there's another one!\n")}, + "8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("+oops might drop a slack token here \n+ \n+github_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\" \n+ \n+yup, just did that\n"), Multi: true}, + "8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("+blah blaj \n+ \n+this is the secret: AKIA2E0A8F3B244C9986 \n+ \n+okay thank you bye\n"), Multi: true}, + "90c75f884c65dc3638ca1610bd9844e668f213c2-aws": {B: []byte("+this is the secret: [Default] \n+Access key Id: AKIAILE3JG6KMS3HZGCA \n+Secret Access Key: 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 \n"), Multi: true}, }, }, } @@ -216,8 +217,9 @@ func TestSource_Chunks_Integration(t *testing.T) { if expectedData, exists := tt.expectedChunkData[key]; !exists { t.Errorf("A chunk exists that was not expected with key %q", key) } else { - (*tt.expectedChunkData[key]).Found = true - if !bytes.Equal(chunk.Data, expectedData.B) { + if bytes.Equal(chunk.Data, expectedData.B) { + (*tt.expectedChunkData[key]).Found = true + } else if !expectedData.Multi { t.Errorf("Got %q: %q, which was not expected", key, string(chunk.Data)) } } diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 5ad28853..231171c3 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -113,7 +113,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, s.conn = &conn s.git = git.NewGit(s.Type(), s.JobID(), s.SourceID(), s.name, s.verify, runtime.NumCPU(), - func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData { + func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData { return &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Github{ Github: &source_metadatapb.Github{ @@ -123,6 +123,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, Repository: sanitizer.UTF8(repository), Link: git.GenerateLink(repository, commit, file), Timestamp: sanitizer.UTF8(timestamp), + Line: line, }, }, } diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go index 33025420..f3643621 100644 --- a/pkg/sources/gitlab/gitlab.go +++ b/pkg/sources/gitlab/gitlab.go @@ -101,7 +101,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, } s.git = git.NewGit(s.Type(), s.JobID(), s.SourceID(), s.name, s.verify, runtime.NumCPU(), - func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData { + func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData { return &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Gitlab{ Gitlab: &source_metadatapb.Gitlab{ @@ -111,6 +111,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, Repository: sanitizer.UTF8(repository), Link: git.GenerateLink(repository, commit, file), Timestamp: sanitizer.UTF8(timestamp), + Line: line, }, }, } diff --git a/proto/source_metadata.proto b/proto/source_metadata.proto index 0f9b5518..39b5d3a7 100644 --- a/proto/source_metadata.proto +++ b/proto/source_metadata.proto @@ -22,7 +22,7 @@ message Bitbucket { string email = 7; string link = 8; string timestamp = 9; - string line = 10; + int64 line = 10; } message Buildkite { @@ -84,7 +84,7 @@ message Git { string email = 3; string repository = 4; string timestamp = 5; - string line = 6; + int64 line = 6; } message Github { @@ -95,7 +95,7 @@ message Github { string email = 5; string file = 6; string timestamp = 7; - string line = 8; + int64 line = 8; } message Gitlab { @@ -105,7 +105,7 @@ message Gitlab { string email = 4; string repository = 5; string timestamp = 6; - string line = 7; + int64 line = 7; } message GCS { @@ -164,7 +164,7 @@ message Gerrit { string email = 3; string project = 4; // projects are what Gerrit calls repositories string timestamp = 5; - string line = 6; + int64 line = 6; } message Test {