actions/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Include line numbers in git metadata (#97)

Browse Source 
* Include line numbers in git metadata

* Update tests for fragments and line numbers
This commit is contained in:
Bill Rich
2022-03-22 08:27:15 -08:00
committed by GitHub
parent 0ea3b93238
commit c2b4f0bc39
12 changed files with 149 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hack/snifftest/main.go
+1 -1
View File
@@ -170,7 +170,7 @@ func main() {
log.Infof("cloned %s", r) log.Infof("cloned %s", r)
s := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "snifftest", false, runtime.NumCPU(), s := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "snifftest", false, runtime.NumCPU(),
func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData { func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{ return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Git{ Data: &source_metadatapb.MetaData_Git{
Git: &source_metadatapb.Git{ Git: &source_metadatapb.Git{
main.go
+1 -2
View File
@@ -4,6 +4,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"gopkg.in/alecthomas/kingpin.v2"
"log" "log"
"net/http" "net/http"
_ "net/http/pprof" _ "net/http/pprof"
@@ -17,8 +18,6 @@ import (
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"gopkg.in/alecthomas/kingpin.v2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders" "github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine" "github.com/trufflesecurity/trufflehog/v3/pkg/engine"
pkg/engine/engine.go
+58 -1
View File
@@ -1,7 +1,10 @@
package engine package engine
import ( import (
"bytes"
"context" "context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"runtime" "runtime"
"strings" "strings"
"sync" "sync"
@@ -174,7 +177,13 @@ func (e *Engine) detectorWorker(ctx context.Context) {
continue continue
} }
for _, result := range results { for _, result := range results {
e.results <- detectors.CopyMetadata(chunk, result) targetChunk := chunk
if isGitSource(chunk.SourceType) {
copyChunk := *chunk
targetChunk = &copyChunk
SetLineNumber(&copyChunk, &result)
}
e.results <- detectors.CopyMetadata(targetChunk, result)
} }
if len(results) > 0 { if len(results) > 0 {
elasped := time.Since(start) elasped := time.Since(start)
@@ -196,3 +205,51 @@ func (e *Engine) detectorWorker(ctx context.Context) {
atomic.AddUint64(&e.chunksScanned, 1) atomic.AddUint64(&e.chunksScanned, 1)
} }
} }
// gitSources is a list of sources that utilize the Git source. It is stored this way because slice consts are not
// supported.
func gitSources() []sourcespb.SourceType {
return []sourcespb.SourceType{
sourcespb.SourceType_SOURCE_TYPE_GIT,
sourcespb.SourceType_SOURCE_TYPE_GITHUB,
sourcespb.SourceType_SOURCE_TYPE_GITLAB,
sourcespb.SourceType_SOURCE_TYPE_BITBUCKET,
sourcespb.SourceType_SOURCE_TYPE_GERRIT,
sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,
sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,
}
}
func isGitSource(sourceType sourcespb.SourceType) bool {
for _, i := range gitSources() {
if i == sourceType {
return true
}
}
return false
}
func SetLineNumber(chunk *sources.Chunk, result *detectors.Result) {
var startingLine *int64
switch metadata := chunk.SourceMetadata.GetData().(type) {
case *source_metadatapb.MetaData_Git:
startingLine = &metadata.Git.Line
case *source_metadatapb.MetaData_Github:
startingLine = &metadata.Github.Line
case *source_metadatapb.MetaData_Gitlab:
startingLine = &metadata.Gitlab.Line
case *source_metadatapb.MetaData_Bitbucket:
startingLine = &metadata.Bitbucket.Line
case *source_metadatapb.MetaData_Gerrit:
startingLine = &metadata.Gerrit.Line
}
lines := bytes.Split(chunk.Data, []byte("\n"))
for i, line := range lines {
if bytes.Contains(line, result.Raw) {
*startingLine = *startingLine + int64(i)
return
}
}
return
}
pkg/engine/git.go
+3 -2
View File
@@ -86,8 +86,8 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,
} }
scanOptions := git.NewScanOptions(opts...) scanOptions := git.NewScanOptions(opts...)
gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "local", true, runtime.NumCPU(), gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "trufflehog - git", true, runtime.NumCPU(),
func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData { func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{ return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Git{ Data: &source_metadatapb.MetaData_Git{
Git: &source_metadatapb.Git{ Git: &source_metadatapb.Git{
@@ -96,6 +96,7 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,
Email: email, Email: email,
Repository: repository, Repository: repository,
Timestamp: timestamp, Timestamp: timestamp,
Line: line,
}, },
}, },
} }
pkg/engine/git_test.go
+18 -10
View File
@@ -11,6 +11,11 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
) )
type expResult struct {
B string
LineNumber int64
}
func TestGitEngine(t *testing.T) { func TestGitEngine(t *testing.T) {
repoUrl := "https://github.com/dustin-decker/secretsandstuff.git" repoUrl := "https://github.com/dustin-decker/secretsandstuff.git"
path, _, err := git.PrepareRepo(repoUrl) path, _, err := git.PrepareRepo(repoUrl)
@@ -22,7 +27,7 @@ func TestGitEngine(t *testing.T) {
ctx, cancel := context.WithCancel(context.TODO()) ctx, cancel := context.WithCancel(context.TODO())
defer cancel() defer cancel()
type testProfile struct { type testProfile struct {
expected map[string]string expected map[string]expResult
branch string branch string
base string base string
maxDepth int maxDepth int
@@ -30,17 +35,17 @@ func TestGitEngine(t *testing.T) {
} }
for tName, tTest := range map[string]testProfile{ for tName, tTest := range map[string]testProfile{
"all_secrets": { "all_secrets": {
expected: map[string]string{ expected: map[string]expResult{
"70001020fab32b1fcf2f1f0e5c66424eae649826": "AKIAXYZDQCEN4B6JSJQI", "70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
"90c75f884c65dc3638ca1610bd9844e668f213c2": "AKIAILE3JG6KMS3HZGCA", "90c75f884c65dc3638ca1610bd9844e668f213c2": {"AKIAILE3JG6KMS3HZGCA", 4},
"8afb0ecd4998b1179e428db5ebbcdc8221214432": "369963c1434c377428ca8531fbc46c0c43d037a0", "8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3},
"27fbead3bf883cdb7de9d7825ed401f28f9398f1": "ffc7e0f9400fb6300167009e42d2f842cd7956e2", "27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7},
}, },
filter: common.FilterEmpty(), filter: common.FilterEmpty(),
}, },
"base_commit": { "base_commit": {
expected: map[string]string{ expected: map[string]expResult{
"70001020fab32b1fcf2f1f0e5c66424eae649826": "AKIAXYZDQCEN4B6JSJQI", "70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
}, },
filter: common.FilterEmpty(), filter: common.FilterEmpty(),
base: "2f251b8c1e72135a375b659951097ec7749d4af9", base: "2f251b8c1e72135a375b659951097ec7749d4af9",
@@ -56,8 +61,11 @@ func TestGitEngine(t *testing.T) {
for result := range e.ResultsChan() { for result := range e.ResultsChan() {
switch meta := result.SourceMetadata.GetData().(type) { switch meta := result.SourceMetadata.GetData().(type) {
case *source_metadatapb.MetaData_Git: case *source_metadatapb.MetaData_Git:
if tTest.expected[meta.Git.Commit] != string(result.Raw) { if tTest.expected[meta.Git.Commit].B != string(result.Raw) {
t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit]) t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit].B)
}
if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
} }
} }
resultCount++ resultCount++
pkg/output/plain.go
+1 -1
View File
@@ -42,7 +42,7 @@ func PrintPlainOutput(r *detectors.ResultWithMetadata) {
printer.Printf("Raw result: %s\n", whitePrinter.Sprint(out.Raw)) printer.Printf("Raw result: %s\n", whitePrinter.Sprint(out.Raw))
for _, data := range meta { for _, data := range meta {
for k, v := range data { for k, v := range data {
printer.Printf("%s: %s\n", strings.Title(k), v) printer.Printf("%s: %v\n", strings.Title(k), v)
} }
} }
fmt.Println("") fmt.Println("")
pkg/pb/source_metadatapb/source_metadata.pb.go
+20 -20
View File
@@ -113,7 +113,7 @@ type Bitbucket struct {
Email string `protobuf:"bytes,7,opt,name=email,proto3" json:"email,omitempty"` Email string `protobuf:"bytes,7,opt,name=email,proto3" json:"email,omitempty"`
Link string `protobuf:"bytes,8,opt,name=link,proto3" json:"link,omitempty"` Link string `protobuf:"bytes,8,opt,name=link,proto3" json:"link,omitempty"`
Timestamp string `protobuf:"bytes,9,opt,name=timestamp,proto3" json:"timestamp,omitempty"` Timestamp string `protobuf:"bytes,9,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
Line string `protobuf:"bytes,10,opt,name=line,proto3" json:"line,omitempty"` Line int64 `protobuf:"varint,10,opt,name=line,proto3" json:"line,omitempty"`
} }
func (x *Bitbucket) Reset() { func (x *Bitbucket) Reset() {
@@ -211,11 +211,11 @@ func (x *Bitbucket) GetTimestamp() string {
return "" return ""
} }
func (x *Bitbucket) GetLine() string { func (x *Bitbucket) GetLine() int64 {
if x != nil { if x != nil {
return x.Line return x.Line
} }
return "" return 0
} }
type Buildkite struct { type Buildkite struct {
@@ -742,7 +742,7 @@ type Git struct {
Email string `protobuf:"bytes,3,opt,name=email,proto3" json:"email,omitempty"` Email string `protobuf:"bytes,3,opt,name=email,proto3" json:"email,omitempty"`
Repository string `protobuf:"bytes,4,opt,name=repository,proto3" json:"repository,omitempty"` Repository string `protobuf:"bytes,4,opt,name=repository,proto3" json:"repository,omitempty"`
Timestamp string `protobuf:"bytes,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"` Timestamp string `protobuf:"bytes,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
Line string `protobuf:"bytes,6,opt,name=line,proto3" json:"line,omitempty"` Line int64 `protobuf:"varint,6,opt,name=line,proto3" json:"line,omitempty"`
} }
func (x *Git) Reset() { func (x *Git) Reset() {
@@ -812,11 +812,11 @@ func (x *Git) GetTimestamp() string {
return "" return ""
} }
func (x *Git) GetLine() string { func (x *Git) GetLine() int64 {
if x != nil { if x != nil {
return x.Line return x.Line
} }
return "" return 0
} }
type Github struct { type Github struct {
@@ -831,7 +831,7 @@ type Github struct {
Email string `protobuf:"bytes,5,opt,name=email,proto3" json:"email,omitempty"` Email string `protobuf:"bytes,5,opt,name=email,proto3" json:"email,omitempty"`
File string `protobuf:"bytes,6,opt,name=file,proto3" json:"file,omitempty"` File string `protobuf:"bytes,6,opt,name=file,proto3" json:"file,omitempty"`
Timestamp string `protobuf:"bytes,7,opt,name=timestamp,proto3" json:"timestamp,omitempty"` Timestamp string `protobuf:"bytes,7,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
Line string `protobuf:"bytes,8,opt,name=line,proto3" json:"line,omitempty"` Line int64 `protobuf:"varint,8,opt,name=line,proto3" json:"line,omitempty"`
} }
func (x *Github) Reset() { func (x *Github) Reset() {
@@ -915,11 +915,11 @@ func (x *Github) GetTimestamp() string {
return "" return ""
} }
func (x *Github) GetLine() string { func (x *Github) GetLine() int64 {
if x != nil { if x != nil {
return x.Line return x.Line
} }
return "" return 0
} }
type Gitlab struct { type Gitlab struct {
@@ -933,7 +933,7 @@ type Gitlab struct {
Email string `protobuf:"bytes,4,opt,name=email,proto3" json:"email,omitempty"` Email string `protobuf:"bytes,4,opt,name=email,proto3" json:"email,omitempty"`
Repository string `protobuf:"bytes,5,opt,name=repository,proto3" json:"repository,omitempty"` Repository string `protobuf:"bytes,5,opt,name=repository,proto3" json:"repository,omitempty"`
Timestamp string `protobuf:"bytes,6,opt,name=timestamp,proto3" json:"timestamp,omitempty"` Timestamp string `protobuf:"bytes,6,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
Line string `protobuf:"bytes,7,opt,name=line,proto3" json:"line,omitempty"` Line int64 `protobuf:"varint,7,opt,name=line,proto3" json:"line,omitempty"`
} }
func (x *Gitlab) Reset() { func (x *Gitlab) Reset() {
@@ -1010,11 +1010,11 @@ func (x *Gitlab) GetTimestamp() string {
return "" return ""
} }
func (x *Gitlab) GetLine() string { func (x *Gitlab) GetLine() int64 {
if x != nil { if x != nil {
return x.Line return x.Line
} }
return "" return 0
} }
type GCS struct { type GCS struct {
@@ -1517,7 +1517,7 @@ type Gerrit struct {
Email string `protobuf:"bytes,3,opt,name=email,proto3" json:"email,omitempty"` Email string `protobuf:"bytes,3,opt,name=email,proto3" json:"email,omitempty"`
Project string `protobuf:"bytes,4,opt,name=project,proto3" json:"project,omitempty"` // projects are what Gerrit calls repositories Project string `protobuf:"bytes,4,opt,name=project,proto3" json:"project,omitempty"` // projects are what Gerrit calls repositories
Timestamp string `protobuf:"bytes,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"` Timestamp string `protobuf:"bytes,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
Line string `protobuf:"bytes,6,opt,name=line,proto3" json:"line,omitempty"` Line int64 `protobuf:"varint,6,opt,name=line,proto3" json:"line,omitempty"`
} }
func (x *Gerrit) Reset() { func (x *Gerrit) Reset() {
@@ -1587,11 +1587,11 @@ func (x *Gerrit) GetTimestamp() string {
return "" return ""
} }
func (x *Gerrit) GetLine() string { func (x *Gerrit) GetLine() int64 {
if x != nil { if x != nil {
return x.Line return x.Line
} }
return "" return 0
} }
type Test struct { type Test struct {
@@ -2073,7 +2073,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b,
0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x09, 0x20, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x09, 0x20,
0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12,
0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69,
0x6e, 0x65, 0x22, 0xa4, 0x01, 0x0a, 0x09, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x6e, 0x65, 0x22, 0xa4, 0x01, 0x0a, 0x09, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65,
0x12, 0x10, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f, 0x12, 0x10, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f,
0x72, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x02, 0x72, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x02,
@@ -2140,7 +2140,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69,
0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74,
0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65,
0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xcc, 0x01, 0x0a, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xcc, 0x01, 0x0a,
0x06, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x06, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x75,
0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75,
@@ -2153,7 +2153,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d,
0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69,
0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18,
0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xb0, 0x01, 0x0a, 0x06, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xb0, 0x01, 0x0a, 0x06,
0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x12, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x12,
0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69,
@@ -2164,7 +2164,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09,
0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52,
0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69,
0x6e, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x5b, 0x6e, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x5b,
0x0a, 0x03, 0x47, 0x43, 0x53, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x0a, 0x03, 0x47, 0x43, 0x53, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a,
0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c,
@@ -2224,7 +2224,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74,
0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09,
0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e,
0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x1a, 0x0a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x1a, 0x0a,
0x04, 0x54, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x04, 0x54, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x81, 0x01, 0x0a, 0x07, 0x4a, 0x65, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x81, 0x01, 0x0a, 0x07, 0x4a, 0x65,
0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74,
pkg/sources/git/git.go
+23 -20
View File
@@ -47,14 +47,14 @@ type Git struct {
sourceName string sourceName string
sourceID int64 sourceID int64
jobID int64 jobID int64
sourceMetadataFunc func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData
verify bool verify bool
// sem is used to limit concurrency // sem is used to limit concurrency
sem *semaphore.Weighted sem *semaphore.Weighted
} }
func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int, func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
sourceMetadataFunc func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData, sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData,
) *Git { ) *Git {
return &Git{ return &Git{
sourceType: sourceType, sourceType: sourceType,
@@ -102,7 +102,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
s.conn = &conn s.conn = &conn
s.git = NewGit(s.Type(), s.jobId, s.sourceId, s.name, s.verify, runtime.NumCPU(), s.git = NewGit(s.Type(), s.jobId, s.sourceId, s.name, s.verify, runtime.NumCPU(),
func(file, email, commit, repository, timestamp string) *source_metadatapb.MetaData { func(file, email, commit, repository, timestamp string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{ return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Git{ Data: &source_metadatapb.MetaData_Git{
Git: &source_metadatapb.Git{ Git: &source_metadatapb.Git{
@@ -111,6 +111,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
Email: sanitizer.UTF8(email), Email: sanitizer.UTF8(email),
Repository: sanitizer.UTF8(repository), Repository: sanitizer.UTF8(repository),
Timestamp: sanitizer.UTF8(timestamp), Timestamp: sanitizer.UTF8(timestamp),
Line: line,
}, },
}, },
} }
@@ -269,14 +270,7 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
if !scanOptions.Filter.Pass(file.NewName) { if !scanOptions.Filter.Pass(file.NewName) {
continue continue
} }
newLines := bytes.Buffer{}
for _, frag := range file.TextFragments {
for _, line := range frag.Lines {
if line.Op == gitdiff.OpAdd {
newLines.WriteString(line.String())
}
}
}
fileName := file.NewName fileName := file.NewName
if fileName == "" { if fileName == "" {
continue continue
@@ -299,14 +293,23 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
return errors.WrapPrefix(err, "couldn't get repo name", 0) return errors.WrapPrefix(err, "couldn't get repo name", 0)
} }
metadata := s.sourceMetadataFunc(fileName, email, hash, when, safeRepo) for _, frag := range file.TextFragments {
chunksChan <- &sources.Chunk{ newLines := bytes.Buffer{}
SourceName: s.sourceName, newLineNumber := frag.NewPosition
SourceID: s.sourceID, for _, line := range frag.Lines {
SourceType: s.sourceType, if line.Op == gitdiff.OpAdd {
SourceMetadata: metadata, newLines.WriteString(strings.ReplaceAll(line.String(), "\n", " ") + "\n")
Data: newLines.Bytes(), }
Verify: s.verify, }
metadata := s.sourceMetadataFunc(fileName, email, hash, when, safeRepo, newLineNumber)
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: newLines.Bytes(),
Verify: s.verify,
}
} }
} }
return nil return nil
@@ -341,7 +344,7 @@ func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunk
continue continue
} }
metadata := s.sourceMetadataFunc( metadata := s.sourceMetadataFunc(
fh, "unstaged", "unstaged", time.Now().String(), safeRepo, fh, "unstaged", "unstaged", time.Now().String(), safeRepo, 0,
) )
fileBuf := bytes.NewBuffer(nil) fileBuf := bytes.NewBuffer(nil)
pkg/sources/git/git_test.go
+15 -13
View File
@@ -149,6 +149,7 @@ func TestSource_Chunks_Integration(t *testing.T) {
type byteCompare struct { type byteCompare struct {
B []byte B []byte
Found bool Found bool
Multi bool
} }
tests := []struct { tests := []struct {
name string name string
@@ -168,17 +169,17 @@ func TestSource_Chunks_Integration(t *testing.T) {
}, },
}, },
expectedChunkData: map[string]*byteCompare{ expectedChunkData: map[string]*byteCompare{
"70001020fab32b1fcf2f1f0e5c66424eae649826-aws": {B: []byte("+[default]\n+aws_access_key_id = AKIAXYZDQCEN4B6JSJQI\n+aws_secret_access_key = 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\n+output = json\n+region = us-east-2\n")}, "70001020fab32b1fcf2f1f0e5c66424eae649826-aws": {B: []byte("+[default] \n+aws_access_key_id = AKIAXYZDQCEN4B6JSJQI \n+aws_secret_access_key = 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 \n+output = json \n+region = us-east-2 \n")},
"a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte("+f\n")}, "a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte("+f \n")},
"07d96d011005fe8296bdd237c13a06a72e96783d-bump": {B: []byte("+ s \n")}, "07d96d011005fe8296bdd237c13a06a72e96783d-bump": {B: []byte("+ s \n")},
"2f251b8c1e72135a375b659951097ec7749d4af9-bump": {B: []byte("+ \n")}, "2f251b8c1e72135a375b659951097ec7749d4af9-bump": {B: []byte("+ \n")},
"e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump": {B: []byte("+oops \n")}, "e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump": {B: []byte("+oops \n")},
"735b52b0eb40610002bb1088e902bd61824eb305-bump": {B: []byte("+oops\n")}, "735b52b0eb40610002bb1088e902bd61824eb305-bump": {B: []byte("+oops \n")},
"ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("+\n")}, "ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("+ \n")},
"27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("+yup, just did that\n+\n+github_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n+\n+oh, goodness. there's another one!")}, "27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("+yup, just did that \n+ \n+github_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\" \n+ \n+oh, goodness. there's another one!\n")},
"8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("+oops might drop a slack token here\n+\n+github_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\"\n+\n+yup, just did that")}, "8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("+oops might drop a slack token here \n+ \n+github_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\" \n+ \n+yup, just did that\n"), Multi: true},
"8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("+blah blaj\n+\n+this is the secret: AKIA2E0A8F3B244C9986\n+\n+okay thank you bye")}, "8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("+blah blaj \n+ \n+this is the secret: AKIA2E0A8F3B244C9986 \n+ \n+okay thank you bye\n"), Multi: true},
"90c75f884c65dc3638ca1610bd9844e668f213c2-aws": {B: []byte("+this is the secret: [Default]\n+Access key Id: AKIAILE3JG6KMS3HZGCA\n+Secret Access Key: 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\n+okay thank you bye\n")}, "90c75f884c65dc3638ca1610bd9844e668f213c2-aws": {B: []byte("+this is the secret: [Default] \n+Access key Id: AKIAILE3JG6KMS3HZGCA \n+Secret Access Key: 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 \n"), Multi: true},
}, },
}, },
} }
@@ -216,8 +217,9 @@ func TestSource_Chunks_Integration(t *testing.T) {
if expectedData, exists := tt.expectedChunkData[key]; !exists { if expectedData, exists := tt.expectedChunkData[key]; !exists {
t.Errorf("A chunk exists that was not expected with key %q", key) t.Errorf("A chunk exists that was not expected with key %q", key)
} else { } else {
(*tt.expectedChunkData[key]).Found = true if bytes.Equal(chunk.Data, expectedData.B) {
if !bytes.Equal(chunk.Data, expectedData.B) { (*tt.expectedChunkData[key]).Found = true
} else if !expectedData.Multi {
t.Errorf("Got %q: %q, which was not expected", key, string(chunk.Data)) t.Errorf("Got %q: %q, which was not expected", key, string(chunk.Data))
} }
} }
pkg/sources/github/github.go
+2 -1
View File
@@ -113,7 +113,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
s.conn = &conn s.conn = &conn
s.git = git.NewGit(s.Type(), s.JobID(), s.SourceID(), s.name, s.verify, runtime.NumCPU(), s.git = git.NewGit(s.Type(), s.JobID(), s.SourceID(), s.name, s.verify, runtime.NumCPU(),
func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData { func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{ return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{ Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{ Github: &source_metadatapb.Github{
@@ -123,6 +123,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
Repository: sanitizer.UTF8(repository), Repository: sanitizer.UTF8(repository),
Link: git.GenerateLink(repository, commit, file), Link: git.GenerateLink(repository, commit, file),
Timestamp: sanitizer.UTF8(timestamp), Timestamp: sanitizer.UTF8(timestamp),
Line: line,
}, },
}, },
} }
pkg/sources/gitlab/gitlab.go
+2 -1
View File
@@ -101,7 +101,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
} }
s.git = git.NewGit(s.Type(), s.JobID(), s.SourceID(), s.name, s.verify, runtime.NumCPU(), s.git = git.NewGit(s.Type(), s.JobID(), s.SourceID(), s.name, s.verify, runtime.NumCPU(),
func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData { func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{ return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Gitlab{ Data: &source_metadatapb.MetaData_Gitlab{
Gitlab: &source_metadatapb.Gitlab{ Gitlab: &source_metadatapb.Gitlab{
@@ -111,6 +111,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
Repository: sanitizer.UTF8(repository), Repository: sanitizer.UTF8(repository),
Link: git.GenerateLink(repository, commit, file), Link: git.GenerateLink(repository, commit, file),
Timestamp: sanitizer.UTF8(timestamp), Timestamp: sanitizer.UTF8(timestamp),
Line: line,
}, },
}, },
} }
proto/source_metadata.proto
+5 -5
View File
@@ -22,7 +22,7 @@ message Bitbucket {
string email = 7; string email = 7;
string link = 8; string link = 8;
string timestamp = 9; string timestamp = 9;
string line = 10; int64 line = 10;
} }
message Buildkite { message Buildkite {
@@ -84,7 +84,7 @@ message Git {
string email = 3; string email = 3;
string repository = 4; string repository = 4;
string timestamp = 5; string timestamp = 5;
string line = 6; int64 line = 6;
} }
message Github { message Github {
@@ -95,7 +95,7 @@ message Github {
string email = 5; string email = 5;
string file = 6; string file = 6;
string timestamp = 7; string timestamp = 7;
string line = 8; int64 line = 8;
} }
message Gitlab { message Gitlab {
@@ -105,7 +105,7 @@ message Gitlab {
string email = 4; string email = 4;
string repository = 5; string repository = 5;
string timestamp = 6; string timestamp = 6;
string line = 7; int64 line = 7;
} }
message GCS { message GCS {
@@ -164,7 +164,7 @@ message Gerrit {
string email = 3; string email = 3;
string project = 4; // projects are what Gerrit calls repositories string project = 4; // projects are what Gerrit calls repositories
string timestamp = 5; string timestamp = 5;
string line = 6; int64 line = 6;
} }
message Test { message Test {