actions/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Include line numbers in git metadata (#97)

Browse Source 
* Include line numbers in git metadata

* Update tests for fragments and line numbers
This commit is contained in:
Bill Rich
2022-03-22 08:27:15 -08:00
committed by GitHub
parent 0ea3b93238
commit c2b4f0bc39
12 changed files with 149 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hack/snifftest/main.go
+1 -1
View File
@@ -170,7 +170,7 @@ func main() {
log.Infof("cloned %s", r)
s := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "snifftest", false, runtime.NumCPU(),
func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData {
func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Git{
Git: &source_metadatapb.Git{
main.go
+1 -2
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"gopkg.in/alecthomas/kingpin.v2"
"log"
"net/http"
_ "net/http/pprof"
@@ -17,8 +18,6 @@ import (
"github.com/gorilla/mux"
"github.com/sirupsen/logrus"
"gopkg.in/alecthomas/kingpin.v2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
pkg/engine/engine.go
+58 -1
View File
@@ -1,7 +1,10 @@
package engine
import (
"bytes"
"context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"runtime"
"strings"
"sync"
@@ -174,7 +177,13 @@ func (e *Engine) detectorWorker(ctx context.Context) {
continue
}
for _, result := range results {
e.results <- detectors.CopyMetadata(chunk, result)
targetChunk := chunk
if isGitSource(chunk.SourceType) {
copyChunk := *chunk
targetChunk = &copyChunk
SetLineNumber(&copyChunk, &result)
}
e.results <- detectors.CopyMetadata(targetChunk, result)
}
if len(results) > 0 {
elasped := time.Since(start)
@@ -196,3 +205,51 @@ func (e *Engine) detectorWorker(ctx context.Context) {
atomic.AddUint64(&e.chunksScanned, 1)
}
}
// gitSources is a list of sources that utilize the Git source. It is stored this way because slice consts are not
// supported.
func gitSources() []sourcespb.SourceType {
return []sourcespb.SourceType{
sourcespb.SourceType_SOURCE_TYPE_GIT,
sourcespb.SourceType_SOURCE_TYPE_GITHUB,
sourcespb.SourceType_SOURCE_TYPE_GITLAB,
sourcespb.SourceType_SOURCE_TYPE_BITBUCKET,
sourcespb.SourceType_SOURCE_TYPE_GERRIT,
sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,
sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,
}
}
func isGitSource(sourceType sourcespb.SourceType) bool {
for _, i := range gitSources() {
if i == sourceType {
return true
}
}
return false
}
func SetLineNumber(chunk *sources.Chunk, result *detectors.Result) {
var startingLine *int64
switch metadata := chunk.SourceMetadata.GetData().(type) {
case *source_metadatapb.MetaData_Git:
startingLine = &metadata.Git.Line
case *source_metadatapb.MetaData_Github:
startingLine = &metadata.Github.Line
case *source_metadatapb.MetaData_Gitlab:
startingLine = &metadata.Gitlab.Line
case *source_metadatapb.MetaData_Bitbucket:
startingLine = &metadata.Bitbucket.Line
case *source_metadatapb.MetaData_Gerrit:
startingLine = &metadata.Gerrit.Line
}
lines := bytes.Split(chunk.Data, []byte("\n"))
for i, line := range lines {
if bytes.Contains(line, result.Raw) {
*startingLine = *startingLine + int64(i)
return
}
}
return
}
pkg/engine/git.go
+3 -2
View File
@@ -86,8 +86,8 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,
}
scanOptions := git.NewScanOptions(opts...)
gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "local", true, runtime.NumCPU(),
func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData {
gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "trufflehog - git", true, runtime.NumCPU(),
func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Git{
Git: &source_metadatapb.Git{
@@ -96,6 +96,7 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,
Email: email,
Repository: repository,
Timestamp: timestamp,
Line: line,
},
},
}
pkg/engine/git_test.go
+18 -10
View File
@@ -11,6 +11,11 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
)
type expResult struct {
B string
LineNumber int64
}
func TestGitEngine(t *testing.T) {
repoUrl := "https://github.com/dustin-decker/secretsandstuff.git"
path, _, err := git.PrepareRepo(repoUrl)
@@ -22,7 +27,7 @@ func TestGitEngine(t *testing.T) {
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
type testProfile struct {
expected map[string]string
expected map[string]expResult
branch string
base string
maxDepth int
@@ -30,17 +35,17 @@ func TestGitEngine(t *testing.T) {
}
for tName, tTest := range map[string]testProfile{
"all_secrets": {
expected: map[string]string{
"70001020fab32b1fcf2f1f0e5c66424eae649826": "AKIAXYZDQCEN4B6JSJQI",
"90c75f884c65dc3638ca1610bd9844e668f213c2": "AKIAILE3JG6KMS3HZGCA",
"8afb0ecd4998b1179e428db5ebbcdc8221214432": "369963c1434c377428ca8531fbc46c0c43d037a0",
"27fbead3bf883cdb7de9d7825ed401f28f9398f1": "ffc7e0f9400fb6300167009e42d2f842cd7956e2",
expected: map[string]expResult{
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
"90c75f884c65dc3638ca1610bd9844e668f213c2": {"AKIAILE3JG6KMS3HZGCA", 4},
"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3},
"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7},
},
filter: common.FilterEmpty(),
},
"base_commit": {
expected: map[string]string{
"70001020fab32b1fcf2f1f0e5c66424eae649826": "AKIAXYZDQCEN4B6JSJQI",
expected: map[string]expResult{
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
},
filter: common.FilterEmpty(),
base: "2f251b8c1e72135a375b659951097ec7749d4af9",
@@ -56,8 +61,11 @@ func TestGitEngine(t *testing.T) {
for result := range e.ResultsChan() {
switch meta := result.SourceMetadata.GetData().(type) {
case *source_metadatapb.MetaData_Git:
if tTest.expected[meta.Git.Commit] != string(result.Raw) {
t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit])
if tTest.expected[meta.Git.Commit].B != string(result.Raw) {
t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit].B)
}
if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
}
}
resultCount++
pkg/output/plain.go
+1 -1
View File
@@ -42,7 +42,7 @@ func PrintPlainOutput(r *detectors.ResultWithMetadata) {
printer.Printf("Raw result: %s\n", whitePrinter.Sprint(out.Raw))
for _, data := range meta {
for k, v := range data {
printer.Printf("%s: %s\n", strings.Title(k), v)
printer.Printf("%s: %v\n", strings.Title(k), v)
}
}
fmt.Println("")
pkg/pb/source_metadatapb/source_metadata.pb.go
+20 -20
View File
@@ -113,7 +113,7 @@ type Bitbucket struct {
Email string `protobuf:"bytes,7,opt,name=email,proto3" json:"email,omitempty"`
Link string `protobuf:"bytes,8,opt,name=link,proto3" json:"link,omitempty"`
Timestamp string `protobuf:"bytes,9,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
Line string `protobuf:"bytes,10,opt,name=line,proto3" json:"line,omitempty"`
Line int64 `protobuf:"varint,10,opt,name=line,proto3" json:"line,omitempty"`
}
func (x *Bitbucket) Reset() {
@@ -211,11 +211,11 @@ func (x *Bitbucket) GetTimestamp() string {
return ""
}
func (x *Bitbucket) GetLine() string {
func (x *Bitbucket) GetLine() int64 {
if x != nil {
return x.Line
}
return ""
return 0
}
type Buildkite struct {
@@ -742,7 +742,7 @@ type Git struct {
Email string `protobuf:"bytes,3,opt,name=email,proto3" json:"email,omitempty"`
Repository string `protobuf:"bytes,4,opt,name=repository,proto3" json:"repository,omitempty"`
Timestamp string `protobuf:"bytes,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
Line string `protobuf:"bytes,6,opt,name=line,proto3" json:"line,omitempty"`
Line int64 `protobuf:"varint,6,opt,name=line,proto3" json:"line,omitempty"`
}
func (x *Git) Reset() {
@@ -812,11 +812,11 @@ func (x *Git) GetTimestamp() string {
return ""
}
func (x *Git) GetLine() string {
func (x *Git) GetLine() int64 {
if x != nil {
return x.Line
}
return ""
return 0
}
type Github struct {
@@ -831,7 +831,7 @@ type Github struct {
Email string `protobuf:"bytes,5,opt,name=email,proto3" json:"email,omitempty"`
File string `protobuf:"bytes,6,opt,name=file,proto3" json:"file,omitempty"`
Timestamp string `protobuf:"bytes,7,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
Line string `protobuf:"bytes,8,opt,name=line,proto3" json:"line,omitempty"`
Line int64 `protobuf:"varint,8,opt,name=line,proto3" json:"line,omitempty"`
}
func (x *Github) Reset() {
@@ -915,11 +915,11 @@ func (x *Github) GetTimestamp() string {
return ""
}
func (x *Github) GetLine() string {
func (x *Github) GetLine() int64 {
if x != nil {
return x.Line
}
return ""
return 0
}
type Gitlab struct {
@@ -933,7 +933,7 @@ type Gitlab struct {
Email string `protobuf:"bytes,4,opt,name=email,proto3" json:"email,omitempty"`
Repository string `protobuf:"bytes,5,opt,name=repository,proto3" json:"repository,omitempty"`
Timestamp string `protobuf:"bytes,6,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
Line string `protobuf:"bytes,7,opt,name=line,proto3" json:"line,omitempty"`
Line int64 `protobuf:"varint,7,opt,name=line,proto3" json:"line,omitempty"`
}
func (x *Gitlab) Reset() {
@@ -1010,11 +1010,11 @@ func (x *Gitlab) GetTimestamp() string {
return ""
}
func (x *Gitlab) GetLine() string {
func (x *Gitlab) GetLine() int64 {
if x != nil {
return x.Line
}
return ""
return 0
}
type GCS struct {
@@ -1517,7 +1517,7 @@ type Gerrit struct {
Email string `protobuf:"bytes,3,opt,name=email,proto3" json:"email,omitempty"`
Project string `protobuf:"bytes,4,opt,name=project,proto3" json:"project,omitempty"` // projects are what Gerrit calls repositories
Timestamp string `protobuf:"bytes,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
Line string `protobuf:"bytes,6,opt,name=line,proto3" json:"line,omitempty"`
Line int64 `protobuf:"varint,6,opt,name=line,proto3" json:"line,omitempty"`
}
func (x *Gerrit) Reset() {
@@ -1587,11 +1587,11 @@ func (x *Gerrit) GetTimestamp() string {
return ""
}
func (x *Gerrit) GetLine() string {
func (x *Gerrit) GetLine() int64 {
if x != nil {
return x.Line
}
return ""
return 0
}
type Test struct {
@@ -2073,7 +2073,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b,
0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x09, 0x20,
0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12,
0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69,
0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69,
0x6e, 0x65, 0x22, 0xa4, 0x01, 0x0a, 0x09, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65,
0x12, 0x10, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f,
0x72, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x02,
@@ -2140,7 +2140,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69,
0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74,
0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65,
0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xcc, 0x01, 0x0a,
0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xcc, 0x01, 0x0a,
0x06, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x75,
0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75,
@@ -2153,7 +2153,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d,
0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69,
0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18,
0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xb0, 0x01, 0x0a, 0x06,
0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xb0, 0x01, 0x0a, 0x06,
0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x12,
0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69,
@@ -2164,7 +2164,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09,
0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52,
0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69,
0x6e, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x5b,
0x6e, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x5b,
0x0a, 0x03, 0x47, 0x43, 0x53, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a,
0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c,
@@ -2224,7 +2224,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74,
0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09,
0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e,
0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x1a, 0x0a,
0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x1a, 0x0a,
0x04, 0x54, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x81, 0x01, 0x0a, 0x07, 0x4a, 0x65,
0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74,
pkg/sources/git/git.go
+23 -20
View File
@@ -47,14 +47,14 @@ type Git struct {
sourceName string
sourceID int64
jobID int64
sourceMetadataFunc func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData
sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData
verify bool
// sem is used to limit concurrency
sem *semaphore.Weighted
}
func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
sourceMetadataFunc func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData,
sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData,
) *Git {
return &Git{
sourceType: sourceType,
@@ -102,7 +102,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
s.conn = &conn
s.git = NewGit(s.Type(), s.jobId, s.sourceId, s.name, s.verify, runtime.NumCPU(),
func(file, email, commit, repository, timestamp string) *source_metadatapb.MetaData {
func(file, email, commit, repository, timestamp string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Git{
Git: &source_metadatapb.Git{
@@ -111,6 +111,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
Email: sanitizer.UTF8(email),
Repository: sanitizer.UTF8(repository),
Timestamp: sanitizer.UTF8(timestamp),
Line: line,
},
},
}
@@ -269,14 +270,7 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
if !scanOptions.Filter.Pass(file.NewName) {
continue
}
newLines := bytes.Buffer{}
for _, frag := range file.TextFragments {
for _, line := range frag.Lines {
if line.Op == gitdiff.OpAdd {
newLines.WriteString(line.String())
}
}
}
fileName := file.NewName
if fileName == "" {
continue
@@ -299,14 +293,23 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
return errors.WrapPrefix(err, "couldn't get repo name", 0)
}
metadata := s.sourceMetadataFunc(fileName, email, hash, when, safeRepo)
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: newLines.Bytes(),
Verify: s.verify,
for _, frag := range file.TextFragments {
newLines := bytes.Buffer{}
newLineNumber := frag.NewPosition
for _, line := range frag.Lines {
if line.Op == gitdiff.OpAdd {
newLines.WriteString(strings.ReplaceAll(line.String(), "\n", " ") + "\n")
}
}
metadata := s.sourceMetadataFunc(fileName, email, hash, when, safeRepo, newLineNumber)
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: newLines.Bytes(),
Verify: s.verify,
}
}
}
return nil
@@ -341,7 +344,7 @@ func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunk
continue
}
metadata := s.sourceMetadataFunc(
fh, "unstaged", "unstaged", time.Now().String(), safeRepo,
fh, "unstaged", "unstaged", time.Now().String(), safeRepo, 0,
)
fileBuf := bytes.NewBuffer(nil)
pkg/sources/git/git_test.go
+15 -13
View File
@@ -149,6 +149,7 @@ func TestSource_Chunks_Integration(t *testing.T) {
type byteCompare struct {
B []byte
Found bool
Multi bool
}
tests := []struct {
name string
@@ -168,17 +169,17 @@ func TestSource_Chunks_Integration(t *testing.T) {
},
},
expectedChunkData: map[string]*byteCompare{
"70001020fab32b1fcf2f1f0e5c66424eae649826-aws": {B: []byte("+[default]\n+aws_access_key_id = AKIAXYZDQCEN4B6JSJQI\n+aws_secret_access_key = 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\n+output = json\n+region = us-east-2\n")},
"a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte("+f\n")},
"07d96d011005fe8296bdd237c13a06a72e96783d-bump": {B: []byte("+ s \n")},
"2f251b8c1e72135a375b659951097ec7749d4af9-bump": {B: []byte("+ \n")},
"e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump": {B: []byte("+oops \n")},
"735b52b0eb40610002bb1088e902bd61824eb305-bump": {B: []byte("+oops\n")},
"ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("+\n")},
"27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("+yup, just did that\n+\n+github_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n+\n+oh, goodness. there's another one!")},
"8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("+oops might drop a slack token here\n+\n+github_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\"\n+\n+yup, just did that")},
"8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("+blah blaj\n+\n+this is the secret: AKIA2E0A8F3B244C9986\n+\n+okay thank you bye")},
"90c75f884c65dc3638ca1610bd9844e668f213c2-aws": {B: []byte("+this is the secret: [Default]\n+Access key Id: AKIAILE3JG6KMS3HZGCA\n+Secret Access Key: 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\n+okay thank you bye\n")},
"70001020fab32b1fcf2f1f0e5c66424eae649826-aws": {B: []byte("+[default] \n+aws_access_key_id = AKIAXYZDQCEN4B6JSJQI \n+aws_secret_access_key = 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 \n+output = json \n+region = us-east-2 \n")},
"a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte("+f \n")},
"07d96d011005fe8296bdd237c13a06a72e96783d-bump": {B: []byte("+ s \n")},
"2f251b8c1e72135a375b659951097ec7749d4af9-bump": {B: []byte("+ \n")},
"e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump": {B: []byte("+oops \n")},
"735b52b0eb40610002bb1088e902bd61824eb305-bump": {B: []byte("+oops \n")},
"ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("+ \n")},
"27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("+yup, just did that \n+ \n+github_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\" \n+ \n+oh, goodness. there's another one!\n")},
"8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("+oops might drop a slack token here \n+ \n+github_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\" \n+ \n+yup, just did that\n"), Multi: true},
"8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("+blah blaj \n+ \n+this is the secret: AKIA2E0A8F3B244C9986 \n+ \n+okay thank you bye\n"), Multi: true},
"90c75f884c65dc3638ca1610bd9844e668f213c2-aws": {B: []byte("+this is the secret: [Default] \n+Access key Id: AKIAILE3JG6KMS3HZGCA \n+Secret Access Key: 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 \n"), Multi: true},
},
},
}
@@ -216,8 +217,9 @@ func TestSource_Chunks_Integration(t *testing.T) {
if expectedData, exists := tt.expectedChunkData[key]; !exists {
t.Errorf("A chunk exists that was not expected with key %q", key)
} else {
(*tt.expectedChunkData[key]).Found = true
if !bytes.Equal(chunk.Data, expectedData.B) {
if bytes.Equal(chunk.Data, expectedData.B) {
(*tt.expectedChunkData[key]).Found = true
} else if !expectedData.Multi {
t.Errorf("Got %q: %q, which was not expected", key, string(chunk.Data))
}
}
pkg/sources/github/github.go
+2 -1
View File
@@ -113,7 +113,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
s.conn = &conn
s.git = git.NewGit(s.Type(), s.JobID(), s.SourceID(), s.name, s.verify, runtime.NumCPU(),
func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData {
func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
@@ -123,6 +123,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
Repository: sanitizer.UTF8(repository),
Link: git.GenerateLink(repository, commit, file),
Timestamp: sanitizer.UTF8(timestamp),
Line: line,
},
},
}
pkg/sources/gitlab/gitlab.go
+2 -1
View File
@@ -101,7 +101,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
}
s.git = git.NewGit(s.Type(), s.JobID(), s.SourceID(), s.name, s.verify, runtime.NumCPU(),
func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData {
func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Gitlab{
Gitlab: &source_metadatapb.Gitlab{
@@ -111,6 +111,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
Repository: sanitizer.UTF8(repository),
Link: git.GenerateLink(repository, commit, file),
Timestamp: sanitizer.UTF8(timestamp),
Line: line,
},
},
}
proto/source_metadata.proto
+5 -5
View File
@@ -22,7 +22,7 @@ message Bitbucket {
string email = 7;
string link = 8;
string timestamp = 9;
string line = 10;
int64 line = 10;
}
message Buildkite {
@@ -84,7 +84,7 @@ message Git {
string email = 3;
string repository = 4;
string timestamp = 5;
string line = 6;
int64 line = 6;
}
message Github {
@@ -95,7 +95,7 @@ message Github {
string email = 5;
string file = 6;
string timestamp = 7;
string line = 8;
int64 line = 8;
}
message Gitlab {
@@ -105,7 +105,7 @@ message Gitlab {
string email = 4;
string repository = 5;
string timestamp = 6;
string line = 7;
int64 line = 7;
}
message GCS {
@@ -164,7 +164,7 @@ message Gerrit {
string email = 3;
string project = 4; // projects are what Gerrit calls repositories
string timestamp = 5;
string line = 6;
int64 line = 6;
}
message Test {