Compare commits

...

3 Commits

Author SHA1 Message Date
Zachary Rice ebe74507b4 remove _ from npm_ and update ahocorasick to our fork 2023-07-25 15:24:14 -05:00
Bill Rich f39303495a Add commitsScanned metrics (#1533)
* Add commitsScanned metrics

* Just keep commit count
2023-07-25 11:31:01 -07:00
ahrav b5b01d3eba [chore] - optimize chunker (#1535)
* Use chunkbytes that includes the size of peek.

* linter.

* continue.

* add TotalChunkSize const.
2023-07-24 19:30:29 -07:00
7 changed files with 52 additions and 19 deletions
+1 -1
View File
@@ -8,7 +8,6 @@ require (
cloud.google.com/go/secretmanager v1.11.1
cloud.google.com/go/storage v1.31.0
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
github.com/BobuSumisu/aho-corasick v1.0.3
github.com/TheZeroSlave/zapsentry v1.17.0
github.com/aws/aws-sdk-go v1.44.83
github.com/bill-rich/disk-buffer-reader v0.1.7
@@ -151,6 +150,7 @@ require (
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/skeema/knownhosts v1.1.1 // indirect
github.com/therootcompany/xz v1.0.1 // indirect
github.com/trufflesecurity/aho-corasick v0.1.0
github.com/ulikunitz/xz v0.5.10 // indirect
github.com/vbatts/tar-split v0.11.3 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
+2 -2
View File
@@ -51,8 +51,6 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g=
github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -447,6 +445,8 @@ github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HP
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
github.com/trufflesecurity/aho-corasick v0.1.0 h1:bwtdjWFkuzDRLWjJi4fCxkBSv2axhYXJryCqsSgMHSg=
github.com/trufflesecurity/aho-corasick v0.1.0/go.mod h1:JKKZLgcVlyWn0LsPezrqEIQPi76i2URv44hC0gJY4Vo=
github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
+1 -1
View File
@@ -29,7 +29,7 @@ var (
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"npm_"}
return []string{"npm"}
}
// FromData will find and optionally verify NpmTokenV2 secrets in a given set of bytes.
+11 -6
View File
@@ -10,7 +10,7 @@ import (
"sync/atomic"
"time"
ahocorasick "github.com/BobuSumisu/aho-corasick"
ahocorasick "github.com/trufflesecurity/aho-corasick"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
@@ -43,7 +43,7 @@ type Engine struct {
// prefilter is a ahocorasick struct used for doing efficient string
// matching given a set of words (keywords from the rules in the config)
prefilter ahocorasick.Trie
prefilter ahocorasick.AhoCorasick
}
type EngineOption func(*Engine)
@@ -150,7 +150,13 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
for _, d := range e.detectors[true] {
keywords = append(keywords, d.Keywords()...)
}
e.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{
AsciiCaseInsensitive: true,
MatchOnlyWholeWords: false,
MatchKind: ahocorasick.StandardMatch,
DFA: false,
})
e.prefilter = builder.Build(keywords)
ctx.Logger().Info("loaded decoders", "count", len(e.decoders))
ctx.Logger().Info("loaded detectors",
@@ -291,8 +297,8 @@ func (e *Engine) detectorWorker(ctx context.Context) {
}
// build a map of all keywords that were matched in the chunk
for _, m := range e.prefilter.MatchString(string(decoded.Data)) {
matchedKeywords[strings.ToLower(m.MatchString())] = struct{}{}
for _, m := range e.prefilter.FindAll(string(decoded.Data)) {
matchedKeywords[strings.ToLower(string(decoded.Data[m.Start():m.End()]))] = struct{}{}
}
for verify, detectorsSet := range e.detectors {
@@ -301,7 +307,6 @@ func (e *Engine) detectorWorker(ctx context.Context) {
for _, kw := range detector.Keywords() {
if _, ok := matchedKeywords[strings.ToLower(kw)]; ok {
chunkContainsKeyword = true
break
}
}
+15 -9
View File
@@ -12,6 +12,8 @@ const (
ChunkSize = 10 * 1024
// PeekSize is the size of the peek into the previous chunk.
PeekSize = 3 * 1024
// TotalChunkSize is the total size of a chunk with peek data.
TotalChunkSize = ChunkSize + PeekSize
)
// Chunker takes a chunk and splits it into chunks of ChunkSize.
@@ -19,27 +21,31 @@ func Chunker(originalChunk *Chunk) chan *Chunk {
chunkChan := make(chan *Chunk)
go func() {
defer close(chunkChan)
if len(originalChunk.Data) <= ChunkSize+PeekSize {
if len(originalChunk.Data) <= TotalChunkSize {
chunkChan <- originalChunk
return
}
r := bytes.NewReader(originalChunk.Data)
reader := bufio.NewReaderSize(bufio.NewReader(r), ChunkSize)
for {
chunkBytes := make([]byte, ChunkSize)
chunkBytes := make([]byte, TotalChunkSize)
chunk := *originalChunk
chunkBytes = chunkBytes[:ChunkSize]
n, err := reader.Read(chunkBytes)
if err != nil && !errors.Is(err, io.EOF) {
break
}
if n == 0 {
if errors.Is(err, io.EOF) {
break
}
continue
}
peekData, _ := reader.Peek(PeekSize)
chunk.Data = append(chunkBytes[:n], peekData...)
if n > 0 {
chunkChan <- &chunk
}
if errors.Is(err, io.EOF) {
break
}
copy(chunkBytes[n:], peekData)
chunk.Data = chunkBytes[:n+len(peekData)]
chunkChan <- &chunk
}
}()
return chunkChan
+12
View File
@@ -72,3 +72,15 @@ func TestChunker(t *testing.T) {
}
}
func BenchmarkChunker(b *testing.B) {
data := bytes.Repeat([]byte("a"), ChunkSize*100)
chunk := &Chunk{
Data: data,
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
for range Chunker(chunk) {
}
}
}
+10
View File
@@ -51,9 +51,14 @@ type Git struct {
jobID int64
sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData
verify bool
metrics metrics
concurrency *semaphore.Weighted
}
type metrics struct {
commitsScanned int
}
func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData,
) *Git {
@@ -339,6 +344,10 @@ func CloneRepoUsingSSH(ctx context.Context, gitUrl string, args ...string) (stri
return CloneRepo(ctx, userInfo, gitUrl, args...)
}
func (s *Git) CommitsScanned() int {
return s.metrics.commitsScanned
}
func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
if err := GitCmdCheck(); err != nil {
return err
@@ -371,6 +380,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
break
}
depth++
s.metrics.commitsScanned++
logger.V(5).Info("scanning commit", "commit", commit.Hash)
for _, diff := range commit.Diffs {
if !scanOptions.Filter.Pass(diff.PathB) {