Compare commits

..

10 Commits

Author SHA1 Message Date
Dustin Decker 065984ee58 review updates 2023-10-18 09:05:07 -07:00
Miccah Castorina fe90effec1 Continue chunking on error 2023-10-16 16:42:33 -07:00
Miccah Castorina e5a34a9809 Remove jobPage loop
ListByBuild does not support pagination, so this was infinitely
repeating. https://developer.travis-ci.com/resource/jobs#find
2023-10-16 16:21:56 -07:00
Dustin Decker ec3c59d025 update test to use sourcestest 2023-10-16 14:00:22 -07:00
Dustin Decker c99e0b93f7 Merge branch 'main' of github.com:trufflesecurity/trufflehog into travis-scanning 2023-10-16 13:42:35 -07:00
Miccah 03dc7cb68d [chore] Add SourceUnitEnumChunker filesystem tests (#1873)
* [chore] Add SourceUnitEnumChunker filesystem tests

* Ensure reported units are exactly what is expected
2023-10-16 10:42:18 -07:00
Corben Leo 072e1f9dcf Detector-Competition-Fix: Add Personal Access Tokens (API Tokens Depr… (#1871)
* Detector-Competition-Fix: Add Personal Access Tokens (API Tokens Depreciation)

* fix(test): fix test debug msg

* remove print
2023-10-16 08:17:12 -05:00
ahrav 5c721d1a73 [bug] - Don't modify global client var (#1890)
* Create a new client within the verify block

* remove unused var
2023-10-13 12:32:21 -07:00
s.shivasurya 040167178c added cody gateway token detection code (#1883)
* added cody gateway token detection code

* resolved conflict
2023-10-13 09:09:04 -06:00
Dustin Decker b960231155 Add TravisCI source 2023-10-08 19:44:13 -07:00
31 changed files with 2678 additions and 1382 deletions
+1
View File
@@ -65,6 +65,7 @@ require (
github.com/prometheus/client_golang v1.16.0
github.com/rabbitmq/amqp091-go v1.8.1
github.com/sergi/go-diff v1.3.1
github.com/shuheiktgw/go-travis v0.3.1
github.com/snowflakedb/gosnowflake v1.6.23
github.com/stretchr/testify v1.8.4
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
+3
View File
@@ -369,6 +369,7 @@ github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5Px
github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=
github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
@@ -605,6 +606,8 @@ github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
github.com/shuheiktgw/go-travis v0.3.1 h1:SAT16mi77ccqogOslnXxBXzXbpeyChaIYUwi2aJpVZY=
github.com/shuheiktgw/go-travis v0.3.1/go.mod h1:avnFFDqJDdRHwlF9tgqvYi3asQCm/HGL8aLxYiKa4Yg=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+7
View File
@@ -139,6 +139,9 @@ var (
dockerScan = cli.Command("docker", "Scan Docker Image")
dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, otherwise a image registry is assumed.").Required().Strings()
travisCiScan = cli.Command("travisci", "Scan TravisCI")
travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()
)
func init() {
@@ -498,6 +501,10 @@ func run(state overseer.State) {
if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
logFatal(err, "Failed to scan CircleCI.")
}
case travisCiScan.FullCommand():
if err := e.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
logFatal(err, "Failed to scan TravisCI.")
}
case gcsScan.FullCommand():
cfg := sources.GCSConfig{
ProjectID: *gcsProjectID,
+2 -4
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"encoding/base64"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
@@ -26,8 +25,7 @@ func init() {
}
}
func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_BASE64}
func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk {
encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, 20, b64CharsetMapping, b64EndChars)
decodedSubstrings := make(map[string][]byte)
@@ -63,7 +61,7 @@ func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
}
result.Write(chunk.Data[start:])
chunk.Data = result.Bytes()
return decodableChunk
return chunk
}
return nil
+1 -9
View File
@@ -1,7 +1,6 @@
package decoders
import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
@@ -14,15 +13,8 @@ func DefaultDecoders() []Decoder {
}
}
// DecodableChunk is a chunk that includes the type of decoder used.
// This allows us to avoid a type assertion on each decoder.
type DecodableChunk struct {
*sources.Chunk
DecoderType detectorspb.DecoderType
}
type Decoder interface {
FromChunk(chunk *sources.Chunk) *DecodableChunk
FromChunk(chunk *sources.Chunk) *sources.Chunk
}
// Fuzz is an entrypoint for go-fuzz, which is an AFL-style fuzzing tool.
+2 -4
View File
@@ -5,24 +5,22 @@ import (
"encoding/binary"
"unicode/utf8"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
type UTF16 struct{}
func (d *UTF16) FromChunk(chunk *sources.Chunk) *DecodableChunk {
func (d *UTF16) FromChunk(chunk *sources.Chunk) *sources.Chunk {
if chunk == nil || len(chunk.Data) == 0 {
return nil
}
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_UTF16}
if utf16Data, err := utf16ToUTF8(chunk.Data); err == nil {
if len(utf16Data) == 0 {
return nil
}
chunk.Data = utf16Data
return decodableChunk
return chunk
}
return nil
+3 -6
View File
@@ -4,25 +4,22 @@ import (
"bytes"
"unicode/utf8"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
type UTF8 struct{}
func (d *UTF8) FromChunk(chunk *sources.Chunk) *DecodableChunk {
func (d *UTF8) FromChunk(chunk *sources.Chunk) *sources.Chunk {
if chunk == nil || len(chunk.Data) == 0 {
return nil
}
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_PLAIN}
if !utf8.Valid(chunk.Data) {
chunk.Data = extractSubstrings(chunk.Data)
return decodableChunk
return chunk
}
return decodableChunk
return chunk
}
// extractSubstrings performs similarly to the strings binutil,
+27 -2
View File
@@ -2,6 +2,7 @@ package airtableapikey
import (
"context"
"encoding/json"
"fmt"
"net/http"
"regexp"
@@ -20,8 +21,9 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
appPat = regexp.MustCompile(`(app[a-zA-Z0-9_-]{14})`) // could be part of url
keyPat = regexp.MustCompile(`\b(key[a-zA-Z0-9_-]{14})\b`)
appPat = regexp.MustCompile(`(app[a-zA-Z0-9_-]{14})`) // could be part of url
keyPat = regexp.MustCompile(`\b(key[a-zA-Z0-9_-]{14})\b`)
personalPat = regexp.MustCompile(`(\bpat[[:alnum:]]{14}\.[[:alnum:]]{64}\b)`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -30,17 +32,31 @@ func (s Scanner) Keywords() []string {
return []string{"airtable"}
}
type response struct {
Error struct {
Type string `json:"type"`
Message string `json:"message"`
} `json:"error"`
}
// FromData will find and optionally verify AirtableApiKey secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
appMatches := appPat.FindAllStringSubmatch(dataStr, -1)
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
personalKeyMatches := personalPat.FindAllStringSubmatch(dataStr, -1)
if len(keyMatches) == 0 {
keyMatches = personalKeyMatches
}
for _, keyMatch := range keyMatches {
if len(keyMatch) != 2 {
continue
}
keyRes := strings.TrimSpace(keyMatch[1])
for _, appMatch := range appMatches {
@@ -67,6 +83,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 403 {
var resp response
if err = json.NewDecoder(res.Body).Decode(&resp); err == nil {
// check if the error is due to invalid permissions or model not found
if resp.Error.Type == "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND" {
// The key is verified as it works, but the user must enumerate the tables or permissions for the key.
s1.Verified = true
}
}
} else {
if detectors.IsKnownFalsePositive(keyRes, detectors.DefaultFalsePositives, true) {
continue
@@ -9,9 +9,10 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -99,8 +100,9 @@ func TestAirtableApiKey_FromChunk(t *testing.T) {
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("AirtableApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Airtable.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
@@ -0,0 +1,91 @@
package sourcegraphcody
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b(slk_[a-f0-9]{64})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"slk_"}
}
// FromData will find and optionally verify Sourcegraphcody secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_SourcegraphCody,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://cody-gateway.sourcegraph.com/v1/limits", nil)
req.Header.Add("Authorization", "Bearer "+resMatch)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 403 || res.StatusCode == 401 {
// 403 when the gateway is not configured to accept the token,
// 401 when the token is invalid.
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_SourcegraphCody
}
@@ -0,0 +1,162 @@
//go:build detectors
// +build detectors
package sourcegraphcody
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestSourcegraphcody_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("SOURCEGRAPHCODY")
inactiveSecret := testSecrets.MustGetField("SOURCEGRAPHCODY_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SourcegraphCody,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SourcegraphCody,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SourcegraphCody,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SourcegraphCody,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Sourcegraphcody.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Sourcegraphcody.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+1 -2
View File
@@ -23,7 +23,6 @@ var (
identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive.
sidPat = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`)
keyPat = regexp.MustCompile(`\b[0-9a-f]{32}\b`)
client = common.SaneHttpClient()
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -59,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
client = s.client
client := s.client
if client == nil {
client = defaultClient
}
-112
View File
@@ -1,112 +0,0 @@
package engine
import (
"strings"
ahocorasick "github.com/BobuSumisu/aho-corasick"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
// detectorKey is used to identify a detector in the keywordsToDetectors map.
// Multiple detectors can have the same detector type but different versions.
// This allows us to identify a detector by its type and version.
type detectorKey struct {
detectorType detectorspb.DetectorType
version int
}
// detectorInfo is used to store a detector and whether it should be verified.
type detectorInfo struct {
detectors.Detector
shouldVerify bool
}
// AhoCorasickCore encapsulates the operations and data structures used for keyword matching via the
// Aho-Corasick algorithm. It is responsible for constructing and managing the trie for efficient
// substring searches, as well as mapping keywords to their associated detectors for rapid lookups.
type AhoCorasickCore struct {
// prefilter is a ahocorasick struct used for doing efficient string
// matching given a set of words. (keywords from the rules in the config)
prefilter ahocorasick.Trie
// Maps for efficient lookups during detection.
detectorTypeToDetectorInfo map[detectorKey]detectorInfo
detectors map[bool][]detectors.Detector
keywordsToDetectors map[string][]detectorKey
}
// NewAhoCorasickCore allocates and initializes a new instance of AhoCorasickCore.
// It creates an empty keyword-to-detectors map for future string matching operations.
// The map detectorTypeToDetectorInfo is pre-allocated based on the size of detectors
// provided, for efficient storage and lookup of detector information.
func NewAhoCorasickCore(detectors map[bool][]detectors.Detector) *AhoCorasickCore {
return &AhoCorasickCore{
keywordsToDetectors: make(map[string][]detectorKey),
detectors: detectors,
detectorTypeToDetectorInfo: make(map[detectorKey]detectorInfo, len(detectors[true])+len(detectors[false])),
}
}
// Setup initializes the internal state of AhoCorasickCore to prepare it for keyword matching.
// This involves pre-filtering setup and lookup optimization, critical for the engine's performance.
func (ac *AhoCorasickCore) Setup(ctx context.Context) {
// Prepare maps for fast detector lookups, instead of scanning through an array of detectors for every chunk.
var keywords []string
for verify, detectorsSet := range ac.detectors {
for _, d := range detectorsSet {
key := createDetectorKey(d)
ac.detectorTypeToDetectorInfo[key] = detectorInfo{Detector: d, shouldVerify: verify}
keywords = ac.extractAndMapKeywords(d, key, keywords)
}
}
// Use the Ahocorasick algorithm to create a trie structure for efficient keyword matching.
// This ensures that we can rapidly match against a vast set of keywords without individually comparing each one.
ac.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
ctx.Logger().V(4).Info("AhoCorasickCore lookups built")
ctx.Logger().V(4).Info("AhoCorasickCore Setup complete")
}
// createDetectorKey creates a unique key for each detector. This key based on type and version,
// it ensures faster lookups and reduces redundancy in our main detector store.
func createDetectorKey(d detectors.Detector) detectorKey {
detectorType := d.Type()
var version int
if v, ok := d.(detectors.Versioner); ok {
version = v.Version()
}
return detectorKey{detectorType: detectorType, version: version}
}
// extractAndMapKeywords captures keywords associated with each detector and maps them.
// This allows us to quickly determine which detectors are relevant based on the presence of certain keywords.
func (ac *AhoCorasickCore) extractAndMapKeywords(d detectors.Detector, key detectorKey, keywords []string) []string {
for _, kw := range d.Keywords() {
kwLower := strings.ToLower(kw)
keywords = append(keywords, kwLower)
ac.keywordsToDetectors[kwLower] = append(ac.keywordsToDetectors[kwLower], key)
}
return keywords
}
// MatchString performs a string match using the Aho-Corasick algorithm, returning an array of matches.
// Designed for internal use within the AhoCorasickCore component.
func (ac *AhoCorasickCore) MatchString(input string) []*ahocorasick.Match {
return ac.prefilter.MatchString(strings.ToLower(input))
}
// PopulateDetectorsByMatch populates the given detectorMap based on the Aho-Corasick match results.
// This method is designed to reuse the same map for performance optimization,
// reducing the need for repeated allocations within each detector worker in the engine.
func (ac *AhoCorasickCore) PopulateDetectorsByMatch(match *ahocorasick.Match, detectors map[detectorspb.DetectorType]detectorInfo) bool {
matchedKeys, ok := ac.keywordsToDetectors[match.MatchString()]
if !ok {
return false
}
for _, key := range matchedKeys {
detectors[key.detectorType] = ac.detectorTypeToDetectorInfo[key]
}
return true
}
+2
View File
@@ -8,6 +8,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraph"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraphcody"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
@@ -1552,6 +1553,7 @@ func DefaultDetectors() []detectors.Detector {
web3storage.Scanner{},
&ramp.Scanner{},
&anthropic.Scanner{},
&sourcegraphcody.Scanner{},
}
}
+111 -98
View File
@@ -3,11 +3,14 @@ package engine
import (
"bytes"
"fmt"
"reflect"
"runtime"
"strings"
"sync"
"sync/atomic"
"time"
ahocorasick "github.com/BobuSumisu/aho-corasick"
lru "github.com/hashicorp/golang-lru"
"google.golang.org/protobuf/proto"
@@ -63,8 +66,9 @@ type Engine struct {
onlyVerified bool
printAvgDetectorTime bool
// ahoCorasickHandler manages the Aho-Corasick trie and related keyword lookups.
ahoCorasickCore *AhoCorasickCore
// prefilter is a ahocorasick struct used for doing efficient string
// matching given a set of words (keywords from the rules in the config)
prefilter ahocorasick.Trie
// Engine synchronization primitives.
sourceManager *sources.SourceManager
@@ -89,10 +93,9 @@ type Engine struct {
dedupeCache *lru.Cache
}
// Option is used to configure the engine during initialization using functional options.
type Option func(*Engine)
type EngineOption func(*Engine)
func WithConcurrency(concurrency uint8) Option {
func WithConcurrency(concurrency uint8) EngineOption {
return func(e *Engine) {
e.concurrency = concurrency
}
@@ -100,7 +103,7 @@ func WithConcurrency(concurrency uint8) Option {
const ignoreTag = "trufflehog:ignore"
func WithDetectors(verify bool, d ...detectors.Detector) Option {
func WithDetectors(verify bool, d ...detectors.Detector) EngineOption {
return func(e *Engine) {
if e.detectors == nil {
e.detectors = make(map[bool][]detectors.Detector)
@@ -113,7 +116,7 @@ func WithDetectors(verify bool, d ...detectors.Detector) Option {
}
}
func WithDecoders(decoders ...decoders.Decoder) Option {
func WithDecoders(decoders ...decoders.Decoder) EngineOption {
return func(e *Engine) {
e.decoders = decoders
}
@@ -121,14 +124,14 @@ func WithDecoders(decoders ...decoders.Decoder) Option {
// WithFilterUnverified sets the filterUnverified flag on the engine. If set to
// true, the engine will only return the first unverified result for a chunk for a detector.
func WithFilterUnverified(filter bool) Option {
func WithFilterUnverified(filter bool) EngineOption {
return func(e *Engine) {
e.filterUnverified = filter
}
}
// WithFilterEntropy filters out unverified results using Shannon entropy.
func WithFilterEntropy(entropy float64) Option {
func WithFilterEntropy(entropy float64) EngineOption {
return func(e *Engine) {
if entropy > 0 {
e.filterEntropy = &entropy
@@ -138,7 +141,7 @@ func WithFilterEntropy(entropy float64) Option {
// WithOnlyVerified sets the onlyVerified flag on the engine. If set to true,
// the engine will only print verified results.
func WithOnlyVerified(onlyVerified bool) Option {
func WithOnlyVerified(onlyVerified bool) EngineOption {
return func(e *Engine) {
e.onlyVerified = onlyVerified
}
@@ -150,7 +153,7 @@ func WithOnlyVerified(onlyVerified bool) Option {
// the engine is configured to print the results.
// Calculating the average time taken by each detector is an expensive operation
// and should be avoided unless specified by the user.
func WithPrintAvgDetectorTime(printAvgDetectorTime bool) Option {
func WithPrintAvgDetectorTime(printAvgDetectorTime bool) EngineOption {
return func(e *Engine) {
e.printAvgDetectorTime = printAvgDetectorTime
}
@@ -160,7 +163,7 @@ func WithPrintAvgDetectorTime(printAvgDetectorTime bool) Option {
// the filterFunc returns true, the detector will be included for scanning.
// This option applies to the existing list of detectors configured, so the
// order this option appears matters. All filtering happens before scanning.
func WithFilterDetectors(filterFunc func(detectors.Detector) bool) Option {
func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption {
return func(e *Engine) {
// If no detectors are configured, do nothing.
if e.detectors == nil {
@@ -172,7 +175,7 @@ func WithFilterDetectors(filterFunc func(detectors.Detector) bool) Option {
}
// WithPrinter sets the Printer on the engine.
func WithPrinter(printer Printer) Option {
func WithPrinter(printer Printer) EngineOption {
return func(e *Engine) {
e.printer = printer
}
@@ -266,60 +269,32 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
return avgTime
}
// Start initializes and activates the engine's processing pipeline.
// It sets up various default configurations, prepares lookup structures for
// detectors, conducts basic sanity checks, and kickstarts all necessary workers.
// Once started, the engine begins processing input data to identify secrets.
func Start(ctx context.Context, options ...Option) (*Engine, error) {
e := &Engine{}
if err := e.initialize(ctx, options...); err != nil {
return nil, err
}
e.setDefaults(ctx)
ctx.Logger().V(4).Info("setting up aho-corasick core")
e.ahoCorasickCore.Setup(ctx)
e.sanityChecks(ctx)
e.startWorkers(ctx)
return e, nil
}
const defaultChannelBuffer = 1
// initialize prepares the engine's internal structures. The LRU cache optimizes
// deduplication efforts, allowing the engine to quickly check if a chunk has
// been processed before, thereby saving computational overhead.
func (e *Engine) initialize(ctx context.Context, options ...Option) error {
// TODO (ahrav): Determine the optimal cache size.
const cacheSize = 512 // number of entries in the LRU cache
// Start the engine with options.
func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
const (
defaultChannelBuffer = 1
// TODO (ahrav): Determine the optimal cache size.
cacheSize = 512 // number of entries in the LRU cache
)
cache, err := lru.New(cacheSize)
if err != nil {
return fmt.Errorf("failed to initialize LRU cache: %w", err)
return nil, fmt.Errorf("failed to initialize LRU cache: %w", err)
}
// Channels are used for communication between different parts of the engine,
// ensuring that data flows smoothly without race conditions.
e.detectableChunksChan = make(chan detectableChunk, defaultChannelBuffer)
e.results = make(chan detectors.ResultWithMetadata, defaultChannelBuffer)
e.dedupeCache = cache
e.printer = new(output.PlainPrinter)
e.metrics = runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}}
e := &Engine{
detectableChunksChan: make(chan detectableChunk, defaultChannelBuffer),
results: make(chan detectors.ResultWithMetadata, defaultChannelBuffer),
dedupeCache: cache,
printer: new(output.PlainPrinter), // default printer
metrics: runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}},
}
for _, option := range options {
option(e)
}
ctx.Logger().V(4).Info("engine initialized")
e.ahoCorasickCore = NewAhoCorasickCore(e.detectors)
return nil
}
// setDefaults ensures that if specific engine properties aren't provided,
// they're set to reasonable default values. It makes the engine robust to
// incomplete configuration.
func (e *Engine) setDefaults(ctx context.Context) {
// Set defaults.
if e.concurrency == 0 {
numCPU := runtime.NumCPU()
ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU)
@@ -327,13 +302,13 @@ func (e *Engine) setDefaults(ctx context.Context) {
}
ctx.Logger().V(3).Info("engine started", "workers", e.concurrency)
// Create SourceManager.
e.sourceManager = sources.NewManager(
sources.WithConcurrentSources(int(e.concurrency)),
sources.WithConcurrentUnits(int(e.concurrency)),
sources.WithBufferedOutput(defaultChannelBuffer),
sources.WithSourceUnits(),
)
// Default decoders handle common encoding formats.
if len(e.decoders) == 0 {
e.decoders = decoders.DefaultDecoders()
}
@@ -343,30 +318,46 @@ func (e *Engine) setDefaults(ctx context.Context) {
e.detectors[true] = DefaultDetectors()
e.detectors[false] = []detectors.Detector{}
}
ctx.Logger().V(4).Info("default engine options set")
}
// Sanity check detectors for duplicate configuration. Only log in case
// a detector has been configured in a way that isn't represented by
// the DetectorID (type and version).
func (e *Engine) sanityChecks(ctx context.Context) {
dets := append(e.detectors[true], e.detectors[false]...)
seenDetectors := make(map[config.DetectorID]struct{}, len(dets))
for _, det := range dets {
id := config.GetDetectorID(det)
if _, ok := seenDetectors[id]; ok && id.ID != detectorspb.DetectorType_CustomRegex {
ctx.Logger().Info("possible duplicate detector configured", "detector", id)
// build ahocorasick prefilter for efficient string matching
// on keywords
keywords := []string{}
for _, d := range e.detectors[false] {
for _, kw := range d.Keywords() {
keywords = append(keywords, strings.ToLower(kw))
}
}
for _, d := range e.detectors[true] {
for _, kw := range d.Keywords() {
keywords = append(keywords, strings.ToLower(kw))
}
}
e.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
ctx.Logger().V(3).Info("loaded decoders", "count", len(e.decoders))
ctx.Logger().V(3).Info("loaded detectors",
"total", len(e.detectors[true])+len(e.detectors[false]),
"verification_enabled", len(e.detectors[true]),
"verification_disabled", len(e.detectors[false]),
)
// Sanity check detectors for duplicate configuration. Only log in case
// a detector has been configured in a way that isn't represented by
// the DetectorID (type and version).
{
dets := append(e.detectors[true], e.detectors[false]...)
seenDetectors := make(map[config.DetectorID]struct{}, len(dets))
for _, det := range dets {
id := config.GetDetectorID(det)
if _, ok := seenDetectors[id]; ok && id.ID != detectorspb.DetectorType_CustomRegex {
ctx.Logger().Info("possible duplicate detector configured", "detector", id)
}
seenDetectors[id] = struct{}{}
}
seenDetectors[id] = struct{}{}
}
}
// startWorkers initiates all necessary workers. Workers handle processing of
// chunks concurrently. Separating the initialization of different types of
// workers helps in scalability and makes it easier to diagnose issues.
func (e *Engine) startWorkers(ctx context.Context) {
// Scanner workers process input data and extract chunks for detectors.
ctx.Logger().V(2).Info("starting scanner workers", "count", e.concurrency)
// Run the Secret scanner workers and Notifier pipelines.
for worker := uint64(0); worker < uint64(e.concurrency); worker++ {
e.workersWg.Add(1)
go func() {
@@ -377,7 +368,6 @@ func (e *Engine) startWorkers(ctx context.Context) {
}()
}
// Detector workers apply keyword matching, regexes and API calls to detect secrets in chunks.
const detectorWorkerMultiplier = 50
ctx.Logger().V(2).Info("starting detector workers", "count", e.concurrency*detectorWorkerMultiplier)
for worker := uint64(0); worker < uint64(e.concurrency*detectorWorkerMultiplier); worker++ {
@@ -390,7 +380,6 @@ func (e *Engine) startWorkers(ctx context.Context) {
}()
}
// Notifier workers communicate detected issues to the user or any downstream systems.
// We want 1/4th of the notifier workers as the number of scanner workers.
const notifierWorkerRatio = 4
maxNotifierWorkers := 1
@@ -407,6 +396,8 @@ func (e *Engine) startWorkers(ctx context.Context) {
e.notifyResults(ctx)
}()
}
return e, nil
}
// Finish waits for running sources to complete and workers to finish scanning
@@ -455,42 +446,64 @@ type detectableChunk struct {
func (e *Engine) detectorWorker(ctx context.Context) {
var wgDetect sync.WaitGroup
// Reuse the same map to avoid allocations.
const avgDetectorsPerChunk = 2
chunkSpecificDetectors := make(map[detectorspb.DetectorType]detectorInfo, avgDetectorsPerChunk)
for originalChunk := range e.ChunksChan() {
for chunk := range sources.Chunker(originalChunk) {
matchedKeywords := make(map[string]struct{})
atomic.AddUint64(&e.metrics.BytesScanned, uint64(len(chunk.Data)))
for _, decoder := range e.decoders {
var decoderType detectorspb.DecoderType
switch decoder.(type) {
case *decoders.UTF8:
decoderType = detectorspb.DecoderType_PLAIN
case *decoders.Base64:
decoderType = detectorspb.DecoderType_BASE64
case *decoders.UTF16:
decoderType = detectorspb.DecoderType_UTF16
default:
ctx.Logger().Info("unknown decoder type", "type", reflect.TypeOf(decoder).String())
decoderType = detectorspb.DecoderType_UNKNOWN
}
decoded := decoder.FromChunk(chunk)
if decoded == nil {
ctx.Logger().V(4).Info("no decoder found for chunk", "chunk", chunk)
continue
}
for _, match := range e.ahoCorasickCore.MatchString(string(decoded.Chunk.Data)) {
if !e.ahoCorasickCore.PopulateDetectorsByMatch(match, chunkSpecificDetectors) {
continue
}
// build a map of all keywords that were matched in the chunk
for _, m := range e.prefilter.MatchString(strings.ToLower(string(decoded.Data))) {
matchedKeywords[strings.ToLower(m.MatchString())] = struct{}{}
}
for k, detector := range chunkSpecificDetectors {
decoded.Chunk.Verify = detector.shouldVerify
wgDetect.Add(1)
e.detectableChunksChan <- detectableChunk{
chunk: *decoded.Chunk,
detector: detector,
decoder: decoded.DecoderType,
wgDoneFn: wgDetect.Done,
for verify, detectorsSet := range e.detectors {
for _, detector := range detectorsSet {
chunkContainsKeyword := false
for _, kw := range detector.Keywords() {
if _, ok := matchedKeywords[strings.ToLower(kw)]; ok {
chunkContainsKeyword = true
break
}
}
if !chunkContainsKeyword {
continue
}
decoded.Verify = verify
wgDetect.Add(1)
e.detectableChunksChan <- detectableChunk{
chunk: *decoded,
detector: detector,
decoder: decoderType,
wgDoneFn: wgDetect.Done,
}
}
delete(chunkSpecificDetectors, k)
}
}
}
atomic.AddUint64(&e.metrics.ChunksScanned, 1)
}
wgDetect.Wait()
ctx.Logger().V(4).Info("finished scanning chunks")
}
func (e *Engine) detectChunks(ctx context.Context) {
+1 -4
View File
@@ -2,7 +2,6 @@ package engine
import (
"os"
"runtime"
"testing"
"github.com/stretchr/testify/assert"
@@ -121,10 +120,9 @@ func BenchmarkGitEngine(b *testing.B) {
defer cancel()
e, err := Start(ctx,
WithConcurrency(uint8(runtime.NumCPU())),
WithConcurrency(1),
WithDecoders(decoders.DefaultDecoders()...),
WithDetectors(false, DefaultDetectors()...),
WithPrinter(new(discardPrinter)),
)
assert.Nil(b, err)
@@ -135,7 +133,6 @@ func BenchmarkGitEngine(b *testing.B) {
}
}()
b.ResetTimer()
for i := 0; i < b.N; i++ {
// TODO: this is measuring the time it takes to initialize the source
// and not to do the full scan
+38
View File
@@ -0,0 +1,38 @@
package engine
import (
"runtime"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/travisci"
)
// ScanTravisCI scans TravisCI logs.
func (e *Engine) ScanTravisCI(ctx context.Context, token string) error {
connection := &sourcespb.TravisCI{
Credential: &sourcespb.TravisCI_Token{
Token: token,
},
}
var conn anypb.Any
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal Travis CI connection")
return err
}
sourceName := "trufflehog - Travis CI"
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, travisci.SourceType)
travisSource := &travisci.Source{}
if err := travisSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
return err
}
_, err = e.sourceManager.Run(ctx, sourceName, travisSource)
return err
}
+11 -7
View File
@@ -1007,6 +1007,7 @@ const (
DetectorType_Anthropic DetectorType = 933
DetectorType_Ramp DetectorType = 934
DetectorType_Klaviyo DetectorType = 935
DetectorType_SourcegraphCody DetectorType = 936
)
// Enum value maps for DetectorType.
@@ -1944,6 +1945,7 @@ var (
933: "Anthropic",
934: "Ramp",
935: "Klaviyo",
936: "SourcegraphCody",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@@ -2878,6 +2880,7 @@ var (
"Anthropic": 933,
"Ramp": 934,
"Klaviyo": 935,
"SourcegraphCody": 936,
}
)
@@ -3256,7 +3259,7 @@ var file_detectors_proto_rawDesc = []byte{
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x90, 0x75, 0x0a, 0x0c, 0x44,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xa6, 0x75, 0x0a, 0x0c, 0x44,
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4193,12 +4196,13 @@ var file_detectors_proto_rawDesc = []byte{
0x12, 0x12, 0x0a, 0x0d, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x74, 0x53, 0x63, 0x61, 0x6c, 0x65, 0x44,
0x62, 0x10, 0xa4, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x41, 0x6e, 0x74, 0x68, 0x72, 0x6f, 0x70, 0x69,
0x63, 0x10, 0xa5, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x52, 0x61, 0x6d, 0x70, 0x10, 0xa6, 0x07, 0x12,
0x0c, 0x0a, 0x07, 0x4b, 0x6c, 0x61, 0x76, 0x69, 0x79, 0x6f, 0x10, 0xa7, 0x07, 0x42, 0x3d, 0x5a,
0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66,
0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66,
0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62,
0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,
0x0c, 0x0a, 0x07, 0x4b, 0x6c, 0x61, 0x76, 0x69, 0x79, 0x6f, 0x10, 0xa7, 0x07, 0x12, 0x14, 0x0a,
0x0f, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x43, 0x6f, 0x64, 0x79,
0x10, 0xa8, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
File diff suppressed because it is too large Load Diff
@@ -486,6 +486,117 @@ var _ interface {
ErrorName() string
} = CircleCIValidationError{}
// Validate checks the field values on TravisCI with the rules defined in the
// proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
func (m *TravisCI) Validate() error {
return m.validate(false)
}
// ValidateAll checks the field values on TravisCI with the rules defined in
// the proto definition for this message. If any rules are violated, the
// result is a list of violation errors wrapped in TravisCIMultiError, or nil
// if none found.
func (m *TravisCI) ValidateAll() error {
return m.validate(true)
}
func (m *TravisCI) validate(all bool) error {
if m == nil {
return nil
}
var errors []error
// no validation rules for Username
// no validation rules for Repository
// no validation rules for BuildNumber
// no validation rules for JobNumber
// no validation rules for Link
// no validation rules for Public
if len(errors) > 0 {
return TravisCIMultiError(errors)
}
return nil
}
// TravisCIMultiError is an error wrapping multiple validation errors returned
// by TravisCI.ValidateAll() if the designated constraints aren't met.
type TravisCIMultiError []error
// Error returns a concatenation of all the error messages it wraps.
func (m TravisCIMultiError) Error() string {
var msgs []string
for _, err := range m {
msgs = append(msgs, err.Error())
}
return strings.Join(msgs, "; ")
}
// AllErrors returns a list of validation violation errors.
func (m TravisCIMultiError) AllErrors() []error { return m }
// TravisCIValidationError is the validation error returned by
// TravisCI.Validate if the designated constraints aren't met.
type TravisCIValidationError struct {
field string
reason string
cause error
key bool
}
// Field function returns field value.
func (e TravisCIValidationError) Field() string { return e.field }
// Reason function returns reason value.
func (e TravisCIValidationError) Reason() string { return e.reason }
// Cause function returns cause value.
func (e TravisCIValidationError) Cause() error { return e.cause }
// Key function returns key value.
func (e TravisCIValidationError) Key() bool { return e.key }
// ErrorName returns error name.
func (e TravisCIValidationError) ErrorName() string { return "TravisCIValidationError" }
// Error satisfies the builtin error interface
func (e TravisCIValidationError) Error() string {
cause := ""
if e.cause != nil {
cause = fmt.Sprintf(" | caused by: %v", e.cause)
}
key := ""
if e.key {
key = "key for "
}
return fmt.Sprintf(
"invalid %sTravisCI.%s: %s%s",
key,
e.field,
e.reason,
cause)
}
var _ error = TravisCIValidationError{}
var _ interface {
Field() string
Reason() string
Key() bool
Cause() error
ErrorName() string
} = TravisCIValidationError{}
// Validate checks the field values on Confluence with the rules defined in the
// proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
@@ -3926,6 +4037,37 @@ func (m *MetaData) validate(all bool) error {
}
}
case *MetaData_TravisCI:
if all {
switch v := interface{}(m.GetTravisCI()).(type) {
case interface{ ValidateAll() error }:
if err := v.ValidateAll(); err != nil {
errors = append(errors, MetaDataValidationError{
field: "TravisCI",
reason: "embedded message failed validation",
cause: err,
})
}
case interface{ Validate() error }:
if err := v.Validate(); err != nil {
errors = append(errors, MetaDataValidationError{
field: "TravisCI",
reason: "embedded message failed validation",
cause: err,
})
}
}
} else if v, ok := interface{}(m.GetTravisCI()).(interface{ Validate() error }); ok {
if err := v.Validate(); err != nil {
return MetaDataValidationError{
field: "TravisCI",
reason: "embedded message failed validation",
cause: err,
}
}
}
}
if len(errors) > 0 {
File diff suppressed because it is too large Load Diff
+118
View File
@@ -667,6 +667,124 @@ var _ interface {
ErrorName() string
} = CircleCIValidationError{}
// Validate checks the field values on TravisCI with the rules defined in the
// proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
func (m *TravisCI) Validate() error {
return m.validate(false)
}
// ValidateAll checks the field values on TravisCI with the rules defined in
// the proto definition for this message. If any rules are violated, the
// result is a list of violation errors wrapped in TravisCIMultiError, or nil
// if none found.
func (m *TravisCI) ValidateAll() error {
return m.validate(true)
}
func (m *TravisCI) validate(all bool) error {
if m == nil {
return nil
}
var errors []error
if _, err := url.Parse(m.GetEndpoint()); err != nil {
err = TravisCIValidationError{
field: "Endpoint",
reason: "value must be a valid URI",
cause: err,
}
if !all {
return err
}
errors = append(errors, err)
}
switch m.Credential.(type) {
case *TravisCI_Token:
// no validation rules for Token
}
if len(errors) > 0 {
return TravisCIMultiError(errors)
}
return nil
}
// TravisCIMultiError is an error wrapping multiple validation errors returned
// by TravisCI.ValidateAll() if the designated constraints aren't met.
type TravisCIMultiError []error
// Error returns a concatenation of all the error messages it wraps.
func (m TravisCIMultiError) Error() string {
var msgs []string
for _, err := range m {
msgs = append(msgs, err.Error())
}
return strings.Join(msgs, "; ")
}
// AllErrors returns a list of validation violation errors.
func (m TravisCIMultiError) AllErrors() []error { return m }
// TravisCIValidationError is the validation error returned by
// TravisCI.Validate if the designated constraints aren't met.
type TravisCIValidationError struct {
field string
reason string
cause error
key bool
}
// Field function returns field value.
func (e TravisCIValidationError) Field() string { return e.field }
// Reason function returns reason value.
func (e TravisCIValidationError) Reason() string { return e.reason }
// Cause function returns cause value.
func (e TravisCIValidationError) Cause() error { return e.cause }
// Key function returns key value.
func (e TravisCIValidationError) Key() bool { return e.key }
// ErrorName returns error name.
func (e TravisCIValidationError) ErrorName() string { return "TravisCIValidationError" }
// Error satisfies the builtin error interface
func (e TravisCIValidationError) Error() string {
cause := ""
if e.cause != nil {
cause = fmt.Sprintf(" | caused by: %v", e.cause)
}
key := ""
if e.key {
key = "key for "
}
return fmt.Sprintf(
"invalid %sTravisCI.%s: %s%s",
key,
e.field,
e.reason,
cause)
}
var _ error = TravisCIValidationError{}
var _ interface {
Field() string
Reason() string
Key() bool
Cause() error
ErrorName() string
} = TravisCIValidationError{}
// Validate checks the field values on Confluence with the rules defined in the
// proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
+2 -2
View File
@@ -1,8 +1,8 @@
package sources
import (
"errors"
"fmt"
"reflect"
"sync"
"testing"
)
@@ -40,7 +40,7 @@ func TestNewScanErrors(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
got := NewScanErrors()
if !reflect.DeepEqual(got, tc.want) {
if !errors.Is(got.Errors(), tc.want.Errors()) {
t.Errorf("got %+v, want %+v", got, tc.want)
}
})
+1 -2
View File
@@ -39,8 +39,7 @@ type Source struct {
// Ensure the Source satisfies the interfaces at compile time
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
var _ sources.SourceUnitEnumerator = (*Source)(nil)
var _ sources.SourceUnitChunker = (*Source)(nil)
var _ sources.SourceUnitEnumChunker = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
+195 -10
View File
@@ -15,6 +15,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest"
)
func TestSource_Scan(t *testing.T) {
@@ -84,23 +85,15 @@ func TestSource_Scan(t *testing.T) {
}
func TestScanFile(t *testing.T) {
tmpfile, err := os.CreateTemp("", "example.txt")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
chunkSize := sources.ChunkSize
secretPart1 := "SECRET"
secretPart2 := "SPLIT"
// Split the secret into two parts and pad the rest of the chunk with A's.
data := strings.Repeat("A", chunkSize-len(secretPart1)) + secretPart1 + secretPart2 + strings.Repeat("A", chunkSize-len(secretPart2))
_, err = tmpfile.Write([]byte(data))
assert.Nil(t, err)
err = tmpfile.Close()
tmpfile, cleanup, err := createTempFile("", data)
assert.Nil(t, err)
defer cleanup()
source := &Source{}
chunksChan := make(chan *sources.Chunk, 2)
@@ -120,3 +113,195 @@ func TestScanFile(t *testing.T) {
assert.Contains(t, foundSecret, secretPart1+secretPart2)
}
func TestEnumerate(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Setup the connection to test enumeration.
units := []string{
"/one", "/two", "/three",
"/path/to/dir/", "/path/to/another/dir/",
}
conn, err := anypb.New(&sourcespb.Filesystem{
Paths: units[0:3],
Directories: units[3:],
})
assert.NoError(t, err)
// Initialize the source.
s := Source{}
err = s.Init(ctx, "test enumerate", 0, 0, true, conn, 1)
assert.NoError(t, err)
reporter := sourcestest.TestReporter{}
err = s.Enumerate(ctx, &reporter)
assert.NoError(t, err)
assert.Equal(t, len(units), len(reporter.Units))
assert.Equal(t, 0, len(reporter.UnitErrs))
for _, unit := range reporter.Units {
assert.Contains(t, units, unit.SourceUnitID())
}
for _, unit := range units {
assert.Contains(t, reporter.Units, sources.CommonSourceUnit{ID: unit})
}
}
func TestChunkUnit(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Setup test file to chunk.
fileContents := "TestChunkUnit"
tmpfile, cleanup, err := createTempFile("", fileContents)
assert.NoError(t, err)
defer cleanup()
tmpdir, cleanup, err := createTempDir("", "foo", "bar", "baz")
assert.NoError(t, err)
defer cleanup()
conn, err := anypb.New(&sourcespb.Filesystem{})
assert.NoError(t, err)
// Initialize the source.
s := Source{}
err = s.Init(ctx, "test chunk unit", 0, 0, true, conn, 1)
assert.NoError(t, err)
// Happy path single file.
reporter := sourcestest.TestReporter{}
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: tmpfile.Name(),
}, &reporter)
assert.NoError(t, err)
// Happy path directory.
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: tmpdir,
}, &reporter)
assert.NoError(t, err)
// Error path.
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: "/file/not/found",
}, &reporter)
assert.NoError(t, err)
assert.Equal(t, 4, len(reporter.Chunks))
assert.Equal(t, 1, len(reporter.ChunkErrs))
dataFound := make(map[string]struct{}, 4)
for _, chunk := range reporter.Chunks {
dataFound[string(chunk.Data)] = struct{}{}
}
assert.Contains(t, dataFound, fileContents)
assert.Contains(t, dataFound, "foo")
assert.Contains(t, dataFound, "bar")
assert.Contains(t, dataFound, "baz")
}
func TestEnumerateReporterErr(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Setup the connection to test enumeration.
units := []string{
"/one", "/two", "/three",
"/path/to/dir/", "/path/to/another/dir/",
}
conn, err := anypb.New(&sourcespb.Filesystem{
Paths: units[0:3],
Directories: units[3:],
})
assert.NoError(t, err)
// Initialize the source.
s := Source{}
err = s.Init(ctx, "test enumerate", 0, 0, true, conn, 1)
assert.NoError(t, err)
// Enumerate should always return an error if the reporter returns an
// error.
reporter := sourcestest.ErrReporter{}
err = s.Enumerate(ctx, &reporter)
assert.Error(t, err)
}
func TestChunkUnitReporterErr(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Setup test file to chunk.
tmpfile, err := os.CreateTemp("", "example.txt")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
fileContents := []byte("TestChunkUnit")
_, err = tmpfile.Write(fileContents)
assert.NoError(t, err)
assert.NoError(t, tmpfile.Close())
conn, err := anypb.New(&sourcespb.Filesystem{})
assert.NoError(t, err)
// Initialize the source.
s := Source{}
err = s.Init(ctx, "test chunk unit", 0, 0, true, conn, 1)
assert.NoError(t, err)
// Happy path. ChunkUnit should always return an error if the reporter
// returns an error.
reporter := sourcestest.ErrReporter{}
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: tmpfile.Name(),
}, &reporter)
assert.Error(t, err)
// Error path. ChunkUnit should always return an error if the reporter
// returns an error.
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: "/file/not/found",
}, &reporter)
assert.Error(t, err)
}
// createTempFile is a helper function to create a temporary file in the given
// directory with the provided contents. If dir is "", the operating system's
// temp directory is used.
func createTempFile(dir string, contents string) (*os.File, func(), error) {
tmpfile, err := os.CreateTemp(dir, "trufflehogtest")
if err != nil {
return nil, nil, err
}
if _, err := tmpfile.Write([]byte(contents)); err != nil {
_ = os.Remove(tmpfile.Name())
return nil, nil, err
}
if err := tmpfile.Close(); err != nil {
_ = os.Remove(tmpfile.Name())
return nil, nil, err
}
return tmpfile, func() { _ = os.Remove(tmpfile.Name()) }, nil
}
// createTempDir is a helper function to create a temporary directory in the
// given directory with files containing the provided contents. If dir is "",
// the operating system's temp directory is used.
func createTempDir(dir string, contents ...string) (string, func(), error) {
tmpdir, err := os.MkdirTemp(dir, "trufflehogtest")
if err != nil {
return "", nil, err
}
for _, content := range contents {
if _, _, err := createTempFile(tmpdir, content); err != nil {
_ = os.RemoveAll(tmpdir)
return "", nil, err
}
}
return tmpdir, func() { _ = os.RemoveAll(tmpdir) }, nil
}
+223
View File
@@ -0,0 +1,223 @@
package travisci
import (
"fmt"
"strconv"
"github.com/go-errors/errors"
"github.com/shuheiktgw/go-travis"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
const (
SourceType = sourcespb.SourceType_SOURCE_TYPE_TRAVISCI
baseURL = "https://api.travis-ci.com/"
pageSize = 100
)
type Source struct {
name string
sourceId sources.SourceID
jobId sources.JobID
verify bool
jobPool *errgroup.Group
sources.Progress
client *travis.Client
sources.CommonSourceUnitUnmarshaller
returnAfterFirstChunk bool
}
// Ensure the Source satisfies the interfaces at compile time.
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
func (s *Source) Type() sourcespb.SourceType {
return SourceType
}
func (s *Source) SourceID() sources.SourceID {
return s.sourceId
}
func (s *Source) JobID() sources.JobID {
return s.jobId
}
// Init returns an initialized TravisCI source.
func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
s.name = name
s.sourceId = sourceId
s.jobId = jobId
s.verify = verify
s.jobPool = &errgroup.Group{}
s.jobPool.SetLimit(concurrency)
var conn sourcespb.TravisCI
if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
}
switch conn.Credential.(type) {
case *sourcespb.TravisCI_Token:
if conn.GetToken() == "" {
return errors.New("token is empty")
}
s.client = travis.NewClient(baseURL, conn.GetToken())
s.client.HTTPClient = common.RetryableHttpClientTimeout(3)
user, _, err := s.client.User.Current(ctx, nil)
if err != nil {
return errors.WrapPrefix(err, "error getting testing travis client", 0)
}
ctx.Logger().V(2).Info("authenticated to Travis CI with user", "username", user.Login)
default:
return errors.New("credential type not implemented for Travis CI")
}
return nil
}
func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {
for repoPage := 0; ; repoPage++ {
repositories, _, err := s.client.Repositories.List(ctx, &travis.RepositoriesOption{
Limit: pageSize,
Offset: repoPage * pageSize,
})
if err != nil {
if repoPage == 0 {
return fmt.Errorf("error listing repositories: %w", err)
}
err = reporter.UnitErr(ctx, err)
if err != nil {
return fmt.Errorf("error reporting error: %w", err)
}
}
if len(repositories) == 0 {
break
}
for _, repo := range repositories {
err = reporter.UnitOk(ctx, sources.CommonSourceUnit{
ID: strconv.Itoa(int(*repo.Id)),
})
if err != nil {
return fmt.Errorf("error reporting unit: %w", err)
}
ctx.Logger().V(2).Info("enumerated repository", "id", repo.Id, "name", repo.Name)
}
}
return nil
}
// ChunkUnit implements SourceUnitChunker interface.
func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
repo, _, err := s.client.Repositories.Find(ctx, unit.SourceUnitID(), nil)
if err != nil {
return fmt.Errorf("error finding repository: %w", err)
}
logger := ctx.Logger().WithValues("repo", *repo.Name)
logger.V(2).Info("scanning repository")
// Counts continuous errors from ListByRepoSlug. Used to quit early in
// case the API always returns an error.
var buildPageErrs int
for buildPage := 0; ; buildPage++ {
builds, _, err := s.client.Builds.ListByRepoSlug(ctx, *repo.Slug, &travis.BuildsByRepoOption{
Limit: pageSize,
Offset: buildPage * pageSize,
})
if err != nil {
if err := reporter.ChunkErr(ctx, err); err != nil {
return err
}
buildPageErrs++
if buildPageErrs >= 5 {
return fmt.Errorf("encountered too many errors listing builds, aborting")
}
continue
}
// Reset the page error counter.
buildPageErrs = 0
if len(builds) == 0 {
break
}
for _, build := range builds {
jobs, _, err := s.client.Jobs.ListByBuild(ctx, *build.Id)
if err != nil {
if err := reporter.ChunkErr(ctx, err); err != nil {
return err
}
continue
}
if len(jobs) == 0 {
break
}
for _, job := range jobs {
log, _, err := s.client.Logs.FindByJobId(ctx, *job.Id)
if err != nil {
if err := reporter.ChunkErr(ctx, err); err != nil {
return err
}
continue
}
logger.V(3).Info("scanning job", "id", *job.Id, "number", *job.Number)
chunk := sources.Chunk{
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
Data: []byte(*log.Content),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_TravisCI{
TravisCI: &source_metadatapb.TravisCI{
Username: *job.Owner.Login,
Repository: *repo.Name,
BuildNumber: *build.Number,
JobNumber: *job.Number,
Link: fmt.Sprintf("https://app.travis-ci.com/github/%s/%s/jobs/%d", *job.Owner.Login, *repo.Name, *job.Id),
Public: !*repo.Private,
},
},
},
Verify: s.verify,
}
if err := reporter.ChunkOk(ctx, chunk); err != nil {
return err
}
if s.returnAfterFirstChunk {
return nil
}
}
}
}
return nil
}
// Chunks emits chunks of bytes over a channel.
// It's a no-op because we've implemented the SourceUnitChunker interface instead.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
return nil
}
+99
View File
@@ -0,0 +1,99 @@
package travisci
import (
"fmt"
"testing"
"github.com/kylelemons/godebug/pretty"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest"
)
func TestSource_Scan(t *testing.T) {
ctx := context.Background()
secret, err := common.GetTestSecret(ctx)
if err != nil {
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
}
token := secret.MustGetField("TRAVISCI_TOKEN")
type init struct {
name string
verify bool
connection *sourcespb.TravisCI
}
tests := []struct {
name string
init init
wantSourceMetadata *source_metadatapb.MetaData
wantErr bool
}{
{
name: "get a chunk",
init: init{
name: "example repo",
connection: &sourcespb.TravisCI{
Credential: &sourcespb.TravisCI_Token{
Token: token,
},
},
verify: true,
},
wantSourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_TravisCI{
TravisCI: &source_metadatapb.TravisCI{
Username: "truffle-sandbox",
Repository: "test-repo",
BuildNumber: "1",
JobNumber: "1.1",
Link: "https://app.travis-ci.com/github/truffle-sandbox/test-repo/jobs/611053994",
Public: false,
},
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Source{}
conn, err := anypb.New(tt.init.connection)
if err != nil {
t.Fatal(err)
}
err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5)
if (err != nil) != tt.wantErr {
t.Fatalf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
}
reporter := sourcestest.TestReporter{}
s.returnAfterFirstChunk = true
err = s.Enumerate(ctx, &reporter)
if (err != nil) != tt.wantErr {
t.Fatalf("Source.Enumerate() error = %v, wantErr %v", err, tt.wantErr)
}
for _, unit := range reporter.Units {
err = s.ChunkUnit(ctx, unit, &reporter)
if (err != nil) != tt.wantErr {
t.Fatalf("Source.ChunkUnit() error = %v, wantErr %v", err, tt.wantErr)
}
}
for _, chunk := range reporter.Chunks {
if diff := pretty.Compare(chunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {
t.Fatalf("comparing chunks: %s diff: (-got +want)\n%s", tt.name, diff)
}
}
})
}
}
+61
View File
@@ -0,0 +1,61 @@
package sourcestest
import (
"fmt"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
type reporter interface {
sources.UnitReporter
sources.ChunkReporter
}
var (
_ reporter = (*TestReporter)(nil)
_ reporter = (*ErrReporter)(nil)
)
// TestReporter is a helper struct that implements both UnitReporter and
// ChunkReporter by simply recording the values passed in the methods.
type TestReporter struct {
Units []sources.SourceUnit
UnitErrs []error
Chunks []sources.Chunk
ChunkErrs []error
}
func (t *TestReporter) UnitOk(_ context.Context, unit sources.SourceUnit) error {
t.Units = append(t.Units, unit)
return nil
}
func (t *TestReporter) UnitErr(_ context.Context, err error) error {
t.UnitErrs = append(t.UnitErrs, err)
return nil
}
func (t *TestReporter) ChunkOk(_ context.Context, chunk sources.Chunk) error {
t.Chunks = append(t.Chunks, chunk)
return nil
}
func (t *TestReporter) ChunkErr(_ context.Context, err error) error {
t.ChunkErrs = append(t.ChunkErrs, err)
return nil
}
// ErrReporter implements UnitReporter and ChunkReporter but always returns an
// error.
type ErrReporter struct{}
func (ErrReporter) UnitOk(context.Context, sources.SourceUnit) error {
return fmt.Errorf("ErrReporter: UnitOk error")
}
func (ErrReporter) UnitErr(context.Context, error) error {
return fmt.Errorf("ErrReporter: UnitErr error")
}
func (ErrReporter) ChunkOk(context.Context, sources.Chunk) error {
return fmt.Errorf("ErrReporter: ChunkOk error")
}
func (ErrReporter) ChunkErr(context.Context, error) error {
return fmt.Errorf("ErrReporter: ChunkErr error")
}
+1
View File
@@ -944,6 +944,7 @@ enum DetectorType {
Anthropic = 933;
Ramp = 934;
Klaviyo = 935;
SourcegraphCody = 936;
}
message Result {
+11
View File
@@ -44,6 +44,16 @@ message CircleCI {
string email = 7;
}
message TravisCI {
string username = 2;
string repository = 3;
string build_number = 4;
string job_number = 5;
string link = 6;
bool public = 7;
}
message Confluence {
string page = 1;
string space = 2;
@@ -294,5 +304,6 @@ message MetaData {
SharePoint sharepoint = 25;
GoogleDrive googleDrive = 26;
AzureRepos azureRepos = 27;
TravisCI travisCI = 28;
}
}
+8
View File
@@ -44,6 +44,7 @@ enum SourceType {
SOURCE_TYPE_SHAREPOINT = 29;
SOURCE_TYPE_GCS_UNAUTHED = 30;
SOURCE_TYPE_AZURE_REPOS = 31;
SOURCE_TYPE_TRAVISCI = 32;
}
message LocalSource {
@@ -83,6 +84,13 @@ message CircleCI {
}
}
message TravisCI {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
string token = 2;
}
}
message Confluence {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {