Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 065984ee58 | |||
| fe90effec1 | |||
| e5a34a9809 | |||
| ec3c59d025 | |||
| c99e0b93f7 | |||
| 03dc7cb68d | |||
| 072e1f9dcf | |||
| 5c721d1a73 | |||
| 040167178c | |||
| b960231155 |
@@ -65,6 +65,7 @@ require (
|
||||
github.com/prometheus/client_golang v1.16.0
|
||||
github.com/rabbitmq/amqp091-go v1.8.1
|
||||
github.com/sergi/go-diff v1.3.1
|
||||
github.com/shuheiktgw/go-travis v0.3.1
|
||||
github.com/snowflakedb/gosnowflake v1.6.23
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
|
||||
@@ -369,6 +369,7 @@ github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5Px
|
||||
github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
|
||||
github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=
|
||||
github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=
|
||||
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
@@ -605,6 +606,8 @@ github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
|
||||
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
|
||||
github.com/shuheiktgw/go-travis v0.3.1 h1:SAT16mi77ccqogOslnXxBXzXbpeyChaIYUwi2aJpVZY=
|
||||
github.com/shuheiktgw/go-travis v0.3.1/go.mod h1:avnFFDqJDdRHwlF9tgqvYi3asQCm/HGL8aLxYiKa4Yg=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
|
||||
@@ -139,6 +139,9 @@ var (
|
||||
|
||||
dockerScan = cli.Command("docker", "Scan Docker Image")
|
||||
dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, otherwise a image registry is assumed.").Required().Strings()
|
||||
|
||||
travisCiScan = cli.Command("travisci", "Scan TravisCI")
|
||||
travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -498,6 +501,10 @@ func run(state overseer.State) {
|
||||
if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
|
||||
logFatal(err, "Failed to scan CircleCI.")
|
||||
}
|
||||
case travisCiScan.FullCommand():
|
||||
if err := e.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
|
||||
logFatal(err, "Failed to scan TravisCI.")
|
||||
}
|
||||
case gcsScan.FullCommand():
|
||||
cfg := sources.GCSConfig{
|
||||
ProjectID: *gcsProjectID,
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
@@ -26,8 +25,7 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_BASE64}
|
||||
func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk {
|
||||
encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, 20, b64CharsetMapping, b64EndChars)
|
||||
decodedSubstrings := make(map[string][]byte)
|
||||
|
||||
@@ -63,7 +61,7 @@ func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
}
|
||||
result.Write(chunk.Data[start:])
|
||||
chunk.Data = result.Bytes()
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package decoders
|
||||
|
||||
import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
@@ -14,15 +13,8 @@ func DefaultDecoders() []Decoder {
|
||||
}
|
||||
}
|
||||
|
||||
// DecodableChunk is a chunk that includes the type of decoder used.
|
||||
// This allows us to avoid a type assertion on each decoder.
|
||||
type DecodableChunk struct {
|
||||
*sources.Chunk
|
||||
DecoderType detectorspb.DecoderType
|
||||
}
|
||||
|
||||
type Decoder interface {
|
||||
FromChunk(chunk *sources.Chunk) *DecodableChunk
|
||||
FromChunk(chunk *sources.Chunk) *sources.Chunk
|
||||
}
|
||||
|
||||
// Fuzz is an entrypoint for go-fuzz, which is an AFL-style fuzzing tool.
|
||||
|
||||
@@ -5,24 +5,22 @@ import (
|
||||
"encoding/binary"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type UTF16 struct{}
|
||||
|
||||
func (d *UTF16) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
func (d *UTF16) FromChunk(chunk *sources.Chunk) *sources.Chunk {
|
||||
if chunk == nil || len(chunk.Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_UTF16}
|
||||
if utf16Data, err := utf16ToUTF8(chunk.Data); err == nil {
|
||||
if len(utf16Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
chunk.Data = utf16Data
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -4,25 +4,22 @@ import (
|
||||
"bytes"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type UTF8 struct{}
|
||||
|
||||
func (d *UTF8) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
func (d *UTF8) FromChunk(chunk *sources.Chunk) *sources.Chunk {
|
||||
if chunk == nil || len(chunk.Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_PLAIN}
|
||||
|
||||
if !utf8.Valid(chunk.Data) {
|
||||
chunk.Data = extractSubstrings(chunk.Data)
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
// extractSubstrings performs similarly to the strings binutil,
|
||||
|
||||
@@ -2,6 +2,7 @@ package airtableapikey
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
@@ -20,8 +21,9 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
appPat = regexp.MustCompile(`(app[a-zA-Z0-9_-]{14})`) // could be part of url
|
||||
keyPat = regexp.MustCompile(`\b(key[a-zA-Z0-9_-]{14})\b`)
|
||||
appPat = regexp.MustCompile(`(app[a-zA-Z0-9_-]{14})`) // could be part of url
|
||||
keyPat = regexp.MustCompile(`\b(key[a-zA-Z0-9_-]{14})\b`)
|
||||
personalPat = regexp.MustCompile(`(\bpat[[:alnum:]]{14}\.[[:alnum:]]{64}\b)`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -30,17 +32,31 @@ func (s Scanner) Keywords() []string {
|
||||
return []string{"airtable"}
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Error struct {
|
||||
Type string `json:"type"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify AirtableApiKey secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
appMatches := appPat.FindAllStringSubmatch(dataStr, -1)
|
||||
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
personalKeyMatches := personalPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
if len(keyMatches) == 0 {
|
||||
keyMatches = personalKeyMatches
|
||||
|
||||
}
|
||||
|
||||
for _, keyMatch := range keyMatches {
|
||||
if len(keyMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
keyRes := strings.TrimSpace(keyMatch[1])
|
||||
|
||||
for _, appMatch := range appMatches {
|
||||
@@ -67,6 +83,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 403 {
|
||||
var resp response
|
||||
if err = json.NewDecoder(res.Body).Decode(&resp); err == nil {
|
||||
// check if the error is due to invalid permissions or model not found
|
||||
if resp.Error.Type == "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND" {
|
||||
// The key is verified as it works, but the user must enumerate the tables or permissions for the key.
|
||||
s1.Verified = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(keyRes, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
|
||||
@@ -9,9 +9,10 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
@@ -99,8 +100,9 @@ func TestAirtableApiKey_FromChunk(t *testing.T) {
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("AirtableApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Airtable.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package sourcegraphcody
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(slk_[a-f0-9]{64})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"slk_"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Sourcegraphcody secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_SourcegraphCody,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://cody-gateway.sourcegraph.com/v1/limits", nil)
|
||||
req.Header.Add("Authorization", "Bearer "+resMatch)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 403 || res.StatusCode == 401 {
|
||||
// 403 when the gateway is not configured to accept the token,
|
||||
// 401 when the token is invalid.
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_SourcegraphCody
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package sourcegraphcody
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestSourcegraphcody_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("SOURCEGRAPHCODY")
|
||||
inactiveSecret := testSecrets.MustGetField("SOURCEGRAPHCODY_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SourcegraphCody,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SourcegraphCody,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SourcegraphCody,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SourcegraphCody,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Sourcegraphcody.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Sourcegraphcody.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -23,7 +23,6 @@ var (
|
||||
identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive.
|
||||
sidPat = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`)
|
||||
keyPat = regexp.MustCompile(`\b[0-9a-f]{32}\b`)
|
||||
client = common.SaneHttpClient()
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -59,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
client = s.client
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
ahocorasick "github.com/BobuSumisu/aho-corasick"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
// detectorKey is used to identify a detector in the keywordsToDetectors map.
|
||||
// Multiple detectors can have the same detector type but different versions.
|
||||
// This allows us to identify a detector by its type and version.
|
||||
type detectorKey struct {
|
||||
detectorType detectorspb.DetectorType
|
||||
version int
|
||||
}
|
||||
|
||||
// detectorInfo is used to store a detector and whether it should be verified.
|
||||
type detectorInfo struct {
|
||||
detectors.Detector
|
||||
shouldVerify bool
|
||||
}
|
||||
|
||||
// AhoCorasickCore encapsulates the operations and data structures used for keyword matching via the
|
||||
// Aho-Corasick algorithm. It is responsible for constructing and managing the trie for efficient
|
||||
// substring searches, as well as mapping keywords to their associated detectors for rapid lookups.
|
||||
type AhoCorasickCore struct {
|
||||
// prefilter is a ahocorasick struct used for doing efficient string
|
||||
// matching given a set of words. (keywords from the rules in the config)
|
||||
prefilter ahocorasick.Trie
|
||||
// Maps for efficient lookups during detection.
|
||||
detectorTypeToDetectorInfo map[detectorKey]detectorInfo
|
||||
detectors map[bool][]detectors.Detector
|
||||
keywordsToDetectors map[string][]detectorKey
|
||||
}
|
||||
|
||||
// NewAhoCorasickCore allocates and initializes a new instance of AhoCorasickCore.
|
||||
// It creates an empty keyword-to-detectors map for future string matching operations.
|
||||
// The map detectorTypeToDetectorInfo is pre-allocated based on the size of detectors
|
||||
// provided, for efficient storage and lookup of detector information.
|
||||
func NewAhoCorasickCore(detectors map[bool][]detectors.Detector) *AhoCorasickCore {
|
||||
return &AhoCorasickCore{
|
||||
keywordsToDetectors: make(map[string][]detectorKey),
|
||||
detectors: detectors,
|
||||
detectorTypeToDetectorInfo: make(map[detectorKey]detectorInfo, len(detectors[true])+len(detectors[false])),
|
||||
}
|
||||
}
|
||||
|
||||
// Setup initializes the internal state of AhoCorasickCore to prepare it for keyword matching.
|
||||
// This involves pre-filtering setup and lookup optimization, critical for the engine's performance.
|
||||
func (ac *AhoCorasickCore) Setup(ctx context.Context) {
|
||||
// Prepare maps for fast detector lookups, instead of scanning through an array of detectors for every chunk.
|
||||
var keywords []string
|
||||
for verify, detectorsSet := range ac.detectors {
|
||||
for _, d := range detectorsSet {
|
||||
key := createDetectorKey(d)
|
||||
ac.detectorTypeToDetectorInfo[key] = detectorInfo{Detector: d, shouldVerify: verify}
|
||||
keywords = ac.extractAndMapKeywords(d, key, keywords)
|
||||
}
|
||||
}
|
||||
|
||||
// Use the Ahocorasick algorithm to create a trie structure for efficient keyword matching.
|
||||
// This ensures that we can rapidly match against a vast set of keywords without individually comparing each one.
|
||||
ac.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
|
||||
ctx.Logger().V(4).Info("AhoCorasickCore lookups built")
|
||||
ctx.Logger().V(4).Info("AhoCorasickCore Setup complete")
|
||||
}
|
||||
|
||||
// createDetectorKey creates a unique key for each detector. This key based on type and version,
|
||||
// it ensures faster lookups and reduces redundancy in our main detector store.
|
||||
func createDetectorKey(d detectors.Detector) detectorKey {
|
||||
detectorType := d.Type()
|
||||
var version int
|
||||
if v, ok := d.(detectors.Versioner); ok {
|
||||
version = v.Version()
|
||||
}
|
||||
return detectorKey{detectorType: detectorType, version: version}
|
||||
}
|
||||
|
||||
// extractAndMapKeywords captures keywords associated with each detector and maps them.
|
||||
// This allows us to quickly determine which detectors are relevant based on the presence of certain keywords.
|
||||
func (ac *AhoCorasickCore) extractAndMapKeywords(d detectors.Detector, key detectorKey, keywords []string) []string {
|
||||
for _, kw := range d.Keywords() {
|
||||
kwLower := strings.ToLower(kw)
|
||||
keywords = append(keywords, kwLower)
|
||||
ac.keywordsToDetectors[kwLower] = append(ac.keywordsToDetectors[kwLower], key)
|
||||
}
|
||||
return keywords
|
||||
}
|
||||
|
||||
// MatchString performs a string match using the Aho-Corasick algorithm, returning an array of matches.
|
||||
// Designed for internal use within the AhoCorasickCore component.
|
||||
func (ac *AhoCorasickCore) MatchString(input string) []*ahocorasick.Match {
|
||||
return ac.prefilter.MatchString(strings.ToLower(input))
|
||||
}
|
||||
|
||||
// PopulateDetectorsByMatch populates the given detectorMap based on the Aho-Corasick match results.
|
||||
// This method is designed to reuse the same map for performance optimization,
|
||||
// reducing the need for repeated allocations within each detector worker in the engine.
|
||||
func (ac *AhoCorasickCore) PopulateDetectorsByMatch(match *ahocorasick.Match, detectors map[detectorspb.DetectorType]detectorInfo) bool {
|
||||
matchedKeys, ok := ac.keywordsToDetectors[match.MatchString()]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
for _, key := range matchedKeys {
|
||||
detectors[key.detectorType] = ac.detectorTypeToDetectorInfo[key]
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraph"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraphcody"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
|
||||
|
||||
@@ -1552,6 +1553,7 @@ func DefaultDetectors() []detectors.Detector {
|
||||
web3storage.Scanner{},
|
||||
&ramp.Scanner{},
|
||||
&anthropic.Scanner{},
|
||||
&sourcegraphcody.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+111
-98
@@ -3,11 +3,14 @@ package engine
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
ahocorasick "github.com/BobuSumisu/aho-corasick"
|
||||
lru "github.com/hashicorp/golang-lru"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
@@ -63,8 +66,9 @@ type Engine struct {
|
||||
onlyVerified bool
|
||||
printAvgDetectorTime bool
|
||||
|
||||
// ahoCorasickHandler manages the Aho-Corasick trie and related keyword lookups.
|
||||
ahoCorasickCore *AhoCorasickCore
|
||||
// prefilter is a ahocorasick struct used for doing efficient string
|
||||
// matching given a set of words (keywords from the rules in the config)
|
||||
prefilter ahocorasick.Trie
|
||||
|
||||
// Engine synchronization primitives.
|
||||
sourceManager *sources.SourceManager
|
||||
@@ -89,10 +93,9 @@ type Engine struct {
|
||||
dedupeCache *lru.Cache
|
||||
}
|
||||
|
||||
// Option is used to configure the engine during initialization using functional options.
|
||||
type Option func(*Engine)
|
||||
type EngineOption func(*Engine)
|
||||
|
||||
func WithConcurrency(concurrency uint8) Option {
|
||||
func WithConcurrency(concurrency uint8) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.concurrency = concurrency
|
||||
}
|
||||
@@ -100,7 +103,7 @@ func WithConcurrency(concurrency uint8) Option {
|
||||
|
||||
const ignoreTag = "trufflehog:ignore"
|
||||
|
||||
func WithDetectors(verify bool, d ...detectors.Detector) Option {
|
||||
func WithDetectors(verify bool, d ...detectors.Detector) EngineOption {
|
||||
return func(e *Engine) {
|
||||
if e.detectors == nil {
|
||||
e.detectors = make(map[bool][]detectors.Detector)
|
||||
@@ -113,7 +116,7 @@ func WithDetectors(verify bool, d ...detectors.Detector) Option {
|
||||
}
|
||||
}
|
||||
|
||||
func WithDecoders(decoders ...decoders.Decoder) Option {
|
||||
func WithDecoders(decoders ...decoders.Decoder) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.decoders = decoders
|
||||
}
|
||||
@@ -121,14 +124,14 @@ func WithDecoders(decoders ...decoders.Decoder) Option {
|
||||
|
||||
// WithFilterUnverified sets the filterUnverified flag on the engine. If set to
|
||||
// true, the engine will only return the first unverified result for a chunk for a detector.
|
||||
func WithFilterUnverified(filter bool) Option {
|
||||
func WithFilterUnverified(filter bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.filterUnverified = filter
|
||||
}
|
||||
}
|
||||
|
||||
// WithFilterEntropy filters out unverified results using Shannon entropy.
|
||||
func WithFilterEntropy(entropy float64) Option {
|
||||
func WithFilterEntropy(entropy float64) EngineOption {
|
||||
return func(e *Engine) {
|
||||
if entropy > 0 {
|
||||
e.filterEntropy = &entropy
|
||||
@@ -138,7 +141,7 @@ func WithFilterEntropy(entropy float64) Option {
|
||||
|
||||
// WithOnlyVerified sets the onlyVerified flag on the engine. If set to true,
|
||||
// the engine will only print verified results.
|
||||
func WithOnlyVerified(onlyVerified bool) Option {
|
||||
func WithOnlyVerified(onlyVerified bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.onlyVerified = onlyVerified
|
||||
}
|
||||
@@ -150,7 +153,7 @@ func WithOnlyVerified(onlyVerified bool) Option {
|
||||
// the engine is configured to print the results.
|
||||
// Calculating the average time taken by each detector is an expensive operation
|
||||
// and should be avoided unless specified by the user.
|
||||
func WithPrintAvgDetectorTime(printAvgDetectorTime bool) Option {
|
||||
func WithPrintAvgDetectorTime(printAvgDetectorTime bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.printAvgDetectorTime = printAvgDetectorTime
|
||||
}
|
||||
@@ -160,7 +163,7 @@ func WithPrintAvgDetectorTime(printAvgDetectorTime bool) Option {
|
||||
// the filterFunc returns true, the detector will be included for scanning.
|
||||
// This option applies to the existing list of detectors configured, so the
|
||||
// order this option appears matters. All filtering happens before scanning.
|
||||
func WithFilterDetectors(filterFunc func(detectors.Detector) bool) Option {
|
||||
func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
// If no detectors are configured, do nothing.
|
||||
if e.detectors == nil {
|
||||
@@ -172,7 +175,7 @@ func WithFilterDetectors(filterFunc func(detectors.Detector) bool) Option {
|
||||
}
|
||||
|
||||
// WithPrinter sets the Printer on the engine.
|
||||
func WithPrinter(printer Printer) Option {
|
||||
func WithPrinter(printer Printer) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.printer = printer
|
||||
}
|
||||
@@ -266,60 +269,32 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
|
||||
return avgTime
|
||||
}
|
||||
|
||||
// Start initializes and activates the engine's processing pipeline.
|
||||
// It sets up various default configurations, prepares lookup structures for
|
||||
// detectors, conducts basic sanity checks, and kickstarts all necessary workers.
|
||||
// Once started, the engine begins processing input data to identify secrets.
|
||||
func Start(ctx context.Context, options ...Option) (*Engine, error) {
|
||||
e := &Engine{}
|
||||
|
||||
if err := e.initialize(ctx, options...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e.setDefaults(ctx)
|
||||
ctx.Logger().V(4).Info("setting up aho-corasick core")
|
||||
e.ahoCorasickCore.Setup(ctx)
|
||||
e.sanityChecks(ctx)
|
||||
e.startWorkers(ctx)
|
||||
|
||||
return e, nil
|
||||
}
|
||||
|
||||
const defaultChannelBuffer = 1
|
||||
|
||||
// initialize prepares the engine's internal structures. The LRU cache optimizes
|
||||
// deduplication efforts, allowing the engine to quickly check if a chunk has
|
||||
// been processed before, thereby saving computational overhead.
|
||||
func (e *Engine) initialize(ctx context.Context, options ...Option) error {
|
||||
// TODO (ahrav): Determine the optimal cache size.
|
||||
const cacheSize = 512 // number of entries in the LRU cache
|
||||
// Start the engine with options.
|
||||
func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
|
||||
const (
|
||||
defaultChannelBuffer = 1
|
||||
// TODO (ahrav): Determine the optimal cache size.
|
||||
cacheSize = 512 // number of entries in the LRU cache
|
||||
)
|
||||
|
||||
cache, err := lru.New(cacheSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize LRU cache: %w", err)
|
||||
return nil, fmt.Errorf("failed to initialize LRU cache: %w", err)
|
||||
}
|
||||
|
||||
// Channels are used for communication between different parts of the engine,
|
||||
// ensuring that data flows smoothly without race conditions.
|
||||
e.detectableChunksChan = make(chan detectableChunk, defaultChannelBuffer)
|
||||
e.results = make(chan detectors.ResultWithMetadata, defaultChannelBuffer)
|
||||
e.dedupeCache = cache
|
||||
e.printer = new(output.PlainPrinter)
|
||||
e.metrics = runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}}
|
||||
e := &Engine{
|
||||
detectableChunksChan: make(chan detectableChunk, defaultChannelBuffer),
|
||||
results: make(chan detectors.ResultWithMetadata, defaultChannelBuffer),
|
||||
dedupeCache: cache,
|
||||
printer: new(output.PlainPrinter), // default printer
|
||||
metrics: runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}},
|
||||
}
|
||||
|
||||
for _, option := range options {
|
||||
option(e)
|
||||
}
|
||||
ctx.Logger().V(4).Info("engine initialized")
|
||||
e.ahoCorasickCore = NewAhoCorasickCore(e.detectors)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// setDefaults ensures that if specific engine properties aren't provided,
|
||||
// they're set to reasonable default values. It makes the engine robust to
|
||||
// incomplete configuration.
|
||||
func (e *Engine) setDefaults(ctx context.Context) {
|
||||
// Set defaults.
|
||||
if e.concurrency == 0 {
|
||||
numCPU := runtime.NumCPU()
|
||||
ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU)
|
||||
@@ -327,13 +302,13 @@ func (e *Engine) setDefaults(ctx context.Context) {
|
||||
}
|
||||
ctx.Logger().V(3).Info("engine started", "workers", e.concurrency)
|
||||
|
||||
// Create SourceManager.
|
||||
e.sourceManager = sources.NewManager(
|
||||
sources.WithConcurrentSources(int(e.concurrency)),
|
||||
sources.WithConcurrentUnits(int(e.concurrency)),
|
||||
sources.WithBufferedOutput(defaultChannelBuffer),
|
||||
sources.WithSourceUnits(),
|
||||
)
|
||||
|
||||
// Default decoders handle common encoding formats.
|
||||
if len(e.decoders) == 0 {
|
||||
e.decoders = decoders.DefaultDecoders()
|
||||
}
|
||||
@@ -343,30 +318,46 @@ func (e *Engine) setDefaults(ctx context.Context) {
|
||||
e.detectors[true] = DefaultDetectors()
|
||||
e.detectors[false] = []detectors.Detector{}
|
||||
}
|
||||
ctx.Logger().V(4).Info("default engine options set")
|
||||
}
|
||||
|
||||
// Sanity check detectors for duplicate configuration. Only log in case
|
||||
// a detector has been configured in a way that isn't represented by
|
||||
// the DetectorID (type and version).
|
||||
func (e *Engine) sanityChecks(ctx context.Context) {
|
||||
dets := append(e.detectors[true], e.detectors[false]...)
|
||||
seenDetectors := make(map[config.DetectorID]struct{}, len(dets))
|
||||
for _, det := range dets {
|
||||
id := config.GetDetectorID(det)
|
||||
if _, ok := seenDetectors[id]; ok && id.ID != detectorspb.DetectorType_CustomRegex {
|
||||
ctx.Logger().Info("possible duplicate detector configured", "detector", id)
|
||||
// build ahocorasick prefilter for efficient string matching
|
||||
// on keywords
|
||||
keywords := []string{}
|
||||
for _, d := range e.detectors[false] {
|
||||
for _, kw := range d.Keywords() {
|
||||
keywords = append(keywords, strings.ToLower(kw))
|
||||
}
|
||||
}
|
||||
for _, d := range e.detectors[true] {
|
||||
for _, kw := range d.Keywords() {
|
||||
keywords = append(keywords, strings.ToLower(kw))
|
||||
}
|
||||
}
|
||||
e.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
|
||||
|
||||
ctx.Logger().V(3).Info("loaded decoders", "count", len(e.decoders))
|
||||
ctx.Logger().V(3).Info("loaded detectors",
|
||||
"total", len(e.detectors[true])+len(e.detectors[false]),
|
||||
"verification_enabled", len(e.detectors[true]),
|
||||
"verification_disabled", len(e.detectors[false]),
|
||||
)
|
||||
|
||||
// Sanity check detectors for duplicate configuration. Only log in case
|
||||
// a detector has been configured in a way that isn't represented by
|
||||
// the DetectorID (type and version).
|
||||
{
|
||||
dets := append(e.detectors[true], e.detectors[false]...)
|
||||
seenDetectors := make(map[config.DetectorID]struct{}, len(dets))
|
||||
for _, det := range dets {
|
||||
id := config.GetDetectorID(det)
|
||||
if _, ok := seenDetectors[id]; ok && id.ID != detectorspb.DetectorType_CustomRegex {
|
||||
ctx.Logger().Info("possible duplicate detector configured", "detector", id)
|
||||
}
|
||||
seenDetectors[id] = struct{}{}
|
||||
}
|
||||
seenDetectors[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// startWorkers initiates all necessary workers. Workers handle processing of
|
||||
// chunks concurrently. Separating the initialization of different types of
|
||||
// workers helps in scalability and makes it easier to diagnose issues.
|
||||
func (e *Engine) startWorkers(ctx context.Context) {
|
||||
// Scanner workers process input data and extract chunks for detectors.
|
||||
ctx.Logger().V(2).Info("starting scanner workers", "count", e.concurrency)
|
||||
// Run the Secret scanner workers and Notifier pipelines.
|
||||
for worker := uint64(0); worker < uint64(e.concurrency); worker++ {
|
||||
e.workersWg.Add(1)
|
||||
go func() {
|
||||
@@ -377,7 +368,6 @@ func (e *Engine) startWorkers(ctx context.Context) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Detector workers apply keyword matching, regexes and API calls to detect secrets in chunks.
|
||||
const detectorWorkerMultiplier = 50
|
||||
ctx.Logger().V(2).Info("starting detector workers", "count", e.concurrency*detectorWorkerMultiplier)
|
||||
for worker := uint64(0); worker < uint64(e.concurrency*detectorWorkerMultiplier); worker++ {
|
||||
@@ -390,7 +380,6 @@ func (e *Engine) startWorkers(ctx context.Context) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Notifier workers communicate detected issues to the user or any downstream systems.
|
||||
// We want 1/4th of the notifier workers as the number of scanner workers.
|
||||
const notifierWorkerRatio = 4
|
||||
maxNotifierWorkers := 1
|
||||
@@ -407,6 +396,8 @@ func (e *Engine) startWorkers(ctx context.Context) {
|
||||
e.notifyResults(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// Finish waits for running sources to complete and workers to finish scanning
|
||||
@@ -455,42 +446,64 @@ type detectableChunk struct {
|
||||
func (e *Engine) detectorWorker(ctx context.Context) {
|
||||
var wgDetect sync.WaitGroup
|
||||
|
||||
// Reuse the same map to avoid allocations.
|
||||
const avgDetectorsPerChunk = 2
|
||||
chunkSpecificDetectors := make(map[detectorspb.DetectorType]detectorInfo, avgDetectorsPerChunk)
|
||||
for originalChunk := range e.ChunksChan() {
|
||||
for chunk := range sources.Chunker(originalChunk) {
|
||||
matchedKeywords := make(map[string]struct{})
|
||||
atomic.AddUint64(&e.metrics.BytesScanned, uint64(len(chunk.Data)))
|
||||
for _, decoder := range e.decoders {
|
||||
var decoderType detectorspb.DecoderType
|
||||
switch decoder.(type) {
|
||||
case *decoders.UTF8:
|
||||
decoderType = detectorspb.DecoderType_PLAIN
|
||||
case *decoders.Base64:
|
||||
decoderType = detectorspb.DecoderType_BASE64
|
||||
case *decoders.UTF16:
|
||||
decoderType = detectorspb.DecoderType_UTF16
|
||||
default:
|
||||
ctx.Logger().Info("unknown decoder type", "type", reflect.TypeOf(decoder).String())
|
||||
decoderType = detectorspb.DecoderType_UNKNOWN
|
||||
}
|
||||
|
||||
decoded := decoder.FromChunk(chunk)
|
||||
|
||||
if decoded == nil {
|
||||
ctx.Logger().V(4).Info("no decoder found for chunk", "chunk", chunk)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, match := range e.ahoCorasickCore.MatchString(string(decoded.Chunk.Data)) {
|
||||
if !e.ahoCorasickCore.PopulateDetectorsByMatch(match, chunkSpecificDetectors) {
|
||||
continue
|
||||
}
|
||||
// build a map of all keywords that were matched in the chunk
|
||||
for _, m := range e.prefilter.MatchString(strings.ToLower(string(decoded.Data))) {
|
||||
matchedKeywords[strings.ToLower(m.MatchString())] = struct{}{}
|
||||
}
|
||||
|
||||
for k, detector := range chunkSpecificDetectors {
|
||||
decoded.Chunk.Verify = detector.shouldVerify
|
||||
wgDetect.Add(1)
|
||||
e.detectableChunksChan <- detectableChunk{
|
||||
chunk: *decoded.Chunk,
|
||||
detector: detector,
|
||||
decoder: decoded.DecoderType,
|
||||
wgDoneFn: wgDetect.Done,
|
||||
for verify, detectorsSet := range e.detectors {
|
||||
for _, detector := range detectorsSet {
|
||||
chunkContainsKeyword := false
|
||||
for _, kw := range detector.Keywords() {
|
||||
if _, ok := matchedKeywords[strings.ToLower(kw)]; ok {
|
||||
chunkContainsKeyword = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !chunkContainsKeyword {
|
||||
continue
|
||||
}
|
||||
|
||||
decoded.Verify = verify
|
||||
wgDetect.Add(1)
|
||||
e.detectableChunksChan <- detectableChunk{
|
||||
chunk: *decoded,
|
||||
detector: detector,
|
||||
decoder: decoderType,
|
||||
wgDoneFn: wgDetect.Done,
|
||||
}
|
||||
}
|
||||
delete(chunkSpecificDetectors, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
atomic.AddUint64(&e.metrics.ChunksScanned, 1)
|
||||
}
|
||||
wgDetect.Wait()
|
||||
ctx.Logger().V(4).Info("finished scanning chunks")
|
||||
}
|
||||
|
||||
func (e *Engine) detectChunks(ctx context.Context) {
|
||||
|
||||
@@ -2,7 +2,6 @@ package engine
|
||||
|
||||
import (
|
||||
"os"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -121,10 +120,9 @@ func BenchmarkGitEngine(b *testing.B) {
|
||||
defer cancel()
|
||||
|
||||
e, err := Start(ctx,
|
||||
WithConcurrency(uint8(runtime.NumCPU())),
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(false, DefaultDetectors()...),
|
||||
WithPrinter(new(discardPrinter)),
|
||||
)
|
||||
assert.Nil(b, err)
|
||||
|
||||
@@ -135,7 +133,6 @@ func BenchmarkGitEngine(b *testing.B) {
|
||||
}
|
||||
}()
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
// TODO: this is measuring the time it takes to initialize the source
|
||||
// and not to do the full scan
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/travisci"
|
||||
)
|
||||
|
||||
// ScanTravisCI scans TravisCI logs.
|
||||
func (e *Engine) ScanTravisCI(ctx context.Context, token string) error {
|
||||
connection := &sourcespb.TravisCI{
|
||||
Credential: &sourcespb.TravisCI_Token{
|
||||
Token: token,
|
||||
},
|
||||
}
|
||||
|
||||
var conn anypb.Any
|
||||
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "failed to marshal Travis CI connection")
|
||||
return err
|
||||
}
|
||||
|
||||
sourceName := "trufflehog - Travis CI"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, travisci.SourceType)
|
||||
|
||||
travisSource := &travisci.Source{}
|
||||
if err := travisSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, travisSource)
|
||||
return err
|
||||
}
|
||||
@@ -1007,6 +1007,7 @@ const (
|
||||
DetectorType_Anthropic DetectorType = 933
|
||||
DetectorType_Ramp DetectorType = 934
|
||||
DetectorType_Klaviyo DetectorType = 935
|
||||
DetectorType_SourcegraphCody DetectorType = 936
|
||||
)
|
||||
|
||||
// Enum value maps for DetectorType.
|
||||
@@ -1944,6 +1945,7 @@ var (
|
||||
933: "Anthropic",
|
||||
934: "Ramp",
|
||||
935: "Klaviyo",
|
||||
936: "SourcegraphCody",
|
||||
}
|
||||
DetectorType_value = map[string]int32{
|
||||
"Alibaba": 0,
|
||||
@@ -2878,6 +2880,7 @@ var (
|
||||
"Anthropic": 933,
|
||||
"Ramp": 934,
|
||||
"Klaviyo": 935,
|
||||
"SourcegraphCody": 936,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -3256,7 +3259,7 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
|
||||
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
|
||||
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x90, 0x75, 0x0a, 0x0c, 0x44,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xa6, 0x75, 0x0a, 0x0c, 0x44,
|
||||
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
|
||||
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
|
||||
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
|
||||
@@ -4193,12 +4196,13 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x12, 0x12, 0x0a, 0x0d, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x74, 0x53, 0x63, 0x61, 0x6c, 0x65, 0x44,
|
||||
0x62, 0x10, 0xa4, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x41, 0x6e, 0x74, 0x68, 0x72, 0x6f, 0x70, 0x69,
|
||||
0x63, 0x10, 0xa5, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x52, 0x61, 0x6d, 0x70, 0x10, 0xa6, 0x07, 0x12,
|
||||
0x0c, 0x0a, 0x07, 0x4b, 0x6c, 0x61, 0x76, 0x69, 0x79, 0x6f, 0x10, 0xa7, 0x07, 0x42, 0x3d, 0x5a,
|
||||
0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66,
|
||||
0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66,
|
||||
0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62,
|
||||
0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
|
||||
0x6f, 0x74, 0x6f, 0x33,
|
||||
0x0c, 0x0a, 0x07, 0x4b, 0x6c, 0x61, 0x76, 0x69, 0x79, 0x6f, 0x10, 0xa7, 0x07, 0x12, 0x14, 0x0a,
|
||||
0x0f, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x43, 0x6f, 0x64, 0x79,
|
||||
0x10, 0xa8, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
|
||||
0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
|
||||
0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
|
||||
0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
|
||||
0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -486,6 +486,117 @@ var _ interface {
|
||||
ErrorName() string
|
||||
} = CircleCIValidationError{}
|
||||
|
||||
// Validate checks the field values on TravisCI with the rules defined in the
|
||||
// proto definition for this message. If any rules are violated, the first
|
||||
// error encountered is returned, or nil if there are no violations.
|
||||
func (m *TravisCI) Validate() error {
|
||||
return m.validate(false)
|
||||
}
|
||||
|
||||
// ValidateAll checks the field values on TravisCI with the rules defined in
|
||||
// the proto definition for this message. If any rules are violated, the
|
||||
// result is a list of violation errors wrapped in TravisCIMultiError, or nil
|
||||
// if none found.
|
||||
func (m *TravisCI) ValidateAll() error {
|
||||
return m.validate(true)
|
||||
}
|
||||
|
||||
func (m *TravisCI) validate(all bool) error {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var errors []error
|
||||
|
||||
// no validation rules for Username
|
||||
|
||||
// no validation rules for Repository
|
||||
|
||||
// no validation rules for BuildNumber
|
||||
|
||||
// no validation rules for JobNumber
|
||||
|
||||
// no validation rules for Link
|
||||
|
||||
// no validation rules for Public
|
||||
|
||||
if len(errors) > 0 {
|
||||
return TravisCIMultiError(errors)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TravisCIMultiError is an error wrapping multiple validation errors returned
|
||||
// by TravisCI.ValidateAll() if the designated constraints aren't met.
|
||||
type TravisCIMultiError []error
|
||||
|
||||
// Error returns a concatenation of all the error messages it wraps.
|
||||
func (m TravisCIMultiError) Error() string {
|
||||
var msgs []string
|
||||
for _, err := range m {
|
||||
msgs = append(msgs, err.Error())
|
||||
}
|
||||
return strings.Join(msgs, "; ")
|
||||
}
|
||||
|
||||
// AllErrors returns a list of validation violation errors.
|
||||
func (m TravisCIMultiError) AllErrors() []error { return m }
|
||||
|
||||
// TravisCIValidationError is the validation error returned by
|
||||
// TravisCI.Validate if the designated constraints aren't met.
|
||||
type TravisCIValidationError struct {
|
||||
field string
|
||||
reason string
|
||||
cause error
|
||||
key bool
|
||||
}
|
||||
|
||||
// Field function returns field value.
|
||||
func (e TravisCIValidationError) Field() string { return e.field }
|
||||
|
||||
// Reason function returns reason value.
|
||||
func (e TravisCIValidationError) Reason() string { return e.reason }
|
||||
|
||||
// Cause function returns cause value.
|
||||
func (e TravisCIValidationError) Cause() error { return e.cause }
|
||||
|
||||
// Key function returns key value.
|
||||
func (e TravisCIValidationError) Key() bool { return e.key }
|
||||
|
||||
// ErrorName returns error name.
|
||||
func (e TravisCIValidationError) ErrorName() string { return "TravisCIValidationError" }
|
||||
|
||||
// Error satisfies the builtin error interface
|
||||
func (e TravisCIValidationError) Error() string {
|
||||
cause := ""
|
||||
if e.cause != nil {
|
||||
cause = fmt.Sprintf(" | caused by: %v", e.cause)
|
||||
}
|
||||
|
||||
key := ""
|
||||
if e.key {
|
||||
key = "key for "
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"invalid %sTravisCI.%s: %s%s",
|
||||
key,
|
||||
e.field,
|
||||
e.reason,
|
||||
cause)
|
||||
}
|
||||
|
||||
var _ error = TravisCIValidationError{}
|
||||
|
||||
var _ interface {
|
||||
Field() string
|
||||
Reason() string
|
||||
Key() bool
|
||||
Cause() error
|
||||
ErrorName() string
|
||||
} = TravisCIValidationError{}
|
||||
|
||||
// Validate checks the field values on Confluence with the rules defined in the
|
||||
// proto definition for this message. If any rules are violated, the first
|
||||
// error encountered is returned, or nil if there are no violations.
|
||||
@@ -3926,6 +4037,37 @@ func (m *MetaData) validate(all bool) error {
|
||||
}
|
||||
}
|
||||
|
||||
case *MetaData_TravisCI:
|
||||
|
||||
if all {
|
||||
switch v := interface{}(m.GetTravisCI()).(type) {
|
||||
case interface{ ValidateAll() error }:
|
||||
if err := v.ValidateAll(); err != nil {
|
||||
errors = append(errors, MetaDataValidationError{
|
||||
field: "TravisCI",
|
||||
reason: "embedded message failed validation",
|
||||
cause: err,
|
||||
})
|
||||
}
|
||||
case interface{ Validate() error }:
|
||||
if err := v.Validate(); err != nil {
|
||||
errors = append(errors, MetaDataValidationError{
|
||||
field: "TravisCI",
|
||||
reason: "embedded message failed validation",
|
||||
cause: err,
|
||||
})
|
||||
}
|
||||
}
|
||||
} else if v, ok := interface{}(m.GetTravisCI()).(interface{ Validate() error }); ok {
|
||||
if err := v.Validate(); err != nil {
|
||||
return MetaDataValidationError{
|
||||
field: "TravisCI",
|
||||
reason: "embedded message failed validation",
|
||||
cause: err,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if len(errors) > 0 {
|
||||
|
||||
+741
-640
File diff suppressed because it is too large
Load Diff
@@ -667,6 +667,124 @@ var _ interface {
|
||||
ErrorName() string
|
||||
} = CircleCIValidationError{}
|
||||
|
||||
// Validate checks the field values on TravisCI with the rules defined in the
|
||||
// proto definition for this message. If any rules are violated, the first
|
||||
// error encountered is returned, or nil if there are no violations.
|
||||
func (m *TravisCI) Validate() error {
|
||||
return m.validate(false)
|
||||
}
|
||||
|
||||
// ValidateAll checks the field values on TravisCI with the rules defined in
|
||||
// the proto definition for this message. If any rules are violated, the
|
||||
// result is a list of violation errors wrapped in TravisCIMultiError, or nil
|
||||
// if none found.
|
||||
func (m *TravisCI) ValidateAll() error {
|
||||
return m.validate(true)
|
||||
}
|
||||
|
||||
func (m *TravisCI) validate(all bool) error {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var errors []error
|
||||
|
||||
if _, err := url.Parse(m.GetEndpoint()); err != nil {
|
||||
err = TravisCIValidationError{
|
||||
field: "Endpoint",
|
||||
reason: "value must be a valid URI",
|
||||
cause: err,
|
||||
}
|
||||
if !all {
|
||||
return err
|
||||
}
|
||||
errors = append(errors, err)
|
||||
}
|
||||
|
||||
switch m.Credential.(type) {
|
||||
|
||||
case *TravisCI_Token:
|
||||
// no validation rules for Token
|
||||
|
||||
}
|
||||
|
||||
if len(errors) > 0 {
|
||||
return TravisCIMultiError(errors)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TravisCIMultiError is an error wrapping multiple validation errors returned
|
||||
// by TravisCI.ValidateAll() if the designated constraints aren't met.
|
||||
type TravisCIMultiError []error
|
||||
|
||||
// Error returns a concatenation of all the error messages it wraps.
|
||||
func (m TravisCIMultiError) Error() string {
|
||||
var msgs []string
|
||||
for _, err := range m {
|
||||
msgs = append(msgs, err.Error())
|
||||
}
|
||||
return strings.Join(msgs, "; ")
|
||||
}
|
||||
|
||||
// AllErrors returns a list of validation violation errors.
|
||||
func (m TravisCIMultiError) AllErrors() []error { return m }
|
||||
|
||||
// TravisCIValidationError is the validation error returned by
|
||||
// TravisCI.Validate if the designated constraints aren't met.
|
||||
type TravisCIValidationError struct {
|
||||
field string
|
||||
reason string
|
||||
cause error
|
||||
key bool
|
||||
}
|
||||
|
||||
// Field function returns field value.
|
||||
func (e TravisCIValidationError) Field() string { return e.field }
|
||||
|
||||
// Reason function returns reason value.
|
||||
func (e TravisCIValidationError) Reason() string { return e.reason }
|
||||
|
||||
// Cause function returns cause value.
|
||||
func (e TravisCIValidationError) Cause() error { return e.cause }
|
||||
|
||||
// Key function returns key value.
|
||||
func (e TravisCIValidationError) Key() bool { return e.key }
|
||||
|
||||
// ErrorName returns error name.
|
||||
func (e TravisCIValidationError) ErrorName() string { return "TravisCIValidationError" }
|
||||
|
||||
// Error satisfies the builtin error interface
|
||||
func (e TravisCIValidationError) Error() string {
|
||||
cause := ""
|
||||
if e.cause != nil {
|
||||
cause = fmt.Sprintf(" | caused by: %v", e.cause)
|
||||
}
|
||||
|
||||
key := ""
|
||||
if e.key {
|
||||
key = "key for "
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"invalid %sTravisCI.%s: %s%s",
|
||||
key,
|
||||
e.field,
|
||||
e.reason,
|
||||
cause)
|
||||
}
|
||||
|
||||
var _ error = TravisCIValidationError{}
|
||||
|
||||
var _ interface {
|
||||
Field() string
|
||||
Reason() string
|
||||
Key() bool
|
||||
Cause() error
|
||||
ErrorName() string
|
||||
} = TravisCIValidationError{}
|
||||
|
||||
// Validate checks the field values on Confluence with the rules defined in the
|
||||
// proto definition for this message. If any rules are violated, the first
|
||||
// error encountered is returned, or nil if there are no violations.
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
@@ -40,7 +40,7 @@ func TestNewScanErrors(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := NewScanErrors()
|
||||
|
||||
if !reflect.DeepEqual(got, tc.want) {
|
||||
if !errors.Is(got.Errors(), tc.want.Errors()) {
|
||||
t.Errorf("got %+v, want %+v", got, tc.want)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -39,8 +39,7 @@ type Source struct {
|
||||
// Ensure the Source satisfies the interfaces at compile time
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
var _ sources.SourceUnitEnumerator = (*Source)(nil)
|
||||
var _ sources.SourceUnitChunker = (*Source)(nil)
|
||||
var _ sources.SourceUnitEnumChunker = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest"
|
||||
)
|
||||
|
||||
func TestSource_Scan(t *testing.T) {
|
||||
@@ -84,23 +85,15 @@ func TestSource_Scan(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestScanFile(t *testing.T) {
|
||||
tmpfile, err := os.CreateTemp("", "example.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.Remove(tmpfile.Name())
|
||||
|
||||
chunkSize := sources.ChunkSize
|
||||
secretPart1 := "SECRET"
|
||||
secretPart2 := "SPLIT"
|
||||
// Split the secret into two parts and pad the rest of the chunk with A's.
|
||||
data := strings.Repeat("A", chunkSize-len(secretPart1)) + secretPart1 + secretPart2 + strings.Repeat("A", chunkSize-len(secretPart2))
|
||||
|
||||
_, err = tmpfile.Write([]byte(data))
|
||||
assert.Nil(t, err)
|
||||
|
||||
err = tmpfile.Close()
|
||||
tmpfile, cleanup, err := createTempFile("", data)
|
||||
assert.Nil(t, err)
|
||||
defer cleanup()
|
||||
|
||||
source := &Source{}
|
||||
chunksChan := make(chan *sources.Chunk, 2)
|
||||
@@ -120,3 +113,195 @@ func TestScanFile(t *testing.T) {
|
||||
|
||||
assert.Contains(t, foundSecret, secretPart1+secretPart2)
|
||||
}
|
||||
|
||||
func TestEnumerate(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
|
||||
// Setup the connection to test enumeration.
|
||||
units := []string{
|
||||
"/one", "/two", "/three",
|
||||
"/path/to/dir/", "/path/to/another/dir/",
|
||||
}
|
||||
conn, err := anypb.New(&sourcespb.Filesystem{
|
||||
Paths: units[0:3],
|
||||
Directories: units[3:],
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Initialize the source.
|
||||
s := Source{}
|
||||
err = s.Init(ctx, "test enumerate", 0, 0, true, conn, 1)
|
||||
assert.NoError(t, err)
|
||||
|
||||
reporter := sourcestest.TestReporter{}
|
||||
err = s.Enumerate(ctx, &reporter)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, len(units), len(reporter.Units))
|
||||
assert.Equal(t, 0, len(reporter.UnitErrs))
|
||||
for _, unit := range reporter.Units {
|
||||
assert.Contains(t, units, unit.SourceUnitID())
|
||||
}
|
||||
for _, unit := range units {
|
||||
assert.Contains(t, reporter.Units, sources.CommonSourceUnit{ID: unit})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkUnit(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
|
||||
// Setup test file to chunk.
|
||||
fileContents := "TestChunkUnit"
|
||||
tmpfile, cleanup, err := createTempFile("", fileContents)
|
||||
assert.NoError(t, err)
|
||||
defer cleanup()
|
||||
|
||||
tmpdir, cleanup, err := createTempDir("", "foo", "bar", "baz")
|
||||
assert.NoError(t, err)
|
||||
defer cleanup()
|
||||
|
||||
conn, err := anypb.New(&sourcespb.Filesystem{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Initialize the source.
|
||||
s := Source{}
|
||||
err = s.Init(ctx, "test chunk unit", 0, 0, true, conn, 1)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Happy path single file.
|
||||
reporter := sourcestest.TestReporter{}
|
||||
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
|
||||
ID: tmpfile.Name(),
|
||||
}, &reporter)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Happy path directory.
|
||||
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
|
||||
ID: tmpdir,
|
||||
}, &reporter)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Error path.
|
||||
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
|
||||
ID: "/file/not/found",
|
||||
}, &reporter)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 4, len(reporter.Chunks))
|
||||
assert.Equal(t, 1, len(reporter.ChunkErrs))
|
||||
dataFound := make(map[string]struct{}, 4)
|
||||
for _, chunk := range reporter.Chunks {
|
||||
dataFound[string(chunk.Data)] = struct{}{}
|
||||
}
|
||||
assert.Contains(t, dataFound, fileContents)
|
||||
assert.Contains(t, dataFound, "foo")
|
||||
assert.Contains(t, dataFound, "bar")
|
||||
assert.Contains(t, dataFound, "baz")
|
||||
}
|
||||
|
||||
func TestEnumerateReporterErr(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
|
||||
// Setup the connection to test enumeration.
|
||||
units := []string{
|
||||
"/one", "/two", "/three",
|
||||
"/path/to/dir/", "/path/to/another/dir/",
|
||||
}
|
||||
conn, err := anypb.New(&sourcespb.Filesystem{
|
||||
Paths: units[0:3],
|
||||
Directories: units[3:],
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Initialize the source.
|
||||
s := Source{}
|
||||
err = s.Init(ctx, "test enumerate", 0, 0, true, conn, 1)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Enumerate should always return an error if the reporter returns an
|
||||
// error.
|
||||
reporter := sourcestest.ErrReporter{}
|
||||
err = s.Enumerate(ctx, &reporter)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestChunkUnitReporterErr(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
|
||||
// Setup test file to chunk.
|
||||
tmpfile, err := os.CreateTemp("", "example.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.Remove(tmpfile.Name())
|
||||
|
||||
fileContents := []byte("TestChunkUnit")
|
||||
_, err = tmpfile.Write(fileContents)
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, tmpfile.Close())
|
||||
|
||||
conn, err := anypb.New(&sourcespb.Filesystem{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Initialize the source.
|
||||
s := Source{}
|
||||
err = s.Init(ctx, "test chunk unit", 0, 0, true, conn, 1)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Happy path. ChunkUnit should always return an error if the reporter
|
||||
// returns an error.
|
||||
reporter := sourcestest.ErrReporter{}
|
||||
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
|
||||
ID: tmpfile.Name(),
|
||||
}, &reporter)
|
||||
assert.Error(t, err)
|
||||
|
||||
// Error path. ChunkUnit should always return an error if the reporter
|
||||
// returns an error.
|
||||
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
|
||||
ID: "/file/not/found",
|
||||
}, &reporter)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
// createTempFile is a helper function to create a temporary file in the given
|
||||
// directory with the provided contents. If dir is "", the operating system's
|
||||
// temp directory is used.
|
||||
func createTempFile(dir string, contents string) (*os.File, func(), error) {
|
||||
tmpfile, err := os.CreateTemp(dir, "trufflehogtest")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if _, err := tmpfile.Write([]byte(contents)); err != nil {
|
||||
_ = os.Remove(tmpfile.Name())
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := tmpfile.Close(); err != nil {
|
||||
_ = os.Remove(tmpfile.Name())
|
||||
return nil, nil, err
|
||||
}
|
||||
return tmpfile, func() { _ = os.Remove(tmpfile.Name()) }, nil
|
||||
}
|
||||
|
||||
// createTempDir is a helper function to create a temporary directory in the
|
||||
// given directory with files containing the provided contents. If dir is "",
|
||||
// the operating system's temp directory is used.
|
||||
func createTempDir(dir string, contents ...string) (string, func(), error) {
|
||||
tmpdir, err := os.MkdirTemp(dir, "trufflehogtest")
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
for _, content := range contents {
|
||||
if _, _, err := createTempFile(tmpdir, content); err != nil {
|
||||
_ = os.RemoveAll(tmpdir)
|
||||
return "", nil, err
|
||||
}
|
||||
}
|
||||
return tmpdir, func() { _ = os.RemoveAll(tmpdir) }, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
package travisci
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/shuheiktgw/go-travis"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const (
|
||||
SourceType = sourcespb.SourceType_SOURCE_TYPE_TRAVISCI
|
||||
|
||||
baseURL = "https://api.travis-ci.com/"
|
||||
|
||||
pageSize = 100
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
verify bool
|
||||
jobPool *errgroup.Group
|
||||
sources.Progress
|
||||
client *travis.Client
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
returnAfterFirstChunk bool
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return SourceType
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
// Init returns an initialized TravisCI source.
|
||||
func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
s.verify = verify
|
||||
s.jobPool = &errgroup.Group{}
|
||||
s.jobPool.SetLimit(concurrency)
|
||||
|
||||
var conn sourcespb.TravisCI
|
||||
if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
|
||||
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
|
||||
}
|
||||
|
||||
switch conn.Credential.(type) {
|
||||
case *sourcespb.TravisCI_Token:
|
||||
if conn.GetToken() == "" {
|
||||
return errors.New("token is empty")
|
||||
}
|
||||
s.client = travis.NewClient(baseURL, conn.GetToken())
|
||||
s.client.HTTPClient = common.RetryableHttpClientTimeout(3)
|
||||
|
||||
user, _, err := s.client.User.Current(ctx, nil)
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "error getting testing travis client", 0)
|
||||
}
|
||||
ctx.Logger().V(2).Info("authenticated to Travis CI with user", "username", user.Login)
|
||||
default:
|
||||
return errors.New("credential type not implemented for Travis CI")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {
|
||||
for repoPage := 0; ; repoPage++ {
|
||||
repositories, _, err := s.client.Repositories.List(ctx, &travis.RepositoriesOption{
|
||||
Limit: pageSize,
|
||||
Offset: repoPage * pageSize,
|
||||
})
|
||||
if err != nil {
|
||||
if repoPage == 0 {
|
||||
return fmt.Errorf("error listing repositories: %w", err)
|
||||
}
|
||||
err = reporter.UnitErr(ctx, err)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reporting error: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(repositories) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, repo := range repositories {
|
||||
err = reporter.UnitOk(ctx, sources.CommonSourceUnit{
|
||||
ID: strconv.Itoa(int(*repo.Id)),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reporting unit: %w", err)
|
||||
}
|
||||
ctx.Logger().V(2).Info("enumerated repository", "id", repo.Id, "name", repo.Name)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ChunkUnit implements SourceUnitChunker interface.
|
||||
func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
|
||||
repo, _, err := s.client.Repositories.Find(ctx, unit.SourceUnitID(), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error finding repository: %w", err)
|
||||
}
|
||||
logger := ctx.Logger().WithValues("repo", *repo.Name)
|
||||
logger.V(2).Info("scanning repository")
|
||||
|
||||
// Counts continuous errors from ListByRepoSlug. Used to quit early in
|
||||
// case the API always returns an error.
|
||||
var buildPageErrs int
|
||||
for buildPage := 0; ; buildPage++ {
|
||||
builds, _, err := s.client.Builds.ListByRepoSlug(ctx, *repo.Slug, &travis.BuildsByRepoOption{
|
||||
Limit: pageSize,
|
||||
Offset: buildPage * pageSize,
|
||||
})
|
||||
if err != nil {
|
||||
if err := reporter.ChunkErr(ctx, err); err != nil {
|
||||
return err
|
||||
}
|
||||
buildPageErrs++
|
||||
if buildPageErrs >= 5 {
|
||||
return fmt.Errorf("encountered too many errors listing builds, aborting")
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Reset the page error counter.
|
||||
buildPageErrs = 0
|
||||
|
||||
if len(builds) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, build := range builds {
|
||||
jobs, _, err := s.client.Jobs.ListByBuild(ctx, *build.Id)
|
||||
if err != nil {
|
||||
if err := reporter.ChunkErr(ctx, err); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if len(jobs) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, job := range jobs {
|
||||
log, _, err := s.client.Logs.FindByJobId(ctx, *job.Id)
|
||||
if err != nil {
|
||||
if err := reporter.ChunkErr(ctx, err); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
logger.V(3).Info("scanning job", "id", *job.Id, "number", *job.Number)
|
||||
|
||||
chunk := sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
Data: []byte(*log.Content),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_TravisCI{
|
||||
TravisCI: &source_metadatapb.TravisCI{
|
||||
Username: *job.Owner.Login,
|
||||
Repository: *repo.Name,
|
||||
BuildNumber: *build.Number,
|
||||
JobNumber: *job.Number,
|
||||
Link: fmt.Sprintf("https://app.travis-ci.com/github/%s/%s/jobs/%d", *job.Owner.Login, *repo.Name, *job.Id),
|
||||
Public: !*repo.Private,
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
|
||||
if err := reporter.ChunkOk(ctx, chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.returnAfterFirstChunk {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
// It's a no-op because we've implemented the SourceUnitChunker interface instead.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package travisci
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest"
|
||||
)
|
||||
|
||||
func TestSource_Scan(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
secret, err := common.GetTestSecret(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||
}
|
||||
token := secret.MustGetField("TRAVISCI_TOKEN")
|
||||
|
||||
type init struct {
|
||||
name string
|
||||
verify bool
|
||||
connection *sourcespb.TravisCI
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
init init
|
||||
wantSourceMetadata *source_metadatapb.MetaData
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "get a chunk",
|
||||
init: init{
|
||||
name: "example repo",
|
||||
connection: &sourcespb.TravisCI{
|
||||
Credential: &sourcespb.TravisCI_Token{
|
||||
Token: token,
|
||||
},
|
||||
},
|
||||
verify: true,
|
||||
},
|
||||
wantSourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_TravisCI{
|
||||
TravisCI: &source_metadatapb.TravisCI{
|
||||
Username: "truffle-sandbox",
|
||||
Repository: "test-repo",
|
||||
BuildNumber: "1",
|
||||
JobNumber: "1.1",
|
||||
Link: "https://app.travis-ci.com/github/truffle-sandbox/test-repo/jobs/611053994",
|
||||
Public: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Source{}
|
||||
|
||||
conn, err := anypb.New(tt.init.connection)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
|
||||
reporter := sourcestest.TestReporter{}
|
||||
s.returnAfterFirstChunk = true
|
||||
|
||||
err = s.Enumerate(ctx, &reporter)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("Source.Enumerate() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
|
||||
for _, unit := range reporter.Units {
|
||||
err = s.ChunkUnit(ctx, unit, &reporter)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("Source.ChunkUnit() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
for _, chunk := range reporter.Chunks {
|
||||
if diff := pretty.Compare(chunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {
|
||||
t.Fatalf("comparing chunks: %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package sourcestest
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type reporter interface {
|
||||
sources.UnitReporter
|
||||
sources.ChunkReporter
|
||||
}
|
||||
|
||||
var (
|
||||
_ reporter = (*TestReporter)(nil)
|
||||
_ reporter = (*ErrReporter)(nil)
|
||||
)
|
||||
|
||||
// TestReporter is a helper struct that implements both UnitReporter and
|
||||
// ChunkReporter by simply recording the values passed in the methods.
|
||||
type TestReporter struct {
|
||||
Units []sources.SourceUnit
|
||||
UnitErrs []error
|
||||
Chunks []sources.Chunk
|
||||
ChunkErrs []error
|
||||
}
|
||||
|
||||
func (t *TestReporter) UnitOk(_ context.Context, unit sources.SourceUnit) error {
|
||||
t.Units = append(t.Units, unit)
|
||||
return nil
|
||||
}
|
||||
func (t *TestReporter) UnitErr(_ context.Context, err error) error {
|
||||
t.UnitErrs = append(t.UnitErrs, err)
|
||||
return nil
|
||||
}
|
||||
func (t *TestReporter) ChunkOk(_ context.Context, chunk sources.Chunk) error {
|
||||
t.Chunks = append(t.Chunks, chunk)
|
||||
return nil
|
||||
}
|
||||
func (t *TestReporter) ChunkErr(_ context.Context, err error) error {
|
||||
t.ChunkErrs = append(t.ChunkErrs, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ErrReporter implements UnitReporter and ChunkReporter but always returns an
|
||||
// error.
|
||||
type ErrReporter struct{}
|
||||
|
||||
func (ErrReporter) UnitOk(context.Context, sources.SourceUnit) error {
|
||||
return fmt.Errorf("ErrReporter: UnitOk error")
|
||||
}
|
||||
func (ErrReporter) UnitErr(context.Context, error) error {
|
||||
return fmt.Errorf("ErrReporter: UnitErr error")
|
||||
}
|
||||
func (ErrReporter) ChunkOk(context.Context, sources.Chunk) error {
|
||||
return fmt.Errorf("ErrReporter: ChunkOk error")
|
||||
}
|
||||
func (ErrReporter) ChunkErr(context.Context, error) error {
|
||||
return fmt.Errorf("ErrReporter: ChunkErr error")
|
||||
}
|
||||
@@ -944,6 +944,7 @@ enum DetectorType {
|
||||
Anthropic = 933;
|
||||
Ramp = 934;
|
||||
Klaviyo = 935;
|
||||
SourcegraphCody = 936;
|
||||
}
|
||||
|
||||
message Result {
|
||||
|
||||
@@ -44,6 +44,16 @@ message CircleCI {
|
||||
string email = 7;
|
||||
}
|
||||
|
||||
message TravisCI {
|
||||
string username = 2;
|
||||
string repository = 3;
|
||||
string build_number = 4;
|
||||
string job_number = 5;
|
||||
string link = 6;
|
||||
bool public = 7;
|
||||
}
|
||||
|
||||
|
||||
message Confluence {
|
||||
string page = 1;
|
||||
string space = 2;
|
||||
@@ -294,5 +304,6 @@ message MetaData {
|
||||
SharePoint sharepoint = 25;
|
||||
GoogleDrive googleDrive = 26;
|
||||
AzureRepos azureRepos = 27;
|
||||
TravisCI travisCI = 28;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ enum SourceType {
|
||||
SOURCE_TYPE_SHAREPOINT = 29;
|
||||
SOURCE_TYPE_GCS_UNAUTHED = 30;
|
||||
SOURCE_TYPE_AZURE_REPOS = 31;
|
||||
SOURCE_TYPE_TRAVISCI = 32;
|
||||
}
|
||||
|
||||
message LocalSource {
|
||||
@@ -83,6 +84,13 @@ message CircleCI {
|
||||
}
|
||||
}
|
||||
|
||||
message TravisCI {
|
||||
string endpoint = 1 [(validate.rules).string.uri_ref = true];
|
||||
oneof credential {
|
||||
string token = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message Confluence {
|
||||
string endpoint = 1 [(validate.rules).string.uri_ref = true];
|
||||
oneof credential {
|
||||
|
||||
Reference in New Issue
Block a user