Compare commits
12 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| fd840b72cb | |||
| 953e7f43bd | |||
| 3f84a6700e | |||
| fdeccf06a0 | |||
| 70cdff915b | |||
| 2c32b14df1 | |||
| e53a72abd2 | |||
| 62ce9bac8b | |||
| e68ace5338 | |||
| 3aa5369608 | |||
| 8d66fde6de | |||
| 2a9f34962d |
@@ -105,11 +105,11 @@ var (
|
||||
filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
|
||||
filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
|
||||
|
||||
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
|
||||
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
|
||||
s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
|
||||
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
|
||||
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
|
||||
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
|
||||
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
|
||||
s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
|
||||
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
|
||||
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
|
||||
s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
|
||||
s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
|
||||
s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()
|
||||
@@ -136,12 +136,6 @@ var (
|
||||
circleCiScan = cli.Command("circleci", "Scan CircleCI")
|
||||
circleCiScanToken = circleCiScan.Flag("token", "CircleCI token. Can also be provided with environment variable").Envar("CIRCLECI_TOKEN").Required().String()
|
||||
|
||||
trelloScan = cli.Command("trello", "Scan Trello")
|
||||
trelloApiKey = trelloScan.Flag("apiKey", "Trello API Key. Can also be provided with environment variable").Envar("TRELLO_API_KEY").Required().String()
|
||||
trelloToken = trelloScan.Flag("token", "Trello token. Can also be provided with environment variable").Envar("TRELLO_TOKEN").Required().String()
|
||||
trelloBoardIDs = trelloScan.Flag("boardIDs", "Trello board IDs to scan. Can also be provided with environment variable").Envar("TRELLO_BOARD_IDS").Required().Strings()
|
||||
|
||||
|
||||
dockerScan = cli.Command("docker", "Scan Docker Image")
|
||||
dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, otherwise a image registry is assumed.").Required().Strings()
|
||||
)
|
||||
@@ -360,6 +354,10 @@ func run(state overseer.State) {
|
||||
printer = new(output.PlainPrinter)
|
||||
}
|
||||
|
||||
if !*jsonLegacy && !*jsonOut {
|
||||
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
|
||||
}
|
||||
|
||||
e, err := engine.Start(ctx,
|
||||
engine.WithConcurrency(uint8(*concurrency)),
|
||||
engine.WithDecoders(decoders.DefaultDecoders()...),
|
||||
@@ -475,7 +473,7 @@ func run(state overseer.State) {
|
||||
Secret: *s3ScanSecret,
|
||||
SessionToken: *s3ScanSessionToken,
|
||||
Buckets: *s3ScanBuckets,
|
||||
Roles: *s3ScanRoleArns,
|
||||
Roles: *s3ScanRoleArns,
|
||||
CloudCred: *s3ScanCloudEnv,
|
||||
MaxObjectSize: int64(*s3ScanMaxObjectSize),
|
||||
}
|
||||
@@ -498,10 +496,6 @@ func run(state overseer.State) {
|
||||
if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
|
||||
logFatal(err, "Failed to scan CircleCI.")
|
||||
}
|
||||
case trelloScan.FullCommand():
|
||||
if err := e.ScanTrello(ctx, *trelloBoardIDs, *trelloApiKey, *trelloToken); err != nil {
|
||||
logFatal(err, "Failed to scan Trello.")
|
||||
}
|
||||
case gcsScan.FullCommand():
|
||||
cfg := sources.GCSConfig{
|
||||
ProjectID: *gcsProjectID,
|
||||
@@ -535,10 +529,6 @@ func run(state overseer.State) {
|
||||
}
|
||||
}
|
||||
|
||||
if !*jsonLegacy && !*jsonOut {
|
||||
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
|
||||
}
|
||||
|
||||
// Wait for all workers to finish.
|
||||
if err = e.Finish(ctx); err != nil {
|
||||
logFatal(err, "engine failed to finish execution")
|
||||
|
||||
+130
-110
@@ -118,118 +118,21 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
// REQUEST VALUES.
|
||||
method := "GET"
|
||||
service := "sts"
|
||||
host := "sts.amazonaws.com"
|
||||
region := "us-east-1"
|
||||
endpoint := "https://sts.amazonaws.com"
|
||||
now := time.Now().UTC()
|
||||
datestamp := now.Format("20060102")
|
||||
amzDate := now.Format("20060102T150405Z0700")
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
// TASK 1: CREATE A CANONICAL REQUEST.
|
||||
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
|
||||
canonicalURI := "/"
|
||||
canonicalHeaders := "host:" + host + "\n"
|
||||
signedHeaders := "host"
|
||||
algorithm := "AWS4-HMAC-SHA256"
|
||||
credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
|
||||
|
||||
params := req.URL.Query()
|
||||
params.Add("Action", "GetCallerIdentity")
|
||||
params.Add("Version", "2011-06-15")
|
||||
params.Add("X-Amz-Algorithm", algorithm)
|
||||
params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
|
||||
params.Add("X-Amz-Date", amzDate)
|
||||
params.Add("X-Amz-Expires", "30")
|
||||
params.Add("X-Amz-SignedHeaders", signedHeaders)
|
||||
|
||||
canonicalQuerystring := params.Encode()
|
||||
payloadHash := GetHash("") // empty payload
|
||||
canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
|
||||
|
||||
// TASK 2: CREATE THE STRING TO SIGN.
|
||||
stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
|
||||
|
||||
// TASK 3: CALCULATE THE SIGNATURE.
|
||||
// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
|
||||
hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
|
||||
hash = GetHMAC(hash, []byte(region))
|
||||
hash = GetHMAC(hash, []byte(service))
|
||||
hash = GetHMAC(hash, []byte("aws4_request"))
|
||||
|
||||
signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
|
||||
signature := hex.EncodeToString(signature2)
|
||||
|
||||
// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
|
||||
params.Add("X-Amz-Signature", signature)
|
||||
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
|
||||
req.URL.RawQuery = params.Encode()
|
||||
|
||||
client := s.verificationClient
|
||||
if client == nil {
|
||||
client = defaultVerificationClient
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
identityInfo := identityRes{}
|
||||
err := json.NewDecoder(res.Body).Decode(&identityInfo)
|
||||
if err == nil {
|
||||
s1.Verified = true
|
||||
s1.ExtraData = map[string]string{
|
||||
"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
|
||||
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
|
||||
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
res.Body.Close()
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
if res.StatusCode == 403 {
|
||||
var body awsErrorResponseBody
|
||||
err = json.NewDecoder(res.Body).Decode(&body)
|
||||
res.Body.Close()
|
||||
if err == nil {
|
||||
// All instances of the code I've seen in the wild are PascalCased but this check is
|
||||
// case-insensitive out of an abundance of caution
|
||||
if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
|
||||
// determinate failure - nothing to do
|
||||
} else {
|
||||
// We see a surprising number of false-negative signature mismatch errors here
|
||||
// (The official SDK somehow elicits even more than just making the request
|
||||
// ourselves)
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
verified, extraData, verificationErr := s.verifyMatch(ctx, resIDMatch, resSecretMatch, true)
|
||||
s1.Verified = verified
|
||||
s1.ExtraData = extraData
|
||||
s1.VerificationError = verificationErr
|
||||
}
|
||||
|
||||
// If the result is unverified and matches something like a git hash, don't include it in the results.
|
||||
if !s1.Verified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
|
||||
continue
|
||||
if !s1.Verified {
|
||||
// Unverified results that contain common test words are probably not secrets
|
||||
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
// Unverified results that look like hashes are probably not secrets
|
||||
if falsePositiveSecretCheck.MatchString(resSecretMatch) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
@@ -242,6 +145,123 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
return awsCustomCleanResults(results), nil
|
||||
}
|
||||
|
||||
func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, retryOn403 bool) (bool, map[string]string, error) {
|
||||
// REQUEST VALUES.
|
||||
method := "GET"
|
||||
service := "sts"
|
||||
host := "sts.amazonaws.com"
|
||||
region := "us-east-1"
|
||||
endpoint := "https://sts.amazonaws.com"
|
||||
now := time.Now().UTC()
|
||||
datestamp := now.Format("20060102")
|
||||
amzDate := now.Format("20060102T150405Z0700")
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
// TASK 1: CREATE A CANONICAL REQUEST.
|
||||
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
|
||||
canonicalURI := "/"
|
||||
canonicalHeaders := "host:" + host + "\n"
|
||||
signedHeaders := "host"
|
||||
algorithm := "AWS4-HMAC-SHA256"
|
||||
credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
|
||||
|
||||
params := req.URL.Query()
|
||||
params.Add("Action", "GetCallerIdentity")
|
||||
params.Add("Version", "2011-06-15")
|
||||
params.Add("X-Amz-Algorithm", algorithm)
|
||||
params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
|
||||
params.Add("X-Amz-Date", amzDate)
|
||||
params.Add("X-Amz-Expires", "30")
|
||||
params.Add("X-Amz-SignedHeaders", signedHeaders)
|
||||
|
||||
canonicalQuerystring := params.Encode()
|
||||
payloadHash := GetHash("") // empty payload
|
||||
canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
|
||||
|
||||
// TASK 2: CREATE THE STRING TO SIGN.
|
||||
stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
|
||||
|
||||
// TASK 3: CALCULATE THE SIGNATURE.
|
||||
// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
|
||||
hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
|
||||
hash = GetHMAC(hash, []byte(region))
|
||||
hash = GetHMAC(hash, []byte(service))
|
||||
hash = GetHMAC(hash, []byte("aws4_request"))
|
||||
|
||||
signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
|
||||
signature := hex.EncodeToString(signature2)
|
||||
|
||||
// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
|
||||
params.Add("X-Amz-Signature", signature)
|
||||
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
|
||||
req.URL.RawQuery = params.Encode()
|
||||
|
||||
client := s.verificationClient
|
||||
if client == nil {
|
||||
client = defaultVerificationClient
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
identityInfo := identityRes{}
|
||||
err := json.NewDecoder(res.Body).Decode(&identityInfo)
|
||||
if err == nil {
|
||||
extraData := map[string]string{
|
||||
"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
|
||||
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
|
||||
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
|
||||
}
|
||||
return true, extraData, nil
|
||||
} else {
|
||||
return false, nil, err
|
||||
}
|
||||
} else if res.StatusCode == 403 {
|
||||
// Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that
|
||||
// share a key ID but are signed with different secrets the second one will be rejected with a 403 that
|
||||
// carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is
|
||||
// valid. Since this is exactly our access pattern, we need to work around it.
|
||||
//
|
||||
// Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The
|
||||
// response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of
|
||||
// the response to the original second request, or read to its end, or the resubmission will also yield a
|
||||
// SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only
|
||||
// SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch
|
||||
// after decoding its response body, which requires reading the entire response body, which disables the
|
||||
// workaround.
|
||||
//
|
||||
// We are clearly deep in the guts of AWS implementation details here, so this all might change with no
|
||||
// notice. If you're here because something in this detector broke, you have my condolences.
|
||||
if retryOn403 {
|
||||
return s.verifyMatch(ctx, resIDMatch, resSecretMatch, false)
|
||||
}
|
||||
var body awsErrorResponseBody
|
||||
err = json.NewDecoder(res.Body).Decode(&body)
|
||||
if err == nil {
|
||||
// All instances of the code I've seen in the wild are PascalCased but this check is
|
||||
// case-insensitive out of an abundance of caution
|
||||
if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
|
||||
return false, nil, nil
|
||||
} else {
|
||||
return false, nil, fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
|
||||
}
|
||||
} else {
|
||||
return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err)
|
||||
}
|
||||
} else {
|
||||
return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
return false, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
|
||||
if len(results) == 0 {
|
||||
return results
|
||||
|
||||
@@ -248,6 +248,28 @@ func TestAWS_FromChunk(t *testing.T) {
|
||||
wantErr: false,
|
||||
wantVerificationError: true,
|
||||
},
|
||||
{
|
||||
name: "verified secret checked directly after unverified secret with same key id",
|
||||
s: scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_AWS,
|
||||
Verified: true,
|
||||
Redacted: "AKIASP2TPHJSQH3FJRUX",
|
||||
ExtraData: map[string]string{
|
||||
"account": "171436882533",
|
||||
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
|
||||
"user_id": "AIDASP2TPHJSUFRSTTZX4",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
resMatch = keyString[len(keyString)-1]
|
||||
}
|
||||
|
||||
secret := detectors.Result{
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
Raw: []byte(match[1]),
|
||||
}
|
||||
@@ -84,26 +84,26 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// 401 is bad key
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK:
|
||||
secret.Verified = true
|
||||
s1.Verified = true
|
||||
case http.StatusForbidden:
|
||||
// Good key but not the right scope
|
||||
secret.Verified = true
|
||||
s1.Verified = true
|
||||
case http.StatusUnauthorized:
|
||||
// Nothing to do; zero values are the ones we want
|
||||
default:
|
||||
secret.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
secret.VerificationError = err
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, secret)
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
|
||||
@@ -6,11 +6,12 @@ package gitlab
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
|
||||
@@ -11,7 +11,10 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{ detectors.EndpointSetter }
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
detectors.EndpointSetter
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interfaces at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
@@ -22,7 +25,8 @@ func (Scanner) Version() int { return 2 }
|
||||
func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" }
|
||||
|
||||
var (
|
||||
keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
|
||||
defaultClient = common.SaneHttpClient()
|
||||
keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -42,7 +46,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
|
||||
secret := detectors.Result{
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
Raw: []byte(match[1]),
|
||||
}
|
||||
@@ -52,7 +56,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// they all grant access to different parts of the API. I couldn't find an endpoint that every
|
||||
// one of these scopes has access to, so we just check an example endpoint for each scope. If any
|
||||
// of them contain data, we know we have a valid key, but if they all fail, we don't
|
||||
client := common.SaneHttpClient()
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
|
||||
// test `read_user` scope
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
|
||||
@@ -67,19 +74,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// 200 means good key and has `read_user` scope
|
||||
// 403 means good key but not the right scope
|
||||
// 401 is bad key
|
||||
if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
|
||||
secret.Verified = true
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK:
|
||||
s1.Verified = true
|
||||
case http.StatusForbidden:
|
||||
// Good key but not the right scope
|
||||
s1.Verified = true
|
||||
case http.StatusUnauthorized:
|
||||
// Nothing to do; zero values are the ones we want
|
||||
default:
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, secret)
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
@@ -9,7 +9,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -18,7 +19,7 @@ import (
|
||||
func TestGitlab_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
@@ -31,18 +32,19 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a aws secret %s within", secret)),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
@@ -58,10 +60,65 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a aws secret %s within", secretInactive)),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified but for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found and valid but unexpected api response",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, good key but wrong scope",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(403, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
@@ -78,8 +135,7 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -88,9 +144,12 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatal("no raw secret present")
|
||||
}
|
||||
got[i].Raw = nil
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
|
||||
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -13,13 +13,15 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
defaultClient = common.SaneHttpClient()
|
||||
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendgrid"}) + `(SG\.[\w\-_]{20,24}\.[\w\-_]{39,50})\b`)
|
||||
)
|
||||
@@ -42,7 +44,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
res := strings.TrimSpace(match[1])
|
||||
|
||||
s := detectors.Result{
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_SendGrid,
|
||||
Raw: []byte(res),
|
||||
}
|
||||
@@ -52,6 +54,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// 403 will be issued if the scope is wrong but the key is correct
|
||||
baseURL := "https://api.sendgrid.com/v3/templates"
|
||||
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
// test `read_user` scope
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", baseURL, nil)
|
||||
if err != nil {
|
||||
@@ -67,16 +74,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// 403 means good key but not the right scope
|
||||
// 401 is bad key
|
||||
if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
|
||||
s.Verified = true
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode != http.StatusUnauthorized {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s)
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
@@ -9,7 +9,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -18,7 +19,7 @@ import (
|
||||
func TestSendgrid_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
@@ -30,11 +31,12 @@ func TestSendgrid_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -50,7 +52,8 @@ func TestSendgrid_FromChunk(t *testing.T) {
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
@@ -66,7 +69,8 @@ func TestSendgrid_FromChunk(t *testing.T) {
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -76,22 +80,54 @@ func TestSendgrid_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SendGrid,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SendGrid,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Sendgrid.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Sendgrid.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -122,6 +122,9 @@ func verifyToken(ctx context.Context, client *http.Client, token string) (bool,
|
||||
if err = json.Unmarshal(bytes, &resp); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(resp) == 0 {
|
||||
return false, fmt.Errorf("unexpected response body: %s", string(bytes))
|
||||
}
|
||||
|
||||
return isVerified, err
|
||||
}
|
||||
|
||||
@@ -123,6 +123,40 @@ func TestSentryToken_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, account deactivated",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(200, reponseAccountDeactivated)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SentryToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, account deactivated",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(200, responseEnmpty)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SentryToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
@@ -159,7 +193,8 @@ func TestSentryToken_FromChunk(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
const responseBody403 = `
|
||||
const (
|
||||
responseBody403 = `
|
||||
[
|
||||
{
|
||||
"organization": {
|
||||
@@ -174,6 +209,9 @@ const responseBody403 = `
|
||||
}
|
||||
]
|
||||
`
|
||||
reponseAccountDeactivated = `{"detail": "Authentication credentials were not provided"}`
|
||||
responseEnmpty = `[]`
|
||||
)
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -13,13 +13,16 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Check that the Slack scanner implements the SecretScanner interface at compile time.
|
||||
var _ detectors.Detector = Scanner{}
|
||||
|
||||
var (
|
||||
tokenPats = map[string]*regexp.Regexp{
|
||||
defaultClient = common.SaneHttpClient()
|
||||
tokenPats = map[string]*regexp.Regexp{
|
||||
"Slack Bot Token": regexp.MustCompile(`xoxb\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
|
||||
"Slack User Token": regexp.MustCompile(`xoxp\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
|
||||
"Slack Workspace Access Token": regexp.MustCompile(`xoxa\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
|
||||
@@ -53,34 +56,54 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
tokens := tokenPat.FindAllString(dataStr, -1)
|
||||
|
||||
for _, token := range tokens {
|
||||
s := detectors.Result{
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Slack,
|
||||
Raw: []byte(token),
|
||||
}
|
||||
if verify {
|
||||
client := common.SaneHttpClient()
|
||||
client := s.client
|
||||
if s.client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", verifyURL, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Add("Content-Type", "application/json; charset=utf-8")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
var authResponse authRes
|
||||
if err := json.NewDecoder(res.Body).Decode(&authResponse); err != nil {
|
||||
continue
|
||||
s1.VerificationError = fmt.Errorf("failed to decode auth response: %w", err)
|
||||
}
|
||||
s.Verified = authResponse.Ok
|
||||
|
||||
if authResponse.Ok {
|
||||
s1.Verified = true
|
||||
// Slack API returns 200 even if the token is invalid. We need to check the error field.
|
||||
} else if authResponse.Error == "invalid_auth" {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected error auth response %+v", authResponse.Error)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s)
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
@@ -9,13 +9,15 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestScanner_FromChunk(t *testing.T) {
|
||||
func TestSlack_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
|
||||
@@ -24,17 +26,28 @@ func TestScanner_FromChunk(t *testing.T) {
|
||||
}
|
||||
secret := testSecrets.MustGetField("SLACK")
|
||||
secretInactive := testSecrets.MustGetField("SLACK_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
data []byte
|
||||
verify bool
|
||||
wantResults []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
wantResults []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
|
||||
verify: true,
|
||||
name: "found, verified",
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
wantResults: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Slack,
|
||||
@@ -44,9 +57,12 @@ func TestScanner_FromChunk(t *testing.T) {
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found but unverified",
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
|
||||
verify: true,
|
||||
name: "found but unverified",
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
|
||||
verify: true,
|
||||
},
|
||||
wantResults: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Slack,
|
||||
@@ -55,23 +71,62 @@ func TestScanner_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
wantResults: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Slack,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "unexpected auth response",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "unexpected_error"}`)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
wantResults: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Slack,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(context.Background(), tt.verify, tt.data)
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Scanner.FromData) error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("Slack.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatal("no raw secret present")
|
||||
}
|
||||
got[i].Raw = nil
|
||||
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.wantResults); diff != "" {
|
||||
t.Errorf("Scanner.FromData) %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.wantResults, ignoreOpts); diff != "" {
|
||||
t.Errorf("Slack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package tailscale
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\btskey-[a-z]+-[0-9A-Za-z_]+-[0-9A-Za-z_]+\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"tskey-", "tskey-api-", "tskey-oauth-"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Tailscaleapi secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
resMatch := strings.TrimSpace(match[0])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Tailscale,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
const u = "https://api.tailscale.com/api/v2/secret-scanning/verify"
|
||||
vals := url.Values{"key": []string{resMatch}}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, strings.NewReader(vals.Encode()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
switch res.StatusCode {
|
||||
case http.StatusNoContent:
|
||||
s1.Verified = true
|
||||
case http.StatusUnauthorized:
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
default:
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Tailscale
|
||||
}
|
||||
@@ -1,4 +1,7 @@
|
||||
package gitlab
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package tailscale
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -7,20 +10,23 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestGitlabv2_FromChunk(t *testing.T) {
|
||||
func TestTailscaleapi_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
if err != nil && false {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("GITLABV2")
|
||||
secretInactive := testSecrets.MustGetField("GITLABV2_INACTIVE")
|
||||
secret := testSecrets.MustGetField("TAILSCALEAPI")
|
||||
inactiveSecret := testSecrets.MustGetField("TAILSCALEAPI_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
@@ -34,16 +40,16 @@ func TestGitlabv2_FromChunk(t *testing.T) {
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found",
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secret)),
|
||||
data: []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
DetectorType: detectorspb.DetectorType_Tailscale,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
@@ -54,12 +60,12 @@ func TestGitlabv2_FromChunk(t *testing.T) {
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
|
||||
data: []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
DetectorType: detectorspb.DetectorType_Tailscale,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
@@ -82,28 +88,27 @@ func TestGitlabv2_FromChunk(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("Tailscaleapi.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatal("no raw secret present")
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
t.Errorf("Tailscaleapi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData2(benchmark *testing.B) {
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraph"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -1542,6 +1543,7 @@ func DefaultDetectors() []detectors.Detector {
|
||||
trufflehogenterprise.Scanner{},
|
||||
salesforce.Scanner{},
|
||||
sourcegraph.Scanner{},
|
||||
tailscale.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+10
-3
@@ -565,11 +565,18 @@ func (e *Engine) notifyResults(ctx context.Context) {
|
||||
}
|
||||
atomic.AddUint32(&e.numFoundResults, 1)
|
||||
|
||||
// Dedupe results by comparing the detector type, raw result, and source metadata.
|
||||
// We want to avoid duplicate results with different decoder types, but we also
|
||||
// want to include duplicate results with the same decoder type.
|
||||
// Duplicate results with the same decoder type SHOULD have their own entry in the
|
||||
// results list, this would happen if the same secret is found multiple times.
|
||||
key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
|
||||
if _, ok := e.dedupeCache.Get(key); ok {
|
||||
continue
|
||||
if val, ok := e.dedupeCache.Get(key); ok {
|
||||
if res, ok := val.(detectorspb.DecoderType); ok && res != r.DecoderType {
|
||||
continue
|
||||
}
|
||||
}
|
||||
e.dedupeCache.Add(key, struct{}{})
|
||||
e.dedupeCache.Add(key, r.DecoderType)
|
||||
|
||||
if r.Verified {
|
||||
atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)
|
||||
|
||||
@@ -2,8 +2,13 @@ package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
@@ -171,3 +176,32 @@ func BenchmarkSupportsLineNumbersLoop(b *testing.B) {
|
||||
_ = SupportsLineNumbers(sourceType)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEngine_DuplicatSecrets is a test that detects ALL duplicate secrets with the same decoder.
|
||||
func TestEngine_DuplicatSecrets(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
absPath, err := filepath.Abs("./testdata")
|
||||
assert.Nil(t, err)
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
e, err := Start(ctx,
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(true, DefaultDetectors()...),
|
||||
WithPrinter(new(discardPrinter)),
|
||||
)
|
||||
assert.Nil(t, err)
|
||||
|
||||
cfg := sources.FilesystemConfig{Paths: []string{absPath}}
|
||||
if err := e.ScanFileSystem(ctx, cfg); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Wait for all the chunks to be processed.
|
||||
assert.Nil(t, e.Finish(ctx))
|
||||
want := uint64(5)
|
||||
assert.Equal(t, want, e.GetMetrics().UnverifiedSecretsFound)
|
||||
}
|
||||
|
||||
Vendored
+6
@@ -0,0 +1,6 @@
|
||||
1 aws AKIAWARWQKZNHMZBLY4I
|
||||
2 secret s6NbZeygUrUdM95K683Lb6IsILWXOJlJ8ZVd1Kw0
|
||||
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
|
||||
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
|
||||
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
|
||||
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
|
||||
@@ -1,50 +0,0 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/trello"
|
||||
)
|
||||
|
||||
// ScanTrello scans Trello boards for secrets.
|
||||
func (e *Engine) ScanTrello(ctx context.Context, apiKey string) error {
|
||||
trelloSource := trello.Source{}
|
||||
boardIDs, err := trelloSource.GetBoardIDs(ctx, apiKey)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "failed to get Trello board IDs")
|
||||
return err
|
||||
}
|
||||
|
||||
connection := &sourcespb.Trello{
|
||||
Auth: &sourcespb.Trello_TrelloAuth{
|
||||
ApiKey: apiKey,
|
||||
},
|
||||
Boards: boardIDs,
|
||||
}
|
||||
|
||||
var conn anypb.Any
|
||||
err = anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "failed to marshal Trello connection")
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Trello", new(trello.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
if err := trelloSource.Init(ctx, "trufflehog - Trello", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &trelloSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
+11
-10
@@ -119,7 +119,18 @@ func GenerateLink(repo, commit, file string, line int64) string {
|
||||
case providerBitbucket:
|
||||
return repo[:len(repo)-4] + "/commits/" + commit
|
||||
|
||||
case providerAzure:
|
||||
baseLink := repo + "/commit/" + commit + "/" + file
|
||||
if line > 0 {
|
||||
baseLink += "?line=" + strconv.FormatInt(line, 10)
|
||||
}
|
||||
return baseLink
|
||||
|
||||
case providerGithub, providerGitlab:
|
||||
// If the provider name isn't one of the cloud defaults, it is probably an on-prem github or gitlab.
|
||||
// So do the same thing.
|
||||
fallthrough
|
||||
default:
|
||||
var baseLink string
|
||||
if file == "" {
|
||||
baseLink = repo[:len(repo)-4] + "/commit/" + commit
|
||||
@@ -130,15 +141,5 @@ func GenerateLink(repo, commit, file string, line int64) string {
|
||||
}
|
||||
}
|
||||
return baseLink
|
||||
|
||||
case providerAzure:
|
||||
baseLink := repo + "/commit/" + commit + "/" + file
|
||||
if line > 0 {
|
||||
baseLink += "?line=" + strconv.FormatInt(line, 10)
|
||||
}
|
||||
return baseLink
|
||||
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,6 +160,24 @@ func TestGenerateLink(t *testing.T) {
|
||||
},
|
||||
want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=20",
|
||||
},
|
||||
{
|
||||
name: "Unknown provider on-prem instance",
|
||||
args: args{
|
||||
repo: "https://onprem.customdomain.com/org/repo.git",
|
||||
commit: "xyz123",
|
||||
file: "main.go",
|
||||
line: int64(30),
|
||||
},
|
||||
want: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L30",
|
||||
},
|
||||
{
|
||||
name: "Unknown provider on-prem instance - no file",
|
||||
args: args{
|
||||
repo: "https://onprem.customdomain.com/org/repo.git",
|
||||
commit: "xyz123",
|
||||
},
|
||||
want: "https://onprem.customdomain.com/org/repo/commit/xyz123",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -1000,6 +1000,7 @@ const (
|
||||
DetectorType_HuggingFace DetectorType = 926
|
||||
DetectorType_Snowflake DetectorType = 927
|
||||
DetectorType_Sourcegraph DetectorType = 928
|
||||
DetectorType_Tailscale DetectorType = 929
|
||||
)
|
||||
|
||||
// Enum value maps for DetectorType.
|
||||
@@ -1930,6 +1931,7 @@ var (
|
||||
926: "HuggingFace",
|
||||
927: "Snowflake",
|
||||
928: "Sourcegraph",
|
||||
929: "Tailscale",
|
||||
}
|
||||
DetectorType_value = map[string]int32{
|
||||
"Alibaba": 0,
|
||||
@@ -2857,6 +2859,7 @@ var (
|
||||
"HuggingFace": 926,
|
||||
"Snowflake": 927,
|
||||
"Sourcegraph": 928,
|
||||
"Tailscale": 929,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -3235,7 +3238,7 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
|
||||
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
|
||||
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x9e, 0x74, 0x0a, 0x0c, 0x44,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xae, 0x74, 0x0a, 0x0c, 0x44,
|
||||
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
|
||||
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
|
||||
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
|
||||
@@ -4165,7 +4168,8 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x6f, 0x72, 0x63, 0x65, 0x10, 0x9d, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69,
|
||||
0x6e, 0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x6e, 0x6f,
|
||||
0x77, 0x66, 0x6c, 0x61, 0x6b, 0x65, 0x10, 0x9f, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x53, 0x6f, 0x75,
|
||||
0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x10, 0xa0, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67,
|
||||
0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x10, 0xa0, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x54,
|
||||
0x61, 0x69, 0x6c, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x10, 0xa1, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67,
|
||||
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
|
||||
0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
|
||||
0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64,
|
||||
|
||||
@@ -60,7 +60,6 @@ const (
|
||||
SourceType_SOURCE_TYPE_SHAREPOINT SourceType = 29
|
||||
SourceType_SOURCE_TYPE_GCS_UNAUTHED SourceType = 30
|
||||
SourceType_SOURCE_TYPE_AZURE_REPOS SourceType = 31
|
||||
SourceType_SOURCE_TYPE_TRELLO SourceType = 32
|
||||
)
|
||||
|
||||
// Enum value maps for SourceType.
|
||||
@@ -98,7 +97,6 @@ var (
|
||||
29: "SOURCE_TYPE_SHAREPOINT",
|
||||
30: "SOURCE_TYPE_GCS_UNAUTHED",
|
||||
31: "SOURCE_TYPE_AZURE_REPOS",
|
||||
32: "SOURCE_TYPE_AZURE_TRELLO",
|
||||
}
|
||||
SourceType_value = map[string]int32{
|
||||
"SOURCE_TYPE_AZURE_STORAGE": 0,
|
||||
@@ -133,7 +131,6 @@ var (
|
||||
"SOURCE_TYPE_SHAREPOINT": 29,
|
||||
"SOURCE_TYPE_GCS_UNAUTHED": 30,
|
||||
"SOURCE_TYPE_AZURE_REPOS": 31,
|
||||
"SOURCE_TYPE_AZURE_TRELLO": 32,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -526,80 +523,6 @@ func (*Bitbucket_Oauth) isBitbucket_Credential() {}
|
||||
|
||||
func (*Bitbucket_BasicAuth) isBitbucket_Credential() {}
|
||||
|
||||
type Trello struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Auth *Trello_TrelloAuth `protobuf:"bytes,1,opt,name=auth,proto3,oneof"`
|
||||
Boards []string `protobuf:"bytes,2,rep,name=boards,proto3" json:"boards,omitempty"`
|
||||
}
|
||||
|
||||
type Trello_TrelloAuth struct {
|
||||
ApiKey string `protobuf:"bytes,1,opt,name=api_key,json=apiKey,proto3,oneof"`
|
||||
Token string `protobuf:"bytes,2,opt,name=token,proto3,oneof"`
|
||||
}
|
||||
|
||||
func (x *Trello) Reset() {
|
||||
*x = Trello{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_sources_proto_msgTypes[4]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *Trello) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*Trello) ProtoMessage() {}
|
||||
|
||||
func (x *Trello) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_sources_proto_msgTypes[4]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
// Deprecated: Use Trello.ProtoReflect.Descriptor instead.
|
||||
func (*Trello) Descriptor() ([]byte, []int) {
|
||||
return file_sources_proto_rawDescGZIP(), []int{4} // Assuming Trello is the 5th message in source.proto file
|
||||
}
|
||||
|
||||
func (x *Trello) GetAuth() *Trello_TrelloAuth {
|
||||
if x != nil {
|
||||
return x.Auth
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Trello_TrelloAuth) GetApiKey() string {
|
||||
if t != nil && t.ApiKey != nil {
|
||||
return t.ApiKey
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (t *Trello_TrelloAuth) GetToken() string {
|
||||
if t != nil && t.Token != nil {
|
||||
return t.Token
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *Trello) GetBoards() []string {
|
||||
if x != nil {
|
||||
return x.Boards
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
type CircleCI struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -3770,7 +3693,6 @@ var file_sources_proto_goTypes = []interface{}{
|
||||
(*SlackRealtime)(nil), // 28: sources.SlackRealtime
|
||||
(*Sharepoint)(nil), // 29: sources.Sharepoint
|
||||
(*AzureRepos)(nil), // 30: sources.AzureRepos
|
||||
(*Trello)(nil), // 45: sources.Trello
|
||||
(*durationpb.Duration)(nil), // 31: google.protobuf.Duration
|
||||
(*anypb.Any)(nil), // 32: google.protobuf.Any
|
||||
(*credentialspb.BasicAuth)(nil), // 33: credentials.BasicAuth
|
||||
@@ -4304,16 +4226,13 @@ func file_sources_proto_init() {
|
||||
(*AzureRepos_Token)(nil),
|
||||
(*AzureRepos_Oauth)(nil),
|
||||
}
|
||||
file_sources_proto_msgTypes[29].OneofWrappers = []interface{}{
|
||||
(*Trello_TrelloAuth)(nil),
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
File: protoimpl.DescBuilder{
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: file_sources_proto_rawDesc,
|
||||
NumEnums: 2,
|
||||
NumMessages: 30,
|
||||
NumMessages: 29,
|
||||
NumExtensions: 0,
|
||||
NumServices: 0,
|
||||
},
|
||||
|
||||
@@ -75,7 +75,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
projects, err := s.projects(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting projects: %w", err)
|
||||
|
||||
@@ -85,7 +85,7 @@ type layerInfo struct {
|
||||
}
|
||||
|
||||
// Chunks emits data over a channel that is decoded and scanned for secrets.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
ctx = context.WithValues(ctx, "source_type", s.Type(), "source_name", s.name)
|
||||
|
||||
workers := new(errgroup.Group)
|
||||
|
||||
@@ -77,7 +77,7 @@ func (s *Source) WithFilter(filter *common.Filter) {
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
for i, path := range s.paths {
|
||||
logger := ctx.Logger().WithValues("path", path)
|
||||
if common.IsDone(ctx) {
|
||||
|
||||
@@ -248,7 +248,7 @@ func (s *Source) enumerate(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
persistableCache := s.setupCache(ctx)
|
||||
|
||||
objectCh, err := s.gcsManager.ListObjects(ctx)
|
||||
|
||||
@@ -153,7 +153,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
if err := s.scanRepos(ctx, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -413,7 +413,7 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
apiEndpoint := s.conn.Endpoint
|
||||
if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) {
|
||||
apiEndpoint = "https://api.github.com"
|
||||
|
||||
@@ -139,7 +139,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
// Start client.
|
||||
apiClient, err := s.newClient()
|
||||
if err != nil {
|
||||
|
||||
@@ -232,7 +232,7 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
|
||||
s.scanBuckets(c, defaultRegionClient, roleArn, buckets, chunksChan)
|
||||
}
|
||||
|
||||
@@ -6,10 +6,11 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
)
|
||||
|
||||
// DummySource implements Source and is used for testing a SourceManager.
|
||||
@@ -31,7 +32,7 @@ func (d *DummySource) GetProgress() *Progress { return nil }
|
||||
|
||||
// Interface to easily test different chunking methods.
|
||||
type chunker interface {
|
||||
Chunks(context.Context, chan *Chunk) error
|
||||
Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error
|
||||
ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
|
||||
Enumerate(ctx context.Context, reporter UnitReporter) error
|
||||
}
|
||||
@@ -42,7 +43,7 @@ type counterChunker struct {
|
||||
count int
|
||||
}
|
||||
|
||||
func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
|
||||
func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
|
||||
for i := 0; i < c.count; i++ {
|
||||
select {
|
||||
case ch <- &Chunk{Data: []byte{c.chunkCounter}}:
|
||||
@@ -75,9 +76,9 @@ func (c *counterChunker) ChunkUnit(ctx context.Context, unit SourceUnit, reporte
|
||||
// Chunk method that always returns an error.
|
||||
type errorChunker struct{ error }
|
||||
|
||||
func (c errorChunker) Chunks(context.Context, chan *Chunk) error { return c }
|
||||
func (c errorChunker) Enumerate(context.Context, UnitReporter) error { return c }
|
||||
func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
|
||||
func (c errorChunker) Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error { return c }
|
||||
func (c errorChunker) Enumerate(context.Context, UnitReporter) error { return c }
|
||||
func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
|
||||
|
||||
// enrollDummy is a helper function to enroll a DummySource with a SourceManager.
|
||||
func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {
|
||||
@@ -176,7 +177,7 @@ type unitChunk struct {
|
||||
|
||||
type unitChunker struct{ steps []unitChunk }
|
||||
|
||||
func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
|
||||
func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
|
||||
for _, step := range c.steps {
|
||||
if step.err != "" {
|
||||
continue
|
||||
@@ -294,7 +295,9 @@ type callbackChunker struct {
|
||||
cb func(context.Context, chan *Chunk) error
|
||||
}
|
||||
|
||||
func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk) error { return c.cb(ctx, ch) }
|
||||
func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
|
||||
return c.cb(ctx, ch)
|
||||
}
|
||||
func (c callbackChunker) Enumerate(context.Context, UnitReporter) error { return nil }
|
||||
func (c callbackChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return nil }
|
||||
|
||||
|
||||
+16
-2
@@ -30,6 +30,16 @@ type Chunk struct {
|
||||
Verify bool
|
||||
}
|
||||
|
||||
// ChunkingTarget specifies criteria for a targeted chunking process.
|
||||
// Instead of collecting data indiscriminately, this struct allows the caller
|
||||
// to specify particular subsets of data they're interested in. This becomes
|
||||
// especially useful when one needs to verify or recheck specific data points
|
||||
// without processing the entire dataset.
|
||||
type ChunkingTarget struct {
|
||||
// QueryCriteria represents specific parameters or conditions to target the chunking process.
|
||||
QueryCriteria source_metadatapb.MetaData
|
||||
}
|
||||
|
||||
// Source defines the interface required to implement a source chunker.
|
||||
type Source interface {
|
||||
// Type returns the source type, used for matching against configuration and jobs.
|
||||
@@ -40,8 +50,12 @@ type Source interface {
|
||||
JobID() int64
|
||||
// Init initializes the source.
|
||||
Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error
|
||||
// Chunks emits data over a channel that is decoded and scanned for secrets.
|
||||
Chunks(ctx context.Context, chunksChan chan *Chunk) error
|
||||
// Chunks emits data over a channel which is then decoded and scanned for secrets.
|
||||
// By default, data is obtained indiscriminately. However, by providing one or more
|
||||
// ChunkingTarget parameters, the caller can direct the function to retrieve
|
||||
// specific chunks of data. This targeted approach allows for efficient and
|
||||
// intentional data processing, beneficial when verifying or rechecking specific data points.
|
||||
Chunks(ctx context.Context, chunksChan chan *Chunk, targets ...ChunkingTarget) error
|
||||
// GetProgress is the completion progress (percentage) for Scanned Source.
|
||||
GetProgress() *Progress
|
||||
}
|
||||
|
||||
@@ -182,7 +182,7 @@ func (s *Source) verifyConnectionConfig() error {
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
switch {
|
||||
case s.conn.TlsCert != nilString || s.conn.TlsKey != nilString:
|
||||
cert, err := tls.X509KeyPair([]byte(s.conn.TlsCert), []byte(s.conn.TlsKey))
|
||||
|
||||
@@ -1,239 +0,0 @@
|
||||
package trello
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const baseURL = "https://api.trello.com/1/"
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
apiKey string
|
||||
token string
|
||||
boardIDs []string
|
||||
sourceId int64
|
||||
jobId int64
|
||||
verify bool
|
||||
jobPool *errgroup.Group
|
||||
sources.Progress
|
||||
client *http.Client
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
// Type returns the type of the source.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return sourcespb.SourceType_SOURCE_TYPE_TRELLO
|
||||
}
|
||||
|
||||
// Init returns an initialized Source.
|
||||
func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
s.verify = verify
|
||||
s.jobPool = &errgroup.Group{}
|
||||
s.jobPool.SetLimit(concurrency)
|
||||
s.client = common.RetryableHttpClientTimeout(3)
|
||||
|
||||
var conn sourcespb.Trello
|
||||
if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
|
||||
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
|
||||
}
|
||||
|
||||
s.apiKey = conn.GetAuth().GetApiKey()
|
||||
s.token = conn.GetAuth().GetToken()
|
||||
s.boardIDs = conn.GetBoards()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
scanErrs := sources.NewScanErrors()
|
||||
|
||||
for _, boardID := range s.boardIDs {
|
||||
board, err := s.getBoard(ctx, boardID)
|
||||
if err != nil {
|
||||
scanErrs.Add(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err = s.chunkBoard(ctx, board, chunksChan); err != nil {
|
||||
scanErrs.Add(err)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
_ = s.jobPool.Wait()
|
||||
if scanErrs.Count() > 0 {
|
||||
ctx.Logger().V(2).Info("encountered errors while scanning", "count", scanErrs.Count(), "errors", scanErrs)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type board struct {
|
||||
Name string `json:"name"`
|
||||
Desc string `json:"desc"`
|
||||
ID string `json:"id"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type card struct {
|
||||
Name string `json:"name"`
|
||||
Desc string `json:"desc"`
|
||||
ID string `json:"id"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type action struct {
|
||||
Type string `json:"type"`
|
||||
Data struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// Add these two methods to Source
|
||||
func (s *Source) getCards(_ context.Context, boardID string) ([]card, error) {
|
||||
reqURL := fmt.Sprintf("%sboards/%s/cards?key=%s&token=%s", baseURL, boardID, s.apiKey, s.token)
|
||||
req, err := http.NewRequest("GET", reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
res, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
var cards []card
|
||||
if err := json.NewDecoder(res.Body).Decode(&cards); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return cards, nil
|
||||
}
|
||||
|
||||
func (s *Source) getComments(_ context.Context, cardID string) ([]action, error) {
|
||||
reqURL := fmt.Sprintf("%scards/%s/actions?key=%s&token=%s", baseURL, cardID, s.apiKey, s.token)
|
||||
req, err := http.NewRequest("GET", reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
res, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
var actions []action
|
||||
if err := json.NewDecoder(res.Body).Decode(&actions); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return actions, nil
|
||||
}
|
||||
|
||||
func (s *Source) getBoard(_ context.Context, boardID string) (*board, error) {
|
||||
reqURL := fmt.Sprintf("%sboards/%s?key=%s&token=%s", baseURL, boardID, s.apiKey, s.token)
|
||||
req, err := http.NewRequest("GET", reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
res, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
var b board
|
||||
if err := json.NewDecoder(res.Body).Decode(&b); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &b, nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkBoard(ctx context.Context, board *board, chunksChan chan *sources.Chunk) error {
|
||||
err := s.chunkItem(ctx, chunksChan, board.ID, board.Desc, board.URL, source_metadatapb.ItemType_ITEM_TYPE_BOARD, board.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cards, err := s.getCards(ctx, board.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, card := range cards {
|
||||
err := s.chunkItem(ctx, chunksChan, card.ID, card.Desc, card.URL, source_metadatapb.ItemType_ITEM_TYPE_CARD, card.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
comments, err := s.getComments(ctx, card.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, comment := range comments {
|
||||
if comment.Type != "commentCard" {
|
||||
continue
|
||||
}
|
||||
|
||||
err := s.chunkItem(ctx, chunksChan, card.ID, comment.Data.Text, card.URL, source_metadatapb.ItemType_ITEM_TYPE_COMMENT, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkItem(ctx context.Context, chunksChan chan *sources.Chunk, id, text, url string, itemType source_metadatapb.ItemType, name string) error {
|
||||
data := []byte(text)
|
||||
|
||||
chunk := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.sourceId,
|
||||
JobID: s.jobId,
|
||||
Data: data,
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Trello{
|
||||
Trello: &source_metadatapb.Trello{
|
||||
Id: id,
|
||||
Url: url,
|
||||
Type: itemType,
|
||||
Name: name
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
|
||||
atomic.AddInt64(&s.scannedBytes, int64(len(data)))
|
||||
|
||||
if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
package trello
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
func TestSource_Scan(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
|
||||
defer cancel()
|
||||
|
||||
secret, err := common.GetTestSecret(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||
}
|
||||
apiKey := secret.MustGetField("TRELLO_API_KEY")
|
||||
token := secret.MustGetField("TRELLO_TOKEN")
|
||||
|
||||
type init struct {
|
||||
name string
|
||||
verify bool
|
||||
connection *sourcespb.Trello
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
init init
|
||||
wantSourceMetadata *source_metadatapb.MetaData
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "get a chunk",
|
||||
init: init{
|
||||
name: "example board",
|
||||
connection: &sourcespb.Trello{
|
||||
Auth: &sourcespb.Trello_TrelloAuth{
|
||||
ApiKey: apiKey,
|
||||
Token: token,
|
||||
},
|
||||
Boards: []string{"boardID1"},
|
||||
},
|
||||
verify: true,
|
||||
},
|
||||
wantSourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Trello{
|
||||
Trello: &source_metadatapb.Trello{
|
||||
BoardId: "boardID1",
|
||||
BoardName: "Example Board",
|
||||
CardId: "cardID1",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Source{}
|
||||
|
||||
conn, err := anypb.New(tt.init.connection)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5); (err != nil) != tt.wantErr {
|
||||
t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
chunksCh := make(chan *sources.Chunk, 1)
|
||||
go func() {
|
||||
if err := s.Chunks(ctx, chunksCh); (err != nil) != tt.wantErr {
|
||||
t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
}()
|
||||
|
||||
gotChunk := <-chunksCh
|
||||
if diff := pretty.Compare(gotChunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {
|
||||
t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -68,7 +68,6 @@ func New(c common.Common) *SourceSelect {
|
||||
OssItem("Filesystem", "Scan your filesystem by selecting what directories to scan."),
|
||||
OssItem("AWS S3", "Scan Amazon S3 buckets."),
|
||||
OssItem("CircleCI", "Scan CircleCI, a CI/CD platform."),
|
||||
OssItem("Trello", "Scan Trello, a ticket system."),
|
||||
OssItem("Syslog", "Scan syslog, event data logs."),
|
||||
OssItem("Docker", "Scan a Docker instance, a containerized application."),
|
||||
OssItem("GCS (Google Cloud Storage)", "Scan a Google Cloud Storage instance."),
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/circleci"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/trello"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/docker"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/filesystem"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/gcs"
|
||||
@@ -53,8 +52,6 @@ func GetSourceFields(sourceName string) CmdModel {
|
||||
return syslog.GetFields()
|
||||
case "circleci":
|
||||
return circleci.GetFields()
|
||||
case "trello":
|
||||
return trello.GetFields()
|
||||
case "docker":
|
||||
return docker.GetFields()
|
||||
}
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
package trello
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs"
|
||||
)
|
||||
|
||||
type trelloCmdModel struct {
|
||||
textinputs.Model
|
||||
}
|
||||
|
||||
func GetFields() trelloCmdModel {
|
||||
apiKey := textinputs.InputConfig{
|
||||
Label: "API Key",
|
||||
Key: "api_key",
|
||||
Required: true,
|
||||
Placeholder: "Your Trello API key",
|
||||
}
|
||||
|
||||
token := textinputs.InputConfig{
|
||||
Label: "Token",
|
||||
Key: "token",
|
||||
Required: true,
|
||||
Placeholder: "Your Trello token",
|
||||
}
|
||||
|
||||
boardIds := textinputs.InputConfig{
|
||||
Label: "Board IDs",
|
||||
Key: "board_ids",
|
||||
Required: true,
|
||||
Placeholder: "Comma-separated list of Trello board ID(s)",
|
||||
}
|
||||
|
||||
return trelloCmdModel{textinputs.New([]textinputs.InputConfig{apiKey, token, boardIds})}
|
||||
}
|
||||
|
||||
func (m trelloCmdModel) Cmd() string {
|
||||
var command []string
|
||||
command = append(command, "trufflehog", "trello")
|
||||
|
||||
inputs := m.GetInputs()
|
||||
|
||||
if inputs["api_key"] != "" {
|
||||
command = append(command, "--api-key="+inputs["api_key"])
|
||||
}
|
||||
if inputs["token"] != "" {
|
||||
command = append(command, "--token="+inputs["token"])
|
||||
}
|
||||
if inputs["board_ids"] != "" {
|
||||
command = append(command, "--board-ids="+inputs["board_ids"])
|
||||
}
|
||||
|
||||
return strings.Join(command, " ")
|
||||
}
|
||||
|
||||
func (m trelloCmdModel) Summary() string {
|
||||
inputs := m.GetInputs()
|
||||
labels := m.GetLabels()
|
||||
keys := []string{"api_key", "token", "board_ids"}
|
||||
|
||||
return common.SummarizeSource(keys, inputs, labels)
|
||||
}
|
||||
@@ -937,6 +937,7 @@ enum DetectorType {
|
||||
HuggingFace = 926;
|
||||
Snowflake = 927;
|
||||
Sourcegraph = 928;
|
||||
Tailscale = 929;
|
||||
}
|
||||
|
||||
message Result {
|
||||
|
||||
@@ -186,17 +186,6 @@ message Test {
|
||||
string file = 1;
|
||||
}
|
||||
|
||||
message Trello {
|
||||
string board_id = 1;
|
||||
string card_id = 2;
|
||||
string list_id = 3;
|
||||
string action_id = 4;
|
||||
string user_id = 5;
|
||||
string username = 6;
|
||||
string timestamp = 7;
|
||||
string link = 8;
|
||||
}
|
||||
|
||||
message Jenkins {
|
||||
string project_name = 1;
|
||||
int64 build_number = 2;
|
||||
@@ -305,6 +294,5 @@ message MetaData {
|
||||
SharePoint sharepoint = 25;
|
||||
GoogleDrive googleDrive = 26;
|
||||
AzureRepos azureRepos = 27;
|
||||
Trello trello = 28;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,7 +44,6 @@ enum SourceType {
|
||||
SOURCE_TYPE_SHAREPOINT = 29;
|
||||
SOURCE_TYPE_GCS_UNAUTHED = 30;
|
||||
SOURCE_TYPE_AZURE_REPOS = 31;
|
||||
SOURCE_TYPE_TRELLO = 32;
|
||||
}
|
||||
|
||||
message LocalSource {
|
||||
@@ -266,17 +265,6 @@ message Jenkins {
|
||||
}
|
||||
}
|
||||
|
||||
message Trello {
|
||||
message TrelloAuth {
|
||||
string api_key = 1;
|
||||
string token = 2;
|
||||
}
|
||||
|
||||
TrelloAuth auth = 1;
|
||||
repeated string boards = 2;
|
||||
}
|
||||
|
||||
|
||||
message Teams {
|
||||
string endpoint = 1 [(validate.rules).string.uri_ref = true];
|
||||
oneof credential {
|
||||
|
||||
Reference in New Issue
Block a user